Merge remote-tracking branch 'origin/master' into topic/bernhard/hyperloglog

Conflicts:
	src/NetVar.cc
	src/NetVar.h
	src/SerialTypes.h
	src/probabilistic/CMakeLists.txt
	testing/btest/scripts/base/frameworks/sumstats/basic-cluster.bro
	testing/btest/scripts/base/frameworks/sumstats/basic.bro

CHANGES

@@ -1,4 +1,94 @@
 
+2.1-1034 | 2013-08-03 20:27:43 -0700
+
+  * A set of DHCP extensions. (Vlad Grigorescu)
+
+     - Leases are logged to dhcp.log as they are seen.
+     - scripts/policy/protocols/dhcp/known-devices-and-hostnames.bro
+     - Added DPD sig.
+
+2.1-1027 | 2013-08-03 01:57:37 -0400
+
+  * Fix a major memory issue in the SumStats framework.
+
+2.1-1026 | 2013-08-02 22:35:09 -0400
+
+  * Fix the SumStats top-k plugin and test. (Seth Hall)
+
+  * Rework of SumStats API to reduce high instantaneous memory
+    use on clusters. (Seth Hall)
+
+  * Large update for the SumStats framework.
+
+     - On-demand access to sumstats results through "return from"
+       functions named SumStats::request and Sumstats::request_key.
+       Both functions are tested in standalone and clustered modes.
+
+     - $name field has returned to SumStats which simplifies cluster
+       code and makes the on-demand access stuff possible.
+
+     - Clustered results can only be collected for 1 minute from their
+       time of creation now instead of time of last read.
+
+     - Thresholds use doubles instead of counts everywhere now.
+
+     - Calculation dependency resolution occurs at start up time now
+       instead of doing it at observation time which provide a minor
+       cpu performance improvement.  A new plugin registration mechanism
+       was created to support this change.
+
+     - AppStats now has a minimal doc string and is broken into hook-based
+       plugins.
+
+     - AppStats and traceroute detection added to local.bro (Seth Hall)
+
+2.1-1009 | 2013-08-02 17:19:08 -0700
+
+  * A number of exec module and raw input reader fixes. (Jon Siwek)
+    
+2.1-1007 | 2013-08-01 15:41:54 -0700
+
+  * More function documentation. (Bernhard Amann)
+
+2.1-1004 | 2013-08-01 14:37:43 -0700
+
+  * Adding a probabilistic data structure for computing "top k"
+    elements. (Bernhard Amann)
+
+    The corresponding functions are:
+
+        topk_init(size: count): opaque of topk
+        topk_add(handle: opaque of topk, value: any)
+        topk_get_top(handle: opaque of topk, k: count)
+        topk_count(handle: opaque of topk, value: any): count
+        topk_epsilon(handle: opaque of topk, value: any): count
+        topk_size(handle: opaque of topk): count
+        topk_sum(handle: opaque of topk): count
+        topk_merge(handle1: opaque of topk, handle2: opaque of topk)
+        topk_merge_prune(handle1: opaque of topk, handle2: opaque of topk)
+
+2.1-971 | 2013-08-01 13:28:32 -0700
+
+  * Fix some build errors. (Jon Siwek)
+
+  * Internal refactoring of how plugin components are tagged/managed.
+    (Jon Siwek)
+
+  * Fix various documentation, mostly related to file analysis. (Jon
+    Siwek)
+
+  * Changing the Bloom filter hashing so that it's independent of
+    CompositeHash. (Robin Sommer)
+
+2.1-951 | 2013-08-01 11:19:23 -0400
+
+  * Small fix to deal with a bug in the SSL log delay mechanism.
+
+2.1-948 | 2013-07-31 20:08:28 -0700
+
+  * Fix segfault caused by merging an empty bloom-filter with a
+    bloom-filter already containing values. (Bernhard Amann)
+
 2.1-945 | 2013-07-30 10:05:10 -0700
 
   * Make hashers serializable. (Matthias Vallentin)

NEWS

@@ -113,6 +113,7 @@ New Functionality
   the frequency of elements. The corresponding functions are:
 
     bloomfilter_basic_init(fp: double, capacity: count, name: string &default=""): opaque of bloomfilter
+    bloomfilter_basic_init2(k: count, cells: count, name: string &default=""): opaque of bloomfilter
     bloomfilter_counting_init(k: count, cells: count, max: count, name: string &default=""): opaque of bloomfilter
     bloomfilter_add(bf: opaque of bloomfilter, x: any)
     bloomfilter_lookup(bf: opaque of bloomfilter, x: any): count
@@ -121,6 +122,21 @@ New Functionality
 
   See <INSERT LINK> for full documentation.
 
+- Bro now provides a probabilistic data structure for computing
+  "top k" elements. The corresponding functions are:
+
+    topk_init(size: count): opaque of topk
+    topk_add(handle: opaque of topk, value: any)
+    topk_get_top(handle: opaque of topk, k: count)
+    topk_count(handle: opaque of topk, value: any): count
+    topk_epsilon(handle: opaque of topk, value: any): count
+    topk_size(handle: opaque of topk): count
+    topk_sum(handle: opaque of topk): count
+    topk_merge(handle1: opaque of topk, handle2: opaque of topk)
+    topk_merge_prune(handle1: opaque of topk, handle2: opaque of topk)
+
+  See <INSERT LINK> for full documentation.
+
 - base/utils/exec.bro provides a module to start external processes
   asynchronously and retrieve their output on termination.
   base/utils/dir.bro uses it to monitor a directory for changes, and

VERSION

@@ -1 +1 @@
-2.1-945
+2.1-1034

aux/bro-aux

@@ -1 +1 @@
-Subproject commit 91d258cc8b2f74cd02fc93dfe61f73ec9f0dd489
+Subproject commit d9963983c0b4d426b24836f8d154d014d5aecbba

aux/broctl

@@ -1 +1 @@
-Subproject commit 52fd91261f41fa1528f7b964837a364d7991889e
+Subproject commit 090d4553ace0f9acf2d86eafab07ecfdcc534878

aux/btest

@@ -1 +1 @@
-Subproject commit ce366206e3407e534a786ad572c342e9f9fef26b
+Subproject commit 69606f8f3cc84d694ca1da14868a5fecd4abbc96

doc/file-analysis.rst

@@ -82,9 +82,9 @@ attached, they start receiving the contents of the file as Bro extracts
 it from an ongoing network connection.  What they do with the file
 contents is up to the particular file analyzer implementation, but
 they'll typically either report further information about the file via
-events (e.g. :bro:see:`FileAnalysis::ANALYZER_MD5` will report the
+events (e.g. :bro:see:`Files::ANALYZER_MD5` will report the
 file's MD5 checksum via :bro:see:`file_hash` once calculated) or they'll
-have some side effect (e.g. :bro:see:`FileAnalysis::ANALYZER_EXTRACT`
+have some side effect (e.g. :bro:see:`Files::ANALYZER_EXTRACT`
 will write the contents of the file out to the local file system).
 
 In the future there may be file analyzers that automatically attach to
@@ -98,7 +98,7 @@ explicit attachment decision:
         {
         print "new file", f$id;
         if ( f?$mime_type && f$mime_type == "text/plain" )
-            FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_MD5]);
+            Files::add_analyzer(f, Files::ANALYZER_MD5);
         }
 
     event file_hash(f: fa_file, kind: string, hash: string)
@@ -113,26 +113,27 @@ output::
     file_hash, Cx92a0ym5R8, md5, 397168fd09991a0e712254df7bc639ac
 
 Some file analyzers might have tunable parameters that need to be
-specified in the call to :bro:see:`FileAnalysis::add_analyzer`:
+specified in the call to :bro:see:`Files::add_analyzer`:
 
 .. code:: bro
 
     event file_new(f: fa_file)
         {
-        FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT,
+        Files::add_analyzer(f, Files::ANALYZER_EXTRACT,
-                                       $extract_filename="./myfile"]);
+                            [$extract_filename="myfile"]);
         }
 
 In this case, the file extraction analyzer doesn't generate any further
-events, but does have the side effect of writing out the file contents
+events, but does have the effect of writing out the file contents to the
-to the local file system at the specified location of ``./myfile``.  Of
+local file system at the location resulting from the concatenation of
-course, for a network with more than a single file being transferred,
+the path specified by :bro:see:`FileExtract::prefix` and the string,
-it's probably preferable to specify a different extraction path for each
+``myfile``.  Of course, for a network with more than a single file being
-file, unlike this example.
+transferred, it's probably preferable to specify a different extraction
+path for each file, unlike this example.
 
 Regardless of which file analyzers end up acting on a file, general
 information about the file (e.g. size, time of last data transferred,
-MIME type, etc.) are logged in ``file_analysis.log``.
+MIME type, etc.) are logged in ``files.log``.
 
 Input Framework Integration
 ===========================
@@ -150,7 +151,7 @@ a network interface it's monitoring.  It only requires a call to
     event file_new(f: fa_file)
         {
         print "new file", f$id;
-        FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_MD5]);
+        Files::add_analyzer(f, Files::ANALYZER_MD5);
         }
 
     event file_state_remove(f: fa_file)

doc/index.rst

@@ -47,6 +47,7 @@ Script Reference
    scripts/index
    scripts/builtins
    scripts/proto-analyzers
+   scripts/file-analyzers
 
 Other Bro Components
 --------------------

doc/scripts/CMakeLists.txt

@@ -124,8 +124,10 @@ endmacro(REST_TARGET)
 # Schedule Bro scripts for which to generate documentation.
 include(DocSourcesList.cmake)
 
-# This reST target is independent of a particular Bro script...
+# Macro for generating reST docs that are independent of any particular Bro
-add_custom_command(OUTPUT proto-analyzers.rst
+# script.
+macro(INDEPENDENT_REST_TARGET reST_file)
+    add_custom_command(OUTPUT ${reST_file}
         # delete any leftover state from previous bro runs
         COMMAND "${CMAKE_COMMAND}"
         ARGS -E remove_directory .state
@@ -137,15 +139,19 @@ add_custom_command(OUTPUT proto-analyzers.rst
         COMMAND "${CMAKE_COMMAND}"
         ARGS -E make_directory ${dstDir}
         COMMAND "${CMAKE_COMMAND}"
-    ARGS -E copy proto-analyzers.rst ${dstDir}
+        ARGS -E copy ${reST_file} ${dstDir}
         # clean up the build directory
         COMMAND rm
         ARGS -rf .state *.log *.rst
         DEPENDS bro
         WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
-    COMMENT "[Bro] Generating reST docs for proto-analyzers.rst"
+        COMMENT "[Bro] Generating reST docs for ${reST_file}"
     )
-list(APPEND ALL_REST_OUTPUTS proto-analyzers.rst)
+    list(APPEND ALL_REST_OUTPUTS ${reST_file})
+endmacro(INDEPENDENT_REST_TARGET)
+
+independent_rest_target(proto-analyzers.rst)
+independent_rest_target(file-analyzers.rst)
 
 # create temporary list of all docs to include in the master policy/index file
 file(WRITE ${MASTER_POLICY_INDEX} "${MASTER_POLICY_INDEX_TEXT}")

doc/scripts/DocSourcesList.cmake

@@ -73,6 +73,7 @@ rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_UDP.events.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ZIP.events.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/reporter.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/strings.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/top-k.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/types.bif.bro)
 rest_target(${psd} base/files/extract/main.bro)
 rest_target(${psd} base/files/hash/main.bro)
@@ -129,6 +130,7 @@ rest_target(${psd} base/frameworks/sumstats/plugins/min.bro)
 rest_target(${psd} base/frameworks/sumstats/plugins/sample.bro)
 rest_target(${psd} base/frameworks/sumstats/plugins/std-dev.bro)
 rest_target(${psd} base/frameworks/sumstats/plugins/sum.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/topk.bro)
 rest_target(${psd} base/frameworks/sumstats/plugins/unique.bro)
 rest_target(${psd} base/frameworks/sumstats/plugins/variance.bro)
 rest_target(${psd} base/frameworks/tunnels/main.bro)
@@ -137,10 +139,14 @@ rest_target(${psd} base/protocols/conn/contents.bro)
 rest_target(${psd} base/protocols/conn/inactivity.bro)
 rest_target(${psd} base/protocols/conn/main.bro)
 rest_target(${psd} base/protocols/conn/polling.bro)
+rest_target(${psd} base/protocols/dhcp/consts.bro)
+rest_target(${psd} base/protocols/dhcp/main.bro)
+rest_target(${psd} base/protocols/dhcp/utils.bro)
 rest_target(${psd} base/protocols/dns/consts.bro)
 rest_target(${psd} base/protocols/dns/main.bro)
 rest_target(${psd} base/protocols/ftp/files.bro)
 rest_target(${psd} base/protocols/ftp/gridftp.bro)
+rest_target(${psd} base/protocols/ftp/info.bro)
 rest_target(${psd} base/protocols/ftp/main.bro)
 rest_target(${psd} base/protocols/ftp/utils-commands.bro)
 rest_target(${psd} base/protocols/ftp/utils.bro)
@@ -203,9 +209,16 @@ rest_target(${psd} policy/frameworks/software/vulnerable.bro)
 rest_target(${psd} policy/integration/barnyard2/main.bro)
 rest_target(${psd} policy/integration/barnyard2/types.bro)
 rest_target(${psd} policy/integration/collective-intel/main.bro)
-rest_target(${psd} policy/misc/app-metrics.bro)
+rest_target(${psd} policy/misc/app-stats/main.bro)
+rest_target(${psd} policy/misc/app-stats/plugins/facebook.bro)
+rest_target(${psd} policy/misc/app-stats/plugins/gmail.bro)
+rest_target(${psd} policy/misc/app-stats/plugins/google.bro)
+rest_target(${psd} policy/misc/app-stats/plugins/netflix.bro)
+rest_target(${psd} policy/misc/app-stats/plugins/pandora.bro)
+rest_target(${psd} policy/misc/app-stats/plugins/youtube.bro)
 rest_target(${psd} policy/misc/capture-loss.bro)
 rest_target(${psd} policy/misc/detect-traceroute/main.bro)
+rest_target(${psd} policy/misc/known-devices.bro)
 rest_target(${psd} policy/misc/load-balancing.bro)
 rest_target(${psd} policy/misc/loaded-scripts.bro)
 rest_target(${psd} policy/misc/profiling.bro)
@@ -215,6 +228,7 @@ rest_target(${psd} policy/misc/trim-trace-file.bro)
 rest_target(${psd} policy/protocols/conn/known-hosts.bro)
 rest_target(${psd} policy/protocols/conn/known-services.bro)
 rest_target(${psd} policy/protocols/conn/weirds.bro)
+rest_target(${psd} policy/protocols/dhcp/known-devices-and-hostnames.bro)
 rest_target(${psd} policy/protocols/dns/auth-addl.bro)
 rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
 rest_target(${psd} policy/protocols/ftp/detect-bruteforcing.bro)

scripts/base/frameworks/files/main.bro

@@ -204,7 +204,7 @@ export {
 	## 
 	## tag: Tag for the protocol analyzer having a callback being registered.
 	##
-	## reg: A :bro:see:`ProtoRegistration` record.
+	## reg: A :bro:see:`Files::ProtoRegistration` record.
 	##
 	## Returns: true if the protocol being registered was not previously registered.
 	global register_protocol: function(tag: Analyzer::Tag, reg: ProtoRegistration): bool;
@@ -228,11 +228,6 @@ redef record fa_file += {
 	info: Info &optional;
 };
 
-redef record AnalyzerArgs += {
-	# This is used interally for the core file analyzer api.
-	tag: Files::Tag &optional;
-};
-
 # Store the callbacks for protocol analyzers that have files.
 global registered_protocols: table[Analyzer::Tag] of ProtoRegistration = table();
 
@@ -275,14 +270,12 @@ function set_timeout_interval(f: fa_file, t: interval): bool
 
 function add_analyzer(f: fa_file, tag: Files::Tag, args: AnalyzerArgs): bool
 	{
-	# This is to construct the correct args for the core API.
-	args$tag = tag;
 	add f$info$analyzers[Files::analyzer_name(tag)];
 
 	if ( tag in analyzer_add_callbacks )
 		analyzer_add_callbacks[tag](f, args);
 
-	if ( ! __add_analyzer(f$id, args) )
+	if ( ! __add_analyzer(f$id, tag, args) )
 		{
 		Reporter::warning(fmt("Analyzer %s not added successfully to file %s.", tag, f$id));
 		return F;
@@ -297,8 +290,7 @@ function register_analyzer_add_callback(tag: Files::Tag, callback: function(f: f
 
 function remove_analyzer(f: fa_file, tag: Files::Tag, args: AnalyzerArgs): bool
 	{
-	args$tag = tag;
+	return __remove_analyzer(f$id, tag, args);
-	return __remove_analyzer(f$id, args);
 	}
 
 function stop(f: fa_file): bool

scripts/base/frameworks/sumstats/cluster.bro

@@ -10,10 +10,6 @@
 module SumStats;
 
 export {
-	## Allows a user to decide how large of result groups the workers should transmit
-	## values for cluster stats aggregation.
-	const cluster_send_in_groups_of = 50 &redef;
-
 	## The percent of the full threshold value that needs to be met on a single worker
 	## for that worker to send the value to its manager in order for it to request a
 	## global view for that value.  There is no requirement that the manager requests
@@ -27,45 +23,46 @@ export {
 	## performed.  In practice this should hopefully have a minimal effect.
 	const max_outstanding_global_views = 10 &redef;
 
-	## Intermediate updates can cause overload situations on very large clusters. This
-	## option may help reduce load and correct intermittent problems. The goal for this
-	## option is also meant to be temporary.
-	const enable_intermediate_updates = T &redef;
-
 	## Event sent by the manager in a cluster to initiate the collection of values for
 	## a sumstat.
-	global cluster_ss_request: event(uid: string, ssid: string);
+	global cluster_ss_request: event(uid: string, ss_name: string, cleanup: bool);
 
 	## Event sent by nodes that are collecting sumstats after receiving a request for
 	## the sumstat from the manager.
-	global cluster_ss_response: event(uid: string, ssid: string, data: ResultTable, done: bool);
+	#global cluster_ss_response: event(uid: string, ss_name: string, data: ResultTable, done: bool, cleanup: bool);
 
 	## This event is sent by the manager in a cluster to initiate the collection of
 	## a single key value from a sumstat.  It's typically used to get intermediate
 	## updates before the break interval triggers to speed detection of a value
 	## crossing a threshold.
-	global cluster_key_request: event(uid: string, ssid: string, key: Key);
+	global cluster_get_result: event(uid: string, ss_name: string, key: Key, cleanup: bool);
 
 	## This event is sent by nodes in response to a
-	## :bro:id:`SumStats::cluster_key_request` event.
+	## :bro:id:`SumStats::cluster_get_result` event.
-	global cluster_key_response: event(uid: string, ssid: string, key: Key, result: Result);
+	global cluster_send_result: event(uid: string, ss_name: string, key: Key, result: Result, cleanup: bool);
 
 	## This is sent by workers to indicate that they crossed the percent
 	## of the current threshold by the percentage defined globally in
 	## :bro:id:`SumStats::cluster_request_global_view_percent`
-	global cluster_key_intermediate_response: event(ssid: string, key: SumStats::Key);
+	global cluster_key_intermediate_response: event(ss_name: string, key: SumStats::Key);
 
 	## This event is scheduled internally on workers to send result chunks.
-	global send_data: event(uid: string, ssid: string, data: ResultTable);
+	global send_data: event(uid: string, ss_name: string, data: ResultTable, cleanup: bool);
+
+	global get_a_key: event(uid: string, ss_name: string, cleanup: bool &default=F);
+
+	global send_a_key: event(uid: string, ss_name: string, key: Key);
+	global send_no_key: event(uid: string, ss_name: string);
 
 	## This event is generated when a threshold is crossed.
-	global cluster_threshold_crossed: event(ssid: string, key: SumStats::Key, thold: Thresholding);
+	global cluster_threshold_crossed: event(ss_name: string, key: SumStats::Key, thold_index: count);
 }
 
 # Add events to the cluster framework to make this work.
-redef Cluster::manager2worker_events += /SumStats::cluster_(ss_request|key_request|threshold_crossed)/;
+redef Cluster::manager2worker_events += /SumStats::cluster_(ss_request|get_result|threshold_crossed)/;
-redef Cluster::manager2worker_events += /SumStats::thresholds_reset/;
+redef Cluster::manager2worker_events += /SumStats::(thresholds_reset|get_a_key)/;
-redef Cluster::worker2manager_events += /SumStats::cluster_(ss_response|key_response|key_intermediate_response)/;
+redef Cluster::worker2manager_events += /SumStats::cluster_(ss_response|send_result|key_intermediate_response)/;
+redef Cluster::worker2manager_events += /SumStats::(send_a_key|send_no_key)/;
 
 @if ( Cluster::local_node_type() != Cluster::MANAGER )
 # This variable is maintained to know what keys have recently sent as
@@ -74,12 +71,9 @@ redef Cluster::worker2manager_events += /SumStats::cluster_(ss_response|key_resp
 # an intermediate result has been received.
 global recent_global_view_keys: table[string, Key] of count &create_expire=1min &default=0;
 
-event bro_init() &priority=-100
+# Result tables indexed on a uid that are currently being sent to the
-	{
+# manager.
-	# The manager is the only host allowed to track these.
+global sending_results: table[string] of ResultTable = table() &create_expire=1min;
-	stats_store = table();
-	reducer_store = table();
-	}
 
 # This is done on all non-manager node types in the event that a sumstat is
 # being collected somewhere other than a worker.
@@ -87,95 +81,151 @@ function data_added(ss: SumStat, key: Key, result: Result)
 	{
 	# If an intermediate update for this value was sent recently, don't send
 	# it again.
-	if ( [ss$id, key] in recent_global_view_keys )
+	if ( [ss$name, key] in recent_global_view_keys )
 		return;
 
 	# If val is 5 and global view % is 0.1 (10%), pct_val will be 50.  If that
 	# crosses the full threshold then it's a candidate to send as an
 	# intermediate update.
-	if ( enable_intermediate_updates &&
+	if ( check_thresholds(ss, key, result, cluster_request_global_view_percent) )
-	     check_thresholds(ss, key, result, cluster_request_global_view_percent) )
 		{
 		# kick off intermediate update
-		event SumStats::cluster_key_intermediate_response(ss$id, key);
+		event SumStats::cluster_key_intermediate_response(ss$name, key);
-		++recent_global_view_keys[ss$id, key];
+		++recent_global_view_keys[ss$name, key];
 		}
 	}
 
-event SumStats::send_data(uid: string, ssid: string, data: ResultTable)
+#event SumStats::send_data(uid: string, ss_name: string, cleanup: bool)
-	{
+#	{
-	#print fmt("WORKER %s: sending data for uid %s...", Cluster::node, uid);
+#	#print fmt("WORKER %s: sending data for uid %s...", Cluster::node, uid);
+#
+#	local local_data: ResultTable = table();
+#	local incoming_data: ResultTable = cleanup ? data : copy(data);
+#
+#	local num_added = 0;
+#	for ( key in incoming_data )
+#		{
+#		local_data[key] = incoming_data[key];
+#		delete incoming_data[key];
+#
+#		# Only send cluster_send_in_groups_of at a time.  Queue another
+#		# event to send the next group.
+#		if ( cluster_send_in_groups_of == ++num_added )
+#			break;
+#		}
+#
+#	local done = F;
+#	# If data is empty, this sumstat is done.
+#	if ( |incoming_data| == 0 )
+#		done = T;
+#
+#	# Note: copy is needed to compensate serialization caching issue. This should be
+#	# changed to something else later. 
+#	event SumStats::cluster_ss_response(uid, ss_name, copy(local_data), done, cleanup);
+#	if ( ! done )
+#		schedule 0.01 sec { SumStats::send_data(uid, T) };
+#	}
 
-	local local_data: ResultTable = table();
+event SumStats::get_a_key(uid: string, ss_name: string, cleanup: bool)
-	local num_added = 0;
-	for ( key in data )
 	{
-		local_data[key] = data[key];
+	if ( uid in sending_results )
-		delete data[key];
+		{
-
+		if ( |sending_results[uid]| == 0 )
-		# Only send cluster_send_in_groups_of at a time.  Queue another
+			{
-		# event to send the next group.
+			event SumStats::send_no_key(uid, ss_name);
-		if ( cluster_send_in_groups_of == ++num_added )
+			}
+		else
+			{
+			for ( key in sending_results[uid] )
+				{
+				event SumStats::send_a_key(uid, ss_name, key);
+				# break to only send one.
 				break;
 				}
-
+			}
-	local done = F;
+		}
-	# If data is empty, this sumstat is done.
+	else if ( !cleanup && ss_name in result_store && |result_store[ss_name]| > 0 )
-	if ( |data| == 0 )
+		{
-		done = T;
+		if ( |result_store[ss_name]| == 0 )
-
+			{
-	# Note: copy is needed to compensate serialization caching issue. This should be
+			event SumStats::send_no_key(uid, ss_name);
-	# changed to something else later. 
+			}
-	event SumStats::cluster_ss_response(uid, ssid, copy(local_data), done);
+		else
-	if ( ! done )
+			{
-		schedule 0.01 sec { SumStats::send_data(uid, ssid, data) };
+			for ( key in result_store[ss_name] )
+				{
+				event SumStats::send_a_key(uid, ss_name, key);
+				# break to only send one.
+				break;
+				}
+			}
+		}
+	else
+		{
+		event SumStats::send_no_key(uid, ss_name);
+		}
 	}
 
-event SumStats::cluster_ss_request(uid: string, ssid: string)
+event SumStats::cluster_ss_request(uid: string, ss_name: string, cleanup: bool)
 	{
 	#print fmt("WORKER %s: received the cluster_ss_request event for %s.", Cluster::node, id);
 
-	# Initiate sending all of the data for the requested stats.
+	# Create a back store for the result
-	if ( ssid in result_store )
+	sending_results[uid] = (ss_name in result_store) ? result_store[ss_name] : table();
-		event SumStats::send_data(uid, ssid, result_store[ssid]);
-	else
-		event SumStats::send_data(uid, ssid, table());
 
 	# Lookup the actual sumstats and reset it, the reference to the data
-	# currently stored will be maintained internally by the send_data event.
+	# currently stored will be maintained internally from the 
-	if ( ssid in stats_store )
+	# sending_results table.
-		reset(stats_store[ssid]);
+	if ( cleanup && ss_name in stats_store )
+		reset(stats_store[ss_name]);
 	}
 
-event SumStats::cluster_key_request(uid: string, ssid: string, key: Key)
+event SumStats::cluster_get_result(uid: string, ss_name: string, key: Key, cleanup: bool)
 	{
-	if ( ssid in result_store && key in result_store[ssid] )
+	#print fmt("WORKER %s: received the cluster_get_result event for %s=%s.", Cluster::node, key2str(key), data);
-		{
-		#print fmt("WORKER %s: received the cluster_key_request event for %s=%s.", Cluster::node, key2str(key), data);
 
+	if ( cleanup ) # data will implicitly be in sending_results (i know this isn't great)
+		{
+		if ( uid in sending_results && key in sending_results[uid] )
+			{
 			# Note: copy is needed to compensate serialization caching issue. This should be
 			# changed to something else later. 
-		event SumStats::cluster_key_response(uid, ssid, key, copy(result_store[ssid][key]));
+			event SumStats::cluster_send_result(uid, ss_name, key, copy(sending_results[uid][key]), cleanup);
+			delete sending_results[uid][key];
 			}
 		else
 			{
 			# We need to send an empty response if we don't have the data so that the manager
 			# can know that it heard back from all of the workers.
-		event SumStats::cluster_key_response(uid, ssid, key, table());
+			event SumStats::cluster_send_result(uid, ss_name, key, table(), cleanup);
 			}
 		}
-
+	else 
-event SumStats::cluster_threshold_crossed(ssid: string, key: SumStats::Key, thold: Thresholding)
 		{
-	if ( ssid !in threshold_tracker )
+		if ( ss_name in result_store && key in result_store[ss_name] )
-		threshold_tracker[ssid] = table();
+			{
-
+			event SumStats::cluster_send_result(uid, ss_name, key, copy(result_store[ss_name][key]), cleanup);
-	threshold_tracker[ssid][key] = thold;
+			}
+		else
+			{
+			# We need to send an empty response if we don't have the data so that the manager
+			# can know that it heard back from all of the workers.
+			event SumStats::cluster_send_result(uid, ss_name, key, table(), cleanup);
+			}
+		}
 	}
 
-event SumStats::thresholds_reset(ssid: string)
+event SumStats::cluster_threshold_crossed(ss_name: string, key: SumStats::Key, thold_index: count)
 	{
-	threshold_tracker[ssid] = table();
+	if ( ss_name !in threshold_tracker )
+		threshold_tracker[ss_name] = table();
+
+	threshold_tracker[ss_name][key] = thold_index;
+	}
+
+event SumStats::thresholds_reset(ss_name: string)
+	{
+	delete threshold_tracker[ss_name];
 	}
 
 @endif
@@ -186,7 +236,7 @@ event SumStats::thresholds_reset(ssid: string)
 # This variable is maintained by manager nodes as they collect and aggregate
 # results.
 # Index on a uid.
-global stats_results: table[string] of ResultTable &read_expire=1min;
+global stats_keys: table[string] of set[Key] &create_expire=1min;
 
 # This variable is maintained by manager nodes to track how many "dones" they
 # collected per collection unique id.  Once the number of results for a uid
@@ -194,18 +244,18 @@ global stats_results: table[string] of ResultTable &read_expire=1min;
 # result is written out and deleted from here.
 # Indexed on a uid.
 # TODO: add an &expire_func in case not all results are received.
-global done_with: table[string] of count &read_expire=1min &default=0;
+global done_with: table[string] of count &create_expire=1min &default=0;
 
 # This variable is maintained by managers to track intermediate responses as
 # they are getting a global view for a certain key.
 # Indexed on a uid.
-global key_requests: table[string] of Result &read_expire=1min;
+global key_requests: table[string] of Result &create_expire=1min;
 
 # This variable is maintained by managers to prevent overwhelming communication due
 # to too many intermediate updates.  Each sumstat is tracked separately so that
 # one won't overwhelm and degrade other quieter sumstats.
 # Indexed on a sumstat id.
-global outstanding_global_views: table[string] of count &default=0;
+global outstanding_global_views: table[string] of count &create_expire=1min &default=0;
 
 const zero_time = double_to_time(0.0);
 # Managers handle logging.
@@ -213,15 +263,19 @@ event SumStats::finish_epoch(ss: SumStat)
 	{
 	if ( network_time() > zero_time )
 		{
-		#print fmt("%.6f MANAGER: breaking %s sumstat for %s sumstat", network_time(), ss$name, ss$id);
+		#print fmt("%.6f MANAGER: breaking %s sumstat", network_time(), ss$name);
 		local uid = unique_id("");
 
-		if ( uid in stats_results )
+		if ( uid in stats_keys )
-			delete stats_results[uid];
+			delete stats_keys[uid];
-		stats_results[uid] = table();
+		stats_keys[uid] = set();
 
 		# Request data from peers.
-		event SumStats::cluster_ss_request(uid, ss$id);
+		event SumStats::cluster_ss_request(uid, ss$name, T);
+
+		done_with[uid] = 0;
+		#print fmt("get_key by uid: %s", uid);
+		event SumStats::get_a_key(uid, ss$name, T);
 		}
 
 	# Schedule the next finish_epoch event.
@@ -235,51 +289,160 @@ function data_added(ss: SumStat, key: Key, result: Result)
 	if ( check_thresholds(ss, key, result, 1.0) )
 		{
 		threshold_crossed(ss, key, result);
-		event SumStats::cluster_threshold_crossed(ss$id, key, threshold_tracker[ss$id][key]);
+		event SumStats::cluster_threshold_crossed(ss$name, key, threshold_tracker[ss$name][key]);
 		}
 	}
 
-event SumStats::cluster_key_response(uid: string, ssid: string, key: Key, result: Result)
+function handle_end_of_result_collection(uid: string, ss_name: string, key: Key, cleanup: bool)
 	{
-	#print fmt("%0.6f MANAGER: receiving key data from %s - %s=%s", network_time(), get_event_peer()$descr, key2str(key), result);
-
-	# We only want to try and do a value merge if there are actually measured datapoints
-	# in the Result.
-	if ( uid in key_requests )
-		key_requests[uid] = compose_results(key_requests[uid], result);
-	else
-		key_requests[uid] = result;
-
-	# Mark that a worker is done.
-	++done_with[uid];
-
 	#print fmt("worker_count:%d :: done_with:%d", Cluster::worker_count, done_with[uid]);
-	if ( Cluster::worker_count == done_with[uid] )
+	local ss = stats_store[ss_name];
-		{
-		local ss = stats_store[ssid];
 	local ir = key_requests[uid];
 	if ( check_thresholds(ss, key, ir, 1.0) )
 		{
 		threshold_crossed(ss, key, ir);
-			event SumStats::cluster_threshold_crossed(ss$id, key, threshold_tracker[ss$id][key]);
+		event SumStats::cluster_threshold_crossed(ss_name, key, threshold_tracker[ss_name][key]);
 		}
 
-		delete done_with[uid];
+	if ( cleanup )
-		delete key_requests[uid];
+		{
-		# Check that there is an outstanding view before subtracting.
+		# This is done here because "cleanup" implicitly means
-		if ( outstanding_global_views[ssid] > 0 )
+		# it's the end of an epoch.
-			--outstanding_global_views[ssid];
+		if ( ss?$epoch_result && |ir| > 0 )
+			{
+			local now = network_time();
+			ss$epoch_result(now, key, ir);
 			}
+
+		# Check that there is an outstanding view before subtracting.
+		# Global views only apply to non-dynamic requests. Dynamic
+		# requests must be serviced.
+		if ( outstanding_global_views[ss_name] > 0 )
+			--outstanding_global_views[ss_name];
+		}
+
+	delete key_requests[uid];
+	delete done_with[uid];
+	}
+
+function request_all_current_keys(uid: string, ss_name: string, cleanup: bool)
+	{
+	#print "request_all_current_keys";
+	if ( uid in stats_keys && |stats_keys[uid]| > 0 )
+		{
+		#print fmt("    -- %d remaining keys here", |stats_keys[uid]|);
+		for ( key in stats_keys[uid] )
+			{
+			done_with[uid] = 0;
+			event SumStats::cluster_get_result(uid, ss_name, key, cleanup);
+			when ( uid in done_with && Cluster::worker_count == done_with[uid] )
+				{
+				#print "done getting result";
+				handle_end_of_result_collection(uid, ss_name, key, cleanup);
+				request_all_current_keys(uid, ss_name, cleanup);
+				}
+			delete stats_keys[uid][key];
+			break; # only a single key
+			}
+		}
+	else
+		{
+		# Get more keys!  And this breaks us out of the evented loop.
+		done_with[uid] = 0;
+		#print fmt("get_key by uid: %s", uid);
+		event SumStats::get_a_key(uid, ss_name, cleanup);
+		}
+	}
+
+event SumStats::send_no_key(uid: string, ss_name: string)
+	{
+	#print "send_no_key";
+	++done_with[uid];
+	if ( Cluster::worker_count == done_with[uid] )
+		{
+		delete done_with[uid];
+
+		if ( |stats_keys[uid]| > 0 )
+			{
+			#print "we need more keys!";
+			# Now that we have a key from each worker, lets
+			# grab all of the results.
+			request_all_current_keys(uid, ss_name, T);
+			}
+		else
+			{
+			#print "we're out of keys!";
+			local ss = stats_store[ss_name];
+			if ( ss?$epoch_finished )
+				ss$epoch_finished(network_time());
+			}
+		}
+	}
+
+event SumStats::send_a_key(uid: string, ss_name: string, key: Key)
+	{
+	#print fmt("send_a_key %s", key);
+	if ( uid !in stats_keys )
+		{
+		# no clue what happened here
+		return;
+		}
+
+	if ( key !in stats_keys[uid] )
+		add stats_keys[uid][key];
+
+	++done_with[uid];
+	if ( Cluster::worker_count == done_with[uid] )
+		{
+		delete done_with[uid];
+
+		if ( |stats_keys[uid]| > 0 )
+			{
+			#print "we need more keys!";
+			# Now that we have a key from each worker, lets
+			# grab all of the results.
+			request_all_current_keys(uid, ss_name, T);
+			}
+		else
+			{
+			#print "we're out of keys!";
+			local ss = stats_store[ss_name];
+			if ( ss?$epoch_finished )
+				ss$epoch_finished(network_time());
+			}
+		}
+	}
+
+event SumStats::cluster_send_result(uid: string, ss_name: string, key: Key, result: Result, cleanup: bool)
+	{
+	#print "cluster_send_result";
+	#print fmt("%0.6f MANAGER: receiving key data from %s - %s=%s", network_time(), get_event_peer()$descr, key2str(key), result);
+
+	# We only want to try and do a value merge if there are actually measured datapoints
+	# in the Result.
+	if ( uid !in key_requests || |key_requests[uid]| == 0 )
+		key_requests[uid] = result;
+	else
+		key_requests[uid] = compose_results(key_requests[uid], result);
+
+	# Mark that a worker is done.
+	++done_with[uid];
+
+	#if ( Cluster::worker_count == done_with[uid] )
+	#	{
+	#	print "done";
+	#	handle_end_of_result_collection(uid, ss_name, key, cleanup);
+	#	}
 	}
 
 # Managers handle intermediate updates here.
-event SumStats::cluster_key_intermediate_response(ssid: string, key: Key)
+event SumStats::cluster_key_intermediate_response(ss_name: string, key: Key)
 	{
 	#print fmt("MANAGER: receiving intermediate key data from %s", get_event_peer()$descr);
 	#print fmt("MANAGER: requesting key data for %s", key2str(key));
 
-	if ( ssid in outstanding_global_views &&
+	if ( ss_name in outstanding_global_views &&
-	     |outstanding_global_views[ssid]| > max_outstanding_global_views )
+	     |outstanding_global_views[ss_name]| > max_outstanding_global_views )
 		{
 		# Don't do this intermediate update.  Perhaps at some point in the future
 		# we will queue and randomly select from these ignored intermediate
@@ -287,60 +450,131 @@ event SumStats::cluster_key_intermediate_response(ssid: string, key: Key)
 		return;
 		}
 
-	++outstanding_global_views[ssid];
+	++outstanding_global_views[ss_name];
 
 	local uid = unique_id("");
-	event SumStats::cluster_key_request(uid, ssid, key);
+	done_with[uid] = 0;
+	event SumStats::cluster_get_result(uid, ss_name, key, F);
+	when ( uid in done_with && Cluster::worker_count == done_with[uid] )
+		{
+		handle_end_of_result_collection(uid, ss_name, key, F);
+		}
+	timeout 1.1min
+		{
+		Reporter::warning(fmt("Dynamic SumStat intermediate key request for %s (%s) took longer than 1 minute and was automatically cancelled.", ss_name, key));
 		}
 
-event SumStats::cluster_ss_response(uid: string, ssid: string, data: ResultTable, done: bool)
-	{
-	#print fmt("MANAGER: receiving results from %s", get_event_peer()$descr);
-
-	# Mark another worker as being "done" for this uid.
-	if ( done )
-		++done_with[uid];
-
-	local local_data = stats_results[uid];
-	local ss = stats_store[ssid];
-
-	for ( key in data )
-		{
-		if ( key in local_data )
-			local_data[key] = compose_results(local_data[key], data[key]);
-		else
-			local_data[key] = data[key];
-
-		# If a stat is done being collected, thresholds for each key
-		# need to be checked so we're doing it here to avoid doubly
-		# iterating over each key.
-		if ( Cluster::worker_count == done_with[uid] )
-			{
-			if ( check_thresholds(ss, key, local_data[key], 1.0) )
-				{
-				threshold_crossed(ss, key, local_data[key]);
-				event SumStats::cluster_threshold_crossed(ss$id, key, threshold_tracker[ss$id][key]);
-				}
-			}
 	}
 
-	# If the data has been collected from all peers, we are done and ready to finish.
+#event SumStats::cluster_ss_response(uid: string, ss_name: string, data: ResultTable, done: bool, cleanup: bool)
-	if ( Cluster::worker_count == done_with[uid] )
+#	{
-		{
+#	#print fmt("MANAGER: receiving results from %s", get_event_peer()$descr);
-		if ( ss?$epoch_finished )
+#
-			ss$epoch_finished(local_data);
+#	# Mark another worker as being "done" for this uid.
+#	if ( done )
+#		++done_with[uid];
+#
+#	# We had better only be getting requests for stuff that exists.
+#	if ( ss_name !in stats_store )
+#		return;
+#
+#	if ( uid !in stats_keys )
+#		stats_keys[uid] = table();
+#
+#	local local_data = stats_keys[uid];
+#	local ss = stats_store[ss_name];
+#
+#	for ( key in data )
+#		{
+#		if ( key in local_data )
+#			local_data[key] = compose_results(local_data[key], data[key]);
+#		else
+#			local_data[key] = data[key];
+#
+#		# If a stat is done being collected, thresholds for each key
+#		# need to be checked so we're doing it here to avoid doubly
+#		# iterating over each key.
+#		if ( Cluster::worker_count == done_with[uid] )
+#			{
+#			if ( check_thresholds(ss, key, local_data[key], 1.0) )
+#				{
+#				threshold_crossed(ss, key, local_data[key]);
+#				event SumStats::cluster_threshold_crossed(ss$name, key, threshold_tracker[ss$name][key]);
+#				}
+#			}
+#		}
+#
+#	# If the data has been collected from all peers, we are done and ready to finish.
+#	if ( cleanup && Cluster::worker_count == done_with[uid] )
+#		{
+#		local now = network_time();
+#		if ( ss?$epoch_result )
+#			{
+#			for ( key in local_data )
+#				ss$epoch_result(now, key, local_data[key]);
+#			}
+#
+#		if ( ss?$epoch_finished )
+#			ss$epoch_finished(now);
+#
+#		# Clean up
+#		delete stats_keys[uid];
+#		delete done_with[uid];
+#		reset(ss);
+#		}
+#	}
 
+#function request(ss_name: string): ResultTable
+#	{
+#	# This only needs to be implemented this way for cluster compatibility.
+#	local uid = unique_id("dyn-");
+#	stats_keys[uid] = table();
+#	done_with[uid] = 0;
+#	event SumStats::cluster_ss_request(uid, ss_name, F);
+#
+#	return when ( uid in done_with && Cluster::worker_count == done_with[uid] )
+#		{
+#		if ( uid in stats_keys )
+#			{
+#			local ss_result = stats_keys[uid];
+#			# Clean up
+#			delete stats_keys[uid];
+#			delete done_with[uid];
+#			reset(stats_store[ss_name]);
+#			return ss_result;
+#			}
+#		else
+#			return table();
+#		}
+#	timeout 1.1min
+#		{
+#		Reporter::warning(fmt("Dynamic SumStat request for %s took longer than 1 minute and was automatically cancelled.", ss_name));
+#		return table();
+#		}
+#	}
+
+function request_key(ss_name: string, key: Key): Result
+	{
+	local uid = unique_id("");
+	done_with[uid] = 0;
+	key_requests[uid] = table();
+
+	event SumStats::cluster_get_result(uid, ss_name, key, F);
+	return when ( uid in done_with && Cluster::worker_count == done_with[uid] )
+		{
+		#print "done with request_key";
+		local result = key_requests[uid];
 		# Clean up
-		delete stats_results[uid];
+		delete key_requests[uid];
 		delete done_with[uid];
-		# Not sure I need to reset the sumstat on the manager.
+
-		reset(ss);
+		return result;
+		}
+	timeout 1.1min
+		{
+		Reporter::warning(fmt("Dynamic SumStat key request for %s (%s) took longer than 1 minute and was automatically cancelled.", ss_name, key));
+		return table();
 		}
 	}
 
-event remote_connection_handshake_done(p: event_peer) &priority=5
-	{
-	send_id(p, "SumStats::stats_store");
-	send_id(p, "SumStats::reducer_store");
-	}
 @endif

scripts/base/frameworks/sumstats/main.bro

@@ -74,10 +74,6 @@ export {
 	## Type to store results for multiple reducers.
 	type Result: table[string] of ResultVal;
 
-	## Type to store a table of sumstats results indexed
-	## by keys.
-	type ResultTable: table[Key] of Result;
-
 	## SumStats represent an aggregation of reducers along with
 	## mechanisms to handle various situations like the epoch ending
 	## or thresholds being crossed.
@@ -87,8 +83,12 @@ export {
 	## is no assurance provided as to where the callbacks
 	## will be executed on clusters.
 	type SumStat: record {
+		## An arbitrary name for the sumstat so that it can 
+		## be referred to later.
+		name:               string;
+		
 		## The interval at which this filter should be "broken"
-		## and the '$epoch_finished' callback called.  The
+		## and the '$epoch_result' callback called.  The
 		## results are also reset at this time so any threshold
 		## based detection needs to be set to a
 		## value that should be expected to happen within
@@ -102,22 +102,28 @@ export {
 		## :bro:see:`SumStats::Result` structure which will be used
 		## for thresholding.
 		## This is required if a $threshold value is given.
-		threshold_val:      function(key: SumStats::Key, result: SumStats::Result): count &optional;
+		threshold_val:      function(key: SumStats::Key, result: SumStats::Result): double &optional;
 
 		## The threshold value for calling the
 		## $threshold_crossed callback.
-		threshold:          count             &optional;
+		threshold:          double            &optional;
 
 		## A series of thresholds for calling the
 		## $threshold_crossed callback.
-		threshold_series:   vector of count   &optional;
+		threshold_series:   vector of double  &optional;
 
 		## A callback that is called when a threshold is crossed.
 		threshold_crossed:  function(key: SumStats::Key, result: SumStats::Result) &optional;
 
-		## A callback with the full collection of Results for
+		## A callback that receives each of the results at the
-		## this SumStat.
+		## end of the analysis epoch.  The function will be 
-		epoch_finished:    function(rt: SumStats::ResultTable) &optional;
+		## called once for each key.
+		epoch_result:       function(ts: time, key: SumStats::Key, result: SumStats::Result) &optional;
+	
+		## A callback that will be called when a single collection 
+		## interval is completed.  The ts value will be the time of 
+		## when the collection started.
+		epoch_finished:     function(ts:time) &optional;
 	};
 
 	## Create a summary statistic.
@@ -134,19 +140,23 @@ export {
 	## obs: The data point to send into the stream.
 	global observe: function(id: string, key: SumStats::Key, obs: SumStats::Observation);
 
-	## This record is primarily used for internal threshold tracking.
+	## Dynamically request a sumstat key.  This function should be
-	type Thresholding: record {
+	## used sparingly and not as a replacement for the callbacks 
-		# Internal use only.  Indicates if a simple threshold was already crossed.
+	## from the :bro:see:`SumStat` record.  The function is only
-		is_threshold_crossed: bool &default=F;
+	## available for use within "when" statements as an asynchronous
-
+	## function.
-		# Internal use only.  Current key for threshold series.
+	##
-		threshold_series_index: count &default=0;
+	## ss_name: SumStat name.
-	};
+	##
+	## key: The SumStat key being requested.
+	##
+	## Returns: The result for the requested sumstat key.
+	global request_key: function(ss_name: string, key: Key): Result;
 
 	## This event is generated when thresholds are reset for a SumStat.
 	##
-	## ssid: SumStats ID that thresholds were reset for.
+	## name: SumStats name that thresholds were reset for.
-	global thresholds_reset: event(ssid: string);
+	global thresholds_reset: event(name: string);
 
 	## Helper function to represent a :bro:type:`SumStats::Key` value as
 	## a simple string.
@@ -157,18 +167,49 @@ export {
 	global key2str: function(key: SumStats::Key): string;
 }
 
+# Type to store a table of sumstats results indexed by keys.
+type ResultTable: table[Key] of Result;
+
+# The function prototype for plugins to do calculations.
+type ObserveFunc: function(r: Reducer, val: double, data: Observation, rv: ResultVal);
+
 redef record Reducer += {
-	# Internal use only.  Provides a reference back to the related SumStats by it's ID.
+	# Internal use only.  Provides a reference back to the related SumStats by its name.
-	sid: string &optional;
+	ssname: string &optional;
+
+	calc_funcs: vector of Calculation &optional;
 };
 
 # Internal use only.  For tracking thresholds per sumstat and key.
-global threshold_tracker: table[string] of table[Key] of Thresholding &optional;
+# In the case of a single threshold, 0 means the threshold isn't crossed.
+# In the case of a threshold series, the number tracks the threshold offset.
+global threshold_tracker: table[string] of table[Key] of count;
 
-redef record SumStat += {
+function increment_threshold_tracker(ss_name: string, key: Key)
-	# Internal use only (mostly for cluster coherency).
+	{
-	id: string &optional;
+	if ( ss_name !in threshold_tracker )
-};
+		threshold_tracker[ss_name] = table();
+	if ( key !in threshold_tracker[ss_name] )
+		threshold_tracker[ss_name][key] = 0;
+
+	++threshold_tracker[ss_name][key];
+	}
+
+function get_threshold_index(ss_name: string, key: Key): count
+	{
+	if ( ss_name !in threshold_tracker )
+		return 0;
+	if ( key !in threshold_tracker[ss_name] )
+		return 0;
+
+	return threshold_tracker[ss_name][key];
+	}
+
+# Prototype the hook point for plugins to initialize any result values.
+global init_resultval_hook: hook(r: Reducer, rv: ResultVal);
+
+# Prototype the hook point for plugins to merge Results.
+global compose_resultvals_hook: hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal);
 
 # Store of sumstats indexed on the sumstat id.
 global stats_store: table[string] of SumStat = table();
@@ -182,20 +223,20 @@ global result_store: table[string] of ResultTable = table();
 # Store of threshold information.
 global thresholds_store: table[string, Key] of bool = table();
 
+# Store the calculations.
+global calc_store: table[Calculation] of ObserveFunc = table();
+
+# Store the dependencies for Calculations.
+global calc_deps: table[Calculation] of vector of Calculation = table();
+
+# Hook for registering observation calculation plugins.
+global register_observe_plugins: hook();
+
 # This is called whenever key values are updated and the new val is given as the
 # `val` argument. It's only prototyped here because cluster and non-cluster have
 # separate  implementations.
 global data_added: function(ss: SumStat, key: Key, result: Result);
 
-# Prototype the hook point for plugins to do calculations.
-global observe_hook: hook(r: Reducer, val: double, data: Observation, rv: ResultVal);
-
-# Prototype the hook point for plugins to initialize any result values.
-global init_resultval_hook: hook(r: Reducer, rv: ResultVal);
-
-# Prototype the hook point for plugins to merge Results.
-global compose_resultvals_hook: hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal);
-
 # Event that is used to "finish" measurements and adapt the measurement
 # framework for clustered or non-clustered usage.
 global finish_epoch: event(ss: SumStat);
@@ -210,6 +251,24 @@ function key2str(key: Key): string
 	return fmt("sumstats_key(%s)", out);
 	}
 
+function register_observe_plugin(calc: Calculation, func: ObserveFunc)
+	{
+	calc_store[calc] = func;
+	}
+
+function add_observe_plugin_dependency(calc: Calculation, depends_on: Calculation)
+	{
+	if ( calc !in calc_deps )
+		calc_deps[calc] = vector();
+	calc_deps[calc][|calc_deps[calc]|] = depends_on;
+	}
+
+event bro_init() &priority=100000
+	{
+	# Call all of the plugin registration hooks
+	hook register_observe_plugins();
+	}
+
 function init_resultval(r: Reducer): ResultVal
 	{
 	local rv: ResultVal = [$begin=network_time(), $end=network_time()];
@@ -234,25 +293,17 @@ function compose_results(r1: Result, r2: Result): Result
 	{
 	local result: Result = table();
 
-	if ( |r1| > |r2| )
+	for ( id in r1 )
 		{
-		for ( data_id in r1 )
+		result[id] = r1[id];
-			{
-			if ( data_id in r2 )
-				result[data_id] = compose_resultvals(r1[data_id], r2[data_id]);
-			else
-				result[data_id] = r1[data_id];
 		}
-		}
+
-	else
+	for ( id in r2 )
 		{
-		for ( data_id in r2 )
+		if ( id in r1 )
-			{
+			result[id] = compose_resultvals(r1[id], r2[id]);
-			if ( data_id in r1 )
-				result[data_id] = compose_resultvals(r1[data_id], r2[data_id]);
 		else
-				result[data_id] = r2[data_id];
+			result[id] = r2[id];
-			}
 		}
 
 	return result;
@@ -261,18 +312,43 @@ function compose_results(r1: Result, r2: Result): Result
 
 function reset(ss: SumStat)
 	{
-	if ( ss$id in result_store )
+	if ( ss$name in result_store )
-		delete result_store[ss$id];
+		delete result_store[ss$name];
 
-	result_store[ss$id] = table();
+	result_store[ss$name] = table();
 
-	if ( ss?$threshold || ss?$threshold_series )
+	if ( ss$name in threshold_tracker )
 		{
-		threshold_tracker[ss$id] = table();
+		delete threshold_tracker[ss$name];
-		event SumStats::thresholds_reset(ss$id);
+		threshold_tracker[ss$name] = table();
+		event SumStats::thresholds_reset(ss$name);
 		}
 	}
 
+# This could potentially recurse forever, but plugin authors 
+# should be making sure they aren't causing reflexive dependencies.
+function add_calc_deps(calcs: vector of Calculation, c: Calculation)
+	{
+	#print fmt("Checking for deps for %s", c);
+	for ( i in calc_deps[c] )
+		{
+		local skip_calc=F;
+		for ( j in calcs )
+			{
+			if ( calcs[j] == calc_deps[c][i] )
+				skip_calc=T;
+			}
+		if ( ! skip_calc )
+			{
+			if ( calc_deps[c][i] in calc_deps )
+				add_calc_deps(calcs, calc_deps[c][i]);
+			calcs[|c|] = calc_deps[c][i];
+			#print fmt("add dep for %s [%s] ", c, calc_deps[c][i]);
+			}
+		}
+
+	}
+
 function create(ss: SumStat)
 	{
 	if ( (ss?$threshold || ss?$threshold_series) && ! ss?$threshold_val )
@@ -280,14 +356,34 @@ function create(ss: SumStat)
 		Reporter::error("SumStats given a threshold with no $threshold_val function");
 		}
 
-	if ( ! ss?$id )
+	stats_store[ss$name] = ss;
-		ss$id=unique_id("");
+
-	threshold_tracker[ss$id] = table();
+	if ( ss?$threshold || ss?$threshold_series )
-	stats_store[ss$id] = ss;
+		threshold_tracker[ss$name] = table();
 
 	for ( reducer in ss$reducers )
 		{
-		reducer$sid = ss$id;
+		reducer$ssname = ss$name;
+		reducer$calc_funcs = vector();
+		for ( calc in reducer$apply )
+			{
+			# Add in dependencies recursively.
+			if ( calc in calc_deps )
+				add_calc_deps(reducer$calc_funcs, calc);
+
+			# Don't add this calculation to the vector if 
+			# it was already added by something else as a 
+			# dependency.
+			local skip_calc=F;
+			for ( j in reducer$calc_funcs )
+				{
+				if ( calc == reducer$calc_funcs[j] )
+					skip_calc=T;
+				}
+			if ( ! skip_calc )
+				reducer$calc_funcs[|reducer$calc_funcs|] = calc;
+			}
+
 		if ( reducer$stream !in reducer_store )
 			reducer_store[reducer$stream] = set();
 		add reducer_store[reducer$stream][reducer];
@@ -313,9 +409,9 @@ function observe(id: string, key: Key, obs: Observation)
 		if ( r?$pred && ! r$pred(key, obs) )
 			next;
 
-		local ss = stats_store[r$sid];
+		local ss = stats_store[r$ssname];
 
-		# If there is a threshold and no epoch_finished callback
+		# If there is a threshold and no epoch_result callback
 		# we don't need to continue counting since the data will
 		# never be accessed.  This was leading
 		# to some state management issues when measuring
@@ -323,18 +419,21 @@ function observe(id: string, key: Key, obs: Observation)
 		# NOTE: this optimization could need removed in the
 		#       future if on demand access is provided to the
 		#       SumStats results.
-		if ( ! ss?$epoch_finished &&
+		if ( ! ss?$epoch_result &&
-		     r$sid in threshold_tracker &&
+			 r$ssname in threshold_tracker &&
-		     key in threshold_tracker[r$sid] &&
 		     ( ss?$threshold &&
-		       threshold_tracker[r$sid][key]$is_threshold_crossed ) ||
+		       key in threshold_tracker[r$ssname] &&
+		       threshold_tracker[r$ssname][key] != 0 ) ||
 		     ( ss?$threshold_series &&
-		       threshold_tracker[r$sid][key]$threshold_series_index+1 == |ss$threshold_series| ) )
+		       key in threshold_tracker[r$ssname] &&
+		       threshold_tracker[r$ssname][key] == |ss$threshold_series| ) )
+			{
 			next;
+			}
 
-		if ( r$sid !in result_store )
+		if ( r$ssname !in result_store )
-			result_store[ss$id] = table();
+			result_store[r$ssname] = table();
-		local results = result_store[r$sid];
+		local results = result_store[r$ssname];
 
 		if ( key !in results )
 			results[key] = table();
@@ -350,10 +449,13 @@ function observe(id: string, key: Key, obs: Observation)
 
 		# If a string was given, fall back to 1.0 as the value.
 		local val = 1.0;
-		if ( obs?$num || obs?$dbl )
+		if ( obs?$num )
-			val = obs?$dbl ? obs$dbl : obs$num;
+			val = obs$num;
+		else if ( obs?$dbl )
+			val = obs$dbl;
 
-		hook observe_hook(r, val, obs, result_val);
+		for ( i in r$calc_funcs )
+			calc_store[r$calc_funcs[i]](r, val, obs, result_val);
 		data_added(ss, key, result);
 		}
 	}
@@ -362,10 +464,12 @@ function observe(id: string, key: Key, obs: Observation)
 # mid-break-interval threshold crossing detection for cluster deployments.
 function check_thresholds(ss: SumStat, key: Key, result: Result, modify_pct: double): bool
 	{
-	if ( ! (ss?$threshold || ss?$threshold_series) )
+	if ( ! (ss?$threshold || ss?$threshold_series || ss?$threshold_crossed) )
 		return F;
 
 	# Add in the extra ResultVals to make threshold_vals easier to write.
+	# This length comparison should work because we just need to make 
+	# sure that we have the same number of reducers and results.
 	if ( |ss$reducers| != |result| )
 		{
 		for ( reducer in ss$reducers )
@@ -378,28 +482,21 @@ function check_thresholds(ss: SumStat, key: Key, result: Result, modify_pct: dou
 	local watch = ss$threshold_val(key, result);
 
 	if ( modify_pct < 1.0 && modify_pct > 0.0 )
-		watch = double_to_count(floor(watch/modify_pct));
+		watch = watch/modify_pct;
 
-	if ( ss$id !in threshold_tracker )
+	local t_index = get_threshold_index(ss$name, key);
-		threshold_tracker[ss$id] = table();
-	local t_tracker = threshold_tracker[ss$id];
 
-	if ( key !in t_tracker )
+	if ( ss?$threshold &&
-		{
+	     t_index == 0 && # Check that the threshold hasn't already been crossed.
-		local ttmp: Thresholding;
+	     watch >= ss$threshold )
-		t_tracker[key] = ttmp;
-		}
-	local tt = t_tracker[key];
-
-	if ( ss?$threshold && ! tt$is_threshold_crossed && watch >= ss$threshold )
 		{
 		# Value crossed the threshold.
 		return T;
 		}
 
 	if ( ss?$threshold_series &&
-	     |ss$threshold_series| >= tt$threshold_series_index &&
+	     |ss$threshold_series| > t_index && # Check if there are more thresholds.
-	     watch >= ss$threshold_series[tt$threshold_series_index] )
+	     watch >= ss$threshold_series[t_index] )
 		{
 		# A threshold series was given and the value crossed the next
 		# value in the series.
@@ -415,6 +512,8 @@ function threshold_crossed(ss: SumStat, key: Key, result: Result)
 	if ( ! ss?$threshold_crossed )
 		return;
 
+	increment_threshold_tracker(ss$name,key);
+
 	# Add in the extra ResultVals to make threshold_crossed callbacks easier to write.
 	if ( |ss$reducers| != |result| )
 		{
@@ -426,11 +525,5 @@ function threshold_crossed(ss: SumStat, key: Key, result: Result)
 		}
 
 	ss$threshold_crossed(key, result);
-	local tt = threshold_tracker[ss$id][key];
-	tt$is_threshold_crossed = T;
-
-	# Bump up to the next threshold series index if a threshold series is being used.
-	if ( ss?$threshold_series )
-		++tt$threshold_series_index;
 	}
 

scripts/base/frameworks/sumstats/non-cluster.bro

@@ -4,11 +4,20 @@ module SumStats;
 
 event SumStats::finish_epoch(ss: SumStat)
 	{
-	if ( ss$id in result_store )
+	if ( ss$name in result_store )
 		{
-		local data = result_store[ss$id];
+		local now = network_time();
+
+		if ( ss?$epoch_result )
+			{
+			local data = result_store[ss$name];
+			# TODO: don't block here.
+			for ( key in data )
+				ss$epoch_result(now, key, data[key]);
+			}
+
 		if ( ss?$epoch_finished )
-			ss$epoch_finished(data);
+			ss$epoch_finished(now);
 
 		reset(ss);
 		}
@@ -16,9 +25,32 @@ event SumStats::finish_epoch(ss: SumStat)
 	schedule ss$epoch { SumStats::finish_epoch(ss) };
 	}
 
-
 function data_added(ss: SumStat, key: Key, result: Result)
 	{
 	if ( check_thresholds(ss, key, result, 1.0) )
 		threshold_crossed(ss, key, result);
 	}
+
+function request(ss_name: string): ResultTable
+	{
+	# This only needs to be implemented this way for cluster compatibility.
+	return when ( T )
+		{
+		if ( ss_name in result_store )
+			return result_store[ss_name];
+		else
+			return table();
+		}
+	}
+
+function request_key(ss_name: string, key: Key): Result
+	{
+	# This only needs to be implemented this way for cluster compatibility.
+	return when ( T )
+		{
+		if ( ss_name in result_store && key in result_store[ss_name] )
+			return result_store[ss_name][key];
+		else
+			return table();
+		}
+	}

scripts/base/frameworks/sumstats/plugins/__load__.bro

@@ -6,5 +6,6 @@
 @load ./sample
 @load ./std-dev
 @load ./sum
+@load ./topk
 @load ./unique
 @load ./variance

scripts/base/frameworks/sumstats/plugins/average.bro

@@ -1,4 +1,4 @@
-@load base/frameworks/sumstats/main
+@load ../main
 
 module SumStats;
 
@@ -14,16 +14,17 @@ export {
 	};
 }
 
-hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+hook register_observe_plugins()
 	{
-	if ( AVERAGE in r$apply )
+	register_observe_plugin(AVERAGE, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 		{
 		if ( ! rv?$average )
 			rv$average = val;
 		else
 			rv$average += (val - rv$average) / rv$num;
+		});
 	}
-	}
+
 
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
 	{

scripts/base/frameworks/sumstats/plugins/hll_unique.bro

@@ -30,23 +30,20 @@ redef record ResultVal += {
 	hll_error_margin: double &optional;
 };
 
-
+hook register_observe_plugins()
-hook init_resultval_hook(r: Reducer, rv: ResultVal)
+	{
+	register_observe_plugin(HLLUNIQUE, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+		{
+		if ( ! rv?$card )
 			{
-	if ( HLLUNIQUE in r$apply && ! rv?$card )
 			rv$card = hll_cardinality_init(r$hll_error_margin);
 			rv$hll_error_margin = r$hll_error_margin;
 			rv$hllunique = 0;
 			}
 
-
-hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
-	{
-	if ( HLLUNIQUE in r$apply )
-		{
 		hll_cardinality_add(rv$card, obs);
 		rv$hllunique = double_to_count(hll_cardinality_estimate(rv$card));		
-		}
+		});
 	}
 
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)

scripts/base/frameworks/sumstats/plugins/last.bro

@@ -33,16 +33,20 @@ function get_last(rv: ResultVal): vector of Observation
 	return s;
 	}
 
-hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+hook register_observe_plugins()
 	{
-	if ( LAST in r$apply && r$num_last_elements > 0 )
+	register_observe_plugin(LAST, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+		{
+		if ( r$num_last_elements > 0 )
 			{
 			if ( ! rv?$last_elements )
 				rv$last_elements = Queue::init([$max_len=r$num_last_elements]);
 			Queue::put(rv$last_elements, obs);
 			}
+		});
 	}
 
+
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
 	{
 	# Merge $samples

scripts/base/frameworks/sumstats/plugins/max.bro

@@ -1,4 +1,4 @@
-@load base/frameworks/sumstats/main
+@load ../main
 
 module SumStats;
 
@@ -14,15 +14,15 @@ export {
 	};
 }
 
-hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+hook register_observe_plugins()
 	{
-	if ( MAX in r$apply )
+	register_observe_plugin(MAX, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 		{
 		if ( ! rv?$max )
 			rv$max = val;
 		else if ( val > rv$max )
 			rv$max = val;
-		}
+		});
 	}
 
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)

scripts/base/frameworks/sumstats/plugins/min.bro

@@ -1,4 +1,4 @@
-@load base/frameworks/sumstats/main
+@load ../main
 
 module SumStats;
 
@@ -14,16 +14,17 @@ export {
 	};
 }
 
-hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+hook register_observe_plugins()
 	{
-	if ( MIN in r$apply )
+	register_observe_plugin(MIN, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 		{
 		if ( ! rv?$min )
 			rv$min = val;
 		else if ( val < rv$min )
 			rv$min = val;
+		});
 	}
-	}
+
 
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
 	{

scripts/base/frameworks/sumstats/plugins/sample.bro

@@ -47,15 +47,14 @@ function sample_add_sample(obs:Observation, rv: ResultVal)
 		if ( ra < rv$num_samples )
 			rv$samples[ra] = obs;
 		}
-
 	}
 
-hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+hook register_observe_plugins()
 	{
-	if ( SAMPLE in r$apply )
+	register_observe_plugin(SAMPLE, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 		{
 		sample_add_sample(obs, rv);
-		}
+		});
 	}
 
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
@@ -75,7 +74,6 @@ hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
 		return;
 		}
 
-
 	if ( |rv1$samples| != num_samples && |rv2$samples| < num_samples )
 		{
 		if ( |rv1$samples| != rv1$sample_elements || |rv2$samples| < rv2$sample_elements )

scripts/base/frameworks/sumstats/plugins/std-dev.bro

@@ -1,5 +1,5 @@
-@load base/frameworks/sumstats/main
 @load ./variance
+@load ../main
 
 module SumStats;
 
@@ -21,13 +21,20 @@ function calc_std_dev(rv: ResultVal)
 		rv$std_dev = sqrt(rv$variance);
 	}
 
-# This depends on the variance plugin which uses priority -5
+hook std_dev_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
-hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) &priority=-10
 	{
-	if ( STD_DEV in r$apply )
 	calc_std_dev(rv);
 	}
 
+hook register_observe_plugins() &priority=-10
+	{
+	register_observe_plugin(STD_DEV, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+		{
+		calc_std_dev(rv);
+		});
+	add_observe_plugin_dependency(STD_DEV, VARIANCE);
+	}
+
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal) &priority=-10
 	{
 	calc_std_dev(result);

scripts/base/frameworks/sumstats/plugins/sum.bro

@@ -1,4 +1,4 @@
-@load base/frameworks/sumstats/main
+@load ../main
 
 module SumStats;
 
@@ -14,19 +14,19 @@ export {
 		sum: double &default=0.0;
 	};
 
-	type threshold_function: function(key: SumStats::Key, result: SumStats::Result): count;
+	#type threshold_function: function(key: SumStats::Key, result: SumStats::Result): count;
-	global sum_threshold: function(data_id: string): threshold_function;
+	#global sum_threshold: function(data_id: string): threshold_function;
 }
 
-function sum_threshold(data_id: string): threshold_function
+#function sum_threshold(data_id: string): threshold_function
-	{
+#	{
-	return function(key: SumStats::Key, result: SumStats::Result): count
+#	return function(key: SumStats::Key, result: SumStats::Result): count
-		{
+#		{
-		print fmt("data_id: %s", data_id);
+#		print fmt("data_id: %s", data_id);
-		print result;
+#		print result;
-		return double_to_count(result[data_id]$sum);
+#		return double_to_count(result[data_id]$sum);
-		};
+#		};
-	}
+#	}
 
 hook init_resultval_hook(r: Reducer, rv: ResultVal)
 	{
@@ -34,10 +34,12 @@ hook init_resultval_hook(r: Reducer, rv: ResultVal)
 		rv$sum = 0;
 	}
 
-hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+hook register_observe_plugins()
+	{
+	register_observe_plugin(SUM, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 		{
-	if ( SUM in r$apply )
 		rv$sum += val;
+		});
 	}
 
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)

scripts/base/frameworks/sumstats/plugins/topk.bro

@@ -0,0 +1,52 @@
+@load base/frameworks/sumstats
+
+module SumStats;
+
+export {
+	redef record Reducer += {
+		## number of elements to keep in the top-k list
+		topk_size: count &default=500;
+	};
+
+	redef enum Calculation += {
+		TOPK
+	};
+
+	redef record ResultVal += {
+		topk: opaque of topk &optional;
+	};
+
+}
+
+hook register_observe_plugins()
+	{
+	register_observe_plugin(TOPK, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+		{
+		topk_add(rv$topk, obs);
+		});
+	}
+
+hook init_resultval_hook(r: Reducer, rv: ResultVal)
+	{
+	if ( TOPK in r$apply && ! rv?$topk )
+		rv$topk = topk_init(r$topk_size);
+	}
+
+hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
+	{
+	if ( rv1?$topk )
+		{
+		result$topk = topk_init(topk_size(rv1$topk));
+
+		topk_merge(result$topk, rv1$topk);
+
+		if ( rv2?$topk )
+			topk_merge(result$topk, rv2$topk);
+		}
+
+	else if ( rv2?$topk )
+		{
+		result$topk = topk_init(topk_size(rv2$topk));
+		topk_merge(result$topk, rv2$topk);
+		}
+	}

scripts/base/frameworks/sumstats/plugins/unique.bro

@@ -1,4 +1,4 @@
-@load base/frameworks/sumstats/main
+@load ../main
 
 module SumStats;
 
@@ -23,15 +23,15 @@ redef record ResultVal += {
 	unique_vals: set[Observation] &optional;
 };
 
-hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+hook register_observe_plugins()
 	{
-	if ( UNIQUE in r$apply )
+	register_observe_plugin(UNIQUE, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 		{
 		if ( ! rv?$unique_vals )
 			rv$unique_vals=set();
 		add rv$unique_vals[obs];
 		rv$unique = |rv$unique_vals|;
-		}
+		});
 	}
 
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)

scripts/base/frameworks/sumstats/plugins/variance.bro

@@ -1,5 +1,5 @@
-@load base/frameworks/sumstats/main
 @load ./average
+@load ../main
 
 module SumStats;
 
@@ -28,17 +28,17 @@ function calc_variance(rv: ResultVal)
 	rv$variance = (rv$num > 1) ? rv$var_s/(rv$num-1) : 0.0;
 	}
 
-# Reduced priority since this depends on the average
+hook register_observe_plugins() &priority=-5
-hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) &priority=-5
 	{
-	if ( VARIANCE in r$apply )
+	register_observe_plugin(VARIANCE, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 		{
 		if ( rv$num > 1 )
 			rv$var_s += ((val - rv$prev_avg) * (val - rv$average));
 
 		calc_variance(rv);
 		rv$prev_avg = rv$average;
-		}
+		});
+	add_observe_plugin_dependency(VARIANCE, AVERAGE);
 	}
 
 # Reduced priority since this depends on the average

scripts/base/init-bare.bro

@@ -531,22 +531,19 @@ type record_field_table: table[string] of record_field;
 # dependent on the names remaining as they are now.
 
 ## Set of BPF capture filters to use for capturing, indexed by a user-definable
-## ID (which must be unique). If Bro is *not* configured to examine
+## ID (which must be unique). If Bro is *not* configured with
-## :bro:id:`PacketFilter::all_packets`, all packets matching at least
+## :bro:id:`PacketFilter::enable_auto_protocol_capture_filters`,
-## one of the filters in this table (and all in :bro:id:`restrict_filters`)
+## all packets matching at least one of the filters in this table (and all in
-## will be analyzed.
+## :bro:id:`restrict_filters`) will be analyzed.
 ##
-## .. bro:see:: PacketFilter PacketFilter::all_packets
+## .. bro:see:: PacketFilter PacketFilter::enable_auto_protocol_capture_filters
 ##    PacketFilter::unrestricted_filter restrict_filters
 global capture_filters: table[string] of string &redef;
 
 ## Set of BPF filters to restrict capturing, indexed by a user-definable ID (which
-## must be unique). If Bro is *not* configured to examine
+## must be unique).
-## :bro:id:`PacketFilter::all_packets`, only packets matching *all* of the
-## filters in this table (and any in :bro:id:`capture_filters`) will be
-## analyzed.
 ##
-## .. bro:see:: PacketFilter PacketFilter::all_packets
+## .. bro:see:: PacketFilter PacketFilter::enable_auto_protocol_capture_filters
 ##    PacketFilter::unrestricted_filter capture_filters
 global restrict_filters: table[string] of string &redef;
 
@@ -3041,6 +3038,11 @@ module GLOBAL;
 ## Number of bytes per packet to capture from live interfaces.
 const snaplen = 8192 &redef;
 
+## Seed for hashes computed internally for probabilistic data structures. Using
+## the same value here will make the hashes compatible between independent Bro
+## instances. If left unset, Bro will use a temporary local seed.
+const global_hash_seed: string = "" &redef;
+
 # Load BiFs defined by plugins.
 @load base/bif/plugins
 

scripts/base/init-default.bro

@@ -39,6 +39,7 @@
 @load base/frameworks/tunnels
 
 @load base/protocols/conn
+@load base/protocols/dhcp
 @load base/protocols/dns
 @load base/protocols/ftp
 @load base/protocols/http

scripts/base/protocols/dhcp/__load__.bro

@@ -0,0 +1,4 @@
+@load ./consts
+@load ./main
+
+@load-sigs ./dpd.sig

scripts/base/protocols/dhcp/consts.bro

@@ -0,0 +1,20 @@
+##! Types, errors, and fields for analyzing DHCP data.  A helper file
+##! for DHCP analysis scripts.
+
+module DHCP;
+
+export {
+
+	## Types of DHCP messages. See RFC 1533.
+	const message_types = {
+		[1] = "DHCP_DISCOVER",
+		[2] = "DHCP_OFFER",
+		[3] = "DHCP_REQUEST",
+		[4] = "DHCP_DECLINE",
+		[5] = "DHCP_ACK",
+		[6] = "DHCP_NAK",
+		[7] = "DHCP_RELEASE",
+		[8] = "DHCP_INFORM",
+	} &default = function(n: count): string { return fmt("unknown-message-type-%d", n); };
+
+}

scripts/base/protocols/dhcp/dpd.sig

@@ -0,0 +1,5 @@
+signature dhcp_cookie {
+  ip-proto == udp
+  payload /^.*\x63\x82\x53\x63/
+  enable "dhcp"
+}

scripts/base/protocols/dhcp/main.bro

@@ -0,0 +1,75 @@
+##! Analyzes DHCP traffic in order to log DHCP leases given to clients.
+##! This script ignores large swaths of the protocol, since it is rather
+##! noisy on most networks, and focuses on the end-result: assigned leases.
+##!
+##! If you'd like to track known DHCP devices and to log the hostname
+##! supplied by the client, see policy/protocols/dhcp/known-devices.bro
+
+@load ./utils.bro
+
+module DHCP;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	## The record type which contains the column fields of the DHCP log.
+	type Info: record {
+		## The earliest time at which a DHCP message over the
+		## associated connection is observed.
+		ts:		time		&log;
+		## A unique identifier of the connection over which DHCP is
+		## occuring.
+		uid:		string		&log;
+		## The connection's 4-tuple of endpoint addresses/ports.
+		id:		conn_id		&log;
+		## Client's hardware address.
+		mac:		string		&log &optional;
+		## Client's actual assigned IP address.
+		assigned_ip:	addr		&log &optional;
+		## IP address lease interval.
+		lease_time:	interval	&log &optional;
+		## A random number choosen by the client for this transaction.
+		trans_id:	count		&log;
+	};
+
+	## Event that can be handled to access the DHCP
+	## record as it is sent on to the logging framework.
+	global log_dhcp: event(rec: Info);
+}
+
+# Add the dhcp info to the connection record
+redef record connection += {
+	dhcp: Info &optional;
+};
+
+# 67/udp is the server's port, 68/udp the client.
+const ports = { 67/udp, 68/udp };
+redef likely_server_ports += { 67/udp };
+
+event bro_init()
+	{
+	Log::create_stream(DHCP::LOG, [$columns=Info, $ev=log_dhcp]);
+	Analyzer::register_for_ports(Analyzer::ANALYZER_DHCP, ports);
+	}
+
+event dhcp_ack(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr, host_name: string)
+	{
+	local info: Info;
+	info$ts          = network_time();
+	info$id          = c$id;
+	info$uid         = c$uid;
+	info$lease_time  = lease;
+	info$trans_id    = msg$xid;
+
+	if ( msg$h_addr != "" )
+		info$mac = msg$h_addr;
+
+	if ( reverse_ip(msg$yiaddr) != 0.0.0.0 )
+		info$assigned_ip = reverse_ip(msg$yiaddr);
+	else
+		info$assigned_ip = c$id$orig_h;
+
+	c$dhcp = info;
+
+	Log::write(DHCP::LOG, c$dhcp);
+	}

scripts/base/protocols/dhcp/utils.bro

@@ -0,0 +1,21 @@
+##! Utilities specific for DHCP processing.
+
+@load ./main
+
+module DHCP;
+
+export {
+	## Reverse the octets of an IPv4 IP.
+	##
+	## ip: An :bro:type:`addr` IPv4 address.
+	##
+	## Returns: A reversed addr.
+	global reverse_ip: function(ip: addr): addr;
+}
+
+function reverse_ip(ip: addr): addr
+	{
+	local octets = split(cat(ip), /\./);
+	return to_addr(cat(octets[4], ".", octets[3], ".", octets[2], ".", octets[1]));
+	}
+

scripts/base/protocols/ftp/__load__.bro

@@ -1,4 +1,5 @@
 @load ./utils-commands
+@load ./info
 @load ./main
 @load ./utils
 @load ./files

scripts/base/protocols/ftp/files.bro

@@ -1,3 +1,4 @@
+@load ./info
 @load ./main
 @load ./utils
 @load base/utils/conn-ids

scripts/base/protocols/ftp/gridftp.bro

@@ -19,6 +19,7 @@
 ##! sizes are not logged, but at the benefit of saving CPU cycles that
 ##! otherwise go to analyzing the large (and likely benign) connections.
 
+@load ./info
 @load ./main
 @load base/protocols/conn
 @load base/protocols/ssl

scripts/base/protocols/ftp/info.bro

@@ -0,0 +1,72 @@
+##! Defines data structures for tracking and logging FTP sessions.
+
+module FTP;
+
+@load ./utils-commands
+
+export {
+
+	## This setting changes if passwords used in FTP sessions are
+	## captured or not.
+	const default_capture_password = F &redef;
+
+	## The expected endpoints of an FTP data channel.
+	type ExpectedDataChannel: record {
+		## Whether PASV mode is toggled for control channel.
+		passive: bool &log;
+		## The host that will be initiating the data connection.
+		orig_h: addr &log;
+		## The host that will be accepting the data connection.
+		resp_h: addr &log;
+		## The port at which the acceptor is listening for the data connection.
+		resp_p: port &log;
+	};
+
+	type Info: record {
+		## Time when the command was sent.
+		ts:               time        &log;
+		## Unique ID for the connection.
+		uid:              string      &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
+		id:               conn_id     &log;
+		## User name for the current FTP session.
+		user:             string      &log &default="<unknown>";
+		## Password for the current FTP session if captured.
+		password:         string      &log &optional;
+		## Command given by the client.
+		command:          string      &log &optional;
+		## Argument for the command if one is given.
+		arg:              string      &log &optional;
+
+		## Libmagic "sniffed" file type if the command indicates a file transfer.
+		mime_type:        string      &log &optional;
+		## Size of the file if the command indicates a file transfer.
+		file_size:        count       &log &optional;
+
+		## Reply code from the server in response to the command.
+		reply_code:       count       &log &optional;
+		## Reply message from the server in response to the command.
+		reply_msg:        string      &log &optional;
+
+		## Expected FTP data channel.
+		data_channel:     ExpectedDataChannel &log &optional;
+
+		## Current working directory that this session is in.  By making
+		## the default value '.', we can indicate that unless something
+		## more concrete is discovered that the existing but unknown
+		## directory is ok to use.
+		cwd:                string  &default=".";
+
+		## Command that is currently waiting for a response.
+		cmdarg:             CmdArg  &optional;
+		## Queue for commands that have been sent but not yet responded to
+		## are tracked here.
+		pending_commands:   PendingCmds;
+
+		## Indicates if the session is in active or passive mode.
+		passive:            bool &default=F;
+
+		## Determines if the password will be captured for this request.
+		capture_password:   bool &default=default_capture_password;
+	};
+}

scripts/base/protocols/ftp/main.bro

@@ -3,6 +3,8 @@
 ##! will take on the full path that the client is at along with the requested
 ##! file name.
 
+@load ./info
+@load ./utils
 @load ./utils-commands
 @load base/utils/paths
 @load base/utils/numbers
@@ -20,72 +22,9 @@ export {
 		"EPSV"
 	} &redef;
 
-	## This setting changes if passwords used in FTP sessions are captured or not.
-	const default_capture_password = F &redef;
-
 	## User IDs that can be considered "anonymous".
 	const guest_ids = { "anonymous", "ftp", "ftpuser", "guest" } &redef;
 
-	## The expected endpoints of an FTP data channel.
-	type ExpectedDataChannel: record {
-		## Whether PASV mode is toggled for control channel.
-		passive: bool &log;
-		## The host that will be initiating the data connection.
-		orig_h: addr &log;
-		## The host that will be accepting the data connection.
-		resp_h: addr &log;
-		## The port at which the acceptor is listening for the data connection.
-		resp_p: port &log;
-	};
-
-	type Info: record {
-		## Time when the command was sent.
-		ts:               time        &log;
-		## Unique ID for the connection.
-		uid:              string      &log;
-		## The connection's 4-tuple of endpoint addresses/ports.
-		id:               conn_id     &log;
-		## User name for the current FTP session.
-		user:             string      &log &default="<unknown>";
-		## Password for the current FTP session if captured.
-		password:         string      &log &optional;
-		## Command given by the client.
-		command:          string      &log &optional;
-		## Argument for the command if one is given.
-		arg:              string      &log &optional;
-
-		## Libmagic "sniffed" file type if the command indicates a file transfer.
-		mime_type:        string      &log &optional;
-		## Size of the file if the command indicates a file transfer.
-		file_size:        count       &log &optional;
-
-		## Reply code from the server in response to the command.
-		reply_code:       count       &log &optional;
-		## Reply message from the server in response to the command.
-		reply_msg:        string      &log &optional;
-
-		## Expected FTP data channel.
-		data_channel:     ExpectedDataChannel &log &optional;
-
-		## Current working directory that this session is in.  By making
-		## the default value '.', we can indicate that unless something
-		## more concrete is discovered that the existing but unknown
-		## directory is ok to use.
-		cwd:                string  &default=".";
-
-		## Command that is currently waiting for a response.
-		cmdarg:             CmdArg  &optional;
-		## Queue for commands that have been sent but not yet responded to
-		## are tracked here.
-		pending_commands:   PendingCmds;
-
-		## Indicates if the session is in active or passive mode.
-		passive:            bool &default=F;
-
-		## Determines if the password will be captured for this request.
-		capture_password:   bool &default=default_capture_password;
-	};
-
 	## This record is to hold a parsed FTP reply code.  For example, for the
 	## 201 status code, the digits would be parsed as: x->2, y->0, z=>1.
 	type ReplyCode: record {
@@ -102,8 +41,6 @@ export {
 	global log_ftp: event(rec: Info);
 }
 
-@load ./utils
-
 # Add the state tracking information variable to the connection record
 redef record connection += {
 	ftp: Info &optional;

scripts/base/protocols/ftp/utils.bro

@@ -1,7 +1,8 @@
 ##! Utilities specific for FTP processing.
 
-@load ./main
+@load ./info
 @load base/utils/addrs
+@load base/utils/paths
 
 module FTP;
 

scripts/base/protocols/ssl/main.bro

@@ -67,11 +67,8 @@ export {
 	## (especially with large file transfers).
 	const disable_analyzer_after_detection = T &redef;
 
-	## The maximum amount of time a script can delay records from being logged.
-	const max_log_delay = 15secs &redef;
-
 	## Delays an SSL record for a specific token: the record will not be logged
-	## as longs the token exists or until :bro:id:`SSL::max_log_delay` elapses.
+	## as longs the token exists or until 15 seconds elapses.
 	global delay_log: function(info: Info, token: string);
 
 	## Undelays an SSL record for a previously inserted token, allowing the
@@ -90,7 +87,7 @@ redef record connection += {
 redef record Info += {
 		# Adding a string "token" to this set will cause the SSL script
 		# to delay logging the record until either the token has been removed or
-		# the record has been delayed for :bro:id:`SSL::max_log_delay`.
+		# the record has been delayed.
 		delay_tokens: set[string] &optional;
 };
 
@@ -138,7 +135,7 @@ function log_record(info: Info)
 			{
 			log_record(info);
 			}
-		timeout SSL::max_log_delay
+		timeout 15secs
 			{
 			Reporter::info(fmt("SSL delay tokens not released in time (%s tokens remaining)",
 			                   |info$delay_tokens|));

scripts/base/utils/dir.bro

@@ -28,7 +28,7 @@ event Dir::monitor_ev(dir: string, last_files: set[string],
                       callback: function(fname: string),
                       poll_interval: interval)
 	{
-	when ( local result = Exec::run([$cmd=fmt("ls -i \"%s/\"", str_shell_escape(dir))]) )
+	when ( local result = Exec::run([$cmd=fmt("ls -i -1 \"%s/\"", str_shell_escape(dir))]) )
 		{
 		if ( result$exit_code != 0 )
 			{

scripts/base/utils/exec.bro

@@ -163,6 +163,7 @@ function run(cmd: Command): Result
 	Input::add_event([$name=cmd$uid,
 	                  $source=fmt("%s |", cmd$cmd),
 	                  $reader=Input::READER_RAW,
+	                  $mode=Input::STREAM,
 	                  $fields=Exec::OneLine,
 	                  $ev=Exec::line,
 	                  $want_record=F,

scripts/policy/frameworks/packet-filter/shunt.bro

@@ -34,8 +34,8 @@ export {
 	global current_shunted_host_pairs: function(): set[conn_id];
 
 	redef enum Notice::Type += {
-		## Indicative that :bro:id:`max_bpf_shunts` connections are already
+		## Indicative that :bro:id:`PacketFilter::max_bpf_shunts` connections
-		## being shunted with BPF filters and no more are allowed.
+		## are already being shunted with BPF filters and no more are allowed.
 		No_More_Conn_Shunts_Available,
 
 		## Limitations in BPF make shunting some connections with BPF impossible.

scripts/policy/misc/app-metrics.bro

@@ -1,109 +0,0 @@
-@load base/protocols/http
-@load base/protocols/ssl
-@load base/frameworks/sumstats
-
-module AppStats;
-
-export {
-	redef enum Log::ID += { LOG };
-
-	type Info: record {
-		## Timestamp when the log line was finished and written.
-		ts:         time   &log;
-		## Time interval that the log line covers.
-		ts_delta:   interval &log;
-		## The name of the "app", like "facebook" or "netflix".
-		app:        string &log;
-		## The number of unique local hosts using the app.
-		uniq_hosts: count  &log;
-		## The number of hits to the app in total.
-		hits:       count  &log;
-		## The total number of bytes received by users of the app.
-		bytes:      count  &log;
-	};
-
-	## The frequency of logging the stats collected by this script.
-	const break_interval = 15mins &redef;
-}
-
-redef record connection += {
-	resp_hostname: string &optional;
-};
-
-event bro_init() &priority=3
-	{
-	Log::create_stream(AppStats::LOG, [$columns=Info]);
-
-	local r1: SumStats::Reducer = [$stream="apps.bytes", $apply=set(SumStats::SUM)];
-	local r2: SumStats::Reducer = [$stream="apps.hits",  $apply=set(SumStats::UNIQUE)];
-	SumStats::create([$epoch=break_interval,
-	                  $reducers=set(r1, r2),
-	                  $epoch_finished(data: SumStats::ResultTable) =
-	                  	{
-	                  	local l: Info;
-	                  	l$ts = network_time();
-	                  	l$ts_delta = break_interval;
-	                  	for ( key in data )
-	                  		{
-	                  		local result = data[key];
-	                  		l$app        = key$str;
-	                  		l$bytes      = double_to_count(floor(result["apps.bytes"]$sum));
-	                  		l$hits       = result["apps.hits"]$num;
-	                  		l$uniq_hosts = result["apps.hits"]$unique;
-	                  		Log::write(LOG, l);
-	                  		}
-	                  	}]);
-	}
-
-function add_sumstats(id: conn_id, hostname: string, size: count)
-	{
-	if ( /\.youtube\.com$/ in hostname && size > 512*1024 )
-		{
-		SumStats::observe("apps.bytes", [$str="youtube"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="youtube"], [$str=cat(id$orig_h)]);
-		}
-	else if ( /(\.facebook\.com|\.fbcdn\.net)$/ in hostname && size > 20 )
-		{
-		SumStats::observe("apps.bytes", [$str="facebook"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="facebook"], [$str=cat(id$orig_h)]);
-		}
-	else if ( /\.google\.com$/ in hostname && size > 20 )
-		{
-		SumStats::observe("apps.bytes", [$str="google"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="google"], [$str=cat(id$orig_h)]);
-		}
-	else if ( /\.nflximg\.com$/ in hostname && size > 200*1024 )
-		{
-		SumStats::observe("apps.bytes", [$str="netflix"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="netflix"], [$str=cat(id$orig_h)]);
-		}
-	else if ( /\.(pandora|p-cdn)\.com$/ in hostname && size > 512*1024 )
-		{
-		SumStats::observe("apps.bytes", [$str="pandora"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="pandora"], [$str=cat(id$orig_h)]);
-		}
-	else if ( /\.gmail\.com$/ in hostname && size > 20 )
-		{
-		SumStats::observe("apps.bytes", [$str="gmail"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="gmail"], [$str=cat(id$orig_h)]);
-		}
-}
-
-
-event ssl_established(c: connection)
-	{
-	if ( c?$ssl && c$ssl?$server_name )
-		c$resp_hostname = c$ssl$server_name;
-	}
-
-event connection_finished(c: connection)
-	{
-	if ( c?$resp_hostname )
-		add_sumstats(c$id, c$resp_hostname, c$resp$size);
-	}
-
-event HTTP::log_http(rec: HTTP::Info)
-	{
-	if( rec?$host )
-		add_sumstats(rec$id, rec$host, rec$response_body_len);
-	}

scripts/policy/misc/app-stats/__load__.bro

@@ -0,0 +1,2 @@
+@load ./main
+@load ./plugins

scripts/policy/misc/app-stats/main.bro

@@ -0,0 +1,77 @@
+#! AppStats collects information about web applications in use
+#! on the network.  
+
+@load base/protocols/http
+@load base/protocols/ssl
+@load base/frameworks/sumstats
+
+module AppStats;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		## Timestamp when the log line was finished and written.
+		ts:         time   &log;
+		## Time interval that the log line covers.
+		ts_delta:   interval &log;
+		## The name of the "app", like "facebook" or "netflix".
+		app:        string &log;
+		## The number of unique local hosts using the app.
+		uniq_hosts: count  &log;
+		## The number of hits to the app in total.
+		hits:       count  &log;
+		## The total number of bytes received by users of the app.
+		bytes:      count  &log;
+	};
+
+	## The frequency of logging the stats collected by this script.
+	const break_interval = 15mins &redef;
+}
+
+redef record connection += {
+	resp_hostname: string &optional;
+};
+
+global add_sumstats: hook(id: conn_id, hostname: string, size: count);
+
+
+event bro_init() &priority=3
+	{
+	Log::create_stream(AppStats::LOG, [$columns=Info]);
+
+	local r1: SumStats::Reducer = [$stream="apps.bytes", $apply=set(SumStats::SUM)];
+	local r2: SumStats::Reducer = [$stream="apps.hits",  $apply=set(SumStats::UNIQUE)];
+	SumStats::create([$name="app-metrics",
+	                  $epoch=break_interval,
+	                  $reducers=set(r1, r2),
+	                  $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	local l: Info;
+	                  	l$ts         = network_time();
+	                  	l$ts_delta   = break_interval;
+	                  	l$app        = key$str;
+	                  	l$bytes      = double_to_count(floor(result["apps.bytes"]$sum));
+	                  	l$hits       = result["apps.hits"]$num;
+	                  	l$uniq_hosts = result["apps.hits"]$unique;
+	                  	Log::write(LOG, l);
+	                  	}]);
+	}
+
+event ssl_established(c: connection)
+	{
+	if ( c?$ssl && c$ssl?$server_name )
+		c$resp_hostname = c$ssl$server_name;
+	}
+
+event connection_finished(c: connection)
+	{
+	if ( c?$resp_hostname )
+		hook add_sumstats(c$id, c$resp_hostname, c$resp$size);
+	}
+
+event HTTP::log_http(rec: HTTP::Info)
+	{
+	if( rec?$host )
+		hook add_sumstats(rec$id, rec$host, rec$response_body_len);
+	}

scripts/policy/misc/app-stats/plugins/__load__.bro

@@ -0,0 +1,6 @@
+@load ./facebook
+@load ./gmail
+@load ./google
+@load ./netflix
+@load ./pandora
+@load ./youtube

scripts/policy/misc/app-stats/plugins/facebook.bro

@@ -0,0 +1,12 @@
+@load ../main
+
+module AppStats;
+
+hook add_sumstats(id: conn_id, hostname: string, size: count)
+	{
+	if ( /\.(facebook\.com|fbcdn\.net)$/ in hostname && size > 20 )
+		{
+		SumStats::observe("apps.bytes", [$str="facebook"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="facebook"], [$str=cat(id$orig_h)]);
+		}
+	}

scripts/policy/misc/app-stats/plugins/gmail.bro

@@ -0,0 +1,12 @@
+@load ../main
+
+module AppStats;
+
+hook add_sumstats(id: conn_id, hostname: string, size: count)
+	{
+	if ( /\.gmail\.com$/ in hostname && size > 20 )
+		{
+		SumStats::observe("apps.bytes", [$str="gmail"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="gmail"], [$str=cat(id$orig_h)]);
+		}
+	}

scripts/policy/misc/app-stats/plugins/google.bro

@@ -0,0 +1,12 @@
+@load ../main
+
+module AppStats;
+
+hook add_sumstats(id: conn_id, hostname: string, size: count)
+	{
+	if ( /\.google\.com$/ in hostname && size > 20 )
+		{
+		SumStats::observe("apps.bytes", [$str="google"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="google"], [$str=cat(id$orig_h)]);
+		}
+	}

scripts/policy/misc/app-stats/plugins/netflix.bro

@@ -0,0 +1,12 @@
+@load ../main
+
+module AppStats;
+
+hook add_sumstats(id: conn_id, hostname: string, size: count)
+	{
+	if ( /\.nflximg\.com$/ in hostname && size > 200*1024 )
+		{
+		SumStats::observe("apps.bytes", [$str="netflix"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="netflix"], [$str=cat(id$orig_h)]);
+		}
+	}

scripts/policy/misc/app-stats/plugins/pandora.bro

@@ -0,0 +1,12 @@
+@load ../main
+
+module AppStats;
+
+hook add_sumstats(id: conn_id, hostname: string, size: count)
+	{
+	if ( /\.(pandora|p-cdn)\.com$/ in hostname && size > 512*1024 )
+		{
+		SumStats::observe("apps.bytes", [$str="pandora"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="pandora"], [$str=cat(id$orig_h)]);
+		}
+	}

scripts/policy/misc/app-stats/plugins/youtube.bro

@@ -0,0 +1,12 @@
+@load ../main
+
+module AppStats;
+
+hook add_sumstats(id: conn_id, hostname: string, size: count)
+	{
+	if ( /\.youtube\.com$/ in hostname && size > 512*1024 )
+		{
+		SumStats::observe("apps.bytes", [$str="youtube"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="youtube"], [$str=cat(id$orig_h)]);
+		}
+	}

scripts/policy/misc/detect-traceroute/main.bro

@@ -29,7 +29,7 @@ export {
 	## Defines the threshold for ICMP Time Exceeded messages for a src-dst pair.
 	## This threshold only comes into play after a host is found to be
 	## sending low ttl packets.
-	const icmp_time_exceeded_threshold = 3 &redef;
+	const icmp_time_exceeded_threshold: double = 3 &redef;
 
 	## Interval at which to watch for the
 	## :bro:id:`Traceroute::icmp_time_exceeded_threshold` variable to be
@@ -57,16 +57,17 @@ event bro_init() &priority=5
 
 	local r1: SumStats::Reducer = [$stream="traceroute.time_exceeded", $apply=set(SumStats::UNIQUE)];
 	local r2: SumStats::Reducer = [$stream="traceroute.low_ttl_packet", $apply=set(SumStats::SUM)];
-	SumStats::create([$epoch=icmp_time_exceeded_interval,
+	SumStats::create([$name="traceroute-detection",
+	                  $epoch=icmp_time_exceeded_interval,
 	                  $reducers=set(r1, r2),
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
 	                  	# Give a threshold value of zero depending on if the host
 	                  	# sends a low ttl packet.
 	                  	if ( require_low_ttl_packets && result["traceroute.low_ttl_packet"]$sum == 0 )
-	                  		return 0;
+	                  		return 0.0;
 	                  	else
-	                  		return result["traceroute.time_exceeded"]$unique;
+	                  		return result["traceroute.time_exceeded"]$unique+0;
 	                  	},
 	                  $threshold=icmp_time_exceeded_threshold,
 	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =

scripts/policy/misc/known-devices.bro

@@ -0,0 +1,41 @@
+##! This script provides infrastructure for logging devices for which Bro has been
+##! able to determine the MAC address, and it logs them once per day (by default).
+##! The log that is output provides an easy way to determine a count of the devices
+##! in use on a network per day.
+##!
+##! ..note::
+##!
+##!     This script will not generate any logs on its own, it needs to be
+##!     supplied with information from elsewhere, such as
+##!     :doc:`policy/protocols/dhcp/known-devices-and-hostnames/scripts/.
+
+module Known;
+
+export {
+	## The known-hosts logging stream identifier.
+	redef enum Log::ID += { DEVICES_LOG };
+
+	## The record type which contains the column fields of the known-devices log.
+	type DevicesInfo: record {
+		## The timestamp at which the host was detected.
+		ts:      time &log;
+		## The MAC address that was detected.
+		mac:     string &log;
+	};
+
+	## The set of all known MAC addresses. It can accessed from other
+        ## to add, and check for, addresses seen in use.
+        ##
+	## We maintain each entry for 24 hours by default so that the existence of
+	## individual addressed is logged each day.
+	global known_devices: set[string] &create_expire=1day &synchronized &redef;
+
+	## An event that can be handled to access the :bro:type:`Known::DevicesInfo`
+	## record as it is sent on to the logging framework.
+	global log_known_devices: event(rec: DevicesInfo);
+}
+
+event bro_init()
+	{
+	Log::create_stream(Known::DEVICES_LOG, [$columns=DevicesInfo, $ev=log_known_devices]);
+	}

scripts/policy/misc/load-balancing.bro

@@ -12,12 +12,12 @@ export {
 		## Apply BPF filters to each worker in a way that causes them to
 		## automatically flow balance traffic between them.
 		AUTO_BPF,
-		## Load balance traffic across the workers by making each one apply
+		# Load balance traffic across the workers by making each one apply
-		## a restrict filter to only listen to a single MAC address.  This
+		# a restrict filter to only listen to a single MAC address.  This
-		## is a somewhat common deployment option for sites doing network
+		# is a somewhat common deployment option for sites doing network
-		## based load balancing with MAC address rewriting and passing the
+		# based load balancing with MAC address rewriting and passing the
-		## traffic to a single interface.  Multiple MAC addresses will show
+		# traffic to a single interface.  Multiple MAC addresses will show
-		## up on the same interface and need filtered to a single address.
+		# up on the same interface and need filtered to a single address.
 		#MAC_ADDR_BPF,
 	};
 

scripts/policy/misc/scan.bro

@@ -40,15 +40,11 @@ export {
 
 	## The threshold of a unique number of hosts a scanning host has to have failed
 	## connections with on a single port.
-	const addr_scan_threshold = 25 &redef;
+	const addr_scan_threshold = 25.0 &redef;
 
 	## The threshold of a number of unique ports a scanning host has to have failed
 	## connections with on a single victim host.
-	const port_scan_threshold = 15 &redef;
+	const port_scan_threshold = 15.0 &redef;
-
-	## Custom thresholds based on service for address scan.  This is primarily
-	## useful for setting reduced thresholds for specific ports.
-	const addr_scan_custom_thresholds: table[port] of count &redef;
 
 	global Scan::addr_scan_policy: hook(scanner: addr, victim: addr, scanned_port: port);
 	global Scan::port_scan_policy: hook(scanner: addr, victim: addr, scanned_port: port);
@@ -57,11 +53,12 @@ export {
 event bro_init() &priority=5
 	{
 	local r1: SumStats::Reducer = [$stream="scan.addr.fail", $apply=set(SumStats::UNIQUE)];
-	SumStats::create([$epoch=addr_scan_interval,
+	SumStats::create([$name="addr-scan",
+	                  $epoch=addr_scan_interval,
 	                  $reducers=set(r1),
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
-	                  	return double_to_count(result["scan.addr.fail"]$unique);
+	                  	return result["scan.addr.fail"]$unique+0.0;
 	                  	},
 	                  #$threshold_func=check_addr_scan_threshold,
 	                  $threshold=addr_scan_threshold,
@@ -81,11 +78,12 @@ event bro_init() &priority=5
 
 	# Note: port scans are tracked similar to: table[src_ip, dst_ip] of set(port);
 	local r2: SumStats::Reducer = [$stream="scan.port.fail", $apply=set(SumStats::UNIQUE)];
-	SumStats::create([$epoch=port_scan_interval,
+	SumStats::create([$name="port-scan",
+	                  $epoch=port_scan_interval,
 	                  $reducers=set(r2),
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
-	                  	return double_to_count(result["scan.port.fail"]$unique);
+	                  	return result["scan.port.fail"]$unique+0.0;
 	                  	},
 	                  $threshold=port_scan_threshold,
 	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =

scripts/policy/protocols/dhcp/known-devices-and-hostnames.bro

@@ -0,0 +1,37 @@
+##! Tracks MAC address with hostnames seen in DHCP traffic. They are logged into
+##! ``devices.log``.
+
+@load policy/misc/known-devices
+
+module Known;
+
+export {
+	redef record DevicesInfo += {
+		## The value of the DHCP host name option, if seen
+		dhcp_host_name:	string		&log &optional;
+	};
+}
+
+event dhcp_request(c: connection, msg: dhcp_msg, req_addr: addr, serv_addr: addr, host_name: string)
+	{
+	if ( msg$h_addr == "" )
+		return;
+
+	if ( msg$h_addr !in known_devices )
+		{
+		add known_devices[msg$h_addr];
+		Log::write(Known::DEVICES_LOG, [$ts=network_time(), $mac=msg$h_addr, $dhcp_host_name=host_name]);
+		}
+	}
+
+event dhcp_inform(c: connection, msg: dhcp_msg, host_name: string)
+	{
+	if ( msg$h_addr == "" )
+		return;
+
+	if ( msg$h_addr !in known_devices )
+		{
+		add known_devices[msg$h_addr];
+		Log::write(Known::DEVICES_LOG, [$ts=network_time(), $mac=msg$h_addr, $dhcp_host_name=host_name]);
+		}
+	}

scripts/policy/protocols/ftp/detect-bruteforcing.bro

@@ -17,7 +17,7 @@ export {
 
 	## How many rejected usernames or passwords are required before being
 	## considered to be bruteforcing.
-	const bruteforce_threshold = 20 &redef;
+	const bruteforce_threshold: double = 20 &redef;
 
 	## The time period in which the threshold needs to be crossed before
 	## being reset.
@@ -28,11 +28,12 @@ export {
 event bro_init()
 	{
 	local r1: SumStats::Reducer = [$stream="ftp.failed_auth", $apply=set(SumStats::UNIQUE)];
-	SumStats::create([$epoch=bruteforce_measurement_interval,
+	SumStats::create([$name="ftp-detect-bruteforcing",
+	                  $epoch=bruteforce_measurement_interval,
 	                  $reducers=set(r1),
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
-	                  	return result["ftp.failed_auth"]$num;
+	                  	return result["ftp.failed_auth"]$num+0.0;
 	                  	},
 	                  $threshold=bruteforce_threshold,
 	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =

scripts/policy/protocols/http/detect-sqli.bro

@@ -28,7 +28,7 @@ export {
 	## Defines the threshold that determines if an SQL injection attack
 	## is ongoing based on the number of requests that appear to be SQL
 	## injection attacks.
-	const sqli_requests_threshold = 50 &redef;
+	const sqli_requests_threshold: double = 50.0 &redef;
 
 	## Interval at which to watch for the
 	## :bro:id:`HTTP::sqli_requests_threshold` variable to be crossed.
@@ -64,11 +64,12 @@ event bro_init() &priority=3
 	# determine when it looks like an actual attack and how to respond when
 	# thresholds are crossed.
 	local r1: SumStats::Reducer = [$stream="http.sqli.attacker", $apply=set(SumStats::SUM, SumStats::SAMPLE), $num_samples=collect_SQLi_samples];
-	SumStats::create([$epoch=sqli_requests_interval,
+	SumStats::create([$name="detect-sqli-attackers",
+	                  $epoch=sqli_requests_interval,
 	                  $reducers=set(r1),
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
-	                  	return double_to_count(result["http.sqli.attacker"]$sum);
+	                  	return result["http.sqli.attacker"]$sum;
 	                  	},
 	                  $threshold=sqli_requests_threshold,
 	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
@@ -82,11 +83,12 @@ event bro_init() &priority=3
 	                  	}]);
 
 	local r2: SumStats::Reducer = [$stream="http.sqli.victim", $apply=set(SumStats::SUM, SumStats::SAMPLE), $num_samples=collect_SQLi_samples];
-	SumStats::create([$epoch=sqli_requests_interval,
+	SumStats::create([$name="detect-sqli-victims",
+	                  $epoch=sqli_requests_interval,
 	                  $reducers=set(r2),
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
-	                  	return double_to_count(result["http.sqli.victim"]$sum);
+	                  	return result["http.sqli.victim"]$sum;
 	                  	},
 	                  $threshold=sqli_requests_threshold,
 	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =

scripts/policy/protocols/ssh/detect-bruteforcing.bro

@@ -27,7 +27,7 @@ export {
 
 	## The number of failed SSH connections before a host is designated as
 	## guessing passwords.
-	const password_guesses_limit = 30 &redef;
+	const password_guesses_limit: double = 30 &redef;
 
 	## The amount of time to remember presumed non-successful logins to build
 	## model of a password guesser.
@@ -42,20 +42,29 @@ export {
 
 event bro_init()
 	{
-	local r1: SumStats::Reducer = [$stream="ssh.login.failure", $apply=set(SumStats::SUM)];
+	local r1: SumStats::Reducer = [$stream="ssh.login.failure", $apply=set(SumStats::SUM, SumStats::SAMPLE), $num_samples=5];
-	SumStats::create([$epoch=guessing_timeout,
+	SumStats::create([$name="detect-ssh-bruteforcing",
+	                  $epoch=guessing_timeout,
 	                  $reducers=set(r1),
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
-	                  	return double_to_count(result["ssh.login.failure"]$sum);
+	                  	return result["ssh.login.failure"]$sum;
 	                  	},
 	                  $threshold=password_guesses_limit,
 	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
 	                  	local r = result["ssh.login.failure"];
+	                  	local sub_msg = fmt("Sampled servers: ");
+	                  	local samples = r$samples;
+	                  	for ( i in samples )
+	                  		{
+	                  		if ( samples[i]?$str )
+	                  			sub_msg = fmt("%s%s %s", sub_msg, i==0 ? "":",", samples[i]$str);
+	                  		}
 	                  	# Generate the notice.
 	                  	NOTICE([$note=Password_Guessing,
 	                  	        $msg=fmt("%s appears to be guessing SSH passwords (seen in %d connections).", key$host, r$num),
+	                  	        $sub=sub_msg,
 	                  	        $src=key$host,
 	                  	        $identifier=cat(key$host)]);
 	                  	}]);
@@ -78,5 +87,5 @@ event SSH::heuristic_failed_login(c: connection)
 	# be ignored.
 	if ( ! (id$orig_h in ignore_guessers &&
 	        id$resp_h in ignore_guessers[id$orig_h]) )
-		SumStats::observe("ssh.login.failure", [$host=id$orig_h], [$num=1]);
+		SumStats::observe("ssh.login.failure", [$host=id$orig_h], [$str=cat(id$resp_h)]);
 	}

scripts/policy/tuning/defaults/packet-fragments.bro

@@ -1,10 +1,10 @@
-## Capture TCP fragments, but not UDP (or ICMP), since those are a lot more
+# Capture TCP fragments, but not UDP (or ICMP), since those are a lot more
-## common due to high-volume, fragmenting protocols such as NFS :-(.
+# common due to high-volume, fragmenting protocols such as NFS :-(.
 
-## This normally isn't used because of the default open packet filter 
+# This normally isn't used because of the default open packet filter 
-## but we set it anyway in case the user is using a packet filter.
+# but we set it anyway in case the user is using a packet filter.
-## Note: This was removed because the default model now is to have a wide
+# Note: This was removed because the default model now is to have a wide
-##       open packet filter.
+#       open packet filter.
 #redef capture_filters += { ["frag"] = "(ip[6:2] & 0x3fff != 0) and tcp" };
 
 ## Shorten the fragment timeout from never expiring to expiring fragments after

scripts/site/local.bro

@@ -11,6 +11,13 @@
 # Load the scan detection script.
 @load misc/scan
 
+# Log some information about web applications being used by users 
+# on your network.
+@load misc/app-stats
+
+# Detect traceroute being run on the network.  
+@load misc/detect-traceroute
+
 # Generate notices when vulnerable versions of software are discovered.
 # The default is to only monitor software found in the address space defined
 # as "local".  Refer to the software framework's documentation for more 

scripts/test-all-policy.bro

@@ -35,10 +35,19 @@
 @load integration/barnyard2/types.bro
 @load integration/collective-intel/__load__.bro
 @load integration/collective-intel/main.bro
-@load misc/app-metrics.bro
+@load misc/app-stats/__load__.bro
+@load misc/app-stats/main.bro
+@load misc/app-stats/plugins/__load__.bro
+@load misc/app-stats/plugins/facebook.bro
+@load misc/app-stats/plugins/gmail.bro
+@load misc/app-stats/plugins/google.bro
+@load misc/app-stats/plugins/netflix.bro
+@load misc/app-stats/plugins/pandora.bro
+@load misc/app-stats/plugins/youtube.bro
 @load misc/capture-loss.bro
 @load misc/detect-traceroute/__load__.bro
 @load misc/detect-traceroute/main.bro
+@load misc/known-devices.bro
 @load misc/load-balancing.bro
 @load misc/loaded-scripts.bro
 @load misc/profiling.bro
@@ -48,6 +57,7 @@
 @load protocols/conn/known-hosts.bro
 @load protocols/conn/known-services.bro
 @load protocols/conn/weirds.bro
+@load protocols/dhcp/known-devices-and-hostnames.bro
 @load protocols/dns/auth-addl.bro
 @load protocols/dns/detect-external-names.bro
 @load protocols/ftp/detect-bruteforcing.bro

src/BroDoc.cc

@@ -11,6 +11,7 @@
 #include "plugin/Manager.h"
 #include "analyzer/Manager.h"
 #include "analyzer/Component.h"
+#include "file_analysis/Manager.h"
 
 BroDoc::BroDoc(const std::string& rel, const std::string& abs)
 	{
@@ -479,6 +480,17 @@ static void WriteAnalyzerComponent(FILE* f, const analyzer::Component* c)
 	fprintf(f, ":bro:enum:`Analyzer::%s`\n\n", tag.c_str());
 	}
 
+static void WriteAnalyzerComponent(FILE* f, const file_analysis::Component* c)
+	{
+	EnumType* atag = file_mgr->GetTagEnumType();
+	string tag = fmt("ANALYZER_%s", c->CanonicalName());
+
+	if ( atag->Lookup("Files", tag.c_str()) < 0 )
+		reporter->InternalError("missing analyzer tag for %s", tag.c_str());
+
+	fprintf(f, ":bro:enum:`Files::%s`\n\n", tag.c_str());
+	}
+
 static void WritePluginComponents(FILE* f, const plugin::Plugin* p)
 	{
 	plugin::Plugin::component_list components = p->Components();
@@ -494,6 +506,10 @@ static void WritePluginComponents(FILE* f, const plugin::Plugin* p)
 			WriteAnalyzerComponent(f,
 			        dynamic_cast<const analyzer::Component*>(*it));
 			break;
+		case plugin::component::FILE_ANALYZER:
+			WriteAnalyzerComponent(f,
+			        dynamic_cast<const file_analysis::Component*>(*it));
+			break;
 		case plugin::component::READER:
 			reporter->InternalError("docs for READER component unimplemented");
 		case plugin::component::WRITER:
@@ -537,30 +553,35 @@ static void WritePluginBifItems(FILE* f, const plugin::Plugin* p,
 		}
 	}
 
-static void WriteAnalyzerTagDefn(FILE* f, EnumType* e)
+static void WriteAnalyzerTagDefn(FILE* f, EnumType* e, const string& module)
 	{
+	string tag_id= module + "::Tag";
 	e = new CommentedEnumType(e);
-	e->SetTypeID(copy_string("Analyzer::Tag"));
+	e->SetTypeID(copy_string(tag_id.c_str()));
 
-	ID* dummy_id = new ID(copy_string("Analyzer::Tag"), SCOPE_GLOBAL, true);
+	ID* dummy_id = new ID(copy_string(tag_id.c_str()), SCOPE_GLOBAL, true);
 	dummy_id->SetType(e);
 	dummy_id->MakeType();
 
 	list<string>* r = new list<string>();
-	r->push_back("Unique identifiers for protocol analyzers.");
+	r->push_back("Unique identifiers for analyzers.");
 
 	BroDocObj bdo(dummy_id, r, true);
 
 	bdo.WriteReST(f);
 	}
 
-static bool IsAnalyzerPlugin(const plugin::Plugin* p)
+static bool ComponentsMatch(const plugin::Plugin* p, plugin::component::Type t,
+                            bool match_empty = false)
 	{
 	plugin::Plugin::component_list components = p->Components();
 	plugin::Plugin::component_list::const_iterator it;
 
+	if ( components.empty() )
+		return match_empty;
+
 	for ( it = components.begin(); it != components.end(); ++it )
-		if ( (*it)->Type() != plugin::component::ANALYZER )
+		if ( (*it)->Type() != t )
 			return false;
 
 	return true;
@@ -573,14 +594,44 @@ void CreateProtoAnalyzerDoc(const char* filename)
 	fprintf(f, "Protocol Analyzer Reference\n");
 	fprintf(f, "===========================\n\n");
 
-	WriteAnalyzerTagDefn(f, analyzer_mgr->GetTagEnumType());
+	WriteAnalyzerTagDefn(f, analyzer_mgr->GetTagEnumType(), "Analyzer");
 
 	plugin::Manager::plugin_list plugins = plugin_mgr->Plugins();
 	plugin::Manager::plugin_list::const_iterator it;
 
 	for ( it = plugins.begin(); it != plugins.end(); ++it )
 		{
-		if ( ! IsAnalyzerPlugin(*it) )
+		if ( ! ComponentsMatch(*it, plugin::component::ANALYZER, true) )
+			continue;
+
+		WritePluginSectionHeading(f, *it);
+		WritePluginComponents(f, *it);
+		WritePluginBifItems(f, *it, plugin::BifItem::CONSTANT,
+		                    "Options/Constants");
+		WritePluginBifItems(f, *it, plugin::BifItem::GLOBAL, "Globals");
+		WritePluginBifItems(f, *it, plugin::BifItem::TYPE, "Types");
+		WritePluginBifItems(f, *it, plugin::BifItem::EVENT, "Events");
+		WritePluginBifItems(f, *it, plugin::BifItem::FUNCTION, "Functions");
+		}
+
+	fclose(f);
+	}
+
+void CreateFileAnalyzerDoc(const char* filename)
+	{
+	FILE* f = fopen(filename, "w");
+
+	fprintf(f, "File Analyzer Reference\n");
+	fprintf(f, "=======================\n\n");
+
+	WriteAnalyzerTagDefn(f, file_mgr->GetTagEnumType(), "Files");
+
+	plugin::Manager::plugin_list plugins = plugin_mgr->Plugins();
+	plugin::Manager::plugin_list::const_iterator it;
+
+	for ( it = plugins.begin(); it != plugins.end(); ++it )
+		{
+		if ( ! ComponentsMatch(*it, plugin::component::FILE_ANALYZER) )
 			continue;
 
 		WritePluginSectionHeading(f, *it);

src/BroDoc.h

@@ -413,4 +413,10 @@ private:
  */
 void CreateProtoAnalyzerDoc(const char* filename);
 
+/**
+ * Writes out plugin index documentation for all file analyzer plugins.
+ * @param filename the name of the file to write.
+ */
+void CreateFileAnalyzerDoc(const char* filename);
+
 #endif

src/CMakeLists.txt

@@ -319,6 +319,7 @@ set(bro_SRCS
     StateAccess.cc
     Stats.cc
     Stmt.cc
+    Tag.cc
     Timer.cc
     Traverse.cc
     Trigger.cc
@@ -362,6 +363,8 @@ set(bro_SRCS
     3rdparty/sqlite3.c
 
     plugin/Component.cc
+    plugin/ComponentManager.h
+    plugin/TaggedComponent.h
     plugin/Manager.cc
     plugin/Plugin.cc
     plugin/Macros.h

src/DebugLogger.cc

@@ -16,7 +16,8 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = {
 	{ "notifiers", 0, false },  { "main-loop", 0, false },
 	{ "dpd", 0, false }, { "tm", 0, false },
 	{ "logging", 0, false }, {"input", 0, false }, 
-	{ "threading", 0, false }, { "file_analysis", 0, false }
+	{ "threading", 0, false }, { "file_analysis", 0, false },
+	{ "plugins", 0, false}
 };
 
 DebugLogger::DebugLogger(const char* filename)

src/DebugLogger.h

@@ -27,6 +27,7 @@ enum DebugStream {
 	DBG_INPUT,	// Input streams
 	DBG_THREADING,	// Threading system
 	DBG_FILE_ANALYSIS,	// File analysis
+	DBG_PLUGINS,
 
 	NUM_DBGS // Has to be last
 };

src/NetVar.cc

@@ -238,11 +238,14 @@ TableType* record_field_table;
 
 StringVal* cmd_line_bpf_filter;
 
+StringVal* global_hash_seed;
+
 OpaqueType* md5_type;
 OpaqueType* sha1_type;
 OpaqueType* sha256_type;
 OpaqueType* entropy_type;
 OpaqueType* cardinality_type;
+OpaqueType* topk_type;
 OpaqueType* bloomfilter_type;
 
 #include "const.bif.netvar_def"
@@ -305,11 +308,14 @@ void init_general_global_var()
 	cmd_line_bpf_filter =
 		internal_val("cmd_line_bpf_filter")->AsStringVal();
 
+	global_hash_seed = opt_internal_string("global_hash_seed");
+
 	md5_type = new OpaqueType("md5");
 	sha1_type = new OpaqueType("sha1");
 	sha256_type = new OpaqueType("sha256");
 	entropy_type = new OpaqueType("entropy");
 	cardinality_type = new OpaqueType("cardinality");
+	topk_type = new OpaqueType("topk");
 	bloomfilter_type = new OpaqueType("bloomfilter");
 	}
 

src/NetVar.h

@@ -242,12 +242,15 @@ extern TableType* record_field_table;
 
 extern StringVal* cmd_line_bpf_filter;
 
+extern StringVal* global_hash_seed;
+
 class OpaqueType;
 extern OpaqueType* md5_type;
 extern OpaqueType* sha1_type;
 extern OpaqueType* sha256_type;
 extern OpaqueType* entropy_type;
 extern OpaqueType* cardinality_type;
+extern OpaqueType* topk_type;
 extern OpaqueType* bloomfilter_type;
 
 // Initializes globals that don't pertain to network/event analysis.

src/OpaqueVal.cc

@@ -569,14 +569,14 @@ BroType* BloomFilterVal::Type() const
 void BloomFilterVal::Add(const Val* val)
 	{
 	HashKey* key = hash->ComputeHash(val, 1);
-	bloom_filter->Add(key->Hash());
+	bloom_filter->Add(key);
 	delete key;
 	}
 
 size_t BloomFilterVal::Count(const Val* val) const
 	{
 	HashKey* key = hash->ComputeHash(val, 1);
-	size_t cnt = bloom_filter->Count(key->Hash());
+	size_t cnt = bloom_filter->Count(key);
 	delete key;
 	return cnt;
 	}
@@ -591,10 +591,17 @@ bool BloomFilterVal::Empty() const
 	return bloom_filter->Empty();
 	}
 
+string BloomFilterVal::InternalState() const
+	{
+	return bloom_filter->InternalState();
+	}
+
 BloomFilterVal* BloomFilterVal::Merge(const BloomFilterVal* x,
 				      const BloomFilterVal* y)
 	{
-	if ( ! same_type(x->Type(), y->Type()) )
+	if ( x->Type() && // any one 0 is ok here
+	     y->Type() &&
+	     ! same_type(x->Type(), y->Type()) )
 		{
 		reporter->Error("cannot merge Bloom filters with different types");
 		return 0;
@@ -616,7 +623,7 @@ BloomFilterVal* BloomFilterVal::Merge(const BloomFilterVal* x,
 
 	BloomFilterVal* merged = new BloomFilterVal(copy);
 
-	if ( ! merged->Typify(x->Type()) )
+	if ( x->Type() && ! merged->Typify(x->Type()) )
 		{
 		reporter->Error("failed to set type on merged Bloom filter");
 		return 0;

src/OpaqueVal.h

@@ -127,6 +127,7 @@ public:
 	size_t Count(const Val* val) const;
 	void Clear();
 	bool Empty() const;
+	string InternalState() const;
 
 	static BloomFilterVal* Merge(const BloomFilterVal* x,
 				     const BloomFilterVal* y);

src/RuleAction.cc

@@ -40,7 +40,7 @@ RuleActionAnalyzer::RuleActionAnalyzer(const char* arg_analyzer)
 	string str(arg_analyzer);
 	string::size_type pos = str.find(':');
 	string arg = str.substr(0, pos);
-	analyzer = analyzer_mgr->GetAnalyzerTag(arg.c_str());
+	analyzer = analyzer_mgr->GetComponentTag(arg.c_str());
 
 	if ( ! analyzer )
 		reporter->Warning("unknown analyzer '%s' specified in rule", arg.c_str());
@@ -48,7 +48,7 @@ RuleActionAnalyzer::RuleActionAnalyzer(const char* arg_analyzer)
 	if ( pos != string::npos )
 		{
 		arg = str.substr(pos + 1);
-		child_analyzer = analyzer_mgr->GetAnalyzerTag(arg.c_str());
+		child_analyzer = analyzer_mgr->GetComponentTag(arg.c_str());
 
 		if ( ! child_analyzer )
 			reporter->Warning("unknown analyzer '%s' specified in rule", arg.c_str());
@@ -60,11 +60,11 @@ RuleActionAnalyzer::RuleActionAnalyzer(const char* arg_analyzer)
 void RuleActionAnalyzer::PrintDebug()
 	{
 	if ( ! child_analyzer )
-		fprintf(stderr, "|%s|\n", analyzer_mgr->GetAnalyzerName(analyzer));
+		fprintf(stderr, "|%s|\n", analyzer_mgr->GetComponentName(analyzer));
 	else
 		fprintf(stderr, "|%s:%s|\n",
-			analyzer_mgr->GetAnalyzerName(analyzer),
+			analyzer_mgr->GetComponentName(analyzer),
-			analyzer_mgr->GetAnalyzerName(child_analyzer));
+			analyzer_mgr->GetComponentName(child_analyzer));
 	}
 
 

src/SerialTypes.h

@@ -108,8 +108,9 @@ SERIAL_VAL(MD5_VAL, 16)
 SERIAL_VAL(SHA1_VAL, 17)
 SERIAL_VAL(SHA256_VAL, 18)
 SERIAL_VAL(ENTROPY_VAL, 19)
-SERIAL_VAL(BLOOMFILTER_VAL, 20)
+SERIAL_VAL(TOPK_VAL, 20)
-SERIAL_VAL(CARDINALITY_VAL, 21)
+SERIAL_VAL(BLOOMFILTER_VAL, 21)
+SERIAL_VAL(CARDINALITY_VAL, 22)
 
 #define SERIAL_EXPR(name, val) SERIAL_CONST(name, val, EXPR)
 SERIAL_EXPR(EXPR, 1)

src/Tag.cc

@@ -0,0 +1,82 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "Tag.h"
+#include "Val.h"
+
+Tag::Tag(EnumType* etype, type_t arg_type, subtype_t arg_subtype)
+	{
+	assert(arg_type > 0);
+
+	type = arg_type;
+	subtype = arg_subtype;
+	int64_t i = (int64)(type) | ((int64)subtype << 31);
+	Ref(etype);
+	val = new EnumVal(i, etype);
+	}
+
+Tag::Tag(EnumVal* arg_val)
+	{
+	assert(arg_val);
+
+	val = arg_val;
+	Ref(val);
+
+	int64 i = val->InternalInt();
+	type = i & 0xffffffff;
+	subtype = (i >> 31) & 0xffffffff;
+	}
+
+Tag::Tag(const Tag& other)
+	{
+	type = other.type;
+	subtype = other.subtype;
+	val = other.val;
+
+	if ( val )
+		Ref(val);
+	}
+
+Tag::Tag()
+	{
+	type = 0;
+	subtype = 0;
+	val = 0;
+	}
+
+Tag::~Tag()
+	{
+	Unref(val);
+	val = 0;
+	}
+
+Tag& Tag::operator=(const Tag& other)
+	{
+	if ( this != &other )
+		{
+		type = other.type;
+		subtype = other.subtype;
+		val = other.val;
+
+		if ( val )
+			Ref(val);
+		}
+
+	return *this;
+	}
+
+EnumVal* Tag::AsEnumVal(EnumType* etype) const
+	{
+	if ( ! val )
+		{
+		assert(type == 0 && subtype == 0);
+		Ref(etype);
+		val = new EnumVal(0, etype);
+		}
+
+	return val;
+	}
+
+std::string Tag::AsString() const
+	{
+	return fmt("%" PRIu32 "/%" PRIu32, type, subtype);
+	}

src/Tag.h

@@ -0,0 +1,138 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef TAG_H
+#define TAG_H
+
+#include "config.h"
+#include "util.h"
+#include "Type.h"
+
+class EnumVal;
+
+/**
+ * Class to identify an analyzer type.
+ *
+ * Each analyzer type gets a tag consisting of a main type and subtype. The
+ * former is an identifier that's unique across all analyzer classes. The latter is
+ * passed through to the analyzer instances for their use, yet not further
+ * interpreted by the analyzer infrastructure; it allows an analyzer to
+ * branch out into a set of sub-analyzers internally. Jointly, main type and
+ * subtype form an analyzer "tag". Each unique tag corresponds to a single
+ * "analyzer" from the user's perspective. At the script layer, these tags
+ * are mapped into enums of type \c Analyzer::Tag or Files::Tag. Internally,
+ * the analyzer::Manager and file_analysis::Manager maintain the mapping of tag
+ * to analyzer (and it also assigns them their main types), and
+ * analyzer::Component and file_analysis::Component create new tag.
+ *
+ * The Tag class supports all operations necessary to act as an index in a
+ * \c std::map.
+ */
+class Tag  {
+public:
+	/**
+	 * Type for the analyzer's main type.
+	 */
+	typedef uint32 type_t;
+
+	/**
+	 * Type for the analyzer's subtype.
+	 */
+	typedef uint32 subtype_t;
+
+	/**
+	 * Returns the tag's main type.
+	 */
+	type_t Type() const 	{ return type; }
+
+	/**
+	 * Returns the tag's subtype.
+	 */
+	subtype_t Subtype() const 	{ return subtype; }
+
+	/**
+	 * Returns the numerical values for main and subtype inside a string
+	 * suitable for printing. This is primarily for debugging.
+	 */
+	std::string AsString() const;
+
+protected:
+	/*
+	 * Copy constructor.
+	 */
+	Tag(const Tag& other);
+
+	/**
+	 * Default constructor. This initializes the tag with an error value
+	 * that will make \c operator \c bool return false.
+	 */
+	Tag();
+
+	/**
+	 * Destructor.
+	 */
+	~Tag();
+
+	/**
+	 * Assignment operator.
+	 */
+	Tag& operator=(const Tag& other);
+
+	/**
+	 * Compares two tags for equality.
+	 */
+	bool operator==(const Tag& other) const
+		{
+		return type == other.type && subtype == other.subtype;
+		}
+
+	/**
+	 * Compares two tags for inequality.
+	 */
+	bool operator!=(const Tag& other) const
+		{
+		return type != other.type || subtype != other.subtype;
+		}
+
+	/**
+	 * Compares two tags for less-than relationship.
+	 */
+	bool operator<(const Tag& other) const
+		{
+		return type != other.type ? type < other.type : (subtype < other.subtype);
+		}
+
+	/**
+	 * Returns the script-layer enum that corresponds to this tag.
+	 * The returned value does not have its ref-count increased.
+	 *
+	 * @param etype the script-layer enum type associated with the tag.
+	 */
+	EnumVal* AsEnumVal(EnumType* etype) const;
+
+	/**
+	 * Constructor.
+	 *
+	 * @param etype the script-layer enum type associated with the tag.
+	 *
+	 * @param type The main type. Note that the manager class manages the
+	 * the value space internally, so noone else should assign main types.
+	 *
+	 * @param subtype The sub type, which is left to an analyzer for
+	 * interpretation. By default it's set to zero.
+	 */
+	Tag(EnumType* etype, type_t type, subtype_t subtype = 0);
+
+	/**
+	 * Constructor.
+	 *
+	 * @param val An enum value of script type \c Analyzer::Tag.
+	 */
+	Tag(EnumVal* val);
+
+private:
+	type_t type;            // Main type.
+	subtype_t subtype;      // Subtype.
+	mutable EnumVal* val;   // Script-layer value.
+};
+
+#endif

src/analyzer/Analyzer.cc

@@ -70,12 +70,12 @@ void AnalyzerTimer::Init(Analyzer* arg_analyzer, analyzer_timer_func arg_timer,
 	Ref(analyzer->Conn());
 	}
 
-analyzer::ID Analyzer::id_counter = 0;;
+analyzer::ID Analyzer::id_counter = 0;
 
 const char* Analyzer::GetAnalyzerName() const
 	{
 	assert(tag);
-	return analyzer_mgr->GetAnalyzerName(tag);
+	return analyzer_mgr->GetComponentName(tag);
 	}
 
 void Analyzer::SetAnalyzerTag(const Tag& arg_tag)
@@ -87,7 +87,7 @@ void Analyzer::SetAnalyzerTag(const Tag& arg_tag)
 bool Analyzer::IsAnalyzer(const char* name)
 	{
 	assert(tag);
-	return strcmp(analyzer_mgr->GetAnalyzerName(tag), name) == 0;
+	return strcmp(analyzer_mgr->GetComponentName(tag), name) == 0;
 	}
 
 // Used in debugging output.
@@ -98,7 +98,7 @@ static string fmt_analyzer(Analyzer* a)
 
 Analyzer::Analyzer(const char* name, Connection* conn)
 	{
-	Tag tag = analyzer_mgr->GetAnalyzerTag(name);
+	Tag tag = analyzer_mgr->GetComponentTag(name);
 
 	if ( ! tag )
 		reporter->InternalError("unknown analyzer name %s; mismatch with tag analyzer::Component?", name);
@@ -494,7 +494,7 @@ Analyzer* Analyzer::FindChild(Tag arg_tag)
 
 Analyzer* Analyzer::FindChild(const char* name)
 	{
-	Tag tag = analyzer_mgr->GetAnalyzerTag(name);
+	Tag tag = analyzer_mgr->GetComponentTag(name);
 	return tag ? FindChild(tag) : 0;
 	}
 

src/analyzer/Component.cc

@@ -8,29 +8,26 @@
 
 using namespace analyzer;
 
-Tag::type_t Component::type_counter = 0;
-
 Component::Component(const char* arg_name, factory_callback arg_factory, Tag::subtype_t arg_subtype, bool arg_enabled, bool arg_partial)
-	: plugin::Component(plugin::component::ANALYZER)
+	: plugin::Component(plugin::component::ANALYZER),
+	  plugin::TaggedComponent<analyzer::Tag>(arg_subtype)
 	{
 	name = copy_string(arg_name);
 	canon_name = canonify_name(arg_name);
 	factory = arg_factory;
 	enabled = arg_enabled;
 	partial = arg_partial;
-
-	tag = analyzer::Tag(++type_counter, arg_subtype);
 	}
 
 Component::Component(const Component& other)
-	: plugin::Component(Type())
+	: plugin::Component(Type()),
+	  plugin::TaggedComponent<analyzer::Tag>(other)
 	{
 	name = copy_string(other.name);
 	canon_name = copy_string(other.canon_name);
 	factory = other.factory;
 	enabled = other.enabled;
 	partial = other.partial;
-	tag = other.tag;
 	}
 
 Component::~Component()
@@ -39,11 +36,6 @@ Component::~Component()
 	delete [] canon_name;
 	}
 
-analyzer::Tag Component::Tag() const
-	{
-	return tag;
-	}
-
 void Component::Describe(ODesc* d) const
 	{
 	plugin::Component::Describe(d);
@@ -63,13 +55,14 @@ void Component::Describe(ODesc* d) const
 
 Component& Component::operator=(const Component& other)
 	{
+	plugin::TaggedComponent<analyzer::Tag>::operator=(other);
+
 	if ( &other != this )
 		{
 		name = copy_string(other.name);
 		factory = other.factory;
 		enabled = other.enabled;
 		partial = other.partial;
-		tag = other.tag;
 		}
 
 	return *this;

src/analyzer/Component.h

@@ -5,6 +5,7 @@
 
 #include "Tag.h"
 #include "plugin/Component.h"
+#include "plugin/TaggedComponent.h"
 
 #include "../config.h"
 #include "../util.h"
@@ -21,7 +22,8 @@ class Analyzer;
  * A plugin can provide a specific protocol analyzer by registering this
  * analyzer component, describing the analyzer.
  */
-class Component : public plugin::Component {
+class Component : public plugin::Component,
+                  public plugin::TaggedComponent<analyzer::Tag> {
 public:
 	typedef Analyzer* (*factory_callback)(Connection* conn);
 
@@ -100,13 +102,6 @@ public:
 	 */
 	bool Enabled() const	{ return enabled; }
 
-	/**
-	 * Returns the analyzer's tag. Note that this is automatically
-	 * generated for each new Components, and hence unique across all of
-	 * them.
-	 */
-	analyzer::Tag Tag() const;
-
 	/**
 	 * Enables or disables this analyzer.
 	 *
@@ -128,11 +123,7 @@ private:
 	const char* canon_name;	// The analyzer's canonical name.
 	factory_callback factory;	// The analyzer's factory callback.
 	bool partial;	// True if the analyzer supports partial connections.
-	analyzer::Tag tag;	// The automatically assigned analyzer tag.
 	bool enabled;	// True if the analyzer is enabled.
-
-	// Global counter used to generate unique tags.
-	static analyzer::Tag::type_t type_counter;
 };
 
 }

src/analyzer/Manager.cc

@@ -60,10 +60,8 @@ bool Manager::ConnIndex::operator<(const ConnIndex& other) const
 	}
 
 Manager::Manager()
+	: plugin::ComponentManager<analyzer::Tag, analyzer::Component>("Analyzer")
 	{
-	tag_enum_type = new EnumType("Analyzer::Tag");
-	::ID* id = install_ID("Tag", "Analyzer", true, true);
-	add_type(id, tag_enum_type, 0, 0);
 	}
 
 Manager::~Manager()
@@ -91,14 +89,14 @@ void Manager::InitPreScript()
 	std::list<Component*> analyzers = plugin_mgr->Components<Component>();
 
 	for ( std::list<Component*>::const_iterator i = analyzers.begin(); i != analyzers.end(); i++ )
-		RegisterAnalyzerComponent(*i);
+		RegisterComponent(*i, "ANALYZER_");
 
 	// Cache these tags.
-	analyzer_backdoor = GetAnalyzerTag("BACKDOOR");
+	analyzer_backdoor = GetComponentTag("BACKDOOR");
-	analyzer_connsize = GetAnalyzerTag("CONNSIZE");
+	analyzer_connsize = GetComponentTag("CONNSIZE");
-	analyzer_interconn = GetAnalyzerTag("INTERCONN");
+	analyzer_interconn = GetComponentTag("INTERCONN");
-	analyzer_stepping = GetAnalyzerTag("STEPPINGSTONE");
+	analyzer_stepping = GetComponentTag("STEPPINGSTONE");
-	analyzer_tcpstats = GetAnalyzerTag("TCPSTATS");
+	analyzer_tcpstats = GetComponentTag("TCPSTATS");
 	}
 
 void Manager::InitPostScript()
@@ -109,8 +107,9 @@ void Manager::DumpDebug()
 	{
 #ifdef DEBUG
 	DBG_LOG(DBG_ANALYZER, "Available analyzers after bro_init():");
-	for ( analyzer_map_by_name::const_iterator i = analyzers_by_name.begin(); i != analyzers_by_name.end(); i++ )
+	list<Component*> all_analyzers = GetComponents();
-		DBG_LOG(DBG_ANALYZER, "    %s (%s)", i->second->Name(), IsEnabled(i->second->Tag()) ? "enabled" : "disabled");
+	for ( list<Component*>::const_iterator i = all_analyzers.begin(); i != all_analyzers.end(); ++i )
+		DBG_LOG(DBG_ANALYZER, "    %s (%s)", (*i)->Name(), IsEnabled((*i)->Tag()) ? "enabled" : "disabled");
 
 	DBG_LOG(DBG_ANALYZER, "");
 	DBG_LOG(DBG_ANALYZER, "Analyzers by port:");
@@ -120,7 +119,7 @@ void Manager::DumpDebug()
 		string s;
 
 		for ( tag_set::const_iterator j = i->second->begin(); j != i->second->end(); j++ )
-			s += string(GetAnalyzerName(*j)) + " ";
+			s += string(GetComponentName(*j)) + " ";
 
 		DBG_LOG(DBG_ANALYZER, "    %d/tcp: %s", i->first, s.c_str());
 		}
@@ -130,7 +129,7 @@ void Manager::DumpDebug()
 		string s;
 
 		for ( tag_set::const_iterator j = i->second->begin(); j != i->second->end(); j++ )
-			s += string(GetAnalyzerName(*j)) + " ";
+			s += string(GetComponentName(*j)) + " ";
 
 		DBG_LOG(DBG_ANALYZER, "    %d/udp: %s", i->first, s.c_str());
 		}
@@ -142,25 +141,6 @@ void Manager::Done()
 	{
 	}
 
-void Manager::RegisterAnalyzerComponent(Component* component)
-	{
-	const char* cname = component->CanonicalName();
-
-	if ( Lookup(cname) )
-		reporter->FatalError("Analyzer %s defined more than once", cname);
-
-	DBG_LOG(DBG_ANALYZER, "Registering analyzer %s (tag %s)",
-		component->Name(), component->Tag().AsString().c_str());
-
-	analyzers_by_name.insert(std::make_pair(cname, component));
-	analyzers_by_tag.insert(std::make_pair(component->Tag(), component));
-	analyzers_by_val.insert(std::make_pair(component->Tag().AsEnumVal()->InternalInt(), component));
-
-	// Install enum "Analyzer::ANALYZER_*"
-	string id = fmt("ANALYZER_%s", cname);
-	tag_enum_type->AddName("Analyzer", id.c_str(), component->Tag().AsEnumVal()->InternalInt(), true);
-	}
-
 bool Manager::EnableAnalyzer(Tag tag)
 	{
 	Component* p = Lookup(tag);
@@ -217,8 +197,9 @@ void Manager::DisableAllAnalyzers()
 	{
 	DBG_LOG(DBG_ANALYZER, "Disabling all analyzers");
 
-	for ( analyzer_map_by_tag::const_iterator i = analyzers_by_tag.begin(); i != analyzers_by_tag.end(); i++ )
+	list<Component*> all_analyzers = GetComponents();
-		i->second->SetEnabled(false);
+	for ( list<Component*>::const_iterator i = all_analyzers.begin(); i != all_analyzers.end(); ++i )
+		(*i)->SetEnabled(false);
 	}
 
 bool Manager::IsEnabled(Tag tag)
@@ -270,7 +251,7 @@ bool Manager::RegisterAnalyzerForPort(Tag tag, TransportProto proto, uint32 port
 	tag_set* l = LookupPort(proto, port, true);
 
 #ifdef DEBUG
-	const char* name = GetAnalyzerName(tag);
+	const char* name = GetComponentName(tag);
 	DBG_LOG(DBG_ANALYZER, "Registering analyzer %s for port %" PRIu32 "/%d", name, port, proto);
 #endif
 
@@ -283,7 +264,7 @@ bool Manager::UnregisterAnalyzerForPort(Tag tag, TransportProto proto, uint32 po
 	tag_set* l = LookupPort(proto, port, true);
 
 #ifdef DEBUG
-	const char* name = GetAnalyzerName(tag);
+	const char* name = GetComponentName(tag);
 	DBG_LOG(DBG_ANALYZER, "Unregistering analyzer %s for port %" PRIu32 "/%d", name, port, proto);
 #endif
 
@@ -302,7 +283,7 @@ Analyzer* Manager::InstantiateAnalyzer(Tag tag, Connection* conn)
 		return 0;
 
 	if ( ! c->Factory() )
-		reporter->InternalError("analyzer %s cannot be instantiated dynamically", GetAnalyzerName(tag));
+		reporter->InternalError("analyzer %s cannot be instantiated dynamically", GetComponentName(tag));
 
 	Analyzer* a = c->Factory()(conn);
 
@@ -316,59 +297,10 @@ Analyzer* Manager::InstantiateAnalyzer(Tag tag, Connection* conn)
 
 Analyzer* Manager::InstantiateAnalyzer(const char* name, Connection* conn)
 	{
-	Tag tag = GetAnalyzerTag(name);
+	Tag tag = GetComponentTag(name);
 	return tag ? InstantiateAnalyzer(tag, conn) : 0;
 	}
 
-const char* Manager::GetAnalyzerName(Tag tag)
-	{
-	static const char* error = "<error>";
-
-	if ( ! tag )
-		return error;
-
-	Component* c = Lookup(tag);
-
-	if ( ! c )
-		reporter->InternalError("request for name of unknown analyzer tag %s", tag.AsString().c_str());
-
-	return c->CanonicalName();
-	}
-
-const char* Manager::GetAnalyzerName(Val* val)
-	{
-	return GetAnalyzerName(Tag(val->AsEnumVal()));
-	}
-
-Tag Manager::GetAnalyzerTag(const char* name)
-	{
-	Component* c = Lookup(name);
-	return c ? c->Tag() : Tag();
-	}
-
-EnumType* Manager::GetTagEnumType()
-	{
-	return tag_enum_type;
-	}
-
-Component* Manager::Lookup(const char* name)
-	{
-	analyzer_map_by_name::const_iterator i = analyzers_by_name.find(to_upper(name));
-	return i != analyzers_by_name.end() ? i->second : 0;
-	}
-
-Component* Manager::Lookup(const Tag& tag)
-	{
-	analyzer_map_by_tag::const_iterator i = analyzers_by_tag.find(tag);
-	return i != analyzers_by_tag.end() ? i->second : 0;
-	}
-
-Component* Manager::Lookup(EnumVal* val)
-	{
-	analyzer_map_by_val::const_iterator i = analyzers_by_val.find(val->InternalInt());
-	return i != analyzers_by_val.end() ? i->second : 0;
-	}
-
 Manager::tag_set* Manager::LookupPort(TransportProto proto, uint32 port, bool add_if_not_found)
 	{
 	analyzer_map_by_port* m = 0;
@@ -461,7 +393,7 @@ bool Manager::BuildInitialAnalyzerTree(Connection* conn)
 			root->AddChildAnalyzer(analyzer, false);
 
 			DBG_ANALYZER_ARGS(conn, "activated %s analyzer as scheduled",
-				     analyzer_mgr->GetAnalyzerName(*i));
+					  analyzer_mgr->GetComponentName(*i));
 			}
 
 		}
@@ -487,7 +419,7 @@ bool Manager::BuildInitialAnalyzerTree(Connection* conn)
 
 					root->AddChildAnalyzer(analyzer, false);
 					DBG_ANALYZER_ARGS(conn, "activated %s analyzer due to port %d",
-						     analyzer_mgr->GetAnalyzerName(*j), resp_port);
+							  analyzer_mgr->GetComponentName(*j), resp_port);
 					}
 				}
 			}
@@ -613,7 +545,7 @@ void Manager::ExpireScheduledAnalyzers()
 			conns.erase(i);
 
 			DBG_LOG(DBG_ANALYZER, "Expiring expected analyzer %s for connection %s",
-				analyzer_mgr->GetAnalyzerName(a->analyzer),
+				analyzer_mgr->GetComponentName(a->analyzer),
 				fmt_conn_id(a->conn.orig, 0, a->conn.resp, a->conn.resp_p));
 
 			delete a;
@@ -655,7 +587,7 @@ void Manager::ScheduleAnalyzer(const IPAddr& orig, const IPAddr& resp,
 			TransportProto proto, const char* analyzer,
 			double timeout)
 	{
-	Tag tag = GetAnalyzerTag(analyzer);
+	Tag tag = GetComponentTag(analyzer);
 
 	if ( tag != Tag() )
 		ScheduleAnalyzer(orig, resp, resp_p, proto, tag, timeout);

src/analyzer/Manager.h

@@ -26,6 +26,7 @@
 #include "Analyzer.h"
 #include "Component.h"
 #include "Tag.h"
+#include "plugin/ComponentManager.h"
 
 #include "../Dict.h"
 #include "../net_util.h"
@@ -49,7 +50,7 @@ namespace analyzer {
  * classes. This allows to external analyzer code to potentially use a
  * different C++ standard library.
  */
-class Manager {
+class Manager : public plugin::ComponentManager<Tag, Component> {
 public:
 	/**
 	 * Constructor.
@@ -231,42 +232,6 @@ public:
 	 */
 	Analyzer* InstantiateAnalyzer(const char* name, Connection* c);
 
-	/**
-	 * Translates an analyzer tag into corresponding analyzer name.
-	 *
-	 * @param tag The analyzer tag.
-	 *
-	 * @return The name, or an empty string if the tag is invalid.
-	 */
-	const char* GetAnalyzerName(Tag tag);
-
-	/**
-	 * Translates an script-level analyzer tag into corresponding
-	 * analyzer name.
-	 *
-	 * @param val The analyzer tag as an script-level enum value of type
-	 * \c Analyzer::Tag.
-	 *
-	 * @return The name, or an empty string if the tag is invalid.
-	 */
-	const char* GetAnalyzerName(Val* val);
-
-	/**
-	 * Translates an analyzer name into the corresponding tag.
-	 *
-	 * @param name The name.
-	 *
-	 * @return The tag. If the name does not correspond to a valid
-	 * analyzer, the returned tag will evaluate to false.
-	 */
-	Tag GetAnalyzerTag(const char* name);
-
-	/**
-	 * Returns the enum type that corresponds to the script-level type \c
-	 * Analyzer::Tag.
-	 */
-	EnumType* GetTagEnumType();
-
 	/**
 	 * Given the first packet of a connection, builds its initial
 	 * analyzer tree.
@@ -350,18 +315,8 @@ public:
 
 private:
 	typedef set<Tag> tag_set;
-	typedef map<string, Component*> analyzer_map_by_name;
-	typedef map<Tag, Component*>  analyzer_map_by_tag;
-	typedef map<int, Component*>  analyzer_map_by_val;
 	typedef map<uint32, tag_set*> analyzer_map_by_port;
 
-	void RegisterAnalyzerComponent(Component* component); // Takes ownership.
-
-	Component* Lookup(const string& name);
-	Component* Lookup(const char* name);
-	Component* Lookup(const Tag& tag);
-	Component* Lookup(EnumVal* val);
-
 	tag_set* LookupPort(PortVal* val, bool add_if_not_found);
 	tag_set* LookupPort(TransportProto proto, uint32 port, bool add_if_not_found);
 
@@ -370,9 +325,6 @@ private:
 
 	analyzer_map_by_port analyzers_by_port_tcp;
 	analyzer_map_by_port analyzers_by_port_udp;
-	analyzer_map_by_name analyzers_by_name;
-	analyzer_map_by_tag  analyzers_by_tag;
-	analyzer_map_by_val  analyzers_by_val;
 
 	Tag analyzer_backdoor;
 	Tag analyzer_connsize;
@@ -380,8 +332,6 @@ private:
 	Tag analyzer_stepping;
 	Tag analyzer_tcpstats;
 
-	EnumType* tag_enum_type;
-
 	//// Data structures to track analyzed scheduled for future connections.
 
 	// The index for a scheduled connection.

src/analyzer/Tag.cc

@@ -3,90 +3,20 @@
 #include "Tag.h"
 #include "Manager.h"
 
-#include "../NetVar.h"
+analyzer::Tag analyzer::Tag::Error;
 
-using namespace analyzer;
+analyzer::Tag::Tag(type_t type, subtype_t subtype)
-
+	: ::Tag(analyzer_mgr->GetTagEnumType(), type, subtype)
-Tag Tag::Error;
-
-Tag::Tag(type_t arg_type, subtype_t arg_subtype)
 	{
-	assert(arg_type > 0);
-
-	type = arg_type;
-	subtype = arg_subtype;
-	int64_t i = (int64)(type) | ((int64)subtype << 31);
-
-	EnumType* etype = analyzer_mgr->GetTagEnumType();
-	Ref(etype);
-	val = new EnumVal(i, etype);
 	}
 
-Tag::Tag(EnumVal* arg_val)
+analyzer::Tag& analyzer::Tag::operator=(const analyzer::Tag& other)
 	{
-	assert(arg_val);
+	::Tag::operator=(other);
-
-	val = arg_val;
-	Ref(val);
-
-	int64 i = val->InternalInt();
-	type = i & 0xffffffff;
-	subtype = (i >> 31) & 0xffffffff;
-	}
-
-Tag::Tag(const Tag& other)
-	{
-	type = other.type;
-	subtype = other.subtype;
-	val = other.val;
-
-	if ( val )
-		Ref(val);
-	}
-
-Tag::Tag()
-	{
-	type = 0;
-	subtype = 0;
-	val = 0;
-	}
-
-Tag::~Tag()
-	{
-	Unref(val);
-	val = 0;
-	}
-
-Tag& Tag::operator=(const Tag& other)
-	{
-	if ( this != &other )
-		{
-		type = other.type;
-		subtype = other.subtype;
-		val = other.val;
-
-		if ( val )
-			Ref(val);
-		}
-
 	return *this;
 	}
 
-EnumVal* Tag::AsEnumVal() const
+EnumVal* analyzer::Tag::AsEnumVal() const
 	{
-	if ( ! val )
+	return ::Tag::AsEnumVal(analyzer_mgr->GetTagEnumType());
-		{
-		assert(analyzer_mgr);
-		assert(type == 0 && subtype == 0);
-		EnumType* etype = analyzer_mgr->GetTagEnumType();
-		Ref(etype);
-		val = new EnumVal(0, etype);
-		}
-
-	return val;
-	}
-
-std::string Tag::AsString() const
-	{
-	return fmt("%" PRIu32 "/%" PRIu32, type, subtype);
 	}

src/analyzer/Tag.h

@@ -5,90 +5,46 @@
 
 #include "config.h"
 #include "util.h"
+#include "../Tag.h"
+#include "plugin/TaggedComponent.h"
+#include "plugin/ComponentManager.h"
 
 class EnumVal;
 
-namespace file_analysis {
-class Manager;
-class Component;
-}
-
 namespace analyzer {
 
 class Manager;
 class Component;
 
 /**
- * Class to identify an analyzer type.
+ * Class to identify a protocol analyzer type.
  *
- * Each analyzer type gets a tag consisting of a main type and subtype. The
+ * The script-layer analogue is Analyzer::Tag.
- * former is an identifier that's unique all analyzer classes. The latter is
- * passed through to the analyzer instances for their use, yet not further
- * interpreted by the analyzer infrastructure; it allows an analyzer to
- * branch out into a set of sub-analyzers internally. Jointly, main type and
- * subtype form an analyzer "tag". Each unique tag corresponds to a single
- * "analyzer" from the user's perspective. At the script layer, these tags
- * are mapped into enums of type \c Analyzer::Tag. Internally, the
- * analyzer::Manager maintains the mapping of tag to analyzer (and it also
- * assigns them their main types), and analyzer::Component creates new
- * tags.
- *
- * The Tag class supports all operations necessary to act as an index in a
- * \c std::map.
  */
-class Tag  {
+class Tag : public ::Tag  {
 public:
-	/**
-	 * Type for the analyzer's main type.
-	 */
-	typedef uint32 type_t;
-
-	/**
-	 * Type for the analyzer's subtype.
-	 */
-	typedef uint32 subtype_t;
-
 	/*
 	 * Copy constructor.
 	 */
-	Tag(const Tag& other);
+	Tag(const Tag& other) : ::Tag(other) {}
 
 	/**
 	 * Default constructor. This initializes the tag with an error value
 	 * that will make \c operator \c bool return false.
 	 */
-	Tag();
+	Tag() : ::Tag() {}
 
 	/**
 	 * Destructor.
 	 */
-	~Tag();
+	~Tag() {}
-
-	/**
-	 * Returns the tag's main type.
-	 */
-	type_t Type() const 	{ return type; }
-
-	/**
-	 * Returns the tag's subtype.
-	 */
-	subtype_t Subtype() const 	{ return subtype; }
-
-	/**
-	 * Returns the \c Analyzer::Tag enum that corresponds to this tag.
-	 * The returned value is \a does not have its ref-count increased.
-	 */
-	EnumVal* AsEnumVal() const;
-
-	/**
-	 * Returns the numerical values for main and subtype inside a string
-	 * suitable for printing. This is primarily for debugging.
-	 */
-	std::string AsString() const;
 
 	/**
 	 * Returns false if the tag represents an error value rather than a
 	 * legal analyzer type.
+	 * TODO: make this conversion operator "explicit" (C++11) or use a
+	 *       "safe bool" idiom (not necessary if "explicit" is available),
+	 *       otherwise this may allow nonsense/undesired comparison operations.
 	 */
 	operator bool() const	{ return *this != Tag(); }
 
@@ -102,7 +58,7 @@ public:
 	 */
 	bool operator==(const Tag& other) const
 		{
-		return type == other.type && subtype == other.subtype;
+		return ::Tag::operator==(other);
 		}
 
 	/**
@@ -110,7 +66,7 @@ public:
 	 */
 	bool operator!=(const Tag& other) const
 		{
-		return type != other.type || subtype != other.subtype;
+		return ::Tag::operator!=(other);
 		}
 
 	/**
@@ -118,23 +74,30 @@ public:
 	 */
 	bool operator<(const Tag& other) const
 		{
-		return type != other.type ? type < other.type : (subtype < other.subtype);
+		return ::Tag::operator<(other);
 		}
 
+	/**
+	 * Returns the \c Analyzer::Tag enum that corresponds to this tag.
+	 * The returned value does not have its ref-count increased.
+	 *
+	 * @param etype the script-layer enum type associated with the tag.
+	 */
+	EnumVal* AsEnumVal() const;
+
 	static Tag Error;
 
 protected:
 	friend class analyzer::Manager;
-	friend class analyzer::Component;
+	friend class plugin::ComponentManager<Tag, Component>;
-	friend class file_analysis::Manager;
+	friend class plugin::TaggedComponent<Tag>;
-	friend class file_analysis::Component;
 
 	/**
 	 * Constructor.
 	 *
 	 * @param type The main type. Note that the \a analyzer::Manager
 	 * manages the value space internally, so noone else should assign
-	 * any main tyoes.
+	 * any main types.
 	 *
 	 * @param subtype The sub type, which is left to an analyzer for
 	 * interpretation. By default it's set to zero.
@@ -144,14 +107,9 @@ protected:
 	/**
 	 * Constructor.
 	 *
-	 * @param val An enuam value of script type \c Analyzer::Tag.
+	 * @param val An enum value of script type \c Analyzer::Tag.
 	 */
-	Tag(EnumVal* val);
+	Tag(EnumVal* val) : ::Tag(val) {}
-
-private:
-	type_t type;		// Main type.
-	subtype_t subtype;	// Subtype.
-	mutable EnumVal* val;	// Analyzer::Tag value.
 };
 
 }

src/analyzer/analyzer.bif

@@ -41,11 +41,11 @@ function Analyzer::__schedule_analyzer%(orig: addr, resp: addr, resp_p: port,
 
 function __name%(atype: Analyzer::Tag%) : string
 	%{
-	return new StringVal(analyzer_mgr->GetAnalyzerName(atype));
+	return new StringVal(analyzer_mgr->GetComponentName(atype));
 	%}
 
 function __tag%(name: string%) : Analyzer::Tag
 	%{
-	analyzer::Tag t = analyzer_mgr->GetAnalyzerTag(name->CheckString());
+	analyzer::Tag t = analyzer_mgr->GetComponentTag(name->CheckString());
 	return t.AsEnumVal()->Ref();
 	%}

src/analyzer/protocol/dhcp/DHCP.cc

@@ -1,4 +1,3 @@
-
 #include "DHCP.h"
 
 #include "events.bif.h"

src/analyzer/protocol/dhcp/dhcp-analyzer.pac

@@ -8,12 +8,10 @@ flow DHCP_Flow(is_orig: bool) {
 
 	%member{
 		BroVal dhcp_msg_val_;
-		BroAnalyzer interp;
 	%}
 
 	%init{
 		dhcp_msg_val_ = 0;
-		interp = connection->bro_analyzer();
 	%}
 
 	%cleanup{
@@ -45,7 +43,7 @@ flow DHCP_Flow(is_orig: bool) {
 			}
 
 		if ( type == 0 )
-			interp->Weird("DHCP_no_type_option");
+			connection()->bro_analyzer()->ProtocolViolation("no DHCP message type option");
 
 		return type;
 		%}
@@ -56,11 +54,12 @@ flow DHCP_Flow(is_orig: bool) {
 
 		// Requested IP address to the server.
 		::uint32 req_addr = 0, serv_addr = 0;
+		StringVal* host_name = 0;
 
-		for ( ptr = options->begin();
+		for ( ptr = options->begin();  ptr != options->end() && ! (*ptr)->last(); ++ptr )
-		       ptr != options->end() && ! (*ptr)->last(); ++ptr )
+			{
+			switch ( (*ptr)->code() )
 				{
-			switch ( (*ptr)->code() ) {
 				case REQ_IP_OPTION:
 					req_addr = htonl((*ptr)->info()->req_addr());
 					break;
@@ -68,40 +67,48 @@ flow DHCP_Flow(is_orig: bool) {
 				case SERV_ID_OPTION:
 					serv_addr = htonl((*ptr)->info()->serv_addr());
 					break;
+
+				case HOST_NAME_OPTION:
+					host_name = new StringVal((*ptr)->info()->host_name().length(),
+								  (const char*) (*ptr)->info()->host_name().begin());
+					break;
 				}
 			}
 
+		if ( host_name == 0 )
+			host_name = new StringVal("");
+
 		switch ( type )
 			{
 			case DHCPDISCOVER:
 				BifEvent::generate_dhcp_discover(connection()->bro_analyzer(),
 								 connection()->bro_analyzer()->Conn(),
-					dhcp_msg_val_->Ref(), new AddrVal(req_addr));
+								 dhcp_msg_val_->Ref(), new AddrVal(req_addr), host_name);
 				break;
 
 			case DHCPREQUEST:
 				BifEvent::generate_dhcp_request(connection()->bro_analyzer(),
 								connection()->bro_analyzer()->Conn(),
 								dhcp_msg_val_->Ref(), new AddrVal(req_addr),
-				new AddrVal(serv_addr));
+								new AddrVal(serv_addr), host_name);
 				break;
 
 			case DHCPDECLINE:
 				BifEvent::generate_dhcp_decline(connection()->bro_analyzer(),
 								connection()->bro_analyzer()->Conn(),
-					dhcp_msg_val_->Ref());
+								dhcp_msg_val_->Ref(), host_name);
 				break;
 
 			case DHCPRELEASE:
 				BifEvent::generate_dhcp_release(connection()->bro_analyzer(),
 								connection()->bro_analyzer()->Conn(),
-					dhcp_msg_val_->Ref());
+								dhcp_msg_val_->Ref(), host_name);
 				break;
 
 			case DHCPINFORM:
 				BifEvent::generate_dhcp_inform(connection()->bro_analyzer(),
 							       connection()->bro_analyzer()->Conn(),
-					dhcp_msg_val_->Ref());
+							       dhcp_msg_val_->Ref(), host_name);
 				break;
 			}
 
@@ -118,11 +125,13 @@ flow DHCP_Flow(is_orig: bool) {
 		::uint32 subnet_mask = 0, serv_addr = 0;
 
 		uint32 lease = 0;
+		StringVal* host_name = 0;
 
 		for ( ptr = options->begin();
 		      ptr != options->end() && ! (*ptr)->last(); ++ptr )
 			{
-			switch ( (*ptr)->code() ) {
+			switch ( (*ptr)->code() )
+				{
 				case SUBNET_OPTION:
 					subnet_mask = htonl((*ptr)->info()->mask());
 					break;
@@ -134,16 +143,16 @@ flow DHCP_Flow(is_orig: bool) {
 					router_list = new TableVal(dhcp_router_list);
 
 						{
-				int num_routers =
+						int num_routers = (*ptr)->info()->router_list()->size();
-					(*ptr)->info()->router_list()->size();
 
 						for ( int i = 0; i < num_routers; ++i )
 							{
-					vector<uint32>* rlist =
+							vector<uint32>* rlist = (*ptr)->info()->router_list();
-						(*ptr)->info()->router_list();
+
 							uint32 raddr = (*rlist)[i];
 							::uint32 tmp_addr;
 							tmp_addr = htonl(raddr);
+
 							// index starting from 1
 							Val* index = new Val(i + 1, TYPE_COUNT);
 							router_list->Assign(index, new AddrVal(tmp_addr));
@@ -159,28 +168,37 @@ flow DHCP_Flow(is_orig: bool) {
 				case SERV_ID_OPTION:
 					serv_addr = htonl((*ptr)->info()->serv_addr());
 					break;
+
+				case HOST_NAME_OPTION:
+					host_name = new StringVal((*ptr)->info()->host_name().length(),
+								  (const char*) (*ptr)->info()->host_name().begin());
+					break;
 				}
 			}
 
-		switch ( type ) {
+			if ( host_name == 0 )
+				host_name = new StringVal("");
+
+		switch ( type )
+			{
 			case DHCPOFFER:
 				BifEvent::generate_dhcp_offer(connection()->bro_analyzer(),
 							      connection()->bro_analyzer()->Conn(),
 							      dhcp_msg_val_->Ref(), new AddrVal(subnet_mask),
-					router_list, lease, new AddrVal(serv_addr));
+							      router_list, lease, new AddrVal(serv_addr), host_name);
 				break;
 
 			case DHCPACK:
 				BifEvent::generate_dhcp_ack(connection()->bro_analyzer(),
 							    connection()->bro_analyzer()->Conn(),
 							    dhcp_msg_val_->Ref(), new AddrVal(subnet_mask),
-					router_list, lease, new AddrVal(serv_addr));
+							    router_list, lease, new AddrVal(serv_addr), host_name);
 				break;
 
 			case DHCPNAK:
 				BifEvent::generate_dhcp_nak(connection()->bro_analyzer(),
 							    connection()->bro_analyzer()->Conn(),
-					dhcp_msg_val_->Ref());
+							    dhcp_msg_val_->Ref(), host_name);
 				break;
 
 			}
@@ -195,7 +213,10 @@ flow DHCP_Flow(is_orig: bool) {
 		// DHCP or BOOTP.  If not, we are unable to interpret
 		// the message options.
 		if ( ${msg.cookie} != 0x63825363 )
+			{
+			connection()->bro_analyzer()->ProtocolViolation(fmt("bad cookie (%d)", ${msg.cookie}));
 			return false;
+			}
 
 		Unref(dhcp_msg_val_);
 		RecordVal* r = new RecordVal(dhcp_msg);
@@ -203,17 +224,15 @@ flow DHCP_Flow(is_orig: bool) {
 		r->Assign(0, new Val(${msg.op}, TYPE_COUNT));
 		r->Assign(1, new Val(${msg.type}, TYPE_COUNT));
 		r->Assign(2, new Val(${msg.xid}, TYPE_COUNT));
-
+		r->Assign(3, new StringVal(fmt_mac(${msg.chaddr}.data(), ${msg.chaddr}.length())));
-		// We want only 6 bytes for Ethernet address.
-		r->Assign(3, new StringVal(6, (const char*) ${msg.chaddr}.begin()));
-
 		r->Assign(4, new AddrVal(${msg.ciaddr}));
 		r->Assign(5, new AddrVal(${msg.yiaddr}));
 
 		dhcp_msg_val_ = r;
 
-		switch ( ${msg.op} ) {
+		switch ( ${msg.op} )
-		case BOOTREQUEST:	// presumablye from client to server
+			{
+			case BOOTREQUEST:	// presumably from client to server
 				if ( ${msg.type} == DHCPDISCOVER ||
 			     	     ${msg.type} == DHCPREQUEST ||
 			     	     ${msg.type} == DHCPDECLINE ||
@@ -221,22 +240,28 @@ flow DHCP_Flow(is_orig: bool) {
 			     	     ${msg.type} == DHCPINFORM )
 					parse_request(${msg.options}, ${msg.type});
 				else
-				interp->Weird("DHCP_wrong_msg_type");
+					connection()->bro_analyzer()->ProtocolViolation(fmt("unknown DHCP message type option for BOOTREQUEST (%d)",
+											    ${msg.type}));
 				break;
 
 			case BOOTREPLY:		// presumably from server to client
 				if ( ${msg.type} == DHCPOFFER ||
-			     ${msg.type} == DHCPACK || ${msg.type} == DHCPNAK )
+			     	     ${msg.type} == DHCPACK ||
+				     ${msg.type} == DHCPNAK )
 					parse_reply(${msg.options}, ${msg.type});
 				else
-				interp->Weird("DHCP_wrong_msg_type");
+					connection()->bro_analyzer()->ProtocolViolation(fmt("unknown DHCP message type option for BOOTREPLY (%d)",
+											    ${msg.type}));
+
 				break;
 
 			default:
-			interp->Weird("DHCP_wrong_op_type");
+				connection()->bro_analyzer()->ProtocolViolation(fmt("unknown DHCP message op code (%d). Known codes: 1=BOOTREQUEST, 2=BOOTREPLY",
+										${msg.op}));
 				break;
 			}
 
+		connection()->bro_analyzer()->ProtocolConfirmation();
 		return true;
 		%}
 };

src/analyzer/protocol/dhcp/dhcp-protocol.pac

@@ -12,6 +12,7 @@ enum OP_type {
 enum OPTION_type {
 	SUBNET_OPTION	 = 1,
 	ROUTER_OPTION	 = 3,
+	HOST_NAME_OPTION = 12,
 	REQ_IP_OPTION	 = 50,
 	LEASE_OPTION	 = 51,
 	MSG_TYPE_OPTION  = 53,
@@ -40,6 +41,7 @@ type Option_Info(code: uint8)  = record {
 		LEASE_OPTION	-> lease	: uint32;
 		MSG_TYPE_OPTION	-> msg_type	: uint8;
 		SERV_ID_OPTION	-> serv_addr	: uint32;
+		HOST_NAME_OPTION-> host_name	: bytestring &length = length;
 		default		-> other	: bytestring &length = length;
 	};
 };

src/analyzer/protocol/dhcp/dhcp.pac

@@ -1,3 +1,4 @@
+%include binpac.pac
 %include bro.pac
 
 %extern{

src/analyzer/protocol/dhcp/events.bif

@@ -1,8 +1,5 @@
-## Generated for DHCP messages of type *discover*.
+## Generated for DHCP messages of type *DHCPDISCOVER* (client broadcast to locate
-##
+## available servers).
-## See `Wikipedia
-## <http://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol>`__ for
-## more information about the DHCP protocol.
 ##
 ## c: The connection record describing the underlying UDP flow.
 ##
@@ -10,33 +7,23 @@
 ##
 ## req_addr: The specific address requested by the client.
 ##
-## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl
+## host_name: The value of the host name option, if specified by the client.
-##    dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply
+##
-##    dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end
+## .. bro:see:: dhcp_discover dhcp_offer dhcp_request dhcp_decline dhcp_ack dhcp_nak
-##    dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name
+##              dhcp_release dhcp_inform
-##    dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply
-##    dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout
-##    dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth
 ##
 ## .. note:: Bro does not support broadcast packets (as used by the DHCP
 ##    protocol). It treats broadcast addresses just like any other and
 ##    associates packets into transport-level flows in the same way as usual.
 ##
-## .. todo:: Bro's current default configuration does not activate the protocol
+event dhcp_discover%(c: connection, msg: dhcp_msg, req_addr: addr, host_name: string%);
-##    analyzer that generates this event; the corresponding script has not yet
-##    been ported to Bro 2.x. To still enable this event, one needs to
-##    register a port for it or add a DPD payload signature.
-event dhcp_discover%(c: connection, msg: dhcp_msg, req_addr: addr%);
 
-## Generated for DHCP messages of type *offer*.
+## Generated for DHCP messages of type *DHCPOFFER* (server to client in response to
-##
+## DHCPDISCOVER with offer of configuration parameters).
-## See `Wikipedia
-## <http://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol>`__ for
-## more information about the DHCP protocol.
 ##
 ## c: The connection record describing the underlying UDP flow.
 ##
-## msg: TODO.
+## msg: The parsed type-independent part of the DHCP message.
 ##
 ## mask: The subnet mask specified by the message.
 ##
@@ -46,28 +33,21 @@ event dhcp_discover%(c: connection, msg: dhcp_msg, req_addr: addr%);
 ##
 ## serv_addr: The server address specified by the message.
 ##
-## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl
+## host_name: The value of the host name option, if specified by the client.
-##    dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply
+##
-##    dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end
+## .. bro:see:: dhcp_discover dhcp_request dhcp_decline dhcp_ack dhcp_nak
-##    dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name
+##              dhcp_release dhcp_inform
-##    dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply
-##    dns_rejected dns_request non_dns_request
 ##
 ## .. note:: Bro does not support broadcast packets (as used by the DHCP
 ##    protocol). It treats broadcast addresses just like any other and
 ##    associates packets into transport-level flows in the same way as usual.
 ##
-## .. todo:: Bro's current default configuration does not activate the protocol
+event dhcp_offer%(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr, host_name: string%);
-##    analyzer that generates this event; the corresponding script has not yet
-##    been ported to Bro 2.x. To still enable this event, one needs to
-##    register a port for it or add a DPD payload signature.
-event dhcp_offer%(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr%);
 
-## Generated for DHCP messages of type *request*.
+## Generated for DHCP messages of type *DHCPREQUEST* (Client message to servers either
-##
+## (a) requesting offered parameters from one server and implicitly declining offers
-## See `Wikipedia
+## from all others, (b) confirming correctness of previously allocated address after,
-## <http://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol>`__ for
+## e.g., system reboot, or (c) extending the lease on a particular network address.)
-## more information about the DHCP protocol.
 ##
 ## c: The connection record describing the underlying UDP flow.
 ##
@@ -77,55 +57,37 @@ event dhcp_offer%(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_
 ##
 ## serv_addr: The server address specified by the message.
 ##
-## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl
+## host_name: The value of the host name option, if specified by the client.
-##    dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply
+##
-##    dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end
+## .. bro:see:: dhcp_discover dhcp_offer dhcp_decline dhcp_ack dhcp_nak
-##    dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name
+##              dhcp_release dhcp_inform
-##    dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply
-##    dns_rejected dns_request non_dns_request
 ##
 ## .. note:: Bro does not support broadcast packets (as used by the DHCP
 ##    protocol). It treats broadcast addresses just like any other and
 ##    associates packets into transport-level flows in the same way as usual.
 ##
-## .. todo:: Bro's current default configuration does not activate the protocol
+event dhcp_request%(c: connection, msg: dhcp_msg, req_addr: addr, serv_addr: addr, host_name: string%);
-##    analyzer that generates this event; the corresponding script has not yet
-##    been ported to Bro 2.x. To still enable this event, one needs to
-##    register a port for it or add a DPD payload signature.
-event dhcp_request%(c: connection, msg: dhcp_msg, req_addr: addr, serv_addr: addr%);
 
-## Generated for DHCP messages of type *decline*.
+## Generated for DHCP messages of type *DHCPDECLINE* (Client to server indicating
-##
+## network address is already in use).
-## See `Wikipedia
-## <http://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol>`__ for
-## more information about the DHCP protocol.
 ##
 ## c: The connection record describing the underlying UDP flow.
 ##
 ## msg: The parsed type-independent part of the DHCP message.
 ##
-## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl
+## host_name: The value of the host name option, if specified by the client.
-##    dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply
+##
-##    dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end
+## .. bro:see:: dhcp_discover dhcp_offer dhcp_request dhcp_ack dhcp_nak
-##    dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name
+##              dhcp_release dhcp_inform
-##    dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply
-##    dns_rejected dns_request non_dns_request
 ##
 ## .. note:: Bro does not support broadcast packets (as used by the DHCP
 ##    protocol). It treats broadcast addresses just like any other and
 ##    associates packets into transport-level flows in the same way as usual.
 ##
-## .. todo:: Bro's current default configuration does not activate the protocol
+event dhcp_decline%(c: connection, msg: dhcp_msg, host_name: string%);
-##    analyzer that generates this event; the corresponding script has not yet
-##    been ported to Bro 2.x. To still enable this event, one needs to
-##    register a port for it or add a DPD payload signature.
-event dhcp_decline%(c: connection, msg: dhcp_msg%);
 
-## Generated for DHCP messages of type *acknowledgment*.
+## Generated for DHCP messages of type *DHCPACK* (Server to client with configuration
-##
+## parameters, including committed network address).
-## See `Wikipedia
-## <http://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol>`__ for
-## more information about the DHCP protocol.
 ##
 ## c: The connection record describing the underlying UDP flow.
 ##
@@ -139,101 +101,62 @@ event dhcp_decline%(c: connection, msg: dhcp_msg%);
 ##
 ## serv_addr: The server address specified by the message.
 ##
-## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl
+## host_name: The value of the host name option, if specified by the client.
-##    dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply
-##    dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end
-##    dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name
-##    dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply
-##    dns_rejected dns_request non_dns_request
 ##
-## .. note:: Bro does not support broadcast packets (as used by the DHCP
+## .. bro:see:: dhcp_discover dhcp_offer dhcp_request dhcp_decline dhcp_nak
-##    protocol). It treats broadcast addresses just like any other and
+##              dhcp_release dhcp_inform
-##    associates packets into transport-level flows in the same way as usual.
 ##
-## .. todo:: Bro's current default configuration does not activate the protocol
+event dhcp_ack%(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr, host_name: string%);
-##    analyzer that generates this event; the corresponding script has not yet
-##    been ported to Bro 2.x. To still enable this event, one needs to
-##    register a port for it or add a DPD payload signature.
-event dhcp_ack%(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr%);
 
-## Generated for DHCP messages of type *negative acknowledgment*.
+## Generated for DHCP messages of type *DHCPNAK* (Server to client indicating client's
-##
+## notion of network address is incorrect (e.g., client has moved to new subnet) or
-## See `Wikipedia
+## client's lease has expired).
-## <http://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol>`__ for
-## more information about the DHCP protocol.
 ##
 ## c: The connection record describing the underlying UDP flow.
 ##
 ## msg: The parsed type-independent part of the DHCP message.
 ##
-## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl
+## host_name: The value of the host name option, if specified by the client.
-##    dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply
+##
-##    dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end
+## .. bro:see:: dhcp_discover dhcp_offer dhcp_request dhcp_decline dhcp_ack dhcp_release
-##    dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name
+##              dhcp_inform
-##    dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply
-##    dns_rejected dns_request non_dns_request
 ##
 ## .. note:: Bro does not support broadcast packets (as used by the DHCP
 ##    protocol). It treats broadcast addresses just like any other and
 ##    associates packets into transport-level flows in the same way as usual.
 ##
-## .. todo:: Bro's current default configuration does not activate the protocol
+event dhcp_nak%(c: connection, msg: dhcp_msg, host_name: string%);
-##    analyzer that generates this event; the corresponding script has not yet
-##    been ported to Bro 2.x. To still enable this event, one needs to
-##    register a port for it or add a DPD payload signature.
-event dhcp_nak%(c: connection, msg: dhcp_msg%);
 
-## Generated for DHCP messages of type *release*.
+## Generated for DHCP messages of type *DHCPRELEASE* (Client to server relinquishing
-##
+## network address and cancelling remaining lease).
-## See `Wikipedia
-## <http://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol>`__ for
-## more information about the DHCP protocol.
 ##
 ## c: The connection record describing the underlying UDP flow.
 ##
 ## msg: The parsed type-independent part of the DHCP message.
 ##
-## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl
+## host_name: The value of the host name option, if specified by the client.
-##    dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply
-##    dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end
-##    dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name
-##    dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply
-##    dns_rejected dns_request non_dns_request
 ##
-## .. note:: Bro does not support broadcast packets (as used by the DHCP
+## .. bro:see:: dhcp_discover dhcp_offer dhcp_request dhcp_decline dhcp_ack dhcp_nak
-##    protocol). It treats broadcast addresses just like any other and
+##              dhcp_inform
-##    associates packets into transport-level flows in the same way as usual.
 ##
-## .. todo:: Bro's current default configuration does not activate the protocol
+event dhcp_release%(c: connection, msg: dhcp_msg, host_name: string%);
-##    analyzer that generates this event; the corresponding script has not yet
-##    been ported to Bro 2.x. To still enable this event, one needs to
-##    register a port for it or add a DPD payload signature.
-event dhcp_release%(c: connection, msg: dhcp_msg%);
 
-## Generated for DHCP messages of type *inform*.
+## Generated for DHCP messages of type *DHCPINFORM* (Client to server, asking only for
-##
+## local configuration parameters; client already has externally configured network
-## See `Wikipedia
+## address).
-## <http://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol>`__ for
-## more information about the DHCP protocol.
 ##
 ## c: The connection record describing the underlying UDP flow.
 ##
 ## msg: The parsed type-independent part of the DHCP message.
 ##
-## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl
+## host_name: The value of the host name option, if specified by the client.
-##    dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply
+##
-##    dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end
+## .. bro:see:: dhcp_discover dhcp_offer dhcp_request dhcp_decline dhcp_ack dhcp_nak
-##    dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name
+##              dhcp_release
-##    dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply
-##    dns_rejected dns_request non_dns_request
 ##
 ## .. note:: Bro does not support broadcast packets (as used by the DHCP
 ##    protocol). It treats broadcast addresses just like any other and
 ##    associates packets into transport-level flows in the same way as usual.
 ##
-## .. todo:: Bro's current default configuration does not activate the protocol
+event dhcp_inform%(c: connection, msg: dhcp_msg, host_name: string%);
-##    analyzer that generates this event; the corresponding script has not yet
-##    been ported to Bro 2.x. To still enable this event, one needs to
-##    register a port for it or add a DPD payload signature.
-event dhcp_inform%(c: connection, msg: dhcp_msg%);
 

src/file_analysis/Analyzer.cc

@@ -0,0 +1,11 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "Analyzer.h"
+#include "Manager.h"
+
+file_analysis::Analyzer::~Analyzer()
+	{
+	DBG_LOG(DBG_FILE_ANALYSIS, "Destroy file analyzer %s",
+	        file_mgr->GetComponentName(tag));
+	Unref(args);
+	}

src/file_analysis/Analyzer.h

@@ -5,14 +5,12 @@
 
 #include "Val.h"
 #include "NetVar.h"
-#include "analyzer/Tag.h"
+#include "Tag.h"
 
 #include "file_analysis/file_analysis.bif.h"
 
 namespace file_analysis {
 
-typedef int FA_Tag;
-
 class File;
 
 /**
@@ -25,11 +23,7 @@ public:
 	 * Destructor.  Nothing special about it. Virtual since we definitely expect
 	 * to delete instances of derived classes via pointers to this class.
 	 */
-	virtual ~Analyzer()
+	virtual ~Analyzer();
-		{
-		DBG_LOG(DBG_FILE_ANALYSIS, "Destroy file analyzer %d", tag);
-		Unref(args);
-		}
 
 	/**
 	 * Subclasses may override this metod to receive file data non-sequentially.
@@ -76,7 +70,7 @@ public:
 	/**
 	 * @return the analyzer type enum value.
 	 */
-	FA_Tag Tag() const { return tag; }
+	file_analysis::Tag Tag() const { return tag; }
 
 	/**
 	 * @return the AnalyzerArgs associated with the analyzer.
@@ -88,18 +82,6 @@ public:
 	 */
 	File* GetFile() const { return file; }
 
-	/**
-	 * Retrieves an analyzer tag field from full analyzer argument record.
-	 * @param args an \c AnalyzerArgs (script-layer type) value.
-	 * @return the analyzer tag equivalent of the 'tag' field from the
-	 *         \c AnalyzerArgs value \a args.
-	 */
-	static FA_Tag ArgsTag(const RecordVal* args)
-		{
-		using BifType::Record::Files::AnalyzerArgs;
-		return args->Lookup(AnalyzerArgs->FieldOffset("tag"))->AsEnum();
-		}
-
 protected:
 
 	/**
@@ -108,15 +90,15 @@ protected:
 	 *        tunable options, if any, related to a particular analyzer type.
 	 * @param arg_file the file to which the the analyzer is being attached.
 	 */
-	Analyzer(RecordVal* arg_args, File* arg_file)
+	Analyzer(file_analysis::Tag arg_tag, RecordVal* arg_args, File* arg_file)
-	    : tag(file_analysis::Analyzer::ArgsTag(arg_args)),
+	    : tag(arg_tag),
 	      args(arg_args->Ref()->AsRecordVal()),
 	      file(arg_file)
 		{}
 
 private:
 
-	FA_Tag tag;	/**< The particular analyzer type of the analyzer instance. */
+	file_analysis::Tag tag;	/**< The particular type of the analyzer instance. */
 	RecordVal* args;	/**< \c AnalyzerArgs val gives tunable analyzer params. */
 	File* file;	/**< The file to which the analyzer is attached. */
 };

src/file_analysis/AnalyzerSet.cc

@@ -15,6 +15,7 @@ static void analyzer_del_func(void* v)
 AnalyzerSet::AnalyzerSet(File* arg_file) : file(arg_file)
 	{
 	TypeList* t = new TypeList();
+	t->Append(file_mgr->GetTagEnumType()->Ref());
 	t->Append(BifType::Record::Files::AnalyzerArgs->Ref());
 	analyzer_hash = new CompositeHash(t);
 	Unref(t);
@@ -34,20 +35,20 @@ AnalyzerSet::~AnalyzerSet()
 	delete analyzer_hash;
 	}
 
-bool AnalyzerSet::Add(RecordVal* args)
+bool AnalyzerSet::Add(file_analysis::Tag tag, RecordVal* args)
 	{
-	HashKey* key = GetKey(args);
+	HashKey* key = GetKey(tag, args);
 
 	if ( analyzer_map.Lookup(key) )
 		{
-		DBG_LOG(DBG_FILE_ANALYSIS, "Instantiate analyzer %d skipped for file id"
+		DBG_LOG(DBG_FILE_ANALYSIS, "Instantiate analyzer %s skipped for file id"
-		        " %s: already exists", file_analysis::Analyzer::ArgsTag(args),
+		        " %s: already exists", file_mgr->GetComponentName(tag),
 		        file->GetID().c_str());
 		delete key;
 		return true;
 		}
 
-	file_analysis::Analyzer* a = InstantiateAnalyzer(args);
+	file_analysis::Analyzer* a = InstantiateAnalyzer(tag, args);
 
 	if ( ! a )
 		{
@@ -60,10 +61,10 @@ bool AnalyzerSet::Add(RecordVal* args)
 	return true;
 	}
 
-bool AnalyzerSet::QueueAdd(RecordVal* args)
+bool AnalyzerSet::QueueAdd(file_analysis::Tag tag, RecordVal* args)
 	{
-	HashKey* key = GetKey(args);
+	HashKey* key = GetKey(tag, args);
-	file_analysis::Analyzer* a = InstantiateAnalyzer(args);
+	file_analysis::Analyzer* a = InstantiateAnalyzer(tag, args);
 
 	if ( ! a )
 		{
@@ -80,8 +81,9 @@ bool AnalyzerSet::AddMod::Perform(AnalyzerSet* set)
 	{
 	if ( set->analyzer_map.Lookup(key) )
 		{
-		DBG_LOG(DBG_FILE_ANALYSIS, "Add analyzer %d skipped for file id"
+		DBG_LOG(DBG_FILE_ANALYSIS, "Add analyzer %s skipped for file id"
-		        " %s: already exists", a->Tag(), a->GetFile()->GetID().c_str());
+		        " %s: already exists", file_mgr->GetComponentName(a->Tag()),
+		        a->GetFile()->GetID().c_str());
 
 		Abort();
 		return true;
@@ -91,12 +93,12 @@ bool AnalyzerSet::AddMod::Perform(AnalyzerSet* set)
 	return true;
 	}
 
-bool AnalyzerSet::Remove(const RecordVal* args)
+bool AnalyzerSet::Remove(file_analysis::Tag tag, RecordVal* args)
 	{
-	return Remove(file_analysis::Analyzer::ArgsTag(args), GetKey(args));
+	return Remove(tag, GetKey(tag, args));
 	}
 
-bool AnalyzerSet::Remove(FA_Tag tag, HashKey* key)
+bool AnalyzerSet::Remove(file_analysis::Tag tag, HashKey* key)
 	{
 	file_analysis::Analyzer* a =
 	    (file_analysis::Analyzer*) analyzer_map.Remove(key);
@@ -105,22 +107,22 @@ bool AnalyzerSet::Remove(FA_Tag tag, HashKey* key)
 
 	if ( ! a )
 		{
-		DBG_LOG(DBG_FILE_ANALYSIS, "Skip remove analyzer %d for file id %s",
+		DBG_LOG(DBG_FILE_ANALYSIS, "Skip remove analyzer %s for file id %s",
-		        tag, file->GetID().c_str());
+		        file_mgr->GetComponentName(tag), file->GetID().c_str());
 		return false;
 		}
 
-	DBG_LOG(DBG_FILE_ANALYSIS, "Remove analyzer %d for file id %s", a->Tag(),
+	DBG_LOG(DBG_FILE_ANALYSIS, "Remove analyzer %s for file id %s",
+	        file_mgr->GetComponentName(tag),
 	        file->GetID().c_str());
 
 	delete a;
 	return true;
 	}
 
-bool AnalyzerSet::QueueRemove(const RecordVal* args)
+bool AnalyzerSet::QueueRemove(file_analysis::Tag tag, RecordVal* args)
 	{
-	HashKey* key = GetKey(args);
+	HashKey* key = GetKey(tag, args);
-	FA_Tag tag = file_analysis::Analyzer::ArgsTag(args);
 
 	mod_queue.push(new RemoveMod(tag, key));
 
@@ -132,24 +134,28 @@ bool AnalyzerSet::RemoveMod::Perform(AnalyzerSet* set)
 	return set->Remove(tag, key);
 	}
 
-HashKey* AnalyzerSet::GetKey(const RecordVal* args) const
+HashKey* AnalyzerSet::GetKey(file_analysis::Tag t, RecordVal* args) const
 	{
-	HashKey* key = analyzer_hash->ComputeHash(args, 1);
+	ListVal* lv = new ListVal(TYPE_ANY);
+	lv->Append(t.AsEnumVal()->Ref());
+	lv->Append(args->Ref());
+	HashKey* key = analyzer_hash->ComputeHash(lv, 1);
+	Unref(lv);
 	if ( ! key )
 		reporter->InternalError("AnalyzerArgs type mismatch");
 
 	return key;
 	}
 
-file_analysis::Analyzer* AnalyzerSet::InstantiateAnalyzer(RecordVal* args) const
+file_analysis::Analyzer* AnalyzerSet::InstantiateAnalyzer(Tag tag,
+                                                          RecordVal* args) const
 	{
-	FA_Tag tag = file_analysis::Analyzer::ArgsTag(args);
 	file_analysis::Analyzer* a = file_mgr->InstantiateAnalyzer(tag, args, file);
 
 	if ( ! a )
 		{
 		reporter->Error("Failed file analyzer %s instantiation for file id %s",
-		                file_mgr->GetAnalyzerName(tag), file->GetID().c_str());
+		                file_mgr->GetComponentName(tag), file->GetID().c_str());
 		return 0;
 		}
 
@@ -158,8 +164,8 @@ file_analysis::Analyzer* AnalyzerSet::InstantiateAnalyzer(RecordVal* args) const
 
 void AnalyzerSet::Insert(file_analysis::Analyzer* a, HashKey* key)
 	{
-	DBG_LOG(DBG_FILE_ANALYSIS, "Add analyzer %d for file id %s", a->Tag(),
+	DBG_LOG(DBG_FILE_ANALYSIS, "Add analyzer %s for file id %s",
-	        file->GetID().c_str());
+	        file_mgr->GetComponentName(a->Tag()), file->GetID().c_str());
 	analyzer_map.Insert(key, a);
 	delete key;
 	}

src/file_analysis/AnalyzerSet.h

@@ -9,6 +9,7 @@
 #include "Dict.h"
 #include "CompHash.h"
 #include "Val.h"
+#include "Tag.h"
 
 namespace file_analysis {
 
@@ -38,31 +39,35 @@ public:
 
 	/**
 	 * Attach an analyzer to #file immediately.
+	 * @param tag the analyzer tag of the file analyzer to add.
 	 * @param args an \c AnalyzerArgs value which specifies an analyzer.
 	 * @return true if analyzer was instantiated/attached, else false.
 	 */
-	bool Add(RecordVal* args);
+	bool Add(file_analysis::Tag tag, RecordVal* args);
 
 	/**
 	 * Queue the attachment of an analyzer to #file.
+	 * @param tag the analyzer tag of the file analyzer to add.
 	 * @param args an \c AnalyzerArgs value which specifies an analyzer.
 	 * @return true if analyzer was able to be instantiated, else false.
 	 */
-	bool QueueAdd(RecordVal* args);
+	bool QueueAdd(file_analysis::Tag tag, RecordVal* args);
 
 	/**
 	 * Remove an analyzer from #file immediately.
+	 * @param tag the analyzer tag of the file analyzer to remove.
 	 * @param args an \c AnalyzerArgs value which specifies an analyzer.
 	 * @return false if analyzer didn't exist and so wasn't removed, else true.
 	 */
-	bool Remove(const RecordVal* args);
+	bool Remove(file_analysis::Tag tag, RecordVal* args);
 
 	/**
 	 * Queue the removal of an analyzer from #file.
+	 * @param tag the analyzer tag of the file analyzer to remove.
 	 * @param args an \c AnalyzerArgs value which specifies an analyzer.
 	 * @return true if analyzer exists at time of call, else false;
 	 */
-	bool QueueRemove(const RecordVal* args);
+	bool QueueRemove(file_analysis::Tag tag, RecordVal* args);
 
 	/**
 	 * Perform all queued modifications to the current analyzer set.
@@ -91,17 +96,20 @@ protected:
 
 	/**
 	 * Get a hash key which represents an analyzer instance.
+	 * @param tag the file analyzer tag.
 	 * @param args an \c AnalyzerArgs value which specifies an analyzer.
 	 * @return the hash key calculated from \a args
 	 */
-	HashKey* GetKey(const RecordVal* args) const;
+	HashKey* GetKey(file_analysis::Tag tag, RecordVal* args) const;
 
 	/**
 	 * Create an instance of a file analyzer.
+	 * @param tag the tag of a file analyzer.
 	 * @param args an \c AnalyzerArgs value which specifies an analyzer.
 	 * @return a new file analyzer instance.
 	 */
-	file_analysis::Analyzer* InstantiateAnalyzer(RecordVal* args) const;
+	file_analysis::Analyzer* InstantiateAnalyzer(file_analysis::Tag tag,
+	                                             RecordVal* args) const;
 
 	/**
 	 * Insert an analyzer instance in to the set.
@@ -116,7 +124,7 @@ protected:
 	 *        just used for debugging messages.
 	 * @param key the hash key which represents the analyzer's \c AnalyzerArgs.
 	 */
-	bool Remove(FA_Tag tag, HashKey* key);
+	bool Remove(file_analysis::Tag tag, HashKey* key);
 
 private:
 
@@ -175,14 +183,14 @@ private:
 		 * @param arg_a an analyzer instance to add to an analyzer set.
 		 * @param arg_key hash key representing the analyzer's \c AnalyzerArgs.
 		 */
-		RemoveMod(FA_Tag arg_tag, HashKey* arg_key)
+		RemoveMod(file_analysis::Tag arg_tag, HashKey* arg_key)
 			: Modification(), tag(arg_tag), key(arg_key) {}
 		virtual ~RemoveMod() {}
 		virtual bool Perform(AnalyzerSet* set);
 		virtual void Abort() { delete key; }
 
 	protected:
-		FA_Tag tag;
+		file_analysis::Tag tag;
 		HashKey* key;
 	};
 

src/file_analysis/CMakeLists.txt

@@ -11,9 +11,10 @@ set(file_analysis_SRCS
     Manager.cc
     File.cc
     FileTimer.cc
-    Analyzer.h
+    Analyzer.cc
     AnalyzerSet.cc
     Component.cc
+    Tag.cc
 )
 
 bif_target(file_analysis.bif)

src/file_analysis/Component.cc

@@ -8,26 +8,22 @@
 
 using namespace file_analysis;
 
-analyzer::Tag::type_t Component::type_counter = 0;
+Component::Component(const char* arg_name, factory_callback arg_factory)
-
+	: plugin::Component(plugin::component::FILE_ANALYZER),
-Component::Component(const char* arg_name, factory_callback arg_factory,
+	  plugin::TaggedComponent<file_analysis::Tag>()
-                     analyzer::Tag::subtype_t arg_subtype)
-	: plugin::Component(plugin::component::FILE_ANALYZER)
 	{
 	name = copy_string(arg_name);
 	canon_name = canonify_name(arg_name);
 	factory = arg_factory;
-
-	tag = analyzer::Tag(++type_counter, arg_subtype);
 	}
 
 Component::Component(const Component& other)
-	: plugin::Component(Type())
+	: plugin::Component(Type()),
+	  plugin::TaggedComponent<file_analysis::Tag>(other)
 	{
 	name = copy_string(other.name);
 	canon_name = copy_string(other.canon_name);
 	factory = other.factory;
-	tag = other.tag;
 	}
 
 Component::~Component()
@@ -36,11 +32,6 @@ Component::~Component()
 	delete [] canon_name;
 	}
 
-analyzer::Tag Component::Tag() const
-	{
-	return tag;
-	}
-
 void Component::Describe(ODesc* d) const
 	{
 	plugin::Component::Describe(d);
@@ -58,11 +49,12 @@ void Component::Describe(ODesc* d) const
 
 Component& Component::operator=(const Component& other)
 	{
+	plugin::TaggedComponent<file_analysis::Tag>::operator=(other);
+
 	if ( &other != this )
 		{
 		name = copy_string(other.name);
 		factory = other.factory;
-		tag = other.tag;
 		}
 
 	return *this;

src/file_analysis/Component.h

@@ -3,8 +3,9 @@
 #ifndef FILE_ANALYZER_PLUGIN_COMPONENT_H
 #define FILE_ANALYZER_PLUGIN_COMPONENT_H
 
-#include "analyzer/Tag.h"
+#include "Tag.h"
 #include "plugin/Component.h"
+#include "plugin/TaggedComponent.h"
 
 #include "Val.h"
 
@@ -22,7 +23,8 @@ class Analyzer;
  * A plugin can provide a specific file analyzer by registering this
  * analyzer component, describing the analyzer.
  */
-class Component : public plugin::Component {
+class Component : public plugin::Component,
+                  public plugin::TaggedComponent<file_analysis::Tag> {
 public:
 	typedef Analyzer* (*factory_callback)(RecordVal* args, File* file);
 
@@ -38,15 +40,8 @@ public:
 	 * from file_analysis::Analyzer. This is typically a static \c
 	 * Instatiate() method inside the class that just allocates and
 	 * returns a new instance.
-	 *
-	 * @param subtype A subtype associated with this component that
-	 * further distinguishes it. The subtype will be integrated into
-	 * the analyzer::Tag that the manager associates with this analyzer,
-	 * and analyzer instances can accordingly access it via analyzer::Tag().
-	 * If not used, leave at zero.
 	 */
-	Component(const char* name, factory_callback factory,
+	Component(const char* name, factory_callback factory);
-	          analyzer::Tag::subtype_t subtype = 0);
 
 	/**
 	 * Copy constructor.
@@ -79,13 +74,6 @@ public:
 	 */
 	factory_callback Factory() const	{ return factory; }
 
-	/**
-	 * Returns the analyzer's tag. Note that this is automatically
-	 * generated for each new Components, and hence unique across all of
-	 * them.
-	 */
-	analyzer::Tag Tag() const;
-
 	/**
 	 * Generates a human-readable description of the component's main
 	 * parameters. This goes into the output of \c "bro -NN".
@@ -98,10 +86,6 @@ private:
 	const char* name;	// The analyzer's name.
 	const char* canon_name;	// The analyzer's canonical name.
 	factory_callback factory;	// The analyzer's factory callback.
-	analyzer::Tag tag;	// The automatically assigned analyzer tag.
-
-	// Global counter used to generate unique tags.
-	static analyzer::Tag::type_t type_counter;
 };
 
 }

src/file_analysis/File.cc

@@ -88,7 +88,7 @@ File::File(const string& file_id, Connection* conn, analyzer::Tag tag,
 	if ( conn )
 		{
 		// add source, connection, is_orig fields
-		SetSource(analyzer_mgr->GetAnalyzerName(tag));
+		SetSource(analyzer_mgr->GetComponentName(tag));
 		val->Assign(is_orig_idx, new Val(is_orig, TYPE_BOOL));
 		UpdateConnectionFields(conn, is_orig);
 		}
@@ -231,14 +231,14 @@ void File::ScheduleInactivityTimer() const
 	timer_mgr->Add(new FileTimer(network_time, id, GetTimeoutInterval()));
 	}
 
-bool File::AddAnalyzer(RecordVal* args)
+bool File::AddAnalyzer(file_analysis::Tag tag, RecordVal* args)
 	{
-	return done ? false : analyzers.QueueAdd(args);
+	return done ? false : analyzers.QueueAdd(tag, args);
 	}
 
-bool File::RemoveAnalyzer(const RecordVal* args)
+bool File::RemoveAnalyzer(file_analysis::Tag tag, RecordVal* args)
 	{
-	return done ? false : analyzers.QueueRemove(args);
+	return done ? false : analyzers.QueueRemove(tag, args);
 	}
 
 bool File::BufferBOF(const u_char* data, uint64 len)
@@ -321,7 +321,7 @@ void File::DataIn(const u_char* data, uint64 len, uint64 offset)
 	while ( (a = analyzers.NextEntry(c)) )
 		{
 		if ( ! a->DeliverChunk(data, len, offset) )
-			analyzers.QueueRemove(a->Args());
+			analyzers.QueueRemove(a->Tag(), a->Args());
 		}
 
 	analyzers.DrainModifications();
@@ -356,7 +356,7 @@ void File::DataIn(const u_char* data, uint64 len)
 		{
 		if ( ! a->DeliverStream(data, len) )
 			{
-			analyzers.QueueRemove(a->Args());
+			analyzers.QueueRemove(a->Tag(), a->Args());
 			continue;
 			}
 
@@ -364,7 +364,7 @@ void File::DataIn(const u_char* data, uint64 len)
 		                LookupFieldDefaultCount(missing_bytes_idx);
 
 		if ( ! a->DeliverChunk(data, len, offset) )
-			analyzers.QueueRemove(a->Args());
+			analyzers.QueueRemove(a->Tag(), a->Args());
 		}
 
 	analyzers.DrainModifications();
@@ -389,7 +389,7 @@ void File::EndOfFile()
 	while ( (a = analyzers.NextEntry(c)) )
 		{
 		if ( ! a->EndOfFile() )
-			analyzers.QueueRemove(a->Args());
+			analyzers.QueueRemove(a->Tag(), a->Args());
 		}
 
 	FileEvent(file_state_remove);
@@ -411,7 +411,7 @@ void File::Gap(uint64 offset, uint64 len)
 	while ( (a = analyzers.NextEntry(c)) )
 		{
 		if ( ! a->Undelivered(offset, len) )
-			analyzers.QueueRemove(a->Args());
+			analyzers.QueueRemove(a->Tag(), a->Args());
 		}
 
 	if ( FileEventAvailable(file_gap) )

src/file_analysis/File.h

@@ -10,6 +10,7 @@
 
 #include "Conn.h"
 #include "Val.h"
+#include "Tag.h"
 #include "AnalyzerSet.h"
 #include "BroString.h"
 
@@ -94,17 +95,19 @@ public:
 	/**
 	 * Queues attaching an analyzer.  Only one analyzer per type can be attached
 	 * at a time unless the arguments differ.
+	 * @param tag the analyzer tag of the file analyzer to add.
 	 * @param args an \c AnalyzerArgs value representing a file analyzer.
 	 * @return false if analyzer can't be instantiated, else true.
 	 */
-	bool AddAnalyzer(RecordVal* args);
+	bool AddAnalyzer(file_analysis::Tag tag, RecordVal* args);
 
 	/**
 	 * Queues removal of an analyzer.
+	 * @param tag the analyzer tag of the file analyzer to remove.
 	 * @param args an \c AnalyzerArgs value representing a file analyzer.
 	 * @return true if analyzer was active at time of call, else false.
 	 */
-	bool RemoveAnalyzer(const RecordVal* args);
+	bool RemoveAnalyzer(file_analysis::Tag tag, RecordVal* args);
 
 	/**
 	 * Pass in non-sequential data and deliver to attached analyzers.

src/file_analysis/FileTimer.cc

@@ -14,7 +14,7 @@ FileTimer::FileTimer(double t, const string& id, double interval)
 
 void FileTimer::Dispatch(double t, int is_expire)
 	{
-	File* file = file_mgr->Lookup(file_id);
+	File* file = file_mgr->LookupFile(file_id);
 
 	if ( ! file )
 		return;