Merge remote-tracking branch 'origin/master' into topic/bernhard/input-update

scripts/base/frameworks/analyzer/__load__.bro

@@ -0,0 +1 @@
+@load ./main

scripts/base/frameworks/analyzer/main.bro

@@ -0,0 +1,181 @@
+##! Framework for managing Bro's protocol analyzers.
+##!
+##! The analyzer framework allows to dynamically enable or disable analyzers, as
+##! well as to manage the well-known ports which automatically activate a
+##! particular analyzer for new connections.
+##!
+##! Protocol analyzers are identified by unique tags of type
+##! :bro:type:`Analyzer::Tag`, such as :bro:enum:`Analyzer::ANALYZER_HTTP` and
+##! :bro:enum:`Analyzer::ANALYZER_HTTP`. These tags are defined internally by
+##! the analyzers themselves, and documented in their analyzer-specific
+##! description along with the events that they generate.
+##!
+##! .. todo: ``The ANALYZER_*`` are in fact not yet documented, we need to
+##!    add that to Broxygen.
+module Analyzer;
+
+export {
+	## If true, all available analyzers are initially disabled at startup. One
+	## can then selectively enable them with
+	## :bro:id:`Analyzer::enable_analyzer`.
+	global disable_all = F &redef;
+
+	## Enables an analyzer. Once enabled, the analyzer may be used for analysis
+	## of future connections as decided by Bro's dynamic protocol detection.
+	##
+	## tag: The tag of the analyzer to enable.
+	##
+	## Returns: True if the analyzer was successfully enabled.
+	global enable_analyzer: function(tag: Analyzer::Tag) : bool;
+
+	## Disables an analyzer. Once disabled, the analyzer will not be used
+	## further for analysis of future connections.
+	##
+	## tag: The tag of the analyzer to disable.
+	##
+	## Returns: True if the analyzer was successfully disabled.
+	global disable_analyzer: function(tag: Analyzer::Tag) : bool;
+
+	## Registers a set of well-known ports for an analyzer. If a future
+	## connection on one of these ports is seen, the analyzer will be
+	## automatically assigned to parsing it. The function *adds* to all ports
+	## already registered, it doesn't replace them.
+	##
+	## tag: The tag of the analyzer.
+	##
+	## ports: The set of well-known ports to associate with the analyzer.
+	##
+	## Returns: True if the ports were sucessfully registered.
+	global register_for_ports: function(tag: Analyzer::Tag, ports: set[port]) : bool;
+
+	## Registers an individual well-known port for an analyzer. If a future
+	## connection on this port is seen, the analyzer will be automatically
+	## assigned to parsing it. The function *adds* to all ports already
+	## registered, it doesn't replace them.
+	##
+	## tag: The tag of the analyzer.
+	##
+	## p: The well-known port to associate with the analyzer.
+	##
+	## Returns: True if the port was sucessfully registered.
+	global register_for_port: function(tag: Analyzer::Tag, p: port) : bool;
+
+	## Returns a set of all well-known ports currently registered for a
+	## specific analyzer.
+	##
+	## tag: The tag of the analyzer.
+	##
+	## Returns: The set of ports.
+	global registered_ports: function(tag: Analyzer::Tag) : set[port];
+
+	## Returns a table of all ports-to-analyzer mappings currently registered.
+	##
+	## Returns: A table mapping each analyzer to the set of ports
+	##          registered for it.
+	global all_registered_ports: function() : table[Analyzer::Tag] of set[port];
+
+	## Translates an analyzer type to a string with the analyzer's name.
+	##
+	## tag: The analyzer tag.
+	##
+	## Returns: The analyzer name corresponding to the tag.
+	global name: function(tag: Analyzer::Tag) : string;
+
+	## Schedules an analyzer for a future connection originating from a given IP
+	## address and port.
+	##
+	## orig: The IP address originating a connection in the future.
+	##       0.0.0.0 can be used as a wildcard to match any originator address.
+	##
+	## resp: The IP address responding to a connection from *orig*.
+	##
+	## resp_p: The destination port at *resp*.
+	##
+	## analyzer: The analyzer ID.
+	##
+	## tout: A timeout interval after which the scheduling request will be
+	##       discarded if the connection has not yet been seen.
+	##
+	## Returns: True if succesful.
+	global schedule_analyzer: function(orig: addr, resp: addr, resp_p: port,
+					   analyzer: Analyzer::Tag, tout: interval) : bool;
+
+	## A set of analyzers to disable by default at startup. The default set
+	## contains legacy analyzers that are no longer supported.
+	global disabled_analyzers: set[Analyzer::Tag] = {
+		ANALYZER_INTERCONN,
+		ANALYZER_STEPPINGSTONE,
+		ANALYZER_BACKDOOR,
+		ANALYZER_TCPSTATS,
+	} &redef;
+}
+
+@load base/bif/analyzer.bif
+
+global ports: table[Analyzer::Tag] of set[port];
+
+event bro_init() &priority=5
+	{
+	if ( disable_all )
+		__disable_all_analyzers();
+
+	for ( a in disabled_analyzers )
+		disable_analyzer(a);
+	}
+
+function enable_analyzer(tag: Analyzer::Tag) : bool
+	{
+	return __enable_analyzer(tag);
+	}
+
+function disable_analyzer(tag: Analyzer::Tag) : bool
+	{
+	return __disable_analyzer(tag);
+	}
+
+function register_for_ports(tag: Analyzer::Tag, ports: set[port]) : bool
+	{
+	local rc = T;
+
+	for ( p in ports )
+		{
+		if ( ! register_for_port(tag, p) )
+			rc = F;
+		}
+
+	return rc;
+	}
+
+function register_for_port(tag: Analyzer::Tag, p: port) : bool
+	{
+	if ( ! __register_for_port(tag, p) )
+		return F;
+
+	if ( tag !in ports )
+		ports[tag] = set();
+
+	add ports[tag][p];
+	return T;
+	}
+
+function registered_ports(tag: Analyzer::Tag) : set[port]
+	{
+	return tag in ports ? ports[tag] : set();
+	}
+
+function all_registered_ports(): table[Analyzer::Tag] of set[port]
+	{
+	return ports;
+	}
+
+function name(atype: Analyzer::Tag) : string
+	{
+	return __name(atype);
+	}
+
+function schedule_analyzer(orig: addr, resp: addr, resp_p: port,
+			   analyzer: Analyzer::Tag, tout: interval) : bool
+	{
+	return __schedule_analyzer(orig, resp, resp_p, analyzer, tout);
+	}
+

scripts/base/frameworks/dpd/main.bro

@@ -23,12 +23,12 @@ export {
 		analyzer:       string          &log;
 		## The textual reason for the analysis failure.
 		failure_reason: string          &log;
-		
-		## Disabled analyzer IDs.  This is only for internal tracking 
+
+		## Disabled analyzer IDs.  This is only for internal tracking
 		## so as to not attempt to disable analyzers multiple times.
 		disabled_aids:  set[count];
 	};
-	
+
 	## Ignore violations which go this many bytes into the connection.
 	## Set to 0 to never ignore protocol violations.
 	const ignore_violations_after = 10 * 1024 &redef;
@@ -41,41 +41,30 @@ redef record connection += {
 event bro_init() &priority=5
 	{
 	Log::create_stream(DPD::LOG, [$columns=Info]);
-	
-	# Populate the internal DPD analysis variable.
-	for ( a in dpd_config )
-		{
-		for ( p in dpd_config[a]$ports )
-			{
-			if ( p !in dpd_analyzer_ports )
-				dpd_analyzer_ports[p] = set();
-			add dpd_analyzer_ports[p][a];
-			}
-		}
 	}
 
-event protocol_confirmation(c: connection, atype: count, aid: count) &priority=10
+event protocol_confirmation(c: connection, atype: Analyzer::Tag, aid: count) &priority=10
 	{
-	local analyzer = analyzer_name(atype);
-	
+	local analyzer = Analyzer::name(atype);
+
 	if ( fmt("-%s",analyzer) in c$service )
 		delete c$service[fmt("-%s", analyzer)];
 
 	add c$service[analyzer];
 	}
 
-event protocol_violation(c: connection, atype: count, aid: count,
+event protocol_violation(c: connection, atype: Analyzer::Tag, aid: count,
                          reason: string) &priority=10
 	{
-	local analyzer = analyzer_name(atype);
+	local analyzer = Analyzer::name(atype);
 	# If the service hasn't been confirmed yet, don't generate a log message
 	# for the protocol violation.
 	if ( analyzer !in c$service )
 		return;
-		
+
 	delete c$service[analyzer];
 	add c$service[fmt("-%s", analyzer)];
-	
+
 	local info: Info;
 	info$ts=network_time();
 	info$uid=c$uid;
@@ -86,7 +75,7 @@ event protocol_violation(c: connection, atype: count, aid: count,
 	c$dpd = info;
 	}
 
-event protocol_violation(c: connection, atype: count, aid: count, reason: string) &priority=5
+event protocol_violation(c: connection, atype: Analyzer::Tag, aid: count, reason: string) &priority=5
 	{
 	if ( !c?$dpd || aid in c$dpd$disabled_aids )
 		return;
@@ -94,13 +83,13 @@ event protocol_violation(c: connection, atype: count, aid: count, reason: string
 	local size = c$orig$size + c$resp$size;
 	if ( ignore_violations_after > 0 && size > ignore_violations_after )
 		return;
-	
+
 	# Disable the analyzer that raised the last core-generated event.
 	disable_analyzer(c$id, aid);
 	add c$dpd$disabled_aids[aid];
 	}
 
-event protocol_violation(c: connection, atype: count, aid: count,
+event protocol_violation(c: connection, atype: Analyzer::Tag, aid: count,
 				reason: string) &priority=-5
 	{
 	if ( c?$dpd )

scripts/base/frameworks/file-analysis/main.bro

@@ -1,7 +1,7 @@
 ##! An interface for driving the analysis of files, possibly independent of
 ##! any network protocol over which they're transported.
 
-@load base/file_analysis.bif
+@load base/bif/file_analysis.bif
 @load base/frameworks/logging
 
 module FileAnalysis;
@@ -104,7 +104,7 @@ export {
 
 	## A table that can be used to disable file analysis completely for
 	## any files transferred over given network protocol analyzers.
-	const disable: table[AnalyzerTag] of bool = table() &redef;
+	const disable: table[Analyzer::Tag] of bool = table() &redef;
 
 	## Event that can be handled to access the Info record as it is sent on
 	## to the logging framework.

scripts/base/frameworks/input/main.bro

@@ -149,7 +149,7 @@ export {
 	global end_of_data: event(name: string, source:string);
 }
 
-@load base/input.bif
+@load base/bif/input.bif
 
 
 module Input;

scripts/base/frameworks/logging/main.bro

@@ -366,7 +366,7 @@ export {
 # We keep a script-level copy of all filters so that we can manipulate them.
 global filters: table[ID, string] of Filter;
 
-@load base/logging.bif # Needs Filter and Stream defined.
+@load base/bif/logging.bif # Needs Filter and Stream defined.
 
 module Log;
 

scripts/base/frameworks/reporter/main.bro

@@ -9,7 +9,7 @@
 ##! Note that this framework deals with the handling of internally generated
 ##! reporter messages, for the interface in to actually creating interface
 ##! into actually creating reporter messages from the scripting layer, use
-##! the built-in functions in :doc:`/scripts/base/reporter.bif`.
+##! the built-in functions in :doc:`/scripts/base/bif/reporter.bif`.
 
 module Reporter;
 

scripts/base/frameworks/tunnels/main.bro

@@ -83,19 +83,17 @@ export {
 }
 
 const ayiya_ports = { 5072/udp };
-redef dpd_config += { [ANALYZER_AYIYA] = [$ports = ayiya_ports] };
-
 const teredo_ports = { 3544/udp };
-redef dpd_config += { [ANALYZER_TEREDO] = [$ports = teredo_ports] };
-
 const gtpv1_ports = { 2152/udp, 2123/udp };
-redef dpd_config += { [ANALYZER_GTPV1] = [$ports = gtpv1_ports] };
-
 redef likely_server_ports += { ayiya_ports, teredo_ports, gtpv1_ports };
 
 event bro_init() &priority=5
 	{
 	Log::create_stream(Tunnel::LOG, [$columns=Info]);
+
+	Analyzer::register_for_ports(Analyzer::ANALYZER_AYIYA, ayiya_ports);
+	Analyzer::register_for_ports(Analyzer::ANALYZER_TEREDO, teredo_ports);
+	Analyzer::register_for_ports(Analyzer::ANALYZER_GTPV1, gtpv1_ports);
 	}
 
 function register_all(ecv: EncapsulatingConnVector)