Merge remote branch 'origin/master' into topic/bernhard/hyperloglog

Conflicts:
	src/3rdparty

.gitmodules

@@ -1,21 +1,24 @@
 [submodule "aux/bro-aux"]
 	path = aux/bro-aux
-	url = git://git.bro-ids.org/bro-aux
+	url = ../bro-aux
 [submodule "aux/binpac"]
 	path = aux/binpac
-	url = git://git.bro-ids.org/binpac
+	url = ../binpac
 [submodule "aux/broccoli"]
 	path = aux/broccoli
-	url = git://git.bro-ids.org/broccoli
+	url = ../broccoli
 [submodule "aux/broctl"]
 	path = aux/broctl
-	url = git://git.bro-ids.org/broctl
+	url = ../broctl
 [submodule "aux/btest"]
 	path = aux/btest
-	url = git://git.bro-ids.org/btest
+	url = ../btest
 [submodule "cmake"]
 	path = cmake
-	url = git://git.bro-ids.org/cmake
+	url = ../cmake
 [submodule "magic"]
 	path = magic
-	url = git://git.bro.org/bromagic
+	url = ../bromagic
+[submodule "src/3rdparty"]
+	path = src/3rdparty
+	url = ../bro-3rdparty

CHANGES

@@ -1,4 +1,93 @@
 
+2.1-1128 | 2013-08-24 10:27:29 -0700
+
+  * Remove code relict in input framework. (Jon Siwek)
+    
+  * Fix documentation for mkdir BIF. (Jon Siwek)
+    
+  * File extraction tweaks. (Jon Siwek)
+
+    - Default extraction limit of 100MB now provided via a tuning
+      script loaded in local.bro so that command-line Bro is unlimited
+      by default.
+
+    - Extraction directory is now created on request of file
+      extraction rather than unconditionally in bro_init(). (Jon
+      Siwek)
+
+2.1-1124 | 2013-08-23 16:33:52 -0700
+
+  * Fixed a number of object bugs DNP3 analyzer. (Hui Lin)
+    
+2.1-1122 | 2013-08-22 16:52:27 -0700
+
+  * Use macros to create file analyzer plugin classes. (Jon Siwek)
+    
+  * Add options to limit extracted file sizes w/ 100MB default. (Jon
+    Siwek)
+
+2.1-1117 | 2013-08-22 08:44:12 -0700
+
+  * A number of input framework fixes and corresponding test stability
+    improvements. (Jon Siwek)
+
+  * Make memory leak tests able to time out.  (Jon Siwek)
+    
+  * Fix a compiler warning regarding strncat misuse. (Jon Siwek)
+    
+2.1-1103 | 2013-08-21 19:11:34 -0400
+
+  * A number of sumstats fixes. (Seth Hall, Vlad Grigorescu)
+    
+  * Fix memory leak w/ when statements. Addresses BIT-1058. (Jon
+    Siwek)
+
+  * Switching to relative submodule paths (Robin Sommer)
+
+2.1-1089 | 2013-08-19 11:25:11 -0700
+
+  * Fix bloom filters' dependence on size_t. (Jon Siwek, Matthias
+    Vallentin).
+
+2.1-1081 | 2013-08-19 11:19:33 -0700
+
+  * New BiF levenshtein_distance() to compute the Levenshtein distance
+    between two strings. (Anthony Kasza)
+
+2.1-1078 | 2013-08-19 09:29:30 -0700
+
+  * Moving sqlite code into new external 3rdparty submodule. (Bernhard
+    Amann)
+
+2.1-1074 | 2013-08-14 10:29:54 -0700
+
+  * Fix timer type enum and timer name array mismatch. (Jon Siwek)
+
+2.1-1072 | 2013-08-14 10:28:51 -0700
+
+  * Adding the unified2 analyzer that reads unified2 files from disk,
+    turning them into events. (Seth Hall)
+
+  * Fixing intel framework tests. (Seth Hall)
+    
+2.1-1059 | 2013-08-13 23:52:41 -0400
+
+  * Add file name support to intel framework. (Seth Hall)
+
+  * Add file support to intel framework and slightly restructure 
+    intel http handling. (Seth Hall)
+
+2.1-1052 | 2013-08-12 14:38:14 -0700
+
+  * Fixing bug in DNP3 analyzer flagged by compiler warning. (Robin
+    Sommer)
+
+2.1-1050 | 2013-08-12 11:37:44 -0700
+
+  * Experimental DNP3 analyzer. This includes only very basic
+    script-level support at the moment, but quite a number of events
+    are provided. (Hui Lin, Robin Sommer)
+
 2.1-1041 | 2013-08-09 15:32:22 -0700
 
   * Update coverage baselines for canonical load order of scripts.

NEWS

@@ -17,6 +17,10 @@ New Functionality
 
 - GridFTP support. TODO: Extend.
 
+- Modbus support. TODO: Extend.
+
+- DNP3 support. TODO: Extend.
+
 - ssl.log now also records the subject client and issuer certificates.
 
 - Hooks: TODO: Briefly summarize the documention from

VERSION

@@ -1 +1 @@
-2.1-1041
+2.1-1128

aux/binpac

@@ -1 +1 @@
-Subproject commit 00674ed07d702252b00675b5060647d9e811cdd7
+Subproject commit ff22fa7299d460d9c4b7f6c7269e2d3aafa06b21

aux/bro-aux

@@ -1 +1 @@
-Subproject commit 0e2d74e488195170e4648037e22b51e122dc7b0e
+Subproject commit 0b6fce59fc503fb22d87d5f0cdfeb162885189e2

aux/broccoli

@@ -1 +1 @@
-Subproject commit 7ddfa3212d1fd0822588d4a96158f1a30c755afe
+Subproject commit a0b18634e5840dca59bd6ca4c486a761f37c076e

aux/broctl

@@ -1 +1 @@
-Subproject commit c9293bad3bf4d6fc3e1808a315e791140a632961
+Subproject commit e94b0d7f831fc58abf84dcddafa143cb95e068f2

aux/btest

@@ -1 +1 @@
-Subproject commit 69606f8f3cc84d694ca1da14868a5fecd4abbc96
+Subproject commit 35bb074c1c5173e44689df680a24ba13fea39a11

doc/scripts/DocSourcesList.cmake

@@ -31,10 +31,13 @@ rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_BitTorrent.events.b
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ConnSize.events.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DCE_RPC.events.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DHCP.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DNP3.events.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DNS.events.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FTP.events.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FTP.functions.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_File.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FileExtract.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FileExtract.functions.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FileHash.events.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Finger.events.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_GTPv1.events.bif.bro)
@@ -70,6 +73,8 @@ rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_TCP.events.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_TCP.functions.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Teredo.events.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_UDP.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Unified2.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Unified2.types.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ZIP.events.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/reporter.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/strings.bif.bro)
@@ -77,6 +82,7 @@ rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/top-k.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/types.bif.bro)
 rest_target(${psd} base/files/extract/main.bro)
 rest_target(${psd} base/files/hash/main.bro)
+rest_target(${psd} base/files/unified2/main.bro)
 rest_target(${psd} base/frameworks/analyzer/main.bro)
 rest_target(${psd} base/frameworks/cluster/main.bro)
 rest_target(${psd} base/frameworks/cluster/nodes/manager.bro)
@@ -142,6 +148,8 @@ rest_target(${psd} base/protocols/conn/polling.bro)
 rest_target(${psd} base/protocols/dhcp/consts.bro)
 rest_target(${psd} base/protocols/dhcp/main.bro)
 rest_target(${psd} base/protocols/dhcp/utils.bro)
+rest_target(${psd} base/protocols/dnp3/consts.bro)
+rest_target(${psd} base/protocols/dnp3/main.bro)
 rest_target(${psd} base/protocols/dns/consts.bro)
 rest_target(${psd} base/protocols/dns/main.bro)
 rest_target(${psd} base/protocols/ftp/files.bro)
@@ -196,9 +204,10 @@ rest_target(${psd} policy/frameworks/files/hash-all-files.bro)
 rest_target(${psd} policy/frameworks/intel/do_notice.bro)
 rest_target(${psd} policy/frameworks/intel/seen/conn-established.bro)
 rest_target(${psd} policy/frameworks/intel/seen/dns.bro)
-rest_target(${psd} policy/frameworks/intel/seen/http-host-header.bro)
+rest_target(${psd} policy/frameworks/intel/seen/file-hashes.bro)
+rest_target(${psd} policy/frameworks/intel/seen/file-names.bro)
+rest_target(${psd} policy/frameworks/intel/seen/http-headers.bro)
 rest_target(${psd} policy/frameworks/intel/seen/http-url.bro)
-rest_target(${psd} policy/frameworks/intel/seen/http-user-agents.bro)
 rest_target(${psd} policy/frameworks/intel/seen/smtp-url-extraction.bro)
 rest_target(${psd} policy/frameworks/intel/seen/smtp.bro)
 rest_target(${psd} policy/frameworks/intel/seen/ssl.bro)
@@ -257,6 +266,7 @@ rest_target(${psd} policy/protocols/ssl/extract-certs-pem.bro)
 rest_target(${psd} policy/protocols/ssl/known-certs.bro)
 rest_target(${psd} policy/protocols/ssl/notary.bro)
 rest_target(${psd} policy/protocols/ssl/validate-certs.bro)
+rest_target(${psd} policy/tuning/defaults/extracted_file_limits.bro)
 rest_target(${psd} policy/tuning/defaults/packet-fragments.bro)
 rest_target(${psd} policy/tuning/defaults/warnings.bro)
 rest_target(${psd} policy/tuning/logs-to-elasticsearch.bro)

scripts/base/files/extract/main.bro

@@ -7,6 +7,10 @@ export {
 	## The prefix where files are extracted to.
 	const prefix = "./extract_files/" &redef;
 
+	## The default max size for extracted files (they won't exceed this
+	## number of bytes), unlimited.
+	const default_limit = 0 &redef;
+
 	redef record Files::Info += {
 		## Local filenames of extracted file.
 		extracted: string &optional &log;
@@ -17,7 +21,30 @@ export {
 		## This field is used in the core by the extraction plugin
 		## to know where to write the file to.  It's also optional
 		extract_filename: string &optional;
+		## The maximum allowed file size in bytes of *extract_filename*.
+		## Once reached, a :bro:see:`file_extraction_limit` event is
+		## raised and the analyzer will be removed unless
+		## :bro:see:`FileExtract::set_limit` is called to increase the
+		## limit.  A value of zero means "no limit".
+		extract_limit: count &default=default_limit;
 	};
+
+	## Sets the maximum allowed extracted file size.
+	##
+	## f: A file that's being extracted.
+	##
+	## args: Arguments that identify a file extraction analyzer.
+	##
+	## n: Allowed number of bytes to be extracted.
+	##
+	## Returns: false if a file extraction analyzer wasn't active for
+	##          the file, else true.
+	global set_limit: function(f: fa_file, args: Files::AnalyzerArgs, n: count): bool;
+}
+
+function set_limit(f: fa_file, args: Files::AnalyzerArgs, n: count): bool
+	{
+	return __set_limit(f$id, args, n);
 	}
 
 function on_add(f: fa_file, args: Files::AnalyzerArgs)
@@ -27,12 +54,10 @@ function on_add(f: fa_file, args: Files::AnalyzerArgs)
 
 	f$info$extracted = args$extract_filename;
 	args$extract_filename = build_path_compressed(prefix, args$extract_filename);
+	mkdir(prefix);
 	}
 
 event bro_init() &priority=10
 	{
 	Files::register_analyzer_add_callback(Files::ANALYZER_EXTRACT, on_add);
-
-	# Create the extraction directory.
-	mkdir(prefix);
 	}

scripts/base/files/unified2/__load__.bro

@@ -0,0 +1 @@
+@load ./main

scripts/base/files/unified2/main.bro

@@ -0,0 +1,244 @@
+
+@load base/utils/dir
+@load base/utils/paths
+
+module Unified2;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	## Directory to watch for Unified2 files.
+	const watch_file = "" &redef;
+
+	## File to watch for Unified2 records.
+	const watch_dir = "" &redef;
+
+	## The sid-msg.map file you would like to use for your alerts.
+	const sid_msg = "" &redef;
+
+	## The gen-msg.map file you would like to use for your alerts.
+	const gen_msg = "" &redef;
+
+	## The classification.config file you would like to use for your alerts.
+	const classification_config = "" &redef;
+
+	## Reconstructed "alert" which combines related events
+	## and packets.
+	global alert: event(f: fa_file, ev: Unified2::IDSEvent, pkt: Unified2::Packet);
+
+	type PacketID: record {
+		src_ip: addr;
+		src_p: port;
+		dst_ip: addr;
+		dst_p: port;
+	} &log;
+
+	type Info: record {
+		## Timestamp attached to the alert.
+		ts:                 time     &log;
+		## Addresses and ports for the connection.
+		id:                 PacketID &log;
+		## Sensor that originated this event.
+		sensor_id:          count    &log;
+		## Sig id for this generator.
+		signature_id:       count    &log;
+		## A string representation of the "signature_id" field if a sid_msg.map file was loaded.
+		signature:          string   &log &optional;
+		## Which generator generated the alert?
+		generator_id:       count    &log;
+		## A string representation of the "generator_id" field if a gen_msg.map file was loaded.
+		generator:          string   &log &optional;
+		## Sig revision for this id.
+		signature_revision: count    &log;
+		## Event classification.
+		classification_id:  count    &log;
+		## Descriptive classification string,
+		classification:     string   &log &optional;
+		## Event priority.
+		priority_id:        count    &log;
+		## Event ID.
+		event_id:           count    &log;
+		## Some of the packet data.
+		packet:             string   &log &optional;
+	} &log;
+
+	## The event for accessing logged records.
+	global log_unified2: event(rec: Info);
+}
+
+# Mappings for extended information from alerts.
+global classification_map: table[count] of string;
+global sid_map: table[count] of string;
+global gen_map: table[count] of string;
+
+# For reading in config files.
+type OneLine: record {
+	line: string;
+};
+
+function create_info(ev: IDSEvent): Info
+	{
+	local info = Info($ts=ev$ts,
+	                  $id=PacketID($src_ip=ev$src_ip, $src_p=ev$src_p,
+	                               $dst_ip=ev$dst_ip, $dst_p=ev$dst_p),
+	                  $sensor_id=ev$sensor_id,
+	                  $signature_id=ev$signature_id,
+	                  $generator_id=ev$generator_id,
+	                  $signature_revision=ev$signature_revision,
+	                  $classification_id=ev$classification_id,
+	                  $priority_id=ev$priority_id,
+	                  $event_id=ev$event_id);
+
+	if ( ev$signature_id in sid_map )
+		info$signature=sid_map[ev$signature_id];
+	if ( ev$generator_id in gen_map )
+		info$generator=gen_map[ev$generator_id];
+	if ( ev$classification_id in classification_map )
+		info$classification=classification_map[ev$classification_id];
+
+	return info;
+	}
+
+redef record fa_file += {
+	## Recently received IDS events.  This is primarily used
+	## for tying together Unified2 events and packets.
+	u2_events: table[count] of Unified2::IDSEvent
+		&optional &create_expire=5sec
+		&expire_func=function(t: table[count] of Unified2::IDSEvent, event_id: count): interval
+			{
+			Log::write(LOG, create_info(t[event_id]));
+			return 0secs;
+			};
+};
+
+event Unified2::read_sid_msg_line(desc: Input::EventDescription, tpe: Input::Event, line: string)
+	{
+	local parts = split_n(line, / \|\| /, F, 100);
+	if ( |parts| >= 2 && /^[0-9]+$/ in parts[1] )
+		sid_map[to_count(parts[1])] = parts[2];
+	}
+
+event Unified2::read_gen_msg_line(desc: Input::EventDescription, tpe: Input::Event, line: string)
+	{
+	local parts = split_n(line, / \|\| /, F, 3);
+	if ( |parts| >= 2 && /^[0-9]+$/ in parts[1] )
+		gen_map[to_count(parts[1])] = parts[3];
+	}
+
+event Unified2::read_classification_line(desc: Input::EventDescription, tpe: Input::Event, line: string)
+	{
+	local parts = split_n(line, /: /, F, 2);
+	if ( |parts| == 2 )
+		{
+		local parts2 = split_n(parts[2], /,/, F, 4);
+		if ( |parts2| > 1 )
+			classification_map[|classification_map|+1] = parts2[1];
+		}
+	}
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(Unified2::LOG, [$columns=Info, $ev=log_unified2]);
+
+	if ( sid_msg != "" )
+		{
+		Input::add_event([$source=sid_msg,
+		                  $reader=Input::READER_RAW,
+		                  $mode=Input::REREAD,
+		                  $name=sid_msg,
+		                  $fields=Unified2::OneLine,
+		                  $want_record=F,
+		                  $ev=Unified2::read_sid_msg_line]);
+		}
+
+	if ( gen_msg != "" )
+		{
+		Input::add_event([$source=gen_msg,
+		                  $name=gen_msg,
+		                  $reader=Input::READER_RAW,
+		                  $mode=Input::REREAD,
+		                  $fields=Unified2::OneLine,
+		                  $want_record=F,
+		                  $ev=Unified2::read_gen_msg_line]);
+		}
+
+	if ( classification_config != "" )
+		{
+		Input::add_event([$source=classification_config,
+		                  $name=classification_config,
+		                  $reader=Input::READER_RAW,
+		                  $mode=Input::REREAD,
+		                  $fields=Unified2::OneLine,
+		                  $want_record=F,
+		                  $ev=Unified2::read_classification_line]);
+		}
+
+	if ( watch_dir != "" )
+		{
+		Dir::monitor(watch_dir, function(fname: string)
+			{
+			Input::add_analysis([$source=fname,
+			                     $reader=Input::READER_BINARY,
+			                     $mode=Input::STREAM,
+			                     $name=fname]);
+			}, 10secs);
+		}
+
+	if ( watch_file != "" )
+		{
+		Input::add_analysis([$source=watch_file,
+		                     $reader=Input::READER_BINARY,
+		                     $mode=Input::STREAM,
+		                     $name=watch_file]);
+		}
+	}
+
+event file_new(f: fa_file)
+	{
+	local file_dir = "";
+	local parts = split_all(f$source, /\/[^\/]*$/);
+	if ( |parts| == 3 )
+		file_dir = parts[1];
+
+	if ( (watch_file != "" && f$source == watch_file) || 
+	     (watch_dir != "" && compress_path(watch_dir) == file_dir) )
+		{
+		Files::add_analyzer(f, Files::ANALYZER_UNIFIED2);
+		f$u2_events = table();
+		}
+	}
+
+event unified2_event(f: fa_file, ev: Unified2::IDSEvent)
+	{
+	f$u2_events[ev$event_id] = ev;
+	}
+
+event unified2_packet(f: fa_file, pkt: Unified2::Packet)
+	{
+	if ( f?$u2_events && pkt$event_id in f$u2_events)
+		{
+		local ev = f$u2_events[pkt$event_id];
+		event Unified2::alert(f, ev, pkt);
+		delete f$u2_events[pkt$event_id];
+		}
+	}
+
+event Unified2::alert(f: fa_file, ev: IDSEvent, pkt: Packet)
+	{
+	local info = create_info(ev);
+	info$packet=pkt$data;
+	Log::write(LOG, info);
+	}
+
+event file_state_remove(f: fa_file)
+	{
+	if ( f?$u2_events )
+		{
+		# In case any events never had matching packets, flush
+		# the extras to the log.
+		for ( i in f$u2_events )
+			{
+			Log::write(LOG, create_info(f$u2_events[i]));
+			}
+		}
+	}

scripts/base/frameworks/intel/main.bro

@@ -27,6 +27,9 @@ export {
 		## File hash which is non-hash type specific.  It's up to the user to query
 		## for any relevant hash types.
 		FILE_HASH,
+		## File names.  Typically with protocols with definite indications
+		## of a file name.
+		FILE_NAME,
 		## Certificate SHA-1 hash.
 		CERT_HASH,
 	};
@@ -80,6 +83,10 @@ export {
 		## If the data was discovered within a connection, the 
 		## connection record should go into get to give context to the data.
 		conn:            connection    &optional;
+
+		## If the data was discovered within a file, the file record
+		## should go here to provide context to the data.
+		f:               fa_file       &optional;
 	};
 
 	## Record used for the logging framework representing a positive
@@ -95,6 +102,16 @@ export {
 		## this is the conn_id for the connection.
 		id:       conn_id        &log &optional;
 
+		## If a file was associated with this intelligence hit,
+		## this is the uid for the file.
+		fuid:           string   &log &optional;
+		## A mime type if the intelligence hit is related to a file.  
+		## If the $f field is provided this will be automatically filled out.
+		file_mime_type: string   &log &optional;
+		## Frequently files can be "described" to give a bit more context.
+		## If the $f field is provided this field will be automatically filled out.
+		file_desc:      string   &log &optional;
+
 		## Where the data was seen.
 		seen:     Seen           &log;
 		## Sources which supplied data that resulted in this match.
@@ -248,7 +265,25 @@ function has_meta(check: MetaData, metas: set[MetaData]): bool
 
 event Intel::match(s: Seen, items: set[Item]) &priority=5
 	{
-	local info: Info = [$ts=network_time(), $seen=s];
+	local info = Info($ts=network_time(), $seen=s);
+
+	if ( s?$f )
+		{
+		if ( s$f?$conns && |s$f$conns| == 1 )
+			{
+			for ( cid in s$f$conns )
+				s$conn = s$f$conns[cid];
+			}
+
+		if ( ! info?$fuid )
+			info$fuid = s$f$id;
+
+		if ( ! info?$file_mime_type && s$f?$mime_type )
+			info$file_mime_type = s$f$mime_type;
+
+		if ( ! info?$file_desc )
+			info$file_desc = Files::describe(s$f);
+		}
 
 	if ( s?$conn )
 		{

scripts/base/frameworks/sumstats/cluster.bro

@@ -60,7 +60,7 @@ export {
 
 # Add events to the cluster framework to make this work.
 redef Cluster::manager2worker_events += /SumStats::cluster_(ss_request|get_result|threshold_crossed)/;
-redef Cluster::manager2worker_events += /SumStats::(thresholds_reset|get_a_key)/;
+redef Cluster::manager2worker_events += /SumStats::(get_a_key)/;
 redef Cluster::worker2manager_events += /SumStats::cluster_(ss_response|send_result|key_intermediate_response)/;
 redef Cluster::worker2manager_events += /SumStats::(send_a_key|send_no_key)/;
 
@@ -95,37 +95,6 @@ function data_added(ss: SumStat, key: Key, result: Result)
 		}
 	}
 
-#event SumStats::send_data(uid: string, ss_name: string, cleanup: bool)
-#	{
-#	#print fmt("WORKER %s: sending data for uid %s...", Cluster::node, uid);
-#
-#	local local_data: ResultTable = table();
-#	local incoming_data: ResultTable = cleanup ? data : copy(data);
-#
-#	local num_added = 0;
-#	for ( key in incoming_data )
-#		{
-#		local_data[key] = incoming_data[key];
-#		delete incoming_data[key];
-#
-#		# Only send cluster_send_in_groups_of at a time.  Queue another
-#		# event to send the next group.
-#		if ( cluster_send_in_groups_of == ++num_added )
-#			break;
-#		}
-#
-#	local done = F;
-#	# If data is empty, this sumstat is done.
-#	if ( |incoming_data| == 0 )
-#		done = T;
-#
-#	# Note: copy is needed to compensate serialization caching issue. This should be
-#	# changed to something else later. 
-#	event SumStats::cluster_ss_response(uid, ss_name, copy(local_data), done, cleanup);
-#	if ( ! done )
-#		schedule 0.01 sec { SumStats::send_data(uid, T) };
-#	}
-
 event SumStats::get_a_key(uid: string, ss_name: string, cleanup: bool)
 	{
 	if ( uid in sending_results )
@@ -204,6 +173,8 @@ event SumStats::cluster_get_result(uid: string, ss_name: string, key: Key, clean
 		{
 		if ( ss_name in result_store && key in result_store[ss_name] )
 			{
+			# Note: copy is needed to compensate serialization caching issue. This should be
+			# changed to something else later. 
 			event SumStats::cluster_send_result(uid, ss_name, key, copy(result_store[ss_name][key]), cleanup);
 			}
 		else
@@ -223,11 +194,6 @@ event SumStats::cluster_threshold_crossed(ss_name: string, key: SumStats::Key, t
 	threshold_tracker[ss_name][key] = thold_index;
 	}
 
-event SumStats::thresholds_reset(ss_name: string)
-	{
-	delete threshold_tracker[ss_name];
-	}
-
 @endif
 
 
@@ -236,7 +202,12 @@ event SumStats::thresholds_reset(ss_name: string)
 # This variable is maintained by manager nodes as they collect and aggregate
 # results.
 # Index on a uid.
-global stats_keys: table[string] of set[Key] &create_expire=1min;
+global stats_keys: table[string] of set[Key] &create_expire=1min 
+	&expire_func=function(s: table[string] of set[Key], idx: string): interval
+		{
+		Reporter::warning(fmt("SumStat key request for the %s SumStat uid took longer than 1 minute and was automatically cancelled.", idx));
+		return 0secs;
+		};
 
 # This variable is maintained by manager nodes to track how many "dones" they
 # collected per collection unique id.  Once the number of results for a uid
@@ -251,11 +222,15 @@ global done_with: table[string] of count &create_expire=1min &default=0;
 # Indexed on a uid.
 global key_requests: table[string] of Result &create_expire=1min;
 
+# Store uids for dynamic requests here to avoid cleanup on the uid.
+# (This needs to be done differently!)
+global dynamic_requests: set[string] &create_expire=1min;
+
 # This variable is maintained by managers to prevent overwhelming communication due
 # to too many intermediate updates.  Each sumstat is tracked separately so that
 # one won't overwhelm and degrade other quieter sumstats.
 # Indexed on a sumstat id.
-global outstanding_global_views: table[string] of count &create_expire=1min &default=0;
+global outstanding_global_views: table[string] of count &read_expire=1min &default=0;
 
 const zero_time = double_to_time(0.0);
 # Managers handle logging.
@@ -274,6 +249,7 @@ event SumStats::finish_epoch(ss: SumStat)
 		event SumStats::cluster_ss_request(uid, ss$name, T);
 
 		done_with[uid] = 0;
+
 		#print fmt("get_key by uid: %s", uid);
 		event SumStats::get_a_key(uid, ss$name, T);
 		}
@@ -295,6 +271,12 @@ function data_added(ss: SumStat, key: Key, result: Result)
 
 function handle_end_of_result_collection(uid: string, ss_name: string, key: Key, cleanup: bool)
 	{
+	if ( uid !in key_requests )
+		{
+		Reporter::warning(fmt("Tried to handle end of result collection with missing uid in key_request sumstat:%s, key:%s.", ss_name, key));
+		return;
+		}
+
 	#print fmt("worker_count:%d :: done_with:%d", Cluster::worker_count, done_with[uid]);
 	local ss = stats_store[ss_name];
 	local ir = key_requests[uid];
@@ -335,12 +317,6 @@ function request_all_current_keys(uid: string, ss_name: string, cleanup: bool)
 			{
 			done_with[uid] = 0;
 			event SumStats::cluster_get_result(uid, ss_name, key, cleanup);
-			when ( uid in done_with && Cluster::worker_count == done_with[uid] )
-				{
-				#print "done getting result";
-				handle_end_of_result_collection(uid, ss_name, key, cleanup);
-				request_all_current_keys(uid, ss_name, cleanup);
-				}
 			delete stats_keys[uid][key];
 			break; # only a single key
 			}
@@ -357,12 +333,16 @@ function request_all_current_keys(uid: string, ss_name: string, cleanup: bool)
 event SumStats::send_no_key(uid: string, ss_name: string)
 	{
 	#print "send_no_key";
+
+	if ( uid !in done_with )
+		done_with[uid] = 0;
+
 	++done_with[uid];
 	if ( Cluster::worker_count == done_with[uid] )
 		{
 		delete done_with[uid];
 
-		if ( |stats_keys[uid]| > 0 )
+		if ( uid in stats_keys && |stats_keys[uid]| > 0 )
 			{
 			#print "we need more keys!";
 			# Now that we have a key from each worker, lets
@@ -375,6 +355,9 @@ event SumStats::send_no_key(uid: string, ss_name: string)
 			local ss = stats_store[ss_name];
 			if ( ss?$epoch_finished )
 				ss$epoch_finished(network_time());
+
+			delete stats_keys[uid];
+			reset(ss);
 			}
 		}
 	}
@@ -384,7 +367,7 @@ event SumStats::send_a_key(uid: string, ss_name: string, key: Key)
 	#print fmt("send_a_key %s", key);
 	if ( uid !in stats_keys )
 		{
-		# no clue what happened here
+		Reporter::warning(fmt("Manager received a uid for an unknown request.  SumStat: %s, Key: %s", ss_name, key));
 		return;
 		}
 
@@ -409,6 +392,8 @@ event SumStats::send_a_key(uid: string, ss_name: string, key: Key)
 			local ss = stats_store[ss_name];
 			if ( ss?$epoch_finished )
 				ss$epoch_finished(network_time());
+
+			reset(ss);
 			}
 		}
 	}
@@ -426,20 +411,27 @@ event SumStats::cluster_send_result(uid: string, ss_name: string, key: Key, resu
 		key_requests[uid] = compose_results(key_requests[uid], result);
 
 	# Mark that a worker is done.
+	if ( uid !in done_with )
+		done_with[uid] = 0;
+	
+	#print fmt("MANAGER: got a result for %s %s from %s", uid, key, get_event_peer()$descr);
 	++done_with[uid];
 
-	#if ( Cluster::worker_count == done_with[uid] )
+	if ( uid !in dynamic_requests &&
-	#	{
+	     uid in done_with && Cluster::worker_count == done_with[uid] )
-	#	print "done";
+		{
-	#	handle_end_of_result_collection(uid, ss_name, key, cleanup);
+		handle_end_of_result_collection(uid, ss_name, key, cleanup);
-	#	}
+
+		if ( cleanup )
+			request_all_current_keys(uid, ss_name, cleanup);
+		}
 	}
 
 # Managers handle intermediate updates here.
 event SumStats::cluster_key_intermediate_response(ss_name: string, key: Key)
 	{
 	#print fmt("MANAGER: receiving intermediate key data from %s", get_event_peer()$descr);
-	#print fmt("MANAGER: requesting key data for %s", key2str(key));
+	#print fmt("MANAGER: requesting key data for %s", key);
 
 	if ( ss_name in outstanding_global_views &&
 	     |outstanding_global_views[ss_name]| > max_outstanding_global_views )
@@ -454,110 +446,16 @@ event SumStats::cluster_key_intermediate_response(ss_name: string, key: Key)
 
 	local uid = unique_id("");
 	done_with[uid] = 0;
+	#print fmt("requesting results for: %s", uid);
 	event SumStats::cluster_get_result(uid, ss_name, key, F);
-	when ( uid in done_with && Cluster::worker_count == done_with[uid] )
-		{
-		handle_end_of_result_collection(uid, ss_name, key, F);
 	}
-	timeout 1.1min
-		{
-		Reporter::warning(fmt("Dynamic SumStat intermediate key request for %s (%s) took longer than 1 minute and was automatically cancelled.", ss_name, key));
-		}
-
-	}
-
-#event SumStats::cluster_ss_response(uid: string, ss_name: string, data: ResultTable, done: bool, cleanup: bool)
-#	{
-#	#print fmt("MANAGER: receiving results from %s", get_event_peer()$descr);
-#
-#	# Mark another worker as being "done" for this uid.
-#	if ( done )
-#		++done_with[uid];
-#
-#	# We had better only be getting requests for stuff that exists.
-#	if ( ss_name !in stats_store )
-#		return;
-#
-#	if ( uid !in stats_keys )
-#		stats_keys[uid] = table();
-#
-#	local local_data = stats_keys[uid];
-#	local ss = stats_store[ss_name];
-#
-#	for ( key in data )
-#		{
-#		if ( key in local_data )
-#			local_data[key] = compose_results(local_data[key], data[key]);
-#		else
-#			local_data[key] = data[key];
-#
-#		# If a stat is done being collected, thresholds for each key
-#		# need to be checked so we're doing it here to avoid doubly
-#		# iterating over each key.
-#		if ( Cluster::worker_count == done_with[uid] )
-#			{
-#			if ( check_thresholds(ss, key, local_data[key], 1.0) )
-#				{
-#				threshold_crossed(ss, key, local_data[key]);
-#				event SumStats::cluster_threshold_crossed(ss$name, key, threshold_tracker[ss$name][key]);
-#				}
-#			}
-#		}
-#
-#	# If the data has been collected from all peers, we are done and ready to finish.
-#	if ( cleanup && Cluster::worker_count == done_with[uid] )
-#		{
-#		local now = network_time();
-#		if ( ss?$epoch_result )
-#			{
-#			for ( key in local_data )
-#				ss$epoch_result(now, key, local_data[key]);
-#			}
-#
-#		if ( ss?$epoch_finished )
-#			ss$epoch_finished(now);
-#
-#		# Clean up
-#		delete stats_keys[uid];
-#		delete done_with[uid];
-#		reset(ss);
-#		}
-#	}
-
-#function request(ss_name: string): ResultTable
-#	{
-#	# This only needs to be implemented this way for cluster compatibility.
-#	local uid = unique_id("dyn-");
-#	stats_keys[uid] = table();
-#	done_with[uid] = 0;
-#	event SumStats::cluster_ss_request(uid, ss_name, F);
-#
-#	return when ( uid in done_with && Cluster::worker_count == done_with[uid] )
-#		{
-#		if ( uid in stats_keys )
-#			{
-#			local ss_result = stats_keys[uid];
-#			# Clean up
-#			delete stats_keys[uid];
-#			delete done_with[uid];
-#			reset(stats_store[ss_name]);
-#			return ss_result;
-#			}
-#		else
-#			return table();
-#		}
-#	timeout 1.1min
-#		{
-#		Reporter::warning(fmt("Dynamic SumStat request for %s took longer than 1 minute and was automatically cancelled.", ss_name));
-#		return table();
-#		}
-#	}
 
 function request_key(ss_name: string, key: Key): Result
 	{
 	local uid = unique_id("");
 	done_with[uid] = 0;
 	key_requests[uid] = table();
+	add dynamic_requests[uid];
 
 	event SumStats::cluster_get_result(uid, ss_name, key, F);
 	return when ( uid in done_with && Cluster::worker_count == done_with[uid] )
@@ -567,13 +465,14 @@ function request_key(ss_name: string, key: Key): Result
 		# Clean up
 		delete key_requests[uid];
 		delete done_with[uid];
+		delete dynamic_requests[uid];
 
 		return result;
 		}
 	timeout 1.1min
 		{
-		Reporter::warning(fmt("Dynamic SumStat key request for %s (%s) took longer than 1 minute and was automatically cancelled.", ss_name, key));
+		Reporter::warning(fmt("Dynamic SumStat key request for %s in SumStat %s took longer than 1 minute and was automatically cancelled.", key, ss_name));
-		return table();
+		return Result();
 		}
 	}
 

scripts/base/frameworks/sumstats/main.bro

@@ -153,11 +153,6 @@ export {
 	## Returns: The result for the requested sumstat key.
 	global request_key: function(ss_name: string, key: Key): Result;
 
-	## This event is generated when thresholds are reset for a SumStat.
-	##
-	## name: SumStats name that thresholds were reset for.
-	global thresholds_reset: event(name: string);
-
 	## Helper function to represent a :bro:type:`SumStats::Key` value as
 	## a simple string.
 	##
@@ -321,7 +316,6 @@ function reset(ss: SumStat)
 		{
 		delete threshold_tracker[ss$name];
 		threshold_tracker[ss$name] = table();
-		event SumStats::thresholds_reset(ss$name);
 		}
 	}
 

scripts/base/init-bare.bro

@@ -2685,6 +2685,42 @@ type ModbusHeaders: record {
 	function_code: count;
 };
 
+module Unified2;
+export {
+	type Unified2::IDSEvent: record {
+		sensor_id:          count;
+		event_id:           count;
+		ts:                 time;
+		signature_id:       count;
+		generator_id:       count;
+		signature_revision: count;
+		classification_id:  count;
+		priority_id:        count;
+		src_ip:             addr;
+		dst_ip:             addr;
+		src_p:              port;
+		dst_p:              port;
+		impact_flag:        count;
+		impact:             count;
+		blocked:            count;
+		## Not available in "legacy" IDS events.
+		mpls_label:         count  &optional;
+		## Not available in "legacy" IDS events.
+		vlan_id:            count  &optional;
+		## Only available in "legacy" IDS events.
+		packet_action:      count  &optional;
+	};
+
+	type Unified2::Packet: record {
+		sensor_id:    count;
+		event_id:     count;
+		event_second: count;
+		packet_ts:    time;
+		link_type:    count;
+		data:         string;
+	};
+}
+
 module SOCKS;
 export {
 	## This record is for a SOCKS client or server to provide either a

scripts/base/init-default.bro

@@ -40,6 +40,7 @@
 
 @load base/protocols/conn
 @load base/protocols/dhcp
+@load base/protocols/dnp3
 @load base/protocols/dns
 @load base/protocols/ftp
 @load base/protocols/http
@@ -55,5 +56,7 @@
 
 @load base/files/hash
 @load base/files/extract
+@load base/files/unified2
+
 
 @load base/misc/find-checksum-offloading

scripts/base/protocols/dnp3/__load__.bro

@@ -0,0 +1,3 @@
+@load ./main
+
+@load-sigs ./dpd.sig

scripts/base/protocols/dnp3/consts.bro

@@ -0,0 +1,49 @@
+
+module DNP3;
+
+export {
+	## Standard defined Modbus function codes.
+	const function_codes = {
+		# Requests.
+		[0x00] = "CONFIRM",
+		[0x01] = "READ",
+		[0x02] = "WRITE",
+		[0x03] = "SELECT",
+		[0x04] = "OPERATE",
+		[0x05] = "DIRECT_OPERATE",
+		[0x06] = "DIRECT_OPERATE_NR",
+		[0x07] = "IMMED_FREEZE",
+		[0x08] = "IMMED_FREEZE_NR",
+		[0x09] = "FREEZE_CLEAR",
+		[0x0a] = "FREEZE_CLEAR_NR",
+		[0x0b] = "FREEZE_AT_TIME",
+		[0x0c] = "FREEZE_AT_TIME_NR",
+		[0x0d] = "COLD_RESTART",
+		[0x0e] = "WARM_RESTART",
+		[0x0f] = "INITIALIZE_DATA",
+		[0x10] = "INITIALIZE_APPL",
+		[0x11] = "START_APPL",
+		[0x12] = "STOP_APPL",
+		[0x13] = "SAVE_CONFIG",
+		[0x14] = "ENABLE_UNSOLICITED",
+		[0x15] = "DISABLE_UNSOLICITED",
+		[0x16] = "ASSIGN_CLASS",
+		[0x17] = "DELAY_MEASURE",
+		[0x18] = "RECORD_CURRENT_TIME",
+		[0x19] = "OPEN_FILE",
+		[0x1a] = "CLOSE_FILE",
+		[0x1b] = "DELETE_FILE",
+		[0x1c] = "GET_FILE_INFO",
+		[0x1d] = "AUTHENTICATE_FILE",
+		[0x1e] = "ABORT_FILE",
+		[0x1f] = "ACTIVATE_CONFIG",
+		[0x20] = "AUTHENTICATE_REQ",
+		[0x21] = "AUTHENTICATE_ERR",
+
+		# Responses.
+		[0x81] = "RESPONSE",
+		[0x82] = "UNSOLICITED_RESPONSE",
+		[0x83] = "AUTHENTICATE_RESP",
+	} &default=function(i: count):string { return fmt("unknown-%d", i); } &redef;
+}
+

scripts/base/protocols/dnp3/dpd.sig

@@ -0,0 +1,9 @@
+
+# DNP3 packets always starts with 0x05 0x64 .
+
+signature dpd_dnp3_server {
+	ip-proto == tcp
+	payload /\x05\x64/
+	tcp-state responder
+ 	enable "dnp3"
+}

scripts/base/protocols/dnp3/main.bro

@@ -0,0 +1,73 @@
+##! A very basic DNP3 analysis script that just logs requests and replies.
+
+module DNP3;
+
+@load ./consts
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		## Time of the request.
+		ts:         time           &log;
+		## Unique identifier for the connnection.
+		uid:        string         &log;
+		## Identifier for the connection.
+		id:         conn_id        &log;
+		## The name of the function message in the request.
+		fc_request: string         &log &optional;
+		## The name of the function message in the reply.
+		fc_reply:   string         &log &optional;
+		## The response's "internal indication number".
+		iin:        count          &log &optional;
+	};
+
+	## Event that can be handled to access the DNP3 record as it is sent on
+	## to the logging framework.
+	global log_dnp3: event(rec: Info);
+}
+
+redef record connection += {
+	dnp3: Info &optional;
+};
+
+const ports = { 20000/tcp };
+redef likely_server_ports += { ports };
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(DNP3::LOG, [$columns=Info, $ev=log_dnp3]);
+	Analyzer::register_for_ports(Analyzer::ANALYZER_DNP3, ports);
+	}
+
+event dnp3_application_request_header(c: connection, is_orig: bool, fc: count)
+	{
+	if ( ! c?$dnp3 )
+		c$dnp3 = [$ts=network_time(), $uid=c$uid, $id=c$id];
+
+	c$dnp3$ts = network_time();
+	c$dnp3$fc_request = function_codes[fc];
+	}
+
+event dnp3_application_response_header(c: connection, is_orig: bool, fc: count, iin: count)
+	{
+	if ( ! c?$dnp3 )
+		c$dnp3 = [$ts=network_time(), $uid=c$uid, $id=c$id];
+
+	c$dnp3$ts = network_time();
+	c$dnp3$fc_reply = function_codes[fc];
+	c$dnp3$iin = iin;
+
+	Log::write(LOG, c$dnp3);
+
+	delete c$dnp3;
+	}
+
+event connection_state_remove(c: connection) &priority=-5
+	{
+	if ( ! c?$dnp3 )
+		return;
+
+	Log::write(LOG, c$dnp3);
+	delete c$dnp3;
+	}

scripts/base/protocols/smtp/entities.bro

@@ -27,7 +27,9 @@ export {
 
 event mime_begin_entity(c: connection) &priority=10
 	{
+	if ( c?$smtp )
 		c$smtp$entity = Entity();
+	if ( c?$smtp_state )
 		++c$smtp_state$mime_depth;
 	}
 

scripts/policy/frameworks/intel/seen/__load__.bro

@@ -1,8 +1,9 @@
 @load ./conn-established
 @load ./dns
-@load ./http-host-header
+@load ./file-hashes
+@load ./file-names
+@load ./http-headers
 @load ./http-url
-@load ./http-user-agents
 @load ./ssl
 @load ./smtp
 @load ./smtp-url-extraction

scripts/policy/frameworks/intel/seen/file-hashes.bro

@@ -0,0 +1,12 @@
+@load base/frameworks/intel
+@load ./where-locations
+
+event file_hash(f: fa_file, kind: string, hash: string)
+	{
+	local seen = Intel::Seen($indicator=hash,
+	                         $indicator_type=Intel::FILE_HASH,
+	                         $f=f,
+	                         $where=Files::IN_HASH);
+	
+	Intel::seen(seen);
+	}

scripts/policy/frameworks/intel/seen/file-names.bro

@@ -0,0 +1,11 @@
+@load base/frameworks/intel
+@load ./where-locations
+
+event file_new(f: fa_file)
+	{
+	if ( f?$info && f$info?$filename )
+		Intel::seen([$indicator=f$info$filename,
+		             $indicator_type=Intel::FILE_NAME,
+		             $f=f,
+		             $where=Files::IN_NAME]);
+	}

scripts/policy/frameworks/intel/seen/http-headers.bro

@@ -0,0 +1,46 @@
+@load base/frameworks/intel
+@load ./where-locations
+
+event http_header(c: connection, is_orig: bool, name: string, value: string)
+	{
+	if ( is_orig )
+		{
+		switch ( name ) 
+			{
+			case "HOST": 
+			Intel::seen([$indicator=value,
+			             $indicator_type=Intel::DOMAIN,
+			             $conn=c,
+			             $where=HTTP::IN_HOST_HEADER]);
+			break;
+
+			case "REFERER":
+			Intel::seen([$indicator=sub(value, /^.*:\/\//, ""),
+			             $indicator_type=Intel::URL,
+			             $conn=c,
+			             $where=HTTP::IN_REFERRER_HEADER]);
+			break;
+
+			case "X-FORWARDED-FOR":
+			if ( is_valid_ip(value) )
+				{
+				local addrs = find_ip_addresses(value);
+				for ( i in addrs )
+					{
+					Intel::seen([$host=to_addr(addrs[i]),
+					             $indicator_type=Intel::ADDR,
+					             $conn=c,
+					             $where=HTTP::IN_X_FORWARDED_FOR_HEADER]);
+					}
+				}
+			break;
+
+			case "USER-AGENT":
+			Intel::seen([$indicator=value,
+			             $indicator_type=Intel::SOFTWARE,
+			             $conn=c,
+			             $where=HTTP::IN_USER_AGENT_HEADER]);
+			break;
+			}
+		}
+	}

scripts/policy/frameworks/intel/seen/http-host-header.bro

@@ -1,11 +0,0 @@
-@load base/frameworks/intel
-@load ./where-locations
-
-event http_header(c: connection, is_orig: bool, name: string, value: string)
-	{
-	if ( is_orig && name == "HOST" )
-		Intel::seen([$indicator=value,
-		             $indicator_type=Intel::DOMAIN,
-		             $conn=c,
-		             $where=HTTP::IN_HOST_HEADER]);
-	}

scripts/policy/frameworks/intel/seen/http-user-agents.bro

@@ -1,12 +0,0 @@
-@load base/frameworks/intel
-@load ./where-locations
-
-event http_header(c: connection, is_orig: bool, name: string, value: string)
-	{
-	if ( is_orig && name == "USER-AGENT" )
-		Intel::seen([$indicator=value,
-		             $indicator_type=Intel::SOFTWARE,
-		             $conn=c,
-		             $where=HTTP::IN_USER_AGENT_HEADER]);
-	}
-

scripts/policy/frameworks/intel/seen/where-locations.bro

@@ -4,10 +4,14 @@ export {
 	redef enum Intel::Where += {
 		Conn::IN_ORIG,
 		Conn::IN_RESP,
+		Files::IN_HASH,
+		Files::IN_NAME,
 		DNS::IN_REQUEST,
 		DNS::IN_RESPONSE,
 		HTTP::IN_HOST_HEADER,
+		HTTP::IN_REFERRER_HEADER,
 		HTTP::IN_USER_AGENT_HEADER,
+		HTTP::IN_X_FORWARDED_FOR_HEADER,
 		HTTP::IN_URL,
 		SMTP::IN_MAIL_FROM,
 		SMTP::IN_RCPT_TO,

scripts/policy/tuning/defaults/__load__.bro

@@ -1,2 +1,3 @@
 @load ./packet-fragments
 @load ./warnings
+@load ./extracted_file_limits.bro

scripts/policy/tuning/defaults/extracted_file_limits.bro

@@ -0,0 +1,4 @@
+@load base/files/extract
+
+# 100 MB.
+redef FileExtract::default_limit = 104857600;

scripts/test-all-policy.bro

@@ -18,9 +18,10 @@
 @load frameworks/intel/seen/__load__.bro
 @load frameworks/intel/seen/conn-established.bro
 @load frameworks/intel/seen/dns.bro
-@load frameworks/intel/seen/http-host-header.bro
+@load frameworks/intel/seen/file-hashes.bro
+@load frameworks/intel/seen/file-names.bro
+@load frameworks/intel/seen/http-headers.bro
 @load frameworks/intel/seen/http-url.bro
-@load frameworks/intel/seen/http-user-agents.bro
 @load frameworks/intel/seen/smtp-url-extraction.bro
 @load frameworks/intel/seen/smtp.bro
 @load frameworks/intel/seen/ssl.bro
@@ -88,6 +89,7 @@
 @load protocols/ssl/validate-certs.bro
 @load tuning/__load__.bro
 @load tuning/defaults/__load__.bro
+@load tuning/defaults/extracted_file_limits.bro
 @load tuning/defaults/packet-fragments.bro
 @load tuning/defaults/warnings.bro
 @load tuning/logs-to-elasticsearch.bro

src/3rdparty

@@ -0,0 +1 @@
+Subproject commit 12b5cb446c8128bb22e5cbd7baa7d53669539487

src/Timer.cc

@@ -16,6 +16,7 @@ const char* TimerNames[] = {
 	"ConnectionInactivityTimer",
 	"ConnectionStatusUpdateTimer",
 	"DNSExpireTimer",
+	"FileAnalysisInactivityTimer",
 	"FragTimer",
 	"IncrementalSendTimer",
 	"IncrementalWriteTimer",

src/Timer.h

@@ -23,7 +23,6 @@ enum TimerType {
 	TIMER_CONN_STATUS_UPDATE,
 	TIMER_DNS_EXPIRE,
 	TIMER_FILE_ANALYSIS_INACTIVITY,
-	TIMER_FILE_ANALYSIS_DRAIN,
 	TIMER_FRAG,
 	TIMER_INCREMENTAL_SEND,
 	TIMER_INCREMENTAL_WRITE,

src/Trigger.cc

@@ -44,7 +44,10 @@ TraversalCode TriggerTraversalCallback::PreExpr(const Expr* expr)
 		BroObj::SuppressErrors no_errors;
 		Val* v = e->Eval(trigger->frame);
 		if ( v )
+			{
 			trigger->Register(v);
+			Unref(v);
+			}
 		break;
 		}
 

src/analyzer/protocol/CMakeLists.txt

@@ -6,6 +6,7 @@ add_subdirectory(bittorrent)
 add_subdirectory(conn-size)
 add_subdirectory(dce-rpc)
 add_subdirectory(dhcp)
+add_subdirectory(dnp3)
 add_subdirectory(dns)
 add_subdirectory(file)
 add_subdirectory(finger)

src/analyzer/protocol/dnp3/CMakeLists.txt

@@ -0,0 +1,10 @@
+
+include(BroPlugin)
+
+include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
+
+bro_plugin_begin(Bro DNP3)
+bro_plugin_cc(DNP3.cc Plugin.cc)
+bro_plugin_bif(events.bif)
+bro_plugin_pac(dnp3.pac dnp3-analyzer.pac dnp3-protocol.pac dnp3-objects.pac)
+bro_plugin_end()

src/analyzer/protocol/dnp3/DNP3.cc

@@ -0,0 +1,376 @@
+//
+// DNP3 was initially used over serial links; it defined its own application
+// layer, transport layer, and data link layer. This hierarchy cannot be
+// mapped to the TCP/IP stack directly. As a result, all three DNP3 layers
+// are packed together as a single application layer payload over the TCP
+// layer. Each DNP3 packet in the application layer may look like this DNP3
+// Packet:
+//
+//    DNP3 Link Layer | DNP3 Transport Layer | DNP3 Application Layer
+//
+//    (This hierarchy can be viewed in the Wireshark visually.)
+//
+// === Background on DNP3
+//
+// 1. Basic structure of DNP3 Protocol over serial links. This information
+//    can be found in detail in
+//
+//        DNP3 Specification Volume 2, Part 1 Basic, Application Layer
+//        DNP3 Specification Volume 4, Data Link Layer
+//
+//    Traditionally, the DNP3 Application Layer in serial links contains a
+//    "DNP3 Application Layer Fragment". The data that is parsed by the end
+//    device and then executed. As the "DNP3 Application Layer Fragment" can
+//    be long (>255 bytes), it may be trunkcated and carried in different
+//    DNP3 Application Layer of more than one DNP3 packets.
+//
+//    So we may find a long DNP3 Application Layer Fragment to be transmitted in the following format
+//
+//        DNP3 Packet #1 : DNP3 Link Layer | DNP3 Transport Layer | DNP3 Application Layer #1
+//        DNP3 Packet #2 : DNP3 Link Layer | DNP3 Transport Layer | DNP3 Application Layer #2
+//        ....
+//        DNP3 Packet #n : DNP3 Link Layer | DNP3 Transport Layer | DNP3 Application Layer #n
+//
+//    So to get the whole DNP3 application layer fragment, we concatenate
+//    each DNP3 Application Layer Data into a logic DNP3 Application Layer
+//    Fragment:
+//
+//       DNP3 Application Layer #1 + DNP3 Application Layer #2 + ... + DNP3 Application Layer #n
+//
+// 2. Packing DNP3 Network Packet into TCP/IP stack
+//
+// We will call the original DNP3 Link Layer, Transport Layer and Application
+// Layer used in serial link as Pseudo Link Layer, Pseudo Transport Layer and
+// Pseudo Application Layer.
+//
+// For a long DNP3 application layer fragment, we may find it tramistted
+// over IP network in the following format:
+//
+//     Network Packet #1 : TCP Header | DNP3 Pseudo Link Layer | DNP3 Pseudo Transport Layer | DNP3 Pseudo Application Layer #1
+//     Network Packet #2 : TCP Header | DNP3 Pseudo Link Layer | DNP3 Pseudo Transport Layer | DNP3 Pseudo Application Layer #2
+//     ....
+//     Network Packet #n : TCP Header | DNP3 Pseudo Link Layer | DNP3 Pseudo Transport Layer | DNP3 Pseudo Application Layer #n
+//
+// === Challenges of Writing DNP3 Analyzer on Binpac ===
+//
+// The detailed structure of the DNP3 Link Layer is:
+//
+//     0x05 0x64 Len Ctrl Dest_LSB Dest_MSB Src_LSB Src_MSB CRC_LSB CRC_MSB
+//
+//     Each field is a byte; LSB: least significant byte; MSB: most significatn byte.
+//
+//     "Len" indicates the length of the byte stream right after this field
+//     (excluding CRC fields) in the current DNP3 packet.
+//
+// Since "Len" is of size one byte, the largest length it can represent is
+// 255 bytes. The larget DNP3 Application Layer size is "255 - 5 + size of
+// all CRC fields". "minus 5" is coming from the 5 bytes after "Len" field in
+// the DNP3 Link Layer, i.e. Ctrl Dest_LSB Dest_MSB Src_LSB Src_MSB Hence,
+// the largest size of a DNP3 Packet (DNP3 Data Link Layer : DNP3 Transport
+// Layer : DNP3 Application Layer) can only be 292 bytes.
+//
+// The "Len" field indicates the length of of a single chunk of DNP3 Psuedo
+// Application Layer data instead of the whole DNP3 Application Layer
+// Fragment. However, we can not know the whole length of the DNP3
+// Application Layer Fragment (which Binpac would normally need) until all
+// chunks of Pseudo Application Layer Data are received.
+//
+// We hence exploit the internal flow_buffer class used in Binpac to buffer
+// the application layer data until all chunk are received, which does
+// require a bit of internal knowledge of the generated code.
+//
+// The binpac analyzer parses the DNP3 Application Layer Fragment. However,
+// we manually add the original Pseudo Link Layer data as an additional
+// header before the DNP3 Application Fragment. This helps to know how many
+// bytes are in the current chunk of DNP3 application layer data (not the
+// whole Application Layer Fragment).
+//
+// Graphically, the procedure is:
+//
+// DNP3 Packet :  DNP3 Pseudo Data Link Layer : DNP3 Pseudo Transport Layer : DNP3 Pseudo Application Layer
+//                                   ||                                    ||
+//                                   || (length field)                     || (original paylad byte stream)
+//                                   \/                                    \/
+//                DNP3 Additional Header              :                  Reassembled DNP3 Pseudo Application Layer Data
+//                                                   ||
+//                                                   \/
+//                                            Binpac DNP3 Analyzer
+
+#include "DNP3.h"
+#include "analyzer/protocol/tcp/TCP_Reassembler.h"
+#include "events.bif.h"
+
+using namespace analyzer::dnp3;
+
+const unsigned int PSEUDO_LENGTH_INDEX = 2;		// index of len field of DNP3 Pseudo Link Layer
+const unsigned int PSEUDO_CONTROL_FIELD_INDEX = 3;	// index of ctrl field of DNP3 Pseudo Link Layer
+const unsigned int PSEUDO_TRANSPORT_INDEX = 10;		// index of DNP3 Pseudo Transport Layer
+const unsigned int PSEUDO_APP_LAYER_INDEX = 11;		// index of first DNP3 app-layer byte.
+const unsigned int PSEUDO_TRANSPORT_LEN = 1;		// length of DNP3 Transport Layer
+const unsigned int PSEUDO_LINK_LAYER_LEN = 8;		// length of DNP3 Pseudo Link Layer
+
+bool DNP3_Analyzer::crc_table_initialized = false;
+unsigned int DNP3_Analyzer::crc_table[256];
+
+DNP3_Analyzer::DNP3_Analyzer(Connection* c) : TCP_ApplicationAnalyzer("DNP3", c)
+	{
+	interp = new binpac::DNP3::DNP3_Conn(this);
+
+	ClearEndpointState(true);
+	ClearEndpointState(false);
+
+	if ( ! crc_table_initialized )
+		PrecomputeCRCTable();
+	}
+
+DNP3_Analyzer::~DNP3_Analyzer()
+	{
+	delete interp;
+	}
+
+void DNP3_Analyzer::Done()
+	{
+	TCP_ApplicationAnalyzer::Done();
+
+	interp->FlowEOF(true);
+	interp->FlowEOF(false);
+	}
+
+void DNP3_Analyzer::DeliverStream(int len, const u_char* data, bool orig)
+	{
+	TCP_ApplicationAnalyzer::DeliverStream(len, data, orig);
+
+	try
+		{
+		if ( ! ProcessData(len, data, orig) )
+			SetSkip(1);
+		}
+
+	catch ( const binpac::Exception& e )
+		{
+		SetSkip(1);
+		throw;
+		}
+	}
+
+void DNP3_Analyzer::Undelivered(int seq, int len, bool orig)
+	{
+	TCP_ApplicationAnalyzer::Undelivered(seq, len, orig);
+	interp->NewGap(orig, len);
+	}
+
+void DNP3_Analyzer::EndpointEOF(bool is_orig)
+	{
+	TCP_ApplicationAnalyzer::EndpointEOF(is_orig);
+	interp->FlowEOF(is_orig);
+	}
+
+bool DNP3_Analyzer::ProcessData(int len, const u_char* data, bool orig)
+	{
+	Endpoint* endp = orig ? &orig_state : &resp_state;
+
+	while ( len )
+		{
+		if ( endp->in_hdr )
+			{
+			// We're parsing the DNP3 header and link layer, get that in full.
+			if ( ! AddToBuffer(endp, PSEUDO_APP_LAYER_INDEX, &data, &len) )
+				return true;
+
+			// The first two bytes must always be 0x0564.
+			if( endp->buffer[0] != 0x05 || endp->buffer[1] != 0x64 )
+				{
+				Weird("dnp3_header_lacks_magic");
+				return false;
+				}
+
+			// Make sure header checksum is correct.
+			if ( ! CheckCRC(PSEUDO_LINK_LAYER_LEN, endp->buffer, endp->buffer + PSEUDO_LINK_LAYER_LEN, "header") )
+				{
+				ProtocolViolation("broken_checksum");
+				return false;
+				}
+
+			// If the checksum works out, we're pretty certainly DNP3.
+			ProtocolConfirmation();
+
+			// DNP3 packets without transport and application
+			// layers can happen, we ignore them.
+			if ( (endp->buffer[PSEUDO_LENGTH_INDEX] + 3) == (char)PSEUDO_LINK_LAYER_LEN  )
+				{
+				ClearEndpointState(orig);
+				return true;
+				}
+
+			// Double check the direction in case the first
+			// received packet is a response.
+			u_char ctrl = endp->buffer[PSEUDO_CONTROL_FIELD_INDEX];
+
+			if ( orig != (bool)(ctrl & 0x80) )
+				Weird("dnp3_unexpected_flow_direction");
+
+			// Update state.
+			endp->pkt_length = endp->buffer[PSEUDO_LENGTH_INDEX];
+			endp->tpflags = endp->buffer[PSEUDO_TRANSPORT_INDEX];
+			endp->in_hdr = false; // Now parsing application layer.
+
+			// For the first packet, we submit the header to
+			// BinPAC.
+			if ( ++endp->pkt_cnt == 1 )
+				interp->NewData(orig, endp->buffer, endp->buffer + PSEUDO_LINK_LAYER_LEN);
+			}
+
+		if ( ! endp->in_hdr )
+			{
+			assert(endp->pkt_length);
+
+			// We're parsing the DNP3 application layer, get that
+			// in full now as well. We calculate the number of
+			// raw bytes the application layer consists of from
+			// the packet length by determining how much 16-byte
+			// chunks fit in there, and then add 2 bytes CRC for
+			// each.
+			int n = PSEUDO_APP_LAYER_INDEX + (endp->pkt_length - 5) + ((endp->pkt_length - 5) / 16) * 2 + 2 - 1;
+
+			if ( ! AddToBuffer(endp, n, &data, &len) )
+				return true;
+
+			// Parse the the application layer data.
+			if ( ! ParseAppLayer(endp) )
+				return false;
+
+			// Done with this packet, prepare for next.
+			endp->buffer_len = 0;
+			endp->in_hdr = true;
+			}
+		}
+
+	return true;
+	}
+
+bool DNP3_Analyzer::AddToBuffer(Endpoint* endp, int target_len, const u_char** data, int* len)
+	{
+	if ( ! target_len )
+		return true;
+
+	int to_copy = min(*len, target_len - endp->buffer_len);
+
+	memcpy(endp->buffer + endp->buffer_len, *data, to_copy);
+	*data += to_copy;
+	*len -= to_copy;
+	endp->buffer_len += to_copy;
+
+	return endp->buffer_len == target_len;
+	}
+
+bool DNP3_Analyzer::ParseAppLayer(Endpoint* endp)
+	{
+	bool orig = (endp == &orig_state);
+	binpac::DNP3::DNP3_Flow* flow = orig ? interp->upflow() : interp->downflow();
+
+	u_char* data = endp->buffer + PSEUDO_TRANSPORT_INDEX; // The transport layer byte counts as app-layer it seems.
+	int len = endp->pkt_length - 5;
+
+	// DNP3 Packet :  DNP3 Pseudo Link Layer | DNP3 Pseudo Transport Layer | DNP3 Pseudo Application Layer
+	// DNP3 Serial Transport Layer data is always 1 byte.
+	// Get FIN FIR seq field in transport header.
+	// FIR indicate whether the following DNP3 Serial Application Layer is first chunk of bytes or not.
+	// FIN indicate whether the following DNP3 Serial Application Layer is last chunk of bytes or not.
+
+	int is_first = (endp->tpflags & 0x40) >> 6; // Initial chunk of data in this packet.
+	int is_last = (endp->tpflags & 0x80) >> 7; // Last chunk of data in this packet.
+
+	int transport = PSEUDO_TRANSPORT_LEN;
+
+	int i = 0;
+	while ( len > 0 )
+		{
+		int n = min(len, 16);
+
+		// Make sure chunk has a correct checksum.
+		if ( ! CheckCRC(n, data, data + n, "app_chunk") )
+			return false;
+
+		// Pass on to BinPAC.
+		assert(data + n < endp->buffer + endp->buffer_len);
+		flow->flow_buffer()->BufferData(data + transport, data + n);
+		transport = 0;
+
+		data += n + 2;
+		len -= n;
+		}
+
+	if ( is_first )
+		endp->encountered_first_chunk = true;
+
+	if ( ! is_first && ! endp->encountered_first_chunk )
+		{
+		// We lost the first chunk.
+		Weird("dnp3_first_application_layer_chunk_missing");
+		return false;
+		}
+
+	if ( is_last )
+		{
+		flow->flow_buffer()->FinishBuffer();
+		flow->FlowEOF();
+		ClearEndpointState(orig);
+		}
+
+	return true;
+	}
+
+void DNP3_Analyzer::ClearEndpointState(bool orig)
+	{
+	Endpoint* endp = orig ? &orig_state : &resp_state;
+	binpac::DNP3::DNP3_Flow* flow = orig ? interp->upflow() : interp->downflow();
+
+	endp->in_hdr = true;
+	endp->encountered_first_chunk = false;
+	endp->buffer_len = 0;
+	endp->pkt_length = 0;
+	endp->tpflags = 0;
+	endp->pkt_cnt = 0;
+	}
+
+bool DNP3_Analyzer::CheckCRC(int len, const u_char* data, const u_char* crc16, const char* where)
+	{
+	unsigned int crc = CalcCRC(len, data);
+
+	if ( crc16[0] == (crc & 0xff) && crc16[1] == (crc & 0xff00) >> 8 )
+		return true;
+
+	Weird(fmt("dnp3_corrupt_%s_checksum", where));
+	return false;
+	}
+
+void DNP3_Analyzer::PrecomputeCRCTable()
+	{
+	for( unsigned int i = 0; i < 256; i++)
+		{
+		unsigned int crc = i;
+
+		for ( unsigned int j = 0; j < 8; ++j )
+			{
+			if ( crc & 0x0001 )
+				crc = (crc >> 1) ^ 0xA6BC; // Generating polynomial.
+			else
+				crc >>= 1;
+			}
+
+		crc_table[i] = crc;
+		}
+	}
+
+unsigned int DNP3_Analyzer::CalcCRC(int len, const u_char* data)
+	{
+	unsigned int crc = 0x0000;
+
+	for ( int i = 0; i < len; i++ )
+		{
+		unsigned int index = (crc ^ data[i]) & 0xFF;
+		crc = crc_table[index] ^ (crc >> 8);
+		}
+
+	return ~crc & 0xFFFF;
+	}

src/analyzer/protocol/dnp3/DNP3.h

@@ -0,0 +1,56 @@
+
+#ifndef ANALYZER_PROTOCOL_DNP3_DNP3_H
+#define ANALYZER_PROTOCOL_DNP3_DNP3_H
+
+#include "analyzer/protocol/tcp/TCP.h"
+#include "dnp3_pac.h"
+
+namespace analyzer { namespace dnp3 {
+
+class DNP3_Analyzer : public tcp::TCP_ApplicationAnalyzer {
+public:
+	DNP3_Analyzer(Connection* conn);
+	virtual ~DNP3_Analyzer();
+
+	virtual void Done();
+	virtual void DeliverStream(int len, const u_char* data, bool orig);
+	virtual void Undelivered(int seq, int len, bool orig);
+	virtual void EndpointEOF(bool is_orig);
+
+	static Analyzer* InstantiateAnalyzer(Connection* conn)
+		{ return new DNP3_Analyzer(conn); }
+
+private:
+	static const int MAX_BUFFER_SIZE = 300;
+
+	struct Endpoint	{
+		u_char buffer[MAX_BUFFER_SIZE];
+		int buffer_len;
+		bool in_hdr;
+		int tpflags;
+		int pkt_length;
+		int pkt_cnt;
+		bool encountered_first_chunk;
+		};
+
+	bool ProcessData(int len, const u_char* data, bool orig);
+	void ClearEndpointState(bool orig);
+	bool AddToBuffer(Endpoint* endp, int target_len, const u_char** data, int* len);
+	bool ParseAppLayer(Endpoint* endp);
+	bool CheckCRC(int len, const u_char* data, const u_char* crc16, const char* where);
+	unsigned int CalcCRC(int len, const u_char* data);
+
+	binpac::DNP3::DNP3_Conn* interp;
+
+	Endpoint orig_state;
+	Endpoint resp_state;
+
+	static void PrecomputeCRCTable();
+
+	static bool crc_table_initialized;
+	static unsigned int crc_table[256];
+};
+
+} } // namespace analyzer::* 
+
+#endif

src/analyzer/protocol/dnp3/Plugin.cc

@@ -0,0 +1,10 @@
+
+#include "plugin/Plugin.h"
+
+#include "DNP3.h"
+
+BRO_PLUGIN_BEGIN(Bro, DNP3)
+	BRO_PLUGIN_DESCRIPTION("DNP3 analyzer");
+	BRO_PLUGIN_ANALYZER("DNP3", dnp3::DNP3_Analyzer);
+	BRO_PLUGIN_BIF_FILE(events);
+BRO_PLUGIN_END

src/analyzer/protocol/dnp3/dnp3-analyzer.pac

@@ -0,0 +1,969 @@
+
+connection DNP3_Conn(bro_analyzer: BroAnalyzer) {
+	upflow = DNP3_Flow(true);
+	downflow = DNP3_Flow(false);
+};
+
+flow DNP3_Flow(is_orig: bool) {
+	flowunit = DNP3_PDU(is_orig) withcontext (connection, this);
+
+	function get_dnp3_header_block(start: uint16, len: uint16, ctrl: uint8, dest_addr: uint16, src_addr: uint16): bool
+		%{
+		if ( ::dnp3_header_block )
+			{
+			BifEvent::generate_dnp3_header_block(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), start, len, ctrl, dest_addr, src_addr);
+			}
+
+		return true;
+		%}
+
+	function get_dnp3_application_request_header(fc: uint8): bool
+		%{
+		if ( ::dnp3_application_request_header )
+			{
+			BifEvent::generate_dnp3_application_request_header(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(),
+				fc
+				);
+			}
+		return true;
+		%}
+
+	function get_dnp3_application_response_header(fc: uint8, iin: uint16): bool
+		%{
+		if ( ::dnp3_application_response_header )
+			{
+			BifEvent::generate_dnp3_application_response_header(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(),
+				fc,
+				iin
+				);
+			}
+		return true;
+		%}
+
+	function get_dnp3_object_header(obj_type: uint16, qua_field: uint8, number: uint32, rf_low: uint32, rf_high: uint32 ): bool
+		%{
+		if ( ::dnp3_object_header )
+			{
+			BifEvent::generate_dnp3_object_header(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), obj_type, qua_field, number, rf_low, rf_high);
+			}
+
+		return true;
+		%}
+
+	function get_dnp3_object_prefix(prefix_value: uint32): bool
+		%{
+		if ( ::dnp3_object_prefix )
+			{
+			BifEvent::generate_dnp3_object_prefix(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), prefix_value);
+			}
+
+		return true;
+		%}
+
+	function get_dnp3_response_data_object(data_value: uint8): bool
+		%{
+		if ( ::dnp3_response_data_object )
+			{
+			BifEvent::generate_dnp3_response_data_object(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), data_value);
+			}
+
+		return true;
+		%}
+
+	#g0
+	function get_dnp3_attribute_common(data_type_code: uint8, leng: uint8, attribute_obj: const_bytestring): bool
+		%{
+		if ( ::dnp3_attribute_common )
+			{
+			BifEvent::generate_dnp3_attribute_common(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), data_type_code, leng, bytestring_to_val(attribute_obj) );
+			}
+
+		return true;
+		%}
+
+	#g12v1
+	function get_dnp3_crob(control_code: uint8, count8: uint8, on_time: uint32, off_time: uint32, status_code: uint8): bool
+		%{
+		if ( ::dnp3_crob )
+			{
+			BifEvent::generate_dnp3_crob(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), control_code, count8, on_time, off_time, status_code);
+			}
+
+		return true;
+		%}
+
+	#g12v2
+	function get_dnp3_pcb(control_code: uint8, count8: uint8, on_time: uint32, off_time: uint32, status_code: uint8): bool
+		%{
+		if ( ::dnp3_pcb )
+			{
+			BifEvent::generate_dnp3_pcb(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), control_code, count8, on_time, off_time, status_code);
+			}
+
+		return true;
+		%}
+
+	# g20v1
+	function get_dnp3_counter_32wFlag(flag: uint8, count_value: uint32): bool
+		%{
+		if ( ::dnp3_counter_32wFlag )
+			{
+			BifEvent::generate_dnp3_counter_32wFlag(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, count_value);
+			}
+
+		return true;
+		%}
+
+	# g20v2
+	function get_dnp3_counter_16wFlag(flag: uint8, count_value: uint16): bool
+		%{
+		if ( ::dnp3_counter_16wFlag )
+			{
+			BifEvent::generate_dnp3_counter_16wFlag(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, count_value);
+			}
+
+		return true;
+		%}
+
+	# g20v5
+	function get_dnp3_counter_32woFlag(count_value: uint32): bool
+		%{
+		if ( ::dnp3_counter_32woFlag )
+			{
+			BifEvent::generate_dnp3_counter_32woFlag(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), count_value);
+			}
+
+		return true;
+		%}
+
+	# g20v6
+	function get_dnp3_counter_16woFlag(count_value: uint16): bool
+		%{
+		if ( ::dnp3_counter_16woFlag )
+			{
+			BifEvent::generate_dnp3_counter_16woFlag(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), count_value);
+			}
+
+		return true;
+		%}
+
+	# g21v1
+	function get_dnp3_frozen_counter_32wFlag(flag: uint8, count_value: uint32): bool
+		%{
+		if ( ::dnp3_frozen_counter_32wFlag )
+			{
+			BifEvent::generate_dnp3_frozen_counter_32wFlag(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, count_value);
+			}
+
+		return true;
+		%}
+
+	# g21v2
+	function get_dnp3_frozen_counter_16wFlag(flag: uint8, count_value: uint16): bool
+		%{
+		if ( ::dnp3_frozen_counter_16wFlag )
+			{
+			BifEvent::generate_dnp3_frozen_counter_16wFlag(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, count_value);
+			}
+
+		return true;
+		%}
+
+	# g21v5
+	function get_dnp3_frozen_counter_32wFlagTime(flag: uint8, count_value: uint32, time48: const_bytestring): bool
+		%{
+		if ( ::dnp3_frozen_counter_32wFlagTime )
+			{
+			BifEvent::generate_dnp3_frozen_counter_32wFlagTime(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, count_value, bytestring_to_val(time48));
+			}
+
+		return true;
+		%}
+
+	# g21v6
+	function get_dnp3_frozen_counter_16wFlagTime(flag: uint8, count_value: uint16, time48: const_bytestring): bool
+		%{
+		if ( ::dnp3_frozen_counter_16wFlagTime )
+			{
+			BifEvent::generate_dnp3_frozen_counter_16wFlagTime(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, count_value, bytestring_to_val(time48));
+			}
+
+		return true;
+		%}
+
+	# g21v9
+	function get_dnp3_frozen_counter_32woFlag(count_value: uint32): bool
+		%{
+		if ( ::dnp3_frozen_counter_32woFlag )
+			{
+			BifEvent::generate_dnp3_frozen_counter_32woFlag(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), count_value);
+			}
+
+		return true;
+		%}
+
+	# g21v10
+	function get_dnp3_frozen_counter_16woFlag(count_value: uint16): bool
+		%{
+		if ( ::dnp3_frozen_counter_16woFlag )
+			{
+			BifEvent::generate_dnp3_frozen_counter_16woFlag(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), count_value);
+			}
+
+		return true;
+		%}
+
+	# g30v1
+	function get_dnp3_analog_input_32wFlag(flag: uint8, value: int32): bool
+		%{
+		if ( ::dnp3_analog_input_32wFlag )
+			{
+			BifEvent::generate_dnp3_analog_input_32wFlag(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, value);
+			}
+
+		return true;
+		%}
+
+	# g30v2
+	function get_dnp3_analog_input_16wFlag(flag: uint8, value: int16): bool
+		%{
+		if ( ::dnp3_analog_input_16wFlag )
+			{
+			BifEvent::generate_dnp3_analog_input_16wFlag(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, value);
+			}
+
+		return true;
+		%}
+
+	# g30v3
+	function get_dnp3_analog_input_32woFlag(value: int32): bool
+		%{
+		if ( ::dnp3_analog_input_32woFlag )
+			{
+			BifEvent::generate_dnp3_analog_input_32woFlag(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), value);
+			}
+
+		return true;
+		%}
+
+	#g30v4
+	function get_dnp3_analog_input_16woFlag(value: int16): bool
+		%{
+		if ( ::dnp3_analog_input_16woFlag )
+			{
+			BifEvent::generate_dnp3_analog_input_16woFlag(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), value);
+			}
+
+		return true;
+		%}
+
+	# g30v5
+	function get_dnp3_analog_input_SPwFlag(flag: uint8, value: uint32): bool
+		%{
+		if ( ::dnp3_analog_input_SPwFlag )
+			{
+			BifEvent::generate_dnp3_analog_input_SPwFlag(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, value);
+			}
+
+		return true;
+		%}
+
+	# g30v6
+	function get_dnp3_analog_input_DPwFlag(flag: uint8, value_low: uint32, value_high: uint32): bool
+		%{
+		if ( ::dnp3_analog_input_DPwFlag )
+			{
+			BifEvent::generate_dnp3_analog_input_DPwFlag(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, value_low, value_high);
+			}
+
+		return true;
+		%}
+
+	# g31v1
+	function get_dnp3_frozen_analog_input_32wFlag(flag: uint8, frozen_value: int32): bool
+		%{
+		if ( ::dnp3_frozen_analog_input_32wFlag )
+			{
+			BifEvent::generate_dnp3_frozen_analog_input_32wFlag(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, frozen_value);
+			}
+
+		return true;
+		%}
+
+	# g31v2
+	function get_dnp3_frozen_analog_input_16wFlag(flag: uint8, frozen_value: int16): bool
+		%{
+		if ( ::dnp3_frozen_analog_input_16wFlag )
+			{
+			BifEvent::generate_dnp3_frozen_analog_input_16wFlag(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, frozen_value);
+			}
+
+		return true;
+		%}
+
+	# g31v3
+	function get_dnp3_frozen_analog_input_32wTime(flag: uint8, frozen_value: int32, time48: const_bytestring): bool
+		%{
+		if ( ::dnp3_frozen_analog_input_32wTime )
+			{
+			BifEvent::generate_dnp3_frozen_analog_input_32wTime(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, frozen_value, bytestring_to_val(time48));
+			}
+
+		return true;
+		%}
+
+	# g31v4
+	function get_dnp3_frozen_analog_input_16wTime(flag: uint8, frozen_value: int16, time48: const_bytestring): bool
+		%{
+		if ( ::dnp3_frozen_analog_input_16wTime )
+			{
+			BifEvent::generate_dnp3_frozen_analog_input_16wTime(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, frozen_value, bytestring_to_val(time48));
+			}
+
+		return true;
+		%}
+
+	# g31v5
+	function get_dnp3_frozen_analog_input_32woFlag(frozen_value: int32): bool
+		%{
+		if ( ::dnp3_frozen_analog_input_32woFlag )
+			{
+			BifEvent::generate_dnp3_frozen_analog_input_32woFlag(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), frozen_value);
+			}
+
+		return true;
+		%}
+
+	# g31v6
+	function get_dnp3_frozen_analog_input_16woFlag(frozen_value: int16): bool
+		%{
+		if ( ::dnp3_frozen_analog_input_16woFlag )
+			{
+			BifEvent::generate_dnp3_frozen_analog_input_16woFlag(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), frozen_value);
+			}
+
+		return true;
+		%}
+
+	# g31v7
+	function get_dnp3_frozen_analog_input_SPwFlag(flag: uint8, frozen_value: uint32): bool
+		%{
+		if ( ::dnp3_frozen_analog_input_SPwFlag )
+			{
+			BifEvent::generate_dnp3_frozen_analog_input_SPwFlag(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, frozen_value);
+			}
+
+		return true;
+		%}
+
+	# g31v8
+	function get_dnp3_frozen_analog_input_DPwFlag(flag: uint8, frozen_value_low: uint32, frozen_value_high: uint32): bool
+		%{
+		if ( ::dnp3_frozen_analog_input_DPwFlag )
+			{
+			BifEvent::generate_dnp3_frozen_analog_input_DPwFlag(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, frozen_value_low, frozen_value_high);
+			}
+
+		return true;
+		%}
+
+	# g32v1
+	function get_dnp3_analog_input_event_32woTime(flag: uint8, value: int32): bool
+		%{
+		if ( ::dnp3_analog_input_event_32woTime )
+			{
+			BifEvent::generate_dnp3_analog_input_event_32woTime(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, value);
+			}
+
+		return true;
+		%}
+
+	# g32v2
+	function get_dnp3_analog_input_event_16woTime(flag: uint8, value: int16): bool
+		%{
+		if ( ::dnp3_analog_input_event_16woTime )
+			{
+			BifEvent::generate_dnp3_analog_input_event_16woTime(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, value);
+			}
+
+		return true;
+		%}
+
+	# g32v3
+	function get_dnp3_analog_input_event_32wTime(flag: uint8, value: int32, time48: const_bytestring): bool
+		%{
+		if ( ::dnp3_analog_input_event_32wTime )
+			{
+			BifEvent::generate_dnp3_analog_input_event_32wTime(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, value, bytestring_to_val(time48));
+			}
+
+		return true;
+		%}
+
+	# g32v4
+	function get_dnp3_analog_input_event_16wTime(flag: uint8, value: int16, time48: const_bytestring): bool
+		%{
+		if ( ::dnp3_analog_input_event_16wTime )
+			{
+			BifEvent::generate_dnp3_analog_input_event_16wTime(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, value, bytestring_to_val(time48));
+			}
+
+		return true;
+		%}
+
+	# g32v5
+	function get_dnp3_analog_input_event_SPwoTime(flag: uint8, value: uint32): bool
+		%{
+		if ( ::dnp3_analog_input_event_SPwoTime )
+			{
+			BifEvent::generate_dnp3_analog_input_event_SPwoTime(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, value);
+			}
+
+		return true;
+		%}
+
+	# g32v6
+	function get_dnp3_analog_input_event_DPwoTime(flag: uint8, value_low: uint32, value_high: uint32): bool
+		%{
+		if ( ::dnp3_analog_input_event_DPwoTime )
+			{
+			BifEvent::generate_dnp3_analog_input_event_DPwoTime(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, value_low, value_high);
+			}
+
+		return true;
+		%}
+
+	# g32v7
+	function get_dnp3_analog_input_event_SPwTime(flag: uint8, value: uint32, time48: const_bytestring): bool
+		%{
+		if ( ::dnp3_analog_input_event_SPwTime )
+			{
+			BifEvent::generate_dnp3_analog_input_event_SPwTime(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, value, bytestring_to_val(time48));
+			}
+
+		return true;
+		%}
+
+	# g32v8
+	function get_dnp3_analog_input_event_DPwTime(flag: uint8, value_low: uint32, value_high: uint32, time48: const_bytestring): bool
+		%{
+		if ( ::dnp3_analog_input_event_DPwTime )
+			{
+			BifEvent::generate_dnp3_analog_input_event_DPwTime(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, value_low, value_high, bytestring_to_val(time48));
+			}
+
+		return true;
+		%}
+
+	# g33v1
+	function get_dnp3_frozen_analog_input_event_32woTime(flag: uint8, frozen_value: int32): bool
+		%{
+		if ( ::dnp3_frozen_analog_input_event_32woTime )
+			{
+			BifEvent::generate_dnp3_frozen_analog_input_event_32woTime(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, frozen_value);
+			}
+
+		return true;
+		%}
+
+	# g33v2
+	function get_dnp3_frozen_analog_input_event_16woTime(flag: uint8, frozen_value: int16): bool
+		%{
+		if ( ::dnp3_frozen_analog_input_event_16woTime )
+			{
+			BifEvent::generate_dnp3_frozen_analog_input_event_16woTime(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, frozen_value);
+			}
+
+		return true;
+		%}
+
+	# g33v3
+	function get_dnp3_frozen_analog_input_event_32wTime(flag: uint8, frozen_value: int32, time48: const_bytestring): bool
+		%{
+		if ( ::dnp3_frozen_analog_input_event_32wTime )
+			{
+			BifEvent::generate_dnp3_frozen_analog_input_event_32wTime(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, frozen_value, bytestring_to_val(time48));
+			}
+
+		return true;
+		%}
+
+	# g33v4
+	function get_dnp3_frozen_analog_input_event_16wTime(flag: uint8, frozen_value: int16, time48: const_bytestring): bool
+		%{
+		if ( ::dnp3_frozen_analog_input_event_16wTime )
+			{
+			BifEvent::generate_dnp3_frozen_analog_input_event_16wTime(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, frozen_value, bytestring_to_val(time48));
+			}
+
+		return true;
+		%}
+
+	# g33v5
+	function get_dnp3_frozen_analog_input_event_SPwoTime(flag: uint8, frozen_value: uint32): bool
+		%{
+		if ( ::dnp3_frozen_analog_input_event_SPwoTime )
+			{
+			BifEvent::generate_dnp3_frozen_analog_input_event_SPwoTime(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, frozen_value);
+			}
+
+		return true;
+		%}
+
+	# g33v6
+	function get_dnp3_frozen_analog_input_event_DPwoTime(flag: uint8, frozen_value_low: uint32, frozen_value_high: uint32): bool
+		%{
+		if ( ::dnp3_frozen_analog_input_event_DPwoTime )
+			{
+			BifEvent::generate_dnp3_frozen_analog_input_event_DPwoTime(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, frozen_value_low, frozen_value_high);
+			}
+
+		return true;
+		%}
+
+	# g33v7
+	function get_dnp3_frozen_analog_input_event_SPwTime(flag: uint8, frozen_value: uint32, time48: const_bytestring): bool
+		%{
+		if ( ::dnp3_frozen_analog_input_event_SPwTime )
+			{
+			BifEvent::generate_dnp3_frozen_analog_input_event_SPwTime(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, frozen_value, bytestring_to_val(time48));
+			}
+
+		return true;
+		%}
+
+	# g33v8
+	function get_dnp3_frozen_analog_input_event_DPwTime(flag: uint8, frozen_value_low: uint32, frozen_value_high: uint32, time48: const_bytestring): bool
+		%{
+		if ( ::dnp3_frozen_analog_input_event_DPwTime )
+			{
+			BifEvent::generate_dnp3_frozen_analog_input_event_DPwTime(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), flag, frozen_value_low, frozen_value_high, bytestring_to_val(time48));
+			}
+
+		return true;
+		%}
+
+	# g70v5
+	function get_dnp3_file_transport(file_handle: uint32, block_num: uint32, file_data: const_bytestring): bool
+		%{
+		if ( ::dnp3_file_transport )
+			{
+			BifEvent::generate_dnp3_file_transport(
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), file_handle, block_num, bytestring_to_val(file_data));
+			}
+
+		return true;
+		%}
+
+#### for debug use or unknown data types used in "case"
+	function get_dnp3_debug_byte(debug: const_bytestring): bool
+		%{
+		if ( ::dnp3_debug_byte )
+			{
+			BifEvent::generate_dnp3_debug_byte (
+				connection()->bro_analyzer(),
+				connection()->bro_analyzer()->Conn(),
+				is_orig(), bytestring_to_val(debug));
+			}
+
+		return true;
+		%}
+
+};
+
+refine typeattr Header_Block += &let {
+	get_header: bool =  $context.flow.get_dnp3_header_block(start, len, ctrl, dest_addr, src_addr);
+};
+
+refine typeattr DNP3_Application_Request_Header += &let {
+	process_request: bool =  $context.flow.get_dnp3_application_request_header(function_code);
+};
+
+refine typeattr DNP3_Application_Response_Header += &let {
+	process_request: bool =  $context.flow.get_dnp3_application_response_header(function_code, internal_indications);
+};
+
+refine typeattr Object_Header += &let {
+	process_request: bool =  $context.flow.get_dnp3_object_header(object_type_field, qualifier_field, number_of_item, rf_value_low, rf_value_high);
+};
+
+refine typeattr Prefix_Type += &let {
+	prefix_called: bool =  $context.flow.get_dnp3_object_prefix(prefix_value);
+};
+
+refine typeattr Response_Data_Object += &let {
+	process_request: bool =  $context.flow.get_dnp3_response_data_object(data_value);
+};
+
+# g0
+refine typeattr AttributeCommon += &let {
+	process_request: bool =  $context.flow.get_dnp3_attribute_common(data_type_code, leng, attribute_obj);
+};
+
+# g12v1
+refine typeattr CROB += &let {
+	process_request: bool =  $context.flow.get_dnp3_crob(control_code, count, on_time, off_time, status_code);
+};
+
+# g12v2
+refine typeattr PCB += &let {
+	process_request: bool =  $context.flow.get_dnp3_pcb(control_code, count, on_time, off_time, status_code);
+};
+
+# g20v1
+refine typeattr Counter32wFlag += &let {
+	process_request: bool =  $context.flow.get_dnp3_counter_32wFlag(flag, count_value);
+};
+
+# g20v2
+refine typeattr Counter16wFlag += &let {
+	process_request: bool =  $context.flow.get_dnp3_counter_16wFlag(flag, count_value);
+};
+
+# g20v5
+refine typeattr Counter32woFlag += &let {
+	process_request: bool =  $context.flow.get_dnp3_counter_32woFlag(count_value);
+};
+
+# g20v6
+refine typeattr Counter16woFlag += &let {
+	process_request: bool =  $context.flow.get_dnp3_counter_16woFlag(count_value);
+};
+
+# g21v1
+refine typeattr FrozenCounter32wFlag += &let {
+	process_request: bool =  $context.flow.get_dnp3_frozen_counter_32wFlag(flag, count_value);
+};
+
+# g21v2
+refine typeattr FrozenCounter16wFlag += &let {
+	process_request: bool =  $context.flow.get_dnp3_frozen_counter_16wFlag(flag, count_value);
+};
+# g21v5
+refine typeattr FrozenCounter32wFlagTime += &let {
+	process_request: bool =  $context.flow.get_dnp3_frozen_counter_32wFlagTime(flag, count_value, time48);
+};
+
+# g21v6
+refine typeattr FrozenCounter16wFlagTime += &let {
+	process_request: bool =  $context.flow.get_dnp3_frozen_counter_16wFlagTime(flag, count_value, time48);
+};
+
+# g21v9
+refine typeattr FrozenCounter32woFlag += &let {
+	process_request: bool =  $context.flow.get_dnp3_frozen_counter_32woFlag(count_value);
+};
+
+# g21v10
+refine typeattr FrozenCounter16woFlag += &let {
+	process_request: bool =  $context.flow.get_dnp3_frozen_counter_16woFlag(count_value);
+};
+
+# g30v1
+refine typeattr AnalogInput32wFlag += &let {
+	process_request: bool =  $context.flow.get_dnp3_analog_input_32wFlag(flag, value);
+};
+
+# g30v2
+refine typeattr AnalogInput16wFlag += &let {
+	process_request: bool =  $context.flow.get_dnp3_analog_input_16wFlag(flag, value);
+};
+
+# g30v3
+refine typeattr AnalogInput32woFlag += &let {
+	process_request: bool =  $context.flow.get_dnp3_analog_input_32woFlag(value);
+};
+
+# g30v4
+refine typeattr AnalogInput16woFlag += &let {
+	process_request: bool =  $context.flow.get_dnp3_analog_input_16woFlag(value);
+};
+
+# g30v5
+refine typeattr AnalogInputSPwFlag += &let {
+	process_request: bool =  $context.flow.get_dnp3_analog_input_SPwFlag(flag, value);
+};
+
+# g30v6
+refine typeattr AnalogInputDPwFlag += &let {
+	process_request: bool =  $context.flow.get_dnp3_analog_input_DPwFlag(flag, value_low, value_high);
+};
+
+# g31v1
+refine typeattr FrozenAnalogInput32wFlag += &let {
+	process_request: bool =  $context.flow.get_dnp3_frozen_analog_input_32wFlag(flag, frozen_value);
+};
+
+# g31v2
+refine typeattr FrozenAnalogInput16wFlag += &let {
+	process_request: bool =  $context.flow.get_dnp3_frozen_analog_input_16wFlag(flag, frozen_value);
+};
+
+# g31v3
+refine typeattr FrozenAnalogInput32wTime += &let {
+	process_request: bool =  $context.flow.get_dnp3_frozen_analog_input_32wTime(flag, frozen_value, time48);
+};
+
+# g31v4
+refine typeattr FrozenAnalogInput16wTime += &let {
+	process_request: bool =  $context.flow.get_dnp3_frozen_analog_input_16wTime(flag, frozen_value, time48);
+};
+
+# g31v5
+refine typeattr FrozenAnalogInput32woFlag += &let {
+	process_request: bool =  $context.flow.get_dnp3_frozen_analog_input_32woFlag(frozen_value);
+};
+
+# g31v6
+refine typeattr FrozenAnalogInput16woFlag += &let {
+	process_request: bool =  $context.flow.get_dnp3_frozen_analog_input_16woFlag(frozen_value);
+};
+
+# g31v7
+refine typeattr FrozenAnalogInputSPwFlag += &let {
+	process_request: bool =  $context.flow.get_dnp3_frozen_analog_input_SPwFlag(flag, frozen_value);
+};
+
+# g31v8
+refine typeattr FrozenAnalogInputDPwFlag += &let {
+	process_request: bool =  $context.flow.get_dnp3_frozen_analog_input_DPwFlag(flag, frozen_value_low, frozen_value_high);
+};
+
+# g32v1
+refine typeattr AnalogInput32woTime += &let {
+	process_request: bool =  $context.flow.get_dnp3_analog_input_event_32woTime(flag, value);
+};
+
+# g32v2
+refine typeattr AnalogInput16woTime += &let {
+	process_request: bool =  $context.flow.get_dnp3_analog_input_event_16woTime(flag, value);
+};
+
+# g32v3
+refine typeattr AnalogInput32wTime += &let {
+	process_request: bool =  $context.flow.get_dnp3_analog_input_event_32wTime(flag, value, time48);
+};
+
+# g32v4
+refine typeattr AnalogInput16wTime += &let {
+	process_request: bool =  $context.flow.get_dnp3_analog_input_event_16wTime(flag, value, time48);
+};
+
+# g32v5
+refine typeattr AnalogInputSPwoTime += &let {
+	process_request: bool =  $context.flow.get_dnp3_analog_input_event_SPwoTime(flag, value);
+};
+
+# g32v6
+refine typeattr AnalogInputDPwoTime += &let {
+	process_request: bool =  $context.flow.get_dnp3_analog_input_event_DPwoTime(flag, value_low, value_high);
+};
+
+# g32v7
+refine typeattr AnalogInputSPwTime += &let {
+	process_request: bool =  $context.flow.get_dnp3_analog_input_event_SPwTime(flag, value, time48);
+};
+
+# g32v8
+refine typeattr AnalogInputDPwTime += &let {
+	process_request: bool =  $context.flow.get_dnp3_analog_input_event_DPwTime(flag, value_low, value_high, time48);
+};
+
+# g33v1
+refine typeattr FrozenAnaInputEve32woTime += &let {
+	process_request: bool =  $context.flow.get_dnp3_frozen_analog_input_event_32woTime(flag, f_value);
+};
+
+# g33v2
+refine typeattr FrozenAnaInputEve16woTime += &let {
+	process_request: bool =  $context.flow.get_dnp3_frozen_analog_input_event_16woTime(flag, f_value);
+};
+
+# g33v3
+refine typeattr FrozenAnaInputEve32wTime += &let {
+	process_request: bool =  $context.flow.get_dnp3_frozen_analog_input_event_32wTime(flag, f_value, time48);
+};
+
+# g33v4
+refine typeattr FrozenAnaInputEve16wTime += &let {
+	process_request: bool =  $context.flow.get_dnp3_frozen_analog_input_event_16wTime(flag, f_value, time48);
+};
+
+# g33v5
+refine typeattr FrozenAnaInputEveSPwoTime += &let {
+	process_request: bool =  $context.flow.get_dnp3_frozen_analog_input_event_SPwoTime(flag, f_value);
+};
+
+# g33v6
+refine typeattr FrozenAnaInputEveDPwoTime += &let {
+	process_request: bool =  $context.flow.get_dnp3_frozen_analog_input_event_DPwoTime(flag, f_value_low, f_value_high);
+};
+
+# g33v7
+refine typeattr FrozenAnaInputEveSPwTime += &let {
+	process_request: bool =  $context.flow.get_dnp3_frozen_analog_input_event_SPwTime(flag, f_value, time48);
+};
+
+# g33v8
+refine typeattr FrozenAnaInputEveDPwTime += &let {
+	process_request: bool =  $context.flow.get_dnp3_frozen_analog_input_event_DPwTime(flag, f_value_low, f_value_high, time48);
+};
+
+# g70v5
+refine typeattr File_Transport += &let {
+        result: bool =  $context.flow.get_dnp3_file_transport(file_handle, block_num, file_data);
+};
+
+refine typeattr Debug_Byte += &let {
+	process_request: bool =  $context.flow.get_dnp3_debug_byte(debug);
+};
+

src/analyzer/protocol/dnp3/dnp3-protocol.pac

@@ -0,0 +1,257 @@
+#
+# This is Binpac code for DNP3 analyzer by Hui Lin.
+#
+
+type DNP3_PDU(is_orig: bool) = case is_orig of {
+	true    ->  request:  DNP3_Request;
+	false   ->  response: DNP3_Response;
+} &byteorder = bigendian;
+
+type Header_Block = record {
+	start: uint16 &check(start == 0x0564);
+	len: uint8;
+	ctrl: uint8;
+	dest_addr: uint16;
+	src_addr: uint16;
+} &byteorder = littleendian;
+
+type DNP3_Request = record {
+	addin_header: Header_Block;  ## added by Hui Lin in Bro code
+	app_header: DNP3_Application_Request_Header;
+	data: case ( app_header.function_code ) of {
+		CONFIRM -> none_coonfirm: empty;
+		READ -> read_requests: Request_Objects(app_header.function_code)[];
+		WRITE -> write_requests: Request_Objects(app_header.function_code)[];
+		SELECT -> select_requests: Request_Objects(app_header.function_code)[];
+		OPERATE -> operate_requests: Request_Objects(app_header.function_code)[];
+		DIRECT_OPERATE -> direct_operate_requests: Request_Objects(app_header.function_code)[];
+		DIRECT_OPERATE_NR -> direct_operate_nr_requests: Request_Objects(app_header.function_code)[];
+		IMMED_FREEZE -> immed_freeze_requests: Request_Objects(app_header.function_code)[];
+		IMMED_FREEZE_NR -> immed_freeze_nr_requests: Request_Objects(app_header.function_code)[];
+		FREEZE_CLEAR -> freeze_clear_requests: Request_Objects(app_header.function_code)[];
+		FREEZE_CLEAR_NR -> freeze_clear_nr_requests: Request_Objects(app_header.function_code)[];
+		FREEZE_AT_TIME -> freeze_time_requests: Request_Objects(app_header.function_code)[];
+		FREEZE_AT_TIME_NR -> freeze_time_nr_requests: Request_Objects(app_header.function_code)[];
+		COLD_RESTART -> cold_restart: empty;
+		WARM_RESTART -> warm_restart: empty;
+		INITIALIZE_DATA -> initilize_data: empty &check(0);  # obsolete
+		INITIALIZE_APPL -> initilize_appl: Request_Objects(app_header.function_code)[];
+		START_APPL -> start_appl: Request_Objects(app_header.function_code)[];
+		STOP_APPL -> stop_appl: Request_Objects(app_header.function_code)[];
+		SAVE_CONFIG -> save_config: empty &check(0);  # depracated
+		ENABLE_UNSOLICITED -> enable_unsolicited: Request_Objects(app_header.function_code)[];
+		DISABLE_UNSOLICITED -> disable_unsolicited: Request_Objects(app_header.function_code)[];
+		ASSIGN_CLASS -> assign_class: Request_Objects(app_header.function_code)[];
+		DELAY_MEASURE -> delay_measure: empty;
+		RECORD_CURRENT_TIME -> record_cur_time: empty;
+		OPEN_FILE -> open_file: Request_Objects(app_header.function_code)[];
+		CLOSE_FILE -> close_file: Request_Objects(app_header.function_code)[];
+		DELETE_FILE -> delete_file: Request_Objects(app_header.function_code)[];
+		ABORT_FILE -> abort_file: Request_Objects(app_header.function_code)[];
+		GET_FILE_INFO -> get_file_info: Request_Objects(app_header.function_code)[];
+		AUTHENTICATE_FILE  -> auth_file: Request_Objects(app_header.function_code)[];
+		ACTIVATE_CONFIG  -> active_config: Request_Objects(app_header.function_code)[];
+		AUTHENTICATE_REQ  -> auth_req: Request_Objects(app_header.function_code)[];
+		AUTHENTICATE_ERR  -> auth_err: Request_Objects(app_header.function_code)[];
+		default -> unknown: bytestring &restofdata;
+	};
+} &byteorder = bigendian
+  &length= 9 + addin_header.len - 5 - 1;
+
+type Debug_Byte = record {
+	debug: bytestring &restofdata;
+};
+
+type DNP3_Response = record {
+	addin_header: Header_Block;
+	app_header: DNP3_Application_Response_Header;
+	data: case ( app_header.function_code ) of {
+		RESPONSE -> response_objects: Response_Objects(app_header.function_code)[];
+		UNSOLICITED_RESPONSE -> unsolicited_response_objects: Response_Objects(app_header.function_code)[];
+		AUTHENTICATE_RESP -> auth_response: Response_Objects(app_header.function_code)[];
+		default -> unknown: Debug_Byte;
+	};
+} &byteorder = bigendian
+  &length= 9 + addin_header.len - 5 - 1'
+
+type DNP3_Application_Request_Header = record {
+	empty: bytestring &length = 0; # Work-around BinPAC problem.
+	application_control : uint8;
+	function_code       : uint8 ;
+} &length = 2;
+
+type DNP3_Application_Response_Header = record {
+	empty: bytestring &length = 0; # Work-around BinPAC problem.
+	application_control  : uint8;
+	function_code	: uint8;
+	internal_indications : uint16;
+} &length = 4;
+
+type Request_Objects(function_code: uint8) = record {
+	object_header: Object_Header(function_code);
+	data: case (object_header.object_type_field) of {
+		0x0c03 -> bocmd_PM: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ ( object_header.number_of_item / 8 ) + 1 ];
+		0x3202 -> time_interval_ojbects: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]
+							&check( object_header.qualifer_field == 0x0f && object_header.number_of_item == 0x01);
+		default -> ojbects: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item];
+	};
+	# dump_data is always empty; I intend to use it for checking some conditions;
+	# However, in the current binpac implementation, &check is not implemented
+	dump_data: case (function_code) of {
+		OPEN_FILE -> open_file_dump: empty &check(object_header.object_type_field == 0x4603);
+		CLOSE_FILE -> close_file_dump: empty &check(object_header.object_type_field == 0x4604);
+		DELETE_FILE -> delete_file_dump: empty &check(object_header.object_type_field == 0x4603);
+		ABORT_FILE -> abort_file_dump: empty &check(object_header.object_type_field == 0x4604);
+		GET_FILE_INFO -> get_file_info: empty &check(object_header.object_type_field == 0x4607);
+		AUTHENTICATE_FILE  -> auth_file: empty &check(object_header.object_type_field == 0x4602);
+		ACTIVATE_CONFIG  -> active_config: empty &check(object_header.object_type_field == 0x4608 || (object_header.object_type_field & 0xFF00) == 0x6E00);
+		default -> default_dump: empty;
+	};
+};
+
+type Response_Objects(function_code: uint8) = record {
+	object_header: Object_Header(function_code);
+	data: case (object_header.object_type_field) of {
+		0x0101 -> biwoflag: Response_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ ( object_header.number_of_item / 8 ) + 1 ];
+		0x0301 -> diwoflag: Response_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ ( object_header.number_of_item / 8 ) + 1 ];
+		0x0a01 -> bowoflag: Response_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ ( object_header.number_of_item / 8 ) + 1 ];
+		0x0c03 -> bocmd_PM: Response_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ ( object_header.number_of_item / 8 ) + 1 ];
+		default -> ojbects: Response_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item];
+	};
+};
+
+type Object_Header(function_code: uint8) = record {
+	object_type_field: uint16 ;
+	qualifier_field: uint8 ;
+	range_field: case ( qualifier_field & 0x0f ) of {
+		0 -> range_field_0: Range_Field_0 &check(range_field_0.stop_index >= range_field_0.start_index);
+		1 -> range_field_1: Range_Field_1 &check(range_field_1.stop_index >= range_field_1.start_index);
+		2 -> range_field_2: Range_Field_2 &check(range_field_2.stop_index >= range_field_2.start_index);
+		3 -> range_field_3: Range_Field_3;
+		4 -> range_field_4: Range_Field_4;
+		5 -> range_field_5: Range_Field_5;
+		6 -> range_field_6: empty;
+		7 -> range_field_7: uint8;
+		8 -> range_field_8: uint16;
+		9 -> range_field_9: uint32;
+		0x0b -> range_field_b: uint8;
+		default -> unknown: bytestring &restofdata &check(0);
+	};
+	# dump_data is always empty; used to check dependency bw object_type_field and qualifier_field
+	dump_data: case ( object_type_field & 0xff00 ) of {
+		0x3C00 -> dump_3c: empty &check( (object_type_field == 0x3C01 || object_type_field == 0x3C02 || object_type_field == 0x3C03 || object_type_field == 0x3C04) && ( qualifier_field == 0x06 ) );
+		default -> dump_def: empty;
+	};
+}
+      &let{
+	number_of_item: int = case (qualifier_field & 0x0f) of {
+		0 -> (range_field_0.stop_index - range_field_0.start_index + 1);
+		1 -> (range_field_1.stop_index - range_field_1.start_index + 1);
+		2 -> (range_field_2.stop_index - range_field_2.start_index + 1);
+		7 -> range_field_7;
+		8 -> ( range_field_8 & 0x0ff )* 0x100 + ( range_field_8 / 0x100 ) ;
+		9 -> ( range_field_9 & 0x000000ff )* 0x1000000 + (range_field_9 & 0x0000ff00) * 0x100 + (range_field_9 & 0x00ff0000) / 0x100 + (range_field_9 & 0xff000000) / 0x1000000 ;
+		0x0b -> range_field_b;
+		default -> 0;
+	};
+	rf_value_low: int = case (qualifier_field & 0x0f) of {
+		0 -> 0 + range_field_0.start_index;
+		1 -> range_field_1.start_index;
+		2 -> range_field_2.start_index;
+		3 -> range_field_3.start_addr;
+		4 -> range_field_4.start_addr;
+		5 -> range_field_5.start_addr;
+		6 -> 0xffff;
+		7 -> range_field_7;
+		8 -> range_field_8;
+		9 -> range_field_9;
+		0x0b -> range_field_b;
+		default -> 0 ;
+	};
+	rf_value_high: int = case (qualifier_field & 0x0f) of {
+		0 -> 0 + range_field_0.stop_index;
+		1 -> range_field_1.stop_index;
+		2 -> range_field_2.stop_index;
+		3 -> range_field_3.stop_addr;
+		4 -> range_field_4.stop_addr;
+		5 -> range_field_5.stop_addr;
+		6 -> 0xffff;
+		default -> 0 ;
+	};
+};
+
+type Range_Field_0 = record {
+	start_index: uint8;
+	stop_index: uint8;
+};
+
+type Range_Field_1 = record {
+	start_index: uint16;
+	stop_index: uint16;
+}
+  &byteorder = littleendian;
+
+type Range_Field_2 = record {
+	start_index: uint32;
+	stop_index: uint32;
+}
+  &byteorder = littleendian;
+
+type Range_Field_3 = record {
+	start_addr: uint8;
+	stop_addr: uint8;
+};
+
+type Range_Field_4 = record {
+	start_addr: uint16;
+	stop_addr: uint16;
+};
+
+type Range_Field_5 = record {
+	start_addr: uint32;
+	stop_addr: uint32;
+};
+
+enum function_codes_value {
+	CONFIRM = 0x00,
+	READ = 0x01,
+	WRITE = 0x02,
+	SELECT = 0x03,
+	OPERATE = 0x04,
+	DIRECT_OPERATE = 0x05,
+	DIRECT_OPERATE_NR = 0x06,
+	IMMED_FREEZE = 0x07,
+	IMMED_FREEZE_NR = 0x08,
+	FREEZE_CLEAR = 0x09,
+	FREEZE_CLEAR_NR = 0x0a,
+	FREEZE_AT_TIME = 0x0b,
+	FREEZE_AT_TIME_NR = 0x0c,
+	COLD_RESTART = 0x0d,
+	WARM_RESTART = 0x0e,
+	INITIALIZE_DATA = 0x0f,
+	INITIALIZE_APPL = 0x10,
+	START_APPL = 0x11,
+	STOP_APPL = 0x12,
+	SAVE_CONFIG = 0x13,
+	ENABLE_UNSOLICITED = 0x14,
+	DISABLE_UNSOLICITED = 0x15,
+	ASSIGN_CLASS = 0x16,
+	DELAY_MEASURE = 0x17,
+	RECORD_CURRENT_TIME = 0x18,
+	OPEN_FILE = 0x19,
+	CLOSE_FILE = 0x1a,
+	DELETE_FILE = 0x1b,
+	GET_FILE_INFO = 0x1c,
+	AUTHENTICATE_FILE = 0x1d,
+	ABORT_FILE = 0x1e,
+	ACTIVATE_CONFIG = 0x1f,
+	AUTHENTICATE_REQ = 0x20,
+	AUTHENTICATE_ERR = 0x21,
+# reserved
+	RESPONSE = 0x81,
+	UNSOLICITED_RESPONSE = 0x82,
+	AUTHENTICATE_RESP = 0x83,
+# reserved
+};
+
+%include dnp3-objects.pac

src/analyzer/protocol/dnp3/dnp3.pac

@@ -0,0 +1,16 @@
+
+%include binpac.pac
+%include bro.pac
+
+%extern{
+#include "events.bif.h"
+%}
+
+analyzer DNP3 withcontext {
+	connection:	DNP3_Conn;
+	flow:		DNP3_Flow;
+};
+
+%include dnp3-protocol.pac
+%include dnp3-analyzer.pac
+

src/analyzer/protocol/dnp3/events.bif

@@ -0,0 +1,240 @@
+
+## Generated for a DNP3 request header.
+##
+## c: The connection the DNP3 communication is part of.
+## is_orig: True if this reflects originator-side activity.
+## fc: function code.
+event dnp3_application_request_header%(c: connection, is_orig: bool, fc: count%);
+
+## Generated for a DNP3 response header.
+##
+## c: The connection the DNP3 communication is part of.
+## is_orig: True if this reflects originator-side activity.
+## fc: function code.
+## iin: internal indication number
+event dnp3_application_response_header%(c: connection, is_orig: bool, fc: count, iin: count%);
+
+## Generated for the object header found in both DNP3 requests and responses.
+##
+## c: The connection the DNP3 communication is part of.
+## is_orig: True if this reflects originator-side activity.
+## obj_type: type of object, which is classified based on an 8-bit group number and an 8-bit variation number
+## qua_field: qualifier field
+## rf_low, rf_high: the structure of the range field depends on the qualified field. In some cases, range field
+##       contain only one logic part, e.g., number of objects, so only rf_low contains the useful values; in some
+##       cases, range field contain two logic parts, e.g., start index and stop index, so rf_low contains the start
+##       index while rf_high contains the stop index
+event dnp3_object_header%(c: connection, is_orig: bool, obj_type: count, qua_field: count, number: count, rf_low: count, rf_high: count%);
+
+## Generated for the prefix before a DNP3 object. The structure and the meaning
+## of the prefix are defined by the qualifier field.
+##
+## c: The connection the DNP3 communication is part of.
+## is_orig: True if this reflects originator-side activity.
+## prefix_value: The prefix.
+event dnp3_object_prefix%(c: connection, is_orig: bool, prefix_value: count%);
+
+## Generated for an additional header that the DNP3 analyzer passes to the
+## script-level. This headers mimics the DNP3 transport-layer yet is only passed
+## once for each sequence of DNP3 records (which are otherwise reassembled and
+## treated as a single entity).
+##
+## c: The connection the DNP3 communication is part of.
+## is_orig: True if this reflects originator-side activity.
+## start: the first two bytes of the DNP3 Pseudo Link Layer; its value is fixed as 0x0564
+## len:   the "length" field in the DNP3 Pseudo Link Layer
+## ctrl:  the "control" field in the DNP3 Pseudo Link Layer
+## dest_addr: the "destination" field in the DNP3 Pseudo Link Layer
+## src_addr: the "source" field in the DNP3 Pseudo Link Layer
+event dnp3_header_block%(c: connection, is_orig: bool, start: count, len: count, ctrl: count, dest_addr: count, src_addr: count%);
+
+## Generated for a DNP3 "Response_Data_Object". The "Response_Data_Object" contains two
+## parts: object prefix and objects data. In most cases, objects data are defined
+## by new record types. But in a few cases, objects data are directly basic types,
+## such as int16, or int8; thus we use a additional data_value to record the values
+## of those object data.
+##
+## c: The connection the DNP3 communication is part of.
+## is_orig: True if this reflects originator-side activity.
+## data_value: The value for those objects that carry their information here
+##             directly.
+event dnp3_response_data_object%(c: connection, is_orig: bool, data_value: count%);
+
+## Generated for DNP3 attributes.
+event dnp3_attribute_common%(c: connection, is_orig: bool, data_type_code: count, leng: count, attribute_obj: string%);
+
+## Generated for DNP3 objects with the group number 12 and variation number 1
+## CROB: control relay output block
+event dnp3_crob%(c: connection, is_orig: bool, control_code: count, count8: count, on_time: count, off_time: count, status_code: count%);
+
+## Generated for DNP3 objects with the group number 12 and variation number 2
+## PCB: Pattern Control Block
+event dnp3_pcb%(c: connection, is_orig: bool, control_code: count, count8: count, on_time: count, off_time: count, status_code: count%);
+
+## Generated for DNP3 objects with the group number 20 and variation number 1
+## counter 32 bit with flag
+event dnp3_counter_32wFlag%(c: connection, is_orig: bool, flag: count, count_value: count%);
+
+## Generated for DNP3 objects with the group number 20 and variation number 2
+## counter 16 bit with flag
+event dnp3_counter_16wFlag%(c: connection, is_orig: bool, flag: count, count_value: count%);
+
+## Generated for DNP3 objects with the group number 20 and variation number 5
+## counter 32 bit without flag
+event dnp3_counter_32woFlag%(c: connection, is_orig: bool, count_value: count%);
+
+## Generated for DNP3 objects with the group number 20 and variation number 6
+## counter 16 bit without flag
+event dnp3_counter_16woFlag%(c: connection, is_orig: bool, count_value: count%);
+
+## Generated for DNP3 objects with the group number 21 and variation number 1
+## frozen counter 32 bit with flag
+event dnp3_frozen_counter_32wFlag%(c: connection, is_orig: bool, flag:count, count_value: count%);
+
+## Generated for DNP3 objects with the group number 21 and variation number 2
+## frozen counter 16 bit with flag
+event dnp3_frozen_counter_16wFlag%(c: connection, is_orig: bool, flag:count, count_value: count%);
+
+## Generated for DNP3 objects with the group number 21 and variation number 5
+## frozen counter 32 bit with flag and time
+event dnp3_frozen_counter_32wFlagTime%(c: connection, is_orig: bool, flag:count, count_value: count, time48: string%);
+
+## Generated for DNP3 objects with the group number 21 and variation number 6
+## frozen counter 16 bit with flag and time
+event dnp3_frozen_counter_16wFlagTime%(c: connection, is_orig: bool, flag:count, count_value: count, time48: string%);
+
+## Generated for DNP3 objects with the group number 21 and variation number 9
+## frozen counter 32 bit without flag
+event dnp3_frozen_counter_32woFlag%(c: connection, is_orig: bool, count_value: count%);
+
+## Generated for DNP3 objects with the group number 21 and variation number 10
+## frozen counter 16 bit without flag
+event dnp3_frozen_counter_16woFlag%(c: connection, is_orig: bool, count_value: count%);
+
+## Generated for DNP3 objects with the group number 30 and variation number 1
+## analog input 32 bit with flag
+event dnp3_analog_input_32wFlag%(c: connection, is_orig: bool, flag: count, value: count%);
+
+## Generated for DNP3 objects with the group number 30 and variation number 2
+## analog input 16 bit with flag
+event dnp3_analog_input_16wFlag%(c: connection, is_orig: bool, flag: count, value: count%);
+
+## Generated for DNP3 objects with the group number 30 and variation number 3
+## analog input 32 bit without flag
+event dnp3_analog_input_32woFlag%(c: connection, is_orig: bool, value: count%);
+
+## Generated for DNP3 objects with the group number 30 and variation number 4
+## analog input 16 bit without flag
+event dnp3_analog_input_16woFlag%(c: connection, is_orig: bool, value: count%);
+
+## Generated for DNP3 objects with the group number 30 and variation number 5
+## analog input single precision, float point with flag
+event dnp3_analog_input_SPwFlag%(c: connection, is_orig: bool, flag: count, value: count%);
+
+## Generated for DNP3 objects with the group number 30 and variation number 6
+## analog input double precision, float point with flag
+event dnp3_analog_input_DPwFlag%(c: connection, is_orig: bool, flag: count, value_low: count, value_high: count%);
+
+## Generated for DNP3 objects with the group number 31 and variation number 1
+## frozen analog input 32 bit with flag
+event dnp3_frozen_analog_input_32wFlag%(c: connection, is_orig: bool, flag: count, frozen_value: count%);
+
+## Generated for DNP3 objects with the group number 31 and variation number 2
+## frozen analog input 16 bit with flag
+event dnp3_frozen_analog_input_16wFlag%(c: connection, is_orig: bool, flag: count, frozen_value: count%);
+
+## Generated for DNP3 objects with the group number 31 and variation number 3
+## frozen analog input 32 bit with time-of-freeze
+event dnp3_frozen_analog_input_32wTime%(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: string%);
+
+## Generated for DNP3 objects with the group number 31 and variation number 4
+## frozen analog input 16 bit with time-of-freeze
+event dnp3_frozen_analog_input_16wTime%(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: string%);
+
+## Generated for DNP3 objects with the group number 31 and variation number 5
+## frozen analog input 32 bit without flag
+event dnp3_frozen_analog_input_32woFlag%(c: connection, is_orig: bool, frozen_value: count%);
+
+## Generated for DNP3 objects with the group number 31 and variation number 6
+## frozen analog input 16 bit without flag
+event dnp3_frozen_analog_input_16woFlag%(c: connection, is_orig: bool, frozen_value: count%);
+
+## Generated for DNP3 objects with the group number 31 and variation number 7
+## frozen analog input single-precision, float point with flag
+event dnp3_frozen_analog_input_SPwFlag%(c: connection, is_orig: bool, flag: count, frozen_value: count%);
+
+## Generated for DNP3 objects with the group number 31 and variation number 8
+## frozen analog input double-precision, float point with flag
+event dnp3_frozen_analog_input_DPwFlag%(c: connection, is_orig: bool, flag: count, frozen_value_low: count, frozen_value_high: count%);
+
+## Generated for DNP3 objects with the group number 32 and variation number 1
+## analog input event 32 bit without time
+event dnp3_analog_input_event_32woTime%(c: connection, is_orig: bool, flag: count, value: count%);
+
+## Generated for DNP3 objects with the group number 32 and variation number 2
+## analog input event 16 bit without time
+event dnp3_analog_input_event_16woTime%(c: connection, is_orig: bool, flag: count, value: count%);
+
+## Generated for DNP3 objects with the group number 32 and variation number 3
+## analog input event 32 bit with time
+event dnp3_analog_input_event_32wTime%(c: connection, is_orig: bool, flag: count, value: count, time48: string%);
+
+## Generated for DNP3 objects with the group number 32 and variation number 4
+## analog input event 16 bit with time
+event dnp3_analog_input_event_16wTime%(c: connection, is_orig: bool, flag: count, value: count, time48: string%);
+
+## Generated for DNP3 objects with the group number 32 and variation number 5
+## analog input event single-precision float point without time
+event dnp3_analog_input_event_SPwoTime%(c: connection, is_orig: bool, flag: count, value: count%);
+
+## Generated for DNP3 objects with the group number 32 and variation number 6
+## analog input event double-precision float point without time
+event dnp3_analog_input_event_DPwoTime%(c: connection, is_orig: bool, flag: count, value_low: count, value_high: count%);
+
+## Generated for DNP3 objects with the group number 32 and variation number 7
+## analog input event single-precision float point with time
+event dnp3_analog_input_event_SPwTime%(c: connection, is_orig: bool, flag: count, value: count, time48: string%);
+
+## Generated for DNP3 objects with the group number 32 and variation number 8
+## analog input event double-precisiion float point with time
+event dnp3_analog_input_event_DPwTime%(c: connection, is_orig: bool, flag: count, value_low: count, value_high: count, time48: string%);
+
+## Generated for DNP3 objects with the group number 33 and variation number 1
+## frozen analog input event 32 bit without time
+event dnp3_frozen_analog_input_event_32woTime%(c: connection, is_orig: bool, flag: count, frozen_value: count%);
+
+## Generated for DNP3 objects with the group number 33 and variation number 2
+## frozen analog input event 16 bit without time
+event dnp3_frozen_analog_input_event_16woTime%(c: connection, is_orig: bool, flag: count, frozen_value: count%);
+
+## Generated for DNP3 objects with the group number 33 and variation number 3
+## frozen analog input event 32 bit with time
+event dnp3_frozen_analog_input_event_32wTime%(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: string%);
+
+## Generated for DNP3 objects with the group number 33 and variation number 4
+## frozen analog input event 16 bit with time
+event dnp3_frozen_analog_input_event_16wTime%(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: string%);
+
+## Generated for DNP3 objects with the group number 33 and variation number 5
+## frozen analog input event single-precision float point without time
+event dnp3_frozen_analog_input_event_SPwoTime%(c: connection, is_orig: bool, flag: count, frozen_value: count%);
+
+## Generated for DNP3 objects with the group number 33 and variation number 6
+## frozen analog input event double-precision float point without time
+event dnp3_frozen_analog_input_event_DPwoTime%(c: connection, is_orig: bool, flag: count, frozen_value_low: count, frozen_value_high: count%);
+
+## Generated for DNP3 objects with the group number 33 and variation number 7
+## frozen analog input event single-precision float point with time
+event dnp3_frozen_analog_input_event_SPwTime%(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: string%);
+
+## Generated for DNP3 objects with the group number 34 and variation number 8
+## frozen analog input event double-precision float point with time
+event dnp3_frozen_analog_input_event_DPwTime%(c: connection, is_orig: bool, flag: count, frozen_value_low: count, frozen_value_high: count, time48: string%);
+
+## g70
+event dnp3_file_transport%(c: connection, is_orig: bool, file_handle: count, block_num: count, file_data: string%);
+
+## Debugging event generated by the DNP3 analyzer. The "Debug_Byte" binpac unit
+## generates this for unknown "cases". The user can use it to debug the byte string
+## to check what cause the malformed network packets.
+event dnp3_debug_byte%(c: connection, is_orig: bool, debug: string%);

src/binpac_bro.h

@@ -12,6 +12,7 @@ namespace analyzer { class Analyzer; }
 #include "event.bif.func_h"
 #include "TunnelEncapsulation.h"
 #include "analyzer/Analyzer.h"
+#include "file_analysis/Analyzer.h"
 #include "Conn.h"
 
 #include "binpac.h"
@@ -19,6 +20,7 @@ namespace analyzer { class Analyzer; }
 namespace binpac {
 
 typedef analyzer::Analyzer* BroAnalyzer;
+typedef file_analysis::Analyzer BroFileAnalyzer;
 typedef Val* BroVal;
 typedef PortVal* BroPortVal;
 typedef StringVal* BroStringVal;

src/bro.bif

@@ -3895,8 +3895,8 @@ function flush_all%(%): bool
 ##
 ## f: The directory name.
 ##
-## Returns: Returns true if the operation succeeds, or false if the
+## Returns: Returns true if the operation succeeds or if *f* already exists,
-##           creation fails or if *f* exists already.
+##          and false if the file creation fails.
 ##
 ## .. bro:see:: active_file open_for_append close write_file
 ##              get_file_name set_buf flush_all enable_raw_output

src/builtin-func.l

@@ -209,8 +209,8 @@ void init_alternative_mode()
 
 	static char guard[1024];
 	getcwd(guard, sizeof(guard));
-	strncat(guard, "/", sizeof(guard));
+	strncat(guard, "/", sizeof(guard) - strlen(guard) - 1);
-	strncat(guard, input_filename, sizeof(guard));
+	strncat(guard, input_filename, sizeof(guard) - strlen(guard) - 1);
 
 	for ( char* p = guard; *p; p++ )
 		{

src/file_analysis/AnalyzerSet.cc

@@ -35,6 +35,14 @@ AnalyzerSet::~AnalyzerSet()
 	delete analyzer_hash;
 	}
 
+Analyzer* AnalyzerSet::Find(file_analysis::Tag tag, RecordVal* args)
+	{
+	HashKey* key = GetKey(tag, args);
+	Analyzer* rval = analyzer_map.Lookup(key);
+	delete key;
+	return rval;
+	}
+
 bool AnalyzerSet::Add(file_analysis::Tag tag, RecordVal* args)
 	{
 	HashKey* key = GetKey(tag, args);

src/file_analysis/AnalyzerSet.h

@@ -37,6 +37,14 @@ public:
 	 */
 	~AnalyzerSet();
 
+	/**
+	 * Looks up an analyzer by its tag and arguments.
+	 * @param tag an analyzer tag.
+	 * @param args an \c AnalyzerArgs record.
+	 * @return pointer to an analyzer instance, or a null pointer if not found.
+	 */
+	Analyzer* Find(file_analysis::Tag tag, RecordVal* args);
+
 	/**
 	 * Attach an analyzer to #file immediately.
 	 * @param tag the analyzer tag of the file analyzer to add.

src/file_analysis/File.cc

@@ -14,6 +14,8 @@
 #include "analyzer/Analyzer.h"
 #include "analyzer/Manager.h"
 
+#include "analyzer/extract/Extract.h"
+
 using namespace file_analysis;
 
 static Val* empty_connection_table()
@@ -203,6 +205,22 @@ void File::SetTimeoutInterval(double interval)
 	val->Assign(timeout_interval_idx, new Val(interval, TYPE_INTERVAL));
 	}
 
+bool File::SetExtractionLimit(RecordVal* args, uint64 bytes)
+	{
+	Analyzer* a = analyzers.Find(file_mgr->GetComponentTag("EXTRACT"), args);
+
+	if ( ! a )
+		return false;
+
+	Extract* e = dynamic_cast<Extract*>(a);
+
+	if ( ! e )
+		return false;
+
+	e->SetLimit(bytes);
+	return true;
+	}
+
 void File::IncrementByteCount(uint64 size, int field_idx)
 	{
 	uint64 old = LookupFieldDefaultCount(field_idx);
@@ -458,7 +476,7 @@ void File::FileEvent(EventHandlerPtr h, val_list* vl)
 			}
 		}
 
-	if ( h == file_new || h == file_timeout )
+	if ( h == file_new || h == file_timeout || h == file_extraction_limit )
 		{
 		// immediate feedback is required for these events.
 		mgr.Drain();

src/file_analysis/File.h

@@ -56,6 +56,14 @@ public:
 	 */
 	void SetTimeoutInterval(double interval);
 
+	/**
+	 * Change the maximum size that an attached extraction analyzer is allowed.
+	 * @param args the file extraction analyzer whose limit needs changed.
+	 * @param bytes new limit.
+	 * @return false if no extraction analyzer is active, else true.
+	 */
+	bool SetExtractionLimit(RecordVal* args, uint64 bytes);
+
 	/**
 	 * @return value of the "id" field from #val record.
 	 */

src/file_analysis/Manager.cc

@@ -184,6 +184,17 @@ bool Manager::SetTimeoutInterval(const string& file_id, double interval) const
 	return true;
 	}
 
+bool Manager::SetExtractionLimit(const string& file_id, RecordVal* args,
+                                 uint64 n) const
+	{
+	File* file = LookupFile(file_id);
+
+	if ( ! file )
+		return false;
+
+	return file->SetExtractionLimit(args, n);
+	}
+
 bool Manager::AddAnalyzer(const string& file_id, file_analysis::Tag tag,
                           RecordVal* args) const
 	{

src/file_analysis/Manager.h

@@ -173,6 +173,19 @@ public:
 	 */
 	bool SetTimeoutInterval(const string& file_id, double interval) const;
 
+	/**
+	 * Sets a limit on the maximum size allowed for extracting the file
+	 * to local disk;
+	 * @param file_id the file identifier/hash.
+	 * @param args a \c AnalyzerArgs value which describes a file analyzer,
+	 *        which should be a file extraction analyzer.
+	 * @param n the new extraction limit, in bytes.
+	 * @return false if file identifier and analyzer did not map to anything,
+	 *         else true.
+	 */
+	bool SetExtractionLimit(const string& file_id, RecordVal* args,
+	                        uint64 n) const;
+
 	/**
 	 * Queue attachment of an analzer to the file identifier.  Multiple
 	 * analyzers of a given type can be attached per file identifier at a time

src/file_analysis/analyzer/CMakeLists.txt

@@ -1,3 +1,4 @@
 add_subdirectory(data_event)
 add_subdirectory(extract)
 add_subdirectory(hash)
+add_subdirectory(unified2)

src/file_analysis/analyzer/data_event/Plugin.cc

@@ -1,26 +1,8 @@
 #include "plugin/Plugin.h"
-#include "file_analysis/Component.h"
 
 #include "DataEvent.h"
 
-namespace plugin { namespace Bro_FileDataEvent {
+BRO_PLUGIN_BEGIN(Bro, FileDataEvent)
-
+	BRO_PLUGIN_DESCRIPTION("Delivers file content via events");
-class Plugin : public plugin::Plugin {
+	BRO_PLUGIN_FILE_ANALYZER("DATA_EVENT", DataEvent);
-protected:
+BRO_PLUGIN_END
-	void InitPreScript()
-		{
-		SetName("Bro::FileDataEvent");
-		SetVersion(-1);
-		SetAPIVersion(BRO_PLUGIN_API_VERSION);
-		SetDynamicPlugin(false);
-
-		SetDescription("Delivers file content via events");
-
-		AddComponent(new ::file_analysis::Component("DATA_EVENT",
-		        ::file_analysis::DataEvent::Instantiate));
-		}
-};
-
-Plugin __plugin;
-
-} }

src/file_analysis/analyzer/extract/CMakeLists.txt

@@ -5,4 +5,6 @@ include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}
 
 bro_plugin_begin(Bro FileExtract)
 bro_plugin_cc(Extract.cc Plugin.cc ../../Analyzer.cc)
+bro_plugin_bif(events.bif)
+bro_plugin_bif(functions.bif)
 bro_plugin_end()

src/file_analysis/analyzer/extract/Extract.cc

@@ -4,13 +4,15 @@
 
 #include "Extract.h"
 #include "util.h"
+#include "Event.h"
 #include "file_analysis/Manager.h"
 
 using namespace file_analysis;
 
-Extract::Extract(RecordVal* args, File* file, const string& arg_filename)
+Extract::Extract(RecordVal* args, File* file, const string& arg_filename,
+                 uint64 arg_limit)
     : file_analysis::Analyzer(file_mgr->GetComponentTag("EXTRACT"), args, file),
-	  filename(arg_filename)
+      filename(arg_filename), limit(arg_limit)
 	{
 	fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666);
 
@@ -29,15 +31,51 @@ Extract::~Extract()
 		safe_close(fd);
 	}
 
-file_analysis::Analyzer* Extract::Instantiate(RecordVal* args, File* file)
+static Val* get_extract_field_val(RecordVal* args, const char* name)
 	{
 	using BifType::Record::Files::AnalyzerArgs;
-	Val* v = args->Lookup(AnalyzerArgs->FieldOffset("extract_filename"));
+	Val* rval = args->Lookup(AnalyzerArgs->FieldOffset(name));
 
-	if ( ! v )
+	if ( ! rval )
+		reporter->Error("File extraction analyzer missing arg field: %s", name);
+
+	return rval;
+	}
+
+file_analysis::Analyzer* Extract::Instantiate(RecordVal* args, File* file)
+	{
+	Val* fname = get_extract_field_val(args, "extract_filename");
+	Val* limit = get_extract_field_val(args, "extract_limit");
+
+	if ( ! fname || ! limit )
 		return 0;
 
-	return new Extract(args, file, v->AsString()->CheckString());
+	return new Extract(args, file, fname->AsString()->CheckString(),
+	                   limit->AsCount());
+	}
+
+static bool check_limit_exceeded(uint64 lim, uint64 off, uint64 len, uint64* n)
+	{
+	if ( lim == 0 )
+		{
+		*n = len;
+		return false;
+		}
+
+	if ( off >= lim )
+		{
+		*n = 0;
+		return true;
+		}
+
+	*n = lim - off;
+
+	if ( len > *n )
+		return true;
+	else
+		*n = len;
+
+	return false;
 	}
 
 bool Extract::DeliverChunk(const u_char* data, uint64 len, uint64 offset)
@@ -45,6 +83,26 @@ bool Extract::DeliverChunk(const u_char* data, uint64 len, uint64 offset)
 	if ( ! fd )
 		return false;
 
-	safe_pwrite(fd, data, len, offset);
+	uint64 towrite = 0;
-	return true;
+	bool limit_exceeded = check_limit_exceeded(limit, offset, len, &towrite);
+
+	if ( limit_exceeded && file_extraction_limit )
+		{
+		File* f = GetFile();
+		val_list* vl = new val_list();
+		vl->append(f->GetVal()->Ref());
+		vl->append(Args()->Ref());
+		vl->append(new Val(limit, TYPE_COUNT));
+		vl->append(new Val(offset, TYPE_COUNT));
+		vl->append(new Val(len, TYPE_COUNT));
+		f->FileEvent(file_extraction_limit, vl);
+
+		// Limit may have been modified by BIF, re-check it.
+		limit_exceeded = check_limit_exceeded(limit, offset, len, &towrite);
+		}
+
+	if ( towrite > 0 )
+		safe_pwrite(fd, data, towrite, offset);
+
+	return ( ! limit_exceeded );
 	}

src/file_analysis/analyzer/extract/Extract.h

@@ -9,6 +9,8 @@
 #include "File.h"
 #include "Analyzer.h"
 
+#include "analyzer/extract/events.bif.h"
+
 namespace file_analysis {
 
 /**
@@ -41,6 +43,13 @@ public:
 	 */
 	static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file);
 
+	/**
+	 * Sets the maximum allowed extracted file size.  A value of zero means
+	 * "no limit".
+	 * @param bytes number of bytes allowed to be extracted
+	 */
+	void SetLimit(uint64 bytes) { limit = bytes; }
+
 protected:
 
 	/**
@@ -49,12 +58,15 @@ protected:
 	 * @param file the file to which the analyzer will be attached.
 	 * @param arg_filename a file system path which specifies the local file
 	 *        to which the contents of the file will be extracted/written.
+	 * @param arg_limit the maximum allowed file size.
 	 */
-	Extract(RecordVal* args, File* file, const string& arg_filename);
+	Extract(RecordVal* args, File* file, const string& arg_filename,
+	        uint64 arg_limit);
 
 private:
 	string filename;
 	int fd;
+	uint64 limit;
 };
 
 } // namespace file_analysis

src/file_analysis/analyzer/extract/Plugin.cc

@@ -1,26 +1,10 @@
 #include "plugin/Plugin.h"
-#include "file_analysis/Component.h"
 
 #include "Extract.h"
 
-namespace plugin { namespace Bro_FileExtract {
+BRO_PLUGIN_BEGIN(Bro, FileExtract)
-
+	BRO_PLUGIN_DESCRIPTION("Extract file content to local file system");
-class Plugin : public plugin::Plugin {
+	BRO_PLUGIN_FILE_ANALYZER("EXTRACT", Extract);
-protected:
+	BRO_PLUGIN_BIF_FILE(events);
-	void InitPreScript()
+	BRO_PLUGIN_BIF_FILE(functions);
-		{
+BRO_PLUGIN_END
-		SetName("Bro::FileExtract");
-		SetVersion(-1);
-		SetAPIVersion(BRO_PLUGIN_API_VERSION);
-		SetDynamicPlugin(false);
-
-		SetDescription("Extract file content to local file system");
-
-		AddComponent(new ::file_analysis::Component("EXTRACT",
-		        ::file_analysis::Extract::Instantiate));
-		}
-};
-
-Plugin __plugin;
-
-} }

src/file_analysis/analyzer/extract/events.bif

@@ -0,0 +1,19 @@
+## This event is generated when a file extraction analyzer is about
+## to exceed the maximum permitted file size allowed by
+## *extract_size_limit* field of :bro:see:`Files::AnalyzerArgs`.
+## The analyzer is automatically removed from file *f*.
+##
+## f: The file.
+##
+## args: Arguments that identify a particular file extraction analyzer.
+##       This is only provided to be able to pass along to
+##       :bro:see:`FileExtract::set_limit`.
+##
+## limit: The limit, in bytes, the extracted file is about to breach.
+##
+## offset: The offset at which a file chunk is about to be written.
+##
+## len:: The length of the file chunk about to be written.
+##
+## .. bro:see:: Files::add_analyzer Files::ANALYZER_EXTRACT
+event file_extraction_limit%(f: fa_file, args: any, limit: count, offset: count, len: count%);

src/file_analysis/analyzer/extract/functions.bif

@@ -0,0 +1,19 @@
+##! Internal functions used by the extraction file analyzer.
+
+module FileExtract;
+
+%%{
+#include "file_analysis/Manager.h"
+%%}
+
+## :bro:see:`FileExtract::set_limit`.
+function FileExtract::__set_limit%(file_id: string, args: any, n: count%): bool
+    %{
+	using BifType::Record::Files::AnalyzerArgs;
+	RecordVal* rv = args->AsRecordVal()->CoerceTo(AnalyzerArgs);
+    bool result = file_mgr->SetExtractionLimit(file_id->CheckString(), rv, n);
+	Unref(rv);
+    return new Val(result, TYPE_BOOL);
+    %}
+
+module GLOBAL;

src/file_analysis/analyzer/hash/Plugin.cc

@@ -1,33 +1,11 @@
 #include "plugin/Plugin.h"
-#include "file_analysis/Component.h"
 
 #include "Hash.h"
 
-namespace plugin { namespace Bro_FileHash {
+BRO_PLUGIN_BEGIN(Bro, FileHash)
-
+	BRO_PLUGIN_DESCRIPTION("Hash file content");
-class Plugin : public plugin::Plugin {
+	BRO_PLUGIN_FILE_ANALYZER("MD5", MD5);
-protected:
+	BRO_PLUGIN_FILE_ANALYZER("SHA1", SHA1);
-	void InitPreScript()
+	BRO_PLUGIN_FILE_ANALYZER("SHA256", SHA256);
-		{
+	BRO_PLUGIN_BIF_FILE(events);
-		SetName("Bro::FileHash");
+BRO_PLUGIN_END
-		SetVersion(-1);
-		SetAPIVersion(BRO_PLUGIN_API_VERSION);
-		SetDynamicPlugin(false);
-
-		SetDescription("Hash file content");
-
-		AddComponent(new ::file_analysis::Component("MD5",
-		        ::file_analysis::MD5::Instantiate));
-		AddComponent(new ::file_analysis::Component("SHA1",
-		        ::file_analysis::SHA1::Instantiate));
-		AddComponent(new ::file_analysis::Component("SHA256",
-		        ::file_analysis::SHA256::Instantiate));
-
-		extern std::list<std::pair<const char*, int> > __bif_events_init();
-		AddBifInitFunction(&__bif_events_init);
-		}
-};
-
-Plugin __plugin;
-
-} }

src/file_analysis/analyzer/unified2/CMakeLists.txt

@@ -0,0 +1,11 @@
+
+include(BroPlugin)
+
+include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}
+                           ${CMAKE_CURRENT_BINARY_DIR})
+
+bro_plugin_begin(Bro Unified2)
+bro_plugin_cc(Unified2.cc Plugin.cc ../../Analyzer.cc)
+bro_plugin_bif(events.bif types.bif)
+bro_plugin_pac(unified2.pac unified2-file.pac unified2-analyzer.pac)
+bro_plugin_end()

src/file_analysis/analyzer/unified2/Plugin.cc

@@ -0,0 +1,12 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "plugin/Plugin.h"
+
+#include "Unified2.h"
+
+BRO_PLUGIN_BEGIN(Bro, Unified2)
+	BRO_PLUGIN_DESCRIPTION("Analyze Unified2 alert files.");
+	BRO_PLUGIN_FILE_ANALYZER("UNIFIED2", Unified2);
+	BRO_PLUGIN_BIF_FILE(events);
+	BRO_PLUGIN_BIF_FILE(types);
+BRO_PLUGIN_END

src/file_analysis/analyzer/unified2/Unified2.cc

@@ -0,0 +1,38 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "Unified2.h"
+#include "file_analysis/Manager.h"
+
+using namespace file_analysis;
+
+Unified2::Unified2(RecordVal* args, File* file)
+    : file_analysis::Analyzer(file_mgr->GetComponentTag("UNIFIED2"), args, file)
+	{
+	interp = new binpac::Unified2::Unified2_Analyzer(this);
+	}
+
+Unified2::~Unified2()
+	{
+	delete interp;
+	}
+
+file_analysis::Analyzer* Unified2::Instantiate(RecordVal* args, File* file)
+	{
+	return new Unified2(args, file);
+	}
+
+bool Unified2::DeliverStream(const u_char* data, uint64 len)
+	{
+	try
+		{
+		interp->NewData(true, data, data + len);
+		}
+
+	catch ( const binpac::Exception& e )
+		{
+		printf("Binpac exception: %s\n", e.c_msg());
+		return false;
+		}
+
+	return true;
+	}

src/file_analysis/analyzer/unified2/Unified2.h

@@ -0,0 +1,38 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef FILE_ANALYSIS_UNIFIED2_H
+#define FILE_ANALYSIS_UNIFIED2_H
+
+#include <string>
+
+#include "Val.h"
+#include "File.h"
+#include "Analyzer.h"
+#include "unified2_pac.h"
+
+namespace file_analysis {
+
+/**
+ * An analyzer to extract content of files from local disk.
+ */
+class Unified2 : public file_analysis::Analyzer {
+public:
+	virtual ~Unified2();
+
+	virtual bool DeliverStream(const u_char* data, uint64 len);
+
+	static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file);
+
+protected:
+	Unified2(RecordVal* args, File* file);
+
+private:
+	binpac::Unified2::Unified2_Analyzer* interp;
+
+	string filename;
+	int fd;
+};
+
+} // namespace file_analysis
+
+#endif

src/file_analysis/analyzer/unified2/events.bif

@@ -0,0 +1,7 @@
+
+## Abstract all of the various Unified2 event formats into 
+## a single event.
+event unified2_event%(f: fa_file, ev: Unified2::IDSEvent%);
+
+## The Unified2 packet format event.
+event unified2_packet%(f: fa_file, pkt: Unified2::Packet%);

src/file_analysis/analyzer/unified2/types.bif

@@ -0,0 +1,2 @@
+type Unified2::IDSEvent: record;
+type Unified2::Packet: record;

src/file_analysis/analyzer/unified2/unified2-analyzer.pac

@@ -0,0 +1,170 @@
+
+%extern{
+#include "Event.h"
+#include "file_analysis/File.h"
+#include "events.bif.h"
+#include "types.bif.h"
+#include "IPAddr.h"
+%}
+
+refine flow Flow += {
+
+	%member{
+	%}
+
+	%init{
+	%}
+
+	%eof{
+	%}
+
+	%cleanup{
+	%}
+
+	function ts_to_double(ts: Time): double
+		%{
+		double t = ${ts.seconds} + (${ts.microseconds} / 1000000);
+		return t;
+		%}
+
+	function unified2_addr_to_bro_addr(a: uint32[]): AddrVal
+		%{
+		if ( a->size() == 1 )
+			{
+			return new AddrVal(IPAddr(IPv4, &(a->at(0)), IPAddr::Host));
+			}
+		else if ( a->size() == 4 )
+			{
+			uint32 tmp[4] = { a->at(0), a->at(1), a->at(2), a->at(3) };
+			return new AddrVal(IPAddr(IPv6, tmp, IPAddr::Host));
+			}
+		else
+			{
+			// Should never reach here.
+			return new AddrVal(1);
+			}
+		%}
+
+	function to_port(n: uint16, p: uint8): PortVal
+		%{
+		TransportProto proto = TRANSPORT_UNKNOWN;
+		switch ( p ) {
+		case 1: proto = TRANSPORT_ICMP; break;
+		case 6: proto = TRANSPORT_TCP; break;
+		case 17: proto = TRANSPORT_UDP; break;
+		}
+
+		return new PortVal(n, proto);
+		%}
+
+	#function proc_record(rec: Record) : bool
+	#	%{
+	#	return true;
+	#	%}
+
+	function proc_ids_event(ev: IDS_Event) : bool
+		%{
+		if ( ::unified2_event )
+			{
+			RecordVal* ids_event = new RecordVal(BifType::Record::Unified2::IDSEvent);
+			ids_event->Assign(0, new Val(${ev.sensor_id}, TYPE_COUNT));
+			ids_event->Assign(1, new Val(${ev.event_id}, TYPE_COUNT));
+			ids_event->Assign(2, new Val(ts_to_double(${ev.ts}), TYPE_TIME));
+			ids_event->Assign(3, new Val(${ev.signature_id}, TYPE_COUNT));
+			ids_event->Assign(4, new Val(${ev.generator_id}, TYPE_COUNT));
+			ids_event->Assign(5, new Val(${ev.signature_revision}, TYPE_COUNT));
+			ids_event->Assign(6, new Val(${ev.classification_id}, TYPE_COUNT));
+			ids_event->Assign(7, new Val(${ev.priority_id}, TYPE_COUNT));
+			ids_event->Assign(8, unified2_addr_to_bro_addr(${ev.src_ip}));
+			ids_event->Assign(9, unified2_addr_to_bro_addr(${ev.dst_ip}));
+			ids_event->Assign(10, to_port(${ev.src_p}, ${ev.protocol}));
+			ids_event->Assign(11, to_port(${ev.dst_p}, ${ev.protocol}));
+			ids_event->Assign(17, new Val(${ev.packet_action}, TYPE_COUNT));
+
+			val_list* vl = new val_list();
+			vl->append(connection()->bro_analyzer()->GetFile()->GetVal()->Ref());
+			vl->append(ids_event);
+			mgr.QueueEvent(::unified2_event, vl, SOURCE_LOCAL);
+			}
+		return true;
+		%}
+
+	function proc_ids_event_2(ev: IDS_Event_2) : bool
+		%{
+		if ( ::unified2_event )
+			{
+			RecordVal* ids_event = new RecordVal(BifType::Record::Unified2::IDSEvent);
+			ids_event->Assign(0, new Val(${ev.sensor_id}, TYPE_COUNT));
+			ids_event->Assign(1, new Val(${ev.event_id}, TYPE_COUNT));
+			ids_event->Assign(2, new Val(ts_to_double(${ev.ts}), TYPE_TIME));
+			ids_event->Assign(3, new Val(${ev.signature_id}, TYPE_COUNT));
+			ids_event->Assign(4, new Val(${ev.generator_id}, TYPE_COUNT));
+			ids_event->Assign(5, new Val(${ev.signature_revision}, TYPE_COUNT));
+			ids_event->Assign(6, new Val(${ev.classification_id}, TYPE_COUNT));
+			ids_event->Assign(7, new Val(${ev.priority_id}, TYPE_COUNT));
+			ids_event->Assign(8, unified2_addr_to_bro_addr(${ev.src_ip}));
+			ids_event->Assign(9, unified2_addr_to_bro_addr(${ev.dst_ip}));
+			ids_event->Assign(10, to_port(${ev.src_p}, ${ev.protocol}));
+			ids_event->Assign(11, to_port(${ev.dst_p}, ${ev.protocol}));
+			ids_event->Assign(12, new Val(${ev.impact_flag}, TYPE_COUNT));
+			ids_event->Assign(13, new Val(${ev.impact}, TYPE_COUNT));
+			ids_event->Assign(14, new Val(${ev.blocked}, TYPE_COUNT));
+			ids_event->Assign(15, new Val(${ev.mpls_label}, TYPE_COUNT));
+			ids_event->Assign(16, new Val(${ev.vlan_id}, TYPE_COUNT));
+
+			val_list* vl = new val_list();
+			vl->append(connection()->bro_analyzer()->GetFile()->GetVal()->Ref());
+			vl->append(ids_event);
+			mgr.QueueEvent(::unified2_event, vl, SOURCE_LOCAL);
+			}
+
+		return true;
+		%}
+
+	function proc_packet(pkt: Packet) : bool
+		%{
+		if ( ::unified2_packet )
+			{
+			RecordVal* packet = new RecordVal(BifType::Record::Unified2::Packet);
+			packet->Assign(0, new Val(${pkt.sensor_id}, TYPE_COUNT));
+			packet->Assign(1, new Val(${pkt.event_id}, TYPE_COUNT));
+			packet->Assign(2, new Val(${pkt.event_second}, TYPE_COUNT));
+			packet->Assign(3, new Val(ts_to_double(${pkt.packet_ts}), TYPE_TIME));
+			packet->Assign(4, new Val(${pkt.link_type}, TYPE_COUNT));
+			packet->Assign(5, bytestring_to_val(${pkt.packet_data}));
+
+			val_list* vl = new val_list();
+			vl->append(connection()->bro_analyzer()->GetFile()->GetVal()->Ref());
+			vl->append(packet);
+			mgr.QueueEvent(::unified2_packet, vl, SOURCE_LOCAL);
+			}
+
+		return true;
+		%}
+
+	#function proc_unknown_record_type(rec: UnknownRecordType) : bool
+	#	%{
+	#	printf("unknown packet type\n");
+	#	return true;
+	#	%}
+};
+
+#refine typeattr Record += &let {
+#	proc : bool = $context.flow.proc_record(this);
+#};
+
+refine typeattr IDS_Event += &let {
+	proc : bool = $context.flow.proc_ids_event(this);
+};
+
+refine typeattr IDS_Event_2 += &let {
+	proc : bool = $context.flow.proc_ids_event_2(this);
+};
+
+refine typeattr Packet += &let {
+	proc : bool = $context.flow.proc_packet(this);
+};
+
+#refine typeattr UnknownRecordType += &let {
+#	proc : bool = $context.flow.proc_unknown_record_type(this);
+#};

src/file_analysis/analyzer/unified2/unified2-file.pac

@@ -0,0 +1,91 @@
+
+enum Types {
+	PACKET           = 2,
+	IDS_EVENT        = 7,
+	IDS_EVENT_IPV6   = 72,
+	IDS_EVENT_2      = 104,
+	IDS_EVENT_IPV6_2 = 105,
+	EXTRA_DATA       = 110,
+};
+
+type Time = record {
+	seconds:      uint32;
+	microseconds: uint32;
+} &byteorder=bigendian;
+
+type Record = record {
+	rtype:   uint32;
+	length:  uint32;
+	data:   case rtype of {
+		PACKET            -> packet:              Packet(this);
+		IDS_EVENT         -> ids_event:           IDS_Event(this, 1);
+		IDS_EVENT_IPV6    -> ids_event_ipv6:      IDS_Event(this, 4);
+		IDS_EVENT_2       -> ids_event_vlan:      IDS_Event_2(this, 1);
+		IDS_EVENT_IPV6_2  -> ids_event_ipv6_vlan: IDS_Event_2(this, 4);
+		#EXTRA_DATA        -> extra_data:          ExtraData(this);
+		default           -> unknown_record_type: UnknownRecordType(this);
+	};
+} &byteorder=bigendian &length=length+8;
+
+type IDS_Event(rec: Record, ip_len: int) = record {
+	sensor_id:          uint32;
+	event_id:           uint32;
+	ts:                 Time;
+	signature_id:       uint32;
+	generator_id:       uint32;
+	signature_revision: uint32;
+	classification_id:  uint32;
+	priority_id:        uint32;
+	src_ip:             uint32[ip_len];
+	dst_ip:             uint32[ip_len];
+	src_p:              uint16;
+	dst_p:              uint16;
+	protocol:           uint8;
+	packet_action:      uint8;
+} &byteorder=bigendian;
+
+type IDS_Event_2(rec: Record, ip_len: int) = record {
+	sensor_id:          uint32;
+	event_id:           uint32;
+	ts:                 Time;
+	signature_id:       uint32;
+	generator_id:       uint32;
+	signature_revision: uint32;
+	classification_id:  uint32;
+	priority_id:        uint32;
+	src_ip:             uint32[ip_len];
+	dst_ip:             uint32[ip_len];
+	src_p:              uint16;
+	dst_p:              uint16;
+	protocol:           uint8;
+	impact_flag:        uint8;
+	impact:             uint8;
+	blocked:            uint8;
+	mpls_label:         uint32;
+	vlan_id:            uint16;
+	pad:                uint16;
+} &byteorder=bigendian;
+
+type Packet(rec: Record) = record {
+	sensor_id:          uint32;
+	event_id:           uint32;
+	event_second:       uint32;
+	packet_ts:          Time;
+	link_type:          uint32;
+	packet_len:         uint32;
+	packet_data:        bytestring &length=packet_len;
+} &byteorder=bigendian;
+
+type ExtraData(rec: Record) = record {
+	sensor_id:          uint32;
+	event_id:           uint32;
+	event_second:       uint32;
+	extra_type:         uint32;
+	data_type:          uint32;
+	blob_len:           uint32;
+	blob:               bytestring &length=blob_len;
+} &byteorder=bigendian &length=rec.length;
+
+type UnknownRecordType(rec: Record) = record {
+	data: bytestring &transient &length=rec.length;
+} &byteorder=bigendian &length=rec.length;

src/file_analysis/analyzer/unified2/unified2.pac

@@ -0,0 +1,21 @@
+
+%include binpac.pac
+%include bro.pac
+
+analyzer Unified2 withcontext {
+	analyzer:   Unified2_Analyzer;
+	flow:       Flow;
+};
+
+analyzer Unified2_Analyzer(bro_analyzer: BroFileAnalyzer) {
+	downflow = Flow;
+	upflow   = Flow;
+};
+
+%include unified2-file.pac
+
+flow Flow {
+	flowunit = Record withcontext(connection, this);
+};
+
+%include unified2-analyzer.pac

src/input/readers/Raw.cc

@@ -19,7 +19,12 @@ using threading::Value;
 using threading::Field;
 
 const int Raw::block_size = 4096; // how big do we expect our chunks of data to be.
+pthread_mutex_t Raw::fork_mutex;
 
+bool Raw::ClassInit()
+	{
+	return pthread_mutex_init(&fork_mutex, 0) == 0;
+	}
 
 Raw::Raw(ReaderFrontend *frontend) : ReaderBackend(frontend)
 	{
@@ -77,10 +82,51 @@ void Raw::DoClose()
 		}
 	}
 
+void Raw::ClosePipeEnd(int i)
+	{
+	if ( pipes[i] == -1 )
+		return;
+
+	safe_close(pipes[i]);
+	pipes[i] = -1;
+	}
+
+bool Raw::LockForkMutex()
+	{
+	int res = pthread_mutex_lock(&fork_mutex);
+	if ( res == 0 )
+		return true;
+
+	Error(Fmt("cannot lock fork mutex: %d", res));
+	return false;
+	}
+
+bool Raw::UnlockForkMutex()
+	{
+	int res = pthread_mutex_unlock(&fork_mutex);
+	if ( res == 0 )
+		return true;
+
+	Error(Fmt("cannot unlock fork mutex: %d", res));
+	return false;
+	}
+
 bool Raw::Execute()
 	{
+	// AFAICT, pipe/fork/exec should be thread-safe, but actually having
+	// multiple threads set up pipes and fork concurrently sometimes
+	// results in problems w/ a stdin pipe not ever getting an EOF even
+	// though both ends of it are closed.  But if the same threads
+	// allocate pipes and fork individually or sequentially, that issue
+	// never crops up... ("never" meaning I haven't seen in it in
+	// hundreds of tests using 50+ threads where before I'd see the issue
+	// w/ just 2 threads ~33% of the time).
+	if ( ! LockForkMutex() )
+		return false;
+
 	if ( pipe(pipes) != 0 || pipe(pipes+2) || pipe(pipes+4) )
 		{
+		UnlockForkMutex();
 		Error(Fmt("Could not open pipe: %d", errno));
 		return false;
 		}
@@ -88,6 +134,7 @@ bool Raw::Execute()
 	childpid = fork();
 	if ( childpid < 0 )
 		{
+		UnlockForkMutex();
 		Error(Fmt("Could not create child process: %d", errno));
 		return false;
 		}
@@ -95,65 +142,83 @@ bool Raw::Execute()
 	else if ( childpid == 0 )
 		{
 		// we are the child.
-		safe_close(pipes[stdout_in]);
+		close(pipes[stdout_in]);
 		if ( dup2(pipes[stdout_out], stdout_fileno) == -1 )
-			Error(Fmt("Error on dup2 stdout_out: %d", errno));
+			_exit(252);
 
-		if ( stdin_towrite )
+		close(pipes[stdout_out]);
-			{
-			safe_close(pipes[stdin_out]);
-			if ( dup2(pipes[stdin_in], stdin_fileno) == -1 )
-				Error(Fmt("Error on dup2 stdin_in: %d", errno));
-			}
 
-		if ( use_stderr )
+		close(pipes[stdin_out]);
-			{
+		if ( stdin_towrite && dup2(pipes[stdin_in], stdin_fileno) == -1 )
-			safe_close(pipes[stderr_in]);
+			_exit(253);
-			if ( dup2(pipes[stderr_out], stderr_fileno) == -1 )
+
-				Error(Fmt("Error on dup2 stderr_out: %d", errno));
+		close(pipes[stdin_in]);
-			}
+
+		close(pipes[stderr_in]);
+		if ( use_stderr && dup2(pipes[stderr_out], stderr_fileno) == -1 )
+			_exit(254);
+
+		close(pipes[stderr_out]);
 
 		execl("/bin/sh", "sh", "-c", fname.c_str(), (char*) NULL);
 		fprintf(stderr, "Exec failed :(......\n");
-		exit(255);
+		_exit(255);
 		}
 	else
 		{
 		// we are the parent
-		safe_close(pipes[stdout_out]);
+		if ( ! UnlockForkMutex() )
-		pipes[stdout_out] = -1;
+			return false;
+
+		ClosePipeEnd(stdout_out);
 
 		if ( Info().mode == MODE_STREAM )
 			fcntl(pipes[stdout_in], F_SETFL, O_NONBLOCK);
 
+		ClosePipeEnd(stdin_in);
+
 		if ( stdin_towrite )
-			{
+			// Ya, just always set this to nonblocking. we do not
-			safe_close(pipes[stdin_in]);
+			// want to block on a program receiving data. Note
-			pipes[stdin_in] = -1;
+			// that there is a small gotcha with it. More data is
-			fcntl(pipes[stdin_out], F_SETFL, O_NONBLOCK); // ya, just always set this to nonblocking. we do not want to block on a program receiving data.
+			// queued when more data is read from the program
-			// note that there is a small gotcha with it. More data is queued when more data is read from the program output. Hence, when having
+			// output. Hence, when having a program in
-			// a program in mode_manual where the first write cannot write everything, the rest will be stuck in a queue that is never emptied.
+			// mode_manual where the first write cannot write
-			}
+			// everything, the rest will be stuck in a queue that
+			// is never emptied.
+			fcntl(pipes[stdin_out], F_SETFL, O_NONBLOCK);
+		else
+			ClosePipeEnd(stdin_out);
+
+		ClosePipeEnd(stderr_out);
 
 		if ( use_stderr )
-			{
-			safe_close(pipes[stderr_out]);
-			pipes[stderr_out] = -1;
 			fcntl(pipes[stderr_in], F_SETFL, O_NONBLOCK); // true for this too.
-			}
+		else
+			ClosePipeEnd(stderr_in);
 
 		file = fdopen(pipes[stdout_in], "r");
-		pipes[stdout_in] = -1; // will be closed by fclose
 
-		if ( use_stderr )
+		if ( ! file )
-			stderrfile = fdopen(pipes[stderr_in], "r");
-			pipes[stderr_in] = -1; // will be closed by fclose
-			if ( file == 0 || (stderrfile == 0 && use_stderr) )
 			{
-				Error("Could not convert fileno to file");
+			Error("Could not convert stdout_in fileno to file");
 			return false;
 			}
 
+		pipes[stdout_in] = -1; // will be closed by fclose
+
+		if ( use_stderr )
+			{
+			stderrfile = fdopen(pipes[stderr_in], "r");
+
+			if ( ! stderrfile )
+				{
+				Error("Could not convert stderr_in fileno to file");
+				return false;
+				}
+
+			pipes[stderr_in] = -1; // will be closed by fclose
+			}
 
 		return true;
 		}
@@ -172,6 +237,7 @@ bool Raw::OpenInput()
 			Error(Fmt("Init: cannot open %s", fname.c_str()));
 			return false;
 			}
+
 		fcntl(fileno(file),  F_SETFD, FD_CLOEXEC);
 		}
 
@@ -194,14 +260,10 @@ bool Raw::CloseInput()
 	if ( use_stderr )
 		fclose(stderrfile);
 
-	if ( execute ) // we do not care if any of those fails. They should all be defined.
+	if ( execute )
 		{
 		for ( int i = 0; i < 6; i ++ )
-			if ( pipes[i] != -1 )
+			ClosePipeEnd(i);
-				{
-				safe_close(pipes[i]);
-				pipes[i] = -1;
-				}
 		}
 
 	file = 0;
@@ -402,10 +464,7 @@ void Raw::WriteToStdin()
 		}
 
 	if ( stdin_towrite == 0 ) // send EOF when we are done.
-		{
+		ClosePipeEnd(stdin_out);
-		safe_close(pipes[stdin_out]);
-		pipes[stdin_out] = -1;
-		}
 
 	if ( Info().mode == MODE_MANUAL && stdin_towrite != 0 )
 		{

src/input/readers/Raw.h

@@ -4,6 +4,7 @@
 #define INPUT_READERS_RAW_H
 
 #include <vector>
+#include <pthread.h>
 
 #include "../ReaderBackend.h"
 
@@ -20,6 +21,8 @@ public:
 
 	static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Raw(frontend); }
 
+	static bool ClassInit();
+
 protected:
 	virtual bool DoInit(const ReaderInfo& info, int arg_num_fields, const threading::Field* const* fields);
 	virtual void DoClose();
@@ -27,6 +30,10 @@ protected:
 	virtual bool DoHeartbeat(double network_time, double current_time);
 
 private:
+	void ClosePipeEnd(int i);
+	bool LockForkMutex();
+	bool UnlockForkMutex();
+
 	bool OpenInput();
 	bool CloseInput();
 	int64_t GetLine(FILE* file);
@@ -44,7 +51,6 @@ private:
 	string separator;
 	unsigned int sep_length; // length of the separator
 
-	static const int block_size;
 	int bufpos;
 	char* buf;
 	char* outbuf;
@@ -72,6 +78,8 @@ private:
 		stderr_out = 5
 	};
 
+	static const int block_size;
+	static pthread_mutex_t fork_mutex;
 };
 
 }

src/main.cc

@@ -57,6 +57,7 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void);
 #include "input/Manager.h"
 #include "logging/Manager.h"
 #include "logging/writers/Ascii.h"
+#include "input/readers/Raw.h"
 #include "analyzer/Manager.h"
 #include "analyzer/Tag.h"
 #include "plugin/Manager.h"
@@ -842,6 +843,8 @@ int main(int argc, char** argv)
 
 	init_event_handlers();
 
+	input::reader::Raw::ClassInit();
+
 	// The leak-checker tends to produce some false
 	// positives (memory which had already been
 	// allocated before we start the checking is
@@ -1151,10 +1154,10 @@ int main(int argc, char** argv)
 		curl_global_cleanup();
 #endif
 
-		sqlite3_shutdown();
-
 		terminate_bro();
 
+		sqlite3_shutdown();
+
 		// Close files after net_delete(), because net_delete()
 		// might write to connection content files.
 		BroFile::CloseCachedFiles();

src/plugin/Macros.h

@@ -9,6 +9,7 @@
 #define PLUGIN_MACROS_H
 
 #include "analyzer/Component.h"
+#include "file_analysis/Component.h"
 
 /**
  * The current plugin API version. Plugins that won't match this version will
@@ -91,6 +92,19 @@
 #define BRO_PLUGIN_ANALYZER(tag, cls) \
 	AddComponent(new ::analyzer::Component(tag, ::analyzer::cls::InstantiateAnalyzer));
 
+/**
+ * Defines a component implementing a file analyzer.
+ *
+ * @param tag A string with the analyzer's tag. This must be unique across
+ * all loaded analyzers and will translate into a corresponding \c ANALYZER_*
+ * constant at the script-layer.
+ *
+ * @param cls The class that implements the analyzer. It must be derived
+ * (directly or indirectly) from file_analysis::Analyzer.
+ */
+#define BRO_PLUGIN_FILE_ANALYZER(tag, cls) \
+	AddComponent(new ::file_analysis::Component(tag, ::file_analysis::cls::Instantiate));
+
 /**
  * Defines a component implementing a protocol analyzer class that will
  * not be instantiated dynamically. This is for two use-cases: (1) abstract

src/probabilistic/BitVector.cc

@@ -492,10 +492,8 @@ BitVector::size_type BitVector::FindNext(size_type i) const
 	return block ? bi * bits_per_block + lowest_bit(block) : find_from(bi + 1);
 	}
 
-size_t BitVector::Hash() const
+uint64 BitVector::Hash() const
 	{
-	size_t hash = 0;
-
 	u_char buf[SHA256_DIGEST_LENGTH];
 	SHA256_CTX ctx;
 	sha256_init(&ctx);
@@ -504,7 +502,7 @@ size_t BitVector::Hash() const
 		sha256_update(&ctx, &bits[i], sizeof(bits[i]));
 
 	sha256_final(&ctx, buf);
-	return *reinterpret_cast<size_t*>(buf); // Use the first bytes as seed.
+	return *reinterpret_cast<uint64*>(buf); // Use the first bytes as digest.
 	}
 
 BitVector::size_type BitVector::lowest_bit(block_type block)

src/probabilistic/BitVector.h

@@ -15,7 +15,7 @@ namespace probabilistic {
  */
 class BitVector : public SerialObj {
 public:
-	typedef size_t block_type;
+	typedef uint64 block_type;
 	typedef size_t size_type;
 	typedef bool const_reference;
 
@@ -281,7 +281,7 @@ public:
 	  *
 	  * @return The hash.
 	  */
-	size_t Hash() const;
+	uint64 Hash() const;
 
 	/**
 	 * Serializes the bit vector.

src/probabilistic/BloomFilter.cc

@@ -111,7 +111,7 @@ BasicBloomFilter* BasicBloomFilter::Clone() const
 
 std::string BasicBloomFilter::InternalState() const
 	{
-	return fmt("%" PRIu64, (uint64_t)bits->Hash());
+	return fmt("%" PRIu64, bits->Hash());
 	}
 
 BasicBloomFilter::BasicBloomFilter()
@@ -219,7 +219,7 @@ CountingBloomFilter* CountingBloomFilter::Clone() const
 
 string CountingBloomFilter::InternalState() const
 	{
-	return fmt("%" PRIu64, (uint64_t)cells->Hash());
+	return fmt("%" PRIu64, cells->Hash());
 	}
 
 IMPLEMENT_SERIAL(CountingBloomFilter, SER_COUNTINGBLOOMFILTER)

src/probabilistic/CounterVector.cc

@@ -153,7 +153,7 @@ CounterVector operator|(const CounterVector& x, const CounterVector& y)
 
 }
 
-size_t CounterVector::Hash() const
+uint64 CounterVector::Hash() const
 	{
 	return bits->Hash();
 	}

src/probabilistic/CounterVector.h

@@ -131,7 +131,7 @@ public:
 	  *
 	  * @return The hash.
 	  */
-	size_t Hash() const;
+	uint64 Hash() const;
 
 	/**
 	 * Serializes the bit vector.

src/probabilistic/Hasher.cc

@@ -10,7 +10,7 @@
 
 using namespace probabilistic;
 
-size_t Hasher::MakeSeed(const void* data, size_t size)
+uint64 Hasher::MakeSeed(const void* data, size_t size)
 	{
 	u_char buf[SHA256_DIGEST_LENGTH];
 	SHA256_CTX ctx;
@@ -29,7 +29,7 @@ size_t Hasher::MakeSeed(const void* data, size_t size)
 		}
 
 	sha256_final(&ctx, buf);
-	return *reinterpret_cast<size_t*>(buf); // Use the first bytes as seed.
+	return *reinterpret_cast<uint64*>(buf); // Use the first bytes as seed.
 	}
 
 Hasher::digest_vector Hasher::Hash(const HashKey* key) const

src/probabilistic/Hasher.h

@@ -30,7 +30,7 @@ public:
 	 *
 	 * @return A seed suitable for hashers.
 	 */
-	static size_t MakeSeed(const void* data, size_t size);
+	static uint64 MakeSeed(const void* data, size_t size);
 
 	/**
 	 * Destructor.

src/strings.bif

@@ -10,6 +10,45 @@ using namespace std;
 #include "SmithWaterman.h"
 %%}
 
+## Calculates the Levenshtein distance between the two strings. See `Wikipedia
+## <http://en.wikipedia.org/wiki/Levenshtein_distance>`_ for more information.
+##
+## s1: The first string.
+##
+## s2: The second string.
+##
+## Returns: The Levenshtein distance of two strings as a count.
+##
+function levenshtein_distance%(s1: string, s2: string%): count
+	%{
+	unsigned int n = s1->Len();
+	unsigned int m = s2->Len();
+
+	if ( ! n )
+		return new Val(m, TYPE_COUNT);
+
+	if ( ! m )
+		return new Val(n, TYPE_COUNT);
+
+	vector<vector<unsigned int> > d(n + 1, vector<unsigned int>(m + 1));
+
+	d[0][0] = 0;
+
+	for ( unsigned int i = 1; i <= n; ++i )
+		d[i][0] = i;
+
+	for ( unsigned int i = 1; i <= m; ++i )
+		d[0][i] = i;
+
+	for ( unsigned int i = 1; i <= n; ++i )
+		{
+		for ( unsigned int j = 1; j <= m; ++j )
+			d[i][j] = min(min(d[i-1][j] + 1, d[i][j-1] + 1),
+				      d[i-1][j-1] + (s1->Bytes()[i-1] == s2->Bytes()[j-1] ? 0 : 1));
+		}
+
+	return new Val(d[n][m], TYPE_COUNT);
+	%}
 
 ## Concatenates all arguments into a single string. The function takes a
 ## variable number of arguments of type string and stitches them together.

testing/btest/Baseline/bifs.bloomfilter/output

@@ -12,8 +12,8 @@ error: false-positive rate must take value between 0 and 1
 1
 1
 1
-0, no fp
+1, fp
-1
+1, fp
 1
 1
 1

testing/btest/Baseline/bifs.levenshtein_distance/out

@@ -0,0 +1,11 @@
+1
+1
+1
+1
+1
+1
+16
+16
+0
+0
+3

testing/btest/Baseline/core.leaks.ip-in-ip/output

@@ -1,13 +0,0 @@
-new_connection: tunnel
-    conn_id: [orig_h=dead::beef, orig_p=30000/udp, resp_h=cafe::babe, resp_p=13000/udp]
-    encap: [[cid=[orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=0/unknown, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=0/unknown], tunnel_type=Tunnel::IP, uid=UWkUyAuUGXf]]
-new_connection: tunnel
-    conn_id: [orig_h=dead::beef, orig_p=30000/udp, resp_h=cafe::babe, resp_p=13000/udp]
-    encap: [[cid=[orig_h=feed::beef, orig_p=0/unknown, resp_h=feed::cafe, resp_p=0/unknown], tunnel_type=Tunnel::IP, uid=UWkUyAuUGXf], [cid=[orig_h=babe::beef, orig_p=0/unknown, resp_h=dead::babe, resp_p=0/unknown], tunnel_type=Tunnel::IP, uid=arKYeMETxOg]]
-new_connection: tunnel
-    conn_id: [orig_h=dead::beef, orig_p=30000/udp, resp_h=cafe::babe, resp_p=13000/udp]
-    encap: [[cid=[orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=0/unknown, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=0/unknown], tunnel_type=Tunnel::IP, uid=UWkUyAuUGXf]]
-tunnel_changed:
-    conn_id: [orig_h=dead::beef, orig_p=30000/udp, resp_h=cafe::babe, resp_p=13000/udp]
-    old: [[cid=[orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=0/unknown, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=0/unknown], tunnel_type=Tunnel::IP, uid=UWkUyAuUGXf]]
-    new: [[cid=[orig_h=feed::beef, orig_p=0/unknown, resp_h=feed::cafe, resp_p=0/unknown], tunnel_type=Tunnel::IP, uid=k6kgXLOoSKl]]

testing/btest/Baseline/core.leaks.ipv6_ext_headers/output

@@ -1,4 +0,0 @@
-weird routing0_hdr from 2001:4f8:4:7:2e0:81ff:fe52:ffff to 2001:78:1:32::2
-[orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=53/udp, resp_h=2001:78:1:32::2, resp_p=53/udp]
-[ip=<uninitialized>, ip6=[class=0, flow=0, len=59, nxt=0, hlim=64, src=2001:4f8:4:7:2e0:81ff:fe52:ffff, dst=2001:4f8:4:7:2e0:81ff:fe52:9a6b, exts=[[id=0, hopopts=[nxt=43, len=0, options=[[otype=1, len=4, data=\0\0\0\0]]], dstopts=<uninitialized>, routing=<uninitialized>, fragment=<uninitialized>, ah=<uninitialized>, esp=<uninitialized>, mobility=<uninitialized>], [id=43, hopopts=<uninitialized>, dstopts=<uninitialized>, routing=[nxt=17, len=4, rtype=0, segleft=2, data=\0\0\0\0 ^A\0x\0^A\02\0\0\0\0\0\0\0^A ^A\0x\0^A\02\0\0\0\0\0\0\0^B], fragment=<uninitialized>, ah=<uninitialized>, esp=<uninitialized>, mobility=<uninitialized>]]], tcp=<uninitialized>, udp=[sport=53/udp, dport=53/udp, ulen=11], icmp=<uninitialized>]
-[2001:78:1:32::1, 2001:78:1:32::2]

testing/btest/Baseline/core.leaks.vector-val-bifs/output

@@ -1,10 +0,0 @@
-[1, 3, 0, 2]
-[2374950123]
-[1, 3, 0, 2]
-[2374950123]
-[1, 3, 0, 2]
-[2374950123]
-[1, 3, 0, 2]
-[3353991673]
-[1, 3, 0, 2]
-[3353991673]

testing/btest/Baseline/core.print-bpf-filters/conn.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
-#open	2013-07-18-00-18-33
+#open	2013-08-12-18-24-50
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
 1278600802.069419	UWkUyAuUGXf	10.20.80.1	50343	10.0.0.15	80	tcp	-	0.004152	9	3429	SF	-	0	ShADadfF	7	381	7	3801	(empty)
-#close	2013-07-18-00-18-33
+#close	2013-08-12-18-24-50

testing/btest/Baseline/core.print-bpf-filters/output

@@ -3,28 +3,28 @@
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
-#open	2013-07-19-02-54-13
+#open	2013-08-12-18-24-49
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1374202453.158981	-	ip or not ip	T	T
+1376331889.617206	-	ip or not ip	T	T
-#close	2013-07-19-02-54-13
+#close	2013-08-12-18-24-49
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
-#open	2013-07-19-02-54-13
+#open	2013-08-12-18-24-49
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1374202453.437816	-	port 42	T	T
+1376331889.904944	-	port 42	T	T
-#close	2013-07-19-02-54-13
+#close	2013-08-12-18-24-49
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
-#open	2013-07-19-02-54-13
+#open	2013-08-12-18-24-50
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1374202453.715717	-	(vlan) and (ip or not ip)	T	T
+1376331890.192875	-	(vlan) and (ip or not ip)	T	T
-#close	2013-07-19-02-54-13
+#close	2013-08-12-18-24-50

testing/btest/Baseline/core.print-bpf-filters/output2

@@ -1,5 +1,6 @@
 2 1080
 1 137
+1 20000
 1 21
 1 2123
 1 2152
@@ -38,8 +39,8 @@
 1 992
 1 993
 1 995
-42 and
+43 and
-41 or
+42 or
-42 port
+43 port
-31 tcp
+32 tcp
 11 udp

testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2013-08-09-16-13-58
+#open	2013-08-14-01-19-27
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -22,8 +22,11 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/plugins/Bro_ConnSize.events.bif.bro
     build/scripts/base/bif/plugins/Bro_DCE_RPC.events.bif.bro
     build/scripts/base/bif/plugins/Bro_DHCP.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_DNP3.events.bif.bro
     build/scripts/base/bif/plugins/Bro_DNS.events.bif.bro
     build/scripts/base/bif/plugins/Bro_File.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_FileExtract.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_FileExtract.functions.bif.bro
     build/scripts/base/bif/plugins/Bro_FileHash.events.bif.bro
     build/scripts/base/bif/plugins/Bro_Finger.events.bif.bro
     build/scripts/base/bif/plugins/Bro_FTP.events.bif.bro
@@ -61,6 +64,8 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/plugins/Bro_TCP.functions.bif.bro
     build/scripts/base/bif/plugins/Bro_Teredo.events.bif.bro
     build/scripts/base/bif/plugins/Bro_UDP.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_Unified2.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_Unified2.types.bif.bro
     build/scripts/base/bif/plugins/Bro_ZIP.events.bif.bro
   scripts/base/frameworks/logging/__load__.bro
     scripts/base/frameworks/logging/main.bro
@@ -94,4 +99,4 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/top-k.bif.bro
 scripts/policy/misc/loaded-scripts.bro
   scripts/base/utils/paths.bro
-#close	2013-08-09-16-13-58
+#close	2013-08-14-01-19-27

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2013-08-09-16-13-37
+#open	2013-08-14-01-19-28
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -22,8 +22,11 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/plugins/Bro_ConnSize.events.bif.bro
     build/scripts/base/bif/plugins/Bro_DCE_RPC.events.bif.bro
     build/scripts/base/bif/plugins/Bro_DHCP.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_DNP3.events.bif.bro
     build/scripts/base/bif/plugins/Bro_DNS.events.bif.bro
     build/scripts/base/bif/plugins/Bro_File.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_FileExtract.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_FileExtract.functions.bif.bro
     build/scripts/base/bif/plugins/Bro_FileHash.events.bif.bro
     build/scripts/base/bif/plugins/Bro_Finger.events.bif.bro
     build/scripts/base/bif/plugins/Bro_FTP.events.bif.bro
@@ -61,6 +64,8 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/plugins/Bro_TCP.functions.bif.bro
     build/scripts/base/bif/plugins/Bro_Teredo.events.bif.bro
     build/scripts/base/bif/plugins/Bro_UDP.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_Unified2.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_Unified2.types.bif.bro
     build/scripts/base/bif/plugins/Bro_ZIP.events.bif.bro
   scripts/base/frameworks/logging/__load__.bro
     scripts/base/frameworks/logging/main.bro
@@ -162,6 +167,9 @@ scripts/base/init-default.bro
     scripts/base/protocols/dhcp/consts.bro
     scripts/base/protocols/dhcp/main.bro
       scripts/base/protocols/dhcp/utils.bro
+  scripts/base/protocols/dnp3/__load__.bro
+    scripts/base/protocols/dnp3/main.bro
+      scripts/base/protocols/dnp3/consts.bro
   scripts/base/protocols/dns/__load__.bro
     scripts/base/protocols/dns/consts.bro
     scripts/base/protocols/dns/main.bro
@@ -206,6 +214,8 @@ scripts/base/init-default.bro
     scripts/base/files/hash/main.bro
   scripts/base/files/extract/__load__.bro
     scripts/base/files/extract/main.bro
+  scripts/base/files/unified2/__load__.bro
+    scripts/base/files/unified2/main.bro
   scripts/base/misc/find-checksum-offloading.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2013-08-09-16-13-37
+#close	2013-08-14-01-19-28

testing/btest/Baseline/scripts.base.files.extract.limit/1.out

@@ -0,0 +1 @@
+file_extraction_limit, 3000, 2896, 1448

testing/btest/Baseline/scripts.base.files.extract.limit/2.out

@@ -0,0 +1,3 @@
+file_extraction_limit, 3000, 2896, 1448
+T
+file_extraction_limit, 6000, 5792, 1448