Merge remote-tracking branch 'origin/master' into topic/robin/radius-merge

Conflicts:
	scripts/base/init-default.bro

scripts/base/files/x509/README

@@ -0,0 +1 @@
+Support for X509 certificates with the file analysis framework.

scripts/base/files/x509/__load__.bro

@@ -0,0 +1 @@
+@load ./main

scripts/base/files/x509/main.bro

@@ -0,0 +1,77 @@
+@load base/frameworks/files
+@load base/files/hash
+
+module X509;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		## Current timestamp.
+		ts: time &log;
+
+		## File id of this certificate.
+		id: string &log;
+
+		## Basic information about the certificate.
+		certificate: X509::Certificate &log;
+
+		## The opaque wrapping the certificate. Mainly used
+		## for the verify operations.
+		handle: opaque of x509;
+
+		## All extensions that were encountered in the certificate.
+		extensions: vector of X509::Extension &default=vector();
+
+		## Subject alternative name extension of the certificate.
+		san: X509::SubjectAlternativeName &optional &log;
+
+		## Basic constraints extension of the certificate.
+		basic_constraints: X509::BasicConstraints &optional &log;
+	};
+
+	## Event for accessing logged records.
+	global log_x509: event(rec: Info);
+}
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(X509::LOG, [$columns=Info, $ev=log_x509]);
+	}
+
+redef record Files::Info += {
+	## Information about X509 certificates. This is used to keep
+	## certificate information until all events have been received.
+	x509: X509::Info &optional;
+};
+
+event x509_certificate(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate) &priority=5
+	{
+	f$info$x509 = [$ts=f$info$ts, $id=f$id, $certificate=cert, $handle=cert_ref];
+	}
+
+event x509_extension(f: fa_file, ext: X509::Extension) &priority=5
+	{
+	if ( f$info?$x509 )
+		f$info$x509$extensions[|f$info$x509$extensions|] = ext;
+	}
+
+event x509_ext_basic_constraints(f: fa_file, ext: X509::BasicConstraints) &priority=5
+	{
+	if ( f$info?$x509 )
+		f$info$x509$basic_constraints = ext;
+	}
+
+event x509_ext_subject_alternative_name(f: fa_file, ext: X509::SubjectAlternativeName) &priority=5
+	{
+	if ( f$info?$x509 )
+		f$info$x509$san = ext;
+	}
+
+event file_state_remove(f: fa_file) &priority=5
+	{
+	if ( ! f$info?$x509 )
+		return;
+
+	Log::write(LOG, f$info$x509);
+	}

scripts/base/frameworks/files/__load__.bro

@@ -1 +1,2 @@
 @load ./main.bro
+@load ./magic

scripts/base/frameworks/files/magic/__load__.bro

@@ -0,0 +1,2 @@
+@load-sigs ./general
+@load-sigs ./libmagic

scripts/base/frameworks/files/magic/general.sig

@@ -0,0 +1,11 @@
+# General purpose file magic signatures.
+
+signature file-plaintext {
+    file-magic /([[:print:][:space:]]{10})/
+    file-mime "text/plain", -20
+}
+
+signature file-tar {
+    file-magic /([[:print:]\x00]){100}(([[:digit:]\x00\x20]){8}){3}/
+    file-mime "application/x-tar", 150
+}

scripts/base/frameworks/files/main.bro

@@ -41,15 +41,15 @@ export {
 		## If this file was transferred over a network
 		## connection this should show the host or hosts that
 		## the data sourced from.
-		tx_hosts: set[addr] &log;
+		tx_hosts: set[addr] &default=addr_set() &log;
 
 		## If this file was transferred over a network
 		## connection this should show the host or hosts that
 		## the data traveled to.
-		rx_hosts: set[addr] &log;
+		rx_hosts: set[addr] &default=addr_set() &log;
 
 		## Connection UIDs over which the file was transferred.
-		conn_uids: set[string] &log;
+		conn_uids: set[string] &default=string_set() &log;
 
 		## An identification of the source of the file data.  E.g. it
 		## may be a network protocol over which it was transferred, or a
@@ -63,12 +63,13 @@ export {
 		depth: count &default=0 &log;
 
 		## A set of analysis types done during the file analysis.
-		analyzers: set[string] &log;
+		analyzers: set[string] &default=string_set() &log;
 
-		## A mime type provided by libmagic against the *bof_buffer*
-		## field of :bro:see:`fa_file`, or in the cases where no
-		## buffering of the beginning of file occurs, an initial
-		## guess of the mime type based on the first data seen.
+		## A mime type provided by the strongest file magic signature
+		## match against the *bof_buffer* field of :bro:see:`fa_file`,
+		## or in the cases where no buffering of the beginning of file
+		## occurs, an initial guess of the mime type based on the first
+		## data seen.
 		mime_type: string &log &optional;
 
 		## A filename for the file if one is available from the source

scripts/base/frameworks/logging/writers/ascii.bro

@@ -5,11 +5,11 @@
 ##! ``config``: setting ``tsv`` to the string ``T`` turns the output into
 ##! "tab-separated-value" mode where only a single header row with the column
 ##! names is printed out as meta information, with no "# fields" prepended; no
-##! other meta data gets included in that mode.   
-##! 
+##! other meta data gets included in that mode.
+##!
 ##! Example filter using this::
-##! 
-##!    local my_filter: Log::Filter = [$name = "my-filter", $writer = Log::WRITER_ASCII, $config = table(["tsv"] = "T")]; 
+##!
+##!    local my_filter: Log::Filter = [$name = "my-filter", $writer = Log::WRITER_ASCII, $config = table(["tsv"] = "T")];
 ##!
 
 module LogAscii;
@@ -17,27 +17,51 @@ module LogAscii;
 export {
 	## If true, output everything to stdout rather than
 	## into files. This is primarily for debugging purposes.
+	##
+	## This option is also available as a per-filter ``$config`` option.
 	const output_to_stdout = F &redef;
 
+	## If true, the default will be to write logs in a JSON format.
+	##
+	## This option is also available as a per-filter ``$config`` option.
+	const use_json = F &redef;
+
+	## Format of timestamps when writing out JSON. By default, the JSON formatter will
+	## use double values for timestamps which represent the number of seconds from the
+	## UNIX epoch.
+	const json_timestamps: JSON::TimestampFormat = JSON::TS_EPOCH &redef;
+
 	## If true, include lines with log meta information such as column names
 	## with types, the values of ASCII logging options that are in use, and
 	## the time when the file was opened and closed (the latter at the end).
+        ##
+	## If writing in JSON format, this is implicitly disabled.
 	const include_meta = T &redef;
 
 	## Prefix for lines with meta information.
+        ##
+	## This option is also available as a per-filter ``$config`` option.
 	const meta_prefix = "#" &redef;
 
 	## Separator between fields.
+	##
+	## This option is also available as a per-filter ``$config`` option.
 	const separator = Log::separator &redef;
 
 	## Separator between set elements.
+	##
+	## This option is also available as a per-filter ``$config`` option.
 	const set_separator = Log::set_separator &redef;
 
 	## String to use for empty fields. This should be different from
-        ## *unset_field* to make the output unambiguous. 
+	## *unset_field* to make the output unambiguous.
+	##
+	## This option is also available as a per-filter ``$config`` option.
 	const empty_field = Log::empty_field &redef;
 
 	## String to use for an unset &optional field.
+	##
+	## This option is also available as a per-filter ``$config`` option.
 	const unset_field = Log::unset_field &redef;
 }
 

scripts/base/frameworks/notice/main.bro

@@ -206,6 +206,38 @@ export {
 	## The maximum amount of time a plugin can delay email from being sent.
 	const max_email_delay     = 15secs &redef;
 
+	## Contains a portion of :bro:see:`fa_file` that's also contained in
+	## :bro:see:`Notice::Info`.
+	type FileInfo: record {
+		fuid: string;            ##< File UID.
+		desc: string;            ##< File description from e.g.
+		                         ##< :bro:see:`Files::describe`.
+		mime: string  &optional; ##< Strongest mime type match for file.
+		cid:  conn_id &optional; ##< Connection tuple over which file is sent.
+		cuid: string  &optional; ##< Connection UID over which file is sent.
+	};
+
+	## Creates a record containing a subset of a full :bro:see:`fa_file` record.
+	##
+	## f: record containing metadata about a file.
+	##
+	## Returns: record containing a subset of fields copied from *f*.
+	global create_file_info: function(f: fa_file): Notice::FileInfo;
+
+	## Populates file-related fields in a notice info record.
+	##
+	## f: record containing metadata about a file.
+	##
+	## n: a notice record that needs file-related fields populated.
+	global populate_file_info: function(f: fa_file, n: Notice::Info);
+
+	## Populates file-related fields in a notice info record.
+	##
+	## fi: record containing metadata about a file.
+	##
+	## n: a notice record that needs file-related fields populated.
+	global populate_file_info2: function(fi: Notice::FileInfo, n: Notice::Info);
+
 	## A log postprocessing function that implements emailing the contents
 	## of a log upon rotation to any configured :bro:id:`Notice::mail_dest`.
 	## The rotated log is removed upon being sent.
@@ -493,6 +525,42 @@ function execute_with_notice(cmd: string, n: Notice::Info)
 	#system_env(cmd, tags);
 	}
 
+function create_file_info(f: fa_file): Notice::FileInfo
+	{
+	local fi: Notice::FileInfo = Notice::FileInfo($fuid = f$id,
+	                                              $desc = Files::describe(f));
+
+	if ( f?$mime_type )
+		fi$mime = f$mime_type;
+
+	if ( f?$conns && |f$conns| == 1 )
+		for ( id in f$conns )
+			{
+			fi$cid = id;
+			fi$cuid = f$conns[id]$uid;
+			}
+
+	return fi;
+	}
+
+function populate_file_info(f: fa_file, n: Notice::Info)
+	{
+	populate_file_info2(create_file_info(f), n);
+	}
+
+function populate_file_info2(fi: Notice::FileInfo, n: Notice::Info)
+	{
+	if ( ! n?$fuid )
+		n$fuid = fi$fuid;
+
+	if ( ! n?$file_mime_type && fi?$mime )
+		n$file_mime_type = fi$mime;
+
+	n$file_desc = fi$desc;
+	n$id = fi$cid;
+	n$uid = fi$cuid;
+	}
+
 # This is run synchronously as a function before all of the other
 # notice related functions and events.  It also modifies the
 # :bro:type:`Notice::Info` record in place.
@@ -503,21 +571,7 @@ function apply_policy(n: Notice::Info)
 		n$ts = network_time();
 
 	if ( n?$f )
-		{
-		if ( ! n?$fuid )
-			n$fuid = n$f$id;
-
-		if ( ! n?$file_mime_type && n$f?$mime_type )
-			n$file_mime_type = n$f$mime_type;
-
-		n$file_desc = Files::describe(n$f);
-
-		if ( n$f?$conns && |n$f$conns| == 1 )
-			{
-			for ( id in n$f$conns )
-				n$conn = n$f$conns[id];
-			}
-		}
+		populate_file_info(n$f, n);
 
 	if ( n?$conn )
 		{

scripts/base/frameworks/notice/weird.bro

@@ -185,6 +185,7 @@ export {
 		["RPC_underflow"]                       = ACTION_LOG,
 		["RST_storm"]                           = ACTION_LOG,
 		["RST_with_data"]                       = ACTION_LOG,
+		["SSL_many_server_names"]               = ACTION_LOG,
 		["simultaneous_open"]                   = ACTION_LOG_PER_CONN,
 		["spontaneous_FIN"]                     = ACTION_IGNORE,
 		["spontaneous_RST"]                     = ACTION_IGNORE,

scripts/base/frameworks/signatures/main.bro

@@ -70,6 +70,9 @@ export {
 		## The network time at which a signature matching type of event
 		## to be logged has occurred.
 		ts:         time         &log;
+		## A unique identifier of the connection which triggered the
+		## signature match event
+		uid:        string       &log &optional;
 		## The host which triggered the signature match event.
 		src_addr:   addr         &log &optional;
 		## The host port on which the signature-matching activity
@@ -167,7 +170,7 @@ event signature_match(state: signature_state, msg: string, data: string)
 	# Trim the matched data down to something reasonable
 	if ( |data| > 140 )
 		data = fmt("%s...", sub_bytes(data, 0, 140));
-		
+	
 	local src_addr: addr;
 	local src_port: port;
 	local dst_addr: addr;
@@ -192,6 +195,7 @@ event signature_match(state: signature_state, msg: string, data: string)
 		{
 		local info: Info = [$ts=network_time(),
 		                    $note=Sensitive_Signature,
+		                    $uid=state$conn$uid,
 		                    $src_addr=src_addr,
 		                    $src_port=src_port,
 		                    $dst_addr=dst_addr,
@@ -212,11 +216,11 @@ event signature_match(state: signature_state, msg: string, data: string)
 		if ( ++count_per_resp[dst,sig_id] in count_thresholds )
 			{
 			NOTICE([$note=Count_Signature, $conn=state$conn,
-				   $msg=msg,
-				   $n=count_per_resp[dst,sig_id],
-				   $sub=fmt("%d matches of signature %s on host %s",
-						count_per_resp[dst,sig_id],
-						sig_id, dst)]);
+			        $msg=msg,
+			        $n=count_per_resp[dst,sig_id],
+			        $sub=fmt("%d matches of signature %s on host %s",
+			                 count_per_resp[dst,sig_id],
+			                 sig_id, dst)]);
 			}
 		}
 
@@ -290,16 +294,16 @@ event signature_match(state: signature_state, msg: string, data: string)
 				orig, vcount, resp);
 
 		Log::write(Signatures::LOG, 
-			[$ts=network_time(),
-			 $note=Multiple_Signatures, 
-			 $src_addr=orig,
-			 $dst_addr=resp, $sig_id=sig_id, $sig_count=vcount,
-			 $event_msg=fmt("%s different signatures triggered", vcount),
-			 $sub_msg=vert_scan_msg]);
+		           [$ts=network_time(),
+		            $note=Multiple_Signatures, 
+		            $src_addr=orig,
+		            $dst_addr=resp, $sig_id=sig_id, $sig_count=vcount,
+		            $event_msg=fmt("%s different signatures triggered", vcount),
+		            $sub_msg=vert_scan_msg]);
 
 		NOTICE([$note=Multiple_Signatures, $src=orig, $dst=resp,
-			$msg=fmt("%s different signatures triggered", vcount),
-			$n=vcount, $sub=vert_scan_msg]);
+		        $msg=fmt("%s different signatures triggered", vcount),
+		        $n=vcount, $sub=vert_scan_msg]);
 
 		last_vthresh[orig] = vcount;
 		}

scripts/base/frameworks/software/main.bro

@@ -287,6 +287,13 @@ function parse_mozilla(unparsed_version: string): Description
 		if ( 2 in parts )
 			v = parse(parts[2])$version;
 		}
+	else if ( / Java\/[0-9]\./ in unparsed_version )
+		{
+		software_name = "Java";
+		parts = split_all(unparsed_version, /Java\/[0-9\._]*/);
+		if ( 2 in parts )
+			v = parse(parts[2])$version;
+		}
 
 	return [$version=v, $unparsed_version=unparsed_version, $name=software_name];
 	}

scripts/base/frameworks/sumstats/cluster.bro

@@ -28,10 +28,6 @@ export {
 	## values for a sumstat.
 	global cluster_ss_request: event(uid: string, ss_name: string, cleanup: bool);
 
-	# Event sent by nodes that are collecting sumstats after receiving a
-	# request for the sumstat from the manager.
-	#global cluster_ss_response: event(uid: string, ss_name: string, data: ResultTable, done: bool, cleanup: bool);
-
 	## This event is sent by the manager in a cluster to initiate the
 	## collection of a single key value from a sumstat.  It's typically used
 	## to get intermediate updates before the break interval triggers to
@@ -62,7 +58,7 @@ export {
 # Add events to the cluster framework to make this work.
 redef Cluster::manager2worker_events += /SumStats::cluster_(ss_request|get_result|threshold_crossed)/;
 redef Cluster::manager2worker_events += /SumStats::(get_a_key)/;
-redef Cluster::worker2manager_events += /SumStats::cluster_(ss_response|send_result|key_intermediate_response)/;
+redef Cluster::worker2manager_events += /SumStats::cluster_(send_result|key_intermediate_response)/;
 redef Cluster::worker2manager_events += /SumStats::(send_a_key|send_no_key)/;
 
 @if ( Cluster::local_node_type() != Cluster::MANAGER )
@@ -74,7 +70,7 @@ global recent_global_view_keys: table[string, Key] of count &create_expire=1min
 
 # Result tables indexed on a uid that are currently being sent to the
 # manager.
-global sending_results: table[string] of ResultTable = table() &create_expire=1min;
+global sending_results: table[string] of ResultTable = table() &read_expire=1min;
 
 # This is done on all non-manager node types in the event that a sumstat is
 # being collected somewhere other than a worker.
@@ -144,7 +140,7 @@ event SumStats::cluster_ss_request(uid: string, ss_name: string, cleanup: bool)
 	sending_results[uid] = (ss_name in result_store) ? result_store[ss_name] : table();
 
 	# Lookup the actual sumstats and reset it, the reference to the data
-	# currently stored will be maintained internally from the 
+	# currently stored will be maintained internally from the
 	# sending_results table.
 	if ( cleanup && ss_name in stats_store )
 		reset(stats_store[ss_name]);
@@ -159,7 +155,7 @@ event SumStats::cluster_get_result(uid: string, ss_name: string, key: Key, clean
 		if ( uid in sending_results && key in sending_results[uid] )
 			{
 			# Note: copy is needed to compensate serialization caching issue. This should be
-			# changed to something else later. 
+			# changed to something else later.
 			event SumStats::cluster_send_result(uid, ss_name, key, copy(sending_results[uid][key]), cleanup);
 			delete sending_results[uid][key];
 			}
@@ -170,12 +166,12 @@ event SumStats::cluster_get_result(uid: string, ss_name: string, key: Key, clean
 			event SumStats::cluster_send_result(uid, ss_name, key, table(), cleanup);
 			}
 		}
-	else 
+	else
 		{
 		if ( ss_name in result_store && key in result_store[ss_name] )
 			{
 			# Note: copy is needed to compensate serialization caching issue. This should be
-			# changed to something else later. 
+			# changed to something else later.
 			event SumStats::cluster_send_result(uid, ss_name, key, copy(result_store[ss_name][key]), cleanup);
 			}
 		else
@@ -195,6 +191,19 @@ event SumStats::cluster_threshold_crossed(ss_name: string, key: SumStats::Key, t
 	threshold_tracker[ss_name][key] = thold_index;
 	}
 
+# request-key is a non-op on the workers.
+# It only should be called by the manager. Due to the fact that we usually run the same scripts on the
+# workers and the manager, it might also be called by the workers, so we just ignore it here.
+#
+# There is a small chance that people will try running it on events that are just thrown on the workers.
+# This does not work at the moment and we cannot throw an error message, because we cannot distinguish it
+# from the "script is running it everywhere" case. But - people should notice that they do not get results.
+# Not entirely pretty, sorry :(
+function request_key(ss_name: string, key: Key): Result
+	{
+	return Result();
+	}
+
 @endif
 
 
@@ -203,7 +212,7 @@ event SumStats::cluster_threshold_crossed(ss_name: string, key: SumStats::Key, t
 # This variable is maintained by manager nodes as they collect and aggregate
 # results.
 # Index on a uid.
-global stats_keys: table[string] of set[Key] &create_expire=1min 
+global stats_keys: table[string] of set[Key] &read_expire=1min 
 	&expire_func=function(s: table[string] of set[Key], idx: string): interval
 		{
 		Reporter::warning(fmt("SumStat key request for the %s SumStat uid took longer than 1 minute and was automatically cancelled.", idx));
@@ -215,17 +224,16 @@ global stats_keys: table[string] of set[Key] &create_expire=1min
 # matches the number of peer nodes that results should be coming from, the
 # result is written out and deleted from here.
 # Indexed on a uid.
-# TODO: add an &expire_func in case not all results are received.
-global done_with: table[string] of count &create_expire=1min &default=0;
+global done_with: table[string] of count &read_expire=1min &default=0;
 
 # This variable is maintained by managers to track intermediate responses as
 # they are getting a global view for a certain key.
 # Indexed on a uid.
-global key_requests: table[string] of Result &create_expire=1min;
+global key_requests: table[string] of Result &read_expire=1min;
 
 # Store uids for dynamic requests here to avoid cleanup on the uid.
 # (This needs to be done differently!)
-global dynamic_requests: set[string] &create_expire=1min;
+global dynamic_requests: set[string] &read_expire=1min;
 
 # This variable is maintained by managers to prevent overwhelming communication due
 # to too many intermediate updates.  Each sumstat is tracked separately so that
@@ -414,7 +422,7 @@ event SumStats::cluster_send_result(uid: string, ss_name: string, key: Key, resu
 	# Mark that a worker is done.
 	if ( uid !in done_with )
 		done_with[uid] = 0;
-	
+
 	#print fmt("MANAGER: got a result for %s %s from %s", uid, key, get_event_peer()$descr);
 	++done_with[uid];
 

scripts/base/frameworks/sumstats/non-cluster.bro

@@ -2,23 +2,59 @@
 
 module SumStats;
 
+event SumStats::process_epoch_result(ss: SumStat, now: time, data: ResultTable)
+	{
+	# TODO: is this the right processing group size?
+	local i = 50;
+	for ( key in data )
+		{
+		ss$epoch_result(now, key, data[key]);
+		delete data[key];
+
+		if ( |data| == 0 )
+			{
+			if ( ss?$epoch_finished )
+				ss$epoch_finished(now);
+
+			# Now that no data is left we can finish.
+			return;
+			}
+
+		i = i-1;
+		if ( i == 0 )
+			{
+			# TODO: is this the right interval?
+			schedule 0.01 secs { process_epoch_result(ss, now, data) };
+			break;
+			}
+		}
+	}
+
 event SumStats::finish_epoch(ss: SumStat)
 	{
 	if ( ss$name in result_store )
 		{
-		local now = network_time();
-
 		if ( ss?$epoch_result )
 			{
 			local data = result_store[ss$name];
-			# TODO: don't block here.
-			for ( key in data )
-				ss$epoch_result(now, key, data[key]);
+			local now = network_time();
+			if ( bro_is_terminating() )
+				{
+				for ( key in data )
+					ss$epoch_result(now, key, data[key]);
+
+				if ( ss?$epoch_finished )
+					ss$epoch_finished(now);
+				}
+			else
+				{
+				event SumStats::process_epoch_result(ss, now, data);
+				}
 			}
-
-		if ( ss?$epoch_finished )
-			ss$epoch_finished(now);
-
+		
+		# We can reset here because we know that the reference
+		# to the data will be maintained by the process_epoch_result
+		# event.
 		reset(ss);
 		}
 

scripts/base/init-bare.bro

@@ -39,6 +39,14 @@ type count_set: set[count];
 ##    directly and then remove this alias.
 type index_vec: vector of count;
 
+## A vector of any, used by some builtin functions to store a list of varying
+## types.
+##
+## .. todo:: We need this type definition only for declaring builtin functions
+##    via ``bifcl``. We should extend ``bifcl`` to understand composite types
+##    directly and then remove this alias.
+type any_vec: vector of any;
+
 ## A vector of strings.
 ##
 ## .. todo:: We need this type definition only for declaring builtin functions
@@ -46,6 +54,13 @@ type index_vec: vector of count;
 ##    directly and then remove this alias.
 type string_vec: vector of string;
 
+## A vector of x509 opaques.
+##
+## .. todo:: We need this type definition only for declaring builtin functions
+##    via ``bifcl``. We should extend ``bifcl`` to understand composite types
+##    directly and then remove this alias.
+type x509_opaque_vector: vector of opaque of x509;
+
 ## A vector of addresses.
 ##
 ## .. todo:: We need this type definition only for declaring builtin functions
@@ -60,6 +75,23 @@ type addr_vec: vector of addr;
 ##    directly and then remove this alias.
 type table_string_of_string: table[string] of string;
 
+## A structure indicating a MIME type and strength of a match against
+## file magic signatures.
+##
+## :bro:see:`file_magic`
+type mime_match: record {
+	strength: int;    ##< How strongly the signature matched.  Used for
+	                  ##< prioritization when multiple file magic signatures
+	                  ##< match.
+	mime:     string; ##< The MIME type of the file magic signature match.
+};
+
+## A vector of file magic signature matches, ordered by strength of
+## the signature, strongest first.
+##
+## :bro:see:`file_magic`
+type mime_matches: vector of mime_match;
+
 ## A connection's transport-layer protocol. Note that Bro uses the term
 ## "connection" broadly, using flow semantics for ICMP and UDP.
 type transport_proto: enum {
@@ -371,10 +403,15 @@ type fa_file: record {
 	## This is also the buffer that's used for file/mime type detection.
 	bof_buffer: string &optional;
 
-	## A mime type provided by libmagic against the *bof_buffer*, or
-	## in the cases where no buffering of the beginning of file occurs,
-	## an initial guess of the mime type based on the first data seen.
+	## The mime type of the strongest file magic signature matches against
+	## the data chunk in *bof_buffer*, or in the cases where no buffering
+	## of the beginning of file occurs, an initial guess of the mime type
+	## based on the first data seen.
 	mime_type: string &optional;
+
+	## All mime types that matched file magic signatures against the data
+	## chunk in *bof_buffer*, in order of their strength value.
+	mime_types: mime_matches &optional;
 } &redef;
 
 ## Fields of a SYN packet.
@@ -1028,13 +1065,6 @@ const rpc_timeout = 24 sec &redef;
 ## means "forever", which resists evasion, but can lead to state accrual.
 const frag_timeout = 0.0 sec &redef;
 
-## Time window for reordering packets. This is used for dealing with timestamp
-## discrepancy between multiple packet sources.
-##
-## .. note:: Setting this can have a major performance impact as now packets
-##    need to be potentially copied and buffered.
-const packet_sort_window = 0 usecs &redef;
-
 ## If positive, indicates the encapsulation header size that should
 ## be skipped. This applies to all packets.
 const encap_hdr_size = 0 &redef;
@@ -2420,29 +2450,6 @@ global dns_skip_all_addl = T &redef;
 ## traffic and do not process it.  Set to 0 to turn off this functionality.
 global dns_max_queries = 5;
 
-## An X509 certificate.
-##
-## .. bro:see:: x509_certificate
-type X509: record {
-	version: count;	##< Version number.
-	serial: string;	##< Serial number.
-	subject: string;	##< Subject.
-	issuer: string;	##< Issuer.
-	not_valid_before: time;	##< Timestamp before when certificate is not valid.
-	not_valid_after: time;	##< Timestamp after when certificate is not valid.
-};
-
-## An X509 extension.
-##
-## .. bro:see:: x509_extension
-type X509_extension_info: record {
-	name: string;	##< Long name of extension; oid if name not known.
-	short_name: string &optional;	##< Short name of extension if known.
-	oid: string;	##< Oid of extension.
-	critical: bool;	##< True if extension is critical.
-	value: string;	##< Extension content parsed to string for known extensions. Raw data otherwise.
-};
-
 ## HTTP session statistics.
 ##
 ## .. bro:see:: http_stats
@@ -2764,6 +2771,55 @@ export {
 	};
 }
 
+module X509;
+export {
+	type Certificate: record {
+		version: count;	##< Version number.
+		serial: string;	##< Serial number.
+		subject: string;	##< Subject.
+		issuer: string;	##< Issuer.
+		not_valid_before: time;	##< Timestamp before when certificate is not valid.
+		not_valid_after: time;	##< Timestamp after when certificate is not valid.
+		key_alg: string;	##< Name of the key algorithm
+		sig_alg: string;	##< Name of the signature algorithm
+		key_type: string &optional;	##< Key type, if key parseable by openssl (either rsa, dsa or ec)
+		key_length: count &optional;	##< Key length in bits
+		exponent: string &optional;	##< Exponent, if RSA-certificate
+		curve: string &optional;	##< Curve, if EC-certificate
+	} &log;
+
+	type Extension: record {
+		name: string;	##< Long name of extension. oid if name not known
+		short_name: string &optional;	##< Short name of extension if known
+		oid: string;	##< Oid of extension
+		critical: bool;	##< True if extension is critical
+		value: string;	##< Extension content parsed to string for known extensions. Raw data otherwise.
+	};
+
+	type BasicConstraints: record {
+		ca: bool;	##< CA flag set?
+		path_len: count &optional;	##< Maximum path length
+	} &log;
+
+	type SubjectAlternativeName: record {
+		dns: string_vec &optional &log;	##< List of DNS entries in SAN
+		uri: string_vec &optional &log;	##< List of URI entries in SAN
+		email: string_vec &optional &log;	##< List of email entries in SAN
+		ip: addr_vec &optional &log;	##< List of IP entries in SAN
+		other_fields: bool;	##< True if the certificate contained other, not recognized or parsed name fields
+	};
+
+	## Result of an X509 certificate chain verification
+	type Result: record {
+		## OpenSSL result code
+		result:	count;
+		## Result as string
+		result_string: string;
+		## References to the final certificate chain, if verification successful. End-host certificate is first.
+		chain_certs: vector of opaque of x509 &optional;
+	};
+}
+
 module SOCKS;
 export {
 	## This record is for a SOCKS client or server to provide either a
@@ -2793,6 +2849,130 @@ export {
 }
 module GLOBAL;
 
+@load base/bif/plugins/Bro_SNMP.types.bif
+
+module SNMP;
+export {
+	## The top-level message data structure of an SNMPv1 datagram, not
+	## including the PDU data.  See :rfc:`1157`.
+	type SNMP::HeaderV1: record {
+		community: string;
+	};
+
+	## The top-level message data structure of an SNMPv2 datagram, not
+	## including the PDU data.  See :rfc:`1901`.
+	type SNMP::HeaderV2: record {
+		community: string;
+	};
+
+	## The ``ScopedPduData`` data structure of an SNMPv3 datagram, not
+	## including the PDU data (i.e. just the "context" fields).
+	## See :rfc:`3412`.
+	type SNMP::ScopedPDU_Context: record {
+		engine_id: string;
+		name:      string;
+	};
+
+	## The top-level message data structure of an SNMPv3 datagram, not
+	## including the PDU data.  See :rfc:`3412`.
+	type SNMP::HeaderV3: record {
+		id:              count;
+		max_size:        count;
+		flags:           count;
+		auth_flag:       bool;
+		priv_flag:       bool;
+		reportable_flag: bool;
+		security_model:  count;
+		security_params: string;
+		pdu_context:     SNMP::ScopedPDU_Context &optional;
+	};
+
+	## A generic SNMP header data structure that may include data from
+	## any version of SNMP.  The value of the ``version`` field
+	## determines what header field is initialized.
+	type SNMP::Header: record {
+		version: count;
+		v1:      SNMP::HeaderV1 &optional; ##< Set when ``version`` is 0.
+		v2:      SNMP::HeaderV2 &optional; ##< Set when ``version`` is 1.
+		v3:      SNMP::HeaderV3 &optional; ##< Set when ``version`` is 3.
+	};
+
+	## A generic SNMP object value, that may include any of the
+	## valid ``ObjectSyntax`` values from :rfc:`1155` or :rfc:`3416`.
+	## The value is decoded whenever possible and assigned to
+	## the appropriate field, which can be determined from the value
+	## of the ``tag`` field.  For tags that can't be mapped to an
+	## appropriate type, the ``octets`` field holds the BER encoded
+	## ASN.1 content if there is any (though, ``octets`` is may also
+	## be used for other tags such as OCTET STRINGS or Opaque).  Null
+	## values will only have their corresponding tag value set.
+	type SNMP::ObjectValue: record {
+		tag:      count;
+		oid:      string &optional;
+		signed:   int    &optional;
+		unsigned: count  &optional;
+		address:  addr   &optional;
+		octets:   string &optional;
+	};
+
+	# These aren't an enum because it's easier to type fields as count.
+	# That way don't have to deal with type conversion, plus doesn't
+	# mislead that these are the only valid tag values (it's just the set
+	# of known tags).
+	const SNMP::OBJ_INTEGER_TAG       : count = 0x02; ##< Signed 64-bit integer.
+	const SNMP::OBJ_OCTETSTRING_TAG   : count = 0x04; ##< An octet string.
+	const SNMP::OBJ_UNSPECIFIED_TAG   : count = 0x05; ##< A NULL value.
+	const SNMP::OBJ_OID_TAG           : count = 0x06; ##< An Object Identifier.
+	const SNMP::OBJ_IPADDRESS_TAG     : count = 0x40; ##< An IP address.
+	const SNMP::OBJ_COUNTER32_TAG     : count = 0x41; ##< Unsigned 32-bit integer.
+	const SNMP::OBJ_UNSIGNED32_TAG    : count = 0x42; ##< Unsigned 32-bit integer.
+	const SNMP::OBJ_TIMETICKS_TAG     : count = 0x43; ##< Unsigned 32-bit integer.
+	const SNMP::OBJ_OPAQUE_TAG        : count = 0x44; ##< An octet string.
+	const SNMP::OBJ_COUNTER64_TAG     : count = 0x46; ##< Unsigned 64-bit integer.
+	const SNMP::OBJ_NOSUCHOBJECT_TAG  : count = 0x80; ##< A NULL value.
+	const SNMP::OBJ_NOSUCHINSTANCE_TAG: count = 0x81; ##< A NULL value.
+	const SNMP::OBJ_ENDOFMIBVIEW_TAG  : count = 0x82; ##< A NULL value.
+
+	## The ``VarBind`` data structure from either :rfc:`1157` or
+	## :rfc:`3416`, which maps an Object Identifier to a value.
+	type SNMP::Binding: record {
+		oid:   string;
+		value: SNMP::ObjectValue;
+	};
+
+	## A ``VarBindList`` data structure from either :rfc:`1157` or :rfc:`3416`.
+	## A sequences of :bro:see:`SNMP::Binding`, which maps an OIDs to values.
+	type SNMP::Bindings: vector of SNMP::Binding;
+
+	## A ``PDU`` data structure from either :rfc:`1157` or :rfc:`3416`.
+	type SNMP::PDU: record {
+		request_id:   int;
+		error_status: int;
+		error_index:  int;
+		bindings:     SNMP::Bindings;
+	};
+
+	## A ``Trap-PDU`` data structure from :rfc:`1157`.
+	type SNMP::TrapPDU: record {
+		enterprise:    string;
+		agent:         addr;
+		generic_trap:  int;
+		specific_trap: int;
+		time_stamp:    count;
+		bindings:      SNMP::Bindings;
+	};
+
+	## A ``BulkPDU`` data structure from :rfc:`3416`.
+	type SNMP::BulkPDU: record {
+		request_id:      int;
+		non_repeaters:   count;
+		max_repititions: count;
+		bindings:        SNMP::Bindings;
+	};
+}
+
+module GLOBAL;
+
 @load base/bif/event.bif
 
 ## BPF filter the user has set via the -f command line options. Empty if none.
@@ -3074,6 +3254,24 @@ const record_all_packets = F &redef;
 ## .. bro:see:: conn_stats
 const ignore_keep_alive_rexmit = F &redef;
 
+module JSON;
+export {
+	type TimestampFormat: enum {
+		## Timestamps will be formatted as UNIX epoch doubles.  This is
+		## the format that Bro typically writes out timestamps.
+		TS_EPOCH,
+		## Timestamps will be formatted as unsigned integers that
+		## represent the number of milliseconds since the UNIX
+		## epoch.
+		TS_MILLIS,
+		## Timestamps will be formatted in the ISO8601 DateTime format.
+		## Subseconds are also included which isn't actually part of the
+		## standard but most consumers that parse ISO8601 seem to be able
+		## to cope with that.
+		TS_ISO8601,
+	};
+}
+
 module Tunnel;
 export {
 	## The maximum depth of a tunnel to decapsulate until giving up.

scripts/base/init-default.bro

@@ -48,6 +48,7 @@
 @load base/protocols/modbus
 @load base/protocols/pop3
 @load base/protocols/radius
+@load base/protocols/snmp
 @load base/protocols/smtp
 @load base/protocols/socks
 @load base/protocols/ssh
@@ -58,7 +59,7 @@
 @load base/files/hash
 @load base/files/extract
 @load base/files/unified2
-
+@load base/files/x509
 
 @load base/misc/find-checksum-offloading
 @load base/misc/find-filtered-trace

scripts/base/protocols/dns/main.bro

@@ -181,10 +181,9 @@ function log_unmatched_msgs_queue(q: Queue::Queue)
 function log_unmatched_msgs(msgs: PendingMessages)
 	{
 	for ( trans_id in msgs )
-		{
 		log_unmatched_msgs_queue(msgs[trans_id]);
-		delete msgs[trans_id];
-		}
+
+	clear_table(msgs);
 	}
 
 function enqueue_new_msg(msgs: PendingMessages, id: count, msg: Info)
@@ -360,7 +359,15 @@ event dns_request(c: connection, msg: dns_msg, query: string, qtype: count, qcla
 	# Note: I'm ignoring the name type for now.  Not sure if this should be
 	#       worked into the query/response in some fashion.
 	if ( c$id$resp_p == 137/udp )
+		{
 		query = decode_netbios_name(query);
+		if ( c$dns$qtype_name == "SRV" )
+			{
+			# The SRV RFC used the ID used for NetBios Status RRs.
+			# So if this is NetBios Name Service we name it correctly.
+			c$dns$qtype_name = "NBSTAT";
+			}
+		}
 	c$dns$query = query;
 	}
 
@@ -375,9 +382,19 @@ event dns_A_reply(c: connection, msg: dns_msg, ans: dns_answer, a: addr) &priori
 	hook DNS::do_reply(c, msg, ans, fmt("%s", a));
 	}
 
-event dns_TXT_reply(c: connection, msg: dns_msg, ans: dns_answer, str: string) &priority=5
+event dns_TXT_reply(c: connection, msg: dns_msg, ans: dns_answer, strs: string_vec) &priority=5
 	{
-	hook DNS::do_reply(c, msg, ans, str);
+	local txt_strings: string = "";
+
+	for ( i in strs )
+		{
+		if ( i > 0 )
+			txt_strings += " ";
+
+		txt_strings += fmt("TXT %d %s", |strs[i]|, strs[i]);
+		}
+
+	hook DNS::do_reply(c, msg, ans, txt_strings);
 	}
 
 event dns_AAAA_reply(c: connection, msg: dns_msg, ans: dns_answer, a: addr) &priority=5
@@ -421,9 +438,9 @@ event dns_WKS_reply(c: connection, msg: dns_msg, ans: dns_answer) &priority=5
 	hook DNS::do_reply(c, msg, ans, "");
 	}
 
-event dns_SRV_reply(c: connection, msg: dns_msg, ans: dns_answer) &priority=5
+event dns_SRV_reply(c: connection, msg: dns_msg, ans: dns_answer, target: string, priority: count, weight: count, p: count) &priority=5
 	{
-	hook DNS::do_reply(c, msg, ans, "");
+	hook DNS::do_reply(c, msg, ans, target);
 	}
 
 # TODO: figure out how to handle these

scripts/base/protocols/http/dpd.sig

@@ -1,6 +1,8 @@
+# List of HTTP headers pulled from:
+#   http://annevankesteren.nl/2007/10/http-methods
 signature dpd_http_client {
   ip-proto == tcp
-  payload /^[[:space:]]*(GET|HEAD|POST)[[:space:]]*/
+  payload /^[[:space:]]*(OPTIONS|GET|HEAD|POST|PUT|DELETE|TRACE|CONNECT|PROPFIND|PROPPATCH|MKCOL|COPY|MOVE|LOCK|UNLOCK|VERSION-CONTROL|REPORT|CHECKOUT|CHECKIN|UNCHECKOUT|MKWORKSPACE|UPDATE|LABEL|MERGE|BASELINE-CONTROL|MKACTIVITY|ORDERPATCH|ACL|PATCH|SEARCH|BCOPY|BDELETE|BMOVE|BPROPFIND|BPROPPATCH|NOTIFY|POLL|SUBSCRIBE|UNSUBSCRIBE|X-MS-ENUMATTS|RPC_OUT_DATA|RPC_IN_DATA)[[:space:]]*/
   tcp-state originator
 }
 

scripts/base/protocols/http/entities.bro

@@ -72,7 +72,7 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori
 
 		if ( f$is_orig )
 			{
-			if ( ! c$http?$orig_mime_types )
+			if ( ! c$http?$orig_fuids )
 				c$http$orig_fuids = string_vec(f$id);
 			else
 				c$http$orig_fuids[|c$http$orig_fuids|] = f$id;
@@ -87,7 +87,7 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori
 			}
 		else
 			{
-			if ( ! c$http?$resp_mime_types )
+			if ( ! c$http?$resp_fuids )
 				c$http$resp_fuids = string_vec(f$id);
 			else
 				c$http$resp_fuids[|c$http$resp_fuids|] = f$id;

scripts/base/protocols/http/main.bro

@@ -4,6 +4,7 @@
 
 @load base/utils/numbers
 @load base/utils/files
+@load base/frameworks/tunnels
 
 module HTTP;
 
@@ -217,6 +218,17 @@ event http_reply(c: connection, version: string, code: count, reason: string) &p
 		c$http$info_code = code;
 		c$http$info_msg = reason;
 		}
+
+	if ( c$http?$method && c$http$method == "CONNECT" && code == 200 )
+		{
+		# Copy this conn_id and set the orig_p to zero because in the case of CONNECT
+		# proxies there will be potentially many source ports since a new proxy connection
+		# is established for each proxied connection.  We treat this as a singular
+		# "tunnel".
+		local tid = copy(c$id);
+		tid$orig_p = 0/tcp;
+		Tunnel::register([$cid=tid, $tunnel_type=Tunnel::HTTP]);
+		}
 	}
 
 event http_header(c: connection, is_orig: bool, name: string, value: string) &priority=5

scripts/base/protocols/irc/dcc-send.bro

@@ -76,7 +76,7 @@ event irc_dcc_message(c: connection, is_orig: bool,
 	dcc_expected_transfers[address, p] = c$irc;
 	}
 
-event expected_connection_seen(c: connection, a: Analyzer::Tag) &priority=10
+event scheduled_analyzer_applied(c: connection, a: Analyzer::Tag) &priority=10
 	{
 	local id = c$id;
 	if ( [id$resp_h, id$resp_p] in dcc_expected_transfers )

scripts/base/protocols/snmp/README

@@ -0,0 +1 @@
+Support for Simple Network Management Protocol (SNMP) analysis.

scripts/base/protocols/snmp/__load__.bro

@@ -0,0 +1 @@
+@load ./main

scripts/base/protocols/snmp/main.bro

@@ -0,0 +1,182 @@
+##! Enables analysis and logging of SNMP datagrams.
+
+module SNMP;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	## Information tracked per SNMP session.
+	type Info: record {
+		## Timestamp of first packet belonging to the SNMP session.
+		ts: time &log;
+		## The unique ID for the connection.
+		uid: string &log;
+		## The connection's 5-tuple of addresses/ports (ports inherently
+		## include transport protocol information)
+		id: conn_id &log;
+		## The amount of time between the first packet beloning to
+		## the SNMP session and the latest one seen.
+		duration: interval &log &default=0secs;
+		## The version of SNMP being used.
+		version: string &log;
+		## The community string of the first SNMP packet associated with
+		## the session.  This is used as part of SNMP's (v1 and v2c)
+		## administrative/security framework.  See :rfc:`1157` or :rfc:`1901`.
+		community: string &log &optional;
+
+		## The number of variable bindings in GetRequest/GetNextRequest PDUs
+		## seen for the session.
+		get_requests:      count &log &default=0;
+		## The number of variable bindings in GetBulkRequest PDUs seen for
+		## the session.
+		get_bulk_requests: count &log &default=0;
+		## The number of variable bindings in GetResponse/Response PDUs seen
+		## for the session.
+		get_responses:     count &log &default=0;
+		## The number of variable bindings in SetRequest PDUs seen for
+		## the session.
+		set_requests: count &log &default=0;
+
+		## A system description of the SNMP responder endpoint.
+		display_string: string &log &optional;
+		## The time at which the SNMP responder endpoint claims it's been
+		## up since.
+		up_since: time &log &optional;
+	};
+
+	## Maps an SNMP version integer to a human readable string.
+	const version_map: table[count] of string = {
+		[0] = "1",
+		[1] = "2c",
+		[3] = "3",
+	} &redef &default="unknown";
+
+	## Event that can be handled to access the SNMP record as it is sent on
+	## to the logging framework.
+	global log_snmp: event(rec: Info);
+}
+
+redef record connection += {
+	snmp: SNMP::Info &optional;
+};
+
+const ports = { 161/udp, 162/udp };
+redef likely_server_ports += { ports };
+
+event bro_init() &priority=5
+	{
+	Analyzer::register_for_ports(Analyzer::ANALYZER_SNMP, ports);
+	Log::create_stream(SNMP::LOG, [$columns=SNMP::Info, $ev=log_snmp]);
+	}
+
+function init_state(c: connection, h: SNMP::Header): Info
+	{
+	if ( ! c?$snmp )
+		{
+		c$snmp = Info($ts=network_time(),
+		              $uid=c$uid, $id=c$id,
+		              $version=version_map[h$version]);
+		}
+
+	local s = c$snmp;
+
+	if ( ! s?$community )
+		{
+		if ( h?$v1 )
+			s$community = h$v1$community;
+		else if ( h?$v2 )
+			s$community = h$v2$community;
+		}
+
+	s$duration = network_time() - s$ts;
+	return s;
+	}
+
+
+event connection_state_remove(c: connection) &priority=-5
+	{
+	if ( c?$snmp )
+		Log::write(LOG, c$snmp);
+	}
+
+event snmp_get_request(c: connection, is_orig: bool, header: SNMP::Header, pdu: SNMP::PDU) &priority=5
+	{
+	local s = init_state(c, header);
+	s$get_requests += |pdu$bindings|;
+	}
+
+event snmp_get_bulk_request(c: connection, is_orig: bool, header: SNMP::Header, pdu: SNMP::BulkPDU) &priority=5
+	{
+	local s = init_state(c, header);
+	s$get_bulk_requests += |pdu$bindings|;
+	}
+
+event snmp_get_next_request(c: connection, is_orig: bool, header: SNMP::Header, pdu: SNMP::PDU) &priority=5
+	{
+	local s = init_state(c, header);
+	s$get_requests += |pdu$bindings|;
+	}
+
+event snmp_response(c: connection, is_orig: bool, header: SNMP::Header, pdu: SNMP::PDU) &priority=5
+	{
+	local s = init_state(c, header);
+	s$get_responses += |pdu$bindings|;
+
+	for ( i in pdu$bindings )
+		{
+		local binding = pdu$bindings[i];
+
+		if ( binding$oid == "1.3.6.1.2.1.1.1.0" && binding$value?$octets )
+			c$snmp$display_string = binding$value$octets;
+		else if ( binding$oid == "1.3.6.1.2.1.1.3.0" && binding$value?$unsigned )
+			{
+			local up_seconds = binding$value$unsigned / 100.0;
+			s$up_since = network_time() - double_to_interval(up_seconds);
+			}
+		}
+	}
+
+event snmp_set_request(c: connection, is_orig: bool, header: SNMP::Header, pdu: SNMP::PDU) &priority=5
+	{
+	local s = init_state(c, header);
+	s$set_requests += |pdu$bindings|;
+	}
+
+event snmp_trap(c: connection, is_orig: bool, header: SNMP::Header, pdu: SNMP::TrapPDU) &priority=5
+	{
+	init_state(c, header);
+	}
+
+event snmp_inform_request(c: connection, is_orig: bool, header: SNMP::Header, pdu: SNMP::PDU) &priority=5
+	{
+	init_state(c, header);
+	}
+
+event snmp_trapV2(c: connection, is_orig: bool, header: SNMP::Header, pdu: SNMP::PDU) &priority=5
+	{
+	init_state(c, header);
+	}
+
+event snmp_report(c: connection, is_orig: bool, header: SNMP::Header, pdu: SNMP::PDU) &priority=5
+	{
+	init_state(c, header);
+	}
+
+event snmp_unknown_pdu(c: connection, is_orig: bool, header: SNMP::Header, tag: count) &priority=5
+	{
+	init_state(c, header);
+	}
+
+event snmp_unknown_scoped_pdu(c: connection, is_orig: bool, header: SNMP::Header, tag: count) &priority=5
+	{
+	init_state(c, header);
+	}
+
+event snmp_encrypted_pdu(c: connection, is_orig: bool, header: SNMP::Header) &priority=5
+	{
+	init_state(c, header);
+	}
+
+#event snmp_unknown_header_version(c: connection, is_orig: bool, version: count) &priority=5
+#	{
+#	}

scripts/base/protocols/ssl/__load__.bro

@@ -1,5 +1,6 @@
 @load ./consts
 @load ./main
 @load ./mozilla-ca-list
+@load ./files
 
 @load-sigs ./dpd.sig

scripts/base/protocols/ssl/consts.bro

@@ -14,15 +14,15 @@ export {
 		[TLSv11] = "TLSv11",
 		[TLSv12] = "TLSv12",
 	} &default=function(i: count):string { return fmt("unknown-%d", i); };
-	
-	## Mapping between numeric codes and human readable strings for alert 
+
+	## Mapping between numeric codes and human readable strings for alert
 	## levels.
 	const alert_levels: table[count] of string = {
 		[1] = "warning",
 		[2] = "fatal",
 	} &default=function(i: count):string { return fmt("unknown-%d", i); };
-	
-	## Mapping between numeric codes and human readable strings for alert 
+
+	## Mapping between numeric codes and human readable strings for alert
 	## descriptions.
 	const alert_descriptions: table[count] of string = {
 		[0] = "close_notify",
@@ -47,6 +47,7 @@ export {
 		[70] = "protocol_version",
 		[71] = "insufficient_security",
 		[80] = "internal_error",
+		[86] = "inappropriate_fallback",
 		[90] = "user_canceled",
 		[100] = "no_renegotiation",
 		[110] = "unsupported_extension",
@@ -55,8 +56,9 @@ export {
 		[113] = "bad_certificate_status_response",
 		[114] = "bad_certificate_hash_value",
 		[115] = "unknown_psk_identity",
+		[120] = "no_application_protocol",
 	} &default=function(i: count):string { return fmt("unknown-%d", i); };
-	
+
 	## Mapping between numeric codes and human readable strings for SSL/TLS
 	## extensions.
 	# More information can be found here:
@@ -87,9 +89,54 @@ export {
 		[13175] = "origin_bound_certificates",
 		[13180] = "encrypted_client_certificates",
 		[30031] = "channel_id",
+		[30032] = "channel_id_new",
+		[35655] = "padding",
 		[65281] = "renegotiation_info"
 	} &default=function(i: count):string { return fmt("unknown-%d", i); };
-	
+
+	## Mapping between numeric codes and human readable string for SSL/TLS elliptic curves.
+	# See http://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml#tls-parameters-8
+	const ec_curves: table[count] of string = {
+		[1] = "sect163k1",
+		[2] = "sect163r1",
+		[3] = "sect163r2",
+		[4] = "sect193r1",
+		[5] = "sect193r2",
+		[6] = "sect233k1",
+		[7] = "sect233r1",
+		[8] = "sect239k1",
+		[9] = "sect283k1",
+		[10] = "sect283r1",
+		[11] = "sect409k1",
+		[12] = "sect409r1",
+		[13] = "sect571k1",
+		[14] = "sect571r1",
+		[15] = "secp160k1",
+		[16] = "secp160r1",
+		[17] = "secp160r2",
+		[18] = "secp192k1",
+		[19] = "secp192r1",
+		[20] = "secp224k1",
+		[21] = "secp224r1",
+		[22] = "secp256k1",
+		[23] = "secp256r1",
+		[24] = "secp384r1",
+		[25] = "secp521r1",
+		[26] = "brainpoolP256r1",
+		[27] = "brainpoolP384r1",
+		[28] = "brainpoolP512r1",
+		[0xFF01] = "arbitrary_explicit_prime_curves",
+		[0xFF02] = "arbitrary_explicit_char2_curves"
+	} &default=function(i: count):string { return fmt("unknown-%d", i); };
+
+	## Mapping between numeric codes and human readable string for SSL/TLC EC point formats.
+	# See http://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml#tls-parameters-9
+	const ec_point_formats: table[count] of string = {
+		[0] = "uncompressed",
+		[1] = "ansiX962_compressed_prime",
+		[2] = "ansiX962_compressed_char2"
+	} &default=function(i: count):string { return fmt("unknown-%d", i); };
+
 	# SSLv2
 	const SSLv20_CK_RC4_128_WITH_MD5 = 0x010080;
 	const SSLv20_CK_RC4_128_EXPORT40_WITH_MD5 = 0x020080;
@@ -263,6 +310,8 @@ export {
 	const TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C3;
 	const TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C4;
 	const TLS_DH_ANON_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C5;
+	# draft-bmoeller-tls-downgrade-scsv-01
+	const TLS_FALLBACK_SCSV = 0x5600;
 	# RFC 4492
 	const TLS_ECDH_ECDSA_WITH_NULL_SHA = 0xC001;
 	const TLS_ECDH_ECDSA_WITH_RC4_128_SHA = 0xC002;
@@ -438,6 +487,10 @@ export {
 	const TLS_PSK_WITH_AES_256_CCM_8 = 0xC0A9;
 	const TLS_PSK_DHE_WITH_AES_128_CCM_8 = 0xC0AA;
 	const TLS_PSK_DHE_WITH_AES_256_CCM_8 = 0xC0AB;
+	const TLS_ECDHE_ECDSA_WITH_AES_128_CCM = 0xC0AC;
+	const TLS_ECDHE_ECDSA_WITH_AES_256_CCM = 0xC0AD;
+	const TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8 = 0xC0AE;
+	const TLS_ECDHE_ECDSA_WITH_AES_256_CCM_8 = 0xC0AF;
 	# draft-agl-tls-chacha20poly1305-02
 	const TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 = 0xCC13;
 	const TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 = 0xCC14;
@@ -452,8 +505,8 @@ export {
 	const SSL_RSA_WITH_DES_CBC_MD5 = 0xFF82;
 	const SSL_RSA_WITH_3DES_EDE_CBC_MD5 = 0xFF83;
 	const TLS_EMPTY_RENEGOTIATION_INFO_SCSV = 0x00FF;
-	
-	## This is a table of all known cipher specs.  It can be used for 
+
+	## This is a table of all known cipher specs.  It can be used for
 	## detecting unknown ciphers and for converting the cipher spec
 	## constants into a human readable format.
 	const cipher_desc: table[count] of string = {
@@ -629,6 +682,7 @@ export {
 		[TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256] = "TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256",
 		[TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256] = "TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256",
 		[TLS_DH_ANON_WITH_CAMELLIA_256_CBC_SHA256] = "TLS_DH_ANON_WITH_CAMELLIA_256_CBC_SHA256",
+		[TLS_FALLBACK_SCSV] = "TLS_FALLBACK_SCSV",
 		[TLS_ECDH_ECDSA_WITH_NULL_SHA] = "TLS_ECDH_ECDSA_WITH_NULL_SHA",
 		[TLS_ECDH_ECDSA_WITH_RC4_128_SHA] = "TLS_ECDH_ECDSA_WITH_RC4_128_SHA",
 		[TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA] = "TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA",
@@ -800,6 +854,10 @@ export {
 		[TLS_PSK_WITH_AES_256_CCM_8] = "TLS_PSK_WITH_AES_256_CCM_8",
 		[TLS_PSK_DHE_WITH_AES_128_CCM_8] = "TLS_PSK_DHE_WITH_AES_128_CCM_8",
 		[TLS_PSK_DHE_WITH_AES_256_CCM_8] = "TLS_PSK_DHE_WITH_AES_256_CCM_8",
+		[TLS_ECDHE_ECDSA_WITH_AES_128_CCM] = "TLS_ECDHE_ECDSA_WITH_AES_128_CCM",
+		[TLS_ECDHE_ECDSA_WITH_AES_256_CCM] = "TLS_ECDHE_ECDSA_WITH_AES_256_CCM",
+		[TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8] = "TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8",
+		[TLS_ECDHE_ECDSA_WITH_AES_256_CCM_8] = "TLS_ECDHE_ECDSA_WITH_AES_256_CCM_8",
 		[TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256] = "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256",
 		[TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256] = "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256",
 		[TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256] = "TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256",
@@ -813,43 +871,5 @@ export {
 		[SSL_RSA_WITH_3DES_EDE_CBC_MD5] = "SSL_RSA_WITH_3DES_EDE_CBC_MD5",
 		[TLS_EMPTY_RENEGOTIATION_INFO_SCSV] = "TLS_EMPTY_RENEGOTIATION_INFO_SCSV",
 	} &default=function(i: count):string { return fmt("unknown-%d", i); };
-	
-	## Mapping between the constants and string values for SSL/TLS errors.
-	const x509_errors: table[count] of string = {
-		[0]  = "ok",
-		[1]  = "unable to get issuer cert",
-		[2]  = "unable to get crl",
-		[3]  = "unable to decrypt cert signature",
-		[4]  = "unable to decrypt crl signature",
-		[5]  = "unable to decode issuer public key",
-		[6]  = "cert signature failure",
-		[7]  = "crl signature failure",
-		[8]  = "cert not yet valid",
-		[9]  = "cert has expired",
-		[10] = "crl not yet valid",
-		[11] = "crl has expired",
-		[12] = "error in cert not before field",
-		[13] = "error in cert not after field",
-		[14] = "error in crl last update field",
-		[15] = "error in crl next update field",
-		[16] = "out of mem",
-		[17] = "depth zero self signed cert",
-		[18] = "self signed cert in chain",
-		[19] = "unable to get issuer cert locally",
-		[20] = "unable to verify leaf signature",
-		[21] = "cert chain too long",
-		[22] = "cert revoked",
-		[23] = "invalid ca",
-		[24] = "path length exceeded",
-		[25] = "invalid purpose",
-		[26] = "cert untrusted",
-		[27] = "cert rejected",
-		[28] = "subject issuer mismatch",
-		[29] = "akid skid mismatch",
-		[30] = "akid issuer serial mismatch",
-		[31] = "keyusage no certsign",
-		[32] = "unable to get crl issuer",
-		[33] = "unhandled critical extension",
-	} &default=function(i: count):string { return fmt("unknown-%d", i); };
 
 }

scripts/base/protocols/ssl/dpd.sig

@@ -1,7 +1,7 @@
 signature dpd_ssl_server {
   ip-proto == tcp
   # Server hello.
-  payload /^(\x16\x03[\x00\x01\x02]..\x02...\x03[\x00\x01\x02]|...?\x04..\x00\x02).*/
+  payload /^(\x16\x03[\x00\x01\x02\x03]..\x02...\x03[\x00\x01\x02\x03]|...?\x04..\x00\x02).*/
   requires-reverse-signature dpd_ssl_client
   enable "ssl"
   tcp-state responder
@@ -10,6 +10,6 @@ signature dpd_ssl_server {
 signature dpd_ssl_client {
   ip-proto == tcp
   # Client hello.
-  payload /^(\x16\x03[\x00\x01\x02]..\x01...\x03[\x00\x01\x02]|...?\x01[\x00\x01\x02][\x02\x03]).*/
+  payload /^(\x16\x03[\x00\x01\x02\x03]..\x01...\x03[\x00\x01\x02\x03]|...?\x01[\x00\x03][\x00\x01\x02\x03]).*/
   tcp-state originator
 }

scripts/base/protocols/ssl/files.bro

@@ -0,0 +1,135 @@
+@load ./main
+@load base/utils/conn-ids
+@load base/frameworks/files
+@load base/files/x509
+
+module SSL;
+
+export {
+	redef record Info += {
+		## Chain of certificates offered by the server to validate its
+		## complete signing chain.
+		cert_chain: vector of Files::Info &optional;
+
+		## An ordered vector of all certicate file unique IDs for the
+		## certificates offered by the server.
+		cert_chain_fuids: vector of string &optional &log;
+
+		## Chain of certificates offered by the client to validate its
+		## complete signing chain.
+		client_cert_chain: vector of Files::Info &optional;
+
+		## An ordered vector of all certicate file unique IDs for the
+		## certificates offered by the client.
+		client_cert_chain_fuids: vector of string &optional &log;
+
+		## Subject of the X.509 certificate offered by the server.
+		subject: string &log &optional;
+
+		## Subject of the signer of the X.509 certificate offered by the
+		## server.
+		issuer: string &log &optional;
+
+		## Subject of the X.509 certificate offered by the client.
+		client_subject: string &log &optional;
+
+		## Subject of the signer of the X.509 certificate offered by the
+		## client.
+		client_issuer: string &log &optional;
+
+		## Current number of certificates seen from either side. Used
+		## to create file handles.
+		server_depth: count &default=0;
+		client_depth: count &default=0;
+	};
+
+	## Default file handle provider for SSL.
+	global get_file_handle: function(c: connection, is_orig: bool): string;
+
+	## Default file describer for SSL.
+	global describe_file: function(f: fa_file): string;
+}
+
+function get_file_handle(c: connection, is_orig: bool): string
+	{
+	# Unused.  File handles are generated in the analyzer.
+	return "";
+	}
+
+function describe_file(f: fa_file): string
+	{
+	if ( f$source != "SSL" || ! f?$info || ! f$info?$x509 || ! f$info$x509?$certificate )
+		return "";
+
+	# It is difficult to reliably describe a certificate - especially since
+	# we do not know when this function is called (hence, if the data structures
+	# are already populated).
+	#
+	# Just return a bit of our connection information and hope that that is good enough.
+	for ( cid in f$conns )
+		{
+		if ( f$conns[cid]?$ssl )
+			{
+			local c = f$conns[cid];
+			return cat(c$id$resp_h, ":", c$id$resp_p);
+			}
+		}
+
+	return cat("Serial: ", f$info$x509$certificate$serial, " Subject: ",
+		f$info$x509$certificate$subject, " Issuer: ",
+		f$info$x509$certificate$issuer);
+	}
+
+event bro_init() &priority=5
+	{
+	Files::register_protocol(Analyzer::ANALYZER_SSL,
+	                         [$get_file_handle = SSL::get_file_handle,
+	                          $describe        = SSL::describe_file]);
+	}
+
+event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5
+	{
+	if ( ! c?$ssl )
+		return;
+
+	if ( ! c$ssl?$cert_chain )
+		{
+		c$ssl$cert_chain = vector();
+		c$ssl$client_cert_chain = vector();
+		c$ssl$cert_chain_fuids = string_vec();
+		c$ssl$client_cert_chain_fuids = string_vec();
+		}
+
+	if ( is_orig )
+		{
+		c$ssl$client_cert_chain[|c$ssl$client_cert_chain|] = f$info;
+		c$ssl$client_cert_chain_fuids[|c$ssl$client_cert_chain_fuids|] = f$id;
+		}
+	else
+		{
+		c$ssl$cert_chain[|c$ssl$cert_chain|] = f$info;
+		c$ssl$cert_chain_fuids[|c$ssl$cert_chain_fuids|] = f$id;
+		}
+
+	Files::add_analyzer(f, Files::ANALYZER_X509);
+	# always calculate hashes. They are not necessary for base scripts
+	# but very useful for identification, and required for policy scripts
+	Files::add_analyzer(f, Files::ANALYZER_MD5);
+	Files::add_analyzer(f, Files::ANALYZER_SHA1);
+	}
+
+event ssl_established(c: connection) &priority=6
+	{
+	# update subject and issuer information
+	if ( c$ssl?$cert_chain && |c$ssl$cert_chain| > 0 )
+		{
+		c$ssl$subject = c$ssl$cert_chain[0]$x509$certificate$subject;
+		c$ssl$issuer = c$ssl$cert_chain[0]$x509$certificate$issuer;
+		}
+
+	if ( c$ssl?$client_cert_chain && |c$ssl$client_cert_chain| > 0 )
+		{
+		c$ssl$client_subject = c$ssl$client_cert_chain[0]$x509$certificate$subject;
+		c$ssl$client_issuer = c$ssl$client_cert_chain[0]$x509$certificate$issuer;
+		}
+	}

scripts/base/protocols/ssl/main.bro

@@ -19,45 +19,28 @@ export {
 		version:          string           &log &optional;
 		## SSL/TLS cipher suite that the server chose.
 		cipher:           string           &log &optional;
+		## Elliptic curve the server chose when using ECDH/ECDHE.
+		curve:            string           &log &optional;
 		## Value of the Server Name Indicator SSL/TLS extension.  It
 		## indicates the server name that the client was requesting.
 		server_name:      string           &log &optional;
 		## Session ID offered by the client for session resumption.
 		session_id:       string           &log &optional;
-		## Subject of the X.509 certificate offered by the server.
-		subject:          string           &log &optional;
-		## Subject of the signer of the X.509 certificate offered by the
-		## server.
-		issuer_subject:   string           &log &optional;
-		## NotValidBefore field value from the server certificate.
-		not_valid_before: time             &log &optional;
-		## NotValidAfter field value from the server certificate.
-		not_valid_after:  time             &log &optional;
 		## Last alert that was seen during the connection.
 		last_alert:       string           &log &optional;
 
-		## Subject of the X.509 certificate offered by the client.
-		client_subject:          string           &log &optional;
-		## Subject of the signer of the X.509 certificate offered by the
-		## client.
-		client_issuer_subject:   string           &log &optional;
-
-		## Full binary server certificate stored in DER format.
-		cert:             string           &optional;
-		## Chain of certificates offered by the server to validate its
-		## complete signing chain.
-		cert_chain:       vector of string &optional;
-
-		## Full binary client certificate stored in DER format.
-		client_cert:             string           &optional;
-		## Chain of certificates offered by the client to validate its
-		## complete signing chain.
-		client_cert_chain:       vector of string &optional;
-
 		## The analyzer ID used for the analyzer instance attached
 		## to each connection.  It is not used for logging since it's a
 		## meaningless arbitrary number.
 		analyzer_id:      count            &optional;
+
+		## Flag to indicate if this ssl session has been established
+		## succesfully, or if it was aborted during the handshake.
+		established:  bool &log &default=F;
+
+		## Flag to indicate if this record already has been logged, to
+		## prevent duplicates.
+		logged:  bool  &default=F;
 	};
 
 	## The default root CA bundle.  By default, the mozilla-ca-list.bro
@@ -108,8 +91,7 @@ event bro_init() &priority=5
 function set_session(c: connection)
 	{
 	if ( ! c?$ssl )
-		c$ssl = [$ts=network_time(), $uid=c$uid, $id=c$id, $cert_chain=vector(),
-		         $client_cert_chain=vector()];
+		c$ssl = [$ts=network_time(), $uid=c$uid, $id=c$id];
 	}
 
 function delay_log(info: Info, token: string)
@@ -127,9 +109,13 @@ function undelay_log(info: Info, token: string)
 
 function log_record(info: Info)
 	{
+	if ( info$logged )
+		return;
+
 	if ( ! info?$delay_tokens || |info$delay_tokens| == 0 )
 		{
 		Log::write(SSL::LOG, info);
+		info$logged = T;
 		}
 	else
 		{
@@ -146,11 +132,16 @@ function log_record(info: Info)
 		}
 	}
 
-function finish(c: connection)
+# remove_analyzer flag is used to prevent disabling analyzer for finished
+# connections.
+function finish(c: connection, remove_analyzer: bool)
 	{
 	log_record(c$ssl);
-	if ( disable_analyzer_after_detection && c?$ssl && c$ssl?$analyzer_id )
+	if ( remove_analyzer && disable_analyzer_after_detection && c?$ssl && c$ssl?$analyzer_id )
+		{
 		disable_analyzer(c$id, c$ssl$analyzer_id);
+		delete c$ssl$analyzer_id;
+		}
 	}
 
 event ssl_client_hello(c: connection, version: count, possible_ts: time, client_random: string, session_id: string, ciphers: index_vec) &priority=5
@@ -170,55 +161,23 @@ event ssl_server_hello(c: connection, version: count, possible_ts: time, server_
 	c$ssl$cipher = cipher_desc[cipher];
 	}
 
-event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string) &priority=5
+event ssl_server_curve(c: connection, curve: count) &priority=5
 	{
 	set_session(c);
 
-	# We aren't doing anything with client certificates yet.
-	if ( is_orig )
-		{
-		if ( chain_idx == 0 )
-			{
-			# Save the primary cert.
-			c$ssl$client_cert = der_cert;
-
-			# Also save other certificate information about the primary cert.
-			c$ssl$client_subject = cert$subject;
-			c$ssl$client_issuer_subject = cert$issuer;
-			}
-		else
-			{
-			# Otherwise, add it to the cert validation chain.
-			c$ssl$client_cert_chain[|c$ssl$client_cert_chain|] = der_cert;
-			}
-		}
-	else
-		{
-		if ( chain_idx == 0 )
-			{
-			# Save the primary cert.
-			c$ssl$cert = der_cert;
-
-			# Also save other certificate information about the primary cert.
-			c$ssl$subject = cert$subject;
-			c$ssl$issuer_subject = cert$issuer;
-			c$ssl$not_valid_before = cert$not_valid_before;
-			c$ssl$not_valid_after = cert$not_valid_after;
-			}
-		else
-			{
-			# Otherwise, add it to the cert validation chain.
-			c$ssl$cert_chain[|c$ssl$cert_chain|] = der_cert;
-			}
-		}
+	c$ssl$curve = ec_curves[curve];
 	}
 
-event ssl_extension(c: connection, is_orig: bool, code: count, val: string) &priority=5
+event ssl_extension_server_name(c: connection, is_orig: bool, names: string_vec) &priority=5
 	{
 	set_session(c);
 
-	if ( is_orig && extensions[code] == "server_name" )
-		c$ssl$server_name = sub_bytes(val, 6, |val|);
+	if ( is_orig && |names| > 0 )
+		{
+		c$ssl$server_name = names[0];
+		if ( |names| > 1 )
+			event conn_weird("SSL_many_server_names", c, cat(names));
+		}
 	}
 
 event ssl_alert(c: connection, is_orig: bool, level: count, desc: count) &priority=5
@@ -228,26 +187,36 @@ event ssl_alert(c: connection, is_orig: bool, level: count, desc: count) &priori
 	c$ssl$last_alert = alert_descriptions[desc];
 	}
 
-event ssl_established(c: connection) &priority=5
+event ssl_established(c: connection) &priority=7
 	{
 	set_session(c);
+	c$ssl$established = T;
 	}
 
 event ssl_established(c: connection) &priority=-5
 	{
-	finish(c);
+	finish(c, T);
+	}
+
+event connection_state_remove(c: connection) &priority=-5
+	{
+	if ( c?$ssl )
+		# called in case a SSL connection that has not been established terminates
+		finish(c, F);
 	}
 
 event protocol_confirmation(c: connection, atype: Analyzer::Tag, aid: count) &priority=5
 	{
-	# Check by checking for existence of c$ssl record.
-	if ( c?$ssl && atype == Analyzer::ANALYZER_SSL )
+	if ( atype == Analyzer::ANALYZER_SSL )
+		{
+		set_session(c);
 		c$ssl$analyzer_id = aid;
+		}
 	}
 
 event protocol_violation(c: connection, atype: Analyzer::Tag, aid: count,
                          reason: string) &priority=5
 	{
 	if ( c?$ssl )
-		finish(c);
+		finish(c, T);
 	}