Merge remote-tracking branch 'origin/master' into topic/vladg/ssh

scripts/base/files/extract/main.bro

@@ -53,7 +53,8 @@ function set_limit(f: fa_file, args: Files::AnalyzerArgs, n: count): bool
 function on_add(f: fa_file, args: Files::AnalyzerArgs)
 	{
 	if ( ! args?$extract_filename )
-		args$extract_filename = cat("extract-", f$source, "-", f$id);
+		args$extract_filename = cat("extract-", f$last_active, "-", f$source,
+		                            "-", f$id);
 
 	f$info$extracted = args$extract_filename;
 	args$extract_filename = build_path_compressed(prefix, args$extract_filename);

scripts/base/frameworks/broker/__load__.bro

@@ -0,0 +1 @@
+@load ./main

scripts/base/frameworks/broker/main.bro

@@ -0,0 +1,103 @@
+##! Various data structure definitions for use with Bro's communication system.
+
+module BrokerComm;
+
+export {
+
+	## A name used to identify this endpoint to peers.
+	## .. bro:see:: BrokerComm::connect BrokerComm::listen
+	const endpoint_name = "" &redef;
+
+	## Change communication behavior.
+	type EndpointFlags: record {
+		## Whether to restrict message topics that can be published to peers.
+		auto_publish: bool &default = T;
+		## Whether to restrict what message topics or data store identifiers
+		## the local endpoint advertises to peers (e.g. subscribing to
+		## events or making a master data store available).
+		auto_advertise: bool &default = T;
+	};
+
+	## Fine-grained tuning of communication behavior for a particular message.
+	type SendFlags: record {
+		## Send the message to the local endpoint.
+		self: bool &default = F;
+		## Send the message to peer endpoints that advertise interest in
+		## the topic associated with the message.
+		peers: bool &default = T;
+		## Send the message to peer endpoints even if they don't advertise
+		## interest in the topic associated with the message.
+		unsolicited: bool &default = F;
+	};
+
+	## Opaque communication data.
+	type Data: record {
+		d: opaque of BrokerComm::Data &optional;
+	};
+
+	## Opaque communication data.
+	type DataVector: vector of BrokerComm::Data;
+
+	## Opaque event communication data.
+	type EventArgs: record {
+		## The name of the event.  Not set if invalid event or arguments.
+		name: string &optional;
+		## The arguments to the event.
+		args: DataVector;
+	};
+
+	## Opaque communication data used as a convenient way to wrap key-value
+	## pairs that comprise table entries.
+	type TableItem : record {
+		key: BrokerComm::Data;
+		val: BrokerComm::Data;
+	};
+}
+
+module BrokerStore;
+
+export {
+
+	## Whether a data store query could be completed or not.
+	type QueryStatus: enum {
+		SUCCESS,
+		FAILURE,
+	};
+
+	## An expiry time for a key-value pair inserted in to a data store.
+	type ExpiryTime: record {
+		## Absolute point in time at which to expire the entry.
+		absolute: time &optional;
+		## A point in time relative to the last modification time at which
+		## to expire the entry.  New modifications will delay the expiration.
+		since_last_modification: interval &optional;
+	};
+
+	## The result of a data store query.
+	type QueryResult: record {
+		## Whether the query completed or not.
+		status: BrokerStore::QueryStatus;
+		## The result of the query.  Certain queries may use a particular
+		## data type (e.g. querying store size always returns a count, but
+		## a lookup may return various data types).
+		result: BrokerComm::Data;
+	};
+
+	## Options to tune the SQLite storage backend.
+	type SQLiteOptions: record {
+		## File system path of the database.
+		path: string &default = "store.sqlite";
+	};
+
+	## Options to tune the RocksDB storage backend.
+	type RocksDBOptions: record {
+		## File system path of the database.
+		path: string &default = "store.rocksdb";
+	};
+
+	## Options to tune the particular storage backends.
+	type BackendOptions: record {
+		sqlite: SQLiteOptions &default = SQLiteOptions();
+		rocksdb: RocksDBOptions &default = RocksDBOptions();
+	};
+}

scripts/base/frameworks/files/main.bro

@@ -267,7 +267,7 @@ export {
 	## mts: The set of MIME types, each in the form "foo/bar" (case-insensitive).
 	##
 	## Returns: True if the MIME types were successfully registered.
-	global register_for_mime_types: function(tag: Analyzer::Tag, mts: set[string]) : bool;
+	global register_for_mime_types: function(tag: Files::Tag, mts: set[string]) : bool;
 
 	## Registers a MIME type for an analyzer. If a future file with this type is seen,
 	## the analyzer will be automatically assigned to parsing it. The function *adds*
@@ -278,20 +278,20 @@ export {
 	## mt: The MIME type in the form "foo/bar" (case-insensitive).
 	##
 	## Returns: True if the MIME type was successfully registered.
-	global register_for_mime_type: function(tag: Analyzer::Tag, mt: string) : bool;
+	global register_for_mime_type: function(tag: Files::Tag, mt: string) : bool;
 
 	## Returns a set of all MIME types currently registered for a specific analyzer.
 	##
 	## tag: The tag of the analyzer.
 	##
 	## Returns: The set of MIME types.
-	global registered_mime_types: function(tag: Analyzer::Tag) : set[string];
+	global registered_mime_types: function(tag: Files::Tag) : set[string];
 
 	## Returns a table of all MIME-type-to-analyzer mappings currently registered.
 	##
 	## Returns: A table mapping each analyzer to the set of MIME types
 	##          registered for it.
-	global all_registered_mime_types: function() : table[Analyzer::Tag] of set[string];
+	global all_registered_mime_types: function() : table[Files::Tag] of set[string];
 
 	## Event that can be handled to access the Info record as it is sent on
 	## to the logging framework.
@@ -306,8 +306,8 @@ redef record fa_file += {
 global registered_protocols: table[Analyzer::Tag] of ProtoRegistration = table();
 
 # Store the MIME type to analyzer mappings.
-global mime_types: table[Analyzer::Tag] of set[string];
-global mime_type_to_analyzers: table[string] of set[Analyzer::Tag];
+global mime_types: table[Files::Tag] of set[string];
+global mime_type_to_analyzers: table[string] of set[Files::Tag];
 
 global analyzer_add_callbacks: table[Files::Tag] of function(f: fa_file, args: AnalyzerArgs) = table();
 
@@ -401,7 +401,7 @@ function register_protocol(tag: Analyzer::Tag, reg: ProtoRegistration): bool
 	return result;
 	}
 
-function register_for_mime_types(tag: Analyzer::Tag, mime_types: set[string]) : bool
+function register_for_mime_types(tag: Files::Tag, mime_types: set[string]) : bool
 	{
 	local rc = T;
 
@@ -414,7 +414,7 @@ function register_for_mime_types(tag: Analyzer::Tag, mime_types: set[string]) :
 	return rc;
 	}
 
-function register_for_mime_type(tag: Analyzer::Tag, mt: string) : bool
+function register_for_mime_type(tag: Files::Tag, mt: string) : bool
 	{
 	if ( tag !in mime_types )
 		{
@@ -431,12 +431,12 @@ function register_for_mime_type(tag: Analyzer::Tag, mt: string) : bool
 	return T;
 	}
 
-function registered_mime_types(tag: Analyzer::Tag) : set[string]
+function registered_mime_types(tag: Files::Tag) : set[string]
 	{
 	return tag in mime_types ? mime_types[tag] : set();
 	}
 
-function all_registered_mime_types(): table[Analyzer::Tag] of set[string]
+function all_registered_mime_types(): table[Files::Tag] of set[string]
 	{
 	return mime_types;
 	}
@@ -451,7 +451,7 @@ function describe(f: fa_file): string
 	return handler$describe(f);
 	}
 
-event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) &priority=5
+event get_file_handle(tag: Files::Tag, c: connection, is_orig: bool) &priority=5
 	{
 	if ( tag !in registered_protocols )
 		return;

scripts/base/init-bare.bro

@@ -440,6 +440,7 @@ type NetStats: record {
 	## packet capture system, this value may not be available and will then
 	## be always set to zero.
 	pkts_link:    count &default=0;
+	bytes_recvd:  count &default=0;	##< Bytes received by Bro.
 };
 
 ## Statistics about Bro's resource consumption.
@@ -2809,19 +2810,20 @@ export {
 module X509;
 export {
 	type Certificate: record {
-		version: count;	##< Version number.
-		serial: string;	##< Serial number.
-		subject: string;	##< Subject.
-		issuer: string;	##< Issuer.
-		not_valid_before: time;	##< Timestamp before when certificate is not valid.
-		not_valid_after: time;	##< Timestamp after when certificate is not valid.
-		key_alg: string;	##< Name of the key algorithm
-		sig_alg: string;	##< Name of the signature algorithm
-		key_type: string &optional;	##< Key type, if key parseable by openssl (either rsa, dsa or ec)
-		key_length: count &optional;	##< Key length in bits
-		exponent: string &optional;	##< Exponent, if RSA-certificate
-		curve: string &optional;	##< Curve, if EC-certificate
-	} &log;
+		version: count &log;	##< Version number.
+		serial: string &log;	##< Serial number.
+		subject: string &log;	##< Subject.
+		issuer: string &log;	##< Issuer.
+		cn: string &optional; ##< Last (most specific) common name.
+		not_valid_before: time &log;	##< Timestamp before when certificate is not valid.
+		not_valid_after: time &log;	##< Timestamp after when certificate is not valid.
+		key_alg: string &log;	##< Name of the key algorithm
+		sig_alg: string &log;	##< Name of the signature algorithm
+		key_type: string &optional &log;	##< Key type, if key parseable by openssl (either rsa, dsa or ec)
+		key_length: count &optional &log;	##< Key length in bits
+		exponent: string &optional &log;	##< Exponent, if RSA-certificate
+		curve: string &optional &log;	##< Curve, if EC-certificate
+	};
 
 	type Extension: record {
 		name: string;	##< Long name of extension. oid if name not known
@@ -3393,6 +3395,7 @@ const bits_per_uid: count = 96 &redef;
 
 # Load these frameworks here because they use fairly deep integration with
 # BiFs and script-land defined types.
+@load base/frameworks/broker
 @load base/frameworks/logging
 @load base/frameworks/input
 @load base/frameworks/analyzer

scripts/base/protocols/conn/main.bro

@@ -62,6 +62,12 @@ export {
 		## field will be left empty at all times.
 		local_orig:   bool            &log &optional;
 
+		## If the connection is responded to locally, this value will be T.
+		## If it was responded to remotely it will be F.  In the case that
+		## the :bro:id:`Site::local_nets` variable is undefined, this
+		## field will be left empty at all times.
+		local_resp:   bool            &log &optional;
+
 		## Indicates the number of bytes missed in content gaps, which
 		## is representative of packet loss.  A value other than zero
 		## will normally cause protocol analysis to fail but some
@@ -201,7 +207,10 @@ function set_conn(c: connection, eoc: bool)
 		add c$conn$tunnel_parents[c$tunnel[|c$tunnel|-1]$uid];
 	c$conn$proto=get_port_transport_proto(c$id$resp_p);
 	if( |Site::local_nets| > 0 )
+		{
 		c$conn$local_orig=Site::is_local_addr(c$id$orig_h);
+		c$conn$local_resp=Site::is_local_addr(c$id$resp_h);
+		}
 
 	if ( eoc )
 		{

scripts/base/protocols/socks/main.bro

@@ -16,8 +16,10 @@ export {
 		id:          conn_id         &log;
 		## Protocol version of SOCKS.
 		version:     count           &log;
-		## Username for the proxy if extracted from the network.
+		## Username used to request a login to the proxy.
 		user:        string          &log &optional;
+		## Password used to request a login to the proxy.
+		password:    string          &log &optional;
 		## Server status for the attempt at using the proxy.
 		status:      string          &log &optional;
 		## Client requested SOCKS address. Could be an address, a name
@@ -91,3 +93,21 @@ event socks_reply(c: connection, version: count, reply: count, sa: SOCKS::Addres
 	if ( "SOCKS" in c$service )
 		Log::write(SOCKS::LOG, c$socks);
 	}
+
+event socks_login_userpass_request(c: connection, user: string, password: string) &priority=5
+	{
+	# Authentication only possible with the version 5.
+	set_session(c, 5); 
+
+	c$socks$user = user;
+	c$socks$password = password;
+	}
+
+event socks_login_userpass_reply(c: connection, code: count) &priority=5
+	{
+	# Authentication only possible with the version 5.
+	set_session(c, 5);
+
+	c$socks$status = v5_status[code];
+	}
+

scripts/broxygen/__load__.bro

@@ -5,6 +5,7 @@
 @load frameworks/communication/listen.bro
 @load frameworks/control/controllee.bro
 @load frameworks/control/controller.bro
+@load frameworks/files/extract-all-files.bro
 @load policy/misc/dump-events.bro
 
 @load ./example.bro

scripts/policy/frameworks/files/extract-all-files.bro

@@ -0,0 +1,8 @@
+##! Extract all files to disk.
+
+@load base/files/extract
+
+event file_new(f: fa_file)
+	{
+	Files::add_analyzer(f, Files::ANALYZER_EXTRACT);
+	}

scripts/policy/frameworks/intel/seen/ssl.bro

@@ -10,3 +10,16 @@ event ssl_extension_server_name(c: connection, is_orig: bool, names: string_vec)
 		             $conn=c,
 		             $where=SSL::IN_SERVER_NAME]);
 	}
+
+event ssl_established(c: connection)
+	{
+	if ( ! c$ssl?$cert_chain || |c$ssl$cert_chain| == 0 ||
+	     ! c$ssl$cert_chain[0]?$x509 )
+		return;
+
+	if ( c$ssl$cert_chain[0]$x509?$certificate && c$ssl$cert_chain[0]$x509$certificate?$cn )
+		Intel::seen([$indicator=c$ssl$cert_chain[0]$x509$certificate$cn,
+			$indicator_type=Intel::DOMAIN,
+			$conn=c,
+			$where=X509::IN_CERT]);
+	}

scripts/policy/frameworks/intel/seen/x509.bro

@@ -2,6 +2,18 @@
 @load base/files/x509
 @load ./where-locations
 
+event x509_ext_subject_alternative_name(f: fa_file, ext: X509::SubjectAlternativeName)
+	{
+	if ( ext?$dns )
+		{
+		for ( i in ext$dns )
+			Intel::seen([$indicator=ext$dns[i],
+				$indicator_type=Intel::DOMAIN,
+				$f=f,
+				$where=X509::IN_CERT]);
+		}
+	}
+
 event x509_certificate(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate)
 	{
 	if ( /emailAddress=/ in cert$subject )

scripts/policy/misc/stats.bro

@@ -39,6 +39,9 @@ export {
 		## Number of packets seen on the link since the last stats
 		## interval if reading live traffic.
 		pkts_link:     count     &log &optional;
+		## Number of bytes received since the last stats interval if
+		## reading live traffic.
+		bytes_recv:   count     &log &optional;
 	};
 
 	## Event to catch stats as they are written to the logging stream.
@@ -74,6 +77,7 @@ event check_stats(last_ts: time, last_ns: NetStats, last_res: bro_resources)
 		info$pkts_recv = ns$pkts_recvd - last_ns$pkts_recvd;
 		info$pkts_dropped = ns$pkts_dropped  - last_ns$pkts_dropped;
 		info$pkts_link = ns$pkts_link  - last_ns$pkts_link;
+		info$bytes_recv = ns$bytes_recvd  - last_ns$bytes_recvd;
 		}
 
 	Log::write(Stats::LOG, info);

scripts/policy/protocols/ssl/validate-certs.bro

@@ -12,16 +12,16 @@ export {
 		## invalid.
 		Invalid_Server_Cert
 	};
-	
+
 	redef record Info += {
 		## Result of certificate validation for this connection.
 		validation_status: string &log &optional;
 	};
-	
+
 	## MD5 hash values for recently validated chains along with the
 	## validation status message are kept in this table to avoid constant
 	## validation every time the same certificate chain is seen.
-	global recently_validated_certs: table[string] of string = table() 
+	global recently_validated_certs: table[string] of string = table()
 		&read_expire=5mins &synchronized &redef;
 }
 
@@ -33,6 +33,7 @@ event ssl_established(c: connection) &priority=3
 		return;
 
 	local chain_id = join_string_vec(c$ssl$cert_chain_fuids, ".");
+	local hash = c$ssl$cert_chain[0]$sha1;
 
 	local chain: vector of opaque of x509 = vector();
 	for ( i in c$ssl$cert_chain )
@@ -57,7 +58,7 @@ event ssl_established(c: connection) &priority=3
 		local message = fmt("SSL certificate validation failed with (%s)", c$ssl$validation_status);
 		NOTICE([$note=Invalid_Server_Cert, $msg=message,
 		        $sub=c$ssl$subject, $conn=c,
-		        $identifier=cat(c$id$resp_h,c$id$resp_p,c$ssl$validation_status)]);
+		        $identifier=cat(c$id$resp_h,c$id$resp_p,hash,c$ssl$validation_status)]);
 		}
 	}
 

scripts/policy/protocols/ssl/validate-ocsp.bro

@@ -34,9 +34,10 @@ event ssl_stapled_ocsp(c: connection, is_orig: bool, response: string) &priority
 
 event ssl_established(c: connection) &priority=3
 	{
-	if ( ! c$ssl?$cert_chain || |c$ssl$cert_chain| == 0 || !c$ssl?$ocsp_response )
+	if ( ! c$ssl?$cert_chain || |c$ssl$cert_chain| == 0 || ! c$ssl$cert_chain[0]?$x509 || !c$ssl?$ocsp_response )
 		return;
 
+	local hash = c$ssl$cert_chain[0]$sha1;
 	local chain: vector of opaque of x509 = vector();
 	for ( i in c$ssl$cert_chain )
 		{

scripts/policy/protocols/ssl/weak-keys.bro

@@ -1,5 +1,5 @@
-##! Generate notices when SSL/TLS connections use certificates or DH parameters
-##! that have potentially unsafe key lengths.
+##! Generate notices when SSL/TLS connections use certificates, DH parameters,
+##! or cipher suites that are deemed to be insecure.
 
 @load base/protocols/ssl
 @load base/frameworks/notice
@@ -11,17 +11,20 @@ export {
 	redef enum Notice::Type += {
 		## Indicates that a server is using a potentially unsafe key.
 		Weak_Key,
+		## Indicates that a server is using a potentially unsafe version
+		Old_Version,
+		## Indicates that a server is using a potentially unsafe cipher
+		Weak_Cipher
 	};
 
-	## The category of hosts you would like to be notified about which have
-	## certificates that are going to be expiring soon.  By default, these
-	## notices will be suppressed by the notice framework for 1 day after a particular
-	## certificate has had a notice generated. Choices are: LOCAL_HOSTS, REMOTE_HOSTS,
-	## ALL_HOSTS, NO_HOSTS
+	## The category of hosts you would like to be notified about which are using weak
+	## keys/ciphers/protocol_versions.  By default, these notices will be suppressed
+	## by the notice framework for 1 day after a particular host has had a notice
+	## generated. Choices are: LOCAL_HOSTS, REMOTE_HOSTS, ALL_HOSTS, NO_HOSTS
 	const notify_weak_keys = LOCAL_HOSTS &redef;
 
 	## The minimal key length in bits that is considered to be safe. Any shorter
-	## (non-EC) key lengths will trigger the notice.
+	## (non-EC) key lengths will trigger a notice.
 	const notify_minimal_key_length = 2048 &redef;
 
 	## Warn if the DH key length is smaller than the certificate key length. This is
@@ -29,6 +32,17 @@ export {
 	## certificate key length. However, it is very common and cannot be avoided in some
 	## settings (e.g. with old jave clients).
 	const notify_dh_length_shorter_cert_length = T &redef;
+
+	## Warn if a server negotiates a SSL session with a protocol version smaller than
+	## the specified version. By default, the minimal version is TLSv10 because SSLv2
+	## and v3 have serious security issued.
+	## See https://tools.ietf.org/html/draft-thomson-sslv3-diediedie-00
+	## To disable, set to SSLv20
+	const tls_minimum_version = TLSv10 &redef;
+
+	## Warn if a server negotiates an unsafe cipher suite. By default, we only warn when
+	## encountering old export cipher suites, or RC4 (see RFC7465).
+	const unsafe_ciphers_regex = /(_EXPORT_)|(_RC4_)/ &redef;
 }
 
 # We check key lengths only for DSA or RSA certificates. For others, we do
@@ -43,6 +57,7 @@ event ssl_established(c: connection) &priority=3
 
 	local fuid = c$ssl$cert_chain_fuids[0];
 	local cert = c$ssl$cert_chain[0]$x509$certificate;
+	local hash = c$ssl$cert_chain[0]$sha1;
 
 	if ( !cert?$key_type || !cert?$key_length )
 		return;
@@ -56,7 +71,32 @@ event ssl_established(c: connection) &priority=3
 		NOTICE([$note=Weak_Key,
 			$msg=fmt("Host uses weak certificate with %d bit key", key_length),
 			$conn=c, $suppress_for=1day,
-			$identifier=cat(c$id$resp_h, c$id$resp_h, key_length)
+			$identifier=cat(c$id$resp_h, c$id$resp_h, hash, key_length)
+		]);
+	}
+
+# Check for old SSL versions and weak connection keys
+event ssl_server_hello(c: connection, version: count, possible_ts: time, server_random: string, session_id: string, cipher: count, comp_method: count) &priority=3
+	{
+	if ( ! addr_matches_host(c$id$resp_h, notify_weak_keys) )
+		return;
+
+	if ( version < tls_minimum_version )
+		{
+		local minimum_string = version_strings[tls_minimum_version];
+		local host_string = version_strings[version];
+		NOTICE([$note=Old_Version,
+			$msg=fmt("Host uses protocol version %s which is lower than the safe minimum %s", host_string, minimum_string),
+			$conn=c, $suppress_for=1day,
+			$identifier=cat(c$id$resp_h, c$id$resp_h)
+		]);
+		}
+
+	if ( unsafe_ciphers_regex in c$ssl$cipher )
+		NOTICE([$note=Weak_Cipher,
+			$msg=fmt("Host established connection using unsafe ciper suite %s", c$ssl$cipher),
+			$conn=c, $suppress_for=1day,
+			$identifier=cat(c$id$resp_h, c$id$resp_h, c$ssl$cipher)
 		]);
 	}
 

scripts/test-all-policy.bro

@@ -28,6 +28,7 @@
 @load frameworks/intel/seen/where-locations.bro
 @load frameworks/intel/seen/x509.bro
 @load frameworks/files/detect-MHR.bro
+#@load frameworks/files/extract-all-files.bro
 @load frameworks/files/hash-all-files.bro
 @load frameworks/packet-filter/shunt.bro
 @load frameworks/software/version-changes.bro