Merge remote-tracking branch 'origin/master' into topic/vladg/ssh

scripts/base/files/extract/main.bro

@@ -53,7 +53,8 @@ function set_limit(f: fa_file, args: Files::AnalyzerArgs, n: count): bool
 function on_add(f: fa_file, args: Files::AnalyzerArgs)
 	{
 	if ( ! args?$extract_filename )
-		args$extract_filename = cat("extract-", f$source, "-", f$id);
+		args$extract_filename = cat("extract-", f$last_active, "-", f$source,
+		                            "-", f$id);
 
 	f$info$extracted = args$extract_filename;
 	args$extract_filename = build_path_compressed(prefix, args$extract_filename);

scripts/base/frameworks/broker/__load__.bro

@@ -0,0 +1 @@
+@load ./main

scripts/base/frameworks/broker/main.bro

@@ -0,0 +1,103 @@
+##! Various data structure definitions for use with Bro's communication system.
+
+module BrokerComm;
+
+export {
+
+	## A name used to identify this endpoint to peers.
+	## .. bro:see:: BrokerComm::connect BrokerComm::listen
+	const endpoint_name = "" &redef;
+
+	## Change communication behavior.
+	type EndpointFlags: record {
+		## Whether to restrict message topics that can be published to peers.
+		auto_publish: bool &default = T;
+		## Whether to restrict what message topics or data store identifiers
+		## the local endpoint advertises to peers (e.g. subscribing to
+		## events or making a master data store available).
+		auto_advertise: bool &default = T;
+	};
+
+	## Fine-grained tuning of communication behavior for a particular message.
+	type SendFlags: record {
+		## Send the message to the local endpoint.
+		self: bool &default = F;
+		## Send the message to peer endpoints that advertise interest in
+		## the topic associated with the message.
+		peers: bool &default = T;
+		## Send the message to peer endpoints even if they don't advertise
+		## interest in the topic associated with the message.
+		unsolicited: bool &default = F;
+	};
+
+	## Opaque communication data.
+	type Data: record {
+		d: opaque of BrokerComm::Data &optional;
+	};
+
+	## Opaque communication data.
+	type DataVector: vector of BrokerComm::Data;
+
+	## Opaque event communication data.
+	type EventArgs: record {
+		## The name of the event.  Not set if invalid event or arguments.
+		name: string &optional;
+		## The arguments to the event.
+		args: DataVector;
+	};
+
+	## Opaque communication data used as a convenient way to wrap key-value
+	## pairs that comprise table entries.
+	type TableItem : record {
+		key: BrokerComm::Data;
+		val: BrokerComm::Data;
+	};
+}
+
+module BrokerStore;
+
+export {
+
+	## Whether a data store query could be completed or not.
+	type QueryStatus: enum {
+		SUCCESS,
+		FAILURE,
+	};
+
+	## An expiry time for a key-value pair inserted in to a data store.
+	type ExpiryTime: record {
+		## Absolute point in time at which to expire the entry.
+		absolute: time &optional;
+		## A point in time relative to the last modification time at which
+		## to expire the entry.  New modifications will delay the expiration.
+		since_last_modification: interval &optional;
+	};
+
+	## The result of a data store query.
+	type QueryResult: record {
+		## Whether the query completed or not.
+		status: BrokerStore::QueryStatus;
+		## The result of the query.  Certain queries may use a particular
+		## data type (e.g. querying store size always returns a count, but
+		## a lookup may return various data types).
+		result: BrokerComm::Data;
+	};
+
+	## Options to tune the SQLite storage backend.
+	type SQLiteOptions: record {
+		## File system path of the database.
+		path: string &default = "store.sqlite";
+	};
+
+	## Options to tune the RocksDB storage backend.
+	type RocksDBOptions: record {
+		## File system path of the database.
+		path: string &default = "store.rocksdb";
+	};
+
+	## Options to tune the particular storage backends.
+	type BackendOptions: record {
+		sqlite: SQLiteOptions &default = SQLiteOptions();
+		rocksdb: RocksDBOptions &default = RocksDBOptions();
+	};
+}

scripts/base/frameworks/files/main.bro

@@ -267,7 +267,7 @@ export {
 	## mts: The set of MIME types, each in the form "foo/bar" (case-insensitive).
 	##
 	## Returns: True if the MIME types were successfully registered.
-	global register_for_mime_types: function(tag: Analyzer::Tag, mts: set[string]) : bool;
+	global register_for_mime_types: function(tag: Files::Tag, mts: set[string]) : bool;
 
 	## Registers a MIME type for an analyzer. If a future file with this type is seen,
 	## the analyzer will be automatically assigned to parsing it. The function *adds*
@@ -278,20 +278,20 @@ export {
 	## mt: The MIME type in the form "foo/bar" (case-insensitive).
 	##
 	## Returns: True if the MIME type was successfully registered.
-	global register_for_mime_type: function(tag: Analyzer::Tag, mt: string) : bool;
+	global register_for_mime_type: function(tag: Files::Tag, mt: string) : bool;
 
 	## Returns a set of all MIME types currently registered for a specific analyzer.
 	##
 	## tag: The tag of the analyzer.
 	##
 	## Returns: The set of MIME types.
-	global registered_mime_types: function(tag: Analyzer::Tag) : set[string];
+	global registered_mime_types: function(tag: Files::Tag) : set[string];
 
 	## Returns a table of all MIME-type-to-analyzer mappings currently registered.
 	##
 	## Returns: A table mapping each analyzer to the set of MIME types
 	##          registered for it.
-	global all_registered_mime_types: function() : table[Analyzer::Tag] of set[string];
+	global all_registered_mime_types: function() : table[Files::Tag] of set[string];
 
 	## Event that can be handled to access the Info record as it is sent on
 	## to the logging framework.
@@ -306,8 +306,8 @@ redef record fa_file += {
 global registered_protocols: table[Analyzer::Tag] of ProtoRegistration = table();
 
 # Store the MIME type to analyzer mappings.
-global mime_types: table[Analyzer::Tag] of set[string];
-global mime_type_to_analyzers: table[string] of set[Analyzer::Tag];
+global mime_types: table[Files::Tag] of set[string];
+global mime_type_to_analyzers: table[string] of set[Files::Tag];
 
 global analyzer_add_callbacks: table[Files::Tag] of function(f: fa_file, args: AnalyzerArgs) = table();
 
@@ -401,7 +401,7 @@ function register_protocol(tag: Analyzer::Tag, reg: ProtoRegistration): bool
 	return result;
 	}
 
-function register_for_mime_types(tag: Analyzer::Tag, mime_types: set[string]) : bool
+function register_for_mime_types(tag: Files::Tag, mime_types: set[string]) : bool
 	{
 	local rc = T;
 
@@ -414,7 +414,7 @@ function register_for_mime_types(tag: Analyzer::Tag, mime_types: set[string]) :
 	return rc;
 	}
 
-function register_for_mime_type(tag: Analyzer::Tag, mt: string) : bool
+function register_for_mime_type(tag: Files::Tag, mt: string) : bool
 	{
 	if ( tag !in mime_types )
 		{
@@ -431,12 +431,12 @@ function register_for_mime_type(tag: Analyzer::Tag, mt: string) : bool
 	return T;
 	}
 
-function registered_mime_types(tag: Analyzer::Tag) : set[string]
+function registered_mime_types(tag: Files::Tag) : set[string]
 	{
 	return tag in mime_types ? mime_types[tag] : set();
 	}
 
-function all_registered_mime_types(): table[Analyzer::Tag] of set[string]
+function all_registered_mime_types(): table[Files::Tag] of set[string]
 	{
 	return mime_types;
 	}
@@ -451,7 +451,7 @@ function describe(f: fa_file): string
 	return handler$describe(f);
 	}
 
-event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) &priority=5
+event get_file_handle(tag: Files::Tag, c: connection, is_orig: bool) &priority=5
 	{
 	if ( tag !in registered_protocols )
 		return;

scripts/base/init-bare.bro

@@ -440,6 +440,7 @@ type NetStats: record {
 	## packet capture system, this value may not be available and will then
 	## be always set to zero.
 	pkts_link:    count &default=0;
+	bytes_recvd:  count &default=0;	##< Bytes received by Bro.
 };
 
 ## Statistics about Bro's resource consumption.
@@ -2809,19 +2810,20 @@ export {
 module X509;
 export {
 	type Certificate: record {
-		version: count;	##< Version number.
-		serial: string;	##< Serial number.
-		subject: string;	##< Subject.
-		issuer: string;	##< Issuer.
-		not_valid_before: time;	##< Timestamp before when certificate is not valid.
-		not_valid_after: time;	##< Timestamp after when certificate is not valid.
-		key_alg: string;	##< Name of the key algorithm
-		sig_alg: string;	##< Name of the signature algorithm
-		key_type: string &optional;	##< Key type, if key parseable by openssl (either rsa, dsa or ec)
-		key_length: count &optional;	##< Key length in bits
-		exponent: string &optional;	##< Exponent, if RSA-certificate
-		curve: string &optional;	##< Curve, if EC-certificate
-	} &log;
+		version: count &log;	##< Version number.
+		serial: string &log;	##< Serial number.
+		subject: string &log;	##< Subject.
+		issuer: string &log;	##< Issuer.
+		cn: string &optional; ##< Last (most specific) common name.
+		not_valid_before: time &log;	##< Timestamp before when certificate is not valid.
+		not_valid_after: time &log;	##< Timestamp after when certificate is not valid.
+		key_alg: string &log;	##< Name of the key algorithm
+		sig_alg: string &log;	##< Name of the signature algorithm
+		key_type: string &optional &log;	##< Key type, if key parseable by openssl (either rsa, dsa or ec)
+		key_length: count &optional &log;	##< Key length in bits
+		exponent: string &optional &log;	##< Exponent, if RSA-certificate
+		curve: string &optional &log;	##< Curve, if EC-certificate
+	};
 
 	type Extension: record {
 		name: string;	##< Long name of extension. oid if name not known
@@ -3393,6 +3395,7 @@ const bits_per_uid: count = 96 &redef;
 
 # Load these frameworks here because they use fairly deep integration with
 # BiFs and script-land defined types.
+@load base/frameworks/broker
 @load base/frameworks/logging
 @load base/frameworks/input
 @load base/frameworks/analyzer

scripts/base/protocols/conn/main.bro

@@ -62,6 +62,12 @@ export {
 		## field will be left empty at all times.
 		local_orig:   bool            &log &optional;
 
+		## If the connection is responded to locally, this value will be T.
+		## If it was responded to remotely it will be F.  In the case that
+		## the :bro:id:`Site::local_nets` variable is undefined, this
+		## field will be left empty at all times.
+		local_resp:   bool            &log &optional;
+
 		## Indicates the number of bytes missed in content gaps, which
 		## is representative of packet loss.  A value other than zero
 		## will normally cause protocol analysis to fail but some
@@ -201,7 +207,10 @@ function set_conn(c: connection, eoc: bool)
 		add c$conn$tunnel_parents[c$tunnel[|c$tunnel|-1]$uid];
 	c$conn$proto=get_port_transport_proto(c$id$resp_p);
 	if( |Site::local_nets| > 0 )
+		{
 		c$conn$local_orig=Site::is_local_addr(c$id$orig_h);
+		c$conn$local_resp=Site::is_local_addr(c$id$resp_h);
+		}
 
 	if ( eoc )
 		{

scripts/base/protocols/socks/main.bro

@@ -16,8 +16,10 @@ export {
 		id:          conn_id         &log;
 		## Protocol version of SOCKS.
 		version:     count           &log;
-		## Username for the proxy if extracted from the network.
+		## Username used to request a login to the proxy.
 		user:        string          &log &optional;
+		## Password used to request a login to the proxy.
+		password:    string          &log &optional;
 		## Server status for the attempt at using the proxy.
 		status:      string          &log &optional;
 		## Client requested SOCKS address. Could be an address, a name
@@ -91,3 +93,21 @@ event socks_reply(c: connection, version: count, reply: count, sa: SOCKS::Addres
 	if ( "SOCKS" in c$service )
 		Log::write(SOCKS::LOG, c$socks);
 	}
+
+event socks_login_userpass_request(c: connection, user: string, password: string) &priority=5
+	{
+	# Authentication only possible with the version 5.
+	set_session(c, 5); 
+
+	c$socks$user = user;
+	c$socks$password = password;
+	}
+
+event socks_login_userpass_reply(c: connection, code: count) &priority=5
+	{
+	# Authentication only possible with the version 5.
+	set_session(c, 5);
+
+	c$socks$status = v5_status[code];
+	}
+