Merge remote-tracking branch 'origin/master' into topic/johanna/config

scripts/base/frameworks/files/main.bro

@@ -135,6 +135,20 @@ export {
 	## The default per-file reassembly buffer size.
 	const reassembly_buffer_size = 524288 &redef;
 
+	## Lookup to see if a particular file id exists and is still valid.
+	##
+	## fuid: the file id.
+	##
+	## Returns: T if the file uid is known.
+	global file_exists: function(fuid: string): bool;
+
+	## Lookup an :bro:see:`fa_file` record with the file id.
+	##
+	## fuid: the file id.
+	##
+	## Returns: the associated :bro:see:`fa_file` record.
+	global lookup_file: function(fuid: string): fa_file;
+
 	## Allows the file reassembler to be used if it's necessary because the
 	## file is transferred out of order.
 	##
@@ -338,6 +352,16 @@ function set_info(f: fa_file)
 		f$info$is_orig = f$is_orig;
 	}
 
+function file_exists(fuid: string): bool
+	{
+	return __file_exists(fuid);
+	}
+
+function lookup_file(fuid: string): fa_file
+	{
+	return __lookup_file(fuid);
+	}
+
 function set_timeout_interval(f: fa_file, t: interval): bool
 	{
 	return __set_timeout_interval(f$id, t);

scripts/base/frameworks/logging/main.bro

@@ -300,7 +300,7 @@ export {
 	##          the correct type.
 	##
 	## .. bro:see:: Log::remove_filter Log::add_default_filter
-	##    Log::remove_default_filter
+	##    Log::remove_default_filter Log::get_filter Log::get_filter_names
 	global add_filter: function(id: ID, filter: Filter) : bool;
 
 	## Removes a filter from an existing logging stream.
@@ -315,9 +315,21 @@ export {
 	##          if no filter associated with *name* was found.
 	##
 	## .. bro:see:: Log::remove_filter Log::add_default_filter
-	##    Log::remove_default_filter
+	##    Log::remove_default_filter Log::get_filter Log::get_filter_names
 	global remove_filter: function(id: ID, name: string) : bool;
 
+	## Gets the names of all filters associated with an existing
+	## logging stream.
+	##
+	## id: The ID of a logging stream from which to obtain the list
+	##     of filter names.
+	##
+	## Returns: The set of filter names associated with the stream.
+	##
+	## ..bro:see:: Log::remove_filter Log::add_default_filter
+	##   Log::remove_default_filter Log::get_filter
+	global get_filter_names: function(id: ID) : set[string];
+
 	## Gets a filter associated with an existing logging stream.
 	##
 	## id: The ID associated with a logging stream from which to
@@ -331,7 +343,7 @@ export {
 	##          :bro:id:`Log::no_filter` sentinel value.
 	##
 	## .. bro:see:: Log::add_filter Log::remove_filter Log::add_default_filter
-	##              Log::remove_default_filter
+	##              Log::remove_default_filter Log::get_filter_names
 	global get_filter: function(id: ID, name: string) : Filter;
 
 	## Writes a new log line/entry to a logging stream.
@@ -432,6 +444,8 @@ export {
 
 global all_streams: table[ID] of Stream = table();
 
+global stream_filters: table[ID] of set[string] = table();
+
 # We keep a script-level copy of all filters so that we can manipulate them.
 global filters: table[ID, string] of Filter;
 
@@ -525,16 +539,41 @@ function remove_stream(id: ID) : bool
 	{
 	delete active_streams[id];
 	delete all_streams[id];
+
+	if ( id in stream_filters )
+		{
+		for ( i in stream_filters[id] )
+			delete filters[id, i];
+
+		delete stream_filters[id];
+		}
 	return __remove_stream(id);
 	}
 
 function disable_stream(id: ID) : bool
 	{
 	delete active_streams[id];
-
 	return __disable_stream(id);
 	}
 
+function enable_stream(id: ID) : bool
+	{
+	if ( ! __enable_stream(id) )
+		return F;
+
+	if ( id in all_streams )
+		active_streams[id] = all_streams[id];
+	}
+
+# convenience function to add a filter name to stream_filters
+function add_stream_filters(id: ID, name: string)
+	{
+	if ( id in stream_filters )
+		add stream_filters[id][name];
+	else
+		stream_filters[id] = set(name);
+	}
+
 function add_filter(id: ID, filter: Filter) : bool
 	{
 	local stream = all_streams[id];
@@ -545,13 +584,22 @@ function add_filter(id: ID, filter: Filter) : bool
 	if ( ! filter?$path && ! filter?$path_func )
 		filter$path_func = default_path_func;
 
-	filters[id, filter$name] = filter;
-	return __add_filter(id, filter);
+	local res = __add_filter(id, filter);
+	if ( res )
+		{
+		add_stream_filters(id, filter$name);
+		filters[id, filter$name] = filter;
+		}
+	return res;
 	}
 
 function remove_filter(id: ID, name: string) : bool
 	{
+	if ( id in stream_filters )
+		delete stream_filters[id][name];
+
 	delete filters[id, name];
+
 	return __remove_filter(id, name);
 	}
 
@@ -563,6 +611,14 @@ function get_filter(id: ID, name: string) : Filter
 	return no_filter;
 	}
 
+function get_filter_names(id: ID) : set[string]
+	{
+	if ( id in stream_filters )
+		return stream_filters[id];
+	else
+		return set();
+	}
+
 function write(id: ID, columns: any) : bool
 	{
 	return __write(id, columns);

scripts/base/init-bare.bro

@@ -531,7 +531,7 @@ type EventStats: record {
 	dispatched: count; ##< Total number of events dispatched so far.
 };
 
-## Summary statistics of all regular expression matchers.
+## Holds statistics for all types of reassembly.
 ##
 ## .. bro:see:: get_reassembler_stats
 type ReassemblerStats: record {

scripts/base/protocols/conn/main.bro

@@ -116,7 +116,7 @@ export {
 		## If this connection was over a tunnel, indicate the
 		## *uid* values for any encapsulating parent connections
 		## used over the lifetime of this inner connection.
-		tunnel_parents: set[string] &log;
+		tunnel_parents: set[string] &log &optional;
 	};
 
 	## Event that can be handled to access the :bro:type:`Conn::Info`
@@ -207,7 +207,11 @@ function set_conn(c: connection, eoc: bool)
 	c$conn$uid=c$uid;
 	c$conn$id=c$id;
 	if ( c?$tunnel && |c$tunnel| > 0 )
+		{
+		if ( ! c$conn?$tunnel_parents )
+			c$conn$tunnel_parents = set();
 		add c$conn$tunnel_parents[c$tunnel[|c$tunnel|-1]$uid];
+		}
 	c$conn$proto=get_port_transport_proto(c$id$resp_p);
 	if( |Site::local_nets| > 0 )
 		{
@@ -253,7 +257,11 @@ event tunnel_changed(c: connection, e: EncapsulatingConnVector) &priority=5
 	{
 	set_conn(c, F);
 	if ( |e| > 0 )
+		{
+		if ( ! c$conn?$tunnel_parents )
+			c$conn$tunnel_parents = set();
 		add c$conn$tunnel_parents[e[|e|-1]$uid];
+		}
 	c$tunnel = e;
 	}
 

scripts/base/protocols/dns/main.bro

@@ -2,7 +2,6 @@
 ##! their responses.
 
 @load base/utils/queue
-@load base/frameworks/notice/weird
 @load ./consts
 
 module DNS;
@@ -177,9 +176,6 @@ function log_unmatched_msgs_queue(q: Queue::Queue)
 
 	for ( i in infos )
 		{
-		local wi = Weird::Info($ts=network_time(), $name="dns_unmatched_msg", $uid=infos[i]$uid,
-		                       $id=infos[i]$id);
-		Weird::weird(wi);
 		Log::write(DNS::LOG, infos[i]);
 		}
 	}
@@ -187,21 +183,19 @@ function log_unmatched_msgs_queue(q: Queue::Queue)
 function log_unmatched_msgs(msgs: PendingMessages)
 	{
 	for ( trans_id in msgs )
+		{
 		log_unmatched_msgs_queue(msgs[trans_id]);
+		}
 
 	clear_table(msgs);
 	}
 
 function enqueue_new_msg(msgs: PendingMessages, id: count, msg: Info)
 	{
-	local wi: Weird::Info;
 	if ( id !in msgs )
 		{
 		if ( |msgs| > max_pending_query_ids )
 			{
-			wi = Weird::Info($ts=network_time(), $name="dns_unmatched_msg", $uid=msg$uid,
-			                       $id=msg$id);
-			Weird::weird(wi);
 			# Throw away all unmatched on assumption they'll never be matched.
 			log_unmatched_msgs(msgs);
 			}
@@ -212,9 +206,6 @@ function enqueue_new_msg(msgs: PendingMessages, id: count, msg: Info)
 		{
 		if ( Queue::len(msgs[id]) > max_pending_msgs )
 			{
-			wi = Weird::Info($ts=network_time(), $name="dns_unmatched_msg_quantity", $uid=msg$uid,
-			                       $id=msg$id);
-			Weird::weird(wi);
 			log_unmatched_msgs_queue(msgs[id]);
 			# Throw away all unmatched on assumption they'll never be matched.
 			msgs[id] = Queue::init();
@@ -271,7 +262,6 @@ hook set_session(c: connection, msg: dns_msg, is_query: bool) &priority=5
 			# Create a new DNS session and put it in the reply queue so
 			# we can wait for a matching query.
 			c$dns = new_session(c, msg$id);
-			event conn_weird("dns_unmatched_reply", c, "");
 			enqueue_new_msg(c$dns_state$pending_replies, msg$id, c$dns);
 			}
 		}

scripts/base/protocols/socks/main.bro

@@ -6,32 +6,37 @@ module SOCKS;
 export {
 	redef enum Log::ID += { LOG };
 
+	## Whether passwords are captured or not.
+	const default_capture_password = F &redef;
+
 	## The record type which contains the fields of the SOCKS log.
 	type Info: record {
 		## Time when the proxy connection was first detected.
-		ts:          time            &log;
+		ts:               time            &log;
 		## Unique ID for the tunnel - may correspond to connection uid
 		## or be non-existent.
-		uid:         string          &log;
+		uid:              string          &log;
 		## The connection's 4-tuple of endpoint addresses/ports.
-		id:          conn_id         &log;
+		id:               conn_id         &log;
 		## Protocol version of SOCKS.
-		version:     count           &log;
+		version:          count           &log;
 		## Username used to request a login to the proxy.
-		user:        string          &log &optional;
+		user:             string          &log &optional;
 		## Password used to request a login to the proxy.
-		password:    string          &log &optional;
+		password:         string          &log &optional;
 		## Server status for the attempt at using the proxy.
-		status:      string          &log &optional;
+		status:           string          &log &optional;
 		## Client requested SOCKS address. Could be an address, a name
 		## or both.
-		request:     SOCKS::Address  &log &optional;
+		request:          SOCKS::Address  &log &optional;
 		## Client requested port.
-		request_p:   port            &log &optional;
+		request_p:        port            &log &optional;
 		## Server bound address. Could be an address, a name or both.
-		bound:       SOCKS::Address  &log &optional;
+		bound:            SOCKS::Address  &log &optional;
 		## Server bound port.
-		bound_p:     port            &log &optional;
+		bound_p:          port            &log &optional;
+		## Determines if the password will be captured for this request.
+		capture_password: bool            &default=default_capture_password;
 	};
 
 	## Event that can be handled to access the SOCKS
@@ -90,10 +95,12 @@ event socks_reply(c: connection, version: count, reply: count, sa: SOCKS::Addres
 event socks_login_userpass_request(c: connection, user: string, password: string) &priority=5
 	{
 	# Authentication only possible with the version 5.
-	set_session(c, 5); 
+	set_session(c, 5);
 
 	c$socks$user = user;
-	c$socks$password = password;
+
+	if ( c$socks$capture_password )
+		c$socks$password = password;
 	}
 
 event socks_login_userpass_reply(c: connection, code: count) &priority=5