Merge remote-tracking branch 'origin/master' into topic/johanna/gh-859

scripts/base/frameworks/analyzer/main.zeek

@@ -124,7 +124,6 @@ export {
 	## A set of analyzers to disable by default at startup. The default set
 	## contains legacy analyzers that are no longer supported.
 	global disabled_analyzers: set[Analyzer::Tag] = {
-		ANALYZER_STEPPINGSTONE,
 		ANALYZER_TCPSTATS,
 	} &redef;
 }

scripts/base/frameworks/broker/main.zeek

@@ -31,7 +31,7 @@ export {
 	## authenticated.
 	const disable_ssl = F &redef;
 
-	## Path to a file containing concatenated trusted certificates 
+	## Path to a file containing concatenated trusted certificates
 	## in PEM format. If set, Zeek will require valid certificates for
 	## all peers.
 	const ssl_cafile = "" &redef;
@@ -122,6 +122,37 @@ export {
 	## done reading the pcap.
 	option peer_counts_as_iosource = T;
 
+	## Port for Broker's metric exporter. Setting this to a valid TCP port causes
+	## Broker to make metrics available to Prometheus scrapers via HTTP. Zeek
+	## overrides any value provided in zeek_init or earlier at startup if the
+	## environment variable BROKER_METRICS_PORT is defined.
+	const metrics_port = 0/unknown &redef;
+
+	## Frequency for publishing scraped metrics to the target topic. Zeek
+	## overrides any value provided in zeek_init or earlier at startup if the
+	## environment variable BROKER_METRICS_EXPORT_INTERVAL is defined.
+	option metrics_export_interval = 1 sec;
+
+	## Target topic for the metrics. Setting a non-empty string starts the
+	## periodic publishing of local metrics. Zeek overrides any value provided in
+	## zeek_init or earlier at startup if the environment variable
+	## BROKER_METRICS_EXPORT_TOPIC is defined.
+	option metrics_export_topic = "";
+
+	## ID for the metrics exporter. When setting a target topic for the
+	## exporter, Broker sets this option to the suffix of the new topic *unless*
+	## the ID is a non-empty string. Since setting a topic starts the periodic
+	## publishing of events, we recommend setting the ID always first or avoid
+	## setting it at all if the topic suffix serves as a good-enough ID. Zeek
+	## overrides any value provided in zeek_init or earlier at startup if the
+	## environment variable BROKER_METRICS_ENDPOINT_NAME is defined.
+	option metrics_export_endpoint_name = "";
+
+	## Selects prefixes from the local metrics. Only metrics with prefixes
+	## listed in this variable are included when publishing local metrics.
+	## Setting an empty vector selects *all* metrics.
+	option metrics_export_prefixes: vector of string = vector();
+
 	## The default topic prefix where logs will be published.  The log's stream
 	## id is appended when writing to a particular stream.
 	const default_log_topic_prefix = "zeek/logs/" &redef;
@@ -385,9 +416,53 @@ event Broker::log_flush() &priority=10
 	schedule Broker::log_batch_interval { Broker::log_flush() };
 	}
 
+function update_metrics_export_interval(id: string, val: interval): interval
+	{
+	Broker::__set_metrics_export_interval(val);
+	return val;
+	}
+
+function update_metrics_export_topic(id: string, val: string): string
+	{
+	Broker::__set_metrics_export_topic(val);
+	return val;
+	}
+
+function update_metrics_export_endpoint_name(id: string, val: string): string
+	{
+	Broker::__set_metrics_export_endpoint_name(val);
+	return val;
+	}
+
+function update_metrics_export_prefixes(id: string, filter: vector of string): vector of string
+	{
+	Broker::__set_metrics_export_prefixes(filter);
+	return filter;
+	}
+
 event zeek_init()
 	{
 	schedule Broker::log_batch_interval { Broker::log_flush() };
+	# interval
+	update_metrics_export_interval("Broker::metrics_export_interval",
+	                               Broker::metrics_export_interval);
+	Option::set_change_handler("Broker::metrics_export_interval",
+	                           update_metrics_export_interval);
+	# topic
+	update_metrics_export_topic("Broker::metrics_export_topic",
+	                            Broker::metrics_export_topic);
+	Option::set_change_handler("Broker::metrics_export_topic",
+	                           update_metrics_export_topic);
+	# endpoint name
+	update_metrics_export_endpoint_name("Broker::metrics_export_endpoint_name",
+	                                    Broker::metrics_export_endpoint_name);
+	Option::set_change_handler("Broker::metrics_export_endpoint_name",
+	                           update_metrics_export_endpoint_name);
+	# prefixes
+	update_metrics_export_prefixes("Broker::metrics_export_prefixes",
+	                               Broker::metrics_export_prefixes);
+	Option::set_change_handler("Broker::metrics_export_prefixes",
+	                           update_metrics_export_prefixes);
 	}
 
 event retry_listen(a: string, p: port, retry: interval)

scripts/base/frameworks/logging/writers/ascii.zeek

@@ -54,6 +54,11 @@ export {
 	## This option is also available as a per-filter ``$config`` option.
 	const gzip_file_extension = "gz" &redef;
 
+	## Define the default logging directory. If empty, logs are written
+	## to the current working directory.
+	##
+	const logdir = "" &redef;
+
 	## Format of timestamps when writing out JSON. By default, the JSON
 	## formatter will use double values for timestamps which represent the
 	## number of seconds from the UNIX epoch.

scripts/base/frameworks/notice/actions/email_admin.zeek

@@ -18,18 +18,15 @@ export {
 	};
 }
 
-hook notice(n: Notice::Info) &priority=-5
+hook notice(n: Notice::Info)
 	{
 	if ( |Site::local_admins| > 0 &&
 	     ACTION_EMAIL_ADMIN in n$actions )
 		{
 		local email = "";
 		if ( n?$src && |Site::get_emails(n$src)| > 0 )
-			email = fmt("%s, %s", email, Site::get_emails(n$src));
+			add n$email_dest[Site::get_emails(n$src)];
 		if ( n?$dst && |Site::get_emails(n$dst)| > 0 )
-			email = fmt("%s, %s", email, Site::get_emails(n$dst));
-		
-		if ( email != "" )
-			email_notice_to(n, email, T);
+			add n$email_dest[Site::get_emails(n$dst)];
 		}
 	}

scripts/base/frameworks/notice/actions/page.zeek

@@ -11,14 +11,14 @@ export {
 		## variable.
 		ACTION_PAGE
 	};
-	
+
 	## Email address to send notices with the :zeek:enum:`Notice::ACTION_PAGE`
 	## action.
 	option mail_page_dest = "";
 }
 
-hook notice(n: Notice::Info) &priority=-5
+hook notice(n: Notice::Info)
 	{
 	if ( ACTION_PAGE in n$actions )
-		email_notice_to(n, mail_page_dest, F);
+		add n$email_dest[mail_page_dest];
 	}

scripts/base/frameworks/notice/main.zeek

@@ -136,6 +136,9 @@ export {
 		## The actions which have been applied to this notice.
 		actions:        ActionSet      &log &default=ActionSet();
 
+		## The email address(es) where to send this notice
+		email_dest:     set[string]    &log &default=set();
+
 		## By adding chunks of text into this element, other scripts
 		## can expand on notices that are being emailed.  The normal
 		## way to add text is to extend the vector by handling the
@@ -510,10 +513,17 @@ hook Notice::policy(n: Notice::Info) &priority=10
 	add n$actions[ACTION_LOG];
 	}
 
-hook Notice::notice(n: Notice::Info) &priority=-5
+hook Notice::notice(n: Notice::Info)
 	{
 	if ( ACTION_EMAIL in n$actions )
-		email_notice_to(n, mail_dest, T);
+		add n$email_dest[mail_dest];
+	}
+
+hook Notice::notice(n: Notice::Info) &priority=-5
+	{
+	for ( dest in n$email_dest )
+		email_notice_to(n, dest, T);
+
 	if ( ACTION_LOG in n$actions )
 		Log::write(Notice::LOG, n);
 	if ( ACTION_ALARM in n$actions )

scripts/base/init-bare.zeek

@@ -635,7 +635,7 @@ type ProcStats: record {
 	real_time: interval;          ##< Elapsed real time since Zeek started running.
 	user_time: interval;          ##< User CPU seconds.
 	system_time: interval;        ##< System CPU seconds.
-	mem: count;                   ##< Maximum memory consumed, in KB.
+	mem: count;                   ##< Maximum memory consumed, in bytes.
 	minor_faults: count;          ##< Page faults not requiring actual I/O.
 	major_faults: count;          ##< Page faults requiring actual I/O.
 	num_swap: count;              ##< Times swapped out.
@@ -1933,6 +1933,7 @@ type gtp_delete_pdp_ctx_response_elements: record {
 @load base/frameworks/supervisor/api
 @load base/bif/supervisor.bif
 @load base/bif/packet_analysis.bif
+@load base/bif/CPP-load.bif
 
 ## Internal function.
 function add_interface(iold: string, inew: string): string
@@ -4117,15 +4118,6 @@ type PE::SectionHeader: record {
 }
 module GLOBAL;
 
-## Internal to the stepping stone detector.
-const stp_delta: interval &redef;
-
-## Internal to the stepping stone detector.
-const stp_idle_min: interval &redef;
-
-## Internal to the stepping stone detector.
-global stp_skip_src: set[addr] &redef;
-
 ## Description of a signature match.
 ##
 ## .. zeek:see:: signature_match

scripts/base/packet-protocols/__load__.zeek

@@ -15,3 +15,6 @@
 @load base/packet-protocols/gre
 @load base/packet-protocols/iptunnel
 @load base/packet-protocols/vntag
+@load base/packet-protocols/udp
+@load base/packet-protocols/tcp
+@load base/packet-protocols/icmp

scripts/base/packet-protocols/icmp/__load__.zeek

@@ -0,0 +1 @@
+@load ./main

scripts/base/packet-protocols/icmp/main.zeek

@@ -0,0 +1,5 @@
+module PacketAnalyzer::ICMP;
+
+#event zeek_init() &priority=20
+#	{
+#	}

scripts/base/packet-protocols/ip/main.zeek

@@ -1,8 +1,22 @@
 module PacketAnalyzer::IP;
 
+const IPPROTO_TCP : count = 6;
+const IPPROTO_UDP : count = 17;
+const IPPROTO_ICMP : count = 1;
+const IPPROTO_ICMP6 : count = 58;
+
+const IPPROTO_IPIP : count = 4;
+const IPPROTO_IPV6 : count = 41;
+const IPPROTO_GRE : count = 47;
+
 event zeek_init() &priority=20
 	{
-	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, 4, PacketAnalyzer::ANALYZER_IPTUNNEL);
-	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, 41, PacketAnalyzer::ANALYZER_IPTUNNEL);
-	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, 47, PacketAnalyzer::ANALYZER_GRE);
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_IPIP, PacketAnalyzer::ANALYZER_IPTUNNEL);
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_IPV6, PacketAnalyzer::ANALYZER_IPTUNNEL);
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_GRE, PacketAnalyzer::ANALYZER_GRE);
+
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_TCP, PacketAnalyzer::ANALYZER_TCP);
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_UDP, PacketAnalyzer::ANALYZER_UDP);
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_ICMP, PacketAnalyzer::ANALYZER_ICMP);
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_ICMP6, PacketAnalyzer::ANALYZER_ICMP);
 	}

scripts/base/packet-protocols/tcp/__load__.zeek

@@ -0,0 +1 @@
+@load ./main

scripts/base/packet-protocols/tcp/main.zeek

@@ -0,0 +1,5 @@
+module PacketAnalyzer::TCP;
+
+#event zeek_init() &priority=20
+#	{
+#	}

scripts/base/packet-protocols/udp/__load__.zeek

@@ -0,0 +1 @@
+@load ./main

scripts/base/packet-protocols/udp/main.zeek

@@ -0,0 +1,5 @@
+module PacketAnalyzer::UDP;
+
+#event zeek_init() &priority=20
+#	{
+#	}

scripts/base/protocols/ssh/main.zeek

@@ -20,8 +20,12 @@ export {
 		uid:             string       &log;
 		## The connection's 4-tuple of endpoint addresses/ports.
 		id:              conn_id      &log;
-		## SSH major version (1 or 2)
-		version:         count        &log;
+		## SSH major version (1, 2, or unset). The version can be unset if the
+		## client and server version strings are unset, malformed or incompatible
+		## so no common version can be extracted. If no version can be extracted
+		## even though both client and server versions are set a weird
+		## will be generated.
+		version:         count        &log &optional;
 		## Authentication result (T=success, F=failure, unset=unknown)
 		auth_success:    bool         &log &optional;
 		## The number of authentication attemps we observed. There's always
@@ -155,65 +159,82 @@ function set_session(c: connection)
 		}
 	}
 
-function set_version(c: connection, version: string)
-    {
-    if ( c$ssh?$server && c$ssh?$client && |c$ssh$client| > 4 && |c$ssh$server| > 4 )
-        {
-        if ( c$ssh$client[4] == "1" && c$ssh$server[4] == "2" )
-            {
-            # SSH199 vs SSH2 -> 2
-            if ( ( |c$ssh$client| > 7 ) && ( c$ssh$client[6] == "9" ) && ( c$ssh$client[7] == "9" ) )
-                c$ssh$version = 2;
-            # SSH1 vs SSH2 -> Undefined
-            else
-                c$ssh$version = 0;
-            }
-        else if ( c$ssh$client[4] == "2" && c$ssh$server[4] == "1" )
-            {
-            # SSH2 vs SSH199 -> 2
-            if ( ( |c$ssh$server| > 7 ) && ( c$ssh$server[6] == "9" ) && ( c$ssh$server[7] == "9" ) )
-                c$ssh$version = 2;
-            else
-            # SSH2 vs SSH1 -> Undefined
-                c$ssh$version = 0;
-            }
-        else if ( c$ssh$client[4] == "1" && c$ssh$server[4] == "1" )
-            {
-            # SSH1 vs SSH199 -> 1
-            if ( ( |c$ssh$server| > 7 ) && ( c$ssh$server[6] == "9" ) && ( c$ssh$server[7] == "9" ) )
-                {
-                # SSH199 vs SSH199
-                if (( |c$ssh$client| > 7 ) && ( c$ssh$client[6] == "9" ) && ( c$ssh$client[7] == "9" ))
-                    c$ssh$version = 2;
-                else
-                    c$ssh$version = 1;
-                }
-            else
-                {
-                # SSH1 vs SSH1 -> 1
-                c$ssh$version = 1;
-                }
-            }
-        # SSH2 vs SSH2
-        else if (c$ssh$client[4] == "2" && c$ssh$server[4] == "2" )
-            {
-            c$ssh$version = 2;
-            }
-        }
-    }
+function set_version(c: connection)
+	{
+	# We always either set the version field to a concrete value, or unset it.
+	delete c$ssh$version;
+
+	# If either the client or server string is unset we cannot compute a
+	# version and return early. We do not raise a weird in this case as we
+	# might arrive here while having only seen one side of the handshake.
+	const has_server = c$ssh?$server && |c$ssh$server| > 0;
+	const has_client = c$ssh?$client && |c$ssh$client| > 0;
+	if ( ! ( has_server && has_client ) )
+		return;
+
+	if ( |c$ssh$client| > 4 && |c$ssh$server| > 4 )
+		{
+		if ( c$ssh$client[4] == "1" && c$ssh$server[4] == "2" )
+			{
+			# SSH199 vs SSH2 -> 2
+			if ( ( |c$ssh$client| > 7 ) && ( c$ssh$client[6] == "9" ) && ( c$ssh$client[7] == "9" ) )
+				c$ssh$version = 2;
+			# SSH1 vs SSH2 -> Undefined
+			else
+				Reporter::conn_weird("SSH_version_mismatch", c, fmt("%s vs %s", c$ssh$server, c$ssh$client));
+				return;
+			}
+		else if ( c$ssh$client[4] == "2" && c$ssh$server[4] == "1" )
+			{
+			# SSH2 vs SSH199 -> 2
+			if ( ( |c$ssh$server| > 7 ) && ( c$ssh$server[6] == "9" ) && ( c$ssh$server[7] == "9" ) )
+				c$ssh$version = 2;
+			else
+				# SSH2 vs SSH1 -> Undefined
+				Reporter::conn_weird("SSH_version_mismatch", c, fmt("%s vs %s", c$ssh$server, c$ssh$client));
+				return;
+			}
+		else if ( c$ssh$client[4] == "1" && c$ssh$server[4] == "1" )
+			{
+			# SSH1 vs SSH199 -> 1
+			if ( ( |c$ssh$server| > 7 ) && ( c$ssh$server[6] == "9" ) && ( c$ssh$server[7] == "9" ) )
+				{
+				# SSH199 vs SSH199
+				if (( |c$ssh$client| > 7 ) && ( c$ssh$client[6] == "9" ) && ( c$ssh$client[7] == "9" ))
+					c$ssh$version = 2;
+				else
+					c$ssh$version = 1;
+				}
+			else
+				{
+				# SSH1 vs SSH1 -> 1
+				c$ssh$version = 1;
+				}
+			}
+		# SSH2 vs SSH2
+		else if (c$ssh$client[4] == "2" && c$ssh$server[4] == "2" )
+			{
+			c$ssh$version = 2;
+			}
+
+		return;
+		}
+
+	Reporter::conn_weird("SSH_cannot_determine_version", c, fmt("%s vs %s", c$ssh$server, c$ssh$client));
+	}
 
 event ssh_server_version(c: connection, version: string)
 	{
 	set_session(c);
 	c$ssh$server = version;
-	set_version(c, version);
+	set_version(c);
 	}
 
 event ssh_client_version(c: connection, version: string)
 	{
 	set_session(c);
 	c$ssh$client = version;
-	set_version(c, version);
+	set_version(c);
 	}
 
 event ssh_auth_attempted(c: connection, authenticated: bool) &priority=5

scripts/base/protocols/ssl/main.zeek

@@ -141,8 +141,8 @@ export {
 	## record as it is sent on to the logging framework.
 	global log_ssl: event(rec: Info);
 
-	# Hook that can be used to perform actions right before the log record
-	# is written.
+	## Hook that can be used to perform actions right before the log record
+	## is written.
 	global ssl_finishing: hook(c: connection);
 
 	## SSL finalization hook.  Remaining SSL info may get logged when it's called.