Merge branch 'master' into topic/vlad/page_hostnames

scripts/base/frameworks/broker/main.zeek

@@ -31,7 +31,7 @@ export {
 	## authenticated.
 	const disable_ssl = F &redef;
 
-	## Path to a file containing concatenated trusted certificates 
+	## Path to a file containing concatenated trusted certificates
 	## in PEM format. If set, Zeek will require valid certificates for
 	## all peers.
 	const ssl_cafile = "" &redef;
@@ -122,6 +122,37 @@ export {
 	## done reading the pcap.
 	option peer_counts_as_iosource = T;
 
+	## Port for Broker's metric exporter. Setting this to a valid TCP port causes
+	## Broker to make metrics available to Prometheus scrapers via HTTP. Zeek
+	## overrides any value provided in zeek_init or earlier at startup if the
+	## environment variable BROKER_METRICS_PORT is defined.
+	const metrics_port = 0/unknown &redef;
+
+	## Frequency for publishing scraped metrics to the target topic. Zeek
+	## overrides any value provided in zeek_init or earlier at startup if the
+	## environment variable BROKER_METRICS_EXPORT_INTERVAL is defined.
+	option metrics_export_interval = 1 sec;
+
+	## Target topic for the metrics. Setting a non-empty string starts the
+	## periodic publishing of local metrics. Zeek overrides any value provided in
+	## zeek_init or earlier at startup if the environment variable
+	## BROKER_METRICS_EXPORT_TOPIC is defined.
+	option metrics_export_topic = "";
+
+	## ID for the metrics exporter. When setting a target topic for the
+	## exporter, Broker sets this option to the suffix of the new topic *unless*
+	## the ID is a non-empty string. Since setting a topic starts the periodic
+	## publishing of events, we recommend setting the ID always first or avoid
+	## setting it at all if the topic suffix serves as a good-enough ID. Zeek
+	## overrides any value provided in zeek_init or earlier at startup if the
+	## environment variable BROKER_METRICS_ENDPOINT_NAME is defined.
+	option metrics_export_endpoint_name = "";
+
+	## Selects prefixes from the local metrics. Only metrics with prefixes
+	## listed in this variable are included when publishing local metrics.
+	## Setting an empty vector selects *all* metrics.
+	option metrics_export_prefixes: vector of string = vector();
+
 	## The default topic prefix where logs will be published.  The log's stream
 	## id is appended when writing to a particular stream.
 	const default_log_topic_prefix = "zeek/logs/" &redef;
@@ -385,9 +416,53 @@ event Broker::log_flush() &priority=10
 	schedule Broker::log_batch_interval { Broker::log_flush() };
 	}
 
+function update_metrics_export_interval(id: string, val: interval): interval
+	{
+	Broker::__set_metrics_export_interval(val);
+	return val;
+	}
+
+function update_metrics_export_topic(id: string, val: string): string
+	{
+	Broker::__set_metrics_export_topic(val);
+	return val;
+	}
+
+function update_metrics_export_endpoint_name(id: string, val: string): string
+	{
+	Broker::__set_metrics_export_endpoint_name(val);
+	return val;
+	}
+
+function update_metrics_export_prefixes(id: string, filter: vector of string): vector of string
+	{
+	Broker::__set_metrics_export_prefixes(filter);
+	return filter;
+	}
+
 event zeek_init()
 	{
 	schedule Broker::log_batch_interval { Broker::log_flush() };
+	# interval
+	update_metrics_export_interval("Broker::metrics_export_interval",
+	                               Broker::metrics_export_interval);
+	Option::set_change_handler("Broker::metrics_export_interval",
+	                           update_metrics_export_interval);
+	# topic
+	update_metrics_export_topic("Broker::metrics_export_topic",
+	                            Broker::metrics_export_topic);
+	Option::set_change_handler("Broker::metrics_export_topic",
+	                           update_metrics_export_topic);
+	# endpoint name
+	update_metrics_export_endpoint_name("Broker::metrics_export_endpoint_name",
+	                                    Broker::metrics_export_endpoint_name);
+	Option::set_change_handler("Broker::metrics_export_endpoint_name",
+	                           update_metrics_export_endpoint_name);
+	# prefixes
+	update_metrics_export_prefixes("Broker::metrics_export_prefixes",
+	                               Broker::metrics_export_prefixes);
+	Option::set_change_handler("Broker::metrics_export_prefixes",
+	                           update_metrics_export_prefixes);
 	}
 
 event retry_listen(a: string, p: port, retry: interval)

scripts/base/frameworks/logging/writers/ascii.zeek

@@ -54,6 +54,11 @@ export {
 	## This option is also available as a per-filter ``$config`` option.
 	const gzip_file_extension = "gz" &redef;
 
+	## Define the default logging directory. If empty, logs are written
+	## to the current working directory.
+	##
+	const logdir = "" &redef;
+
 	## Format of timestamps when writing out JSON. By default, the JSON
 	## formatter will use double values for timestamps which represent the
 	## number of seconds from the UNIX epoch.

scripts/base/frameworks/tunnels/main.zeek

@@ -93,7 +93,7 @@ export {
 const ayiya_ports = { 5072/udp };
 const teredo_ports = { 3544/udp };
 const gtpv1_ports = { 2152/udp, 2123/udp };
-redef likely_server_ports += { ayiya_ports, teredo_ports, gtpv1_ports, vxlan_ports };
+redef likely_server_ports += { ayiya_ports, teredo_ports, gtpv1_ports, vxlan_ports, geneve_ports };
 
 event zeek_init() &priority=5
 	{
@@ -103,6 +103,7 @@ event zeek_init() &priority=5
 	Analyzer::register_for_ports(Analyzer::ANALYZER_TEREDO, teredo_ports);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_GTPV1, gtpv1_ports);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_VXLAN, vxlan_ports);
+	Analyzer::register_for_ports(Analyzer::ANALYZER_GENEVE, geneve_ports);
 	}
 
 function register_all(ecv: EncapsulatingConnVector)

scripts/base/init-bare.zeek

@@ -635,7 +635,7 @@ type ProcStats: record {
 	real_time: interval;          ##< Elapsed real time since Zeek started running.
 	user_time: interval;          ##< User CPU seconds.
 	system_time: interval;        ##< System CPU seconds.
-	mem: count;                   ##< Maximum memory consumed, in KB.
+	mem: count;                   ##< Maximum memory consumed, in bytes.
 	minor_faults: count;          ##< Page faults not requiring actual I/O.
 	major_faults: count;          ##< Page faults requiring actual I/O.
 	num_swap: count;              ##< Times swapped out.
@@ -1933,6 +1933,7 @@ type gtp_delete_pdp_ctx_response_elements: record {
 @load base/frameworks/supervisor/api
 @load base/bif/supervisor.bif
 @load base/bif/packet_analysis.bif
+@load base/bif/CPP-load.bif
 
 ## Internal function.
 function add_interface(iold: string, inew: string): string
@@ -5029,6 +5030,12 @@ export {
 	## if you customize this, you may still want to manually ensure that
 	## :zeek:see:`likely_server_ports` also gets populated accordingly.
 	const vxlan_ports: set[port] = { 4789/udp } &redef;
+
+	## The set of UDP ports used for Geneve traffic.  Traffic using this
+	## UDP destination port will attempt to be decapsulated.  Note that if
+	## if you customize this, you may still want to manually ensure that
+	## :zeek:see:`likely_server_ports` also gets populated accordingly.
+	const geneve_ports: set[port] = { 6081/udp } &redef;
 } # end export
 
 module Reporter;

scripts/base/packet-protocols/__load__.zeek

@@ -15,3 +15,6 @@
 @load base/packet-protocols/gre
 @load base/packet-protocols/iptunnel
 @load base/packet-protocols/vntag
+@load base/packet-protocols/udp
+@load base/packet-protocols/tcp
+@load base/packet-protocols/icmp

scripts/base/packet-protocols/icmp/__load__.zeek

@@ -0,0 +1 @@
+@load ./main

scripts/base/packet-protocols/icmp/main.zeek

@@ -0,0 +1,5 @@
+module PacketAnalyzer::ICMP;
+
+#event zeek_init() &priority=20
+#	{
+#	}

scripts/base/packet-protocols/ip/main.zeek

@@ -1,8 +1,22 @@
 module PacketAnalyzer::IP;
 
+const IPPROTO_TCP : count = 6;
+const IPPROTO_UDP : count = 17;
+const IPPROTO_ICMP : count = 1;
+const IPPROTO_ICMP6 : count = 58;
+
+const IPPROTO_IPIP : count = 4;
+const IPPROTO_IPV6 : count = 41;
+const IPPROTO_GRE : count = 47;
+
 event zeek_init() &priority=20
 	{
-	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, 4, PacketAnalyzer::ANALYZER_IPTUNNEL);
-	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, 41, PacketAnalyzer::ANALYZER_IPTUNNEL);
-	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, 47, PacketAnalyzer::ANALYZER_GRE);
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_IPIP, PacketAnalyzer::ANALYZER_IPTUNNEL);
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_IPV6, PacketAnalyzer::ANALYZER_IPTUNNEL);
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_GRE, PacketAnalyzer::ANALYZER_GRE);
+
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_TCP, PacketAnalyzer::ANALYZER_TCP);
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_UDP, PacketAnalyzer::ANALYZER_UDP);
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_ICMP, PacketAnalyzer::ANALYZER_ICMP);
+	PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_ICMP6, PacketAnalyzer::ANALYZER_ICMP);
 	}

scripts/base/packet-protocols/tcp/__load__.zeek

@@ -0,0 +1 @@
+@load ./main

scripts/base/packet-protocols/tcp/main.zeek

@@ -0,0 +1,5 @@
+module PacketAnalyzer::TCP;
+
+#event zeek_init() &priority=20
+#	{
+#	}

scripts/base/packet-protocols/udp/__load__.zeek

@@ -0,0 +1 @@
+@load ./main

scripts/base/packet-protocols/udp/main.zeek

@@ -0,0 +1,5 @@
+module PacketAnalyzer::UDP;
+
+#event zeek_init() &priority=20
+#	{
+#	}

scripts/base/protocols/dns/main.zeek

@@ -434,7 +434,11 @@ event dns_request(c: connection, msg: dns_msg, query: string, qtype: count, qcla
 	#       worked into the query/response in some fashion.
 	if ( c$id$resp_p == 137/udp )
 		{
-		query = decode_netbios_name(query);
+		local decoded_query = decode_netbios_name(query);
+
+		if ( |decoded_query| != 0 )
+			query = decoded_query;
+
 		if ( c$dns$qtype_name == "SRV" )
 			{
 			# The SRV RFC used the ID used for NetBios Status RRs.

scripts/base/protocols/ssl/main.zeek

@@ -111,8 +111,8 @@ export {
 	## record as it is sent on to the logging framework.
 	global log_ssl: event(rec: Info);
 
-	# Hook that can be used to perform actions right before the log record
-	# is written.
+	## Hook that can be used to perform actions right before the log record
+	## is written.
 	global ssl_finishing: hook(c: connection);
 
 	## SSL finalization hook.  Remaining SSL info may get logged when it's called.