Merge branch 'known_services_multiprotocols' of https://github.com/mauropalumbo75/zeek

* 'known_services_multiprotocols' of https://github.com/mauropalumbo75/zeek:
  improve logging with broker store
  drop services starting with -
  remove service from key for Cluster::publish_hrw
  remove check for empty services
  update tests
  order list of services in store key
  remove repeated services in logs if already seen
  add multiprotocol known_services when Known::use_service_store = T
  remove hyphen in front of some services (for example -HTTP, -SSL)   In some cases, there is an hyphen before the protocol name in the field   connection$service. This can cause problems in known_services and   is removed here. It originates probably in some analyzer where it   would be better removed in the future.
  add multiprotocol known_services when Known::use_service_store = F

Changes during merge:
  * whitespace
  * add unit test

CHANGES

@@ -1,4 +1,11 @@
 
+3.1.0-dev.18 | 2019-08-09 10:43:28 -0700
+
+  * GH-419: improve multi-protocol logging in known_services.log (Mauro Palumbo)
+
+    Previously, when multiple protocols were detected on a given addr/port
+    pair, not all protocols were always logged.
+
 3.1.0-dev.7 | 2019-08-09 09:56:06 -0700
 
   * Remove empty services from known_services.log (Mauro Palumbo)

NEWS

@@ -3,6 +3,25 @@ This document summarizes the most important changes in the current Zeek
 release. For an exhaustive list of changes, see the ``CHANGES`` file
 (note that submodules, such as Broker, come with their own ``CHANGES``.)
 
+Zeek 3.1.0
+==========
+
+New Functionality
+-----------------
+
+Changed Functionality
+---------------------
+
+- The key type of ``Known::service_store`` has changed to
+  ``Known::AddrPortServTriplet`` and ``Known::services`` is now a table
+  instead of just a set.
+
+Removed Functionality
+---------------------
+
+Deprecated Functionality
+------------------------
+
 Zeek 3.0.0
 ==========
 

VERSION

@@ -1 +1 @@
-3.1.0-dev.7
+3.1.0-dev.18

doc

@@ -1 +1 @@
-Subproject commit 1cd9867ca6c8ab73db1f2b665ec69332fe88734c
+Subproject commit 56019a5297cd5d167e0febc6b0b33a32a9dbe96e

scripts/policy/protocols/conn/known-services.zeek

@@ -37,13 +37,14 @@ export {
 	## See :zeek:type:`Host` for possible choices.
 	option service_tracking = LOCAL_HOSTS;
 
-	type AddrPortPair: record {
+	type AddrPortServTriplet: record {
 		host: addr;
 		p: port;
+		serv: string;
 	};
 
 	## Holds the set of all known services.  Keys in the store are
-	## :zeek:type:`Known::AddrPortPair` and their associated value is
+	## :zeek:type:`Known::AddrPortServTriplet` and their associated value is
 	## always the boolean value of "true".
 	global service_store: Cluster::StoreInfo;
 
@@ -62,11 +63,11 @@ export {
 	## of duplicates, but can also be inspected by other scripts for
 	## different purposes.
 	##
-	## In cluster operation, this set is uniformly distributed across
+	## In cluster operation, this table is uniformly distributed across
 	## proxy nodes.
 	##
-	## This set is automatically populated and shouldn't be directly modified.
-	global services: set[addr, port] &create_expire=1day;
+	## This table is automatically populated and shouldn't be directly modified.
+	global services: table[addr, port] of set[string] &create_expire=1day;
 
 	## Event that can be handled to access the :zeek:type:`Known::ServicesInfo`
 	## record as it is sent on to the logging framework.
@@ -79,6 +80,20 @@ redef record connection += {
 	known_services_done: bool &default=F;
 };
 
+# Check if the triplet (host,port_num,service) is already in Known::services
+function check(info: ServicesInfo) : bool
+	{
+	if ( [info$host, info$port_num] !in Known::services )
+		return F;
+
+	for ( s in info$service )
+		{
+		if ( s !in Known::services[info$host, info$port_num] )
+			return F;
+		}
+
+	return T;
+	}
 
 event zeek_init()
 	{
@@ -89,28 +104,34 @@ event zeek_init()
 	}
 
 event service_info_commit(info: ServicesInfo)
-                          
 	{
 	if ( ! Known::use_service_store )
 		return;
 
-	local key = AddrPortPair($host = info$host, $p = info$port_num);
+	local tempservs = info$service;
 
-	when ( local r = Broker::put_unique(Known::service_store$store, key,
-	                                    T, Known::service_store_expiry) )
+	for ( s in tempservs )
 		{
-		if ( r$status == Broker::SUCCESS )
+		local key = AddrPortServTriplet($host = info$host, $p = info$port_num, $serv = s);
+
+		when ( local r = Broker::put_unique(Known::service_store$store, key,
+		                                    T, Known::service_store_expiry) )
 			{
-			if ( r$result as bool )
-				Log::write(Known::SERVICES_LOG, info);
+			if ( r$status == Broker::SUCCESS )
+				{
+				if ( r$result as bool ) {
+					info$service = set(s);	# log one service at the time if multiservice
+					Log::write(Known::SERVICES_LOG, info);
+					}
+				}
+			else
+				Reporter::error(fmt("%s: data store put_unique failure",
+				                    Known::service_store_name));
+			}
+		timeout Known::service_store_timeout
+			{
+			Log::write(Known::SERVICES_LOG, info);
 			}
-		else
-			Reporter::error(fmt("%s: data store put_unique failure",
-			                    Known::service_store_name));
-		}
-	timeout Known::service_store_timeout
-		{
-		Log::write(Known::SERVICES_LOG, info);
 		}
 	}
 
@@ -119,14 +140,32 @@ event known_service_add(info: ServicesInfo)
 	if ( Known::use_service_store )
 		return;
 
-	if ( [info$host, info$port_num] in Known::services )
+	if ( check(info) )
 		return;
 
-	add Known::services[info$host, info$port_num];
+	if ( [info$host, info$port_num] !in Known::services )
+		Known::services[info$host, info$port_num] = set();
+
+	 # service to log can be a subset of info$service if some were already seen
+	local info_to_log: ServicesInfo;
+	info_to_log$ts = info$ts;
+	info_to_log$host = info$host;
+	info_to_log$port_num = info$port_num;
+	info_to_log$port_proto = info$port_proto;
+	info_to_log$service = set();
+
+	for ( s in info$service )
+		{
+		if ( s !in Known::services[info$host, info$port_num] )
+			{
+			add Known::services[info$host, info$port_num][s];
+			add info_to_log$service[s];
+			}
+		}
 
 	@if ( ! Cluster::is_enabled() ||
 	      Cluster::local_node_type() == Cluster::PROXY )
-		Log::write(Known::SERVICES_LOG, info);
+		Log::write(Known::SERVICES_LOG, info_to_log);
 	@endif
 	}
 
@@ -139,7 +178,7 @@ event Cluster::node_up(name: string, id: string)
 		return;
 
 	# Drop local suppression cache on workers to force HRW key repartitioning.
-	Known::services = set();
+	Known::services = table();
 	}
 
 event Cluster::node_down(name: string, id: string)
@@ -151,7 +190,7 @@ event Cluster::node_down(name: string, id: string)
 		return;
 
 	# Drop local suppression cache on workers to force HRW key repartitioning.
-	Known::services = set();
+	Known::services = table();
 	}
 
 event service_info_commit(info: ServicesInfo)
@@ -159,7 +198,7 @@ event service_info_commit(info: ServicesInfo)
 	if ( Known::use_service_store )
 		return;
 
-	if ( [info$host, info$port_num] in Known::services )
+	if ( check(info) )
 		return;
 
 	local key = cat(info$host, info$port_num);
@@ -186,10 +225,16 @@ function known_services_done(c: connection)
 			return;
 		}
 
+	# Drop services starting with "-" (confirmed-but-then-violated protocol)
+	local tempservs: set[string];
+		for (s in c$service)
+			if ( s[0] != "-" )
+				add tempservs[s];
+
 	local info = ServicesInfo($ts = network_time(), $host = id$resp_h,
 	                          $port_num = id$resp_p,
 	                          $port_proto = get_port_transport_proto(id$resp_p),
-	                          $service = c$service);
+	                          $service = tempservs);
 
 	# If no protocol was detected, wait a short time before attempting to log
 	# in case a protocol is detected on another connection.

testing/btest/Baseline/scripts.policy.protocols.conn.known-services-multi/known_services.log

@@ -0,0 +1,11 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	known_services
+#open	2019-08-09-17-38-17
+#fields	ts	host	port_num	port_proto	service
+#types	time	addr	port	enum	set[string]
+1455718916.856316	192.168.2.230	443	tcp	SSH
+1455718922.796688	192.168.2.230	443	tcp	SSL
+#close	2019-08-09-17-38-18

testing/btest/scripts/policy/protocols/conn/known-services-multi.zeek

@@ -0,0 +1,7 @@
+# A test case for when more than a single service is detected for a given
+# (addr, port) pair.
+
+# @TEST-EXEC: zeek -C -r $TRACES/ssl-and-ssh-using-sslh.trace %INPUT "Known::service_tracking = ALL_HOSTS"
+# @TEST-EXEC: btest-diff known_services.log
+
+@load protocols/conn/known-services

testing/external/commit-hash.zeek-testing

@@ -1 +1 @@
-5e5a5e8dbb94215a7ca1def810f4bbe0322bc72e
+dc6a8f1de9f3b298406051282abcaa6e8f198695

testing/external/commit-hash.zeek-testing-private

@@ -1 +1 @@
-b7cf5aa8224fb39baf7497d187f48165fad050da
+fdfcdffd464fd2114be03feacfd075d73a8b1ef9