From 9faabe99917a6a7fe2dd4db25946ed7fdc742a0b Mon Sep 17 00:00:00 2001
From: Mauro Palumbo <mauropalumbo75@gmail.com>
Date: Thu, 25 Jul 2019 16:40:55 +0200
Subject: [PATCH 01/10] add multiprotocol known_services when
 Known::use_service_store = F

---
 .../policy/protocols/conn/known-services.zeek | 48 ++++++++++++++-----
 1 file changed, 37 insertions(+), 11 deletions(-)

diff --git a/scripts/policy/protocols/conn/known-services.zeek b/scripts/policy/protocols/conn/known-services.zeek
index 24774586dc..5765d94f47 100644
--- a/scripts/policy/protocols/conn/known-services.zeek
+++ b/scripts/policy/protocols/conn/known-services.zeek
@@ -62,11 +62,11 @@ export {
 	## of duplicates, but can also be inspected by other scripts for
 	## different purposes.
 	##
-	## In cluster operation, this set is uniformly distributed across
+	## In cluster operation, this table is uniformly distributed across
 	## proxy nodes.
 	##
-	## This set is automatically populated and shouldn't be directly modified.
-	global services: set[addr, port] &create_expire=1day;
+	## This table is automatically populated and shouldn't be directly modified.
+	global services: table[addr, port] of set[string] &create_expire=1day;
 
 	## Event that can be handled to access the :zeek:type:`Known::ServicesInfo`
 	## record as it is sent on to the logging framework.
@@ -79,6 +79,24 @@ redef record connection += {
 	known_services_done: bool &default=F;
 };
 
+# Check if the triplet (host,port_num,service) is already in Known::services
+function check(info: ServicesInfo) : bool
+{
+
+        if ( [info$host, info$port_num] !in Known::services ) 
+                return F;
+
+	if ( |info$service| == 0 )
+		return T;  # don't log empty service
+
+        for(s in info$service)
+                {
+                if ( s !in Known::services[info$host, info$port_num] )
+                        return F;
+                }
+
+        return T;
+}
 
 event zeek_init()
 	{
@@ -89,7 +107,6 @@ event zeek_init()
 	}
 
 event service_info_commit(info: ServicesInfo)
-                          
 	{
 	if ( ! Known::use_service_store )
 		return;
@@ -119,10 +136,19 @@ event known_service_add(info: ServicesInfo)
 	if ( Known::use_service_store )
 		return;
 
-	if ( [info$host, info$port_num] in Known::services )
-		return;
+        if ( check(info) )
+                return;
 
-	add Known::services[info$host, info$port_num];
+	if([info$host, info$port_num] !in Known::services) 
+		Known::services[info$host, info$port_num] = set();
+	
+	for(s in info$service)
+		{
+		if ( s !in Known::services[info$host, info$port_num] )
+			{
+			add Known::services[info$host, info$port_num][s];
+			}
+		}
 
 	@if ( ! Cluster::is_enabled() ||
 	      Cluster::local_node_type() == Cluster::PROXY )
@@ -139,7 +165,7 @@ event Cluster::node_up(name: string, id: string)
 		return;
 
 	# Drop local suppression cache on workers to force HRW key repartitioning.
-	Known::services = set();
+	Known::services = table();
 	}
 
 event Cluster::node_down(name: string, id: string)
@@ -151,7 +177,7 @@ event Cluster::node_down(name: string, id: string)
 		return;
 
 	# Drop local suppression cache on workers to force HRW key repartitioning.
-	Known::services = set();
+	Known::services = table();
 	}
 
 event service_info_commit(info: ServicesInfo)
@@ -159,10 +185,10 @@ event service_info_commit(info: ServicesInfo)
 	if ( Known::use_service_store )
 		return;
 
-	if ( [info$host, info$port_num] in Known::services )
+	if ( check(info) )
 		return;
 
-	local key = cat(info$host, info$port_num);
+	local key = cat(info$host, info$port_num, info$service);
 	Cluster::publish_hrw(Cluster::proxy_pool, key, known_service_add, info);
 	event known_service_add(info);
 	}

From 98f8eb6317b860b7eb8ebe177c0736abd1944b33 Mon Sep 17 00:00:00 2001
From: Mauro Palumbo <mauropalumbo75@gmail.com>
Date: Thu, 25 Jul 2019 16:50:03 +0200
Subject: [PATCH 02/10] remove hyphen in front of some services (for example
 -HTTP, -SSL)   In some cases, there is an hyphen before the protocol name in
 the field   connection$service. This can cause problems in known_services and
   is removed here. It originates probably in some analyzer where it   would
 be better removed in the future.

---
 .../policy/protocols/conn/known-services.zeek  | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/scripts/policy/protocols/conn/known-services.zeek b/scripts/policy/protocols/conn/known-services.zeek
index 5765d94f47..bc23fd9a9f 100644
--- a/scripts/policy/protocols/conn/known-services.zeek
+++ b/scripts/policy/protocols/conn/known-services.zeek
@@ -139,9 +139,9 @@ event known_service_add(info: ServicesInfo)
         if ( check(info) )
                 return;
 
-	if([info$host, info$port_num] !in Known::services) 
+	if([info$host, info$port_num] !in Known::services)
 		Known::services[info$host, info$port_num] = set();
-	
+
 	for(s in info$service)
 		{
 		if ( s !in Known::services[info$host, info$port_num] )
@@ -212,10 +212,22 @@ function known_services_done(c: connection)
 			return;
 		}
 
+	# TODO: this is a temporary patch, because sometimes in c$service the protocol name is written with "-" 
+	# at the beginning. This comes from the analyzers (I've seen it for HTTP and SSL), but causes problems
+	# when checking for known_services on triplets (host, port, services). The service starting with "-" (i.e. -HTTP) is 
+	# reconized as different from the normal one (HTTP).
+	# It would be better to correct the analyzers some time later...
+	local tempservs : set[string];
+        for (s in c$service)
+        	if ( s[0] == "-" )
+			add tempservs[s[1:]];
+		else
+			add tempservs[s];
+              		
 	local info = ServicesInfo($ts = network_time(), $host = id$resp_h,
 	                          $port_num = id$resp_p,
 	                          $port_proto = get_port_transport_proto(id$resp_p),
-	                          $service = c$service);
+	                          $service = tempservs);
 
 	# If no protocol was detected, wait a short time before attempting to log
 	# in case a protocol is detected on another connection.

From cc0f0e2f0961525adc482455cf0b96d99c945429 Mon Sep 17 00:00:00 2001
From: Mauro Palumbo <mauropalumbo75@gmail.com>
Date: Thu, 25 Jul 2019 17:35:32 +0200
Subject: [PATCH 03/10] add multiprotocol known_services when
 Known::use_service_store = T

---
 scripts/policy/protocols/conn/known-services.zeek | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scripts/policy/protocols/conn/known-services.zeek b/scripts/policy/protocols/conn/known-services.zeek
index bc23fd9a9f..9408e9a5b6 100644
--- a/scripts/policy/protocols/conn/known-services.zeek
+++ b/scripts/policy/protocols/conn/known-services.zeek
@@ -37,13 +37,14 @@ export {
 	## See :zeek:type:`Host` for possible choices.
 	option service_tracking = LOCAL_HOSTS;
 
-	type AddrPortPair: record {
+	type AddrPortServTriplet: record {
 		host: addr;
 		p: port;
+		serv: set[string];
 	};
 
 	## Holds the set of all known services.  Keys in the store are
-	## :zeek:type:`Known::AddrPortPair` and their associated value is
+	## :zeek:type:`Known::AddrPortServTriplet` and their associated value is
 	## always the boolean value of "true".
 	global service_store: Cluster::StoreInfo;
 
@@ -111,7 +112,7 @@ event service_info_commit(info: ServicesInfo)
 	if ( ! Known::use_service_store )
 		return;
 
-	local key = AddrPortPair($host = info$host, $p = info$port_num);
+	local key = AddrPortServTriplet($host = info$host, $p = info$port_num, $serv = info$service);
 
 	when ( local r = Broker::put_unique(Known::service_store$store, key,
 	                                    T, Known::service_store_expiry) )

From ddf2d2d8a9fefb0dd4fb7bf6e6970cd89942bda4 Mon Sep 17 00:00:00 2001
From: Mauro Palumbo <mauropalumbo75@gmail.com>
Date: Mon, 29 Jul 2019 17:48:29 +0200
Subject: [PATCH 04/10] remove repeated services in logs if already seen

---
 .../policy/protocols/conn/known-services.zeek | 21 +++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/scripts/policy/protocols/conn/known-services.zeek b/scripts/policy/protocols/conn/known-services.zeek
index 9408e9a5b6..ef216e4289 100644
--- a/scripts/policy/protocols/conn/known-services.zeek
+++ b/scripts/policy/protocols/conn/known-services.zeek
@@ -87,9 +87,6 @@ function check(info: ServicesInfo) : bool
         if ( [info$host, info$port_num] !in Known::services ) 
                 return F;
 
-	if ( |info$service| == 0 )
-		return T;  # don't log empty service
-
         for(s in info$service)
                 {
                 if ( s !in Known::services[info$host, info$port_num] )
@@ -137,23 +134,31 @@ event known_service_add(info: ServicesInfo)
 	if ( Known::use_service_store )
 		return;
 
-        if ( check(info) )
-                return;
+	if ( check(info) )
+		return;
 
 	if([info$host, info$port_num] !in Known::services)
 		Known::services[info$host, info$port_num] = set();
 
+	local info_to_log : ServicesInfo; # service to log can be a subset of info$service if some were already seen
+	info_to_log$ts = info$ts;
+	info_to_log$host = info$host;
+	info_to_log$port_num = info$port_num;
+	info_to_log$port_proto = info$port_proto;
+	info_to_log$service = set();
+
 	for(s in info$service)
 		{
 		if ( s !in Known::services[info$host, info$port_num] )
 			{
 			add Known::services[info$host, info$port_num][s];
+			add info_to_log$service[s];
 			}
 		}
 
 	@if ( ! Cluster::is_enabled() ||
 	      Cluster::local_node_type() == Cluster::PROXY )
-		Log::write(Known::SERVICES_LOG, info);
+		Log::write(Known::SERVICES_LOG, info_to_log);
 	@endif
 	}
 
@@ -202,6 +207,10 @@ function known_services_done(c: connection)
 	if ( ! addr_matches_host(id$resp_h, service_tracking) )
 		return;
 
+	# don't log empty service
+	if ( |c$service| == 0 )
+		return;
+
 	if ( |c$service| == 1 )
 		{
 		if ( "ftp-data" in c$service )

From 9e1e1776215b37f67e46a60f8ab4bbb2fcba1629 Mon Sep 17 00:00:00 2001
From: Mauro Palumbo <mauropalumbo75@gmail.com>
Date: Tue, 30 Jul 2019 15:58:47 +0200
Subject: [PATCH 05/10] order list of services in store key

---
 scripts/policy/protocols/conn/known-services.zeek | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/scripts/policy/protocols/conn/known-services.zeek b/scripts/policy/protocols/conn/known-services.zeek
index ef216e4289..aa1ae58700 100644
--- a/scripts/policy/protocols/conn/known-services.zeek
+++ b/scripts/policy/protocols/conn/known-services.zeek
@@ -40,7 +40,7 @@ export {
 	type AddrPortServTriplet: record {
 		host: addr;
 		p: port;
-		serv: set[string];
+		serv: vector of string;
 	};
 
 	## Holds the set of all known services.  Keys in the store are
@@ -109,7 +109,12 @@ event service_info_commit(info: ServicesInfo)
 	if ( ! Known::use_service_store )
 		return;
 
-	local key = AddrPortServTriplet($host = info$host, $p = info$port_num, $serv = info$service);
+	local v : vector of string;
+	for ( s in info$service )
+		v += s;
+	sort(v, strcmp);	# sort the vector for proper key comparison in put_unique
+
+	local key = AddrPortServTriplet($host = info$host, $p = info$port_num, $serv = v);
 
 	when ( local r = Broker::put_unique(Known::service_store$store, key,
 	                                    T, Known::service_store_expiry) )

From b4ac0b54fe6a6fed36188d6a08de18f76adf31c1 Mon Sep 17 00:00:00 2001
From: Mauro Palumbo <mauropalumbo75@gmail.com>
Date: Tue, 30 Jul 2019 17:03:52 +0200
Subject: [PATCH 06/10] update tests

---
 .../knownhosts-all.log                                      | 4 ++--
 .../knownhosts-local.log                                    | 4 ++--
 .../knownhosts-remote.log                                   | 4 ++--
 .../knownservices-all.log                                   | 4 ++--
 .../knownservices-local.log                                 | 6 +++---
 .../knownservices-remote.log                                | 4 ++--
 .../scripts.policy.protocols.conn.mac-logging/conn1.log     | 4 ++--
 .../scripts.policy.protocols.conn.mac-logging/conn2.log     | 4 ++--
 .../scripts.policy.protocols.conn.mac-logging/conn3.log     | 4 ++--
 .../scripts.policy.protocols.conn.vlan-logging/conn.log     | 4 ++--
 10 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log
index 08899cd565..8013dbb0d2 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log
@@ -3,11 +3,11 @@
 #empty_field	(empty)
 #unset_field	-
 #path	known_hosts
-#open	2016-07-13-16-17-24
+#open	2019-07-30-14-59-50
 #fields	ts	host
 #types	time	addr
 1300475168.783842	208.80.152.118
 1300475168.783842	141.142.220.118
 1300475168.915940	208.80.152.3
 1300475168.962628	208.80.152.2
-#close	2016-07-13-16-17-24
+#close	2019-07-30-14-59-50
diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log
index 766167a010..9f60d52495 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	known_hosts
-#open	2016-07-13-16-17-22
+#open	2019-07-30-14-59-48
 #fields	ts	host
 #types	time	addr
 1300475168.783842	141.142.220.118
-#close	2016-07-13-16-17-22
+#close	2019-07-30-14-59-48
diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log
index 688d482308..50b10109f5 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log
@@ -3,10 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	known_hosts
-#open	2016-07-13-16-17-23
+#open	2019-07-30-14-59-49
 #fields	ts	host
 #types	time	addr
 1300475168.783842	208.80.152.118
 1300475168.915940	208.80.152.3
 1300475168.962628	208.80.152.2
-#close	2016-07-13-16-17-23
+#close	2019-07-30-14-59-49
diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log
index 8cf5b98527..d8225a60d1 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	known_services
-#open	2019-06-15-23-46-49
+#open	2019-07-31-13-46-10
 #fields	ts	host	port_num	port_proto	service
 #types	time	addr	port	enum	set[string]
 1308930691.089263	172.16.238.131	22	tcp	SSH
@@ -12,4 +12,4 @@
 1308930718.361665	172.16.238.131	21	tcp	FTP
 1308930726.889624	141.142.192.39	22	tcp	SSH
 1308930727.236071	69.50.219.51	123	udp	NTP
-#close	2019-06-15-23-46-49
+#close	2019-07-31-13-46-10
diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log
index ac271018e2..9eedad7fa4 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log
@@ -3,10 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	known_services
-#open	2014-04-01-23-16-20
+#open	2019-07-31-13-46-08
 #fields	ts	host	port_num	port_proto	service
 #types	time	addr	port	enum	set[string]
-1308930691.049431	172.16.238.131	22	tcp	SSH
+1308930691.089263	172.16.238.131	22	tcp	SSH
 1308930694.550308	172.16.238.131	80	tcp	HTTP
 1308930718.361665	172.16.238.131	21	tcp	FTP
-#close	2014-04-01-23-16-20
+#close	2019-07-31-13-46-08
diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log
index 6441c5255e..1ad6308068 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log
@@ -3,10 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	known_services
-#open	2019-06-15-23-44-01
+#open	2019-07-31-13-46-09
 #fields	ts	host	port_num	port_proto	service
 #types	time	addr	port	enum	set[string]
 1308930716.462556	74.125.225.81	80	tcp	HTTP
 1308930726.889624	141.142.192.39	22	tcp	SSH
 1308930727.236071	69.50.219.51	123	udp	NTP
-#close	2019-06-15-23-44-01
+#close	2019-07-31-13-46-09
diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.mac-logging/conn1.log b/testing/btest/Baseline/scripts.policy.protocols.conn.mac-logging/conn1.log
index 3d415916f2..7f539a133a 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.mac-logging/conn1.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.mac-logging/conn1.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
-#open	2018-01-12-21-44-59
+#open	2019-07-30-14-59-56
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	local_resp	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents	orig_l2_addr	resp_l2_addr
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	bool	count	string	count	count	count	count	set[string]	string	string
 1300475169.780331	C3eiCBGOLw3VtHfOj	173.192.163.128	80	141.142.220.235	6705	tcp	-	-	-	-	OTH	-	-	0	H	1	48	0	0	-	00:13:7f:be:8c:ff	00:e0:db:01:cf:4b
@@ -40,4 +40,4 @@
 1300475168.893988	C8rquZ3DjgNW06JGLl	141.142.220.118	45000	141.142.2.2	53	udp	-	0.000384	38	89	SF	-	-	0	Dd	1	66	1	117	-	00:24:7e:e0:1d:b5	00:13:7f:be:8c:ff
 1300475168.857956	CzrZOtXqhwwndQva3	141.142.220.118	32902	141.142.2.2	53	udp	-	0.000317	38	89	SF	-	-	0	Dd	1	66	1	117	-	00:24:7e:e0:1d:b5	00:13:7f:be:8c:ff
 1300475168.891644	CaGCc13FffXe6RkQl9	141.142.220.118	58206	141.142.2.2	53	udp	-	0.000339	38	89	SF	-	-	0	Dd	1	66	1	117	-	00:24:7e:e0:1d:b5	00:13:7f:be:8c:ff
-#close	2018-01-12-21-44-59
+#close	2019-07-30-14-59-56
diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.mac-logging/conn2.log b/testing/btest/Baseline/scripts.policy.protocols.conn.mac-logging/conn2.log
index 0cfe1b9e4e..9ff8e9f493 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.mac-logging/conn2.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.mac-logging/conn2.log
@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
-#open	2018-01-12-21-45-00
+#open	2019-07-30-14-59-56
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	local_resp	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents	orig_l2_addr	resp_l2_addr
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	bool	count	string	count	count	count	count	set[string]	string	string
 1439902891.705224	CHhAvVGS1DHFjwGM9	172.17.156.76	61738	208.67.220.220	53	udp	-	0.041654	35	128	SF	-	-	0	Dd	1	63	1	156	-	90:72:40:97:b6:f5	44:2b:03:aa:ab:8d
 1439903050.580632	ClEkJM2Vm5giqnMf4h	fe80::a667:6ff:fef7:ec54	5353	ff02::fb	5353	udp	-	-	-	-	S0	-	-	0	D	1	328	0	0	-	a4:67:06:f7:ec:54	33:33:00:00:00:fb
-#close	2018-01-12-21-45-00
+#close	2019-07-30-14-59-56
diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.mac-logging/conn3.log b/testing/btest/Baseline/scripts.policy.protocols.conn.mac-logging/conn3.log
index b738f14754..c5ce634a1d 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.mac-logging/conn3.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.mac-logging/conn3.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
-#open	2018-01-12-21-45-00
+#open	2019-07-30-14-59-56
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	local_resp	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents	orig_l2_addr	resp_l2_addr
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	bool	count	string	count	count	count	count	set[string]	string	string
 826191058.128321	CHhAvVGS1DHFjwGM9	128.3.140.132	2035	194.140.136.34	80	tcp	-	-	-	-	RSTOS0	-	-	0	R	1	40	0	0	-	-	-
@@ -1339,4 +1339,4 @@
 826277279.235554	CBP3Hu4RKc79x58Y2	128.3.140.132	2035	194.140.136.34	80	tcp	-	-	-	-	RSTOS0	-	-	0	R	1	40	0	0	-	-	-
 826277339.221727	CUbAnm2k9C1iEtTmgd	128.3.140.132	2035	194.140.136.34	80	tcp	-	-	-	-	RSTOS0	-	-	0	R	1	40	0	0	-	-	-
 826277399.202051	CkWokd3nscpygp5lIc	128.3.140.132	2035	194.140.136.34	80	tcp	-	-	-	-	RSTOS0	-	-	0	R	1	40	0	0	-	-	-
-#close	2018-01-12-21-45-00
+#close	2019-07-30-14-59-57
diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.vlan-logging/conn.log b/testing/btest/Baseline/scripts.policy.protocols.conn.vlan-logging/conn.log
index ae6d54784f..21a7c84374 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.vlan-logging/conn.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.vlan-logging/conn.log
@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
-#open	2016-07-13-16-17-26
+#open	2019-07-30-14-59-58
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	local_resp	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents	vlan	inner_vlan
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	bool	count	string	count	count	count	count	set[string]	int	int
 1363900699.548138	CHhAvVGS1DHFjwGM9	172.19.51.37	47808	172.19.51.63	47808	udp	-	0.000100	36	0	S0	-	-	0	D	2	92	0	0	-	13	10
 1363900699.549647	ClEkJM2Vm5giqnMf4h	193.1.186.60	9875	224.2.127.254	9875	udp	-	0.000139	552	0	S0	-	-	0	D	2	608	0	0	-	13	10
-#close	2016-07-13-16-17-26
+#close	2019-07-30-14-59-58

From 55013fa128886621c5ae009d44adb5942ee604d1 Mon Sep 17 00:00:00 2001
From: Mauro Palumbo <mauropalumbo75@gmail.com>
Date: Wed, 31 Jul 2019 16:08:36 +0200
Subject: [PATCH 07/10] remove check for empty services

---
 scripts/policy/protocols/conn/known-services.zeek | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/scripts/policy/protocols/conn/known-services.zeek b/scripts/policy/protocols/conn/known-services.zeek
index aa1ae58700..373995beae 100644
--- a/scripts/policy/protocols/conn/known-services.zeek
+++ b/scripts/policy/protocols/conn/known-services.zeek
@@ -212,10 +212,6 @@ function known_services_done(c: connection)
 	if ( ! addr_matches_host(id$resp_h, service_tracking) )
 		return;
 
-	# don't log empty service
-	if ( |c$service| == 0 )
-		return;
-
 	if ( |c$service| == 1 )
 		{
 		if ( "ftp-data" in c$service )

From f7a8e8c8fbfc3cc1f29439f0cbd0b46e4b5fd563 Mon Sep 17 00:00:00 2001
From: Mauro Palumbo <mauropalumbo75@gmail.com>
Date: Wed, 31 Jul 2019 16:28:25 +0200
Subject: [PATCH 08/10] remove service from key for Cluster::publish_hrw

---
 scripts/policy/protocols/conn/known-services.zeek | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/policy/protocols/conn/known-services.zeek b/scripts/policy/protocols/conn/known-services.zeek
index 373995beae..8cdc474864 100644
--- a/scripts/policy/protocols/conn/known-services.zeek
+++ b/scripts/policy/protocols/conn/known-services.zeek
@@ -199,7 +199,7 @@ event service_info_commit(info: ServicesInfo)
 	if ( check(info) )
 		return;
 
-	local key = cat(info$host, info$port_num, info$service);
+	local key = cat(info$host, info$port_num);
 	Cluster::publish_hrw(Cluster::proxy_pool, key, known_service_add, info);
 	event known_service_add(info);
 	}

From 1f7f42daeab33cbee218b9251427d1e7372872a1 Mon Sep 17 00:00:00 2001
From: Mauro Palumbo <mauropalumbo75@gmail.com>
Date: Wed, 31 Jul 2019 17:07:10 +0200
Subject: [PATCH 09/10] drop services starting with -

---
 scripts/policy/protocols/conn/known-services.zeek | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/scripts/policy/protocols/conn/known-services.zeek b/scripts/policy/protocols/conn/known-services.zeek
index 8cdc474864..8b55163703 100644
--- a/scripts/policy/protocols/conn/known-services.zeek
+++ b/scripts/policy/protocols/conn/known-services.zeek
@@ -223,16 +223,10 @@ function known_services_done(c: connection)
 			return;
 		}
 
-	# TODO: this is a temporary patch, because sometimes in c$service the protocol name is written with "-" 
-	# at the beginning. This comes from the analyzers (I've seen it for HTTP and SSL), but causes problems
-	# when checking for known_services on triplets (host, port, services). The service starting with "-" (i.e. -HTTP) is 
-	# reconized as different from the normal one (HTTP).
-	# It would be better to correct the analyzers some time later...
+	# Drop services starting with "-"
 	local tempservs : set[string];
         for (s in c$service)
-        	if ( s[0] == "-" )
-			add tempservs[s[1:]];
-		else
+        	if ( s[0] != "-" )
 			add tempservs[s];
               		
 	local info = ServicesInfo($ts = network_time(), $host = id$resp_h,

From e206347d1a612a07cdca57a38f7b5981c6b3b8b8 Mon Sep 17 00:00:00 2001
From: Mauro Palumbo <mauropalumbo75@gmail.com>
Date: Wed, 31 Jul 2019 17:40:02 +0200
Subject: [PATCH 10/10] improve logging with broker store

---
 .../policy/protocols/conn/known-services.zeek | 37 ++++++++++---------
 1 file changed, 19 insertions(+), 18 deletions(-)

diff --git a/scripts/policy/protocols/conn/known-services.zeek b/scripts/policy/protocols/conn/known-services.zeek
index 8b55163703..cf1af574a4 100644
--- a/scripts/policy/protocols/conn/known-services.zeek
+++ b/scripts/policy/protocols/conn/known-services.zeek
@@ -40,7 +40,7 @@ export {
 	type AddrPortServTriplet: record {
 		host: addr;
 		p: port;
-		serv: vector of string;
+		serv: string;
 	};
 
 	## Holds the set of all known services.  Keys in the store are
@@ -109,28 +109,29 @@ event service_info_commit(info: ServicesInfo)
 	if ( ! Known::use_service_store )
 		return;
 
-	local v : vector of string;
-	for ( s in info$service )
-		v += s;
-	sort(v, strcmp);	# sort the vector for proper key comparison in put_unique
+	local tempservs = info$service;
+	for ( s in tempservs ) {
 
-	local key = AddrPortServTriplet($host = info$host, $p = info$port_num, $serv = v);
+		local key = AddrPortServTriplet($host = info$host, $p = info$port_num, $serv = s);
 
-	when ( local r = Broker::put_unique(Known::service_store$store, key,
+		when ( local r = Broker::put_unique(Known::service_store$store, key,
 	                                    T, Known::service_store_expiry) )
-		{
-		if ( r$status == Broker::SUCCESS )
 			{
-			if ( r$result as bool )
-				Log::write(Known::SERVICES_LOG, info);
+			if ( r$status == Broker::SUCCESS )
+				{
+				if ( r$result as bool ) {
+					info$service = set(s);	# log one service at the time if multiservice
+					Log::write(Known::SERVICES_LOG, info);
+					}
+				}
+			else
+				Reporter::error(fmt("%s: data store put_unique failure",
+				                    Known::service_store_name));
+			}
+		timeout Known::service_store_timeout
+			{
+			Log::write(Known::SERVICES_LOG, info);
 			}
-		else
-			Reporter::error(fmt("%s: data store put_unique failure",
-			                    Known::service_store_name));
-		}
-	timeout Known::service_store_timeout
-		{
-		Log::write(Known::SERVICES_LOG, info);
 		}
 	}