Modify known-services policy script to add storage framework support

This adds a flag to enable using the storage framework instead of Broker stores,
plus a btest for both broker and the the storage framework.

scripts/policy/protocols/conn/known-services.zeek

@@ -9,6 +9,10 @@
 @load base/utils/directions-and-hosts
 @load base/frameworks/cluster
 
+@load base/frameworks/storage/async
+@load base/frameworks/storage/sync
+@load policy/frameworks/storage/backend/sqlite
+
 module Known;
 
 export {
@@ -39,6 +43,11 @@ export {
 	## operation.
 	const use_service_store = F &redef;
 
+	## Switches to the version of this script that uses the storage
+	## framework instead of Broker stores. This will default to ``T``
+	## in v8.1.
+	const use_storage_framework = F &redef;
+
 	## Require UDP server to respond before considering it an "active service".
 	option service_udp_requires_response = T;
 
@@ -52,20 +61,46 @@ export {
 		serv: string;
 	};
 
+	## Storage configuration for Broker stores
+
 	## Holds the set of all known services.  Keys in the store are
 	## :zeek:type:`Known::AddrPortServTriplet` and their associated value is
 	## always the boolean value of "true".
-	global service_store: Cluster::StoreInfo;
+	global service_broker_store: Cluster::StoreInfo;
 
-	## The Broker topic name to use for :zeek:see:`Known::service_store`.
+	## The Broker topic name to use for :zeek:see:`Known::service_broker_store`.
 	const service_store_name = "zeek/known/services" &redef;
 
-	## The expiry interval of new entries in :zeek:see:`Known::service_store`.
+	## Storage configuration for storage framework stores
-	## This also changes the interval at which services get logged.
+
+	## This requires setting a configuration in local.zeek that sets the
+	## Known::use_storage_framework boolean to T, and optionally sets different
+	## values in the Known::service_store_backend_options record.
+
+	## Backend to use for storing known services data using the storage framework.
+	global service_store_backend: opaque of Storage::BackendHandle;
+
+	## The name to use for :zeek:see:`Known::service_store_backend`. This will be used
+	## by the backends to differentiate tables/keys. This should be alphanumeric so
+	## that it can be used as the table name for the storage framework.
+	const service_store_prefix = "zeekknownservices" &redef;
+
+	## The type of storage backend to open.
+	const service_store_backend_type : Storage::Backend = Storage::STORAGE_BACKEND_SQLITE &redef;
+
+	## The options for the service store. This should be redef'd in local.zeek to set
+	## connection information for the backend. The options default to a memory store.
+	const service_store_backend_options : Storage::BackendOptions = [ $sqlite = [
+		$database_path=":memory:", $table_name=Known::service_store_name ]] &redef;
+
+	## The expiry interval of new entries in :zeek:see:`Known::service_broker_store`
+	## and :zeek:see:`Known::service_store_backend`.  This also changes the interval
+	## at which services get logged.
 	const service_store_expiry = 1day &redef;
 
 	## The timeout interval to use for operations against
-	## :zeek:see:`Known::service_store`.
+	## :zeek:see:`Known::service_broker_store` and
+	## :zeek:see:`Known::service_store_backend`.
 	option service_store_timeout = 15sec;
 
 	## Tracks the set of daily-detected services for preventing the logging
@@ -109,7 +144,16 @@ event zeek_init()
 	if ( ! Known::use_service_store )
 		return;
 
-	Known::service_store = Cluster::create_store(Known::service_store_name);
+	if ( Known::use_storage_framework )
+		{
+		local res = Storage::Sync::open_backend(Known::service_store_backend_type, Known::service_store_backend_options, Known::AddrPortServTriplet, bool);
+		if ( res$code == Storage::SUCCESS )
+			Known::service_store_backend = res$value;
+		else
+			Reporter::error(fmt("%s: Failed to open backend connection: %s", Known::service_store_prefix, res$error_str));
+		}
+	else
+		Known::service_broker_store = Cluster::create_store(Known::service_store_name);
 	}
 
 event service_info_commit(info: ServicesInfo)
@@ -123,23 +167,45 @@ event service_info_commit(info: ServicesInfo)
 		{
 		local key = AddrPortServTriplet($host = info$host, $p = info$port_num, $serv = s);
 
-		when [info, s, key] ( local r = Broker::put_unique(Known::service_store$store, key,
+		if ( Known::use_storage_framework )
-		                                    T, Known::service_store_expiry) )
+		{
-			{
+			when [info, s, key] ( local put_res = Storage::Async::put(Known::service_store_backend, [$key=key, $value=T, $overwrite=F,
-			if ( r$status == Broker::SUCCESS )
+			                                                    $expire_time=Known::service_store_expiry]) )
 				{
-				if ( r$result as bool ) {
+				if ( put_res$code == Storage::SUCCESS )
+					{
 					info$service = set(s);	# log one service at the time if multiservice
 					Log::write(Known::SERVICES_LOG, info);
 					}
+				else if ( put_res$code != Storage::KEY_EXISTS )
+					Reporter::error(fmt("%s: data store put_unique failure: %s",
+					                    Known::service_store_name, put_res$error_str));
+				}
+			timeout Known::service_store_timeout
+				{
+				Log::write(Known::SERVICES_LOG, info);
 				}
-			else
-				Reporter::error(fmt("%s: data store put_unique failure",
-				                    Known::service_store_name));
 			}
-		timeout Known::service_store_timeout
+		else
 			{
-			Log::write(Known::SERVICES_LOG, info);
+			when [info, s, key] ( local r = Broker::put_unique(Known::service_broker_store$store, key,
+			                                    T, Known::service_store_expiry) )
+				{
+				if ( r$status == Broker::SUCCESS )
+					{
+					if ( r$result as bool ) {
+						info$service = set(s);	# log one service at the time if multiservice
+						Log::write(Known::SERVICES_LOG, info);
+						}
+					}
+				else
+					Reporter::error(fmt("%s: data store put_unique failure",
+					                    Known::service_store_name));
+				}
+			timeout Known::service_store_timeout
+				{
+				Log::write(Known::SERVICES_LOG, info);
+				}
 			}
 		}
 	}
@@ -155,7 +221,7 @@ event known_service_add(info: ServicesInfo)
 	if ( [info$host, info$port_num] !in Known::services )
 		Known::services[info$host, info$port_num] = set();
 
-	 # service to log can be a subset of info$service if some were already seen
+	# service to log can be a subset of info$service if some were already seen
 	local info_to_log: ServicesInfo;
 	info_to_log$ts = info$ts;
 	info_to_log$host = info$host;

testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-broker-store.log

@@ -0,0 +1,17 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	known_services
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	host	port_num	port_proto	service
+#types	time	addr	port	enum	set[string]
+XXXXXXXXXX.XXXXXX	172.16.238.131	22	tcp	SSH
+XXXXXXXXXX.XXXXXX	172.16.238.131	80	tcp	HTTP
+XXXXXXXXXX.XXXXXX	172.16.238.2	53	udp	DNS
+XXXXXXXXXX.XXXXXX	74.125.225.81	80	tcp	HTTP
+XXXXXXXXXX.XXXXXX	172.16.238.131	21	tcp	FTP
+XXXXXXXXXX.XXXXXX	141.142.192.39	22	tcp	SSH
+XXXXXXXXXX.XXXXXX	69.50.219.51	123	udp	NTP
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-storage-framework.log

@@ -0,0 +1,17 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	known_services
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	host	port_num	port_proto	service
+#types	time	addr	port	enum	set[string]
+XXXXXXXXXX.XXXXXX	172.16.238.131	22	tcp	SSH
+XXXXXXXXXX.XXXXXX	172.16.238.131	80	tcp	HTTP
+XXXXXXXXXX.XXXXXX	172.16.238.2	53	udp	DNS
+XXXXXXXXXX.XXXXXX	74.125.225.81	80	tcp	HTTP
+XXXXXXXXXX.XXXXXX	172.16.238.131	21	tcp	FTP
+XXXXXXXXXX.XXXXXX	141.142.192.39	22	tcp	SSH
+XXXXXXXXXX.XXXXXX	69.50.219.51	123	udp	NTP
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.policy.protocols.conn.known-services/logs-diff.txt

@@ -0,0 +1 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

testing/btest/scripts/policy/protocols/conn/known-services.zeek

@@ -1,4 +1,4 @@
-# A basic test of the known-services script's logging and asset_tracking options
+# @TEST-DOC: A basic test of the known-services script's logging and asset_tracking options
 
 # Don't run for C++ scripts because there's no script to compile.
 # @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1"
@@ -18,6 +18,41 @@
 # @TEST-EXEC: zeek -r $TRACES/var-services-std-ports.trace %INPUT Known::service_tracking=NO_HOSTS
 # @TEST-EXEC: test '!' -e known_services.log
 
+# @TEST-EXEC: zeek -r $TRACES/var-services-std-ports.trace %INPUT broker-store-config.zeek
+# @TEST-EXEC: mv known_services.log knownservices-broker-store.log
+# @TEST-EXEC: btest-diff knownservices-broker-store.log
+
+# @TEST-EXEC: zeek -r $TRACES/var-services-std-ports.trace %INPUT storage-framework-config.zeek
+# @TEST-EXEC: mv known_services.log knownservices-storage-framework.log
+# @TEST-EXEC: btest-diff knownservices-storage-framework.log
+
+# @TEST-EXEC: cat knownservices-broker-store.log | $SCRIPTS/diff-remove-timestamps > broker-store.log
+# @TEST-EXEC: cat knownservices-storage-framework.log | $SCRIPTS/diff-remove-timestamps > storage-framework.log
+# @TEST-EXEC: diff broker-store.log storage-framework.log > logs-diff.txt
+# @TEST-EXEC: btest-diff logs-diff.txt
+
 @load protocols/conn/known-services
 
 redef Site::local_nets += {172.16.238.0/24};
+
+# @TEST-START-FILE broker-store-config.zeek
+
+redef Known::service_tracking=ALL_HOSTS;
+redef Known::use_service_store=T;
+redef Known::use_storage_framework=F;
+
+# @TEST-END-FILE
+
+# @TEST-START-FILE storage-framework-config.zeek
+
+@load policy/frameworks/storage/backend/sqlite
+
+redef Known::service_tracking=ALL_HOSTS;
+redef Known::use_service_store=T;
+redef Known::use_storage_framework=T;
+
+redef Known::service_store_backend_type = Storage::STORAGE_BACKEND_SQLITE;
+redef Known::service_store_backend_options = [ $sqlite = [
+    $database_path="test.sqlite", $table_name=Known::service_store_prefix ]];
+
+# @TEST-END-FILE