diff --git a/scripts/policy/protocols/conn/known-services.zeek b/scripts/policy/protocols/conn/known-services.zeek index e868868f70..e8d6e8b5b3 100644 --- a/scripts/policy/protocols/conn/known-services.zeek +++ b/scripts/policy/protocols/conn/known-services.zeek @@ -9,6 +9,10 @@ @load base/utils/directions-and-hosts @load base/frameworks/cluster +@load base/frameworks/storage/async +@load base/frameworks/storage/sync +@load policy/frameworks/storage/backend/sqlite + module Known; export { @@ -39,6 +43,11 @@ export { ## operation. const use_service_store = F &redef; + ## Switches to the version of this script that uses the storage + ## framework instead of Broker stores. This will default to ``T`` + ## in v8.1. + const use_storage_framework = F &redef; + ## Require UDP server to respond before considering it an "active service". option service_udp_requires_response = T; @@ -52,20 +61,46 @@ export { serv: string; }; + ## Storage configuration for Broker stores + ## Holds the set of all known services. Keys in the store are ## :zeek:type:`Known::AddrPortServTriplet` and their associated value is ## always the boolean value of "true". - global service_store: Cluster::StoreInfo; + global service_broker_store: Cluster::StoreInfo; - ## The Broker topic name to use for :zeek:see:`Known::service_store`. + ## The Broker topic name to use for :zeek:see:`Known::service_broker_store`. const service_store_name = "zeek/known/services" &redef; - ## The expiry interval of new entries in :zeek:see:`Known::service_store`. - ## This also changes the interval at which services get logged. + ## Storage configuration for storage framework stores + + ## This requires setting a configuration in local.zeek that sets the + ## Known::use_storage_framework boolean to T, and optionally sets different + ## values in the Known::service_store_backend_options record. + + ## Backend to use for storing known services data using the storage framework. + global service_store_backend: opaque of Storage::BackendHandle; + + ## The name to use for :zeek:see:`Known::service_store_backend`. This will be used + ## by the backends to differentiate tables/keys. This should be alphanumeric so + ## that it can be used as the table name for the storage framework. + const service_store_prefix = "zeekknownservices" &redef; + + ## The type of storage backend to open. + const service_store_backend_type : Storage::Backend = Storage::STORAGE_BACKEND_SQLITE &redef; + + ## The options for the service store. This should be redef'd in local.zeek to set + ## connection information for the backend. The options default to a memory store. + const service_store_backend_options : Storage::BackendOptions = [ $sqlite = [ + $database_path=":memory:", $table_name=Known::service_store_name ]] &redef; + + ## The expiry interval of new entries in :zeek:see:`Known::service_broker_store` + ## and :zeek:see:`Known::service_store_backend`. This also changes the interval + ## at which services get logged. const service_store_expiry = 1day &redef; ## The timeout interval to use for operations against - ## :zeek:see:`Known::service_store`. + ## :zeek:see:`Known::service_broker_store` and + ## :zeek:see:`Known::service_store_backend`. option service_store_timeout = 15sec; ## Tracks the set of daily-detected services for preventing the logging @@ -109,7 +144,16 @@ event zeek_init() if ( ! Known::use_service_store ) return; - Known::service_store = Cluster::create_store(Known::service_store_name); + if ( Known::use_storage_framework ) + { + local res = Storage::Sync::open_backend(Known::service_store_backend_type, Known::service_store_backend_options, Known::AddrPortServTriplet, bool); + if ( res$code == Storage::SUCCESS ) + Known::service_store_backend = res$value; + else + Reporter::error(fmt("%s: Failed to open backend connection: %s", Known::service_store_prefix, res$error_str)); + } + else + Known::service_broker_store = Cluster::create_store(Known::service_store_name); } event service_info_commit(info: ServicesInfo) @@ -123,23 +167,45 @@ event service_info_commit(info: ServicesInfo) { local key = AddrPortServTriplet($host = info$host, $p = info$port_num, $serv = s); - when [info, s, key] ( local r = Broker::put_unique(Known::service_store$store, key, - T, Known::service_store_expiry) ) - { - if ( r$status == Broker::SUCCESS ) + if ( Known::use_storage_framework ) + { + when [info, s, key] ( local put_res = Storage::Async::put(Known::service_store_backend, [$key=key, $value=T, $overwrite=F, + $expire_time=Known::service_store_expiry]) ) { - if ( r$result as bool ) { + if ( put_res$code == Storage::SUCCESS ) + { info$service = set(s); # log one service at the time if multiservice Log::write(Known::SERVICES_LOG, info); } + else if ( put_res$code != Storage::KEY_EXISTS ) + Reporter::error(fmt("%s: data store put_unique failure: %s", + Known::service_store_name, put_res$error_str)); + } + timeout Known::service_store_timeout + { + Log::write(Known::SERVICES_LOG, info); } - else - Reporter::error(fmt("%s: data store put_unique failure", - Known::service_store_name)); } - timeout Known::service_store_timeout + else { - Log::write(Known::SERVICES_LOG, info); + when [info, s, key] ( local r = Broker::put_unique(Known::service_broker_store$store, key, + T, Known::service_store_expiry) ) + { + if ( r$status == Broker::SUCCESS ) + { + if ( r$result as bool ) { + info$service = set(s); # log one service at the time if multiservice + Log::write(Known::SERVICES_LOG, info); + } + } + else + Reporter::error(fmt("%s: data store put_unique failure", + Known::service_store_name)); + } + timeout Known::service_store_timeout + { + Log::write(Known::SERVICES_LOG, info); + } } } } @@ -155,7 +221,7 @@ event known_service_add(info: ServicesInfo) if ( [info$host, info$port_num] !in Known::services ) Known::services[info$host, info$port_num] = set(); - # service to log can be a subset of info$service if some were already seen + # service to log can be a subset of info$service if some were already seen local info_to_log: ServicesInfo; info_to_log$ts = info$ts; info_to_log$host = info$host; diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-broker-store.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-broker-store.log new file mode 100644 index 0000000000..7827f4ef11 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-broker-store.log @@ -0,0 +1,17 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path known_services +#open XXXX-XX-XX-XX-XX-XX +#fields ts host port_num port_proto service +#types time addr port enum set[string] +XXXXXXXXXX.XXXXXX 172.16.238.131 22 tcp SSH +XXXXXXXXXX.XXXXXX 172.16.238.131 80 tcp HTTP +XXXXXXXXXX.XXXXXX 172.16.238.2 53 udp DNS +XXXXXXXXXX.XXXXXX 74.125.225.81 80 tcp HTTP +XXXXXXXXXX.XXXXXX 172.16.238.131 21 tcp FTP +XXXXXXXXXX.XXXXXX 141.142.192.39 22 tcp SSH +XXXXXXXXXX.XXXXXX 69.50.219.51 123 udp NTP +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-storage-framework.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-storage-framework.log new file mode 100644 index 0000000000..7827f4ef11 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-storage-framework.log @@ -0,0 +1,17 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path known_services +#open XXXX-XX-XX-XX-XX-XX +#fields ts host port_num port_proto service +#types time addr port enum set[string] +XXXXXXXXXX.XXXXXX 172.16.238.131 22 tcp SSH +XXXXXXXXXX.XXXXXX 172.16.238.131 80 tcp HTTP +XXXXXXXXXX.XXXXXX 172.16.238.2 53 udp DNS +XXXXXXXXXX.XXXXXX 74.125.225.81 80 tcp HTTP +XXXXXXXXXX.XXXXXX 172.16.238.131 21 tcp FTP +XXXXXXXXXX.XXXXXX 141.142.192.39 22 tcp SSH +XXXXXXXXXX.XXXXXX 69.50.219.51 123 udp NTP +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/logs-diff.txt b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/logs-diff.txt new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/logs-diff.txt @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/scripts/policy/protocols/conn/known-services.zeek b/testing/btest/scripts/policy/protocols/conn/known-services.zeek index 5a621cbfcd..168ea08807 100644 --- a/testing/btest/scripts/policy/protocols/conn/known-services.zeek +++ b/testing/btest/scripts/policy/protocols/conn/known-services.zeek @@ -1,4 +1,4 @@ -# A basic test of the known-services script's logging and asset_tracking options +# @TEST-DOC: A basic test of the known-services script's logging and asset_tracking options # Don't run for C++ scripts because there's no script to compile. # @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1" @@ -18,6 +18,41 @@ # @TEST-EXEC: zeek -r $TRACES/var-services-std-ports.trace %INPUT Known::service_tracking=NO_HOSTS # @TEST-EXEC: test '!' -e known_services.log +# @TEST-EXEC: zeek -r $TRACES/var-services-std-ports.trace %INPUT broker-store-config.zeek +# @TEST-EXEC: mv known_services.log knownservices-broker-store.log +# @TEST-EXEC: btest-diff knownservices-broker-store.log + +# @TEST-EXEC: zeek -r $TRACES/var-services-std-ports.trace %INPUT storage-framework-config.zeek +# @TEST-EXEC: mv known_services.log knownservices-storage-framework.log +# @TEST-EXEC: btest-diff knownservices-storage-framework.log + +# @TEST-EXEC: cat knownservices-broker-store.log | $SCRIPTS/diff-remove-timestamps > broker-store.log +# @TEST-EXEC: cat knownservices-storage-framework.log | $SCRIPTS/diff-remove-timestamps > storage-framework.log +# @TEST-EXEC: diff broker-store.log storage-framework.log > logs-diff.txt +# @TEST-EXEC: btest-diff logs-diff.txt + @load protocols/conn/known-services redef Site::local_nets += {172.16.238.0/24}; + +# @TEST-START-FILE broker-store-config.zeek + +redef Known::service_tracking=ALL_HOSTS; +redef Known::use_service_store=T; +redef Known::use_storage_framework=F; + +# @TEST-END-FILE + +# @TEST-START-FILE storage-framework-config.zeek + +@load policy/frameworks/storage/backend/sqlite + +redef Known::service_tracking=ALL_HOSTS; +redef Known::use_service_store=T; +redef Known::use_storage_framework=T; + +redef Known::service_store_backend_type = Storage::STORAGE_BACKEND_SQLITE; +redef Known::service_store_backend_options = [ $sqlite = [ + $database_path="test.sqlite", $table_name=Known::service_store_prefix ]]; + +# @TEST-END-FILE