mirror of
https://github.com/zeek/zeek.git
synced 2025-10-02 06:38:20 +00:00
Modify known-services policy script to add storage framework support
This adds a flag to enable using the storage framework instead of Broker stores, plus a btest for both broker and the the storage framework.
This commit is contained in:
Tim Wojtulewicz
parent
6446bdf0dc
commit
1f64bb2870
5 changed files with 154 additions and 18 deletions
|
|
@ -9,6 +9,10 @@
|
|||
@load base/utils/directions-and-hosts
|
||||
@load base/frameworks/cluster
|
||||
|
||||
@load base/frameworks/storage/async
|
||||
@load base/frameworks/storage/sync
|
||||
@load policy/frameworks/storage/backend/sqlite
|
||||
|
||||
module Known;
|
||||
|
||||
export {
|
||||
|
|
@ -39,6 +43,11 @@ export {
|
|||
## operation.
|
||||
const use_service_store = F &redef;
|
||||
|
||||
## Switches to the version of this script that uses the storage
|
||||
## framework instead of Broker stores. This will default to ``T``
|
||||
## in v8.1.
|
||||
const use_storage_framework = F &redef;
|
||||
|
||||
## Require UDP server to respond before considering it an "active service".
|
||||
option service_udp_requires_response = T;
|
||||
|
||||
|
|
@ -52,20 +61,46 @@ export {
|
|||
serv: string;
|
||||
};
|
||||
|
||||
## Storage configuration for Broker stores
|
||||
|
||||
## Holds the set of all known services. Keys in the store are
|
||||
## :zeek:type:`Known::AddrPortServTriplet` and their associated value is
|
||||
## always the boolean value of "true".
|
||||
global service_store: Cluster::StoreInfo;
|
||||
global service_broker_store: Cluster::StoreInfo;
|
||||
|
||||
## The Broker topic name to use for :zeek:see:`Known::service_store`.
|
||||
## The Broker topic name to use for :zeek:see:`Known::service_broker_store`.
|
||||
const service_store_name = "zeek/known/services" &redef;
|
||||
|
||||
## The expiry interval of new entries in :zeek:see:`Known::service_store`.
|
||||
## This also changes the interval at which services get logged.
|
||||
## Storage configuration for storage framework stores
|
||||
|
||||
## This requires setting a configuration in local.zeek that sets the
|
||||
## Known::use_storage_framework boolean to T, and optionally sets different
|
||||
## values in the Known::service_store_backend_options record.
|
||||
|
||||
## Backend to use for storing known services data using the storage framework.
|
||||
global service_store_backend: opaque of Storage::BackendHandle;
|
||||
|
||||
## The name to use for :zeek:see:`Known::service_store_backend`. This will be used
|
||||
## by the backends to differentiate tables/keys. This should be alphanumeric so
|
||||
## that it can be used as the table name for the storage framework.
|
||||
const service_store_prefix = "zeekknownservices" &redef;
|
||||
|
||||
## The type of storage backend to open.
|
||||
const service_store_backend_type : Storage::Backend = Storage::STORAGE_BACKEND_SQLITE &redef;
|
||||
|
||||
## The options for the service store. This should be redef'd in local.zeek to set
|
||||
## connection information for the backend. The options default to a memory store.
|
||||
const service_store_backend_options : Storage::BackendOptions = [ $sqlite = [
|
||||
$database_path=":memory:", $table_name=Known::service_store_name ]] &redef;
|
||||
|
||||
## The expiry interval of new entries in :zeek:see:`Known::service_broker_store`
|
||||
## and :zeek:see:`Known::service_store_backend`. This also changes the interval
|
||||
## at which services get logged.
|
||||
const service_store_expiry = 1day &redef;
|
||||
|
||||
## The timeout interval to use for operations against
|
||||
## :zeek:see:`Known::service_store`.
|
||||
## :zeek:see:`Known::service_broker_store` and
|
||||
## :zeek:see:`Known::service_store_backend`.
|
||||
option service_store_timeout = 15sec;
|
||||
|
||||
## Tracks the set of daily-detected services for preventing the logging
|
||||
|
|
@ -109,7 +144,16 @@ event zeek_init()
|
|||
if ( ! Known::use_service_store )
|
||||
return;
|
||||
|
||||
Known::service_store = Cluster::create_store(Known::service_store_name);
|
||||
if ( Known::use_storage_framework )
|
||||
{
|
||||
local res = Storage::Sync::open_backend(Known::service_store_backend_type, Known::service_store_backend_options, Known::AddrPortServTriplet, bool);
|
||||
if ( res$code == Storage::SUCCESS )
|
||||
Known::service_store_backend = res$value;
|
||||
else
|
||||
Reporter::error(fmt("%s: Failed to open backend connection: %s", Known::service_store_prefix, res$error_str));
|
||||
}
|
||||
else
|
||||
Known::service_broker_store = Cluster::create_store(Known::service_store_name);
|
||||
}
|
||||
|
||||
event service_info_commit(info: ServicesInfo)
|
||||
|
|
@ -123,7 +167,28 @@ event service_info_commit(info: ServicesInfo)
|
|||
{
|
||||
local key = AddrPortServTriplet($host = info$host, $p = info$port_num, $serv = s);
|
||||
|
||||
when [info, s, key] ( local r = Broker::put_unique(Known::service_store$store, key,
|
||||
if ( Known::use_storage_framework )
|
||||
{
|
||||
when [info, s, key] ( local put_res = Storage::Async::put(Known::service_store_backend, [$key=key, $value=T, $overwrite=F,
|
||||
$expire_time=Known::service_store_expiry]) )
|
||||
{
|
||||
if ( put_res$code == Storage::SUCCESS )
|
||||
{
|
||||
info$service = set(s); # log one service at the time if multiservice
|
||||
Log::write(Known::SERVICES_LOG, info);
|
||||
}
|
||||
else if ( put_res$code != Storage::KEY_EXISTS )
|
||||
Reporter::error(fmt("%s: data store put_unique failure: %s",
|
||||
Known::service_store_name, put_res$error_str));
|
||||
}
|
||||
timeout Known::service_store_timeout
|
||||
{
|
||||
Log::write(Known::SERVICES_LOG, info);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
when [info, s, key] ( local r = Broker::put_unique(Known::service_broker_store$store, key,
|
||||
T, Known::service_store_expiry) )
|
||||
{
|
||||
if ( r$status == Broker::SUCCESS )
|
||||
|
|
@ -143,6 +208,7 @@ event service_info_commit(info: ServicesInfo)
|
|||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
event known_service_add(info: ServicesInfo)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
#separator \x09
|
||||
#set_separator ,
|
||||
#empty_field (empty)
|
||||
#unset_field -
|
||||
#path known_services
|
||||
#open XXXX-XX-XX-XX-XX-XX
|
||||
#fields ts host port_num port_proto service
|
||||
#types time addr port enum set[string]
|
||||
XXXXXXXXXX.XXXXXX 172.16.238.131 22 tcp SSH
|
||||
XXXXXXXXXX.XXXXXX 172.16.238.131 80 tcp HTTP
|
||||
XXXXXXXXXX.XXXXXX 172.16.238.2 53 udp DNS
|
||||
XXXXXXXXXX.XXXXXX 74.125.225.81 80 tcp HTTP
|
||||
XXXXXXXXXX.XXXXXX 172.16.238.131 21 tcp FTP
|
||||
XXXXXXXXXX.XXXXXX 141.142.192.39 22 tcp SSH
|
||||
XXXXXXXXXX.XXXXXX 69.50.219.51 123 udp NTP
|
||||
#close XXXX-XX-XX-XX-XX-XX
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
#separator \x09
|
||||
#set_separator ,
|
||||
#empty_field (empty)
|
||||
#unset_field -
|
||||
#path known_services
|
||||
#open XXXX-XX-XX-XX-XX-XX
|
||||
#fields ts host port_num port_proto service
|
||||
#types time addr port enum set[string]
|
||||
XXXXXXXXXX.XXXXXX 172.16.238.131 22 tcp SSH
|
||||
XXXXXXXXXX.XXXXXX 172.16.238.131 80 tcp HTTP
|
||||
XXXXXXXXXX.XXXXXX 172.16.238.2 53 udp DNS
|
||||
XXXXXXXXXX.XXXXXX 74.125.225.81 80 tcp HTTP
|
||||
XXXXXXXXXX.XXXXXX 172.16.238.131 21 tcp FTP
|
||||
XXXXXXXXXX.XXXXXX 141.142.192.39 22 tcp SSH
|
||||
XXXXXXXXXX.XXXXXX 69.50.219.51 123 udp NTP
|
||||
#close XXXX-XX-XX-XX-XX-XX
|
||||
|
|
@ -0,0 +1 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
# A basic test of the known-services script's logging and asset_tracking options
|
||||
# @TEST-DOC: A basic test of the known-services script's logging and asset_tracking options
|
||||
|
||||
# Don't run for C++ scripts because there's no script to compile.
|
||||
# @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1"
|
||||
|
|
@ -18,6 +18,41 @@
|
|||
# @TEST-EXEC: zeek -r $TRACES/var-services-std-ports.trace %INPUT Known::service_tracking=NO_HOSTS
|
||||
# @TEST-EXEC: test '!' -e known_services.log
|
||||
|
||||
# @TEST-EXEC: zeek -r $TRACES/var-services-std-ports.trace %INPUT broker-store-config.zeek
|
||||
# @TEST-EXEC: mv known_services.log knownservices-broker-store.log
|
||||
# @TEST-EXEC: btest-diff knownservices-broker-store.log
|
||||
|
||||
# @TEST-EXEC: zeek -r $TRACES/var-services-std-ports.trace %INPUT storage-framework-config.zeek
|
||||
# @TEST-EXEC: mv known_services.log knownservices-storage-framework.log
|
||||
# @TEST-EXEC: btest-diff knownservices-storage-framework.log
|
||||
|
||||
# @TEST-EXEC: cat knownservices-broker-store.log | $SCRIPTS/diff-remove-timestamps > broker-store.log
|
||||
# @TEST-EXEC: cat knownservices-storage-framework.log | $SCRIPTS/diff-remove-timestamps > storage-framework.log
|
||||
# @TEST-EXEC: diff broker-store.log storage-framework.log > logs-diff.txt
|
||||
# @TEST-EXEC: btest-diff logs-diff.txt
|
||||
|
||||
@load protocols/conn/known-services
|
||||
|
||||
redef Site::local_nets += {172.16.238.0/24};
|
||||
|
||||
# @TEST-START-FILE broker-store-config.zeek
|
||||
|
||||
redef Known::service_tracking=ALL_HOSTS;
|
||||
redef Known::use_service_store=T;
|
||||
redef Known::use_storage_framework=F;
|
||||
|
||||
# @TEST-END-FILE
|
||||
|
||||
# @TEST-START-FILE storage-framework-config.zeek
|
||||
|
||||
@load policy/frameworks/storage/backend/sqlite
|
||||
|
||||
redef Known::service_tracking=ALL_HOSTS;
|
||||
redef Known::use_service_store=T;
|
||||
redef Known::use_storage_framework=T;
|
||||
|
||||
redef Known::service_store_backend_type = Storage::STORAGE_BACKEND_SQLITE;
|
||||
redef Known::service_store_backend_options = [ $sqlite = [
|
||||
$database_path="test.sqlite", $table_name=Known::service_store_prefix ]];
|
||||
|
||||
# @TEST-END-FILE
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue