mirror of
https://github.com/zeek/zeek.git
synced 2025-10-02 06:38:20 +00:00
intel: Add indicator_inserted and indicator_removed hooks
This change adds two new hooks to the Intel framework that can be used to intercept added and removed indicators and their type. These hooks are fairly low-level. One immediate use-case is to count the number of indicators loaded per Intel::Type and enable and disable the corresponding event groups of the intel/seen scripts. I attempted to gauge the overhead and while it's definitely there, loading a file with ~500k DOMAIN entries takes somewhere around ~0.5 seconds hooks when populated via the min_data_store store mechanism. While that doesn't sound great, it actually takes the manager on my system 2.5 seconds to serialize and Cluster::publish() the min_data_store alone and its doing that serially for every active worker. Mostly to say that the bigger overhead in that area on the manager doing redundant work per worker. Co-authored-by: Mohan Dhawan <mohan@corelight.com>
This commit is contained in:
Arne Welzel
parent
224519c11a
commit
7eb849ddf4
19 changed files with 611 additions and 1 deletions
10
NEWS
10
NEWS
|
|
@ -66,6 +66,16 @@ New Functionality
|
|||
|
||||
- The FTP analyzer now supports explicit TLS via AUTH TLS.
|
||||
|
||||
- Two new script-level hooks in the Intel framework have been added.
|
||||
|
||||
hook indicator_inserted(indicator_value: string, indicator_type: Intel::Type)
|
||||
|
||||
hook indicator_removed(indicator_value: string, indicator_type: Intel::Type)
|
||||
|
||||
These are reliably invoked on worker and manager nodes the first time an
|
||||
indicator value is inserted into the store and once it has been completely
|
||||
removed from the store.
|
||||
|
||||
Changed Functionality
|
||||
---------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -105,10 +105,30 @@ event Intel::insert_indicator(item: Intel::Item) &priority=5
|
|||
Intel::_insert(item, F);
|
||||
}
|
||||
|
||||
function invoke_indicator_hook(store: MinDataStore, h: hook(v: string, t: Intel::Type))
|
||||
{
|
||||
for ( a in store$host_data )
|
||||
hook h(cat(a), Intel::ADDR);
|
||||
|
||||
for ( sn in store$subnet_data)
|
||||
hook h(cat(sn), Intel::SUBNET);
|
||||
|
||||
for ( [indicator_value, indicator_type] in store$string_data )
|
||||
hook h(indicator_value, indicator_type);
|
||||
}
|
||||
|
||||
# Handling of a complete MinDataStore snapshot
|
||||
#
|
||||
# Invoke the removed and inserted hooks using the old and new min data store
|
||||
# instances, respectively. The way this event is used, the original
|
||||
# min_data_store should essentially be empty.
|
||||
event new_min_data_store(store: MinDataStore)
|
||||
{
|
||||
invoke_indicator_hook(min_data_store, Intel::indicator_removed);
|
||||
|
||||
min_data_store = store;
|
||||
|
||||
invoke_indicator_hook(min_data_store, Intel::indicator_inserted);
|
||||
}
|
||||
@endif
|
||||
|
||||
|
|
|
|||
|
|
@ -207,6 +207,35 @@ export {
|
|||
## item: The intel item that should be inserted.
|
||||
global filter_item: hook(item: Intel::Item);
|
||||
|
||||
## This hook is invoked when a new indicator has been inserted into
|
||||
## the min data store for the first time.
|
||||
##
|
||||
## Calls to :zeek:see:`Intel::seen` with a matching indicator value
|
||||
## and type will result in matches.
|
||||
##
|
||||
## Subsequent inserts of the same indicator type and value do not
|
||||
## invoke this hook. Breaking from this hook has no effect.
|
||||
##
|
||||
## indicator: The indicator value.
|
||||
##
|
||||
## indicator_type: The indicator type.
|
||||
##
|
||||
## .. zeek::see:: Intel::indicator_removed
|
||||
global indicator_inserted: hook(indicator: string, indiator_type: Type);
|
||||
|
||||
## This hook is invoked when an indicator has been removed from
|
||||
## the min data store.
|
||||
##
|
||||
## After this hooks runs, :zeek:see:`Intel::seen` for the indicator
|
||||
## will not return any matches. Breaking from this hook has no effect.
|
||||
##
|
||||
## indicator: The indicator value.
|
||||
##
|
||||
## indicator_type: The indicator type.
|
||||
##
|
||||
## .. zeek::see:: Intel::indicator_inserted
|
||||
global indicator_removed: hook(indicator: string, indiator_type: Type);
|
||||
|
||||
global log_intel: event(rec: Info);
|
||||
}
|
||||
|
||||
|
|
@ -507,18 +536,44 @@ function _insert(item: Item, first_dispatch: bool &default = T)
|
|||
# All intelligence is case insensitive at the moment.
|
||||
local lower_indicator = to_lower(item$indicator);
|
||||
|
||||
# Track if the indicator was inserted into the min_data_store.
|
||||
# It's tempting to just use is_new above, but it seems that only works
|
||||
# correctly on a worker if the manager never spuriously sends a
|
||||
# Intel::insert_item(), so better to determine this locally based
|
||||
# on the actual contents of the min_data_store.
|
||||
local inserted = F;
|
||||
local inserted_value = "";
|
||||
|
||||
# Insert indicator into MinDataStore (might exist already).
|
||||
switch ( item$indicator_type )
|
||||
{
|
||||
case ADDR:
|
||||
local host = to_addr(item$indicator);
|
||||
if ( host !in min_data_store$host_data )
|
||||
{
|
||||
inserted = T;
|
||||
inserted_value = cat(host);
|
||||
}
|
||||
|
||||
add min_data_store$host_data[host];
|
||||
break;
|
||||
case SUBNET:
|
||||
local net = to_subnet(item$indicator);
|
||||
if ( net !in min_data_store$subnet_data )
|
||||
{
|
||||
inserted = T;
|
||||
inserted_value = cat(net);
|
||||
}
|
||||
|
||||
add min_data_store$subnet_data[net];
|
||||
break;
|
||||
default:
|
||||
if ( [lower_indicator, item$indicator_type] !in min_data_store$string_data )
|
||||
{
|
||||
inserted = T;
|
||||
inserted_value = lower_indicator;
|
||||
}
|
||||
|
||||
add min_data_store$string_data[lower_indicator, item$indicator_type];
|
||||
break;
|
||||
}
|
||||
|
|
@ -533,6 +588,9 @@ function _insert(item: Item, first_dispatch: bool &default = T)
|
|||
# Announce a (possibly) new item if this is the first dispatch and
|
||||
# we know it is new or have to assume that on a worker.
|
||||
event Intel::new_item(item);
|
||||
|
||||
if ( inserted )
|
||||
hook Intel::indicator_inserted(inserted_value, item$indicator_type);
|
||||
}
|
||||
|
||||
function insert(item: Item)
|
||||
|
|
@ -632,18 +690,43 @@ function remove(item: Item, purge_indicator: bool)
|
|||
# Handling of indicator removal in minimal data stores.
|
||||
event remove_indicator(item: Item)
|
||||
{
|
||||
local removed = F;
|
||||
local removed_value = "";
|
||||
|
||||
switch ( item$indicator_type )
|
||||
{
|
||||
case ADDR:
|
||||
local host = to_addr(item$indicator);
|
||||
if ( host in min_data_store$host_data )
|
||||
{
|
||||
removed = T;
|
||||
removed_value = cat(host);
|
||||
}
|
||||
|
||||
delete min_data_store$host_data[host];
|
||||
break;
|
||||
case SUBNET:
|
||||
local net = to_subnet(item$indicator);
|
||||
if ( net in min_data_store$subnet_data )
|
||||
{
|
||||
removed = T;
|
||||
removed_value = cat(net);
|
||||
}
|
||||
|
||||
delete min_data_store$subnet_data[net];
|
||||
break;
|
||||
default:
|
||||
delete min_data_store$string_data[to_lower(item$indicator), item$indicator_type];
|
||||
local indicator_value = to_lower(item$indicator);
|
||||
if ( [indicator_value, item$indicator_type] in min_data_store$string_data )
|
||||
{
|
||||
removed = T;
|
||||
removed_value = indicator_value;
|
||||
}
|
||||
|
||||
delete min_data_store$string_data[indicator_value, item$indicator_type];
|
||||
break;
|
||||
}
|
||||
|
||||
if ( removed )
|
||||
hook Intel::indicator_removed(removed_value, item$indicator_type);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
received termination signal
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
|
||||
Intel::indicator_inserted, 192.168.0.0/16, Intel::SUBNET
|
||||
Intel::indicator_inserted, putty, Intel::SOFTWARE
|
||||
Intel::indicator_inserted, putty2, Intel::SOFTWARE
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
Intel::new_min_data_store pre, [host_data={\x0a\x0a}, subnet_data={\x0a\x0a}, string_data={\x0a\x0a}]
|
||||
Intel::new_min_data_store post, [host_data={\x0a\x0a}, subnet_data={\x0a\x0a}, string_data={\x0a\x0a}]
|
||||
Intel::insert_indicator, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
|
||||
Intel::insert_indicator, 1.2.3.5, Intel::ADDR
|
||||
Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
|
||||
Intel::insert_indicator, 192.168.0.0/16, Intel::SUBNET
|
||||
Intel::indicator_inserted, 192.168.0.0/16, Intel::SUBNET
|
||||
Intel::insert_indicator, putty, Intel::SOFTWARE
|
||||
Intel::indicator_inserted, putty, Intel::SOFTWARE
|
||||
Intel::insert_indicator, putty2, Intel::SOFTWARE
|
||||
Intel::indicator_inserted, putty2, Intel::SOFTWARE
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
Intel::new_min_data_store pre, [host_data={\x0a\x0a}, subnet_data={\x0a\x0a}, string_data={\x0a\x0a}]
|
||||
Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
|
||||
Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_inserted, 192.168.0.0/16, Intel::SUBNET
|
||||
Intel::indicator_inserted, putty, Intel::SOFTWARE
|
||||
Intel::indicator_inserted, putty2, Intel::SOFTWARE
|
||||
Intel::new_min_data_store post, [host_data={\x0a\x091.2.3.5,\x0a\x091.2.3.4\x0a}, subnet_data={\x0a\x09192.168.0.0/16\x0a}, string_data={\x0a\x09[putty, Intel::SOFTWARE] ,\x0a\x09[putty2, Intel::SOFTWARE] \x0a}]
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
Intel::new_min_data_store pre, [host_data={\x0a\x0a}, subnet_data={\x0a\x0a}, string_data={\x0a\x0a}]
|
||||
Intel::new_min_data_store post, [host_data={\x0a\x0a}, subnet_data={\x0a\x0a}, string_data={\x0a\x0a}]
|
||||
Intel::insert_indicator, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
|
||||
Intel::insert_indicator, 1.2.3.5, Intel::ADDR
|
||||
Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
|
||||
Intel::insert_indicator, 192.168.0.0/16, Intel::SUBNET
|
||||
Intel::indicator_inserted, 192.168.0.0/16, Intel::SUBNET
|
||||
Intel::insert_indicator, putty, Intel::SOFTWARE
|
||||
Intel::indicator_inserted, putty, Intel::SOFTWARE
|
||||
Intel::insert_indicator, putty2, Intel::SOFTWARE
|
||||
Intel::indicator_inserted, putty2, Intel::SOFTWARE
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
insert_addr, 1.2.3.4, from-manager
|
||||
Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
|
||||
Intel::indicator_removed, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_removed, 1.2.3.5, Intel::ADDR
|
||||
====
|
||||
insert_addr, 1.2.3.4, from-manager
|
||||
Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
|
||||
Intel::indicator_removed, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_removed, 1.2.3.5, Intel::ADDR
|
||||
publish_do_terminate()
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
|
||||
insert_addr, 1.2.3.4, from-worker
|
||||
insert_addr, 1.2.3.5, from-worker
|
||||
Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
|
||||
insert_addr, 1.2.3.5, from-manager
|
||||
Intel::indicator_removed, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_removed, 1.2.3.5, Intel::ADDR
|
||||
====
|
||||
Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
|
||||
insert_addr, 1.2.3.4, from-worker
|
||||
insert_addr, 1.2.3.5, from-worker
|
||||
Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
|
||||
insert_addr, 1.2.3.5, from-manager
|
||||
Intel::indicator_removed, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_removed, 1.2.3.5, Intel::ADDR
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
|
||||
remove_addr, 1.2.3.4, from-manager
|
||||
remove_addr, 1.2.3.5, from-manager
|
||||
remove_addr, 1.2.3.4, from-worker
|
||||
remove_addr, 1.2.3.5, from-worker
|
||||
Intel::indicator_removed, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_removed, 1.2.3.5, Intel::ADDR
|
||||
====
|
||||
Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
|
||||
remove_addr, 1.2.3.4, from-manager
|
||||
remove_addr, 1.2.3.5, from-manager
|
||||
remove_addr, 1.2.3.4, from-worker
|
||||
remove_addr, 1.2.3.5, from-worker
|
||||
Intel::indicator_removed, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_removed, 1.2.3.5, Intel::ADDR
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
insert_software, software-1.2.3.4, from-manager
|
||||
Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE
|
||||
Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE
|
||||
Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE
|
||||
Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE
|
||||
====
|
||||
insert_software, software-1.2.3.4, from-manager
|
||||
Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE
|
||||
Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE
|
||||
Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE
|
||||
Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE
|
||||
publish_do_terminate()
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE
|
||||
insert_software, software-1.2.3.4, from-worker
|
||||
insert_software, software-1.2.3.5, from-worker
|
||||
Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE
|
||||
insert_software, software-1.2.3.5, from-manager
|
||||
Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE
|
||||
Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE
|
||||
====
|
||||
Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE
|
||||
insert_software, software-1.2.3.4, from-worker
|
||||
insert_software, software-1.2.3.5, from-worker
|
||||
Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE
|
||||
insert_software, software-1.2.3.5, from-manager
|
||||
Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE
|
||||
Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE
|
||||
Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE
|
||||
remove_software, software-1.2.3.4, from-manager
|
||||
remove_software, software-1.2.3.5, from-manager
|
||||
remove_software, software-1.2.3.4, from-worker
|
||||
remove_software, software-1.2.3.5, from-worker
|
||||
Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE
|
||||
Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE
|
||||
====
|
||||
Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE
|
||||
Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE
|
||||
remove_software, software-1.2.3.4, from-manager
|
||||
remove_software, software-1.2.3.5, from-manager
|
||||
remove_software, software-1.2.3.4, from-worker
|
||||
remove_software, software-1.2.3.5, from-worker
|
||||
Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE
|
||||
Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
insert_subnet, 1.2.3.4/32, from-manager
|
||||
Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET
|
||||
Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET
|
||||
Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET
|
||||
Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET
|
||||
====
|
||||
insert_subnet, 1.2.3.4/32, from-manager
|
||||
Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET
|
||||
Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET
|
||||
Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET
|
||||
Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET
|
||||
publish_do_terminate()
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET
|
||||
insert_subnet, 1.2.3.4/32, from-worker
|
||||
insert_subnet, 1.2.3.5/32, from-worker
|
||||
Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET
|
||||
insert_subnet, 1.2.3.5/32, from-manager
|
||||
Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET
|
||||
Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET
|
||||
====
|
||||
Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET
|
||||
insert_subnet, 1.2.3.4/32, from-worker
|
||||
insert_subnet, 1.2.3.5/32, from-worker
|
||||
Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET
|
||||
insert_subnet, 1.2.3.5/32, from-manager
|
||||
Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET
|
||||
Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET
|
||||
Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET
|
||||
remove_subnet, 1.2.3.4/32, from-manager
|
||||
remove_subnet, 1.2.3.5/32, from-manager
|
||||
remove_subnet, 1.2.3.4/32, from-worker
|
||||
remove_subnet, 1.2.3.5/32, from-worker
|
||||
Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET
|
||||
Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET
|
||||
====
|
||||
Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET
|
||||
Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET
|
||||
remove_subnet, 1.2.3.4/32, from-manager
|
||||
remove_subnet, 1.2.3.5/32, from-manager
|
||||
remove_subnet, 1.2.3.4/32, from-worker
|
||||
remove_subnet, 1.2.3.5/32, from-worker
|
||||
Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET
|
||||
Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET
|
||||
|
|
@ -0,0 +1,121 @@
|
|||
# @TEST-DOC: Verify behavior of Intel:indicator_inserted() and Intel::indicator_removed() when a worker node restarts.
|
||||
#
|
||||
# @TEST-PORT: BROKER_MANAGER_PORT
|
||||
# @TEST-PORT: BROKER_WORKER1_PORT
|
||||
# @TEST-PORT: BROKER_WORKER2_PORT
|
||||
#
|
||||
# @TEST-EXEC: cp $FILES/broker/cluster-layout.zeek .
|
||||
#
|
||||
# @TEST-EXEC: zeek --parse-only %INPUT
|
||||
# @TEST-EXEC: btest-bg-run manager ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager zeek -b %INPUT
|
||||
# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b %INPUT
|
||||
# @TEST-EXEC: btest-bg-run worker-2 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b %INPUT
|
||||
# @TEST-EXEC: wait-for-file worker-1/DONE 30
|
||||
# @TEST-EXEC: mv worker-1 worker-1-run-1
|
||||
# @TEST-EXEC: rm -rf worker-1
|
||||
# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b %INPUT
|
||||
# @TEST-EXEC: wait-for-file worker-1/DONE 30
|
||||
# @TEST-EXEC: btest-bg-wait 30
|
||||
# @TEST-EXEC: mv worker-1 worker-1-run-2
|
||||
|
||||
# @TEST-EXEC: btest-diff manager/.stdout
|
||||
# @TEST-EXEC: btest-diff manager/.stderr
|
||||
# @TEST-EXEC: btest-diff worker-1-run-1/.stdout
|
||||
# @TEST-EXEC: btest-diff worker-1-run-2/.stdout
|
||||
# @TEST-EXEC: btest-diff worker-2/.stdout
|
||||
#
|
||||
@load policy/frameworks/cluster/experimental
|
||||
@load base/frameworks/intel
|
||||
|
||||
# Ttoal of 5 unique indicators
|
||||
# @TEST-START-FILE intel.dat
|
||||
#fields indicator indicator_type meta.source meta.desc meta.url
|
||||
1.2.3.4 Intel::ADDR source1 this host is bad http://some-data-distributor.com/1
|
||||
1.2.3.4 Intel::ADDR source2 this host is bad http://some-data-distributor.com/1
|
||||
1.2.3.5 Intel::ADDR source1 this host is bad http://some-data-distributor.com/1
|
||||
192.168.0.0/16 Intel::SUBNET source1 this network is bad http://some-data-distributor.com/2
|
||||
putty Intel::SOFTWARE source1 this software is bad http://some-data-distributor.com/2
|
||||
putty Intel::SOFTWARE source2 this software is bad http://some-data-distributor.com/2
|
||||
putty2 Intel::SOFTWARE source3 this software is bad http://some-data-distributor.com/2
|
||||
# @TEST-END-FILE
|
||||
|
||||
redef Log::default_rotation_interval = 0sec;
|
||||
|
||||
event Cluster::Experimental::cluster_started()
|
||||
{
|
||||
# Start reading the intel file on the manager once all workers are up.
|
||||
if ( Cluster::node == "manager" )
|
||||
{
|
||||
local source = "../intel.dat";
|
||||
Input::add_event([$source=source,
|
||||
$reader=Input::READER_ASCII,
|
||||
$mode=Input::REREAD,
|
||||
$name=cat("intel-", source),
|
||||
$fields=Intel::Item,
|
||||
$ev=Intel::read_entry,
|
||||
$error_ev=Intel::read_error]);
|
||||
}
|
||||
}
|
||||
|
||||
# Send by manager for termination purposes.
|
||||
event do_terminate()
|
||||
{
|
||||
terminate();
|
||||
}
|
||||
|
||||
global worker1_down = 0;
|
||||
global nodes_down = 0;
|
||||
|
||||
event Cluster::node_down(name: string, id: string)
|
||||
{
|
||||
++nodes_down;
|
||||
if ( name == "worker-1")
|
||||
++worker1_down;
|
||||
|
||||
if ( worker1_down == 2 )
|
||||
Cluster::publish(Cluster::worker_topic, do_terminate);
|
||||
|
||||
if ( nodes_down >= 3 )
|
||||
terminate();
|
||||
}
|
||||
|
||||
global total_indicators = 0;
|
||||
|
||||
hook Intel::indicator_inserted(indicator: string, indicator_type: Intel::Type)
|
||||
{
|
||||
print "Intel::indicator_inserted", indicator, indicator_type;
|
||||
++total_indicators;
|
||||
|
||||
# Once worker-1 has seen all the 5 indicators, write a DONE file and terminate()
|
||||
if ( Cluster::node == "worker-1" && total_indicators == 5 )
|
||||
{
|
||||
if ( ! piped_exec("touch DONE", "") )
|
||||
exit(1);
|
||||
|
||||
terminate();
|
||||
}
|
||||
}
|
||||
|
||||
hook Intel::indicator_removed(indicator: string, indicator_type: Intel::Type)
|
||||
{
|
||||
print "Intel::indicator_removed", indicator, indicator_type;
|
||||
}
|
||||
|
||||
module Intel;
|
||||
|
||||
# Internal events for easier grasping behavior.
|
||||
|
||||
event Intel::insert_indicator(item: Intel::Item) &priority=10
|
||||
{
|
||||
print "Intel::insert_indicator", item$indicator, item$indicator_type;
|
||||
}
|
||||
|
||||
event Intel::new_min_data_store(store: Intel::MinDataStore) &priority=10
|
||||
{
|
||||
print "Intel::new_min_data_store pre", cat(Intel::min_data_store);
|
||||
}
|
||||
|
||||
event Intel::new_min_data_store(store: Intel::MinDataStore) &priority=-10
|
||||
{
|
||||
print "Intel::new_min_data_store post", cat(Intel::min_data_store);
|
||||
}
|
||||
|
|
@ -0,0 +1,193 @@
|
|||
# @TEST-DOC: Verify Intel::indicator_inserted() and Intel::indicator_removed() in a cluster setup with three different types of indicators.
|
||||
#
|
||||
# @TEST-PORT: BROKER_MANAGER_PORT
|
||||
# @TEST-PORT: BROKER_WORKER1_PORT
|
||||
# @TEST-PORT: BROKER_WORKER2_PORT
|
||||
#
|
||||
# @TEST-EXEC: cp $FILES/broker/cluster-layout.zeek .
|
||||
#
|
||||
# @TEST-EXEC: btest-bg-run manager ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager zeek -b ../addr-indicator.zeek %INPUT
|
||||
# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b ../addr-indicator.zeek %INPUT
|
||||
# @TEST-EXEC: btest-bg-run worker-2 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b ../addr-indicator.zeek %INPUT
|
||||
# @TEST-EXEC: btest-bg-wait 30
|
||||
# @TEST-EXEC: mkdir addr-indicator/; cp -R manager worker-1 worker-2 ./addr-indicator/
|
||||
# @TEST-EXEC: btest-diff addr-indicator/manager/.stdout
|
||||
# @TEST-EXEC: btest-diff addr-indicator/worker-1/.stdout
|
||||
# @TEST-EXEC: btest-diff addr-indicator/worker-2/.stdout
|
||||
#
|
||||
# @TEST-EXEC: rm -rf manager worker-1 worker-2
|
||||
#
|
||||
# @TEST-EXEC: btest-bg-run manager ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager zeek -b ../subnet-indicator.zeek %INPUT
|
||||
# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b ../subnet-indicator.zeek %INPUT
|
||||
# @TEST-EXEC: btest-bg-run worker-2 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b ../subnet-indicator.zeek %INPUT
|
||||
# @TEST-EXEC: btest-bg-wait 30
|
||||
# @TEST-EXEC: mkdir subnet-indicator/; cp -R manager worker-1 worker-2 ./subnet-indicator/
|
||||
# @TEST-EXEC: btest-diff subnet-indicator/manager/.stdout
|
||||
# @TEST-EXEC: btest-diff subnet-indicator/worker-1/.stdout
|
||||
# @TEST-EXEC: btest-diff subnet-indicator/worker-2/.stdout
|
||||
|
||||
# @TEST-EXEC: rm -rf manager worker-1 worker-2
|
||||
#
|
||||
# @TEST-EXEC: btest-bg-run manager ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager zeek -b ../software-indicator.zeek %INPUT
|
||||
# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b ../software-indicator.zeek %INPUT
|
||||
# @TEST-EXEC: btest-bg-run worker-2 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b ../software-indicator.zeek %INPUT
|
||||
# @TEST-EXEC: btest-bg-wait 30
|
||||
# @TEST-EXEC: mkdir software-indicator/; cp -R manager worker-1 worker-2 ./software-indicator/
|
||||
# @TEST-EXEC: btest-diff software-indicator/manager/.stdout
|
||||
# @TEST-EXEC: btest-diff software-indicator/worker-1/.stdout
|
||||
# @TEST-EXEC: btest-diff software-indicator/worker-2/.stdout
|
||||
#
|
||||
# @TEST-EXEC: rm -rf manager worker-1 worker-2
|
||||
|
||||
@load policy/frameworks/cluster/experimental
|
||||
@load base/frameworks/intel
|
||||
|
||||
redef Log::default_rotation_interval = 0sec;
|
||||
|
||||
# @TEST-START-FILE addr-indicator.zeek
|
||||
@load base/frameworks/intel
|
||||
|
||||
const test_intel_type = Intel::ADDR;
|
||||
|
||||
event remove_indicator(value: string, source: string)
|
||||
{
|
||||
print"remove_addr", value, source;
|
||||
Intel::remove([$indicator=value, $indicator_type=Intel::ADDR, $meta=[$source=source]], F);
|
||||
}
|
||||
|
||||
event insert_indicator(value: string, source: string)
|
||||
{
|
||||
print "insert_addr", value, source;
|
||||
Intel::insert([$indicator=value, $indicator_type=Intel::ADDR, $meta=[$source=source]]);
|
||||
}
|
||||
# @TEST-END-FILE
|
||||
|
||||
# @TEST-START-FILE subnet-indicator.zeek
|
||||
@load base/frameworks/intel
|
||||
|
||||
const test_intel_type = Intel::SUBNET;
|
||||
|
||||
event remove_indicator(value: string, source: string)
|
||||
{
|
||||
value = value + "/32"; # make the IP value from generic code a valid subnet
|
||||
print"remove_subnet", value, source;
|
||||
Intel::remove([$indicator=value, $indicator_type=Intel::SUBNET, $meta=[$source=source]], F);
|
||||
}
|
||||
|
||||
event insert_indicator(value: string, source: string)
|
||||
{
|
||||
value = value + "/32"; # make the IP value from generic code a valid subnet
|
||||
print "insert_subnet", value , source;
|
||||
Intel::insert([$indicator=value, $indicator_type=Intel::SUBNET, $meta=[$source=source]]);
|
||||
}
|
||||
# @TEST-END-FILE
|
||||
|
||||
# @TEST-START-FILE software-indicator.zeek
|
||||
@load base/frameworks/intel
|
||||
|
||||
const test_intel_type = Intel::SOFTWARE;
|
||||
|
||||
event remove_indicator(value: string, source: string)
|
||||
{
|
||||
value = "software-" + value;
|
||||
print"remove_software", value, source;
|
||||
Intel::remove([$indicator=value, $indicator_type=Intel::SOFTWARE, $meta=[$source=source]], F);
|
||||
}
|
||||
|
||||
event insert_indicator(value: string, source: string)
|
||||
{
|
||||
value = "software-" + value;
|
||||
print "insert_software", value , source;
|
||||
Intel::insert([$indicator=value, $indicator_type=Intel::SOFTWARE, $meta=[$source=source]]);
|
||||
}
|
||||
# @TEST-END-FILE
|
||||
|
||||
# Helper event for printing on manager and worker.
|
||||
event next_round()
|
||||
{
|
||||
print "====";
|
||||
if ( Cluster::node == "manager" )
|
||||
Cluster::publish(Cluster::worker_topic, next_round);
|
||||
}
|
||||
|
||||
# Send by manager for termination purposes.
|
||||
event do_terminate()
|
||||
{
|
||||
terminate();
|
||||
}
|
||||
|
||||
event publish_do_terminate()
|
||||
{
|
||||
print "publish_do_terminate()";
|
||||
Cluster::publish(Cluster::worker_topic, do_terminate);
|
||||
}
|
||||
|
||||
# Ensure the manager terminates eventually.
|
||||
global nodes_down = 0;
|
||||
event Cluster::node_down(name: string, id: string)
|
||||
{
|
||||
++nodes_down;
|
||||
if ( nodes_down >= 2)
|
||||
terminate();
|
||||
}
|
||||
|
||||
# Gets the ball rolling
|
||||
event Cluster::Experimental::cluster_started()
|
||||
{
|
||||
if ( Cluster::local_node_type() == Cluster::MANAGER )
|
||||
event insert_indicator("1.2.3.4", "from-manager");
|
||||
}
|
||||
|
||||
global indicators_inserted: table[Intel::Type] of count &default=0;
|
||||
global indicator_removed: table[Intel::Type] of count &default=0;
|
||||
|
||||
hook Intel::indicator_inserted(indicator: string, indicator_type: Intel::Type)
|
||||
{
|
||||
print "Intel::indicator_inserted", indicator, indicator_type;
|
||||
++indicators_inserted[indicator_type];
|
||||
|
||||
# If worker-1 sees the first addr indicator (1.2.3.4), it re-inserts
|
||||
# it with a different source (from-worker) and also inserts a second
|
||||
# indicatore 1.2.3.5 with with sources "from-worker" and "from-manager";
|
||||
if ( Cluster::node == "worker-1" )
|
||||
{
|
||||
if ( indicators_inserted[test_intel_type] == 1 || indicators_inserted[test_intel_type] == 3 )
|
||||
{
|
||||
event insert_indicator("1.2.3.4", "from-worker");
|
||||
event insert_indicator("1.2.3.5", "from-worker");
|
||||
event insert_indicator("1.2.3.5", "from-manager");
|
||||
}
|
||||
}
|
||||
|
||||
# Once worker-2 has observed two or four indicators, it removes
|
||||
# all of them again!
|
||||
if ( Cluster::node == "worker-2" )
|
||||
{
|
||||
if ( indicators_inserted[test_intel_type] == 2 || indicators_inserted[test_intel_type] == 4 )
|
||||
{
|
||||
event remove_indicator("1.2.3.4", "from-manager");
|
||||
event remove_indicator("1.2.3.5", "from-manager");
|
||||
event remove_indicator("1.2.3.4", "from-worker");
|
||||
event remove_indicator("1.2.3.5", "from-worker");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
hook Intel::indicator_removed(indicator: string, indicator_type: Intel::Type)
|
||||
{
|
||||
print "Intel::indicator_removed", indicator, indicator_type;
|
||||
++indicator_removed[indicator_type];
|
||||
if ( Cluster::node == "manager" )
|
||||
{
|
||||
if ( indicator_removed[test_intel_type] == 2 )
|
||||
{
|
||||
# Trigger another round of inserts at the workers!
|
||||
event next_round();
|
||||
event insert_indicator("1.2.3.4", "from-manager");
|
||||
}
|
||||
else if ( indicator_removed[test_intel_type] == 4 )
|
||||
{
|
||||
event publish_do_terminate();
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue