mirror of
https://github.com/zeek/zeek.git
synced 2025-10-02 06:38:20 +00:00
Merge remote-tracking branch 'origin/topic/awelzel/intel-indicator-hooks'
* origin/topic/awelzel/intel-indicator-hooks: intel/seen/manage-event-groups: Policy script for toggling intel event groups intel: Add indicator_inserted and indicator_removed hooks
This commit is contained in:
Arne Welzel
commit
41f04eda72
28 changed files with 923 additions and 2 deletions
26
CHANGES
26
CHANGES
|
|
@ -1,3 +1,29 @@
|
|||
8.0.0-dev.303 | 2025-06-02 09:51:53 +0200
|
||||
|
||||
* intel/seen/manage-event-groups: Policy script for toggling intel event groups (Arne Welzel, Corelight)
|
||||
|
||||
Co-authored-by: Mohan Dhawan <mohan@corelight.com>
|
||||
|
||||
* intel: Add indicator_inserted and indicator_removed hooks (Arne Welzel, Corelight)
|
||||
|
||||
This change adds two new hooks to the Intel framework that can be used
|
||||
to intercept added and removed indicators and their type.
|
||||
|
||||
These hooks are fairly low-level. One immediate use-case is to count the
|
||||
number of indicators loaded per Intel::Type and enable and disable the
|
||||
corresponding event groups of the intel/seen scripts.
|
||||
|
||||
I attempted to gauge the overhead and while it's definitely there, loading
|
||||
a file with ~500k DOMAIN entries takes somewhere around ~0.5 seconds hooks
|
||||
when populated via the min_data_store store mechanism. While that
|
||||
doesn't sound great, it actually takes the manager on my system 2.5
|
||||
seconds to serialize and Cluster::publish() the min_data_store alone
|
||||
and its doing that serially for every active worker. Mostly to say that
|
||||
the bigger overhead in that area on the manager doing redundant work
|
||||
per worker.
|
||||
|
||||
Co-authored-by: Mohan Dhawan <mohan@corelight.com>
|
||||
|
||||
8.0.0-dev.299 | 2025-05-30 13:06:42 -0700
|
||||
|
||||
* fix for crash when interpreting transformed ASTs that include multi-field record assignments/additions (Vern Paxson, Corelight)
|
||||
|
|
|
|||
28
NEWS
28
NEWS
|
|
@ -66,6 +66,34 @@ New Functionality
|
|||
|
||||
- The FTP analyzer now supports explicit TLS via AUTH TLS.
|
||||
|
||||
- Two new script-level hooks in the Intel framework have been added.
|
||||
|
||||
hook indicator_inserted(indicator_value: string, indicator_type: Intel::Type)
|
||||
|
||||
hook indicator_removed(indicator_value: string, indicator_type: Intel::Type)
|
||||
|
||||
These are reliably invoked on worker and manager nodes the first time an
|
||||
indicator value is inserted into the store and once it has been completely
|
||||
removed from the store.
|
||||
|
||||
- The ``frameworks/intel/seen`` scripts have been annotated with event groups
|
||||
and a new ``frameworks/intel/seen/manage-event-groups`` policy script added.
|
||||
|
||||
The motivation is to allow Zeek distributors to load the ``intel/seen`` scripts
|
||||
by default without incurring their event overhead when no Intel indicators are
|
||||
loaded. Corresponding event handlers are enabled once the first Intel indicator
|
||||
of a given ``Intel::Type`` is added. Event handlers are disabled when the last
|
||||
indicator is removed, again.
|
||||
|
||||
Note that the ``manage-event-groups`` script interacts with the ``Intel::seen_policy``
|
||||
hook: If no indicators for a given ``Intel::Type`` are loaded, the ``Intel::seen_policy``
|
||||
will not be invoked as the event handlers extracting indicators aren't executed.
|
||||
|
||||
If you rely on the ``Intel::seen_policy`` hook to be invoked regardless of the
|
||||
contents of the Intel store, do not load the ``manage-event-groups`` or set:
|
||||
|
||||
redef Intel::manage_seen_event_groups = F;
|
||||
|
||||
Changed Functionality
|
||||
---------------------
|
||||
|
||||
|
|
|
|||
2
VERSION
2
VERSION
|
|
@ -1 +1 @@
|
|||
8.0.0-dev.299
|
||||
8.0.0-dev.303
|
||||
|
|
|
|||
|
|
@ -105,10 +105,30 @@ event Intel::insert_indicator(item: Intel::Item) &priority=5
|
|||
Intel::_insert(item, F);
|
||||
}
|
||||
|
||||
function invoke_indicator_hook(store: MinDataStore, h: hook(v: string, t: Intel::Type))
|
||||
{
|
||||
for ( a in store$host_data )
|
||||
hook h(cat(a), Intel::ADDR);
|
||||
|
||||
for ( sn in store$subnet_data)
|
||||
hook h(cat(sn), Intel::SUBNET);
|
||||
|
||||
for ( [indicator_value, indicator_type] in store$string_data )
|
||||
hook h(indicator_value, indicator_type);
|
||||
}
|
||||
|
||||
# Handling of a complete MinDataStore snapshot
|
||||
#
|
||||
# Invoke the removed and inserted hooks using the old and new min data store
|
||||
# instances, respectively. The way this event is used, the original
|
||||
# min_data_store should essentially be empty.
|
||||
event new_min_data_store(store: MinDataStore)
|
||||
{
|
||||
invoke_indicator_hook(min_data_store, Intel::indicator_removed);
|
||||
|
||||
min_data_store = store;
|
||||
|
||||
invoke_indicator_hook(min_data_store, Intel::indicator_inserted);
|
||||
}
|
||||
@endif
|
||||
|
||||
|
|
|
|||
|
|
@ -207,6 +207,35 @@ export {
|
|||
## item: The intel item that should be inserted.
|
||||
global filter_item: hook(item: Intel::Item);
|
||||
|
||||
## This hook is invoked when a new indicator has been inserted into
|
||||
## the min data store for the first time.
|
||||
##
|
||||
## Calls to :zeek:see:`Intel::seen` with a matching indicator value
|
||||
## and type will result in matches.
|
||||
##
|
||||
## Subsequent inserts of the same indicator type and value do not
|
||||
## invoke this hook. Breaking from this hook has no effect.
|
||||
##
|
||||
## indicator: The indicator value.
|
||||
##
|
||||
## indicator_type: The indicator type.
|
||||
##
|
||||
## .. zeek::see:: Intel::indicator_removed
|
||||
global indicator_inserted: hook(indicator: string, indiator_type: Type);
|
||||
|
||||
## This hook is invoked when an indicator has been removed from
|
||||
## the min data store.
|
||||
##
|
||||
## After this hooks runs, :zeek:see:`Intel::seen` for the indicator
|
||||
## will not return any matches. Breaking from this hook has no effect.
|
||||
##
|
||||
## indicator: The indicator value.
|
||||
##
|
||||
## indicator_type: The indicator type.
|
||||
##
|
||||
## .. zeek::see:: Intel::indicator_inserted
|
||||
global indicator_removed: hook(indicator: string, indiator_type: Type);
|
||||
|
||||
global log_intel: event(rec: Info);
|
||||
}
|
||||
|
||||
|
|
@ -507,18 +536,44 @@ function _insert(item: Item, first_dispatch: bool &default = T)
|
|||
# All intelligence is case insensitive at the moment.
|
||||
local lower_indicator = to_lower(item$indicator);
|
||||
|
||||
# Track if the indicator was inserted into the min_data_store.
|
||||
# It's tempting to just use is_new above, but it seems that only works
|
||||
# correctly on a worker if the manager never spuriously sends a
|
||||
# Intel::insert_item(), so better to determine this locally based
|
||||
# on the actual contents of the min_data_store.
|
||||
local inserted = F;
|
||||
local inserted_value = "";
|
||||
|
||||
# Insert indicator into MinDataStore (might exist already).
|
||||
switch ( item$indicator_type )
|
||||
{
|
||||
case ADDR:
|
||||
local host = to_addr(item$indicator);
|
||||
if ( host !in min_data_store$host_data )
|
||||
{
|
||||
inserted = T;
|
||||
inserted_value = cat(host);
|
||||
}
|
||||
|
||||
add min_data_store$host_data[host];
|
||||
break;
|
||||
case SUBNET:
|
||||
local net = to_subnet(item$indicator);
|
||||
if ( net !in min_data_store$subnet_data )
|
||||
{
|
||||
inserted = T;
|
||||
inserted_value = cat(net);
|
||||
}
|
||||
|
||||
add min_data_store$subnet_data[net];
|
||||
break;
|
||||
default:
|
||||
if ( [lower_indicator, item$indicator_type] !in min_data_store$string_data )
|
||||
{
|
||||
inserted = T;
|
||||
inserted_value = lower_indicator;
|
||||
}
|
||||
|
||||
add min_data_store$string_data[lower_indicator, item$indicator_type];
|
||||
break;
|
||||
}
|
||||
|
|
@ -533,6 +588,9 @@ function _insert(item: Item, first_dispatch: bool &default = T)
|
|||
# Announce a (possibly) new item if this is the first dispatch and
|
||||
# we know it is new or have to assume that on a worker.
|
||||
event Intel::new_item(item);
|
||||
|
||||
if ( inserted )
|
||||
hook Intel::indicator_inserted(inserted_value, item$indicator_type);
|
||||
}
|
||||
|
||||
function insert(item: Item)
|
||||
|
|
@ -632,18 +690,43 @@ function remove(item: Item, purge_indicator: bool)
|
|||
# Handling of indicator removal in minimal data stores.
|
||||
event remove_indicator(item: Item)
|
||||
{
|
||||
local removed = F;
|
||||
local removed_value = "";
|
||||
|
||||
switch ( item$indicator_type )
|
||||
{
|
||||
case ADDR:
|
||||
local host = to_addr(item$indicator);
|
||||
if ( host in min_data_store$host_data )
|
||||
{
|
||||
removed = T;
|
||||
removed_value = cat(host);
|
||||
}
|
||||
|
||||
delete min_data_store$host_data[host];
|
||||
break;
|
||||
case SUBNET:
|
||||
local net = to_subnet(item$indicator);
|
||||
if ( net in min_data_store$subnet_data )
|
||||
{
|
||||
removed = T;
|
||||
removed_value = cat(net);
|
||||
}
|
||||
|
||||
delete min_data_store$subnet_data[net];
|
||||
break;
|
||||
default:
|
||||
delete min_data_store$string_data[to_lower(item$indicator), item$indicator_type];
|
||||
local indicator_value = to_lower(item$indicator);
|
||||
if ( [indicator_value, item$indicator_type] in min_data_store$string_data )
|
||||
{
|
||||
removed = T;
|
||||
removed_value = indicator_value;
|
||||
}
|
||||
|
||||
delete min_data_store$string_data[indicator_value, item$indicator_type];
|
||||
break;
|
||||
}
|
||||
|
||||
if ( removed )
|
||||
hook Intel::indicator_removed(removed_value, item$indicator_type);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,74 @@
|
|||
@load frameworks/intel/seen
|
||||
@load base/frameworks/reporter
|
||||
|
||||
module Intel;
|
||||
|
||||
export {
|
||||
## Whether Intel event groups for the seen scripts are managed.
|
||||
##
|
||||
## When loading this script, by default, all :zeek:see:`Intel::Type`
|
||||
## event groups are disabled at startup and only enabled when indicators
|
||||
## of corresponding types are loaded into the Intel framework's store.
|
||||
## This allows to load the ``frameworks/intel/seen`` scripts without
|
||||
## incurring event handling overhead when no Intel indicators are loaded.
|
||||
##
|
||||
## One caveat is that the :zeek:see:`Intel::seen_policy` hook will not
|
||||
## be invoked for indicator types that are not at all in the Intel
|
||||
## framework's store. If you rely on :zeek:see:`Intel::seen_policy` to
|
||||
## find unmatched indicators, do not not load this script, set this
|
||||
## variable to ``F``, or insert dummy values of the types using
|
||||
## :zeek:see:`Intel::insert`.
|
||||
const manage_seen_event_groups = T &redef;
|
||||
}
|
||||
|
||||
global intel_type_counts: table[Intel::Type] of count &default=0;
|
||||
|
||||
event zeek_init()
|
||||
{
|
||||
# If the feature is disabled, don't act.
|
||||
if ( ! manage_seen_event_groups )
|
||||
return;
|
||||
|
||||
# Disable all Intel related event groups at startup. These
|
||||
# are enabled again as soon as at least one indicator of the
|
||||
# type is inserted.
|
||||
for ( name in enum_names(Intel::Type) )
|
||||
{
|
||||
if ( has_event_group(name) )
|
||||
disable_event_group(name);
|
||||
}
|
||||
}
|
||||
|
||||
hook Intel::indicator_inserted(v: string, t: Intel::Type)
|
||||
{
|
||||
++intel_type_counts[t];
|
||||
|
||||
if ( ! manage_seen_event_groups )
|
||||
return;
|
||||
|
||||
|
||||
if ( intel_type_counts[t] == 1 )
|
||||
{
|
||||
local name = cat(t);
|
||||
|
||||
if ( has_event_group(name) )
|
||||
enable_event_group(name);
|
||||
}
|
||||
}
|
||||
|
||||
hook Intel::indicator_removed(v: string, t: Intel::Type)
|
||||
{
|
||||
--intel_type_counts[t];
|
||||
|
||||
if ( ! manage_seen_event_groups )
|
||||
return;
|
||||
|
||||
|
||||
if ( intel_type_counts[t] == 0 )
|
||||
{
|
||||
local name = cat(t);
|
||||
|
||||
if ( has_event_group(name) )
|
||||
disable_event_group(name);
|
||||
}
|
||||
}
|
||||
|
|
@ -59,6 +59,7 @@
|
|||
@load frameworks/intel/seen/file-names.zeek
|
||||
@load frameworks/intel/seen/http-headers.zeek
|
||||
@load frameworks/intel/seen/http-url.zeek
|
||||
@load frameworks/intel/seen/manage-event-groups.zeek
|
||||
@load frameworks/intel/seen/pubkey-hashes.zeek
|
||||
@load frameworks/intel/seen/smb-filenames.zeek
|
||||
@load frameworks/intel/seen/smtp-url-extraction.zeek
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
received termination signal
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
|
||||
Intel::indicator_inserted, 192.168.0.0/16, Intel::SUBNET
|
||||
Intel::indicator_inserted, putty, Intel::SOFTWARE
|
||||
Intel::indicator_inserted, putty2, Intel::SOFTWARE
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
Intel::new_min_data_store pre, [host_data={\x0a\x0a}, subnet_data={\x0a\x0a}, string_data={\x0a\x0a}]
|
||||
Intel::new_min_data_store post, [host_data={\x0a\x0a}, subnet_data={\x0a\x0a}, string_data={\x0a\x0a}]
|
||||
Intel::insert_indicator, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
|
||||
Intel::insert_indicator, 1.2.3.5, Intel::ADDR
|
||||
Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
|
||||
Intel::insert_indicator, 192.168.0.0/16, Intel::SUBNET
|
||||
Intel::indicator_inserted, 192.168.0.0/16, Intel::SUBNET
|
||||
Intel::insert_indicator, putty, Intel::SOFTWARE
|
||||
Intel::indicator_inserted, putty, Intel::SOFTWARE
|
||||
Intel::insert_indicator, putty2, Intel::SOFTWARE
|
||||
Intel::indicator_inserted, putty2, Intel::SOFTWARE
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
Intel::new_min_data_store pre, [host_data={\x0a\x0a}, subnet_data={\x0a\x0a}, string_data={\x0a\x0a}]
|
||||
Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
|
||||
Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_inserted, 192.168.0.0/16, Intel::SUBNET
|
||||
Intel::indicator_inserted, putty, Intel::SOFTWARE
|
||||
Intel::indicator_inserted, putty2, Intel::SOFTWARE
|
||||
Intel::new_min_data_store post, [host_data={\x0a\x091.2.3.5,\x0a\x091.2.3.4\x0a}, subnet_data={\x0a\x09192.168.0.0/16\x0a}, string_data={\x0a\x09[putty, Intel::SOFTWARE] ,\x0a\x09[putty2, Intel::SOFTWARE] \x0a}]
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
Intel::new_min_data_store pre, [host_data={\x0a\x0a}, subnet_data={\x0a\x0a}, string_data={\x0a\x0a}]
|
||||
Intel::new_min_data_store post, [host_data={\x0a\x0a}, subnet_data={\x0a\x0a}, string_data={\x0a\x0a}]
|
||||
Intel::insert_indicator, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
|
||||
Intel::insert_indicator, 1.2.3.5, Intel::ADDR
|
||||
Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
|
||||
Intel::insert_indicator, 192.168.0.0/16, Intel::SUBNET
|
||||
Intel::indicator_inserted, 192.168.0.0/16, Intel::SUBNET
|
||||
Intel::insert_indicator, putty, Intel::SOFTWARE
|
||||
Intel::indicator_inserted, putty, Intel::SOFTWARE
|
||||
Intel::insert_indicator, putty2, Intel::SOFTWARE
|
||||
Intel::indicator_inserted, putty2, Intel::SOFTWARE
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
insert_addr, 1.2.3.4, from-manager
|
||||
Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
|
||||
Intel::indicator_removed, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_removed, 1.2.3.5, Intel::ADDR
|
||||
====
|
||||
insert_addr, 1.2.3.4, from-manager
|
||||
Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
|
||||
Intel::indicator_removed, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_removed, 1.2.3.5, Intel::ADDR
|
||||
publish_do_terminate()
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
|
||||
insert_addr, 1.2.3.4, from-worker
|
||||
insert_addr, 1.2.3.5, from-worker
|
||||
Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
|
||||
insert_addr, 1.2.3.5, from-manager
|
||||
Intel::indicator_removed, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_removed, 1.2.3.5, Intel::ADDR
|
||||
====
|
||||
Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
|
||||
insert_addr, 1.2.3.4, from-worker
|
||||
insert_addr, 1.2.3.5, from-worker
|
||||
Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
|
||||
insert_addr, 1.2.3.5, from-manager
|
||||
Intel::indicator_removed, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_removed, 1.2.3.5, Intel::ADDR
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
|
||||
remove_addr, 1.2.3.4, from-manager
|
||||
remove_addr, 1.2.3.5, from-manager
|
||||
remove_addr, 1.2.3.4, from-worker
|
||||
remove_addr, 1.2.3.5, from-worker
|
||||
Intel::indicator_removed, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_removed, 1.2.3.5, Intel::ADDR
|
||||
====
|
||||
Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
|
||||
remove_addr, 1.2.3.4, from-manager
|
||||
remove_addr, 1.2.3.5, from-manager
|
||||
remove_addr, 1.2.3.4, from-worker
|
||||
remove_addr, 1.2.3.5, from-worker
|
||||
Intel::indicator_removed, 1.2.3.4, Intel::ADDR
|
||||
Intel::indicator_removed, 1.2.3.5, Intel::ADDR
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
insert_software, software-1.2.3.4, from-manager
|
||||
Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE
|
||||
Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE
|
||||
Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE
|
||||
Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE
|
||||
====
|
||||
insert_software, software-1.2.3.4, from-manager
|
||||
Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE
|
||||
Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE
|
||||
Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE
|
||||
Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE
|
||||
publish_do_terminate()
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE
|
||||
insert_software, software-1.2.3.4, from-worker
|
||||
insert_software, software-1.2.3.5, from-worker
|
||||
Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE
|
||||
insert_software, software-1.2.3.5, from-manager
|
||||
Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE
|
||||
Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE
|
||||
====
|
||||
Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE
|
||||
insert_software, software-1.2.3.4, from-worker
|
||||
insert_software, software-1.2.3.5, from-worker
|
||||
Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE
|
||||
insert_software, software-1.2.3.5, from-manager
|
||||
Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE
|
||||
Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE
|
||||
Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE
|
||||
remove_software, software-1.2.3.4, from-manager
|
||||
remove_software, software-1.2.3.5, from-manager
|
||||
remove_software, software-1.2.3.4, from-worker
|
||||
remove_software, software-1.2.3.5, from-worker
|
||||
Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE
|
||||
Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE
|
||||
====
|
||||
Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE
|
||||
Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE
|
||||
remove_software, software-1.2.3.4, from-manager
|
||||
remove_software, software-1.2.3.5, from-manager
|
||||
remove_software, software-1.2.3.4, from-worker
|
||||
remove_software, software-1.2.3.5, from-worker
|
||||
Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE
|
||||
Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
insert_subnet, 1.2.3.4/32, from-manager
|
||||
Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET
|
||||
Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET
|
||||
Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET
|
||||
Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET
|
||||
====
|
||||
insert_subnet, 1.2.3.4/32, from-manager
|
||||
Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET
|
||||
Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET
|
||||
Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET
|
||||
Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET
|
||||
publish_do_terminate()
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET
|
||||
insert_subnet, 1.2.3.4/32, from-worker
|
||||
insert_subnet, 1.2.3.5/32, from-worker
|
||||
Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET
|
||||
insert_subnet, 1.2.3.5/32, from-manager
|
||||
Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET
|
||||
Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET
|
||||
====
|
||||
Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET
|
||||
insert_subnet, 1.2.3.4/32, from-worker
|
||||
insert_subnet, 1.2.3.5/32, from-worker
|
||||
Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET
|
||||
insert_subnet, 1.2.3.5/32, from-manager
|
||||
Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET
|
||||
Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET
|
||||
Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET
|
||||
remove_subnet, 1.2.3.4/32, from-manager
|
||||
remove_subnet, 1.2.3.5/32, from-manager
|
||||
remove_subnet, 1.2.3.4/32, from-worker
|
||||
remove_subnet, 1.2.3.5/32, from-worker
|
||||
Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET
|
||||
Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET
|
||||
====
|
||||
Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET
|
||||
Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET
|
||||
remove_subnet, 1.2.3.4/32, from-manager
|
||||
remove_subnet, 1.2.3.5/32, from-manager
|
||||
remove_subnet, 1.2.3.4/32, from-worker
|
||||
remove_subnet, 1.2.3.5/32, from-worker
|
||||
Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET
|
||||
Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
running do_step(2)
|
||||
Intel::indicator_inserted 1.2.3.4 Intel::ADDR
|
||||
publish do_step(3) to zeek/cluster/worker
|
||||
Intel::match_remote: 1.2.3.4 Intel::ADDR
|
||||
Intel::match: 1.2.3.4 Intel::ADDR
|
||||
running do_step(4)
|
||||
Intel::indicator_removed 1.2.3.4 Intel::ADDR
|
||||
publish do_step(5) to zeek/cluster/worker
|
||||
running do_step(6)
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
#separator \x09
|
||||
#set_separator ,
|
||||
#empty_field (empty)
|
||||
#unset_field -
|
||||
#path intel
|
||||
#open XXXX-XX-XX-XX-XX-XX
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p seen.indicator seen.indicator_type seen.where seen.node matched sources fuid file_mime_type file_desc
|
||||
#types time string addr port addr port string enum enum string set[enum] set[string] string string string
|
||||
XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 1.1.1.1 1 2.2.2.2 2 1.2.3.4 Intel::ADDR SMTP::IN_RECEIVED_HEADER worker-1 Intel::ADDR source1 - - -
|
||||
#close XXXX-XX-XX-XX-XX-XX
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
|
||||
running do_step(1)
|
||||
publish do_step(2) to zeek/cluster/manager
|
||||
Intel::indicator_inserted 1.2.3.4 Intel::ADDR
|
||||
running do_step(3)
|
||||
Intel::seen_policy(1.2.3.4 of Intel::ADDR, T)
|
||||
Intel::seen_policy(1.2.3.5 of Intel::ADDR, F)
|
||||
Intel::match_remote: 1.2.3.4 Intel::ADDR
|
||||
Intel::indicator_removed 1.2.3.4 Intel::ADDR
|
||||
running do_step(5)
|
||||
publish do_step(6) to zeek/cluster/manager
|
||||
|
|
@ -0,0 +1,121 @@
|
|||
# @TEST-DOC: Verify behavior of Intel:indicator_inserted() and Intel::indicator_removed() when a worker node restarts.
|
||||
#
|
||||
# @TEST-PORT: BROKER_MANAGER_PORT
|
||||
# @TEST-PORT: BROKER_WORKER1_PORT
|
||||
# @TEST-PORT: BROKER_WORKER2_PORT
|
||||
#
|
||||
# @TEST-EXEC: cp $FILES/broker/cluster-layout.zeek .
|
||||
#
|
||||
# @TEST-EXEC: zeek --parse-only %INPUT
|
||||
# @TEST-EXEC: btest-bg-run manager ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager zeek -b %INPUT
|
||||
# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b %INPUT
|
||||
# @TEST-EXEC: btest-bg-run worker-2 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b %INPUT
|
||||
# @TEST-EXEC: wait-for-file worker-1/DONE 30
|
||||
# @TEST-EXEC: mv worker-1 worker-1-run-1
|
||||
# @TEST-EXEC: rm -rf worker-1
|
||||
# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b %INPUT
|
||||
# @TEST-EXEC: wait-for-file worker-1/DONE 30
|
||||
# @TEST-EXEC: btest-bg-wait 30
|
||||
# @TEST-EXEC: mv worker-1 worker-1-run-2
|
||||
|
||||
# @TEST-EXEC: btest-diff manager/.stdout
|
||||
# @TEST-EXEC: btest-diff manager/.stderr
|
||||
# @TEST-EXEC: btest-diff worker-1-run-1/.stdout
|
||||
# @TEST-EXEC: btest-diff worker-1-run-2/.stdout
|
||||
# @TEST-EXEC: btest-diff worker-2/.stdout
|
||||
#
|
||||
@load policy/frameworks/cluster/experimental
|
||||
@load base/frameworks/intel
|
||||
|
||||
# Ttoal of 5 unique indicators
|
||||
# @TEST-START-FILE intel.dat
|
||||
#fields indicator indicator_type meta.source meta.desc meta.url
|
||||
1.2.3.4 Intel::ADDR source1 this host is bad http://some-data-distributor.com/1
|
||||
1.2.3.4 Intel::ADDR source2 this host is bad http://some-data-distributor.com/1
|
||||
1.2.3.5 Intel::ADDR source1 this host is bad http://some-data-distributor.com/1
|
||||
192.168.0.0/16 Intel::SUBNET source1 this network is bad http://some-data-distributor.com/2
|
||||
putty Intel::SOFTWARE source1 this software is bad http://some-data-distributor.com/2
|
||||
putty Intel::SOFTWARE source2 this software is bad http://some-data-distributor.com/2
|
||||
putty2 Intel::SOFTWARE source3 this software is bad http://some-data-distributor.com/2
|
||||
# @TEST-END-FILE
|
||||
|
||||
redef Log::default_rotation_interval = 0sec;
|
||||
|
||||
event Cluster::Experimental::cluster_started()
|
||||
{
|
||||
# Start reading the intel file on the manager once all workers are up.
|
||||
if ( Cluster::node == "manager" )
|
||||
{
|
||||
local source = "../intel.dat";
|
||||
Input::add_event([$source=source,
|
||||
$reader=Input::READER_ASCII,
|
||||
$mode=Input::REREAD,
|
||||
$name=cat("intel-", source),
|
||||
$fields=Intel::Item,
|
||||
$ev=Intel::read_entry,
|
||||
$error_ev=Intel::read_error]);
|
||||
}
|
||||
}
|
||||
|
||||
# Send by manager for termination purposes.
|
||||
event do_terminate()
|
||||
{
|
||||
terminate();
|
||||
}
|
||||
|
||||
global worker1_down = 0;
|
||||
global nodes_down = 0;
|
||||
|
||||
event Cluster::node_down(name: string, id: string)
|
||||
{
|
||||
++nodes_down;
|
||||
if ( name == "worker-1")
|
||||
++worker1_down;
|
||||
|
||||
if ( worker1_down == 2 )
|
||||
Cluster::publish(Cluster::worker_topic, do_terminate);
|
||||
|
||||
if ( nodes_down >= 3 )
|
||||
terminate();
|
||||
}
|
||||
|
||||
global total_indicators = 0;
|
||||
|
||||
hook Intel::indicator_inserted(indicator: string, indicator_type: Intel::Type)
|
||||
{
|
||||
print "Intel::indicator_inserted", indicator, indicator_type;
|
||||
++total_indicators;
|
||||
|
||||
# Once worker-1 has seen all the 5 indicators, write a DONE file and terminate()
|
||||
if ( Cluster::node == "worker-1" && total_indicators == 5 )
|
||||
{
|
||||
if ( ! piped_exec("touch DONE", "") )
|
||||
exit(1);
|
||||
|
||||
terminate();
|
||||
}
|
||||
}
|
||||
|
||||
hook Intel::indicator_removed(indicator: string, indicator_type: Intel::Type)
|
||||
{
|
||||
print "Intel::indicator_removed", indicator, indicator_type;
|
||||
}
|
||||
|
||||
module Intel;
|
||||
|
||||
# Internal events for easier grasping behavior.
|
||||
|
||||
event Intel::insert_indicator(item: Intel::Item) &priority=10
|
||||
{
|
||||
print "Intel::insert_indicator", item$indicator, item$indicator_type;
|
||||
}
|
||||
|
||||
event Intel::new_min_data_store(store: Intel::MinDataStore) &priority=10
|
||||
{
|
||||
print "Intel::new_min_data_store pre", cat(Intel::min_data_store);
|
||||
}
|
||||
|
||||
event Intel::new_min_data_store(store: Intel::MinDataStore) &priority=-10
|
||||
{
|
||||
print "Intel::new_min_data_store post", cat(Intel::min_data_store);
|
||||
}
|
||||
|
|
@ -0,0 +1,193 @@
|
|||
# @TEST-DOC: Verify Intel::indicator_inserted() and Intel::indicator_removed() in a cluster setup with three different types of indicators.
|
||||
#
|
||||
# @TEST-PORT: BROKER_MANAGER_PORT
|
||||
# @TEST-PORT: BROKER_WORKER1_PORT
|
||||
# @TEST-PORT: BROKER_WORKER2_PORT
|
||||
#
|
||||
# @TEST-EXEC: cp $FILES/broker/cluster-layout.zeek .
|
||||
#
|
||||
# @TEST-EXEC: btest-bg-run manager ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager zeek -b ../addr-indicator.zeek %INPUT
|
||||
# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b ../addr-indicator.zeek %INPUT
|
||||
# @TEST-EXEC: btest-bg-run worker-2 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b ../addr-indicator.zeek %INPUT
|
||||
# @TEST-EXEC: btest-bg-wait 30
|
||||
# @TEST-EXEC: mkdir addr-indicator/; cp -R manager worker-1 worker-2 ./addr-indicator/
|
||||
# @TEST-EXEC: btest-diff addr-indicator/manager/.stdout
|
||||
# @TEST-EXEC: btest-diff addr-indicator/worker-1/.stdout
|
||||
# @TEST-EXEC: btest-diff addr-indicator/worker-2/.stdout
|
||||
#
|
||||
# @TEST-EXEC: rm -rf manager worker-1 worker-2
|
||||
#
|
||||
# @TEST-EXEC: btest-bg-run manager ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager zeek -b ../subnet-indicator.zeek %INPUT
|
||||
# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b ../subnet-indicator.zeek %INPUT
|
||||
# @TEST-EXEC: btest-bg-run worker-2 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b ../subnet-indicator.zeek %INPUT
|
||||
# @TEST-EXEC: btest-bg-wait 30
|
||||
# @TEST-EXEC: mkdir subnet-indicator/; cp -R manager worker-1 worker-2 ./subnet-indicator/
|
||||
# @TEST-EXEC: btest-diff subnet-indicator/manager/.stdout
|
||||
# @TEST-EXEC: btest-diff subnet-indicator/worker-1/.stdout
|
||||
# @TEST-EXEC: btest-diff subnet-indicator/worker-2/.stdout
|
||||
|
||||
# @TEST-EXEC: rm -rf manager worker-1 worker-2
|
||||
#
|
||||
# @TEST-EXEC: btest-bg-run manager ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager zeek -b ../software-indicator.zeek %INPUT
|
||||
# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b ../software-indicator.zeek %INPUT
|
||||
# @TEST-EXEC: btest-bg-run worker-2 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b ../software-indicator.zeek %INPUT
|
||||
# @TEST-EXEC: btest-bg-wait 30
|
||||
# @TEST-EXEC: mkdir software-indicator/; cp -R manager worker-1 worker-2 ./software-indicator/
|
||||
# @TEST-EXEC: btest-diff software-indicator/manager/.stdout
|
||||
# @TEST-EXEC: btest-diff software-indicator/worker-1/.stdout
|
||||
# @TEST-EXEC: btest-diff software-indicator/worker-2/.stdout
|
||||
#
|
||||
# @TEST-EXEC: rm -rf manager worker-1 worker-2
|
||||
|
||||
@load policy/frameworks/cluster/experimental
|
||||
@load base/frameworks/intel
|
||||
|
||||
redef Log::default_rotation_interval = 0sec;
|
||||
|
||||
# @TEST-START-FILE addr-indicator.zeek
|
||||
@load base/frameworks/intel
|
||||
|
||||
const test_intel_type = Intel::ADDR;
|
||||
|
||||
event remove_indicator(value: string, source: string)
|
||||
{
|
||||
print"remove_addr", value, source;
|
||||
Intel::remove([$indicator=value, $indicator_type=Intel::ADDR, $meta=[$source=source]], F);
|
||||
}
|
||||
|
||||
event insert_indicator(value: string, source: string)
|
||||
{
|
||||
print "insert_addr", value, source;
|
||||
Intel::insert([$indicator=value, $indicator_type=Intel::ADDR, $meta=[$source=source]]);
|
||||
}
|
||||
# @TEST-END-FILE
|
||||
|
||||
# @TEST-START-FILE subnet-indicator.zeek
|
||||
@load base/frameworks/intel
|
||||
|
||||
const test_intel_type = Intel::SUBNET;
|
||||
|
||||
event remove_indicator(value: string, source: string)
|
||||
{
|
||||
value = value + "/32"; # make the IP value from generic code a valid subnet
|
||||
print"remove_subnet", value, source;
|
||||
Intel::remove([$indicator=value, $indicator_type=Intel::SUBNET, $meta=[$source=source]], F);
|
||||
}
|
||||
|
||||
event insert_indicator(value: string, source: string)
|
||||
{
|
||||
value = value + "/32"; # make the IP value from generic code a valid subnet
|
||||
print "insert_subnet", value , source;
|
||||
Intel::insert([$indicator=value, $indicator_type=Intel::SUBNET, $meta=[$source=source]]);
|
||||
}
|
||||
# @TEST-END-FILE
|
||||
|
||||
# @TEST-START-FILE software-indicator.zeek
|
||||
@load base/frameworks/intel
|
||||
|
||||
const test_intel_type = Intel::SOFTWARE;
|
||||
|
||||
event remove_indicator(value: string, source: string)
|
||||
{
|
||||
value = "software-" + value;
|
||||
print"remove_software", value, source;
|
||||
Intel::remove([$indicator=value, $indicator_type=Intel::SOFTWARE, $meta=[$source=source]], F);
|
||||
}
|
||||
|
||||
event insert_indicator(value: string, source: string)
|
||||
{
|
||||
value = "software-" + value;
|
||||
print "insert_software", value , source;
|
||||
Intel::insert([$indicator=value, $indicator_type=Intel::SOFTWARE, $meta=[$source=source]]);
|
||||
}
|
||||
# @TEST-END-FILE
|
||||
|
||||
# Helper event for printing on manager and worker.
|
||||
event next_round()
|
||||
{
|
||||
print "====";
|
||||
if ( Cluster::node == "manager" )
|
||||
Cluster::publish(Cluster::worker_topic, next_round);
|
||||
}
|
||||
|
||||
# Send by manager for termination purposes.
|
||||
event do_terminate()
|
||||
{
|
||||
terminate();
|
||||
}
|
||||
|
||||
event publish_do_terminate()
|
||||
{
|
||||
print "publish_do_terminate()";
|
||||
Cluster::publish(Cluster::worker_topic, do_terminate);
|
||||
}
|
||||
|
||||
# Ensure the manager terminates eventually.
|
||||
global nodes_down = 0;
|
||||
event Cluster::node_down(name: string, id: string)
|
||||
{
|
||||
++nodes_down;
|
||||
if ( nodes_down >= 2)
|
||||
terminate();
|
||||
}
|
||||
|
||||
# Gets the ball rolling
|
||||
event Cluster::Experimental::cluster_started()
|
||||
{
|
||||
if ( Cluster::local_node_type() == Cluster::MANAGER )
|
||||
event insert_indicator("1.2.3.4", "from-manager");
|
||||
}
|
||||
|
||||
global indicators_inserted: table[Intel::Type] of count &default=0;
|
||||
global indicator_removed: table[Intel::Type] of count &default=0;
|
||||
|
||||
hook Intel::indicator_inserted(indicator: string, indicator_type: Intel::Type)
|
||||
{
|
||||
print "Intel::indicator_inserted", indicator, indicator_type;
|
||||
++indicators_inserted[indicator_type];
|
||||
|
||||
# If worker-1 sees the first addr indicator (1.2.3.4), it re-inserts
|
||||
# it with a different source (from-worker) and also inserts a second
|
||||
# indicatore 1.2.3.5 with with sources "from-worker" and "from-manager";
|
||||
if ( Cluster::node == "worker-1" )
|
||||
{
|
||||
if ( indicators_inserted[test_intel_type] == 1 || indicators_inserted[test_intel_type] == 3 )
|
||||
{
|
||||
event insert_indicator("1.2.3.4", "from-worker");
|
||||
event insert_indicator("1.2.3.5", "from-worker");
|
||||
event insert_indicator("1.2.3.5", "from-manager");
|
||||
}
|
||||
}
|
||||
|
||||
# Once worker-2 has observed two or four indicators, it removes
|
||||
# all of them again!
|
||||
if ( Cluster::node == "worker-2" )
|
||||
{
|
||||
if ( indicators_inserted[test_intel_type] == 2 || indicators_inserted[test_intel_type] == 4 )
|
||||
{
|
||||
event remove_indicator("1.2.3.4", "from-manager");
|
||||
event remove_indicator("1.2.3.5", "from-manager");
|
||||
event remove_indicator("1.2.3.4", "from-worker");
|
||||
event remove_indicator("1.2.3.5", "from-worker");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
hook Intel::indicator_removed(indicator: string, indicator_type: Intel::Type)
|
||||
{
|
||||
print "Intel::indicator_removed", indicator, indicator_type;
|
||||
++indicator_removed[indicator_type];
|
||||
if ( Cluster::node == "manager" )
|
||||
{
|
||||
if ( indicator_removed[test_intel_type] == 2 )
|
||||
{
|
||||
# Trigger another round of inserts at the workers!
|
||||
event next_round();
|
||||
event insert_indicator("1.2.3.4", "from-manager");
|
||||
}
|
||||
else if ( indicator_removed[test_intel_type] == 4 )
|
||||
{
|
||||
event publish_do_terminate();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,155 @@
|
|||
# @TEST-DOC: Smoke test that the seen/smtp mime_end_entity() only runs when Intel::ADDR indicators are loaded and mime_end_entity() runs for a SMTP connection.
|
||||
#
|
||||
# @TEST-PORT: BROKER_MANAGER_PORT
|
||||
# @TEST-PORT: BROKER_WORKER1_PORT
|
||||
#
|
||||
# @TEST-EXEC: cp $FILES/broker/cluster-layout.zeek .
|
||||
#
|
||||
# @TEST-EXEC: zeek --parse-only %INPUT
|
||||
#
|
||||
# @TEST-EXEC: btest-bg-run manager ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager zeek -b %INPUT
|
||||
# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b %INPUT
|
||||
# @TEST-EXEC: btest-bg-wait 30
|
||||
|
||||
# @TEST-EXEC: btest-diff manager/.stdout
|
||||
# @TEST-EXEC: btest-diff worker-1/.stdout
|
||||
# @TEST-EXEC: btest-diff manager/intel.log
|
||||
|
||||
@load base/frameworks/cluster
|
||||
@load base/frameworks/intel
|
||||
|
||||
@load frameworks/intel/seen
|
||||
@load frameworks/intel/seen/manage-event-groups
|
||||
|
||||
@load frameworks/cluster/experimental
|
||||
|
||||
redef Log::default_rotation_interval = 0secs;
|
||||
|
||||
global addr_indicator = Intel::Item(
|
||||
$indicator="1.2.3.4",
|
||||
$indicator_type=Intel::ADDR,
|
||||
$meta=Intel::MetaData($source="source1"),
|
||||
);
|
||||
|
||||
function make_conn(): connection
|
||||
{
|
||||
local c = connection(
|
||||
$id = conn_id($orig_h=1.1.1.1, $orig_p=1/tcp,
|
||||
$resp_h=2.2.2.2, $resp_p=2/tcp, $proto=6),
|
||||
$orig = endpoint($size=1, $state=4, $flow_label=0),
|
||||
$resp = endpoint($size=1, $state=4, $flow_label=0),
|
||||
$start_time=double_to_time(1747323991.0),
|
||||
$duration=1sec,
|
||||
$service=set("smtp"),
|
||||
$history="ShAdDa",
|
||||
$uid="CHhAvVGS1DHFjwGM9",
|
||||
);
|
||||
|
||||
c$smtp = SMTP::Info(
|
||||
$ts=c$start_time,
|
||||
$uid=c$uid,
|
||||
$id=c$id,
|
||||
$trans_depth=1,
|
||||
$path=vector(1.2.3.4)
|
||||
);
|
||||
|
||||
c$smtp$x_originating_ip = 1.2.3.5;
|
||||
|
||||
return c;
|
||||
}
|
||||
|
||||
global current_step = 0;
|
||||
|
||||
global do_step: event(step: count);
|
||||
|
||||
event publish_do_step(step: count)
|
||||
{
|
||||
local topic = Cluster::local_node_type() == Cluster::MANAGER ? Cluster::worker_topic : Cluster::manager_topic;
|
||||
print fmt("publish do_step(%s) to %s", step, topic);
|
||||
Cluster::publish(topic, do_step, step);
|
||||
}
|
||||
|
||||
# Log Intel::seen_policy() invocations.
|
||||
#
|
||||
# The idea here is that if the seen event groups are disabled,
|
||||
# the Intel::seen_policy() hook isn't invoked at all. So we can
|
||||
# use that to verify that the corresponding events have actually
|
||||
# been disabled.
|
||||
hook Intel::seen_policy(s: Intel::Seen, found: bool)
|
||||
{
|
||||
print fmt("Intel::seen_policy(%s of %s, %s)",
|
||||
s?$host ? cat(s$host) : s$indicator,
|
||||
s?$host ? "Intel::ADDR" : cat(s$indicator_type),
|
||||
found);
|
||||
}
|
||||
|
||||
event Intel::match(s: Intel::Seen, items: set[Intel::Item])
|
||||
{
|
||||
print fmt("Intel::match: %s %s", s$indicator, s$indicator_type);
|
||||
if ( current_step == 2 )
|
||||
event do_step(4);
|
||||
}
|
||||
|
||||
event do_step(step: count)
|
||||
{
|
||||
current_step = step;
|
||||
print fmt("running do_step(%s)", step);
|
||||
|
||||
switch ( step ) {
|
||||
case 1: # worker
|
||||
local c1 = make_conn();
|
||||
event mime_end_entity(c1);
|
||||
event publish_do_step(2);
|
||||
break;
|
||||
case 2: # manager, insert a intel indicator
|
||||
Intel::insert(addr_indicator);
|
||||
event publish_do_step(3);
|
||||
break;
|
||||
case 3: # worker - should have an addr indicator now, match it.
|
||||
local c2 = make_conn();
|
||||
event mime_end_entity(c2);
|
||||
# no publish of step 4, see Intel::match() that drives it
|
||||
break;
|
||||
case 4: # manager waits for the match
|
||||
Intel::remove(addr_indicator);
|
||||
event publish_do_step(5);
|
||||
break;
|
||||
case 5: # worker - the ADDR groups are disabled again.
|
||||
local c3 = make_conn();
|
||||
event mime_end_entity(c3);
|
||||
event publish_do_step(6);
|
||||
break;
|
||||
case 6: # manager, done
|
||||
terminate();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
event Cluster::Experimental::cluster_started()
|
||||
{
|
||||
if ( Cluster::node == "worker-1" )
|
||||
event do_step(1);
|
||||
}
|
||||
|
||||
event Cluster::node_down(name: string, id: string)
|
||||
{
|
||||
terminate();
|
||||
}
|
||||
|
||||
# Output a few internal things for sanity. These aren't testing functionality,
|
||||
# but nice to have.
|
||||
module Intel;
|
||||
event Intel::match_remote(s: Intel::Seen)
|
||||
{
|
||||
print fmt("Intel::match_remote: %s %s", s$indicator, s$indicator_type);
|
||||
}
|
||||
|
||||
hook Intel::indicator_inserted(indicator: string, indicator_type: Intel::Type)
|
||||
{
|
||||
print fmt("Intel::indicator_inserted %s %s", indicator, indicator_type);
|
||||
}
|
||||
|
||||
hook Intel::indicator_removed(indicator: string, indicator_type: Intel::Type)
|
||||
{
|
||||
print fmt("Intel::indicator_removed %s %s", indicator, indicator_type);
|
||||
}
|
||||
5
testing/external/scripts/testing-setup.zeek
vendored
5
testing/external/scripts/testing-setup.zeek
vendored
|
|
@ -17,6 +17,11 @@ redef DPD::track_removed_services_in_connection=T;
|
|||
redef LogAscii::use_json = F;
|
||||
@endif
|
||||
|
||||
# The tests don't load intel data and so all Intel event groups are disabled
|
||||
# due to intel/seen/manage-event-groups being loaded by default. Disable that
|
||||
# functionality by default to cover execution in the intel/seen scripts.
|
||||
redef Intel::manage_seen_event_groups = F;
|
||||
|
||||
# The IMAP analyzer includes absolute filenames in its error messages,
|
||||
# exclude it for now from analyzer.log.
|
||||
# https://github.com/zeek/zeek/issues/2659
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue