diff --git a/CHANGES b/CHANGES index 68091d4919..4c7e23126b 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,29 @@ +8.0.0-dev.303 | 2025-06-02 09:51:53 +0200 + + * intel/seen/manage-event-groups: Policy script for toggling intel event groups (Arne Welzel, Corelight) + + Co-authored-by: Mohan Dhawan + + * intel: Add indicator_inserted and indicator_removed hooks (Arne Welzel, Corelight) + + This change adds two new hooks to the Intel framework that can be used + to intercept added and removed indicators and their type. + + These hooks are fairly low-level. One immediate use-case is to count the + number of indicators loaded per Intel::Type and enable and disable the + corresponding event groups of the intel/seen scripts. + + I attempted to gauge the overhead and while it's definitely there, loading + a file with ~500k DOMAIN entries takes somewhere around ~0.5 seconds hooks + when populated via the min_data_store store mechanism. While that + doesn't sound great, it actually takes the manager on my system 2.5 + seconds to serialize and Cluster::publish() the min_data_store alone + and its doing that serially for every active worker. Mostly to say that + the bigger overhead in that area on the manager doing redundant work + per worker. + + Co-authored-by: Mohan Dhawan + 8.0.0-dev.299 | 2025-05-30 13:06:42 -0700 * fix for crash when interpreting transformed ASTs that include multi-field record assignments/additions (Vern Paxson, Corelight) diff --git a/NEWS b/NEWS index 02cb56fd06..65ca985ca9 100644 --- a/NEWS +++ b/NEWS @@ -66,6 +66,34 @@ New Functionality - The FTP analyzer now supports explicit TLS via AUTH TLS. +- Two new script-level hooks in the Intel framework have been added. + + hook indicator_inserted(indicator_value: string, indicator_type: Intel::Type) + + hook indicator_removed(indicator_value: string, indicator_type: Intel::Type) + + These are reliably invoked on worker and manager nodes the first time an + indicator value is inserted into the store and once it has been completely + removed from the store. + +- The ``frameworks/intel/seen`` scripts have been annotated with event groups + and a new ``frameworks/intel/seen/manage-event-groups`` policy script added. + + The motivation is to allow Zeek distributors to load the ``intel/seen`` scripts + by default without incurring their event overhead when no Intel indicators are + loaded. Corresponding event handlers are enabled once the first Intel indicator + of a given ``Intel::Type`` is added. Event handlers are disabled when the last + indicator is removed, again. + + Note that the ``manage-event-groups`` script interacts with the ``Intel::seen_policy`` + hook: If no indicators for a given ``Intel::Type`` are loaded, the ``Intel::seen_policy`` + will not be invoked as the event handlers extracting indicators aren't executed. + + If you rely on the ``Intel::seen_policy`` hook to be invoked regardless of the + contents of the Intel store, do not load the ``manage-event-groups`` or set: + + redef Intel::manage_seen_event_groups = F; + Changed Functionality --------------------- diff --git a/VERSION b/VERSION index 72db25f0e8..1abe084d22 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -8.0.0-dev.299 +8.0.0-dev.303 diff --git a/scripts/base/frameworks/intel/cluster.zeek b/scripts/base/frameworks/intel/cluster.zeek index cd8a1c845c..4f7be311d9 100644 --- a/scripts/base/frameworks/intel/cluster.zeek +++ b/scripts/base/frameworks/intel/cluster.zeek @@ -105,10 +105,30 @@ event Intel::insert_indicator(item: Intel::Item) &priority=5 Intel::_insert(item, F); } +function invoke_indicator_hook(store: MinDataStore, h: hook(v: string, t: Intel::Type)) + { + for ( a in store$host_data ) + hook h(cat(a), Intel::ADDR); + + for ( sn in store$subnet_data) + hook h(cat(sn), Intel::SUBNET); + + for ( [indicator_value, indicator_type] in store$string_data ) + hook h(indicator_value, indicator_type); + } + # Handling of a complete MinDataStore snapshot +# +# Invoke the removed and inserted hooks using the old and new min data store +# instances, respectively. The way this event is used, the original +# min_data_store should essentially be empty. event new_min_data_store(store: MinDataStore) { + invoke_indicator_hook(min_data_store, Intel::indicator_removed); + min_data_store = store; + + invoke_indicator_hook(min_data_store, Intel::indicator_inserted); } @endif diff --git a/scripts/base/frameworks/intel/main.zeek b/scripts/base/frameworks/intel/main.zeek index 221b1be1d7..8231e4be8e 100644 --- a/scripts/base/frameworks/intel/main.zeek +++ b/scripts/base/frameworks/intel/main.zeek @@ -207,6 +207,35 @@ export { ## item: The intel item that should be inserted. global filter_item: hook(item: Intel::Item); + ## This hook is invoked when a new indicator has been inserted into + ## the min data store for the first time. + ## + ## Calls to :zeek:see:`Intel::seen` with a matching indicator value + ## and type will result in matches. + ## + ## Subsequent inserts of the same indicator type and value do not + ## invoke this hook. Breaking from this hook has no effect. + ## + ## indicator: The indicator value. + ## + ## indicator_type: The indicator type. + ## + ## .. zeek::see:: Intel::indicator_removed + global indicator_inserted: hook(indicator: string, indiator_type: Type); + + ## This hook is invoked when an indicator has been removed from + ## the min data store. + ## + ## After this hooks runs, :zeek:see:`Intel::seen` for the indicator + ## will not return any matches. Breaking from this hook has no effect. + ## + ## indicator: The indicator value. + ## + ## indicator_type: The indicator type. + ## + ## .. zeek::see:: Intel::indicator_inserted + global indicator_removed: hook(indicator: string, indiator_type: Type); + global log_intel: event(rec: Info); } @@ -507,18 +536,44 @@ function _insert(item: Item, first_dispatch: bool &default = T) # All intelligence is case insensitive at the moment. local lower_indicator = to_lower(item$indicator); + # Track if the indicator was inserted into the min_data_store. + # It's tempting to just use is_new above, but it seems that only works + # correctly on a worker if the manager never spuriously sends a + # Intel::insert_item(), so better to determine this locally based + # on the actual contents of the min_data_store. + local inserted = F; + local inserted_value = ""; + # Insert indicator into MinDataStore (might exist already). switch ( item$indicator_type ) { case ADDR: local host = to_addr(item$indicator); + if ( host !in min_data_store$host_data ) + { + inserted = T; + inserted_value = cat(host); + } + add min_data_store$host_data[host]; break; case SUBNET: local net = to_subnet(item$indicator); + if ( net !in min_data_store$subnet_data ) + { + inserted = T; + inserted_value = cat(net); + } + add min_data_store$subnet_data[net]; break; default: + if ( [lower_indicator, item$indicator_type] !in min_data_store$string_data ) + { + inserted = T; + inserted_value = lower_indicator; + } + add min_data_store$string_data[lower_indicator, item$indicator_type]; break; } @@ -533,6 +588,9 @@ function _insert(item: Item, first_dispatch: bool &default = T) # Announce a (possibly) new item if this is the first dispatch and # we know it is new or have to assume that on a worker. event Intel::new_item(item); + + if ( inserted ) + hook Intel::indicator_inserted(inserted_value, item$indicator_type); } function insert(item: Item) @@ -632,18 +690,43 @@ function remove(item: Item, purge_indicator: bool) # Handling of indicator removal in minimal data stores. event remove_indicator(item: Item) { + local removed = F; + local removed_value = ""; + switch ( item$indicator_type ) { case ADDR: local host = to_addr(item$indicator); + if ( host in min_data_store$host_data ) + { + removed = T; + removed_value = cat(host); + } + delete min_data_store$host_data[host]; break; case SUBNET: local net = to_subnet(item$indicator); + if ( net in min_data_store$subnet_data ) + { + removed = T; + removed_value = cat(net); + } + delete min_data_store$subnet_data[net]; break; default: - delete min_data_store$string_data[to_lower(item$indicator), item$indicator_type]; + local indicator_value = to_lower(item$indicator); + if ( [indicator_value, item$indicator_type] in min_data_store$string_data ) + { + removed = T; + removed_value = indicator_value; + } + + delete min_data_store$string_data[indicator_value, item$indicator_type]; break; } + + if ( removed ) + hook Intel::indicator_removed(removed_value, item$indicator_type); } diff --git a/scripts/policy/frameworks/intel/seen/manage-event-groups.zeek b/scripts/policy/frameworks/intel/seen/manage-event-groups.zeek new file mode 100644 index 0000000000..7ef05c1776 --- /dev/null +++ b/scripts/policy/frameworks/intel/seen/manage-event-groups.zeek @@ -0,0 +1,74 @@ +@load frameworks/intel/seen +@load base/frameworks/reporter + +module Intel; + +export { + ## Whether Intel event groups for the seen scripts are managed. + ## + ## When loading this script, by default, all :zeek:see:`Intel::Type` + ## event groups are disabled at startup and only enabled when indicators + ## of corresponding types are loaded into the Intel framework's store. + ## This allows to load the ``frameworks/intel/seen`` scripts without + ## incurring event handling overhead when no Intel indicators are loaded. + ## + ## One caveat is that the :zeek:see:`Intel::seen_policy` hook will not + ## be invoked for indicator types that are not at all in the Intel + ## framework's store. If you rely on :zeek:see:`Intel::seen_policy` to + ## find unmatched indicators, do not not load this script, set this + ## variable to ``F``, or insert dummy values of the types using + ## :zeek:see:`Intel::insert`. + const manage_seen_event_groups = T &redef; +} + +global intel_type_counts: table[Intel::Type] of count &default=0; + +event zeek_init() + { + # If the feature is disabled, don't act. + if ( ! manage_seen_event_groups ) + return; + + # Disable all Intel related event groups at startup. These + # are enabled again as soon as at least one indicator of the + # type is inserted. + for ( name in enum_names(Intel::Type) ) + { + if ( has_event_group(name) ) + disable_event_group(name); + } + } + +hook Intel::indicator_inserted(v: string, t: Intel::Type) + { + ++intel_type_counts[t]; + + if ( ! manage_seen_event_groups ) + return; + + + if ( intel_type_counts[t] == 1 ) + { + local name = cat(t); + + if ( has_event_group(name) ) + enable_event_group(name); + } + } + +hook Intel::indicator_removed(v: string, t: Intel::Type) + { + --intel_type_counts[t]; + + if ( ! manage_seen_event_groups ) + return; + + + if ( intel_type_counts[t] == 0 ) + { + local name = cat(t); + + if ( has_event_group(name) ) + disable_event_group(name); + } + } diff --git a/scripts/test-all-policy.zeek b/scripts/test-all-policy.zeek index b3cf8a5a9e..2358afda5b 100644 --- a/scripts/test-all-policy.zeek +++ b/scripts/test-all-policy.zeek @@ -59,6 +59,7 @@ @load frameworks/intel/seen/file-names.zeek @load frameworks/intel/seen/http-headers.zeek @load frameworks/intel/seen/http-url.zeek +@load frameworks/intel/seen/manage-event-groups.zeek @load frameworks/intel/seen/pubkey-hashes.zeek @load frameworks/intel/seen/smb-filenames.zeek @load frameworks/intel/seen/smtp-url-extraction.zeek diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/manager..stderr b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/manager..stderr new file mode 100644 index 0000000000..e3f6131b1d --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/manager..stderr @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +received termination signal diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/manager..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/manager..stdout new file mode 100644 index 0000000000..a6deb08e51 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/manager..stdout @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Intel::indicator_inserted, 1.2.3.4, Intel::ADDR +Intel::indicator_inserted, 1.2.3.5, Intel::ADDR +Intel::indicator_inserted, 192.168.0.0/16, Intel::SUBNET +Intel::indicator_inserted, putty, Intel::SOFTWARE +Intel::indicator_inserted, putty2, Intel::SOFTWARE diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/worker-1-run-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/worker-1-run-1..stdout new file mode 100644 index 0000000000..76dadf9ea9 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/worker-1-run-1..stdout @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Intel::new_min_data_store pre, [host_data={\x0a\x0a}, subnet_data={\x0a\x0a}, string_data={\x0a\x0a}] +Intel::new_min_data_store post, [host_data={\x0a\x0a}, subnet_data={\x0a\x0a}, string_data={\x0a\x0a}] +Intel::insert_indicator, 1.2.3.4, Intel::ADDR +Intel::indicator_inserted, 1.2.3.4, Intel::ADDR +Intel::insert_indicator, 1.2.3.5, Intel::ADDR +Intel::indicator_inserted, 1.2.3.5, Intel::ADDR +Intel::insert_indicator, 192.168.0.0/16, Intel::SUBNET +Intel::indicator_inserted, 192.168.0.0/16, Intel::SUBNET +Intel::insert_indicator, putty, Intel::SOFTWARE +Intel::indicator_inserted, putty, Intel::SOFTWARE +Intel::insert_indicator, putty2, Intel::SOFTWARE +Intel::indicator_inserted, putty2, Intel::SOFTWARE diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/worker-1-run-2..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/worker-1-run-2..stdout new file mode 100644 index 0000000000..34a62b5535 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/worker-1-run-2..stdout @@ -0,0 +1,8 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Intel::new_min_data_store pre, [host_data={\x0a\x0a}, subnet_data={\x0a\x0a}, string_data={\x0a\x0a}] +Intel::indicator_inserted, 1.2.3.5, Intel::ADDR +Intel::indicator_inserted, 1.2.3.4, Intel::ADDR +Intel::indicator_inserted, 192.168.0.0/16, Intel::SUBNET +Intel::indicator_inserted, putty, Intel::SOFTWARE +Intel::indicator_inserted, putty2, Intel::SOFTWARE +Intel::new_min_data_store post, [host_data={\x0a\x091.2.3.5,\x0a\x091.2.3.4\x0a}, subnet_data={\x0a\x09192.168.0.0/16\x0a}, string_data={\x0a\x09[putty, Intel::SOFTWARE] ,\x0a\x09[putty2, Intel::SOFTWARE] \x0a}] diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/worker-2..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/worker-2..stdout new file mode 100644 index 0000000000..76dadf9ea9 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/worker-2..stdout @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Intel::new_min_data_store pre, [host_data={\x0a\x0a}, subnet_data={\x0a\x0a}, string_data={\x0a\x0a}] +Intel::new_min_data_store post, [host_data={\x0a\x0a}, subnet_data={\x0a\x0a}, string_data={\x0a\x0a}] +Intel::insert_indicator, 1.2.3.4, Intel::ADDR +Intel::indicator_inserted, 1.2.3.4, Intel::ADDR +Intel::insert_indicator, 1.2.3.5, Intel::ADDR +Intel::indicator_inserted, 1.2.3.5, Intel::ADDR +Intel::insert_indicator, 192.168.0.0/16, Intel::SUBNET +Intel::indicator_inserted, 192.168.0.0/16, Intel::SUBNET +Intel::insert_indicator, putty, Intel::SOFTWARE +Intel::indicator_inserted, putty, Intel::SOFTWARE +Intel::insert_indicator, putty2, Intel::SOFTWARE +Intel::indicator_inserted, putty2, Intel::SOFTWARE diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/addr-indicator.manager..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/addr-indicator.manager..stdout new file mode 100644 index 0000000000..056bfcffb1 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/addr-indicator.manager..stdout @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +insert_addr, 1.2.3.4, from-manager +Intel::indicator_inserted, 1.2.3.4, Intel::ADDR +Intel::indicator_inserted, 1.2.3.5, Intel::ADDR +Intel::indicator_removed, 1.2.3.4, Intel::ADDR +Intel::indicator_removed, 1.2.3.5, Intel::ADDR +==== +insert_addr, 1.2.3.4, from-manager +Intel::indicator_inserted, 1.2.3.4, Intel::ADDR +Intel::indicator_inserted, 1.2.3.5, Intel::ADDR +Intel::indicator_removed, 1.2.3.4, Intel::ADDR +Intel::indicator_removed, 1.2.3.5, Intel::ADDR +publish_do_terminate() diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/addr-indicator.worker-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/addr-indicator.worker-1..stdout new file mode 100644 index 0000000000..e8124c5d40 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/addr-indicator.worker-1..stdout @@ -0,0 +1,16 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Intel::indicator_inserted, 1.2.3.4, Intel::ADDR +insert_addr, 1.2.3.4, from-worker +insert_addr, 1.2.3.5, from-worker +Intel::indicator_inserted, 1.2.3.5, Intel::ADDR +insert_addr, 1.2.3.5, from-manager +Intel::indicator_removed, 1.2.3.4, Intel::ADDR +Intel::indicator_removed, 1.2.3.5, Intel::ADDR +==== +Intel::indicator_inserted, 1.2.3.4, Intel::ADDR +insert_addr, 1.2.3.4, from-worker +insert_addr, 1.2.3.5, from-worker +Intel::indicator_inserted, 1.2.3.5, Intel::ADDR +insert_addr, 1.2.3.5, from-manager +Intel::indicator_removed, 1.2.3.4, Intel::ADDR +Intel::indicator_removed, 1.2.3.5, Intel::ADDR diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/addr-indicator.worker-2..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/addr-indicator.worker-2..stdout new file mode 100644 index 0000000000..3f3b1a0d22 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/addr-indicator.worker-2..stdout @@ -0,0 +1,18 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Intel::indicator_inserted, 1.2.3.4, Intel::ADDR +Intel::indicator_inserted, 1.2.3.5, Intel::ADDR +remove_addr, 1.2.3.4, from-manager +remove_addr, 1.2.3.5, from-manager +remove_addr, 1.2.3.4, from-worker +remove_addr, 1.2.3.5, from-worker +Intel::indicator_removed, 1.2.3.4, Intel::ADDR +Intel::indicator_removed, 1.2.3.5, Intel::ADDR +==== +Intel::indicator_inserted, 1.2.3.4, Intel::ADDR +Intel::indicator_inserted, 1.2.3.5, Intel::ADDR +remove_addr, 1.2.3.4, from-manager +remove_addr, 1.2.3.5, from-manager +remove_addr, 1.2.3.4, from-worker +remove_addr, 1.2.3.5, from-worker +Intel::indicator_removed, 1.2.3.4, Intel::ADDR +Intel::indicator_removed, 1.2.3.5, Intel::ADDR diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/software-indicator.manager..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/software-indicator.manager..stdout new file mode 100644 index 0000000000..979daf317c --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/software-indicator.manager..stdout @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +insert_software, software-1.2.3.4, from-manager +Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE +Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE +Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE +Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE +==== +insert_software, software-1.2.3.4, from-manager +Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE +Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE +Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE +Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE +publish_do_terminate() diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/software-indicator.worker-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/software-indicator.worker-1..stdout new file mode 100644 index 0000000000..3bdb022ec1 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/software-indicator.worker-1..stdout @@ -0,0 +1,16 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE +insert_software, software-1.2.3.4, from-worker +insert_software, software-1.2.3.5, from-worker +Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE +insert_software, software-1.2.3.5, from-manager +Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE +Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE +==== +Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE +insert_software, software-1.2.3.4, from-worker +insert_software, software-1.2.3.5, from-worker +Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE +insert_software, software-1.2.3.5, from-manager +Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE +Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/software-indicator.worker-2..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/software-indicator.worker-2..stdout new file mode 100644 index 0000000000..e68492f713 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/software-indicator.worker-2..stdout @@ -0,0 +1,18 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE +Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE +remove_software, software-1.2.3.4, from-manager +remove_software, software-1.2.3.5, from-manager +remove_software, software-1.2.3.4, from-worker +remove_software, software-1.2.3.5, from-worker +Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE +Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE +==== +Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE +Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE +remove_software, software-1.2.3.4, from-manager +remove_software, software-1.2.3.5, from-manager +remove_software, software-1.2.3.4, from-worker +remove_software, software-1.2.3.5, from-worker +Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE +Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/subnet-indicator.manager..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/subnet-indicator.manager..stdout new file mode 100644 index 0000000000..4b00589ebe --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/subnet-indicator.manager..stdout @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +insert_subnet, 1.2.3.4/32, from-manager +Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET +Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET +Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET +Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET +==== +insert_subnet, 1.2.3.4/32, from-manager +Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET +Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET +Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET +Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET +publish_do_terminate() diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/subnet-indicator.worker-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/subnet-indicator.worker-1..stdout new file mode 100644 index 0000000000..d00e765dd6 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/subnet-indicator.worker-1..stdout @@ -0,0 +1,16 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET +insert_subnet, 1.2.3.4/32, from-worker +insert_subnet, 1.2.3.5/32, from-worker +Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET +insert_subnet, 1.2.3.5/32, from-manager +Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET +Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET +==== +Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET +insert_subnet, 1.2.3.4/32, from-worker +insert_subnet, 1.2.3.5/32, from-worker +Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET +insert_subnet, 1.2.3.5/32, from-manager +Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET +Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/subnet-indicator.worker-2..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/subnet-indicator.worker-2..stdout new file mode 100644 index 0000000000..0e1780e0cc --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/subnet-indicator.worker-2..stdout @@ -0,0 +1,18 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET +Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET +remove_subnet, 1.2.3.4/32, from-manager +remove_subnet, 1.2.3.5/32, from-manager +remove_subnet, 1.2.3.4/32, from-worker +remove_subnet, 1.2.3.5/32, from-worker +Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET +Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET +==== +Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET +Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET +remove_subnet, 1.2.3.4/32, from-manager +remove_subnet, 1.2.3.5/32, from-manager +remove_subnet, 1.2.3.4/32, from-worker +remove_subnet, 1.2.3.5/32, from-worker +Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET +Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET diff --git a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.manage-event-groups.basic-smtp/manager..stdout b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.manage-event-groups.basic-smtp/manager..stdout new file mode 100644 index 0000000000..f593ee46d9 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.manage-event-groups.basic-smtp/manager..stdout @@ -0,0 +1,10 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +running do_step(2) +Intel::indicator_inserted 1.2.3.4 Intel::ADDR +publish do_step(3) to zeek/cluster/worker +Intel::match_remote: 1.2.3.4 Intel::ADDR +Intel::match: 1.2.3.4 Intel::ADDR +running do_step(4) +Intel::indicator_removed 1.2.3.4 Intel::ADDR +publish do_step(5) to zeek/cluster/worker +running do_step(6) diff --git a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.manage-event-groups.basic-smtp/manager.intel.log b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.manage-event-groups.basic-smtp/manager.intel.log new file mode 100644 index 0000000000..0494365703 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.manage-event-groups.basic-smtp/manager.intel.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path intel +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p seen.indicator seen.indicator_type seen.where seen.node matched sources fuid file_mime_type file_desc +#types time string addr port addr port string enum enum string set[enum] set[string] string string string +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 1.1.1.1 1 2.2.2.2 2 1.2.3.4 Intel::ADDR SMTP::IN_RECEIVED_HEADER worker-1 Intel::ADDR source1 - - - +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.manage-event-groups.basic-smtp/worker-1..stdout b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.manage-event-groups.basic-smtp/worker-1..stdout new file mode 100644 index 0000000000..73378256ee --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.manage-event-groups.basic-smtp/worker-1..stdout @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +running do_step(1) +publish do_step(2) to zeek/cluster/manager +Intel::indicator_inserted 1.2.3.4 Intel::ADDR +running do_step(3) +Intel::seen_policy(1.2.3.4 of Intel::ADDR, T) +Intel::seen_policy(1.2.3.5 of Intel::ADDR, F) +Intel::match_remote: 1.2.3.4 Intel::ADDR +Intel::indicator_removed 1.2.3.4 Intel::ADDR +running do_step(5) +publish do_step(6) to zeek/cluster/manager diff --git a/testing/btest/scripts/base/frameworks/intel/cluster-indicator-inserted-new-min-store.zeek b/testing/btest/scripts/base/frameworks/intel/cluster-indicator-inserted-new-min-store.zeek new file mode 100644 index 0000000000..81c171f5da --- /dev/null +++ b/testing/btest/scripts/base/frameworks/intel/cluster-indicator-inserted-new-min-store.zeek @@ -0,0 +1,121 @@ +# @TEST-DOC: Verify behavior of Intel:indicator_inserted() and Intel::indicator_removed() when a worker node restarts. +# +# @TEST-PORT: BROKER_MANAGER_PORT +# @TEST-PORT: BROKER_WORKER1_PORT +# @TEST-PORT: BROKER_WORKER2_PORT +# +# @TEST-EXEC: cp $FILES/broker/cluster-layout.zeek . +# +# @TEST-EXEC: zeek --parse-only %INPUT +# @TEST-EXEC: btest-bg-run manager ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager zeek -b %INPUT +# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b %INPUT +# @TEST-EXEC: btest-bg-run worker-2 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b %INPUT +# @TEST-EXEC: wait-for-file worker-1/DONE 30 +# @TEST-EXEC: mv worker-1 worker-1-run-1 +# @TEST-EXEC: rm -rf worker-1 +# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b %INPUT +# @TEST-EXEC: wait-for-file worker-1/DONE 30 +# @TEST-EXEC: btest-bg-wait 30 +# @TEST-EXEC: mv worker-1 worker-1-run-2 + +# @TEST-EXEC: btest-diff manager/.stdout +# @TEST-EXEC: btest-diff manager/.stderr +# @TEST-EXEC: btest-diff worker-1-run-1/.stdout +# @TEST-EXEC: btest-diff worker-1-run-2/.stdout +# @TEST-EXEC: btest-diff worker-2/.stdout +# +@load policy/frameworks/cluster/experimental +@load base/frameworks/intel + +# Ttoal of 5 unique indicators +# @TEST-START-FILE intel.dat +#fields indicator indicator_type meta.source meta.desc meta.url +1.2.3.4 Intel::ADDR source1 this host is bad http://some-data-distributor.com/1 +1.2.3.4 Intel::ADDR source2 this host is bad http://some-data-distributor.com/1 +1.2.3.5 Intel::ADDR source1 this host is bad http://some-data-distributor.com/1 +192.168.0.0/16 Intel::SUBNET source1 this network is bad http://some-data-distributor.com/2 +putty Intel::SOFTWARE source1 this software is bad http://some-data-distributor.com/2 +putty Intel::SOFTWARE source2 this software is bad http://some-data-distributor.com/2 +putty2 Intel::SOFTWARE source3 this software is bad http://some-data-distributor.com/2 +# @TEST-END-FILE + +redef Log::default_rotation_interval = 0sec; + +event Cluster::Experimental::cluster_started() + { + # Start reading the intel file on the manager once all workers are up. + if ( Cluster::node == "manager" ) + { + local source = "../intel.dat"; + Input::add_event([$source=source, + $reader=Input::READER_ASCII, + $mode=Input::REREAD, + $name=cat("intel-", source), + $fields=Intel::Item, + $ev=Intel::read_entry, + $error_ev=Intel::read_error]); + } + } + +# Send by manager for termination purposes. +event do_terminate() + { + terminate(); + } + +global worker1_down = 0; +global nodes_down = 0; + +event Cluster::node_down(name: string, id: string) + { + ++nodes_down; + if ( name == "worker-1") + ++worker1_down; + + if ( worker1_down == 2 ) + Cluster::publish(Cluster::worker_topic, do_terminate); + + if ( nodes_down >= 3 ) + terminate(); + } + +global total_indicators = 0; + +hook Intel::indicator_inserted(indicator: string, indicator_type: Intel::Type) + { + print "Intel::indicator_inserted", indicator, indicator_type; + ++total_indicators; + + # Once worker-1 has seen all the 5 indicators, write a DONE file and terminate() + if ( Cluster::node == "worker-1" && total_indicators == 5 ) + { + if ( ! piped_exec("touch DONE", "") ) + exit(1); + + terminate(); + } + } + +hook Intel::indicator_removed(indicator: string, indicator_type: Intel::Type) + { + print "Intel::indicator_removed", indicator, indicator_type; + } + +module Intel; + +# Internal events for easier grasping behavior. + +event Intel::insert_indicator(item: Intel::Item) &priority=10 + { + print "Intel::insert_indicator", item$indicator, item$indicator_type; + } + +event Intel::new_min_data_store(store: Intel::MinDataStore) &priority=10 + { + print "Intel::new_min_data_store pre", cat(Intel::min_data_store); + } + +event Intel::new_min_data_store(store: Intel::MinDataStore) &priority=-10 + { + print "Intel::new_min_data_store post", cat(Intel::min_data_store); + } diff --git a/testing/btest/scripts/base/frameworks/intel/cluster-indicator-inserted.zeek b/testing/btest/scripts/base/frameworks/intel/cluster-indicator-inserted.zeek new file mode 100644 index 0000000000..93e986f7dc --- /dev/null +++ b/testing/btest/scripts/base/frameworks/intel/cluster-indicator-inserted.zeek @@ -0,0 +1,193 @@ +# @TEST-DOC: Verify Intel::indicator_inserted() and Intel::indicator_removed() in a cluster setup with three different types of indicators. +# +# @TEST-PORT: BROKER_MANAGER_PORT +# @TEST-PORT: BROKER_WORKER1_PORT +# @TEST-PORT: BROKER_WORKER2_PORT +# +# @TEST-EXEC: cp $FILES/broker/cluster-layout.zeek . +# +# @TEST-EXEC: btest-bg-run manager ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager zeek -b ../addr-indicator.zeek %INPUT +# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b ../addr-indicator.zeek %INPUT +# @TEST-EXEC: btest-bg-run worker-2 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b ../addr-indicator.zeek %INPUT +# @TEST-EXEC: btest-bg-wait 30 +# @TEST-EXEC: mkdir addr-indicator/; cp -R manager worker-1 worker-2 ./addr-indicator/ +# @TEST-EXEC: btest-diff addr-indicator/manager/.stdout +# @TEST-EXEC: btest-diff addr-indicator/worker-1/.stdout +# @TEST-EXEC: btest-diff addr-indicator/worker-2/.stdout +# +# @TEST-EXEC: rm -rf manager worker-1 worker-2 +# +# @TEST-EXEC: btest-bg-run manager ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager zeek -b ../subnet-indicator.zeek %INPUT +# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b ../subnet-indicator.zeek %INPUT +# @TEST-EXEC: btest-bg-run worker-2 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b ../subnet-indicator.zeek %INPUT +# @TEST-EXEC: btest-bg-wait 30 +# @TEST-EXEC: mkdir subnet-indicator/; cp -R manager worker-1 worker-2 ./subnet-indicator/ +# @TEST-EXEC: btest-diff subnet-indicator/manager/.stdout +# @TEST-EXEC: btest-diff subnet-indicator/worker-1/.stdout +# @TEST-EXEC: btest-diff subnet-indicator/worker-2/.stdout + +# @TEST-EXEC: rm -rf manager worker-1 worker-2 +# +# @TEST-EXEC: btest-bg-run manager ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager zeek -b ../software-indicator.zeek %INPUT +# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b ../software-indicator.zeek %INPUT +# @TEST-EXEC: btest-bg-run worker-2 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b ../software-indicator.zeek %INPUT +# @TEST-EXEC: btest-bg-wait 30 +# @TEST-EXEC: mkdir software-indicator/; cp -R manager worker-1 worker-2 ./software-indicator/ +# @TEST-EXEC: btest-diff software-indicator/manager/.stdout +# @TEST-EXEC: btest-diff software-indicator/worker-1/.stdout +# @TEST-EXEC: btest-diff software-indicator/worker-2/.stdout +# +# @TEST-EXEC: rm -rf manager worker-1 worker-2 + +@load policy/frameworks/cluster/experimental +@load base/frameworks/intel + +redef Log::default_rotation_interval = 0sec; + +# @TEST-START-FILE addr-indicator.zeek +@load base/frameworks/intel + +const test_intel_type = Intel::ADDR; + +event remove_indicator(value: string, source: string) + { + print"remove_addr", value, source; + Intel::remove([$indicator=value, $indicator_type=Intel::ADDR, $meta=[$source=source]], F); + } + +event insert_indicator(value: string, source: string) + { + print "insert_addr", value, source; + Intel::insert([$indicator=value, $indicator_type=Intel::ADDR, $meta=[$source=source]]); + } +# @TEST-END-FILE + +# @TEST-START-FILE subnet-indicator.zeek +@load base/frameworks/intel + +const test_intel_type = Intel::SUBNET; + +event remove_indicator(value: string, source: string) + { + value = value + "/32"; # make the IP value from generic code a valid subnet + print"remove_subnet", value, source; + Intel::remove([$indicator=value, $indicator_type=Intel::SUBNET, $meta=[$source=source]], F); + } + +event insert_indicator(value: string, source: string) + { + value = value + "/32"; # make the IP value from generic code a valid subnet + print "insert_subnet", value , source; + Intel::insert([$indicator=value, $indicator_type=Intel::SUBNET, $meta=[$source=source]]); + } +# @TEST-END-FILE + +# @TEST-START-FILE software-indicator.zeek +@load base/frameworks/intel + +const test_intel_type = Intel::SOFTWARE; + +event remove_indicator(value: string, source: string) + { + value = "software-" + value; + print"remove_software", value, source; + Intel::remove([$indicator=value, $indicator_type=Intel::SOFTWARE, $meta=[$source=source]], F); + } + +event insert_indicator(value: string, source: string) + { + value = "software-" + value; + print "insert_software", value , source; + Intel::insert([$indicator=value, $indicator_type=Intel::SOFTWARE, $meta=[$source=source]]); + } +# @TEST-END-FILE + +# Helper event for printing on manager and worker. +event next_round() + { + print "===="; + if ( Cluster::node == "manager" ) + Cluster::publish(Cluster::worker_topic, next_round); + } + +# Send by manager for termination purposes. +event do_terminate() + { + terminate(); + } + +event publish_do_terminate() + { + print "publish_do_terminate()"; + Cluster::publish(Cluster::worker_topic, do_terminate); + } + +# Ensure the manager terminates eventually. +global nodes_down = 0; +event Cluster::node_down(name: string, id: string) + { + ++nodes_down; + if ( nodes_down >= 2) + terminate(); + } + +# Gets the ball rolling +event Cluster::Experimental::cluster_started() + { + if ( Cluster::local_node_type() == Cluster::MANAGER ) + event insert_indicator("1.2.3.4", "from-manager"); + } + +global indicators_inserted: table[Intel::Type] of count &default=0; +global indicator_removed: table[Intel::Type] of count &default=0; + +hook Intel::indicator_inserted(indicator: string, indicator_type: Intel::Type) + { + print "Intel::indicator_inserted", indicator, indicator_type; + ++indicators_inserted[indicator_type]; + + # If worker-1 sees the first addr indicator (1.2.3.4), it re-inserts + # it with a different source (from-worker) and also inserts a second + # indicatore 1.2.3.5 with with sources "from-worker" and "from-manager"; + if ( Cluster::node == "worker-1" ) + { + if ( indicators_inserted[test_intel_type] == 1 || indicators_inserted[test_intel_type] == 3 ) + { + event insert_indicator("1.2.3.4", "from-worker"); + event insert_indicator("1.2.3.5", "from-worker"); + event insert_indicator("1.2.3.5", "from-manager"); + } + } + + # Once worker-2 has observed two or four indicators, it removes + # all of them again! + if ( Cluster::node == "worker-2" ) + { + if ( indicators_inserted[test_intel_type] == 2 || indicators_inserted[test_intel_type] == 4 ) + { + event remove_indicator("1.2.3.4", "from-manager"); + event remove_indicator("1.2.3.5", "from-manager"); + event remove_indicator("1.2.3.4", "from-worker"); + event remove_indicator("1.2.3.5", "from-worker"); + } + } + } + +hook Intel::indicator_removed(indicator: string, indicator_type: Intel::Type) + { + print "Intel::indicator_removed", indicator, indicator_type; + ++indicator_removed[indicator_type]; + if ( Cluster::node == "manager" ) + { + if ( indicator_removed[test_intel_type] == 2 ) + { + # Trigger another round of inserts at the workers! + event next_round(); + event insert_indicator("1.2.3.4", "from-manager"); + } + else if ( indicator_removed[test_intel_type] == 4 ) + { + event publish_do_terminate(); + } + } + } diff --git a/testing/btest/scripts/policy/frameworks/intel/seen/manage-event-groups/basic-smtp.zeek b/testing/btest/scripts/policy/frameworks/intel/seen/manage-event-groups/basic-smtp.zeek new file mode 100644 index 0000000000..fd779b11d2 --- /dev/null +++ b/testing/btest/scripts/policy/frameworks/intel/seen/manage-event-groups/basic-smtp.zeek @@ -0,0 +1,155 @@ +# @TEST-DOC: Smoke test that the seen/smtp mime_end_entity() only runs when Intel::ADDR indicators are loaded and mime_end_entity() runs for a SMTP connection. +# +# @TEST-PORT: BROKER_MANAGER_PORT +# @TEST-PORT: BROKER_WORKER1_PORT +# +# @TEST-EXEC: cp $FILES/broker/cluster-layout.zeek . +# +# @TEST-EXEC: zeek --parse-only %INPUT +# +# @TEST-EXEC: btest-bg-run manager ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager zeek -b %INPUT +# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b %INPUT +# @TEST-EXEC: btest-bg-wait 30 + +# @TEST-EXEC: btest-diff manager/.stdout +# @TEST-EXEC: btest-diff worker-1/.stdout +# @TEST-EXEC: btest-diff manager/intel.log + +@load base/frameworks/cluster +@load base/frameworks/intel + +@load frameworks/intel/seen +@load frameworks/intel/seen/manage-event-groups + +@load frameworks/cluster/experimental + +redef Log::default_rotation_interval = 0secs; + +global addr_indicator = Intel::Item( + $indicator="1.2.3.4", + $indicator_type=Intel::ADDR, + $meta=Intel::MetaData($source="source1"), +); + +function make_conn(): connection + { + local c = connection( + $id = conn_id($orig_h=1.1.1.1, $orig_p=1/tcp, + $resp_h=2.2.2.2, $resp_p=2/tcp, $proto=6), + $orig = endpoint($size=1, $state=4, $flow_label=0), + $resp = endpoint($size=1, $state=4, $flow_label=0), + $start_time=double_to_time(1747323991.0), + $duration=1sec, + $service=set("smtp"), + $history="ShAdDa", + $uid="CHhAvVGS1DHFjwGM9", + ); + + c$smtp = SMTP::Info( + $ts=c$start_time, + $uid=c$uid, + $id=c$id, + $trans_depth=1, + $path=vector(1.2.3.4) + ); + + c$smtp$x_originating_ip = 1.2.3.5; + + return c; + } + +global current_step = 0; + +global do_step: event(step: count); + +event publish_do_step(step: count) + { + local topic = Cluster::local_node_type() == Cluster::MANAGER ? Cluster::worker_topic : Cluster::manager_topic; + print fmt("publish do_step(%s) to %s", step, topic); + Cluster::publish(topic, do_step, step); + } + +# Log Intel::seen_policy() invocations. +# +# The idea here is that if the seen event groups are disabled, +# the Intel::seen_policy() hook isn't invoked at all. So we can +# use that to verify that the corresponding events have actually +# been disabled. +hook Intel::seen_policy(s: Intel::Seen, found: bool) + { + print fmt("Intel::seen_policy(%s of %s, %s)", + s?$host ? cat(s$host) : s$indicator, + s?$host ? "Intel::ADDR" : cat(s$indicator_type), + found); + } + +event Intel::match(s: Intel::Seen, items: set[Intel::Item]) + { + print fmt("Intel::match: %s %s", s$indicator, s$indicator_type); + if ( current_step == 2 ) + event do_step(4); + } + +event do_step(step: count) + { + current_step = step; + print fmt("running do_step(%s)", step); + + switch ( step ) { + case 1: # worker + local c1 = make_conn(); + event mime_end_entity(c1); + event publish_do_step(2); + break; + case 2: # manager, insert a intel indicator + Intel::insert(addr_indicator); + event publish_do_step(3); + break; + case 3: # worker - should have an addr indicator now, match it. + local c2 = make_conn(); + event mime_end_entity(c2); + # no publish of step 4, see Intel::match() that drives it + break; + case 4: # manager waits for the match + Intel::remove(addr_indicator); + event publish_do_step(5); + break; + case 5: # worker - the ADDR groups are disabled again. + local c3 = make_conn(); + event mime_end_entity(c3); + event publish_do_step(6); + break; + case 6: # manager, done + terminate(); + break; + } + } + +event Cluster::Experimental::cluster_started() + { + if ( Cluster::node == "worker-1" ) + event do_step(1); + } + +event Cluster::node_down(name: string, id: string) + { + terminate(); + } + +# Output a few internal things for sanity. These aren't testing functionality, +# but nice to have. +module Intel; +event Intel::match_remote(s: Intel::Seen) + { + print fmt("Intel::match_remote: %s %s", s$indicator, s$indicator_type); + } + +hook Intel::indicator_inserted(indicator: string, indicator_type: Intel::Type) + { + print fmt("Intel::indicator_inserted %s %s", indicator, indicator_type); + } + +hook Intel::indicator_removed(indicator: string, indicator_type: Intel::Type) + { + print fmt("Intel::indicator_removed %s %s", indicator, indicator_type); + } diff --git a/testing/external/scripts/testing-setup.zeek b/testing/external/scripts/testing-setup.zeek index 56d80ae873..06f161347d 100644 --- a/testing/external/scripts/testing-setup.zeek +++ b/testing/external/scripts/testing-setup.zeek @@ -17,6 +17,11 @@ redef DPD::track_removed_services_in_connection=T; redef LogAscii::use_json = F; @endif +# The tests don't load intel data and so all Intel event groups are disabled +# due to intel/seen/manage-event-groups being loaded by default. Disable that +# functionality by default to cover execution in the intel/seen scripts. +redef Intel::manage_seen_event_groups = F; + # The IMAP analyzer includes absolute filenames in its error messages, # exclude it for now from analyzer.log. # https://github.com/zeek/zeek/issues/2659