diff --git a/NEWS b/NEWS index 02cb56fd06..38bdf6a316 100644 --- a/NEWS +++ b/NEWS @@ -66,6 +66,16 @@ New Functionality - The FTP analyzer now supports explicit TLS via AUTH TLS. +- Two new script-level hooks in the Intel framework have been added. + + hook indicator_inserted(indicator_value: string, indicator_type: Intel::Type) + + hook indicator_removed(indicator_value: string, indicator_type: Intel::Type) + + These are reliably invoked on worker and manager nodes the first time an + indicator value is inserted into the store and once it has been completely + removed from the store. + Changed Functionality --------------------- diff --git a/scripts/base/frameworks/intel/cluster.zeek b/scripts/base/frameworks/intel/cluster.zeek index cd8a1c845c..4f7be311d9 100644 --- a/scripts/base/frameworks/intel/cluster.zeek +++ b/scripts/base/frameworks/intel/cluster.zeek @@ -105,10 +105,30 @@ event Intel::insert_indicator(item: Intel::Item) &priority=5 Intel::_insert(item, F); } +function invoke_indicator_hook(store: MinDataStore, h: hook(v: string, t: Intel::Type)) + { + for ( a in store$host_data ) + hook h(cat(a), Intel::ADDR); + + for ( sn in store$subnet_data) + hook h(cat(sn), Intel::SUBNET); + + for ( [indicator_value, indicator_type] in store$string_data ) + hook h(indicator_value, indicator_type); + } + # Handling of a complete MinDataStore snapshot +# +# Invoke the removed and inserted hooks using the old and new min data store +# instances, respectively. The way this event is used, the original +# min_data_store should essentially be empty. event new_min_data_store(store: MinDataStore) { + invoke_indicator_hook(min_data_store, Intel::indicator_removed); + min_data_store = store; + + invoke_indicator_hook(min_data_store, Intel::indicator_inserted); } @endif diff --git a/scripts/base/frameworks/intel/main.zeek b/scripts/base/frameworks/intel/main.zeek index 221b1be1d7..8231e4be8e 100644 --- a/scripts/base/frameworks/intel/main.zeek +++ b/scripts/base/frameworks/intel/main.zeek @@ -207,6 +207,35 @@ export { ## item: The intel item that should be inserted. global filter_item: hook(item: Intel::Item); + ## This hook is invoked when a new indicator has been inserted into + ## the min data store for the first time. + ## + ## Calls to :zeek:see:`Intel::seen` with a matching indicator value + ## and type will result in matches. + ## + ## Subsequent inserts of the same indicator type and value do not + ## invoke this hook. Breaking from this hook has no effect. + ## + ## indicator: The indicator value. + ## + ## indicator_type: The indicator type. + ## + ## .. zeek::see:: Intel::indicator_removed + global indicator_inserted: hook(indicator: string, indiator_type: Type); + + ## This hook is invoked when an indicator has been removed from + ## the min data store. + ## + ## After this hooks runs, :zeek:see:`Intel::seen` for the indicator + ## will not return any matches. Breaking from this hook has no effect. + ## + ## indicator: The indicator value. + ## + ## indicator_type: The indicator type. + ## + ## .. zeek::see:: Intel::indicator_inserted + global indicator_removed: hook(indicator: string, indiator_type: Type); + global log_intel: event(rec: Info); } @@ -507,18 +536,44 @@ function _insert(item: Item, first_dispatch: bool &default = T) # All intelligence is case insensitive at the moment. local lower_indicator = to_lower(item$indicator); + # Track if the indicator was inserted into the min_data_store. + # It's tempting to just use is_new above, but it seems that only works + # correctly on a worker if the manager never spuriously sends a + # Intel::insert_item(), so better to determine this locally based + # on the actual contents of the min_data_store. + local inserted = F; + local inserted_value = ""; + # Insert indicator into MinDataStore (might exist already). switch ( item$indicator_type ) { case ADDR: local host = to_addr(item$indicator); + if ( host !in min_data_store$host_data ) + { + inserted = T; + inserted_value = cat(host); + } + add min_data_store$host_data[host]; break; case SUBNET: local net = to_subnet(item$indicator); + if ( net !in min_data_store$subnet_data ) + { + inserted = T; + inserted_value = cat(net); + } + add min_data_store$subnet_data[net]; break; default: + if ( [lower_indicator, item$indicator_type] !in min_data_store$string_data ) + { + inserted = T; + inserted_value = lower_indicator; + } + add min_data_store$string_data[lower_indicator, item$indicator_type]; break; } @@ -533,6 +588,9 @@ function _insert(item: Item, first_dispatch: bool &default = T) # Announce a (possibly) new item if this is the first dispatch and # we know it is new or have to assume that on a worker. event Intel::new_item(item); + + if ( inserted ) + hook Intel::indicator_inserted(inserted_value, item$indicator_type); } function insert(item: Item) @@ -632,18 +690,43 @@ function remove(item: Item, purge_indicator: bool) # Handling of indicator removal in minimal data stores. event remove_indicator(item: Item) { + local removed = F; + local removed_value = ""; + switch ( item$indicator_type ) { case ADDR: local host = to_addr(item$indicator); + if ( host in min_data_store$host_data ) + { + removed = T; + removed_value = cat(host); + } + delete min_data_store$host_data[host]; break; case SUBNET: local net = to_subnet(item$indicator); + if ( net in min_data_store$subnet_data ) + { + removed = T; + removed_value = cat(net); + } + delete min_data_store$subnet_data[net]; break; default: - delete min_data_store$string_data[to_lower(item$indicator), item$indicator_type]; + local indicator_value = to_lower(item$indicator); + if ( [indicator_value, item$indicator_type] in min_data_store$string_data ) + { + removed = T; + removed_value = indicator_value; + } + + delete min_data_store$string_data[indicator_value, item$indicator_type]; break; } + + if ( removed ) + hook Intel::indicator_removed(removed_value, item$indicator_type); } diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/manager..stderr b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/manager..stderr new file mode 100644 index 0000000000..e3f6131b1d --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/manager..stderr @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +received termination signal diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/manager..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/manager..stdout new file mode 100644 index 0000000000..a6deb08e51 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/manager..stdout @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Intel::indicator_inserted, 1.2.3.4, Intel::ADDR +Intel::indicator_inserted, 1.2.3.5, Intel::ADDR +Intel::indicator_inserted, 192.168.0.0/16, Intel::SUBNET +Intel::indicator_inserted, putty, Intel::SOFTWARE +Intel::indicator_inserted, putty2, Intel::SOFTWARE diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/worker-1-run-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/worker-1-run-1..stdout new file mode 100644 index 0000000000..76dadf9ea9 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/worker-1-run-1..stdout @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Intel::new_min_data_store pre, [host_data={\x0a\x0a}, subnet_data={\x0a\x0a}, string_data={\x0a\x0a}] +Intel::new_min_data_store post, [host_data={\x0a\x0a}, subnet_data={\x0a\x0a}, string_data={\x0a\x0a}] +Intel::insert_indicator, 1.2.3.4, Intel::ADDR +Intel::indicator_inserted, 1.2.3.4, Intel::ADDR +Intel::insert_indicator, 1.2.3.5, Intel::ADDR +Intel::indicator_inserted, 1.2.3.5, Intel::ADDR +Intel::insert_indicator, 192.168.0.0/16, Intel::SUBNET +Intel::indicator_inserted, 192.168.0.0/16, Intel::SUBNET +Intel::insert_indicator, putty, Intel::SOFTWARE +Intel::indicator_inserted, putty, Intel::SOFTWARE +Intel::insert_indicator, putty2, Intel::SOFTWARE +Intel::indicator_inserted, putty2, Intel::SOFTWARE diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/worker-1-run-2..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/worker-1-run-2..stdout new file mode 100644 index 0000000000..34a62b5535 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/worker-1-run-2..stdout @@ -0,0 +1,8 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Intel::new_min_data_store pre, [host_data={\x0a\x0a}, subnet_data={\x0a\x0a}, string_data={\x0a\x0a}] +Intel::indicator_inserted, 1.2.3.5, Intel::ADDR +Intel::indicator_inserted, 1.2.3.4, Intel::ADDR +Intel::indicator_inserted, 192.168.0.0/16, Intel::SUBNET +Intel::indicator_inserted, putty, Intel::SOFTWARE +Intel::indicator_inserted, putty2, Intel::SOFTWARE +Intel::new_min_data_store post, [host_data={\x0a\x091.2.3.5,\x0a\x091.2.3.4\x0a}, subnet_data={\x0a\x09192.168.0.0/16\x0a}, string_data={\x0a\x09[putty, Intel::SOFTWARE] ,\x0a\x09[putty2, Intel::SOFTWARE] \x0a}] diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/worker-2..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/worker-2..stdout new file mode 100644 index 0000000000..76dadf9ea9 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/worker-2..stdout @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Intel::new_min_data_store pre, [host_data={\x0a\x0a}, subnet_data={\x0a\x0a}, string_data={\x0a\x0a}] +Intel::new_min_data_store post, [host_data={\x0a\x0a}, subnet_data={\x0a\x0a}, string_data={\x0a\x0a}] +Intel::insert_indicator, 1.2.3.4, Intel::ADDR +Intel::indicator_inserted, 1.2.3.4, Intel::ADDR +Intel::insert_indicator, 1.2.3.5, Intel::ADDR +Intel::indicator_inserted, 1.2.3.5, Intel::ADDR +Intel::insert_indicator, 192.168.0.0/16, Intel::SUBNET +Intel::indicator_inserted, 192.168.0.0/16, Intel::SUBNET +Intel::insert_indicator, putty, Intel::SOFTWARE +Intel::indicator_inserted, putty, Intel::SOFTWARE +Intel::insert_indicator, putty2, Intel::SOFTWARE +Intel::indicator_inserted, putty2, Intel::SOFTWARE diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/addr-indicator.manager..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/addr-indicator.manager..stdout new file mode 100644 index 0000000000..056bfcffb1 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/addr-indicator.manager..stdout @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +insert_addr, 1.2.3.4, from-manager +Intel::indicator_inserted, 1.2.3.4, Intel::ADDR +Intel::indicator_inserted, 1.2.3.5, Intel::ADDR +Intel::indicator_removed, 1.2.3.4, Intel::ADDR +Intel::indicator_removed, 1.2.3.5, Intel::ADDR +==== +insert_addr, 1.2.3.4, from-manager +Intel::indicator_inserted, 1.2.3.4, Intel::ADDR +Intel::indicator_inserted, 1.2.3.5, Intel::ADDR +Intel::indicator_removed, 1.2.3.4, Intel::ADDR +Intel::indicator_removed, 1.2.3.5, Intel::ADDR +publish_do_terminate() diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/addr-indicator.worker-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/addr-indicator.worker-1..stdout new file mode 100644 index 0000000000..e8124c5d40 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/addr-indicator.worker-1..stdout @@ -0,0 +1,16 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Intel::indicator_inserted, 1.2.3.4, Intel::ADDR +insert_addr, 1.2.3.4, from-worker +insert_addr, 1.2.3.5, from-worker +Intel::indicator_inserted, 1.2.3.5, Intel::ADDR +insert_addr, 1.2.3.5, from-manager +Intel::indicator_removed, 1.2.3.4, Intel::ADDR +Intel::indicator_removed, 1.2.3.5, Intel::ADDR +==== +Intel::indicator_inserted, 1.2.3.4, Intel::ADDR +insert_addr, 1.2.3.4, from-worker +insert_addr, 1.2.3.5, from-worker +Intel::indicator_inserted, 1.2.3.5, Intel::ADDR +insert_addr, 1.2.3.5, from-manager +Intel::indicator_removed, 1.2.3.4, Intel::ADDR +Intel::indicator_removed, 1.2.3.5, Intel::ADDR diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/addr-indicator.worker-2..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/addr-indicator.worker-2..stdout new file mode 100644 index 0000000000..3f3b1a0d22 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/addr-indicator.worker-2..stdout @@ -0,0 +1,18 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Intel::indicator_inserted, 1.2.3.4, Intel::ADDR +Intel::indicator_inserted, 1.2.3.5, Intel::ADDR +remove_addr, 1.2.3.4, from-manager +remove_addr, 1.2.3.5, from-manager +remove_addr, 1.2.3.4, from-worker +remove_addr, 1.2.3.5, from-worker +Intel::indicator_removed, 1.2.3.4, Intel::ADDR +Intel::indicator_removed, 1.2.3.5, Intel::ADDR +==== +Intel::indicator_inserted, 1.2.3.4, Intel::ADDR +Intel::indicator_inserted, 1.2.3.5, Intel::ADDR +remove_addr, 1.2.3.4, from-manager +remove_addr, 1.2.3.5, from-manager +remove_addr, 1.2.3.4, from-worker +remove_addr, 1.2.3.5, from-worker +Intel::indicator_removed, 1.2.3.4, Intel::ADDR +Intel::indicator_removed, 1.2.3.5, Intel::ADDR diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/software-indicator.manager..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/software-indicator.manager..stdout new file mode 100644 index 0000000000..979daf317c --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/software-indicator.manager..stdout @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +insert_software, software-1.2.3.4, from-manager +Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE +Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE +Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE +Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE +==== +insert_software, software-1.2.3.4, from-manager +Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE +Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE +Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE +Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE +publish_do_terminate() diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/software-indicator.worker-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/software-indicator.worker-1..stdout new file mode 100644 index 0000000000..3bdb022ec1 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/software-indicator.worker-1..stdout @@ -0,0 +1,16 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE +insert_software, software-1.2.3.4, from-worker +insert_software, software-1.2.3.5, from-worker +Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE +insert_software, software-1.2.3.5, from-manager +Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE +Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE +==== +Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE +insert_software, software-1.2.3.4, from-worker +insert_software, software-1.2.3.5, from-worker +Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE +insert_software, software-1.2.3.5, from-manager +Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE +Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/software-indicator.worker-2..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/software-indicator.worker-2..stdout new file mode 100644 index 0000000000..e68492f713 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/software-indicator.worker-2..stdout @@ -0,0 +1,18 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE +Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE +remove_software, software-1.2.3.4, from-manager +remove_software, software-1.2.3.5, from-manager +remove_software, software-1.2.3.4, from-worker +remove_software, software-1.2.3.5, from-worker +Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE +Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE +==== +Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE +Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE +remove_software, software-1.2.3.4, from-manager +remove_software, software-1.2.3.5, from-manager +remove_software, software-1.2.3.4, from-worker +remove_software, software-1.2.3.5, from-worker +Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE +Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/subnet-indicator.manager..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/subnet-indicator.manager..stdout new file mode 100644 index 0000000000..4b00589ebe --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/subnet-indicator.manager..stdout @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +insert_subnet, 1.2.3.4/32, from-manager +Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET +Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET +Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET +Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET +==== +insert_subnet, 1.2.3.4/32, from-manager +Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET +Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET +Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET +Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET +publish_do_terminate() diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/subnet-indicator.worker-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/subnet-indicator.worker-1..stdout new file mode 100644 index 0000000000..d00e765dd6 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/subnet-indicator.worker-1..stdout @@ -0,0 +1,16 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET +insert_subnet, 1.2.3.4/32, from-worker +insert_subnet, 1.2.3.5/32, from-worker +Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET +insert_subnet, 1.2.3.5/32, from-manager +Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET +Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET +==== +Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET +insert_subnet, 1.2.3.4/32, from-worker +insert_subnet, 1.2.3.5/32, from-worker +Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET +insert_subnet, 1.2.3.5/32, from-manager +Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET +Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/subnet-indicator.worker-2..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/subnet-indicator.worker-2..stdout new file mode 100644 index 0000000000..0e1780e0cc --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/subnet-indicator.worker-2..stdout @@ -0,0 +1,18 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET +Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET +remove_subnet, 1.2.3.4/32, from-manager +remove_subnet, 1.2.3.5/32, from-manager +remove_subnet, 1.2.3.4/32, from-worker +remove_subnet, 1.2.3.5/32, from-worker +Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET +Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET +==== +Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET +Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET +remove_subnet, 1.2.3.4/32, from-manager +remove_subnet, 1.2.3.5/32, from-manager +remove_subnet, 1.2.3.4/32, from-worker +remove_subnet, 1.2.3.5/32, from-worker +Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET +Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET diff --git a/testing/btest/scripts/base/frameworks/intel/cluster-indicator-inserted-new-min-store.zeek b/testing/btest/scripts/base/frameworks/intel/cluster-indicator-inserted-new-min-store.zeek new file mode 100644 index 0000000000..81c171f5da --- /dev/null +++ b/testing/btest/scripts/base/frameworks/intel/cluster-indicator-inserted-new-min-store.zeek @@ -0,0 +1,121 @@ +# @TEST-DOC: Verify behavior of Intel:indicator_inserted() and Intel::indicator_removed() when a worker node restarts. +# +# @TEST-PORT: BROKER_MANAGER_PORT +# @TEST-PORT: BROKER_WORKER1_PORT +# @TEST-PORT: BROKER_WORKER2_PORT +# +# @TEST-EXEC: cp $FILES/broker/cluster-layout.zeek . +# +# @TEST-EXEC: zeek --parse-only %INPUT +# @TEST-EXEC: btest-bg-run manager ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager zeek -b %INPUT +# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b %INPUT +# @TEST-EXEC: btest-bg-run worker-2 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b %INPUT +# @TEST-EXEC: wait-for-file worker-1/DONE 30 +# @TEST-EXEC: mv worker-1 worker-1-run-1 +# @TEST-EXEC: rm -rf worker-1 +# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b %INPUT +# @TEST-EXEC: wait-for-file worker-1/DONE 30 +# @TEST-EXEC: btest-bg-wait 30 +# @TEST-EXEC: mv worker-1 worker-1-run-2 + +# @TEST-EXEC: btest-diff manager/.stdout +# @TEST-EXEC: btest-diff manager/.stderr +# @TEST-EXEC: btest-diff worker-1-run-1/.stdout +# @TEST-EXEC: btest-diff worker-1-run-2/.stdout +# @TEST-EXEC: btest-diff worker-2/.stdout +# +@load policy/frameworks/cluster/experimental +@load base/frameworks/intel + +# Ttoal of 5 unique indicators +# @TEST-START-FILE intel.dat +#fields indicator indicator_type meta.source meta.desc meta.url +1.2.3.4 Intel::ADDR source1 this host is bad http://some-data-distributor.com/1 +1.2.3.4 Intel::ADDR source2 this host is bad http://some-data-distributor.com/1 +1.2.3.5 Intel::ADDR source1 this host is bad http://some-data-distributor.com/1 +192.168.0.0/16 Intel::SUBNET source1 this network is bad http://some-data-distributor.com/2 +putty Intel::SOFTWARE source1 this software is bad http://some-data-distributor.com/2 +putty Intel::SOFTWARE source2 this software is bad http://some-data-distributor.com/2 +putty2 Intel::SOFTWARE source3 this software is bad http://some-data-distributor.com/2 +# @TEST-END-FILE + +redef Log::default_rotation_interval = 0sec; + +event Cluster::Experimental::cluster_started() + { + # Start reading the intel file on the manager once all workers are up. + if ( Cluster::node == "manager" ) + { + local source = "../intel.dat"; + Input::add_event([$source=source, + $reader=Input::READER_ASCII, + $mode=Input::REREAD, + $name=cat("intel-", source), + $fields=Intel::Item, + $ev=Intel::read_entry, + $error_ev=Intel::read_error]); + } + } + +# Send by manager for termination purposes. +event do_terminate() + { + terminate(); + } + +global worker1_down = 0; +global nodes_down = 0; + +event Cluster::node_down(name: string, id: string) + { + ++nodes_down; + if ( name == "worker-1") + ++worker1_down; + + if ( worker1_down == 2 ) + Cluster::publish(Cluster::worker_topic, do_terminate); + + if ( nodes_down >= 3 ) + terminate(); + } + +global total_indicators = 0; + +hook Intel::indicator_inserted(indicator: string, indicator_type: Intel::Type) + { + print "Intel::indicator_inserted", indicator, indicator_type; + ++total_indicators; + + # Once worker-1 has seen all the 5 indicators, write a DONE file and terminate() + if ( Cluster::node == "worker-1" && total_indicators == 5 ) + { + if ( ! piped_exec("touch DONE", "") ) + exit(1); + + terminate(); + } + } + +hook Intel::indicator_removed(indicator: string, indicator_type: Intel::Type) + { + print "Intel::indicator_removed", indicator, indicator_type; + } + +module Intel; + +# Internal events for easier grasping behavior. + +event Intel::insert_indicator(item: Intel::Item) &priority=10 + { + print "Intel::insert_indicator", item$indicator, item$indicator_type; + } + +event Intel::new_min_data_store(store: Intel::MinDataStore) &priority=10 + { + print "Intel::new_min_data_store pre", cat(Intel::min_data_store); + } + +event Intel::new_min_data_store(store: Intel::MinDataStore) &priority=-10 + { + print "Intel::new_min_data_store post", cat(Intel::min_data_store); + } diff --git a/testing/btest/scripts/base/frameworks/intel/cluster-indicator-inserted.zeek b/testing/btest/scripts/base/frameworks/intel/cluster-indicator-inserted.zeek new file mode 100644 index 0000000000..93e986f7dc --- /dev/null +++ b/testing/btest/scripts/base/frameworks/intel/cluster-indicator-inserted.zeek @@ -0,0 +1,193 @@ +# @TEST-DOC: Verify Intel::indicator_inserted() and Intel::indicator_removed() in a cluster setup with three different types of indicators. +# +# @TEST-PORT: BROKER_MANAGER_PORT +# @TEST-PORT: BROKER_WORKER1_PORT +# @TEST-PORT: BROKER_WORKER2_PORT +# +# @TEST-EXEC: cp $FILES/broker/cluster-layout.zeek . +# +# @TEST-EXEC: btest-bg-run manager ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager zeek -b ../addr-indicator.zeek %INPUT +# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b ../addr-indicator.zeek %INPUT +# @TEST-EXEC: btest-bg-run worker-2 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b ../addr-indicator.zeek %INPUT +# @TEST-EXEC: btest-bg-wait 30 +# @TEST-EXEC: mkdir addr-indicator/; cp -R manager worker-1 worker-2 ./addr-indicator/ +# @TEST-EXEC: btest-diff addr-indicator/manager/.stdout +# @TEST-EXEC: btest-diff addr-indicator/worker-1/.stdout +# @TEST-EXEC: btest-diff addr-indicator/worker-2/.stdout +# +# @TEST-EXEC: rm -rf manager worker-1 worker-2 +# +# @TEST-EXEC: btest-bg-run manager ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager zeek -b ../subnet-indicator.zeek %INPUT +# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b ../subnet-indicator.zeek %INPUT +# @TEST-EXEC: btest-bg-run worker-2 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b ../subnet-indicator.zeek %INPUT +# @TEST-EXEC: btest-bg-wait 30 +# @TEST-EXEC: mkdir subnet-indicator/; cp -R manager worker-1 worker-2 ./subnet-indicator/ +# @TEST-EXEC: btest-diff subnet-indicator/manager/.stdout +# @TEST-EXEC: btest-diff subnet-indicator/worker-1/.stdout +# @TEST-EXEC: btest-diff subnet-indicator/worker-2/.stdout + +# @TEST-EXEC: rm -rf manager worker-1 worker-2 +# +# @TEST-EXEC: btest-bg-run manager ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager zeek -b ../software-indicator.zeek %INPUT +# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b ../software-indicator.zeek %INPUT +# @TEST-EXEC: btest-bg-run worker-2 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b ../software-indicator.zeek %INPUT +# @TEST-EXEC: btest-bg-wait 30 +# @TEST-EXEC: mkdir software-indicator/; cp -R manager worker-1 worker-2 ./software-indicator/ +# @TEST-EXEC: btest-diff software-indicator/manager/.stdout +# @TEST-EXEC: btest-diff software-indicator/worker-1/.stdout +# @TEST-EXEC: btest-diff software-indicator/worker-2/.stdout +# +# @TEST-EXEC: rm -rf manager worker-1 worker-2 + +@load policy/frameworks/cluster/experimental +@load base/frameworks/intel + +redef Log::default_rotation_interval = 0sec; + +# @TEST-START-FILE addr-indicator.zeek +@load base/frameworks/intel + +const test_intel_type = Intel::ADDR; + +event remove_indicator(value: string, source: string) + { + print"remove_addr", value, source; + Intel::remove([$indicator=value, $indicator_type=Intel::ADDR, $meta=[$source=source]], F); + } + +event insert_indicator(value: string, source: string) + { + print "insert_addr", value, source; + Intel::insert([$indicator=value, $indicator_type=Intel::ADDR, $meta=[$source=source]]); + } +# @TEST-END-FILE + +# @TEST-START-FILE subnet-indicator.zeek +@load base/frameworks/intel + +const test_intel_type = Intel::SUBNET; + +event remove_indicator(value: string, source: string) + { + value = value + "/32"; # make the IP value from generic code a valid subnet + print"remove_subnet", value, source; + Intel::remove([$indicator=value, $indicator_type=Intel::SUBNET, $meta=[$source=source]], F); + } + +event insert_indicator(value: string, source: string) + { + value = value + "/32"; # make the IP value from generic code a valid subnet + print "insert_subnet", value , source; + Intel::insert([$indicator=value, $indicator_type=Intel::SUBNET, $meta=[$source=source]]); + } +# @TEST-END-FILE + +# @TEST-START-FILE software-indicator.zeek +@load base/frameworks/intel + +const test_intel_type = Intel::SOFTWARE; + +event remove_indicator(value: string, source: string) + { + value = "software-" + value; + print"remove_software", value, source; + Intel::remove([$indicator=value, $indicator_type=Intel::SOFTWARE, $meta=[$source=source]], F); + } + +event insert_indicator(value: string, source: string) + { + value = "software-" + value; + print "insert_software", value , source; + Intel::insert([$indicator=value, $indicator_type=Intel::SOFTWARE, $meta=[$source=source]]); + } +# @TEST-END-FILE + +# Helper event for printing on manager and worker. +event next_round() + { + print "===="; + if ( Cluster::node == "manager" ) + Cluster::publish(Cluster::worker_topic, next_round); + } + +# Send by manager for termination purposes. +event do_terminate() + { + terminate(); + } + +event publish_do_terminate() + { + print "publish_do_terminate()"; + Cluster::publish(Cluster::worker_topic, do_terminate); + } + +# Ensure the manager terminates eventually. +global nodes_down = 0; +event Cluster::node_down(name: string, id: string) + { + ++nodes_down; + if ( nodes_down >= 2) + terminate(); + } + +# Gets the ball rolling +event Cluster::Experimental::cluster_started() + { + if ( Cluster::local_node_type() == Cluster::MANAGER ) + event insert_indicator("1.2.3.4", "from-manager"); + } + +global indicators_inserted: table[Intel::Type] of count &default=0; +global indicator_removed: table[Intel::Type] of count &default=0; + +hook Intel::indicator_inserted(indicator: string, indicator_type: Intel::Type) + { + print "Intel::indicator_inserted", indicator, indicator_type; + ++indicators_inserted[indicator_type]; + + # If worker-1 sees the first addr indicator (1.2.3.4), it re-inserts + # it with a different source (from-worker) and also inserts a second + # indicatore 1.2.3.5 with with sources "from-worker" and "from-manager"; + if ( Cluster::node == "worker-1" ) + { + if ( indicators_inserted[test_intel_type] == 1 || indicators_inserted[test_intel_type] == 3 ) + { + event insert_indicator("1.2.3.4", "from-worker"); + event insert_indicator("1.2.3.5", "from-worker"); + event insert_indicator("1.2.3.5", "from-manager"); + } + } + + # Once worker-2 has observed two or four indicators, it removes + # all of them again! + if ( Cluster::node == "worker-2" ) + { + if ( indicators_inserted[test_intel_type] == 2 || indicators_inserted[test_intel_type] == 4 ) + { + event remove_indicator("1.2.3.4", "from-manager"); + event remove_indicator("1.2.3.5", "from-manager"); + event remove_indicator("1.2.3.4", "from-worker"); + event remove_indicator("1.2.3.5", "from-worker"); + } + } + } + +hook Intel::indicator_removed(indicator: string, indicator_type: Intel::Type) + { + print "Intel::indicator_removed", indicator, indicator_type; + ++indicator_removed[indicator_type]; + if ( Cluster::node == "manager" ) + { + if ( indicator_removed[test_intel_type] == 2 ) + { + # Trigger another round of inserts at the workers! + event next_round(); + event insert_indicator("1.2.3.4", "from-manager"); + } + else if ( indicator_removed[test_intel_type] == 4 ) + { + event publish_do_terminate(); + } + } + }