From e5367ba820b95dba7acc018d4d865b1c2f91163d Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Thu, 15 May 2025 09:34:14 +0200 Subject: [PATCH] intel/seen/manage-event-groups: Policy script for toggling intel event groups Co-authored-by: Mohan Dhawan (cherry picked from commit e90f947bee572189a82ed81f91c505d9db474588) --- .../intel/seen/manage-event-groups.zeek | 71 ++++++++ scripts/test-all-policy.zeek | 1 + .../manager..stdout | 10 ++ .../manager.intel.log | 11 ++ .../worker-1..stdout | 11 ++ .../seen/manage-event-groups/basic-smtp.zeek | 155 ++++++++++++++++++ 6 files changed, 259 insertions(+) create mode 100644 scripts/policy/frameworks/intel/seen/manage-event-groups.zeek create mode 100644 testing/btest/Baseline/scripts.policy.frameworks.intel.seen.manage-event-groups.basic-smtp/manager..stdout create mode 100644 testing/btest/Baseline/scripts.policy.frameworks.intel.seen.manage-event-groups.basic-smtp/manager.intel.log create mode 100644 testing/btest/Baseline/scripts.policy.frameworks.intel.seen.manage-event-groups.basic-smtp/worker-1..stdout create mode 100644 testing/btest/scripts/policy/frameworks/intel/seen/manage-event-groups/basic-smtp.zeek diff --git a/scripts/policy/frameworks/intel/seen/manage-event-groups.zeek b/scripts/policy/frameworks/intel/seen/manage-event-groups.zeek new file mode 100644 index 0000000000..1c1b32ad84 --- /dev/null +++ b/scripts/policy/frameworks/intel/seen/manage-event-groups.zeek @@ -0,0 +1,71 @@ +@load frameworks/intel/seen +@load base/frameworks/reporter + +module Intel; + +export { + ## Whether Intel event groups for the seen scripts are managed. + ## + ## When loading this script, by default, all :zeek:see:`Intel::Type` + ## event groups are disabled at startup and only enabled when indicators + ## of corresponding types are loaded into the Intel framework's store. + ## This allows to load the ``frameworks/intel/seen`` scripts by default, + ## without incurring overhead when no indicators are loaded. + ## + ## One caveat is that the :zeek:see:`Intel::seen_policy` hook will not + ## be invoked for indicator types that are not at all in the Intel + ## framework's store. You should not load this script, set this variable + ## to ``F``, or insert dummy values of the types using + ## :zeek:see:`Intel::insert`. + const manage_seen_event_groups = T &redef; +} + +global intel_type_counts: table[Intel::Type] of count &default=0; + +event zeek_init() + { + # If the feature is disabled, don't act. + if ( ! manage_seen_event_groups ) + return; + + # Disable all Intel related event groups at startup. These + # are enabled again as soon as at least one indicator of the + # type is inserted. + for ( name in enum_names(Intel::Type) ) + { + if ( has_event_group(name) ) + disable_event_group(name); + } + } + +hook Intel::indicator_inserted(v: string, t: Intel::Type) + { + ++intel_type_counts[t]; + + if ( ! manage_seen_event_groups ) + return; + + local name = cat(t); + + if ( intel_type_counts[t] == 1 ) + { + if ( has_event_group(name) ) + enable_event_group(name); + } + } + +hook Intel::indicator_removed(v: string, t: Intel::Type) + { + --intel_type_counts[t]; + + if ( ! manage_seen_event_groups ) + return; + + local name = cat(t); + + if ( intel_type_counts[t] == 0 ) + { + if ( has_event_group(name) ) + disable_event_group(name); + } + } diff --git a/scripts/test-all-policy.zeek b/scripts/test-all-policy.zeek index cf75ed5aa0..1fc9612fe7 100644 --- a/scripts/test-all-policy.zeek +++ b/scripts/test-all-policy.zeek @@ -54,6 +54,7 @@ @load frameworks/intel/seen/file-names.zeek @load frameworks/intel/seen/http-headers.zeek @load frameworks/intel/seen/http-url.zeek +@load frameworks/intel/seen/manage-event-groups.zeek @load frameworks/intel/seen/pubkey-hashes.zeek @load frameworks/intel/seen/smb-filenames.zeek @load frameworks/intel/seen/smtp-url-extraction.zeek diff --git a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.manage-event-groups.basic-smtp/manager..stdout b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.manage-event-groups.basic-smtp/manager..stdout new file mode 100644 index 0000000000..f593ee46d9 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.manage-event-groups.basic-smtp/manager..stdout @@ -0,0 +1,10 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +running do_step(2) +Intel::indicator_inserted 1.2.3.4 Intel::ADDR +publish do_step(3) to zeek/cluster/worker +Intel::match_remote: 1.2.3.4 Intel::ADDR +Intel::match: 1.2.3.4 Intel::ADDR +running do_step(4) +Intel::indicator_removed 1.2.3.4 Intel::ADDR +publish do_step(5) to zeek/cluster/worker +running do_step(6) diff --git a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.manage-event-groups.basic-smtp/manager.intel.log b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.manage-event-groups.basic-smtp/manager.intel.log new file mode 100644 index 0000000000..0494365703 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.manage-event-groups.basic-smtp/manager.intel.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path intel +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p seen.indicator seen.indicator_type seen.where seen.node matched sources fuid file_mime_type file_desc +#types time string addr port addr port string enum enum string set[enum] set[string] string string string +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 1.1.1.1 1 2.2.2.2 2 1.2.3.4 Intel::ADDR SMTP::IN_RECEIVED_HEADER worker-1 Intel::ADDR source1 - - - +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.manage-event-groups.basic-smtp/worker-1..stdout b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.manage-event-groups.basic-smtp/worker-1..stdout new file mode 100644 index 0000000000..73378256ee --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.manage-event-groups.basic-smtp/worker-1..stdout @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +running do_step(1) +publish do_step(2) to zeek/cluster/manager +Intel::indicator_inserted 1.2.3.4 Intel::ADDR +running do_step(3) +Intel::seen_policy(1.2.3.4 of Intel::ADDR, T) +Intel::seen_policy(1.2.3.5 of Intel::ADDR, F) +Intel::match_remote: 1.2.3.4 Intel::ADDR +Intel::indicator_removed 1.2.3.4 Intel::ADDR +running do_step(5) +publish do_step(6) to zeek/cluster/manager diff --git a/testing/btest/scripts/policy/frameworks/intel/seen/manage-event-groups/basic-smtp.zeek b/testing/btest/scripts/policy/frameworks/intel/seen/manage-event-groups/basic-smtp.zeek new file mode 100644 index 0000000000..440803be1d --- /dev/null +++ b/testing/btest/scripts/policy/frameworks/intel/seen/manage-event-groups/basic-smtp.zeek @@ -0,0 +1,155 @@ +# @TEST-DOC: Smoke test that the seen/smtp mime_end_entity() only runs when Intel::ADDR indicators are loaded and mime_end_entity() runs for a SMTP connection. +# +# @TEST-PORT: BROKER_MANAGER_PORT +# @TEST-PORT: BROKER_WORKER1_PORT +# +# @TEST-EXEC: cp $FILES/broker/cluster-layout.zeek . +# +# @TEST-EXEC: zeek --parse-only %INPUT +# +# @TEST-EXEC: btest-bg-run manager ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager zeek -b %INPUT +# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b %INPUT +# @TEST-EXEC: btest-bg-wait 10 + +# @TEST-EXEC: btest-diff manager/.stdout +# @TEST-EXEC: btest-diff worker-1/.stdout +# @TEST-EXEC: btest-diff manager/intel.log + +@load base/frameworks/cluster +@load base/frameworks/intel + +@load frameworks/intel/seen +@load frameworks/intel/seen/manage-event-groups + +@load frameworks/cluster/experimental + +redef Log::default_rotation_interval = 0secs; + +global addr_indicator = Intel::Item( + $indicator="1.2.3.4", + $indicator_type=Intel::ADDR, + $meta=Intel::MetaData($source="source1"), +); + +function make_conn(): connection + { + local c = connection( + $id = conn_id($orig_h=1.1.1.1, $orig_p=1/tcp, + $resp_h=2.2.2.2, $resp_p=2/tcp), + $orig = endpoint($size=1, $state=4, $flow_label=0), + $resp = endpoint($size=1, $state=4, $flow_label=0), + $start_time=double_to_time(1747323991.0), + $duration=1sec, + $service=set("smtp"), + $history="ShAdDa", + $uid="CHhAvVGS1DHFjwGM9", + ); + + c$smtp = SMTP::Info( + $ts=c$start_time, + $uid=c$uid, + $id=c$id, + $trans_depth=1, + $path=vector(1.2.3.4) + ); + + c$smtp$x_originating_ip = 1.2.3.5; + + return c; + } + +global current_step = 0; + +global do_step: event(step: count); + +event publish_do_step(step: count) + { + local topic = Cluster::local_node_type() == Cluster::MANAGER ? Cluster::worker_topic : Cluster::manager_topic; + print fmt("publish do_step(%s) to %s", step, topic); + Broker::publish(topic, do_step, step); + } + +# Log Intel::seen_policy() invocations. +# +# The idea here is that if the seen event groups are disabled, +# the Intel::seen_policy() hook isn't invoked at all. So we can +# use that to verify that the corresponding events have actually +# been disabled. +hook Intel::seen_policy(s: Intel::Seen, found: bool) + { + print fmt("Intel::seen_policy(%s of %s, %s)", + s?$host ? cat(s$host) : s$indicator, + s?$host ? "Intel::ADDR" : cat(s$indicator_type), + found); + } + +event Intel::match(s: Intel::Seen, items: set[Intel::Item]) + { + print fmt("Intel::match: %s %s", s$indicator, s$indicator_type); + if ( current_step == 2 ) + event do_step(4); + } + +event do_step(step: count) + { + current_step = step; + print fmt("running do_step(%s)", step); + + switch ( step ) { + case 1: # worker + local c1 = make_conn(); + event mime_end_entity(c1); + event publish_do_step(2); + break; + case 2: # manager, insert a intel indicator + Intel::insert(addr_indicator); + event publish_do_step(3); + break; + case 3: # worker - should have an addr indicator now, match it. + local c2 = make_conn(); + event mime_end_entity(c2); + # no publish of step 4, see Intel::match() that drives it + break; + case 4: # manager waits for the match + Intel::remove(addr_indicator); + event publish_do_step(5); + break; + case 5: # worker - the ADDR groups are disabled again. + local c3 = make_conn(); + event mime_end_entity(c3); + event publish_do_step(6); + break; + case 6: # manager, done + terminate(); + break; + } + } + +event Cluster::Experimental::cluster_started() + { + if ( Cluster::node == "worker-1" ) + event do_step(1); + } + +event Cluster::node_down(name: string, id: string) + { + terminate(); + } + +# Output a few internal things for sanity. These aren't testing functionality, +# but nice to have. +module Intel; +event Intel::match_remote(s: Intel::Seen) + { + print fmt("Intel::match_remote: %s %s", s$indicator, s$indicator_type); + } + +hook Intel::indicator_inserted(indicator: string, indicator_type: Intel::Type) + { + print fmt("Intel::indicator_inserted %s %s", indicator, indicator_type); + } + +hook Intel::indicator_removed(indicator: string, indicator_type: Intel::Type) + { + print fmt("Intel::indicator_removed %s %s", indicator, indicator_type); + }