mirror of
https://github.com/zeek/zeek.git
synced 2025-10-17 14:08:20 +00:00
Merge branch 'topic/robin/intel-framework-merge'
* topic/robin/intel-framework-merge: (22 commits) Fixing tests after intel-framework merge. Extracting URLs from message bodies over SMTP and sending them to Intel framework. Small comment updates in the Intel framework CIF support. Intelligence framework documentation first draft. Only the manager tries to read files with the input framework now. Initial support for Bro's Intel framework with the Collective Intelligence Framework. Initial API for Intel framework is complete. Fixed an issue with cluster data distribution. Updating some intel framework test baselines. Reworked cluster intelligence data distribution mechanism and fixed tests. Lots more intelligence checking in SMTP traffic. Added intelligence check for "Received" path checking and a bit of reshuffling. Added sources to the intel log. Fixing a problem with intel distribution on clusters. Updated intel framework test to include matching. Restructuring the scripts that feed data into the intel framework slightly. One test for cluster transparency of the intel framework. Fixed a cluster support bug. Intelligence framework checkpoint Major updates to fix the Intel framework API. ... Closes #914.
This commit is contained in:
Robin Sommer
commit
a40b00d4ab
40 changed files with 1039 additions and 337 deletions
|
|
@ -40,6 +40,7 @@ scripts/base/init-default.bro
|
|||
scripts/base/utils/paths.bro
|
||||
scripts/base/utils/strings.bro
|
||||
scripts/base/utils/thresholds.bro
|
||||
scripts/base/utils/urls.bro
|
||||
scripts/base/frameworks/notice/__load__.bro
|
||||
scripts/base/frameworks/notice/./main.bro
|
||||
scripts/base/frameworks/notice/./weird.bro
|
||||
|
|
@ -69,6 +70,7 @@ scripts/base/init-default.bro
|
|||
scripts/base/frameworks/metrics/./non-cluster.bro
|
||||
scripts/base/frameworks/intel/__load__.bro
|
||||
scripts/base/frameworks/intel/./main.bro
|
||||
scripts/base/frameworks/intel/./input.bro
|
||||
scripts/base/frameworks/reporter/__load__.bro
|
||||
scripts/base/frameworks/reporter/./main.bro
|
||||
scripts/base/frameworks/tunnels/__load__.bro
|
||||
|
|
|
|||
|
|
@ -2,5 +2,6 @@
|
|||
-./frameworks/cluster/nodes/proxy.bro
|
||||
-./frameworks/cluster/nodes/worker.bro
|
||||
-./frameworks/cluster/setup-connections.bro
|
||||
-./frameworks/intel/cluster.bro
|
||||
-./frameworks/metrics/cluster.bro
|
||||
-./frameworks/notice/cluster.bro
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
cluster_new_item: 123.123.123.123 inserted by worker-1 (from peer: worker-1)
|
||||
cluster_new_item: 4.3.2.1 inserted by worker-2 (from peer: worker-2)
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
#separator \x09
|
||||
#set_separator ,
|
||||
#empty_field (empty)
|
||||
#unset_field -
|
||||
#path intel
|
||||
#open 2012-10-03-20-20-39
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p seen.host seen.str seen.str_type seen.where sources
|
||||
#types time string addr port addr port addr string enum enum table[string]
|
||||
1349295639.424940 - - - - - 123.123.123.123 - - Intel::IN_ANYWHERE worker-1
|
||||
#close 2012-10-03-20-20-49
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
cluster_new_item: 1.2.3.4 inserted by manager (from peer: manager-1)
|
||||
cluster_new_item: 123.123.123.123 inserted by worker-1 (from peer: manager-1)
|
||||
cluster_new_item: 4.3.2.1 inserted by worker-2 (from peer: manager-1)
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
cluster_new_item: 1.2.3.4 inserted by manager (from peer: manager-1)
|
||||
cluster_new_item: 123.123.123.123 inserted by worker-1 (from peer: manager-1)
|
||||
cluster_new_item: 4.3.2.1 inserted by worker-2 (from peer: manager-1)
|
||||
Doing a lookup
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
#separator \x09
|
||||
#set_separator ,
|
||||
#empty_field (empty)
|
||||
#unset_field -
|
||||
#path intel
|
||||
#open 2012-10-03-20-18-05
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p seen.host seen.str seen.str_type seen.where sources
|
||||
#types time string addr port addr port addr string enum enum table[string]
|
||||
1349295485.114156 - - - - - - e@mail.com Intel::EMAIL SOMEWHERE source1
|
||||
1349295485.114156 - - - - - 1.2.3.4 - - SOMEWHERE source1
|
||||
#close 2012-10-03-20-18-05
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
VALID
|
||||
VALID
|
||||
VALID
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
#separator \x09
|
||||
#set_separator ,
|
||||
#empty_field (empty)
|
||||
#unset_field -
|
||||
#path intel
|
||||
#open 2012-10-10-15-05-23
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p seen.host seen.str seen.str_type seen.where sources
|
||||
#types time string addr port addr port addr string enum enum table[string]
|
||||
1349881523.548946 - - - - - 1.2.3.4 - - Intel::IN_A_TEST source1
|
||||
1349881523.548946 - - - - - - e@mail.com Intel::EMAIL Intel::IN_A_TEST source1
|
||||
1349881524.567896 - - - - - 1.2.3.4 - - Intel::IN_A_TEST source1
|
||||
1349881524.567896 - - - - - - e@mail.com Intel::EMAIL Intel::IN_A_TEST source1
|
||||
#close 2012-10-10-15-05-24
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
# @TEST-SERIALIZE: comm
|
||||
#
|
||||
# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
|
||||
# @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
|
||||
# @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
|
||||
# @TEST-EXEC: btest-bg-wait -k 10
|
||||
# @TEST-EXEC: btest-diff manager-1/.stdout
|
||||
# @TEST-EXEC: btest-diff manager-1/intel.log
|
||||
# @TEST-EXEC: btest-diff worker-1/.stdout
|
||||
# @TEST-EXEC: btest-diff worker-2/.stdout
|
||||
|
||||
@TEST-START-FILE cluster-layout.bro
|
||||
redef Cluster::nodes = {
|
||||
["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")],
|
||||
["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1"],
|
||||
["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1"],
|
||||
};
|
||||
@TEST-END-FILE
|
||||
|
||||
@load base/frameworks/control
|
||||
|
||||
module Intel;
|
||||
|
||||
redef Log::default_rotation_interval=0sec;
|
||||
|
||||
event remote_connection_handshake_done(p: event_peer)
|
||||
{
|
||||
# Insert the data once both workers are connected.
|
||||
if ( Cluster::local_node_type() == Cluster::MANAGER && Cluster::worker_count == 2 )
|
||||
{
|
||||
Intel::insert([$host=1.2.3.4,$meta=[$source="manager"]]);
|
||||
}
|
||||
}
|
||||
|
||||
global worker2_data = 0;
|
||||
global sent_data = F;
|
||||
event Intel::cluster_new_item(item: Intel::Item)
|
||||
{
|
||||
if ( ! is_remote_event() )
|
||||
return;
|
||||
|
||||
print fmt("cluster_new_item: %s inserted by %s (from peer: %s)", item$host, item$meta$source, get_event_peer()$descr);
|
||||
|
||||
if ( ! sent_data )
|
||||
{
|
||||
# We wait to insert data here because we can now be sure the
|
||||
# full cluster is constructed.
|
||||
sent_data = T;
|
||||
if ( Cluster::node == "worker-1" )
|
||||
Intel::insert([$host=123.123.123.123,$meta=[$source="worker-1"]]);
|
||||
if ( Cluster::node == "worker-2" )
|
||||
Intel::insert([$host=4.3.2.1,$meta=[$source="worker-2"]]);
|
||||
}
|
||||
|
||||
# We're forcing worker-2 to do a lookup when it has three intelligence items
|
||||
# which were distributed over the cluster (data inserted locally is resent).
|
||||
if ( Cluster::node == "worker-2" )
|
||||
{
|
||||
++worker2_data;
|
||||
if ( worker2_data == 3 )
|
||||
{
|
||||
# Now that everything is inserted, see if we can match on the data inserted
|
||||
# by worker-1.
|
||||
print "Doing a lookup";
|
||||
Intel::seen([$host=123.123.123.123, $where=Intel::IN_ANYWHERE]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
event Intel::log_intel(rec: Intel::Info)
|
||||
{
|
||||
event Control::shutdown_request();
|
||||
}
|
||||
|
||||
event remote_connection_closed(p: event_peer)
|
||||
{
|
||||
# Cascading termination
|
||||
#print fmt("disconnected from: %s", p);
|
||||
terminate_communication();
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
# @TEST-SERIALIZE: comm
|
||||
|
||||
# @TEST-EXEC: btest-bg-run broproc bro %INPUT
|
||||
# @TEST-EXEC: btest-bg-wait -k 5
|
||||
# @TEST-EXEC: btest-diff broproc/intel.log
|
||||
|
||||
@TEST-START-FILE intel.dat
|
||||
#fields host net str str_type meta.source meta.desc meta.url
|
||||
1.2.3.4 - - - source1 this host is just plain baaad http://some-data-distributor.com/1234
|
||||
1.2.3.4 - - - source1 this host is just plain baaad http://some-data-distributor.com/1234
|
||||
- - e@mail.com Intel::EMAIL source1 Phishing email source http://some-data-distributor.com/100000
|
||||
@TEST-END-FILE
|
||||
|
||||
@load frameworks/communication/listen
|
||||
|
||||
redef Intel::read_files += { "../intel.dat" };
|
||||
redef enum Intel::Where += { SOMEWHERE };
|
||||
|
||||
event do_it()
|
||||
{
|
||||
Intel::seen([$str="e@mail.com",
|
||||
$str_type=Intel::EMAIL,
|
||||
$where=SOMEWHERE]);
|
||||
|
||||
Intel::seen([$host=1.2.3.4,
|
||||
$where=SOMEWHERE]);
|
||||
}
|
||||
|
||||
global log_lines = 0;
|
||||
event Intel::log_intel(rec: Intel::Info)
|
||||
{
|
||||
++log_lines;
|
||||
if ( log_lines == 2 )
|
||||
terminate();
|
||||
}
|
||||
|
||||
event bro_init() &priority=-10
|
||||
{
|
||||
schedule 1sec { do_it() };
|
||||
}
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
#
|
||||
# @TEST-EXEC: bro %INPUT >out
|
||||
# @TEST-EXEC: btest-diff out
|
||||
|
||||
event bro_init()
|
||||
{
|
||||
Intel::insert([$ip=1.2.3.4, $tags=set("zeustracker.abuse.ch", "malicious")]);
|
||||
Intel::insert([$str="http://www.google.com/", $subtype="url", $tags=set("infrastructure", "google")]);
|
||||
Intel::insert([$str="Ab439G32F...", $subtype="x509_cert", $tags=set("bad")]);
|
||||
Intel::insert([$str="Ab439G32F...", $tags=set("bad")]);
|
||||
}
|
||||
|
||||
event bro_done()
|
||||
{
|
||||
local orig_h = 1.2.3.4;
|
||||
|
||||
if ( Intel::matcher([$ip=orig_h, $and_tags=set("malicious")]) )
|
||||
print "VALID";
|
||||
|
||||
if ( Intel::matcher([$ip=orig_h, $and_tags=set("don't match")]) )
|
||||
print "INVALID";
|
||||
|
||||
if ( Intel::matcher([$ip=orig_h, $pred=function(meta: Intel::MetaData): bool { return T; } ]) )
|
||||
print "VALID";
|
||||
|
||||
if ( Intel::matcher([$ip=orig_h, $pred=function(meta: Intel::MetaData): bool { return F; } ]) )
|
||||
print "INVALID";
|
||||
|
||||
if ( Intel::matcher([$str="http://www.google.com/", $subtype="url", $tags=set("google")]) )
|
||||
print "VALID";
|
||||
|
||||
if ( Intel::matcher([$str="http://www.example.com", $subtype="url"]) )
|
||||
print "INVALID";
|
||||
}
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
# @TEST-SERIALIZE: comm
|
||||
#
|
||||
# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
|
||||
# @TEST-EXEC: sleep 2
|
||||
# @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
|
||||
# @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
|
||||
# @TEST-EXEC: btest-bg-wait -k 10
|
||||
# @TEST-EXEC: btest-diff manager-1/.stdout
|
||||
# @TEST-EXEC: btest-diff manager-1/intel.log
|
||||
# @TEST-EXEC: btest-diff worker-1/.stdout
|
||||
# @TEST-EXEC: btest-diff worker-2/.stdout
|
||||
|
||||
@TEST-START-FILE cluster-layout.bro
|
||||
redef Cluster::nodes = {
|
||||
["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")],
|
||||
["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1"],
|
||||
["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1"],
|
||||
};
|
||||
@TEST-END-FILE
|
||||
|
||||
@TEST-START-FILE intel.dat
|
||||
#fields host net str str_type meta.source meta.desc meta.url
|
||||
1.2.3.4 - - - source1 this host is just plain baaad http://some-data-distributor.com/1234
|
||||
1.2.3.4 - - - source1 this host is just plain baaad http://some-data-distributor.com/1234
|
||||
- - e@mail.com Intel::EMAIL source1 Phishing email source http://some-data-distributor.com/100000
|
||||
@TEST-END-FILE
|
||||
|
||||
@load base/frameworks/control
|
||||
redef Log::default_rotation_interval=0sec;
|
||||
|
||||
module Intel;
|
||||
|
||||
@if ( Cluster::local_node_type() == Cluster::MANAGER )
|
||||
redef Intel::read_files += { "../intel.dat" };
|
||||
@endif
|
||||
|
||||
redef enum Intel::Where += {
|
||||
Intel::IN_A_TEST,
|
||||
};
|
||||
|
||||
event do_it()
|
||||
{
|
||||
Intel::seen([$host=1.2.3.4, $where=Intel::IN_A_TEST]);
|
||||
Intel::seen([$str="e@mail.com", $str_type=Intel::EMAIL, $where=Intel::IN_A_TEST]);
|
||||
}
|
||||
|
||||
event bro_init()
|
||||
{
|
||||
# Delay the workers searching for hits briefly to allow for the data distribution
|
||||
# mechanism to distribute the data to the workers.
|
||||
if ( Cluster::local_node_type() == Cluster::WORKER )
|
||||
schedule 2sec { do_it() };
|
||||
}
|
||||
|
||||
global intel_hits=0;
|
||||
event Intel::log_intel(rec: Intel::Info)
|
||||
{
|
||||
++intel_hits;
|
||||
# There should be 4 hits since each worker is "seeing" 2 things.
|
||||
if ( intel_hits == 4 )
|
||||
{
|
||||
# We're delaying shutdown for a second here to make sure that no other
|
||||
# matches happen (which would be wrong!).
|
||||
schedule 1sec { Control::shutdown_request() };
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue