Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-16 21:48:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Reworked cluster intelligence data distribution mechanism and fixed tests.

Browse source 
- Intel data distribution on clusters is now pushed in whole
  by the manager when a worker connects.  Additions after that point
  are managed by the normal single-item distribution mechanism already
  built into the intelligence framework.

- The manager maintains the complete "minimal" data store that the
  workers use to do their matching so that full "minimal" data
  distribution is very easy.

- Tests are cleaned up and work.
This commit is contained in:
Seth Hall 2012-10-03 16:25:02 -04:00
parent 38468f9daa
commit bf9651b323
16 changed files with 84 additions and 177 deletions
Download patch file Download diff file Expand all files Collapse all files

2
testing/btest/scripts/base/frameworks/intel/cluster-transparency.bro
View file

@ -39,7 +39,7 @@ event Intel::cluster_new_item(item: Intel::Item)
if ( ! is_remote_event() )
return;
print fmt("cluster_new_item: %s from source %s (from peer: %s)", item$host, item$meta$source, get_event_peer()$descr);
print fmt("cluster_new_item: %s inserted by %s (from peer: %s)", item$host, item$meta$source, get_event_peer()$descr);
if ( ! sent_data )
{

52
testing/btest/scripts/base/frameworks/intel/input-and-match.bro
View file

@ -1,36 +1,40 @@
# @TEST-EXEC: bro %INPUT >out
# @TEST-EXEC: btest-diff out
# @TEST-SERIALIZE: comm
# @TEST-EXEC: btest-bg-run broproc bro %INPUT
# @TEST-EXEC: btest-bg-wait -k 5
# @TEST-EXEC: btest-diff broproc/intel.log
@TEST-START-FILE intel.dat
#fields ip net str subtype meta.source meta.class meta.desc meta.url meta.tags
1.2.3.4 - - - source1 Intel::MALICIOUS this host is just plain baaad http://some-data-distributor.com/1234 foo,bar
1.2.3.4 - - - source1 Intel::MALICIOUS this host is just plain baaad http://some-data-distributor.com/1234 foo,bar
- - e@mail.com Intel::EMAIL source1 Intel::MALICIOUS Phishing email source http://some-data-distributor.com/100000 -
#fields host net str str_type meta.source meta.desc meta.url
1.2.3.4 - - - source1 this host is just plain baaad http://some-data-distributor.com/1234
1.2.3.4 - - - source1 this host is just plain baaad http://some-data-distributor.com/1234
- - e@mail.com Intel::EMAIL source1 Phishing email source http://some-data-distributor.com/100000
@TEST-END-FILE
@load frameworks/communication/listen
redef Intel::read_files += { "intel.dat" };
redef Intel::read_files += { "../intel.dat" };
redef enum Intel::Where += { SOMEWHERE };
event do_it(allowed_loops: count)
event do_it()
{
if ( Intel::matcher([$str="e@mail.com", $subtype=Intel::EMAIL, $class=Intel::MALICIOUS]) &&
Intel::matcher([$ip=1.2.3.4, $class=Intel::MALICIOUS]) )
{
# Once the match happens a single time we print and shutdown.
print "Matched it!";
terminate_communication();
return;
}
if ( allowed_loops > 0 )
schedule 100msecs { do_it(allowed_loops-1) };
else
terminate_communication();
Intel::seen([$str="e@mail.com",
$str_type=Intel::EMAIL,
$where=SOMEWHERE]);
Intel::seen([$host=1.2.3.4,
$where=SOMEWHERE]);
}
event bro_init()
global log_lines = 0;
event Intel::log_intel(rec: Intel::Info)
{
event do_it(20);
++log_lines;
if ( log_lines == 2 )
terminate();
}
event bro_init() &priority=-10
{
schedule 1sec { do_it() };
}

23
testing/btest/scripts/base/frameworks/intel/item-merge.bro
View file

@ -1,23 +0,0 @@
# @TEST-EXEC: bro %INPUT >out
# @TEST-EXEC: btest-diff out
event bro_init()
{
Intel::insert([$ip=1.2.3.4, $meta=[$source="source1-feed1", $class=Intel::MALICIOUS, $tags=set("foo")]]);
Intel::insert([$ip=1.2.3.4, $meta=[$source="source2-special-sauce", $class=Intel::MALICIOUS, $tags=set("foo","bar")]]);
# Lookup should return the items matching the query.
local items = Intel::lookup([$ip=1.2.3.4]);
print fmt("Number of matching intel items: %d (should be 2)", |items|);
# This can be considered an update of a previous value since the
# data, source, and class are the matching points for determining sameness.
Intel::insert([$ip=1.2.3.4, $meta=[$source="source2-special-sauce", $class=Intel::MALICIOUS, $tags=set("foobar", "testing")]]);
items = Intel::lookup([$ip=1.2.3.4]);
print fmt("Number of matching intel items: %d (should still be 2)", |items|);
# This is a new value.
Intel::insert([$ip=1.2.3.4, $meta=[$source="source3", $class=Intel::MALICIOUS]]);
items = Intel::lookup([$ip=1.2.3.4]);
print fmt("Number of matching intel items: %d (should be 3)", |items|);
}

38
testing/btest/scripts/base/frameworks/intel/matching.bro
View file

@ -1,38 +0,0 @@
#
# @TEST-EXEC: bro %INPUT >out
# @TEST-EXEC: btest-diff out
event bro_init()
{
Intel::insert([$ip=1.2.3.4, $meta=[$source="zeus-tracker", $class=Intel::MALICIOUS, $tags=set("example-tag1", "example-tag2")]]);
Intel::insert([$str="http://www.google.com/", $subtype=Intel::URL, $meta=[$source="source2", $class=Intel::MALICIOUS, $tags=set("infrastructure", "google")]]);
}
event bro_done()
{
local orig_h = 1.2.3.4;
if ( Intel::matcher([$ip=orig_h, $and_tags=set("example-tag1", "example-tag2")]) )
print "VALID";
if ( Intel::matcher([$ip=orig_h, $and_tags=set("don't match")]) )
print "INVALID";
if ( Intel::matcher([$ip=orig_h, $pred=function(meta: Intel::Item): bool { return T; } ]) )
print "VALID";
if ( Intel::matcher([$ip=4.3.2.1, $pred=function(meta: Intel::Item): bool { return T; } ]) )
print "INVALID";
if ( Intel::matcher([$ip=orig_h, $pred=function(meta: Intel::Item): bool { return F; } ]) )
print "INVALID";
if ( Intel::matcher([$str="http://www.google.com/", $subtype=Intel::URL, $and_tags=set("google")]) )
print "VALID";
if ( Intel::matcher([$str="http://www.google.com/", $subtype=Intel::URL, $and_tags=set("woah")]) )
print "INVALID";
if ( Intel::matcher([$str="http://www.example.com", $subtype=Intel::URL]) )
print "INVALID";
}