mirror of
https://github.com/zeek/zeek.git
synced 2025-10-04 07:38:19 +00:00
Merge branch 'topic/seth/intel-update-merge'
Closes github's #77 and closes BIT-1606 * topic/seth/intel-update-merge: Tiny scoping updates and test baseline updates for Intel framework. Minor documentation cleanups. Fixed insertion of nested subnets. Refactored FAF integration of intel framework. Added expiration for intelligence items. Improved intel notices. Added hook to allow extending the intel log. Added remove function to intel-framework. Added support for subnets to intel-framework. Refactoring of meta data handling for intel. Added testcase for intel updates.
This commit is contained in:
Seth Hall
commit
eeb8c0cbb0
31 changed files with 930 additions and 213 deletions
46
testing/btest/scripts/base/frameworks/intel/expire-item.bro
Normal file
46
testing/btest/scripts/base/frameworks/intel/expire-item.bro
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
# @TEST-EXEC: btest-bg-run broproc bro %INPUT
|
||||
# @TEST-EXEC: btest-bg-wait -k 7
|
||||
# @TEST-EXEC: cat broproc/intel.log > output
|
||||
# @TEST-EXEC: cat broproc/.stdout >> output
|
||||
# @TEST-EXEC: btest-diff output
|
||||
|
||||
# @TEST-START-FILE intel.dat
|
||||
#fields indicator indicator_type meta.source meta.desc meta.url
|
||||
1.2.3.4 Intel::ADDR source1 this host is bad http://some-data-distributor.com/1
|
||||
# @TEST-END-FILE
|
||||
|
||||
@load frameworks/communication/listen
|
||||
@load frameworks/intel/do_expire
|
||||
|
||||
redef Intel::read_files += { "../intel.dat" };
|
||||
redef enum Intel::Where += { SOMEWHERE };
|
||||
redef Intel::item_expiration = 3sec;
|
||||
redef table_expire_interval = 1sec;
|
||||
|
||||
global runs = 0;
|
||||
event do_it()
|
||||
{
|
||||
print "Trigger: 1.2.3.4";
|
||||
Intel::seen([$host=1.2.3.4,
|
||||
$where=SOMEWHERE]);
|
||||
|
||||
++runs;
|
||||
if ( runs < 6 )
|
||||
schedule 1sec { do_it() };
|
||||
}
|
||||
|
||||
event Intel::match(s: Intel::Seen, items: set[Intel::Item])
|
||||
{
|
||||
print fmt("Seen: %s", s$indicator);
|
||||
}
|
||||
|
||||
hook Intel::item_expired(indicator: string, indicator_type: Intel::Type,
|
||||
metas: set[Intel::MetaData])
|
||||
{
|
||||
print fmt("Expired: %s", indicator);
|
||||
}
|
||||
|
||||
event bro_init() &priority=-10
|
||||
{
|
||||
schedule 1sec { do_it() };
|
||||
}
|
||||
51
testing/btest/scripts/base/frameworks/intel/match-subnet.bro
Normal file
51
testing/btest/scripts/base/frameworks/intel/match-subnet.bro
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
# @TEST-EXEC: btest-bg-run broproc bro %INPUT
|
||||
# @TEST-EXEC: btest-bg-wait -k 5
|
||||
# @TEST-EXEC: cat broproc/intel.log > output
|
||||
# @TEST-EXEC: cat broproc/.stdout >> output
|
||||
# @TEST-EXEC: btest-diff output
|
||||
|
||||
# @TEST-START-FILE intel.dat
|
||||
#fields indicator indicator_type meta.source meta.desc meta.url
|
||||
192.168.1.1 Intel::ADDR source1 this host is just plain baaad http://some-data-distributor.com/1
|
||||
192.168.2.0/24 Intel::SUBNET source1 this subnetwork is just plain baaad http://some-data-distributor.com/2
|
||||
192.168.142.1 Intel::ADDR source1 this host is just plain baaad http://some-data-distributor.com/3
|
||||
192.168.142.0/24 Intel::SUBNET source1 this subnetwork is baaad http://some-data-distributor.com/4
|
||||
192.168.142.0/26 Intel::SUBNET source1 this subnetwork is inside http://some-data-distributor.com/4
|
||||
192.168.128.0/18 Intel::SUBNET source1 this subnetwork might be baaad http://some-data-distributor.com/5
|
||||
# @TEST-END-FILE
|
||||
|
||||
@load frameworks/communication/listen
|
||||
|
||||
redef Intel::read_files += { "../intel.dat" };
|
||||
redef enum Intel::Where += { SOMEWHERE };
|
||||
|
||||
event do_it()
|
||||
{
|
||||
Intel::seen([$host=192.168.1.1,
|
||||
$where=SOMEWHERE]);
|
||||
Intel::seen([$host=192.168.2.1,
|
||||
$where=SOMEWHERE]);
|
||||
Intel::seen([$host=192.168.142.1,
|
||||
$where=SOMEWHERE]);
|
||||
}
|
||||
|
||||
event bro_init() &priority=-10
|
||||
{
|
||||
schedule 1sec { do_it() };
|
||||
}
|
||||
|
||||
global log_lines = 0;
|
||||
event Intel::log_intel(rec: Intel::Info)
|
||||
{
|
||||
++log_lines;
|
||||
if ( log_lines == 2 )
|
||||
terminate();
|
||||
}
|
||||
|
||||
event Intel::match(s: Intel::Seen, items: set[Intel::Item])
|
||||
{
|
||||
print "";
|
||||
print fmt("Seen: %s", s);
|
||||
for ( item in items )
|
||||
print fmt("Item: %s", item);
|
||||
}
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
# @TEST-SERIALIZE: comm
|
||||
#
|
||||
# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
|
||||
# @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
|
||||
# @TEST-EXEC: btest-bg-wait -k 10
|
||||
# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff manager-1/.stdout
|
||||
# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff worker-1/.stdout
|
||||
# @TEST-EXEC: btest-diff manager-1/intel.log
|
||||
|
||||
# @TEST-START-FILE cluster-layout.bro
|
||||
redef Cluster::nodes = {
|
||||
["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1")],
|
||||
["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1"],
|
||||
};
|
||||
# @TEST-END-FILE
|
||||
|
||||
@load base/frameworks/control
|
||||
|
||||
module Intel;
|
||||
|
||||
redef Log::default_rotation_interval=0sec;
|
||||
|
||||
event test_manager()
|
||||
{
|
||||
Intel::remove([$indicator="192.168.0.1", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]);
|
||||
Intel::seen([$host=192.168.0.1, $where=Intel::IN_ANYWHERE]);
|
||||
Intel::remove([$indicator="192.168.0.2", $indicator_type=Intel::ADDR, $meta=[$source="source1"]], T);
|
||||
Intel::seen([$host=192.168.0.2, $where=Intel::IN_ANYWHERE]);
|
||||
}
|
||||
|
||||
event test_worker()
|
||||
{
|
||||
Intel::remove([$indicator="192.168.1.2", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]);
|
||||
Intel::remove([$indicator="192.168.1.2", $indicator_type=Intel::ADDR, $meta=[$source="source2"]]);
|
||||
Intel::seen([$host=192.168.1.2, $where=Intel::IN_ANYWHERE]);
|
||||
# Trigger shutdown by matching data that should be present
|
||||
Intel::seen([$host=10.10.10.10, $where=Intel::IN_ANYWHERE]);
|
||||
}
|
||||
|
||||
event remote_connection_handshake_done(p: event_peer)
|
||||
{
|
||||
# Insert the data once all workers are connected.
|
||||
if ( Cluster::local_node_type() == Cluster::MANAGER && Cluster::worker_count == 1 )
|
||||
{
|
||||
Intel::insert([$indicator="192.168.0.1", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]);
|
||||
Intel::insert([$indicator="192.168.0.2", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]);
|
||||
Intel::insert([$indicator="192.168.0.2", $indicator_type=Intel::ADDR, $meta=[$source="source2"]]);
|
||||
Intel::insert([$indicator="192.168.1.2", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]);
|
||||
Intel::insert([$indicator="192.168.1.2", $indicator_type=Intel::ADDR, $meta=[$source="source2"]]);
|
||||
Intel::insert([$indicator="10.10.10.10", $indicator_type=Intel::ADDR, $meta=[$source="end"]]);
|
||||
|
||||
event test_manager();
|
||||
}
|
||||
}
|
||||
|
||||
global worker_data = 0;
|
||||
event Intel::cluster_new_item(item: Intel::Item)
|
||||
{
|
||||
# Run test on worker-1 when all items have been inserted
|
||||
if ( Cluster::node == "worker-1" )
|
||||
{
|
||||
++worker_data;
|
||||
if ( worker_data == 4 )
|
||||
event test_worker();
|
||||
}
|
||||
}
|
||||
|
||||
event Intel::remove_item(item: Item, purge_indicator: bool)
|
||||
{
|
||||
print fmt("Removing %s (source: %s).", item$indicator, item$meta$source);
|
||||
}
|
||||
|
||||
event purge_item(item: Item)
|
||||
{
|
||||
print fmt("Purging %s.", item$indicator);
|
||||
}
|
||||
|
||||
event Intel::log_intel(rec: Intel::Info)
|
||||
{
|
||||
print "Logging intel hit!";
|
||||
event Control::shutdown_request();
|
||||
}
|
||||
|
||||
event remote_connection_closed(p: event_peer)
|
||||
{
|
||||
# Cascading termination
|
||||
terminate_communication();
|
||||
}
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
# @TEST-SERIALIZE: comm
|
||||
|
||||
# @TEST-EXEC: cp intel1.dat intel.dat
|
||||
# @TEST-EXEC: btest-bg-run broproc bro %INPUT
|
||||
# @TEST-EXEC: sleep 2
|
||||
# @TEST-EXEC: cp intel2.dat intel.dat
|
||||
# @TEST-EXEC: sleep 2
|
||||
# @TEST-EXEC: cp intel3.dat intel.dat
|
||||
# @TEST-EXEC: btest-bg-wait 6
|
||||
# @TEST-EXEC: cat broproc/intel.log > output
|
||||
# @TEST-EXEC: cat broproc/notice.log >> output
|
||||
# @TEST-EXEC: btest-diff output
|
||||
|
||||
# @TEST-START-FILE intel1.dat
|
||||
#fields indicator indicator_type meta.source meta.desc meta.url meta.do_notice
|
||||
1.2.3.4 Intel::ADDR source1 this host is just plain baaad http://some-data-distributor.com/1234 F
|
||||
# @TEST-END-FILE
|
||||
|
||||
# @TEST-START-FILE intel2.dat
|
||||
#fields indicator indicator_type meta.source meta.desc meta.url meta.do_notice
|
||||
1.2.3.4 Intel::ADDR source2 this host is just plain baaad http://some-data-distributor.com/1234 F
|
||||
4.3.2.1 Intel::ADDR source2 this host might also be baaad http://some-data-distributor.com/4321 F
|
||||
# @TEST-END-FILE
|
||||
|
||||
# @TEST-START-FILE intel3.dat
|
||||
#fields indicator indicator_type meta.source meta.desc meta.url meta.do_notice
|
||||
1.2.3.4 Intel::ADDR source2 this host is just plain baaad http://some-data-distributor.com/1234 T
|
||||
4.3.2.1 Intel::ADDR source2 this host might also be baaad http://some-data-distributor.com/4321 T
|
||||
# @TEST-END-FILE
|
||||
|
||||
@load base/frameworks/communication # let network-time run
|
||||
@load frameworks/intel/do_notice
|
||||
|
||||
redef exit_only_after_terminate = T;
|
||||
redef Intel::read_files += { "../intel.dat" };
|
||||
redef enum Intel::Where += { SOMEWHERE };
|
||||
|
||||
global runs = 0;
|
||||
event do_it()
|
||||
{
|
||||
Intel::seen([$host=1.2.3.4,
|
||||
$where=SOMEWHERE]);
|
||||
Intel::seen([$host=4.3.2.1,
|
||||
$where=SOMEWHERE]);
|
||||
|
||||
++runs;
|
||||
if ( runs < 3 )
|
||||
schedule 3sec { do_it() };
|
||||
}
|
||||
|
||||
global log_lines = 0;
|
||||
event Intel::log_intel(rec: Intel::Info)
|
||||
{
|
||||
++log_lines;
|
||||
if ( log_lines == 5 )
|
||||
terminate();
|
||||
}
|
||||
|
||||
event bro_init() &priority=-10
|
||||
{
|
||||
schedule 1sec { do_it() };
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
# @TEST-EXEC: bro -r $TRACES/tls/ecdsa-cert.pcap %INPUT
|
||||
# @TEST-EXEC: bro -Cr $TRACES/tls/ecdsa-cert.pcap %INPUT
|
||||
# @TEST-EXEC: cat intel.log > intel-all.log
|
||||
# @TEST-EXEC: bro -r $TRACES/tls/ssl.v3.trace %INPUT
|
||||
# @TEST-EXEC: cat intel.log >> intel-all.log
|
||||
|
|
|
|||
|
|
@ -0,0 +1,39 @@
|
|||
# @TEST-EXEC: bro -Cr $TRACES/wikipedia.trace %INPUT
|
||||
# @TEST-EXEC: btest-diff intel.log
|
||||
|
||||
#@TEST-START-FILE intel.dat
|
||||
#fields indicator indicator_type meta.source meta.desc meta.url
|
||||
upload.wikimedia.org Intel::DOMAIN source1 somehow bad http://some-data-distributor.com/1
|
||||
meta.wikimedia.org Intel::DOMAIN source1 also bad http://some-data-distributor.com/1
|
||||
#@TEST-END-FILE
|
||||
|
||||
#@TEST-START-FILE whitelist.dat
|
||||
#fields indicator indicator_type meta.source meta.desc meta.whitelist meta.url
|
||||
meta.wikimedia.org Intel::DOMAIN source2 also bad T http://some-data-distributor.com/1
|
||||
#@TEST-END-FILE
|
||||
|
||||
@load base/frameworks/intel
|
||||
@load frameworks/intel/whitelist
|
||||
@load frameworks/intel/seen
|
||||
|
||||
redef Intel::read_files += {
|
||||
"intel.dat",
|
||||
"whitelist.dat",
|
||||
};
|
||||
|
||||
global total_files_read = 0;
|
||||
|
||||
event bro_init()
|
||||
{
|
||||
suspend_processing();
|
||||
}
|
||||
|
||||
event Input::end_of_data(name: string, source: string)
|
||||
{
|
||||
# Wait until both intel files are read.
|
||||
if ( /^intel-/ in name && (++total_files_read == 2) )
|
||||
{
|
||||
continue_processing();
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue