mirror of
https://github.com/zeek/zeek.git
synced 2025-10-02 06:38:20 +00:00
Added support for subnets to intel-framework.
The intel-framework now supports the new indicator type Intel::SUBNET. As subnets are matched against seen addresses, the field matched was introduced to indicate which indicator types caused the hit. A testcase for subents was added and the old ones have been updated accordingly.
This commit is contained in:
Jan Grashoefer
parent
06faee2cc8
commit
cafae5351b
8 changed files with 158 additions and 49 deletions
|
|
@ -14,6 +14,8 @@ export {
|
|||
type Type: enum {
|
||||
## An IP address.
|
||||
ADDR,
|
||||
## A subnet in CIDR notation.
|
||||
SUBNET,
|
||||
## A complete URL without the prefix ``"http://"``.
|
||||
URL,
|
||||
## Software name.
|
||||
|
|
@ -35,6 +37,8 @@ export {
|
|||
## Public key MD5 hash. (SSH server host keys are a good example.)
|
||||
PUBKEY_HASH,
|
||||
};
|
||||
## Set of intelligence data types.
|
||||
type TypeSet: set[Type];
|
||||
|
||||
## Data about an :bro:type:`Intel::Item`.
|
||||
type MetaData: record {
|
||||
|
|
@ -123,6 +127,8 @@ export {
|
|||
|
||||
## Where the data was seen.
|
||||
seen: Seen &log;
|
||||
## Which indicator types matched.
|
||||
matched: TypeSet &log;
|
||||
## Sources which supplied data that resulted in this match.
|
||||
sources: set[string] &log &default=string_set();
|
||||
};
|
||||
|
|
@ -162,6 +168,7 @@ type MetaDataTable: table[string] of MetaData;
|
|||
# The in memory data structure for holding intelligence.
|
||||
type DataStore: record {
|
||||
host_data: table[addr] of MetaDataTable;
|
||||
subnet_data: table[subnet] of MetaDataTable;
|
||||
string_data: table[string, Type] of MetaDataTable;
|
||||
};
|
||||
global data_store: DataStore &redef;
|
||||
|
|
@ -171,6 +178,7 @@ global data_store: DataStore &redef;
|
|||
# a minimal amount of data for the full match to happen on the manager.
|
||||
type MinDataStore: record {
|
||||
host_data: set[addr];
|
||||
subnet_data: set[subnet];
|
||||
string_data: set[string, Type];
|
||||
};
|
||||
global min_data_store: MinDataStore &redef;
|
||||
|
|
@ -186,12 +194,11 @@ function find(s: Seen): bool
|
|||
if ( s?$host )
|
||||
{
|
||||
return ((s$host in min_data_store$host_data) ||
|
||||
(have_full_data && s$host in data_store$host_data));
|
||||
(|matching_subnets(addr_to_subnet(s$host), min_data_store$subnet_data)| > 0));
|
||||
}
|
||||
else
|
||||
{
|
||||
return (([to_lower(s$indicator), s$indicator_type] in min_data_store$string_data) ||
|
||||
(have_full_data && [to_lower(s$indicator), s$indicator_type] in data_store$string_data));
|
||||
return ([to_lower(s$indicator), s$indicator_type] in min_data_store$string_data);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -219,6 +226,17 @@ function get_items(s: Seen): set[Item]
|
|||
add return_data[Item($indicator=cat(s$host), $indicator_type=ADDR, $meta=mt[m])];
|
||||
}
|
||||
}
|
||||
# See if the host is part of a known subnet, which has meta values
|
||||
local nets: table[subnet] of MetaDataTable;
|
||||
nets = filter_subnet_table(addr_to_subnet(s$host), data_store$subnet_data);
|
||||
for ( n in nets )
|
||||
{
|
||||
mt = nets[n];
|
||||
for ( m in mt )
|
||||
{
|
||||
add return_data[Item($indicator=cat(n), $indicator_type=SUBNET, $meta=mt[m])];
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
@ -266,7 +284,7 @@ function Intel::seen(s: Seen)
|
|||
|
||||
event Intel::match(s: Seen, items: set[Item]) &priority=5
|
||||
{
|
||||
local info = Info($ts=network_time(), $seen=s);
|
||||
local info = Info($ts=network_time(), $seen=s, $matched=TypeSet());
|
||||
|
||||
if ( s?$f )
|
||||
{
|
||||
|
|
@ -293,7 +311,10 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5
|
|||
}
|
||||
|
||||
for ( item in items )
|
||||
{
|
||||
add info$sources[item$meta$source];
|
||||
add info$matched[item$indicator_type];
|
||||
}
|
||||
|
||||
Log::write(Intel::LOG, info);
|
||||
}
|
||||
|
|
@ -323,6 +344,21 @@ function insert(item: Item)
|
|||
|
||||
add min_data_store$host_data[host];
|
||||
}
|
||||
else if ( item$indicator_type == SUBNET )
|
||||
{
|
||||
local net = to_subnet(item$indicator);
|
||||
if ( have_full_data )
|
||||
{
|
||||
if ( net !in data_store$subnet_data )
|
||||
data_store$subnet_data[net] = table();
|
||||
else
|
||||
is_new = F;
|
||||
|
||||
meta_tbl = data_store$subnet_data[net];
|
||||
}
|
||||
|
||||
add min_data_store$subnet_data[net];
|
||||
}
|
||||
else
|
||||
{
|
||||
if ( have_full_data )
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@
|
|||
#empty_field (empty)
|
||||
#unset_field -
|
||||
#path intel
|
||||
#open 2014-09-23-16-13-39
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node sources
|
||||
#types time string addr port addr port string string string string enum enum string set[string]
|
||||
1411488819.555114 - - - - - - - - 123.123.123.123 Intel::ADDR Intel::IN_ANYWHERE worker-2 worker-1
|
||||
#close 2014-09-23-16-13-49
|
||||
#open 2016-03-22-18-11-20
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node matched sources
|
||||
#types time string addr port addr port string string string string enum enum string set[enum] set[string]
|
||||
1458670280.078658 - - - - - - - - 123.123.123.123 Intel::ADDR Intel::IN_ANYWHERE worker-2 Intel::ADDR worker-1
|
||||
#close 2016-03-22-18-11-29
|
||||
|
|
|
|||
|
|
@ -3,9 +3,9 @@
|
|||
#empty_field (empty)
|
||||
#unset_field -
|
||||
#path intel
|
||||
#open 2014-09-23-16-14-49
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node sources
|
||||
#types time string addr port addr port string string string string enum enum string set[string]
|
||||
1411488889.571819 - - - - - - - - e@mail.com Intel::EMAIL SOMEWHERE bro source1
|
||||
1411488889.571819 - - - - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro source1
|
||||
#close 2014-09-23-16-14-49
|
||||
#open 2016-03-22-18-11-32
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node matched sources
|
||||
#types time string addr port addr port string string string string enum enum string set[enum] set[string]
|
||||
1458670292.167298 - - - - - - - - e@mail.com Intel::EMAIL SOMEWHERE bro Intel::EMAIL source1
|
||||
1458670292.167298 - - - - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro Intel::ADDR source1
|
||||
#close 2016-03-22-18-11-32
|
||||
|
|
|
|||
|
|
@ -0,0 +1,23 @@
|
|||
#separator \x09
|
||||
#set_separator ,
|
||||
#empty_field (empty)
|
||||
#unset_field -
|
||||
#path intel
|
||||
#open 2016-03-22-18-11-34
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node matched sources
|
||||
#types time string addr port addr port string string string string enum enum string set[enum] set[string]
|
||||
1458670294.227182 - - - - - - - - 192.168.1.1 Intel::ADDR SOMEWHERE bro Intel::ADDR source1
|
||||
1458670294.227182 - - - - - - - - 192.168.2.1 Intel::ADDR SOMEWHERE bro Intel::SUBNET source1
|
||||
1458670294.227182 - - - - - - - - 192.168.142.1 Intel::ADDR SOMEWHERE bro Intel::ADDR,Intel::SUBNET source1
|
||||
#close 2016-03-22-18-11-34
|
||||
|
||||
Seen: [indicator=192.168.1.1, indicator_type=Intel::ADDR, host=192.168.1.1, where=SOMEWHERE, node=bro, conn=<uninitialized>, f=<uninitialized>]
|
||||
Item: [indicator=192.168.1.1, indicator_type=Intel::ADDR, meta=[source=source1, desc=this host is just plain baaad, url=http://some-data-distributor.com/1]]
|
||||
|
||||
Seen: [indicator=192.168.2.1, indicator_type=Intel::ADDR, host=192.168.2.1, where=SOMEWHERE, node=bro, conn=<uninitialized>, f=<uninitialized>]
|
||||
Item: [indicator=192.168.2.0/24, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is just plain baaad, url=http://some-data-distributor.com/2]]
|
||||
|
||||
Seen: [indicator=192.168.142.1, indicator_type=Intel::ADDR, host=192.168.142.1, where=SOMEWHERE, node=bro, conn=<uninitialized>, f=<uninitialized>]
|
||||
Item: [indicator=192.168.142.0/24, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is baaad, url=http://some-data-distributor.com/4]]
|
||||
Item: [indicator=192.168.142.1, indicator_type=Intel::ADDR, meta=[source=source1, desc=this host is just plain baaad, url=http://some-data-distributor.com/3]]
|
||||
Item: [indicator=192.168.128.0/18, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork might be baaad, url=http://some-data-distributor.com/5]]
|
||||
|
|
@ -3,11 +3,11 @@
|
|||
#empty_field (empty)
|
||||
#unset_field -
|
||||
#path intel
|
||||
#open 2014-09-23-16-15-00
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node sources
|
||||
#types time string addr port addr port string string string string enum enum string set[string]
|
||||
1411488900.900403 - - - - - - - - 1.2.3.4 Intel::ADDR Intel::IN_A_TEST worker-1 source1
|
||||
1411488900.900403 - - - - - - - - e@mail.com Intel::EMAIL Intel::IN_A_TEST worker-1 source1
|
||||
1411488901.923543 - - - - - - - - 1.2.3.4 Intel::ADDR Intel::IN_A_TEST worker-2 source1
|
||||
1411488901.923543 - - - - - - - - e@mail.com Intel::EMAIL Intel::IN_A_TEST worker-2 source1
|
||||
#close 2014-09-23-16-15-09
|
||||
#open 2016-03-22-18-11-40
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node matched sources
|
||||
#types time string addr port addr port string string string string enum enum string set[enum] set[string]
|
||||
1458670300.363597 - - - - - - - - 1.2.3.4 Intel::ADDR Intel::IN_A_TEST worker-1 Intel::ADDR source1
|
||||
1458670300.363597 - - - - - - - - e@mail.com Intel::EMAIL Intel::IN_A_TEST worker-1 Intel::EMAIL source1
|
||||
1458670301.370555 - - - - - - - - 1.2.3.4 Intel::ADDR Intel::IN_A_TEST worker-2 Intel::ADDR source1
|
||||
1458670301.370555 - - - - - - - - e@mail.com Intel::EMAIL Intel::IN_A_TEST worker-2 Intel::EMAIL source1
|
||||
#close 2016-03-22-18-11-49
|
||||
|
|
|
|||
|
|
@ -3,23 +3,23 @@
|
|||
#empty_field (empty)
|
||||
#unset_field -
|
||||
#path intel
|
||||
#open 2016-03-19-16-01-52
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node sources
|
||||
#types time string addr port addr port string string string string enum enum string set[string]
|
||||
1458403312.669166 - - - - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro source1
|
||||
1458403315.672095 - - - - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro source2,source1
|
||||
1458403315.672095 - - - - - - - - 4.3.2.1 Intel::ADDR SOMEWHERE bro source2
|
||||
1458403318.675592 - - - - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro source2,source1
|
||||
1458403318.675592 - - - - - - - - 4.3.2.1 Intel::ADDR SOMEWHERE bro source2
|
||||
#close 2016-03-19-16-01-58
|
||||
#open 2016-03-22-18-11-51
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node matched sources
|
||||
#types time string addr port addr port string string string string enum enum string set[enum] set[string]
|
||||
1458670311.505318 - - - - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro Intel::ADDR source1
|
||||
1458670314.509318 - - - - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro Intel::ADDR source2,source1
|
||||
1458670314.509318 - - - - - - - - 4.3.2.1 Intel::ADDR SOMEWHERE bro Intel::ADDR source2
|
||||
1458670317.513183 - - - - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro Intel::ADDR source2,source1
|
||||
1458670317.513183 - - - - - - - - 4.3.2.1 Intel::ADDR SOMEWHERE bro Intel::ADDR source2
|
||||
#close 2016-03-22-18-11-57
|
||||
#separator \x09
|
||||
#set_separator ,
|
||||
#empty_field (empty)
|
||||
#unset_field -
|
||||
#path notice
|
||||
#open 2016-03-19-16-01-58
|
||||
#open 2016-03-22-18-11-57
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude
|
||||
#types time string addr port addr port string string string enum enum string string addr addr port count string set[enum] interval bool string string string double double
|
||||
1458403318.675592 - - - - - - - - - Intel::Notice Intel hit on 1.2.3.4 at SOMEWHERE 1.2.3.4 - - - - bro Notice::ACTION_LOG 3600.000000 F - - - - -
|
||||
1458403318.675592 - - - - - - - - - Intel::Notice Intel hit on 4.3.2.1 at SOMEWHERE 4.3.2.1 - - - - bro Notice::ACTION_LOG 3600.000000 F - - - - -
|
||||
#close 2016-03-19-16-01-58
|
||||
1458670317.513183 - - - - - - - - - Intel::Notice Intel hit on 1.2.3.4 at SOMEWHERE 1.2.3.4 - - - - bro Notice::ACTION_LOG 3600.000000 F - - - - -
|
||||
1458670317.513183 - - - - - - - - - Intel::Notice Intel hit on 4.3.2.1 at SOMEWHERE 4.3.2.1 - - - - bro Notice::ACTION_LOG 3600.000000 F - - - - -
|
||||
#close 2016-03-22-18-11-57
|
||||
|
|
|
|||
|
|
@ -3,20 +3,20 @@
|
|||
#empty_field (empty)
|
||||
#unset_field -
|
||||
#path intel
|
||||
#open 2015-03-14-01-47-46
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node sources
|
||||
#types time string addr port addr port string string string string enum enum string set[string]
|
||||
1416942644.593119 CXWv6p3arKYeMETxOg 192.168.4.149 49422 23.92.19.75 443 F0txuw2pvrkZOn04a8 application/pkix-cert 23.92.19.75:443/tcp www.pantz.org Intel::DOMAIN X509::IN_CERT bro source1
|
||||
#close 2015-03-14-01-47-46
|
||||
#open 2016-03-22-18-09-35
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node matched sources
|
||||
#types time string addr port addr port string string string string enum enum string set[enum] set[string]
|
||||
1416942644.593119 CXWv6p3arKYeMETxOg 192.168.4.149 49422 23.92.19.75 443 F0txuw2pvrkZOn04a8 application/pkix-cert 23.92.19.75:443/tcp www.pantz.org Intel::DOMAIN X509::IN_CERT bro Intel::DOMAIN source1
|
||||
#close 2016-03-22-18-09-35
|
||||
#separator \x09
|
||||
#set_separator ,
|
||||
#empty_field (empty)
|
||||
#unset_field -
|
||||
#path intel
|
||||
#open 2015-03-14-01-47-46
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node sources
|
||||
#types time string addr port addr port string string string string enum enum string set[string]
|
||||
1170717505.934612 CXWv6p3arKYeMETxOg 192.150.187.164 58868 194.127.84.106 443 - - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro source1
|
||||
1170717509.082241 CjhGID4nQcgTWjvg4c 192.150.187.164 58869 194.127.84.106 443 - - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro source1
|
||||
1170717512.108799 CCvvfg3TEfuqmmG4bh 192.150.187.164 58870 194.127.84.106 443 - - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro source1
|
||||
#close 2015-03-14-01-47-46
|
||||
#open 2016-03-22-18-09-35
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node matched sources
|
||||
#types time string addr port addr port string string string string enum enum string set[enum] set[string]
|
||||
1170717505.934612 CXWv6p3arKYeMETxOg 192.150.187.164 58868 194.127.84.106 443 - - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro Intel::DOMAIN source1
|
||||
1170717509.082241 CjhGID4nQcgTWjvg4c 192.150.187.164 58869 194.127.84.106 443 - - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro Intel::DOMAIN source1
|
||||
1170717512.108799 CCvvfg3TEfuqmmG4bh 192.150.187.164 58870 194.127.84.106 443 - - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro Intel::DOMAIN source1
|
||||
#close 2016-03-22-18-09-35
|
||||
|
|
|
|||
50
testing/btest/scripts/base/frameworks/intel/match-subnet.bro
Normal file
50
testing/btest/scripts/base/frameworks/intel/match-subnet.bro
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
# @TEST-EXEC: btest-bg-run broproc bro %INPUT
|
||||
# @TEST-EXEC: btest-bg-wait -k 5
|
||||
# @TEST-EXEC: cat broproc/intel.log > output
|
||||
# @TEST-EXEC: cat broproc/.stdout >> output
|
||||
# @TEST-EXEC: btest-diff output
|
||||
|
||||
# @TEST-START-FILE intel.dat
|
||||
#fields indicator indicator_type meta.source meta.desc meta.url
|
||||
192.168.1.1 Intel::ADDR source1 this host is just plain baaad http://some-data-distributor.com/1
|
||||
192.168.2.0/24 Intel::SUBNET source1 this subnetwork is just plain baaad http://some-data-distributor.com/2
|
||||
192.168.142.1 Intel::ADDR source1 this host is just plain baaad http://some-data-distributor.com/3
|
||||
192.168.142.0/24 Intel::SUBNET source1 this subnetwork is baaad http://some-data-distributor.com/4
|
||||
192.168.128.0/18 Intel::SUBNET source1 this subnetwork might be baaad http://some-data-distributor.com/5
|
||||
# @TEST-END-FILE
|
||||
|
||||
@load frameworks/communication/listen
|
||||
|
||||
redef Intel::read_files += { "../intel.dat" };
|
||||
redef enum Intel::Where += { SOMEWHERE };
|
||||
|
||||
event do_it()
|
||||
{
|
||||
Intel::seen([$host=192.168.1.1,
|
||||
$where=SOMEWHERE]);
|
||||
Intel::seen([$host=192.168.2.1,
|
||||
$where=SOMEWHERE]);
|
||||
Intel::seen([$host=192.168.142.1,
|
||||
$where=SOMEWHERE]);
|
||||
}
|
||||
|
||||
event bro_init() &priority=-10
|
||||
{
|
||||
schedule 1sec { do_it() };
|
||||
}
|
||||
|
||||
global log_lines = 0;
|
||||
event Intel::log_intel(rec: Intel::Info)
|
||||
{
|
||||
++log_lines;
|
||||
if ( log_lines == 2 )
|
||||
terminate();
|
||||
}
|
||||
|
||||
event Intel::match(s: Intel::Seen, items: set[Intel::Item])
|
||||
{
|
||||
print "";
|
||||
print fmt("Seen: %s", s);
|
||||
for ( item in items )
|
||||
print fmt("Item: %s", item);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue