Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 06:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Added support for subnets to intel-framework.

Browse source 
The intel-framework now supports the new indicator type Intel::SUBNET.
As subnets are matched against seen addresses, the field matched was
introduced to indicate which indicator types caused the hit. A testcase
for subents was added and the old ones have been updated accordingly.
This commit is contained in:
Jan Grashoefer 2016-03-22 19:16:51 +01:00
parent 06faee2cc8
commit cafae5351b
8 changed files with 158 additions and 49 deletions
Download patch file Download diff file Expand all files Collapse all files

46
scripts/base/frameworks/intel/main.bro
View file

@ -14,6 +14,8 @@ export {
type Type: enum { type Type: enum {
## An IP address. ## An IP address.
ADDR, ADDR,
## A subnet in CIDR notation.
SUBNET,
## A complete URL without the prefix ``"http://"``. ## A complete URL without the prefix ``"http://"``.
URL, URL,
## Software name. ## Software name.
@ -35,7 +37,9 @@ export {
## Public key MD5 hash. (SSH server host keys are a good example.) ## Public key MD5 hash. (SSH server host keys are a good example.)
PUBKEY_HASH, PUBKEY_HASH,
}; };
## Set of intelligence data types.
type TypeSet: set[Type];
## Data about an :bro:type:`Intel::Item`. ## Data about an :bro:type:`Intel::Item`.
type MetaData: record { type MetaData: record {
## An arbitrary string value representing the data source. ## An arbitrary string value representing the data source.
@ -123,6 +127,8 @@ export {
## Where the data was seen. ## Where the data was seen.
seen: Seen &log; seen: Seen &log;
## Which indicator types matched.
matched: TypeSet &log;
## Sources which supplied data that resulted in this match. ## Sources which supplied data that resulted in this match.
sources: set[string] &log &default=string_set(); sources: set[string] &log &default=string_set();
}; };
@ -162,6 +168,7 @@ type MetaDataTable: table[string] of MetaData;
# The in memory data structure for holding intelligence. # The in memory data structure for holding intelligence.
type DataStore: record { type DataStore: record {
host_data: table[addr] of MetaDataTable; host_data: table[addr] of MetaDataTable;
subnet_data: table[subnet] of MetaDataTable;
string_data: table[string, Type] of MetaDataTable; string_data: table[string, Type] of MetaDataTable;
}; };
global data_store: DataStore &redef; global data_store: DataStore &redef;
@ -171,6 +178,7 @@ global data_store: DataStore &redef;
# a minimal amount of data for the full match to happen on the manager. # a minimal amount of data for the full match to happen on the manager.
type MinDataStore: record { type MinDataStore: record {
host_data: set[addr]; host_data: set[addr];
subnet_data: set[subnet];
string_data: set[string, Type]; string_data: set[string, Type];
}; };
global min_data_store: MinDataStore &redef; global min_data_store: MinDataStore &redef;
@ -186,12 +194,11 @@ function find(s: Seen): bool
if ( s?$host ) if ( s?$host )
{ {
return ((s$host in min_data_store$host_data) || return ((s$host in min_data_store$host_data) ||
(have_full_data && s$host in data_store$host_data)); (|matching_subnets(addr_to_subnet(s$host), min_data_store$subnet_data)| > 0));
} }
else else
{ {
return (([to_lower(s$indicator), s$indicator_type] in min_data_store$string_data) || return ([to_lower(s$indicator), s$indicator_type] in min_data_store$string_data);
(have_full_data && [to_lower(s$indicator), s$indicator_type] in data_store$string_data));
} }
} }
@ -219,6 +226,17 @@ function get_items(s: Seen): set[Item]
add return_data[Item($indicator=cat(s$host), $indicator_type=ADDR, $meta=mt[m])]; add return_data[Item($indicator=cat(s$host), $indicator_type=ADDR, $meta=mt[m])];
} }
} }
# See if the host is part of a known subnet, which has meta values
local nets: table[subnet] of MetaDataTable;
nets = filter_subnet_table(addr_to_subnet(s$host), data_store$subnet_data);
for ( n in nets )
{
mt = nets[n];
for ( m in mt )
{
add return_data[Item($indicator=cat(n), $indicator_type=SUBNET, $meta=mt[m])];
}
}
} }
else else
{ {
@ -266,7 +284,7 @@ function Intel::seen(s: Seen)
event Intel::match(s: Seen, items: set[Item]) &priority=5 event Intel::match(s: Seen, items: set[Item]) &priority=5
{ {
local info = Info($ts=network_time(), $seen=s); local info = Info($ts=network_time(), $seen=s, $matched=TypeSet());
if ( s?$f ) if ( s?$f )
{ {
@ -293,7 +311,10 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5
} }
for ( item in items ) for ( item in items )
{
add info$sources[item$meta$source]; add info$sources[item$meta$source];
add info$matched[item$indicator_type];
}
Log::write(Intel::LOG, info); Log::write(Intel::LOG, info);
} }
@ -323,6 +344,21 @@ function insert(item: Item)
add min_data_store$host_data[host]; add min_data_store$host_data[host];
} }
else if ( item$indicator_type == SUBNET )
{
local net = to_subnet(item$indicator);
if ( have_full_data )
{
if ( net !in data_store$subnet_data )
data_store$subnet_data[net] = table();
else
is_new = F;
meta_tbl = data_store$subnet_data[net];
}
add min_data_store$subnet_data[net];
}
else else
{ {
if ( have_full_data ) if ( have_full_data )

10
testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1.intel.log
View file

@ -3,8 +3,8 @@
#empty_field (empty) #empty_field (empty)
#unset_field - #unset_field -
#path intel #path intel
#open 2014-09-23-16-13-39 #open 2016-03-22-18-11-20
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node sources #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node matched sources
#types time string addr port addr port string string string string enum enum string set[string] #types time string addr port addr port string string string string enum enum string set[enum] set[string]
1411488819.555114 - - - - - - - - 123.123.123.123 Intel::ADDR Intel::IN_ANYWHERE worker-2 worker-1 1458670280.078658 - - - - - - - - 123.123.123.123 Intel::ADDR Intel::IN_ANYWHERE worker-2 Intel::ADDR worker-1
#close 2014-09-23-16-13-49 #close 2016-03-22-18-11-29

12
testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/broproc.intel.log
View file

@ -3,9 +3,9 @@
#empty_field (empty) #empty_field (empty)
#unset_field - #unset_field -
#path intel #path intel
#open 2014-09-23-16-14-49 #open 2016-03-22-18-11-32
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node sources #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node matched sources
#types time string addr port addr port string string string string enum enum string set[string] #types time string addr port addr port string string string string enum enum string set[enum] set[string]
1411488889.571819 - - - - - - - - e@mail.com Intel::EMAIL SOMEWHERE bro source1 1458670292.167298 - - - - - - - - e@mail.com Intel::EMAIL SOMEWHERE bro Intel::EMAIL source1
1411488889.571819 - - - - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro source1 1458670292.167298 - - - - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro Intel::ADDR source1
#close 2014-09-23-16-14-49 #close 2016-03-22-18-11-32

23
testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output Normal file
View file

@ -0,0 +1,23 @@
#separator \x09
#set_separator ,
#empty_field (empty)
#unset_field -
#path intel
#open 2016-03-22-18-11-34
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node matched sources
#types time string addr port addr port string string string string enum enum string set[enum] set[string]
1458670294.227182 - - - - - - - - 192.168.1.1 Intel::ADDR SOMEWHERE bro Intel::ADDR source1
1458670294.227182 - - - - - - - - 192.168.2.1 Intel::ADDR SOMEWHERE bro Intel::SUBNET source1
1458670294.227182 - - - - - - - - 192.168.142.1 Intel::ADDR SOMEWHERE bro Intel::ADDR,Intel::SUBNET source1
#close 2016-03-22-18-11-34
Seen: [indicator=192.168.1.1, indicator_type=Intel::ADDR, host=192.168.1.1, where=SOMEWHERE, node=bro, conn=<uninitialized>, f=<uninitialized>]
Item: [indicator=192.168.1.1, indicator_type=Intel::ADDR, meta=[source=source1, desc=this host is just plain baaad, url=http://some-data-distributor.com/1]]
Seen: [indicator=192.168.2.1, indicator_type=Intel::ADDR, host=192.168.2.1, where=SOMEWHERE, node=bro, conn=<uninitialized>, f=<uninitialized>]
Item: [indicator=192.168.2.0/24, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is just plain baaad, url=http://some-data-distributor.com/2]]
Seen: [indicator=192.168.142.1, indicator_type=Intel::ADDR, host=192.168.142.1, where=SOMEWHERE, node=bro, conn=<uninitialized>, f=<uninitialized>]
Item: [indicator=192.168.142.0/24, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is baaad, url=http://some-data-distributor.com/4]]
Item: [indicator=192.168.142.1, indicator_type=Intel::ADDR, meta=[source=source1, desc=this host is just plain baaad, url=http://some-data-distributor.com/3]]
Item: [indicator=192.168.128.0/18, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork might be baaad, url=http://some-data-distributor.com/5]]

16
testing/btest/Baseline/scripts.base.frameworks.intel.read-file-dist-cluster/manager-1.intel.log
View file

@ -3,11 +3,11 @@
#empty_field (empty) #empty_field (empty)
#unset_field - #unset_field -
#path intel #path intel
#open 2014-09-23-16-15-00 #open 2016-03-22-18-11-40
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node sources #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node matched sources
#types time string addr port addr port string string string string enum enum string set[string] #types time string addr port addr port string string string string enum enum string set[enum] set[string]
1411488900.900403 - - - - - - - - 1.2.3.4 Intel::ADDR Intel::IN_A_TEST worker-1 source1 1458670300.363597 - - - - - - - - 1.2.3.4 Intel::ADDR Intel::IN_A_TEST worker-1 Intel::ADDR source1
1411488900.900403 - - - - - - - - e@mail.com Intel::EMAIL Intel::IN_A_TEST worker-1 source1 1458670300.363597 - - - - - - - - e@mail.com Intel::EMAIL Intel::IN_A_TEST worker-1 Intel::EMAIL source1
1411488901.923543 - - - - - - - - 1.2.3.4 Intel::ADDR Intel::IN_A_TEST worker-2 source1 1458670301.370555 - - - - - - - - 1.2.3.4 Intel::ADDR Intel::IN_A_TEST worker-2 Intel::ADDR source1
1411488901.923543 - - - - - - - - e@mail.com Intel::EMAIL Intel::IN_A_TEST worker-2 source1 1458670301.370555 - - - - - - - - e@mail.com Intel::EMAIL Intel::IN_A_TEST worker-2 Intel::EMAIL source1
#close 2014-09-23-16-15-09 #close 2016-03-22-18-11-49

26
testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output
View file

@ -3,23 +3,23 @@
#empty_field (empty) #empty_field (empty)
#unset_field - #unset_field -
#path intel #path intel
#open 2016-03-19-16-01-52 #open 2016-03-22-18-11-51
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node sources #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node matched sources
#types time string addr port addr port string string string string enum enum string set[string] #types time string addr port addr port string string string string enum enum string set[enum] set[string]
1458403312.669166 - - - - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro source1 1458670311.505318 - - - - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro Intel::ADDR source1
1458403315.672095 - - - - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro source2,source1 1458670314.509318 - - - - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro Intel::ADDR source2,source1
1458403315.672095 - - - - - - - - 4.3.2.1 Intel::ADDR SOMEWHERE bro source2 1458670314.509318 - - - - - - - - 4.3.2.1 Intel::ADDR SOMEWHERE bro Intel::ADDR source2
1458403318.675592 - - - - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro source2,source1 1458670317.513183 - - - - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro Intel::ADDR source2,source1
1458403318.675592 - - - - - - - - 4.3.2.1 Intel::ADDR SOMEWHERE bro source2 1458670317.513183 - - - - - - - - 4.3.2.1 Intel::ADDR SOMEWHERE bro Intel::ADDR source2
#close 2016-03-19-16-01-58 #close 2016-03-22-18-11-57
#separator \x09 #separator \x09
#set_separator , #set_separator ,
#empty_field (empty) #empty_field (empty)
#unset_field - #unset_field -
#path notice #path notice
#open 2016-03-19-16-01-58 #open 2016-03-22-18-11-57
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude
#types time string addr port addr port string string string enum enum string string addr addr port count string set[enum] interval bool string string string double double #types time string addr port addr port string string string enum enum string string addr addr port count string set[enum] interval bool string string string double double
1458403318.675592 - - - - - - - - - Intel::Notice Intel hit on 1.2.3.4 at SOMEWHERE 1.2.3.4 - - - - bro Notice::ACTION_LOG 3600.000000 F - - - - - 1458670317.513183 - - - - - - - - - Intel::Notice Intel hit on 1.2.3.4 at SOMEWHERE 1.2.3.4 - - - - bro Notice::ACTION_LOG 3600.000000 F - - - - -
1458403318.675592 - - - - - - - - - Intel::Notice Intel hit on 4.3.2.1 at SOMEWHERE 4.3.2.1 - - - - bro Notice::ACTION_LOG 3600.000000 F - - - - - 1458670317.513183 - - - - - - - - - Intel::Notice Intel hit on 4.3.2.1 at SOMEWHERE 4.3.2.1 - - - - bro Notice::ACTION_LOG 3600.000000 F - - - - -
#close 2016-03-19-16-01-58 #close 2016-03-22-18-11-57

24
testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log
View file

@ -3,20 +3,20 @@
#empty_field (empty) #empty_field (empty)
#unset_field - #unset_field -
#path intel #path intel
#open 2015-03-14-01-47-46 #open 2016-03-22-18-09-35
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node sources #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node matched sources
#types time string addr port addr port string string string string enum enum string set[string] #types time string addr port addr port string string string string enum enum string set[enum] set[string]
1416942644.593119 CXWv6p3arKYeMETxOg 192.168.4.149 49422 23.92.19.75 443 F0txuw2pvrkZOn04a8 application/pkix-cert 23.92.19.75:443/tcp www.pantz.org Intel::DOMAIN X509::IN_CERT bro source1 1416942644.593119 CXWv6p3arKYeMETxOg 192.168.4.149 49422 23.92.19.75 443 F0txuw2pvrkZOn04a8 application/pkix-cert 23.92.19.75:443/tcp www.pantz.org Intel::DOMAIN X509::IN_CERT bro Intel::DOMAIN source1
#close 2015-03-14-01-47-46 #close 2016-03-22-18-09-35
#separator \x09 #separator \x09
#set_separator , #set_separator ,
#empty_field (empty) #empty_field (empty)
#unset_field - #unset_field -
#path intel #path intel
#open 2015-03-14-01-47-46 #open 2016-03-22-18-09-35
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node sources #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node matched sources
#types time string addr port addr port string string string string enum enum string set[string] #types time string addr port addr port string string string string enum enum string set[enum] set[string]
1170717505.934612 CXWv6p3arKYeMETxOg 192.150.187.164 58868 194.127.84.106 443 - - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro source1 1170717505.934612 CXWv6p3arKYeMETxOg 192.150.187.164 58868 194.127.84.106 443 - - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro Intel::DOMAIN source1
1170717509.082241 CjhGID4nQcgTWjvg4c 192.150.187.164 58869 194.127.84.106 443 - - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro source1 1170717509.082241 CjhGID4nQcgTWjvg4c 192.150.187.164 58869 194.127.84.106 443 - - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro Intel::DOMAIN source1
1170717512.108799 CCvvfg3TEfuqmmG4bh 192.150.187.164 58870 194.127.84.106 443 - - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro source1 1170717512.108799 CCvvfg3TEfuqmmG4bh 192.150.187.164 58870 194.127.84.106 443 - - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro Intel::DOMAIN source1
#close 2015-03-14-01-47-46 #close 2016-03-22-18-09-35

50
testing/btest/scripts/base/frameworks/intel/match-subnet.bro Normal file
View file

@ -0,0 +1,50 @@
# @TEST-EXEC: btest-bg-run broproc bro %INPUT
# @TEST-EXEC: btest-bg-wait -k 5
# @TEST-EXEC: cat broproc/intel.log > output
# @TEST-EXEC: cat broproc/.stdout >> output
# @TEST-EXEC: btest-diff output
# @TEST-START-FILE intel.dat
#fields indicator indicator_type meta.source meta.desc meta.url
192.168.1.1 Intel::ADDR source1 this host is just plain baaad http://some-data-distributor.com/1
192.168.2.0/24 Intel::SUBNET source1 this subnetwork is just plain baaad http://some-data-distributor.com/2
192.168.142.1 Intel::ADDR source1 this host is just plain baaad http://some-data-distributor.com/3
192.168.142.0/24 Intel::SUBNET source1 this subnetwork is baaad http://some-data-distributor.com/4
192.168.128.0/18 Intel::SUBNET source1 this subnetwork might be baaad http://some-data-distributor.com/5
# @TEST-END-FILE
@load frameworks/communication/listen
redef Intel::read_files += { "../intel.dat" };
redef enum Intel::Where += { SOMEWHERE };
event do_it()
{
Intel::seen([$host=192.168.1.1,
$where=SOMEWHERE]);
Intel::seen([$host=192.168.2.1,
$where=SOMEWHERE]);
Intel::seen([$host=192.168.142.1,
$where=SOMEWHERE]);
}
event bro_init() &priority=-10
{
schedule 1sec { do_it() };
}
global log_lines = 0;
event Intel::log_intel(rec: Intel::Info)
{
++log_lines;
if ( log_lines == 2 )
terminate();
}
event Intel::match(s: Intel::Seen, items: set[Intel::Item])
{
print "";
print fmt("Seen: %s", s);
for ( item in items )
print fmt("Item: %s", item);
}