diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro index 32b48a089b..f3bceec25e 100644 --- a/scripts/base/frameworks/intel/main.bro +++ b/scripts/base/frameworks/intel/main.bro @@ -14,6 +14,8 @@ export { type Type: enum { ## An IP address. ADDR, + ## A subnet in CIDR notation. + SUBNET, ## A complete URL without the prefix ``"http://"``. URL, ## Software name. @@ -35,7 +37,9 @@ export { ## Public key MD5 hash. (SSH server host keys are a good example.) PUBKEY_HASH, }; - + ## Set of intelligence data types. + type TypeSet: set[Type]; + ## Data about an :bro:type:`Intel::Item`. type MetaData: record { ## An arbitrary string value representing the data source. @@ -123,6 +127,8 @@ export { ## Where the data was seen. seen: Seen &log; + ## Which indicator types matched. + matched: TypeSet &log; ## Sources which supplied data that resulted in this match. sources: set[string] &log &default=string_set(); }; @@ -162,6 +168,7 @@ type MetaDataTable: table[string] of MetaData; # The in memory data structure for holding intelligence. type DataStore: record { host_data: table[addr] of MetaDataTable; + subnet_data: table[subnet] of MetaDataTable; string_data: table[string, Type] of MetaDataTable; }; global data_store: DataStore &redef; @@ -171,6 +178,7 @@ global data_store: DataStore &redef; # a minimal amount of data for the full match to happen on the manager. type MinDataStore: record { host_data: set[addr]; + subnet_data: set[subnet]; string_data: set[string, Type]; }; global min_data_store: MinDataStore &redef; @@ -186,12 +194,11 @@ function find(s: Seen): bool if ( s?$host ) { return ((s$host in min_data_store$host_data) || - (have_full_data && s$host in data_store$host_data)); + (|matching_subnets(addr_to_subnet(s$host), min_data_store$subnet_data)| > 0)); } else { - return (([to_lower(s$indicator), s$indicator_type] in min_data_store$string_data) || - (have_full_data && [to_lower(s$indicator), s$indicator_type] in data_store$string_data)); + return ([to_lower(s$indicator), s$indicator_type] in min_data_store$string_data); } } @@ -219,6 +226,17 @@ function get_items(s: Seen): set[Item] add return_data[Item($indicator=cat(s$host), $indicator_type=ADDR, $meta=mt[m])]; } } + # See if the host is part of a known subnet, which has meta values + local nets: table[subnet] of MetaDataTable; + nets = filter_subnet_table(addr_to_subnet(s$host), data_store$subnet_data); + for ( n in nets ) + { + mt = nets[n]; + for ( m in mt ) + { + add return_data[Item($indicator=cat(n), $indicator_type=SUBNET, $meta=mt[m])]; + } + } } else { @@ -266,7 +284,7 @@ function Intel::seen(s: Seen) event Intel::match(s: Seen, items: set[Item]) &priority=5 { - local info = Info($ts=network_time(), $seen=s); + local info = Info($ts=network_time(), $seen=s, $matched=TypeSet()); if ( s?$f ) { @@ -293,7 +311,10 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5 } for ( item in items ) + { add info$sources[item$meta$source]; + add info$matched[item$indicator_type]; + } Log::write(Intel::LOG, info); } @@ -323,6 +344,21 @@ function insert(item: Item) add min_data_store$host_data[host]; } + else if ( item$indicator_type == SUBNET ) + { + local net = to_subnet(item$indicator); + if ( have_full_data ) + { + if ( net !in data_store$subnet_data ) + data_store$subnet_data[net] = table(); + else + is_new = F; + + meta_tbl = data_store$subnet_data[net]; + } + + add min_data_store$subnet_data[net]; + } else { if ( have_full_data ) diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1.intel.log b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1.intel.log index ba19f4e8d7..015d2f21bd 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1.intel.log +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1.intel.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path intel -#open 2014-09-23-16-13-39 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node sources -#types time string addr port addr port string string string string enum enum string set[string] -1411488819.555114 - - - - - - - - 123.123.123.123 Intel::ADDR Intel::IN_ANYWHERE worker-2 worker-1 -#close 2014-09-23-16-13-49 +#open 2016-03-22-18-11-20 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node matched sources +#types time string addr port addr port string string string string enum enum string set[enum] set[string] +1458670280.078658 - - - - - - - - 123.123.123.123 Intel::ADDR Intel::IN_ANYWHERE worker-2 Intel::ADDR worker-1 +#close 2016-03-22-18-11-29 diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/broproc.intel.log b/testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/broproc.intel.log index 33c97c0c1e..4436253d96 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/broproc.intel.log +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/broproc.intel.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path intel -#open 2014-09-23-16-14-49 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node sources -#types time string addr port addr port string string string string enum enum string set[string] -1411488889.571819 - - - - - - - - e@mail.com Intel::EMAIL SOMEWHERE bro source1 -1411488889.571819 - - - - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro source1 -#close 2014-09-23-16-14-49 +#open 2016-03-22-18-11-32 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node matched sources +#types time string addr port addr port string string string string enum enum string set[enum] set[string] +1458670292.167298 - - - - - - - - e@mail.com Intel::EMAIL SOMEWHERE bro Intel::EMAIL source1 +1458670292.167298 - - - - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro Intel::ADDR source1 +#close 2016-03-22-18-11-32 diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output b/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output new file mode 100644 index 0000000000..3fbd90949f --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output @@ -0,0 +1,23 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path intel +#open 2016-03-22-18-11-34 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node matched sources +#types time string addr port addr port string string string string enum enum string set[enum] set[string] +1458670294.227182 - - - - - - - - 192.168.1.1 Intel::ADDR SOMEWHERE bro Intel::ADDR source1 +1458670294.227182 - - - - - - - - 192.168.2.1 Intel::ADDR SOMEWHERE bro Intel::SUBNET source1 +1458670294.227182 - - - - - - - - 192.168.142.1 Intel::ADDR SOMEWHERE bro Intel::ADDR,Intel::SUBNET source1 +#close 2016-03-22-18-11-34 + +Seen: [indicator=192.168.1.1, indicator_type=Intel::ADDR, host=192.168.1.1, where=SOMEWHERE, node=bro, conn=, f=] +Item: [indicator=192.168.1.1, indicator_type=Intel::ADDR, meta=[source=source1, desc=this host is just plain baaad, url=http://some-data-distributor.com/1]] + +Seen: [indicator=192.168.2.1, indicator_type=Intel::ADDR, host=192.168.2.1, where=SOMEWHERE, node=bro, conn=, f=] +Item: [indicator=192.168.2.0/24, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is just plain baaad, url=http://some-data-distributor.com/2]] + +Seen: [indicator=192.168.142.1, indicator_type=Intel::ADDR, host=192.168.142.1, where=SOMEWHERE, node=bro, conn=, f=] +Item: [indicator=192.168.142.0/24, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is baaad, url=http://some-data-distributor.com/4]] +Item: [indicator=192.168.142.1, indicator_type=Intel::ADDR, meta=[source=source1, desc=this host is just plain baaad, url=http://some-data-distributor.com/3]] +Item: [indicator=192.168.128.0/18, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork might be baaad, url=http://some-data-distributor.com/5]] diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.read-file-dist-cluster/manager-1.intel.log b/testing/btest/Baseline/scripts.base.frameworks.intel.read-file-dist-cluster/manager-1.intel.log index d8e2d43674..074e6a0122 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.intel.read-file-dist-cluster/manager-1.intel.log +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.read-file-dist-cluster/manager-1.intel.log @@ -3,11 +3,11 @@ #empty_field (empty) #unset_field - #path intel -#open 2014-09-23-16-15-00 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node sources -#types time string addr port addr port string string string string enum enum string set[string] -1411488900.900403 - - - - - - - - 1.2.3.4 Intel::ADDR Intel::IN_A_TEST worker-1 source1 -1411488900.900403 - - - - - - - - e@mail.com Intel::EMAIL Intel::IN_A_TEST worker-1 source1 -1411488901.923543 - - - - - - - - 1.2.3.4 Intel::ADDR Intel::IN_A_TEST worker-2 source1 -1411488901.923543 - - - - - - - - e@mail.com Intel::EMAIL Intel::IN_A_TEST worker-2 source1 -#close 2014-09-23-16-15-09 +#open 2016-03-22-18-11-40 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node matched sources +#types time string addr port addr port string string string string enum enum string set[enum] set[string] +1458670300.363597 - - - - - - - - 1.2.3.4 Intel::ADDR Intel::IN_A_TEST worker-1 Intel::ADDR source1 +1458670300.363597 - - - - - - - - e@mail.com Intel::EMAIL Intel::IN_A_TEST worker-1 Intel::EMAIL source1 +1458670301.370555 - - - - - - - - 1.2.3.4 Intel::ADDR Intel::IN_A_TEST worker-2 Intel::ADDR source1 +1458670301.370555 - - - - - - - - e@mail.com Intel::EMAIL Intel::IN_A_TEST worker-2 Intel::EMAIL source1 +#close 2016-03-22-18-11-49 diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output b/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output index 3e7fa97f40..be3604f541 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output @@ -3,23 +3,23 @@ #empty_field (empty) #unset_field - #path intel -#open 2016-03-19-16-01-52 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node sources -#types time string addr port addr port string string string string enum enum string set[string] -1458403312.669166 - - - - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro source1 -1458403315.672095 - - - - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro source2,source1 -1458403315.672095 - - - - - - - - 4.3.2.1 Intel::ADDR SOMEWHERE bro source2 -1458403318.675592 - - - - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro source2,source1 -1458403318.675592 - - - - - - - - 4.3.2.1 Intel::ADDR SOMEWHERE bro source2 -#close 2016-03-19-16-01-58 +#open 2016-03-22-18-11-51 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node matched sources +#types time string addr port addr port string string string string enum enum string set[enum] set[string] +1458670311.505318 - - - - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro Intel::ADDR source1 +1458670314.509318 - - - - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro Intel::ADDR source2,source1 +1458670314.509318 - - - - - - - - 4.3.2.1 Intel::ADDR SOMEWHERE bro Intel::ADDR source2 +1458670317.513183 - - - - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro Intel::ADDR source2,source1 +1458670317.513183 - - - - - - - - 4.3.2.1 Intel::ADDR SOMEWHERE bro Intel::ADDR source2 +#close 2016-03-22-18-11-57 #separator \x09 #set_separator , #empty_field (empty) #unset_field - #path notice -#open 2016-03-19-16-01-58 +#open 2016-03-22-18-11-57 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude #types time string addr port addr port string string string enum enum string string addr addr port count string set[enum] interval bool string string string double double -1458403318.675592 - - - - - - - - - Intel::Notice Intel hit on 1.2.3.4 at SOMEWHERE 1.2.3.4 - - - - bro Notice::ACTION_LOG 3600.000000 F - - - - - -1458403318.675592 - - - - - - - - - Intel::Notice Intel hit on 4.3.2.1 at SOMEWHERE 4.3.2.1 - - - - bro Notice::ACTION_LOG 3600.000000 F - - - - - -#close 2016-03-19-16-01-58 +1458670317.513183 - - - - - - - - - Intel::Notice Intel hit on 1.2.3.4 at SOMEWHERE 1.2.3.4 - - - - bro Notice::ACTION_LOG 3600.000000 F - - - - - +1458670317.513183 - - - - - - - - - Intel::Notice Intel hit on 4.3.2.1 at SOMEWHERE 4.3.2.1 - - - - bro Notice::ACTION_LOG 3600.000000 F - - - - - +#close 2016-03-22-18-11-57 diff --git a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log index ba1afe4239..ac88efb3d6 100644 --- a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log +++ b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log @@ -3,20 +3,20 @@ #empty_field (empty) #unset_field - #path intel -#open 2015-03-14-01-47-46 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node sources -#types time string addr port addr port string string string string enum enum string set[string] -1416942644.593119 CXWv6p3arKYeMETxOg 192.168.4.149 49422 23.92.19.75 443 F0txuw2pvrkZOn04a8 application/pkix-cert 23.92.19.75:443/tcp www.pantz.org Intel::DOMAIN X509::IN_CERT bro source1 -#close 2015-03-14-01-47-46 +#open 2016-03-22-18-09-35 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node matched sources +#types time string addr port addr port string string string string enum enum string set[enum] set[string] +1416942644.593119 CXWv6p3arKYeMETxOg 192.168.4.149 49422 23.92.19.75 443 F0txuw2pvrkZOn04a8 application/pkix-cert 23.92.19.75:443/tcp www.pantz.org Intel::DOMAIN X509::IN_CERT bro Intel::DOMAIN source1 +#close 2016-03-22-18-09-35 #separator \x09 #set_separator , #empty_field (empty) #unset_field - #path intel -#open 2015-03-14-01-47-46 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node sources -#types time string addr port addr port string string string string enum enum string set[string] -1170717505.934612 CXWv6p3arKYeMETxOg 192.150.187.164 58868 194.127.84.106 443 - - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro source1 -1170717509.082241 CjhGID4nQcgTWjvg4c 192.150.187.164 58869 194.127.84.106 443 - - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro source1 -1170717512.108799 CCvvfg3TEfuqmmG4bh 192.150.187.164 58870 194.127.84.106 443 - - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro source1 -#close 2015-03-14-01-47-46 +#open 2016-03-22-18-09-35 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node matched sources +#types time string addr port addr port string string string string enum enum string set[enum] set[string] +1170717505.934612 CXWv6p3arKYeMETxOg 192.150.187.164 58868 194.127.84.106 443 - - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro Intel::DOMAIN source1 +1170717509.082241 CjhGID4nQcgTWjvg4c 192.150.187.164 58869 194.127.84.106 443 - - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro Intel::DOMAIN source1 +1170717512.108799 CCvvfg3TEfuqmmG4bh 192.150.187.164 58870 194.127.84.106 443 - - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro Intel::DOMAIN source1 +#close 2016-03-22-18-09-35 diff --git a/testing/btest/scripts/base/frameworks/intel/match-subnet.bro b/testing/btest/scripts/base/frameworks/intel/match-subnet.bro new file mode 100644 index 0000000000..924fa947b6 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/intel/match-subnet.bro @@ -0,0 +1,50 @@ +# @TEST-EXEC: btest-bg-run broproc bro %INPUT +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: cat broproc/intel.log > output +# @TEST-EXEC: cat broproc/.stdout >> output +# @TEST-EXEC: btest-diff output + +# @TEST-START-FILE intel.dat +#fields indicator indicator_type meta.source meta.desc meta.url +192.168.1.1 Intel::ADDR source1 this host is just plain baaad http://some-data-distributor.com/1 +192.168.2.0/24 Intel::SUBNET source1 this subnetwork is just plain baaad http://some-data-distributor.com/2 +192.168.142.1 Intel::ADDR source1 this host is just plain baaad http://some-data-distributor.com/3 +192.168.142.0/24 Intel::SUBNET source1 this subnetwork is baaad http://some-data-distributor.com/4 +192.168.128.0/18 Intel::SUBNET source1 this subnetwork might be baaad http://some-data-distributor.com/5 +# @TEST-END-FILE + +@load frameworks/communication/listen + +redef Intel::read_files += { "../intel.dat" }; +redef enum Intel::Where += { SOMEWHERE }; + +event do_it() + { + Intel::seen([$host=192.168.1.1, + $where=SOMEWHERE]); + Intel::seen([$host=192.168.2.1, + $where=SOMEWHERE]); + Intel::seen([$host=192.168.142.1, + $where=SOMEWHERE]); + } + +event bro_init() &priority=-10 + { + schedule 1sec { do_it() }; + } + +global log_lines = 0; +event Intel::log_intel(rec: Intel::Info) + { + ++log_lines; + if ( log_lines == 2 ) + terminate(); + } + +event Intel::match(s: Intel::Seen, items: set[Intel::Item]) + { + print ""; + print fmt("Seen: %s", s); + for ( item in items ) + print fmt("Item: %s", item); + } \ No newline at end of file