mirror of
https://github.com/zeek/zeek.git
synced 2025-10-02 14:48:21 +00:00
Merge remote-tracking branch 'origin/topic/seth/unified2-analyzer' into topic/robin/unified2-analyzer-merge
* origin/topic/seth/unified2-analyzer: Fixed a problem where the Unified2 analyzer was attached to every file. Fixing intel framework tests. Updating submodule(s). Add file name support to intel framework. Add file support to intel framework and slightly restructure intel http handling. Conflicts: CHANGES VERSION scripts/base/files/unified2/main.bro testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
This commit is contained in:
Robin Sommer
commit
a646fde884
18 changed files with 155 additions and 58 deletions
11
CHANGES
11
CHANGES
|
|
@ -1,9 +1,18 @@
|
|||
|
||||
2.1-1063 | 2013-08-13 18:36:53 -0700
|
||||
2.1-1071 | 2013-08-14 10:25:39 -0700
|
||||
|
||||
* Adding the unified2 analyzer that reads unified2 files from disk,
|
||||
turning them into events. (Seth Hall)
|
||||
|
||||
* Fixing intel framework tests. (Seth Hall)
|
||||
|
||||
2.1-1059 | 2013-08-13 23:52:41 -0400
|
||||
|
||||
* Add file name support to intel framework. (Seth Hall)
|
||||
|
||||
* Add file support to intel framework and slightly restructure
|
||||
intel http handling. (Seth Hall)
|
||||
|
||||
2.1-1052 | 2013-08-12 14:38:14 -0700
|
||||
|
||||
* Fixing bug in DNP3 analyzer flagged by compiler warning. (Robin
|
||||
|
|
|
|||
2
VERSION
2
VERSION
|
|
@ -1 +1 @@
|
|||
2.1-1063
|
||||
2.1-1071
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
Subproject commit 1b114401e5eeea0ab2f0ba266f5c79f1e8060f34
|
||||
Subproject commit be7c653dcdc30384d4d17359d19d94540fdedaa5
|
||||
|
|
@ -202,9 +202,10 @@ rest_target(${psd} policy/frameworks/files/hash-all-files.bro)
|
|||
rest_target(${psd} policy/frameworks/intel/do_notice.bro)
|
||||
rest_target(${psd} policy/frameworks/intel/seen/conn-established.bro)
|
||||
rest_target(${psd} policy/frameworks/intel/seen/dns.bro)
|
||||
rest_target(${psd} policy/frameworks/intel/seen/http-host-header.bro)
|
||||
rest_target(${psd} policy/frameworks/intel/seen/file-hashes.bro)
|
||||
rest_target(${psd} policy/frameworks/intel/seen/file-names.bro)
|
||||
rest_target(${psd} policy/frameworks/intel/seen/http-headers.bro)
|
||||
rest_target(${psd} policy/frameworks/intel/seen/http-url.bro)
|
||||
rest_target(${psd} policy/frameworks/intel/seen/http-user-agents.bro)
|
||||
rest_target(${psd} policy/frameworks/intel/seen/smtp-url-extraction.bro)
|
||||
rest_target(${psd} policy/frameworks/intel/seen/smtp.bro)
|
||||
rest_target(${psd} policy/frameworks/intel/seen/ssl.bro)
|
||||
|
|
|
|||
|
|
@ -136,7 +136,7 @@ event Unified2::read_classification_line(desc: Input::EventDescription, tpe: Inp
|
|||
}
|
||||
}
|
||||
|
||||
event bro_init()
|
||||
event bro_init() &priority=5
|
||||
{
|
||||
Log::create_stream(Unified2::LOG, [$columns=Info, $ev=log_unified2]);
|
||||
|
||||
|
|
@ -200,8 +200,8 @@ event file_new(f: fa_file)
|
|||
if ( |parts| == 3 )
|
||||
file_dir = parts[1];
|
||||
|
||||
if ( f$source in watch_file ||
|
||||
compress_path(watch_dir) == file_dir )
|
||||
if ( (watch_file != "" && f$source == watch_file) ||
|
||||
(watch_dir != "" && compress_path(watch_dir) == file_dir) )
|
||||
{
|
||||
Files::add_analyzer(f, Files::ANALYZER_UNIFIED2);
|
||||
f$u2_events = table();
|
||||
|
|
|
|||
|
|
@ -27,6 +27,9 @@ export {
|
|||
## File hash which is non-hash type specific. It's up to the user to query
|
||||
## for any relevant hash types.
|
||||
FILE_HASH,
|
||||
## File names. Typically with protocols with definite indications
|
||||
## of a file name.
|
||||
FILE_NAME,
|
||||
## Certificate SHA-1 hash.
|
||||
CERT_HASH,
|
||||
};
|
||||
|
|
@ -80,6 +83,10 @@ export {
|
|||
## If the data was discovered within a connection, the
|
||||
## connection record should go into get to give context to the data.
|
||||
conn: connection &optional;
|
||||
|
||||
## If the data was discovered within a file, the file record
|
||||
## should go here to provide context to the data.
|
||||
f: fa_file &optional;
|
||||
};
|
||||
|
||||
## Record used for the logging framework representing a positive
|
||||
|
|
@ -95,6 +102,16 @@ export {
|
|||
## this is the conn_id for the connection.
|
||||
id: conn_id &log &optional;
|
||||
|
||||
## If a file was associated with this intelligence hit,
|
||||
## this is the uid for the file.
|
||||
fuid: string &log &optional;
|
||||
## A mime type if the intelligence hit is related to a file.
|
||||
## If the $f field is provided this will be automatically filled out.
|
||||
file_mime_type: string &log &optional;
|
||||
## Frequently files can be "described" to give a bit more context.
|
||||
## If the $f field is provided this field will be automatically filled out.
|
||||
file_desc: string &log &optional;
|
||||
|
||||
## Where the data was seen.
|
||||
seen: Seen &log;
|
||||
## Sources which supplied data that resulted in this match.
|
||||
|
|
@ -248,7 +265,25 @@ function has_meta(check: MetaData, metas: set[MetaData]): bool
|
|||
|
||||
event Intel::match(s: Seen, items: set[Item]) &priority=5
|
||||
{
|
||||
local info: Info = [$ts=network_time(), $seen=s];
|
||||
local info = Info($ts=network_time(), $seen=s);
|
||||
|
||||
if ( s?$f )
|
||||
{
|
||||
if ( s$f?$conns && |s$f$conns| == 1 )
|
||||
{
|
||||
for ( cid in s$f$conns )
|
||||
s$conn = s$f$conns[cid];
|
||||
}
|
||||
|
||||
if ( ! info?$fuid )
|
||||
info$fuid = s$f$id;
|
||||
|
||||
if ( ! info?$file_mime_type && s$f?$mime_type )
|
||||
info$file_mime_type = s$f$mime_type;
|
||||
|
||||
if ( ! info?$file_desc )
|
||||
info$file_desc = Files::describe(s$f);
|
||||
}
|
||||
|
||||
if ( s?$conn )
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
@load ./conn-established
|
||||
@load ./dns
|
||||
@load ./http-host-header
|
||||
@load ./file-hashes
|
||||
@load ./file-names
|
||||
@load ./http-headers
|
||||
@load ./http-url
|
||||
@load ./http-user-agents
|
||||
@load ./ssl
|
||||
@load ./smtp
|
||||
@load ./smtp-url-extraction
|
||||
12
scripts/policy/frameworks/intel/seen/file-hashes.bro
Normal file
12
scripts/policy/frameworks/intel/seen/file-hashes.bro
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
@load base/frameworks/intel
|
||||
@load ./where-locations
|
||||
|
||||
event file_hash(f: fa_file, kind: string, hash: string)
|
||||
{
|
||||
local seen = Intel::Seen($indicator=hash,
|
||||
$indicator_type=Intel::FILE_HASH,
|
||||
$f=f,
|
||||
$where=Files::IN_HASH);
|
||||
|
||||
Intel::seen(seen);
|
||||
}
|
||||
11
scripts/policy/frameworks/intel/seen/file-names.bro
Normal file
11
scripts/policy/frameworks/intel/seen/file-names.bro
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
@load base/frameworks/intel
|
||||
@load ./where-locations
|
||||
|
||||
event file_new(f: fa_file)
|
||||
{
|
||||
if ( f?$info && f$info?$filename )
|
||||
Intel::seen([$indicator=f$info$filename,
|
||||
$indicator_type=Intel::FILE_NAME,
|
||||
$f=f,
|
||||
$where=Files::IN_NAME]);
|
||||
}
|
||||
46
scripts/policy/frameworks/intel/seen/http-headers.bro
Normal file
46
scripts/policy/frameworks/intel/seen/http-headers.bro
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
@load base/frameworks/intel
|
||||
@load ./where-locations
|
||||
|
||||
event http_header(c: connection, is_orig: bool, name: string, value: string)
|
||||
{
|
||||
if ( is_orig )
|
||||
{
|
||||
switch ( name )
|
||||
{
|
||||
case "HOST":
|
||||
Intel::seen([$indicator=value,
|
||||
$indicator_type=Intel::DOMAIN,
|
||||
$conn=c,
|
||||
$where=HTTP::IN_HOST_HEADER]);
|
||||
break;
|
||||
|
||||
case "REFERER":
|
||||
Intel::seen([$indicator=sub(value, /^.*:\/\//, ""),
|
||||
$indicator_type=Intel::URL,
|
||||
$conn=c,
|
||||
$where=HTTP::IN_REFERRER_HEADER]);
|
||||
break;
|
||||
|
||||
case "X-FORWARDED-FOR":
|
||||
if ( is_valid_ip(value) )
|
||||
{
|
||||
local addrs = find_ip_addresses(value);
|
||||
for ( i in addrs )
|
||||
{
|
||||
Intel::seen([$host=to_addr(addrs[i]),
|
||||
$indicator_type=Intel::ADDR,
|
||||
$conn=c,
|
||||
$where=HTTP::IN_X_FORWARDED_FOR_HEADER]);
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
case "USER-AGENT":
|
||||
Intel::seen([$indicator=value,
|
||||
$indicator_type=Intel::SOFTWARE,
|
||||
$conn=c,
|
||||
$where=HTTP::IN_USER_AGENT_HEADER]);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
@load base/frameworks/intel
|
||||
@load ./where-locations
|
||||
|
||||
event http_header(c: connection, is_orig: bool, name: string, value: string)
|
||||
{
|
||||
if ( is_orig && name == "HOST" )
|
||||
Intel::seen([$indicator=value,
|
||||
$indicator_type=Intel::DOMAIN,
|
||||
$conn=c,
|
||||
$where=HTTP::IN_HOST_HEADER]);
|
||||
}
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
@load base/frameworks/intel
|
||||
@load ./where-locations
|
||||
|
||||
event http_header(c: connection, is_orig: bool, name: string, value: string)
|
||||
{
|
||||
if ( is_orig && name == "USER-AGENT" )
|
||||
Intel::seen([$indicator=value,
|
||||
$indicator_type=Intel::SOFTWARE,
|
||||
$conn=c,
|
||||
$where=HTTP::IN_USER_AGENT_HEADER]);
|
||||
}
|
||||
|
||||
|
|
@ -4,10 +4,14 @@ export {
|
|||
redef enum Intel::Where += {
|
||||
Conn::IN_ORIG,
|
||||
Conn::IN_RESP,
|
||||
Files::IN_HASH,
|
||||
Files::IN_NAME,
|
||||
DNS::IN_REQUEST,
|
||||
DNS::IN_RESPONSE,
|
||||
HTTP::IN_HOST_HEADER,
|
||||
HTTP::IN_REFERRER_HEADER,
|
||||
HTTP::IN_USER_AGENT_HEADER,
|
||||
HTTP::IN_X_FORWARDED_FOR_HEADER,
|
||||
HTTP::IN_URL,
|
||||
SMTP::IN_MAIL_FROM,
|
||||
SMTP::IN_RCPT_TO,
|
||||
|
|
|
|||
|
|
@ -18,9 +18,10 @@
|
|||
@load frameworks/intel/seen/__load__.bro
|
||||
@load frameworks/intel/seen/conn-established.bro
|
||||
@load frameworks/intel/seen/dns.bro
|
||||
@load frameworks/intel/seen/http-host-header.bro
|
||||
@load frameworks/intel/seen/file-hashes.bro
|
||||
@load frameworks/intel/seen/file-names.bro
|
||||
@load frameworks/intel/seen/http-headers.bro
|
||||
@load frameworks/intel/seen/http-url.bro
|
||||
@load frameworks/intel/seen/http-user-agents.bro
|
||||
@load frameworks/intel/seen/smtp-url-extraction.bro
|
||||
@load frameworks/intel/seen/smtp.bro
|
||||
@load frameworks/intel/seen/ssl.bro
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@
|
|||
#empty_field (empty)
|
||||
#unset_field -
|
||||
#path files
|
||||
#open 2013-08-12-19-07-37
|
||||
#open 2013-08-14-04-50-17
|
||||
#fields ts fuid tx_hosts rx_hosts conn_uids source depth analyzers mime_type filename duration local_orig is_orig seen_bytes total_bytes missing_bytes overflow_bytes timedout parent_fuid md5 sha1 sha256 extracted
|
||||
#types time string table[addr] table[addr] table[string] string count table[string] string string interval bool bool count count count count bool string string string string string
|
||||
1362692527.009721 G75mcAsU764 192.150.187.43 141.142.228.5 UWkUyAuUGXf HTTP 0 UNIFIED2,SHA256,DATA_EVENT,MD5,EXTRACT,SHA1 text/plain - 0.000054 - F 4705 4705 0 0 F - 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18 G75mcAsU764-file
|
||||
#close 2013-08-12-19-07-37
|
||||
1362692527.009721 G75mcAsU764 192.150.187.43 141.142.228.5 UWkUyAuUGXf HTTP 0 SHA256,DATA_EVENT,MD5,EXTRACT,SHA1 text/plain - 0.000054 - F 4705 4705 0 0 F - 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18 G75mcAsU764-file
|
||||
#close 2013-08-14-04-50-17
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@
|
|||
#empty_field (empty)
|
||||
#unset_field -
|
||||
#path intel
|
||||
#open 2013-07-19-17-05-48
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p seen.indicator seen.indicator_type seen.where sources
|
||||
#types time string addr port addr port string enum enum table[string]
|
||||
1374253548.038580 - - - - - 123.123.123.123 Intel::ADDR Intel::IN_ANYWHERE worker-1
|
||||
#close 2013-07-19-17-05-57
|
||||
#open 2013-08-14-03-46-32
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where sources
|
||||
#types time string addr port addr port string string string string enum enum table[string]
|
||||
1376451992.872806 - - - - - - - - 123.123.123.123 Intel::ADDR Intel::IN_ANYWHERE worker-1
|
||||
#close 2013-08-14-03-46-42
|
||||
|
|
|
|||
|
|
@ -3,9 +3,9 @@
|
|||
#empty_field (empty)
|
||||
#unset_field -
|
||||
#path intel
|
||||
#open 2013-07-19-17-04-26
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p seen.indicator seen.indicator_type seen.where sources
|
||||
#types time string addr port addr port string enum enum table[string]
|
||||
1374253466.857185 - - - - - e@mail.com Intel::EMAIL SOMEWHERE source1
|
||||
1374253466.857185 - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE source1
|
||||
#close 2013-07-19-17-04-26
|
||||
#open 2013-08-14-03-47-03
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where sources
|
||||
#types time string addr port addr port string string string string enum enum table[string]
|
||||
1376452023.137179 - - - - - - - - e@mail.com Intel::EMAIL SOMEWHERE source1
|
||||
1376452023.137179 - - - - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE source1
|
||||
#close 2013-08-14-03-47-03
|
||||
|
|
|
|||
|
|
@ -3,11 +3,11 @@
|
|||
#empty_field (empty)
|
||||
#unset_field -
|
||||
#path intel
|
||||
#open 2013-07-19-17-06-57
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p seen.indicator seen.indicator_type seen.where sources
|
||||
#types time string addr port addr port string enum enum table[string]
|
||||
1374253617.312158 - - - - - 1.2.3.4 Intel::ADDR Intel::IN_A_TEST source1
|
||||
1374253617.312158 - - - - - e@mail.com Intel::EMAIL Intel::IN_A_TEST source1
|
||||
1374253618.332565 - - - - - 1.2.3.4 Intel::ADDR Intel::IN_A_TEST source1
|
||||
1374253618.332565 - - - - - e@mail.com Intel::EMAIL Intel::IN_A_TEST source1
|
||||
#close 2013-07-19-17-07-06
|
||||
#open 2013-08-14-03-47-23
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where sources
|
||||
#types time string addr port addr port string string string string enum enum table[string]
|
||||
1376452043.835810 - - - - - - - - 1.2.3.4 Intel::ADDR Intel::IN_A_TEST source1
|
||||
1376452043.835810 - - - - - - - - e@mail.com Intel::EMAIL Intel::IN_A_TEST source1
|
||||
1376452044.855238 - - - - - - - - 1.2.3.4 Intel::ADDR Intel::IN_A_TEST source1
|
||||
1376452044.855238 - - - - - - - - e@mail.com Intel::EMAIL Intel::IN_A_TEST source1
|
||||
#close 2013-08-14-03-47-32
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue