Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 14:48:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Improve performance of MHR script, addresses BIT-1139.

Browse source 
The MHR script involves a "when" statement which can be expensive due to
the way it clones frames/vals.  In this case, the fa_file record is
expensive to clone, but this change works around that by unrolling only
the necessary fields from it that are needed to populate a Notice::Info
record.  A drawback to this is that the full fa_file or connection
records aren't available in the Notice::Info record when evaluating
Notice::policy hooks for MHR hit notices (though they can possibly be
recovered by using e.g. the lookup_connection() builtin_function).
This commit is contained in:
Jon Siwek 2014-03-11 13:18:14 -05:00
parent 18eb31a6df
commit d3f88ba9d1
2 changed files with 93 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

84
scripts/base/frameworks/notice/main.bro
View file

@ -206,6 +206,38 @@ export {
## The maximum amount of time a plugin can delay email from being sent. ## The maximum amount of time a plugin can delay email from being sent.
const max_email_delay = 15secs &redef; const max_email_delay = 15secs &redef;
## Contains a portion of :bro:see:`fa_file` that's also contained in
## :bro:see:`Notice::Info`.
type FileInfo: record {
fuid: string; ##< File UID.
desc: string; ##< File description from e.g.
##< :bro:see:`Files::describe`.
mime: string &optional; ##< Strongest mime type match for file.
cid: conn_id &optional; ##< Connection tuple over which file is sent.
cuid: string &optional; ##< Connection UID over which file is sent.
};
## Creates a record containing a subset of a full :bro:see:`fa_file` record.
##
## f: record containing metadata about a file.
##
## Returns: record containing a subset of fields copied from *f*.
global create_file_info: function(f: fa_file): Notice::FileInfo;
## Populates file-related fields in a notice info record.
##
## f: record containing metadata about a file.
##
## n: a notice record that needs file-related fields populated.
global populate_file_info: function(f: fa_file, n: Notice::Info);
## Populates file-related fields in a notice info record.
##
## fi: record containing metadata about a file.
##
## n: a notice record that needs file-related fields populated.
global populate_file_info2: function(fi: Notice::FileInfo, n: Notice::Info);
## A log postprocessing function that implements emailing the contents ## A log postprocessing function that implements emailing the contents
## of a log upon rotation to any configured :bro:id:`Notice::mail_dest`. ## of a log upon rotation to any configured :bro:id:`Notice::mail_dest`.
## The rotated log is removed upon being sent. ## The rotated log is removed upon being sent.
@ -493,6 +525,42 @@ function execute_with_notice(cmd: string, n: Notice::Info)
#system_env(cmd, tags); #system_env(cmd, tags);
} }
function create_file_info(f: fa_file): Notice::FileInfo
{
local fi: Notice::FileInfo = Notice::FileInfo($fuid = f$id,
$desc = Files::describe(f));
if ( f?$mime_type )
fi$mime = f$mime_type;
if ( f?$conns && |f$conns| == 1 )
for ( id in f$conns )
{
fi$cid = id;
fi$cuid = f$conns[id]$uid;
}
return fi;
}
function populate_file_info(f: fa_file, n: Notice::Info)
{
populate_file_info2(create_file_info(f), n);
}
function populate_file_info2(fi: Notice::FileInfo, n: Notice::Info)
{
if ( ! n?$fuid )
n$fuid = fi$fuid;
if ( ! n?$file_mime_type && fi?$mime )
n$file_mime_type = fi$mime;
n$file_desc = fi$desc;
n$id = fi$cid;
n$uid = fi$cuid;
}
# This is run synchronously as a function before all of the other # This is run synchronously as a function before all of the other
# notice related functions and events. It also modifies the # notice related functions and events. It also modifies the
# :bro:type:`Notice::Info` record in place. # :bro:type:`Notice::Info` record in place.
@ -503,21 +571,7 @@ function apply_policy(n: Notice::Info)
n$ts = network_time(); n$ts = network_time();
if ( n?$f ) if ( n?$f )
{ populate_file_info(n$f, n);
if ( ! n?$fuid )
n$fuid = n$f$id;
if ( ! n?$file_mime_type && n$f?$mime_type )
n$file_mime_type = n$f$mime_type;
n$file_desc = Files::describe(n$f);
if ( n$f?$conns && |n$f$conns| == 1 )
{
for ( id in n$f$conns )
n$conn = n$f$conns[id];
}
}
if ( n?$conn ) if ( n?$conn )
{ {

39
scripts/policy/frameworks/files/detect-MHR.bro
View file

@ -35,28 +35,37 @@ export {
const notice_threshold = 10 &redef; const notice_threshold = 10 &redef;
} }
event file_hash(f: fa_file, kind: string, hash: string) function do_mhr_lookup(hash: string, fi: Notice::FileInfo)
{ {
if ( kind=="sha1" && match_file_types in f$mime_type ) local hash_domain = fmt("%s.malware.hash.cymru.com", hash);
when ( local MHR_result = lookup_hostname_txt(hash_domain) )
{ {
local hash_domain = fmt("%s.malware.hash.cymru.com", hash); # Data is returned as "<dateFirstDetected> <detectionRate>"
when ( local MHR_result = lookup_hostname_txt(hash_domain) ) local MHR_answer = split1(MHR_result, / /);
if ( |MHR_answer| == 2 )
{ {
# Data is returned as "<dateFirstDetected> <detectionRate>" local mhr_detect_rate = to_count(MHR_answer[2]);
local MHR_answer = split1(MHR_result, / /);
if ( |MHR_answer| == 2 ) if ( mhr_detect_rate >= notice_threshold )
{ {
local mhr_first_detected = double_to_time(to_double(MHR_answer[1])); local mhr_first_detected = double_to_time(to_double(MHR_answer[1]));
local mhr_detect_rate = to_count(MHR_answer[2]);
local readable_first_detected = strftime("%Y-%m-%d %H:%M:%S", mhr_first_detected); local readable_first_detected = strftime("%Y-%m-%d %H:%M:%S", mhr_first_detected);
if ( mhr_detect_rate >= notice_threshold ) local message = fmt("Malware Hash Registry Detection rate: %d%% Last seen: %s", mhr_detect_rate, readable_first_detected);
{ local virustotal_url = fmt(match_sub_url, hash);
local message = fmt("Malware Hash Registry Detection rate: %d%% Last seen: %s", mhr_detect_rate, readable_first_detected); # We don't have the full fa_file record here in order to
local virustotal_url = fmt(match_sub_url, hash); # avoid the "when" statement cloning it (expensive!).
NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]); local n: Notice::Info = Notice::Info($note=Match, $msg=message, $sub=virustotal_url);
} Notice::populate_file_info2(fi, n);
NOTICE(n);
} }
} }
} }
} }
event file_hash(f: fa_file, kind: string, hash: string)
{
if ( kind=="sha1" && match_file_types in f$mime_type )
do_mhr_lookup(hash, Notice::create_file_info(f));
}