Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-10 10:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge remote-tracking branch 'origin/topic/jsiwek/faster-mhr'

Browse source 
* origin/topic/jsiwek/faster-mhr:
  Improve performance of MHR script, addresses BIT-1139.
This commit is contained in:
Seth Hall 2014-03-19 17:20:55 -04:00
commit 0f3ed1a553
4 changed files with 99 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

5
CHANGES
View file

@ -1,4 +1,9 @@
2.2-250 | 2014-03-19 17:20:55 -0400
* Improve performance of MHR script by reducing cloned Vals in
a "when" scope. (Jon Siwek)
2.2-248 | 2014-03-19 14:47:40 -0400
* Make SumStats work incrementally and non-blocking in non-cluster

2
VERSION
View file

@ -1 +1 @@
2.2-248
2.2-250

84
scripts/base/frameworks/notice/main.bro
View file

@ -206,6 +206,38 @@ export {
## The maximum amount of time a plugin can delay email from being sent.
const max_email_delay = 15secs &redef;
## Contains a portion of :bro:see:`fa_file` that's also contained in
## :bro:see:`Notice::Info`.
type FileInfo: record {
fuid: string; ##< File UID.
desc: string; ##< File description from e.g.
##< :bro:see:`Files::describe`.
mime: string &optional; ##< Strongest mime type match for file.
cid: conn_id &optional; ##< Connection tuple over which file is sent.
cuid: string &optional; ##< Connection UID over which file is sent.
};
## Creates a record containing a subset of a full :bro:see:`fa_file` record.
##
## f: record containing metadata about a file.
##
## Returns: record containing a subset of fields copied from *f*.
global create_file_info: function(f: fa_file): Notice::FileInfo;
## Populates file-related fields in a notice info record.
##
## f: record containing metadata about a file.
##
## n: a notice record that needs file-related fields populated.
global populate_file_info: function(f: fa_file, n: Notice::Info);
## Populates file-related fields in a notice info record.
##
## fi: record containing metadata about a file.
##
## n: a notice record that needs file-related fields populated.
global populate_file_info2: function(fi: Notice::FileInfo, n: Notice::Info);
## A log postprocessing function that implements emailing the contents
## of a log upon rotation to any configured :bro:id:`Notice::mail_dest`.
## The rotated log is removed upon being sent.
@ -493,6 +525,42 @@ function execute_with_notice(cmd: string, n: Notice::Info)
#system_env(cmd, tags);
}
function create_file_info(f: fa_file): Notice::FileInfo
{
local fi: Notice::FileInfo = Notice::FileInfo($fuid = f$id,
$desc = Files::describe(f));
if ( f?$mime_type )
fi$mime = f$mime_type;
if ( f?$conns && |f$conns| == 1 )
for ( id in f$conns )
{
fi$cid = id;
fi$cuid = f$conns[id]$uid;
}
return fi;
}
function populate_file_info(f: fa_file, n: Notice::Info)
{
populate_file_info2(create_file_info(f), n);
}
function populate_file_info2(fi: Notice::FileInfo, n: Notice::Info)
{
if ( ! n?$fuid )
n$fuid = fi$fuid;
if ( ! n?$file_mime_type && fi?$mime )
n$file_mime_type = fi$mime;
n$file_desc = fi$desc;
n$id = fi$cid;
n$uid = fi$cuid;
}
# This is run synchronously as a function before all of the other
# notice related functions and events. It also modifies the
# :bro:type:`Notice::Info` record in place.
@ -503,21 +571,7 @@ function apply_policy(n: Notice::Info)
n$ts = network_time();
if ( n?$f )
{
if ( ! n?$fuid )
n$fuid = n$f$id;
if ( ! n?$file_mime_type && n$f?$mime_type )
n$file_mime_type = n$f$mime_type;
n$file_desc = Files::describe(n$f);
if ( n$f?$conns && |n$f$conns| == 1 )
{
for ( id in n$f$conns )
n$conn = n$f$conns[id];
}
}
populate_file_info(n$f, n);
if ( n?$conn )
{

39
scripts/policy/frameworks/files/detect-MHR.bro
View file

@ -35,28 +35,37 @@ export {
const notice_threshold = 10 &redef;
}
event file_hash(f: fa_file, kind: string, hash: string)
function do_mhr_lookup(hash: string, fi: Notice::FileInfo)
{
if ( kind=="sha1" && match_file_types in f$mime_type )
local hash_domain = fmt("%s.malware.hash.cymru.com", hash);
when ( local MHR_result = lookup_hostname_txt(hash_domain) )
{
local hash_domain = fmt("%s.malware.hash.cymru.com", hash);
when ( local MHR_result = lookup_hostname_txt(hash_domain) )
# Data is returned as "<dateFirstDetected> <detectionRate>"
local MHR_answer = split1(MHR_result, / /);
if ( |MHR_answer| == 2 )
{
# Data is returned as "<dateFirstDetected> <detectionRate>"
local MHR_answer = split1(MHR_result, / /);
if ( |MHR_answer| == 2 )
local mhr_detect_rate = to_count(MHR_answer[2]);
if ( mhr_detect_rate >= notice_threshold )
{
local mhr_first_detected = double_to_time(to_double(MHR_answer[1]));
local mhr_detect_rate = to_count(MHR_answer[2]);
local readable_first_detected = strftime("%Y-%m-%d %H:%M:%S", mhr_first_detected);
if ( mhr_detect_rate >= notice_threshold )
{
local message = fmt("Malware Hash Registry Detection rate: %d%% Last seen: %s", mhr_detect_rate, readable_first_detected);
local virustotal_url = fmt(match_sub_url, hash);
NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]);
}
local message = fmt("Malware Hash Registry Detection rate: %d%% Last seen: %s", mhr_detect_rate, readable_first_detected);
local virustotal_url = fmt(match_sub_url, hash);
# We don't have the full fa_file record here in order to
# avoid the "when" statement cloning it (expensive!).
local n: Notice::Info = Notice::Info($note=Match, $msg=message, $sub=virustotal_url);
Notice::populate_file_info2(fi, n);
NOTICE(n);
}
}
}
}
event file_hash(f: fa_file, kind: string, hash: string)
{
if ( kind=="sha1" && match_file_types in f$mime_type )
do_mhr_lookup(hash, Notice::create_file_info(f));
}