Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 14:48:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Include timestamp in default extracted file names.

Browse source 
And add a policy script to extract all files.

BIT-1335 #close
This commit is contained in:
Jon Siwek 2015-03-13 14:16:04 -05:00
parent 6fbceb6a98
commit 0b957cbe75
8 changed files with 40 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

11
CHANGES
View file

@ -1,4 +1,15 @@
2.3-539 | 2015-03-13 14:19:27 -0500
* BIT-1335: Include timestamp in default extracted file names.
And add a policy script to extract all files. (Jon Siwek)
* BIT-1311: Identify GRE tunnels as Tunnel::GRE, not Tunnel::IP.
(Jon Siwek)
* BIT-1309: Add Connection class getter methods for flow labels.
(Jon Siwek)
2.3-536 | 2015-03-12 16:16:24 -0500
* Fix Broker leak tests. (Jon Siwek)

3
NEWS
View file

@ -97,6 +97,9 @@ Changed Functionality
- GRE tunnels are now identified as ``Tunnel::GRE`` instead of
``Tunnel::IP``.
- The default name for extracted files changed from extract-protocol-id
to extract-timestamp-protocol-id.
- [TODO] Add changed BroControl features.
Deprecated Functionality

2
VERSION
View file

@ -1 +1 @@
2.3-536
2.3-539

3
scripts/base/files/extract/main.bro
View file

@ -53,7 +53,8 @@ function set_limit(f: fa_file, args: Files::AnalyzerArgs, n: count): bool
function on_add(f: fa_file, args: Files::AnalyzerArgs)
{
if ( ! args?$extract_filename )
args$extract_filename = cat("extract-", f$source, "-", f$id);
args$extract_filename = cat("extract-", f$last_active, "-", f$source,
"-", f$id);
f$info$extracted = args$extract_filename;
args$extract_filename = build_path_compressed(prefix, args$extract_filename);

8
scripts/policy/frameworks/files/extract-all-files.bro Normal file
View file

@ -0,0 +1,8 @@
##! Extract all files to disk.
@load base/files/extract
event file_new(f: fa_file)
{
Files::add_analyzer(f, Files::ANALYZER_EXTRACT);
}

1
scripts/test-all-policy.bro
View file

@ -28,6 +28,7 @@
@load frameworks/intel/seen/where-locations.bro
@load frameworks/intel/seen/x509.bro
@load frameworks/files/detect-MHR.bro
#@load frameworks/files/extract-all-files.bro
@load frameworks/files/hash-all-files.bro
@load frameworks/packet-filter/shunt.bro
@load frameworks/software/version-changes.bro

24
testing/btest/Baseline/plugins.hooks/output
View file

@ -124,7 +124,7 @@
0.000000 MetaHookPost CallFunction(Analyzer::register_for_ports, <frame>, (Analyzer::ANALYZER_TEREDO, {3544/udp})) -> <no result>
0.000000 MetaHookPost CallFunction(Cluster::is_enabled, <frame>, ()) -> <no result>
0.000000 MetaHookPost CallFunction(Cluster::is_enabled, <null>, ()) -> <no result>
0.000000 MetaHookPost CallFunction(Files::register_analyzer_add_callback, <frame>, (Files::ANALYZER_EXTRACT, FileExtract::on_add{ if (!FileExtract::args?$extract_filename) FileExtract::args$extract_filename = cat(extract-, FileExtract::f$source, -, FileExtract::f$id)FileExtract::f$info$extracted = FileExtract::args$extract_filenameFileExtract::args$extract_filename = build_path_compressed(FileExtract::prefix, FileExtract::args$extract_filename)mkdir(FileExtract::prefix)})) -> <no result>
0.000000 MetaHookPost CallFunction(Files::register_analyzer_add_callback, <frame>, (Files::ANALYZER_EXTRACT, FileExtract::on_add{ if (!FileExtract::args?$extract_filename) FileExtract::args$extract_filename = cat(extract-, FileExtract::f$last_active, -, FileExtract::f$source, -, FileExtract::f$id)FileExtract::f$info$extracted = FileExtract::args$extract_filenameFileExtract::args$extract_filename = build_path_compressed(FileExtract::prefix, FileExtract::args$extract_filename)mkdir(FileExtract::prefix)})) -> <no result>
0.000000 MetaHookPost CallFunction(Files::register_protocol, <frame>, (Analyzer::ANALYZER_FTP_DATA, [get_file_handle=FTP::get_file_handle{ if (!FTP::c$id$resp_h, FTP::c$id$resp_p in FTP::ftp_data_expected) return ()return (cat(Analyzer::ANALYZER_FTP_DATA, FTP::c$start_time, FTP::c$id, FTP::is_orig))}, describe=FTP::describe_file{ <init> FTP::cid{ if (FTP::f$source != FTP) return ()for ([FTP::cid] in FTP::f$conns) { if (FTP::f$conns[FTP::cid]?$ftp) return (FTP::describe(FTP::f$conns[FTP::cid]$ftp))}return ()}}])) -> <no result>
0.000000 MetaHookPost CallFunction(Files::register_protocol, <frame>, (Analyzer::ANALYZER_HTTP, [get_file_handle=HTTP::get_file_handle{ if (!HTTP::c?$http) return ()if (HTTP::c$http$range_request && !HTTP::is_orig) { return (cat(Analyzer::ANALYZER_HTTP, HTTP::is_orig, HTTP::c$id$orig_h, HTTP::build_url(HTTP::c$http)))}else{ HTTP::mime_depth = HTTP::is_orig ? HTTP::c$http$orig_mime_depth : HTTP::c$http$resp_mime_depthreturn (cat(Analyzer::ANALYZER_HTTP, HTTP::c$start_time, HTTP::is_orig, HTTP::c$http$trans_depth, HTTP::mime_depth, id_string(HTTP::c$id)))}}, describe=HTTP::describe_file{ <init> HTTP::cid{ if (HTTP::f$source != HTTP) return ()for ([HTTP::cid] in HTTP::f$conns) { if (HTTP::f$conns[HTTP::cid]?$http) return (HTTP::build_url_http(HTTP::f$conns[HTTP::cid]$http))}return ()}}])) -> <no result>
0.000000 MetaHookPost CallFunction(Files::register_protocol, <frame>, (Analyzer::ANALYZER_IRC_DATA, [get_file_handle=IRC::get_file_handle{ return (cat(Analyzer::ANALYZER_IRC_DATA, IRC::c$start_time, IRC::c$id, IRC::is_orig))}, describe=anonymous-function{ return ()}])) -> <no result>
@ -192,7 +192,7 @@
0.000000 MetaHookPost CallFunction(Log::__create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird])) -> <no result>
0.000000 MetaHookPost CallFunction(Log::__create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509])) -> <no result>
0.000000 MetaHookPost CallFunction(Log::__create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql])) -> <no result>
0.000000 MetaHookPost CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1425596289.27327, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
0.000000 MetaHookPost CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1426273629.648148, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
0.000000 MetaHookPost CallFunction(Log::add_default_filter, <frame>, (Cluster::LOG)) -> <no result>
0.000000 MetaHookPost CallFunction(Log::add_default_filter, <frame>, (Communication::LOG)) -> <no result>
0.000000 MetaHookPost CallFunction(Log::add_default_filter, <frame>, (Conn::LOG)) -> <no result>
@ -286,8 +286,8 @@
0.000000 MetaHookPost CallFunction(Log::create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird])) -> <no result>
0.000000 MetaHookPost CallFunction(Log::create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509])) -> <no result>
0.000000 MetaHookPost CallFunction(Log::create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql])) -> <no result>
0.000000 MetaHookPost CallFunction(Log::default_path_func, <null>, (PacketFilter::LOG, , [ts=1425596289.27327, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
0.000000 MetaHookPost CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1425596289.27327, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
0.000000 MetaHookPost CallFunction(Log::default_path_func, <null>, (PacketFilter::LOG, , [ts=1426273629.648148, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
0.000000 MetaHookPost CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1426273629.648148, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
0.000000 MetaHookPost CallFunction(Notice::want_pp, <frame>, ()) -> <no result>
0.000000 MetaHookPost CallFunction(PacketFilter::build, <frame>, ()) -> <no result>
0.000000 MetaHookPost CallFunction(PacketFilter::combine_filters, <frame>, (ip or not ip, and, )) -> <no result>
@ -669,7 +669,7 @@
0.000000 MetaHookPre CallFunction(Analyzer::register_for_ports, <frame>, (Analyzer::ANALYZER_TEREDO, {3544/udp}))
0.000000 MetaHookPre CallFunction(Cluster::is_enabled, <frame>, ())
0.000000 MetaHookPre CallFunction(Cluster::is_enabled, <null>, ())
0.000000 MetaHookPre CallFunction(Files::register_analyzer_add_callback, <frame>, (Files::ANALYZER_EXTRACT, FileExtract::on_add{ if (!FileExtract::args?$extract_filename) FileExtract::args$extract_filename = cat(extract-, FileExtract::f$source, -, FileExtract::f$id)FileExtract::f$info$extracted = FileExtract::args$extract_filenameFileExtract::args$extract_filename = build_path_compressed(FileExtract::prefix, FileExtract::args$extract_filename)mkdir(FileExtract::prefix)}))
0.000000 MetaHookPre CallFunction(Files::register_analyzer_add_callback, <frame>, (Files::ANALYZER_EXTRACT, FileExtract::on_add{ if (!FileExtract::args?$extract_filename) FileExtract::args$extract_filename = cat(extract-, FileExtract::f$last_active, -, FileExtract::f$source, -, FileExtract::f$id)FileExtract::f$info$extracted = FileExtract::args$extract_filenameFileExtract::args$extract_filename = build_path_compressed(FileExtract::prefix, FileExtract::args$extract_filename)mkdir(FileExtract::prefix)}))
0.000000 MetaHookPre CallFunction(Files::register_protocol, <frame>, (Analyzer::ANALYZER_FTP_DATA, [get_file_handle=FTP::get_file_handle{ if (!FTP::c$id$resp_h, FTP::c$id$resp_p in FTP::ftp_data_expected) return ()return (cat(Analyzer::ANALYZER_FTP_DATA, FTP::c$start_time, FTP::c$id, FTP::is_orig))}, describe=FTP::describe_file{ <init> FTP::cid{ if (FTP::f$source != FTP) return ()for ([FTP::cid] in FTP::f$conns) { if (FTP::f$conns[FTP::cid]?$ftp) return (FTP::describe(FTP::f$conns[FTP::cid]$ftp))}return ()}}]))
0.000000 MetaHookPre CallFunction(Files::register_protocol, <frame>, (Analyzer::ANALYZER_HTTP, [get_file_handle=HTTP::get_file_handle{ if (!HTTP::c?$http) return ()if (HTTP::c$http$range_request && !HTTP::is_orig) { return (cat(Analyzer::ANALYZER_HTTP, HTTP::is_orig, HTTP::c$id$orig_h, HTTP::build_url(HTTP::c$http)))}else{ HTTP::mime_depth = HTTP::is_orig ? HTTP::c$http$orig_mime_depth : HTTP::c$http$resp_mime_depthreturn (cat(Analyzer::ANALYZER_HTTP, HTTP::c$start_time, HTTP::is_orig, HTTP::c$http$trans_depth, HTTP::mime_depth, id_string(HTTP::c$id)))}}, describe=HTTP::describe_file{ <init> HTTP::cid{ if (HTTP::f$source != HTTP) return ()for ([HTTP::cid] in HTTP::f$conns) { if (HTTP::f$conns[HTTP::cid]?$http) return (HTTP::build_url_http(HTTP::f$conns[HTTP::cid]$http))}return ()}}]))
0.000000 MetaHookPre CallFunction(Files::register_protocol, <frame>, (Analyzer::ANALYZER_IRC_DATA, [get_file_handle=IRC::get_file_handle{ return (cat(Analyzer::ANALYZER_IRC_DATA, IRC::c$start_time, IRC::c$id, IRC::is_orig))}, describe=anonymous-function{ return ()}]))
@ -737,7 +737,7 @@
0.000000 MetaHookPre CallFunction(Log::__create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird]))
0.000000 MetaHookPre CallFunction(Log::__create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509]))
0.000000 MetaHookPre CallFunction(Log::__create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql]))
0.000000 MetaHookPre CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1425596289.27327, node=bro, filter=ip or not ip, init=T, success=T]))
0.000000 MetaHookPre CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1426273629.648148, node=bro, filter=ip or not ip, init=T, success=T]))
0.000000 MetaHookPre CallFunction(Log::add_default_filter, <frame>, (Cluster::LOG))
0.000000 MetaHookPre CallFunction(Log::add_default_filter, <frame>, (Communication::LOG))
0.000000 MetaHookPre CallFunction(Log::add_default_filter, <frame>, (Conn::LOG))
@ -831,8 +831,8 @@
0.000000 MetaHookPre CallFunction(Log::create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird]))
0.000000 MetaHookPre CallFunction(Log::create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509]))
0.000000 MetaHookPre CallFunction(Log::create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql]))
0.000000 MetaHookPre CallFunction(Log::default_path_func, <null>, (PacketFilter::LOG, , [ts=1425596289.27327, node=bro, filter=ip or not ip, init=T, success=T]))
0.000000 MetaHookPre CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1425596289.27327, node=bro, filter=ip or not ip, init=T, success=T]))
0.000000 MetaHookPre CallFunction(Log::default_path_func, <null>, (PacketFilter::LOG, , [ts=1426273629.648148, node=bro, filter=ip or not ip, init=T, success=T]))
0.000000 MetaHookPre CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1426273629.648148, node=bro, filter=ip or not ip, init=T, success=T]))
0.000000 MetaHookPre CallFunction(Notice::want_pp, <frame>, ())
0.000000 MetaHookPre CallFunction(PacketFilter::build, <frame>, ())
0.000000 MetaHookPre CallFunction(PacketFilter::combine_filters, <frame>, (ip or not ip, and, ))
@ -1213,7 +1213,7 @@
0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_SYSLOG, {514/udp})
0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_TEREDO, {3544/udp})
0.000000 | HookCallFunction Cluster::is_enabled()
0.000000 | HookCallFunction Files::register_analyzer_add_callback(Files::ANALYZER_EXTRACT, FileExtract::on_add{ if (!FileExtract::args?$extract_filename) FileExtract::args$extract_filename = cat(extract-, FileExtract::f$source, -, FileExtract::f$id)FileExtract::f$info$extracted = FileExtract::args$extract_filenameFileExtract::args$extract_filename = build_path_compressed(FileExtract::prefix, FileExtract::args$extract_filename)mkdir(FileExtract::prefix)})
0.000000 | HookCallFunction Files::register_analyzer_add_callback(Files::ANALYZER_EXTRACT, FileExtract::on_add{ if (!FileExtract::args?$extract_filename) FileExtract::args$extract_filename = cat(extract-, FileExtract::f$last_active, -, FileExtract::f$source, -, FileExtract::f$id)FileExtract::f$info$extracted = FileExtract::args$extract_filenameFileExtract::args$extract_filename = build_path_compressed(FileExtract::prefix, FileExtract::args$extract_filename)mkdir(FileExtract::prefix)})
0.000000 | HookCallFunction Files::register_protocol(Analyzer::ANALYZER_FTP_DATA, [get_file_handle=FTP::get_file_handle{ if (!FTP::c$id$resp_h, FTP::c$id$resp_p in FTP::ftp_data_expected) return ()return (cat(Analyzer::ANALYZER_FTP_DATA, FTP::c$start_time, FTP::c$id, FTP::is_orig))}, describe=FTP::describe_file{ <init> FTP::cid{ if (FTP::f$source != FTP) return ()for ([FTP::cid] in FTP::f$conns) { if (FTP::f$conns[FTP::cid]?$ftp) return (FTP::describe(FTP::f$conns[FTP::cid]$ftp))}return ()}}])
0.000000 | HookCallFunction Files::register_protocol(Analyzer::ANALYZER_HTTP, [get_file_handle=HTTP::get_file_handle{ if (!HTTP::c?$http) return ()if (HTTP::c$http$range_request && !HTTP::is_orig) { return (cat(Analyzer::ANALYZER_HTTP, HTTP::is_orig, HTTP::c$id$orig_h, HTTP::build_url(HTTP::c$http)))}else{ HTTP::mime_depth = HTTP::is_orig ? HTTP::c$http$orig_mime_depth : HTTP::c$http$resp_mime_depthreturn (cat(Analyzer::ANALYZER_HTTP, HTTP::c$start_time, HTTP::is_orig, HTTP::c$http$trans_depth, HTTP::mime_depth, id_string(HTTP::c$id)))}}, describe=HTTP::describe_file{ <init> HTTP::cid{ if (HTTP::f$source != HTTP) return ()for ([HTTP::cid] in HTTP::f$conns) { if (HTTP::f$conns[HTTP::cid]?$http) return (HTTP::build_url_http(HTTP::f$conns[HTTP::cid]$http))}return ()}}])
0.000000 | HookCallFunction Files::register_protocol(Analyzer::ANALYZER_IRC_DATA, [get_file_handle=IRC::get_file_handle{ return (cat(Analyzer::ANALYZER_IRC_DATA, IRC::c$start_time, IRC::c$id, IRC::is_orig))}, describe=anonymous-function{ return ()}])
@ -1281,7 +1281,7 @@
0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=<no value description>, ev=Weird::log_weird])
0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=<no value description>, ev=X509::log_x509])
0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql])
0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1425596289.27327, node=bro, filter=ip or not ip, init=T, success=T])
0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1426273629.648148, node=bro, filter=ip or not ip, init=T, success=T])
0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG)
0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG)
0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG)
@ -1375,8 +1375,8 @@
0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=<no value description>, ev=Weird::log_weird])
0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=<no value description>, ev=X509::log_x509])
0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql])
0.000000 | HookCallFunction Log::default_path_func(PacketFilter::LOG, , [ts=1425596289.27327, node=bro, filter=ip or not ip, init=T, success=T])
0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1425596289.27327, node=bro, filter=ip or not ip, init=T, success=T])
0.000000 | HookCallFunction Log::default_path_func(PacketFilter::LOG, , [ts=1426273629.648148, node=bro, filter=ip or not ip, init=T, success=T])
0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1426273629.648148, node=bro, filter=ip or not ip, init=T, success=T])
0.000000 | HookCallFunction Notice::want_pp()
0.000000 | HookCallFunction PacketFilter::build()
0.000000 | HookCallFunction PacketFilter::combine_filters(ip or not ip, and, )

2
testing/btest/scripts/policy/frameworks/files/extract-all.bro Normal file
View file

@ -0,0 +1,2 @@
# @TEST-EXEC: bro -r $TRACES/http/get.trace frameworks/files/extract-all-files
# @TEST-EXEC: grep -q EXTRACT files.log