diff --git a/CHANGES b/CHANGES
index 926b30c9c0..84f64034ea 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,15 @@
+2.3-539 | 2015-03-13 14:19:27 -0500
+
+ * BIT-1335: Include timestamp in default extracted file names.
+ And add a policy script to extract all files. (Jon Siwek)
+
+ * BIT-1311: Identify GRE tunnels as Tunnel::GRE, not Tunnel::IP.
+ (Jon Siwek)
+
+ * BIT-1309: Add Connection class getter methods for flow labels.
+ (Jon Siwek)
+
2.3-536 | 2015-03-12 16:16:24 -0500
* Fix Broker leak tests. (Jon Siwek)
diff --git a/NEWS b/NEWS
index 981af20370..4d1539b33c 100644
--- a/NEWS
+++ b/NEWS
@@ -97,6 +97,9 @@ Changed Functionality
- GRE tunnels are now identified as ``Tunnel::GRE`` instead of
``Tunnel::IP``.
+- The default name for extracted files changed from extract-protocol-id
+ to extract-timestamp-protocol-id.
+
- [TODO] Add changed BroControl features.
Deprecated Functionality
diff --git a/VERSION b/VERSION
index c168eac2bd..64cd9fa66f 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.3-536
+2.3-539
diff --git a/scripts/base/files/extract/main.bro b/scripts/base/files/extract/main.bro
index 765263a4d8..7f68a8bcce 100644
--- a/scripts/base/files/extract/main.bro
+++ b/scripts/base/files/extract/main.bro
@@ -53,7 +53,8 @@ function set_limit(f: fa_file, args: Files::AnalyzerArgs, n: count): bool
function on_add(f: fa_file, args: Files::AnalyzerArgs)
{
if ( ! args?$extract_filename )
- args$extract_filename = cat("extract-", f$source, "-", f$id);
+ args$extract_filename = cat("extract-", f$last_active, "-", f$source,
+ "-", f$id);
f$info$extracted = args$extract_filename;
args$extract_filename = build_path_compressed(prefix, args$extract_filename);
diff --git a/scripts/policy/frameworks/files/extract-all-files.bro b/scripts/policy/frameworks/files/extract-all-files.bro
new file mode 100644
index 0000000000..7bd7b300e9
--- /dev/null
+++ b/scripts/policy/frameworks/files/extract-all-files.bro
@@ -0,0 +1,8 @@
+##! Extract all files to disk.
+
+@load base/files/extract
+
+event file_new(f: fa_file)
+ {
+ Files::add_analyzer(f, Files::ANALYZER_EXTRACT);
+ }
diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro
index 0fb74f91cf..dc85986172 100644
--- a/scripts/test-all-policy.bro
+++ b/scripts/test-all-policy.bro
@@ -28,6 +28,7 @@
@load frameworks/intel/seen/where-locations.bro
@load frameworks/intel/seen/x509.bro
@load frameworks/files/detect-MHR.bro
+#@load frameworks/files/extract-all-files.bro
@load frameworks/files/hash-all-files.bro
@load frameworks/packet-filter/shunt.bro
@load frameworks/software/version-changes.bro
diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output
index 6956f013bc..63f0a87742 100644
--- a/testing/btest/Baseline/plugins.hooks/output
+++ b/testing/btest/Baseline/plugins.hooks/output
@@ -124,7 +124,7 @@
0.000000 MetaHookPost CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_TEREDO, {3544/udp})) ->
0.000000 MetaHookPost CallFunction(Cluster::is_enabled, , ()) ->
0.000000 MetaHookPost CallFunction(Cluster::is_enabled, , ()) ->
-0.000000 MetaHookPost CallFunction(Files::register_analyzer_add_callback, , (Files::ANALYZER_EXTRACT, FileExtract::on_add{ if (!FileExtract::args?$extract_filename) FileExtract::args$extract_filename = cat(extract-, FileExtract::f$source, -, FileExtract::f$id)FileExtract::f$info$extracted = FileExtract::args$extract_filenameFileExtract::args$extract_filename = build_path_compressed(FileExtract::prefix, FileExtract::args$extract_filename)mkdir(FileExtract::prefix)})) ->
+0.000000 MetaHookPost CallFunction(Files::register_analyzer_add_callback, , (Files::ANALYZER_EXTRACT, FileExtract::on_add{ if (!FileExtract::args?$extract_filename) FileExtract::args$extract_filename = cat(extract-, FileExtract::f$last_active, -, FileExtract::f$source, -, FileExtract::f$id)FileExtract::f$info$extracted = FileExtract::args$extract_filenameFileExtract::args$extract_filename = build_path_compressed(FileExtract::prefix, FileExtract::args$extract_filename)mkdir(FileExtract::prefix)})) ->
0.000000 MetaHookPost CallFunction(Files::register_protocol, , (Analyzer::ANALYZER_FTP_DATA, [get_file_handle=FTP::get_file_handle{ if (!FTP::c$id$resp_h, FTP::c$id$resp_p in FTP::ftp_data_expected) return ()return (cat(Analyzer::ANALYZER_FTP_DATA, FTP::c$start_time, FTP::c$id, FTP::is_orig))}, describe=FTP::describe_file{ FTP::cid{ if (FTP::f$source != FTP) return ()for ([FTP::cid] in FTP::f$conns) { if (FTP::f$conns[FTP::cid]?$ftp) return (FTP::describe(FTP::f$conns[FTP::cid]$ftp))}return ()}}])) ->
0.000000 MetaHookPost CallFunction(Files::register_protocol, , (Analyzer::ANALYZER_HTTP, [get_file_handle=HTTP::get_file_handle{ if (!HTTP::c?$http) return ()if (HTTP::c$http$range_request && !HTTP::is_orig) { return (cat(Analyzer::ANALYZER_HTTP, HTTP::is_orig, HTTP::c$id$orig_h, HTTP::build_url(HTTP::c$http)))}else{ HTTP::mime_depth = HTTP::is_orig ? HTTP::c$http$orig_mime_depth : HTTP::c$http$resp_mime_depthreturn (cat(Analyzer::ANALYZER_HTTP, HTTP::c$start_time, HTTP::is_orig, HTTP::c$http$trans_depth, HTTP::mime_depth, id_string(HTTP::c$id)))}}, describe=HTTP::describe_file{ HTTP::cid{ if (HTTP::f$source != HTTP) return ()for ([HTTP::cid] in HTTP::f$conns) { if (HTTP::f$conns[HTTP::cid]?$http) return (HTTP::build_url_http(HTTP::f$conns[HTTP::cid]$http))}return ()}}])) ->
0.000000 MetaHookPost CallFunction(Files::register_protocol, , (Analyzer::ANALYZER_IRC_DATA, [get_file_handle=IRC::get_file_handle{ return (cat(Analyzer::ANALYZER_IRC_DATA, IRC::c$start_time, IRC::c$id, IRC::is_orig))}, describe=anonymous-function{ return ()}])) ->
@@ -192,7 +192,7 @@
0.000000 MetaHookPost CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird])) ->
0.000000 MetaHookPost CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509])) ->
0.000000 MetaHookPost CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql])) ->
-0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1425596289.27327, node=bro, filter=ip or not ip, init=T, success=T])) ->
+0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1426273629.648148, node=bro, filter=ip or not ip, init=T, success=T])) ->
0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Cluster::LOG)) ->
0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Communication::LOG)) ->
0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Conn::LOG)) ->
@@ -286,8 +286,8 @@
0.000000 MetaHookPost CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird])) ->
0.000000 MetaHookPost CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509])) ->
0.000000 MetaHookPost CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql])) ->
-0.000000 MetaHookPost CallFunction(Log::default_path_func, , (PacketFilter::LOG, , [ts=1425596289.27327, node=bro, filter=ip or not ip, init=T, success=T])) ->
-0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1425596289.27327, node=bro, filter=ip or not ip, init=T, success=T])) ->
+0.000000 MetaHookPost CallFunction(Log::default_path_func, , (PacketFilter::LOG, , [ts=1426273629.648148, node=bro, filter=ip or not ip, init=T, success=T])) ->
+0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1426273629.648148, node=bro, filter=ip or not ip, init=T, success=T])) ->
0.000000 MetaHookPost CallFunction(Notice::want_pp, , ()) ->
0.000000 MetaHookPost CallFunction(PacketFilter::build, , ()) ->
0.000000 MetaHookPost CallFunction(PacketFilter::combine_filters, , (ip or not ip, and, )) ->
@@ -669,7 +669,7 @@
0.000000 MetaHookPre CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_TEREDO, {3544/udp}))
0.000000 MetaHookPre CallFunction(Cluster::is_enabled, , ())
0.000000 MetaHookPre CallFunction(Cluster::is_enabled, , ())
-0.000000 MetaHookPre CallFunction(Files::register_analyzer_add_callback, , (Files::ANALYZER_EXTRACT, FileExtract::on_add{ if (!FileExtract::args?$extract_filename) FileExtract::args$extract_filename = cat(extract-, FileExtract::f$source, -, FileExtract::f$id)FileExtract::f$info$extracted = FileExtract::args$extract_filenameFileExtract::args$extract_filename = build_path_compressed(FileExtract::prefix, FileExtract::args$extract_filename)mkdir(FileExtract::prefix)}))
+0.000000 MetaHookPre CallFunction(Files::register_analyzer_add_callback, , (Files::ANALYZER_EXTRACT, FileExtract::on_add{ if (!FileExtract::args?$extract_filename) FileExtract::args$extract_filename = cat(extract-, FileExtract::f$last_active, -, FileExtract::f$source, -, FileExtract::f$id)FileExtract::f$info$extracted = FileExtract::args$extract_filenameFileExtract::args$extract_filename = build_path_compressed(FileExtract::prefix, FileExtract::args$extract_filename)mkdir(FileExtract::prefix)}))
0.000000 MetaHookPre CallFunction(Files::register_protocol, , (Analyzer::ANALYZER_FTP_DATA, [get_file_handle=FTP::get_file_handle{ if (!FTP::c$id$resp_h, FTP::c$id$resp_p in FTP::ftp_data_expected) return ()return (cat(Analyzer::ANALYZER_FTP_DATA, FTP::c$start_time, FTP::c$id, FTP::is_orig))}, describe=FTP::describe_file{ FTP::cid{ if (FTP::f$source != FTP) return ()for ([FTP::cid] in FTP::f$conns) { if (FTP::f$conns[FTP::cid]?$ftp) return (FTP::describe(FTP::f$conns[FTP::cid]$ftp))}return ()}}]))
0.000000 MetaHookPre CallFunction(Files::register_protocol, , (Analyzer::ANALYZER_HTTP, [get_file_handle=HTTP::get_file_handle{ if (!HTTP::c?$http) return ()if (HTTP::c$http$range_request && !HTTP::is_orig) { return (cat(Analyzer::ANALYZER_HTTP, HTTP::is_orig, HTTP::c$id$orig_h, HTTP::build_url(HTTP::c$http)))}else{ HTTP::mime_depth = HTTP::is_orig ? HTTP::c$http$orig_mime_depth : HTTP::c$http$resp_mime_depthreturn (cat(Analyzer::ANALYZER_HTTP, HTTP::c$start_time, HTTP::is_orig, HTTP::c$http$trans_depth, HTTP::mime_depth, id_string(HTTP::c$id)))}}, describe=HTTP::describe_file{ HTTP::cid{ if (HTTP::f$source != HTTP) return ()for ([HTTP::cid] in HTTP::f$conns) { if (HTTP::f$conns[HTTP::cid]?$http) return (HTTP::build_url_http(HTTP::f$conns[HTTP::cid]$http))}return ()}}]))
0.000000 MetaHookPre CallFunction(Files::register_protocol, , (Analyzer::ANALYZER_IRC_DATA, [get_file_handle=IRC::get_file_handle{ return (cat(Analyzer::ANALYZER_IRC_DATA, IRC::c$start_time, IRC::c$id, IRC::is_orig))}, describe=anonymous-function{ return ()}]))
@@ -737,7 +737,7 @@
0.000000 MetaHookPre CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird]))
0.000000 MetaHookPre CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509]))
0.000000 MetaHookPre CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql]))
-0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1425596289.27327, node=bro, filter=ip or not ip, init=T, success=T]))
+0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1426273629.648148, node=bro, filter=ip or not ip, init=T, success=T]))
0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Cluster::LOG))
0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Communication::LOG))
0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Conn::LOG))
@@ -831,8 +831,8 @@
0.000000 MetaHookPre CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird]))
0.000000 MetaHookPre CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509]))
0.000000 MetaHookPre CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql]))
-0.000000 MetaHookPre CallFunction(Log::default_path_func, , (PacketFilter::LOG, , [ts=1425596289.27327, node=bro, filter=ip or not ip, init=T, success=T]))
-0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1425596289.27327, node=bro, filter=ip or not ip, init=T, success=T]))
+0.000000 MetaHookPre CallFunction(Log::default_path_func, , (PacketFilter::LOG, , [ts=1426273629.648148, node=bro, filter=ip or not ip, init=T, success=T]))
+0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1426273629.648148, node=bro, filter=ip or not ip, init=T, success=T]))
0.000000 MetaHookPre CallFunction(Notice::want_pp, , ())
0.000000 MetaHookPre CallFunction(PacketFilter::build, , ())
0.000000 MetaHookPre CallFunction(PacketFilter::combine_filters, , (ip or not ip, and, ))
@@ -1213,7 +1213,7 @@
0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_SYSLOG, {514/udp})
0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_TEREDO, {3544/udp})
0.000000 | HookCallFunction Cluster::is_enabled()
-0.000000 | HookCallFunction Files::register_analyzer_add_callback(Files::ANALYZER_EXTRACT, FileExtract::on_add{ if (!FileExtract::args?$extract_filename) FileExtract::args$extract_filename = cat(extract-, FileExtract::f$source, -, FileExtract::f$id)FileExtract::f$info$extracted = FileExtract::args$extract_filenameFileExtract::args$extract_filename = build_path_compressed(FileExtract::prefix, FileExtract::args$extract_filename)mkdir(FileExtract::prefix)})
+0.000000 | HookCallFunction Files::register_analyzer_add_callback(Files::ANALYZER_EXTRACT, FileExtract::on_add{ if (!FileExtract::args?$extract_filename) FileExtract::args$extract_filename = cat(extract-, FileExtract::f$last_active, -, FileExtract::f$source, -, FileExtract::f$id)FileExtract::f$info$extracted = FileExtract::args$extract_filenameFileExtract::args$extract_filename = build_path_compressed(FileExtract::prefix, FileExtract::args$extract_filename)mkdir(FileExtract::prefix)})
0.000000 | HookCallFunction Files::register_protocol(Analyzer::ANALYZER_FTP_DATA, [get_file_handle=FTP::get_file_handle{ if (!FTP::c$id$resp_h, FTP::c$id$resp_p in FTP::ftp_data_expected) return ()return (cat(Analyzer::ANALYZER_FTP_DATA, FTP::c$start_time, FTP::c$id, FTP::is_orig))}, describe=FTP::describe_file{ FTP::cid{ if (FTP::f$source != FTP) return ()for ([FTP::cid] in FTP::f$conns) { if (FTP::f$conns[FTP::cid]?$ftp) return (FTP::describe(FTP::f$conns[FTP::cid]$ftp))}return ()}}])
0.000000 | HookCallFunction Files::register_protocol(Analyzer::ANALYZER_HTTP, [get_file_handle=HTTP::get_file_handle{ if (!HTTP::c?$http) return ()if (HTTP::c$http$range_request && !HTTP::is_orig) { return (cat(Analyzer::ANALYZER_HTTP, HTTP::is_orig, HTTP::c$id$orig_h, HTTP::build_url(HTTP::c$http)))}else{ HTTP::mime_depth = HTTP::is_orig ? HTTP::c$http$orig_mime_depth : HTTP::c$http$resp_mime_depthreturn (cat(Analyzer::ANALYZER_HTTP, HTTP::c$start_time, HTTP::is_orig, HTTP::c$http$trans_depth, HTTP::mime_depth, id_string(HTTP::c$id)))}}, describe=HTTP::describe_file{ HTTP::cid{ if (HTTP::f$source != HTTP) return ()for ([HTTP::cid] in HTTP::f$conns) { if (HTTP::f$conns[HTTP::cid]?$http) return (HTTP::build_url_http(HTTP::f$conns[HTTP::cid]$http))}return ()}}])
0.000000 | HookCallFunction Files::register_protocol(Analyzer::ANALYZER_IRC_DATA, [get_file_handle=IRC::get_file_handle{ return (cat(Analyzer::ANALYZER_IRC_DATA, IRC::c$start_time, IRC::c$id, IRC::is_orig))}, describe=anonymous-function{ return ()}])
@@ -1281,7 +1281,7 @@
0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=, ev=Weird::log_weird])
0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=, ev=X509::log_x509])
0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql])
-0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1425596289.27327, node=bro, filter=ip or not ip, init=T, success=T])
+0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1426273629.648148, node=bro, filter=ip or not ip, init=T, success=T])
0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG)
0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG)
0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG)
@@ -1375,8 +1375,8 @@
0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=, ev=Weird::log_weird])
0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=, ev=X509::log_x509])
0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql])
-0.000000 | HookCallFunction Log::default_path_func(PacketFilter::LOG, , [ts=1425596289.27327, node=bro, filter=ip or not ip, init=T, success=T])
-0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1425596289.27327, node=bro, filter=ip or not ip, init=T, success=T])
+0.000000 | HookCallFunction Log::default_path_func(PacketFilter::LOG, , [ts=1426273629.648148, node=bro, filter=ip or not ip, init=T, success=T])
+0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1426273629.648148, node=bro, filter=ip or not ip, init=T, success=T])
0.000000 | HookCallFunction Notice::want_pp()
0.000000 | HookCallFunction PacketFilter::build()
0.000000 | HookCallFunction PacketFilter::combine_filters(ip or not ip, and, )
diff --git a/testing/btest/scripts/policy/frameworks/files/extract-all.bro b/testing/btest/scripts/policy/frameworks/files/extract-all.bro
new file mode 100644
index 0000000000..f54b2e299d
--- /dev/null
+++ b/testing/btest/scripts/policy/frameworks/files/extract-all.bro
@@ -0,0 +1,2 @@
+# @TEST-EXEC: bro -r $TRACES/http/get.trace frameworks/files/extract-all-files
+# @TEST-EXEC: grep -q EXTRACT files.log