diff --git a/scripts/base/frameworks/analyzer/main.bro b/scripts/base/frameworks/analyzer/main.bro index c4ee5c943b..e266eb8c7a 100644 --- a/scripts/base/frameworks/analyzer/main.bro +++ b/scripts/base/frameworks/analyzer/main.bro @@ -81,6 +81,13 @@ export { ## Returns: The analyzer name corresponding to the tag. global name: function(tag: Analyzer::Tag) : string; + ## Translates an analyzer's name to a tag enum value. + ## + ## name: The analyzer name. + ## + ## Returns: The analyzer tag corresponding to the name. + global get_tag: function(name: string): Analyzer::Tag; + ## Schedules an analyzer for a future connection originating from a given IP ## address and port. ## @@ -187,6 +194,11 @@ function name(atype: Analyzer::Tag) : string return __name(atype); } +function get_tag(name: string): Analyzer::Tag + { + return __tag(name); + } + function schedule_analyzer(orig: addr, resp: addr, resp_p: port, analyzer: Analyzer::Tag, tout: interval) : bool { diff --git a/scripts/base/frameworks/files/main.bro b/scripts/base/frameworks/files/main.bro index 8dd07fcb53..cc92932bbf 100644 --- a/scripts/base/frameworks/files/main.bro +++ b/scripts/base/frameworks/files/main.bro @@ -2,6 +2,7 @@ ##! any network protocol over which they're transported. @load base/bif/file_analysis.bif +@load base/frameworks/analyzer @load base/frameworks/logging @load base/utils/site @@ -173,17 +174,36 @@ export { ## Returns: The analyzer name corresponding to the tag. global analyzer_name: function(tag: Files::Tag): string; + ## Provides a text description regarding metadata of the file. + ## For example, with HTTP it would return a URL. + ## + ## f: The file to be described. + ## + ## Returns a text description regarding metadata of the file. + global describe: function(f: fa_file): string; + + type ProtoRegistration: record { + ## A callback to generate a file handle on demand when + ## one is needed by the core. + get_file_handle: function(c: connection, is_orig: bool): string; + + ## A callback to "describe" a file. In the case of an HTTP + ## transfer the most obvious description would be the URL. + ## It's like an extremely compressed version of the normal log. + describe: function(f: fa_file): string + &default=function(f: fa_file): string { return ""; }; + }; + ## Register callbacks for protocols that work with the Files framework. ## The callbacks must uniquely identify a file and each protocol can ## only have a single callback registered for it. ## ## tag: Tag for the protocol analyzer having a callback being registered. ## - ## callback: Function that can generate a file handle for the protocol analyzer - ## defined previously. + ## reg: A :bro:see:`ProtoRegistration` record. ## ## Returns: true if the protocol being registered was not previously registered. - global register_protocol: function(tag: Files::Tag, callback: function(c: connection, is_orig: bool): string): bool; + global register_protocol: function(tag: Analyzer::Tag, reg: ProtoRegistration): bool; ## Register a callback for file analyzers to use if they need to do some manipulation ## when they are being added to a file before the core code takes over. This is @@ -210,8 +230,7 @@ redef record AnalyzerArgs += { }; # Store the callbacks for protocol analyzers that have files. -global registered_protocols: table[Files::Tag] of function(c: connection, is_orig: bool): string = table() - &default=function(c: connection, is_orig: bool): string { return cat(c$uid, is_orig); }; +global registered_protocols: table[Analyzer::Tag] of ProtoRegistration = table(); global analyzer_add_callbacks: table[Files::Tag] of function(f: fa_file, args: AnalyzerArgs) = table(); @@ -321,15 +340,28 @@ event file_state_remove(f: fa_file) &priority=-10 Log::write(Files::LOG, f$info); } -function register_protocol(tag: Files::Tag, callback: function(c: connection, is_orig: bool): string): bool +function register_protocol(tag: Analyzer::Tag, reg: ProtoRegistration): bool { local result = (tag !in registered_protocols); - registered_protocols[tag] = callback; + registered_protocols[tag] = reg; return result; } -event get_file_handle(tag: Files::Tag, c: connection, is_orig: bool) &priority=5 +function describe(f: fa_file): string { + local tag = Analyzer::get_tag(f$source); + if ( tag !in registered_protocols ) + return ""; + local handler = registered_protocols[tag]; - set_file_handle(handler(c, is_orig)); + return handler$describe(f); + } + +event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) &priority=5 + { + if ( tag !in registered_protocols ) + return; + + local handler = registered_protocols[tag]; + set_file_handle(handler$get_file_handle(c, is_orig)); } diff --git a/scripts/base/frameworks/notice/main.bro b/scripts/base/frameworks/notice/main.bro index f47ed79940..5bd01e0982 100644 --- a/scripts/base/frameworks/notice/main.bro +++ b/scripts/base/frameworks/notice/main.bro @@ -79,7 +79,13 @@ export { ## A mime type if the notice is related to a file. If the $f field ## is provided, this will be automatically filled out. - mime_type: string &log &optional; + file_mime_type: string &log &optional; + + ## Frequently files can be "described" to give a bit more context. + ## This field will typically be automatically filled out from an + ## fa_file record. For example, if a notice was related to a + ## file over HTTP, the URL of the request would be shown. + file_desc: string &log &optional; ## The transport protocol. Filled automatically when either conn, iconn ## or p is specified. @@ -477,9 +483,13 @@ function apply_policy(n: Notice::Info) { if ( ! n?$fuid ) n$fuid = n$f$id; - if ( ! n?$mime_type && n$f?$mime_type ) - n$mime_type = n$f$mime_type; - if ( |n$f$conns| == 1 ) + + if ( ! n?$file_mime_type && n$f?$mime_type ) + n$file_mime_type = n$f$mime_type; + + n$file_desc = Files::describe(n$f); + + if ( n$f?$conns && |n$f$conns| == 1 ) { for ( id in n$f$conns ) n$conn = n$f$conns[id]; @@ -490,6 +500,7 @@ function apply_policy(n: Notice::Info) { if ( ! n?$id ) n$id = n$conn$id; + if ( ! n?$uid ) n$uid = n$conn$uid; } diff --git a/scripts/base/protocols/ftp/__load__.bro b/scripts/base/protocols/ftp/__load__.bro index bc68f61cea..ebb09e702c 100644 --- a/scripts/base/protocols/ftp/__load__.bro +++ b/scripts/base/protocols/ftp/__load__.bro @@ -1,5 +1,6 @@ @load ./utils-commands @load ./main +@load ./utils @load ./files @load ./gridftp diff --git a/scripts/base/protocols/ftp/files.bro b/scripts/base/protocols/ftp/files.bro index c68717c8a2..1d7b7670f4 100644 --- a/scripts/base/protocols/ftp/files.bro +++ b/scripts/base/protocols/ftp/files.bro @@ -12,6 +12,9 @@ export { ## Default file handle provider for FTP. global get_file_handle: function(c: connection, is_orig: bool): string; + + ## Describe the file being transferred. + global describe_file: function(f: fa_file): string; } function get_file_handle(c: connection, is_orig: bool): string @@ -22,9 +25,25 @@ function get_file_handle(c: connection, is_orig: bool): string return cat(Analyzer::ANALYZER_FTP_DATA, c$start_time, c$id, is_orig); } +function describe_file(f: fa_file): string + { + # This shouldn't be needed, but just in case... + if ( f$source != "FTP" ) + return ""; + + for ( cid in f$conns ) + { + if ( f$conns[cid]?$ftp ) + return FTP::describe(f$conns[cid]$ftp); + } + return ""; + } + event bro_init() &priority=5 { - Files::register_protocol(Analyzer::ANALYZER_FTP_DATA, FTP::get_file_handle); + Files::register_protocol(Analyzer::ANALYZER_FTP_DATA, + [$get_file_handle = FTP::get_file_handle, + $describe = FTP::describe_file]); } diff --git a/scripts/base/protocols/ftp/main.bro b/scripts/base/protocols/ftp/main.bro index 7bf9d6cc4c..f525c7792b 100644 --- a/scripts/base/protocols/ftp/main.bro +++ b/scripts/base/protocols/ftp/main.bro @@ -63,8 +63,6 @@ export { reply_code: count &log &optional; ## Reply message from the server in response to the command. reply_msg: string &log &optional; - ## Arbitrary tags that may indicate a particular attribute of this command. - tags: set[string] &log; ## Expected FTP data channel. data_channel: ExpectedDataChannel &log &optional; @@ -171,37 +169,22 @@ function set_ftp_session(c: connection) function ftp_message(s: Info) { - # If it either has a tag associated with it (something detected) - # or it's a deliberately logged command. - if ( |s$tags| > 0 || (s?$cmdarg && s$cmdarg$cmd in logged_commands) ) + s$ts=s$cmdarg$ts; + s$command=s$cmdarg$cmd; + s$arg=s$cmdarg$arg; + if ( s$arg == "" ) + delete s$arg; + + if ( s?$password && + ! s$capture_password && + to_lower(s$user) !in guest_ids ) { - if ( s?$password && - ! s$capture_password && - to_lower(s$user) !in guest_ids ) - { - s$password = ""; - } - - local arg = s$cmdarg$arg; - if ( s$cmdarg$cmd in file_cmds ) - { - local comp_path = build_path_compressed(s$cwd, arg); - if ( comp_path[0] != "/" ) - comp_path = cat("/", comp_path); - - arg = fmt("ftp://%s%s", addr_to_uri(s$id$resp_h), comp_path); - } - - s$ts=s$cmdarg$ts; - s$command=s$cmdarg$cmd; - if ( arg == "" ) - delete s$arg; - else - s$arg=arg; - - Log::write(FTP::LOG, s); + s$password = ""; } + if ( s?$cmdarg && s$command in logged_commands) + Log::write(FTP::LOG, s); + # The MIME and file_size fields are specific to file transfer commands # and may not be used in all commands so they need reset to "blank" # values after logging. @@ -209,8 +192,6 @@ function ftp_message(s: Info) delete s$file_size; # Same with data channel. delete s$data_channel; - # Tags are cleared everytime too. - s$tags = set(); } function add_expected_data_channel(s: Info, chan: ExpectedDataChannel) @@ -218,8 +199,9 @@ function add_expected_data_channel(s: Info, chan: ExpectedDataChannel) s$passive = chan$passive; s$data_channel = chan; ftp_data_expected[chan$resp_h, chan$resp_p] = s; - Analyzer::schedule_analyzer(chan$orig_h, chan$resp_h, chan$resp_p, Analyzer::ANALYZER_FTP_DATA, - 5mins); + Analyzer::schedule_analyzer(chan$orig_h, chan$resp_h, chan$resp_p, + Analyzer::ANALYZER_FTP_DATA, + 5mins); } event ftp_request(c: connection, command: string, arg: string) &priority=5 diff --git a/scripts/base/protocols/http/files.bro b/scripts/base/protocols/http/files.bro index 09324b5f45..fd07dc096a 100644 --- a/scripts/base/protocols/http/files.bro +++ b/scripts/base/protocols/http/files.bro @@ -8,6 +8,9 @@ module HTTP; export { ## Default file handle provider for HTTP. global get_file_handle: function(c: connection, is_orig: bool): string; + + ## Default file describer for HTTP. + global describe_file: function(f: fa_file): string; } function get_file_handle(c: connection, is_orig: bool): string @@ -27,7 +30,23 @@ function get_file_handle(c: connection, is_orig: bool): string } } +function describe_file(f: fa_file): string + { + # This shouldn't be needed, but just in case... + if ( f$source != "HTTP" ) + return ""; + + for ( cid in f$conns ) + { + if ( f$conns[cid]?$http ) + return build_url_http(f$conns[cid]$http); + } + return ""; + } + event bro_init() &priority=5 { - Files::register_protocol(Analyzer::ANALYZER_HTTP, HTTP::get_file_handle); + Files::register_protocol(Analyzer::ANALYZER_HTTP, + [$get_file_handle = HTTP::get_file_handle, + $describe = HTTP::describe_file]); } diff --git a/scripts/base/protocols/http/utils.bro b/scripts/base/protocols/http/utils.bro index a74a2fe696..fe8c076780 100644 --- a/scripts/base/protocols/http/utils.bro +++ b/scripts/base/protocols/http/utils.bro @@ -32,6 +32,9 @@ export { ## ## Returns: A URL prefixed with "http://". global build_url_http: function(rec: Info): string; + + ## Create an extremely shortened representation of a log line. + global describe: function(rec: Info): string; } @@ -62,3 +65,8 @@ function build_url_http(rec: Info): string { return fmt("http://%s", build_url(rec)); } + +function describe(rec: Info): string + { + return build_url_http(rec); + } diff --git a/scripts/base/protocols/irc/file-analysis.bro b/scripts/base/protocols/irc/file-analysis.bro deleted file mode 100644 index f2e84fbc22..0000000000 --- a/scripts/base/protocols/irc/file-analysis.bro +++ /dev/null @@ -1,23 +0,0 @@ -@load ./dcc-send -@load base/utils/conn-ids -@load base/frameworks/files - -module IRC; - -export { - ## Default file handle provider for IRC. - global get_file_handle: function(c: connection, is_orig: bool): string; -} - -function get_file_handle(c: connection, is_orig: bool): string - { - if ( [c$id$resp_h, c$id$resp_p] !in dcc_expected_transfers ) - return ""; - - return cat(ANALYZER_IRC_DATA, c$start_time, c$id, is_orig); - } - -event bro_init() &priority=5 - { - Files::register_protocol(ANALYZER_IRC_DATA, IRC::get_file_handle); - } diff --git a/scripts/base/protocols/irc/files.bro b/scripts/base/protocols/irc/files.bro index 8708270bfd..a6321d3f2f 100644 --- a/scripts/base/protocols/irc/files.bro +++ b/scripts/base/protocols/irc/files.bro @@ -24,7 +24,8 @@ function get_file_handle(c: connection, is_orig: bool): string event bro_init() &priority=5 { - Files::register_protocol(Analyzer::ANALYZER_IRC_DATA, IRC::get_file_handle); + Files::register_protocol(Analyzer::ANALYZER_IRC_DATA, + [$get_file_handle = IRC::get_file_handle]); } event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5 diff --git a/scripts/base/protocols/smtp/files.bro b/scripts/base/protocols/smtp/files.bro index 1cf9ec01e1..f9ae2ab05f 100644 --- a/scripts/base/protocols/smtp/files.bro +++ b/scripts/base/protocols/smtp/files.bro @@ -14,6 +14,9 @@ export { ## Default file handle provider for SMTP. global get_file_handle: function(c: connection, is_orig: bool): string; + + ## Default file describer for SMTP. + global describe_file: function(f: fa_file): string; } function get_file_handle(c: connection, is_orig: bool): string @@ -22,9 +25,25 @@ function get_file_handle(c: connection, is_orig: bool): string c$smtp_state$mime_depth); } +function describe_file(f: fa_file): string + { + # This shouldn't be needed, but just in case... + if ( f$source != "SMTP" ) + return ""; + + for ( cid in f$conns ) + { + local c = f$conns[cid]; + return SMTP::describe(c$smtp); + } + return ""; + } + event bro_init() &priority=5 { - Files::register_protocol(Analyzer::ANALYZER_SMTP, SMTP::get_file_handle); + Files::register_protocol(Analyzer::ANALYZER_SMTP, + [$get_file_handle = SMTP::get_file_handle, + $describe = SMTP::describe_file]); } event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5 diff --git a/scripts/base/protocols/smtp/main.bro b/scripts/base/protocols/smtp/main.bro index d53128b06c..702cb9fc0e 100644 --- a/scripts/base/protocols/smtp/main.bro +++ b/scripts/base/protocols/smtp/main.bro @@ -72,7 +72,10 @@ export { ## ALL_HOSTS - always capture the entire path. ## NO_HOSTS - never capture the path. const mail_path_capture = ALL_HOSTS &redef; - + + ## Create an extremely shortened representation of a log line. + global describe: function(rec: Info): string; + global log_smtp: event(rec: Info); } @@ -268,3 +271,29 @@ event connection_state_remove(c: connection) &priority=-5 if ( c?$smtp ) smtp_message(c); } + +function describe(rec: Info): string + { + if ( rec?$mailfrom && rec?$rcptto ) + { + local one_to = ""; + for ( to in rec$rcptto ) + { + one_to = to; + break; + } + local abbrev_subject = ""; + if ( rec?$subject ) + { + if ( |rec$subject| > 20 ) + { + abbrev_subject = rec$subject[0:20] + "..."; + } + } + + return fmt("%s -> %s%s%s", rec$mailfrom, one_to, + (|rec$rcptto|>1 ? fmt(" (plus %d others)", |rec$rcptto|-1) : ""), + (abbrev_subject != "" ? fmt(": %s", abbrev_subject) : "")); + } + return ""; + } \ No newline at end of file diff --git a/src/analyzer/analyzer.bif b/src/analyzer/analyzer.bif index 7f3cc6ed94..8b5a85956c 100644 --- a/src/analyzer/analyzer.bif +++ b/src/analyzer/analyzer.bif @@ -43,3 +43,8 @@ function __name%(atype: Analyzer::Tag%) : string %{ return new StringVal(analyzer_mgr->GetAnalyzerName(atype)); %} + +function __tag%(name: string%) : Analyzer::Tag + %{ + return new Val(analyzer_mgr->GetAnalyzerTag(name->CheckString()), TYPE_ENUM); + %}