Added protocol description functions that provide a super compressed log representation.

scripts/base/frameworks/analyzer/main.bro

@@ -81,6 +81,13 @@ export {
 	## Returns: The analyzer name corresponding to the tag.
 	global name: function(tag: Analyzer::Tag) : string;
 
+	## Translates an analyzer's name to a tag enum value.
+	##
+	## name: The analyzer name.
+	##
+	## Returns: The analyzer tag corresponding to the name.
+	global get_tag: function(name: string): Analyzer::Tag;
+
 	## Schedules an analyzer for a future connection originating from a given IP
 	## address and port.
 	##
@@ -187,6 +194,11 @@ function name(atype: Analyzer::Tag) : string
 	return __name(atype);
 	}
 
+function get_tag(name: string): Analyzer::Tag
+	{
+	return __tag(name);
+	}
+
 function schedule_analyzer(orig: addr, resp: addr, resp_p: port,
 			   analyzer: Analyzer::Tag, tout: interval) : bool
 	{

scripts/base/frameworks/files/main.bro

@@ -2,6 +2,7 @@
 ##! any network protocol over which they're transported.
 
 @load base/bif/file_analysis.bif
+@load base/frameworks/analyzer
 @load base/frameworks/logging
 @load base/utils/site
 
@@ -173,17 +174,36 @@ export {
 	## Returns: The analyzer name corresponding to the tag.
 	global analyzer_name: function(tag: Files::Tag): string;
 
+	## Provides a text description regarding metadata of the file.
+	## For example, with HTTP it would return a URL.
+	##
+	## f: The file to be described.
+	##
+	## Returns a text description regarding metadata of the file.
+	global describe: function(f: fa_file): string;
+
+	type ProtoRegistration: record {
+		## A callback to generate a file handle on demand when
+		## one is needed by the core.
+		get_file_handle: function(c: connection, is_orig: bool): string;
+		
+		## A callback to "describe" a file.  In the case of an HTTP
+		## transfer the most obvious description would be the URL.
+		## It's like an extremely compressed version of the normal log.
+		describe: function(f: fa_file): string
+				&default=function(f: fa_file): string { return ""; };
+	};
+
 	## Register callbacks for protocols that work with the Files framework.  
 	## The callbacks must uniquely identify a file and each protocol can 
 	## only have a single callback registered for it.
 	## 
 	## tag: Tag for the protocol analyzer having a callback being registered.
 	##
-	## callback: Function that can generate a file handle for the protocol analyzer
+	## reg: A :bro:see:`ProtoRegistration` record.
-	##           defined previously.
 	##
 	## Returns: true if the protocol being registered was not previously registered.
-	global register_protocol: function(tag: Files::Tag, callback: function(c: connection, is_orig: bool): string): bool;
+	global register_protocol: function(tag: Analyzer::Tag, reg: ProtoRegistration): bool;
 
 	## Register a callback for file analyzers to use if they need to do some manipulation
 	## when they are being added to a file before the core code takes over.  This is 
@@ -210,8 +230,7 @@ redef record AnalyzerArgs += {
 };
 
 # Store the callbacks for protocol analyzers that have files.
-global registered_protocols: table[Files::Tag] of function(c: connection, is_orig: bool): string = table()
+global registered_protocols: table[Analyzer::Tag] of ProtoRegistration = table();
-		&default=function(c: connection, is_orig: bool): string { return cat(c$uid, is_orig); };
 
 global analyzer_add_callbacks: table[Files::Tag] of function(f: fa_file, args: AnalyzerArgs) = table();
 
@@ -321,15 +340,28 @@ event file_state_remove(f: fa_file) &priority=-10
 	Log::write(Files::LOG, f$info);
 	}
 
-function register_protocol(tag: Files::Tag, callback: function(c: connection, is_orig: bool): string): bool
+function register_protocol(tag: Analyzer::Tag, reg: ProtoRegistration): bool
 	{
 	local result = (tag !in registered_protocols);
-	registered_protocols[tag] = callback;
+	registered_protocols[tag] = reg;
 	return result;
 	}
 
-event get_file_handle(tag: Files::Tag, c: connection, is_orig: bool) &priority=5
+function describe(f: fa_file): string
 	{
+	local tag = Analyzer::get_tag(f$source);
+	if ( tag !in registered_protocols )
+		return "";
+
 	local handler = registered_protocols[tag];
-	set_file_handle(handler(c, is_orig));
+	return handler$describe(f);
+	}
+
+event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) &priority=5
+	{
+	if ( tag !in registered_protocols )
+		return;
+
+	local handler = registered_protocols[tag];
+	set_file_handle(handler$get_file_handle(c, is_orig));
 	}

scripts/base/frameworks/notice/main.bro

@@ -79,7 +79,13 @@ export {
 
 		## A mime type if the notice is related to a file.  If the $f field
 		## is provided, this will be automatically filled out.
-		mime_type:      string          &log &optional;
+		file_mime_type: string          &log &optional;
+
+		## Frequently files can be "described" to give a bit more context.
+		## This field will typically be automatically filled out from an
+		## fa_file record.  For example, if a notice was related to a 
+		## file over HTTP, the URL of the request would be shown.
+		file_desc:      string          &log &optional;
 
 		## The transport protocol. Filled automatically when either conn, iconn
 		## or p is specified.
@@ -477,9 +483,13 @@ function apply_policy(n: Notice::Info)
 		{
 		if ( ! n?$fuid )
 			n$fuid = n$f$id;
-		if ( ! n?$mime_type && n$f?$mime_type )
+		
-			n$mime_type = n$f$mime_type;
+		if ( ! n?$file_mime_type && n$f?$mime_type )
-		if ( |n$f$conns| == 1 )
+			n$file_mime_type = n$f$mime_type;
+
+		n$file_desc = Files::describe(n$f);
+		
+		if ( n$f?$conns && |n$f$conns| == 1 )
 			{
 			for ( id in n$f$conns )
 				n$conn = n$f$conns[id];
@@ -490,6 +500,7 @@ function apply_policy(n: Notice::Info)
 		{
 		if ( ! n?$id )
 			n$id = n$conn$id;
+
 		if ( ! n?$uid )
 			n$uid = n$conn$uid;
 		}

scripts/base/protocols/ftp/__load__.bro

@@ -1,5 +1,6 @@
 @load ./utils-commands
 @load ./main
+@load ./utils
 @load ./files
 @load ./gridftp
 

scripts/base/protocols/ftp/files.bro

@@ -12,6 +12,9 @@ export {
 
 	## Default file handle provider for FTP.
 	global get_file_handle: function(c: connection, is_orig: bool): string;
+
+	## Describe the file being transferred.
+	global describe_file: function(f: fa_file): string;
 }
 
 function get_file_handle(c: connection, is_orig: bool): string
@@ -22,9 +25,25 @@ function get_file_handle(c: connection, is_orig: bool): string
 	return cat(Analyzer::ANALYZER_FTP_DATA, c$start_time, c$id, is_orig);
 	}
 
+function describe_file(f: fa_file): string
+	{
+	# This shouldn't be needed, but just in case...
+	if ( f$source != "FTP" )
+		return "";
+
+	for ( cid in f$conns )
+		{
+		if ( f$conns[cid]?$ftp )
+			return FTP::describe(f$conns[cid]$ftp);
+		}
+	return "";
+	}
+
 event bro_init() &priority=5
 	{
-	Files::register_protocol(Analyzer::ANALYZER_FTP_DATA, FTP::get_file_handle);
+	Files::register_protocol(Analyzer::ANALYZER_FTP_DATA,
+	                         [$get_file_handle = FTP::get_file_handle,
+	                          $describe        = FTP::describe_file]);
 	}
 
 

scripts/base/protocols/ftp/main.bro

@@ -63,8 +63,6 @@ export {
 		reply_code:       count       &log &optional;
 		## Reply message from the server in response to the command.
 		reply_msg:        string      &log &optional;
-		## Arbitrary tags that may indicate a particular attribute of this command.
-		tags:             set[string] &log;
 
 		## Expected FTP data channel.
 		data_channel:     ExpectedDataChannel &log &optional;
@@ -171,10 +169,12 @@ function set_ftp_session(c: connection)
 
 function ftp_message(s: Info)
 	{
-	# If it either has a tag associated with it (something detected)
+	s$ts=s$cmdarg$ts;
-	# or it's a deliberately logged command.
+	s$command=s$cmdarg$cmd;
-	if ( |s$tags| > 0 || (s?$cmdarg && s$cmdarg$cmd in logged_commands) )
+	s$arg=s$cmdarg$arg;
-		{
+	if ( s$arg == "" )
+		delete s$arg;
+
 	if ( s?$password &&
 	     ! s$capture_password &&
 	     to_lower(s$user) !in guest_ids )
@@ -182,25 +182,8 @@ function ftp_message(s: Info)
 		s$password = "<hidden>";
 		}
 
-		local arg = s$cmdarg$arg;
+	if ( s?$cmdarg && s$command in logged_commands)
-		if ( s$cmdarg$cmd in file_cmds )
-			{
-			local comp_path = build_path_compressed(s$cwd, arg);
-			if ( comp_path[0] != "/" )
-				comp_path = cat("/", comp_path);
-
-			arg = fmt("ftp://%s%s", addr_to_uri(s$id$resp_h), comp_path);
-			}
-
-		s$ts=s$cmdarg$ts;
-		s$command=s$cmdarg$cmd;
-		if ( arg == "" )
-			delete s$arg;
-		else
-			s$arg=arg;
-
 		Log::write(FTP::LOG, s);
-		}
 
 	# The MIME and file_size fields are specific to file transfer commands
 	# and may not be used in all commands so they need reset to "blank"
@@ -209,8 +192,6 @@ function ftp_message(s: Info)
 	delete s$file_size;
 	# Same with data channel.
 	delete s$data_channel;
-	# Tags are cleared everytime too.
-	s$tags = set();
 	}
 
 function add_expected_data_channel(s: Info, chan: ExpectedDataChannel)
@@ -218,7 +199,8 @@ function add_expected_data_channel(s: Info, chan: ExpectedDataChannel)
 	s$passive = chan$passive;
 	s$data_channel = chan;
 	ftp_data_expected[chan$resp_h, chan$resp_p] = s;
-	Analyzer::schedule_analyzer(chan$orig_h, chan$resp_h, chan$resp_p, Analyzer::ANALYZER_FTP_DATA,
+	Analyzer::schedule_analyzer(chan$orig_h, chan$resp_h, chan$resp_p, 
+	                            Analyzer::ANALYZER_FTP_DATA,
 	                            5mins);
 	}
 

scripts/base/protocols/http/files.bro

@@ -8,6 +8,9 @@ module HTTP;
 export {
 	## Default file handle provider for HTTP.
 	global get_file_handle: function(c: connection, is_orig: bool): string;
+
+	## Default file describer for HTTP.
+	global describe_file: function(f: fa_file): string;
 }
 
 function get_file_handle(c: connection, is_orig: bool): string
@@ -27,7 +30,23 @@ function get_file_handle(c: connection, is_orig: bool): string
 		}
 	}
 
+function describe_file(f: fa_file): string
+	{
+	# This shouldn't be needed, but just in case...
+	if ( f$source != "HTTP" )
+		return "";
+
+	for ( cid in f$conns )
+		{
+		if ( f$conns[cid]?$http )
+			return build_url_http(f$conns[cid]$http);
+		}
+	return "";
+	}
+
 event bro_init() &priority=5
 	{
-	Files::register_protocol(Analyzer::ANALYZER_HTTP, HTTP::get_file_handle);
+	Files::register_protocol(Analyzer::ANALYZER_HTTP,
+	                         [$get_file_handle = HTTP::get_file_handle,
+	                          $describe        = HTTP::describe_file]);
 	}

scripts/base/protocols/http/utils.bro

@@ -32,6 +32,9 @@ export {
 	##
 	## Returns: A URL prefixed with "http://".
 	global build_url_http: function(rec: Info): string;
+
+	## Create an extremely shortened representation of a log line.
+	global describe: function(rec: Info): string;
 }
 
 
@@ -62,3 +65,8 @@ function build_url_http(rec: Info): string
 	{
 	return fmt("http://%s", build_url(rec));
 	}
+
+function describe(rec: Info): string
+	{
+	return build_url_http(rec);
+	}

scripts/base/protocols/irc/file-analysis.bro

@@ -1,23 +0,0 @@
-@load ./dcc-send
-@load base/utils/conn-ids
-@load base/frameworks/files
-
-module IRC;
-
-export {
-	## Default file handle provider for IRC.
-	global get_file_handle: function(c: connection, is_orig: bool): string;
-}
-
-function get_file_handle(c: connection, is_orig: bool): string
-	{
-	if ( [c$id$resp_h, c$id$resp_p] !in dcc_expected_transfers ) 
-		return "";
-
-	return cat(ANALYZER_IRC_DATA, c$start_time, c$id, is_orig);
-	}
-
-event bro_init() &priority=5
-	{
-	Files::register_protocol(ANALYZER_IRC_DATA, IRC::get_file_handle);
-	}

scripts/base/protocols/irc/files.bro

@@ -24,7 +24,8 @@ function get_file_handle(c: connection, is_orig: bool): string
 
 event bro_init() &priority=5
 	{
-	Files::register_protocol(Analyzer::ANALYZER_IRC_DATA, IRC::get_file_handle);
+	Files::register_protocol(Analyzer::ANALYZER_IRC_DATA,
+	                         [$get_file_handle = IRC::get_file_handle]);
 	}
 
 event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5

scripts/base/protocols/smtp/files.bro

@@ -14,6 +14,9 @@ export {
 
 	## Default file handle provider for SMTP.
 	global get_file_handle: function(c: connection, is_orig: bool): string;
+
+	## Default file describer for SMTP.
+	global describe_file: function(f: fa_file): string;
 }
 
 function get_file_handle(c: connection, is_orig: bool): string
@@ -22,9 +25,25 @@ function get_file_handle(c: connection, is_orig: bool): string
 	           c$smtp_state$mime_depth);
 	}
 
+function describe_file(f: fa_file): string
+	{
+	# This shouldn't be needed, but just in case...
+	if ( f$source != "SMTP" )
+		return "";
+
+	for ( cid in f$conns )
+		{
+		local c = f$conns[cid];
+		return SMTP::describe(c$smtp);
+		}
+	return "";
+	}
+
 event bro_init() &priority=5
 	{
-	Files::register_protocol(Analyzer::ANALYZER_SMTP, SMTP::get_file_handle);
+	Files::register_protocol(Analyzer::ANALYZER_SMTP, 
+	                         [$get_file_handle = SMTP::get_file_handle,
+	                          $describe        = SMTP::describe_file]);
 	}
 
 event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5

scripts/base/protocols/smtp/main.bro

@@ -73,6 +73,9 @@ export {
 	##    NO_HOSTS - never capture the path.
 	const mail_path_capture = ALL_HOSTS &redef;
 	
+	## Create an extremely shortened representation of a log line.
+	global describe: function(rec: Info): string;
+
 	global log_smtp: event(rec: Info);
 }
 
@@ -268,3 +271,29 @@ event connection_state_remove(c: connection) &priority=-5
 	if ( c?$smtp )
 		smtp_message(c);
 	}
+
+function describe(rec: Info): string
+	{
+	if ( rec?$mailfrom && rec?$rcptto )
+		{
+		local one_to = "";
+		for ( to in rec$rcptto )
+			{
+			one_to = to;
+			break;
+			}
+		local abbrev_subject = "";
+		if ( rec?$subject )
+			{
+			if ( |rec$subject| > 20 )
+				{
+				abbrev_subject = rec$subject[0:20] + "...";
+				}
+			}
+
+		return fmt("%s -> %s%s%s", rec$mailfrom, one_to,
+			(|rec$rcptto|>1 ? fmt(" (plus %d others)", |rec$rcptto|-1) : ""),
+			(abbrev_subject != "" ? fmt(": %s", abbrev_subject) : ""));
+		}
+		return "";
+	}

src/analyzer/analyzer.bif

@@ -43,3 +43,8 @@ function __name%(atype: Analyzer::Tag%) : string
 	%{
 	return new StringVal(analyzer_mgr->GetAnalyzerName(atype));
 	%}
+
+function __tag%(name: string%) : Analyzer::Tag
+	%{
+	return new Val(analyzer_mgr->GetAnalyzerTag(name->CheckString()), TYPE_ENUM);
+	%}