FileAnalysis: add logging, file_analysis.log.

scripts/base/frameworks/file-analysis/main.bro

@@ -84,16 +84,16 @@ export {
 		## The number of bytes at the beginning of a file to save for later
 		## inspection in *bof_buffer* field of
 		## :bro:see:`FileAnalysis::ActionResults`.
-		bof_buffer_size: count &default=default_bof_buffer_size;
+		bof_buffer_size: count &log &default=default_bof_buffer_size;
 
 		## The content of the beginning of a file up to *bof_buffer_size* bytes.
 		## This is also the buffer that's used for file/mime type detection.
 		bof_buffer: string &optional;
 
 		## An initial guess at file type.
-		file_type: string &optional;
+		file_type: string &log &optional;
 		## An initial guess at mime type.
-		mime_type: string &optional;
+		mime_type: string &log &optional;
 
 		## Actions that have been added to the analysis of this file.
 		## Not meant to be modified directly by scripts.
@@ -110,4 +110,61 @@ export {
 	global get_handle: function(c: connection, is_orig: bool): string &redef;
 
 	# TODO: wrapper functions for BiFs ?
+
+	## Event that can be handled to access the Info record as it is sent on
+	## to the logging framework.
+	global log_file_analysis: event(rec: Info);
 }
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(FileAnalysis::LOG,
+	                   [$columns=Info, $ev=log_file_analysis]);
+	}
+
+redef record FileAnalysis::Info += {
+	actions_taken: set[Action] &log &optional;
+	extracted_files: set[string] &log &optional;
+	md5: string &log &optional;
+	sha1: string &log &optional;
+	sha256: string &log &optional;
+};
+
+hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+	&priority=-10
+	{
+	if ( trig != FileAnalysis::TRIGGER_EOF &&
+	     trig != FileAnalysis::TRIGGER_DONE ) return;
+
+	info$actions_taken = set();
+	info$extracted_files = set();
+
+	for ( act in info$actions )
+		{
+		add info$actions_taken[act$act];
+		local result: FileAnalysis::ActionResults = info$actions[act];
+
+		switch ( act$act ) {
+		case FileAnalysis::ACTION_EXTRACT:
+			add info$extracted_files[act$extract_filename];
+			break;
+		case FileAnalysis::ACTION_MD5:
+			if ( result?$md5 )
+				info$md5 = result$md5;
+			break;
+		case FileAnalysis::ACTION_SHA1:
+			if ( result?$sha1 )
+				info$sha1 = result$sha1;
+			break;
+		case FileAnalysis::ACTION_SHA256:
+			if ( result?$sha256 )
+				info$sha256 = result$sha256;
+			break;
+		case FileAnalysis::ACTION_DATA_EVENT:
+			# no direct result
+			break;
+		}
+		}
+
+	Log::write(FileAnalysis::LOG, info);
+	}

testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/file_analysis.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	file_analysis
+#open	2013-03-20-18-29-14
+#fields	file_id	parent_file_id	source	conn_uids	seen_bytes	total_bytes	missing_bytes	overflow_bytes	timeout_interval	bof_buffer_size	file_type	mime_type	actions_taken	extracted_files	md5	sha1	sha256
+#types	string	string	string	table[string]	count	count	count	count	interval	count	string	string	table[enum]	table[string]	string	string	string
+Cx92a0ym5R8	-	HTTP	UWkUyAuUGXf	4705	4705	0	0	120.000000	1024	set	set	FileAnalysis::ACTION_SHA1,FileAnalysis::ACTION_EXTRACT,FileAnalysis::ACTION_DATA_EVENT,FileAnalysis::ACTION_MD5,FileAnalysis::ACTION_SHA256	Cx92a0ym5R8-file	397168fd09991a0e712254df7bc639ac	1dd7ac0398df6cbc0696445a91ec681facf4dc47	4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18
+#close	2013-03-20-18-29-14

testing/btest/scripts/base/frameworks/file-analysis/logging.bro

@@ -0,0 +1,99 @@
+# @TEST-EXEC: bro -r $TRACES/http/get.trace %INPUT
+# @TEST-EXEC: btest-diff file_analysis.log
+
+global actions: set[FileAnalysis::ActionArgs];
+
+event file_chunk(info: FileAnalysis::Info, data: string, off: count)
+	{
+	print "file_chunk", info$file_id, |data|, off, data;
+	}
+
+event file_stream(info: FileAnalysis::Info, data: string)
+	{
+	print "file_stream", info$file_id, |data|, data;
+	}
+
+hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+	{
+	print trig;
+
+	switch ( trig ) {
+	case FileAnalysis::TRIGGER_NEW:
+		print info$file_id, info$seen_bytes, info$missing_bytes;
+
+		if ( info$source == "HTTP" )
+			{
+			for ( act in actions )
+				FileAnalysis::add_action(info$file_id, act);
+			local filename: string = fmt("%s-file", info$file_id);
+			FileAnalysis::add_action(info$file_id,
+			                         [$act=FileAnalysis::ACTION_EXTRACT,
+			                          $extract_filename=filename]);
+			FileAnalysis::add_action(info$file_id,
+			                         [$act=FileAnalysis::ACTION_DATA_EVENT,
+			                          $chunk_event=file_chunk,
+			                          $stream_event=file_stream]);
+
+			}
+		break;
+
+	case FileAnalysis::TRIGGER_BOF_BUFFER:
+		if ( info?$bof_buffer )
+			print info$bof_buffer[0:10];
+		break;
+
+	case FileAnalysis::TRIGGER_TYPE:
+		# not actually printing the values due to libmagic variances
+		if ( info?$file_type )
+			print "file type is set";
+		if ( info?$mime_type )
+			print "mime type is set";
+		break;
+
+	case FileAnalysis::TRIGGER_EOF:
+		fallthrough;
+	case FileAnalysis::TRIGGER_DONE:
+
+		print info$file_id, info$seen_bytes, info$missing_bytes;
+		print info$conn_uids;
+		print info$conn_ids;
+
+		if ( info?$total_bytes )
+			print "total bytes: " + fmt("%s", info$total_bytes);
+		if ( info?$source )
+			print "source: " + info$source;
+
+		for ( act in info$actions )
+			switch ( act$act ) {
+			case FileAnalysis::ACTION_MD5:
+				print fmt("MD5: %s", info$actions[act]$md5);
+				break;
+			case FileAnalysis::ACTION_SHA1:
+				print fmt("SHA1: %s", info$actions[act]$sha1);
+				break;
+			case FileAnalysis::ACTION_SHA256:
+				print fmt("SHA256: %s", info$actions[act]$sha256);
+				break;
+			}
+		break;
+	}
+	}
+
+hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+	&priority=-5
+	{
+	if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
+
+	# avoids libmagic variances across systems
+	if ( info?$mime_type )
+		info$mime_type = "set";
+	if ( info?$file_type )
+		info$file_type = "set";
+	}
+
+event bro_init()
+	{
+	add actions[[$act=FileAnalysis::ACTION_MD5]];
+	add actions[[$act=FileAnalysis::ACTION_SHA1]];
+	add actions[[$act=FileAnalysis::ACTION_SHA256]];
+	}