FileAnalysis: replace script-layer IRC file analysis.

scripts/base/protocols/ftp/file-extract.bro

@@ -77,7 +77,7 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
 	}
 
 hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
-	&priority=5
+	&priority=-5
 	{
 	if ( trig != FileAnalysis::TRIGGER_EOF &&
 	     trig != FileAnalysis::TRIGGER_DONE ) return;

scripts/base/protocols/ftp/main.bro

@@ -359,9 +359,7 @@ event file_transferred(c: connection, prefix: string, descr: string,
 	
 event connection_state_remove(c: connection) &priority=-5
 	{
-	local id = c$id;
+	delete ftp_data_expected[c$id$resp_h, c$id$resp_p];
-	if ( [id$resp_h, id$resp_p] in ftp_data_expected )
-		delete ftp_data_expected[id$resp_h, id$resp_p];
 	}
 
 # Use state remove event to cover connections terminated by RST.

scripts/base/protocols/irc/dcc-send.bro

@@ -28,69 +28,146 @@ export {
 		dcc_file_size:         count  &log &optional;
 		## Sniffed mime type of the file.
 		dcc_mime_type:         string &log &optional;
-		
+
 		## The file handle for the file to be extracted
-		extraction_file:       file &log &optional;
+		extraction_file:       string &log &optional;
-		
+
 		## A boolean to indicate if the current file transfer should be extracted.
 		extract_file:          bool &default=F;
-		
-		## The count of the number of file that have been extracted during the session.
-		num_extracted_files:   count &default=0;
 	};
 }
 
-global dcc_expected_transfers: table[addr, port] of Info = table();
+global dcc_expected_transfers: table[addr, port] of Info &read_expire=5mins;
 
-event file_transferred(c: connection, prefix: string, descr: string,
+global extract_count: count = 0;
-                       mime_type: string) &priority=3
+
+hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+	&priority=5
 	{
-	local id = c$id;
+	if ( trig != FileAnalysis::TRIGGER_NEW ) return;
-	if ( [id$resp_h, id$resp_p] !in dcc_expected_transfers )
+	if ( ! info?$source ) return;
-		return;
+	if ( info$source != "IRC_DATA" ) return;
-		
+	if ( ! info?$conns ) return;
-	local irc = dcc_expected_transfers[id$resp_h, id$resp_p];
-	
-	irc$dcc_mime_type = split1(mime_type, /;/)[1];
 
-	if ( extract_file_types == irc$dcc_mime_type )
+	local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id,
+	                          extract_count);
+	local extracting: bool = F;
+
+	for ( cid in info$conns )
 		{
-		irc$extract_file = T;
+		local c: connection = info$conns[cid];
-		}
+
-		
+		if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next;
-	if ( irc$extract_file )
+
-		{
+		local s = dcc_expected_transfers[cid$resp_h, cid$resp_p];
-		local suffix = fmt("%d.dat", ++irc$num_extracted_files);
+
-		local fname = generate_extraction_filename(extraction_prefix, c, suffix);
+		if ( ! s$extract_file ) next;
-		irc$extraction_file = open(fname);
+
+		if ( ! extracting )
+			{
+			FileAnalysis::add_action(info$file_id,
+			                         [$act=FileAnalysis::ACTION_EXTRACT,
+			                          $extract_filename=fname]);
+			extracting = T;
+			++extract_count;
+			}
+
+		s$extraction_file = fname;
 		}
 	}
 
-event file_transferred(c: connection, prefix: string, descr: string,
+function set_dcc_mime(info: FileAnalysis::Info)
-			mime_type: string) &priority=-4
 	{
-	local id = c$id;
+	if ( ! info?$conns ) return;
-	if ( [id$resp_h, id$resp_p] !in dcc_expected_transfers )
+
+	for ( cid in info$conns )
+		{
+		local c: connection = info$conns[cid];
+
+		if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next;
+
+		local s = dcc_expected_transfers[cid$resp_h, cid$resp_p];
+
+		s$dcc_mime_type = info$mime_type;
+		}
+	}
+
+function set_dcc_extraction_file(info: FileAnalysis::Info, filename: string)
+	{
+	if ( ! info?$conns ) return;
+
+	for ( cid in info$conns )
+		{
+		local c: connection = info$conns[cid];
+
+		if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next;
+
+		local s = dcc_expected_transfers[cid$resp_h, cid$resp_p];
+
+		s$extraction_file = filename;
+		}
+	}
+
+function log_dcc(info: FileAnalysis::Info)
+	{
+	if ( ! info?$conns ) return;
+
+	for ( cid in info$conns )
+		{
+		local c: connection = info$conns[cid];
+
+		if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next;
+
+		local irc = dcc_expected_transfers[cid$resp_h, cid$resp_p];
+
+		local tmp = irc$command;
+		irc$command = "DCC";
+		Log::write(IRC::LOG, irc);
+		irc$command = tmp;
+
+		# Delete these values in case another DCC transfer 
+		# happens during the IRC session.
+		delete irc$extract_file;
+		delete irc$extraction_file;
+		delete irc$dcc_file_name;
+		delete irc$dcc_file_size;
+		delete irc$dcc_mime_type;
+
 		return;
+		}
+	}
 
-	local irc = dcc_expected_transfers[id$resp_h, id$resp_p];
+hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+	&priority=5
+	{
+	if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
+	if ( ! info?$mime_type ) return;
+	if ( ! info?$source ) return;
+	if ( info$source != "IRC_DATA" ) return;
 
-	local tmp = irc$command;
+	set_dcc_mime(info);
-	irc$command = "DCC";
-	Log::write(IRC::LOG, irc);
-	irc$command = tmp;
 
-	if ( irc?$extraction_file )
+	if ( extract_file_types !in info$mime_type ) return;
-		set_contents_file(id, CONTENTS_RESP, irc$extraction_file);
 
-	# Delete these values in case another DCC transfer 
+	for ( act in info$actions )
-	# happens during the IRC session.
+		if ( act$act == FileAnalysis::ACTION_EXTRACT ) return;
-	delete irc$extract_file;
+
-	delete irc$extraction_file;
+	local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id,
-	delete irc$dcc_file_name;
+	                          extract_count);
-	delete irc$dcc_file_size;
+	++extract_count;
-	delete irc$dcc_mime_type;
+	FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_EXTRACT,
-	delete dcc_expected_transfers[id$resp_h, id$resp_p];
+	                                        $extract_filename=fname]);
+	set_dcc_extraction_file(info, fname);
+	}
+
+hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
+	&priority=-5
+	{
+	if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
+	if ( ! info?$source ) return;
+	if ( info$source != "IRC_DATA" ) return;
+
+	log_dcc(info);
 	}
 
 event irc_dcc_message(c: connection, is_orig: bool,
@@ -100,7 +177,7 @@ event irc_dcc_message(c: connection, is_orig: bool,
 	{
 	set_session(c);
 	if ( dcc_type != "SEND" )
-            return;
+		return;
 	c$irc$dcc_file_name = argument;
 	c$irc$dcc_file_size = size;
 	local p = count_to_port(dest_port, tcp);
@@ -114,3 +191,8 @@ event expected_connection_seen(c: connection, a: count) &priority=10
 	if ( [id$resp_h, id$resp_p] in dcc_expected_transfers )
 		add c$service["irc-dcc-data"];
 	}
+
+event connection_state_remove(c: connection) &priority=-5
+	{
+	delete dcc_expected_transfers[c$id$resp_h, c$id$resp_p];
+	}

testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log

@@ -3,11 +3,11 @@
 #empty_field	(empty)
 #unset_field	-
 #path	irc
-#open	2011-07-20-19-12-44
+#open	2013-03-27-18-51-40
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	nick	user	command	value	addl	dcc_file_name	dcc_file_size	extraction_file
-#types	time	string	addr	port	addr	port	string	string	string	string	string	string	count	file
+#types	time	string	addr	port	addr	port	string	string	string	string	string	string	count	string
 1311189164.119437	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	-	-	NICK	bloed	-	-	-	-
 1311189164.119437	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	-	USER	sdkfje	sdkfje Montreal.QC.CA.Undernet.org dkdkrwq	-	-	-
 1311189174.474127	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	sdkfje	JOIN	#easymovies	(empty)	-	-	-
 1311189316.326025	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	sdkfje	DCC	#easymovies	(empty)	ladyvampress-default(2011-07-07)-OS.zip	42208	-
-#close	2011-07-20-19-15-42
+#close	2013-03-27-18-51-40

testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log

@@ -3,11 +3,11 @@
 #empty_field	(empty)
 #unset_field	-
 #path	irc
-#open	2011-07-20-19-12-44
+#open	2013-03-27-18-49-16
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	nick	user	command	value	addl	dcc_file_name	dcc_file_size	dcc_mime_type	extraction_file
-#types	time	string	addr	port	addr	port	string	string	string	string	string	string	count	string	file
+#types	time	string	addr	port	addr	port	string	string	string	string	string	string	count	string	string
 1311189164.119437	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	-	-	NICK	bloed	-	-	-	-	-
 1311189164.119437	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	-	USER	sdkfje	sdkfje Montreal.QC.CA.Undernet.org dkdkrwq	-	-	-	-
 1311189174.474127	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	sdkfje	JOIN	#easymovies	(empty)	-	-	-	-
-1311189316.326025	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	sdkfje	DCC	#easymovies	(empty)	ladyvampress-default(2011-07-07)-OS.zip	42208	FAKE_MIME	irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat
+1311189316.326025	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	sdkfje	DCC	#easymovies	(empty)	ladyvampress-default(2011-07-07)-OS.zip	42208	FAKE_MIME	irc-dcc-item-wqKMAamJVSb-0.dat
-#close	2011-07-20-19-15-42
+#close	2013-03-27-18-49-16

testing/btest/scripts/base/protocols/irc/dcc-extract.test

@@ -4,9 +4,9 @@
 
 # @TEST-EXEC: bro -r $TRACES/irc-dcc-send.trace %INPUT
 # @TEST-EXEC: btest-diff irc.log
-# @TEST-EXEC: btest-diff irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat
+# @TEST-EXEC: btest-diff irc-dcc-item-wqKMAamJVSb-0.dat
 # @TEST-EXEC: bro -r $TRACES/irc-dcc-send.trace %INPUT IRC::extraction_prefix="test"
-# @TEST-EXEC: test -e test_192.168.1.77:57655-209.197.168.151:1024_1.dat
+# @TEST-EXEC: test -e test-wqKMAamJVSb-0.dat
 
 redef IRC::extract_file_types=/.*/;