Merge remote-tracking branch 'origin/topic/dnthayer/ftp-ipv6'

* origin/topic/dnthayer/ftp-ipv6:
  Add test case for FTP over IPv4
  Fix IPv6 URLs
  Add a test for FTP over IPv6
  Update FTP EPSV response processing for IPv6
  Fix parsing of FTP EPRT command and EPSV response

Conflicts:
	src/bro.bif

Closes #778.

CHANGES

@@ -1,4 +1,14 @@
 
+2.0-104 | 2012-02-24 14:59:12 -0800
+
+  * Add test case for FTP over IPv4. (Daniel Thayer)
+
+  * Fix IPv6 URLs in ftp.log. (Daniel Thayer)
+
+  * Add a test for FTP over IPv6 (Daniel Thayer)
+
+  * Fix parsing of FTP EPRT command and EPSV response. (Daniel Thayer)
+
 2.0-95 | 2012-02-22 05:27:34 -0800
 
   * GeoIP installation documentation update. (Seth Hall)

VERSION

@@ -1 +1 @@
-2.0-95
+2.0-104

scripts/base/protocols/ftp/main.bro

@@ -165,7 +165,12 @@ function ftp_message(s: Info)
 		
 		local arg = s$cmdarg$arg;
 		if ( s$cmdarg$cmd in file_cmds )
+			{
+			if ( is_v4_addr(s$id$resp_h) )
 				arg = fmt("ftp://%s%s", s$id$resp_h, build_path_compressed(s$cwd, arg));
+			else
+				arg = fmt("ftp://[%s]%s", s$id$resp_h, build_path_compressed(s$cwd, arg));
+			}
 		
 		s$ts=s$cmdarg$ts;
 		s$command=s$cmdarg$cmd;
@@ -270,7 +275,7 @@ event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) &prior
 			{
 			c$ftp$passive=T;
 			
-			if ( code == 229 && data$h == 0.0.0.0 )
+			if ( code == 229 && data$h == :: )
 				data$h = id$resp_h;
 			
 			ftp_data_expected[data$h, data$p] = c$ftp;

src/bro.bif

@@ -2541,7 +2541,7 @@ static Val* parse_eftp(const char* line)
 	RecordVal* r = new RecordVal(ftp_port);
 
 	int net_proto = 0;	// currently not used
-	uint32 addr = 0;
+	IPAddr addr;	// unspecified IPv6 address (all 128 bits zero)
 	int port = 0;
 	int good = 0;
 
@@ -2551,35 +2551,53 @@ static Val* parse_eftp(const char* line)
 			++line;
 
 		char delimiter = *line;
-		good = 1;
 		char* next_delim;
 
-		++line;	// cut off delimiter
+		if ( *line )
-		net_proto = strtol(line, &next_delim, 10);	// currently ignored
+			{
+			good = 1;
+			++line;	// skip delimiter
+
+			net_proto = strtol(line, &next_delim, 10);
 			if ( *next_delim != delimiter )
 				good = 0;
 
-		line = next_delim + 1;
+			line = next_delim;
-		if ( *line != delimiter )	// default of 0 is ok
+			if ( *line )
+				++line;
+
+			if ( *line && *line != delimiter )
 				{
-			string s(line);
+				const char* nptr = strchr(line, delimiter);
-			IPAddr tmp(s);
+				if ( nptr == NULL )
-			const uint32* bytes;
+					{
-			tmp.GetBytes(&bytes);
+					nptr = line + strlen(line);
-			addr = *bytes;
-			if ( addr == 0 )
 					good = 0;
 					}
 
-		// FIXME: check for garbage between IP and delimiter.
+				string s(line, nptr-line); // extract IP address
+				IPAddr tmp(s);
+				// on error, "tmp" will have all 128 bits zero
+				if ( tmp == addr )
+					good = 0;
+
+				addr = tmp;
+				}
+
 			line = strchr(line, delimiter);
 
+			if ( line != NULL )
+				{
 				++line;	// now the port
 				port = strtol(line, &next_delim, 10);
 				if ( *next_delim != delimiter )
 					good = 0;
 				}
 
+			}
+
+		}
+
 	r->Assign(0, new AddrVal(addr));
 	r->Assign(1, new PortVal(port, TRANSPORT_TCP));
 	r->Assign(2, new Val(good, TYPE_BOOL));
@@ -2605,7 +2623,7 @@ function parse_ftp_port%(s: string%): ftp_port
 ## The format is ``EPRT<space><d><net-prt><d><net-addr><d><tcp-port><d>``,
 ## where ``<d>`` is a delimiter in the ASCII range 33-126 (usually ``|``).
 ##
-## s: The string of the FTP PORT command, e.g., ``"10,0,0,1,4,31"``.
+## s: The string of the FTP EPRT command, e.g., ``"|1|10.0.0.1|1055|"``.
 ##
 ## Returns: The FTP PORT, e.g., ``[h=10.0.0.1, p=1055/tcp, valid=T]``
 ##
@@ -2645,7 +2663,7 @@ function parse_ftp_pasv%(str: string%): ftp_port
 ## The format is ``<text> (<d><d><d><tcp-port><d>)``, where ``<d>`` is a
 ## delimiter in the ASCII range 33-126 (usually ``|``).
 ##
-## str: The string containing the result of the FTP PASV command.
+## str: The string containing the result of the FTP EPSV command.
 ##
 ## Returns: The FTP PORT, e.g., ``[h=10.0.0.1, p=1055/tcp, valid=T]``
 ##

testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/conn.log

@@ -0,0 +1,12 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	conn
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count
+1329843175.736107	arKYeMETxOg	141.142.220.235	37604	199.233.217.249	56666	tcp	ftp-data	0.112432	0	342	SF	-	0	ShAdfFa	4	216	4	562
+1329843179.871641	k6kgXLOoSKl	141.142.220.235	59378	199.233.217.249	56667	tcp	ftp-data	0.111218	0	77	SF	-	0	ShAdfFa	4	216	4	297
+1329843194.151526	nQcgTWjvg4c	199.233.217.249	61920	141.142.220.235	33582	tcp	ftp-data	0.056211	342	0	SF	-	0	ShADaFf	5	614	3	164
+1329843197.783443	j4u32Pc5bif	199.233.217.249	61918	141.142.220.235	37835	tcp	ftp-data	0.056005	77	0	SF	-	0	ShADaFf	5	349	3	164
+1329843161.968492	UWkUyAuUGXf	141.142.220.235	50003	199.233.217.249	21	tcp	ftp	38.055625	180	3146	SF	-	0	ShAdDfFa	38	2164	25	4458

testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/ftp.log

@@ -0,0 +1,9 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	ftp
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	user	password	command	arg	mime_type	mime_desc	file_size	reply_code	reply_msg	tags	extraction_file
+#types	time	string	addr	port	addr	port	string	string	string	string	string	string	count	count	string	table[string]	file
+1329843179.926563	UWkUyAuUGXf	141.142.220.235	50003	199.233.217.249	21	anonymous	test	RETR	ftp://199.233.217.249/./robots.txt	text/plain	ASCII text	77	226	Transfer complete.	-	-
+1329843197.727769	UWkUyAuUGXf	141.142.220.235	50003	199.233.217.249	21	anonymous	test	RETR	ftp://199.233.217.249/./robots.txt	text/plain	ASCII text, with CRLF line terminators	77	226	Transfer complete.	-	-

testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/conn.log

@@ -0,0 +1,13 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	conn
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count
+1329327783.316897	arKYeMETxOg	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49186	2001:470:4867:99::21	57086	tcp	ftp-data	0.219721	0	342	SF	-	0	ShAdfFa	5	372	4	642
+1329327786.524332	k6kgXLOoSKl	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49187	2001:470:4867:99::21	57087	tcp	ftp-data	0.217501	0	43	SF	-	0	ShAdfFa	5	372	4	343
+1329327787.289095	nQcgTWjvg4c	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49188	2001:470:4867:99::21	57088	tcp	ftp-data	0.217941	0	77	SF	-	0	ShAdfFa	5	372	4	377
+1329327795.571921	j4u32Pc5bif	2001:470:4867:99::21	55785	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49189	tcp	ftp-data	0.109813	77	0	SF	-	0	ShADFaf	5	449	4	300
+1329327800.017649	TEfuqmmG4bh	2001:470:4867:99::21	55647	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49190	tcp	ftp-data	0.109181	342	0	SF	-	0	ShADFaf	5	714	4	300
+1329327777.822004	UWkUyAuUGXf	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49185	2001:470:4867:99::21	21	tcp	ftp	26.658219	310	3448	SF	-	0	ShAdDfFa	57	4426	34	5908

testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/ftp.log

@@ -0,0 +1,9 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	ftp
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	user	password	command	arg	mime_type	mime_desc	file_size	reply_code	reply_msg	tags	extraction_file
+#types	time	string	addr	port	addr	port	string	string	string	string	string	string	count	count	string	table[string]	file
+1329327787.396984	UWkUyAuUGXf	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49185	2001:470:4867:99::21	21	anonymous	test	RETR	ftp://[2001:470:4867:99::21]/robots.txt	-	-	77	226	Transfer complete.	-	-
+1329327795.463946	UWkUyAuUGXf	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49185	2001:470:4867:99::21	21	anonymous	test	RETR	ftp://[2001:470:4867:99::21]/robots.txt	-	-	77	226	Transfer complete.	-	-

testing/btest/scripts/base/protocols/ftp/ftp-ipv4.bro

@@ -0,0 +1,6 @@
+# This tests both active and passive FTP over IPv4.
+#
+# @TEST-EXEC: bro -r $TRACES/ftp-ipv4.trace
+# @TEST-EXEC: btest-diff conn.log
+# @TEST-EXEC: btest-diff ftp.log
+

testing/btest/scripts/base/protocols/ftp/ftp-ipv6.bro

@@ -0,0 +1,6 @@
+# This tests both active and passive FTP over IPv6.
+#
+# @TEST-EXEC: bro -r $TRACES/ipv6-ftp.trace
+# @TEST-EXEC: btest-diff conn.log
+# @TEST-EXEC: btest-diff ftp.log
+