Merge remote-tracking branch 'origin/topic/jsiwek/ipv6-output-format'

* origin/topic/jsiwek/ipv6-output-format:
  Change IPv6 output format to no longer automatically be bracketed.
  Change IPv6 address/prefix output format to be bracketed.

Closes #818.

CHANGES

@@ -1,4 +1,17 @@
 
+2.0-330 | 2012-05-14 17:05:56 -0700
+
+  * Add `addr_to_uri` script-level function that adds brackets to an
+    address if it's IPv6 and will be included in a URI or when a
+    ":<port>" needs to be appended to it. (Jon Siwek)
+
+  * Also add a test case for content extraction. (Jon Siwek)
+
+  * Fix typos and improve INSTALL document. (Daniel Thayer)
+    
+  * Switching to new btest command TEST-SERIALIZE for communication
+    tests. (Robin Sommer)
+
 2.0-323 | 2012-05-04 21:04:34 -0700
 
   * Add SHA1 and SHA256 hashing BIFs. Addresses #542.

VERSION

@@ -1 +1 @@
-2.0-323
+2.0-330

scripts/base/protocols/ftp/main.bro

@@ -6,6 +6,7 @@
 @load ./utils-commands
 @load base/utils/paths
 @load base/utils/numbers
+@load base/utils/addrs
 
 module FTP;
 
@@ -169,12 +170,7 @@ function ftp_message(s: Info)
 		
 		local arg = s$cmdarg$arg;
 		if ( s$cmdarg$cmd in file_cmds )
-			{
+			arg = fmt("ftp://%s%s", addr_to_uri(s$id$resp_h), build_path_compressed(s$cwd, arg));
-			if ( is_v4_addr(s$id$resp_h) )
-				arg = fmt("ftp://%s%s", s$id$resp_h, build_path_compressed(s$cwd, arg));
-			else
-				arg = fmt("ftp://[%s]%s", s$id$resp_h, build_path_compressed(s$cwd, arg));
-			}
 		
 		s$ts=s$cmdarg$ts;
 		s$command=s$cmdarg$cmd;

scripts/base/protocols/http/utils.bro

@@ -1,6 +1,7 @@
 ##! Utilities specific for HTTP processing.
 
 @load ./main
+@load base/utils/addrs
 
 module HTTP;
 
@@ -51,7 +52,7 @@ function extract_keys(data: string, kv_splitter: pattern): string_vec
 function build_url(rec: Info): string
 	{
 	local uri  = rec?$uri ? rec$uri : "/<missed_request>";
-	local host = rec?$host ? rec$host : fmt("%s", rec$id$resp_h);
+	local host = rec?$host ? rec$host : addr_to_uri(rec$id$resp_h);
 	if ( rec$id$resp_p != 80/tcp )
 		host = fmt("%s:%s", host, rec$id$resp_p);
 	return fmt("%s%s", host, uri);

scripts/base/utils/addrs.bro

@@ -98,3 +98,18 @@ function find_ip_addresses(input: string): string_array
 		}
 	return output;
 	}
+
+## Returns the string representation of an IP address suitable for inclusion
+## in a URI.  For IPv4, this does no special formatting, but for IPv6, the
+## address is included in square brackets.
+##
+## a: the address to make suitable for URI inclusion.
+##
+## Returns: the string representation of *a* suitable for URI inclusion.
+function addr_to_uri(a: addr): string
+	{
+	if ( is_v4_addr(a) )
+		return fmt("%s", a);
+	else
+		return fmt("[%s]", a);
+	}

scripts/base/utils/files.bro

@@ -1,10 +1,11 @@
+@load ./addrs
 
 ## This function can be used to generate a consistent filename for when
 ## contents of a file, stream, or connection are being extracted to disk.
 function generate_extraction_filename(prefix: string, c: connection, suffix: string): string
 	{
-	local conn_info = fmt("%s:%d-%s:%d", 
+	local conn_info = fmt("%s:%d-%s:%d", addr_to_uri(c$id$orig_h), c$id$orig_p,
-	                      c$id$orig_h, c$id$orig_p, c$id$resp_h, c$id$resp_p);
+	                      addr_to_uri(c$id$resp_h), c$id$resp_p);
 	
 	if ( prefix != "" )
 		conn_info = fmt("%s_%s", prefix, conn_info);

src/logging/WriterBackend.cc

@@ -1,6 +1,7 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
 #include "util.h"
+#include "bro_inet_ntop.h"
 #include "threading/SerialTypes.h"
 
 #include "WriterBackend.h"
@@ -248,7 +249,7 @@ string WriterBackend::Render(const threading::Value::addr_t& addr) const
 		{
 		char s[INET_ADDRSTRLEN];
 
-		if ( inet_ntop(AF_INET, &addr.in.in4, s, INET_ADDRSTRLEN) == NULL )
+		if ( ! bro_inet_ntop(AF_INET, &addr.in.in4, s, INET_ADDRSTRLEN) )
 			return "<bad IPv4 address conversion>";
 		else
 			return s;
@@ -257,7 +258,7 @@ string WriterBackend::Render(const threading::Value::addr_t& addr) const
 		{
 		char s[INET6_ADDRSTRLEN];
 
-		if ( inet_ntop(AF_INET6, &addr.in.in6, s, INET6_ADDRSTRLEN) == NULL )
+		if ( ! bro_inet_ntop(AF_INET6, &addr.in.in6, s, INET6_ADDRSTRLEN) )
 			return "<bad IPv6 address conversion>";
 		else
 			return s;

testing/btest/Baseline/istate.pybroccoli/bro..stdout

@@ -1,7 +1,7 @@
 ==== atomic
 -10
 2
-1330035434.516896
+1336411585.166009
 2.0 mins
 F
 1.5

testing/btest/Baseline/istate.pybroccoli/python..stdout.filtered

@@ -1,7 +1,7 @@
 ==== atomic a 1 ====
 -4L -4
 42 42
-1330035434.5180
+1336411585.1711
 60.0
 True True
 3.14
@@ -14,7 +14,7 @@ True True
 ==== atomic a 2 ====
 -10L -10
 2 2
-1330035434.5169
+1336411585.1660
 120.0
 False False
 1.5
@@ -27,7 +27,7 @@ False False
 ==== atomic b 2 ====
 -10L -10
 <broccoli.count instance at > 2
-<broccoli.time instance at > 1330035434.5169
+<broccoli.time instance at > 1336411585.1660
 <broccoli.interval instance at > 120.0
 False False
 1.5

testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/local.log

@@ -5,15 +5,15 @@
 #path	local
 #fields	ts	id.orig_h
 #types	time	addr
-1300475168.855330	141.142.220.118
+1300475168.859163	141.142.220.118
 1300475168.652003	141.142.220.118
 1300475168.895267	141.142.220.118
+1300475168.902635	141.142.220.118
+1300475168.892936	141.142.220.118
 1300475168.855305	141.142.220.118
-1300475168.859163	141.142.220.118
 1300475168.892913	141.142.220.118
 1300475168.724007	141.142.220.118
-1300475168.892936	141.142.220.118
+1300475168.855330	141.142.220.118
-1300475168.902635	141.142.220.118
 1300475168.891644	141.142.220.118
 1300475170.862384	141.142.220.226
 1300475168.853899	141.142.220.118

testing/btest/Baseline/scripts.base.protocols.conn.contents-default-extract/contents_[2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185-[2001:470:4867:99::21]:21_orig.dat

@@ -0,0 +1,22 @@
+USER anonymous
+PASS test
+SYST
+FEAT
+PWD
+EPSV
+LIST
+EPSV
+NLST
+TYPE I
+SIZE robots.txt
+EPSV
+RETR robots.txt
+MDTM robots.txt
+SIZE robots.txt
+EPRT |2|2001:470:1f11:81f:c999:d94:aa7c:2e3e|49189|
+RETR robots.txt
+MDTM robots.txt
+TYPE A
+EPRT |2|2001:470:1f11:81f:c999:d94:aa7c:2e3e|49190|
+LIST
+QUIT

testing/btest/Baseline/scripts.base.protocols.conn.contents-default-extract/contents_[2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185-[2001:470:4867:99::21]:21_resp.dat

@@ -0,0 +1,73 @@
+220 ftp.NetBSD.org FTP server (NetBSD-ftpd 20100320) ready.
+331 Guest login ok, type your name as password.
+230-
+    The NetBSD Project FTP Server located in Redwood City, CA, USA
+    1 Gbps connectivity courtesy of                          ,        ,
+    Internet Systems Consortium                 WELCOME!    /(        )`
+                                                            \ \___   / |
+      +--- Currently Supported Platforms ----+              /- _  `-/  '
+      |  acorn[26,32], algor, alpha, amd64,  |             (/\/ \ \   /\
+      |   amiga[,ppc], arc, atari, bebox,    |             / /   | `    \
+      |   cats, cesfic, cobalt, dreamcast,   |             O O   ) /    |
+      |  evb[arm,mips,ppc,sh3], hp[300,700], |             `-^--'`<     '
+      |       hpc[arm,mips,sh], i386,        |            (_.)  _  )   /
+      |      ibmnws, iyonix, luna68k,        |              .___/`    /
+      |    mac[m68k,ppc], mipsco, mmeye,     |               `-----' /
+      |      mvme[m68k,ppc], netwinders,     |  <----.     __ / __   \
+      |   news[m68k,mips], next68k, ofppc,   |  <----|====O)))==) \) /====
+      | playstation2, pmax, prep, sandpoint, |  <----'    `--' `.__,' \
+      |  sbmips, sgimips, shark, sparc[,64], |               |        |
+      |      sun[2,3], vax, x68k, xen        |                \       /
+      +--------------------------------------+           ______( (_  / \_____
+      See our website at http://www.NetBSD.org/        ,'  ,-----'   |       \
+       We log all FTP transfers and commands.          `--{__________)  (FL) \/
+230-
+    EXPORT NOTICE
+    
+    Please note that portions of this FTP site contain cryptographic
+    software controlled under the Export Administration Regulations (EAR).
+    
+    None of this software may be downloaded or otherwise exported or
+    re-exported into (or to a national or resident of) Cuba, Iran, Libya,
+    Sudan, North Korea, Syria or any other country to which the U.S. has
+    embargoed goods.
+    
+    By downloading or using said software, you are agreeing to the
+    foregoing and you are representing and warranting that you are not
+    located in, under the control of, or a national or resident of any
+    such country or on any such list.
+230 Guest login ok, access restrictions apply.
+215 UNIX Type: L8 Version: NetBSD-ftpd 20100320
+211-Features supported
+ MDTM
+ MLST Type*;Size*;Modify*;Perm*;Unique*;
+ REST STREAM
+ SIZE
+ TVFS
+211 End
+257 "/" is the current directory.
+229 Entering Extended Passive Mode (|||57086|)
+150 Opening ASCII mode data connection for '/bin/ls'.
+226 Transfer complete.
+229 Entering Extended Passive Mode (|||57087|)
+150 Opening ASCII mode data connection for 'file list'.
+226 Transfer complete.
+200 Type set to I.
+213 77
+229 Entering Extended Passive Mode (|||57088|)
+150 Opening BINARY mode data connection for 'robots.txt' (77 bytes).
+226 Transfer complete.
+213 20090816112038
+213 77
+200 EPRT command successful.
+150 Opening BINARY mode data connection for 'robots.txt' (77 bytes).
+226 Transfer complete.
+213 20090816112038
+200 Type set to A.
+200 EPRT command successful.
+150 Opening ASCII mode data connection for '/bin/ls'.
+226 Transfer complete.
+221-
+    Data traffic for this session was 154 bytes in 2 files.
+    Total traffic for this session was 4512 bytes in 5 transfers.
+221 Thank you for using the FTP service on ftp.NetBSD.org.

testing/btest/scripts/base/protocols/conn/contents-default-extract.test

@@ -0,0 +1,3 @@
+# @TEST-EXEC: bro -f "tcp port 21" -r $TRACES/ipv6-ftp.trace "Conn::default_extract=T"
+# @TEST-EXEC: btest-diff contents_[2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185-[2001:470:4867:99::21]:21_orig.dat
+# @TEST-EXEC: btest-diff contents_[2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185-[2001:470:4867:99::21]:21_resp.dat