Merge branch 'master' into topic/tunnels

Conflicts:
	scripts/base/init-bare.bro
	src/Conn.cc
	src/Conn.h
	src/Sessions.cc
	src/Sessions.h
	src/const.bif
	src/types.bif

scripts/base/frameworks/cluster/main.bro

@@ -77,6 +77,9 @@ export {
 		node_type:    NodeType;
 		## The IP address of the cluster node.
 		ip:           addr;
+		## If the *ip* field is a non-global IPv6 address, this field
+		## can specify a particular :rfc:`4007` ``zone_id``.
+		zone_id:      string      &default="";
 		## The port to which the this local node can connect when
 		## establishing communication.
 		p:            port;

scripts/base/frameworks/cluster/setup-connections.bro

@@ -19,23 +19,26 @@ event bro_init() &priority=9
 		# Connections from the control node for runtime control and update events.
 		# Every node in a cluster is eligible for control from this host.
 		if ( n$node_type == CONTROL )
-			Communication::nodes["control"] = [$host=n$ip, $connect=F,
-			                                   $class="control", $events=control_events];
+			Communication::nodes["control"] = [$host=n$ip, $zone_id=n$zone_id,
+			                                   $connect=F, $class="control",
+			                                   $events=control_events];
 		
 		if ( me$node_type == MANAGER )
 			{
 			if ( n$node_type == WORKER && n$manager == node )
 				Communication::nodes[i] =
-				    [$host=n$ip, $connect=F,
+				    [$host=n$ip, $zone_id=n$zone_id, $connect=F,
 				     $class=i, $events=worker2manager_events, $request_logs=T];
 			
 			if ( n$node_type == PROXY && n$manager == node )
 				Communication::nodes[i] =
-				    [$host=n$ip, $connect=F,
+				    [$host=n$ip, $zone_id=n$zone_id, $connect=F,
 				     $class=i, $events=proxy2manager_events, $request_logs=T];
 				
 			if ( n$node_type == TIME_MACHINE && me?$time_machine && me$time_machine == i )
-				Communication::nodes["time-machine"] = [$host=nodes[i]$ip, $p=nodes[i]$p,
+				Communication::nodes["time-machine"] = [$host=nodes[i]$ip,
+				                                        $zone_id=nodes[i]$zone_id,
+				                                        $p=nodes[i]$p,
 				                                        $connect=T, $retry=1min,
 				                                        $events=tm2manager_events];
 			}
@@ -44,7 +47,8 @@ event bro_init() &priority=9
 			{
 			if ( n$node_type == WORKER && n$proxy == node )
 				Communication::nodes[i] =
-					[$host=n$ip, $connect=F, $class=i, $sync=T, $auth=T, $events=worker2proxy_events];
+					[$host=n$ip, $zone_id=n$zone_id, $connect=F, $class=i,
+					 $sync=T, $auth=T, $events=worker2proxy_events];
 			
 			# accepts connections from the previous one. 
 			# (This is not ideal for setups with many proxies)
@@ -53,16 +57,18 @@ event bro_init() &priority=9
 				{
 				if ( n?$proxy )
 					Communication::nodes[i]
-					     = [$host=n$ip, $p=n$p,
+					     = [$host=n$ip, $zone_id=n$zone_id, $p=n$p,
 					        $connect=T, $auth=F, $sync=T, $retry=1mins];
 				else if ( me?$proxy && me$proxy == i )
 					Communication::nodes[me$proxy]
-					     = [$host=nodes[i]$ip, $connect=F, $auth=T, $sync=T];
+					     = [$host=nodes[i]$ip, $zone_id=nodes[i]$zone_id,
+					        $connect=F, $auth=T, $sync=T];
 				}
 			
 			# Finally the manager, to send it status updates.
 			if ( n$node_type == MANAGER && me$manager == i )
 				Communication::nodes["manager"] = [$host=nodes[i]$ip, 
+				                                   $zone_id=nodes[i]$zone_id, 
 				                                   $p=nodes[i]$p, 
 				                                   $connect=T, $retry=1mins, 
 				                                   $class=node,
@@ -72,6 +78,7 @@ event bro_init() &priority=9
 			{
 			if ( n$node_type == MANAGER && me$manager == i )
 				Communication::nodes["manager"] = [$host=nodes[i]$ip, 
+				                                   $zone_id=nodes[i]$zone_id,
 				                                   $p=nodes[i]$p,
 				                                   $connect=T, $retry=1mins, 
 				                                   $class=node, 
@@ -79,6 +86,7 @@ event bro_init() &priority=9
 			
 			if ( n$node_type == PROXY && me$proxy == i )
 				Communication::nodes["proxy"] = [$host=nodes[i]$ip, 
+				                                 $zone_id=nodes[i]$zone_id,
 				                                 $p=nodes[i]$p,
 				                                 $connect=T, $retry=1mins, 
 				                                 $sync=T, $class=node, 
@@ -87,6 +95,7 @@ event bro_init() &priority=9
 			if ( n$node_type == TIME_MACHINE && 
 			     me?$time_machine && me$time_machine == i )
 				Communication::nodes["time-machine"] = [$host=nodes[i]$ip, 
+				                                        $zone_id=nodes[i]$zone_id,
 				                                        $p=nodes[i]$p,
 				                                        $connect=T, 
 				                                        $retry=1min,

scripts/base/frameworks/communication/main.bro

@@ -2,6 +2,7 @@
 ##! and/or transfer events.
 
 @load base/frameworks/packet-filter
+@load base/utils/addrs
 
 module Communication;
 
@@ -10,7 +11,7 @@ export {
 	## The communication logging stream identifier.
 	redef enum Log::ID += { LOG };
 	
-	## Which interface to listen on (0.0.0.0 for any interface).
+	## Which interface to listen on (``0.0.0.0`` or ``[::]`` are wildcards).
 	const listen_interface = 0.0.0.0 &redef;
 	
 	## Which port to listen on.
@@ -19,6 +20,19 @@ export {
 	## This defines if a listening socket should use SSL.
 	const listen_ssl = F &redef;
 
+	## Defines if a listening socket can bind to IPv6 addresses.
+	const listen_ipv6 = F &redef;
+
+	## If :bro:id:`Communication::listen_interface` is a non-global
+	## IPv6 address and requires a specific :rfc:`4007` ``zone_id``,
+	## it can be specified here.
+	const listen_ipv6_zone_id = "" &redef;
+
+	## Defines the interval at which to retry binding to
+	## :bro:id:`Communication::listen_interface` on
+	## :bro:id:`Communication::listen_port` if it's already in use.
+	const listen_retry = 30 secs &redef;
+
 	## Default compression level.  Compression level is 0-9, with 0 = no 
 	## compression.
 	global compression_level = 0 &redef;
@@ -51,6 +65,10 @@ export {
 	type Node: record {
 		## Remote address.
 		host: addr;
+
+		## If the *host* field is a non-global IPv6 address, this field
+		## can specify a particular :rfc:`4007` ``zone_id``.
+		zone_id: string &optional;
 		
 		## Port of the remote Bro communication endpoint if we are initiating
 		## the connection based on the :bro:id:`connect` field.
@@ -160,7 +178,7 @@ event remote_log(level: count, src: count, msg: string)
 # This is a core generated event.
 event remote_log_peer(p: event_peer, level: count, src: count, msg: string)
 	{
-	local rmsg = fmt("[#%d/%s:%d] %s", p$id, p$host, p$p, msg);
+	local rmsg = fmt("[#%d/%s:%d] %s", p$id, addr_to_uri(p$host), p$p, msg);
 	do_script_log_common(level, src, rmsg);
 	}
 
@@ -178,7 +196,8 @@ function connect_peer(peer: string)
 		p = node$p;
 
 	local class = node?$class ? node$class : "";
-	local id = connect(node$host, p, class, node$retry, node$ssl);
+	local zone_id = node?$zone_id ? node$zone_id : "";
+	local id = connect(node$host, zone_id, p, class, node$retry, node$ssl);
     
 	if ( id == PEER_ID_NONE )
 		Log::write(Communication::LOG, [$ts = network_time(), 

scripts/base/frameworks/control/main.bro

@@ -11,6 +11,10 @@ export {
 	## The port of the host that will be controlled.
 	const host_port = 0/tcp &redef;
 
+	## If :bro:id:`Control::host` is a non-global IPv6 address and
+	## requires a specific :rfc:`4007` ``zone_id``, it can be set here.
+	const zone_id = "" &redef;
+
 	## The command that is being done.  It's typically set on the
 	## command line.
 	const cmd = "" &redef;

scripts/base/frameworks/input/__load__.bro

@@ -0,0 +1,5 @@
+@load ./main
+@load ./readers/ascii
+@load ./readers/raw
+@load ./readers/benchmark
+

scripts/base/frameworks/input/main.bro

@@ -0,0 +1,139 @@
+##! The input framework provides a way to read previously stored data either
+##! as an event stream or into a bro table.
+
+module Input;
+
+export {
+
+	## The default input reader used. Defaults to `READER_ASCII`.
+	const default_reader = READER_ASCII &redef;
+
+	const default_mode = MANUAL &redef;
+
+	## TableFilter description type used for the `table` method.
+	type TableDescription: record {
+		## Common definitions for tables and events
+
+		## String that allows the reader to find the source.
+		## For `READER_ASCII`, this is the filename.
+		source: string;
+
+		## Reader to use for this stream
+		reader: Reader &default=default_reader;
+
+		## Read mode to use for this stream
+		mode: Mode &default=default_mode;
+
+		## Descriptive name. Used to remove a stream at a later time
+		name: string;
+
+		# Special definitions for tables
+
+		## Table which will receive the data read by the input framework
+		destination: any;
+
+		## Record that defines the values used as the index of the table
+		idx: any;
+
+		## Record that defines the values used as the elements of the table
+		## If val is undefined, destination has to be a set.
+		val: any &optional;
+
+		## Defines if the value of the table is a record (default), or a single  value. Val
+		## can only contain one element when this is set to false.
+		want_record: bool &default=T;
+
+		## The event that is raised each time a value is added to, changed in or removed
+		## from the table. The event will receive an Input::Event enum as the first
+		## argument, the idx record as the second argument and the value (record) as the
+		## third argument.
+		ev: any &optional; # event containing idx, val as values.
+
+		## Predicate function that can decide if an insertion, update or removal should
+		## really be executed. Parameters are the same as for the event. If true is
+		## returned, the update is performed. If false is returned, it is skipped.
+		pred: function(typ: Input::Event, left: any, right: any): bool &optional;
+	};
+
+	## EventFilter description type used for the `event` method.
+	type EventDescription: record {
+		## Common definitions for tables and events
+
+		## String that allows the reader to find the source.
+		## For `READER_ASCII`, this is the filename.
+		source: string;
+
+		## Reader to use for this steam
+		reader: Reader &default=default_reader;
+
+		## Read mode to use for this stream
+		mode: Mode &default=default_mode;
+
+		## Descriptive name. Used to remove a stream at a later time
+		name: string;
+
+		# Special definitions for events
+
+		## Record describing the fields to be retrieved from the source input.
+		fields: any;
+
+		## If want_record if false (default), the event receives each value in fields as a seperate argument.
+		## If it is set to true, the event receives all fields in a signle record value.
+		want_record: bool &default=F;
+
+		## The event that is rised each time a new line is received from the reader.
+		## The event will receive an Input::Event enum as the first element, and the fields as the following arguments.
+		ev: any;
+
+	};
+
+	## Create a new table input from a given source. Returns true on success.
+	##
+	## description: `TableDescription` record describing the source.
+	global add_table: function(description: Input::TableDescription) : bool;
+
+	## Create a new event input from a given source. Returns true on success.
+	##
+	## description: `TableDescription` record describing the source.
+	global add_event: function(description: Input::EventDescription) : bool;
+
+	## Remove a input stream. Returns true on success and false if the named stream was not found.
+	##
+	## id: string value identifying the stream to be removed
+	global remove: function(id: string) : bool;
+
+	## Forces the current input to be checked for changes.
+	## Returns true on success and false if the named stream was not found
+	##
+	## id: string value identifying the stream
+	global force_update: function(id: string) : bool;
+
+	## Event that is called, when the update of a specific source is finished
+	global update_finished: event(name: string, source:string);
+}
+
+@load base/input.bif
+
+
+module Input;
+
+function add_table(description: Input::TableDescription) : bool
+	{
+	return __create_table_stream(description);
+	}
+
+function add_event(description: Input::EventDescription) : bool
+	{
+	return __create_event_stream(description);
+	}
+
+function remove(id: string) : bool
+	{
+	return __remove_stream(id);
+	}
+
+function force_update(id: string) : bool
+	{
+	return __force_update(id);
+	}
+

scripts/base/frameworks/input/readers/ascii.bro

@@ -0,0 +1,21 @@
+##! Interface for the ascii input reader.
+##!
+##! The defaults are set to match Bro's ASCII output.
+
+module InputAscii;
+
+export {
+	## Separator between fields.
+	## Please note that the separator has to be exactly one character long
+	const separator = "\t" &redef;
+
+	## Separator between set elements.
+	## Please note that the separator has to be exactly one character long
+	const set_separator = "," &redef;
+
+	## String to use for empty fields.
+	const empty_field = "(empty)" &redef;
+
+	## String to use for an unset &optional field.
+	const unset_field = "-" &redef;
+}

scripts/base/frameworks/input/readers/benchmark.bro

@@ -0,0 +1,23 @@
+##! Interface for the ascii input reader.
+
+module InputBenchmark;
+
+export {
+	## multiplication factor for each second
+	const factor = 1.0 &redef;
+
+	## spread factor between lines
+	const spread = 0 &redef;
+
+	## spreading where usleep = 1000000 / autospread * num_lines
+	const autospread = 0.0 &redef;
+
+	## addition factor for each heartbeat
+	const addfactor = 0 &redef;
+
+	## stop spreading at x lines per heartbeat
+	const stopspreadat = 0 &redef;
+
+	## 1 -> enable timed spreading
+	const timedspread = 0.0 &redef;
+}

scripts/base/frameworks/input/readers/raw.bro

@@ -0,0 +1,9 @@
+##! Interface for the raw input reader.
+
+module InputRaw;
+
+export {
+	## Separator between input records.
+	## Please note that the separator has to be exactly one character long
+	const record_separator = "\n" &redef;
+}

scripts/base/frameworks/logging/writers/dataseries.bro

@@ -10,7 +10,7 @@ export {
 	## 'lzo'  -- LZO compression.   Very fast decompression times.
 	## 'gz'   -- GZIP compression.  Slower than LZF, but also produces smaller output.
 	## 'bz2'  -- BZIP2 compression. Slower than GZIP, but also produces smaller output.
-	const compression = "lzo" &redef;
+	const compression = "gz" &redef;
 
 	## The extent buffer size.
 	## Larger values here lead to better compression and more efficient writes, but