Merge remote-tracking branch 'origin/topic/jsiwek/reduce-ftp-cluster-msg-sizes'

* origin/topic/jsiwek/reduce-ftp-cluster-msg-sizes:
  Minimize data published for expected FTP data channel analysis

CHANGES

@@ -1,4 +1,28 @@
 
+3.2.0-dev.794 | 2020-06-18 20:04:06 +0000
+
+  * Minimize data published for expected FTP data channel analysis
+
+    Previously, more data than could effectively be utilized by any remote
+    Zeek was published (e.g. full list of pending commands or other
+    transient state that may add up to non-trivial amount of bytes). (Jon Siwek, Corelight)
+
+  * GH-998: Fix Reporter::conn_weird() to handle expired connections
+
+    This introduces a new sampling state-map for expired connections to fix
+    segfaults that previously occured when passing in a `connection` record
+    to `Reporter::conn_weird()` for which the internal `Connection` object
+    had already been expired and deleted.  This also introduces a new event
+    called `expired_conn_weird`, which is similar to `conn_weird`, except
+    the full `connection` record is no longer available, just the `conn_id`
+    and UID string. (Jon Siwek, Corelight)
+
+  * Place build file in explicit location for benchmarking to work correctly (Tim Wojtulewicz, Corelight)
+
+  * Update highwayhash submodule to upstream.
+    This includes a patch by us, which fixes the compile on AARCH64.
+    Fixes GH-1006. (Johanna Amann, Corelight)
+
 3.2.0-dev.788 | 2020-06-15 11:04:20 -0700
 
   * Fix location where CI places build.tgz (Jon Siwek, Corelight)

NEWS

@@ -51,6 +51,10 @@ New Functionality
 - Add ``flags`` parameters to ``rdp_connect_request``,
   ``rdp_negotiation_response``, and ``rdp_negotiation_failure`` events.
 
+- ``Reporter::conn_weird`` now correctly handles weirds for expired connections,
+  for which no connection state information is available in the core anymore. These
+  cases will raise the new ``expired_conn_weird`` event.
+
 Changed Functionality
 ---------------------
 

VERSION

@@ -1 +1 @@
-3.2.0-dev.788
+3.2.0-dev.794

scripts/base/protocols/ftp/main.zeek

@@ -60,6 +60,20 @@ event zeek_init() &priority=5
 # Establish the variable for tracking expected connections.
 global ftp_data_expected: table[addr, port] of Info &read_expire=5mins;
 
+function minimize_info(info: Info): Info
+	{
+	# Just minimal data for sending to other remote Zeek processes.
+	# Generally, only data that's consistent across an entire FTP session or
+	# relevant to an expected data transfer would even be usable.
+	local rval: Info;
+	rval$ts = info$ts;
+	rval$uid= info$uid;
+	rval$id= info$id;
+	rval$user = info$user;
+	rval$passive = info$passive;
+	rval$pending_commands = PendingCmds();
+	}
+
 ## A set of commands where the argument can be expected to refer
 ## to a file or directory.
 const file_cmds = {
@@ -153,7 +167,7 @@ event sync_add_expected_data(s: Info, chan: ExpectedDataChannel)
 	{
 @if ( Cluster::local_node_type() == Cluster::PROXY ||
       Cluster::local_node_type() == Cluster::MANAGER )
-	Broker::publish(Cluster::worker_topic, sync_add_expected_data, s, chan);
+	Broker::publish(Cluster::worker_topic, sync_add_expected_data, minimize_info(s), chan);
 @else
 	ftp_data_expected[chan$resp_h, chan$resp_p] = s;
 	Analyzer::schedule_analyzer(chan$orig_h, chan$resp_h, chan$resp_p,
@@ -181,7 +195,7 @@ function add_expected_data_channel(s: Info, chan: ExpectedDataChannel)
 	                            Analyzer::ANALYZER_FTP_DATA,
 	                            5mins);
 @if ( Cluster::is_enabled() )
-	Broker::publish(ftp_relay_topic(), sync_add_expected_data, s, chan);
+	Broker::publish(ftp_relay_topic(), sync_add_expected_data, minimize_info(s), chan);
 @endif
 	}