From 1b696490d0c152b0c164ec82f47f3ad667e39618 Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <grigorescu@gmail.com>
Date: Thu, 8 Oct 2020 09:36:18 -0500
Subject: [PATCH 01/38] Whitespace fixes only [nomail] [skip ci]

---
 scripts/policy/misc/capture-loss.zeek | 30 +++++++++++++--------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/scripts/policy/misc/capture-loss.zeek b/scripts/policy/misc/capture-loss.zeek
index 0b827db544..05fdee3011 100644
--- a/scripts/policy/misc/capture-loss.zeek
+++ b/scripts/policy/misc/capture-loss.zeek
@@ -1,10 +1,10 @@
 ##! This script logs evidence regarding the degree to which the packet
-##! capture process suffers from measurement loss.  
-##! The loss could be due to overload on the host or NIC performing 
-##! the packet capture or it could even be beyond the host.  If you are 
-##! capturing from a switch with a SPAN port, it's very possible that 
+##! capture process suffers from measurement loss.
+##! The loss could be due to overload on the host or NIC performing
+##! the packet capture or it could even be beyond the host.  If you are
+##! capturing from a switch with a SPAN port, it's very possible that
 ##! the switch itself could be overloaded and dropping packets.
-##! Reported loss is computed in terms of the number of "gap events" (ACKs 
+##! Reported loss is computed in terms of the number of "gap events" (ACKs
 ##! for a sequence number that's above a gap).
 
 @load base/frameworks/notice
@@ -13,7 +13,7 @@ module CaptureLoss;
 
 export {
 	redef enum Log::ID += { LOG };
-	
+
 	global log_policy: Log::PolicyHook;
 
 	redef enum Notice::Type += {
@@ -21,7 +21,7 @@ export {
 		## threshold.
 		Too_Much_Loss
 	};
-	
+
 	type Info: record {
 		## Timestamp for when the measurement occurred.
 		ts:           time     &log;
@@ -38,11 +38,11 @@ export {
 		## Percentage of ACKs seen where the data being ACKed wasn't seen.
 		percent_lost: double   &log;
 	};
-	
+
 	## The interval at which capture loss reports are created.
 	option watch_interval = 15mins;
-	
-	## The percentage of missed data that is considered "too much" 
+
+	## The percentage of missed data that is considered "too much"
 	## when the :zeek:enum:`CaptureLoss::Too_Much_Loss` notice should be
 	## generated. The value is expressed as a double between 0 and 1 with 1
 	## being 100%.
@@ -56,7 +56,7 @@ event CaptureLoss::take_measurement(last_ts: time, last_acks: count, last_gaps:
 		schedule watch_interval { CaptureLoss::take_measurement(network_time(), 0, 0) };
 		return;
 		}
-	
+
 	local now = network_time();
 	local g = get_gap_stats();
 	local acks = g$ack_events - last_acks;
@@ -65,13 +65,13 @@ event CaptureLoss::take_measurement(last_ts: time, last_acks: count, last_gaps:
 	local info: Info = [$ts=now,
 	                    $ts_delta=now-last_ts,
 	                    $peer=peer_description,
-	                    $acks=acks, $gaps=gaps, 
+	                    $acks=acks, $gaps=gaps,
 	                    $percent_lost=pct_lost];
-	
+
 	if ( pct_lost >= too_much_loss*100 )
-		NOTICE([$note=Too_Much_Loss, 
+		NOTICE([$note=Too_Much_Loss,
 		        $msg=fmt("The capture loss script detected an estimated loss rate above %.3f%%", pct_lost)]);
-	
+
 	Log::write(LOG, info);
 	schedule watch_interval { CaptureLoss::take_measurement(now, g$ack_events, g$gap_events) };
 	}

From bb3527c95513210c7125e5a49447e5c4e76da303 Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <grigorescu@gmail.com>
Date: Thu, 8 Oct 2020 09:38:26 -0500
Subject: [PATCH 02/38] Documentation update, reference the threshold variable.
 [nomail] [skip ci]

---
 scripts/policy/misc/capture-loss.zeek | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/policy/misc/capture-loss.zeek b/scripts/policy/misc/capture-loss.zeek
index 05fdee3011..5e0e49bc9d 100644
--- a/scripts/policy/misc/capture-loss.zeek
+++ b/scripts/policy/misc/capture-loss.zeek
@@ -18,7 +18,7 @@ export {
 
 	redef enum Notice::Type += {
 		## Report if the detected capture loss exceeds the percentage
-		## threshold.
+		## threshold defined in :zeek:id:`CaptureLoss::too_much_loss`.
 		Too_Much_Loss
 	};
 

From cb9d419fa4846a09bafac8fa5564940abbf7483e Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <grigorescu@gmail.com>
Date: Thu, 8 Oct 2020 09:41:29 -0500
Subject: [PATCH 03/38] Add CaptureLoss::initial_watch_interval for a quick
 read on cluster health after startup.

---
 scripts/policy/misc/capture-loss.zeek | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/scripts/policy/misc/capture-loss.zeek b/scripts/policy/misc/capture-loss.zeek
index 5e0e49bc9d..a6fff664e0 100644
--- a/scripts/policy/misc/capture-loss.zeek
+++ b/scripts/policy/misc/capture-loss.zeek
@@ -39,9 +39,14 @@ export {
 		percent_lost: double   &log;
 	};
 
-	## The interval at which capture loss reports are created.
+	## The interval at which capture loss reports are created in a
+	## running cluster (that is, after the first report).
 	option watch_interval = 15mins;
 
+	## For faster feedback on cluster health, the first capture loss
+	## report is generated this many minutes after startup.
+	option initial_watch_interval = 1mins;
+
 	## The percentage of missed data that is considered "too much"
 	## when the :zeek:enum:`CaptureLoss::Too_Much_Loss` notice should be
 	## generated. The value is expressed as a double between 0 and 1 with 1
@@ -82,5 +87,5 @@ event zeek_init() &priority=5
 
 	# We only schedule the event if we are capturing packets.
 	if ( reading_live_traffic() || reading_traces() )
-		schedule watch_interval { CaptureLoss::take_measurement(network_time(), 0, 0) };
+		schedule initial_watch_interval { CaptureLoss::take_measurement(network_time(), 0, 0) };
 	}

From 59620ed75dc27e24f453d5d4d2979303537ef6d6 Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <grigorescu@gmail.com>
Date: Thu, 8 Oct 2020 11:26:46 -0500
Subject: [PATCH 04/38] Add CaptureLoss::Too_Little_Traffic

---
 scripts/policy/misc/capture-loss.zeek | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/scripts/policy/misc/capture-loss.zeek b/scripts/policy/misc/capture-loss.zeek
index a6fff664e0..e836dc31ab 100644
--- a/scripts/policy/misc/capture-loss.zeek
+++ b/scripts/policy/misc/capture-loss.zeek
@@ -19,7 +19,10 @@ export {
 	redef enum Notice::Type += {
 		## Report if the detected capture loss exceeds the percentage
 		## threshold defined in :zeek:id:`CaptureLoss::too_much_loss`.
-		Too_Much_Loss
+		Too_Much_Loss,
+		## Report if the traffic seen by a peer within a given watch
+		## interval is less than :zeek:id:`CaptureLoss::minimum_acks`.
+		Too_Little_Traffic,
 	};
 
 	type Info: record {
@@ -52,6 +55,11 @@ export {
 	## generated. The value is expressed as a double between 0 and 1 with 1
 	## being 100%.
 	option too_much_loss: double = 0.1;
+
+	## The minimum number of ACKs expected for a single peer in a
+	## watch interval. If the number seen is less than this,
+	## :zeek:enum:`CaptureLoss::Too_Little_Traffic` is raised.
+	option minimum_acks: count = 1;
 }
 
 event CaptureLoss::take_measurement(last_ts: time, last_acks: count, last_gaps: count)
@@ -77,6 +85,10 @@ event CaptureLoss::take_measurement(last_ts: time, last_acks: count, last_gaps:
 		NOTICE([$note=Too_Much_Loss,
 		        $msg=fmt("The capture loss script detected an estimated loss rate above %.3f%%", pct_lost)]);
 
+	if ( acks < minimum_acks )
+		NOTICE([$note=Too_Little_Traffic,
+		        $msg=fmt("The worker only observed %d ACKs and was expecting at least %d.", acks, minimum_acks)]);
+
 	Log::write(LOG, info);
 	schedule watch_interval { CaptureLoss::take_measurement(now, g$ack_events, g$gap_events) };
 	}

From c58cf698902874d824c86ec16ef4601ec38f456c Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <grigorescu@gmail.com>
Date: Thu, 8 Oct 2020 11:34:04 -0500
Subject: [PATCH 05/38] Add test for CaptureLoss::Too_Little_Traffic

---
 .../capture_loss.log                                 | 10 ++++++++++
 .../scripts.policy.misc.capture-loss/notice.log      | 10 ++++++++++
 testing/btest/scripts/policy/misc/capture-loss.zeek  | 12 ++++++++++++
 3 files changed, 32 insertions(+)
 create mode 100644 testing/btest/Baseline/scripts.policy.misc.capture-loss/capture_loss.log
 create mode 100644 testing/btest/Baseline/scripts.policy.misc.capture-loss/notice.log
 create mode 100644 testing/btest/scripts/policy/misc/capture-loss.zeek

diff --git a/testing/btest/Baseline/scripts.policy.misc.capture-loss/capture_loss.log b/testing/btest/Baseline/scripts.policy.misc.capture-loss/capture_loss.log
new file mode 100644
index 0000000000..ef5dca2f4b
--- /dev/null
+++ b/testing/btest/Baseline/scripts.policy.misc.capture-loss/capture_loss.log
@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	capture_loss
+#open	2020-10-08-16-33-05
+#fields	ts	ts_delta	peer	gaps	acks	percent_lost
+#types	time	interval	string	count	count	double
+964953086.310131	0.000000	zeek	0	0	0.0
+#close	2020-10-08-16-33-05
diff --git a/testing/btest/Baseline/scripts.policy.misc.capture-loss/notice.log b/testing/btest/Baseline/scripts.policy.misc.capture-loss/notice.log
new file mode 100644
index 0000000000..7c585c0301
--- /dev/null
+++ b/testing/btest/Baseline/scripts.policy.misc.capture-loss/notice.log
@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	notice
+#open	2020-10-08-16-33-05
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	suppress_for	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
+#types	time	string	addr	port	addr	port	string	string	string	enum	enum	string	string	addr	addr	port	count	string	set[enum]	interval	string	string	string	double	double
+964953086.310131	-	-	-	-	-	-	-	-	-	CaptureLoss::Too_Little_Traffic	The worker only observed 0 ACKs and was expecting at least 1.	-	-	-	-	-	-	Notice::ACTION_LOG	3600.000000	-	-	-	-	-
+#close	2020-10-08-16-33-05
diff --git a/testing/btest/scripts/policy/misc/capture-loss.zeek b/testing/btest/scripts/policy/misc/capture-loss.zeek
new file mode 100644
index 0000000000..d51576a5b4
--- /dev/null
+++ b/testing/btest/scripts/policy/misc/capture-loss.zeek
@@ -0,0 +1,12 @@
+# @TEST-EXEC: zeek -b -r $TRACES/dns53.pcap %INPUT
+# @TEST-EXEC: btest-diff capture_loss.log
+# @TEST-EXEC: btest-diff notice.log
+
+@load misc/capture-loss
+
+module CaptureLoss;
+
+event zeek_init()
+	{
+	event take_measurement(network_time(), 0, 0);
+	}

From 2bdc56dfcdd2cf0296b19301d307b97fa33a630c Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@corelight.com>
Date: Thu, 8 Oct 2020 15:46:28 -0400
Subject: [PATCH 06/38] Make it possible to pass command line options through
 to scripts.

The feature is documented with the zeek_script_args variable in
init-bare.zeek.
---
 scripts/base/init-bare.zeek | 12 ++++++++++
 src/NetVar.cc               |  2 ++
 src/Options.cc              | 45 +++++++++++++++++++++++++++++++++++--
 src/Options.h               |  2 ++
 src/zeek-setup.cc           | 10 +++++++++
 5 files changed, 69 insertions(+), 2 deletions(-)

diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek
index b7a2b1b80c..02887b38a5 100644
--- a/scripts/base/init-bare.zeek
+++ b/scripts/base/init-bare.zeek
@@ -454,6 +454,18 @@ type connection: record {
 	inner_vlan: int &optional;
 };
 
+## Arguments given to Zeek from the command line. In order to use this, Zeek
+## must use the "--" command line argument, then give the script name immediately
+## after the double hyphens and the provide the arguments after that. For example:
+## 
+##   zeek --bare-mode -- myscript.zeek -a -b -c
+## 
+## To use Zeek as an executable interpreter, include a line at the top of a script
+## like the following and make the script executable:
+## 
+##   #!/usr/local/zeek/bin/zeek -- 
+const zeek_script_args: vector of string = vector();
+
 ## Default amount of time a file can be inactive before the file analysis
 ## gives up and discards any internal state related to the file.
 option default_file_timeout_interval: interval = 2 mins;
diff --git a/src/NetVar.cc b/src/NetVar.cc
index dfc52e0ea4..b8985456ae 100644
--- a/src/NetVar.cc
+++ b/src/NetVar.cc
@@ -30,6 +30,8 @@ zeek::VectorType* index_vec;
 zeek::VectorType* mime_matches;
 zeek::RecordType* mime_match;
 
+zeek::VectorVal* zeek_script_args;
+
 zeek::RecordType* socks_address;
 
 zeek::TableVal* tcp_reassembler_ports_orig;
diff --git a/src/Options.cc b/src/Options.cc
index a9e4f40db7..cdc99993d1 100644
--- a/src/Options.cc
+++ b/src/Options.cc
@@ -5,6 +5,7 @@
 #include "Options.h"
 
 #include <algorithm>
+#include <sstream>
 
 #include <unistd.h>
 
@@ -186,8 +187,48 @@ Options parse_cmdline(int argc, char** argv)
 		}
 	else
 		{
-		for ( auto i = 0; i < argc; ++i )
-			zeek_args.emplace_back(argv[i]);
+		if ( argc > 1 )
+			{
+			auto endsWith = [](const std::string& str, const std::string& suffix)
+				{
+				return str.size() >= suffix.size() && 0 == str.compare(str.size()-suffix.size(), suffix.size(), suffix);
+				};
+
+			auto i = 0;
+			for ( ; i < argc && ! endsWith(argv[i], "--"); ++i )
+				{
+				zeek_args.emplace_back(argv[i]);
+				}
+
+			// If a script is invoked with Zeek as the interpreter, the arguments provided
+			// directly in the interpreter line of the script won't be broken apart in the 
+			// argv on Linux so we split it up here.
+			if ( endsWith(argv[i], "--") && zeek_args.size() == 1 )
+				{
+				std::istringstream iss(argv[i]);
+				for ( std::string s; iss >> s; )
+					{
+					if ( ! endsWith(s, "--") )
+						{
+						zeek_args.emplace_back(s);
+						}
+					}
+				}
+
+			if ( i < argc )
+				{
+				// There is an additional increment here to skip over the "--" if it was found.
+				if ( endsWith(argv[i], "--") )
+					++i;
+
+				// The first argument after the double hyphens in implicitly a script name.
+				rval.scripts_to_load.emplace_back(argv[i++]);
+
+				// If there are more argument, grab them for script arguments
+				for ( ; i < argc; ++i )
+					rval.script_args.emplace_back(argv[i]);
+				}
+			}
 		}
 
 	constexpr struct option long_opts[] = {
diff --git a/src/Options.h b/src/Options.h
index d70e37d65c..3f949e675f 100644
--- a/src/Options.h
+++ b/src/Options.h
@@ -74,6 +74,8 @@ struct Options {
 	std::set<std::string> plugins_to_load;
 	std::vector<std::string> scripts_to_load;
 	std::vector<std::string> script_options_to_set;
+
+	std::vector<std::string> script_args;
 };
 
 /**
diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc
index 600597988d..2b6a4e37e4 100644
--- a/src/zeek-setup.cc
+++ b/src/zeek-setup.cc
@@ -660,6 +660,16 @@ SetupResult setup(int argc, char** argv, Options* zopts)
 	init_net_var();
 	run_bif_initializers();
 
+	// Assign the script_args for command line processing in Zeek scripts.
+	if ( ! options.script_args.empty() )
+		{
+		auto script_args_val = zeek::id::find_val("zeek_script_args")->AsVectorVal();
+		for ( const string& script_arg: options.script_args )
+			{
+			script_args_val->Assign(script_args_val->Size(), zeek::make_intrusive<zeek::StringVal>(script_arg));
+			}
+		}
+
 	// Must come after plugin activation (and also after hash
 	// initialization).
 	binpac::FlowBuffer::Policy flowbuffer_policy;

From 97f7bf784b4cd81382f1f5be02ba434a94312829 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@corelight.com>
Date: Thu, 8 Oct 2020 16:11:25 -0400
Subject: [PATCH 07/38] Fixed an option processing bug

---
 src/Options.cc | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/Options.cc b/src/Options.cc
index cdc99993d1..951c55831b 100644
--- a/src/Options.cc
+++ b/src/Options.cc
@@ -200,23 +200,23 @@ Options parse_cmdline(int argc, char** argv)
 				zeek_args.emplace_back(argv[i]);
 				}
 
-			// If a script is invoked with Zeek as the interpreter, the arguments provided
-			// directly in the interpreter line of the script won't be broken apart in the 
-			// argv on Linux so we split it up here.
-			if ( endsWith(argv[i], "--") && zeek_args.size() == 1 )
-				{
-				std::istringstream iss(argv[i]);
-				for ( std::string s; iss >> s; )
-					{
-					if ( ! endsWith(s, "--") )
-						{
-						zeek_args.emplace_back(s);
-						}
-					}
-				}
-
 			if ( i < argc )
 				{
+				// If a script is invoked with Zeek as the interpreter, the arguments provided
+				// directly in the interpreter line of the script won't be broken apart in the 
+				// argv on Linux so we split it up here.
+				if ( endsWith(argv[i], "--") && zeek_args.size() == 1 )
+					{
+					std::istringstream iss(argv[i]);
+					for ( std::string s; iss >> s; )
+						{
+						if ( ! endsWith(s, "--") )
+							{
+							zeek_args.emplace_back(s);
+							}
+						}
+					}
+
 				// There is an additional increment here to skip over the "--" if it was found.
 				if ( endsWith(argv[i], "--") )
 					++i;

From ce590ae05fa6c93f86f8d7f75f610d7175eb312e Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@corelight.com>
Date: Thu, 8 Oct 2020 16:32:54 -0400
Subject: [PATCH 08/38] Add a test for script args.

---
 testing/btest/Baseline/core.scripts-args/.stdout | 1 +
 testing/btest/core/scripts-args.zeek             | 9 +++++++++
 2 files changed, 10 insertions(+)
 create mode 100644 testing/btest/Baseline/core.scripts-args/.stdout
 create mode 100644 testing/btest/core/scripts-args.zeek

diff --git a/testing/btest/Baseline/core.scripts-args/.stdout b/testing/btest/Baseline/core.scripts-args/.stdout
new file mode 100644
index 0000000000..c7b3e4ce24
--- /dev/null
+++ b/testing/btest/Baseline/core.scripts-args/.stdout
@@ -0,0 +1 @@
+[-a, -b, -c]
diff --git a/testing/btest/core/scripts-args.zeek b/testing/btest/core/scripts-args.zeek
new file mode 100644
index 0000000000..4541ef8e0d
--- /dev/null
+++ b/testing/btest/core/scripts-args.zeek
@@ -0,0 +1,9 @@
+# @TEST-EXEC: zeek -b -- %INPUT -a -b -c 
+
+# @TEST-EXEC: btest-diff .stdout
+
+event zeek_init()
+	{
+	print zeek_script_args;
+	}
+

From d9f4f9b371223a858ea72311f8c56ddbe0754f24 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Thu, 8 Oct 2020 17:57:56 -0700
Subject: [PATCH 09/38] Silence Clang's warning about ignoring GCC's
 maybe-uninitialized warning

Clang supports `#pragma GCC diagnostic` for "compatibility", but not
`-Wmaybe-uninitialized`, so was emitting `warning: unknown warning group
'-Wmaybe-uninitialized'`
---
 src/Val.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/Val.cc b/src/Val.cc
index 56f4c15009..fe479a6ad6 100644
--- a/src/Val.cc
+++ b/src/Val.cc
@@ -2192,10 +2192,14 @@ void TableVal::SendToStore(const Val* index, const TableEntryVal* new_entry_val,
 			case ELEMENT_NEW:
 			case ELEMENT_CHANGED:
 				{
+				#ifndef __clang__
 				#pragma GCC diagnostic push
 				#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
+				#endif
 				broker::optional<broker::timespan> expiry;
+				#ifndef __clang__
 				#pragma GCC diagnostic pop
+				#endif
 
 				auto expire_time = GetExpireTime();
 				if ( expire_time == 0 )

From 39177ce8c977a3ed0db3f47c66c3b72485113a37 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@corelight.com>
Date: Fri, 9 Oct 2020 12:16:44 -0400
Subject: [PATCH 10/38] Apply suggestions from code review

Co-authored-by: Jon Siwek <jsiwek@corelight.com>
---
 src/zeek-setup.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc
index 2b6a4e37e4..c155e6fdd3 100644
--- a/src/zeek-setup.cc
+++ b/src/zeek-setup.cc
@@ -664,9 +664,9 @@ SetupResult setup(int argc, char** argv, Options* zopts)
 	if ( ! options.script_args.empty() )
 		{
 		auto script_args_val = zeek::id::find_val("zeek_script_args")->AsVectorVal();
-		for ( const string& script_arg: options.script_args )
+		for ( const string& script_arg : options.script_args )
 			{
-			script_args_val->Assign(script_args_val->Size(), zeek::make_intrusive<zeek::StringVal>(script_arg));
+			script_args_val->Assign(script_args_val->Size(), make_intrusive<StringVal>(script_arg));
 			}
 		}
 

From 7b77c7e523f9855a73b1e6a494dd7fe45ad5f431 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Fri, 9 Oct 2020 12:41:15 -0700
Subject: [PATCH 11/38] Rename signature parser tokens to not be TCP-specific

---
 src/rule-parse.y | 14 +++++++-------
 src/rule-scan.l  |  8 ++++----
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/rule-parse.y b/src/rule-parse.y
index eb79250bf5..c99104e0e3 100644
--- a/src/rule-parse.y
+++ b/src/rule-parse.y
@@ -59,14 +59,14 @@ static uint8_t ip4_mask_to_len(uint32_t mask)
 %token TOK_SRC_PORT
 %token TOK_TCP_STATE
 %token TOK_STRING
-%token TOK_TCP_STATE_SYM
+%token TOK_STATE_SYM
 %token TOK_ACTIVE
 %token TOK_BOOL
 %token TOK_POLICY_SYMBOL
 
 %type <str> TOK_STRING TOK_IDENT TOK_POLICY_SYMBOL TOK_PATTERN pattern string
-%type <val> TOK_INT TOK_TCP_STATE_SYM TOK_IP_OPTION_SYM TOK_COMP
-%type <val> integer ipoption_list tcpstate_list opt_strength
+%type <val> TOK_INT TOK_STATE_SYM TOK_IP_OPTION_SYM TOK_COMP
+%type <val> integer ipoption_list state_list opt_strength
 %type <rule> rule
 %type <bl> TOK_BOOL opt_negate
 %type <hdr_test> hdr_expr
@@ -246,7 +246,7 @@ rule_attr:
 				(zeek::detail::RuleHdrTest::Comp) $2, $3));
 			}
 
-	|	TOK_TCP_STATE tcpstate_list
+	|	TOK_TCP_STATE state_list
 			{
 			current_rule->AddCondition(new zeek::detail::RuleConditionTCPState($2));
 			}
@@ -382,10 +382,10 @@ ipoption_list:
 			{ $$ = $1; }
 	;
 
-tcpstate_list:
-		tcpstate_list ',' TOK_TCP_STATE_SYM
+state_list:
+		state_list ',' TOK_STATE_SYM
 			{ $$ = $1 | $3; }
-	|	TOK_TCP_STATE_SYM
+	|	TOK_STATE_SYM
 			{ $$ = $1; }
 	;
 
diff --git a/src/rule-scan.l b/src/rule-scan.l
index ff4c6b0dfa..c7ccb90009 100644
--- a/src/rule-scan.l
+++ b/src/rule-scan.l
@@ -76,22 +76,22 @@ false	{ rules_lval.val = false; return TOK_BOOL; }
 
 established	{
 		rules_lval.val = zeek::detail::RuleConditionTCPState::STATE_ESTABLISHED;
-		return TOK_TCP_STATE_SYM;
+		return TOK_STATE_SYM;
 		}
 
 originator	{
 		rules_lval.val = zeek::detail::RuleConditionTCPState::STATE_ORIG;
-		return TOK_TCP_STATE_SYM;
+		return TOK_STATE_SYM;
 		}
 
 responder	{
 		rules_lval.val = zeek::detail::RuleConditionTCPState::STATE_RESP;
-		return TOK_TCP_STATE_SYM;
+		return TOK_STATE_SYM;
 		}
 
 stateless	{
 		rules_lval.val = zeek::detail::RuleConditionTCPState::STATE_STATELESS;
-		return TOK_TCP_STATE_SYM;
+		return TOK_STATE_SYM;
 		}
 
 lsrr		{

From 4d998742e2c70e7c583f3bb5424f04cf6bae0750 Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <grigorescu@users.noreply.github.com>
Date: Fri, 9 Oct 2020 14:43:04 -0500
Subject: [PATCH 12/38] Fix scheduling due to network_time being 0 in zeek_init

Co-authored-by: Jon Siwek <jsiwek@corelight.com>
---
 scripts/policy/misc/capture-loss.zeek | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/policy/misc/capture-loss.zeek b/scripts/policy/misc/capture-loss.zeek
index e836dc31ab..645bbb51b4 100644
--- a/scripts/policy/misc/capture-loss.zeek
+++ b/scripts/policy/misc/capture-loss.zeek
@@ -66,7 +66,7 @@ event CaptureLoss::take_measurement(last_ts: time, last_acks: count, last_gaps:
 	{
 	if ( last_ts == 0 )
 		{
-		schedule watch_interval { CaptureLoss::take_measurement(network_time(), 0, 0) };
+		schedule initial_watch_interval { CaptureLoss::take_measurement(network_time(), 0, 0) };
 		return;
 		}
 

From 7556beac20ae4798f85e20e872d7262026f7b0af Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Fri, 9 Oct 2020 12:56:23 -0700
Subject: [PATCH 13/38] Rename RuleConditionTCPState::TCPState enum values

---
 src/RuleCondition.cc |  8 ++++----
 src/RuleCondition.h  | 14 +++++++-------
 src/rule-scan.l      |  8 ++++----
 3 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/RuleCondition.cc b/src/RuleCondition.cc
index 977eb9f8e2..0650ea309b 100644
--- a/src/RuleCondition.cc
+++ b/src/RuleCondition.cc
@@ -32,16 +32,16 @@ bool RuleConditionTCPState::DoMatch(Rule* rule, RuleEndpointState* state,
 
 	auto* ta = static_cast<analyzer::tcp::TCP_Analyzer*>(root);
 
-	if ( tcpstates & STATE_STATELESS )
+	if ( tcpstates & RULE_STATE_STATELESS )
 		return true;
 
-	if ( (tcpstates & STATE_ORIG) && ! state->IsOrig() )
+	if ( (tcpstates & RULE_STATE_ORIG) && ! state->IsOrig() )
 		return false;
 
-	if ( (tcpstates & STATE_RESP) && state->IsOrig() )
+	if ( (tcpstates & RULE_STATE_RESP) && state->IsOrig() )
 		return false;
 
-	if ( (tcpstates & STATE_ESTABLISHED ) &&
+	if ( (tcpstates & RULE_STATE_ESTABLISHED ) &&
 		! (is_established(ta->Orig()) &&
 		   is_established(ta->Resp())))
 		return false;
diff --git a/src/RuleCondition.h b/src/RuleCondition.h
index aa735cf854..5951b86eaf 100644
--- a/src/RuleCondition.h
+++ b/src/RuleCondition.h
@@ -22,16 +22,16 @@ public:
 	virtual void PrintDebug() = 0;
 };
 
+enum RuleStateKind {
+	RULE_STATE_ESTABLISHED = 1,
+	RULE_STATE_ORIG = 2,
+	RULE_STATE_RESP = 4,
+	RULE_STATE_STATELESS = 8
+};
+
 // Implements the "tcp-state" keyword.
 class RuleConditionTCPState : public RuleCondition {
 public:
-	enum TCPState {
-		STATE_ESTABLISHED = 1,
-		STATE_ORIG = 2,
-		STATE_RESP = 4,
-		STATE_STATELESS = 8
-	};
-
 	explicit RuleConditionTCPState(int arg_tcpstates)
 		{ tcpstates = arg_tcpstates; }
 
diff --git a/src/rule-scan.l b/src/rule-scan.l
index c7ccb90009..0660105513 100644
--- a/src/rule-scan.l
+++ b/src/rule-scan.l
@@ -75,22 +75,22 @@ true	{ rules_lval.val = true; return TOK_BOOL; }
 false	{ rules_lval.val = false; return TOK_BOOL; }
 
 established	{
-		rules_lval.val = zeek::detail::RuleConditionTCPState::STATE_ESTABLISHED;
+		rules_lval.val = zeek::detail::RULE_STATE_ESTABLISHED;
 		return TOK_STATE_SYM;
 		}
 
 originator	{
-		rules_lval.val = zeek::detail::RuleConditionTCPState::STATE_ORIG;
+		rules_lval.val = zeek::detail::RULE_STATE_ORIG;
 		return TOK_STATE_SYM;
 		}
 
 responder	{
-		rules_lval.val = zeek::detail::RuleConditionTCPState::STATE_RESP;
+		rules_lval.val = zeek::detail::RULE_STATE_RESP;
 		return TOK_STATE_SYM;
 		}
 
 stateless	{
-		rules_lval.val = zeek::detail::RuleConditionTCPState::STATE_STATELESS;
+		rules_lval.val = zeek::detail::RULE_STATE_STATELESS;
 		return TOK_STATE_SYM;
 		}
 

From 5904d0708f7c350ffd92eeac34165fdd5aebdd39 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Fri, 9 Oct 2020 13:43:17 -0700
Subject: [PATCH 14/38] GH-779: Add "udp-state" signature condition

It accepts "originator" or "responder" states as a way to enforce that
the signature only matches packets in the associated direction.
The "established" state is rejected as an error since it doesn't
have a useful meaning like it does for the "tcp-state" condition.
---
 src/RuleCondition.cc                          | 25 +++++++++++++
 src/RuleCondition.h                           | 14 ++++++++
 src/rule-parse.y                              |  9 +++++
 src/rule-scan.l                               |  1 +
 .../btest/Baseline/signatures.udp-state/out   | 10 ++++++
 .../Baseline/signatures.udp-state/reject      |  2 ++
 testing/btest/signatures/udp-state.zeek       | 36 +++++++++++++++++++
 7 files changed, 97 insertions(+)
 create mode 100644 testing/btest/Baseline/signatures.udp-state/out
 create mode 100644 testing/btest/Baseline/signatures.udp-state/reject
 create mode 100644 testing/btest/signatures/udp-state.zeek

diff --git a/src/RuleCondition.cc b/src/RuleCondition.cc
index 0650ea309b..789c94cd88 100644
--- a/src/RuleCondition.cc
+++ b/src/RuleCondition.cc
@@ -54,6 +54,31 @@ void RuleConditionTCPState::PrintDebug()
 	fprintf(stderr, "	RuleConditionTCPState: 0x%x\n", tcpstates);
 	}
 
+bool RuleConditionUDPState::DoMatch(Rule* rule, RuleEndpointState* state,
+                                    const u_char* data, int len)
+	{
+	analyzer::Analyzer* root = state->GetAnalyzer()->Conn()->GetRootAnalyzer();
+
+	if ( ! root || ! root->IsAnalyzer("UDP") )
+		return false;
+
+	if ( states & RULE_STATE_STATELESS )
+		return true;
+
+	if ( (states & RULE_STATE_ORIG) && ! state->IsOrig() )
+		return false;
+
+	if ( (states & RULE_STATE_RESP) && state->IsOrig() )
+		return false;
+
+	return true;
+	}
+
+void RuleConditionUDPState::PrintDebug()
+	{
+	fprintf(stderr, "	RuleConditionUDPState: 0x%x\n", states);
+	}
+
 void RuleConditionIPOptions::PrintDebug()
 	{
 	fprintf(stderr, "	RuleConditionIPOptions: 0x%x\n", options);
diff --git a/src/RuleCondition.h b/src/RuleCondition.h
index 5951b86eaf..6e73da4d1d 100644
--- a/src/RuleCondition.h
+++ b/src/RuleCondition.h
@@ -46,6 +46,20 @@ private:
 	int tcpstates;
 };
 
+// Implements the "udp-state" keyword.
+class RuleConditionUDPState : public RuleCondition {
+public:
+	explicit RuleConditionUDPState(int arg_states)
+		{ states = arg_states; }
+
+	bool DoMatch(Rule* rule, RuleEndpointState* state, const u_char* data,
+	             int len) override;
+
+	void PrintDebug() override;
+
+private:
+	int states;
+};
 
 // Implements "ip-options".
 class RuleConditionIPOptions : public RuleCondition {
diff --git a/src/rule-parse.y b/src/rule-parse.y
index c99104e0e3..cb555c0c6d 100644
--- a/src/rule-parse.y
+++ b/src/rule-parse.y
@@ -58,6 +58,7 @@ static uint8_t ip4_mask_to_len(uint32_t mask)
 %token TOK_SRC_IP
 %token TOK_SRC_PORT
 %token TOK_TCP_STATE
+%token TOK_UDP_STATE
 %token TOK_STRING
 %token TOK_STATE_SYM
 %token TOK_ACTIVE
@@ -251,6 +252,14 @@ rule_attr:
 			current_rule->AddCondition(new zeek::detail::RuleConditionTCPState($2));
 			}
 
+	|	TOK_UDP_STATE state_list
+			{
+			if ( $2 & zeek::detail::RULE_STATE_ESTABLISHED )
+				rules_error("'established' is not a valid 'udp-state'");
+
+			current_rule->AddCondition(new zeek::detail::RuleConditionUDPState($2));
+			}
+
 	|	TOK_ACTIVE TOK_BOOL
 			{ current_rule->SetActiveStatus($2); }
 	;
diff --git a/src/rule-scan.l b/src/rule-scan.l
index 0660105513..8cf698563e 100644
--- a/src/rule-scan.l
+++ b/src/rule-scan.l
@@ -132,6 +132,7 @@ same-ip		return TOK_SAME_IP;
 src-ip		return TOK_SRC_IP;
 src-port	return TOK_SRC_PORT;
 tcp-state	return TOK_TCP_STATE;
+udp-state	return TOK_UDP_STATE;
 active		return TOK_ACTIVE;
 
 file-magic { rules_lval.val = zeek::detail::Rule::FILE_MAGIC; return TOK_PATTERN_TYPE; }
diff --git a/testing/btest/Baseline/signatures.udp-state/out b/testing/btest/Baseline/signatures.udp-state/out
new file mode 100644
index 0000000000..eb4d4bc78a
--- /dev/null
+++ b/testing/btest/Baseline/signatures.udp-state/out
@@ -0,0 +1,10 @@
+signature_match [orig_h=192.168.17.58, orig_p=58755/udp, resp_h=8.8.8.8, resp_p=53/udp] - my_sig_udp_orig
+0000  35 5e 01 00 00 01 00 00  00 00 00 00 06 67 6f 6f  5^...... .....goo
+0010  67 6c 65 03 63 6f 6d 00  01 01 00 01              gle.com. ....
+
+signature_match [orig_h=192.168.17.58, orig_p=58755/udp, resp_h=8.8.8.8, resp_p=53/udp] - my_sig_udp_resp
+0000  35 5e 81 80 00 01 00 01  00 00 00 00 06 67 6f 6f  5^...... .....goo
+0010  67 6c 65 03 63 6f 6d 00  01 01 00 01 c0 0c 01 01  gle.com. ........
+0020  00 01 00 00 54 49 00 13  00 05 69 73 73 75 65 73  ....TI.. ..issues
+0030  79 6d 61 6e 74 65 63 2e  63 6f 6d                 ymantec. com
+
diff --git a/testing/btest/Baseline/signatures.udp-state/reject b/testing/btest/Baseline/signatures.udp-state/reject
new file mode 100644
index 0000000000..b495e79aa4
--- /dev/null
+++ b/testing/btest/Baseline/signatures.udp-state/reject
@@ -0,0 +1,2 @@
+error: Error in signature (udp-established.sig:5): 'established' is not a valid 'udp-state'
+
diff --git a/testing/btest/signatures/udp-state.zeek b/testing/btest/signatures/udp-state.zeek
new file mode 100644
index 0000000000..13cb3d1b8a
--- /dev/null
+++ b/testing/btest/signatures/udp-state.zeek
@@ -0,0 +1,36 @@
+# @TEST-EXEC: zeek -b -s udp-states.sig -r $TRACES/dns-caa.pcap %INPUT >out
+# @TEST-EXEC-FAIL: zeek -b -s udp-established.sig -r $TRACES/dns-caa.pcap %INPUT >reject 2>&1
+# @TEST-EXEC: btest-diff out
+# @TEST-EXEC: btest-diff reject
+
+@TEST-START-FILE udp-states.sig
+signature my_sig_udp_orig {
+	ip-proto == udp
+	payload /.+/
+	udp-state originator
+	event "my_sig_udp_orig"
+}
+
+signature my_sig_udp_resp {
+	ip-proto == udp
+	payload /.+/
+	udp-state responder
+	event "my_sig_udp_resp"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE udp-established.sig
+signature my_sig_udp_est {
+  ip-proto == udp
+  payload /.+/
+  udp-state established
+  event "my_sig_udp_est"
+}
+@TEST-END-FILE
+
+event signature_match(state: signature_state, msg: string, data: string)
+	{
+	print fmt("signature_match %s - %s", state$conn$id, msg);
+	local s = split_string(hexdump(data), /\n/);
+	for ( i in s ) print s[i];
+	}

From cf8671d07885984156809eabb7d5bc9fa26b1e86 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@corelight.com>
Date: Mon, 12 Oct 2020 10:46:28 -0400
Subject: [PATCH 15/38] Make defining a port number for hosts in a cluster that
 only connect outbound optional

---
 scripts/base/frameworks/cluster/main.zeek            |  4 ++--
 .../base/frameworks/cluster/setup-connections.zeek   | 12 ++++++++----
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/scripts/base/frameworks/cluster/main.zeek b/scripts/base/frameworks/cluster/main.zeek
index 02c63562b6..f0ece62741 100644
--- a/scripts/base/frameworks/cluster/main.zeek
+++ b/scripts/base/frameworks/cluster/main.zeek
@@ -1,4 +1,4 @@
-##! A framework for establishing and controlling a cluster of Zeek instances.
+#hh#! A framework for establishing and controlling a cluster of Zeek instances.
 ##! In order to use the cluster framework, a script named
 ##! ``cluster-layout.zeek`` must exist somewhere in Zeek's script search path
 ##! which has a cluster definition of the :zeek:id:`Cluster::nodes` variable.
@@ -162,7 +162,7 @@ export {
 		## can specify a particular :rfc:`4007` ``zone_id``.
 		zone_id:      string      &default="";
 		## The port that this node will listen on for peer connections.
-		p:            port;
+		p:            port        &optional;
 		## Identifier for the interface a worker is sniffing.
 		interface:    string      &optional;
 		## Name of the manager node this node uses.  For workers and proxies.
diff --git a/scripts/base/frameworks/cluster/setup-connections.zeek b/scripts/base/frameworks/cluster/setup-connections.zeek
index 2abd57b142..7e470037a6 100644
--- a/scripts/base/frameworks/cluster/setup-connections.zeek
+++ b/scripts/base/frameworks/cluster/setup-connections.zeek
@@ -89,11 +89,15 @@ event zeek_init() &priority=-10
 	Broker::subscribe(nodeid_topic(Broker::node_id()));
 	Broker::subscribe(node_topic(node));
 
-	Broker::listen(Broker::default_listen_address,
-	               self$p,
-	               Broker::default_listen_retry);
+	if ( self?$p )
+		{
+		Broker::listen(Broker::default_listen_address,
+		               self$p,
+		               Broker::default_listen_retry);
+
+		Cluster::log(fmt("listening on %s:%s", Broker::default_listen_address, self$p));
+		}
 
-	Cluster::log(fmt("listening on %s:%s", Broker::default_listen_address, self$p));
 
 	switch ( self$node_type ) {
 	case MANAGER:

From 7bcbc57401df4d6c1066acc22fdc099dafb9b9c4 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@corelight.com>
Date: Mon, 12 Oct 2020 12:47:23 -0400
Subject: [PATCH 16/38] New bif to wrap pcap_findalldevs

---
 scripts/base/init-bare.zeek | 17 ++++++++++++
 src/iosource/pcap/pcap.bif  | 55 +++++++++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+)

diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek
index b7a2b1b80c..e040e7710f 100644
--- a/scripts/base/init-bare.zeek
+++ b/scripts/base/init-bare.zeek
@@ -4976,6 +4976,23 @@ export {
 	## Number of Mbytes to provide as buffer space when capturing from live
 	## interfaces.
 	const bufsize = 128 &redef;
+
+	## The definition of a "pcap interface".
+	type Interface: record {
+		name: string;
+		description: string &optional;
+		addrs: set[addr];
+		is_loopback: bool;
+		
+		extended_flags: bool &default=F;
+		# If the "extended_flags" field is set to T, then these next two 
+		# flags will have valid settings.  Otherwise, the following
+		# two fields are explicitly false.
+		is_up: bool &default=F;
+		is_running: bool &default=F;
+	};
+
+	type Interfaces: set[Pcap::Interface];
 } # end export
 
 module DCE_RPC;
diff --git a/src/iosource/pcap/pcap.bif b/src/iosource/pcap/pcap.bif
index 5655a8fac9..c4b050761d 100644
--- a/src/iosource/pcap/pcap.bif
+++ b/src/iosource/pcap/pcap.bif
@@ -1,10 +1,14 @@
 
 module Pcap;
 
+type Interface: record;
+
 const snaplen: count;
 const bufsize: count;
 
 %%{
+#include "pcap.h"
+
 #include "iosource/Manager.h"
 %%}
 
@@ -102,3 +106,54 @@ function error%(%): string
 
 	return zeek::make_intrusive<zeek::StringVal>("no error");
 	%}
+
+function findalldevs%(%): Pcap::Interfaces
+	%{
+	pcap_if_t *alldevs, *d;
+	char errbuf[PCAP_ERRBUF_SIZE];
+
+	int ret = pcap_findalldevs(&alldevs, errbuf);
+
+	static auto ifaces_type = id::find_type<TableType>("Pcap::Interfaces");
+	auto pcap_interfaces = make_intrusive<TableVal>(ifaces_type);
+
+	int i=0;
+	RecordVal *r;
+	static auto iface_type = id::find_type<RecordType>("Pcap::Interface");
+	for ( d=alldevs; d; d=d->next )
+		{
+		auto r = make_intrusive<RecordVal>(iface_type);
+
+		r->Assign(0, make_intrusive<StringVal>(d->name));
+		if ( d->description )
+			r->Assign(1, make_intrusive<StringVal>(d->description));
+
+		auto addrs = make_intrusive<ListVal>(TYPE_ADDR);
+		for ( auto addr = d->addresses; addr != NULL; addr = addr->next ) 
+			{
+			if ( addr->addr->sa_family == AF_INET ) 
+				{
+				IPAddr a(reinterpret_cast<struct sockaddr_in *>(addr->addr)->sin_addr);
+				addrs->Append(make_intrusive<AddrVal>(a));
+				}
+			else if ( addr->addr->sa_family == AF_INET6 )
+				{
+				IPAddr a(reinterpret_cast<struct sockaddr_in6 *>(addr->addr)->sin6_addr);
+				addrs->Append(make_intrusive<AddrVal>(a));
+				}
+			}
+		r->Assign(2, addrs->ToSetVal());
+		r->Assign(3, val_mgr->Bool(d->flags & PCAP_IF_LOOPBACK));
+#ifdef PCAP_IF_UP
+		r->Assign(4, val_mgr->True(); // <-- "extended" vals set.
+		r->Assign(5, val_mgr->Bool(d->flags & PCAP_IF_UP));
+		r->Assign(6, val_mgr->Bool(d->flags & PCAP_IF_RUNNING));
+#endif
+		
+		pcap_interfaces->Assign(std::move(r), 0);
+		}
+
+	pcap_freealldevs(alldevs);
+	return pcap_interfaces;
+	%}
+

From 36d75a02964c0dd059341072f599564c6f3d43a8 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@corelight.com>
Date: Mon, 12 Oct 2020 12:59:40 -0400
Subject: [PATCH 17/38] I accidentally missed a paren

---
 src/iosource/pcap/pcap.bif | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/iosource/pcap/pcap.bif b/src/iosource/pcap/pcap.bif
index c4b050761d..3428e6895f 100644
--- a/src/iosource/pcap/pcap.bif
+++ b/src/iosource/pcap/pcap.bif
@@ -145,7 +145,7 @@ function findalldevs%(%): Pcap::Interfaces
 		r->Assign(2, addrs->ToSetVal());
 		r->Assign(3, val_mgr->Bool(d->flags & PCAP_IF_LOOPBACK));
 #ifdef PCAP_IF_UP
-		r->Assign(4, val_mgr->True(); // <-- "extended" vals set.
+		r->Assign(4, val_mgr->True()); // <-- "extended" vals set.
 		r->Assign(5, val_mgr->Bool(d->flags & PCAP_IF_UP));
 		r->Assign(6, val_mgr->Bool(d->flags & PCAP_IF_RUNNING));
 #endif

From cbe47650d1b0bb5ad9fcb7ba0cc5afd05427fa0d Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Mon, 12 Oct 2020 11:20:50 -0700
Subject: [PATCH 18/38] Remove superfluous RuleCondition destructors

---
 src/RuleCondition.h | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/src/RuleCondition.h b/src/RuleCondition.h
index 6e73da4d1d..2d44816832 100644
--- a/src/RuleCondition.h
+++ b/src/RuleCondition.h
@@ -35,8 +35,6 @@ public:
 	explicit RuleConditionTCPState(int arg_tcpstates)
 		{ tcpstates = arg_tcpstates; }
 
-	~RuleConditionTCPState() override { }
-
 	bool DoMatch(Rule* rule, RuleEndpointState* state,
 				const u_char* data, int len) override;
 
@@ -73,9 +71,6 @@ public:
 
 	explicit RuleConditionIPOptions(int arg_options)	{ options = arg_options; }
 
-	~RuleConditionIPOptions() override
-		{ }
-
 	bool DoMatch(Rule* rule, RuleEndpointState* state,
 				const u_char* data, int len) override;
 
@@ -89,7 +84,6 @@ private:
 class RuleConditionSameIP : public RuleCondition {
 public:
 	RuleConditionSameIP()	{ }
-	~RuleConditionSameIP() override {}
 
 	bool DoMatch(Rule* rule, RuleEndpointState* state,
 				const u_char* data, int len) override;
@@ -105,8 +99,6 @@ public:
 	RuleConditionPayloadSize(uint32_t arg_val, Comp arg_comp)
 		{ val = arg_val; comp = arg_comp; }
 
-	~RuleConditionPayloadSize() override {}
-
 	bool DoMatch(Rule* rule, RuleEndpointState* state,
 				const u_char* data, int len) override;
 
@@ -121,7 +113,6 @@ private:
 class RuleConditionEval : public RuleCondition {
 public:
 	explicit RuleConditionEval(const char* func);
-	~RuleConditionEval() override {}
 
 	bool DoMatch(Rule* rule, RuleEndpointState* state,
 				const u_char* data, int len) override;

From 8c85f2135e4a06d73f0c381da5c96beebd010d7c Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Mon, 12 Oct 2020 16:19:19 -0700
Subject: [PATCH 19/38] GH-1211: Improve error message for already-defined
 functions

---
 src/Var.cc                                            |  6 +++++-
 .../Baseline/language.function-already-defined/out    |  2 ++
 testing/btest/language/function-already-defined.zeek  | 11 +++++++++++
 3 files changed, 18 insertions(+), 1 deletion(-)
 create mode 100644 testing/btest/Baseline/language.function-already-defined/out
 create mode 100644 testing/btest/language/function-already-defined.zeek

diff --git a/src/Var.cc b/src/Var.cc
index fa1db6c3d1..a0adcb8f4f 100644
--- a/src/Var.cc
+++ b/src/Var.cc
@@ -583,7 +583,7 @@ void begin_func(IDPtr id, const char* module_name,
 
 		case FUNC_FLAVOR_FUNCTION:
 			if ( ! id->IsRedefinable() )
-				id->Error("already defined");
+				id->Error("already defined", t.get());
 			break;
 
 		default:
@@ -594,6 +594,10 @@ void begin_func(IDPtr id, const char* module_name,
 	else
 		id->SetType(t);
 
+	if ( IsErrorType(id->GetType()->Tag()) )
+		reporter->FatalError("invalid definition of '%s' (see previous errors)",
+		                     id->Name());
+
 	const auto& args = t->Params();
 	const auto& canon_args = id->GetType()->AsFuncType()->Params();
 
diff --git a/testing/btest/Baseline/language.function-already-defined/out b/testing/btest/Baseline/language.function-already-defined/out
new file mode 100644
index 0000000000..fcbb88a3e5
--- /dev/null
+++ b/testing/btest/Baseline/language.function-already-defined/out
@@ -0,0 +1,2 @@
+error in /home/jon/pro/zeek/zeek/testing/btest/.tmp/language.function-already-defined/function-already-defined.zeek, line 4 and /home/jon/pro/zeek/zeek/testing/btest/.tmp/language.function-already-defined/function-already-defined.zeek, line 7: already defined (foo)
+fatal error in /home/jon/pro/zeek/zeek/testing/btest/.tmp/language.function-already-defined/function-already-defined.zeek, line 8: invalid definition of 'foo' (see previous errors)
diff --git a/testing/btest/language/function-already-defined.zeek b/testing/btest/language/function-already-defined.zeek
new file mode 100644
index 0000000000..9c609a68ac
--- /dev/null
+++ b/testing/btest/language/function-already-defined.zeek
@@ -0,0 +1,11 @@
+# @TEST-EXEC-FAIL: zeek -b %INPUT >out 2>&1
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
+
+function foo(a: string)
+	{ print a; }
+
+function foo(a: string)
+	{ }
+
+event zeek_init()
+	{ foo("hello"); }

From 38af38beaebd41f17c3ad6b0971b0fc7e19946e5 Mon Sep 17 00:00:00 2001
From: zeek-bot <info@zeek.org>
Date: Tue, 13 Oct 2020 00:37:59 +0000
Subject: [PATCH 20/38] Update doc submodule [nomail] [skip ci]

---
 doc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc b/doc
index fd3e22bf5e..ddec4e6750 160000
--- a/doc
+++ b/doc
@@ -1 +1 @@
-Subproject commit fd3e22bf5efc42a572e83346031aec4493b3c803
+Subproject commit ddec4e675013eff12746064fa4b6b723e86b5804

From d827e8b2d22e5480b5419af6705be5d3d04b03a6 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Mon, 12 Oct 2020 17:43:15 -0700
Subject: [PATCH 21/38] Improve documentation for zeek_init event scheduling
 pitfalls

---
 CHANGES       | 4 ++++
 VERSION       | 2 +-
 doc           | 2 +-
 src/event.bif | 8 ++++----
 4 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/CHANGES b/CHANGES
index 8ee3c16f8e..19976358be 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,8 @@
 
+3.3.0-dev.390 | 2020-10-12 17:43:15 -0700
+
+  * Improve documentation for zeek_init event scheduling pitfalls (Jon Siwek, Corelight)
+
 3.3.0-dev.388 | 2020-10-12 17:02:20 -0700
 
   * Add CaptureLoss::Too_Little_Traffic notice (Vlad Grigorescu)
diff --git a/VERSION b/VERSION
index 85691726f1..ea0e8675c9 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-3.3.0-dev.388
+3.3.0-dev.390
diff --git a/doc b/doc
index ddec4e6750..4f37431a11 160000
--- a/doc
+++ b/doc
@@ -1 +1 @@
-Subproject commit ddec4e675013eff12746064fa4b6b723e86b5804
+Subproject commit 4f37431a116bd8951c5a63bb4d1e3ffb065152f4
diff --git a/src/event.bif b/src/event.bif
index 362cfa42ff..4271d01ca2 100644
--- a/src/event.bif
+++ b/src/event.bif
@@ -41,10 +41,10 @@
 ##
 ##    When a ``zeek_init`` handler executes, Zeek has not yet seen any input
 ##    packets and therefore :zeek:id:`network_time` is not initialized yet. An
-##    artifact of that is that any timer installed in a ``zeek_init`` handler
-##    will fire immediately with the first packet. The standard way to work
-##    around that is to ignore the first time the timer fires and immediately
-##    reschedule.
+##    artifact of that is that any timer installed in a ``zeek_init`` handler,
+##    like with :zeek:keyword:`schedule`, will fire immediately with the first
+##    packet. The standard way to work around that is to ignore the first time
+##    the timer fires and immediately reschedule.
 ##
 event zeek_init%(%);
 

From e532991bf299468263971dc9d5a2770b479a8fbb Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@corelight.com>
Date: Tue, 13 Oct 2020 08:09:58 -0400
Subject: [PATCH 22/38] Update src/iosource/pcap/pcap.bif

Co-authored-by: Jon Siwek <jsiwek@corelight.com>
---
 src/iosource/pcap/pcap.bif | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/iosource/pcap/pcap.bif b/src/iosource/pcap/pcap.bif
index 3428e6895f..d40c4531e8 100644
--- a/src/iosource/pcap/pcap.bif
+++ b/src/iosource/pcap/pcap.bif
@@ -1,7 +1,6 @@
 
 module Pcap;
 
-type Interface: record;
 
 const snaplen: count;
 const bufsize: count;
@@ -156,4 +155,3 @@ function findalldevs%(%): Pcap::Interfaces
 	pcap_freealldevs(alldevs);
 	return pcap_interfaces;
 	%}
-

From dfa21d54c893939e892812a637a924fa89af8f5f Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@corelight.com>
Date: Tue, 13 Oct 2020 08:12:20 -0400
Subject: [PATCH 23/38] Update scripts/base/init-bare.zeek

Co-authored-by: Jon Siwek <jsiwek@corelight.com>
---
 scripts/base/init-bare.zeek | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek
index e040e7710f..277bab9cc0 100644
--- a/scripts/base/init-bare.zeek
+++ b/scripts/base/init-bare.zeek
@@ -4979,9 +4979,14 @@ export {
 
 	## The definition of a "pcap interface".
 	type Interface: record {
+		## The interface/device name.
 		name: string;
+		## A human-readable description of the device.
 		description: string &optional;
+		## The network addresses associated with the device.
 		addrs: set[addr];
+		## Whether the device is a loopback interface.  E.g. addresses
+		## of ``127.0.0.1`` or ``[::1]`` are used by loopback interfaces.
 		is_loopback: bool;
 		
 		extended_flags: bool &default=F;

From 928faeaad34d40d298a61e08550c209f8f867058 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@corelight.com>
Date: Tue, 13 Oct 2020 08:12:50 -0400
Subject: [PATCH 24/38] Update src/iosource/pcap/pcap.bif

Co-authored-by: Jon Siwek <jsiwek@corelight.com>
---
 src/iosource/pcap/pcap.bif | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/iosource/pcap/pcap.bif b/src/iosource/pcap/pcap.bif
index d40c4531e8..382a059961 100644
--- a/src/iosource/pcap/pcap.bif
+++ b/src/iosource/pcap/pcap.bif
@@ -149,7 +149,7 @@ function findalldevs%(%): Pcap::Interfaces
 		r->Assign(6, val_mgr->Bool(d->flags & PCAP_IF_RUNNING));
 #endif
 		
-		pcap_interfaces->Assign(std::move(r), 0);
+		pcap_interfaces->Assign(std::move(r), nullptr);
 		}
 
 	pcap_freealldevs(alldevs);

From 5d6800f6bd4b5427008742d95cf469d80ebe42a3 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@corelight.com>
Date: Tue, 13 Oct 2020 08:12:57 -0400
Subject: [PATCH 25/38] Update src/iosource/pcap/pcap.bif

Co-authored-by: Jon Siwek <jsiwek@corelight.com>
---
 src/iosource/pcap/pcap.bif | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/iosource/pcap/pcap.bif b/src/iosource/pcap/pcap.bif
index 382a059961..089c77779b 100644
--- a/src/iosource/pcap/pcap.bif
+++ b/src/iosource/pcap/pcap.bif
@@ -116,8 +116,6 @@ function findalldevs%(%): Pcap::Interfaces
 	static auto ifaces_type = id::find_type<TableType>("Pcap::Interfaces");
 	auto pcap_interfaces = make_intrusive<TableVal>(ifaces_type);
 
-	int i=0;
-	RecordVal *r;
 	static auto iface_type = id::find_type<RecordType>("Pcap::Interface");
 	for ( d=alldevs; d; d=d->next )
 		{

From 92eb7c10da20423fe30e6c5065a7be8e407b5e74 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@corelight.com>
Date: Tue, 13 Oct 2020 08:35:45 -0400
Subject: [PATCH 26/38] Finishing changes from code review.

---
 scripts/base/init-bare.zeek | 10 ++++------
 src/iosource/pcap/pcap.bif  | 16 +++++++++++-----
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek
index 277bab9cc0..b5a629d9ad 100644
--- a/scripts/base/init-bare.zeek
+++ b/scripts/base/init-bare.zeek
@@ -4989,12 +4989,10 @@ export {
 		## of ``127.0.0.1`` or ``[::1]`` are used by loopback interfaces.
 		is_loopback: bool;
 		
-		extended_flags: bool &default=F;
-		# If the "extended_flags" field is set to T, then these next two 
-		# flags will have valid settings.  Otherwise, the following
-		# two fields are explicitly false.
-		is_up: bool &default=F;
-		is_running: bool &default=F;
+		## Whether the device is up.  Not set when that info is unavailable.
+		is_up: bool &optional;
+		## Whether the device is running.  Not set when that info is unavailable.
+		is_running: bool &optional;
 	};
 
 	type Interfaces: set[Pcap::Interface];
diff --git a/src/iosource/pcap/pcap.bif b/src/iosource/pcap/pcap.bif
index 089c77779b..b694330bc7 100644
--- a/src/iosource/pcap/pcap.bif
+++ b/src/iosource/pcap/pcap.bif
@@ -111,11 +111,17 @@ function findalldevs%(%): Pcap::Interfaces
 	pcap_if_t *alldevs, *d;
 	char errbuf[PCAP_ERRBUF_SIZE];
 
-	int ret = pcap_findalldevs(&alldevs, errbuf);
-
 	static auto ifaces_type = id::find_type<TableType>("Pcap::Interfaces");
 	auto pcap_interfaces = make_intrusive<TableVal>(ifaces_type);
 
+	int ret = pcap_findalldevs(&alldevs, errbuf);
+	if ( ret == PCAP_ERROR )
+		{
+		emit_builtin_error(util::fmt("Error calling pcap_findalldevs: %s", errbuf));
+		// Return an empty set in case of failure.
+		return pcap_interfaces;
+		}
+
 	static auto iface_type = id::find_type<RecordType>("Pcap::Interface");
 	for ( d=alldevs; d; d=d->next )
 		{
@@ -142,9 +148,9 @@ function findalldevs%(%): Pcap::Interfaces
 		r->Assign(2, addrs->ToSetVal());
 		r->Assign(3, val_mgr->Bool(d->flags & PCAP_IF_LOOPBACK));
 #ifdef PCAP_IF_UP
-		r->Assign(4, val_mgr->True()); // <-- "extended" vals set.
-		r->Assign(5, val_mgr->Bool(d->flags & PCAP_IF_UP));
-		r->Assign(6, val_mgr->Bool(d->flags & PCAP_IF_RUNNING));
+		// These didn't become available until libpcap 1.6.1
+		r->Assign(4, val_mgr->Bool(d->flags & PCAP_IF_UP));
+		r->Assign(5, val_mgr->Bool(d->flags & PCAP_IF_RUNNING));
 #endif
 		
 		pcap_interfaces->Assign(std::move(r), nullptr);

From 08339f071e6b72d94412b9865fcd97901054adf9 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Tue, 13 Oct 2020 10:38:01 -0700
Subject: [PATCH 27/38] Add reference to network_time_init from zeek_init docs

---
 doc           | 2 +-
 src/event.bif | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/doc b/doc
index 4f37431a11..8522dbfadc 160000
--- a/doc
+++ b/doc
@@ -1 +1 @@
-Subproject commit 4f37431a116bd8951c5a63bb4d1e3ffb065152f4
+Subproject commit 8522dbfadcdc3adf3eca9a068e4b2a10cfd06ea9
diff --git a/src/event.bif b/src/event.bif
index 4271d01ca2..bf82df43f9 100644
--- a/src/event.bif
+++ b/src/event.bif
@@ -44,7 +44,8 @@
 ##    artifact of that is that any timer installed in a ``zeek_init`` handler,
 ##    like with :zeek:keyword:`schedule`, will fire immediately with the first
 ##    packet. The standard way to work around that is to ignore the first time
-##    the timer fires and immediately reschedule.
+##    the timer fires and immediately reschedule or to instead schedule the
+##    first event from with the :zeek:see:`network_time_init` event.
 ##
 event zeek_init%(%);
 

From 8a7730f4c894c77fce435e5681ee2a6aa14779ad Mon Sep 17 00:00:00 2001
From: Tim Wojtulewicz <tim@corelight.com>
Date: Tue, 13 Oct 2020 11:13:02 -0700
Subject: [PATCH 28/38] Remove unused LoginConn type and variable in Conn.h

---
 src/Conn.cc | 3 ---
 src/Conn.h  | 4 ----
 2 files changed, 7 deletions(-)

diff --git a/src/Conn.cc b/src/Conn.cc
index 048cab06ae..175cc8053a 100644
--- a/src/Conn.cc
+++ b/src/Conn.cc
@@ -94,8 +94,6 @@ Connection::Connection(NetSessions* s, const detail::ConnIDKey& k, double t,
 	vlan = pkt->vlan;
 	inner_vlan = pkt->inner_vlan;
 
-	login_conn = nullptr;
-
 	is_active = 1;
 	skip = 0;
 	weird = 0;
@@ -610,7 +608,6 @@ unsigned int Connection::MemoryAllocation() const
 		+ (timers.MemoryAllocation() - padded_sizeof(timers))
 		+ (conn_val ? conn_val->MemoryAllocation() : 0)
 		+ (root_analyzer ? root_analyzer->MemoryAllocation(): 0)
-		// login_conn is just a casted 'this'.
 		// primary_PIA is already contained in the analyzer tree.
 		;
 	}
diff --git a/src/Conn.h b/src/Conn.h
index 9be0afc8fe..5b33b5eeed 100644
--- a/src/Conn.h
+++ b/src/Conn.h
@@ -24,7 +24,6 @@
 ZEEK_FORWARD_DECLARE_NAMESPACED(Connection, zeek);
 ZEEK_FORWARD_DECLARE_NAMESPACED(ConnectionTimer, zeek::detail);
 ZEEK_FORWARD_DECLARE_NAMESPACED(NetSessions, zeek);
-class LoginConn;
 ZEEK_FORWARD_DECLARE_NAMESPACED(EncapsulationStack, zeek);
 
 ZEEK_FORWARD_DECLARE_NAMESPACED(Specific_RE_Matcher, zeek::detail);
@@ -173,8 +172,6 @@ public:
 
 	void AppendAddl(const char* str);
 
-	LoginConn* AsLoginConn()		{ return login_conn; }
-
 	void Match(detail::Rule::PatternType type, const u_char* data, int len,
 	           bool is_orig, bool bol, bool eol, bool clear_state);
 
@@ -354,7 +351,6 @@ protected:
 	double start_time, last_time;
 	double inactivity_timeout;
 	RecordValPtr conn_val;
-	LoginConn* login_conn;	// either nil, or this
 	const EncapsulationStack* encapsulation; // tunnels
 	int suppress_event;	// suppress certain events to once per conn.
 

From 2d3b4dab748be36dd2e9cc2b992b6ac7f28258aa Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Tue, 13 Oct 2020 12:09:03 -0700
Subject: [PATCH 29/38] Improve zeek_script_args test case and documentation

---
 NEWS                                               |  7 +++++++
 scripts/base/init-bare.zeek                        | 14 +++++++-------
 src/NetVar.cc                                      |  2 --
 src/zeek-setup.cc                                  |  2 +-
 .../.stdout => core.script-args/out}               |  1 +
 testing/btest/core/script-args.zeek                | 14 ++++++++++++++
 testing/btest/core/scripts-args.zeek               |  9 ---------
 7 files changed, 30 insertions(+), 19 deletions(-)
 rename testing/btest/Baseline/{core.scripts-args/.stdout => core.script-args/out} (50%)
 create mode 100644 testing/btest/core/script-args.zeek
 delete mode 100644 testing/btest/core/scripts-args.zeek

diff --git a/NEWS b/NEWS
index 8e183219db..ea6ca636bd 100644
--- a/NEWS
+++ b/NEWS
@@ -34,6 +34,13 @@ New Functionality
   See https://docs.zeek.org/en/master/frameworks/logging.html#filter-log-records
   for more details.
 
+- A new ``zeek_script_args`` variable contains a list of arguments passed
+  to a script.  E.g. either when explicitly executing Zeek like
+  ``zeek -- myscript.zeek -arg1 -arg2``, or when using Zeek as to interpret
+  executable scripts that contain a hashbang line at the top like::
+
+    #!/usr/local/zeek/bin/zeek --
+
 Changed Functionality
 ---------------------
 
diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek
index 02887b38a5..40bda1d217 100644
--- a/scripts/base/init-bare.zeek
+++ b/scripts/base/init-bare.zeek
@@ -455,15 +455,15 @@ type connection: record {
 };
 
 ## Arguments given to Zeek from the command line. In order to use this, Zeek
-## must use the "--" command line argument, then give the script name immediately
-## after the double hyphens and the provide the arguments after that. For example:
-## 
+## must use a ``--`` command line argument immediately followed by a script
+## file and additional arguments after that. For example::
+##
 ##   zeek --bare-mode -- myscript.zeek -a -b -c
-## 
+##
 ## To use Zeek as an executable interpreter, include a line at the top of a script
-## like the following and make the script executable:
-## 
-##   #!/usr/local/zeek/bin/zeek -- 
+## like the following and make the script executable::
+##
+##   #!/usr/local/zeek/bin/zeek --
 const zeek_script_args: vector of string = vector();
 
 ## Default amount of time a file can be inactive before the file analysis
diff --git a/src/NetVar.cc b/src/NetVar.cc
index b8985456ae..dfc52e0ea4 100644
--- a/src/NetVar.cc
+++ b/src/NetVar.cc
@@ -30,8 +30,6 @@ zeek::VectorType* index_vec;
 zeek::VectorType* mime_matches;
 zeek::RecordType* mime_match;
 
-zeek::VectorVal* zeek_script_args;
-
 zeek::RecordType* socks_address;
 
 zeek::TableVal* tcp_reassembler_ports_orig;
diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc
index c155e6fdd3..4d731a507c 100644
--- a/src/zeek-setup.cc
+++ b/src/zeek-setup.cc
@@ -663,7 +663,7 @@ SetupResult setup(int argc, char** argv, Options* zopts)
 	// Assign the script_args for command line processing in Zeek scripts.
 	if ( ! options.script_args.empty() )
 		{
-		auto script_args_val = zeek::id::find_val("zeek_script_args")->AsVectorVal();
+		auto script_args_val = id::find_val<VectorVal>("zeek_script_args");
 		for ( const string& script_arg : options.script_args )
 			{
 			script_args_val->Assign(script_args_val->Size(), make_intrusive<StringVal>(script_arg));
diff --git a/testing/btest/Baseline/core.scripts-args/.stdout b/testing/btest/Baseline/core.script-args/out
similarity index 50%
rename from testing/btest/Baseline/core.scripts-args/.stdout
rename to testing/btest/Baseline/core.script-args/out
index c7b3e4ce24..e7be66720c 100644
--- a/testing/btest/Baseline/core.scripts-args/.stdout
+++ b/testing/btest/Baseline/core.script-args/out
@@ -1 +1,2 @@
 [-a, -b, -c]
+[-d, -e, -f]
diff --git a/testing/btest/core/script-args.zeek b/testing/btest/core/script-args.zeek
new file mode 100644
index 0000000000..2e38f411cc
--- /dev/null
+++ b/testing/btest/core/script-args.zeek
@@ -0,0 +1,14 @@
+# @TEST-EXEC: printf '#!' > test.zeek
+# @TEST-EXEC: printf "$BUILD/src/zeek -b --\n" >> test.zeek
+# @TEST-EXEC: cat %INPUT >> test.zeek
+# @TEST-EXEC: chmod u+x test.zeek
+
+# @TEST-EXEC: zeek -b -- %INPUT -a -b -c >out
+# @TEST-EXEC: $(dirname %INPUT)/test.zeek -d -e -f >>out
+
+# @TEST-EXEC: btest-diff out
+
+event zeek_init()
+	{
+	print zeek_script_args;
+	}
diff --git a/testing/btest/core/scripts-args.zeek b/testing/btest/core/scripts-args.zeek
deleted file mode 100644
index 4541ef8e0d..0000000000
--- a/testing/btest/core/scripts-args.zeek
+++ /dev/null
@@ -1,9 +0,0 @@
-# @TEST-EXEC: zeek -b -- %INPUT -a -b -c 
-
-# @TEST-EXEC: btest-diff .stdout
-
-event zeek_init()
-	{
-	print zeek_script_args;
-	}
-

From e78386d6e5921e1beaa0ec6b92ff82c378a8910d Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@corelight.com>
Date: Tue, 13 Oct 2020 16:46:26 -0400
Subject: [PATCH 30/38] Update scripts/base/frameworks/cluster/main.zeek

Co-authored-by: Jon Siwek <jsiwek@corelight.com>
---
 scripts/base/frameworks/cluster/main.zeek | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/base/frameworks/cluster/main.zeek b/scripts/base/frameworks/cluster/main.zeek
index f0ece62741..85e1f3a4f9 100644
--- a/scripts/base/frameworks/cluster/main.zeek
+++ b/scripts/base/frameworks/cluster/main.zeek
@@ -1,4 +1,4 @@
-#hh#! A framework for establishing and controlling a cluster of Zeek instances.
+##! A framework for establishing and controlling a cluster of Zeek instances.
 ##! In order to use the cluster framework, a script named
 ##! ``cluster-layout.zeek`` must exist somewhere in Zeek's script search path
 ##! which has a cluster definition of the :zeek:id:`Cluster::nodes` variable.

From a608015338c574b3487de90f0c5e7a29825d5f40 Mon Sep 17 00:00:00 2001
From: Tim Wojtulewicz <tim@corelight.com>
Date: Tue, 13 Oct 2020 10:14:09 -0700
Subject: [PATCH 31/38] GH-1063: Update libkqueue to fix pf_ring-zc failures

---
 auxil/libkqueue | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/auxil/libkqueue b/auxil/libkqueue
index 7636475614..6c1717dea2 160000
--- a/auxil/libkqueue
+++ b/auxil/libkqueue
@@ -1 +1 @@
-Subproject commit 7636475614b583180229a528c470a45f68285e46
+Subproject commit 6c1717dea2dc34a91d32e07d2cae34b1afa0a84e

From 6ef55db5535ee18ff18901fa46530e4f781ced1e Mon Sep 17 00:00:00 2001
From: Tim Wojtulewicz <tim@corelight.com>
Date: Tue, 13 Oct 2020 12:31:27 -0700
Subject: [PATCH 32/38] Update cmake submodule to pull in fix for building
 libkqueue

---
 cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake b/cmake
index 47935075eb..cf652b8459 160000
--- a/cmake
+++ b/cmake
@@ -1 +1 @@
-Subproject commit 47935075ebcc4546494d9e0005eff5e9b3a6735d
+Subproject commit cf652b845908a15c02e11dca3162f3eecca0a9c5

From cd330c801d461d35be9fb3acde6582bac9f733be Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@corelight.com>
Date: Tue, 13 Oct 2020 16:48:15 -0400
Subject: [PATCH 33/38] Apply suggestions from code review

Co-authored-by: Jon Siwek <jsiwek@corelight.com>
---
 scripts/base/frameworks/cluster/main.zeek              | 3 ++-
 scripts/base/frameworks/cluster/setup-connections.zeek | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/scripts/base/frameworks/cluster/main.zeek b/scripts/base/frameworks/cluster/main.zeek
index 85e1f3a4f9..110f4ad3af 100644
--- a/scripts/base/frameworks/cluster/main.zeek
+++ b/scripts/base/frameworks/cluster/main.zeek
@@ -162,7 +162,8 @@ export {
 		## can specify a particular :rfc:`4007` ``zone_id``.
 		zone_id:      string      &default="";
 		## The port that this node will listen on for peer connections.
-		p:            port        &optional;
+		## A value of ``0/unknown`` means the node is not pre-configured to listen.
+		p:            port        &default=0/unknown;
 		## Identifier for the interface a worker is sniffing.
 		interface:    string      &optional;
 		## Name of the manager node this node uses.  For workers and proxies.
diff --git a/scripts/base/frameworks/cluster/setup-connections.zeek b/scripts/base/frameworks/cluster/setup-connections.zeek
index 7e470037a6..55ab8a460a 100644
--- a/scripts/base/frameworks/cluster/setup-connections.zeek
+++ b/scripts/base/frameworks/cluster/setup-connections.zeek
@@ -89,7 +89,7 @@ event zeek_init() &priority=-10
 	Broker::subscribe(nodeid_topic(Broker::node_id()));
 	Broker::subscribe(node_topic(node));
 
-	if ( self?$p )
+	if ( self$p != 0/unknown )
 		{
 		Broker::listen(Broker::default_listen_address,
 		               self$p,

From 3c4de51f40cd68deb470c3dbc827d989d47b5ecd Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Tue, 13 Oct 2020 14:59:11 -0700
Subject: [PATCH 34/38] GH-1208: Use Dictionary validity assertions only during
 CI

---
 .cirrus.yml | 1 +
 src/Dict.cc | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 7a50417e0e..2604595b92 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -161,6 +161,7 @@ sanitizer_task:
   << : *CI_TEMPLATE
   test_fuzzers_script: ./ci/test-fuzzers.sh
   env:
+    CXXFLAGS: -DZEEK_DICT_DEBUG
     ZEEK_CI_CONFIGURE_FLAGS: *SANITIZER_CONFIG
     ZEEK_TAILORED_UB_CHECKS: 1
     UBSAN_OPTIONS: print_stacktrace=1
diff --git a/src/Dict.cc b/src/Dict.cc
index f4144fb464..06aa643ee0 100644
--- a/src/Dict.cc
+++ b/src/Dict.cc
@@ -17,7 +17,7 @@
 
 #include "3rdparty/doctest.h"
 
-#ifdef DEBUG
+#if defined(DEBUG) && defined(ZEEK_DICT_DEBUG)
 #define ASSERT_VALID(o)	o->AssertValid()
 #else
 #define ASSERT_VALID(o)

From 1a8bb30127a0b37c2bc9556a0ee9e1f417fe4083 Mon Sep 17 00:00:00 2001
From: Tim Wojtulewicz <tim@corelight.com>
Date: Tue, 13 Oct 2020 15:25:39 -0700
Subject: [PATCH 35/38] Update submodule(s) [nomail]

---
 auxil/bifcl   | 2 +-
 auxil/binpac  | 2 +-
 auxil/broker  | 2 +-
 auxil/zeekctl | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/auxil/bifcl b/auxil/bifcl
index 857261518a..1eaa6aff1d 160000
--- a/auxil/bifcl
+++ b/auxil/bifcl
@@ -1 +1 @@
-Subproject commit 857261518ad7716af951eaf6894f22ed4e79d00d
+Subproject commit 1eaa6aff1d991307b134d85b64e1ab7b68c89c92
diff --git a/auxil/binpac b/auxil/binpac
index 84c306afe2..bc719c1565 160000
--- a/auxil/binpac
+++ b/auxil/binpac
@@ -1 +1 @@
-Subproject commit 84c306afe235c3032ff1ef682fbaf86cf485eb1c
+Subproject commit bc719c1565de9454b04a4b9aade14460268bcfbe
diff --git a/auxil/broker b/auxil/broker
index 830d2e5632..ffac36b4f2 160000
--- a/auxil/broker
+++ b/auxil/broker
@@ -1 +1 @@
-Subproject commit 830d2e5632c0d268ab085ad32f53b3848a466988
+Subproject commit ffac36b4f2b56a55ae6e298f58d07fe10c13a824
diff --git a/auxil/zeekctl b/auxil/zeekctl
index 60f06da517..f99e3265c5 160000
--- a/auxil/zeekctl
+++ b/auxil/zeekctl
@@ -1 +1 @@
-Subproject commit 60f06da51721b043927543fdb11b04877d9eb0f5
+Subproject commit f99e3265c5e7d6c45361b7d8dc03e772f66b0d4b

From 399badfb51dcca4a00df1c3e26bf0adeae906713 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Tue, 13 Oct 2020 15:32:44 -0700
Subject: [PATCH 36/38] Update submodule(s) [nomail]

---
 auxil/zeek-aux | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/auxil/zeek-aux b/auxil/zeek-aux
index 85b087fc8e..fbb5a21719 160000
--- a/auxil/zeek-aux
+++ b/auxil/zeek-aux
@@ -1 +1 @@
-Subproject commit 85b087fc8eeeeb2c0e0d241e33e78e39fc099aab
+Subproject commit fbb5a21719d4d00244bdd9f0d0a2f8543580a016

From b905e221356511e93fa60430643974204f0dbcc9 Mon Sep 17 00:00:00 2001
From: zeek-bot <info@zeek.org>
Date: Wed, 14 Oct 2020 00:38:25 +0000
Subject: [PATCH 37/38] Update doc submodule [nomail] [skip ci]

---
 doc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc b/doc
index 8522dbfadc..f13e396db3 160000
--- a/doc
+++ b/doc
@@ -1 +1 @@
-Subproject commit 8522dbfadcdc3adf3eca9a068e4b2a10cfd06ea9
+Subproject commit f13e396db352f3c8ad1d8035fa2dd9e5f0b8ba65

From 9ad47a31413a0de7dd0e873d681d08981f2e5d9f Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Wed, 14 Oct 2020 10:46:08 -0700
Subject: [PATCH 38/38] Update submodule(s) [nomail]

---
 doc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc b/doc
index f13e396db3..5666bf7d6d 160000
--- a/doc
+++ b/doc
@@ -1 +1 @@
-Subproject commit f13e396db352f3c8ad1d8035fa2dd9e5f0b8ba65
+Subproject commit 5666bf7d6d1abad60c83fcdaaf2e43eb92958f77