Merge branch 'master' into topic/vern/cpp-prep-profiling

.cirrus.yml

@@ -161,6 +161,13 @@ debian9_32bit_task:
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
 
+opensuse_leap_15_2_task:
+  container:
+    # Opensuse Leap 15.2 EOL: Dec 2021
+    dockerfile: ci/opensuse-leap-15.2/Dockerfile
+    << : *RESOURCES_TEMPLATE
+  << : *CI_TEMPLATE
+
 ubuntu20_task:
   container:
     # Ubuntu 20.04 EOL: April 2025
@@ -182,6 +189,15 @@ ubuntu16_task:
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
 
+alpine_task:
+  container:
+    # Alpine releases typically happen every 6 months w/ support for 2 years.
+    # The Dockerfile simply tracks latest Alpine release and shouldn't
+    # generally need updating based on particular Alpine release timelines.
+    dockerfile: ci/alpine/Dockerfile
+    << : *RESOURCES_TEMPLATE
+  << : *CI_TEMPLATE
+
 # Apple doesn't publish official long-term support timelines.
 # We aim to support both the current and previous macOS release.
 macos_big_sur_task:

CHANGES

@@ -1,4 +1,133 @@
 
+4.1.0-dev.451 | 2021-03-31 11:58:08 -0700
+
+  * Add ssh to Alpine Dockerfile for retrieving external test repos (Jon Siwek, Corelight)
+
+4.1.0-dev.449 | 2021-03-31 10:47:22 -0700
+
+  * Change ci/init-external-repos.sh to use `base64 -d` (Jon Siwek, Corelight)
+
+    The BusyBox version of `base64` does not have `--decode`.
+
+4.1.0-dev.448 | 2021-03-31 10:29:17 -0700
+
+  * Teach ci/init-external-repos.sh to consider user permissions (Jon Siwek, Corelight)
+
+    Any errors while setting up external/private test repo should fail the
+    task for PRs submitted by a user with write/admin permission.
+
+4.1.0-dev.447 | 2021-03-31 09:55:05 -0700
+
+  * CI: Add OpenSUSE Leap 15.2 (Johanna Amann, Corelight)
+
+  * Add CI task for Alpine Linux (Jon Siwek, Corelight)
+
+  * Separate stdout from stderr in btest baselines (Jon Siwek, Corelight)
+
+    Redirecting both to the same file can show platform-specific differences
+    (e.g. Alpine), likely due to different buffering defaults.
+
+  * Remove newline-eof canonification attempt in diff-remove-timestamps (Jon Siwek, Corelight)
+
+  * Change a <sys/errno.h> include to <errno.h> (Jon Siwek, Corelight)
+
+    Some systems (e.g. Alpine) may warn that that the former is incorrect.
+
+4.1.0-dev.440 | 2021-03-30 14:57:07 -0700
+
+  * Fix incomplete-type for struct timeval (Andrew Benson)
+
+    This fixes building on musl (e.g. Void, Alpine, etc.)
+
+4.1.0-dev.438 | 2021-03-29 15:42:25 -0700
+
+  * "balance" tests with multiple Zeek scripts to load the same elements (Vern Paxson, Corelight)
+
+  * put global statements into a quasi-function to support script optimization (Vern Paxson, Corelight)
+
+4.1.0-dev.434 | 2021-03-29 13:18:18 -0700
+
+  * Fix sign-compare compiler warning in coerce_to_record() (Jon Siwek, Corelight)
+
+  * Fix maybe-uninitialized warning in ZVal::ToVal() (Jon Siwek, Corelight)
+
+  * Change RecordVal::GetFieldAs() to use std::vector::operator[] (Jon Siwek, Corelight)
+
+    Since the method claims it's up to the user to ensure the field exists
+    before calling, the extra bounds-checking done by std::vector::at()
+    isn't needed.
+
+  * Add RecordVal::AssignField() and use it in supervisor code (Jon Siwek, Corelight)
+
+    This is a convenience method to assign a known record field value by
+    field name.  May also be useful to reduce warnings from static analysis
+    (e.g. Coverity) about not checking for negative return values before
+    assigning since that now flows through a [[noreturn]] error path.
+
+  * GH-960: Fix include order of bundled header files (Jon Siwek, Corelight)
+
+    Previously, a system-wide installation of any bundled auxil/ software
+    (like CAF) may get found/included rather than the bundled version and
+    possibly break the build.
+
+4.1.0-dev.427 | 2021-03-27 14:18:16 -0700
+
+  * Update bundled CAF to 0.18.2 (Jon Siwek, Corelight)
+
+4.1.0-dev.426 | 2021-03-26 17:17:45 -0700
+
+  * GH-1463: Rename supervisor_rotation_format_func to archiver_rotation_format_func (Vlad Grigorescu)
+
+    And expose it for non-supervised setups.  Also deprecates the old name.
+
+4.1.0-dev.423 | 2021-03-26 16:44:26 -0700
+
+  * GH-1454: Heartbleed: fix substraction order. (Johanna Amann)
+
+    Fixes incorrect/overflowed `n` value for `SSL_Heartbeat_Many_Requests`
+    notices where number of server heartbeats is greater than number of client
+    heartbeats.
+
+    The larger number was substracted from the smaller one leading to an
+    integer overflow. However, no information was lost due to everything
+    also being present in the notice message.
+
+4.1.0-dev.421 | 2021-03-26 16:39:22 -0700
+
+  * Fix crash in Analyzer::ForwardPacket due to recursive analyzer calls. (Tim Wojtulewicz, Corelight)
+
+    The change in 44f558df7b5a85bae40945de653bcb2448e0a7f4 that made analyzer_list
+    a std::vector instead of a std::list doesn't take into account that in some
+    cases an analyzer may chain back into itself, such as with UDP-in-UDP tunnels.
+    In these cases, the second call to ForwardPacket may cause iterator
+    invalidation, leading to a crash, so this reverts back to using an std::list.
+
+  * Include git sha in request to benchmark host (Tim Wojtulewicz, Corelight)
+
+4.1.0-dev.417 | 2021-03-25 11:37:55 -0700
+
+  * test suite update due to factoring out coerce_to_record() (Vern Paxson, Corelight)
+
+  * removal of vestigial #include's (breaking an include loop in the process) (Vern Paxson, Corelight)
+
+  * better method name: HasCopySemantics() (Vern Paxson, Corelight)
+
+  * Use STL functionality to initialize coercion map (Vern Paxson, Corelight)
+
+  * comments for factored-out index slice functions (Vern Paxson, Corelight)
+
+  * support for subclassing ScriptFunc's, esp. for alternate lambda closures (Vern Paxson, Corelight)
+
+  * factor out record coercion; modernize management of coercion "map" (Vern Paxson, Corelight)
+
+  * lower-level method for adding fields to records (Vern Paxson, Corelight)
+
+  * factor out "print" statement's execution functionality (Vern Paxson, Corelight)
+
+  * functions for indexing slices and strings (Vern Paxson, Corelight)
+
+  * new function for getting the location, if any, associated with the current call (Vern Paxson, Corelight)
+
 4.1.0-dev.404 | 2021-03-24 16:58:50 -0700
 
   * Fix missing `user_agent` existence check in smtp/software.zeek (Michael Dopheide)

CMakeLists.txt

@@ -376,9 +376,6 @@ endif ()
 # we have to care about CAF here because Broker headers can pull in CAF
 # headers.
 set(zeekdeps ${zeekdeps} ${CAF_LIBRARIES})
-include_directories(BEFORE ${broker_includes} ${CAF_INCLUDE_DIRS})
-include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/auxil/paraglob/include)
-include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/auxil/rapidjson/include)
 include_directories(BEFORE
                     ${PCAP_INCLUDE_DIR}
                     ${BIND_INCLUDE_DIR}
@@ -441,9 +438,6 @@ endif ()
 # if one specifies --with-openssl (which may be common).
 include_directories(BEFORE ${OPENSSL_INCLUDE_DIR})
 
-# Make everyone find the highwayhash includes
-include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/auxil/highwayhash)
-
 # Determine if libfts is external to libc, i.e. musl
 find_package(FTS)
 if ( FTS_FOUND )
@@ -451,6 +445,14 @@ if ( FTS_FOUND )
     include_directories(BEFORE ${FTS_INCLUDE_DIR})
 endif ()
 
+# Any headers that are possibly bundled in the Zeek source-tree and that are supposed
+# to have priority over any pre-existing/system-wide headers need to appear early in
+# compiler search path.
+include_directories(BEFORE ${broker_includes} ${CAF_INCLUDE_DIRS})
+include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/auxil/highwayhash)
+include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/auxil/paraglob/include)
+include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/auxil/rapidjson/include)
+
 set(zeekdeps ${zeekdeps}
     ${BinPAC_LIBRARY}
     ${PCAP_LIBRARY}

NEWS

@@ -49,11 +49,6 @@ New Functionality
 Changed Functionality
 ---------------------
 
-- The ``zeek::analyzer::analyzer_list`` type-alias changed from an
-  ``std::list`` to ``std::vector`` which, in practice, is not expected to be
-  used from plugins in API-incompatible way and may result in ~1-2% overall
-  performance benefit.
-
 Removed Functionality
 ---------------------
 
@@ -79,6 +74,8 @@ Deprecated Functionality
   environment variables prefixed by ``BRO_`` is now removed and calling
   ``getenv()`` directly with ``ZEEK_`` environment variables can be done.
 
+- ``supervisor_rotation_format_func`` is renamed to ``archiver_rotation_format_func``
+
 Zeek 4.0.0
 ==========
 

VERSION

@@ -1 +1 @@
-4.1.0-dev.404
+4.1.0-dev.451

auxil/broker

@@ -1 +1 @@
-Subproject commit 7810ec9936d1aa65c7c7d21b87e43c7d67481da0
+Subproject commit 938fd7de65220a986361ec596f2335ad471102d7

auxil/package-manager

@@ -1 +1 @@
-Subproject commit cd89d32f1f856544479e4c3d3d994eadf6759034
+Subproject commit ccdd3f305cb56040e212dab9ec9239a4bd0cc62b

ci/alpine/Dockerfile

@@ -0,0 +1,24 @@
+FROM alpine:latest
+
+RUN apk add --no-cache \
+  cmake \
+  make \
+  g++ \
+  python3 \
+  python3-dev \
+  flex \
+  bison \
+  libpcap-dev \
+  openssl-dev \
+  zlib-dev \
+  swig \
+  bash \
+  bsd-compat-headers \
+  linux-headers \
+  fts-dev \
+  git \
+  curl \
+  openssh-client \
+  py3-pip
+
+RUN pip3 install junit2html

ci/benchmark.sh

@@ -35,7 +35,7 @@ TARGET="https://${ZEEK_BENCHMARK_HOST}:${ZEEK_BENCHMARK_PORT}${ZEEK_BENCHMARK_EN
 set +e
 
 # Make a request to the benchmark host.
-RESULTS=$(curl -sS --stderr - --fail --insecure -X POST -H "Zeek-HMAC: ${HMAC_DIGEST}" -H "Zeek-HMAC-Timestamp: ${TIMESTAMP}" "${TARGET}?branch=${CIRRUS_BRANCH}&build=${BUILD_URL}&build_hash=${BUILD_HASH}")
+RESULTS=$(curl -sS --stderr - --fail --insecure -X POST -H "Zeek-HMAC: ${HMAC_DIGEST}" -H "Zeek-HMAC-Timestamp: ${TIMESTAMP}" "${TARGET}?branch=${CIRRUS_BRANCH}&build=${BUILD_URL}&build_hash=${BUILD_HASH}&commit=${CIRRUS_CHANGE_IN_REPO}")
 STATUS=$?
 
 # If we got a bad status back from the host, we want to make sure to mask the host

ci/init-external-repos.sh

@@ -39,14 +39,20 @@ if [[ -n "${CIRRUS_CI}" ]] && [[ ! -d zeek-testing-private ]]; then
     # the key is also available in PRs for people with write access to the
     # repo, so we can still try for those cases).
     if [[ -n "${CIRRUS_PR}" ]]; then
+        if [[ "${CIRRUS_USER_PERMISSION}" == "write" ]]; then
+            set -e
+        elif [[ "${CIRRUS_USER_PERMISSION}" == "admin" ]]; then
+            set -e
+        else
             set +e
+        fi
     else
         set -e
     fi
 
     banner "Trying to clone zeek-testing-private git repo"
     echo "${ZEEK_TESTING_PRIVATE_SSH_KEY}" > cirrus_key.b64
-    base64 --decode cirrus_key.b64 > cirrus_key
+    base64 -d cirrus_key.b64 > cirrus_key
     rm cirrus_key.b64
     chmod 600 cirrus_key
     git --version

ci/opensuse-leap-15.2/Dockerfile

@@ -0,0 +1,25 @@
+FROM opensuse/leap:15.2
+
+RUN zypper in -y \
+  cmake \
+  make \
+  gcc \
+  gcc-c++ \
+  python3 \
+  python3-devel \
+  flex \
+  bison \
+  libpcap-devel \
+  libopenssl-devel \
+  zlib-devel \
+  swig \
+  git \
+  curl \
+  python3-pip \
+  which \
+  gzip \
+  tar \
+  && rm -rf /var/cache/zypp
+
+
+RUN pip3 install junit2html

doc

@@ -1 +1 @@
-Subproject commit afd337f4b00477401514ae3dc9c20476229318dc
+Subproject commit 4bd4cf0233ba4d7585ce664f686eb4aff5ccb8dc

scripts/base/frameworks/cluster/nodes/logger.zeek

@@ -22,11 +22,10 @@ redef Log::default_rotation_interval = 1 hrs;
 ## Alarm summary mail interval.
 redef Log::default_mail_alarms_interval = 24 hrs;
 
-@if ( Supervisor::is_supervised() )
+## This function will rotate logs in a format compatible with zeek-archiver.
-
+## If you're using the Supervisor framework, this function will be used,
-redef Log::default_rotation_dir = "log-queue";
+## if not, you can set :zeek:see:`Log::rotation_format_func` to this function.
-
+function archiver_rotation_format_func(ri: Log::RotationFmtInfo): Log::RotationPath
-function supervisor_rotation_format_func(ri: Log::RotationFmtInfo): Log::RotationPath
 	{
 	local open_str = strftime(Log::default_rotation_date_format, ri$open);
 	local close_str = strftime(Log::default_rotation_date_format, ri$close);
@@ -35,7 +34,13 @@ function supervisor_rotation_format_func(ri: Log::RotationFmtInfo): Log::Rotatio
 	return rval;
 	}
 
-redef Log::rotation_format_func = supervisor_rotation_format_func;
+@if ( Supervisor::is_supervised() )
+
+global supervisor_rotation_format_func = archiver_rotation_format_func &deprecated="Remove in v5.1.  Use 'archiver_rotation_format_func'.";
+
+redef Log::default_rotation_dir = "log-queue";
+
+redef Log::rotation_format_func = archiver_rotation_format_func;
 
 redef LogAscii::enable_leftover_log_rotation = T;
 

scripts/policy/protocols/ssl/heartbleed.zeek

@@ -154,7 +154,7 @@ event ssl_encrypted_heartbeat(c: connection, is_orig: bool, length: count)
 			NOTICE([$note=SSL_Heartbeat_Many_Requests,
 				$msg=fmt("Server sending more heartbeat responses than requests seen. Possible attack. Client count: %d, server count: %d", c$ssl$originator_heartbeats, c$ssl$responder_heartbeats),
 				$conn=c,
-				$n=(c$ssl$originator_heartbeats-c$ssl$responder_heartbeats),
+				$n=(c$ssl$responder_heartbeats-c$ssl$originator_heartbeats),
 				$identifier=fmt("%s%d", c$uid, c$ssl$responder_heartbeats/1000) # re-throw every 1000 heartbeats
 				]);
 

src/Expr.cc

@@ -2936,24 +2936,7 @@ ValPtr IndexExpr::Fold(Val* v1, Val* v2) const
 		if ( lv->Length() == 1 )
 			v = vect->ValAt(lv->Idx(0)->CoerceToUnsigned());
 		else
-			{
+			return index_slice(vect, lv);
-			size_t len = vect->Size();
-			auto result = make_intrusive<VectorVal>(vect->GetType<VectorType>());
-
-			bro_int_t first = get_slice_index(lv->Idx(0)->CoerceToInt(), len);
-			bro_int_t last = get_slice_index(lv->Idx(1)->CoerceToInt(), len);
-			bro_int_t sub_length = last - first;
-
-			if ( sub_length >= 0 )
-				{
-				result->Resize(sub_length);
-
-				for ( bro_int_t idx = first; idx < last; idx++ )
-					result->Assign(idx - first, vect->ValAt(idx));
-				}
-
-			return result;
-			}
 		}
 		break;
 
@@ -2962,9 +2945,22 @@ ValPtr IndexExpr::Fold(Val* v1, Val* v2) const
 		break;
 
 	case TYPE_STRING:
+		return index_string(v1->AsString(), v2->AsListVal());
+
+	default:
+		RuntimeError("type cannot be indexed");
+		break;
+	}
+
+	if ( v )
+		return v;
+
+	RuntimeError("no such index");
+	return nullptr;
+	}
+
+StringValPtr index_string(const String* s, const ListVal* lv)
 	{
-		const ListVal* lv = v2->AsListVal();
-		const String* s = v1->AsString();
 	int len = s->Len();
 	String* substring = nullptr;
 
@@ -2993,16 +2989,31 @@ ValPtr IndexExpr::Fold(Val* v1, Val* v2) const
 	return make_intrusive<StringVal>(substring ? substring : new String(""));
 	}
 
-	default:
+VectorValPtr index_slice(VectorVal* vect, const ListVal* lv)
-		RuntimeError("type cannot be indexed");
+	{
-		break;
+	auto first = lv->Idx(0)->CoerceToInt();
+	auto last = lv->Idx(1)->CoerceToInt();
+	return index_slice(vect, first, last);
 	}
 
-	if ( v )
+VectorValPtr index_slice(VectorVal* vect, int _first, int _last)
-		return v;
+	{
+	size_t len = vect->Size();
+	auto result = make_intrusive<VectorVal>(vect->GetType<VectorType>());
 
-	RuntimeError("no such index");
+	bro_int_t first = get_slice_index(_first, len);
-	return nullptr;
+	bro_int_t last = get_slice_index(_last, len);
+	bro_int_t sub_length = last - first;
+
+	if ( sub_length >= 0 )
+		{
+		result->Resize(sub_length);
+
+		for ( bro_int_t idx = first; idx < last; idx++ )
+			result->Assign(idx - first, vect->ValAt(idx));
+		}
+
+	return result;
 	}
 
 void IndexExpr::Assign(Frame* f, ValPtr v)
@@ -3748,8 +3759,7 @@ ValPtr ArithCoerceExpr::Fold(Val* v) const
 	}
 
 RecordCoerceExpr::RecordCoerceExpr(ExprPtr arg_op, RecordTypePtr r)
-	: UnaryExpr(EXPR_RECORD_COERCE, std::move(arg_op)),
+	: UnaryExpr(EXPR_RECORD_COERCE, std::move(arg_op))
-	  map(nullptr), map_size(0)
 	{
 	if ( IsError() )
 		return;
@@ -3767,13 +3777,10 @@ RecordCoerceExpr::RecordCoerceExpr(ExprPtr arg_op, RecordTypePtr r)
 		RecordType* t_r = type->AsRecordType();
 		RecordType* sub_r = op->GetType()->AsRecordType();
 
-		map_size = t_r->NumFields();
+		int map_size = t_r->NumFields();
-		map = new int[map_size];
+		map.resize(map_size, -1);	// -1 = field is not mapped
 
 		int i;
-		for ( i = 0; i < map_size; ++i )
-			map[i] = -1;	// -1 = field is not mapped
-
 		for ( i = 0; i < sub_r->NumFields(); ++i )
 			{
 			int t_i = t_r->FieldOffset(sub_r->FieldName(i));
@@ -3854,11 +3861,6 @@ RecordCoerceExpr::RecordCoerceExpr(ExprPtr arg_op, RecordTypePtr r)
 		}
 	}
 
-RecordCoerceExpr::~RecordCoerceExpr()
-	{
-	delete [] map;
-	}
-
 ValPtr RecordCoerceExpr::InitVal(const zeek::Type* t, ValPtr aggr) const
 	{
 	if ( auto v = Eval(nullptr) )
@@ -3881,7 +3883,15 @@ ValPtr RecordCoerceExpr::Fold(Val* v) const
 	if ( same_type(GetType(), Op()->GetType()) )
 		return IntrusivePtr{NewRef{}, v};
 
-	auto val = make_intrusive<RecordVal>(GetType<RecordType>());
+	auto rt = cast_intrusive<RecordType>(GetType());
+	return coerce_to_record(rt, v, map);
+	}
+
+RecordValPtr coerce_to_record(RecordTypePtr rt, Val* v,
+				const std::vector<int>& map)
+	{
+	int map_size = map.size();
+	auto val = make_intrusive<RecordVal>(rt);
 	RecordType* val_type = val->GetType()->AsRecordType();
 
 	RecordVal* rv = v->AsRecordVal();
@@ -3894,14 +3904,15 @@ ValPtr RecordCoerceExpr::Fold(Val* v) const
 
 			if ( ! rhs )
 				{
-				const auto& def = rv->GetType()->AsRecordType()->FieldDecl(
+				auto rv_rt = rv->GetType()->AsRecordType();
-					map[i])->GetAttr(ATTR_DEFAULT);
+				const auto& def = rv_rt->FieldDecl(map[i])->
+							GetAttr(ATTR_DEFAULT);
 
 				if ( def )
 					rhs = def->GetExpr()->Eval(nullptr);
 				}
 
-			assert(rhs || GetType()->AsRecordType()->FieldDecl(i)->GetAttr(ATTR_OPTIONAL));
+			assert(rhs || rt->FieldDecl(i)->GetAttr(ATTR_OPTIONAL));
 
 			if ( ! rhs )
 				{
@@ -3923,21 +3934,19 @@ ValPtr RecordCoerceExpr::Fold(Val* v) const
 			else if ( BothArithmetic(rhs_type->Tag(), field_type->Tag()) &&
 			          ! same_type(rhs_type, field_type) )
 				{
-				if ( auto new_val = check_and_promote(rhs, field_type.get(), false, op->GetLocationInfo()) )
+				auto new_val = check_and_promote(rhs, field_type.get(), false);
 				rhs = std::move(new_val);
-				else
-					RuntimeError("Failed type conversion");
 				}
 
 			val->Assign(i, std::move(rhs));
 			}
 		else
 			{
-			if ( const auto& def = GetType()->AsRecordType()->FieldDecl(i)->GetAttr(ATTR_DEFAULT) )
+			if ( const auto& def = rt->FieldDecl(i)->GetAttr(ATTR_DEFAULT) )
 				{
 				auto def_val = def->GetExpr()->Eval(nullptr);
 				const auto& def_type = def_val->GetType();
-				const auto& field_type = GetType()->AsRecordType()->GetFieldType(i);
+				const auto& field_type = rt->GetFieldType(i);
 
 				if ( def_type->Tag() == TYPE_RECORD &&
 				     field_type->Tag() == TYPE_RECORD &&

src/Expr.h

@@ -941,6 +941,22 @@ protected:
 	bool is_slice;
 };
 
+// The following execute the heart of IndexExpr functionality for
+// vector slices and strings.
+
+// Extracts a slice of a vector, where the span of the slice is specified
+// by a list of (exactly) two values.  This is how the interpreter develops
+// the components of a slice.
+extern VectorValPtr index_slice(VectorVal* vect, const ListVal* lv);
+
+// Lower-level access to the slice, where its span is expressed
+// directly as integers.
+extern VectorValPtr index_slice(VectorVal* vect, int first, int last);
+
+// Returns a subset of a string, with the span specified by a list of
+// (exactly) two values.
+extern StringValPtr index_string(const String* s, const ListVal* lv);
+
 class IndexExprWhen final : public IndexExpr {
 public:
 	static inline std::vector<ValPtr> results = {};
@@ -1166,21 +1182,24 @@ protected:
 class RecordCoerceExpr final : public UnaryExpr {
 public:
 	RecordCoerceExpr(ExprPtr op, RecordTypePtr r);
-	~RecordCoerceExpr() override;
 
 	// Optimization-related:
 	ExprPtr Duplicate() override;
 
+	const std::vector<int>& Map() const	{ return map; }
+
 protected:
 	ValPtr InitVal(const zeek::Type* t, ValPtr aggr) const override;
 	ValPtr Fold(Val* v) const override;
 
 	// For each super-record slot, gives subrecord slot with which to
 	// fill it.
-	int* map;
+	std::vector<int> map;
-	int map_size;	// equivalent to Type()->AsRecordType()->NumFields()
 };
 
+extern RecordValPtr coerce_to_record(RecordTypePtr rt, Val* v,
+					const std::vector<int>& map);
+
 class TableCoerceExpr final : public UnaryExpr {
 public:
 	TableCoerceExpr(ExprPtr op, TableTypePtr r);

src/File.h

@@ -12,8 +12,7 @@
 #include <krb5.h>
 #endif // NEED_KRB5_H
 
-#include "zeek/Obj.h"
+#include "zeek/Val.h"
-#include "zeek/IntrusivePtr.h"
 #include "zeek/util.h"
 
 namespace zeek {
@@ -23,6 +22,8 @@ namespace detail {
 class PrintStmt;
 class Attributes;
 
+extern void do_print_stmt(const std::vector<ValPtr>& vals);
+
 } // namespace detail;
 
 class RecordVal;
@@ -84,7 +85,7 @@ public:
 
 protected:
 
-	friend class detail::PrintStmt;
+	friend void detail::do_print_stmt(const std::vector<ValPtr>& vals);
 
 	File()	{ Init(); }
 	void Init();

src/Frame.cc

@@ -616,6 +616,11 @@ void Frame::CaptureClosure(Frame* c, IDPList arg_outer_ids)
 	// if (c) closure = c->SelectiveClone(outer_ids);
 	}
 
+const detail::Location* Frame::GetCallLocation() const
+	{
+	return call ? call->GetLocationInfo() : nullptr;
+	}
+
 void Frame::SetTrigger(trigger::TriggerPtr arg_trigger)
 	{
 	trigger = std::move(arg_trigger);

src/Frame.h

@@ -262,6 +262,7 @@ public:
 	void SetCall(const CallExpr* arg_call)	{ call = arg_call; }
 	void ClearCall()			{ call = nullptr; }
 	const CallExpr* GetCall() const		{ return call; }
+	const detail::Location* GetCallLocation() const;
 
 	void SetDelayed()	{ delayed = true; }
 	bool HasDelayed() const	{ return delayed; }

src/Func.cc

@@ -587,6 +587,11 @@ bool ScriptFunc::StrengthenClosureReference(Frame* f)
 	return true;
 	}
 
+bool ScriptFunc::HasCopySemantics() const
+	{
+	return type->GetCaptures().has_value();
+	}
+
 void ScriptFunc::SetClosureFrame(Frame* f)
 	{
 	if ( closure )

src/Func.h

@@ -145,7 +145,7 @@ protected:
 
 namespace detail {
 
-class ScriptFunc final : public Func {
+class ScriptFunc : public Func {
 public:
 	ScriptFunc(const IDPtr& id, StmtPtr body,
 	        const std::vector<IDPtr>& inits,
@@ -210,12 +210,17 @@ public:
 	 */
 	bool StrengthenClosureReference(Frame* f);
 
+	/**
+	 * Whether the function's closure uses copy semantics.
+	 */
+	virtual bool HasCopySemantics() const;
+
 	/**
 	 * Serializes this function's closure or capture frame.
 	 *
 	 * @return a serialized version of the function's closure/capture frame.
 	 */
-	broker::expected<broker::data> SerializeClosure() const;
+	virtual broker::expected<broker::data> SerializeClosure() const;
 
 	/**
 	 * Sets the captures frame to one built from *data*.
@@ -257,6 +262,7 @@ public:
 
 protected:
 	ScriptFunc() : Func(SCRIPT_FUNC)	{}
+
 	StmtPtr AddInits(
 		StmtPtr body,
 		const std::vector<IDPtr>& inits);
@@ -280,7 +286,7 @@ protected:
 	 *
 	 * @param f  the frame holding the values of capture variables
 	 */
-	void SetCaptures(Frame* f);
+	virtual void SetCaptures(Frame* f);
 
 private:
 	size_t frame_size;

src/ScannedFile.cc

@@ -1,6 +1,6 @@
 #include "zeek/ScannedFile.h"
 
-#include <sys/errno.h>
+#include <errno.h>
 #include <limits.h> // for PATH_MAX
 
 #include "zeek/DebugLogger.h"

src/Stmt.cc

@@ -306,18 +306,23 @@ ValPtr PrintStmt::DoExec(std::vector<ValPtr> vals,
                          StmtFlowType& /* flow */)
 	{
 	RegisterAccess();
+	do_print_stmt(vals);
+	return nullptr;
+	}
 
+void do_print_stmt(const std::vector<ValPtr>& vals)
+	{
 	if ( ! print_stdout )
 		print_stdout = new File(stdout);
 
 	File* f = print_stdout;
 	int offset = 0;
 
-	if ( vals.size() > 0 && (vals)[0]->GetType()->Tag() == TYPE_FILE )
+	if ( vals.size() > 0 && vals[0] && vals[0]->GetType()->Tag() == TYPE_FILE )
 		{
 		f = (vals)[0]->AsFile();
 		if ( ! f->IsOpen() )
-			return nullptr;
+			return;
 
 		++offset;
 		}
@@ -331,7 +336,7 @@ ValPtr PrintStmt::DoExec(std::vector<ValPtr> vals,
 	case BifEnum::Log::REDIRECT_ALL:
 		{
 		print_log(vals);
-		return nullptr;
+		return;
 		}
 	case BifEnum::Log::REDIRECT_STDOUT:
 		if ( f->FileHandle() == stdout )
@@ -339,7 +344,7 @@ ValPtr PrintStmt::DoExec(std::vector<ValPtr> vals,
 			// Should catch even printing to a "manually opened" stdout file,
 			// like "/dev/stdout" or "-".
 			print_log(vals);
-			return nullptr;
+			return;
 			}
 		break;
 	default:
@@ -368,8 +373,6 @@ ValPtr PrintStmt::DoExec(std::vector<ValPtr> vals,
 		describe_vals(vals, &d, offset);
 		f->Write("\n", 1);
 		}
-
-	return nullptr;
 	}
 
 ExprStmt::ExprStmt(ExprPtr arg_e) : Stmt(STMT_EXPR), e(std::move(arg_e))

src/Stmt.h

@@ -62,6 +62,8 @@ protected:
 	StmtPtr DoSubclassReduce(ListExprPtr singletons, Reducer* c) override;
 };
 
+extern void do_print_stmt(const std::vector<ValPtr>& vals);
+
 class ExprStmt : public Stmt {
 public:
 	explicit ExprStmt(ExprPtr e);

src/Type.cc

@@ -1043,6 +1043,17 @@ const char* RecordType::AddFields(const type_decl_list& others,
 
 	TableVal::SaveParseTimeTableState(this);
 
+	AddFieldsDirectly(others, add_log_attr);
+
+	RecordVal::ResizeParseTimeRecords(this);
+	TableVal::RebuildParseTimeTables();
+
+	return nullptr;
+	}
+
+void RecordType::AddFieldsDirectly(const type_decl_list& others,
+					bool add_log_attr)
+	{
 	for ( const auto& td : others )
 		{
 		if ( add_log_attr )
@@ -1059,9 +1070,6 @@ const char* RecordType::AddFields(const type_decl_list& others,
 		}
 
 	num_fields = types->length();
-	RecordVal::ResizeParseTimeRecords(this);
-	TableVal::RebuildParseTimeTables();
-	return nullptr;
 	}
 
 void RecordType::DescribeFields(ODesc* d) const

src/Type.h

@@ -623,6 +623,9 @@ public:
 	const char* AddFields(const type_decl_list& types,
 	                      bool add_log_attr = false);
 
+	void AddFieldsDirectly(const type_decl_list& types,
+				bool add_log_attr = false);
+
 	void Describe(ODesc* d) const override;
 	void DescribeReST(ODesc* d, bool roles_only = false) const override;
 	void DescribeFields(ODesc* d) const;

src/Val.h

@@ -1152,6 +1152,19 @@ public:
 	void Assign(int field, String* new_val)
 		{ Assign(field, new StringVal(new_val)); }
 
+	/**
+	 * Assign a value of type @c T to a record field of the given name.
+	 * A fatal error occurs if the no such field name exists.
+	 */
+	template <class T>
+	void AssignField(const char* field_name, T&& val)
+		{
+		int idx = GetType()->AsRecordType()->FieldOffset(field_name);
+		if ( idx < 0 )
+			reporter->InternalError("missing record field: %s", field_name);
+		Assign(idx, std::forward<T>(val));
+		}
+
 	/**
 	 * Appends a value to the record's fields.  The caller is responsible
 	 * for ensuring that fields are appended in the correct order and
@@ -1288,33 +1301,33 @@ public:
 		if constexpr ( std::is_same_v<T, BoolVal> ||
 		               std::is_same_v<T, IntVal> ||
 		               std::is_same_v<T, EnumVal> )
-			return record_val->at(field).int_val;
+			return record_val->operator[](field).int_val;
 		else if constexpr ( std::is_same_v<T, CountVal> )
-			return record_val->at(field).uint_val;
+			return record_val->operator[](field).uint_val;
 		else if constexpr ( std::is_same_v<T, DoubleVal> ||
 		                    std::is_same_v<T, TimeVal> ||
 		                    std::is_same_v<T, IntervalVal> )
-			return record_val->at(field).double_val;
+			return record_val->operator[](field).double_val;
 		else if constexpr ( std::is_same_v<T, PortVal> )
 			return val_mgr->Port(record_val->at(field).uint_val);
 		else if constexpr ( std::is_same_v<T, StringVal> )
-			return record_val->at(field).string_val->Get();
+			return record_val->operator[](field).string_val->Get();
 		else if constexpr ( std::is_same_v<T, AddrVal> )
-			return record_val->at(field).addr_val->Get();
+			return record_val->operator[](field).addr_val->Get();
 		else if constexpr ( std::is_same_v<T, SubNetVal> )
-			return record_val->at(field).subnet_val->Get();
+			return record_val->operator[](field).subnet_val->Get();
 		else if constexpr ( std::is_same_v<T, File> )
-			return *(record_val->at(field).file_val);
+			return *(record_val->operator[](field).file_val);
 		else if constexpr ( std::is_same_v<T, Func> )
-			return *(record_val->at(field).func_val);
+			return *(record_val->operator[](field).func_val);
 		else if constexpr ( std::is_same_v<T, PatternVal> )
-			return record_val->at(field).re_val->Get();
+			return record_val->operator[](field).re_val->Get();
 		else if constexpr ( std::is_same_v<T, RecordVal> )
-			return record_val->at(field).record_val;
+			return record_val->operator[](field).record_val;
 		else if constexpr ( std::is_same_v<T, VectorVal> )
-			return record_val->at(field).vector_val;
+			return record_val->operator[](field).vector_val;
 		else if constexpr ( std::is_same_v<T, TableVal> )
-			return record_val->at(field).table_val->Get();
+			return record_val->operator[](field).table_val->Get();
 		else
 			{
 			// It's an error to reach here, although because of
@@ -1329,12 +1342,12 @@ public:
 	T GetFieldAs(int field) const
 		{
 		if constexpr ( std::is_integral_v<T> && std::is_signed_v<T> )
-			return record_val->at(field).int_val;
+			return record_val->operator[](field).int_val;
 		else if constexpr ( std::is_integral_v<T> &&
 					std::is_unsigned_v<T> )
-			return record_val->at(field).uint_val;
+			return record_val->operator[](field).uint_val;
 		else if constexpr ( std::is_floating_point_v<T> )
-			return record_val->at(field).double_val;
+			return record_val->operator[](field).double_val;
 
 		// Note: we could add other types here using type traits,
 		// such as is_same_v<T, std::string>, etc.

src/ZVal.cc

@@ -5,7 +5,6 @@
 #include "zeek/Func.h"
 #include "zeek/OpaqueVal.h"
 #include "zeek/Reporter.h"
-#include "zeek/Desc.h"
 
 using namespace zeek;
 
@@ -264,8 +263,9 @@ ValPtr ZVal::ToVal(const TypePtr& t) const
 	case TYPE_TIMER:
 	case TYPE_UNION:
 	case TYPE_VOID:
+	default:
 		v = nullptr;
-		reporter->InternalError("bad ret type return tag");
+		reporter->InternalError("bad type in ZVal::ToVal: %s", type_name(t->Tag()));
 	}
 
 	if ( v )

src/ZVal.h

@@ -6,11 +6,6 @@
 
 #include "zeek/zeek-config.h"
 
-#include <unordered_set>
-
-#include "zeek/Dict.h"
-#include "zeek/Expr.h"
-
 namespace zeek {
 
 class StringVal;

src/analyzer/Analyzer.h

@@ -38,7 +38,12 @@ class AnalyzerTimer;
 class SupportAnalyzer;
 class OutputHandler;
 
-using analyzer_list = std::vector<Analyzer*>;
+// This needs to remain a std::list because of the usage of iterators in the
+// Analyzer::Forward methods. These methods have the chance to loop back
+// into the same analyzer in the case of tunnels. If the recursive call adds
+// to the children list, it can invalidate iterators in the outer call,
+// causing a crash.
+using analyzer_list = std::list<Analyzer*>;
 typedef uint32_t ID;
 typedef void (Analyzer::*analyzer_timer_func)(double t);
 

src/broker/Data.cc

@@ -392,6 +392,17 @@ struct val_converter {
 
 			if ( a.size() == 2 ) // we have a closure/capture frame
 				{
+				// Note, seems if we already have a separate
+				// instance of the same lambda, then unless
+				// we use a cloned value, we'll step on that
+				// one's captures, too.  This is because
+				// the capture mapping lives with the Func
+				// object rather than the FuncVal.  However,
+				// we can't readily Clone() here because
+				// rval is const (and, grrr, Clone() is not).
+				// -VP
+				// rval = rval->Clone();
+
 				auto frame = broker::get_if<broker::vector>(a[1]);
 				if ( ! frame )
 					return nullptr;
@@ -400,9 +411,7 @@ struct val_converter {
 				if ( ! b )
 					return nullptr;
 
-				auto copy_semantics = b->GetType()->GetCaptures().has_value();
+				if ( b->HasCopySemantics() )
-
-				if ( copy_semantics )
 					{
 					if ( ! b->DeserializeCaptures(*frame) )
 						return nullptr;
@@ -1177,7 +1186,7 @@ broker::data& opaque_field_to_data(RecordVal* v, zeek::detail::Frame* f)
 	const auto& d = v->GetField(0);
 
 	if ( ! d )
-		reporter->RuntimeError(f->GetCall()->GetLocationInfo(),
+		reporter->RuntimeError(f->GetCallLocation(),
 		                             "Broker::Data's opaque field is not set");
 
 	// RuntimeError throws an exception which causes this line to never exceute.

src/broker/Data.h

@@ -194,7 +194,7 @@ T& require_data_type(broker::data& d, zeek::TypeTag tag, zeek::detail::Frame* f)
 	{
 	auto ptr = caf::get_if<T>(&d);
 	if ( ! ptr )
-		zeek::reporter->RuntimeError(f->GetCall()->GetLocationInfo(),
+		zeek::reporter->RuntimeError(f->GetCallLocation(),
 		                             "data is of type '%s' not of type '%s'",
 		                             caf::visit(type_name_getter{tag}, d),
 		                             zeek::type_name(tag));

src/broker/Manager.cc

@@ -74,7 +74,7 @@ int Manager::script_scope = 0;
 struct scoped_reporter_location {
 	scoped_reporter_location(zeek::detail::Frame* frame)
 		{
-		reporter->PushLocation(frame->GetCall()->GetLocationInfo());
+		reporter->PushLocation(frame->GetCallLocation());
 		}
 
 	~scoped_reporter_location()

src/parse.y

@@ -266,7 +266,7 @@ static bool expr_is_table_type_name(const zeek::detail::Expr* expr)
 
 %%
 
-bro:
+zeek:
 		decl_list stmt_list
 			{
 			if ( zeek::detail::stmts )

src/reporter.bif

@@ -22,7 +22,7 @@ module Reporter;
 ## .. zeek:see:: reporter_info
 function Reporter::info%(msg: string%): bool
 	%{
-	reporter->PushLocation(frame->GetCall()->GetLocationInfo());
+	reporter->PushLocation(frame->GetCallLocation());
 	reporter->Info("%s", msg->CheckString());
 	reporter->PopLocation();
 	return zeek::val_mgr->True();
@@ -37,7 +37,7 @@ function Reporter::info%(msg: string%): bool
 ## .. zeek:see:: reporter_warning
 function Reporter::warning%(msg: string%): bool
 	%{
-	reporter->PushLocation(frame->GetCall()->GetLocationInfo());
+	reporter->PushLocation(frame->GetCallLocation());
 	reporter->Warning("%s", msg->CheckString());
 	reporter->PopLocation();
 	return zeek::val_mgr->True();
@@ -53,7 +53,7 @@ function Reporter::warning%(msg: string%): bool
 ## .. zeek:see:: reporter_error
 function Reporter::error%(msg: string%): bool
 	%{
-	reporter->PushLocation(frame->GetCall()->GetLocationInfo());
+	reporter->PushLocation(frame->GetCallLocation());
 	reporter->Error("%s", msg->CheckString());
 	reporter->PopLocation();
 	return zeek::val_mgr->True();
@@ -66,7 +66,7 @@ function Reporter::error%(msg: string%): bool
 ## Returns: Always true.
 function Reporter::fatal%(msg: string%): bool
 	%{
-	reporter->PushLocation(frame->GetCall()->GetLocationInfo());
+	reporter->PushLocation(frame->GetCallLocation());
 	reporter->FatalError("%s", msg->CheckString());
 	reporter->PopLocation();
 	return zeek::val_mgr->True();
@@ -80,7 +80,7 @@ function Reporter::fatal%(msg: string%): bool
 ## Returns: Always true.
 function Reporter::fatal_error_with_core%(msg: string%): bool
 	%{
-	reporter->PushLocation(frame->GetCall()->GetLocationInfo());
+	reporter->PushLocation(frame->GetCallLocation());
 	reporter->FatalErrorWithCore("%s", msg->CheckString());
 	reporter->PopLocation();
 	return zeek::val_mgr->True();

src/script_opt/ScriptOpt.cc

@@ -3,6 +3,7 @@
 #include "zeek/Options.h"
 #include "zeek/Reporter.h"
 #include "zeek/Desc.h"
+#include "zeek/module_util.h"
 #include "zeek/script_opt/ScriptOpt.h"
 #include "zeek/script_opt/ProfileFunc.h"
 #include "zeek/script_opt/Inline.h"
@@ -153,6 +154,27 @@ void analyze_func(ScriptFuncPtr f)
 	funcs.emplace_back(f, ScopePtr{NewRef{}, f->GetScope()}, f->CurrentBody());
 	}
 
+const FuncInfo* analyze_global_stmts(Stmt* stmts)
+	{
+	// We ignore analysis_options.only_func - if it's in use, later
+	// logic will keep this function from being compiled, but it's handy
+	// now to enter it into "funcs" so we have a FuncInfo to return.
+
+	auto id = install_ID("<global-stmts>", GLOBAL_MODULE_NAME, true, false);
+	auto empty_args_t = make_intrusive<RecordType>(nullptr);
+	auto func_t = make_intrusive<FuncType>(empty_args_t, nullptr, FUNC_FLAVOR_FUNCTION);
+	id->SetType(func_t);
+
+	auto sc = current_scope();
+	std::vector<IDPtr> empty_inits;
+	StmtPtr stmts_p{NewRef{}, stmts};
+	auto sf = make_intrusive<ScriptFunc>(id, stmts_p, empty_inits, sc->Length(), 0);
+
+	funcs.emplace_back(sf, ScopePtr{NewRef{}, sc}, stmts_p);
+
+	return &funcs.back();
+	}
+
 static void check_env_opt(const char* opt, bool& opt_flag)
 	{
 	if ( getenv(opt) )

src/script_opt/ScriptOpt.h

@@ -106,6 +106,11 @@ extern std::unordered_set<const Func*> non_recursive_funcs;
 // Analyze a given function for optimization.
 extern void analyze_func(ScriptFuncPtr f);
 
+// Analyze the given top-level statement(s) for optimization.  Returns
+// a pointer to a FuncInfo for an argument-less quasi-function that can
+// be Invoked, or its body executed directly, to execute the statements.
+extern const FuncInfo* analyze_global_stmts(Stmt* stmts);
+
 // Analyze all of the parsed scripts collectively for optimization.
 extern void analyze_scripts();
 

src/supervisor/Supervisor.cc

@@ -1348,22 +1348,22 @@ RecordValPtr Supervisor::NodeConfig::ToRecord() const
 	{
 	const auto& rt = BifType::Record::Supervisor::NodeConfig;
 	auto rval = make_intrusive<RecordVal>(rt);
-	rval->Assign(rt->FieldOffset("name"), name);
+	rval->AssignField("name", name);
 
 	if ( interface )
-		rval->Assign(rt->FieldOffset("interface"), *interface);
+		rval->AssignField("interface", *interface);
 
 	if ( directory )
-		rval->Assign(rt->FieldOffset("directory"), *directory);
+		rval->AssignField("directory", *directory);
 
 	if ( stdout_file )
-		rval->Assign(rt->FieldOffset("stdout_file"), *stdout_file);
+		rval->AssignField("stdout_file", *stdout_file);
 
 	if ( stderr_file )
-		rval->Assign(rt->FieldOffset("stderr_file"), *stderr_file);
+		rval->AssignField("stderr_file", *stderr_file);
 
 	if ( cpu_affinity )
-		rval->Assign(rt->FieldOffset("cpu_affinity"), *cpu_affinity);
+		rval->AssignField("cpu_affinity", *cpu_affinity);
 
 	auto st = rt->GetFieldType<VectorType>("scripts");
 	auto scripts_val = make_intrusive<VectorVal>(std::move(st));
@@ -1371,11 +1371,11 @@ RecordValPtr Supervisor::NodeConfig::ToRecord() const
 	for ( const auto& s : scripts )
 		scripts_val->Assign(scripts_val->Size(), make_intrusive<StringVal>(s));
 
-	rval->Assign(rt->FieldOffset("scripts"), std::move(scripts_val));
+	rval->AssignField("scripts", std::move(scripts_val));
 
 	auto tt = rt->GetFieldType<TableType>("cluster");
 	auto cluster_val = make_intrusive<TableVal>(std::move(tt));
-	rval->Assign(rt->FieldOffset("cluster"), cluster_val);
+	rval->AssignField("cluster", cluster_val);
 
 	for ( const auto& e : cluster )
 		{
@@ -1385,12 +1385,12 @@ RecordValPtr Supervisor::NodeConfig::ToRecord() const
 		const auto& ept = BifType::Record::Supervisor::ClusterEndpoint;
 		auto val = make_intrusive<RecordVal>(ept);
 
-		val->Assign(ept->FieldOffset("role"), BifType::Enum::Supervisor::ClusterRole->GetEnumVal(ep.role));
+		val->AssignField("role", BifType::Enum::Supervisor::ClusterRole->GetEnumVal(ep.role));
-		val->Assign(ept->FieldOffset("host"), make_intrusive<AddrVal>(ep.host));
+		val->AssignField("host", make_intrusive<AddrVal>(ep.host));
-		val->Assign(ept->FieldOffset("p"), val_mgr->Port(ep.port, TRANSPORT_TCP));
+		val->AssignField("p", val_mgr->Port(ep.port, TRANSPORT_TCP));
 
 		if ( ep.interface )
-			val->Assign(ept->FieldOffset("interface"), *ep.interface);
+			val->AssignField("interface", *ep.interface);
 
 		cluster_val->Assign(std::move(key), std::move(val));
 		}
@@ -1403,10 +1403,10 @@ RecordValPtr SupervisorNode::ToRecord() const
 	const auto& rt = BifType::Record::Supervisor::NodeStatus;
 	auto rval = make_intrusive<RecordVal>(rt);
 
-	rval->Assign(rt->FieldOffset("node"), config.ToRecord());
+	rval->AssignField("node", config.ToRecord());
 
 	if ( pid )
-		rval->Assign(rt->FieldOffset("pid"), pid);
+		rval->AssignField("pid", pid);
 
 	return rval;
 	}
@@ -1458,17 +1458,15 @@ bool SupervisedNode::InitCluster() const
 		auto val = make_intrusive<RecordVal>(cluster_node_type);
 
 		auto node_type = supervisor_role_to_cluster_node_type(ep.role);
-		val->Assign(cluster_node_type->FieldOffset("node_type"), std::move(node_type));
+		val->AssignField("node_type", std::move(node_type));
-		val->Assign(cluster_node_type->FieldOffset("ip"), make_intrusive<AddrVal>(ep.host));
+		val->AssignField("ip", make_intrusive<AddrVal>(ep.host));
-		val->Assign(cluster_node_type->FieldOffset("p"), val_mgr->Port(ep.port, TRANSPORT_TCP));
+		val->AssignField("p", val_mgr->Port(ep.port, TRANSPORT_TCP));
 
 		if ( ep.interface )
-			val->Assign(cluster_node_type->FieldOffset("interface"),
+			val->AssignField("interface", *ep.interface);
-			            *ep.interface);
 
 		if ( manager_name && ep.role != BifEnum::Supervisor::MANAGER )
-			val->Assign(cluster_node_type->FieldOffset("manager"),
+			val->AssignField("manager", *manager_name);
-			            *manager_name);
 
 		cluster_nodes->Assign(std::move(key), std::move(val));
 		}

src/supervisor/supervisor.bif

@@ -86,7 +86,7 @@ function Supervisor::__node%(%): Supervisor::NodeConfig
 		zeek::emit_builtin_error("not a supervised process");
 		const auto& rt = zeek::BifType::Record::Supervisor::NodeConfig;
 		auto rval = zeek::make_intrusive<zeek::RecordVal>(rt);
-		rval->Assign(rt->FieldOffset("name"), "<invalid>");
+		rval->AssignField("name", "<invalid>");
 		return rval;
 		}
 

src/util.cc

@@ -5,17 +5,6 @@
 
 #include "zeek/util.h"
 
-#ifdef TIME_WITH_SYS_TIME
-# include <sys/time.h>
-# include <time.h>
-#else
-# ifdef HAVE_SYS_TIME_H
-#  include <sys/time.h>
-# else
-#  include <time.h>
-# endif
-#endif
-
 #ifdef HAVE_DARWIN
 #include <mach/task.h>
 #include <mach/mach_init.h>

src/util.h

@@ -29,6 +29,17 @@
 #include <vector>
 #include <memory> // std::unique_ptr
 
+#ifdef TIME_WITH_SYS_TIME
+# include <sys/time.h>
+# include <time.h>
+#else
+# ifdef HAVE_SYS_TIME_H
+#  include <sys/time.h>
+# else
+#  include <time.h>
+# endif
+#endif
+
 #ifdef DEBUG
 
 #include <assert.h>

src/zeek-setup.cc

@@ -752,6 +752,8 @@ SetupResult setup(int argc, char** argv, Options* zopts)
 			}
 		}
 
+	auto init_stmts = stmts ? analyze_global_stmts(stmts) : nullptr;
+
 	analyze_scripts();
 
 	if ( analysis_options.report_recursive )
@@ -838,15 +840,15 @@ SetupResult setup(int argc, char** argv, Options* zopts)
 	// cause more severe problems.
 	ZEEK_LSAN_ENABLE();
 
-	if ( stmts )
+	if ( init_stmts )
 		{
 		StmtFlowType flow;
-		Frame f(current_scope()->Length(), nullptr, nullptr);
+		Frame f(init_stmts->Scope()->Length(), nullptr, nullptr);
 		g_frame_stack.push_back(&f);
 
 		try
 			{
-			stmts->Exec(&f, flow);
+			init_stmts->Body()->Exec(&f, flow);
 			}
 		catch ( InterpreterException& )
 			{

testing/btest/Baseline/bifs.bloomfilter/err

@@ -0,0 +1,7 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+error: incompatible Bloom filter types
+error: incompatible Bloom filter types
+error: incompatible Bloom filter types
+error: incompatible Bloom filter types
+error: false-positive rate must take value between 0 and 1
+error: false-positive rate must take value between 0 and 1

testing/btest/Baseline/bifs.bloomfilter/output

@@ -1,10 +1,4 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error: incompatible Bloom filter types
-error: incompatible Bloom filter types
-error: incompatible Bloom filter types
-error: incompatible Bloom filter types
-error: false-positive rate must take value between 0 and 1
-error: false-positive rate must take value between 0 and 1
 0
 1
 1

testing/btest/Baseline/bifs.strptime/err

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+warning: strptime conversion failed: fmt:%m d:1980-10-24

testing/btest/Baseline/bifs.strptime/out

@@ -1,4 +1,3 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-warning: strptime conversion failed: fmt:%m d:1980-10-24
 XXXXXXXXXX.XXXXXX
 0.0

testing/btest/Baseline/core.init-error/err

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+expression error in <...>/init-error.zeek, line 16: no such index (v[10])
+fatal error: errors occurred while initializing

testing/btest/Baseline/core.init-error/out

@@ -1,6 +1,4 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-expression error in <...>/init-error.zeek, line 15: no such index (v[10])
-fatal error: errors occurred while initializing
 1st event
 2nd event
 3rd event

testing/btest/Baseline/core.pcap.dumper/output

@@ -1,5 +1,8 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-2c2
+@@ -1,5 +1,5 @@
-< 00000010  ff ff 00 00 01 00 00 00  1d a2 b2 4e 73 00 07 00  |...........Ns...|
+ 00000000  d4 c3 b2 a1 02 00 04 00  00 00 00 00 00 00 00 00  |................|
----
+-00000010  ff ff 00 00 01 00 00 00  1d a2 b2 4e 73 00 07 00  |...........Ns...|
-> 00000010  00 24 00 00 01 00 00 00  1d a2 b2 4e 73 00 07 00  |.$.........Ns...|
++00000010  00 24 00 00 01 00 00 00  1d a2 b2 4e 73 00 07 00  |.$.........Ns...|
+ 00000020  4a 00 00 00 4a 00 00 00  52 54 00 12 35 02 08 00  |J...J...RT..5...|
+ 00000030  27 87 94 24 08 00 45 00  00 3c c1 d8 40 00 40 06  |'..$..E..<..@.@.|
+ 00000040  f1 12 0a 00 02 0f c0 96  bb 2b c0 82 00 50 01 28  |.........+...P.(|

testing/btest/Baseline/core.reporter-error-in-handler/err

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+expression error in <...>/reporter-error-in-handler.zeek, line 29: no such index (a[1])
+expression error in <...>/reporter-error-in-handler.zeek, line 23: no such index (a[2])

testing/btest/Baseline/core.reporter-error-in-handler/output

@@ -1,4 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-expression error in <...>/reporter-error-in-handler.zeek, line 28: no such index (a[1])
-expression error in <...>/reporter-error-in-handler.zeek, line 22: no such index (a[2])
 1st error printed on script level

testing/btest/Baseline/core.tunnels.teredo-udp-in-udp/tunnel.log

@@ -0,0 +1,12 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	tunnel
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	tunnel_type	action
+#types	time	string	addr	port	addr	port	enum	enum
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	::38:5f55:6265:726b	25977	2090:9090:9090:9090:9090:9000::	25964	Tunnel::TEREDO	Tunnel::DISCOVER
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	::38:5f55:6265:726b	25977	2090:9090:9090:9090:9090:9000::	25964	Tunnel::TEREDO	Tunnel::CLOSE
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/language.common-mistakes/1.err

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+expression error in ./1.zeek, line 9: field value missing (mr$f)

testing/btest/Baseline/language.common-mistakes/1.out

@@ -1,5 +1,4 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-expression error in ./1.zeek, line 9: field value missing (mr$f)
 bar start
 foo start
 other zeek_init

testing/btest/Baseline/language.common-mistakes/2.err

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+expression error in ./2.zeek, line 7: no such index (t[nope])

testing/btest/Baseline/language.common-mistakes/2.out

@@ -1,3 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-expression error in ./2.zeek, line 7: no such index (t[nope])
 in foo

testing/btest/Baseline/language.common-mistakes/3.err

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+expression error in ./3.zeek, line 5: type-checking failed in vector append (v += ok)

testing/btest/Baseline/language.common-mistakes/3.out

@@ -1,3 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-expression error in ./3.zeek, line 5: type-checking failed in vector append (v += ok)
 in foo

testing/btest/Baseline/language.invalid_index/err

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+expression error in <...>/invalid_index.zeek, line 11: no such index (foo[1])
+expression error in <...>/invalid_index.zeek, line 17: no such index (foo2[1])

testing/btest/Baseline/language.invalid_index/out

@@ -1,6 +1,4 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-expression error in <...>/invalid_index.zeek, line 10: no such index (foo[1])
-expression error in <...>/invalid_index.zeek, line 16: no such index (foo2[1])
 foo[0], 42
 foo2[0], 13
 done

testing/btest/Baseline/language.subnet-errors/err

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+expression error in <...>/subnet-errors.zeek, line 10: bad IPv4 subnet prefix length: 33 (1.2.3.4 / i)
+expression error in <...>/subnet-errors.zeek, line 19: bad IPv6 subnet prefix length: 129 (:: / i)

testing/btest/Baseline/language.subnet-errors/out

@@ -1,6 +1,4 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-expression error in <...>/subnet-errors.zeek, line 9: bad IPv4 subnet prefix length: 33 (1.2.3.4 / i)
-expression error in <...>/subnet-errors.zeek, line 18: bad IPv6 subnet prefix length: 129 (:: / i)
 1.2.3.4/32
 ::/128
 init last

testing/btest/Baseline/language.table-set-iterator-invalidation/err

@@ -0,0 +1,5 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+warning in <...>/table-set-iterator-invalidation.zeek, line 22: possible loop/iterator invalidation caused by expression: t[4]
+warning in <...>/table-set-iterator-invalidation.zeek, line 31: possible loop/iterator invalidation caused by expression: t[4]
+warning in <...>/table-set-iterator-invalidation.zeek, line 54: possible loop/iterator invalidation caused by expression: s[4]
+warning in <...>/table-set-iterator-invalidation.zeek, line 63: possible loop/iterator invalidation caused by expression: s[4]

testing/btest/Baseline/language.table-set-iterator-invalidation/out

@@ -1,8 +1,4 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-warning in <...>/table-set-iterator-invalidation.zeek, line 21: possible loop/iterator invalidation caused by expression: t[4]
-warning in <...>/table-set-iterator-invalidation.zeek, line 30: possible loop/iterator invalidation caused by expression: t[4]
-warning in <...>/table-set-iterator-invalidation.zeek, line 53: possible loop/iterator invalidation caused by expression: s[4]
-warning in <...>/table-set-iterator-invalidation.zeek, line 62: possible loop/iterator invalidation caused by expression: s[4]
 {
 [2] = 2,
 [1] = 1,

testing/btest/Baseline/language.type-cast-error-dynamic/err

@@ -0,0 +1,4 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+expression error in <...>/type-cast-error-dynamic.zeek, line 12: invalid cast of value with type 'count' to type 'string' (a as string)
+expression error in <...>/type-cast-error-dynamic.zeek, line 12: invalid cast of value with type 'record { a:addr; b:port; }' to type 'string' (a as string)
+expression error in <...>/type-cast-error-dynamic.zeek, line 12: invalid cast of value with type 'record { data:opaque of Broker::Data &optional; }' to type 'string' (nil $data field) (a as string)

testing/btest/Baseline/language.type-cast-error-dynamic/output

@@ -1,5 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-expression error in <...>/type-cast-error-dynamic.zeek, line 11: invalid cast of value with type 'count' to type 'string' (a as string)
-expression error in <...>/type-cast-error-dynamic.zeek, line 11: invalid cast of value with type 'record { a:addr; b:port; }' to type 'string' (a as string)
-expression error in <...>/type-cast-error-dynamic.zeek, line 11: invalid cast of value with type 'record { data:opaque of Broker::Data &optional; }' to type 'string' (nil $data field) (a as string)
 data is string, F

testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure1.err

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+error in ./double_convert_failure1.zeek, line 7 and double: type clash for field "cc" ((coerce [$cc=5.0] to myrecord) and double)

testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure1.out

@@ -1,2 +1 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error in ./double_convert_failure1.zeek, line 7 and double: type clash for field "cc" ((coerce [$cc=5.0] to myrecord) and double)

testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure2.err

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+error in ./double_convert_failure2.zeek, line 7 and double: type clash for field "cc" ((coerce [$cc=-5.0] to myrecord) and double)

testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure2.out

@@ -1,2 +1 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error in ./double_convert_failure2.zeek, line 7 and double: type clash for field "cc" ((coerce [$cc=-5.0] to myrecord) and double)

testing/btest/Baseline/language.type-coerce-numerics/first_set.err

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+error in int: overflow promoting from unsigned/double to signed arithmetic value (int and 9223372036854775808)

testing/btest/Baseline/language.type-coerce-numerics/first_set.out

@@ -1,6 +1,4 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error in int and ./first_set.zeek, line 46: overflow promoting from unsigned/double to signed arithmetic value (int and 9223372036854775808)
-expression error in ./first_set.zeek, line 46: Failed type conversion ((coerce [$ii=9223372036854775808] to record { ii:int &optional; cc:count &optional; dd:double &optional; }))
 3
 int
 4

testing/btest/Baseline/language.type-coerce-numerics/int_convert_failure.err

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+error in ./int_convert_failure.zeek, line 7 and int: type clash for field "cc" ((coerce [$cc=-5] to myrecord) and int)

testing/btest/Baseline/language.type-coerce-numerics/int_convert_failure.out

@@ -1,2 +1 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error in ./int_convert_failure.zeek, line 7 and int: type clash for field "cc" ((coerce [$cc=-5] to myrecord) and int)

testing/btest/Baseline/language.type-coerce-numerics/vectors.err

@@ -0,0 +1 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

testing/btest/Baseline/language.uninitialized-local2/err

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+expression error in <...>/uninitialized-local2.zeek, line 20: value used but not set (var_b)

testing/btest/Baseline/language.uninitialized-local2/out

@@ -1,3 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-expression error in <...>/uninitialized-local2.zeek, line 19: value used but not set (var_b)
 var_a is, baz

testing/btest/Baseline/language.uninitialized-local3/err

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+warning in <...>/uninitialized-local3.zeek, line 39: possibly used without definition (x4)
+expression error in <...>/uninitialized-local3.zeek, line 39: value used but not set (x4)

testing/btest/Baseline/language.uninitialized-local3/out

@@ -1,11 +1,9 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-warning in <...>/uninitialized-local3.zeek, line 38: possibly used without definition (x4)
+x$a (x <...>/uninitialized-local3.zeek, line 21) possibly used without being set
-expression error in <...>/uninitialized-local3.zeek, line 38: value used but not set (x4)
+x$e (x <...>/uninitialized-local3.zeek, line 21) possibly used without being set
-x$a (x <...>/uninitialized-local3.zeek, line 20) possibly used without being set
+x$e (x <...>/uninitialized-local3.zeek, line 25) possibly used without being set
-x$e (x <...>/uninitialized-local3.zeek, line 20) possibly used without being set
+x2$worries$a (x2 <...>/uninitialized-local3.zeek, line 28) possibly used without being set
-x$e (x <...>/uninitialized-local3.zeek, line 24) possibly used without being set
+x2$worries$e (x2 <...>/uninitialized-local3.zeek, line 28) possibly used without being set
-x2$worries$a (x2 <...>/uninitialized-local3.zeek, line 27) possibly used without being set
-x2$worries$e (x2 <...>/uninitialized-local3.zeek, line 27) possibly used without being set
 [a=<uninitialized>, b=<uninitialized>, c=9, d=<uninitialized>, e=<uninitialized>]
 [a=<uninitialized>, b=<uninitialized>, c=9, d=<uninitialized>, e=<uninitialized>]
 [no_worries=[a=<uninitialized>, b=<uninitialized>, c=9, d=<uninitialized>, e=<uninitialized>], worries=[a=<uninitialized>, b=<uninitialized>, c=9, d=<uninitialized>, e=<uninitialized>]]

testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.file_exists_lookup_file/.stdout

@@ -1,6 +1,4 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error: file ID asdf not a known file
-warning: non-void function returning without a value: Files::lookup_file
 This should fail but not crash
 This should return F
 F

testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.file_exists_lookup_file/err

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+error: file ID asdf not a known file
+warning: non-void function returning without a value: Files::lookup_file

testing/btest/bifs/bloomfilter.zeek

@@ -1,5 +1,6 @@
-# @TEST-EXEC: zeek -b %INPUT >output 2>&1
+# @TEST-EXEC: zeek -b %INPUT >output 2>err
 # @TEST-EXEC: btest-diff output
+# @TEST-EXEC: btest-diff err
 
 function test_basic_bloom_filter()
   {

testing/btest/bifs/strptime.zeek

@@ -1,6 +1,7 @@
 #
-# @TEST-EXEC: zeek -b %INPUT >out 2>&1
+# @TEST-EXEC: zeek -b %INPUT >out 2>err
 # @TEST-EXEC: btest-diff out
+# @TEST-EXEC: btest-diff err
 
 event zeek_init()
 	{

testing/btest/core/init-error.zeek

@@ -1,7 +1,8 @@
 # The default is for an initialization error to be a hard failure.
 
-# @TEST-EXEC-FAIL: unset ZEEK_ALLOW_INIT_ERRORS && zeek -b %INPUT >out 2>&1
+# @TEST-EXEC-FAIL: unset ZEEK_ALLOW_INIT_ERRORS && zeek -b %INPUT >out 2>err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff err
 
 event zeek_init() &priority=10
 	{

testing/btest/core/pcap/dumper.zeek

@@ -2,7 +2,9 @@
 # @TEST-EXEC: zeek -b -r $TRACES/workshop_2011_browse.trace -w dump
 # @TEST-EXEC: hexdump -C $TRACES/workshop_2011_browse.trace >1
 # @TEST-EXEC: hexdump -C dump >2
-# @TEST-EXEC: diff 1 2 >output || true
+# @TEST-EXEC: diff -U3 1 2 >diff.output || true
+# Remove first two lines of diff since they may contain timestamps
+# @TEST-EXEC: awk 'NR>2' diff.output >output
 
 # Note that we're diff'ing the diff because there is an expected
 # difference in the pcaps: namely, the snaplen setting stored in the

testing/btest/core/reporter-error-in-handler.zeek

@@ -2,8 +2,9 @@
 # This test procudes a recursive error: the error handler is itself broken. Rather
 # than looping indefinitly, the error inside the handler should reported to stderr.
 #
-# @TEST-EXEC: zeek -b %INPUT >output 2>&1
+# @TEST-EXEC: zeek -b %INPUT >output 2>err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff output
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff err
 
 global a: table[count] of count;
 

testing/btest/core/tunnels/teredo-udp-in-udp.zeek

@@ -0,0 +1,4 @@
+# @TEST-EXEC: zeek -r $TRACES/tunnels/teredo-udp-in-udp.pcap %INPUT
+# @TEST-EXEC: btest-diff tunnel.log
+
+@load base/frameworks/tunnels

testing/btest/language/common-mistakes.zeek

@@ -2,14 +2,17 @@
 # handled internally by way of throwing an exception to unwind out
 # of the current event handler body.
 
-# @TEST-EXEC: zeek -b 1.zeek >1.out 2>&1
+# @TEST-EXEC: zeek -b 1.zeek >1.out 2>1.err
 # @TEST-EXEC: btest-diff 1.out
+# @TEST-EXEC: btest-diff 1.err
 
-# @TEST-EXEC: zeek -b 2.zeek >2.out 2>&1
+# @TEST-EXEC: zeek -b 2.zeek >2.out 2>2.err
 # @TEST-EXEC: btest-diff 2.out
+# @TEST-EXEC: btest-diff 2.err
 
-# @TEST-EXEC: zeek -b 3.zeek >3.out 2>&1
+# @TEST-EXEC: zeek -b 3.zeek >3.out 2>3.err
 # @TEST-EXEC: btest-diff 3.out
+# @TEST-EXEC: btest-diff 3.err
 
 @TEST-START-FILE 1.zeek
 type myrec: record {

testing/btest/language/invalid_index.zeek

@@ -1,5 +1,6 @@
-# @TEST-EXEC: zeek -b %INPUT >out 2>&1
+# @TEST-EXEC: zeek -b %INPUT >out 2>err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff err
 
 global foo: vector of count = { 42 };
 global foo2: table[count] of count = { [0] = 13 };

testing/btest/language/subnet-errors.zeek

@@ -1,5 +1,6 @@
-# @TEST-EXEC: zeek -b %INPUT >out 2>&1
+# @TEST-EXEC: zeek -b %INPUT >out 2>err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff err
 
 event zeek_init()
 	{

testing/btest/language/table-set-iterator-invalidation.zeek

@@ -1,5 +1,6 @@
-# @TEST-EXEC: zeek -b %INPUT >out 2>&1
+# @TEST-EXEC: zeek -b %INPUT >out 2>err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff err
 
 # Note that while modifying container membership during for-loop iteration is
 # supposed to be undefined-behavior, it should be practically ok to have this

testing/btest/language/type-cast-error-dynamic.zeek

@@ -1,5 +1,6 @@
-# @TEST-EXEC: zeek -b %INPUT >output 2>&1
+# @TEST-EXEC: zeek -b %INPUT >output 2>err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff output
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff err
 
 type X: record  {
     a: addr;

testing/btest/language/type-coerce-numerics.zeek

@@ -1,17 +1,22 @@
-# @TEST-EXEC: zeek -b first_set.zeek >first_set.out 2>&1
+# @TEST-EXEC: zeek -b first_set.zeek >first_set.out 2>first_set.err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff first_set.out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff first_set.err
 
-# @TEST-EXEC-FAIL: zeek -b double_convert_failure1.zeek >double_convert_failure1.out 2>&1
+# @TEST-EXEC-FAIL: zeek -b double_convert_failure1.zeek >double_convert_failure1.out 2>double_convert_failure1.err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff double_convert_failure1.out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff double_convert_failure1.err
 
-# @TEST-EXEC-FAIL: zeek -b double_convert_failure2.zeek >double_convert_failure2.out 2>&1
+# @TEST-EXEC-FAIL: zeek -b double_convert_failure2.zeek >double_convert_failure2.out 2>double_convert_failure2.err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff double_convert_failure2.out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff double_convert_failure2.err
 
-# @TEST-EXEC-FAIL: zeek -b int_convert_failure.zeek >int_convert_failure.out 2>&1
+# @TEST-EXEC-FAIL: zeek -b int_convert_failure.zeek >int_convert_failure.out 2>int_convert_failure.err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff int_convert_failure.out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff int_convert_failure.err
 
-# @TEST-EXEC: zeek -b vectors.zeek >vectors.out 2>&1
+# @TEST-EXEC: zeek -b vectors.zeek >vectors.out 2>vectors.err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff vectors.out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff vectors.err
 
 @TEST-START-FILE first_set.zeek
 type myrecord : record {

testing/btest/language/uninitialized-local2.zeek

@@ -1,5 +1,6 @@
-# @TEST-EXEC: zeek -b %INPUT >out 2>&1
+# @TEST-EXEC: zeek -b %INPUT >out 2>err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff err
 
 event test()
 	{

testing/btest/language/uninitialized-local3.zeek

@@ -1,5 +1,6 @@
-# @TEST-EXEC: ZEEK_USAGE_ISSUES=2 zeek -b %INPUT >out 2>&1
+# @TEST-EXEC: ZEEK_USAGE_ISSUES=2 zeek -b %INPUT >out 2>err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff err
 
 type r: record {
 	a: count;

testing/btest/scripts/base/frameworks/file-analysis/bifs/file_exists_lookup_file.zeek

@@ -1,5 +1,6 @@
-# @TEST-EXEC: zeek -b -r $TRACES/http/get.trace %INPUT 2>&1
+# @TEST-EXEC: zeek -b -r $TRACES/http/get.trace %INPUT 2>err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stdout
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff err
 
 @load base/protocols/http
 

testing/btest/scripts/base/frameworks/netcontrol/acld-hook.zeek

@@ -9,6 +9,7 @@
 @TEST-START-FILE send.zeek
 
 @load base/frameworks/netcontrol
+@load base/frameworks/broker
 
 redef exit_only_after_terminate = T;
 global have_peer = F;

testing/btest/scripts/base/frameworks/openflow/broker-basic.zeek

@@ -82,6 +82,7 @@ event OpenFlow::flow_mod_failure(name: string, match: OpenFlow::ofp_match, flow_
 
 @TEST-START-FILE recv.zeek
 
+@load base/protocols/conn
 @load base/frameworks/openflow
 
 redef exit_only_after_terminate = T;

testing/scripts/diff-remove-timestamps

@@ -9,6 +9,4 @@ else
    sed="sed -E"
 fi
 
-# $a\ adds a newline to the end of the file if it does not exist.
+$sed -e 's/(^|[^0-9])([0-9]{9,10}\.[0-9]{1,8})/\1XXXXXXXXXX.XXXXXX/g' -e  's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g'
-# This unifies the behavior of BSD and GNU sed
-$sed -e 's/(^|[^0-9])([0-9]{9,10}\.[0-9]{1,8})/\1XXXXXXXXXX.XXXXXX/g' -e  's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g' -e '$a\'