From 8f2637decb3a9a9622b2118c27d17d656142351c Mon Sep 17 00:00:00 2001
From: Vern Paxson <vern@corelight.com>
Date: Thu, 18 Mar 2021 10:00:01 -0700
Subject: [PATCH 01/36] new function for getting the location, if any,
 associated with the current call

---
 src/Frame.cc          |  5 +++++
 src/Frame.h           |  1 +
 src/broker/Data.cc    |  2 +-
 src/broker/Data.h     |  2 +-
 src/broker/Manager.cc |  2 +-
 src/reporter.bif      | 10 +++++-----
 6 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/src/Frame.cc b/src/Frame.cc
index c58d830d89..dd6e391adb 100644
--- a/src/Frame.cc
+++ b/src/Frame.cc
@@ -616,6 +616,11 @@ void Frame::CaptureClosure(Frame* c, IDPList arg_outer_ids)
 	// if (c) closure = c->SelectiveClone(outer_ids);
 	}
 
+const detail::Location* Frame::GetCallLocation() const
+	{
+	return call ? call->GetLocationInfo() : nullptr;
+	}
+
 void Frame::SetTrigger(trigger::TriggerPtr arg_trigger)
 	{
 	trigger = std::move(arg_trigger);
diff --git a/src/Frame.h b/src/Frame.h
index 36d775ef9f..4188dded27 100644
--- a/src/Frame.h
+++ b/src/Frame.h
@@ -262,6 +262,7 @@ public:
 	void SetCall(const CallExpr* arg_call)	{ call = arg_call; }
 	void ClearCall()			{ call = nullptr; }
 	const CallExpr* GetCall() const		{ return call; }
+	const detail::Location* GetCallLocation() const;
 
 	void SetDelayed()	{ delayed = true; }
 	bool HasDelayed() const	{ return delayed; }
diff --git a/src/broker/Data.cc b/src/broker/Data.cc
index a54e30287c..2af93460bd 100644
--- a/src/broker/Data.cc
+++ b/src/broker/Data.cc
@@ -1177,7 +1177,7 @@ broker::data& opaque_field_to_data(RecordVal* v, zeek::detail::Frame* f)
 	const auto& d = v->GetField(0);
 
 	if ( ! d )
-		reporter->RuntimeError(f->GetCall()->GetLocationInfo(),
+		reporter->RuntimeError(f->GetCallLocation(),
 		                             "Broker::Data's opaque field is not set");
 
 	// RuntimeError throws an exception which causes this line to never exceute.
diff --git a/src/broker/Data.h b/src/broker/Data.h
index a3db6c1514..feb58e3806 100644
--- a/src/broker/Data.h
+++ b/src/broker/Data.h
@@ -194,7 +194,7 @@ T& require_data_type(broker::data& d, zeek::TypeTag tag, zeek::detail::Frame* f)
 	{
 	auto ptr = caf::get_if<T>(&d);
 	if ( ! ptr )
-		zeek::reporter->RuntimeError(f->GetCall()->GetLocationInfo(),
+		zeek::reporter->RuntimeError(f->GetCallLocation(),
 		                             "data is of type '%s' not of type '%s'",
 		                             caf::visit(type_name_getter{tag}, d),
 		                             zeek::type_name(tag));
diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc
index 3d682ea4ab..0ce23d9508 100644
--- a/src/broker/Manager.cc
+++ b/src/broker/Manager.cc
@@ -74,7 +74,7 @@ int Manager::script_scope = 0;
 struct scoped_reporter_location {
 	scoped_reporter_location(zeek::detail::Frame* frame)
 		{
-		reporter->PushLocation(frame->GetCall()->GetLocationInfo());
+		reporter->PushLocation(frame->GetCallLocation());
 		}
 
 	~scoped_reporter_location()
diff --git a/src/reporter.bif b/src/reporter.bif
index 950b58f846..523195debb 100644
--- a/src/reporter.bif
+++ b/src/reporter.bif
@@ -22,7 +22,7 @@ module Reporter;
 ## .. zeek:see:: reporter_info
 function Reporter::info%(msg: string%): bool
 	%{
-	reporter->PushLocation(frame->GetCall()->GetLocationInfo());
+	reporter->PushLocation(frame->GetCallLocation());
 	reporter->Info("%s", msg->CheckString());
 	reporter->PopLocation();
 	return zeek::val_mgr->True();
@@ -37,7 +37,7 @@ function Reporter::info%(msg: string%): bool
 ## .. zeek:see:: reporter_warning
 function Reporter::warning%(msg: string%): bool
 	%{
-	reporter->PushLocation(frame->GetCall()->GetLocationInfo());
+	reporter->PushLocation(frame->GetCallLocation());
 	reporter->Warning("%s", msg->CheckString());
 	reporter->PopLocation();
 	return zeek::val_mgr->True();
@@ -53,7 +53,7 @@ function Reporter::warning%(msg: string%): bool
 ## .. zeek:see:: reporter_error
 function Reporter::error%(msg: string%): bool
 	%{
-	reporter->PushLocation(frame->GetCall()->GetLocationInfo());
+	reporter->PushLocation(frame->GetCallLocation());
 	reporter->Error("%s", msg->CheckString());
 	reporter->PopLocation();
 	return zeek::val_mgr->True();
@@ -66,7 +66,7 @@ function Reporter::error%(msg: string%): bool
 ## Returns: Always true.
 function Reporter::fatal%(msg: string%): bool
 	%{
-	reporter->PushLocation(frame->GetCall()->GetLocationInfo());
+	reporter->PushLocation(frame->GetCallLocation());
 	reporter->FatalError("%s", msg->CheckString());
 	reporter->PopLocation();
 	return zeek::val_mgr->True();
@@ -80,7 +80,7 @@ function Reporter::fatal%(msg: string%): bool
 ## Returns: Always true.
 function Reporter::fatal_error_with_core%(msg: string%): bool
 	%{
-	reporter->PushLocation(frame->GetCall()->GetLocationInfo());
+	reporter->PushLocation(frame->GetCallLocation());
 	reporter->FatalErrorWithCore("%s", msg->CheckString());
 	reporter->PopLocation();
 	return zeek::val_mgr->True();

From b792feccab1f8b107d329d83bf638e23cbb886e4 Mon Sep 17 00:00:00 2001
From: Vern Paxson <vern@corelight.com>
Date: Thu, 18 Mar 2021 10:02:22 -0700
Subject: [PATCH 02/36] functions for indexing slices and strings

---
 src/Expr.cc | 107 +++++++++++++++++++++++++++++-----------------------
 src/Expr.h  |   4 ++
 2 files changed, 63 insertions(+), 48 deletions(-)

diff --git a/src/Expr.cc b/src/Expr.cc
index 1f5afbf8a7..24cfde992e 100644
--- a/src/Expr.cc
+++ b/src/Expr.cc
@@ -2936,24 +2936,7 @@ ValPtr IndexExpr::Fold(Val* v1, Val* v2) const
 		if ( lv->Length() == 1 )
 			v = vect->ValAt(lv->Idx(0)->CoerceToUnsigned());
 		else
-			{
-			size_t len = vect->Size();
-			auto result = make_intrusive<VectorVal>(vect->GetType<VectorType>());
-
-			bro_int_t first = get_slice_index(lv->Idx(0)->CoerceToInt(), len);
-			bro_int_t last = get_slice_index(lv->Idx(1)->CoerceToInt(), len);
-			bro_int_t sub_length = last - first;
-
-			if ( sub_length >= 0 )
-				{
-				result->Resize(sub_length);
-
-				for ( bro_int_t idx = first; idx < last; idx++ )
-					result->Assign(idx - first, vect->ValAt(idx));
-				}
-
-			return result;
-			}
+			return index_slice(vect, lv);
 		}
 		break;
 
@@ -2962,36 +2945,7 @@ ValPtr IndexExpr::Fold(Val* v1, Val* v2) const
 		break;
 
 	case TYPE_STRING:
-		{
-		const ListVal* lv = v2->AsListVal();
-		const String* s = v1->AsString();
-		int len = s->Len();
-		String* substring = nullptr;
-
-		if ( lv->Length() == 1 )
-			{
-			bro_int_t idx = lv->Idx(0)->AsInt();
-
-			if ( idx < 0 )
-				idx += len;
-
-			// Out-of-range index will return null pointer.
-			substring = s->GetSubstring(idx, 1);
-			}
-		else
-			{
-			bro_int_t first = get_slice_index(lv->Idx(0)->AsInt(), len);
-			bro_int_t last = get_slice_index(lv->Idx(1)->AsInt(), len);
-			bro_int_t substring_len = last - first;
-
-			if ( substring_len < 0 )
-				substring = nullptr;
-			else
-				substring = s->GetSubstring(first, substring_len);
-			}
-
-		return make_intrusive<StringVal>(substring ? substring : new String(""));
-		}
+		return index_string(v1->AsString(), v2->AsListVal());
 
 	default:
 		RuntimeError("type cannot be indexed");
@@ -3005,6 +2959,63 @@ ValPtr IndexExpr::Fold(Val* v1, Val* v2) const
 	return nullptr;
 	}
 
+StringValPtr index_string(const String* s, const ListVal* lv)
+	{
+	int len = s->Len();
+	String* substring = nullptr;
+
+	if ( lv->Length() == 1 )
+		{
+		bro_int_t idx = lv->Idx(0)->AsInt();
+
+		if ( idx < 0 )
+			idx += len;
+
+		// Out-of-range index will return null pointer.
+		substring = s->GetSubstring(idx, 1);
+		}
+	else
+		{
+		bro_int_t first = get_slice_index(lv->Idx(0)->AsInt(), len);
+		bro_int_t last = get_slice_index(lv->Idx(1)->AsInt(), len);
+		bro_int_t substring_len = last - first;
+
+		if ( substring_len < 0 )
+			substring = nullptr;
+		else
+			substring = s->GetSubstring(first, substring_len);
+		}
+
+	return make_intrusive<StringVal>(substring ? substring : new String(""));
+	}
+
+VectorValPtr index_slice(VectorVal* vect, const ListVal* lv)
+	{
+	auto first = lv->Idx(0)->CoerceToInt();
+	auto last = lv->Idx(1)->CoerceToInt();
+	return index_slice(vect, first, last);
+	}
+
+VectorValPtr index_slice(VectorVal* vect, int _first, int _last)
+	{
+	size_t len = vect->Size();
+	auto result = make_intrusive<VectorVal>(vect->GetType<VectorType>());
+
+	bro_int_t first = get_slice_index(_first, len);
+	bro_int_t last = get_slice_index(_last, len);
+	bro_int_t sub_length = last - first;
+
+	if ( sub_length >= 0 )
+		{
+		result->Resize(sub_length);
+
+		for ( bro_int_t idx = first; idx < last; idx++ )
+			result->Assign(idx - first, vect->ValAt(idx));
+		}
+
+	return result;
+	}
+
 void IndexExpr::Assign(Frame* f, ValPtr v)
 	{
 	if ( IsError() )
diff --git a/src/Expr.h b/src/Expr.h
index ae275a7a84..6b99b1572a 100644
--- a/src/Expr.h
+++ b/src/Expr.h
@@ -941,6 +941,10 @@ protected:
 	bool is_slice;
 };
 
+extern VectorValPtr index_slice(VectorVal* vect, const ListVal* lv);
+extern VectorValPtr index_slice(VectorVal* vect, int first, int last);
+extern StringValPtr index_string(const String* s, const ListVal* lv);
+
 class IndexExprWhen final : public IndexExpr {
 public:
 	static inline std::vector<ValPtr> results = {};

From 22776167bb0f40caa64a19787f082ff981321a27 Mon Sep 17 00:00:00 2001
From: Vern Paxson <vern@corelight.com>
Date: Thu, 18 Mar 2021 10:04:21 -0700
Subject: [PATCH 03/36] factor out "print" statement's execution functionality

---
 src/File.h  |  7 ++++---
 src/Stmt.cc | 15 +++++++++------
 src/Stmt.h  |  2 ++
 3 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/src/File.h b/src/File.h
index 8ed5d0a13e..78524826b2 100644
--- a/src/File.h
+++ b/src/File.h
@@ -12,8 +12,7 @@
 #include <krb5.h>
 #endif // NEED_KRB5_H
 
-#include "zeek/Obj.h"
-#include "zeek/IntrusivePtr.h"
+#include "zeek/Val.h"
 #include "zeek/util.h"
 
 namespace zeek {
@@ -23,6 +22,8 @@ namespace detail {
 class PrintStmt;
 class Attributes;
 
+extern void do_print_stmt(const std::vector<ValPtr>& vals);
+
 } // namespace detail;
 
 class RecordVal;
@@ -84,7 +85,7 @@ public:
 
 protected:
 
-	friend class detail::PrintStmt;
+	friend void detail::do_print_stmt(const std::vector<ValPtr>& vals);
 
 	File()	{ Init(); }
 	void Init();
diff --git a/src/Stmt.cc b/src/Stmt.cc
index fed57f9496..77beb1c9e9 100644
--- a/src/Stmt.cc
+++ b/src/Stmt.cc
@@ -306,18 +306,23 @@ ValPtr PrintStmt::DoExec(std::vector<ValPtr> vals,
                          StmtFlowType& /* flow */)
 	{
 	RegisterAccess();
+	do_print_stmt(vals);
+	return nullptr;
+	}
 
+void do_print_stmt(const std::vector<ValPtr>& vals)
+	{
 	if ( ! print_stdout )
 		print_stdout = new File(stdout);
 
 	File* f = print_stdout;
 	int offset = 0;
 
-	if ( vals.size() > 0 && (vals)[0]->GetType()->Tag() == TYPE_FILE )
+	if ( vals.size() > 0 && vals[0] && vals[0]->GetType()->Tag() == TYPE_FILE )
 		{
 		f = (vals)[0]->AsFile();
 		if ( ! f->IsOpen() )
-			return nullptr;
+			return;
 
 		++offset;
 		}
@@ -331,7 +336,7 @@ ValPtr PrintStmt::DoExec(std::vector<ValPtr> vals,
 	case BifEnum::Log::REDIRECT_ALL:
 		{
 		print_log(vals);
-		return nullptr;
+		return;
 		}
 	case BifEnum::Log::REDIRECT_STDOUT:
 		if ( f->FileHandle() == stdout )
@@ -339,7 +344,7 @@ ValPtr PrintStmt::DoExec(std::vector<ValPtr> vals,
 			// Should catch even printing to a "manually opened" stdout file,
 			// like "/dev/stdout" or "-".
 			print_log(vals);
-			return nullptr;
+			return;
 			}
 		break;
 	default:
@@ -368,8 +373,6 @@ ValPtr PrintStmt::DoExec(std::vector<ValPtr> vals,
 		describe_vals(vals, &d, offset);
 		f->Write("\n", 1);
 		}
-
-	return nullptr;
 	}
 
 ExprStmt::ExprStmt(ExprPtr arg_e) : Stmt(STMT_EXPR), e(std::move(arg_e))
diff --git a/src/Stmt.h b/src/Stmt.h
index bd26492afe..9bf13d9c9a 100644
--- a/src/Stmt.h
+++ b/src/Stmt.h
@@ -62,6 +62,8 @@ protected:
 	StmtPtr DoSubclassReduce(ListExprPtr singletons, Reducer* c) override;
 };
 
+extern void do_print_stmt(const std::vector<ValPtr>& vals);
+
 class ExprStmt : public Stmt {
 public:
 	explicit ExprStmt(ExprPtr e);

From 5f5ba3881c9bb65bcb13bf61d71ab6a209700f53 Mon Sep 17 00:00:00 2001
From: Vern Paxson <vern@corelight.com>
Date: Thu, 18 Mar 2021 10:07:15 -0700
Subject: [PATCH 04/36] lower-level method for adding fields to records

---
 src/Type.cc | 14 +++++++++++---
 src/Type.h  |  3 +++
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/Type.cc b/src/Type.cc
index 6b39ef9198..97fb94ea03 100644
--- a/src/Type.cc
+++ b/src/Type.cc
@@ -1043,6 +1043,17 @@ const char* RecordType::AddFields(const type_decl_list& others,
 
 	TableVal::SaveParseTimeTableState(this);
 
+	AddFieldsDirectly(others, add_log_attr);
+
+	RecordVal::ResizeParseTimeRecords(this);
+	TableVal::RebuildParseTimeTables();
+
+	return nullptr;
+	}
+
+void RecordType::AddFieldsDirectly(const type_decl_list& others,
+					bool add_log_attr)
+	{
 	for ( const auto& td : others )
 		{
 		if ( add_log_attr )
@@ -1059,9 +1070,6 @@ const char* RecordType::AddFields(const type_decl_list& others,
 		}
 
 	num_fields = types->length();
-	RecordVal::ResizeParseTimeRecords(this);
-	TableVal::RebuildParseTimeTables();
-	return nullptr;
 	}
 
 void RecordType::DescribeFields(ODesc* d) const
diff --git a/src/Type.h b/src/Type.h
index 09689ecf90..5a16463263 100644
--- a/src/Type.h
+++ b/src/Type.h
@@ -623,6 +623,9 @@ public:
 	const char* AddFields(const type_decl_list& types,
 	                      bool add_log_attr = false);
 
+	void AddFieldsDirectly(const type_decl_list& types,
+				bool add_log_attr = false);
+
 	void Describe(ODesc* d) const override;
 	void DescribeReST(ODesc* d, bool roles_only = false) const override;
 	void DescribeFields(ODesc* d) const;

From 91c2e050992937f7967e33b4e38dc1f098135cac Mon Sep 17 00:00:00 2001
From: Vern Paxson <vern@corelight.com>
Date: Thu, 18 Mar 2021 10:08:31 -0700
Subject: [PATCH 05/36] factor out record coercion; modernize management of
 coercion "map"

---
 src/Expr.cc                                   | 41 ++++++++++---------
 src/Expr.h                                    |  9 ++--
 .../first_set.out                             |  1 -
 3 files changed, 27 insertions(+), 24 deletions(-)

diff --git a/src/Expr.cc b/src/Expr.cc
index 24cfde992e..1b2562c2ee 100644
--- a/src/Expr.cc
+++ b/src/Expr.cc
@@ -3759,8 +3759,7 @@ ValPtr ArithCoerceExpr::Fold(Val* v) const
 	}
 
 RecordCoerceExpr::RecordCoerceExpr(ExprPtr arg_op, RecordTypePtr r)
-	: UnaryExpr(EXPR_RECORD_COERCE, std::move(arg_op)),
-	  map(nullptr), map_size(0)
+	: UnaryExpr(EXPR_RECORD_COERCE, std::move(arg_op))
 	{
 	if ( IsError() )
 		return;
@@ -3778,12 +3777,12 @@ RecordCoerceExpr::RecordCoerceExpr(ExprPtr arg_op, RecordTypePtr r)
 		RecordType* t_r = type->AsRecordType();
 		RecordType* sub_r = op->GetType()->AsRecordType();
 
-		map_size = t_r->NumFields();
-		map = new int[map_size];
+		int map_size = t_r->NumFields();
+		map.reserve(map_size);
 
 		int i;
 		for ( i = 0; i < map_size; ++i )
-			map[i] = -1;	// -1 = field is not mapped
+			map.emplace_back(-1);	// -1 = field is not mapped
 
 		for ( i = 0; i < sub_r->NumFields(); ++i )
 			{
@@ -3865,11 +3864,6 @@ RecordCoerceExpr::RecordCoerceExpr(ExprPtr arg_op, RecordTypePtr r)
 		}
 	}
 
-RecordCoerceExpr::~RecordCoerceExpr()
-	{
-	delete [] map;
-	}
-
 ValPtr RecordCoerceExpr::InitVal(const zeek::Type* t, ValPtr aggr) const
 	{
 	if ( auto v = Eval(nullptr) )
@@ -3892,7 +3886,15 @@ ValPtr RecordCoerceExpr::Fold(Val* v) const
 	if ( same_type(GetType(), Op()->GetType()) )
 		return IntrusivePtr{NewRef{}, v};
 
-	auto val = make_intrusive<RecordVal>(GetType<RecordType>());
+	auto rt = cast_intrusive<RecordType>(GetType());
+	return coerce_to_record(rt, v, map);
+	}
+
+RecordValPtr coerce_to_record(RecordTypePtr rt, Val* v,
+				const std::vector<int>& map)
+	{
+	auto map_size = map.size();
+	auto val = make_intrusive<RecordVal>(rt);
 	RecordType* val_type = val->GetType()->AsRecordType();
 
 	RecordVal* rv = v->AsRecordVal();
@@ -3905,14 +3907,15 @@ ValPtr RecordCoerceExpr::Fold(Val* v) const
 
 			if ( ! rhs )
 				{
-				const auto& def = rv->GetType()->AsRecordType()->FieldDecl(
-					map[i])->GetAttr(ATTR_DEFAULT);
+				auto rv_rt = rv->GetType()->AsRecordType();
+				const auto& def = rv_rt->FieldDecl(map[i])->
+							GetAttr(ATTR_DEFAULT);
 
 				if ( def )
 					rhs = def->GetExpr()->Eval(nullptr);
 				}
 
-			assert(rhs || GetType()->AsRecordType()->FieldDecl(i)->GetAttr(ATTR_OPTIONAL));
+			assert(rhs || rt->FieldDecl(i)->GetAttr(ATTR_OPTIONAL));
 
 			if ( ! rhs )
 				{
@@ -3934,21 +3937,19 @@ ValPtr RecordCoerceExpr::Fold(Val* v) const
 			else if ( BothArithmetic(rhs_type->Tag(), field_type->Tag()) &&
 			          ! same_type(rhs_type, field_type) )
 				{
-				if ( auto new_val = check_and_promote(rhs, field_type.get(), false, op->GetLocationInfo()) )
-					rhs = std::move(new_val);
-				else
-					RuntimeError("Failed type conversion");
+				auto new_val = check_and_promote(rhs, field_type.get(), false);
+				rhs = std::move(new_val);
 				}
 
 			val->Assign(i, std::move(rhs));
 			}
 		else
 			{
-			if ( const auto& def = GetType()->AsRecordType()->FieldDecl(i)->GetAttr(ATTR_DEFAULT) )
+			if ( const auto& def = rt->FieldDecl(i)->GetAttr(ATTR_DEFAULT) )
 				{
 				auto def_val = def->GetExpr()->Eval(nullptr);
 				const auto& def_type = def_val->GetType();
-				const auto& field_type = GetType()->AsRecordType()->GetFieldType(i);
+				const auto& field_type = rt->GetFieldType(i);
 
 				if ( def_type->Tag() == TYPE_RECORD &&
 				     field_type->Tag() == TYPE_RECORD &&
diff --git a/src/Expr.h b/src/Expr.h
index 6b99b1572a..33810656e5 100644
--- a/src/Expr.h
+++ b/src/Expr.h
@@ -1170,21 +1170,24 @@ protected:
 class RecordCoerceExpr final : public UnaryExpr {
 public:
 	RecordCoerceExpr(ExprPtr op, RecordTypePtr r);
-	~RecordCoerceExpr() override;
 
 	// Optimization-related:
 	ExprPtr Duplicate() override;
 
+	const std::vector<int>& Map() const	{ return map; }
+
 protected:
 	ValPtr InitVal(const zeek::Type* t, ValPtr aggr) const override;
 	ValPtr Fold(Val* v) const override;
 
 	// For each super-record slot, gives subrecord slot with which to
 	// fill it.
-	int* map;
-	int map_size;	// equivalent to Type()->AsRecordType()->NumFields()
+	std::vector<int> map;
 };
 
+extern RecordValPtr coerce_to_record(RecordTypePtr rt, Val* v,
+					const std::vector<int>& map);
+
 class TableCoerceExpr final : public UnaryExpr {
 public:
 	TableCoerceExpr(ExprPtr op, TableTypePtr r);
diff --git a/testing/btest/Baseline/language.type-coerce-numerics/first_set.out b/testing/btest/Baseline/language.type-coerce-numerics/first_set.out
index f053ab7046..7af2ad4a31 100644
--- a/testing/btest/Baseline/language.type-coerce-numerics/first_set.out
+++ b/testing/btest/Baseline/language.type-coerce-numerics/first_set.out
@@ -1,6 +1,5 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
 error in int and ./first_set.zeek, line 46: overflow promoting from unsigned/double to signed arithmetic value (int and 9223372036854775808)
-expression error in ./first_set.zeek, line 46: Failed type conversion ((coerce [$ii=9223372036854775808] to record { ii:int &optional; cc:count &optional; dd:double &optional; }))
 3
 int
 4

From 78f18b4fa6458fd483f2a1ae524d8d6a4b140a18 Mon Sep 17 00:00:00 2001
From: Vern Paxson <vern@corelight.com>
Date: Thu, 18 Mar 2021 10:10:44 -0700
Subject: [PATCH 06/36] support for subclassing ScriptFunc's, esp. for
 alternate lambda closures

---
 src/Func.cc        |  5 +++++
 src/Func.h         | 12 +++++++++---
 src/broker/Data.cc | 15 ++++++++++++---
 3 files changed, 26 insertions(+), 6 deletions(-)

diff --git a/src/Func.cc b/src/Func.cc
index 319868395e..f24154d1a5 100644
--- a/src/Func.cc
+++ b/src/Func.cc
@@ -587,6 +587,11 @@ bool ScriptFunc::StrengthenClosureReference(Frame* f)
 	return true;
 	}
 
+bool ScriptFunc::CopySemantics() const
+	{
+	return type->GetCaptures().has_value();
+	}
+
 void ScriptFunc::SetClosureFrame(Frame* f)
 	{
 	if ( closure )
diff --git a/src/Func.h b/src/Func.h
index db15fde849..9729fbca1d 100644
--- a/src/Func.h
+++ b/src/Func.h
@@ -145,7 +145,7 @@ protected:
 
 namespace detail {
 
-class ScriptFunc final : public Func {
+class ScriptFunc : public Func {
 public:
 	ScriptFunc(const IDPtr& id, StmtPtr body,
 	        const std::vector<IDPtr>& inits,
@@ -210,12 +210,17 @@ public:
 	 */
 	bool StrengthenClosureReference(Frame* f);
 
+	/**
+	 * Whether the function's closure uses copy semantics.
+	 */
+	virtual bool CopySemantics() const;
+
 	/**
 	 * Serializes this function's closure or capture frame.
 	 *
 	 * @return a serialized version of the function's closure/capture frame.
 	 */
-	broker::expected<broker::data> SerializeClosure() const;
+	virtual broker::expected<broker::data> SerializeClosure() const;
 
 	/**
 	 * Sets the captures frame to one built from *data*.
@@ -257,6 +262,7 @@ public:
 
 protected:
 	ScriptFunc() : Func(SCRIPT_FUNC)	{}
+
 	StmtPtr AddInits(
 		StmtPtr body,
 		const std::vector<IDPtr>& inits);
@@ -280,7 +286,7 @@ protected:
 	 *
 	 * @param f  the frame holding the values of capture variables
 	 */
-	void SetCaptures(Frame* f);
+	virtual void SetCaptures(Frame* f);
 
 private:
 	size_t frame_size;
diff --git a/src/broker/Data.cc b/src/broker/Data.cc
index 2af93460bd..db372de6fe 100644
--- a/src/broker/Data.cc
+++ b/src/broker/Data.cc
@@ -392,6 +392,17 @@ struct val_converter {
 
 			if ( a.size() == 2 ) // we have a closure/capture frame
 				{
+				// Note, seems if we already have a separate
+				// instance of the same lambda, then unless
+				// we use a cloned value, we'll step on that
+				// one's captures, too.  This is because
+				// the capture mapping lives with the Func
+				// object rather than the FuncVal.  However,
+				// we can't readily Clone() here because
+				// rval is const (and, grrr, Clone() is not).
+				// -VP
+				// rval = rval->Clone();
+
 				auto frame = broker::get_if<broker::vector>(a[1]);
 				if ( ! frame )
 					return nullptr;
@@ -400,9 +411,7 @@ struct val_converter {
 				if ( ! b )
 					return nullptr;
 
-				auto copy_semantics = b->GetType()->GetCaptures().has_value();
-
-				if ( copy_semantics )
+				if ( b->CopySemantics() )
 					{
 					if ( ! b->DeserializeCaptures(*frame) )
 						return nullptr;

From 8782894d7d516ee29fe74d256fc4d2573509812a Mon Sep 17 00:00:00 2001
From: Vern Paxson <vern@corelight.com>
Date: Thu, 18 Mar 2021 16:59:08 -0700
Subject: [PATCH 07/36] comments for factored-out index slice functions

---
 src/Expr.h | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/Expr.h b/src/Expr.h
index 33810656e5..3cdecfbb5e 100644
--- a/src/Expr.h
+++ b/src/Expr.h
@@ -941,8 +941,20 @@ protected:
 	bool is_slice;
 };
 
+// The following execute the heart of IndexExpr functionality for
+// vector slices and strings.
+
+// Extracts a slice of a vector, where the span of the slice is specified
+// by a list of (exactly) two values.  This is how the interpreter develops
+// the components of a slice.
 extern VectorValPtr index_slice(VectorVal* vect, const ListVal* lv);
+
+// Lower-level access to the slice, where its span is expressed
+// directly as integers.
 extern VectorValPtr index_slice(VectorVal* vect, int first, int last);
+
+// Returns a subset of a string, with the span specified by a list of
+// (exactly) two values.
 extern StringValPtr index_string(const String* s, const ListVal* lv);
 
 class IndexExprWhen final : public IndexExpr {

From a7fcc7a5d8e9e47bc397faabd7df3ec6da69778e Mon Sep 17 00:00:00 2001
From: Vern Paxson <vern@corelight.com>
Date: Thu, 18 Mar 2021 17:08:12 -0700
Subject: [PATCH 08/36] Use STL functionality to initialize coercion map

---
 src/Expr.cc | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/Expr.cc b/src/Expr.cc
index 1b2562c2ee..5778df4235 100644
--- a/src/Expr.cc
+++ b/src/Expr.cc
@@ -3778,12 +3778,9 @@ RecordCoerceExpr::RecordCoerceExpr(ExprPtr arg_op, RecordTypePtr r)
 		RecordType* sub_r = op->GetType()->AsRecordType();
 
 		int map_size = t_r->NumFields();
-		map.reserve(map_size);
+		map.resize(map_size, -1);	// -1 = field is not mapped
 
 		int i;
-		for ( i = 0; i < map_size; ++i )
-			map.emplace_back(-1);	// -1 = field is not mapped
-
 		for ( i = 0; i < sub_r->NumFields(); ++i )
 			{
 			int t_i = t_r->FieldOffset(sub_r->FieldName(i));

From 3e07564fb914b55c96d75072aadbe2925986840b Mon Sep 17 00:00:00 2001
From: Vern Paxson <vern@corelight.com>
Date: Thu, 18 Mar 2021 17:12:23 -0700
Subject: [PATCH 09/36] better method name: HasCopySemantics()

---
 src/Func.cc        | 2 +-
 src/Func.h         | 2 +-
 src/broker/Data.cc | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Func.cc b/src/Func.cc
index f24154d1a5..a01362c05e 100644
--- a/src/Func.cc
+++ b/src/Func.cc
@@ -587,7 +587,7 @@ bool ScriptFunc::StrengthenClosureReference(Frame* f)
 	return true;
 	}
 
-bool ScriptFunc::CopySemantics() const
+bool ScriptFunc::HasCopySemantics() const
 	{
 	return type->GetCaptures().has_value();
 	}
diff --git a/src/Func.h b/src/Func.h
index 9729fbca1d..b76603eba0 100644
--- a/src/Func.h
+++ b/src/Func.h
@@ -213,7 +213,7 @@ public:
 	/**
 	 * Whether the function's closure uses copy semantics.
 	 */
-	virtual bool CopySemantics() const;
+	virtual bool HasCopySemantics() const;
 
 	/**
 	 * Serializes this function's closure or capture frame.
diff --git a/src/broker/Data.cc b/src/broker/Data.cc
index db372de6fe..6395796801 100644
--- a/src/broker/Data.cc
+++ b/src/broker/Data.cc
@@ -411,7 +411,7 @@ struct val_converter {
 				if ( ! b )
 					return nullptr;
 
-				if ( b->CopySemantics() )
+				if ( b->HasCopySemantics() )
 					{
 					if ( ! b->DeserializeCaptures(*frame) )
 						return nullptr;

From a2e7dd82c40ce2743e0e084fc969654c2ef6a79f Mon Sep 17 00:00:00 2001
From: Vern Paxson <vern@corelight.com>
Date: Wed, 24 Mar 2021 15:15:19 -0700
Subject: [PATCH 10/36] removal of vestigial #include's (breaking an include
 loop in the process)

---
 src/ZVal.cc | 1 -
 src/ZVal.h  | 5 -----
 2 files changed, 6 deletions(-)

diff --git a/src/ZVal.cc b/src/ZVal.cc
index a5dfea8a7e..119d8c3f2e 100644
--- a/src/ZVal.cc
+++ b/src/ZVal.cc
@@ -5,7 +5,6 @@
 #include "zeek/Func.h"
 #include "zeek/OpaqueVal.h"
 #include "zeek/Reporter.h"
-#include "zeek/Desc.h"
 
 using namespace zeek;
 
diff --git a/src/ZVal.h b/src/ZVal.h
index 7cfb228047..1e1af6e900 100644
--- a/src/ZVal.h
+++ b/src/ZVal.h
@@ -6,11 +6,6 @@
 
 #include "zeek/zeek-config.h"
 
-#include <unordered_set>
-
-#include "zeek/Dict.h"
-#include "zeek/Expr.h"
-
 namespace zeek {
 
 class StringVal;

From a288800b41e5261acb04d9aedd4e215c96e73cc0 Mon Sep 17 00:00:00 2001
From: Vern Paxson <vern@corelight.com>
Date: Wed, 24 Mar 2021 15:16:03 -0700
Subject: [PATCH 11/36] test suite update due to factoring out
 coerce_to_record()

---
 doc                                                             | 2 +-
 .../btest/Baseline/language.type-coerce-numerics/first_set.out  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc b/doc
index afd337f4b0..412a46b035 160000
--- a/doc
+++ b/doc
@@ -1 +1 @@
-Subproject commit afd337f4b00477401514ae3dc9c20476229318dc
+Subproject commit 412a46b035a646f890933de84ac2e7b71c6473a7
diff --git a/testing/btest/Baseline/language.type-coerce-numerics/first_set.out b/testing/btest/Baseline/language.type-coerce-numerics/first_set.out
index 7af2ad4a31..4809986d53 100644
--- a/testing/btest/Baseline/language.type-coerce-numerics/first_set.out
+++ b/testing/btest/Baseline/language.type-coerce-numerics/first_set.out
@@ -1,5 +1,5 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error in int and ./first_set.zeek, line 46: overflow promoting from unsigned/double to signed arithmetic value (int and 9223372036854775808)
+error in int: overflow promoting from unsigned/double to signed arithmetic value (int and 9223372036854775808)
 3
 int
 4

From 95b89be57160c54efb1adc35d5aa71f5fd43af0d Mon Sep 17 00:00:00 2001
From: Vern Paxson <vern@corelight.com>
Date: Wed, 24 Mar 2021 21:22:03 -0700
Subject: [PATCH 12/36] put global statements into a quasi-function to support
 script optimization

---
 src/parse.y                 |  2 +-
 src/script_opt/ScriptOpt.cc | 23 +++++++++++++++++++++++
 src/script_opt/ScriptOpt.h  |  5 +++++
 src/zeek-setup.cc           |  8 +++++---
 4 files changed, 34 insertions(+), 4 deletions(-)

diff --git a/src/parse.y b/src/parse.y
index ac7a924e40..03a9f4cb68 100644
--- a/src/parse.y
+++ b/src/parse.y
@@ -266,7 +266,7 @@ static bool expr_is_table_type_name(const zeek::detail::Expr* expr)
 
 %%
 
-bro:
+zeek:
 		decl_list stmt_list
 			{
 			if ( zeek::detail::stmts )
diff --git a/src/script_opt/ScriptOpt.cc b/src/script_opt/ScriptOpt.cc
index 78d99ff0e5..3873b43386 100644
--- a/src/script_opt/ScriptOpt.cc
+++ b/src/script_opt/ScriptOpt.cc
@@ -3,6 +3,7 @@
 #include "zeek/Options.h"
 #include "zeek/Reporter.h"
 #include "zeek/Desc.h"
+#include "zeek/module_util.h"
 #include "zeek/script_opt/ScriptOpt.h"
 #include "zeek/script_opt/ProfileFunc.h"
 #include "zeek/script_opt/Inline.h"
@@ -153,6 +154,28 @@ void analyze_func(ScriptFuncPtr f)
 	funcs.emplace_back(f, ScopePtr{NewRef{}, f->GetScope()}, f->CurrentBody());
 	}
 
+const FuncInfo* analyze_global_stmts(Stmt* stmts)
+	{
+	// We ignore analysis_options.only_func - if it's in use, later
+	// logic will keep this function from being compiled, but it's handy
+	// now to enter it into "funcs" so we have a FuncInfo to return.
+
+	auto id = install_ID("<global-stmts>", GLOBAL_MODULE_NAME, true, false);
+	auto empty_args_t = make_intrusive<RecordType>(nullptr);
+	auto func_t = make_intrusive<FuncType>(empty_args_t, nullptr, FUNC_FLAVOR_FUNCTION);
+	id->SetType(func_t);
+
+	auto sc = current_scope();
+	std::vector<IDPtr> empty_inits;
+	StmtPtr stmts_p{NewRef{}, stmts};
+	auto sf = make_intrusive<ScriptFunc>(id, stmts_p, empty_inits,
+						sc->Length(), 0);
+
+	funcs.emplace_back(sf, ScopePtr{NewRef{}, sc}, stmts_p);
+
+	return &funcs.back();
+	}
+
 static void check_env_opt(const char* opt, bool& opt_flag)
 	{
 	if ( getenv(opt) )
diff --git a/src/script_opt/ScriptOpt.h b/src/script_opt/ScriptOpt.h
index 242f9d987a..4c56bbc52f 100644
--- a/src/script_opt/ScriptOpt.h
+++ b/src/script_opt/ScriptOpt.h
@@ -96,6 +96,11 @@ extern std::unordered_set<const Func*> non_recursive_funcs;
 // Analyze a given function for optimization.
 extern void analyze_func(ScriptFuncPtr f);
 
+// Analyze the given top-level statement(s) for optimization.  Returns
+// a pointer to a FuncInfo for an argument-less quasi-function that can
+// be Invoked, or its body executed directly, to execute the statements.
+extern const FuncInfo* analyze_global_stmts(Stmt* stmts);
+
 // Analyze all of the parsed scripts collectively for optimization.
 extern void analyze_scripts();
 
diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc
index 51e3162c2b..a03f9b3696 100644
--- a/src/zeek-setup.cc
+++ b/src/zeek-setup.cc
@@ -752,6 +752,8 @@ SetupResult setup(int argc, char** argv, Options* zopts)
 			}
 		}
 
+	auto init_stmts = stmts ? analyze_global_stmts(stmts) : nullptr;
+
 	analyze_scripts();
 
 	if ( analysis_options.report_recursive )
@@ -838,15 +840,15 @@ SetupResult setup(int argc, char** argv, Options* zopts)
 	// cause more severe problems.
 	ZEEK_LSAN_ENABLE();
 
-	if ( stmts )
+	if ( init_stmts )
 		{
 		StmtFlowType flow;
-		Frame f(current_scope()->Length(), nullptr, nullptr);
+		Frame f(init_stmts->Scope()->Length(), nullptr, nullptr);
 		g_frame_stack.push_back(&f);
 
 		try
 			{
-			stmts->Exec(&f, flow);
+			init_stmts->Body()->Exec(&f, flow);
 			}
 		catch ( InterpreterException& )
 			{

From 3139cf259441524ab7c443122312706f86cc5eba Mon Sep 17 00:00:00 2001
From: Vern Paxson <vern@corelight.com>
Date: Wed, 24 Mar 2021 21:37:26 -0700
Subject: [PATCH 13/36] "balance" tests with multiple Zeek scripts to load the
 same elements

---
 testing/btest/scripts/base/frameworks/netcontrol/acld-hook.zeek  | 1 +
 testing/btest/scripts/base/frameworks/openflow/broker-basic.zeek | 1 +
 2 files changed, 2 insertions(+)

diff --git a/testing/btest/scripts/base/frameworks/netcontrol/acld-hook.zeek b/testing/btest/scripts/base/frameworks/netcontrol/acld-hook.zeek
index 2698a3bfab..0b0bab2e6b 100644
--- a/testing/btest/scripts/base/frameworks/netcontrol/acld-hook.zeek
+++ b/testing/btest/scripts/base/frameworks/netcontrol/acld-hook.zeek
@@ -9,6 +9,7 @@
 @TEST-START-FILE send.zeek
 
 @load base/frameworks/netcontrol
+@load base/frameworks/broker
 
 redef exit_only_after_terminate = T;
 global have_peer = F;
diff --git a/testing/btest/scripts/base/frameworks/openflow/broker-basic.zeek b/testing/btest/scripts/base/frameworks/openflow/broker-basic.zeek
index e1d05db7a5..f6f166013b 100644
--- a/testing/btest/scripts/base/frameworks/openflow/broker-basic.zeek
+++ b/testing/btest/scripts/base/frameworks/openflow/broker-basic.zeek
@@ -82,6 +82,7 @@ event OpenFlow::flow_mod_failure(name: string, match: OpenFlow::ofp_match, flow_
 
 @TEST-START-FILE recv.zeek
 
+@load base/protocols/conn
 @load base/frameworks/openflow
 
 redef exit_only_after_terminate = T;

From e5a0b2c04c4463aaf74cbf437a5e899985644a6f Mon Sep 17 00:00:00 2001
From: Vern Paxson <vern@corelight.com>
Date: Wed, 24 Mar 2021 21:52:20 -0700
Subject: [PATCH 14/36] whitespace adjustment

---
 src/script_opt/ScriptOpt.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/script_opt/ScriptOpt.cc b/src/script_opt/ScriptOpt.cc
index 3873b43386..65e2cd9266 100644
--- a/src/script_opt/ScriptOpt.cc
+++ b/src/script_opt/ScriptOpt.cc
@@ -168,8 +168,7 @@ const FuncInfo* analyze_global_stmts(Stmt* stmts)
 	auto sc = current_scope();
 	std::vector<IDPtr> empty_inits;
 	StmtPtr stmts_p{NewRef{}, stmts};
-	auto sf = make_intrusive<ScriptFunc>(id, stmts_p, empty_inits,
-						sc->Length(), 0);
+	auto sf = make_intrusive<ScriptFunc>(id, stmts_p, empty_inits, sc->Length(), 0);
 
 	funcs.emplace_back(sf, ScopePtr{NewRef{}, sc}, stmts_p);
 

From c23e3ca1056c03be347da6163a8e447670ce06b1 Mon Sep 17 00:00:00 2001
From: Johanna Amann <johanna@icir.org>
Date: Thu, 25 Mar 2021 13:41:12 +0000
Subject: [PATCH 15/36] Heartbleed: fix substraction order.

The larger number was substracted from the smaller one leading to an
integer overflow. However, no information was lost due to everything
also being present in the notice message.

Fixes GH-1454
---
 scripts/policy/protocols/ssl/heartbleed.zeek | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/policy/protocols/ssl/heartbleed.zeek b/scripts/policy/protocols/ssl/heartbleed.zeek
index 483c1f4ce1..aabafbff14 100644
--- a/scripts/policy/protocols/ssl/heartbleed.zeek
+++ b/scripts/policy/protocols/ssl/heartbleed.zeek
@@ -154,7 +154,7 @@ event ssl_encrypted_heartbeat(c: connection, is_orig: bool, length: count)
 			NOTICE([$note=SSL_Heartbeat_Many_Requests,
 				$msg=fmt("Server sending more heartbeat responses than requests seen. Possible attack. Client count: %d, server count: %d", c$ssl$originator_heartbeats, c$ssl$responder_heartbeats),
 				$conn=c,
-				$n=(c$ssl$originator_heartbeats-c$ssl$responder_heartbeats),
+				$n=(c$ssl$responder_heartbeats-c$ssl$originator_heartbeats),
 				$identifier=fmt("%s%d", c$uid, c$ssl$responder_heartbeats/1000) # re-throw every 1000 heartbeats
 				]);
 

From 4015beb732bcacac1debfc00366c777f02a9855a Mon Sep 17 00:00:00 2001
From: Tim Wojtulewicz <tim@corelight.com>
Date: Wed, 24 Mar 2021 13:31:36 -0700
Subject: [PATCH 16/36] Fix crash in Analyzer::ForwardPacket due to recursive
 analyzer calls.

The change in 44f558df7b5a85bae40945de653bcb2448e0a7f4 that made analyzer_list
a std::vector instead of a std::list doesn't take into account that in some
cases an analyzer may chain back into itself, such as with UDP-in-UDP tunnels.
In these cases, the second call to ForwardPacket may cause iterator
invalidation, leading to a crash.
---
 src/analyzer/Analyzer.h                           |   7 ++++++-
 .../core.tunnels.teredo-udp-in-udp/tunnel.log     |  12 ++++++++++++
 .../btest/Traces/tunnels/teredo-udp-in-udp.pcap   | Bin 0 -> 175 bytes
 testing/btest/core/tunnels/teredo-udp-in-udp.zeek |   4 ++++
 4 files changed, 22 insertions(+), 1 deletion(-)
 create mode 100644 testing/btest/Baseline/core.tunnels.teredo-udp-in-udp/tunnel.log
 create mode 100644 testing/btest/Traces/tunnels/teredo-udp-in-udp.pcap
 create mode 100644 testing/btest/core/tunnels/teredo-udp-in-udp.zeek

diff --git a/src/analyzer/Analyzer.h b/src/analyzer/Analyzer.h
index 8de42eeff7..5999063fe6 100644
--- a/src/analyzer/Analyzer.h
+++ b/src/analyzer/Analyzer.h
@@ -38,7 +38,12 @@ class AnalyzerTimer;
 class SupportAnalyzer;
 class OutputHandler;
 
-using analyzer_list = std::vector<Analyzer*>;
+// This needs to remain a std::list because of the usage of iterators in the
+// Analyzer::Forward methods. These methods have the chance to loop back
+// into the same analyzer in the case of tunnels. If the recursive call adds
+// to the children list, it can invalidate iterators in the outer call,
+// causing a crash.
+using analyzer_list = std::list<Analyzer*>;
 typedef uint32_t ID;
 typedef void (Analyzer::*analyzer_timer_func)(double t);
 
diff --git a/testing/btest/Baseline/core.tunnels.teredo-udp-in-udp/tunnel.log b/testing/btest/Baseline/core.tunnels.teredo-udp-in-udp/tunnel.log
new file mode 100644
index 0000000000..e33e982a23
--- /dev/null
+++ b/testing/btest/Baseline/core.tunnels.teredo-udp-in-udp/tunnel.log
@@ -0,0 +1,12 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	tunnel
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	tunnel_type	action
+#types	time	string	addr	port	addr	port	enum	enum
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	::38:5f55:6265:726b	25977	2090:9090:9090:9090:9090:9000::	25964	Tunnel::TEREDO	Tunnel::DISCOVER
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	::38:5f55:6265:726b	25977	2090:9090:9090:9090:9090:9000::	25964	Tunnel::TEREDO	Tunnel::CLOSE
+#close XXXX-XX-XX-XX-XX-XX
diff --git a/testing/btest/Traces/tunnels/teredo-udp-in-udp.pcap b/testing/btest/Traces/tunnels/teredo-udp-in-udp.pcap
new file mode 100644
index 0000000000000000000000000000000000000000..3f08887024b9bb8d32a068ded9ab794878de8948
GIT binary patch
literal 175
zcmca|c+)~A1{MYcU}Rtfa`KC!6BaCFW@rbpK^P2}z$9ba-2@OLP|yKHfq_MQXi{oX
tw!#EB01Ky9rsgmNa?EF7kOc9WQ&UQTOd|D3W|=S~mZYYHg0!YWNB~3@Bg_B*

literal 0
HcmV?d00001

diff --git a/testing/btest/core/tunnels/teredo-udp-in-udp.zeek b/testing/btest/core/tunnels/teredo-udp-in-udp.zeek
new file mode 100644
index 0000000000..25caf94c5d
--- /dev/null
+++ b/testing/btest/core/tunnels/teredo-udp-in-udp.zeek
@@ -0,0 +1,4 @@
+# @TEST-EXEC: zeek -r $TRACES/tunnels/teredo-udp-in-udp.pcap %INPUT
+# @TEST-EXEC: btest-diff tunnel.log
+
+@load base/frameworks/tunnels

From cbd84dbe523c00411dbbd04eb4cb15e6bb275cc6 Mon Sep 17 00:00:00 2001
From: zeek-bot <info@zeek.org>
Date: Fri, 26 Mar 2021 00:35:30 +0000
Subject: [PATCH 17/36] Update doc submodule [nomail] [skip ci]

---
 doc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc b/doc
index 412a46b035..4bd4cf0233 160000
--- a/doc
+++ b/doc
@@ -1 +1 @@
-Subproject commit 412a46b035a646f890933de84ac2e7b71c6473a7
+Subproject commit 4bd4cf0233ba4d7585ce664f686eb4aff5ccb8dc

From fdf34c1f7b9b0b58c21418dc8780c8af7bf0f224 Mon Sep 17 00:00:00 2001
From: Tim Wojtulewicz <tim@corelight.com>
Date: Thu, 25 Mar 2021 17:38:02 -0700
Subject: [PATCH 18/36] Include git sha in request to benchmark host

---
 ci/benchmark.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/benchmark.sh b/ci/benchmark.sh
index 7823915f63..71c65ab55b 100755
--- a/ci/benchmark.sh
+++ b/ci/benchmark.sh
@@ -35,7 +35,7 @@ TARGET="https://${ZEEK_BENCHMARK_HOST}:${ZEEK_BENCHMARK_PORT}${ZEEK_BENCHMARK_EN
 set +e
 
 # Make a request to the benchmark host.
-RESULTS=$(curl -sS --stderr - --fail --insecure -X POST -H "Zeek-HMAC: ${HMAC_DIGEST}" -H "Zeek-HMAC-Timestamp: ${TIMESTAMP}" "${TARGET}?branch=${CIRRUS_BRANCH}&build=${BUILD_URL}&build_hash=${BUILD_HASH}")
+RESULTS=$(curl -sS --stderr - --fail --insecure -X POST -H "Zeek-HMAC: ${HMAC_DIGEST}" -H "Zeek-HMAC-Timestamp: ${TIMESTAMP}" "${TARGET}?branch=${CIRRUS_BRANCH}&build=${BUILD_URL}&build_hash=${BUILD_HASH}&commit=${CIRRUS_CHANGE_IN_REPO}")
 STATUS=$?
 
 # If we got a bad status back from the host, we want to make sure to mask the host

From acfb21c5a6070f1366a127e49137bfe79b28df30 Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <grigorescu@gmail.com>
Date: Fri, 26 Mar 2021 15:26:48 -0500
Subject: [PATCH 19/36] Rename supervisor_rotation_format_func to
 archiver_rotation_format_func, and expose it for non-supervised setups

Closes #1463
---
 scripts/base/frameworks/cluster/nodes/logger.zeek | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/scripts/base/frameworks/cluster/nodes/logger.zeek b/scripts/base/frameworks/cluster/nodes/logger.zeek
index 772d512658..1021cee100 100644
--- a/scripts/base/frameworks/cluster/nodes/logger.zeek
+++ b/scripts/base/frameworks/cluster/nodes/logger.zeek
@@ -22,11 +22,10 @@ redef Log::default_rotation_interval = 1 hrs;
 ## Alarm summary mail interval.
 redef Log::default_mail_alarms_interval = 24 hrs;
 
-@if ( Supervisor::is_supervised() )
-
-redef Log::default_rotation_dir = "log-queue";
-
-function supervisor_rotation_format_func(ri: Log::RotationFmtInfo): Log::RotationPath
+## This function will rotate logs in a format compatible with zeek-archiver.
+## If you're using the Supervisor framework, this function will be used,
+## if not, you can set :zeek:see:`Log::rotation_format_func` to this function.
+function archiver_rotation_format_func(ri: Log::RotationFmtInfo): Log::RotationPath
 	{
 	local open_str = strftime(Log::default_rotation_date_format, ri$open);
 	local close_str = strftime(Log::default_rotation_date_format, ri$close);
@@ -35,7 +34,11 @@ function supervisor_rotation_format_func(ri: Log::RotationFmtInfo): Log::Rotatio
 	return rval;
 	}
 
-redef Log::rotation_format_func = supervisor_rotation_format_func;
+@if ( Supervisor::is_supervised() )
+
+redef Log::default_rotation_dir = "log-queue";
+
+redef Log::rotation_format_func = archiver_rotation_format_func;
 
 redef LogAscii::enable_leftover_log_rotation = T;
 

From 444fd8ff43d645ea7e114889a8e720cb806c1999 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Fri, 26 Mar 2021 16:26:49 -0700
Subject: [PATCH 20/36] GH-960: Fix include order of bundled header files

Previously, a system-wide installation of any bundled auxil/ software
(like CAF) may get found/included rather than the bundled version and
possibly break the build.
---
 CMakeLists.txt | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 17edb97e8c..6c3112c527 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -376,9 +376,6 @@ endif ()
 # we have to care about CAF here because Broker headers can pull in CAF
 # headers.
 set(zeekdeps ${zeekdeps} ${CAF_LIBRARIES})
-include_directories(BEFORE ${broker_includes} ${CAF_INCLUDE_DIRS})
-include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/auxil/paraglob/include)
-include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/auxil/rapidjson/include)
 include_directories(BEFORE
                     ${PCAP_INCLUDE_DIR}
                     ${BIND_INCLUDE_DIR}
@@ -441,9 +438,6 @@ endif ()
 # if one specifies --with-openssl (which may be common).
 include_directories(BEFORE ${OPENSSL_INCLUDE_DIR})
 
-# Make everyone find the highwayhash includes
-include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/auxil/highwayhash)
-
 # Determine if libfts is external to libc, i.e. musl
 find_package(FTS)
 if ( FTS_FOUND )
@@ -451,6 +445,14 @@ if ( FTS_FOUND )
     include_directories(BEFORE ${FTS_INCLUDE_DIR})
 endif ()
 
+# Any headers that are possibly bundled in the Zeek source-tree and that are supposed
+# to have priority over any pre-existing/system-wide headers need to appear early in
+# compiler search path.
+include_directories(BEFORE ${broker_includes} ${CAF_INCLUDE_DIRS})
+include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/auxil/highwayhash)
+include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/auxil/paraglob/include)
+include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/auxil/rapidjson/include)
+
 set(zeekdeps ${zeekdeps}
     ${BinPAC_LIBRARY}
     ${PCAP_LIBRARY}

From 0eaeca6569111c243c455bc7b393fe773ebb5011 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Fri, 26 Mar 2021 16:51:25 -0700
Subject: [PATCH 21/36] Update submodule(s) [nomail] [skip ci]

---
 auxil/package-manager | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/auxil/package-manager b/auxil/package-manager
index cd89d32f1f..ac410375c5 160000
--- a/auxil/package-manager
+++ b/auxil/package-manager
@@ -1 +1 @@
-Subproject commit cd89d32f1f856544479e4c3d3d994eadf6759034
+Subproject commit ac410375c5bc98b58523d7a4c15e36fc90154559

From 2855df63ce69661356002974bf9069c24e5a85c2 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Fri, 26 Mar 2021 18:57:42 -0700
Subject: [PATCH 22/36] Add RecordVal::AssignField() and use it in supervisor
 code

This is a convenience method to assign a known record field value by
field name.  May also be useful to reduce warnings from static analysis
(e.g. Coverity) about not checking for negative return values before
assigning since that now flows through a [[noreturn]] error path.
---
 src/Val.h                     | 13 ++++++++++++
 src/supervisor/Supervisor.cc  | 40 +++++++++++++++++------------------
 src/supervisor/supervisor.bif |  2 +-
 3 files changed, 33 insertions(+), 22 deletions(-)

diff --git a/src/Val.h b/src/Val.h
index 4d4984db0a..9b402b0d2d 100644
--- a/src/Val.h
+++ b/src/Val.h
@@ -1152,6 +1152,19 @@ public:
 	void Assign(int field, String* new_val)
 		{ Assign(field, new StringVal(new_val)); }
 
+	/**
+	 * Assign a value of type @c T to a record field of the given name.
+	 * A fatal error occurs if the no such field name exists.
+	 */
+	template <class T>
+	void AssignField(const char* field_name, T&& val)
+		{
+		int idx = GetType()->AsRecordType()->FieldOffset(field_name);
+		if ( idx < 0 )
+			reporter->InternalError("missing record field: %s", field_name);
+		Assign(idx, std::forward<T>(val));
+		}
+
 	/**
 	 * Appends a value to the record's fields.  The caller is responsible
 	 * for ensuring that fields are appended in the correct order and
diff --git a/src/supervisor/Supervisor.cc b/src/supervisor/Supervisor.cc
index ce3fe8d220..d751387a1e 100644
--- a/src/supervisor/Supervisor.cc
+++ b/src/supervisor/Supervisor.cc
@@ -1348,22 +1348,22 @@ RecordValPtr Supervisor::NodeConfig::ToRecord() const
 	{
 	const auto& rt = BifType::Record::Supervisor::NodeConfig;
 	auto rval = make_intrusive<RecordVal>(rt);
-	rval->Assign(rt->FieldOffset("name"), name);
+	rval->AssignField("name", name);
 
 	if ( interface )
-		rval->Assign(rt->FieldOffset("interface"), *interface);
+		rval->AssignField("interface", *interface);
 
 	if ( directory )
-		rval->Assign(rt->FieldOffset("directory"), *directory);
+		rval->AssignField("directory", *directory);
 
 	if ( stdout_file )
-		rval->Assign(rt->FieldOffset("stdout_file"), *stdout_file);
+		rval->AssignField("stdout_file", *stdout_file);
 
 	if ( stderr_file )
-		rval->Assign(rt->FieldOffset("stderr_file"), *stderr_file);
+		rval->AssignField("stderr_file", *stderr_file);
 
 	if ( cpu_affinity )
-		rval->Assign(rt->FieldOffset("cpu_affinity"), *cpu_affinity);
+		rval->AssignField("cpu_affinity", *cpu_affinity);
 
 	auto st = rt->GetFieldType<VectorType>("scripts");
 	auto scripts_val = make_intrusive<VectorVal>(std::move(st));
@@ -1371,11 +1371,11 @@ RecordValPtr Supervisor::NodeConfig::ToRecord() const
 	for ( const auto& s : scripts )
 		scripts_val->Assign(scripts_val->Size(), make_intrusive<StringVal>(s));
 
-	rval->Assign(rt->FieldOffset("scripts"), std::move(scripts_val));
+	rval->AssignField("scripts", std::move(scripts_val));
 
 	auto tt = rt->GetFieldType<TableType>("cluster");
 	auto cluster_val = make_intrusive<TableVal>(std::move(tt));
-	rval->Assign(rt->FieldOffset("cluster"), cluster_val);
+	rval->AssignField("cluster", cluster_val);
 
 	for ( const auto& e : cluster )
 		{
@@ -1385,12 +1385,12 @@ RecordValPtr Supervisor::NodeConfig::ToRecord() const
 		const auto& ept = BifType::Record::Supervisor::ClusterEndpoint;
 		auto val = make_intrusive<RecordVal>(ept);
 
-		val->Assign(ept->FieldOffset("role"), BifType::Enum::Supervisor::ClusterRole->GetEnumVal(ep.role));
-		val->Assign(ept->FieldOffset("host"), make_intrusive<AddrVal>(ep.host));
-		val->Assign(ept->FieldOffset("p"), val_mgr->Port(ep.port, TRANSPORT_TCP));
+		val->AssignField("role", BifType::Enum::Supervisor::ClusterRole->GetEnumVal(ep.role));
+		val->AssignField("host", make_intrusive<AddrVal>(ep.host));
+		val->AssignField("p", val_mgr->Port(ep.port, TRANSPORT_TCP));
 
 		if ( ep.interface )
-			val->Assign(ept->FieldOffset("interface"), *ep.interface);
+			val->AssignField("interface", *ep.interface);
 
 		cluster_val->Assign(std::move(key), std::move(val));
 		}
@@ -1403,10 +1403,10 @@ RecordValPtr SupervisorNode::ToRecord() const
 	const auto& rt = BifType::Record::Supervisor::NodeStatus;
 	auto rval = make_intrusive<RecordVal>(rt);
 
-	rval->Assign(rt->FieldOffset("node"), config.ToRecord());
+	rval->AssignField("node", config.ToRecord());
 
 	if ( pid )
-		rval->Assign(rt->FieldOffset("pid"), pid);
+		rval->AssignField("pid", pid);
 
 	return rval;
 	}
@@ -1458,17 +1458,15 @@ bool SupervisedNode::InitCluster() const
 		auto val = make_intrusive<RecordVal>(cluster_node_type);
 
 		auto node_type = supervisor_role_to_cluster_node_type(ep.role);
-		val->Assign(cluster_node_type->FieldOffset("node_type"), std::move(node_type));
-		val->Assign(cluster_node_type->FieldOffset("ip"), make_intrusive<AddrVal>(ep.host));
-		val->Assign(cluster_node_type->FieldOffset("p"), val_mgr->Port(ep.port, TRANSPORT_TCP));
+		val->AssignField("node_type", std::move(node_type));
+		val->AssignField("ip", make_intrusive<AddrVal>(ep.host));
+		val->AssignField("p", val_mgr->Port(ep.port, TRANSPORT_TCP));
 
 		if ( ep.interface )
-			val->Assign(cluster_node_type->FieldOffset("interface"),
-			            *ep.interface);
+			val->AssignField("interface", *ep.interface);
 
 		if ( manager_name && ep.role != BifEnum::Supervisor::MANAGER )
-			val->Assign(cluster_node_type->FieldOffset("manager"),
-			            *manager_name);
+			val->AssignField("manager", *manager_name);
 
 		cluster_nodes->Assign(std::move(key), std::move(val));
 		}
diff --git a/src/supervisor/supervisor.bif b/src/supervisor/supervisor.bif
index c143bc1dcf..3d65ab31a8 100644
--- a/src/supervisor/supervisor.bif
+++ b/src/supervisor/supervisor.bif
@@ -86,7 +86,7 @@ function Supervisor::__node%(%): Supervisor::NodeConfig
 		zeek::emit_builtin_error("not a supervised process");
 		const auto& rt = zeek::BifType::Record::Supervisor::NodeConfig;
 		auto rval = zeek::make_intrusive<zeek::RecordVal>(rt);
-		rval->Assign(rt->FieldOffset("name"), "<invalid>");
+		rval->AssignField("name", "<invalid>");
 		return rval;
 		}
 

From 7047eb92d8bf610570ef16d4a911916c73180a87 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Fri, 26 Mar 2021 19:02:24 -0700
Subject: [PATCH 23/36] Change RecordVal::GetFieldAs() to use
 std::vector::operator[]

Since the method claims it's up to the user to ensure the field exists
before calling, the extra bounds-checking done by std::vector::at()
isn't needed.
---
 src/Val.h | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/Val.h b/src/Val.h
index 9b402b0d2d..bfbca2b6a6 100644
--- a/src/Val.h
+++ b/src/Val.h
@@ -1301,33 +1301,33 @@ public:
 		if constexpr ( std::is_same_v<T, BoolVal> ||
 		               std::is_same_v<T, IntVal> ||
 		               std::is_same_v<T, EnumVal> )
-			return record_val->at(field).int_val;
+			return record_val->operator[](field).int_val;
 		else if constexpr ( std::is_same_v<T, CountVal> )
-			return record_val->at(field).uint_val;
+			return record_val->operator[](field).uint_val;
 		else if constexpr ( std::is_same_v<T, DoubleVal> ||
 		                    std::is_same_v<T, TimeVal> ||
 		                    std::is_same_v<T, IntervalVal> )
-			return record_val->at(field).double_val;
+			return record_val->operator[](field).double_val;
 		else if constexpr ( std::is_same_v<T, PortVal> )
 			return val_mgr->Port(record_val->at(field).uint_val);
 		else if constexpr ( std::is_same_v<T, StringVal> )
-			return record_val->at(field).string_val->Get();
+			return record_val->operator[](field).string_val->Get();
 		else if constexpr ( std::is_same_v<T, AddrVal> )
-			return record_val->at(field).addr_val->Get();
+			return record_val->operator[](field).addr_val->Get();
 		else if constexpr ( std::is_same_v<T, SubNetVal> )
-			return record_val->at(field).subnet_val->Get();
+			return record_val->operator[](field).subnet_val->Get();
 		else if constexpr ( std::is_same_v<T, File> )
-			return *(record_val->at(field).file_val);
+			return *(record_val->operator[](field).file_val);
 		else if constexpr ( std::is_same_v<T, Func> )
-			return *(record_val->at(field).func_val);
+			return *(record_val->operator[](field).func_val);
 		else if constexpr ( std::is_same_v<T, PatternVal> )
-			return record_val->at(field).re_val->Get();
+			return record_val->operator[](field).re_val->Get();
 		else if constexpr ( std::is_same_v<T, RecordVal> )
-			return record_val->at(field).record_val;
+			return record_val->operator[](field).record_val;
 		else if constexpr ( std::is_same_v<T, VectorVal> )
-			return record_val->at(field).vector_val;
+			return record_val->operator[](field).vector_val;
 		else if constexpr ( std::is_same_v<T, TableVal> )
-			return record_val->at(field).table_val->Get();
+			return record_val->operator[](field).table_val->Get();
 		else
 			{
 			// It's an error to reach here, although because of
@@ -1342,12 +1342,12 @@ public:
 	T GetFieldAs(int field) const
 		{
 		if constexpr ( std::is_integral_v<T> && std::is_signed_v<T> )
-			return record_val->at(field).int_val;
+			return record_val->operator[](field).int_val;
 		else if constexpr ( std::is_integral_v<T> &&
 					std::is_unsigned_v<T> )
-			return record_val->at(field).uint_val;
+			return record_val->operator[](field).uint_val;
 		else if constexpr ( std::is_floating_point_v<T> )
-			return record_val->at(field).double_val;
+			return record_val->operator[](field).double_val;
 
 		// Note: we could add other types here using type traits,
 		// such as is_same_v<T, std::string>, etc.

From db975ac08ec45f35437abd125e4b7bac94b1a2c5 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Fri, 26 Mar 2021 19:06:29 -0700
Subject: [PATCH 24/36] Fix maybe-uninitialized warning in ZVal::ToVal()

Some compilers warn that 'v' may be used uninitialized but shouldn't be
the case in practice since all cases are handled, making it impossible.
---
 src/ZVal.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/ZVal.cc b/src/ZVal.cc
index 119d8c3f2e..0adaa70941 100644
--- a/src/ZVal.cc
+++ b/src/ZVal.cc
@@ -263,8 +263,9 @@ ValPtr ZVal::ToVal(const TypePtr& t) const
 	case TYPE_TIMER:
 	case TYPE_UNION:
 	case TYPE_VOID:
+	default:
 		v = nullptr;
-		reporter->InternalError("bad ret type return tag");
+		reporter->InternalError("bad type in ZVal::ToVal: %s", type_name(t->Tag()));
 	}
 
 	if ( v )

From 43d9bda0077ef21ac80693f02158bc61321e2e3e Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Fri, 26 Mar 2021 21:59:30 -0700
Subject: [PATCH 25/36] Fix sign-compare compiler warning in coerce_to_record()

---
 src/Expr.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Expr.cc b/src/Expr.cc
index 5778df4235..c23fdb6b16 100644
--- a/src/Expr.cc
+++ b/src/Expr.cc
@@ -3890,7 +3890,7 @@ ValPtr RecordCoerceExpr::Fold(Val* v) const
 RecordValPtr coerce_to_record(RecordTypePtr rt, Val* v,
 				const std::vector<int>& map)
 	{
-	auto map_size = map.size();
+	int map_size = map.size();
 	auto val = make_intrusive<RecordVal>(rt);
 	RecordType* val_type = val->GetType()->AsRecordType();
 

From a5e7b2fec76c1083c50134a62e5f4da16b52e387 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Sat, 27 Mar 2021 14:18:16 -0700
Subject: [PATCH 26/36] Update bundled CAF to 0.18.2

---
 CHANGES      | 4 ++++
 VERSION      | 2 +-
 auxil/broker | 2 +-
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/CHANGES b/CHANGES
index ba04905de6..5c6701d981 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,8 @@
 
+4.1.0-dev.427 | 2021-03-27 14:18:16 -0700
+
+  * Update bundled CAF to 0.18.2 (Jon Siwek, Corelight)
+
 4.1.0-dev.426 | 2021-03-26 17:17:45 -0700
 
   * GH-1463: Rename supervisor_rotation_format_func to archiver_rotation_format_func (Vlad Grigorescu)
diff --git a/VERSION b/VERSION
index b335514ae2..39e1e04dc4 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-4.1.0-dev.426
+4.1.0-dev.427
diff --git a/auxil/broker b/auxil/broker
index 7810ec9936..938fd7de65 160000
--- a/auxil/broker
+++ b/auxil/broker
@@ -1 +1 @@
-Subproject commit 7810ec9936d1aa65c7c7d21b87e43c7d67481da0
+Subproject commit 938fd7de65220a986361ec596f2335ad471102d7

From 2ad482535ee36c99d7f9ea3bcd3077cb9c81bc78 Mon Sep 17 00:00:00 2001
From: Andrew Benson <abenson@gmail.com>
Date: Mon, 29 Mar 2021 22:41:31 -0500
Subject: [PATCH 27/36] Fix incomplete-type for struct timeval

---
 src/util.cc | 11 -----------
 src/util.h  | 11 +++++++++++
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/util.cc b/src/util.cc
index ce4f74cc0b..9c73c33deb 100644
--- a/src/util.cc
+++ b/src/util.cc
@@ -5,17 +5,6 @@
 
 #include "zeek/util.h"
 
-#ifdef TIME_WITH_SYS_TIME
-# include <sys/time.h>
-# include <time.h>
-#else
-# ifdef HAVE_SYS_TIME_H
-#  include <sys/time.h>
-# else
-#  include <time.h>
-# endif
-#endif
-
 #ifdef HAVE_DARWIN
 #include <mach/task.h>
 #include <mach/mach_init.h>
diff --git a/src/util.h b/src/util.h
index a4d5c6a524..4da4a10735 100644
--- a/src/util.h
+++ b/src/util.h
@@ -29,6 +29,17 @@
 #include <vector>
 #include <memory> // std::unique_ptr
 
+#ifdef TIME_WITH_SYS_TIME
+# include <sys/time.h>
+# include <time.h>
+#else
+# ifdef HAVE_SYS_TIME_H
+#  include <sys/time.h>
+# else
+#  include <time.h>
+# endif
+#endif
+
 #ifdef DEBUG
 
 #include <assert.h>

From a15d726f955a54767da7b1935c80f116d32591d0 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Tue, 30 Mar 2021 16:03:14 -0700
Subject: [PATCH 28/36] Change a <sys/errno.h> include to <errno.h>

Some systems (e.g. Alpine) may warn that that the former is incorrect.
---
 src/ScannedFile.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ScannedFile.cc b/src/ScannedFile.cc
index 77a6f1cb60..c7c7cafb0e 100644
--- a/src/ScannedFile.cc
+++ b/src/ScannedFile.cc
@@ -1,6 +1,6 @@
 #include "zeek/ScannedFile.h"
 
-#include <sys/errno.h>
+#include <errno.h>
 #include <limits.h> // for PATH_MAX
 
 #include "zeek/DebugLogger.h"

From 8e47494dffd41b5f02a8969bb2daaf894ca4ac3f Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Tue, 30 Mar 2021 16:04:34 -0700
Subject: [PATCH 29/36] Remove newline-eof canonification attempt in
 diff-remove-timestamps

On Alpine (BusyBox `sed`), the previous `sed -e '$a\'` invocation always
added a newline, breaking most every diff.  There doesn't seem to be a
need to attempt normalizing EOF newlines at the moment and doing it that
way doesn't seem to be portable anyway.  If canonifiers need to be
portable, `sed` should be treated as a text-processing tool and POSIX
definition of text-file is zero or more newline-terminated
character-sequences, so if canonification of Baselines via `sed` is
required, those Baselines should always end with a newline to be
considered text files.  I.e. that's not the job of this canonifier, and
changing it also doesn't necessarily generalize since it could be
considered coincidental that diff-remove-timestamps in particular is the
default canonifier that's commonly used while there's still others that
also make use of `sed`.
---
 testing/scripts/diff-remove-timestamps | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/testing/scripts/diff-remove-timestamps b/testing/scripts/diff-remove-timestamps
index 5c5956a0ed..0bcc458739 100755
--- a/testing/scripts/diff-remove-timestamps
+++ b/testing/scripts/diff-remove-timestamps
@@ -9,6 +9,4 @@ else
    sed="sed -E"
 fi
 
-# $a\ adds a newline to the end of the file if it does not exist.
-# This unifies the behavior of BSD and GNU sed
-$sed -e 's/(^|[^0-9])([0-9]{9,10}\.[0-9]{1,8})/\1XXXXXXXXXX.XXXXXX/g' -e  's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g' -e '$a\'
+$sed -e 's/(^|[^0-9])([0-9]{9,10}\.[0-9]{1,8})/\1XXXXXXXXXX.XXXXXX/g' -e  's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g'

From 3e9ac00f21c8b98ff446615edb4928780fabe9c4 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Tue, 30 Mar 2021 16:23:23 -0700
Subject: [PATCH 30/36] Separate stdout from stderr in btest baselines

Redirecting both to the same file can show platform-specific differences
(e.g. Alpine), likely due to different buffering defaults.
---
 testing/btest/Baseline/bifs.bloomfilter/err       |  7 +++++++
 testing/btest/Baseline/bifs.bloomfilter/output    |  6 ------
 testing/btest/Baseline/bifs.strptime/err          |  2 ++
 testing/btest/Baseline/bifs.strptime/out          |  1 -
 testing/btest/Baseline/core.init-error/err        |  3 +++
 testing/btest/Baseline/core.init-error/out        |  2 --
 testing/btest/Baseline/core.pcap.dumper/output    | 11 +++++++----
 .../Baseline/core.reporter-error-in-handler/err   |  3 +++
 .../core.reporter-error-in-handler/output         |  2 --
 .../btest/Baseline/language.common-mistakes/1.err |  2 ++
 .../btest/Baseline/language.common-mistakes/1.out |  1 -
 .../btest/Baseline/language.common-mistakes/2.err |  2 ++
 .../btest/Baseline/language.common-mistakes/2.out |  1 -
 .../btest/Baseline/language.common-mistakes/3.err |  2 ++
 .../btest/Baseline/language.common-mistakes/3.out |  1 -
 testing/btest/Baseline/language.invalid_index/err |  3 +++
 testing/btest/Baseline/language.invalid_index/out |  2 --
 testing/btest/Baseline/language.subnet-errors/err |  3 +++
 testing/btest/Baseline/language.subnet-errors/out |  2 --
 .../language.table-set-iterator-invalidation/err  |  5 +++++
 .../language.table-set-iterator-invalidation/out  |  4 ----
 .../Baseline/language.type-cast-error-dynamic/err |  4 ++++
 .../language.type-cast-error-dynamic/output       |  3 ---
 .../double_convert_failure1.err                   |  2 ++
 .../double_convert_failure1.out                   |  1 -
 .../double_convert_failure2.err                   |  2 ++
 .../double_convert_failure2.out                   |  1 -
 .../language.type-coerce-numerics/first_set.err   |  2 ++
 .../language.type-coerce-numerics/first_set.out   |  1 -
 .../int_convert_failure.err                       |  2 ++
 .../int_convert_failure.out                       |  1 -
 .../language.type-coerce-numerics/vectors.err     |  1 +
 .../Baseline/language.uninitialized-local2/err    |  2 ++
 .../Baseline/language.uninitialized-local2/out    |  1 -
 .../Baseline/language.uninitialized-local3/err    |  3 +++
 .../Baseline/language.uninitialized-local3/out    | 12 +++++-------
 .../.stdout                                       |  2 --
 .../err                                           |  3 +++
 testing/btest/bifs/bloomfilter.zeek               |  3 ++-
 testing/btest/bifs/strptime.zeek                  |  3 ++-
 testing/btest/core/init-error.zeek                |  3 ++-
 testing/btest/core/pcap/dumper.zeek               |  4 +++-
 testing/btest/core/reporter-error-in-handler.zeek |  3 ++-
 testing/btest/language/common-mistakes.zeek       |  9 ++++++---
 testing/btest/language/invalid_index.zeek         |  3 ++-
 testing/btest/language/subnet-errors.zeek         |  3 ++-
 .../language/table-set-iterator-invalidation.zeek |  3 ++-
 .../btest/language/type-cast-error-dynamic.zeek   |  3 ++-
 testing/btest/language/type-coerce-numerics.zeek  | 15 ++++++++++-----
 testing/btest/language/uninitialized-local2.zeek  |  3 ++-
 testing/btest/language/uninitialized-local3.zeek  |  3 ++-
 .../bifs/file_exists_lookup_file.zeek             |  3 ++-
 52 files changed, 106 insertions(+), 63 deletions(-)
 create mode 100644 testing/btest/Baseline/bifs.bloomfilter/err
 create mode 100644 testing/btest/Baseline/bifs.strptime/err
 create mode 100644 testing/btest/Baseline/core.init-error/err
 create mode 100644 testing/btest/Baseline/core.reporter-error-in-handler/err
 create mode 100644 testing/btest/Baseline/language.common-mistakes/1.err
 create mode 100644 testing/btest/Baseline/language.common-mistakes/2.err
 create mode 100644 testing/btest/Baseline/language.common-mistakes/3.err
 create mode 100644 testing/btest/Baseline/language.invalid_index/err
 create mode 100644 testing/btest/Baseline/language.subnet-errors/err
 create mode 100644 testing/btest/Baseline/language.table-set-iterator-invalidation/err
 create mode 100644 testing/btest/Baseline/language.type-cast-error-dynamic/err
 create mode 100644 testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure1.err
 create mode 100644 testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure2.err
 create mode 100644 testing/btest/Baseline/language.type-coerce-numerics/first_set.err
 create mode 100644 testing/btest/Baseline/language.type-coerce-numerics/int_convert_failure.err
 create mode 100644 testing/btest/Baseline/language.type-coerce-numerics/vectors.err
 create mode 100644 testing/btest/Baseline/language.uninitialized-local2/err
 create mode 100644 testing/btest/Baseline/language.uninitialized-local3/err
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.file_exists_lookup_file/err

diff --git a/testing/btest/Baseline/bifs.bloomfilter/err b/testing/btest/Baseline/bifs.bloomfilter/err
new file mode 100644
index 0000000000..2a4c6d7420
--- /dev/null
+++ b/testing/btest/Baseline/bifs.bloomfilter/err
@@ -0,0 +1,7 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+error: incompatible Bloom filter types
+error: incompatible Bloom filter types
+error: incompatible Bloom filter types
+error: incompatible Bloom filter types
+error: false-positive rate must take value between 0 and 1
+error: false-positive rate must take value between 0 and 1
diff --git a/testing/btest/Baseline/bifs.bloomfilter/output b/testing/btest/Baseline/bifs.bloomfilter/output
index 1bbe4592d2..2e6dfc8abe 100644
--- a/testing/btest/Baseline/bifs.bloomfilter/output
+++ b/testing/btest/Baseline/bifs.bloomfilter/output
@@ -1,10 +1,4 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error: incompatible Bloom filter types
-error: incompatible Bloom filter types
-error: incompatible Bloom filter types
-error: incompatible Bloom filter types
-error: false-positive rate must take value between 0 and 1
-error: false-positive rate must take value between 0 and 1
 0
 1
 1
diff --git a/testing/btest/Baseline/bifs.strptime/err b/testing/btest/Baseline/bifs.strptime/err
new file mode 100644
index 0000000000..7770adb5ba
--- /dev/null
+++ b/testing/btest/Baseline/bifs.strptime/err
@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+warning: strptime conversion failed: fmt:%m d:1980-10-24
diff --git a/testing/btest/Baseline/bifs.strptime/out b/testing/btest/Baseline/bifs.strptime/out
index a6c49aa77c..8784f8dc3e 100644
--- a/testing/btest/Baseline/bifs.strptime/out
+++ b/testing/btest/Baseline/bifs.strptime/out
@@ -1,4 +1,3 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-warning: strptime conversion failed: fmt:%m d:1980-10-24
 XXXXXXXXXX.XXXXXX
 0.0
diff --git a/testing/btest/Baseline/core.init-error/err b/testing/btest/Baseline/core.init-error/err
new file mode 100644
index 0000000000..2d1ce60bf3
--- /dev/null
+++ b/testing/btest/Baseline/core.init-error/err
@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+expression error in <...>/init-error.zeek, line 16: no such index (v[10])
+fatal error: errors occurred while initializing
diff --git a/testing/btest/Baseline/core.init-error/out b/testing/btest/Baseline/core.init-error/out
index a8f510074e..14ab5a1282 100644
--- a/testing/btest/Baseline/core.init-error/out
+++ b/testing/btest/Baseline/core.init-error/out
@@ -1,6 +1,4 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-expression error in <...>/init-error.zeek, line 15: no such index (v[10])
-fatal error: errors occurred while initializing
 1st event
 2nd event
 3rd event
diff --git a/testing/btest/Baseline/core.pcap.dumper/output b/testing/btest/Baseline/core.pcap.dumper/output
index 9ccfcee235..0bfe0ff5ac 100644
--- a/testing/btest/Baseline/core.pcap.dumper/output
+++ b/testing/btest/Baseline/core.pcap.dumper/output
@@ -1,5 +1,8 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-2c2
-< 00000010  ff ff 00 00 01 00 00 00  1d a2 b2 4e 73 00 07 00  |...........Ns...|
----
-> 00000010  00 24 00 00 01 00 00 00  1d a2 b2 4e 73 00 07 00  |.$.........Ns...|
+@@ -1,5 +1,5 @@
+ 00000000  d4 c3 b2 a1 02 00 04 00  00 00 00 00 00 00 00 00  |................|
+-00000010  ff ff 00 00 01 00 00 00  1d a2 b2 4e 73 00 07 00  |...........Ns...|
++00000010  00 24 00 00 01 00 00 00  1d a2 b2 4e 73 00 07 00  |.$.........Ns...|
+ 00000020  4a 00 00 00 4a 00 00 00  52 54 00 12 35 02 08 00  |J...J...RT..5...|
+ 00000030  27 87 94 24 08 00 45 00  00 3c c1 d8 40 00 40 06  |'..$..E..<..@.@.|
+ 00000040  f1 12 0a 00 02 0f c0 96  bb 2b c0 82 00 50 01 28  |.........+...P.(|
diff --git a/testing/btest/Baseline/core.reporter-error-in-handler/err b/testing/btest/Baseline/core.reporter-error-in-handler/err
new file mode 100644
index 0000000000..1c7f00473e
--- /dev/null
+++ b/testing/btest/Baseline/core.reporter-error-in-handler/err
@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+expression error in <...>/reporter-error-in-handler.zeek, line 29: no such index (a[1])
+expression error in <...>/reporter-error-in-handler.zeek, line 23: no such index (a[2])
diff --git a/testing/btest/Baseline/core.reporter-error-in-handler/output b/testing/btest/Baseline/core.reporter-error-in-handler/output
index f233ff3d18..4ffc7f805d 100644
--- a/testing/btest/Baseline/core.reporter-error-in-handler/output
+++ b/testing/btest/Baseline/core.reporter-error-in-handler/output
@@ -1,4 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-expression error in <...>/reporter-error-in-handler.zeek, line 28: no such index (a[1])
-expression error in <...>/reporter-error-in-handler.zeek, line 22: no such index (a[2])
 1st error printed on script level
diff --git a/testing/btest/Baseline/language.common-mistakes/1.err b/testing/btest/Baseline/language.common-mistakes/1.err
new file mode 100644
index 0000000000..571bb4a831
--- /dev/null
+++ b/testing/btest/Baseline/language.common-mistakes/1.err
@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+expression error in ./1.zeek, line 9: field value missing (mr$f)
diff --git a/testing/btest/Baseline/language.common-mistakes/1.out b/testing/btest/Baseline/language.common-mistakes/1.out
index 5496a099cd..9169b4f45e 100644
--- a/testing/btest/Baseline/language.common-mistakes/1.out
+++ b/testing/btest/Baseline/language.common-mistakes/1.out
@@ -1,5 +1,4 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-expression error in ./1.zeek, line 9: field value missing (mr$f)
 bar start
 foo start
 other zeek_init
diff --git a/testing/btest/Baseline/language.common-mistakes/2.err b/testing/btest/Baseline/language.common-mistakes/2.err
new file mode 100644
index 0000000000..c8e7c86b12
--- /dev/null
+++ b/testing/btest/Baseline/language.common-mistakes/2.err
@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+expression error in ./2.zeek, line 7: no such index (t[nope])
diff --git a/testing/btest/Baseline/language.common-mistakes/2.out b/testing/btest/Baseline/language.common-mistakes/2.out
index db87dbf7ec..8e1dd6eefb 100644
--- a/testing/btest/Baseline/language.common-mistakes/2.out
+++ b/testing/btest/Baseline/language.common-mistakes/2.out
@@ -1,3 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-expression error in ./2.zeek, line 7: no such index (t[nope])
 in foo
diff --git a/testing/btest/Baseline/language.common-mistakes/3.err b/testing/btest/Baseline/language.common-mistakes/3.err
new file mode 100644
index 0000000000..41ae45fdce
--- /dev/null
+++ b/testing/btest/Baseline/language.common-mistakes/3.err
@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+expression error in ./3.zeek, line 5: type-checking failed in vector append (v += ok)
diff --git a/testing/btest/Baseline/language.common-mistakes/3.out b/testing/btest/Baseline/language.common-mistakes/3.out
index c6fc21fcc3..8e1dd6eefb 100644
--- a/testing/btest/Baseline/language.common-mistakes/3.out
+++ b/testing/btest/Baseline/language.common-mistakes/3.out
@@ -1,3 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-expression error in ./3.zeek, line 5: type-checking failed in vector append (v += ok)
 in foo
diff --git a/testing/btest/Baseline/language.invalid_index/err b/testing/btest/Baseline/language.invalid_index/err
new file mode 100644
index 0000000000..5910767e32
--- /dev/null
+++ b/testing/btest/Baseline/language.invalid_index/err
@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+expression error in <...>/invalid_index.zeek, line 11: no such index (foo[1])
+expression error in <...>/invalid_index.zeek, line 17: no such index (foo2[1])
diff --git a/testing/btest/Baseline/language.invalid_index/out b/testing/btest/Baseline/language.invalid_index/out
index 80fe32a510..a36cdc6472 100644
--- a/testing/btest/Baseline/language.invalid_index/out
+++ b/testing/btest/Baseline/language.invalid_index/out
@@ -1,6 +1,4 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-expression error in <...>/invalid_index.zeek, line 10: no such index (foo[1])
-expression error in <...>/invalid_index.zeek, line 16: no such index (foo2[1])
 foo[0], 42
 foo2[0], 13
 done
diff --git a/testing/btest/Baseline/language.subnet-errors/err b/testing/btest/Baseline/language.subnet-errors/err
new file mode 100644
index 0000000000..9bd82b682e
--- /dev/null
+++ b/testing/btest/Baseline/language.subnet-errors/err
@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+expression error in <...>/subnet-errors.zeek, line 10: bad IPv4 subnet prefix length: 33 (1.2.3.4 / i)
+expression error in <...>/subnet-errors.zeek, line 19: bad IPv6 subnet prefix length: 129 (:: / i)
diff --git a/testing/btest/Baseline/language.subnet-errors/out b/testing/btest/Baseline/language.subnet-errors/out
index 06dac194d1..b7c2e2e895 100644
--- a/testing/btest/Baseline/language.subnet-errors/out
+++ b/testing/btest/Baseline/language.subnet-errors/out
@@ -1,6 +1,4 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-expression error in <...>/subnet-errors.zeek, line 9: bad IPv4 subnet prefix length: 33 (1.2.3.4 / i)
-expression error in <...>/subnet-errors.zeek, line 18: bad IPv6 subnet prefix length: 129 (:: / i)
 1.2.3.4/32
 ::/128
 init last
diff --git a/testing/btest/Baseline/language.table-set-iterator-invalidation/err b/testing/btest/Baseline/language.table-set-iterator-invalidation/err
new file mode 100644
index 0000000000..e33c980fc3
--- /dev/null
+++ b/testing/btest/Baseline/language.table-set-iterator-invalidation/err
@@ -0,0 +1,5 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+warning in <...>/table-set-iterator-invalidation.zeek, line 22: possible loop/iterator invalidation caused by expression: t[4]
+warning in <...>/table-set-iterator-invalidation.zeek, line 31: possible loop/iterator invalidation caused by expression: t[4]
+warning in <...>/table-set-iterator-invalidation.zeek, line 54: possible loop/iterator invalidation caused by expression: s[4]
+warning in <...>/table-set-iterator-invalidation.zeek, line 63: possible loop/iterator invalidation caused by expression: s[4]
diff --git a/testing/btest/Baseline/language.table-set-iterator-invalidation/out b/testing/btest/Baseline/language.table-set-iterator-invalidation/out
index 38cadd94a2..3c6cc09e27 100644
--- a/testing/btest/Baseline/language.table-set-iterator-invalidation/out
+++ b/testing/btest/Baseline/language.table-set-iterator-invalidation/out
@@ -1,8 +1,4 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-warning in <...>/table-set-iterator-invalidation.zeek, line 21: possible loop/iterator invalidation caused by expression: t[4]
-warning in <...>/table-set-iterator-invalidation.zeek, line 30: possible loop/iterator invalidation caused by expression: t[4]
-warning in <...>/table-set-iterator-invalidation.zeek, line 53: possible loop/iterator invalidation caused by expression: s[4]
-warning in <...>/table-set-iterator-invalidation.zeek, line 62: possible loop/iterator invalidation caused by expression: s[4]
 {
 [2] = 2,
 [1] = 1,
diff --git a/testing/btest/Baseline/language.type-cast-error-dynamic/err b/testing/btest/Baseline/language.type-cast-error-dynamic/err
new file mode 100644
index 0000000000..bc7a985010
--- /dev/null
+++ b/testing/btest/Baseline/language.type-cast-error-dynamic/err
@@ -0,0 +1,4 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+expression error in <...>/type-cast-error-dynamic.zeek, line 12: invalid cast of value with type 'count' to type 'string' (a as string)
+expression error in <...>/type-cast-error-dynamic.zeek, line 12: invalid cast of value with type 'record { a:addr; b:port; }' to type 'string' (a as string)
+expression error in <...>/type-cast-error-dynamic.zeek, line 12: invalid cast of value with type 'record { data:opaque of Broker::Data &optional; }' to type 'string' (nil $data field) (a as string)
diff --git a/testing/btest/Baseline/language.type-cast-error-dynamic/output b/testing/btest/Baseline/language.type-cast-error-dynamic/output
index bf6438e3e6..ec70675fe2 100644
--- a/testing/btest/Baseline/language.type-cast-error-dynamic/output
+++ b/testing/btest/Baseline/language.type-cast-error-dynamic/output
@@ -1,5 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-expression error in <...>/type-cast-error-dynamic.zeek, line 11: invalid cast of value with type 'count' to type 'string' (a as string)
-expression error in <...>/type-cast-error-dynamic.zeek, line 11: invalid cast of value with type 'record { a:addr; b:port; }' to type 'string' (a as string)
-expression error in <...>/type-cast-error-dynamic.zeek, line 11: invalid cast of value with type 'record { data:opaque of Broker::Data &optional; }' to type 'string' (nil $data field) (a as string)
 data is string, F
diff --git a/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure1.err b/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure1.err
new file mode 100644
index 0000000000..833eb76999
--- /dev/null
+++ b/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure1.err
@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+error in ./double_convert_failure1.zeek, line 7 and double: type clash for field "cc" ((coerce [$cc=5.0] to myrecord) and double)
diff --git a/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure1.out b/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure1.out
index 833eb76999..49d861c74c 100644
--- a/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure1.out
+++ b/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure1.out
@@ -1,2 +1 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error in ./double_convert_failure1.zeek, line 7 and double: type clash for field "cc" ((coerce [$cc=5.0] to myrecord) and double)
diff --git a/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure2.err b/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure2.err
new file mode 100644
index 0000000000..c89e0282e1
--- /dev/null
+++ b/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure2.err
@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+error in ./double_convert_failure2.zeek, line 7 and double: type clash for field "cc" ((coerce [$cc=-5.0] to myrecord) and double)
diff --git a/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure2.out b/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure2.out
index c89e0282e1..49d861c74c 100644
--- a/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure2.out
+++ b/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure2.out
@@ -1,2 +1 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error in ./double_convert_failure2.zeek, line 7 and double: type clash for field "cc" ((coerce [$cc=-5.0] to myrecord) and double)
diff --git a/testing/btest/Baseline/language.type-coerce-numerics/first_set.err b/testing/btest/Baseline/language.type-coerce-numerics/first_set.err
new file mode 100644
index 0000000000..20e1c94aae
--- /dev/null
+++ b/testing/btest/Baseline/language.type-coerce-numerics/first_set.err
@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+error in int: overflow promoting from unsigned/double to signed arithmetic value (int and 9223372036854775808)
diff --git a/testing/btest/Baseline/language.type-coerce-numerics/first_set.out b/testing/btest/Baseline/language.type-coerce-numerics/first_set.out
index 4809986d53..b31869c4eb 100644
--- a/testing/btest/Baseline/language.type-coerce-numerics/first_set.out
+++ b/testing/btest/Baseline/language.type-coerce-numerics/first_set.out
@@ -1,5 +1,4 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error in int: overflow promoting from unsigned/double to signed arithmetic value (int and 9223372036854775808)
 3
 int
 4
diff --git a/testing/btest/Baseline/language.type-coerce-numerics/int_convert_failure.err b/testing/btest/Baseline/language.type-coerce-numerics/int_convert_failure.err
new file mode 100644
index 0000000000..9bca49e323
--- /dev/null
+++ b/testing/btest/Baseline/language.type-coerce-numerics/int_convert_failure.err
@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+error in ./int_convert_failure.zeek, line 7 and int: type clash for field "cc" ((coerce [$cc=-5] to myrecord) and int)
diff --git a/testing/btest/Baseline/language.type-coerce-numerics/int_convert_failure.out b/testing/btest/Baseline/language.type-coerce-numerics/int_convert_failure.out
index 9bca49e323..49d861c74c 100644
--- a/testing/btest/Baseline/language.type-coerce-numerics/int_convert_failure.out
+++ b/testing/btest/Baseline/language.type-coerce-numerics/int_convert_failure.out
@@ -1,2 +1 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error in ./int_convert_failure.zeek, line 7 and int: type clash for field "cc" ((coerce [$cc=-5] to myrecord) and int)
diff --git a/testing/btest/Baseline/language.type-coerce-numerics/vectors.err b/testing/btest/Baseline/language.type-coerce-numerics/vectors.err
new file mode 100644
index 0000000000..49d861c74c
--- /dev/null
+++ b/testing/btest/Baseline/language.type-coerce-numerics/vectors.err
@@ -0,0 +1 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
diff --git a/testing/btest/Baseline/language.uninitialized-local2/err b/testing/btest/Baseline/language.uninitialized-local2/err
new file mode 100644
index 0000000000..bf4ff14005
--- /dev/null
+++ b/testing/btest/Baseline/language.uninitialized-local2/err
@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+expression error in <...>/uninitialized-local2.zeek, line 20: value used but not set (var_b)
diff --git a/testing/btest/Baseline/language.uninitialized-local2/out b/testing/btest/Baseline/language.uninitialized-local2/out
index 8d041a13c0..ba5c5cef5e 100644
--- a/testing/btest/Baseline/language.uninitialized-local2/out
+++ b/testing/btest/Baseline/language.uninitialized-local2/out
@@ -1,3 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-expression error in <...>/uninitialized-local2.zeek, line 19: value used but not set (var_b)
 var_a is, baz
diff --git a/testing/btest/Baseline/language.uninitialized-local3/err b/testing/btest/Baseline/language.uninitialized-local3/err
new file mode 100644
index 0000000000..ae81d66dbd
--- /dev/null
+++ b/testing/btest/Baseline/language.uninitialized-local3/err
@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+warning in <...>/uninitialized-local3.zeek, line 39: possibly used without definition (x4)
+expression error in <...>/uninitialized-local3.zeek, line 39: value used but not set (x4)
diff --git a/testing/btest/Baseline/language.uninitialized-local3/out b/testing/btest/Baseline/language.uninitialized-local3/out
index e65bf87f36..ee2f49d506 100644
--- a/testing/btest/Baseline/language.uninitialized-local3/out
+++ b/testing/btest/Baseline/language.uninitialized-local3/out
@@ -1,11 +1,9 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-warning in <...>/uninitialized-local3.zeek, line 38: possibly used without definition (x4)
-expression error in <...>/uninitialized-local3.zeek, line 38: value used but not set (x4)
-x$a (x <...>/uninitialized-local3.zeek, line 20) possibly used without being set
-x$e (x <...>/uninitialized-local3.zeek, line 20) possibly used without being set
-x$e (x <...>/uninitialized-local3.zeek, line 24) possibly used without being set
-x2$worries$a (x2 <...>/uninitialized-local3.zeek, line 27) possibly used without being set
-x2$worries$e (x2 <...>/uninitialized-local3.zeek, line 27) possibly used without being set
+x$a (x <...>/uninitialized-local3.zeek, line 21) possibly used without being set
+x$e (x <...>/uninitialized-local3.zeek, line 21) possibly used without being set
+x$e (x <...>/uninitialized-local3.zeek, line 25) possibly used without being set
+x2$worries$a (x2 <...>/uninitialized-local3.zeek, line 28) possibly used without being set
+x2$worries$e (x2 <...>/uninitialized-local3.zeek, line 28) possibly used without being set
 [a=<uninitialized>, b=<uninitialized>, c=9, d=<uninitialized>, e=<uninitialized>]
 [a=<uninitialized>, b=<uninitialized>, c=9, d=<uninitialized>, e=<uninitialized>]
 [no_worries=[a=<uninitialized>, b=<uninitialized>, c=9, d=<uninitialized>, e=<uninitialized>], worries=[a=<uninitialized>, b=<uninitialized>, c=9, d=<uninitialized>, e=<uninitialized>]]
diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.file_exists_lookup_file/.stdout b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.file_exists_lookup_file/.stdout
index 4ac2117737..ccf66e3400 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.file_exists_lookup_file/.stdout
+++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.file_exists_lookup_file/.stdout
@@ -1,6 +1,4 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error: file ID asdf not a known file
-warning: non-void function returning without a value: Files::lookup_file
 This should fail but not crash
 This should return F
 F
diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.file_exists_lookup_file/err b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.file_exists_lookup_file/err
new file mode 100644
index 0000000000..f92621dcef
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.file_exists_lookup_file/err
@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+error: file ID asdf not a known file
+warning: non-void function returning without a value: Files::lookup_file
diff --git a/testing/btest/bifs/bloomfilter.zeek b/testing/btest/bifs/bloomfilter.zeek
index c0ccbd7080..26b5cdbf5f 100644
--- a/testing/btest/bifs/bloomfilter.zeek
+++ b/testing/btest/bifs/bloomfilter.zeek
@@ -1,5 +1,6 @@
-# @TEST-EXEC: zeek -b %INPUT >output 2>&1
+# @TEST-EXEC: zeek -b %INPUT >output 2>err
 # @TEST-EXEC: btest-diff output
+# @TEST-EXEC: btest-diff err
 
 function test_basic_bloom_filter()
   {
diff --git a/testing/btest/bifs/strptime.zeek b/testing/btest/bifs/strptime.zeek
index 3923ced4c0..e870b5c0f1 100644
--- a/testing/btest/bifs/strptime.zeek
+++ b/testing/btest/bifs/strptime.zeek
@@ -1,6 +1,7 @@
 #
-# @TEST-EXEC: zeek -b %INPUT >out 2>&1
+# @TEST-EXEC: zeek -b %INPUT >out 2>err
 # @TEST-EXEC: btest-diff out
+# @TEST-EXEC: btest-diff err
 
 event zeek_init()
 	{
diff --git a/testing/btest/core/init-error.zeek b/testing/btest/core/init-error.zeek
index 82226e9dfa..70c9239bad 100644
--- a/testing/btest/core/init-error.zeek
+++ b/testing/btest/core/init-error.zeek
@@ -1,7 +1,8 @@
 # The default is for an initialization error to be a hard failure.
 
-# @TEST-EXEC-FAIL: unset ZEEK_ALLOW_INIT_ERRORS && zeek -b %INPUT >out 2>&1
+# @TEST-EXEC-FAIL: unset ZEEK_ALLOW_INIT_ERRORS && zeek -b %INPUT >out 2>err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff err
 
 event zeek_init() &priority=10
 	{
diff --git a/testing/btest/core/pcap/dumper.zeek b/testing/btest/core/pcap/dumper.zeek
index b53c01bc37..21c2148a3b 100644
--- a/testing/btest/core/pcap/dumper.zeek
+++ b/testing/btest/core/pcap/dumper.zeek
@@ -2,7 +2,9 @@
 # @TEST-EXEC: zeek -b -r $TRACES/workshop_2011_browse.trace -w dump
 # @TEST-EXEC: hexdump -C $TRACES/workshop_2011_browse.trace >1
 # @TEST-EXEC: hexdump -C dump >2
-# @TEST-EXEC: diff 1 2 >output || true
+# @TEST-EXEC: diff -U3 1 2 >diff.output || true
+# Remove first two lines of diff since they may contain timestamps
+# @TEST-EXEC: awk 'NR>2' diff.output >output
 
 # Note that we're diff'ing the diff because there is an expected
 # difference in the pcaps: namely, the snaplen setting stored in the
diff --git a/testing/btest/core/reporter-error-in-handler.zeek b/testing/btest/core/reporter-error-in-handler.zeek
index 398c275c32..0b15e445b5 100644
--- a/testing/btest/core/reporter-error-in-handler.zeek
+++ b/testing/btest/core/reporter-error-in-handler.zeek
@@ -2,8 +2,9 @@
 # This test procudes a recursive error: the error handler is itself broken. Rather
 # than looping indefinitly, the error inside the handler should reported to stderr.
 #
-# @TEST-EXEC: zeek -b %INPUT >output 2>&1
+# @TEST-EXEC: zeek -b %INPUT >output 2>err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff output
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff err
 
 global a: table[count] of count;
 
diff --git a/testing/btest/language/common-mistakes.zeek b/testing/btest/language/common-mistakes.zeek
index b829b5315b..b810dd22ea 100644
--- a/testing/btest/language/common-mistakes.zeek
+++ b/testing/btest/language/common-mistakes.zeek
@@ -2,14 +2,17 @@
 # handled internally by way of throwing an exception to unwind out
 # of the current event handler body.
 
-# @TEST-EXEC: zeek -b 1.zeek >1.out 2>&1
+# @TEST-EXEC: zeek -b 1.zeek >1.out 2>1.err
 # @TEST-EXEC: btest-diff 1.out
+# @TEST-EXEC: btest-diff 1.err
 
-# @TEST-EXEC: zeek -b 2.zeek >2.out 2>&1
+# @TEST-EXEC: zeek -b 2.zeek >2.out 2>2.err
 # @TEST-EXEC: btest-diff 2.out
+# @TEST-EXEC: btest-diff 2.err
 
-# @TEST-EXEC: zeek -b 3.zeek >3.out 2>&1
+# @TEST-EXEC: zeek -b 3.zeek >3.out 2>3.err
 # @TEST-EXEC: btest-diff 3.out
+# @TEST-EXEC: btest-diff 3.err
 
 @TEST-START-FILE 1.zeek
 type myrec: record {
diff --git a/testing/btest/language/invalid_index.zeek b/testing/btest/language/invalid_index.zeek
index 80f294c68b..831de104df 100644
--- a/testing/btest/language/invalid_index.zeek
+++ b/testing/btest/language/invalid_index.zeek
@@ -1,5 +1,6 @@
-# @TEST-EXEC: zeek -b %INPUT >out 2>&1
+# @TEST-EXEC: zeek -b %INPUT >out 2>err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff err
 
 global foo: vector of count = { 42 };
 global foo2: table[count] of count = { [0] = 13 };
diff --git a/testing/btest/language/subnet-errors.zeek b/testing/btest/language/subnet-errors.zeek
index 875817c433..4226232cb3 100644
--- a/testing/btest/language/subnet-errors.zeek
+++ b/testing/btest/language/subnet-errors.zeek
@@ -1,5 +1,6 @@
-# @TEST-EXEC: zeek -b %INPUT >out 2>&1
+# @TEST-EXEC: zeek -b %INPUT >out 2>err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff err
 
 event zeek_init()
 	{
diff --git a/testing/btest/language/table-set-iterator-invalidation.zeek b/testing/btest/language/table-set-iterator-invalidation.zeek
index aee45a6cea..6347c69125 100644
--- a/testing/btest/language/table-set-iterator-invalidation.zeek
+++ b/testing/btest/language/table-set-iterator-invalidation.zeek
@@ -1,5 +1,6 @@
-# @TEST-EXEC: zeek -b %INPUT >out 2>&1
+# @TEST-EXEC: zeek -b %INPUT >out 2>err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff err
 
 # Note that while modifying container membership during for-loop iteration is
 # supposed to be undefined-behavior, it should be practically ok to have this
diff --git a/testing/btest/language/type-cast-error-dynamic.zeek b/testing/btest/language/type-cast-error-dynamic.zeek
index 1edf9e3d2a..c636eee0b8 100644
--- a/testing/btest/language/type-cast-error-dynamic.zeek
+++ b/testing/btest/language/type-cast-error-dynamic.zeek
@@ -1,5 +1,6 @@
-# @TEST-EXEC: zeek -b %INPUT >output 2>&1
+# @TEST-EXEC: zeek -b %INPUT >output 2>err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff output
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff err
 
 type X: record  {
     a: addr;
diff --git a/testing/btest/language/type-coerce-numerics.zeek b/testing/btest/language/type-coerce-numerics.zeek
index 7633c02b62..d31d877b35 100644
--- a/testing/btest/language/type-coerce-numerics.zeek
+++ b/testing/btest/language/type-coerce-numerics.zeek
@@ -1,17 +1,22 @@
-# @TEST-EXEC: zeek -b first_set.zeek >first_set.out 2>&1
+# @TEST-EXEC: zeek -b first_set.zeek >first_set.out 2>first_set.err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff first_set.out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff first_set.err
 
-# @TEST-EXEC-FAIL: zeek -b double_convert_failure1.zeek >double_convert_failure1.out 2>&1
+# @TEST-EXEC-FAIL: zeek -b double_convert_failure1.zeek >double_convert_failure1.out 2>double_convert_failure1.err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff double_convert_failure1.out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff double_convert_failure1.err
 
-# @TEST-EXEC-FAIL: zeek -b double_convert_failure2.zeek >double_convert_failure2.out 2>&1
+# @TEST-EXEC-FAIL: zeek -b double_convert_failure2.zeek >double_convert_failure2.out 2>double_convert_failure2.err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff double_convert_failure2.out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff double_convert_failure2.err
 
-# @TEST-EXEC-FAIL: zeek -b int_convert_failure.zeek >int_convert_failure.out 2>&1
+# @TEST-EXEC-FAIL: zeek -b int_convert_failure.zeek >int_convert_failure.out 2>int_convert_failure.err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff int_convert_failure.out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff int_convert_failure.err
 
-# @TEST-EXEC: zeek -b vectors.zeek >vectors.out 2>&1
+# @TEST-EXEC: zeek -b vectors.zeek >vectors.out 2>vectors.err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff vectors.out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff vectors.err
 
 @TEST-START-FILE first_set.zeek
 type myrecord : record {
diff --git a/testing/btest/language/uninitialized-local2.zeek b/testing/btest/language/uninitialized-local2.zeek
index 4b8f0c8275..118ab77c83 100644
--- a/testing/btest/language/uninitialized-local2.zeek
+++ b/testing/btest/language/uninitialized-local2.zeek
@@ -1,5 +1,6 @@
-# @TEST-EXEC: zeek -b %INPUT >out 2>&1
+# @TEST-EXEC: zeek -b %INPUT >out 2>err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff err
 
 event test()
 	{
diff --git a/testing/btest/language/uninitialized-local3.zeek b/testing/btest/language/uninitialized-local3.zeek
index f3aae38f5b..79629d2138 100644
--- a/testing/btest/language/uninitialized-local3.zeek
+++ b/testing/btest/language/uninitialized-local3.zeek
@@ -1,5 +1,6 @@
-# @TEST-EXEC: ZEEK_USAGE_ISSUES=2 zeek -b %INPUT >out 2>&1
+# @TEST-EXEC: ZEEK_USAGE_ISSUES=2 zeek -b %INPUT >out 2>err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff err
 
 type r: record {
 	a: count;
diff --git a/testing/btest/scripts/base/frameworks/file-analysis/bifs/file_exists_lookup_file.zeek b/testing/btest/scripts/base/frameworks/file-analysis/bifs/file_exists_lookup_file.zeek
index 1a8c6e27a1..07e2cccbea 100644
--- a/testing/btest/scripts/base/frameworks/file-analysis/bifs/file_exists_lookup_file.zeek
+++ b/testing/btest/scripts/base/frameworks/file-analysis/bifs/file_exists_lookup_file.zeek
@@ -1,5 +1,6 @@
-# @TEST-EXEC: zeek -b -r $TRACES/http/get.trace %INPUT 2>&1
+# @TEST-EXEC: zeek -b -r $TRACES/http/get.trace %INPUT 2>err
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stdout
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff err
 
 @load base/protocols/http
 

From 3f95abd0dd770f7f5d6ba51681d6f09801038cae Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Tue, 30 Mar 2021 16:36:54 -0700
Subject: [PATCH 31/36] Add CI task for Alpine Linux

---
 .cirrus.yml          |  9 +++++++++
 ci/alpine/Dockerfile | 23 +++++++++++++++++++++++
 2 files changed, 32 insertions(+)
 create mode 100644 ci/alpine/Dockerfile

diff --git a/.cirrus.yml b/.cirrus.yml
index 6cde730bb3..cd7a541d71 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -182,6 +182,15 @@ ubuntu16_task:
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
 
+alpine_task:
+  container:
+    # Alpine releases typically happen every 6 months w/ support for 2 years.
+    # The Dockerfile simply tracks latest Alpine release and shouldn't
+    # generally need updating based on particular Alpine release timelines.
+    dockerfile: ci/alpine/Dockerfile
+    << : *RESOURCES_TEMPLATE
+  << : *CI_TEMPLATE
+
 # Apple doesn't publish official long-term support timelines.
 # We aim to support both the current and previous macOS release.
 macos_big_sur_task:
diff --git a/ci/alpine/Dockerfile b/ci/alpine/Dockerfile
new file mode 100644
index 0000000000..5239c957ff
--- /dev/null
+++ b/ci/alpine/Dockerfile
@@ -0,0 +1,23 @@
+FROM alpine:latest
+
+RUN apk add --no-cache \
+  cmake \
+  make \
+  g++ \
+  python3 \
+  python3-dev \
+  flex \
+  bison \
+  libpcap-dev \
+  openssl-dev \
+  zlib-dev \
+  swig \
+  bash \
+  bsd-compat-headers \
+  linux-headers \
+  fts-dev \
+  git \
+  curl \
+  py3-pip
+
+RUN pip3 install junit2html

From 2f39b66828f26568d5ef395802a5ebb7f275a25e Mon Sep 17 00:00:00 2001
From: Johanna Amann <johanna@corelight.com>
Date: Wed, 31 Mar 2021 15:21:49 +0100
Subject: [PATCH 32/36] CI: Add OpenSUSE Leap 15.2

---
 .cirrus.yml                      |  7 +++++++
 ci/opensuse-leap-15.2/Dockerfile | 25 +++++++++++++++++++++++++
 2 files changed, 32 insertions(+)
 create mode 100644 ci/opensuse-leap-15.2/Dockerfile

diff --git a/.cirrus.yml b/.cirrus.yml
index 6cde730bb3..b6952634ea 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -161,6 +161,13 @@ debian9_32bit_task:
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
 
+opensuse_leap_15_2_task:
+  container:
+    # Opensuse Leap 15.2 EOL: Dec 2021
+    dockerfile: ci/opensuse-leap-15.2/Dockerfile
+    << : *RESOURCES_TEMPLATE
+  << : *CI_TEMPLATE
+
 ubuntu20_task:
   container:
     # Ubuntu 20.04 EOL: April 2025
diff --git a/ci/opensuse-leap-15.2/Dockerfile b/ci/opensuse-leap-15.2/Dockerfile
new file mode 100644
index 0000000000..7b23204cde
--- /dev/null
+++ b/ci/opensuse-leap-15.2/Dockerfile
@@ -0,0 +1,25 @@
+FROM opensuse/leap:15.2
+
+RUN zypper in -y \
+  cmake \
+  make \
+  gcc \
+  gcc-c++ \
+  python3 \
+  python3-devel \
+  flex \
+  bison \
+  libpcap-devel \
+  libopenssl-devel \
+  zlib-devel \
+  swig \
+  git \
+  curl \
+  python3-pip \
+  which \
+  gzip \
+  tar \
+  && rm -rf /var/cache/zypp
+
+
+RUN pip3 install junit2html

From ade37c1dfcf92d0301289d4903716a849f1b456b Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Wed, 31 Mar 2021 10:29:17 -0700
Subject: [PATCH 33/36] Teach ci/init-external-repos.sh to consider user
 permissions

Any errors while setting up external/private test repo should fail the
task for PRs submitted by a user with write/admin permission.
---
 CHANGES                   | 7 +++++++
 VERSION                   | 2 +-
 ci/init-external-repos.sh | 8 +++++++-
 3 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/CHANGES b/CHANGES
index 6c339b6202..c10ed9990d 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,11 @@
 
+4.1.0-dev.448 | 2021-03-31 10:29:17 -0700
+
+  * Teach ci/init-external-repos.sh to consider user permissions (Jon Siwek, Corelight)
+
+    Any errors while setting up external/private test repo should fail the
+    task for PRs submitted by a user with write/admin permission.
+
 4.1.0-dev.447 | 2021-03-31 09:55:05 -0700
 
   * CI: Add OpenSUSE Leap 15.2 (Johanna Amann, Corelight)
diff --git a/VERSION b/VERSION
index 6dde708b58..24757066bf 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-4.1.0-dev.447
+4.1.0-dev.448
diff --git a/ci/init-external-repos.sh b/ci/init-external-repos.sh
index 40bc07c243..562c567cee 100755
--- a/ci/init-external-repos.sh
+++ b/ci/init-external-repos.sh
@@ -39,7 +39,13 @@ if [[ -n "${CIRRUS_CI}" ]] && [[ ! -d zeek-testing-private ]]; then
     # the key is also available in PRs for people with write access to the
     # repo, so we can still try for those cases).
     if [[ -n "${CIRRUS_PR}" ]]; then
-        set +e
+        if [[ "${CIRRUS_USER_PERMISSION}" == "write" ]]; then
+            set -e
+        elif [[ "${CIRRUS_USER_PERMISSION}" == "admin" ]]; then
+            set -e
+        else
+            set +e
+        fi
     else
         set -e
     fi

From 631f19377e5e0b3c38e3f1bdd5266bd622ff1305 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Wed, 31 Mar 2021 10:47:22 -0700
Subject: [PATCH 34/36] Change ci/init-external-repos.sh to use `base64 -d`

The BusyBox version of `base64` does not have `--decode`.
---
 CHANGES                   | 6 ++++++
 VERSION                   | 2 +-
 ci/init-external-repos.sh | 2 +-
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/CHANGES b/CHANGES
index c10ed9990d..29cbd091a2 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,10 @@
 
+4.1.0-dev.449 | 2021-03-31 10:47:22 -0700
+
+  * Change ci/init-external-repos.sh to use `base64 -d` (Jon Siwek, Corelight)
+
+    The BusyBox version of `base64` does not have `--decode`.
+
 4.1.0-dev.448 | 2021-03-31 10:29:17 -0700
 
   * Teach ci/init-external-repos.sh to consider user permissions (Jon Siwek, Corelight)
diff --git a/VERSION b/VERSION
index 24757066bf..f2eff5fe92 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-4.1.0-dev.448
+4.1.0-dev.449
diff --git a/ci/init-external-repos.sh b/ci/init-external-repos.sh
index 562c567cee..277e372eb5 100755
--- a/ci/init-external-repos.sh
+++ b/ci/init-external-repos.sh
@@ -52,7 +52,7 @@ if [[ -n "${CIRRUS_CI}" ]] && [[ ! -d zeek-testing-private ]]; then
 
     banner "Trying to clone zeek-testing-private git repo"
     echo "${ZEEK_TESTING_PRIVATE_SSH_KEY}" > cirrus_key.b64
-    base64 --decode cirrus_key.b64 > cirrus_key
+    base64 -d cirrus_key.b64 > cirrus_key
     rm cirrus_key.b64
     chmod 600 cirrus_key
     git --version

From 3404e4a3d172fe981c8a02120af5ce997b85a75f Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Wed, 31 Mar 2021 11:09:02 -0700
Subject: [PATCH 35/36] Update submodule(s) [nomail] [skip ci]

---
 auxil/package-manager | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/auxil/package-manager b/auxil/package-manager
index ac410375c5..ccdd3f305c 160000
--- a/auxil/package-manager
+++ b/auxil/package-manager
@@ -1 +1 @@
-Subproject commit ac410375c5bc98b58523d7a4c15e36fc90154559
+Subproject commit ccdd3f305cb56040e212dab9ec9239a4bd0cc62b

From 2beaee931814fe01e5fdd7a6ddfc93555d34f466 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@corelight.com>
Date: Wed, 31 Mar 2021 11:58:08 -0700
Subject: [PATCH 36/36] Add ssh to Alpine Dockerfile for retrieving external
 test repos

---
 CHANGES              | 4 ++++
 VERSION              | 2 +-
 ci/alpine/Dockerfile | 1 +
 3 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/CHANGES b/CHANGES
index 29cbd091a2..b3988bb2e4 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,8 @@
 
+4.1.0-dev.451 | 2021-03-31 11:58:08 -0700
+
+  * Add ssh to Alpine Dockerfile for retrieving external test repos (Jon Siwek, Corelight)
+
 4.1.0-dev.449 | 2021-03-31 10:47:22 -0700
 
   * Change ci/init-external-repos.sh to use `base64 -d` (Jon Siwek, Corelight)
diff --git a/VERSION b/VERSION
index f2eff5fe92..d8df332c52 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-4.1.0-dev.449
+4.1.0-dev.451
diff --git a/ci/alpine/Dockerfile b/ci/alpine/Dockerfile
index 5239c957ff..be93e0c3d0 100644
--- a/ci/alpine/Dockerfile
+++ b/ci/alpine/Dockerfile
@@ -18,6 +18,7 @@ RUN apk add --no-cache \
   fts-dev \
   git \
   curl \
+  openssh-client \
   py3-pip
 
 RUN pip3 install junit2html