Merge remote-tracking branch 'origin/master' into topic/jsiwek/gh-893-intrusive-ptr-migration

.github/workflows/coverity-scan.yml

@@ -67,6 +67,7 @@ jobs:
         run: |
           export PATH=`pwd`/coverity-tools/bin:$PATH
           ( cd build && cov-build --dir cov-int make -j 3 )
+          cat build/cov-int/build-log.txt
 
       - name: Submit
         env:

CHANGES

@@ -1,4 +1,59 @@
 
+3.2.0-dev.561 | 2020-05-21 13:05:39 -0700
+
+  * GH-983: Fix opaque Broker types lacking a Type after (de)serialization (Jon Siwek, Corelight)
+
+3.2.0-dev.559 | 2020-05-21 13:04:19 -0700
+
+  * Make SendEvent callable from all threads
+
+    This commit refactors the SendEvent call and moves it from the Input
+    ReaderBackend to to MsgThread. This allows all other types of threads
+    to access this functionality.
+
+    This necessitated a few more changes. Most importantly, one of the
+    ValueToVal methods was moved over to SerialTypes. Whereit arguably
+    belongs - there was nothing that was input-framework specific in
+    that method - and the functionality could come in useful in a number
+    of cases. (Johanna Amann, Corelight)
+
+3.2.0-dev.557 | 2020-05-21 11:41:12 -0700
+
+  * Speed up FuzzBuffer ChunkCount validity check (Justin Azoff, Corelight)
+
+3.2.0-dev.555 | 2020-05-20 11:19:08 -0700
+
+  * Disable output of Reporter messages to stderr in fuzz targets (Jon Siwek, Corelight)
+
+3.2.0-dev.554 | 2020-05-20 10:56:46 -0700
+
+  * Improve standalone fuzz driver timing output (Jon Siwek, Corelight)
+
+  * Skip fuzz inputs that have more than 64 chunks (Justin Azoff, Corelight)
+
+3.2.0-dev.550 | 2020-05-19 10:50:42 -0700
+
+  * Upgrade to latest Broker changes for CAF 0.18 (Dominik Charousset, Corelight)
+
+  * Include pcap.h instead of pcap/dlt.h in packet-fuzzer (Tim Wojtulewicz, Corelight)
+
+3.2.0-dev.547 | 2020-05-18 10:47:54 -0700
+
+  * add packet fuzzer (Justin Azoff)
+
+  * Fix building fuzz targets on macOS (Jon Siwek, Corelight)
+
+  * Highwayhash: small build fix (Johanna Amann, Corelight)
+
+    Turns out that hh_neon should not be compiled on generic arm CPUs.
+    That one is only for aarch64.
+
+3.2.0-dev.539 | 2020-05-15 19:47:55 +0000
+
+  * Replace bzero() with memset() (Noah Treuhaft)
+
+  * Change Coverity action to output build log (Jon Siwek, Corelight)
+
 3.2.0-dev.530 | 2020-05-13 15:05:31 -0700
 
   * GH-906: Fix the regex in url.zeek to better match for find_all_urls (Tim Wojtulewicz, Corelight)

VERSION

@@ -1 +1 @@
-3.2.0-dev.530
+3.2.0-dev.561

aux/binpac

@@ -1 +1 @@
-Subproject commit cdc808b07ba0caf05ed4bd83d906125b10feb695
+Subproject commit 4d3d10cd54b1aa64f30d2fd433252f353c6ea6e0

aux/broker

@@ -1 +1 @@
-Subproject commit d360a228c31a6c6d58afdc9827308471c28cd126
+Subproject commit 6974924007765f70d95e5cf123b6256048ae3af7

aux/zeek-aux

@@ -1 +1 @@
-Subproject commit 43fa1b7f01b1d77ccc1d05cc50f1a1c3ea58b6fa
+Subproject commit 8615abced86b5559fa3203264be55e664b887094

aux/zeekctl

@@ -1 +1 @@
-Subproject commit 2bd666a0c13f8efa6dcbc2d1f125ccc73e333522
+Subproject commit bee11c63923b69a4468838d56067f15bc6ae3a52

cmake

@@ -1 +1 @@
-Subproject commit 2b3cd252b4ef80b5d96c6f8aaac8d63eee86f5da
+Subproject commit 23f4b88f91c537c59ef9a3ad56ec08f021ec2b2c

src/CMakeLists.txt

@@ -338,9 +338,6 @@ set(HH_SRCS
 if (${COMPILER_ARCHITECTURE} STREQUAL "arm")
   set_source_files_properties(${HH_SRCS} PROPERTIES COMPILE_FLAGS
                               -mfloat-abi=hard -march=armv7-a -mfpu=neon)
-  list(APPEND HH_SRCS
-       ../aux/highwayhash/highwayhash/hh_neon.cc
-  )
 elseif (${COMPILER_ARCHITECTURE} STREQUAL "aarch64")
   list(APPEND HH_SRCS
        ../aux/highwayhash/highwayhash/hh_neon.cc

src/Conn.cc

@@ -80,12 +80,12 @@ Connection::Connection(NetSessions* s, const ConnIDKey& k, double t, const ConnI
 	if ( pkt->l2_src )
 		memcpy(orig_l2_addr, pkt->l2_src, sizeof(orig_l2_addr));
 	else
-		bzero(orig_l2_addr, sizeof(orig_l2_addr));
+		memset(orig_l2_addr, 0, sizeof(orig_l2_addr));
 
 	if ( pkt->l2_dst )
 		memcpy(resp_l2_addr, pkt->l2_dst, sizeof(resp_l2_addr));
 	else
-		bzero(resp_l2_addr, sizeof(resp_l2_addr));
+		memset(resp_l2_addr, 0, sizeof(resp_l2_addr));
 
 	vlan = pkt->vlan;
 	inner_vlan = pkt->inner_vlan;

src/IntSet.h

@@ -27,14 +27,11 @@ private:
 	unsigned char* set;
 	};
 
-
-#define bzero(p, size) memset(p, 0, size)
-
 inline IntSet::IntSet(unsigned int n)
 	{
 	size = n / 8 + 1;
 	set = new unsigned char[size];
-	bzero(set, size);
+	memset(set, 0, size);
 	}
 
 inline IntSet::~IntSet()
@@ -65,5 +62,5 @@ inline bool IntSet::Contains(unsigned int i) const
 
 inline void IntSet::Clear()
 	{
-	bzero(set, size);
+	memset(set, 0, size);
 	}

src/broker/Data.h

@@ -116,6 +116,7 @@ public:
 
 protected:
 	DataVal()
+		: OpaqueVal(bro_broker::opaque_of_data_type)
 		{}
 
 	DECLARE_OPAQUE_VALUE(bro_broker::DataVal)
@@ -236,7 +237,10 @@ public:
 	broker::set::iterator it;
 
 protected:
-	SetIterator()	{}
+	SetIterator()
+		: OpaqueVal(bro_broker::opaque_of_set_iterator)
+		{}
+
 	DECLARE_OPAQUE_VALUE(bro_broker::SetIterator)
 };
 
@@ -253,7 +257,10 @@ public:
 	broker::table::iterator it;
 
 protected:
-	TableIterator()	{}
+	TableIterator()
+		: OpaqueVal(bro_broker::opaque_of_table_iterator)
+		{}
+
 	DECLARE_OPAQUE_VALUE(bro_broker::TableIterator)
 };
 
@@ -270,7 +277,10 @@ public:
 	broker::vector::iterator it;
 
 protected:
-	VectorIterator()	{}
+	VectorIterator()
+		: OpaqueVal(bro_broker::opaque_of_vector_iterator)
+		{}
+
 	DECLARE_OPAQUE_VALUE(bro_broker::VectorIterator)
 };
 
@@ -287,7 +297,10 @@ public:
 	broker::vector::iterator it;
 
 protected:
-	RecordIterator()	{}
+	RecordIterator()
+		: OpaqueVal(bro_broker::opaque_of_record_iterator)
+		{}
+
 	DECLARE_OPAQUE_VALUE(bro_broker::RecordIterator)
 };
 

src/broker/Store.cc

@@ -86,13 +86,13 @@ broker::backend to_backend_type(BifEnum::Broker::BackendType type)
 	{
 	switch ( type ) {
 	case BifEnum::Broker::MEMORY:
-		return broker::memory;
+		return broker::backend::memory;
 
 	case BifEnum::Broker::SQLITE:
-		return broker::sqlite;
+		return broker::backend::sqlite;
 
 	case BifEnum::Broker::ROCKSDB:
-		return broker::rocksdb;
+		return broker::backend::rocksdb;
 	}
 
 	throw std::runtime_error("unknown broker backend");
@@ -102,14 +102,14 @@ broker::backend_options to_backend_options(broker::backend backend,
                                            RecordVal* options)
 	{
 	switch ( backend ) {
-	case broker::sqlite:
+	case broker::backend::sqlite:
 		{
 		auto path = options->GetField(0)->AsRecordVal()
 			->GetField(0)->AsStringVal()->CheckString();
 		return {{"path", path}};
 		}
 
-	case broker::rocksdb:
+	case broker::backend::rocksdb:
 		{
 		auto path = options->GetField(1)->AsRecordVal()
 			->GetField(0)->AsStringVal()->CheckString();

src/broker/Store.h

@@ -103,7 +103,9 @@ public:
 	broker::store::proxy proxy;
 
 protected:
-	StoreHandleVal() = default;
+	StoreHandleVal()
+		: OpaqueVal(bro_broker::opaque_of_store_handle)
+		{}
 
 	DECLARE_OPAQUE_VALUE(StoreHandleVal)
 };

src/fuzzers/CMakeLists.txt

@@ -22,16 +22,26 @@ if ( NOT DEFINED ZEEK_FUZZING_ENGINE AND DEFINED ENV{LIB_FUZZING_ENGINE} )
     endif ()
 endif ()
 
+# The bind library is handled a bit hack-ishly since it defaults to linking it
+# as static library by default on Linux, but at least on one common distro,
+# that static library wasn't compiled with -fPIC and so not usable in the
+# shared library we're trying to build.  So instead, the fuzzer executable, not
+# the shared lib, links it.
+string(REGEX MATCH ".*\\.a$" _have_static_bind_lib "${BIND_LIBRARY}")
+
 macro(ADD_FUZZ_TARGET _name)
     set(_fuzz_target zeek-${_name}-fuzzer)
     set(_fuzz_source ${_name}-fuzzer.cc)
 
     add_executable(${_fuzz_target} ${_fuzz_source} ${ARGN})
 
-    target_link_libraries(${_fuzz_target}
+    target_link_libraries(${_fuzz_target} zeek_fuzzer_shared)
-                          zeek_fuzzer_shared
+
-                          ${BIND_LIBRARY}
+    if ( _have_static_bind_lib )
-                          ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS})
+        target_link_libraries(${_fuzz_target} ${BIND_LIBRARY})
+    endif ()
+
+    target_link_libraries(${_fuzz_target} ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS})
 
     if ( DEFINED ZEEK_FUZZING_ENGINE )
         target_link_libraries(${_fuzz_target} ${ZEEK_FUZZING_ENGINE})
@@ -55,12 +65,11 @@ add_library(zeek_fuzzer_shared SHARED
 set(zeek_fuzzer_shared_deps)
 
 foreach(_dep ${zeekdeps} )
-    # The bind library is handled a bit hack-ishly since it defaults to
+    if ( "${_dep}" STREQUAL "${BIND_LIBRARY}" )
-    # linking it as static library by default on Linux, but at least
+        if ( NOT _have_static_bind_lib )
-    # on one common distro, that static library wasn't compiled with -fPIC
+            set(zeek_fuzzer_shared_deps ${zeek_fuzzer_shared_deps} ${_dep})
-    # and so not usable in the shared library we're trying to build.
+        endif ()
-    # So instead, the fuzzer executable, not the shared lib, links it.
+    else ()
-    if ( NOT "${_dep}" STREQUAL "${BIND_LIBRARY}" )
         set(zeek_fuzzer_shared_deps ${zeek_fuzzer_shared_deps} ${_dep})
     endif ()
 endforeach ()
@@ -70,3 +79,4 @@ target_link_libraries(zeek_fuzzer_shared
                       ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS})
 
 add_fuzz_target(pop3)
+add_fuzz_target(packet)

src/fuzzers/FuzzBuffer.cc

@@ -6,7 +6,7 @@
 
 #include "FuzzBuffer.h"
 
-bool zeek::detail::FuzzBuffer::Valid() const
+bool zeek::detail::FuzzBuffer::Valid(int chunk_count_limit) const
 	{
 	if ( end - begin < PKT_MAGIC_LEN + 2 )
 		return false;
@@ -14,9 +14,31 @@ bool zeek::detail::FuzzBuffer::Valid() const
 	if ( memcmp(begin, PKT_MAGIC, PKT_MAGIC_LEN) != 0)
 		return false;
 
+	if ( ExceedsChunkLimit(chunk_count_limit) )
+		return false;
+
 	return true;
 	}
 
+int zeek::detail::FuzzBuffer::ChunkCount(int chunk_count_limit) const
+	{
+	auto pos = begin;
+	int chunks = 0;
+
+	while ( pos < end && (chunks < chunk_count_limit || chunk_count_limit == 0) )
+		{
+		pos = (const unsigned char*)memmem(pos, end - pos,
+		                                   PKT_MAGIC, PKT_MAGIC_LEN);
+		if ( ! pos )
+			break;
+
+		pos += PKT_MAGIC_LEN + 1;
+		chunks++;
+		}
+
+	return chunks;
+	}
+
 std::optional<zeek::detail::FuzzBuffer::Chunk> zeek::detail::FuzzBuffer::Next()
 	{
 	if ( begin == end )

src/fuzzers/FuzzBuffer.h

@@ -27,6 +27,7 @@ public:
 
 	static constexpr int PKT_MAGIC_LEN = 4;
 	static constexpr unsigned char PKT_MAGIC[PKT_MAGIC_LEN + 1] = "\1PKT";
+	static constexpr int MAX_CHUNK_COUNT = 64;
 
 	/**
 	 * Initialize fuzz buffer.
@@ -39,9 +40,25 @@ public:
 
 	/**
 	 * @return  whether the fuzz buffer object is valid --  has enough bytes
-	 * to Deliver to an analyzer and starts with a *PKT_MAGIC* bytestring.
+	 * to Deliver to an analyzer, starts with a *PKT_MAGIC* bytestring, and
+	 * contains less than the limiting number of chunk.
+	 * .
 	 */
-	bool Valid() const;
+	bool Valid(int chunk_count_limit = MAX_CHUNK_COUNT) const;
+
+	/**
+	 * @param chunk_count_limit  Number of chunks to stop counting at (zero
+	 * means "never stop").
+	 * @return  the number of chunks in the fuzz buffer object
+	 */
+	int ChunkCount(int chunk_count_limit = 0) const;
+
+	/**
+	 * @param  Maximum number of chunks to permit the FuzzBuffer to have.
+	 * @return  Whether the FuzzBuffer exceeds the desired chunk count limit.
+	 */
+	bool ExceedsChunkLimit(int chunk_count_limit) const
+		{ return ChunkCount(chunk_count_limit + 1) > chunk_count_limit; }
 
 	/**
 	 * @return  the next chunk to deliver, if one could be extracted

src/fuzzers/fuzzer-setup.h

@@ -32,6 +32,9 @@ extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv)
 	options.scripts_to_load.emplace_back("local.zeek");
 	options.script_options_to_set.emplace_back("Site::local_nets={10.0.0.0/8}");
 	options.script_options_to_set.emplace_back("Log::default_writer=Log::WRITER_NONE");
+	options.script_options_to_set.emplace_back("Reporter::info_to_stderr=F");
+	options.script_options_to_set.emplace_back("Reporter::warnings_to_stderr=F");
+	options.script_options_to_set.emplace_back("Reporter::errors_to_stderr=F");
 	options.deterministic_mode = true;
 	options.ignore_checksums = true;
 	options.abort_on_scripting_errors = true;

src/fuzzers/packet-fuzzer.cc

@@ -0,0 +1,47 @@
+#include "binpac.h"
+
+#include "iosource/Packet.h"
+#include "Event.h"
+#include "Sessions.h"
+
+#include "FuzzBuffer.h"
+#include "fuzzer-setup.h"
+
+extern "C" {
+#include <pcap.h>
+}
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
+	{
+	zeek::detail::FuzzBuffer fb{data, size};
+
+	if ( ! fb.Valid() )
+		return 0;
+
+	for ( ; ;  )
+		{
+		auto chunk = fb.Next();
+
+		if ( ! chunk )
+			break;
+
+		Packet pkt;
+		auto timestamp = 42;
+		pkt_timeval ts = {timestamp, 0};
+		pkt.Init(DLT_RAW, &ts, chunk->size, chunk->size, chunk->data.get(), false, "");
+
+		try
+			{
+			sessions->NextPacket(timestamp, &pkt);
+			}
+		catch ( binpac::Exception const &e )
+			{
+			}
+
+		chunk = {};
+		mgr.Drain();
+		}
+
+	zeek::detail::fuzzer_cleanup_one_input();
+	return 0;
+	}

src/fuzzers/standalone-driver.cc

@@ -17,6 +17,7 @@ int main(int argc, char** argv)
 	printf("Standalone fuzzer processing %d inputs\n", num_inputs);
 
 	LLVMFuzzerInitialize(&argc, &argv);
+	auto fuzz_start = high_resolution_clock::now();
 
 	for ( auto i = 0; i < num_inputs; ++i )
 		{
@@ -60,5 +61,7 @@ int main(int argc, char** argv)
 
 	auto agg_stop = high_resolution_clock::now();
 	auto agg_dt = duration<double>(agg_stop - agg_start).count();
-	printf("Processed %d inputs in %fs\n", num_inputs, agg_dt);
+	auto fuzz_dt = duration<double>(agg_stop - fuzz_start).count();
+	printf("Processed %d inputs in %fs (%fs w/ initialization), avg = %fs\n",
+	       num_inputs, fuzz_dt, agg_dt, fuzz_dt / num_inputs);
 	}

src/input/Manager.cc

@@ -26,14 +26,6 @@ using namespace std;
 using threading::Value;
 using threading::Field;
 
-static void delete_value_ptr_array(Value** vals, int num_fields)
-	{
-	for ( int i = 0; i < num_fields; ++i )
-		delete vals[i];
-
-	delete [] vals;
-	}
-
 /**
  * InputHashes are used as Dictionaries to store the value and index hashes
  * for all lines currently stored in a table. Index hash is stored as
@@ -1091,7 +1083,7 @@ void Manager::SendEntry(ReaderFrontend* reader, Value* *vals)
 	else
 		assert(false);
 
-	delete_value_ptr_array(vals, readFields);
+	Value::delete_value_ptr_array(vals, readFields);
 	}
 
 int Manager::SendEntryTable(Stream* i, const Value* const *vals)
@@ -1458,7 +1450,7 @@ void Manager::Put(ReaderFrontend* reader, Value* *vals)
 	else
 		assert(false);
 
-	delete_value_ptr_array(vals, readFields);
+	Value::delete_value_ptr_array(vals, readFields);
 	}
 
 int Manager::SendEventStreamEvent(Stream* i, EnumVal* type, const Value* const *vals)
@@ -1762,7 +1754,7 @@ bool Manager::Delete(ReaderFrontend* reader, Value* *vals)
 		return false;
 		}
 
-	delete_value_ptr_array(vals, readVals);
+	Value::delete_value_ptr_array(vals, readVals);
 	return success;
 	}
 
@@ -1787,68 +1779,6 @@ bool Manager::CallPred(Func* pred_func, const int numvals, ...) const
 	return result;
 	}
 
-// Raise everything in here as warnings so it is passed to scriptland without
-// looking "fatal". In addition to these warnings, ReaderBackend will queue
-// one reporter message.
-bool Manager::SendEvent(ReaderFrontend* reader, const string& name, const int num_vals, Value* *vals) const
-	{
-	Stream *i = FindStream(reader);
-	if ( i == nullptr )
-		{
-		reporter->InternalWarning("Unknown reader %s in SendEvent for event %s", reader->Name(), name.c_str());
-		delete_value_ptr_array(vals, num_vals);
-		return false;
-		}
-
-	EventHandler* handler = event_registry->Lookup(name);
-	if ( handler == nullptr )
-		{
-		Warning(i, "Event %s not found", name.c_str());
-		delete_value_ptr_array(vals, num_vals);
-		return false;
-		}
-
-#ifdef DEBUG
-	DBG_LOG(DBG_INPUT, "SendEvent for event %s with %d vals",
-		name.c_str(), num_vals);
-#endif
-
-	const auto& type = handler->GetType()->Params();
-	int num_event_vals = type->NumFields();
-	if ( num_vals != num_event_vals )
-		{
-		Warning(i, "Wrong number of values for event %s", name.c_str());
-		delete_value_ptr_array(vals, num_vals);
-		return false;
-		}
-
-	bool convert_error = false;
-
-	zeek::Args vl;
-	vl.reserve(num_vals);
-
-	for ( int j = 0; j < num_vals; j++)
-		{
-		Val* v = ValueToVal(i, vals[j], convert_error);
-		vl.emplace_back(AdoptRef{}, v);
-
-		if ( v && ! convert_error && ! same_type(type->GetFieldType(j).get(), v->GetType().get()) )
-			{
-			convert_error = true;
-			type->GetFieldType(j)->Error("SendEvent types do not match", v->GetType().get());
-			}
-		}
-
-	delete_value_ptr_array(vals, num_vals);
-
-	if ( convert_error )
-		return false;
-	else if ( handler )
-		mgr.Enqueue(handler, std::move(vl), SOURCE_LOCAL);
-
-	return true;
-}
-
 void Manager::SendEvent(EventHandlerPtr ev, const int numvals, ...) const
 	{
 	zeek::Args vl;
@@ -2228,9 +2158,9 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) c
 	}
 
 // convert threading value to Bro value
-// have_error is a reference to a boolean which is set to true as soon as an error occured.
+// have_error is a reference to a boolean which is set to true as soon as an error occurs.
 // When have_error is set to true at the beginning of the function, it is assumed that
-// an error already occured in the past and processing is aborted.
+// an error already occurred in the past and processing is aborted.
 Val* Manager::ValueToVal(const Stream* i, const Value* val, BroType* request_type, bool& have_error) const
 	{
 	if ( have_error )
@@ -2383,204 +2313,6 @@ Val* Manager::ValueToVal(const Stream* i, const Value* val, BroType* request_typ
 	return nullptr;
 	}
 
-Val* Manager::ValueToVal(const Stream* i, const Value* val, bool& have_error) const
-	{
-	if ( have_error )
-		return nullptr;
-
-	if ( ! val->present )
-		return nullptr; // unset field
-
-	switch ( val->type ) {
-	case TYPE_BOOL:
-		return val_mgr->Bool(val->val.int_val)->Ref();
-
-	case TYPE_INT:
-		return val_mgr->Int(val->val.int_val).release();
-
-	case TYPE_COUNT:
-	case TYPE_COUNTER:
-		return val_mgr->Count(val->val.int_val).release();
-
-	case TYPE_DOUBLE:
-	case TYPE_TIME:
-	case TYPE_INTERVAL:
-		return new Val(val->val.double_val, val->type);
-
-	case TYPE_STRING:
-		{
-		BroString *s = new BroString((const u_char*)val->val.string_val.data, val->val.string_val.length, true);
-		return new StringVal(s);
-		}
-
-	case TYPE_PORT:
-		return val_mgr->Port(val->val.port_val.port, val->val.port_val.proto)->Ref();
-
-	case TYPE_ADDR:
-		{
-		IPAddr* addr = nullptr;
-		switch ( val->val.addr_val.family ) {
-		case IPv4:
-			addr = new IPAddr(val->val.addr_val.in.in4);
-			break;
-
-		case IPv6:
-			addr = new IPAddr(val->val.addr_val.in.in6);
-			break;
-
-		default:
-			assert(false);
-		}
-
-		AddrVal* addrval = new AddrVal(*addr);
-		delete addr;
-		return addrval;
-		}
-
-	case TYPE_SUBNET:
-		{
-		IPAddr* addr = nullptr;
-		switch ( val->val.subnet_val.prefix.family ) {
-		case IPv4:
-			addr = new IPAddr(val->val.subnet_val.prefix.in.in4);
-			break;
-
-		case IPv6:
-			addr = new IPAddr(val->val.subnet_val.prefix.in.in6);
-			break;
-
-		default:
-			assert(false);
-		}
-
-		SubNetVal* subnetval = new SubNetVal(*addr, val->val.subnet_val.length);
-		delete addr;
-		return subnetval;
-		}
-
-	case TYPE_PATTERN:
-		{
-		RE_Matcher* re = new RE_Matcher(val->val.pattern_text_val);
-		re->Compile();
-		return new PatternVal(re);
-		}
-
-	case TYPE_TABLE:
-		{
-		IntrusivePtr<TypeList> set_index;
-		if ( val->val.set_val.size == 0 && val->subtype == TYPE_VOID )
-			// don't know type - unspecified table.
-			set_index = make_intrusive<TypeList>();
-		else
-			{
-			// all entries have to have the same type...
-			TypeTag stag = val->subtype;
-			if ( stag == TYPE_VOID )
-				TypeTag stag = val->val.set_val.vals[0]->type;
-
-			IntrusivePtr<BroType> index_type;
-
-			if ( stag == TYPE_ENUM )
-				{
-				// Enums are not a base-type, so need to look it up.
-				const auto& sv = val->val.set_val.vals[0]->val.string_val;
-				std::string enum_name(sv.data, sv.length);
-				const auto& enum_id = global_scope()->Find(enum_name);
-
-				if ( ! enum_id )
-					{
-					Warning(i, "Value '%s' for stream '%s' is not a valid enum.",
-					        enum_name.data(), i->name.c_str());
-
-					have_error = true;
-					return nullptr;
-					}
-
-				index_type = enum_id->GetType();
-				}
-			else
-				index_type = base_type(stag);
-
-			set_index = make_intrusive<TypeList>(index_type);
-			set_index->Append(std::move(index_type));
-			}
-
-		auto s = make_intrusive<SetType>(std::move(set_index), nullptr);
-		TableVal* t = new TableVal(std::move(s));
-		for ( int j = 0; j < val->val.set_val.size; j++ )
-			{
-			Val* assignval = ValueToVal(i, val->val.set_val.vals[j], have_error);
-
-			t->Assign({AdoptRef{}, assignval}, nullptr);
-			}
-
-		return t;
-		}
-
-	case TYPE_VECTOR:
-		{
-		IntrusivePtr<BroType> type;
-
-		if ( val->val.vector_val.size == 0  && val->subtype == TYPE_VOID )
-			// don't know type - unspecified table.
-			type = base_type(TYPE_ANY);
-		else
-			{
-			// all entries have to have the same type...
-			if ( val->subtype == TYPE_VOID )
-				type = base_type(val->val.vector_val.vals[0]->type);
-			else
-				type = base_type(val->subtype);
-			}
-
-		auto vt = make_intrusive<VectorType>(std::move(type));
-		auto v = make_intrusive<VectorVal>(std::move(vt));
-
-		for ( int j = 0; j < val->val.vector_val.size; j++ )
-			v->Assign(j, ValueToVal(i, val->val.vector_val.vals[j], have_error));
-
-		return v.release();
-		}
-
-	case TYPE_ENUM: {
-		// Convert to string first to not have to deal with missing
-		// \0's...
-		string enum_string(val->val.string_val.data, val->val.string_val.length);
-
-		// let's try looking it up by global ID.
-		auto id = lookup_ID(enum_string.c_str(), GLOBAL_MODULE_NAME);
-		if ( ! id || ! id->IsEnumConst() )
-			{
-			Warning(i, "Value '%s' for stream '%s' is not a valid enum.",
-			                        enum_string.c_str(), i->name.c_str());
-
-			have_error = true;
-			return nullptr;
-			}
-
-		EnumType* t = id->GetType()->AsEnumType();
-		int intval = t->Lookup(id->ModuleName(), id->Name());
-		if ( intval < 0 )
-			{
-			Warning(i, "Enum value '%s' for stream '%s' not found.",
-			                        enum_string.c_str(), i->name.c_str());
-
-			have_error = true;
-			return nullptr;
-			}
-
-		auto rval = t->GetVal(intval);
-		return rval.release();
-		}
-
-	default:
-		reporter->InternalError("Unsupported type for input_read in stream %s", i->name.c_str());
-	}
-
-	assert(false);
-	return nullptr;
-	}
-
 Manager::Stream* Manager::FindStream(const string &name) const
 	{
 	for ( auto s = readers.begin(); s != readers.end(); ++s )

src/input/Manager.h

@@ -118,7 +118,6 @@ protected:
 	friend class PutMessage;
 	friend class DeleteMessage;
 	friend class ClearMessage;
-	friend class SendEventMessage;
 	friend class SendEntryMessage;
 	friend class EndCurrentSendMessage;
 	friend class ReaderClosedMessage;
@@ -142,11 +141,6 @@ protected:
 	void SendEntry(ReaderFrontend* reader, threading::Value* *vals);
 	void EndCurrentSend(ReaderFrontend* reader);
 
-	// Allows readers to directly send Bro events. The num_vals and vals
-	// must be the same the named event expects. Takes ownership of
-	// threading::Value fields.
-	bool SendEvent(ReaderFrontend* reader, const std::string& name, const int num_vals, threading::Value* *vals) const;
-
 	// Instantiates a new ReaderBackend of the given type (note that
 	// doing so creates a new thread!).
 	ReaderBackend* CreateBackend(ReaderFrontend* frontend, EnumVal* tag);
@@ -227,14 +221,9 @@ private:
 	// startpos.
 	int CopyValue(char *data, const int startpos, const threading::Value* val) const;
 
-	// Convert Threading::Value to an internal Bro Type (works also with
+	// Convert Threading::Value to an internal Bro Type (works with Records).
-	// Records).
 	Val* ValueToVal(const Stream* i, const threading::Value* val, BroType* request_type, bool& have_error) const;
 
-	// Convert Threading::Value to an internal Bro type just using the information given in the threading::Value.
-	// This allows more flexibility, especially given structures in script-land that contain any types.
-	Val* ValueToVal(const Stream* i, const threading::Value* val, bool& have_error) const;
-
 	// Convert Threading::Value to an internal Bro list type.
 	Val* ValueToIndexVal(const Stream* i, int num_fields, const RecordType* type, const threading::Value* const *vals, bool& have_error) const;
 

src/input/ReaderBackend.cc

@@ -54,30 +54,6 @@ public:
 private:
 };
 
-class SendEventMessage final : public threading::OutputMessage<ReaderFrontend> {
-public:
-	SendEventMessage(ReaderFrontend* reader, const char* name, const int num_vals, Value* *val)
-		: threading::OutputMessage<ReaderFrontend>("SendEvent", reader),
-		name(copy_string(name)), num_vals(num_vals), val(val) {}
-
-	~SendEventMessage() override	{ delete [] name; }
-
-	bool Process() override
-		{
-		bool success = input_mgr->SendEvent(Object(), name, num_vals, val);
-
-		if ( ! success )
-			reporter->Error("SendEvent for event %s failed", name);
-
-		return true; // We do not want to die if sendEvent fails because the event did not return.
-		}
-
-private:
-	const char* name;
-	const int num_vals;
-	Value* *val;
-};
-
 class ReaderErrorMessage final : public threading::OutputMessage<ReaderFrontend>
 {
 public:
@@ -229,11 +205,6 @@ void ReaderBackend::Clear()
 	SendOut(new ClearMessage(frontend));
 	}
 
-void ReaderBackend::SendEvent(const char* name, const int num_vals, Value* *vals)
-	{
-	SendOut(new SendEventMessage(frontend, name, num_vals, vals));
-	}
-
 void ReaderBackend::EndCurrentSend()
 	{
 	SendOut(new EndCurrentSendMessage(frontend));

src/input/ReaderBackend.h

@@ -280,18 +280,6 @@ protected:
 	 */
 	virtual bool DoHeartbeat(double network_time, double current_time) = 0;
 
-	/**
-	 * Method allowing a reader to send a specified Bro event. Vals must
-	 * match the values expected by the bro event.
-	 *
-	 * @param name name of the bro event to send
-	 *
-	 * @param num_vals number of entries in \a vals
-	 *
-	 * @param vals the values to be given to the event
-	 */
-	void SendEvent(const char* name, const int num_vals, threading::Value* *vals);
-
 	// Content-sending-functions (simple mode). Include table-specific
 	// functionality that simply is not used if we have no table.
 

src/threading/BasicThread.h

@@ -37,7 +37,7 @@ public:
 
 	/**
 	 * Returns a descriptive name for the thread. If not set via
-	 * SetName(). If not set, a default name is choosen automatically.
+	 * SetName(), a default name is chosen automatically.
 	 *
 	 * This method is safe to call from any thread.
 	 */

src/threading/Manager.cc

@@ -5,6 +5,7 @@
 
 #include "NetVar.h"
 #include "iosource/Manager.h"
+#include "Event.h"
 
 using namespace threading;
 
@@ -128,6 +129,60 @@ void Manager::StartHeartbeatTimer()
 	timer_mgr->Add(new HeartbeatTimer(network_time + zeek::BifConst::Threading::heartbeat_interval));
 	}
 
+// Raise everything in here as warnings so it is passed to scriptland without
+// looking "fatal". In addition to these warnings, ReaderBackend will queue
+// one reporter message.
+bool Manager::SendEvent(MsgThread* thread, const std::string& name, const int num_vals, Value* *vals) const
+	{
+	EventHandler* handler = event_registry->Lookup(name);
+	if ( handler == nullptr )
+		{
+		reporter->Warning("Thread %s: Event %s not found", thread->Name(), name.c_str());
+		Value::delete_value_ptr_array(vals, num_vals);
+		return false;
+		}
+
+#ifdef DEBUG
+	DBG_LOG(DBG_INPUT, "Thread %s: SendEvent for event %s with %d vals",
+	        thread->Name(), name.c_str(), num_vals);
+#endif
+
+	const auto& type = handler->GetType()->Params();
+	int num_event_vals = type->NumFields();
+	if ( num_vals != num_event_vals )
+		{
+		reporter->Warning("Thread %s: Wrong number of values for event %s", thread->Name(), name.c_str());
+		Value::delete_value_ptr_array(vals, num_vals);
+		return false;
+		}
+
+	bool convert_error = false;
+
+	zeek::Args vl;
+	vl.reserve(num_vals);
+
+	for ( int j = 0; j < num_vals; j++)
+		{
+		Val* v = Value::ValueToVal(std::string("thread ") + thread->Name(), vals[j], convert_error);
+		vl.emplace_back(AdoptRef{}, v);
+
+		if ( v && ! convert_error && ! same_type(type->GetFieldType(j).get(), v->GetType().get()) )
+			{
+			convert_error = true;
+			type->GetFieldType(j)->Error("SendEvent types do not match", v->GetType().get());
+			}
+		}
+
+	Value::delete_value_ptr_array(vals, num_vals);
+
+	if ( convert_error )
+		return false;
+	else if ( handler )
+		mgr.Enqueue(handler, std::move(vl), SOURCE_LOCAL);
+
+	return true;
+	}
+
 void Manager::Flush()
 	{
 	bool do_beat = false;

src/threading/Manager.h

@@ -88,6 +88,18 @@ public:
 	 */
 	void KillThreads();
 
+	/**
+	 * Allows threads to directly send Zeek events. The num_vals and vals must be
+	 * the same the named event expects. Takes ownership of threading::Value fields.
+	 *
+	 * @param thread Thread raising the event
+	 * @param name Name of event being raised
+	 * @param num_vals Number of values passed to the event
+	 * @param vals Values passed to the event
+	 * @returns True on success false on failure.
+	 */
+	bool SendEvent(MsgThread* thread, const std::string& name, const int num_vals, threading::Value* *vals) const;
+
 protected:
 	friend class BasicThread;
 	friend class MsgThread;

src/threading/MsgThread.cc

@@ -126,6 +126,31 @@ private:
 
 }
 
+// An event that the child wants to pass into the main event queue
+class SendEventMessage final : public OutputMessage<MsgThread> {
+public:
+	SendEventMessage(MsgThread* thread, const char* name, const int num_vals, Value* *val)
+		: OutputMessage<MsgThread>("SendEvent", thread),
+	  name(copy_string(name)), num_vals(num_vals), val(val) {}
+
+	~SendEventMessage() override	{ delete [] name; }
+
+	bool Process() override
+		{
+		bool success = thread_mgr->SendEvent(Object(), name, num_vals, val);
+
+		if ( ! success )
+			reporter->Error("SendEvent for event %s failed", name);
+
+		return true; // We do not want to die if sendEvent fails because the event did not return.
+		}
+
+private:
+	const char* name;
+	const int num_vals;
+	Value* *val;
+};
+
 ////// Methods.
 
 Message::~Message()
@@ -363,6 +388,11 @@ void MsgThread::SendOut(BasicOutputMessage* msg, bool force)
 	flare.Fire();
 	}
 
+void MsgThread::SendEvent(const char* name, const int num_vals, Value* *vals)
+	{
+	SendOut(new SendEventMessage(this, name, num_vals, vals));
+	}
+
 BasicOutputMessage* MsgThread::RetrieveOut()
 	{
 	BasicOutputMessage* msg = queue_out.Get();

src/threading/MsgThread.h

@@ -60,6 +60,18 @@ public:
 	 */
 	void SendOut(BasicOutputMessage* msg)	{ return SendOut(msg, false); }
 
+	/**
+	 * Allows the child thread to send a specified Zeek event. The given Vals
+	 * must match the values expected by the event.
+	 *
+	 * @param name name of the bro event to send
+	 *
+	 * @param num_vals number of entries in \a vals
+	 *
+	 * @param vals the values to be given to the event
+	 */
+	void SendEvent(const char* name, const int num_vals, threading::Value* *vals);
+
 	/**
 	 * Reports an informational message from the child thread. The main
 	 * thread will pass this to the Reporter once received.
@@ -393,7 +405,7 @@ protected:
 };
 
 /**
- * A paremeterized InputMessage that stores a pointer to an argument object.
+ * A parameterized InputMessage that stores a pointer to an argument object.
  * Normally, the objects will be used from the Process() callback.
  */
 template<typename O>

src/threading/SerialTypes.cc

@@ -4,6 +4,14 @@
 #include "SerialTypes.h"
 #include "SerializationFormat.h"
 #include "Reporter.h"
+// The following are required for ValueToVal.
+#include "Val.h"
+#include "BroString.h"
+#include "RE.h"
+#include "module_util.h"
+#include "ID.h"
+#include "Expr.h"
+#include "Scope.h"
 
 using namespace threading;
 
@@ -346,7 +354,6 @@ bool Value::Write(SerializationFormat* fmt) const
 		case IPv6:
 			return fmt->Write((char)6, "addr-family")
 				&& fmt->Write(val.addr_val.in.in6, "addr-in6");
-			break;
 		}
 
 		// Can't be reached.
@@ -366,7 +373,6 @@ bool Value::Write(SerializationFormat* fmt) const
 		case IPv6:
 			return fmt->Write((char)6, "subnet-family")
 				&& fmt->Write(val.subnet_val.prefix.in.in6, "subnet-in6");
-			break;
 		}
 
 		// Can't be reached.
@@ -417,5 +423,212 @@ bool Value::Write(SerializationFormat* fmt) const
 		                        type_name(type));
 	}
 
+	// unreachable
 	return false;
 	}
+
+void Value::delete_value_ptr_array(Value** vals, int num_fields)
+	{
+	for ( int i = 0; i < num_fields; ++i )
+		delete vals[i];
+
+	delete [] vals;
+	}
+
+Val* Value::ValueToVal(const std::string& source, const Value* val, bool& have_error)
+	{
+	if ( have_error )
+		return nullptr;
+
+	if ( ! val->present )
+		return nullptr; // unset field
+
+	switch ( val->type ) {
+		case TYPE_BOOL:
+			return val_mgr->Bool(val->val.int_val)->Ref();
+
+		case TYPE_INT:
+			return val_mgr->Int(val->val.int_val).release();
+
+		case TYPE_COUNT:
+		case TYPE_COUNTER:
+			return val_mgr->Count(val->val.int_val).release();
+
+		case TYPE_DOUBLE:
+		case TYPE_TIME:
+		case TYPE_INTERVAL:
+			return new Val(val->val.double_val, val->type);
+
+		case TYPE_STRING:
+			{
+			BroString *s = new BroString((const u_char*)val->val.string_val.data, val->val.string_val.length, true);
+			return new StringVal(s);
+			}
+
+		case TYPE_PORT:
+			return val_mgr->Port(val->val.port_val.port, val->val.port_val.proto)->Ref();
+
+		case TYPE_ADDR:
+			{
+			IPAddr* addr = nullptr;
+			switch ( val->val.addr_val.family ) {
+				case IPv4:
+					addr = new IPAddr(val->val.addr_val.in.in4);
+					break;
+
+				case IPv6:
+					addr = new IPAddr(val->val.addr_val.in.in6);
+					break;
+
+				default:
+					assert(false);
+				}
+
+			AddrVal* addrval = new AddrVal(*addr);
+			delete addr;
+			return addrval;
+			}
+
+		case TYPE_SUBNET:
+			{
+			IPAddr* addr = nullptr;
+			switch ( val->val.subnet_val.prefix.family ) {
+				case IPv4:
+					addr = new IPAddr(val->val.subnet_val.prefix.in.in4);
+					break;
+
+				case IPv6:
+					addr = new IPAddr(val->val.subnet_val.prefix.in.in6);
+					break;
+
+				default:
+					assert(false);
+				}
+
+			SubNetVal* subnetval = new SubNetVal(*addr, val->val.subnet_val.length);
+			delete addr;
+			return subnetval;
+			}
+
+		case TYPE_PATTERN:
+			{
+			RE_Matcher* re = new RE_Matcher(val->val.pattern_text_val);
+			re->Compile();
+			return new PatternVal(re);
+			}
+
+		case TYPE_TABLE:
+			{
+			IntrusivePtr<TypeList> set_index;
+			if ( val->val.set_val.size == 0 && val->subtype == TYPE_VOID )
+				// don't know type - unspecified table.
+				set_index = make_intrusive<TypeList>();
+			else
+				{
+				// all entries have to have the same type...
+				TypeTag stag = val->subtype;
+				if ( stag == TYPE_VOID )
+					TypeTag stag = val->val.set_val.vals[0]->type;
+
+				IntrusivePtr<BroType> index_type;
+
+				if ( stag == TYPE_ENUM )
+					{
+					// Enums are not a base-type, so need to look it up.
+					const auto& sv = val->val.set_val.vals[0]->val.string_val;
+					std::string enum_name(sv.data, sv.length);
+					const auto& enum_id = global_scope()->Find(enum_name);
+
+					if ( ! enum_id )
+						{
+						reporter->Warning("Value '%s' of source '%s' is not a valid enum.",
+						        enum_name.data(), source.c_str());
+
+						have_error = true;
+						return nullptr;
+						}
+
+					index_type = enum_id->GetType();
+					}
+				else
+					index_type = base_type(stag);
+
+				set_index = make_intrusive<TypeList>(index_type);
+				set_index->Append(std::move(index_type));
+				}
+
+			auto s = make_intrusive<SetType>(std::move(set_index), nullptr);
+			TableVal* t = new TableVal(std::move(s));
+			for ( int j = 0; j < val->val.set_val.size; j++ )
+				{
+				Val* assignval = ValueToVal(source, val->val.set_val.vals[j], have_error);
+				t->Assign({AdoptRef{}, assignval}, nullptr);
+				}
+
+			return t;
+			}
+
+		case TYPE_VECTOR:
+			{
+			IntrusivePtr<BroType> type;
+
+			if ( val->val.vector_val.size == 0  && val->subtype == TYPE_VOID )
+				// don't know type - unspecified table.
+				type = base_type(TYPE_ANY);
+			else
+				{
+				// all entries have to have the same type...
+				if ( val->subtype == TYPE_VOID )
+					type = base_type(val->val.vector_val.vals[0]->type);
+				else
+					type = base_type(val->subtype);
+				}
+
+			auto vt = make_intrusive<VectorType>(std::move(type));
+			auto v = make_intrusive<VectorVal>(std::move(vt));
+
+			for ( int j = 0; j < val->val.vector_val.size; j++ )
+				v->Assign(j, ValueToVal(source, val->val.vector_val.vals[j], have_error));
+
+			return v.release();
+			}
+
+		case TYPE_ENUM: {
+			// Convert to string first to not have to deal with missing
+			// \0's...
+			std::string enum_string(val->val.string_val.data, val->val.string_val.length);
+
+			// let's try looking it up by global ID.
+			const auto& id = lookup_ID(enum_string.c_str(), GLOBAL_MODULE_NAME);
+
+			if ( ! id || ! id->IsEnumConst() )
+				{
+				reporter->Warning("Value '%s' for source '%s' is not a valid enum.",
+				        enum_string.c_str(), source.c_str());
+
+				have_error = true;
+				return nullptr;
+				}
+
+			EnumType* t = id->GetType()->AsEnumType();
+			int intval = t->Lookup(id->ModuleName(), id->Name());
+			if ( intval < 0 )
+				{
+				reporter->Warning("Enum value '%s' for source '%s' not found.",
+				        enum_string.c_str(), source.c_str());
+
+				have_error = true;
+				return nullptr;
+				}
+
+			auto rval = t->GetVal(intval);
+			return rval.release();
+			}
+
+		default:
+			reporter->InternalError("Unsupported type in SerialTypes::ValueToVal from source %s", source.c_str());
+		}
+
+	assert(false);
+	return nullptr;
+	}

src/threading/SerialTypes.h

@@ -187,6 +187,26 @@ struct Value {
 	 * method is thread-safe. */
 	static bool IsCompatibleType(BroType* t, bool atomic_only=false);
 
+	/**
+	 * Convenience function to delete an array of value pointers.
+	 * @param vals Array of values
+	 * @param num_fields Number of members
+	 */
+	static void delete_value_ptr_array(Value** vals, int num_fields);
+
+	/**
+	 * Convert threading::Value to an internal Zeek type, just using the information given in the threading::Value.
+	 *
+	 * @param source Name of the source of this threading value. This is used for warnings that are raised
+	 *               in case an error occurs.
+	 * @param val Threading Value to convert to a Zeek Val.
+	 * @param have_error Reference to a boolean. This should be set to false when passed in and is set to true
+	 *                   in case an error occurs. If this is set to false when the function is called, the function
+	 *                   immediately aborts.
+	 * @return Val representation of the threading::Value. nullptr on error.
+	 */
+	static Val* ValueToVal(const std::string& source, const threading::Value* val, bool& have_error);
+
 private:
 	friend class ::IPAddr;
 	Value(const Value& other)	{ } // Disabled.

testing/btest/Baseline/broker.opaque/out

@@ -1,53 +1,62 @@
 ============ Topk
+opaque of topk
 [b, a, c]
 [b, a, c]
 ============ HLL
 3.000069
+opaque of cardinality
 3.000069
 3.000069
 ============ Bloom
 0
 1
+opaque of bloomfilter
 0
 1
 ============ Hashes
+opaque of md5
 5b9164ad6f496d9dee12ec7634ce253f
 5b9164ad6f496d9dee12ec7634ce253f
+opaque of sha1
 30ae97492ce1da88d0e7117ace0a60a6f9e1e0bc
 30ae97492ce1da88d0e7117ace0a60a6f9e1e0bc
+opaque of sha256
 25b6746d5172ed6352966a013d93ac846e1110d5a25e8f183b5931f4688842a1
 25b6746d5172ed6352966a013d93ac846e1110d5a25e8f183b5931f4688842a1
 ============ X509
+opaque of x509
 [version=3, serial=040000000001154B5AC394, subject=CN=GlobalSign Root CA,OU=Root CA,O=GlobalSign nv-sa,C=BE, issuer=CN=GlobalSign Root CA,OU=Root CA,O=GlobalSign nv-sa,C=BE, cn=GlobalSign Root CA, not_valid_before=904651200.0, not_valid_after=1832673600.0, key_alg=rsaEncryption, sig_alg=sha1WithRSAEncryption, key_type=rsa, key_length=2048, exponent=65537, curve=<uninitialized>]
 [version=3, serial=040000000001154B5AC394, subject=CN=GlobalSign Root CA,OU=Root CA,O=GlobalSign nv-sa,C=BE, issuer=CN=GlobalSign Root CA,OU=Root CA,O=GlobalSign nv-sa,C=BE, cn=GlobalSign Root CA, not_valid_before=904651200.0, not_valid_after=1832673600.0, key_alg=rsaEncryption, sig_alg=sha1WithRSAEncryption, key_type=rsa, key_length=2048, exponent=65537, curve=<uninitialized>]
 ============ Entropy
+opaque of entropy
 [entropy=4.715374, chi_square=591.981818, mean=75.472727, monte_carlo_pi=4.0, serial_correlation=-0.11027]
 [entropy=4.715374, chi_square=591.981818, mean=75.472727, monte_carlo_pi=4.0, serial_correlation=-0.11027]
 ============ broker::Data
+opaque of Broker::Data
 broker::data{{hi, there}}
 broker::data{{hi, there}}
 T
 ============ broker::Set
-| [data=broker::data{!}] | [data=broker::data{!}]
+| [data=broker::data{!}] | [data=broker::data{!}] | opaque of Broker::SetIterator
   > [data=broker::data{hi}] | [data=broker::data{hi}]
-| [data=broker::data{hi}] | [data=broker::data{hi}]
+| [data=broker::data{hi}] | [data=broker::data{hi}] | opaque of Broker::SetIterator
   > [data=broker::data{there}] | [data=broker::data{there}]
-| [data=broker::data{there}] | [data=broker::data{there}]
+| [data=broker::data{there}] | [data=broker::data{there}] | opaque of Broker::SetIterator
 ============ broker::Table
-| [key=[data=broker::data{!}], val=[data=broker::data{30}]] | [key=[data=broker::data{!}], val=[data=broker::data{30}]]
+| [key=[data=broker::data{!}], val=[data=broker::data{30}]] | [key=[data=broker::data{!}], val=[data=broker::data{30}]] | opaque of Broker::TableIterator
   > [key=[data=broker::data{hi}], val=[data=broker::data{10}]] | [key=[data=broker::data{hi}], val=[data=broker::data{10}]]
-| [key=[data=broker::data{hi}], val=[data=broker::data{10}]] | [key=[data=broker::data{hi}], val=[data=broker::data{10}]]
+| [key=[data=broker::data{hi}], val=[data=broker::data{10}]] | [key=[data=broker::data{hi}], val=[data=broker::data{10}]] | opaque of Broker::TableIterator
   > [key=[data=broker::data{there}], val=[data=broker::data{20}]] | [key=[data=broker::data{there}], val=[data=broker::data{20}]]
-| [key=[data=broker::data{there}], val=[data=broker::data{20}]] | [key=[data=broker::data{there}], val=[data=broker::data{20}]]
+| [key=[data=broker::data{there}], val=[data=broker::data{20}]] | [key=[data=broker::data{there}], val=[data=broker::data{20}]] | opaque of Broker::TableIterator
 ============ broker::Vector
-| [data=broker::data{hi}] | [data=broker::data{hi}]
+| [data=broker::data{hi}] | [data=broker::data{hi}] | opaque of Broker::VectorIterator
   > [data=broker::data{there}] | [data=broker::data{there}]
-| [data=broker::data{there}] | [data=broker::data{there}]
+| [data=broker::data{there}] | [data=broker::data{there}] | opaque of Broker::VectorIterator
   > [data=broker::data{!}] | [data=broker::data{!}]
-| [data=broker::data{!}] | [data=broker::data{!}]
+| [data=broker::data{!}] | [data=broker::data{!}] | opaque of Broker::VectorIterator
 ============ broker::Record
-| [data=broker::data{hi}] | [data=broker::data{hi}]
+| [data=broker::data{hi}] | [data=broker::data{hi}] | opaque of Broker::RecordIterator
   > [data=broker::data{there}] | [data=broker::data{there}]
-| [data=broker::data{there}] | [data=broker::data{there}]
+| [data=broker::data{there}] | [data=broker::data{there}] | opaque of Broker::RecordIterator
   > [data=broker::data{!}] | [data=broker::data{!}]
-| [data=broker::data{!}] | [data=broker::data{!}]
+| [data=broker::data{!}] | [data=broker::data{!}] | opaque of Broker::RecordIterator

testing/btest/Baseline/scripts.base.frameworks.input.config.errors/errout

@@ -5,7 +5,7 @@ warning: ../configfile/Input::READER_CONFIG: Field: testbool Invalid value for b
 warning: ../configfile/Input::READER_CONFIG: Could not convert line 'testbool A B' to value. Ignoring line.
 warning: ../configfile/Input::READER_CONFIG: String 'A' contained no parseable number
 warning: ../configfile/Input::READER_CONFIG: Could not convert line 'testcount A' to value. Ignoring line.
-warning: Value 'unknown' for stream 'configuration' is not a valid enum.
+warning: Value 'unknown' for source 'thread ../configfile/Input::READER_CONFIG' is not a valid enum.
 error: SendEvent for event InputConfig::new_value failed
 warning: ../configfile/Input::READER_CONFIG: Option 'testbooool' does not exist. Ignoring line.
 warning: ../configfile/Input::READER_CONFIG: Option 'test_any' has type 'any', which is not supported for file input. Ignoring line.

testing/btest/broker/opaque.zeek

@@ -11,6 +11,7 @@ event zeek_init()
 	topk_add(k1, "b");
 	topk_add(k1, "c");
 	local k2 = Broker::__opaque_clone_through_serialization(k1);
+	print type_name(k2);
 	print topk_get_top(k1, 5);
 	topk_add(k1, "shoulnotshowup");
 	print topk_get_top(k2, 5);
@@ -23,6 +24,7 @@ event zeek_init()
 
 	print hll_cardinality_estimate(c1);
 	local c2 = Broker::__opaque_clone_through_serialization(c1);
+	print type_name(c2);
 	hll_cardinality_add(c1, 2004);
 	print hll_cardinality_estimate(c2);
 
@@ -38,6 +40,7 @@ event zeek_init()
 	print bloomfilter_lookup(bf_cnt, 0);
 	print bloomfilter_lookup(bf_cnt, 42);
 	local bf_copy = Broker::__opaque_clone_through_serialization(bf_cnt);
+	print type_name(bf_copy);
 	bloomfilter_add(bf_cnt, 0);
 	print bloomfilter_lookup(bf_copy, 0);
 	print bloomfilter_lookup(bf_copy, 42);
@@ -48,6 +51,7 @@ event zeek_init()
 	local md5a = md5_hash_init();
 	md5_hash_update(md5a, "one");
 	local md5b = Broker::__opaque_clone_through_serialization(md5a);
+	print type_name(md5b);
 	md5_hash_update(md5a, "two");
 	md5_hash_update(md5b, "two");
 	print md5_hash_finish(md5a);
@@ -56,6 +60,7 @@ event zeek_init()
 	local sha1a = sha1_hash_init();
 	sha1_hash_update(sha1a, "one");
 	local sha1b = Broker::__opaque_clone_through_serialization(sha1a);
+	print type_name(sha1b);
 	sha1_hash_update(sha1a, "two");
 	sha1_hash_update(sha1b, "two");
 	print sha1_hash_finish(sha1a);
@@ -64,6 +69,7 @@ event zeek_init()
 	local sha256a = sha256_hash_init();
 	sha256_hash_update(sha256a, "one");
 	local sha256b = Broker::__opaque_clone_through_serialization(sha256a);
+	print type_name(sha256b);
 	sha256_hash_update(sha256a, "two");
 	sha256_hash_update(sha256b, "two");
 	print sha256_hash_finish(sha256a);
@@ -72,6 +78,7 @@ event zeek_init()
 	print "============ X509";
 	local x509 = x509_from_der("\x30\x82\x03\x75\x30\x82\x02\x5D\xA0\x03\x02\x01\x02\x02\x0B\x04\x00\x00\x00\x00\x01\x15\x4B\x5A\xC3\x94\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x57\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x42\x45\x31\x19\x30\x17\x06\x03\x55\x04\x0A\x13\x10\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x6E\x76\x2D\x73\x61\x31\x10\x30\x0E\x06\x03\x55\x04\x0B\x13\x07\x52\x6F\x6F\x74\x20\x43\x41\x31\x1B\x30\x19\x06\x03\x55\x04\x03\x13\x12\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x1E\x17\x0D\x39\x38\x30\x39\x30\x31\x31\x32\x30\x30\x30\x30\x5A\x17\x0D\x32\x38\x30\x31\x32\x38\x31\x32\x30\x30\x30\x30\x5A\x30\x57\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x42\x45\x31\x19\x30\x17\x06\x03\x55\x04\x0A\x13\x10\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x6E\x76\x2D\x73\x61\x31\x10\x30\x0E\x06\x03\x55\x04\x0B\x13\x07\x52\x6F\x6F\x74\x20\x43\x41\x31\x1B\x30\x19\x06\x03\x55\x04\x03\x13\x12\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xDA\x0E\xE6\x99\x8D\xCE\xA3\xE3\x4F\x8A\x7E\xFB\xF1\x8B\x83\x25\x6B\xEA\x48\x1F\xF1\x2A\xB0\xB9\x95\x11\x04\xBD\xF0\x63\xD1\xE2\x67\x66\xCF\x1C\xDD\xCF\x1B\x48\x2B\xEE\x8D\x89\x8E\x9A\xAF\x29\x80\x65\xAB\xE9\xC7\x2D\x12\xCB\xAB\x1C\x4C\x70\x07\xA1\x3D\x0A\x30\xCD\x15\x8D\x4F\xF8\xDD\xD4\x8C\x50\x15\x1C\xEF\x50\xEE\xC4\x2E\xF7\xFC\xE9\x52\xF2\x91\x7D\xE0\x6D\xD5\x35\x30\x8E\x5E\x43\x73\xF2\x41\xE9\xD5\x6A\xE3\xB2\x89\x3A\x56\x39\x38\x6F\x06\x3C\x88\x69\x5B\x2A\x4D\xC5\xA7\x54\xB8\x6C\x89\xCC\x9B\xF9\x3C\xCA\xE5\xFD\x89\xF5\x12\x3C\x92\x78\x96\xD6\xDC\x74\x6E\x93\x44\x61\xD1\x8D\xC7\x46\xB2\x75\x0E\x86\xE8\x19\x8A\xD5\x6D\x6C\xD5\x78\x16\x95\xA2\xE9\xC8\x0A\x38\xEB\xF2\x24\x13\x4F\x73\x54\x93\x13\x85\x3A\x1B\xBC\x1E\x34\xB5\x8B\x05\x8C\xB9\x77\x8B\xB1\xDB\x1F\x20\x91\xAB\x09\x53\x6E\x90\xCE\x7B\x37\x74\xB9\x70\x47\x91\x22\x51\x63\x16\x79\xAE\xB1\xAE\x41\x26\x08\xC8\x19\x2B\xD1\x46\xAA\x48\xD6\x64\x2A\xD7\x83\x34\xFF\x2C\x2A\xC1\x6C\x19\x43\x4A\x07\x85\xE7\xD3\x7C\xF6\x21\x68\xEF\xEA\xF2\x52\x9F\x7F\x93\x90\xCF\x02\x03\x01\x00\x01\xA3\x42\x30\x40\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x60\x7B\x66\x1A\x45\x0D\x97\xCA\x89\x50\x2F\x7D\x04\xCD\x34\xA8\xFF\xFC\xFD\x4B\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\xD6\x73\xE7\x7C\x4F\x76\xD0\x8D\xBF\xEC\xBA\xA2\xBE\x34\xC5\x28\x32\xB5\x7C\xFC\x6C\x9C\x2C\x2B\xBD\x09\x9E\x53\xBF\x6B\x5E\xAA\x11\x48\xB6\xE5\x08\xA3\xB3\xCA\x3D\x61\x4D\xD3\x46\x09\xB3\x3E\xC3\xA0\xE3\x63\x55\x1B\xF2\xBA\xEF\xAD\x39\xE1\x43\xB9\x38\xA3\xE6\x2F\x8A\x26\x3B\xEF\xA0\x50\x56\xF9\xC6\x0A\xFD\x38\xCD\xC4\x0B\x70\x51\x94\x97\x98\x04\xDF\xC3\x5F\x94\xD5\x15\xC9\x14\x41\x9C\xC4\x5D\x75\x64\x15\x0D\xFF\x55\x30\xEC\x86\x8F\xFF\x0D\xEF\x2C\xB9\x63\x46\xF6\xAA\xFC\xDF\xBC\x69\xFD\x2E\x12\x48\x64\x9A\xE0\x95\xF0\xA6\xEF\x29\x8F\x01\xB1\x15\xB5\x0C\x1D\xA5\xFE\x69\x2C\x69\x24\x78\x1E\xB3\xA7\x1C\x71\x62\xEE\xCA\xC8\x97\xAC\x17\x5D\x8A\xC2\xF8\x47\x86\x6E\x2A\xC4\x56\x31\x95\xD0\x67\x89\x85\x2B\xF9\x6C\xA6\x5D\x46\x9D\x0C\xAA\x82\xE4\x99\x51\xDD\x70\xB7\xDB\x56\x3D\x61\xE4\x6A\xE1\x5C\xD6\xF6\xFE\x3D\xDE\x41\xCC\x07\xAE\x63\x52\xBF\x53\x53\xF4\x2B\xE9\xC7\xFD\xB6\xF7\x82\x5F\x85\xD2\x41\x18\xDB\x81\xB3\x04\x1C\xC5\x1F\xA4\x80\x6F\x15\x20\xC9\xDE\x0C\x88\x0A\x1D\xD6\x66\x55\xE2\xFC\x48\xC9\x29\x26\x69\xE0");
 	local x5092 = Broker::__opaque_clone_through_serialization(x509);
+	print type_name(x5092);
 	print x509_parse(x509);
 	print x509_parse(x5092);
 
@@ -79,6 +86,7 @@ event zeek_init()
 	local handle = entropy_test_init();
 	entropy_test_add(handle, "dh3Hie02uh^s#Sdf9L3frd243h$d78r2G4cM6*Q05d(7rh46f!0|4-f");
 	local handle2 = Broker::__opaque_clone_through_serialization(handle);
+	print type_name(handle2);
 	print entropy_test_finish(handle);
 	print entropy_test_finish(handle2);
 
@@ -87,6 +95,7 @@ event zeek_init()
 	Broker::set_insert(s1, "hi");
 	Broker::set_insert(s1, "there");
 	local d2 = Broker::__opaque_clone_through_serialization(s1$data);
+	print type_name(d2);
 	print s1$data;
 	print d2;
 	print same_object(s1$data, d2) == F;
@@ -101,7 +110,7 @@ event zeek_init()
 	while ( ! Broker::set_iterator_last(i) )
 		{
 		local ci = Broker::__opaque_clone_through_serialization(i);
-		print fmt("| %s | %s", Broker::set_iterator_value(i), Broker::set_iterator_value(ci));
+		print fmt("| %s | %s | %s", Broker::set_iterator_value(i), Broker::set_iterator_value(ci), type_name(ci));
 		Broker::set_iterator_next(i);
 		Broker::set_iterator_next(ci);
 		if ( ! Broker::set_iterator_last(i) )
@@ -118,7 +127,7 @@ event zeek_init()
 	while ( ! Broker::table_iterator_last(j) )
 		{
 		local cj = Broker::__opaque_clone_through_serialization(j);
-		print fmt("| %s | %s", Broker::table_iterator_value(j), Broker::table_iterator_value(cj));
+		print fmt("| %s | %s | %s", Broker::table_iterator_value(j), Broker::table_iterator_value(cj), type_name(cj));
 		Broker::table_iterator_next(j);
 		Broker::table_iterator_next(cj);
 		if ( ! Broker::table_iterator_last(j) )
@@ -135,7 +144,7 @@ event zeek_init()
 	while ( ! Broker::vector_iterator_last(k) )
 		{
 		local ck = Broker::__opaque_clone_through_serialization(k);
-		print fmt("| %s | %s", Broker::vector_iterator_value(k), Broker::vector_iterator_value(ck));
+		print fmt("| %s | %s | %s", Broker::vector_iterator_value(k), Broker::vector_iterator_value(ck), type_name(ck));
 		Broker::vector_iterator_next(k);
 		Broker::vector_iterator_next(ck);
 		if ( ! Broker::vector_iterator_last(k) )
@@ -152,7 +161,7 @@ event zeek_init()
 	while ( ! Broker::record_iterator_last(l) )
 		{
 		local cl = Broker::__opaque_clone_through_serialization(l);
-		print fmt("| %s | %s", Broker::record_iterator_value(l), Broker::record_iterator_value(cl));
+		print fmt("| %s | %s | %s", Broker::record_iterator_value(l), Broker::record_iterator_value(cl), type_name(cl));
 		Broker::record_iterator_next(l);
 		Broker::record_iterator_next(cl);
 		if ( ! Broker::record_iterator_last(l) )