Merge remote-tracking branch 'origin/master' into topic/johanna/table-changes

CHANGES

@@ -1,4 +1,24 @@
 
+3.2.0-dev.557 | 2020-05-21 11:41:12 -0700
+
+  * Speed up FuzzBuffer ChunkCount validity check (Justin Azoff, Corelight)
+
+3.2.0-dev.555 | 2020-05-20 11:19:08 -0700
+
+  * Disable output of Reporter messages to stderr in fuzz targets (Jon Siwek, Corelight)
+
+3.2.0-dev.554 | 2020-05-20 10:56:46 -0700
+
+  * Improve standalone fuzz driver timing output (Jon Siwek, Corelight)
+
+  * Skip fuzz inputs that have more than 64 chunks (Justin Azoff, Corelight)
+
+3.2.0-dev.550 | 2020-05-19 10:50:42 -0700
+
+  * Upgrade to latest Broker changes for CAF 0.18 (Dominik Charousset, Corelight)
+
+  * Include pcap.h instead of pcap/dlt.h in packet-fuzzer (Tim Wojtulewicz, Corelight)
+
 3.2.0-dev.547 | 2020-05-18 10:47:54 -0700
 
   * add packet fuzzer (Justin Azoff)

VERSION

@@ -1 +1 @@
-3.2.0-dev.547
+3.2.0-dev.557

aux/bifcl

@@ -1 +1 @@
-Subproject commit abd57da8ba6c9337a5ed79cabbf962f71b38d62d
+Subproject commit 95fece382d34bca72572cc863e1182b31a1b9945

aux/binpac

@@ -1 +1 @@
-Subproject commit 07307976115c90095af183ce950364bc1982027c
+Subproject commit 4d3d10cd54b1aa64f30d2fd433252f353c6ea6e0

aux/broker

@@ -1 +1 @@
-Subproject commit 74d168964369f8b5c51238520d6ad35173791110
+Subproject commit 6974924007765f70d95e5cf123b6256048ae3af7

aux/zeek-aux

@@ -1 +1 @@
-Subproject commit f4a659ee89e95f93ff2900cde7984f032a8113e9
+Subproject commit 8615abced86b5559fa3203264be55e664b887094

aux/zeekctl

@@ -1 +1 @@
-Subproject commit 881fe0083a512b3731738fb0272241e47b272e09
+Subproject commit bee11c63923b69a4468838d56067f15bc6ae3a52

cmake

@@ -1 +1 @@
-Subproject commit 7666cd462888b0eee173040c6888f41930a8be0b
+Subproject commit 23f4b88f91c537c59ef9a3ad56ec08f021ec2b2c

src/broker/Store.cc

@@ -85,13 +85,13 @@ broker::backend to_backend_type(BifEnum::Broker::BackendType type)
 	{
 	switch ( type ) {
 	case BifEnum::Broker::MEMORY:
-		return broker::memory;
+		return broker::backend::memory;
 
 	case BifEnum::Broker::SQLITE:
-		return broker::sqlite;
+		return broker::backend::sqlite;
 
 	case BifEnum::Broker::ROCKSDB:
-		return broker::rocksdb;
+		return broker::backend::rocksdb;
 	}
 
 	throw std::runtime_error("unknown broker backend");
@@ -101,14 +101,14 @@ broker::backend_options to_backend_options(broker::backend backend,
                                            RecordVal* options)
 	{
 	switch ( backend ) {
-	case broker::sqlite:
+	case broker::backend::sqlite:
 		{
 		auto path = options->Lookup(0)->AsRecordVal()
 			->Lookup(0)->AsStringVal()->CheckString();
 		return {{"path", path}};
 		}
 
-	case broker::rocksdb:
+	case broker::backend::rocksdb:
 		{
 		auto path = options->Lookup(1)->AsRecordVal()
 			->Lookup(0)->AsStringVal()->CheckString();

src/fuzzers/FuzzBuffer.cc

@@ -6,7 +6,7 @@
 
 #include "FuzzBuffer.h"
 
-bool zeek::detail::FuzzBuffer::Valid() const
+bool zeek::detail::FuzzBuffer::Valid(int chunk_count_limit) const
 	{
 	if ( end - begin < PKT_MAGIC_LEN + 2 )
 		return false;
@@ -14,9 +14,31 @@ bool zeek::detail::FuzzBuffer::Valid() const
 	if ( memcmp(begin, PKT_MAGIC, PKT_MAGIC_LEN) != 0)
 		return false;
 
+	if ( ExceedsChunkLimit(chunk_count_limit) )
+		return false;
+
 	return true;
 	}
 
+int zeek::detail::FuzzBuffer::ChunkCount(int chunk_count_limit) const
+	{
+	auto pos = begin;
+	int chunks = 0;
+
+	while ( pos < end && (chunks < chunk_count_limit || chunk_count_limit == 0) )
+		{
+		pos = (const unsigned char*)memmem(pos, end - pos,
+		                                   PKT_MAGIC, PKT_MAGIC_LEN);
+		if ( ! pos )
+			break;
+
+		pos += PKT_MAGIC_LEN + 1;
+		chunks++;
+		}
+
+	return chunks;
+	}
+
 std::optional<zeek::detail::FuzzBuffer::Chunk> zeek::detail::FuzzBuffer::Next()
 	{
 	if ( begin == end )

src/fuzzers/FuzzBuffer.h

@@ -27,6 +27,7 @@ public:
 
 	static constexpr int PKT_MAGIC_LEN = 4;
 	static constexpr unsigned char PKT_MAGIC[PKT_MAGIC_LEN + 1] = "\1PKT";
+	static constexpr int MAX_CHUNK_COUNT = 64;
 
 	/**
 	 * Initialize fuzz buffer.
@@ -39,9 +40,25 @@ public:
 
 	/**
 	 * @return  whether the fuzz buffer object is valid --  has enough bytes
-	 * to Deliver to an analyzer and starts with a *PKT_MAGIC* bytestring.
+	 * to Deliver to an analyzer, starts with a *PKT_MAGIC* bytestring, and
+	 * contains less than the limiting number of chunk.
+	 * .
 	 */
-	bool Valid() const;
+	bool Valid(int chunk_count_limit = MAX_CHUNK_COUNT) const;
+
+	/**
+	 * @param chunk_count_limit  Number of chunks to stop counting at (zero
+	 * means "never stop").
+	 * @return  the number of chunks in the fuzz buffer object
+	 */
+	int ChunkCount(int chunk_count_limit = 0) const;
+
+	/**
+	 * @param  Maximum number of chunks to permit the FuzzBuffer to have.
+	 * @return  Whether the FuzzBuffer exceeds the desired chunk count limit.
+	 */
+	bool ExceedsChunkLimit(int chunk_count_limit) const
+		{ return ChunkCount(chunk_count_limit + 1) > chunk_count_limit; }
 
 	/**
 	 * @return  the next chunk to deliver, if one could be extracted

src/fuzzers/fuzzer-setup.h

@@ -32,6 +32,9 @@ extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv)
 	options.scripts_to_load.emplace_back("local.zeek");
 	options.script_options_to_set.emplace_back("Site::local_nets={10.0.0.0/8}");
 	options.script_options_to_set.emplace_back("Log::default_writer=Log::WRITER_NONE");
+	options.script_options_to_set.emplace_back("Reporter::info_to_stderr=F");
+	options.script_options_to_set.emplace_back("Reporter::warnings_to_stderr=F");
+	options.script_options_to_set.emplace_back("Reporter::errors_to_stderr=F");
 	options.deterministic_mode = true;
 	options.ignore_checksums = true;
 	options.abort_on_scripting_errors = true;

src/fuzzers/standalone-driver.cc

@@ -17,6 +17,7 @@ int main(int argc, char** argv)
 	printf("Standalone fuzzer processing %d inputs\n", num_inputs);
 
 	LLVMFuzzerInitialize(&argc, &argv);
+	auto fuzz_start = high_resolution_clock::now();
 
 	for ( auto i = 0; i < num_inputs; ++i )
 		{
@@ -60,5 +61,7 @@ int main(int argc, char** argv)
 
 	auto agg_stop = high_resolution_clock::now();
 	auto agg_dt = duration<double>(agg_stop - agg_start).count();
-	printf("Processed %d inputs in %fs\n", num_inputs, agg_dt);
+	auto fuzz_dt = duration<double>(agg_stop - fuzz_start).count();
+	printf("Processed %d inputs in %fs (%fs w/ initialization), avg = %fs\n",
+	       num_inputs, fuzz_dt, agg_dt, fuzz_dt / num_inputs);
 	}