diff --git a/CHANGES b/CHANGES index 5447be40fb..f3d71e5a3c 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,24 @@ +3.2.0-dev.557 | 2020-05-21 11:41:12 -0700 + + * Speed up FuzzBuffer ChunkCount validity check (Justin Azoff, Corelight) + +3.2.0-dev.555 | 2020-05-20 11:19:08 -0700 + + * Disable output of Reporter messages to stderr in fuzz targets (Jon Siwek, Corelight) + +3.2.0-dev.554 | 2020-05-20 10:56:46 -0700 + + * Improve standalone fuzz driver timing output (Jon Siwek, Corelight) + + * Skip fuzz inputs that have more than 64 chunks (Justin Azoff, Corelight) + +3.2.0-dev.550 | 2020-05-19 10:50:42 -0700 + + * Upgrade to latest Broker changes for CAF 0.18 (Dominik Charousset, Corelight) + + * Include pcap.h instead of pcap/dlt.h in packet-fuzzer (Tim Wojtulewicz, Corelight) + 3.2.0-dev.547 | 2020-05-18 10:47:54 -0700 * add packet fuzzer (Justin Azoff) diff --git a/VERSION b/VERSION index 6f92b88996..69911ac90f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.2.0-dev.547 +3.2.0-dev.557 diff --git a/aux/bifcl b/aux/bifcl index abd57da8ba..95fece382d 160000 --- a/aux/bifcl +++ b/aux/bifcl @@ -1 +1 @@ -Subproject commit abd57da8ba6c9337a5ed79cabbf962f71b38d62d +Subproject commit 95fece382d34bca72572cc863e1182b31a1b9945 diff --git a/aux/binpac b/aux/binpac index 0730797611..4d3d10cd54 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 07307976115c90095af183ce950364bc1982027c +Subproject commit 4d3d10cd54b1aa64f30d2fd433252f353c6ea6e0 diff --git a/aux/broker b/aux/broker index 74d1689643..6974924007 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 74d168964369f8b5c51238520d6ad35173791110 +Subproject commit 6974924007765f70d95e5cf123b6256048ae3af7 diff --git a/aux/zeek-aux b/aux/zeek-aux index f4a659ee89..8615abced8 160000 --- a/aux/zeek-aux +++ b/aux/zeek-aux @@ -1 +1 @@ -Subproject commit f4a659ee89e95f93ff2900cde7984f032a8113e9 +Subproject commit 8615abced86b5559fa3203264be55e664b887094 diff --git a/aux/zeekctl b/aux/zeekctl index 881fe0083a..bee11c6392 160000 --- a/aux/zeekctl +++ b/aux/zeekctl @@ -1 +1 @@ -Subproject commit 881fe0083a512b3731738fb0272241e47b272e09 +Subproject commit bee11c63923b69a4468838d56067f15bc6ae3a52 diff --git a/cmake b/cmake index 7666cd4628..23f4b88f91 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 7666cd462888b0eee173040c6888f41930a8be0b +Subproject commit 23f4b88f91c537c59ef9a3ad56ec08f021ec2b2c diff --git a/src/broker/Store.cc b/src/broker/Store.cc index 81138edcd9..1d1ff7addb 100644 --- a/src/broker/Store.cc +++ b/src/broker/Store.cc @@ -85,13 +85,13 @@ broker::backend to_backend_type(BifEnum::Broker::BackendType type) { switch ( type ) { case BifEnum::Broker::MEMORY: - return broker::memory; + return broker::backend::memory; case BifEnum::Broker::SQLITE: - return broker::sqlite; + return broker::backend::sqlite; case BifEnum::Broker::ROCKSDB: - return broker::rocksdb; + return broker::backend::rocksdb; } throw std::runtime_error("unknown broker backend"); @@ -101,14 +101,14 @@ broker::backend_options to_backend_options(broker::backend backend, RecordVal* options) { switch ( backend ) { - case broker::sqlite: + case broker::backend::sqlite: { auto path = options->Lookup(0)->AsRecordVal() ->Lookup(0)->AsStringVal()->CheckString(); return {{"path", path}}; } - case broker::rocksdb: + case broker::backend::rocksdb: { auto path = options->Lookup(1)->AsRecordVal() ->Lookup(0)->AsStringVal()->CheckString(); diff --git a/src/fuzzers/FuzzBuffer.cc b/src/fuzzers/FuzzBuffer.cc index ebae1a1db1..def5d34086 100644 --- a/src/fuzzers/FuzzBuffer.cc +++ b/src/fuzzers/FuzzBuffer.cc @@ -6,7 +6,7 @@ #include "FuzzBuffer.h" -bool zeek::detail::FuzzBuffer::Valid() const +bool zeek::detail::FuzzBuffer::Valid(int chunk_count_limit) const { if ( end - begin < PKT_MAGIC_LEN + 2 ) return false; @@ -14,9 +14,31 @@ bool zeek::detail::FuzzBuffer::Valid() const if ( memcmp(begin, PKT_MAGIC, PKT_MAGIC_LEN) != 0) return false; + if ( ExceedsChunkLimit(chunk_count_limit) ) + return false; + return true; } +int zeek::detail::FuzzBuffer::ChunkCount(int chunk_count_limit) const + { + auto pos = begin; + int chunks = 0; + + while ( pos < end && (chunks < chunk_count_limit || chunk_count_limit == 0) ) + { + pos = (const unsigned char*)memmem(pos, end - pos, + PKT_MAGIC, PKT_MAGIC_LEN); + if ( ! pos ) + break; + + pos += PKT_MAGIC_LEN + 1; + chunks++; + } + + return chunks; + } + std::optional zeek::detail::FuzzBuffer::Next() { if ( begin == end ) diff --git a/src/fuzzers/FuzzBuffer.h b/src/fuzzers/FuzzBuffer.h index 1b9cdb58a3..69ba821242 100644 --- a/src/fuzzers/FuzzBuffer.h +++ b/src/fuzzers/FuzzBuffer.h @@ -27,6 +27,7 @@ public: static constexpr int PKT_MAGIC_LEN = 4; static constexpr unsigned char PKT_MAGIC[PKT_MAGIC_LEN + 1] = "\1PKT"; + static constexpr int MAX_CHUNK_COUNT = 64; /** * Initialize fuzz buffer. @@ -39,9 +40,25 @@ public: /** * @return whether the fuzz buffer object is valid -- has enough bytes - * to Deliver to an analyzer and starts with a *PKT_MAGIC* bytestring. + * to Deliver to an analyzer, starts with a *PKT_MAGIC* bytestring, and + * contains less than the limiting number of chunk. + * . */ - bool Valid() const; + bool Valid(int chunk_count_limit = MAX_CHUNK_COUNT) const; + + /** + * @param chunk_count_limit Number of chunks to stop counting at (zero + * means "never stop"). + * @return the number of chunks in the fuzz buffer object + */ + int ChunkCount(int chunk_count_limit = 0) const; + + /** + * @param Maximum number of chunks to permit the FuzzBuffer to have. + * @return Whether the FuzzBuffer exceeds the desired chunk count limit. + */ + bool ExceedsChunkLimit(int chunk_count_limit) const + { return ChunkCount(chunk_count_limit + 1) > chunk_count_limit; } /** * @return the next chunk to deliver, if one could be extracted diff --git a/src/fuzzers/fuzzer-setup.h b/src/fuzzers/fuzzer-setup.h index e692ddd3c4..976aa9126e 100644 --- a/src/fuzzers/fuzzer-setup.h +++ b/src/fuzzers/fuzzer-setup.h @@ -32,6 +32,9 @@ extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv) options.scripts_to_load.emplace_back("local.zeek"); options.script_options_to_set.emplace_back("Site::local_nets={10.0.0.0/8}"); options.script_options_to_set.emplace_back("Log::default_writer=Log::WRITER_NONE"); + options.script_options_to_set.emplace_back("Reporter::info_to_stderr=F"); + options.script_options_to_set.emplace_back("Reporter::warnings_to_stderr=F"); + options.script_options_to_set.emplace_back("Reporter::errors_to_stderr=F"); options.deterministic_mode = true; options.ignore_checksums = true; options.abort_on_scripting_errors = true; diff --git a/src/fuzzers/standalone-driver.cc b/src/fuzzers/standalone-driver.cc index 69ca7202cf..64cabaefc8 100644 --- a/src/fuzzers/standalone-driver.cc +++ b/src/fuzzers/standalone-driver.cc @@ -17,6 +17,7 @@ int main(int argc, char** argv) printf("Standalone fuzzer processing %d inputs\n", num_inputs); LLVMFuzzerInitialize(&argc, &argv); + auto fuzz_start = high_resolution_clock::now(); for ( auto i = 0; i < num_inputs; ++i ) { @@ -60,5 +61,7 @@ int main(int argc, char** argv) auto agg_stop = high_resolution_clock::now(); auto agg_dt = duration(agg_stop - agg_start).count(); - printf("Processed %d inputs in %fs\n", num_inputs, agg_dt); + auto fuzz_dt = duration(agg_stop - fuzz_start).count(); + printf("Processed %d inputs in %fs (%fs w/ initialization), avg = %fs\n", + num_inputs, fuzz_dt, agg_dt, fuzz_dt / num_inputs); }