From d7927622e93e7ff72bfa3782eef2b696fb6a162e Mon Sep 17 00:00:00 2001 From: Dominik Charousset Date: Thu, 14 May 2020 11:51:46 +0200 Subject: [PATCH 1/6] Upgrade to latest Broker changes for CAF 0.18 --- aux/broker | 2 +- src/broker/Store.cc | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/aux/broker b/aux/broker index 70fa315e8b..1abf27595e 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 70fa315e8b777056073a9e67f42b99573ef18231 +Subproject commit 1abf27595e0c6cd21ecc1b69412514983d966a5c diff --git a/src/broker/Store.cc b/src/broker/Store.cc index 81138edcd9..1d1ff7addb 100644 --- a/src/broker/Store.cc +++ b/src/broker/Store.cc @@ -85,13 +85,13 @@ broker::backend to_backend_type(BifEnum::Broker::BackendType type) { switch ( type ) { case BifEnum::Broker::MEMORY: - return broker::memory; + return broker::backend::memory; case BifEnum::Broker::SQLITE: - return broker::sqlite; + return broker::backend::sqlite; case BifEnum::Broker::ROCKSDB: - return broker::rocksdb; + return broker::backend::rocksdb; } throw std::runtime_error("unknown broker backend"); @@ -101,14 +101,14 @@ broker::backend_options to_backend_options(broker::backend backend, RecordVal* options) { switch ( backend ) { - case broker::sqlite: + case broker::backend::sqlite: { auto path = options->Lookup(0)->AsRecordVal() ->Lookup(0)->AsStringVal()->CheckString(); return {{"path", path}}; } - case broker::rocksdb: + case broker::backend::rocksdb: { auto path = options->Lookup(1)->AsRecordVal() ->Lookup(0)->AsStringVal()->CheckString(); From fda8b98ac783fe14830a8e1477e5f5f043382009 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 19 May 2020 11:36:03 -0700 Subject: [PATCH 2/6] Update submodule(s) [nomail] --- aux/bifcl | 2 +- aux/binpac | 2 +- aux/broker | 2 +- aux/zeek-aux | 2 +- aux/zeekctl | 2 +- cmake | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/aux/bifcl b/aux/bifcl index abd57da8ba..95fece382d 160000 --- a/aux/bifcl +++ b/aux/bifcl @@ -1 +1 @@ -Subproject commit abd57da8ba6c9337a5ed79cabbf962f71b38d62d +Subproject commit 95fece382d34bca72572cc863e1182b31a1b9945 diff --git a/aux/binpac b/aux/binpac index 0730797611..4d3d10cd54 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 07307976115c90095af183ce950364bc1982027c +Subproject commit 4d3d10cd54b1aa64f30d2fd433252f353c6ea6e0 diff --git a/aux/broker b/aux/broker index 5d5e4f02ef..6974924007 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 5d5e4f02ef3db2415c0272beaaa795ee3799272d +Subproject commit 6974924007765f70d95e5cf123b6256048ae3af7 diff --git a/aux/zeek-aux b/aux/zeek-aux index f4a659ee89..8615abced8 160000 --- a/aux/zeek-aux +++ b/aux/zeek-aux @@ -1 +1 @@ -Subproject commit f4a659ee89e95f93ff2900cde7984f032a8113e9 +Subproject commit 8615abced86b5559fa3203264be55e664b887094 diff --git a/aux/zeekctl b/aux/zeekctl index 881fe0083a..bee11c6392 160000 --- a/aux/zeekctl +++ b/aux/zeekctl @@ -1 +1 @@ -Subproject commit 881fe0083a512b3731738fb0272241e47b272e09 +Subproject commit bee11c63923b69a4468838d56067f15bc6ae3a52 diff --git a/cmake b/cmake index 7666cd4628..23f4b88f91 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 7666cd462888b0eee173040c6888f41930a8be0b +Subproject commit 23f4b88f91c537c59ef9a3ad56ec08f021ec2b2c From 1e4374bd2761ca42b01ffbb667b0cb0177071b84 Mon Sep 17 00:00:00 2001 From: Justin Azoff Date: Wed, 20 May 2020 09:44:06 -0400 Subject: [PATCH 3/6] Skip fuzz inputs that have more than 64 chunks --- src/fuzzers/FuzzBuffer.cc | 17 +++++++++++++++++ src/fuzzers/FuzzBuffer.h | 5 +++++ src/fuzzers/packet-fuzzer.cc | 3 +++ src/fuzzers/pop3-fuzzer.cc | 3 +++ 4 files changed, 28 insertions(+) diff --git a/src/fuzzers/FuzzBuffer.cc b/src/fuzzers/FuzzBuffer.cc index ebae1a1db1..098ce92f5c 100644 --- a/src/fuzzers/FuzzBuffer.cc +++ b/src/fuzzers/FuzzBuffer.cc @@ -17,6 +17,23 @@ bool zeek::detail::FuzzBuffer::Valid() const return true; } +int zeek::detail::FuzzBuffer::ChunkCount() const + { + auto pos = begin; + int chunks = 0; + while (pos < end) + { + pos = (const unsigned char*)memmem(pos, end - pos, + PKT_MAGIC, PKT_MAGIC_LEN); + if ( ! pos ) + break; + pos += PKT_MAGIC_LEN + 1; + chunks++; + } + + return chunks; + } + std::optional zeek::detail::FuzzBuffer::Next() { if ( begin == end ) diff --git a/src/fuzzers/FuzzBuffer.h b/src/fuzzers/FuzzBuffer.h index 1b9cdb58a3..f79f20091e 100644 --- a/src/fuzzers/FuzzBuffer.h +++ b/src/fuzzers/FuzzBuffer.h @@ -43,6 +43,11 @@ public: */ bool Valid() const; + /** + * @return the number of chunks in the fuzz buffer object + */ + int ChunkCount() const; + /** * @return the next chunk to deliver, if one could be extracted */ diff --git a/src/fuzzers/packet-fuzzer.cc b/src/fuzzers/packet-fuzzer.cc index d3bb28b806..59002f6c79 100644 --- a/src/fuzzers/packet-fuzzer.cc +++ b/src/fuzzers/packet-fuzzer.cc @@ -18,6 +18,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) if ( ! fb.Valid() ) return 0; + if ( fb.ChunkCount() > 64 ) + return 0; + for ( ; ; ) { auto chunk = fb.Next(); diff --git a/src/fuzzers/pop3-fuzzer.cc b/src/fuzzers/pop3-fuzzer.cc index 5cc4776aad..2c6e01fc5c 100644 --- a/src/fuzzers/pop3-fuzzer.cc +++ b/src/fuzzers/pop3-fuzzer.cc @@ -50,6 +50,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) if ( ! fb.Valid() ) return 0; + if ( fb.ChunkCount() > 64 ) + return 0; + auto conn = add_connection(); auto a = add_analyzer(conn); From 6d0599c5373566f3aed0f79d1faa1b4b845e0a63 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 20 May 2020 10:56:46 -0700 Subject: [PATCH 4/6] Improve standalone fuzz driver timing output --- CHANGES | 6 ++++++ VERSION | 2 +- src/fuzzers/standalone-driver.cc | 5 ++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index f20fcc5603..3f87234127 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,10 @@ +3.2.0-dev.554 | 2020-05-20 10:56:46 -0700 + + * Improve standalone fuzz driver timing output (Jon Siwek, Corelight) + + * Skip fuzz inputs that have more than 64 chunks (Justin Azoff, Corelight) + 3.2.0-dev.550 | 2020-05-19 10:50:42 -0700 * Upgrade to latest Broker changes for CAF 0.18 (Dominik Charousset, Corelight) diff --git a/VERSION b/VERSION index 9730e9d65d..ee2820da49 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.2.0-dev.550 +3.2.0-dev.554 diff --git a/src/fuzzers/standalone-driver.cc b/src/fuzzers/standalone-driver.cc index 69ca7202cf..64cabaefc8 100644 --- a/src/fuzzers/standalone-driver.cc +++ b/src/fuzzers/standalone-driver.cc @@ -17,6 +17,7 @@ int main(int argc, char** argv) printf("Standalone fuzzer processing %d inputs\n", num_inputs); LLVMFuzzerInitialize(&argc, &argv); + auto fuzz_start = high_resolution_clock::now(); for ( auto i = 0; i < num_inputs; ++i ) { @@ -60,5 +61,7 @@ int main(int argc, char** argv) auto agg_stop = high_resolution_clock::now(); auto agg_dt = duration(agg_stop - agg_start).count(); - printf("Processed %d inputs in %fs\n", num_inputs, agg_dt); + auto fuzz_dt = duration(agg_stop - fuzz_start).count(); + printf("Processed %d inputs in %fs (%fs w/ initialization), avg = %fs\n", + num_inputs, fuzz_dt, agg_dt, fuzz_dt / num_inputs); } From 7d98d816d83c276faaaddaf83b3991ec9f0b80d1 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 20 May 2020 11:19:08 -0700 Subject: [PATCH 5/6] Disable output of Reporter messages to stderr in fuzz targets --- CHANGES | 4 ++++ VERSION | 2 +- src/fuzzers/fuzzer-setup.h | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 3f87234127..f1e089d93b 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +3.2.0-dev.555 | 2020-05-20 11:19:08 -0700 + + * Disable output of Reporter messages to stderr in fuzz targets (Jon Siwek, Corelight) + 3.2.0-dev.554 | 2020-05-20 10:56:46 -0700 * Improve standalone fuzz driver timing output (Jon Siwek, Corelight) diff --git a/VERSION b/VERSION index ee2820da49..18a3b72a3c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.2.0-dev.554 +3.2.0-dev.555 diff --git a/src/fuzzers/fuzzer-setup.h b/src/fuzzers/fuzzer-setup.h index e692ddd3c4..976aa9126e 100644 --- a/src/fuzzers/fuzzer-setup.h +++ b/src/fuzzers/fuzzer-setup.h @@ -32,6 +32,9 @@ extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv) options.scripts_to_load.emplace_back("local.zeek"); options.script_options_to_set.emplace_back("Site::local_nets={10.0.0.0/8}"); options.script_options_to_set.emplace_back("Log::default_writer=Log::WRITER_NONE"); + options.script_options_to_set.emplace_back("Reporter::info_to_stderr=F"); + options.script_options_to_set.emplace_back("Reporter::warnings_to_stderr=F"); + options.script_options_to_set.emplace_back("Reporter::errors_to_stderr=F"); options.deterministic_mode = true; options.ignore_checksums = true; options.abort_on_scripting_errors = true; From 6aa6eea7bc44993c00ee9d969b328f8347ec41b2 Mon Sep 17 00:00:00 2001 From: Justin Azoff Date: Thu, 21 May 2020 12:55:00 -0400 Subject: [PATCH 6/6] Speed up ChunkCount validity check When counting chunks for the purpose of a Valid check, only count up to chunk_count_limit + 1 chunks. This speeds up the skipping of the 70,000 chunk test file considerably. Before: Processed 1 inputs in 0.025517s After: Processed 1 inputs in 0.000620s --- src/fuzzers/FuzzBuffer.cc | 6 +++--- src/fuzzers/FuzzBuffer.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/fuzzers/FuzzBuffer.cc b/src/fuzzers/FuzzBuffer.cc index 0eb166c786..393e830d78 100644 --- a/src/fuzzers/FuzzBuffer.cc +++ b/src/fuzzers/FuzzBuffer.cc @@ -14,18 +14,18 @@ bool zeek::detail::FuzzBuffer::Valid(int chunk_count_limit) const if ( memcmp(begin, PKT_MAGIC, PKT_MAGIC_LEN) != 0) return false; - if ( ChunkCount() > chunk_count_limit ) + if ( ChunkCount(chunk_count_limit + 1) > chunk_count_limit ) return false; return true; } -int zeek::detail::FuzzBuffer::ChunkCount() const +int zeek::detail::FuzzBuffer::ChunkCount(int chunk_count_limit) const { auto pos = begin; int chunks = 0; - while ( pos < end ) + while ( pos < end && (chunk_count_limit == 0 || chunks < chunk_count_limit) ) { pos = (const unsigned char*)memmem(pos, end - pos, PKT_MAGIC, PKT_MAGIC_LEN); diff --git a/src/fuzzers/FuzzBuffer.h b/src/fuzzers/FuzzBuffer.h index 1959e6f18b..830ed65528 100644 --- a/src/fuzzers/FuzzBuffer.h +++ b/src/fuzzers/FuzzBuffer.h @@ -49,7 +49,7 @@ public: /** * @return the number of chunks in the fuzz buffer object */ - int ChunkCount() const; + int ChunkCount(int chunk_count_limit = 0) const; /** * @return the next chunk to deliver, if one could be extracted