Merge remote-tracking branch 'origin/topic/timw/2142-more-fuzzers'

* origin/topic/timw/2142-more-fuzzers:
  Fix a crash related to a broken IPv6 chain
  Add a couple of ICMP files to corpus for packet-fuzzer
  Trick event handlers into returning that they exist during fuzzing
  Add http, ftp, imap, and smtp fuzzers and corpora
  Add section to fuzzer README about generating corpus from pcaps
  Rename fuzzers/README to README.rst so github renders it

CHANGES

@@ -1,3 +1,17 @@
+5.1.0-dev.380 | 2022-08-11 10:30:33 -0700
+
+  * Fix a crash related to a broken IPv6 chain (Tim Wojtulewicz)
+
+  * Add a couple of ICMP files to corpus for packet-fuzzer (Tim Wojtulewicz, Corelight)
+
+  * Trick event handlers into returning that they exist during fuzzing (Tim Wojtulewicz, Corelight)
+
+  * Add http, ftp, imap, and smtp fuzzers and corpora (Tim Wojtulewicz, Corelight)
+
+  * Add section to fuzzer README about generating corpus from pcaps (Tim Wojtulewicz, Corelight)
+
+  * Rename fuzzers/README to README.rst so github renders it (Tim Wojtulewicz, Corelight)
+
 5.1.0-dev.373 | 2022-08-11 08:49:22 -0700
 
   * Add table_keys function (AmazingPP)

VERSION

@@ -1 +1 @@
-5.1.0-dev.373
+5.1.0-dev.380

src/EventRegistry.cc

@@ -136,4 +136,14 @@ void EventRegistry::SetErrorHandler(std::string_view name)
 	                          std::string(name).c_str());
 	}
 
+void EventRegistry::ActivateAllHandlers()
+	{
+	auto event_names = AllHandlers();
+	for ( const auto& name : event_names )
+		{
+		if ( auto event = Lookup(name) )
+			event->SetGenerateAlways();
+		}
+	}
+
 	} // namespace zeek

src/EventRegistry.h

@@ -60,6 +60,17 @@ public:
 
 	void PrintDebug();
 
+	/**
+	 * Marks all event handlers as active.
+	 *
+	 * By default, zeek does not generate (raise) events that have not handled by
+	 * any scripts. This means that these events will be invisible to a lot of other
+	 * event handlers - and will not raise :zeek:id:`new_event`. Calling this
+	 * function will cause all event handlers to be raised. This is likely only
+	 * useful for debugging and fuzzing, and likely causes reduced performance.
+	 */
+	void ActivateAllHandlers();
+
 private:
 	std::map<std::string, std::unique_ptr<EventHandler>, std::less<>> handlers;
 	// Tracks whether a given event handler was registered in a

src/IP.cc

@@ -39,6 +39,10 @@ static VectorValPtr BuildOptionsVal(const u_char* data, int len)
 			{
 			// PadN or other option
 			uint16_t off = 2 * sizeof(uint8_t);
+
+			if ( len < opt->ip6o_len + off )
+				break;
+
 			rv->Assign(1, opt->ip6o_len);
 			rv->Assign(2, new String(data + off, opt->ip6o_len, true));
 			data += opt->ip6o_len + off;

src/fuzzers/CMakeLists.txt

@@ -81,3 +81,7 @@ target_link_libraries(zeek_fuzzer_shared
 add_fuzz_target(dns)
 add_fuzz_target(pop3)
 add_fuzz_target(packet)
+add_fuzz_target(http)
+add_fuzz_target(imap)
+add_fuzz_target(smtp)
+add_fuzz_target(ftp)

src/fuzzers/README.rst

@@ -23,6 +23,9 @@ First configure and build for fuzzing (with libFuzzer) and code coverage::
    variable may be changed to use another flag or direct path to fuzzing engine
    library to link against.
 
+Text/Dictionary-based Corpus
+````````````````````````````
+
 Now start fuzzing to generate an initial corpus (this uses the POP3 fuzzer as
 an example)::
 
@@ -54,6 +57,24 @@ commit)::
 
     zip -j ../src/fuzzers/pop3-corpus.zip min-corpus/*
 
+pcap-based Corpus
+`````````````````
+
+A corpus can also be generated from representative pcp files using the
+``pcap-to-pkt`` application from pcap_simplify_. The fuzzers only handle a
+single connection at a time, so pcap files with multiple connections will
+need to be split using ``PcapSplitter`` from PcapPlusPlus_ or something
+similar. Once the file has been split, the individual connections can be
+converted into separate pkt files. The ``http`` fuzzer is a good example
+of a fuzzer using such files. The corpus for that fuzzer was initially
+generated from a subset of the pcap files located in ``testing/btest/Traces/http``.
+
+.. _pcap_simplify: https://github.com/JustinAzoff/pcap_simplify
+.. _PcapPlusPlus: https://github.com/seladb/PcapPlusPlus
+
+The converted pkt files can then be zipped as in the text-based section
+above.
+
 Example Build: Run Standalone Fuzz Targets
 ------------------------------------------
 

src/fuzzers/ftp-fuzzer.cc

@@ -0,0 +1,78 @@
+#include <binpac.h>
+
+#include "zeek/Conn.h"
+#include "zeek/RunState.h"
+#include "zeek/analyzer/Analyzer.h"
+#include "zeek/analyzer/Manager.h"
+#include "zeek/analyzer/protocol/pia/PIA.h"
+#include "zeek/analyzer/protocol/tcp/TCP.h"
+#include "zeek/fuzzers/FuzzBuffer.h"
+#include "zeek/fuzzers/fuzzer-setup.h"
+#include "zeek/packet_analysis/protocol/tcp/TCPSessionAdapter.h"
+#include "zeek/session/Manager.h"
+
+static constexpr auto ZEEK_FUZZ_ANALYZER = "ftp";
+
+static zeek::Connection* add_connection()
+	{
+	static constexpr double network_time_start = 1439471031;
+	zeek::run_state::detail::update_network_time(network_time_start);
+
+	zeek::Packet p;
+	zeek::ConnTuple conn_id;
+	conn_id.src_addr = zeek::IPAddr("1.2.3.4");
+	conn_id.dst_addr = zeek::IPAddr("5.6.7.8");
+	conn_id.src_port = htons(23132);
+	conn_id.dst_port = htons(80);
+	conn_id.is_one_way = false;
+	conn_id.proto = TRANSPORT_TCP;
+	zeek::detail::ConnKey key(conn_id);
+	zeek::Connection* conn = new zeek::Connection(key, network_time_start, &conn_id, 1, &p);
+	conn->SetTransport(TRANSPORT_TCP);
+	zeek::session_mgr->Insert(conn);
+	return conn;
+	}
+
+static zeek::analyzer::Analyzer* add_analyzer(zeek::Connection* conn)
+	{
+	auto* tcp = new zeek::packet_analysis::TCP::TCPSessionAdapter(conn);
+	auto* pia = new zeek::analyzer::pia::PIA_TCP(conn);
+	auto a = zeek::analyzer_mgr->InstantiateAnalyzer(ZEEK_FUZZ_ANALYZER, conn);
+	tcp->AddChildAnalyzer(a);
+	tcp->AddChildAnalyzer(pia->AsAnalyzer());
+	conn->SetSessionAdapter(tcp, pia);
+	return a;
+	}
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
+	{
+	zeek::detail::FuzzBuffer fb{data, size};
+
+	if ( ! fb.Valid() )
+		return 0;
+
+	auto conn = add_connection();
+	auto a = add_analyzer(conn);
+
+	for ( ;; )
+		{
+		auto chunk = fb.Next();
+
+		if ( ! chunk )
+			break;
+
+		try
+			{
+			a->DeliverStream(chunk->size, chunk->data.get(), chunk->is_orig);
+			}
+		catch ( const binpac::Exception& e )
+			{
+			}
+
+		chunk = {};
+		zeek::event_mgr.Drain();
+		}
+
+	zeek::detail::fuzzer_cleanup_one_input();
+	return 0;
+	}

src/fuzzers/fuzzer-setup.h

@@ -4,6 +4,7 @@
 #include <cstdlib>
 
 #include "zeek/Event.h"
+#include "zeek/EventRegistry.h"
 #include "zeek/broker/Manager.h"
 #include "zeek/file_analysis/Manager.h"
 #include "zeek/session/Manager.h"
@@ -41,6 +42,11 @@ extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv)
 	if ( zeek::detail::setup(*argc, *argv, &options).code )
 		abort();
 
+	// We have to trick the event handlers into returning true that they exist here
+	// even if they don't, because otherwise we lose a bit of coverage where if
+	// statements return false that would otherwise not.
+	zeek::event_registry->ActivateAllHandlers();
+
 	return 0;
 	}
 

src/fuzzers/http-fuzzer.cc

@@ -0,0 +1,78 @@
+#include <binpac.h>
+
+#include "zeek/Conn.h"
+#include "zeek/RunState.h"
+#include "zeek/analyzer/Analyzer.h"
+#include "zeek/analyzer/Manager.h"
+#include "zeek/analyzer/protocol/pia/PIA.h"
+#include "zeek/analyzer/protocol/tcp/TCP.h"
+#include "zeek/fuzzers/FuzzBuffer.h"
+#include "zeek/fuzzers/fuzzer-setup.h"
+#include "zeek/packet_analysis/protocol/tcp/TCPSessionAdapter.h"
+#include "zeek/session/Manager.h"
+
+static constexpr auto ZEEK_FUZZ_ANALYZER = "http";
+
+static zeek::Connection* add_connection()
+	{
+	static constexpr double network_time_start = 1439471031;
+	zeek::run_state::detail::update_network_time(network_time_start);
+
+	zeek::Packet p;
+	zeek::ConnTuple conn_id;
+	conn_id.src_addr = zeek::IPAddr("1.2.3.4");
+	conn_id.dst_addr = zeek::IPAddr("5.6.7.8");
+	conn_id.src_port = htons(23132);
+	conn_id.dst_port = htons(80);
+	conn_id.is_one_way = false;
+	conn_id.proto = TRANSPORT_TCP;
+	zeek::detail::ConnKey key(conn_id);
+	zeek::Connection* conn = new zeek::Connection(key, network_time_start, &conn_id, 1, &p);
+	conn->SetTransport(TRANSPORT_TCP);
+	zeek::session_mgr->Insert(conn);
+	return conn;
+	}
+
+static zeek::analyzer::Analyzer* add_analyzer(zeek::Connection* conn)
+	{
+	auto* tcp = new zeek::packet_analysis::TCP::TCPSessionAdapter(conn);
+	auto* pia = new zeek::analyzer::pia::PIA_TCP(conn);
+	auto a = zeek::analyzer_mgr->InstantiateAnalyzer(ZEEK_FUZZ_ANALYZER, conn);
+	tcp->AddChildAnalyzer(a);
+	tcp->AddChildAnalyzer(pia->AsAnalyzer());
+	conn->SetSessionAdapter(tcp, pia);
+	return a;
+	}
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
+	{
+	zeek::detail::FuzzBuffer fb{data, size};
+
+	if ( ! fb.Valid() )
+		return 0;
+
+	auto conn = add_connection();
+	auto a = add_analyzer(conn);
+
+	for ( ;; )
+		{
+		auto chunk = fb.Next();
+
+		if ( ! chunk )
+			break;
+
+		try
+			{
+			a->DeliverStream(chunk->size, chunk->data.get(), chunk->is_orig);
+			}
+		catch ( const binpac::Exception& e )
+			{
+			}
+
+		chunk = {};
+		zeek::event_mgr.Drain();
+		}
+
+	zeek::detail::fuzzer_cleanup_one_input();
+	return 0;
+	}

src/fuzzers/imap-fuzzer.cc

@@ -0,0 +1,78 @@
+#include <binpac.h>
+
+#include "zeek/Conn.h"
+#include "zeek/RunState.h"
+#include "zeek/analyzer/Analyzer.h"
+#include "zeek/analyzer/Manager.h"
+#include "zeek/analyzer/protocol/pia/PIA.h"
+#include "zeek/analyzer/protocol/tcp/TCP.h"
+#include "zeek/fuzzers/FuzzBuffer.h"
+#include "zeek/fuzzers/fuzzer-setup.h"
+#include "zeek/packet_analysis/protocol/tcp/TCPSessionAdapter.h"
+#include "zeek/session/Manager.h"
+
+static constexpr auto ZEEK_FUZZ_ANALYZER = "imap";
+
+static zeek::Connection* add_connection()
+	{
+	static constexpr double network_time_start = 1439471031;
+	zeek::run_state::detail::update_network_time(network_time_start);
+
+	zeek::Packet p;
+	zeek::ConnTuple conn_id;
+	conn_id.src_addr = zeek::IPAddr("1.2.3.4");
+	conn_id.dst_addr = zeek::IPAddr("5.6.7.8");
+	conn_id.src_port = htons(23132);
+	conn_id.dst_port = htons(80);
+	conn_id.is_one_way = false;
+	conn_id.proto = TRANSPORT_TCP;
+	zeek::detail::ConnKey key(conn_id);
+	zeek::Connection* conn = new zeek::Connection(key, network_time_start, &conn_id, 1, &p);
+	conn->SetTransport(TRANSPORT_TCP);
+	zeek::session_mgr->Insert(conn);
+	return conn;
+	}
+
+static zeek::analyzer::Analyzer* add_analyzer(zeek::Connection* conn)
+	{
+	auto* tcp = new zeek::packet_analysis::TCP::TCPSessionAdapter(conn);
+	auto* pia = new zeek::analyzer::pia::PIA_TCP(conn);
+	auto a = zeek::analyzer_mgr->InstantiateAnalyzer(ZEEK_FUZZ_ANALYZER, conn);
+	tcp->AddChildAnalyzer(a);
+	tcp->AddChildAnalyzer(pia->AsAnalyzer());
+	conn->SetSessionAdapter(tcp, pia);
+	return a;
+	}
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
+	{
+	zeek::detail::FuzzBuffer fb{data, size};
+
+	if ( ! fb.Valid() )
+		return 0;
+
+	auto conn = add_connection();
+	auto a = add_analyzer(conn);
+
+	for ( ;; )
+		{
+		auto chunk = fb.Next();
+
+		if ( ! chunk )
+			break;
+
+		try
+			{
+			a->DeliverStream(chunk->size, chunk->data.get(), chunk->is_orig);
+			}
+		catch ( const binpac::Exception& e )
+			{
+			}
+
+		chunk = {};
+		zeek::event_mgr.Drain();
+		}
+
+	zeek::detail::fuzzer_cleanup_one_input();
+	return 0;
+	}

src/fuzzers/smtp-fuzzer.cc

@@ -0,0 +1,78 @@
+#include <binpac.h>
+
+#include "zeek/Conn.h"
+#include "zeek/RunState.h"
+#include "zeek/analyzer/Analyzer.h"
+#include "zeek/analyzer/Manager.h"
+#include "zeek/analyzer/protocol/pia/PIA.h"
+#include "zeek/analyzer/protocol/tcp/TCP.h"
+#include "zeek/fuzzers/FuzzBuffer.h"
+#include "zeek/fuzzers/fuzzer-setup.h"
+#include "zeek/packet_analysis/protocol/tcp/TCPSessionAdapter.h"
+#include "zeek/session/Manager.h"
+
+static constexpr auto ZEEK_FUZZ_ANALYZER = "smtp";
+
+static zeek::Connection* add_connection()
+	{
+	static constexpr double network_time_start = 1439471031;
+	zeek::run_state::detail::update_network_time(network_time_start);
+
+	zeek::Packet p;
+	zeek::ConnTuple conn_id;
+	conn_id.src_addr = zeek::IPAddr("1.2.3.4");
+	conn_id.dst_addr = zeek::IPAddr("5.6.7.8");
+	conn_id.src_port = htons(23132);
+	conn_id.dst_port = htons(80);
+	conn_id.is_one_way = false;
+	conn_id.proto = TRANSPORT_TCP;
+	zeek::detail::ConnKey key(conn_id);
+	zeek::Connection* conn = new zeek::Connection(key, network_time_start, &conn_id, 1, &p);
+	conn->SetTransport(TRANSPORT_TCP);
+	zeek::session_mgr->Insert(conn);
+	return conn;
+	}
+
+static zeek::analyzer::Analyzer* add_analyzer(zeek::Connection* conn)
+	{
+	auto* tcp = new zeek::packet_analysis::TCP::TCPSessionAdapter(conn);
+	auto* pia = new zeek::analyzer::pia::PIA_TCP(conn);
+	auto a = zeek::analyzer_mgr->InstantiateAnalyzer(ZEEK_FUZZ_ANALYZER, conn);
+	tcp->AddChildAnalyzer(a);
+	tcp->AddChildAnalyzer(pia->AsAnalyzer());
+	conn->SetSessionAdapter(tcp, pia);
+	return a;
+	}
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
+	{
+	zeek::detail::FuzzBuffer fb{data, size};
+
+	if ( ! fb.Valid() )
+		return 0;
+
+	auto conn = add_connection();
+	auto a = add_analyzer(conn);
+
+	for ( ;; )
+		{
+		auto chunk = fb.Next();
+
+		if ( ! chunk )
+			break;
+
+		try
+			{
+			a->DeliverStream(chunk->size, chunk->data.get(), chunk->is_orig);
+			}
+		catch ( const binpac::Exception& e )
+			{
+			}
+
+		chunk = {};
+		zeek::event_mgr.Drain();
+		}
+
+	zeek::detail::fuzzer_cleanup_one_input();
+	return 0;
+	}

src/zeek.bif

@@ -5366,16 +5366,7 @@ function match_signatures%(c: connection, pattern_type: int, s: string,
 ## only useful for debugging and causes reduced performance.
 function generate_all_events%(%) : bool
 	%{
-	auto event_names = event_registry->AllHandlers();
+	event_registry->ActivateAllHandlers();
-	for ( const auto& name: event_names )
-		{
-		auto event = event_registry->Lookup(name);
-		if ( event == nullptr )
-			continue;
-
-		event->SetGenerateAlways();
-		}
-
 	return zeek::val_mgr->True();
 	%}