Add command-line option to write unprocessed packets to a file

This commit also changes the PcapDumper to automatically flush after
every called to Dump(). This is because pcap_dump has an internal buffer
of some sort that only writes to the file after a set amount of bytes.
When using the new option on a low-traffic network, it might be a while
before you see any packets written since it has to overcome that buffer
limit first.

src/Options.cc

@@ -90,6 +90,8 @@ void usage(const char* prog, int code)
 	        "    -a|--parse-only                | exit immediately after parsing scripts\n");
 	fprintf(stderr,
 	        "    -b|--bare-mode                 | don't load scripts from the base/ directory\n");
+	fprintf(stderr,
+	        "    -c|--capture-unprocessed <file>| write unprocessed packets to a tcpdump file\n");
 	fprintf(stderr, "    -d|--debug-script              | activate Zeek script debugging\n");
 	fprintf(stderr, "    -e|--exec <zeek code>          | augment loaded scripts by given code\n");
 	fprintf(stderr, "    -f|--filter <filter>           | tcpdump filter\n");
@@ -103,8 +105,9 @@ void usage(const char* prog, int code)
 	                "allowed, pass '-' as the filename to read from stdin)\n");
 	fprintf(stderr, "    -s|--rulefile <rulefile>       | read rules from given file\n");
 	fprintf(stderr, "    -t|--tracefile <tracefile>     | activate execution tracing\n");
-	fprintf(stderr, "    -u|--usage-issues              | find variable usage issues and exit; use "
+	fprintf(stderr,
-	                "-uu for deeper/more expensive analysis\n");
+	        "    -u|--usage-issues              | find variable usage issues and exit; use "
+	        "-uu for deeper/more expensive analysis\n");
 	fprintf(stderr, "    -v|--version                   | print version and exit\n");
 	fprintf(stderr, "    -w|--writefile <writefile>     | write to given tcpdump file\n");
 #ifdef DEBUG
@@ -165,9 +168,8 @@ void usage(const char* prog, int code)
 	        getenv("ZEEK_DNS_RESOLVER")
 	            ? getenv("ZEEK_DNS_RESOLVER")
 	            : "not set, will use first IPv4 address from /etc/resolv.conf");
-	fprintf(
+	fprintf(stderr, "    $ZEEK_DEBUG_LOG_STDERR         | Use stderr for debug logs generated via "
-		stderr,
+	                "the -B flag");
-		"    $ZEEK_DEBUG_LOG_STDERR         | Use stderr for debug logs generated via the -B flag");
 
 	fprintf(stderr, "\n");
 
@@ -362,6 +364,7 @@ Options parse_cmdline(int argc, char** argv)
 	constexpr struct option long_opts[] = {
 		{"parse-only", no_argument, nullptr, 'a'},
 		{"bare-mode", no_argument, nullptr, 'b'},
+		{"capture-unprocessed", required_argument, nullptr, 'c'},
 		{"debug-script", no_argument, nullptr, 'd'},
 		{"exec", required_argument, nullptr, 'e'},
 		{"filter", required_argument, nullptr, 'f'},
@@ -405,7 +408,7 @@ Options parse_cmdline(int argc, char** argv)
 	};
 
 	char opts[256];
-	util::safe_strncpy(opts, "B:e:f:G:H:I:i:j::n:O:o:p:r:s:T:t:U:w:X:CDFMNPQSWabdhmuv",
+	util::safe_strncpy(opts, "B:c:e:f:G:H:I:i:j::n:O:o:p:r:s:T:t:U:w:X:CDFMNPQSWabdhmuv",
 	                   sizeof(opts));
 
 	int op;
@@ -428,6 +431,9 @@ Options parse_cmdline(int argc, char** argv)
 			case 'b':
 				rval.bare_mode = true;
 				break;
+			case 'c':
+				rval.unprocessed_output_file = optarg;
+				break;
 			case 'd':
 				rval.debug_scripts = true;
 				break;

src/Options.h

@@ -72,6 +72,7 @@ struct Options
 	std::optional<std::string> random_seed_output_file;
 	std::optional<std::string> process_status_file;
 	std::optional<std::string> zeekygen_config_file;
+	std::optional<std::string> unprocessed_output_file;
 
 	std::set<std::string> plugins_to_load;
 	std::vector<std::string> scripts_to_load;

src/iosource/pcap/Dumper.cc

@@ -110,6 +110,7 @@ bool PcapDumper::Dump(const Packet* pkt)
 	const struct pcap_pkthdr phdr = {.ts = pkt->ts, .caplen = pkt->cap_len, .len = pkt->len};
 
 	pcap_dump((u_char*)dumper, &phdr, pkt->data);
+	pcap_dump_flush(dumper);
 	return true;
 	}
 

src/packet_analysis/Manager.cc

@@ -4,6 +4,7 @@
 
 #include "zeek/RunState.h"
 #include "zeek/Stats.h"
+#include "zeek/iosource/Manager.h"
 #include "zeek/iosource/PktDumper.h"
 #include "zeek/packet_analysis/Analyzer.h"
 #include "zeek/packet_analysis/Dispatcher.h"
@@ -24,7 +25,7 @@ Manager::~Manager()
 	delete pkt_filter;
 	}
 
-void Manager::InitPostScript()
+void Manager::InitPostScript(const std::string& unprocessed_output_file)
 	{
 	// Instantiate objects for all available analyzers
 	for ( const auto& analyzerComponent : GetComponents() )
@@ -49,6 +50,10 @@ void Manager::InitPostScript()
 	unknown_sampling_threshold = id::find_val("UnknownProtocol::sampling_threshold")->AsCount();
 	unknown_sampling_duration = id::find_val("UnknownProtocol::sampling_duration")->AsInterval();
 	unknown_first_bytes_count = id::find_val("UnknownProtocol::first_bytes_count")->AsCount();
+
+	if ( ! unprocessed_output_file.empty() )
+		// This gets automatically cleaned up by iosource_mgr. No need to delete it locally.
+		unprocessed_dumper = iosource_mgr->OpenPktDumper(unprocessed_output_file, true);
 	}
 
 void Manager::Done() { }
@@ -114,6 +119,9 @@ void Manager::ProcessPacket(Packet* packet)
 
 		plugin_mgr->HookUnprocessedPacket(packet);
 
+		if ( unprocessed_dumper )
+			unprocessed_dumper->Dump(packet);
+
 		total_not_processed++;
 		}
 

src/packet_analysis/Manager.h

@@ -18,6 +18,11 @@ namespace detail
 class PacketProfiler;
 	}
 
+namespace iosource
+	{
+class PktDumper;
+	}
+
 namespace packet_analysis
 	{
 
@@ -40,8 +45,12 @@ public:
 	/**
 	 * Second-stage initialization of the manager. This is called late
 	 * during Zeek's initialization after any scripts are processed.
+	 *
+	 * @param unprocessed_output_file A path to a file where unprocessed
+	 * packets will be written. This can be an empty string to disable
+	 * writing packets.
 	 */
-	void InitPostScript();
+	void InitPostScript(const std::string& unprocessed_output_file);
 
 	/**
 	 * Finished the manager's operations.
@@ -172,6 +181,7 @@ private:
 	uint64_t unknown_first_bytes_count = 0;
 
 	uint64_t total_not_processed = 0;
+	iosource::PktDumper* unprocessed_dumper = nullptr;
 	};
 
 	} // namespace packet_analysis

src/zeek-setup.cc

@@ -708,7 +708,7 @@ SetupResult setup(int argc, char** argv, Options* zopts)
 			exit(success ? 0 : 1);
 			}
 
-		packet_mgr->InitPostScript();
+		packet_mgr->InitPostScript(options.unprocessed_output_file.value_or(""));
 		analyzer_mgr->InitPostScript();
 		file_mgr->InitPostScript();
 		dns_mgr->InitPostScript();