GH-1221: Add unknown_protocols.log for logging packet analyzer lookup failures

src/packet_analysis/Analyzer.cc

@@ -80,7 +80,7 @@ bool Analyzer::ForwardPacket(size_t len, const uint8_t* data, Packet* packet,
 		{
 		DBG_LOG(DBG_PACKET_ANALYSIS, "Analysis in %s failed, could not find analyzer for identifier %#x.",
 				GetAnalyzerName(), identifier);
-		packet->Weird("no_suitable_analyzer_found");
+		packet_mgr->ReportUnknownProtocol(GetAnalyzerName(), identifier, data, len);
 		return false;
 		}
 

src/packet_analysis/Manager.cc

@@ -43,6 +43,14 @@ void Manager::InitPostScript()
 		pkt_profiler = new detail::PacketProfiler(detail::pkt_profile_mode,
 		                                          detail::pkt_profile_freq,
 		                                          pkt_profile_file->AsFile());
+
+	if ( unknown_protocol )
+		{
+		unknown_sampling_rate = id::find_val("UnknownProtocol::sampling_rate")->AsCount();
+		unknown_sampling_threshold = id::find_val("UnknownProtocol::sampling_threshold")->AsCount();
+		unknown_sampling_duration = id::find_val("UnknownProtocol::sampling_duration")->AsInterval();
+		unknown_first_bytes_count = id::find_val("UnknownProtocol::first_bytes_count")->AsCount();
+		}
 	}
 
 void Manager::Done()
@@ -171,3 +179,62 @@ void Manager::DumpPacket(const Packet *pkt, int len)
 
 	run_state::detail::pkt_dumper->Dump(pkt);
 	}
+
+class UnknownProtocolTimer final : public zeek::detail::Timer {
+public:
+	// Represents a combination of an analyzer name and protocol identifier, where the identifier was
+	// reported as unknown by the analyzer.
+	using UnknownProtocolPair = std::pair<std::string, uint32_t>;
+
+	UnknownProtocolTimer(double t, UnknownProtocolPair p, double timeout)
+		: zeek::detail::Timer(t + timeout, zeek::detail::TIMER_UNKNOWN_PROTOCOL_EXPIRE),
+		  unknown_protocol(std::move(p))
+		{}
+
+	void Dispatch(double t, bool is_expire) override
+		{ zeek::packet_mgr->ResetUnknownProtocolTimer(unknown_protocol.first, unknown_protocol.second); }
+
+	UnknownProtocolPair unknown_protocol;
+};
+
+void Manager::ResetUnknownProtocolTimer(const std::string& analyzer, uint32_t protocol)
+	{
+	unknown_protocols.erase(std::make_pair(analyzer, protocol));
+	}
+
+bool Manager::PermitUnknownProtocol(const std::string& analyzer, uint32_t protocol)
+	{
+	auto p = std::make_pair(analyzer, protocol);
+	uint64_t& count = unknown_protocols[p];
+	++count;
+
+	if ( count == 1 )
+		detail::timer_mgr->Add(new UnknownProtocolTimer(run_state::network_time, p,
+		                                                unknown_sampling_duration));
+
+	if ( count < unknown_sampling_threshold )
+		return true;
+
+	auto num_above_threshold = count - unknown_sampling_threshold;
+	if ( unknown_sampling_rate )
+		return num_above_threshold % unknown_sampling_rate == 0;
+
+	return false;
+	}
+
+void Manager::ReportUnknownProtocol(const std::string& analyzer, uint32_t protocol,
+                                    const uint8_t* data, size_t len)
+	{
+	if ( unknown_protocol )
+		{
+		if ( PermitUnknownProtocol(analyzer, protocol ) )
+			{
+			int bytes_len = std::min(unknown_first_bytes_count, static_cast<uint64_t>(len));
+
+			event_mgr.Enqueue(unknown_protocol,
+			                  make_intrusive<StringVal>(analyzer),
+			                  val_mgr->Count(protocol),
+			                  make_intrusive<StringVal>(bytes_len, (const char*) data));
+			}
+		}
+	}

src/packet_analysis/Manager.h

@@ -92,6 +92,25 @@ public:
 	 */
 	void DumpPacket(const Packet *pkt, int len=0);
 
+	/**
+	 * Attempts to write an entry to unknown_protocols.log, rate-limited to avoid
+	 * spamming the log with duplicates.
+	 *
+	 * @param analyzer The name of the analyzer that was trying to forward the packet.
+	 * @param protocol The protocol of the next header that couldn't be forwarded.
+	 * @param data A pointer to the data of the next header being processed. If this
+	 * is passed as a nullptr, the first_bytes log column will be blank.
+	 * @param len The remaining length of the data in the packet being processed.
+	 */
+	void ReportUnknownProtocol(const std::string& analyzer, uint32_t protocol,
+	                           const uint8_t* data=nullptr, size_t len=0);
+
+	/**
+	 * Callback method for UnknownProtocolTimer to remove an analyzer/protocol
+	 * pair from the map so that it can be logged again.
+	 */
+	void ResetUnknownProtocolTimer(const std::string& analyzer, uint32_t protocol);
+
 private:
 	/**
 	 * Instantiates a new analyzer instance.
@@ -113,11 +132,21 @@ private:
 	 */
 	AnalyzerPtr InstantiateAnalyzer(const std::string& name);
 
+	bool PermitUnknownProtocol(const std::string& analyzer, uint32_t protocol);
+
 	std::map<std::string, AnalyzerPtr> analyzers;
 	AnalyzerPtr root_analyzer = nullptr;
 
 	uint64_t num_packets_processed = 0;
 	detail::PacketProfiler* pkt_profiler = nullptr;
+
+	using UnknownProtocolPair = std::pair<std::string, uint32_t>;
+	std::map<UnknownProtocolPair, uint64_t> unknown_protocols;
+
+	uint64_t unknown_sampling_threshold;
+	uint64_t unknown_sampling_rate;
+	double unknown_sampling_duration;
+	uint64_t unknown_first_bytes_count;
 };
 
 } // namespace packet_analysis