Merge remote-tracking branch 'origin/topic/awelzel/expose-num-packets-unprocessed'

* origin/topic/awelzel/expose-num-packets-unprocessed:
  ConnStats: Expose num_packets_unprocessed
  packet_analysis/Manager: Rename GetUnprocessedCount() to PacketsUnprocessed()

CHANGES

@@ -1,3 +1,15 @@
+8.0.0-dev.778 | 2025-07-29 10:12:18 +0200
+
+  * ConnStats: Expose num_packets_unprocessed (Arne Welzel, Corelight)
+
+    Not sure it's the best place to put, but we don't have packet analysis stats
+    bif and also num_packets is already there, so seems reasonable to put the
+    num_packets_unprocessed into
+
+  * packet_analysis/Manager: Rename GetUnprocessedCount() to PacketsUnprocessed() (Arne Welzel, Corelight)
+
+    Mostly to align with PacketsProcessed()
+
 8.0.0-dev.775 | 2025-07-29 10:05:20 +0200
 
   * Update .git-blame-ignore-revs (Arne Welzel, Corelight)

NEWS

@@ -274,6 +274,11 @@ New Functionality
   The ``onn/pppoe-session-id-logging.zeek`` policy script adds pppoe session IDs to the
   connection log.
 
+  The ``get_conn_stats()`` function's return value now includes the number of packets
+  that have not been processed by any analyzer. Using data from ``get_conn_stats()`` and
+  ``get_net_stats()``, it's possible to determine the number of packets that have
+  been received and accepted by Zeek, but eventually discarded without processing.
+
 Changed Functionality
 ---------------------
 

VERSION

@@ -1 +1 @@
-8.0.0-dev.775
+8.0.0-dev.778

scripts/base/init-bare.zeek

@@ -1092,6 +1092,7 @@ type ConnStats: record {
 	num_icmp_conns: count;        ##< Current number of ICMP flows in memory.
 	max_icmp_conns: count;        ##< Maximum number of concurrent ICMP flows so far.
 	cumulative_icmp_conns: count; ##< Total number of ICMP flows so far.
+	num_packets_unprocessed: count; ##< Total number of packets not processed by any analyzer.
 
 	killed_by_inactivity: count;
 };

src/RunState.cc

@@ -357,8 +357,8 @@ void get_final_stats() {
 
         double dropped_pct = s.dropped > 0 ? pct(s.dropped, s.received) : 0.0;
 
-        uint64_t not_processed = packet_mgr->GetUnprocessedCount();
-        double unprocessed_pct = not_processed > 0 ? ((double)not_processed / (double)s.received) * 100.0 : 0.0;
+        uint64_t unprocessed = packet_mgr->PacketsUnprocessed();
+        double unprocessed_pct = unprocessed > 0 ? ((double)unprocessed / (double)s.received) * 100.0 : 0.0;
 
         std::string filtered = "";
         if ( s.filtered ) {
@@ -368,7 +368,7 @@ void get_final_stats() {
 
         reporter->Info("%" PRIu64 " packets received on interface %s, %" PRIu64 " (%.2f%%) dropped, %" PRIu64
                        " (%.2f%%) not processed%s",
-                       s.received, ps->Path().c_str(), s.dropped, dropped_pct, not_processed, unprocessed_pct,
+                       s.received, ps->Path().c_str(), s.dropped, dropped_pct, unprocessed, unprocessed_pct,
                        filtered.c_str());
     }
 }

src/packet_analysis/Manager.h

@@ -169,11 +169,16 @@ public:
         return pkt_filter;
     }
 
+    [[deprecated("Remove in v8.1: Use PacketsUnprocessed() instead.")]]
+    uint64_t GetUnprocessedCount() const {
+        return PacketsUnprocessed();
+    }
+
     /**
      * Returns the total number of packets received that weren't considered
      * processed by some analyzer.
      */
-    uint64_t GetUnprocessedCount() const { return total_not_processed; }
+    uint64_t PacketsUnprocessed() const { return total_not_processed; }
 
     /**
      * Tracks the given analyzer for the current packet's analyzer history.

src/session/Manager.cc

@@ -218,6 +218,7 @@ void Manager::GetStats(Stats& s) {
     s.num_fragments = zeek::detail::fragment_mgr->Size();
     s.max_fragments = zeek::detail::fragment_mgr->MaxFragments();
     s.num_packets = packet_mgr->PacketsProcessed();
+    s.num_packets_unprocessed = packet_mgr->PacketsUnprocessed();
 }
 
 void Manager::Weird(const char* name, const Packet* pkt, const char* addl, const char* source) {

src/session/Manager.h

@@ -50,6 +50,7 @@ struct Stats {
     size_t num_fragments;
     size_t max_fragments;
     uint64_t num_packets;
+    uint64_t num_packets_unprocessed;
 };
 
 class Manager final {

src/stats.bif

@@ -101,10 +101,11 @@ function get_conn_stats%(%): ConnStats
 		r->Assign(n++, static_cast<uint64_t>(s.num_ICMP_conns));
 		r->Assign(n++, static_cast<uint64_t>(s.max_ICMP_conns));
 		r->Assign(n++, static_cast<uint64_t>(s.cumulative_ICMP_conns));
+		r->Assign(n++, static_cast<uint64_t>(s.num_packets_unprocessed));
 	}
 	else {
 		// Skip all of the fields that would be set from session_mgr data.
-		n += 13;
+		n += 14;
 	}
 
 	r->Assign(n++, zeek::detail::killed_by_inactivity);

testing/btest/Baseline/core.conn-stats/.stdout

@@ -1,2 +1,7 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-[total_conns=9, current_conns=5, sess_current_conns=5, num_packets=125, num_fragments=0, max_fragments=0, num_tcp_conns=5, max_tcp_conns=5, cumulative_tcp_conns=6, num_udp_conns=0, max_udp_conns=2, cumulative_udp_conns=2, num_icmp_conns=0, max_icmp_conns=1, cumulative_icmp_conns=1, killed_by_inactivity=3]
+pcap smtp.trace
+[total_conns=9, current_conns=5, sess_current_conns=5, num_packets=125, num_fragments=0, max_fragments=0, num_tcp_conns=5, max_tcp_conns=5, cumulative_tcp_conns=6, num_udp_conns=0, max_udp_conns=2, cumulative_udp_conns=2, num_icmp_conns=0, max_icmp_conns=1, cumulative_icmp_conns=1, num_packets_unprocessed=0, killed_by_inactivity=3]
+pcap dns-edns-ecs.pcap
+[total_conns=69, current_conns=9, sess_current_conns=9, num_packets=89, num_fragments=0, max_fragments=1, num_tcp_conns=1, max_tcp_conns=6, cumulative_tcp_conns=8, num_udp_conns=8, max_udp_conns=37, cumulative_udp_conns=61, num_icmp_conns=0, max_icmp_conns=0, cumulative_icmp_conns=0, num_packets_unprocessed=4, killed_by_inactivity=59]
+pcap contentline-irc-5k-line.pcap
+[total_conns=0, current_conns=0, sess_current_conns=0, num_packets=118, num_fragments=0, max_fragments=0, num_tcp_conns=0, max_tcp_conns=0, cumulative_tcp_conns=0, num_udp_conns=0, max_udp_conns=0, cumulative_udp_conns=0, num_icmp_conns=0, max_icmp_conns=0, cumulative_icmp_conns=0, num_packets_unprocessed=118, killed_by_inactivity=0]

testing/btest/core/conn-stats.zeek

@@ -1,6 +1,14 @@
 # @TEST-EXEC: zeek -b -r $TRACES/smtp.trace %INPUT
+# @TEST-EXEC: zeek -b -r $TRACES/dns-edns-ecs.pcap %INPUT
+# @TEST-EXEC: zeek -b -r $TRACES/contentline-irc-5k-line.pcap %INPUT
+#
 # @TEST-EXEC: btest-diff .stdout
 
+event zeek_init()
+	{
+	print fmt("pcap %s", split_string(packet_source()$path, /\//)[-1]);
+	}
+
 event net_done(t: time)
 	{
 	print get_conn_stats();