Merge remote-tracking branch 'jgras/topic/jgras/fix-packet-lag'

* jgras/topic/jgras/fix-packet-lag:
  Introduce get_packet_lag()
  Add btest for get_current_packet_ts()
  Introduce get_current_packet_ts to fix packet lag

CHANGES

@@ -1,3 +1,15 @@
+7.1.0-dev.738 | 2024-12-10 18:26:31 +0100
+
+  * Introduce get_packet_lag() (Jan Grashoefer, Corelight)
+
+  * Add btest for get_current_packet_ts() (Jan Grashoefer, Corelight)
+
+  * Introduce get_current_packet_ts to fix packet lag (Jan Grashoefer, Corelight)
+
+    Using network_time to calculate packet lag will produce wrong results
+    when there is no packet available but network time does not (yet) fall
+    back to wall clock.
+
 7.1.0-dev.734 | 2024-12-10 09:56:46 +0100
 
   * DNS/dns_binds_rr: Fix complte to complete typo, switch to count (Arne Welzel, Corelight)

VERSION

@@ -1 +1 @@
-7.1.0-dev.734
+7.1.0-dev.738

scripts/base/utils/time.zeek

@@ -1,3 +1,4 @@
+##! Time-related functions.
 
 ## Given an interval, returns a string representing the minutes and seconds
 ## in the interval (for example, "3m34s").
@@ -6,3 +7,22 @@ function duration_to_mins_secs(dur: interval): string
 	local dur_count = double_to_count(interval_to_double(dur));
 	return fmt("%dm%ds", dur_count/60, dur_count%60);
 	}
+
+## Time value representing the 0 timestamp.
+const null_ts = double_to_time(0);
+
+## Calculate the packet lag, i.e. the difference between wall clock and the
+## timestamp of the currently processed packet. If Zeek is not processing a
+## packet, the function returns a 0 interval value.
+function get_packet_lag(): interval
+	{
+	# We use get_current_packet_ts() instead of network_time() here, because
+	# network time does not immediately fall back to wall clock if there is
+	# no packet. Instead, network time remains set to the last seen packet's
+	# timestamp for ``packet_source_inactivity_timeout``.
+	local pkt_ts = get_current_packet_ts();
+	if (pkt_ts == null_ts)
+		return 0 sec;
+
+	return current_time() - pkt_ts;
+	}

scripts/policy/misc/stats.zeek

@@ -2,6 +2,7 @@
 
 @load base/frameworks/notice
 @load base/frameworks/telemetry
+@load base/utils/time
 
 module Stats;
 
@@ -214,7 +215,7 @@ event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: Pr
 
 	if ( reading_live_traffic() )
 		{
-		info$pkt_lag = current_time() - nettime;
+		info$pkt_lag = get_packet_lag();
 		info$pkts_dropped = ns$pkts_dropped  - last_ns$pkts_dropped;
 		info$pkts_link = ns$pkts_link  - last_ns$pkts_link;
 

src/zeek.bif

@@ -3854,6 +3854,31 @@ function get_current_packet_header%(%) : raw_pkt_hdr
 	return std::move(hdr);
 	%}
 
+## Returns the currently processed PCAP packet's timestamp or a 0 timestamp if
+## there is no packet being processed at the moment.
+##
+## Returns: The currently processed packet's timestamp.
+##
+## .. zeek:see:: get_current_packet get_current_packet_header network_time
+##
+## .. note::
+##
+##      When there is no packet being processed, ``get_current_packet_ts()``
+##      will return a 0 timestamp, while ``network_time()`` will return the
+##      timestamp of the last processed packet until it falls back to tracking
+##      wall clock after ``packet_source_inactivity_timeout``.
+function get_current_packet_ts%(%) : time
+	%{
+	double ts = 0;
+	const Packet* p = nullptr;
+	zeek::iosource::PktSrc* pkt_src = zeek::run_state::detail::current_packet_source();
+
+	if ( pkt_src && pkt_src->GetCurrentPacket(&p) )
+		ts = p->time;
+
+	return zeek::make_intrusive<zeek::TimeVal>(ts);
+	%}
+
 ## Writes a given packet to a file.
 ##
 ## pkt: The PCAP packet.

testing/btest/Baseline/core.network_time.packet_ts/output

@@ -0,0 +1,5 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+network_time_init network time: 1362692526.869344
+network_time_init packet ts:    1362692526.869344
+conn_state_remove network time: 1362692527.080972
+conn_state_remove packet ts:    0.0

testing/btest/core/network_time/packet_ts.zeek

@@ -0,0 +1,17 @@
+# @TEST-DOC: Test get_current_packet_ts() in comparison with network_time().
+# @TEST-EXEC: zeek -b -r $TRACES/http/get.trace %INPUT > output
+# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff output
+
+event network_time_init()
+	{
+	print fmt("network_time_init network time: %s", network_time());
+	print fmt("network_time_init packet ts:    %s", get_current_packet_ts());
+	}
+
+# Note: Gracefully closed connections will be actually removed after
+# tcp_close_delay (default 5 secs).
+event connection_state_remove(c: connection)
+	{
+	print fmt("conn_state_remove network time: %s", network_time());
+	print fmt("conn_state_remove packet ts:    %s", get_current_packet_ts());
+	}

testing/external/commit-hash.zeek-testing

@@ -1 +1 @@
-2a63b457f24133a845c2020a321b7cbc05262291
+4359bd2c0e776dce08f7eca30d3d34cfe3e1d98b