Merge remote-tracking branch 'origin/topic/seth/stats-improvement'

(Cleaned up some code a little bit.)

* origin/topic/seth/stats-improvement:
  Fixing tests for stats improvements
  Rename the reporting interval variable for stats.
  Removing more broken functionality due to changed stats apis.
  Removing some references to resource_usage()
  Removing Broker stats, it was broken and incomplete.
  Fixing default stats collection interval to every 5 minutes.
  Add DNS stats to the stats.log
  Small stats script tweaks and beginning broker stats.
  Continued stats cleanup and extension.
  More stats collection extensions.
  More stats improvements
  Slight change to Mach API for collecting memory usage.
  Fixing some small mistakes.
  Updating the cmake submodule for the stats updates.
  Fix memory usage collection on Mac OS X.
  Cleaned up stats collection.

BIT-1581 #merged

CHANGES

@@ -1,4 +1,25 @@
 
+2.4-541 | 2016-05-06 17:58:45 -0700
+
+  * A set of new built-in function for gathering execution statistics:
+
+      get_net_stats(), get_conn_stats(), get_proc_stats(),
+      get_event_stats(), get_reassembler_stats(), get_dns_stats(),
+      get_timer_stats(), get_file_analysis_stats(), get_thread_stats(),
+      get_gap_stats(), get_matcher_stats().
+
+    net_stats() resource_usage() have been superseded by these. (Seth
+    Hall)
+
+  * New policy script misc/stats.bro that records Bro execution
+    statistics in a standard Bro log file. (Seth Hall)
+
+  * A series of documentation improvements. (Daniel Thayer)
+
+  * Rudimentary XMPP StartTLS analyzer. It parses certificates out of
+    XMPP connections using StartTLS. It aborts processing if StartTLS
+    is not found. (Johanna Amann)
+
 2.4-507 | 2016-05-03 11:18:16 -0700
 
   * Fix incorrect type tags in Bro broker source code. These are just

NEWS

@@ -41,6 +41,9 @@ New Functionality
 - Bro now tracks VLAN IDs. To record them inside the connection log,
   load protocols/conn/vlan-logging.bro.
 
+- The new misc/stats.bro records Bro executions statistics in a
+  standard Bro log file.
+
 - A new dns_CAA_reply event gives access to DNS Certification Authority
   Authorization replies.
 
@@ -83,6 +86,13 @@ New Functionality
 - The IRC analyzer now recognizes StartTLS sessions and enable the SSL
   analyzer for them.
 
+- A set of new built-in function for gathering execution statistics:
+
+      get_net_stats(), get_conn_stats(), get_proc_stats(),
+      get_event_stats(), get_reassembler_stats(), get_dns_stats(),
+      get_timer_stats(), get_file_analysis_stats(), get_thread_stats(),
+      get_gap_stats(), get_matcher_stats(),
+
 - New Bro plugins in aux/plugins:
 
     - af_packet: Native AF_PACKET support.
@@ -102,6 +112,9 @@ Changed Functionality
 - ``SSH::skip_processing_after_detection`` was removed. The functionality was
   replaced by ``SSH::disable_analyzer_after_detection``.
 
+- ``net_stats()`` and ``resource_usage()`` have been superseded by the
+  new execution statistics functions (see above).
+
 - Some script-level identifier have changed their names:
 
       snaplen                  -> Pcap::snaplen

VERSION

@@ -1 +1 @@
-2.4-507
+2.4-541

bro-config.h.in

@@ -14,6 +14,9 @@
 /* We are on a Linux system */
 #cmakedefine HAVE_LINUX
 
+/* We are on a Mac OS X (Darwin) system */
+#cmakedefine HAVE_DARWIN
+
 /* Define if you have the `mallinfo' function. */
 #cmakedefine HAVE_MALLINFO
 

scripts/base/frameworks/packet-filter/netstats.bro

@@ -18,7 +18,7 @@ export {
 
 event net_stats_update(last_stat: NetStats)
 	{
-	local ns = net_stats();
+	local ns = get_net_stats();
 	local new_dropped = ns$pkts_dropped - last_stat$pkts_dropped;
 	if ( new_dropped > 0 )
 		{
@@ -38,5 +38,5 @@ event bro_init()
 	# Since this currently only calculates packet drops, let's skip the stats
 	# collection if reading traces.
 	if ( ! reading_traces() )
-		schedule stats_collection_interval { net_stats_update(net_stats()) };
+		schedule stats_collection_interval { net_stats_update(get_net_stats()) };
 	}

scripts/base/init-bare.bro

@@ -474,14 +474,38 @@ type NetStats: record {
 	bytes_recvd:  count &default=0;	##< Bytes received by Bro.
 };
 
-## Statistics about Bro's resource consumption.
+type ConnStats: record {
+	total_conns: count;           ##<
+	current_conns: count;         ##<
+	current_conns_extern: count;  ##<
+	sess_current_conns: count;    ##<
+
+	num_packets: count;
+	num_fragments: count;
+	max_fragments: count;
+
+	num_tcp_conns: count;         ##< Current number of TCP connections in memory.
+	max_tcp_conns: count;         ##< Maximum number of concurrent TCP connections so far.
+	cumulative_tcp_conns: count;  ##< Total number of TCP connections so far.
+
+	num_udp_conns: count;         ##< Current number of UDP flows in memory.
+	max_udp_conns: count;         ##< Maximum number of concurrent UDP flows so far.
+	cumulative_udp_conns: count;  ##< Total number of UDP flows so far.
+
+	num_icmp_conns: count;        ##< Current number of ICMP flows in memory.
+	max_icmp_conns: count;        ##< Maximum number of concurrent ICMP flows so far.
+	cumulative_icmp_conns: count; ##< Total number of ICMP flows so far.
+
+	killed_by_inactivity: count;
+};
+
+## Statistics about Bro's process.
 ##
-## .. bro:see:: resource_usage
+## .. bro:see:: get_proc_stats
 ##
 ## .. note:: All process-level values refer to Bro's main process only, not to
 ##    the child process it spawns for doing communication.
-type bro_resources: record {
+type ProcStats: record {
-	version: string;	##< Bro version string.
 	debug: bool;                  ##< True if compiled with --enable-debug.
 	start_time: time;             ##< Start time of process.
 	real_time: interval;          ##< Elapsed real time since Bro started running.
@@ -494,46 +518,85 @@ type bro_resources: record {
 	blocking_input: count;        ##< Blocking input operations.
 	blocking_output: count;       ##< Blocking output operations.
 	num_context: count;           ##< Number of involuntary context switches.
+};
 
-	num_TCP_conns: count;	##< Current number of TCP connections in memory.
+type EventStats: record {
-	num_UDP_conns: count;	##< Current number of UDP flows in memory.
+	queued:     count; ##< Total number of events queued so far.
-	num_ICMP_conns: count;	##< Current number of ICMP flows in memory.
+	dispatched: count; ##< Total number of events dispatched so far.
-	num_fragments: count;	##< Current number of fragments pending reassembly.
-	num_packets: count;	##< Total number of packets processed to date.
-	num_timers: count;	##< Current number of pending timers.
-	num_events_queued: count;	##< Total number of events queued so far.
-	num_events_dispatched: count;	##< Total number of events dispatched so far.
-
-	max_TCP_conns: count;	##< Maximum number of concurrent TCP connections so far.
-	max_UDP_conns: count;	##< Maximum number of concurrent UDP connections so far.
-	max_ICMP_conns: count;	##< Maximum number of concurrent ICMP connections so far.
-	max_fragments: count;	##< Maximum number of concurrently buffered fragments so far.
-	max_timers: count;	##< Maximum number of concurrent timers pending so far.
 };
 
 ## Summary statistics of all regular expression matchers.
 ##
+## .. bro:see:: get_reassembler_stats
+type ReassemblerStats: record {
+	file_size:    count;  ##< Byte size of File reassembly tracking.
+	frag_size:    count;  ##< Byte size of Fragment reassembly tracking.
+	tcp_size:     count;  ##< Byte size of TCP reassembly tracking.
+	unknown_size: count;  ##< Byte size of reassembly tracking for unknown purposes.
+};
+
+## Statistics of all regular expression matchers.
+##
 ## .. bro:see:: get_matcher_stats
-type matcher_stats: record {
+type MatcherStats: record {
 	matchers: count;    ##< Number of distinct RE matchers.
+	nfa_states: count;  ##< Number of NFA states across all matchers.
 	dfa_states: count;  ##< Number of DFA states across all matchers.
 	computed: count;    ##< Number of computed DFA state transitions.
 	mem: count;         ##< Number of bytes used by DFA states.
 	hits: count;        ##< Number of cache hits.
 	misses: count;      ##< Number of cache misses.
-	avg_nfa_states: count;	##< Average number of NFA states across all matchers.
+};
+
+## Statistics of timers.
+##
+## .. bro:see:: get_timer_stats
+type TimerStats: record {
+	current:    count; ##< Current number of pending timers.
+	max:        count; ##< Maximum number of concurrent timers pending so far.
+	cumulative: count; ##< Cumulative number of timers scheduled.
+};
+
+## Statistics of file analysis.
+##
+## .. bro:see:: get_file_analysis_stats
+type FileAnalysisStats: record {
+	current:    count; ##< Current number of files being analyzed.
+	max:        count; ##< Maximum number of concurrent files so far.
+	cumulative: count; ##< Cumulative number of files analyzed.
+};
+
+## Statistics related to Bro's active use of DNS.  These numbers are
+## about Bro performing DNS queries on it's own, not traffic
+## being seen.
+##
+## .. bro:see:: get_dns_stats
+type DNSStats: record {
+	requests:         count; ##< Number of DNS requests made
+	successful:       count; ##< Number of successful DNS replies.
+	failed:           count; ##< Number of DNS reply failures.
+	pending:          count; ##< Current pending queries.
+	cached_hosts:     count; ##< Number of cached hosts.
+	cached_addresses: count; ##< Number of cached addresses.
 };
 
 ## Statistics about number of gaps in TCP connections.
 ##
-## .. bro:see:: gap_report get_gap_summary
+## .. bro:see:: get_gap_stats
-type gap_info: record {
+type GapStats: record {
 	ack_events: count;  ##< How many ack events *could* have had gaps.
 	ack_bytes: count;   ##< How many bytes those covered.
 	gap_events: count;  ##< How many *did* have gaps.
 	gap_bytes: count;   ##< How many bytes were missing in the gaps.
 };
 
+## Statistics about threads.
+##
+## .. bro:see:: get_thread_stats
+type ThreadStats: record {
+	num_threads: count;
+};
+
 ## Deprecated.
 ##
 ## .. todo:: Remove. It's still declared internally but doesn't seem  used anywhere
@@ -3435,23 +3498,17 @@ global pkt_profile_file: file &redef;
 ## .. bro:see:: load_sample
 global load_sample_freq = 20 &redef;
 
-## Rate at which to generate :bro:see:`gap_report` events assessing to what
-## degree the measurement process appears to exhibit loss.
-##
-## .. bro:see:: gap_report
-const gap_report_freq = 1.0 sec &redef;
-
 ## Whether to attempt to automatically detect SYN/FIN/RST-filtered trace
 ## and not report missing segments for such connections.
 ## If this is enabled, then missing data at the end of connections may not
 ## be reported via :bro:see:`content_gap`.
 const detect_filtered_trace = F &redef;
 
-## Whether we want :bro:see:`content_gap` and :bro:see:`gap_report` for partial
+## Whether we want :bro:see:`content_gap` and :bro:see:`get_gap_summary` for partial
 ## connections. A connection is partial if it is missing a full handshake. Note
 ## that gap reports for partial connections might not be reliable.
 ##
-## .. bro:see:: content_gap gap_report partial_connection
+## .. bro:see:: content_gap get_gap_summary partial_connection
 const report_gaps_for_partial = F &redef;
 
 ## Flag to prevent Bro from exiting automatically when input is exhausted.

scripts/base/misc/find-checksum-offloading.bro

@@ -26,7 +26,7 @@ event ChecksumOffloading::check()
 	if ( done )
 		return;
 
-	local pkts_recvd = net_stats()$pkts_recvd;
+	local pkts_recvd = get_net_stats()$pkts_recvd;
 	local bad_ip_checksum_pct = (pkts_recvd != 0) ? (bad_ip_checksums*1.0 / pkts_recvd*1.0) : 0;
 	local bad_tcp_checksum_pct = (pkts_recvd != 0) ? (bad_tcp_checksums*1.0 / pkts_recvd*1.0) : 0;
 	local bad_udp_checksum_pct = (pkts_recvd != 0) ? (bad_udp_checksums*1.0 / pkts_recvd*1.0) : 0;

scripts/policy/frameworks/control/controllee.bro

@@ -22,30 +22,10 @@ event Control::id_value_request(id: string)
 
 event Control::peer_status_request()
 	{
-	local status = "";
-	for ( p in Communication::nodes )
-		{
-		local peer = Communication::nodes[p];
-		if ( ! peer$connected )
-			next;
-			
-		local res = resource_usage();
-		status += fmt("%.6f peer=%s host=%s events_in=%s events_out=%s ops_in=%s ops_out=%s bytes_in=? bytes_out=?\n",
-					network_time(),
-					peer$peer$descr, peer$host,
-					res$num_events_queued, res$num_events_dispatched,
-					res$blocking_input, res$blocking_output);
-		}
-
-	event Control::peer_status_response(status);
 	}
 
 event Control::net_stats_request()
 	{
-	local ns = net_stats();
-	local reply = fmt("%.6f recvd=%d dropped=%d link=%d\n", network_time(), 
-	                  ns$pkts_recvd, ns$pkts_dropped, ns$pkts_link);
-	event Control::net_stats_response(reply);
 	}
 
 event Control::configuration_update_request()

scripts/policy/misc/capture-loss.bro

@@ -56,7 +56,7 @@ event CaptureLoss::take_measurement(last_ts: time, last_acks: count, last_gaps:
 		}
 	
 	local now = network_time();
-	local g = get_gap_summary();
+	local g = get_gap_stats();
 	local acks = g$ack_events - last_acks;
 	local gaps = g$gap_events - last_gaps;
 	local pct_lost = (acks == 0) ? 0.0 : (100 * (1.0 * gaps) / (1.0 * acks));

scripts/policy/misc/stats.bro

@@ -1,6 +1,4 @@
-##! Log memory/packet/lag statistics.  Differs from
+##! Log memory/packet/lag statistics.
-##! :doc:`/scripts/policy/misc/profiling.bro` in that this
-##! is lighter-weight (much less info, and less load to generate).
 
 @load base/frameworks/notice
 
@@ -10,7 +8,7 @@ export {
 	redef enum Log::ID += { LOG };
 
 	## How often stats are reported.
-	const stats_report_interval = 1min &redef;
+	const report_interval = 5min &redef;
 
 	type Info: record {
 		## Timestamp for the measurement.
@@ -21,27 +19,63 @@ export {
 		mem:           count     &log;
 		## Number of packets processed since the last stats interval.
 		pkts_proc:     count     &log;
-		## Number of events processed since the last stats interval.
+		## Number of bytes received since the last stats interval if
-		events_proc:   count     &log;
-		## Number of events that have been queued since the last stats
-		## interval.
-		events_queued: count     &log;
-
-		## Lag between the wall clock and packet timestamps if reading
-		## live traffic.
-		lag:           interval  &log &optional;
-		## Number of packets received since the last stats interval if
 		## reading live traffic.
-		pkts_recv:     count     &log &optional;
+		bytes_recv:    count     &log;
+
 		## Number of packets dropped since the last stats interval if
 		## reading live traffic.
 		pkts_dropped:  count     &log &optional;
 		## Number of packets seen on the link since the last stats
 		## interval if reading live traffic.
 		pkts_link:     count     &log &optional;
-		## Number of bytes received since the last stats interval if
+		## Lag between the wall clock and packet timestamps if reading
-		## reading live traffic.
+		## live traffic.
-		bytes_recv:   count     &log &optional;
+		pkt_lag:       interval  &log &optional;
+
+		## Number of events processed since the last stats interval.
+		events_proc:   count     &log;
+		## Number of events that have been queued since the last stats
+		## interval.
+		events_queued: count     &log;
+
+		## TCP connections currently in memory.
+		active_tcp_conns: count  &log;
+		## UDP connections currently in memory.
+		active_udp_conns: count &log;
+		## ICMP connections currently in memory.
+		active_icmp_conns: count &log;
+
+		## TCP connections seen since last stats interval.
+		tcp_conns:        count  &log;
+		## UDP connections seen since last stats interval.
+		udp_conns:        count &log;
+		## ICMP connections seen since last stats interval.
+		icmp_conns:        count &log;
+
+		## Number of timers scheduled since last stats interval.
+		timers: count &log;
+		## Current number of scheduled timers.
+		active_timers: count &log;
+
+		## Number of files seen since last stats interval.
+		files: count &log;
+		## Current number of files actively being seen.
+		active_files: count &log;
+
+		## Number of DNS requests seen since last stats interval.
+		dns_requests: count &log;
+		## Current number of DNS requests awaiting a reply.
+		active_dns_requests: count &log;
+
+		## Current size of TCP data in reassembly.
+		reassem_tcp_size: count &log;
+		## Current size of File data in reassembly.
+		reassem_file_size: count &log;
+		## Current size of packet fragment data in reassembly.
+		reassem_frag_size: count &log;
+		## Current size of unkown data in reassembly (this is only PIA buffer right now).
+		reassem_unknown_size: count &log;
 	};
 
 	## Event to catch stats as they are written to the logging stream.
@@ -53,38 +87,69 @@ event bro_init() &priority=5
 	Log::create_stream(Stats::LOG, [$columns=Info, $ev=log_stats, $path="stats"]);
 	}
 
-event check_stats(last_ts: time, last_ns: NetStats, last_res: bro_resources)
+event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: ProcStats, last_es: EventStats, last_rs: ReassemblerStats, last_ts: TimerStats, last_fs: FileAnalysisStats, last_ds: DNSStats)
 	{
-	local now = current_time();
+	local nettime = network_time();
-	local ns = net_stats();
+	local ns = get_net_stats();
-	local res = resource_usage();
+	local cs = get_conn_stats();
+	local ps = get_proc_stats();
+	local es = get_event_stats();
+	local rs = get_reassembler_stats();
+	local ts = get_timer_stats();
+	local fs = get_file_analysis_stats();
+	local ds = get_dns_stats();
 
 	if ( bro_is_terminating() )
 		# No more stats will be written or scheduled when Bro is
 		# shutting down.
 		return;
 
-	local info: Info = [$ts=now, $peer=peer_description, $mem=res$mem/1000000,
+	local info: Info = [$ts=nettime,
-	                    $pkts_proc=res$num_packets - last_res$num_packets,
+			    $peer=peer_description,
-	                    $events_proc=res$num_events_dispatched - last_res$num_events_dispatched,
+			    $mem=ps$mem/1048576,
-	                    $events_queued=res$num_events_queued - last_res$num_events_queued];
+			    $pkts_proc=ns$pkts_recvd - last_ns$pkts_recvd,
+			    $bytes_recv = ns$bytes_recvd  - last_ns$bytes_recvd,
+
+			    $active_tcp_conns=cs$num_tcp_conns,
+			    $tcp_conns=cs$cumulative_tcp_conns - last_cs$cumulative_tcp_conns,
+			    $active_udp_conns=cs$num_udp_conns,
+			    $udp_conns=cs$cumulative_udp_conns - last_cs$cumulative_udp_conns,
+			    $active_icmp_conns=cs$num_icmp_conns,
+			    $icmp_conns=cs$cumulative_icmp_conns - last_cs$cumulative_icmp_conns,
+
+			    $reassem_tcp_size=rs$tcp_size,
+			    $reassem_file_size=rs$file_size,
+			    $reassem_frag_size=rs$frag_size,
+			    $reassem_unknown_size=rs$unknown_size,
+
+			    $events_proc=es$dispatched - last_es$dispatched,
+			    $events_queued=es$queued - last_es$queued,
+
+			    $timers=ts$cumulative - last_ts$cumulative,
+			    $active_timers=ts$current,
+
+			    $files=fs$cumulative - last_fs$cumulative,
+			    $active_files=fs$current,
+
+			    $dns_requests=ds$requests - last_ds$requests,
+			    $active_dns_requests=ds$pending
+			    ];
+
+	# Someone's going to have to explain what this is and add a field to the Info record.
+	# info$util = 100.0*((ps$user_time + ps$system_time) - (last_ps$user_time + last_ps$system_time))/(now-then);
 
 	if ( reading_live_traffic() )
 		{
-		info$lag = now - network_time();
+		info$pkt_lag = current_time() - nettime;
-		# Someone's going to have to explain what this is and add a field to the Info record.
-		# info$util = 100.0*((res$user_time + res$system_time) - (last_res$user_time + last_res$system_time))/(now-last_ts);
-		info$pkts_recv = ns$pkts_recvd - last_ns$pkts_recvd;
 		info$pkts_dropped = ns$pkts_dropped  - last_ns$pkts_dropped;
 		info$pkts_link = ns$pkts_link  - last_ns$pkts_link;
-		info$bytes_recv = ns$bytes_recvd  - last_ns$bytes_recvd;
 		}
 
 	Log::write(Stats::LOG, info);
-	schedule stats_report_interval { check_stats(now, ns, res) };
+	schedule report_interval { check_stats(nettime, ns, cs, ps, es, rs, ts, fs, ds) };
 	}
 
 event bro_init()
 	{
-	schedule stats_report_interval { check_stats(current_time(), net_stats(), resource_usage()) };
+	schedule report_interval { check_stats(network_time(), get_net_stats(), get_conn_stats(), get_proc_stats(), get_event_stats(), get_reassembler_stats(), get_timer_stats(), get_file_analysis_stats(), get_dns_stats()) };
 	}

src/CMakeLists.txt

@@ -118,6 +118,7 @@ include(BifCl)
 
 set(BIF_SRCS
     bro.bif
+    stats.bif
     event.bif
     const.bif
     types.bif

src/Conn.cc

@@ -108,9 +108,9 @@ bool ConnectionTimer::DoUnserialize(UnserialInfo* info)
 	return true;
 	}
 
-unsigned int Connection::total_connections = 0;
+uint64 Connection::total_connections = 0;
-unsigned int Connection::current_connections = 0;
+uint64 Connection::current_connections = 0;
-unsigned int Connection::external_connections = 0;
+uint64 Connection::external_connections = 0;
 
 IMPLEMENT_SERIAL(Connection, SER_CONNECTION);
 

src/Conn.h

@@ -220,11 +220,11 @@ public:
 	unsigned int MemoryAllocation() const;
 	unsigned int MemoryAllocationConnVal() const;
 
-	static unsigned int TotalConnections()
+	static uint64 TotalConnections()
 		{ return total_connections; }
-	static unsigned int CurrentConnections()
+	static uint64 CurrentConnections()
 		{ return current_connections; }
-	static unsigned int CurrentExternalConnections()
+	static uint64 CurrentExternalConnections()
 		{ return external_connections; }
 
 	// Returns true if the history was already seen, false otherwise.
@@ -315,9 +315,9 @@ protected:
 	unsigned int saw_first_orig_packet:1, saw_first_resp_packet:1;
 
 	// Count number of connections.
-	static unsigned int total_connections;
+	static uint64 total_connections;
-	static unsigned int current_connections;
+	static uint64 current_connections;
-	static unsigned int external_connections;
+	static uint64 external_connections;
 
 	string history;
 	uint32 hist_seen;

src/DFA.cc

@@ -346,6 +346,7 @@ DFA_State* DFA_State_Cache::Lookup(const NFA_state_list& nfas,
 		++misses;
 		return 0;
 		}
+	++hits;
 
 	delete *hash;
 	*hash = 0;
@@ -433,19 +434,6 @@ void DFA_Machine::Dump(FILE* f)
 	start_state->ClearMarks();
 	}
 
-void DFA_Machine::DumpStats(FILE* f)
-	{
-	DFA_State_Cache::Stats stats;
-	dfa_state_cache->GetStats(&stats);
-
-	fprintf(f, "Computed dfa_states = %d; Classes = %d; Computed trans. = %d; Uncomputed trans. = %d\n",
-		stats.dfa_states, EC()->NumClasses(),
-		stats.computed, stats.uncomputed);
-
-	fprintf(f, "DFA cache hits = %d; misses = %d\n",
-		stats.hits, stats.misses);
-	}
-
 unsigned int DFA_Machine::MemoryAllocation() const
 	{
 	DFA_State_Cache::Stats s;

src/DFA.h

@@ -89,10 +89,9 @@ public:
 	int NumEntries() const	{ return states.Length(); }
 
 	struct Stats {
-		unsigned int dfa_states;
+		// Sum of all NFA states
-
-		// Sum over all NFA states per DFA state.
 		unsigned int nfa_states;
+		unsigned int dfa_states;
 		unsigned int computed;
 		unsigned int uncomputed;
 		unsigned int mem;
@@ -132,7 +131,6 @@ public:
 
 	void Describe(ODesc* d) const;
 	void Dump(FILE* f);
-	void DumpStats(FILE* f);
 
 	unsigned int MemoryAllocation() const;
 

src/Dict.cc

@@ -66,6 +66,7 @@ Dictionary::Dictionary(dict_order ordering, int initial_size)
 	delete_func = 0;
 	tbl_next_ind = 0;
 
+	cumulative_entries = 0;
 	num_buckets2 = num_entries2 = max_num_entries2 = thresh_entries2 = 0;
 	den_thresh2 = 0;
 	}
@@ -444,6 +445,7 @@ void* Dictionary::Insert(DictEntry* new_entry, int copy_key)
 	// on lists than prepending.
 	chain->append(new_entry);
 
+	++cumulative_entries;
 	if ( *max_num_entries_ptr < ++*num_entries_ptr )
 		*max_num_entries_ptr = *num_entries_ptr;
 

src/Dict.h

@@ -71,6 +71,12 @@ public:
 			max_num_entries + max_num_entries2 : max_num_entries;
 		}
 
+	// Total number of entries ever.
+	uint64 NumCumulativeInserts() const
+		{
+		return cumulative_entries;
+		}
+
 	// True if the dictionary is ordered, false otherwise.
 	int IsOrdered() const		{ return order != 0; }
 
@@ -166,6 +172,7 @@ private:
 	int num_buckets;
 	int num_entries;
 	int max_num_entries;
+	uint64 cumulative_entries;
 	double den_thresh;
 	int thresh_entries;
 

src/Event.cc

@@ -10,8 +10,8 @@
 
 EventMgr mgr;
 
-int num_events_queued = 0;
+uint64 num_events_queued = 0;
-int num_events_dispatched = 0;
+uint64 num_events_dispatched = 0;
 
 Event::Event(EventHandlerPtr arg_handler, val_list* arg_args,
 		SourceID arg_src, analyzer::ID arg_aid, TimerMgr* arg_mgr,

src/Event.h

@@ -72,8 +72,8 @@ protected:
 	Event* next_event;
 };
 
-extern int num_events_queued;
+extern uint64 num_events_queued;
-extern int num_events_dispatched;
+extern uint64 num_events_dispatched;
 
 class EventMgr : public BroObj {
 public:

src/Frag.cc

@@ -28,7 +28,7 @@ void FragTimer::Dispatch(double t, int /* is_expire */)
 FragReassembler::FragReassembler(NetSessions* arg_s,
 			const IP_Hdr* ip, const u_char* pkt,
 			HashKey* k, double t)
-	: Reassembler(0)
+	: Reassembler(0, REASSEM_FRAG)
 	{
 	s = arg_s;
 	key = k;

src/Func.cc

@@ -628,10 +628,12 @@ void builtin_error(const char* msg, BroObj* arg)
 	}
 
 #include "bro.bif.func_h"
+#include "stats.bif.func_h"
 #include "reporter.bif.func_h"
 #include "strings.bif.func_h"
 
 #include "bro.bif.func_def"
+#include "stats.bif.func_def"
 #include "reporter.bif.func_def"
 #include "strings.bif.func_def"
 
@@ -640,13 +642,22 @@ void builtin_error(const char* msg, BroObj* arg)
 
 void init_builtin_funcs()
 	{
-	bro_resources = internal_type("bro_resources")->AsRecordType();
+	ProcStats = internal_type("ProcStats")->AsRecordType();
-	net_stats = internal_type("NetStats")->AsRecordType();
+	NetStats = internal_type("NetStats")->AsRecordType();
-	matcher_stats = internal_type("matcher_stats")->AsRecordType();
+	MatcherStats = internal_type("MatcherStats")->AsRecordType();
+	ConnStats = internal_type("ConnStats")->AsRecordType();
+	ReassemblerStats = internal_type("ReassemblerStats")->AsRecordType();
+	DNSStats = internal_type("DNSStats")->AsRecordType();
+	GapStats = internal_type("GapStats")->AsRecordType();
+	EventStats = internal_type("EventStats")->AsRecordType();
+	TimerStats = internal_type("TimerStats")->AsRecordType();
+	FileAnalysisStats = internal_type("FileAnalysisStats")->AsRecordType();
+	ThreadStats = internal_type("ThreadStats")->AsRecordType();
+
 	var_sizes = internal_type("var_sizes")->AsTableType();
-	gap_info = internal_type("gap_info")->AsRecordType();
 
 #include "bro.bif.func_init"
+#include "stats.bif.func_init"
 #include "reporter.bif.func_init"
 #include "strings.bif.func_init"
 

src/NFA.cc

@@ -285,11 +285,6 @@ void NFA_Machine::Dump(FILE* f)
 	first_state->ClearMarks();
 	}
 
-void NFA_Machine::DumpStats(FILE* f)
-	{
-	fprintf(f, "highest NFA state ID is %d\n", nfa_state_id);
-	}
-
 NFA_Machine* make_alternate(NFA_Machine* m1, NFA_Machine* m2)
 	{
 	if ( ! m1 )

src/NFA.h

@@ -105,7 +105,6 @@ public:
 
 	void Describe(ODesc* d) const;
 	void Dump(FILE* f);
-	void DumpStats(FILE* f);
 
 	unsigned int MemoryAllocation() const
 		{ return padded_sizeof(*this) + first_state->TotalMemoryAllocation(); }

src/NetVar.cc

@@ -199,7 +199,6 @@ Val* pkt_profile_file;
 int load_sample_freq;
 
 double gap_report_freq;
-RecordType* gap_info;
 
 int packet_filter_default;
 

src/NetVar.h

@@ -202,9 +202,6 @@ extern Val* pkt_profile_file;
 
 extern int load_sample_freq;
 
-extern double gap_report_freq;
-extern RecordType* gap_info;
-
 extern int packet_filter_default;
 
 extern int sig_max_group_size;

src/PriorityQueue.cc

@@ -13,7 +13,7 @@ PriorityQueue::PriorityQueue(int initial_size)
 	{
 	max_heap_size = initial_size;
 	heap = new PQ_Element*[max_heap_size];
-	peak_heap_size = heap_size = 0;
+	peak_heap_size = heap_size = cumulative_num = 0;
 	}
 
 PriorityQueue::~PriorityQueue()
@@ -62,6 +62,8 @@ int PriorityQueue::Add(PQ_Element* e)
 
 	BubbleUp(heap_size);
 
+	++cumulative_num;
+
 	if ( ++heap_size > peak_heap_size )
 		peak_heap_size = heap_size;
 

src/PriorityQueue.h

@@ -4,6 +4,7 @@
 #define __PriorityQueue__
 
 #include <math.h>
+#include "util.h"
 
 class PriorityQueue;
 
@@ -53,6 +54,7 @@ public:
 
 	int Size() const	{ return heap_size; }
 	int PeakSize() const	{ return peak_heap_size; }
+	uint64 CumulativeNum() const { return cumulative_num; }
 
 protected:
 	int Resize(int new_size);
@@ -92,6 +94,7 @@ protected:
 	int heap_size;
 	int peak_heap_size;
 	int max_heap_size;
+	uint64 cumulative_num;
 };
 
 #endif

src/Reassem.cc

@@ -1,6 +1,7 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
 #include <algorithm>
+#include <vector>
 
 #include "bro-config.h"
 
@@ -10,7 +11,8 @@
 static const bool DEBUG_reassem = false;
 
 DataBlock::DataBlock(const u_char* data, uint64 size, uint64 arg_seq,
-			DataBlock* arg_prev, DataBlock* arg_next)
+		     DataBlock* arg_prev, DataBlock* arg_next,
+		     ReassemblerType reassem_type)
 	{
 	seq = arg_seq;
 	upper = seq + size;
@@ -26,17 +28,21 @@ DataBlock::DataBlock(const u_char* data, uint64 size, uint64 arg_seq,
 	if ( next )
 		next->prev = this;
 
+	rtype = reassem_type;
+	Reassembler::sizes[rtype] += pad_size(size) + padded_sizeof(DataBlock);
 	Reassembler::total_size += pad_size(size) + padded_sizeof(DataBlock);
 	}
 
 uint64 Reassembler::total_size = 0;
+uint64 Reassembler::sizes[REASSEM_NUM];
 
-Reassembler::Reassembler(uint64 init_seq)
+Reassembler::Reassembler(uint64 init_seq, ReassemblerType reassem_type)
 	{
 	blocks = last_block = 0;
 	old_blocks = last_old_block = 0;
 	total_old_blocks = max_old_blocks = 0;
 	trim_seq = last_reassem_seq = init_seq;
+	rtype = reassem_type;
 	}
 
 Reassembler::~Reassembler()
@@ -110,7 +116,7 @@ void Reassembler::NewBlock(double t, uint64 seq, uint64 len, const u_char* data)
 
 	if ( ! blocks )
 		blocks = last_block = start_block =
-			new DataBlock(data, len, seq, 0, 0);
+			new DataBlock(data, len, seq, 0, 0, rtype);
 	else
 		start_block = AddAndCheck(blocks, seq, upper_seq, data);
 
@@ -275,7 +281,7 @@ DataBlock* Reassembler::AddAndCheck(DataBlock* b, uint64 seq, uint64 upper,
 	if ( last_block && seq == last_block->upper )
 		{
 		last_block = new DataBlock(data, upper - seq, seq,
-						last_block, 0);
+					   last_block, 0, rtype);
 		return last_block;
 		}
 
@@ -288,7 +294,7 @@ DataBlock* Reassembler::AddAndCheck(DataBlock* b, uint64 seq, uint64 upper,
 		{
 		// b is the last block, and it comes completely before
 		// the new block.
-		last_block = new DataBlock(data, upper - seq, seq, b, 0);
+		last_block = new DataBlock(data, upper - seq, seq, b, 0, rtype);
 		return last_block;
 		}
 
@@ -297,7 +303,7 @@ DataBlock* Reassembler::AddAndCheck(DataBlock* b, uint64 seq, uint64 upper,
 	if ( upper <= b->seq )
 		{
 		// The new block comes completely before b.
-		new_b = new DataBlock(data, upper - seq, seq, b->prev, b);
+		new_b = new DataBlock(data, upper - seq, seq, b->prev, b, rtype);
 		if ( b == blocks )
 			blocks = new_b;
 		return new_b;
@@ -308,7 +314,7 @@ DataBlock* Reassembler::AddAndCheck(DataBlock* b, uint64 seq, uint64 upper,
 		{
 		// The new block has a prefix that comes before b.
 		uint64 prefix_len = b->seq - seq;
-		new_b = new DataBlock(data, prefix_len, seq, b->prev, b);
+		new_b = new DataBlock(data, prefix_len, seq, b->prev, b, rtype);
 		if ( b == blocks )
 			blocks = new_b;
 
@@ -342,6 +348,11 @@ DataBlock* Reassembler::AddAndCheck(DataBlock* b, uint64 seq, uint64 upper,
 	return new_b;
 	}
 
+uint64 Reassembler::MemoryAllocation(ReassemblerType rtype)
+	{
+	return Reassembler::sizes[rtype];
+	}
+
 bool Reassembler::Serialize(SerialInfo* info) const
 	{
 	return SerialObj::Serialize(info);

src/Reassem.h

@@ -6,10 +6,23 @@
 #include "Obj.h"
 #include "IPAddr.h"
 
+// Whenever subclassing the Reassembler class
+// you should add to this for known subclasses.
+enum ReassemblerType {
+	REASSEM_UNKNOWN,
+	REASSEM_TCP,
+	REASSEM_FRAG,
+	REASSEM_FILE,
+
+	// Terminal value. Add new above.
+	REASSEM_NUM,
+};
+
 class DataBlock {
 public:
 	DataBlock(const u_char* data, uint64 size, uint64 seq,
-			DataBlock* prev, DataBlock* next);
+		  DataBlock* prev, DataBlock* next,
+		  ReassemblerType reassem_type = REASSEM_UNKNOWN);
 
 	~DataBlock();
 
@@ -19,13 +32,12 @@ public:
 	DataBlock* prev;	// previous block with lower seq #
 	uint64 seq, upper;
 	u_char* block;
+	ReassemblerType rtype;
 };
 
-
-
 class Reassembler : public BroObj {
 public:
-	Reassembler(uint64 init_seq);
+	Reassembler(uint64 init_seq, ReassemblerType reassem_type = REASSEM_UNKNOWN);
 	virtual ~Reassembler();
 
 	void NewBlock(double t, uint64 seq, uint64 len, const u_char* data);
@@ -51,6 +63,9 @@ public:
 	// Sum over all data buffered in some reassembler.
 	static uint64 TotalMemoryAllocation()	{ return total_size; }
 
+	// Data buffered by type of reassembler.
+	static uint64 MemoryAllocation(ReassemblerType rtype);
+
 	void SetMaxOldBlocks(uint32 count)	{ max_old_blocks = count; }
 
 protected:
@@ -82,12 +97,16 @@ protected:
 	uint32 max_old_blocks;
 	uint32 total_old_blocks;
 
+	ReassemblerType rtype;
+
 	static uint64 total_size;
+	static uint64 sizes[REASSEM_NUM];
 };
 
 inline DataBlock::~DataBlock()
 	{
 	Reassembler::total_size -= pad_size(upper - seq) + padded_sizeof(DataBlock);
+	Reassembler::sizes[rtype] -= pad_size(upper - seq) + padded_sizeof(DataBlock);
 	delete [] block;
 	}
 

src/RuleMatcher.cc

@@ -1174,7 +1174,7 @@ void RuleMatcher::GetStats(Stats* stats, RuleHdrTest* hdr_test)
 		stats->mem = 0;
 		stats->hits = 0;
 		stats->misses = 0;
-		stats->avg_nfa_states = 0;
+		stats->nfa_states = 0;
 		hdr_test = root;
 		}
 
@@ -1195,15 +1195,10 @@ void RuleMatcher::GetStats(Stats* stats, RuleHdrTest* hdr_test)
 			stats->mem += cstats.mem;
 			stats->hits += cstats.hits;
 			stats->misses += cstats.misses;
-			stats->avg_nfa_states += cstats.nfa_states;
+			stats->nfa_states += cstats.nfa_states;
 			}
 		}
 
-	if (  stats->dfa_states )
-		stats->avg_nfa_states /= stats->dfa_states;
-	else
-		stats->avg_nfa_states = 0;
-
 	for ( RuleHdrTest* h = hdr_test->child; h; h = h->sibling )
 		GetStats(stats, h);
 	}

src/RuleMatcher.h

@@ -297,6 +297,9 @@ public:
 	struct Stats {
 		unsigned int matchers;	// # distinct RE matchers
 
+		// NFA states across all matchers.
+		unsigned int nfa_states;
+
 		// # DFA states across all matchers
 		unsigned int dfa_states;
 		unsigned int computed;	// # computed DFA state transitions
@@ -305,9 +308,6 @@ public:
 		// # cache hits (sampled, multiply by MOVE_TO_FRONT_SAMPLE_SIZE)
 		unsigned int hits;
 		unsigned int misses;	// # cache misses
-
-		// Average # NFA states per DFA state.
-		unsigned int avg_nfa_states;
 	};
 
 	Val* BuildRuleStateValue(const Rule* rule,

src/Sessions.cc

@@ -1156,19 +1156,18 @@ void NetSessions::Drain()
 void NetSessions::GetStats(SessionStats& s) const
 	{
 	s.num_TCP_conns = tcp_conns.Length();
+	s.cumulative_TCP_conns = tcp_conns.NumCumulativeInserts();
 	s.num_UDP_conns = udp_conns.Length();
+	s.cumulative_UDP_conns = udp_conns.NumCumulativeInserts();
 	s.num_ICMP_conns = icmp_conns.Length();
+	s.cumulative_ICMP_conns = icmp_conns.NumCumulativeInserts();
 	s.num_fragments = fragments.Length();
 	s.num_packets = num_packets_processed;
-	s.num_timers = timer_mgr->Size();
-	s.num_events_queued = num_events_queued;
-	s.num_events_dispatched = num_events_dispatched;
 
 	s.max_TCP_conns = tcp_conns.MaxLength();
 	s.max_UDP_conns = udp_conns.MaxLength();
 	s.max_ICMP_conns = icmp_conns.MaxLength();
 	s.max_fragments = fragments.MaxLength();
-	s.max_timers = timer_mgr->PeakSize();
 	}
 
 Connection* NetSessions::NewConn(HashKey* k, double t, const ConnID* id,

src/Sessions.h

@@ -32,19 +32,20 @@ namespace analyzer { namespace arp { class ARP_Analyzer; } }
 
 struct SessionStats {
 	int num_TCP_conns;
-	int num_UDP_conns;
-	int num_ICMP_conns;
-	int num_fragments;
-	int num_packets;
-	int num_timers;
-	int num_events_queued;
-	int num_events_dispatched;
-
 	int max_TCP_conns;
+	uint64 cumulative_TCP_conns;
+
+	int num_UDP_conns;
 	int max_UDP_conns;
+	uint64 cumulative_UDP_conns;
+
+	int num_ICMP_conns;
 	int max_ICMP_conns;
+	uint64 cumulative_ICMP_conns;
+
+	int num_fragments;
 	int max_fragments;
-	int max_timers;
+	uint64 num_packets;
 };
 
 // Drains and deletes a timer manager if it hasn't seen any advances
@@ -242,7 +243,7 @@ protected:
 	OSFingerprint* SYN_OS_Fingerprinter;
 	int build_backdoor_analyzer;
 	int dump_this_packet;	// if true, current packet should be recorded
-	int num_packets_processed;
+	uint64 num_packets_processed;
 	PacketProfiler* pkt_profiler;
 
 	// We may use independent timer managers for different sets of related

src/Stats.cc

@@ -14,7 +14,7 @@
 #include "broker/Manager.h"
 #endif
 
-int killed_by_inactivity = 0;
+uint64 killed_by_inactivity = 0;
 
 uint64 tot_ack_events = 0;
 uint64 tot_ack_bytes = 0;
@@ -82,7 +82,7 @@ void ProfileLogger::Log()
 	struct timeval tv_utime = r.ru_utime;
 	struct timeval tv_stime = r.ru_stime;
 
-	unsigned int total, malloced;
+	uint64 total, malloced;
 	get_memory_usage(&total, &malloced);
 
 	static unsigned int first_total = 0;
@@ -110,7 +110,7 @@ void ProfileLogger::Log()
 		file->Write(fmt("\n%.06f ------------------------\n", network_time));
 		}
 
-	file->Write(fmt("%.06f Memory: total=%dK total_adj=%dK malloced: %dK\n",
+	file->Write(fmt("%.06f Memory: total=%" PRId64 "K total_adj=%" PRId64 "K malloced: %" PRId64 "K\n",
 		network_time, total / 1024, (total - first_total) / 1024,
 		malloced / 1024));
 
@@ -120,7 +120,7 @@ void ProfileLogger::Log()
 
 	int conn_mem_use = expensive ? sessions->ConnectionMemoryUsage() : 0;
 
-	file->Write(fmt("%.06f Conns: total=%d current=%d/%d ext=%d mem=%dK avg=%.1f table=%dK connvals=%dK\n",
+	file->Write(fmt("%.06f Conns: total=%" PRIu64 " current=%" PRIu64 "/%" PRIi32 " ext=%" PRIu64 " mem=%" PRIi32 "K avg=%.1f table=%" PRIu32 "K connvals=%" PRIu32 "K\n",
 		network_time,
 		Connection::TotalConnections(),
 		Connection::CurrentConnections(),
@@ -161,10 +161,10 @@ void ProfileLogger::Log()
 		       ));
 	*/
 
-	file->Write(fmt("%.06f Connections expired due to inactivity: %d\n",
+	file->Write(fmt("%.06f Connections expired due to inactivity: %" PRIu64 "\n",
 		network_time, killed_by_inactivity));
 
-	file->Write(fmt("%.06f Total reassembler data: %" PRIu64"K\n", network_time,
+	file->Write(fmt("%.06f Total reassembler data: %" PRIu64 "K\n", network_time,
 		Reassembler::TotalMemoryAllocation() / 1024));
 
 	// Signature engine.
@@ -173,9 +173,9 @@ void ProfileLogger::Log()
 		RuleMatcher::Stats stats;
 		rule_matcher->GetStats(&stats);
 
-		file->Write(fmt("%06f RuleMatcher: matchers=%d dfa_states=%d ncomputed=%d "
+		file->Write(fmt("%06f RuleMatcher: matchers=%d nfa_states=%d dfa_states=%d "
-			"mem=%dK avg_nfa_states=%d\n", network_time, stats.matchers,
+			"ncomputed=%d mem=%dK\n", network_time, stats.matchers,
-			stats.dfa_states, stats.computed, stats.mem / 1024, stats.avg_nfa_states));
+			stats.nfa_states, stats.dfa_states, stats.computed, stats.mem / 1024));
 		}
 
 	file->Write(fmt("%.06f Timers: current=%d max=%d mem=%dK lag=%.2fs\n",
@@ -469,10 +469,10 @@ void PacketProfiler::ProfilePkt(double t, unsigned int bytes)
 		double curr_Rtime =
 			ptimestamp.tv_sec + ptimestamp.tv_usec / 1e6;
 
-		unsigned int curr_mem;
+		uint64 curr_mem;
 		get_memory_usage(&curr_mem, 0);
 
-		file->Write(fmt("%.06f %.03f %d %d %.03f %.03f %.03f %d\n",
+		file->Write(fmt("%.06f %.03f %" PRIu64 " %" PRIu64 " %.03f %.03f %.03f %" PRIu64 "\n",
 				t, time-last_timestamp, pkt_cnt, byte_cnt,
 				curr_Rtime - last_Rtime,
 				curr_Utime - last_Utime,

src/Stats.h

@@ -102,7 +102,7 @@ extern ProfileLogger* segment_logger;
 extern SampleLogger* sample_logger;
 
 // Connection statistics.
-extern int killed_by_inactivity;
+extern uint64 killed_by_inactivity;
 
 // Content gap statistics.
 extern uint64 tot_ack_events;
@@ -127,9 +127,9 @@ protected:
 	double update_freq;
 	double last_Utime, last_Stime, last_Rtime;
 	double last_timestamp, time;
-	unsigned int last_mem;
+	uint64 last_mem;
-	unsigned int pkt_cnt;
+	uint64 pkt_cnt;
-	unsigned int byte_cnt;
+	uint64 byte_cnt;
 };
 
 #endif

src/Timer.h

@@ -109,6 +109,7 @@ public:
 
 	virtual int Size() const = 0;
 	virtual int PeakSize() const = 0;
+	virtual uint64 CumulativeNum() const = 0;
 
 	double LastTimestamp() const	{ return last_timestamp; }
 	// Returns time of last advance in global network time.
@@ -148,6 +149,7 @@ public:
 
 	int Size() const	{ return q->Size(); }
 	int PeakSize() const	{ return q->PeakSize(); }
+	uint64 CumulativeNum() const	{ return q->CumulativeNum(); }
 	unsigned int MemoryUsage() const;
 
 protected:
@@ -170,6 +172,7 @@ public:
 
 	int Size() const	{ return cq_size(cq); }
 	int PeakSize() const	{ return cq_max_size(cq); }
+	uint64 CumulativeNum() const	{ return cq_cumulative_num(cq); }
 	unsigned int MemoryUsage() const;
 
 protected:

src/analyzer/protocol/tcp/TCP_Reassembler.cc

@@ -5,9 +5,6 @@
 #include "analyzer/protocol/tcp/TCP.h"
 #include "TCP_Endpoint.h"
 
-// Only needed for gap_report events.
-#include "Event.h"
-
 #include "events.bif.h"
 
 using namespace analyzer::tcp;
@@ -18,17 +15,11 @@ const bool DEBUG_tcp_contents = false;
 const bool DEBUG_tcp_connection_close = false;
 const bool DEBUG_tcp_match_undelivered = false;
 
-static double last_gap_report = 0.0;
-static uint64 last_ack_events = 0;
-static uint64 last_ack_bytes = 0;
-static uint64 last_gap_events = 0;
-static uint64 last_gap_bytes = 0;
-
 TCP_Reassembler::TCP_Reassembler(analyzer::Analyzer* arg_dst_analyzer,
 				TCP_Analyzer* arg_tcp_analyzer,
 				TCP_Reassembler::Type arg_type,
 				TCP_Endpoint* arg_endp)
-	: Reassembler(1)
+	: Reassembler(1, REASSEM_TCP)
 	{
 	dst_analyzer = arg_dst_analyzer;
 	tcp_analyzer = arg_tcp_analyzer;
@@ -45,7 +36,7 @@ TCP_Reassembler::TCP_Reassembler(analyzer::Analyzer* arg_dst_analyzer,
 	if ( tcp_max_old_segments )
 		SetMaxOldBlocks(tcp_max_old_segments);
 
-	if ( tcp_contents )
+	if ( ::tcp_contents )
 		{
 		// Val dst_port_val(ntohs(Conn()->RespPort()), TYPE_PORT);
 		PortVal dst_port_val(ntohs(tcp_analyzer->Conn()->RespPort()),
@@ -387,7 +378,6 @@ void TCP_Reassembler::BlockInserted(DataBlock* start_block)
 			{ // New stuff.
 			uint64 len = b->Size();
 			uint64 seq = last_reassem_seq;
-
 			last_reassem_seq += len;
 
 			if ( record_contents_file )
@@ -548,35 +538,6 @@ void TCP_Reassembler::AckReceived(uint64 seq)
 			tot_gap_bytes += num_missing;
 			tcp_analyzer->Event(ack_above_hole);
 			}
-
-		double dt = network_time - last_gap_report;
-
-		if ( gap_report && gap_report_freq > 0.0 &&
-		     dt >= gap_report_freq )
-			{
-			uint64 devents = tot_ack_events - last_ack_events;
-			uint64 dbytes = tot_ack_bytes - last_ack_bytes;
-			uint64 dgaps = tot_gap_events - last_gap_events;
-			uint64 dgap_bytes = tot_gap_bytes - last_gap_bytes;
-
-			RecordVal* r = new RecordVal(gap_info);
-			r->Assign(0, new Val(devents, TYPE_COUNT));
-			r->Assign(1, new Val(dbytes, TYPE_COUNT));
-			r->Assign(2, new Val(dgaps, TYPE_COUNT));
-			r->Assign(3, new Val(dgap_bytes, TYPE_COUNT));
-
-			val_list* vl = new val_list;
-			vl->append(new IntervalVal(dt, Seconds));
-			vl->append(r);
-
-			mgr.QueueEvent(gap_report, vl);
-
-			last_gap_report = network_time;
-			last_ack_events = tot_ack_events;
-			last_ack_bytes = tot_ack_bytes;
-			last_gap_events = tot_gap_events;
-			last_gap_bytes = tot_gap_bytes;
-			}
 		}
 
 	// Check EOF here because t_reassem->LastReassemSeq() may have

src/analyzer/protocol/tcp/functions.bif

@@ -63,26 +63,6 @@ function get_resp_seq%(cid: conn_id%): count
 		}
 	%}
 
-## Returns statistics about TCP gaps.
-##
-## Returns: A record with TCP gap statistics.
-##
-## .. bro:see:: do_profiling
-##              net_stats
-##              resource_usage
-##              dump_rule_stats
-##              get_matcher_stats
-function get_gap_summary%(%): gap_info
-	%{
-	RecordVal* r = new RecordVal(gap_info);
-	r->Assign(0, new Val(tot_ack_events, TYPE_COUNT));
-	r->Assign(1, new Val(tot_ack_bytes, TYPE_COUNT));
-	r->Assign(2, new Val(tot_gap_events, TYPE_COUNT));
-	r->Assign(3, new Val(tot_gap_bytes, TYPE_COUNT));
-
-	return r;
-	%}
-
 ## Associates a file handle with a connection for writing TCP byte stream
 ## contents.
 ##

src/bro.bif

@@ -26,15 +26,8 @@
 
 using namespace std;
 
-RecordType* net_stats;
-RecordType* bro_resources;
-RecordType* matcher_stats;
 TableType* var_sizes;
 
-// This one is extern, since it's used beyond just built-ins,
-// and hence it's declared in NetVar.{h,cc}.
-extern RecordType* gap_info;
-
 static iosource::PktDumper* addl_pkt_dumper = 0;
 
 bro_int_t parse_int(const char*& fmt)
@@ -1725,156 +1718,6 @@ function reading_traces%(%): bool
 	return new Val(reading_traces, TYPE_BOOL);
 	%}
 
-## Returns packet capture statistics. Statistics include the number of
-## packets *(i)* received by Bro, *(ii)* dropped, and *(iii)* seen on the
-## link (not always available).
-##
-## Returns: A record of packet statistics.
-##
-## .. bro:see:: do_profiling
-##              resource_usage
-##              get_matcher_stats
-##              dump_rule_stats
-##              get_gap_summary
-function net_stats%(%): NetStats
-	%{
-	unsigned int recv = 0;
-	unsigned int drop = 0;
-	unsigned int link = 0;
-	unsigned int bytes_recv = 0;
-
-	const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs());
-
-	for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin();
-	      i != pkt_srcs.end(); i++ )
-		{
-		iosource::PktSrc* ps = *i;
-
-		struct iosource::PktSrc::Stats stat;
-		ps->Statistics(&stat);
-		recv += stat.received;
-		drop += stat.dropped;
-		link += stat.link;
-		bytes_recv += stat.bytes_received;
-		}
-
-	RecordVal* ns = new RecordVal(net_stats);
-	ns->Assign(0, new Val(recv, TYPE_COUNT));
-	ns->Assign(1, new Val(drop, TYPE_COUNT));
-	ns->Assign(2, new Val(link, TYPE_COUNT));
-	ns->Assign(3, new Val(bytes_recv, TYPE_COUNT));
-
-	return ns;
-	%}
-
-## Returns Bro process statistics. Statistics include real/user/sys CPU time,
-## memory usage, page faults, number of TCP/UDP/ICMP connections, timers,
-## and events queued/dispatched.
-##
-## Returns: A record with resource usage statistics.
-##
-## .. bro:see:: do_profiling
-##              net_stats
-##              get_matcher_stats
-##              dump_rule_stats
-##              get_gap_summary
-function resource_usage%(%): bro_resources
-	%{
-	struct rusage r;
-
-	if ( getrusage(RUSAGE_SELF, &r) < 0 )
-		reporter->InternalError("getrusage() failed in bro_resource_usage()");
-
-	double elapsed_time = current_time() - bro_start_time;
-
-	double user_time =
-		double(r.ru_utime.tv_sec) + double(r.ru_utime.tv_usec) / 1e6;
-	double system_time =
-		double(r.ru_stime.tv_sec) + double(r.ru_stime.tv_usec) / 1e6;
-
-	RecordVal* res = new RecordVal(bro_resources);
-	int n = 0;
-
-	res->Assign(n++, new StringVal(bro_version()));
-
-#ifdef DEBUG
-	res->Assign(n++, new Val(1, TYPE_COUNT));
-#else
-	res->Assign(n++, new Val(0, TYPE_COUNT));
-#endif
-
-	res->Assign(n++, new Val(bro_start_time, TYPE_TIME));
-
-	res->Assign(n++, new IntervalVal(elapsed_time, Seconds));
-	res->Assign(n++, new IntervalVal(user_time, Seconds));
-	res->Assign(n++, new IntervalVal(system_time, Seconds));
-
-	unsigned int total_mem;
-	get_memory_usage(&total_mem, 0);
-	res->Assign(n++, new Val(unsigned(total_mem), TYPE_COUNT));
-
-	res->Assign(n++, new Val(unsigned(r.ru_minflt), TYPE_COUNT));
-	res->Assign(n++, new Val(unsigned(r.ru_majflt), TYPE_COUNT));
-	res->Assign(n++, new Val(unsigned(r.ru_nswap), TYPE_COUNT));
-	res->Assign(n++, new Val(unsigned(r.ru_inblock), TYPE_COUNT));
-	res->Assign(n++, new Val(unsigned(r.ru_oublock), TYPE_COUNT));
-	res->Assign(n++, new Val(unsigned(r.ru_nivcsw), TYPE_COUNT));
-
-	SessionStats s;
-	if ( sessions )
-		sessions->GetStats(s);
-
-#define ADD_STAT(x) \
-	res->Assign(n++, new Val(unsigned(sessions ? x : 0), TYPE_COUNT));
-
-	ADD_STAT(s.num_TCP_conns);
-	ADD_STAT(s.num_UDP_conns);
-	ADD_STAT(s.num_ICMP_conns);
-	ADD_STAT(s.num_fragments);
-	ADD_STAT(s.num_packets);
-	ADD_STAT(s.num_timers);
-	ADD_STAT(s.num_events_queued);
-	ADD_STAT(s.num_events_dispatched);
-	ADD_STAT(s.max_TCP_conns);
-	ADD_STAT(s.max_UDP_conns);
-	ADD_STAT(s.max_ICMP_conns);
-	ADD_STAT(s.max_fragments);
-	ADD_STAT(s.max_timers);
-
-	return res;
-	%}
-
-## Returns statistics about the regular expression engine. Statistics include
-## the number of distinct matchers, DFA states, DFA state transitions, memory
-## usage of DFA states, cache hits/misses, and average number of NFA states
-## across all matchers.
-##
-## Returns: A record with matcher statistics.
-##
-## .. bro:see:: do_profiling
-##              net_stats
-##              resource_usage
-##              dump_rule_stats
-##              get_gap_summary
-function get_matcher_stats%(%): matcher_stats
-	%{
-	RuleMatcher::Stats s;
-	memset(&s, 0, sizeof(s));
-
-	if ( rule_matcher )
-		rule_matcher->GetStats(&s);
-
-	RecordVal* r = new RecordVal(matcher_stats);
-	r->Assign(0, new Val(s.matchers, TYPE_COUNT));
-	r->Assign(1, new Val(s.dfa_states, TYPE_COUNT));
-	r->Assign(2, new Val(s.computed, TYPE_COUNT));
-	r->Assign(3, new Val(s.mem, TYPE_COUNT));
-	r->Assign(4, new Val(s.hits, TYPE_COUNT));
-	r->Assign(5, new Val(s.misses, TYPE_COUNT));
-	r->Assign(6, new Val(s.avg_nfa_states, TYPE_COUNT));
-
-	return r;
-	%}
 
 ## Generates a table of the size of all global variables. The table index is
 ## the variable name and the value is the variable size in bytes.
@@ -2012,11 +1855,17 @@ function record_fields%(rec: any%): record_field_table
 ## timers, and script-level state. The script variable :bro:id:`profiling_file`
 ## holds the name of the file.
 ##
-## .. bro:see:: net_stats
+## .. bro:see:: get_conn_stats
-##              resource_usage
+##              get_dns_stats
+##              get_event_stats
+##              get_file_analysis_stats
+##              get_gap_stats
 ##              get_matcher_stats
-##              dump_rule_stats
+##              get_net_stats
-##              get_gap_summary
+##              get_proc_stats
+##              get_reassembler_stats
+##              get_thread_stats
+##              get_timer_stats
 function do_profiling%(%) : any
 	%{
 	if ( profiling_logger )
@@ -2078,13 +1927,7 @@ function is_local_interface%(ip: addr%) : bool
 ##
 ## Returns: True (unconditionally).
 ##
-## .. bro:see:: do_profiling
+## .. bro:see:: get_matcher_stats
-##              resource_usage
-##              get_matcher_stats
-##              net_stats
-##              get_gap_summary
-##
-## .. todo:: The return value should be changed to any or check appropriately.
 function dump_rule_stats%(f: file%): bool
 	%{
 	if ( rule_matcher )

src/cq.c

@@ -42,6 +42,7 @@ struct cq_handle {
 	int lowmark;			/* low bucket threshold */
 	int nextbucket;			/* next bucket to check */
 	int noresize;			/* don't resize while we're resizing */
+	uint64_t cumulative_num;	/* cumulative entries ever enqueued */
 	double lastpri;			/* last priority */
 	double ysize;			/* length of a year */
 	double bwidth;			/* width of each bucket */
@@ -175,6 +176,9 @@ cq_enqueue(register struct cq_handle *hp, register double pri,
 	}
 	bp->pri = pri;
 	bp->cookie = cookie;
+
+	++hp->cumulative_num;
+
 	if (++hp->qlen > hp->max_qlen)
 		hp->max_qlen = hp->qlen;
 #ifdef DEBUG
@@ -414,6 +418,12 @@ cq_max_size(struct cq_handle *hp)
 	return hp->max_qlen;
 }
 
+uint64_t
+cq_cumulative_num(struct cq_handle *hp)
+{
+	return hp->cumulative_num;
+}
+
 /* Return without doing anything if we fail to allocate a new bucket array */
 static int
 cq_resize(register struct cq_handle *hp, register int grow)

src/cq.h

@@ -1,3 +1,6 @@
+
+#include <stdint.h>
+
 struct cq_handle *cq_init(double, double);
 void cq_destroy(struct cq_handle *);
 int cq_enqueue(struct cq_handle *, double, void *);
@@ -5,6 +8,7 @@ void *cq_dequeue(struct cq_handle *, double);
 void *cq_remove(struct cq_handle *, double, void *);
 int cq_size(struct cq_handle *);
 int cq_max_size(struct cq_handle *);
+uint64_t cq_cumulative_num(struct cq_handle *);
 unsigned int cq_memory_allocation(void);
 #ifdef DEBUG
 void cq_debug(struct cq_handle *, int);

src/event.bif

@@ -366,26 +366,6 @@ event ack_above_hole%(c: connection%);
 ##    the two.
 event content_gap%(c: connection, is_orig: bool, seq: count, length: count%);
 
-## Summarizes the amount of missing TCP payload at regular intervals.
-## Internally, Bro tracks (1) the number of :bro:id:`ack_above_hole` events,
-## including the number of bytes missing; and (2) the total number of TCP
-## acks seen, with the total volume of bytes that have been acked. This event
-## reports these statistics in :bro:id:`gap_report_freq` intervals for the
-## purpose of determining packet loss.
-##
-## dt: The time that has passed since the last ``gap_report`` interval.
-##
-## info: The gap statistics.
-##
-## .. bro:see:: content_gap ack_above_hole
-##
-## .. note::
-##
-##    Bro comes with a script :doc:`/scripts/policy/misc/capture-loss.bro` that
-##    uses this event to estimate packet loss and report when a predefined
-##    threshold is exceeded.
-event gap_report%(dt: interval, info: gap_info%);
-
 ## Generated when a protocol analyzer confirms that a connection is indeed
 ## using that protocol. Bro's dynamic protocol detection heuristically activates
 ## analyzers as soon as it believes a connection *could* be using a particular

src/file_analysis/FileReassembler.cc

@@ -8,7 +8,7 @@ namespace file_analysis {
 class File;
 
 FileReassembler::FileReassembler(File *f, uint64 starting_offset)
-	: Reassembler(starting_offset), the_file(f), flushing(false)
+	: Reassembler(starting_offset, REASSEM_FILE), the_file(f), flushing(false)
 	{
 	}
 

src/file_analysis/Manager.h

@@ -302,6 +302,15 @@ public:
 	 */
 	std::string DetectMIME(const u_char* data, uint64 len) const;
 
+	uint64 CurrentFiles()
+		{ return id_map.Length(); }
+
+	uint64 MaxFiles()
+		{ return id_map.MaxLength(); }
+
+	uint64 CumulativeFiles()
+		{ return id_map.NumCumulativeInserts(); }
+
 protected:
 	friend class FileTimer;
 

src/main.cc

@@ -1172,8 +1172,8 @@ int main(int argc, char** argv)
 
 		double time_net_start = current_time(true);;
 
-		unsigned int mem_net_start_total;
+		uint64 mem_net_start_total;
-		unsigned int mem_net_start_malloced;
+		uint64 mem_net_start_malloced;
 
 		if ( time_bro )
 			{
@@ -1181,7 +1181,7 @@ int main(int argc, char** argv)
 
 			fprintf(stderr, "# initialization %.6f\n", time_net_start - time_start);
 
-			fprintf(stderr, "# initialization %uM/%uM\n",
+			fprintf(stderr, "# initialization %" PRIu64 "M/%" PRIu64 "M\n",
 				mem_net_start_total / 1024 / 1024,
 				mem_net_start_malloced / 1024 / 1024);
 			}
@@ -1190,8 +1190,8 @@ int main(int argc, char** argv)
 
 		double time_net_done = current_time(true);;
 
-		unsigned int mem_net_done_total;
+		uint64 mem_net_done_total;
-		unsigned int mem_net_done_malloced;
+		uint64 mem_net_done_malloced;
 
 		if ( time_bro )
 			{
@@ -1200,7 +1200,7 @@ int main(int argc, char** argv)
 			fprintf(stderr, "# total time %.6f, processing %.6f\n",
 				time_net_done - time_start, time_net_done - time_net_start);
 
-			fprintf(stderr, "# total mem %uM/%uM, processing %uM/%uM\n",
+			fprintf(stderr, "# total mem %" PRId64 "M/%" PRId64 "M, processing %" PRId64 "M/%" PRId64 "M\n",
 				mem_net_done_total / 1024 / 1024,
 				mem_net_done_malloced / 1024 / 1024,
 				(mem_net_done_total - mem_net_start_total) / 1024 / 1024,

src/stats.bif

@@ -0,0 +1,422 @@
+
+%%{ // C segment
+#include "util.h"
+#include "threading/Manager.h"
+
+RecordType* ProcStats;
+RecordType* NetStats;
+RecordType* MatcherStats;
+RecordType* ReassemblerStats;
+RecordType* DNSStats;
+RecordType* ConnStats;
+RecordType* GapStats;
+RecordType* EventStats;
+RecordType* ThreadStats;
+RecordType* TimerStats;
+RecordType* FileAnalysisStats;
+%%}
+
+## Returns packet capture statistics. Statistics include the number of
+## packets *(i)* received by Bro, *(ii)* dropped, and *(iii)* seen on the
+## link (not always available).
+##
+## Returns: A record of packet statistics.
+##
+## .. bro:see:: get_conn_stats
+##              get_dns_stats
+##              get_event_stats
+##              get_file_analysis_stats
+##              get_gap_stats
+##              get_matcher_stats
+##              get_proc_stats
+##              get_reassembler_stats
+##              get_thread_stats
+##              get_timer_stats
+function get_net_stats%(%): NetStats
+	%{
+	uint64 recv = 0;
+	uint64 drop = 0;
+	uint64 link = 0;
+	uint64 bytes_recv = 0;
+
+	const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs());
+
+	for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin();
+	      i != pkt_srcs.end(); i++ )
+		{
+		iosource::PktSrc* ps = *i;
+
+		struct iosource::PktSrc::Stats stat;
+		ps->Statistics(&stat);
+		recv += stat.received;
+		drop += stat.dropped;
+		link += stat.link;
+		bytes_recv += stat.bytes_received;
+		}
+
+	RecordVal* r = new RecordVal(NetStats);
+	int n = 0;
+
+	r->Assign(n++, new Val(recv, TYPE_COUNT));
+	r->Assign(n++, new Val(drop, TYPE_COUNT));
+	r->Assign(n++, new Val(link, TYPE_COUNT));
+	r->Assign(n++, new Val(bytes_recv, TYPE_COUNT));
+
+	return r;
+	%}
+
+## Returns Bro traffic statistics.
+##
+## Returns: A record with connection and packet statistics.
+##
+## .. bro:see:: get_dns_stats
+##              get_event_stats
+##              get_file_analysis_stats
+##              get_gap_stats
+##              get_matcher_stats
+##              get_net_stats
+##              get_proc_stats
+##              get_reassembler_stats
+##              get_thread_stats
+##              get_timer_stats
+function get_conn_stats%(%): ConnStats
+	%{
+	RecordVal* r = new RecordVal(ConnStats);
+	int n = 0;
+
+	r->Assign(n++, new Val(Connection::TotalConnections(), TYPE_COUNT));
+	r->Assign(n++, new Val(Connection::CurrentConnections(), TYPE_COUNT));
+	r->Assign(n++, new Val(Connection::CurrentExternalConnections(), TYPE_COUNT));
+	r->Assign(n++, new Val(sessions->CurrentConnections(), TYPE_COUNT));
+
+	SessionStats s;
+	if ( sessions )
+		sessions->GetStats(s);
+
+#define ADD_STAT(x) \
+	r->Assign(n++, new Val(unsigned(sessions ? x : 0), TYPE_COUNT));
+
+	ADD_STAT(s.num_packets);
+	ADD_STAT(s.num_fragments);
+	ADD_STAT(s.max_fragments);
+	ADD_STAT(s.num_TCP_conns);
+	ADD_STAT(s.max_TCP_conns);
+	ADD_STAT(s.cumulative_TCP_conns);
+	ADD_STAT(s.num_UDP_conns);
+	ADD_STAT(s.max_UDP_conns);
+	ADD_STAT(s.cumulative_UDP_conns);
+	ADD_STAT(s.num_ICMP_conns);
+	ADD_STAT(s.max_ICMP_conns);
+	ADD_STAT(s.cumulative_ICMP_conns);
+
+	r->Assign(n++, new Val(killed_by_inactivity, TYPE_COUNT));
+
+	return r;
+	%}
+
+## Returns Bro process statistics.
+##
+## Returns: A record with process statistics.
+##
+## .. bro:see:: get_conn_stats
+##              get_dns_stats
+##              get_event_stats
+##              get_file_analysis_stats
+##              get_gap_stats
+##              get_matcher_stats
+##              get_net_stats
+##              get_reassembler_stats
+##              get_thread_stats
+##              get_timer_stats
+function get_proc_stats%(%): ProcStats
+	%{
+	struct rusage ru;
+	if ( getrusage(RUSAGE_SELF, &ru) < 0 )
+		reporter->InternalError("getrusage() failed in get_proc_stats()");
+
+	RecordVal* r = new RecordVal(ProcStats);
+	int n = 0;
+
+	double elapsed_time = current_time() - bro_start_time;
+	double user_time =
+		double(ru.ru_utime.tv_sec) + double(ru.ru_utime.tv_usec) / 1e6;
+	double system_time =
+		double(ru.ru_stime.tv_sec) + double(ru.ru_stime.tv_usec) / 1e6;
+
+#ifdef DEBUG
+	r->Assign(n++, new Val(1, TYPE_COUNT));
+#else
+	r->Assign(n++, new Val(0, TYPE_COUNT));
+#endif
+
+	r->Assign(n++, new Val(bro_start_time, TYPE_TIME));
+
+	r->Assign(n++, new IntervalVal(elapsed_time, Seconds));
+	r->Assign(n++, new IntervalVal(user_time, Seconds));
+	r->Assign(n++, new IntervalVal(system_time, Seconds));
+
+	uint64 total_mem;
+	get_memory_usage(&total_mem, NULL);
+	r->Assign(n++, new Val(unsigned(total_mem), TYPE_COUNT));
+
+	r->Assign(n++, new Val(unsigned(ru.ru_minflt), TYPE_COUNT));
+	r->Assign(n++, new Val(unsigned(ru.ru_majflt), TYPE_COUNT));
+	r->Assign(n++, new Val(unsigned(ru.ru_nswap), TYPE_COUNT));
+	r->Assign(n++, new Val(unsigned(ru.ru_inblock), TYPE_COUNT));
+	r->Assign(n++, new Val(unsigned(ru.ru_oublock), TYPE_COUNT));
+	r->Assign(n++, new Val(unsigned(ru.ru_nivcsw), TYPE_COUNT));
+
+	return r;
+	%}
+
+## Returns statistics about the event engine.
+##
+## Returns: A record with event engine statistics.
+##
+## .. bro:see:: get_conn_stats
+##              get_dns_stats
+##              get_file_analysis_stats
+##              get_gap_stats
+##              get_matcher_stats
+##              get_net_stats
+##              get_proc_stats
+##              get_reassembler_stats
+##              get_thread_stats
+##              get_timer_stats
+function get_event_stats%(%): EventStats
+	%{
+	RecordVal* r = new RecordVal(EventStats);
+	int n = 0;
+
+	r->Assign(n++, new Val(num_events_queued, TYPE_COUNT));
+	r->Assign(n++, new Val(num_events_dispatched, TYPE_COUNT));
+
+	return r;
+	%}
+
+## Returns statistics about reassembler usage.
+##
+## Returns: A record with reassembler statistics.
+##
+## .. bro:see:: get_conn_stats
+##              get_dns_stats
+##              get_event_stats
+##              get_file_analysis_stats
+##              get_gap_stats
+##              get_matcher_stats
+##              get_net_stats
+##              get_proc_stats
+##              get_thread_stats
+##              get_timer_stats
+function get_reassembler_stats%(%): ReassemblerStats
+	%{
+	RecordVal* r = new RecordVal(ReassemblerStats);
+	int n = 0;
+
+	r->Assign(n++, new Val(Reassembler::MemoryAllocation(REASSEM_FILE), TYPE_COUNT));
+	r->Assign(n++, new Val(Reassembler::MemoryAllocation(REASSEM_FRAG), TYPE_COUNT));
+	r->Assign(n++, new Val(Reassembler::MemoryAllocation(REASSEM_TCP), TYPE_COUNT));
+	r->Assign(n++, new Val(Reassembler::MemoryAllocation(REASSEM_UNKNOWN), TYPE_COUNT));
+
+	return r;
+	%}
+
+## Returns statistics about DNS lookup activity.
+##
+## Returns: A record with DNS lookup statistics.
+##
+## .. bro:see:: get_conn_stats
+##              get_event_stats
+##              get_file_analysis_stats
+##              get_gap_stats
+##              get_matcher_stats
+##              get_net_stats
+##              get_proc_stats
+##              get_reassembler_stats
+##              get_thread_stats
+##              get_timer_stats
+function get_dns_stats%(%): DNSStats
+	%{
+	RecordVal* r = new RecordVal(DNSStats);
+	int n = 0;
+
+	DNS_Mgr::Stats dstats;
+	dns_mgr->GetStats(&dstats);
+
+	r->Assign(n++, new Val(unsigned(dstats.requests), TYPE_COUNT));
+	r->Assign(n++, new Val(unsigned(dstats.successful), TYPE_COUNT));
+	r->Assign(n++, new Val(unsigned(dstats.failed), TYPE_COUNT));
+	r->Assign(n++, new Val(unsigned(dstats.pending), TYPE_COUNT));
+	r->Assign(n++, new Val(unsigned(dstats.cached_hosts), TYPE_COUNT));
+	r->Assign(n++, new Val(unsigned(dstats.cached_addresses), TYPE_COUNT));
+
+	return r;
+	%}
+
+## Returns statistics about timer usage.
+##
+## Returns: A record with timer usage statistics.
+##
+## .. bro:see:: get_conn_stats
+##              get_dns_stats
+##              get_event_stats
+##              get_file_analysis_stats
+##              get_gap_stats
+##              get_matcher_stats
+##              get_net_stats
+##              get_proc_stats
+##              get_reassembler_stats
+##              get_thread_stats
+function get_timer_stats%(%): TimerStats
+	%{
+	RecordVal* r = new RecordVal(TimerStats);
+	int n = 0;
+
+	r->Assign(n++, new Val(unsigned(timer_mgr->Size()), TYPE_COUNT));
+	r->Assign(n++, new Val(unsigned(timer_mgr->PeakSize()), TYPE_COUNT));
+	r->Assign(n++, new Val(timer_mgr->CumulativeNum(), TYPE_COUNT));
+
+	return r;
+	%}
+
+## Returns statistics about file analysis.
+##
+## Returns: A record with file analysis statistics.
+##
+## .. bro:see:: get_conn_stats
+##              get_dns_stats
+##              get_event_stats
+##              get_gap_stats
+##              get_matcher_stats
+##              get_net_stats
+##              get_proc_stats
+##              get_reassembler_stats
+##              get_thread_stats
+##              get_timer_stats
+function get_file_analysis_stats%(%): FileAnalysisStats
+	%{
+	RecordVal* r = new RecordVal(FileAnalysisStats);
+	int n = 0;
+
+	r->Assign(n++, new Val(file_mgr->CurrentFiles(), TYPE_COUNT));
+	r->Assign(n++, new Val(file_mgr->MaxFiles(), TYPE_COUNT));
+	r->Assign(n++, new Val(file_mgr->CumulativeFiles(), TYPE_COUNT));
+
+	return r;
+	%}
+
+## Returns statistics about thread usage.
+##
+## Returns: A record with thread usage statistics.
+##
+## .. bro:see:: get_conn_stats
+##              get_dns_stats
+##              get_event_stats
+##              get_file_analysis_stats
+##              get_gap_stats
+##              get_matcher_stats
+##              get_net_stats
+##              get_proc_stats
+##              get_reassembler_stats
+##              get_timer_stats
+function get_thread_stats%(%): ThreadStats
+	%{
+	RecordVal* r = new RecordVal(ThreadStats);
+	int n = 0;
+
+	r->Assign(n++, new Val(thread_mgr->NumThreads(), TYPE_COUNT));
+
+	return r;
+	%}
+
+## Returns statistics about TCP gaps.
+##
+## Returns: A record with TCP gap statistics.
+##
+## .. bro:see:: get_conn_stats
+##              get_dns_stats
+##              get_event_stats
+##              get_file_analysis_stats
+##              get_matcher_stats
+##              get_net_stats
+##              get_proc_stats
+##              get_reassembler_stats
+##              get_thread_stats
+##              get_timer_stats
+function get_gap_stats%(%): GapStats
+	%{
+	RecordVal* r = new RecordVal(GapStats);
+	int n = 0;
+
+	r->Assign(n++, new Val(tot_ack_events, TYPE_COUNT));
+	r->Assign(n++, new Val(tot_ack_bytes, TYPE_COUNT));
+	r->Assign(n++, new Val(tot_gap_events, TYPE_COUNT));
+	r->Assign(n++, new Val(tot_gap_bytes, TYPE_COUNT));
+
+	return r;
+	%}
+
+## Returns statistics about the regular expression engine. Statistics include
+## the number of distinct matchers, DFA states, DFA state transitions, memory
+## usage of DFA states, cache hits/misses, and average number of NFA states
+## across all matchers.
+##
+## Returns: A record with matcher statistics.
+##
+## .. bro:see:: get_conn_stats
+##              get_dns_stats
+##              get_event_stats
+##              get_file_analysis_stats
+##              get_gap_stats
+##              get_net_stats
+##              get_proc_stats
+##              get_reassembler_stats
+##              get_thread_stats
+##              get_timer_stats
+function get_matcher_stats%(%): MatcherStats
+	%{
+	RecordVal* r = new RecordVal(MatcherStats);
+	int n = 0;
+
+	RuleMatcher::Stats s;
+	memset(&s, 0, sizeof(s));
+	if ( rule_matcher )
+		rule_matcher->GetStats(&s);
+
+	r->Assign(n++, new Val(s.matchers, TYPE_COUNT));
+	r->Assign(n++, new Val(s.nfa_states, TYPE_COUNT));
+	r->Assign(n++, new Val(s.dfa_states, TYPE_COUNT));
+	r->Assign(n++, new Val(s.computed, TYPE_COUNT));
+	r->Assign(n++, new Val(s.mem, TYPE_COUNT));
+	r->Assign(n++, new Val(s.hits, TYPE_COUNT));
+	r->Assign(n++, new Val(s.misses, TYPE_COUNT));
+
+	return r;
+	%}
+
+# function get_broker_stats%(%): BrokerStats
+# 	%{
+# 	RecordVal* r = new RecordVal(CommunicationStats);
+# 	int n = 0;
+#
+# #ifdef ENABLE_BROKER
+# 	auto cs = broker_mgr->ConsumeStatistics();
+#
+# 	r->Assign(n++, new Val(cs.outgoing_peer_count, TYPE_COUNT));
+# 	r->Assign(n++, new Val(cs.data_store_count, TYPE_COUNT));
+# 	r->Assign(n++, new Val(cs.pending_query_count, TYPE_COUNT));
+# 	r->Assign(n++, new Val(cs.response_count, TYPE_COUNT));
+# 	r->Assign(n++, new Val(cs.outgoing_conn_status_count, TYPE_COUNT));
+# 	r->Assign(n++, new Val(cs.incoming_conn_status_count, TYPE_COUNT));
+# 	r->Assign(n++, new Val(cs.report_count, TYPE_COUNT));
+#
+# 	//for ( const auto& s : cs.print_count )
+# 	//	file->Write(fmt("    %-25s prints dequeued=%zu\n", s.first.data(), s.second));
+# 	//for ( const auto& s : cs.event_count )
+# 	//	file->Write(fmt("    %-25s events dequeued=%zu\n", s.first.data(), s.second));
+# 	//for ( const auto& s : cs.log_count )
+# 	//	file->Write(fmt("    %-25s logs dequeued=%zu\n", s.first.data(), s.second));
+# #endif
+#
+# 	return r;
+# 	%}

src/util.cc

@@ -14,6 +14,11 @@
 # endif
 #endif
 
+#ifdef HAVE_DARWIN
+#include <mach/task.h>
+#include <mach/mach_init.h>
+#endif
+
 #include <string>
 #include <vector>
 #include <algorithm>
@@ -1611,23 +1616,35 @@ extern "C" void out_of_memory(const char* where)
 	abort();
 	}
 
-void get_memory_usage(unsigned int* total, unsigned int* malloced)
+void get_memory_usage(uint64* total, uint64* malloced)
 	{
-	unsigned int ret_total;
+	uint64 ret_total;
 
 #ifdef HAVE_MALLINFO
 	struct mallinfo mi = mallinfo();
 
 	if ( malloced )
 		*malloced = mi.uordblks;
-
 #endif
 
+#ifdef HAVE_DARWIN
+	struct mach_task_basic_info t_info;
+	mach_msg_type_number_t t_info_count = MACH_TASK_BASIC_INFO;
+
+	if ( KERN_SUCCESS != task_info(mach_task_self(),
+	                               MACH_TASK_BASIC_INFO,
+	                               (task_info_t)&t_info,
+	                               &t_info_count) )
+		ret_total = 0;
+	else
+		ret_total = t_info.resident_size;
+#else
 	struct rusage r;
 	getrusage(RUSAGE_SELF, &r);
 
 	// In KB.
 	ret_total = r.ru_maxrss * 1024;
+#endif
 
 	if ( total )
 		*total = ret_total;

src/util.h

@@ -499,8 +499,7 @@ inline int safe_vsnprintf(char* str, size_t size, const char* format, va_list al
 
 // Returns total memory allocations and (if available) amount actually
 // handed out by malloc.
-extern void get_memory_usage(unsigned int* total,
+extern void get_memory_usage(uint64* total, uint64* malloced);
-			     unsigned int* malloced);
 
 // Class to be used as a third argument for STL maps to be able to use
 // char*'s as keys. Otherwise the pointer values will be compared instead of

testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2016-04-29-20-49-16
+#open	2016-05-02-20-39-26
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -50,6 +50,7 @@ scripts/base/init-bare.bro
         scripts/base/utils/patterns.bro
     scripts/base/frameworks/files/magic/__load__.bro
   build/scripts/base/bif/__load__.bro
+    build/scripts/base/bif/stats.bif.bro
     build/scripts/base/bif/broxygen.bif.bro
     build/scripts/base/bif/functions.bif.bro
     build/scripts/base/bif/bloom-filter.bif.bro
@@ -133,4 +134,4 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro
 scripts/policy/misc/loaded-scripts.bro
   scripts/base/utils/paths.bro
-#close	2016-04-29-20-49-16
+#close	2016-05-02-20-39-26

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -50,6 +50,7 @@ scripts/base/init-bare.bro
         scripts/base/utils/patterns.bro
     scripts/base/frameworks/files/magic/__load__.bro
   build/scripts/base/bif/__load__.bro
+    build/scripts/base/bif/stats.bif.bro
     build/scripts/base/bif/broxygen.bif.bro
     build/scripts/base/bif/functions.bif.bro
     build/scripts/base/bif/bloom-filter.bif.bro
@@ -308,4 +309,4 @@ scripts/base/init-default.bro
   scripts/base/misc/find-checksum-offloading.bro
   scripts/base/misc/find-filtered-trace.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2016-04-29-20-49-25
+#close	2016-05-02-20-39-35

testing/btest/Baseline/plugins.hooks/output

@@ -238,7 +238,7 @@
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])) -> <no result>
-0.000000   MetaHookPost  CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1461962978.799805, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
+0.000000   MetaHookPost  CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1462646849.582646, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, <frame>, (Cluster::LOG)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, <frame>, (Communication::LOG)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, <frame>, (Conn::LOG)) -> <no result>
@@ -359,7 +359,7 @@
 0.000000   MetaHookPost  CallFunction(Log::create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])) -> <no result>
-0.000000   MetaHookPost  CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1461962978.799805, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
+0.000000   MetaHookPost  CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1462646849.582646, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
 0.000000   MetaHookPost  CallFunction(NetControl::check_plugins, <frame>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(NetControl::init, <null>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(Notice::want_pp, <frame>, ()) -> <no result>
@@ -533,6 +533,7 @@
 0.000000   MetaHookPost  LoadFile(./sftp) -> -1
 0.000000   MetaHookPost  LoadFile(./shunt) -> -1
 0.000000   MetaHookPost  LoadFile(./site) -> -1
+0.000000   MetaHookPost  LoadFile(./stats.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./std-dev) -> -1
 0.000000   MetaHookPost  LoadFile(./store) -> -1
 0.000000   MetaHookPost  LoadFile(./store.bif.bro) -> -1
@@ -894,7 +895,7 @@
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird]))
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509]))
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql]))
-0.000000   MetaHookPre   CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1461962978.799805, node=bro, filter=ip or not ip, init=T, success=T]))
+0.000000   MetaHookPre   CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1462646849.582646, node=bro, filter=ip or not ip, init=T, success=T]))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, <frame>, (Cluster::LOG))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, <frame>, (Communication::LOG))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, <frame>, (Conn::LOG))
@@ -1015,7 +1016,7 @@
 0.000000   MetaHookPre   CallFunction(Log::create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird]))
 0.000000   MetaHookPre   CallFunction(Log::create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509]))
 0.000000   MetaHookPre   CallFunction(Log::create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql]))
-0.000000   MetaHookPre   CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1461962978.799805, node=bro, filter=ip or not ip, init=T, success=T]))
+0.000000   MetaHookPre   CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1462646849.582646, node=bro, filter=ip or not ip, init=T, success=T]))
 0.000000   MetaHookPre   CallFunction(NetControl::check_plugins, <frame>, ())
 0.000000   MetaHookPre   CallFunction(NetControl::init, <null>, ())
 0.000000   MetaHookPre   CallFunction(Notice::want_pp, <frame>, ())
@@ -1189,6 +1190,7 @@
 0.000000   MetaHookPre   LoadFile(./sftp)
 0.000000   MetaHookPre   LoadFile(./shunt)
 0.000000   MetaHookPre   LoadFile(./site)
+0.000000   MetaHookPre   LoadFile(./stats.bif.bro)
 0.000000   MetaHookPre   LoadFile(./std-dev)
 0.000000   MetaHookPre   LoadFile(./store)
 0.000000   MetaHookPre   LoadFile(./store.bif.bro)
@@ -1549,7 +1551,7 @@
 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])
 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])
 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])
-0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1461962978.799805, node=bro, filter=ip or not ip, init=T, success=T])
+0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1462646849.582646, node=bro, filter=ip or not ip, init=T, success=T])
 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG)
 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG)
 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG)
@@ -1670,7 +1672,7 @@
 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])
 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])
 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])
-0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1461962978.799805, node=bro, filter=ip or not ip, init=T, success=T])
+0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1462646849.582646, node=bro, filter=ip or not ip, init=T, success=T])
 0.000000 | HookCallFunction NetControl::check_plugins()
 0.000000 | HookCallFunction NetControl::init()
 0.000000 | HookCallFunction Notice::want_pp()
@@ -1718,7 +1720,7 @@
 1362692526.869344   MetaHookPost  CallFunction(ChecksumOffloading::check, <null>, ()) -> <no result>
 1362692526.869344   MetaHookPost  CallFunction(NetControl::check_conn, <frame>, (141.142.228.5)) -> <no result>
 1362692526.869344   MetaHookPost  CallFunction(filter_change_tracking, <null>, ()) -> <no result>
-1362692526.869344   MetaHookPost  CallFunction(net_stats, <frame>, ()) -> <no result>
+1362692526.869344   MetaHookPost  CallFunction(get_net_stats, <frame>, ()) -> <no result>
 1362692526.869344   MetaHookPost  CallFunction(new_connection, <null>, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/tcp], orig=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0], resp=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0], start_time=1362692526.869344, duration=0.0, service={}, history=, uid=CXWv6p3arKYeMETxOg, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, krb=<uninitialized>, modbus=<uninitialized>, mysql=<uninitialized>, radius=<uninitialized>, rdp=<uninitialized>, rfb=<uninitialized>, sip=<uninitialized>, sip_state=<uninitialized>, snmp=<uninitialized>, smtp=<uninitialized>, smtp_state=<uninitialized>, socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>])) -> <no result>
 1362692526.869344   MetaHookPost  DrainEvents() -> <void>
 1362692526.869344   MetaHookPost  QueueEvent(ChecksumOffloading::check()) -> false
@@ -1729,7 +1731,7 @@
 1362692526.869344   MetaHookPre   CallFunction(ChecksumOffloading::check, <null>, ())
 1362692526.869344   MetaHookPre   CallFunction(NetControl::check_conn, <frame>, (141.142.228.5))
 1362692526.869344   MetaHookPre   CallFunction(filter_change_tracking, <null>, ())
-1362692526.869344   MetaHookPre   CallFunction(net_stats, <frame>, ())
+1362692526.869344   MetaHookPre   CallFunction(get_net_stats, <frame>, ())
 1362692526.869344   MetaHookPre   CallFunction(new_connection, <null>, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/tcp], orig=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0], resp=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0], start_time=1362692526.869344, duration=0.0, service={}, history=, uid=CXWv6p3arKYeMETxOg, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, krb=<uninitialized>, modbus=<uninitialized>, mysql=<uninitialized>, radius=<uninitialized>, rdp=<uninitialized>, rfb=<uninitialized>, sip=<uninitialized>, sip_state=<uninitialized>, snmp=<uninitialized>, smtp=<uninitialized>, smtp_state=<uninitialized>, socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>]))
 1362692526.869344   MetaHookPre   DrainEvents()
 1362692526.869344   MetaHookPre   QueueEvent(ChecksumOffloading::check())
@@ -1741,7 +1743,7 @@
 1362692526.869344 | HookCallFunction ChecksumOffloading::check()
 1362692526.869344 | HookCallFunction NetControl::check_conn(141.142.228.5)
 1362692526.869344 | HookCallFunction filter_change_tracking()
-1362692526.869344 | HookCallFunction net_stats()
+1362692526.869344 | HookCallFunction get_net_stats()
 1362692526.869344 | HookCallFunction new_connection([id=[orig_h=141.142.228.5, orig_p=59856<...>/tcp], orig=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0], resp=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0], start_time=1362692526.869344, duration=0.0, service={}, history=, uid=CXWv6p3arKYeMETxOg, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, dpd=<uninitialized>, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, dhcp=<uninitialized>, dnp3=<uninitialized>, dns=<uninitialized>, dns_state=<uninitialized>, ftp=<uninitialized>, ftp_data_reuse=F, ssl=<uninitialized>, http=<uninitialized>, http_state=<uninitialized>, irc=<uninitialized>, krb=<uninitialized>, modbus=<uninitialized>, mysql=<uninitialized>, radius=<uninitialized>, rdp=<uninitialized>, rfb=<uninitialized>, sip=<uninitialized>, sip_state=<uninitialized>, snmp=<uninitialized>, smtp=<uninitialized>, smtp_state=<uninitialized>, socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>])
 1362692526.869344 | HookDrainEvents
 1362692526.869344 | HookQueueEvent ChecksumOffloading::check()
@@ -2146,11 +2148,11 @@
 1362692527.080972   MetaHookPost  CallFunction(filter_change_tracking, <null>, ()) -> <no result>
 1362692527.080972   MetaHookPost  CallFunction(fmt, <frame>, (%s:%d > %s:%d, 141.142.228.5, 59856<...>/tcp)) -> <no result>
 1362692527.080972   MetaHookPost  CallFunction(get_file_handle, <null>, (Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=<uninitialized>, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=<uninitialized>, krb=<uninitialized>, modbus=<uninitialized>, mysql=<uninitialized>, radius=<uninitialized>, rdp=<uninitialized>, rfb=<uninitialized>, sip=<uninitialized>, sip_state=<uninitialized>, snmp=<uninitialized>, smtp=<uninitialized>, smtp_state=<uninitialized>, socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>], T)) -> <no result>
+1362692527.080972   MetaHookPost  CallFunction(get_net_stats, <frame>, ()) -> <no result>
 1362692527.080972   MetaHookPost  CallFunction(get_port_transport_proto, <frame>, (80/tcp)) -> <no result>
 1362692527.080972   MetaHookPost  CallFunction(id_string, <frame>, ([orig_h=141.142.228.5, orig_p=59856<...>/tcp])) -> <no result>
 1362692527.080972   MetaHookPost  CallFunction(is_tcp_port, <frame>, (59856/tcp)) -> <no result>
 1362692527.080972   MetaHookPost  CallFunction(net_done, <null>, (1362692527.080972)) -> <no result>
-1362692527.080972   MetaHookPost  CallFunction(net_stats, <frame>, ()) -> <no result>
 1362692527.080972   MetaHookPost  CallFunction(reading_traces, <frame>, ()) -> <no result>
 1362692527.080972   MetaHookPost  CallFunction(set_file_handle, <frame>, (Analyzer::ANALYZER_HTTP1362692526.869344T11141.142.228.5:59856 > 192.150.187.43:80)) -> <no result>
 1362692527.080972   MetaHookPost  CallFunction(sub_bytes, <frame>, (HTTP, 0, 1)) -> <no result>
@@ -2176,11 +2178,11 @@
 1362692527.080972   MetaHookPre   CallFunction(filter_change_tracking, <null>, ())
 1362692527.080972   MetaHookPre   CallFunction(fmt, <frame>, (%s:%d > %s:%d, 141.142.228.5, 59856<...>/tcp))
 1362692527.080972   MetaHookPre   CallFunction(get_file_handle, <null>, (Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=<uninitialized>, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=<uninitialized>, krb=<uninitialized>, modbus=<uninitialized>, mysql=<uninitialized>, radius=<uninitialized>, rdp=<uninitialized>, rfb=<uninitialized>, sip=<uninitialized>, sip_state=<uninitialized>, snmp=<uninitialized>, smtp=<uninitialized>, smtp_state=<uninitialized>, socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>], T))
+1362692527.080972   MetaHookPre   CallFunction(get_net_stats, <frame>, ())
 1362692527.080972   MetaHookPre   CallFunction(get_port_transport_proto, <frame>, (80/tcp))
 1362692527.080972   MetaHookPre   CallFunction(id_string, <frame>, ([orig_h=141.142.228.5, orig_p=59856<...>/tcp]))
 1362692527.080972   MetaHookPre   CallFunction(is_tcp_port, <frame>, (59856/tcp))
 1362692527.080972   MetaHookPre   CallFunction(net_done, <null>, (1362692527.080972))
-1362692527.080972   MetaHookPre   CallFunction(net_stats, <frame>, ())
 1362692527.080972   MetaHookPre   CallFunction(reading_traces, <frame>, ())
 1362692527.080972   MetaHookPre   CallFunction(set_file_handle, <frame>, (Analyzer::ANALYZER_HTTP1362692526.869344T11141.142.228.5:59856 > 192.150.187.43:80))
 1362692527.080972   MetaHookPre   CallFunction(sub_bytes, <frame>, (HTTP, 0, 1))
@@ -2207,11 +2209,11 @@
 1362692527.080972 | HookCallFunction filter_change_tracking()
 1362692527.080972 | HookCallFunction fmt(%s:%d > %s:%d, 141.142.228.5, 59856<...>/tcp)
 1362692527.080972 | HookCallFunction get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=<uninitialized>, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=<uninitialized>, krb=<uninitialized>, modbus=<uninitialized>, mysql=<uninitialized>, radius=<uninitialized>, rdp=<uninitialized>, rfb=<uninitialized>, sip=<uninitialized>, sip_state=<uninitialized>, snmp=<uninitialized>, smtp=<uninitialized>, smtp_state=<uninitialized>, socks=<uninitialized>, ssh=<uninitialized>, syslog=<uninitialized>], T)
+1362692527.080972 | HookCallFunction get_net_stats()
 1362692527.080972 | HookCallFunction get_port_transport_proto(80/tcp)
 1362692527.080972 | HookCallFunction id_string([orig_h=141.142.228.5, orig_p=59856<...>/tcp])
 1362692527.080972 | HookCallFunction is_tcp_port(59856/tcp)
 1362692527.080972 | HookCallFunction net_done(1362692527.080972)
-1362692527.080972 | HookCallFunction net_stats()
 1362692527.080972 | HookCallFunction reading_traces()
 1362692527.080972 | HookCallFunction set_file_handle(Analyzer::ANALYZER_HTTP1362692526.869344T11141.142.228.5:59856 > 192.150.187.43:80)
 1362692527.080972 | HookCallFunction sub_bytes(HTTP, 0, 1)

testing/btest/bifs/net_stats_trace.test

@@ -4,5 +4,5 @@
 
 event bro_done()
 	{
-	print net_stats();
+	print get_net_stats();
 	}

testing/btest/bifs/resource_usage.bro

@@ -1,9 +0,0 @@
-#
-# @TEST-EXEC: bro -b %INPUT
-
-event bro_init()
-	{
-	local a = resource_usage();
-	if ( a$version != bro_version() )
-		exit(1);
-	}