From 2b0a28686a6f5eebf7580618f196d9cfa90250a9 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 4 Jan 2016 00:55:52 -0500 Subject: [PATCH 01/16] Cleaned up stats collection. - Removed the gap_report event. It wasn't used anymore and functionally no more capable that scheduling events and using the get_gap_summary bif. - Added functionality to Dictionaries to count cumulative numbers of inserts performed. This is further used to measure the total number of connections of various types. Previously only the number of active connections was available. - The Reassembler base class now tracks active reassembly size for all subclasses (File/TCP/Frag & unknown). - Improvements to the stats.log. Mostly, more information. --- scripts/base/init-bare.bro | 81 +++++++++++--------- scripts/policy/misc/stats.bro | 37 +++++++-- src/Dict.cc | 2 + src/Dict.h | 7 ++ src/Event.cc | 4 +- src/Event.h | 4 +- src/Frag.cc | 2 +- src/Reassem.cc | 34 ++++++-- src/Reassem.h | 26 ++++++- src/Sessions.cc | 3 + src/Sessions.h | 23 +++--- src/analyzer/protocol/tcp/TCP_Reassembler.cc | 43 +---------- src/bro.bif | 38 ++++++--- src/file_analysis/FileReassembler.cc | 2 +- 14 files changed, 189 insertions(+), 117 deletions(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 94b6ed33e5..337052178d 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -462,34 +462,51 @@ type NetStats: record { ## .. note:: All process-level values refer to Bro's main process only, not to ## the child process it spawns for doing communication. type bro_resources: record { - version: string; ##< Bro version string. - debug: bool; ##< True if compiled with --enable-debug. - start_time: time; ##< Start time of process. - real_time: interval; ##< Elapsed real time since Bro started running. - user_time: interval; ##< User CPU seconds. - system_time: interval; ##< System CPU seconds. - mem: count; ##< Maximum memory consumed, in KB. - minor_faults: count; ##< Page faults not requiring actual I/O. - major_faults: count; ##< Page faults requiring actual I/O. - num_swap: count; ##< Times swapped out. - blocking_input: count; ##< Blocking input operations. - blocking_output: count; ##< Blocking output operations. - num_context: count; ##< Number of involuntary context switches. + version: string; ##< Bro version string. + debug: bool; ##< True if compiled with --enable-debug. + start_time: time; ##< Start time of process. + real_time: interval; ##< Elapsed real time since Bro started running. + user_time: interval; ##< User CPU seconds. + system_time: interval; ##< System CPU seconds. + mem: count; ##< Maximum memory consumed, in KB. + minor_faults: count; ##< Page faults not requiring actual I/O. + major_faults: count; ##< Page faults requiring actual I/O. + num_swap: count; ##< Times swapped out. + blocking_input: count; ##< Blocking input operations. + blocking_output: count; ##< Blocking output operations. + num_context: count; ##< Number of involuntary context switches. + + num_packets: count; ##< Total number of packets processed to date. + num_fragments: count; ##< Current number of fragments pending reassembly. + max_fragments: count; ##< Maximum number of concurrently buffered fragments so far. + + num_tcp_conns: count; ##< Current number of TCP connections in memory. + max_tcp_conns: count; ##< Maximum number of concurrent TCP connections so far. + cumulative_tcp_conns: count; ##< - num_TCP_conns: count; ##< Current number of TCP connections in memory. - num_UDP_conns: count; ##< Current number of UDP flows in memory. - num_ICMP_conns: count; ##< Current number of ICMP flows in memory. - num_fragments: count; ##< Current number of fragments pending reassembly. - num_packets: count; ##< Total number of packets processed to date. - num_timers: count; ##< Current number of pending timers. - num_events_queued: count; ##< Total number of events queued so far. - num_events_dispatched: count; ##< Total number of events dispatched so far. + num_udp_conns: count; ##< Current number of UDP flows in memory. + max_udp_conns: count; ##< Maximum number of concurrent UDP connections so far. + cumulative_udp_conns: count; ##< - max_TCP_conns: count; ##< Maximum number of concurrent TCP connections so far. - max_UDP_conns: count; ##< Maximum number of concurrent UDP connections so far. - max_ICMP_conns: count; ##< Maximum number of concurrent ICMP connections so far. - max_fragments: count; ##< Maximum number of concurrently buffered fragments so far. - max_timers: count; ##< Maximum number of concurrent timers pending so far. + num_icmp_conns: count; ##< Current number of ICMP flows in memory. + max_icmp_conns: count; ##< Maximum number of concurrent ICMP connections so far. + cumulative_icmp_conns: count; ##< + + num_timers: count; ##< Current number of pending timers. + max_timers: count; ##< Maximum number of concurrent timers pending so far. + + num_events_queued: count; ##< Total number of events queued so far. + num_events_dispatched: count; ##< Total number of events dispatched so far. + + total_conns: count; ##< + current_conns: count; ##< + current_conns_extern: count; ##< + sess_current_conns: count; ##< + + reassem_file_size: count; ##< Size of File reassembly tracking. + reassem_frag_size: count; ##< Size of Fragment reassembly tracking. + reassem_tcp_size: count; ##< Size of TCP reassembly tracking. + reassem_unknown_size: count; ##< Size of reassembly tracking for unknown purposes. }; ## Summary statistics of all regular expression matchers. @@ -507,7 +524,7 @@ type matcher_stats: record { ## Statistics about number of gaps in TCP connections. ## -## .. bro:see:: gap_report get_gap_summary +## .. bro:see:: get_gap_summary type gap_info: record { ack_events: count; ##< How many ack events *could* have had gaps. ack_bytes: count; ##< How many bytes those covered. @@ -3416,23 +3433,17 @@ global pkt_profile_file: file &redef; ## .. bro:see:: load_sample global load_sample_freq = 20 &redef; -## Rate at which to generate :bro:see:`gap_report` events assessing to what -## degree the measurement process appears to exhibit loss. -## -## .. bro:see:: gap_report -const gap_report_freq = 1.0 sec &redef; - ## Whether to attempt to automatically detect SYN/FIN/RST-filtered trace ## and not report missing segments for such connections. ## If this is enabled, then missing data at the end of connections may not ## be reported via :bro:see:`content_gap`. const detect_filtered_trace = F &redef; -## Whether we want :bro:see:`content_gap` and :bro:see:`gap_report` for partial +## Whether we want :bro:see:`content_gap` and :bro:see:`get_gap_summary` for partial ## connections. A connection is partial if it is missing a full handshake. Note ## that gap reports for partial connections might not be reliable. ## -## .. bro:see:: content_gap gap_report partial_connection +## .. bro:see:: content_gap get_gap_summary partial_connection const report_gaps_for_partial = F &redef; ## Flag to prevent Bro from exiting automatically when input is exhausted. diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index 215a3bb9de..484267898c 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -10,7 +10,7 @@ export { redef enum Log::ID += { LOG }; ## How often stats are reported. - const stats_report_interval = 1min &redef; + const stats_report_interval = 5min &redef; type Info: record { ## Timestamp for the measurement. @@ -27,6 +27,22 @@ export { ## interval. events_queued: count &log; + ## TCP connections seen since last stats interval. + tcp_conns: count &log; + ## UDP connections seen since last stats interval. + udp_conns: count &log; + ## ICMP connections seen since last stats interval. + icmp_conns: count &log; + + ## Current size of TCP data in reassembly. + reassem_tcp_size: count &log; + ## Current size of File data in reassembly. + reassem_file_size: count &log; + ## Current size of packet fragment data in reassembly. + reassem_frag_size: count &log; + ## Current size of unkown data in reassembly (this is only PIA buffer right now). + reassem_unknown_size: count &log; + ## Lag between the wall clock and packet timestamps if reading ## live traffic. lag: interval &log &optional; @@ -64,16 +80,27 @@ event check_stats(last_ts: time, last_ns: NetStats, last_res: bro_resources) # shutting down. return; - local info: Info = [$ts=now, $peer=peer_description, $mem=res$mem/1000000, + local info: Info = [$ts=now, + $peer=peer_description, + $mem=res$mem/1000000, $pkts_proc=res$num_packets - last_res$num_packets, $events_proc=res$num_events_dispatched - last_res$num_events_dispatched, - $events_queued=res$num_events_queued - last_res$num_events_queued]; + $events_queued=res$num_events_queued - last_res$num_events_queued, + $tcp_conns=res$cumulative_tcp_conns - last_res$cumulative_tcp_conns, + $udp_conns=res$cumulative_udp_conns - last_res$cumulative_udp_conns, + $icmp_conns=res$cumulative_icmp_conns - last_res$cumulative_icmp_conns, + $reassem_tcp_size=res$reassem_tcp_size, + $reassem_file_size=res$reassem_file_size, + $reassem_frag_size=res$reassem_frag_size, + $reassem_unknown_size=res$reassem_unknown_size + ]; + + # Someone's going to have to explain what this is and add a field to the Info record. + # info$util = 100.0*((res$user_time + res$system_time) - (last_res$user_time + last_res$system_time))/(now-last_ts); if ( reading_live_traffic() ) { info$lag = now - network_time(); - # Someone's going to have to explain what this is and add a field to the Info record. - # info$util = 100.0*((res$user_time + res$system_time) - (last_res$user_time + last_res$system_time))/(now-last_ts); info$pkts_recv = ns$pkts_recvd - last_ns$pkts_recvd; info$pkts_dropped = ns$pkts_dropped - last_ns$pkts_dropped; info$pkts_link = ns$pkts_link - last_ns$pkts_link; diff --git a/src/Dict.cc b/src/Dict.cc index 1d32eccde3..9e68d64089 100644 --- a/src/Dict.cc +++ b/src/Dict.cc @@ -66,6 +66,7 @@ Dictionary::Dictionary(dict_order ordering, int initial_size) delete_func = 0; tbl_next_ind = 0; + cumulative_entries = 0; num_buckets2 = num_entries2 = max_num_entries2 = thresh_entries2 = 0; den_thresh2 = 0; } @@ -444,6 +445,7 @@ void* Dictionary::Insert(DictEntry* new_entry, int copy_key) // on lists than prepending. chain->append(new_entry); + ++cumulative_entries; if ( *max_num_entries_ptr < ++*num_entries_ptr ) *max_num_entries_ptr = *num_entries_ptr; diff --git a/src/Dict.h b/src/Dict.h index 3a2239ef54..2def5ea28f 100644 --- a/src/Dict.h +++ b/src/Dict.h @@ -71,6 +71,12 @@ public: max_num_entries + max_num_entries2 : max_num_entries; } + // Total number of entries ever. + uint64 NumCumulativeInserts() const + { + return cumulative_entries; + } + // True if the dictionary is ordered, false otherwise. int IsOrdered() const { return order != 0; } @@ -166,6 +172,7 @@ private: int num_buckets; int num_entries; int max_num_entries; + uint64 cumulative_entries; double den_thresh; int thresh_entries; diff --git a/src/Event.cc b/src/Event.cc index 89e745361f..5d54752a5a 100644 --- a/src/Event.cc +++ b/src/Event.cc @@ -10,8 +10,8 @@ EventMgr mgr; -int num_events_queued = 0; -int num_events_dispatched = 0; +uint64 num_events_queued = 0; +uint64 num_events_dispatched = 0; Event::Event(EventHandlerPtr arg_handler, val_list* arg_args, SourceID arg_src, analyzer::ID arg_aid, TimerMgr* arg_mgr, diff --git a/src/Event.h b/src/Event.h index 6f9c9d10c3..0d004d526c 100644 --- a/src/Event.h +++ b/src/Event.h @@ -72,8 +72,8 @@ protected: Event* next_event; }; -extern int num_events_queued; -extern int num_events_dispatched; +extern uint64 num_events_queued; +extern uint64 num_events_dispatched; class EventMgr : public BroObj { public: diff --git a/src/Frag.cc b/src/Frag.cc index 6a8b901a73..842059e218 100644 --- a/src/Frag.cc +++ b/src/Frag.cc @@ -28,7 +28,7 @@ void FragTimer::Dispatch(double t, int /* is_expire */) FragReassembler::FragReassembler(NetSessions* arg_s, const IP_Hdr* ip, const u_char* pkt, HashKey* k, double t) - : Reassembler(0) + : Reassembler(0, REASSEM_FRAG) { s = arg_s; key = k; diff --git a/src/Reassem.cc b/src/Reassem.cc index 54f27bd895..35f491f8ed 100644 --- a/src/Reassem.cc +++ b/src/Reassem.cc @@ -1,6 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. #include +#include #include "bro-config.h" @@ -10,7 +11,8 @@ static const bool DEBUG_reassem = false; DataBlock::DataBlock(const u_char* data, uint64 size, uint64 arg_seq, - DataBlock* arg_prev, DataBlock* arg_next) + DataBlock* arg_prev, DataBlock* arg_next, + ReassemblerType reassem_type) { seq = arg_seq; upper = seq + size; @@ -26,17 +28,24 @@ DataBlock::DataBlock(const u_char* data, uint64 size, uint64 arg_seq, if ( next ) next->prev = this; + if ( Reassembler::sizes.size() == 0 ) + Reassembler::sizes.resize(REASSEM_TERM, 0); + + rtype = reassem_type; + Reassembler::sizes[rtype] += pad_size(size) + padded_sizeof(DataBlock); Reassembler::total_size += pad_size(size) + padded_sizeof(DataBlock); } uint64 Reassembler::total_size = 0; +std::vector Reassembler::sizes; -Reassembler::Reassembler(uint64 init_seq) +Reassembler::Reassembler(uint64 init_seq, ReassemblerType reassem_type) { blocks = last_block = 0; old_blocks = last_old_block = 0; total_old_blocks = max_old_blocks = 0; trim_seq = last_reassem_seq = init_seq; + rtype = reassem_type; } Reassembler::~Reassembler() @@ -110,7 +119,7 @@ void Reassembler::NewBlock(double t, uint64 seq, uint64 len, const u_char* data) if ( ! blocks ) blocks = last_block = start_block = - new DataBlock(data, len, seq, 0, 0); + new DataBlock(data, len, seq, 0, 0, rtype); else start_block = AddAndCheck(blocks, seq, upper_seq, data); @@ -275,7 +284,7 @@ DataBlock* Reassembler::AddAndCheck(DataBlock* b, uint64 seq, uint64 upper, if ( last_block && seq == last_block->upper ) { last_block = new DataBlock(data, upper - seq, seq, - last_block, 0); + last_block, 0, rtype); return last_block; } @@ -288,7 +297,7 @@ DataBlock* Reassembler::AddAndCheck(DataBlock* b, uint64 seq, uint64 upper, { // b is the last block, and it comes completely before // the new block. - last_block = new DataBlock(data, upper - seq, seq, b, 0); + last_block = new DataBlock(data, upper - seq, seq, b, 0, rtype); return last_block; } @@ -297,7 +306,7 @@ DataBlock* Reassembler::AddAndCheck(DataBlock* b, uint64 seq, uint64 upper, if ( upper <= b->seq ) { // The new block comes completely before b. - new_b = new DataBlock(data, upper - seq, seq, b->prev, b); + new_b = new DataBlock(data, upper - seq, seq, b->prev, b, rtype); if ( b == blocks ) blocks = new_b; return new_b; @@ -308,7 +317,7 @@ DataBlock* Reassembler::AddAndCheck(DataBlock* b, uint64 seq, uint64 upper, { // The new block has a prefix that comes before b. uint64 prefix_len = b->seq - seq; - new_b = new DataBlock(data, prefix_len, seq, b->prev, b); + new_b = new DataBlock(data, prefix_len, seq, b->prev, b, rtype); if ( b == blocks ) blocks = new_b; @@ -342,6 +351,17 @@ DataBlock* Reassembler::AddAndCheck(DataBlock* b, uint64 seq, uint64 upper, return new_b; } +uint64 Reassembler::MemoryAllocation(ReassemblerType rtype) + { + if (Reassembler::sizes.size() == 0 ) + Reassembler::sizes.resize(REASSEM_TERM, 0); + + if ( rtype < REASSEM_TERM ) + return Reassembler::sizes[rtype]; + else + return 0; + } + bool Reassembler::Serialize(SerialInfo* info) const { return SerialObj::Serialize(info); diff --git a/src/Reassem.h b/src/Reassem.h index e55c809990..d371b998bd 100644 --- a/src/Reassem.h +++ b/src/Reassem.h @@ -6,10 +6,23 @@ #include "Obj.h" #include "IPAddr.h" +// Whenever subclassing the Reassembler class +// you should add to this for known subclasses. +enum ReassemblerType { + REASSEM_UNKNOWN, + REASSEM_TCP, + REASSEM_FRAG, + REASSEM_FILE, + + // Terminal value. Add new above. + REASSEM_TERM, +}; + class DataBlock { public: DataBlock(const u_char* data, uint64 size, uint64 seq, - DataBlock* prev, DataBlock* next); + DataBlock* prev, DataBlock* next, + ReassemblerType reassem_type = REASSEM_UNKNOWN); ~DataBlock(); @@ -19,13 +32,12 @@ public: DataBlock* prev; // previous block with lower seq # uint64 seq, upper; u_char* block; + ReassemblerType rtype; }; - - class Reassembler : public BroObj { public: - Reassembler(uint64 init_seq); + Reassembler(uint64 init_seq, ReassemblerType reassem_type = REASSEM_UNKNOWN); virtual ~Reassembler(); void NewBlock(double t, uint64 seq, uint64 len, const u_char* data); @@ -51,6 +63,9 @@ public: // Sum over all data buffered in some reassembler. static uint64 TotalMemoryAllocation() { return total_size; } + // Data buffered by type of reassembler. + static uint64 MemoryAllocation(ReassemblerType rtype); + void SetMaxOldBlocks(uint32 count) { max_old_blocks = count; } protected: @@ -82,12 +97,15 @@ protected: uint32 max_old_blocks; uint32 total_old_blocks; + ReassemblerType rtype; static uint64 total_size; + static std::vector sizes; }; inline DataBlock::~DataBlock() { Reassembler::total_size -= pad_size(upper - seq) + padded_sizeof(DataBlock); + Reassembler::sizes[rtype] -= pad_size(upper - seq) + padded_sizeof(DataBlock); delete [] block; } diff --git a/src/Sessions.cc b/src/Sessions.cc index b8bfe82b34..3194985515 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -1156,8 +1156,11 @@ void NetSessions::Drain() void NetSessions::GetStats(SessionStats& s) const { s.num_TCP_conns = tcp_conns.Length(); + s.cumulative_TCP_conns = tcp_conns.NumCumulativeInserts(); s.num_UDP_conns = udp_conns.Length(); + s.cumulative_UDP_conns = udp_conns.NumCumulativeInserts(); s.num_ICMP_conns = icmp_conns.Length(); + s.cumulative_ICMP_conns = icmp_conns.NumCumulativeInserts(); s.num_fragments = fragments.Length(); s.num_packets = num_packets_processed; s.num_timers = timer_mgr->Size(); diff --git a/src/Sessions.h b/src/Sessions.h index 2aca292789..e8c53256ff 100644 --- a/src/Sessions.h +++ b/src/Sessions.h @@ -32,19 +32,24 @@ namespace analyzer { namespace arp { class ARP_Analyzer; } } struct SessionStats { int num_TCP_conns; - int num_UDP_conns; - int num_ICMP_conns; - int num_fragments; - int num_packets; - int num_timers; - int num_events_queued; - int num_events_dispatched; - int max_TCP_conns; + uint64 cumulative_TCP_conns; + + int num_UDP_conns; int max_UDP_conns; + uint64 cumulative_UDP_conns; + + int num_ICMP_conns; int max_ICMP_conns; + uint64 cumulative_ICMP_conns; + + int num_fragments; int max_fragments; + uint64 num_packets; + int num_timers; int max_timers; + uint64 num_events_queued; + uint64 num_events_dispatched; }; // Drains and deletes a timer manager if it hasn't seen any advances @@ -242,7 +247,7 @@ protected: OSFingerprint* SYN_OS_Fingerprinter; int build_backdoor_analyzer; int dump_this_packet; // if true, current packet should be recorded - int num_packets_processed; + uint64 num_packets_processed; PacketProfiler* pkt_profiler; // We may use independent timer managers for different sets of related diff --git a/src/analyzer/protocol/tcp/TCP_Reassembler.cc b/src/analyzer/protocol/tcp/TCP_Reassembler.cc index 5b88d2dafb..0095947071 100644 --- a/src/analyzer/protocol/tcp/TCP_Reassembler.cc +++ b/src/analyzer/protocol/tcp/TCP_Reassembler.cc @@ -5,9 +5,6 @@ #include "analyzer/protocol/tcp/TCP.h" #include "TCP_Endpoint.h" -// Only needed for gap_report events. -#include "Event.h" - #include "events.bif.h" using namespace analyzer::tcp; @@ -18,17 +15,11 @@ const bool DEBUG_tcp_contents = false; const bool DEBUG_tcp_connection_close = false; const bool DEBUG_tcp_match_undelivered = false; -static double last_gap_report = 0.0; -static uint64 last_ack_events = 0; -static uint64 last_ack_bytes = 0; -static uint64 last_gap_events = 0; -static uint64 last_gap_bytes = 0; - TCP_Reassembler::TCP_Reassembler(analyzer::Analyzer* arg_dst_analyzer, TCP_Analyzer* arg_tcp_analyzer, TCP_Reassembler::Type arg_type, TCP_Endpoint* arg_endp) - : Reassembler(1) + : Reassembler(1, REASSEM_TCP) { dst_analyzer = arg_dst_analyzer; tcp_analyzer = arg_tcp_analyzer; @@ -45,7 +36,7 @@ TCP_Reassembler::TCP_Reassembler(analyzer::Analyzer* arg_dst_analyzer, if ( tcp_max_old_segments ) SetMaxOldBlocks(tcp_max_old_segments); - if ( tcp_contents ) + if ( ::tcp_contents ) { // Val dst_port_val(ntohs(Conn()->RespPort()), TYPE_PORT); PortVal dst_port_val(ntohs(tcp_analyzer->Conn()->RespPort()), @@ -387,7 +378,6 @@ void TCP_Reassembler::BlockInserted(DataBlock* start_block) { // New stuff. uint64 len = b->Size(); uint64 seq = last_reassem_seq; - last_reassem_seq += len; if ( record_contents_file ) @@ -548,35 +538,6 @@ void TCP_Reassembler::AckReceived(uint64 seq) tot_gap_bytes += num_missing; tcp_analyzer->Event(ack_above_hole); } - - double dt = network_time - last_gap_report; - - if ( gap_report && gap_report_freq > 0.0 && - dt >= gap_report_freq ) - { - uint64 devents = tot_ack_events - last_ack_events; - uint64 dbytes = tot_ack_bytes - last_ack_bytes; - uint64 dgaps = tot_gap_events - last_gap_events; - uint64 dgap_bytes = tot_gap_bytes - last_gap_bytes; - - RecordVal* r = new RecordVal(gap_info); - r->Assign(0, new Val(devents, TYPE_COUNT)); - r->Assign(1, new Val(dbytes, TYPE_COUNT)); - r->Assign(2, new Val(dgaps, TYPE_COUNT)); - r->Assign(3, new Val(dgap_bytes, TYPE_COUNT)); - - val_list* vl = new val_list; - vl->append(new IntervalVal(dt, Seconds)); - vl->append(r); - - mgr.QueueEvent(gap_report, vl); - - last_gap_report = network_time; - last_ack_events = tot_ack_events; - last_ack_bytes = tot_ack_bytes; - last_gap_events = tot_gap_events; - last_gap_bytes = tot_gap_bytes; - } } // Check EOF here because t_reassem->LastReassemSeq() may have diff --git a/src/bro.bif b/src/bro.bif index b0465b9609..89e132ca24 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1763,20 +1763,38 @@ function resource_usage%(%): bro_resources #define ADD_STAT(x) \ res->Assign(n++, new Val(unsigned(sessions ? x : 0), TYPE_COUNT)); - ADD_STAT(s.num_TCP_conns); - ADD_STAT(s.num_UDP_conns); - ADD_STAT(s.num_ICMP_conns); - ADD_STAT(s.num_fragments); ADD_STAT(s.num_packets); - ADD_STAT(s.num_timers); - ADD_STAT(s.num_events_queued); - ADD_STAT(s.num_events_dispatched); - ADD_STAT(s.max_TCP_conns); - ADD_STAT(s.max_UDP_conns); - ADD_STAT(s.max_ICMP_conns); + ADD_STAT(s.num_fragments); ADD_STAT(s.max_fragments); + + ADD_STAT(s.num_TCP_conns); + ADD_STAT(s.max_TCP_conns); + ADD_STAT(s.cumulative_TCP_conns); + + ADD_STAT(s.num_UDP_conns); + ADD_STAT(s.max_UDP_conns); + ADD_STAT(s.cumulative_UDP_conns); + + ADD_STAT(s.num_ICMP_conns); + ADD_STAT(s.max_ICMP_conns); + ADD_STAT(s.cumulative_ICMP_conns); + + ADD_STAT(s.num_timers); ADD_STAT(s.max_timers); + ADD_STAT(s.mem); + ADD_STAT(s.num_events_dispatched); + + ADD_STAT(Connection::TotalConnections()); + ADD_STAT(Connection::CurrentConnections()); + ADD_STAT(Connection::CurrentExternalConnections()); + ADD_STAT(sessions->CurrentConnections()); + + ADD_STAT(Reassembler::MemoryAllocation(REASSEM_FILE)); + ADD_STAT(Reassembler::MemoryAllocation(REASSEM_FRAG)); + ADD_STAT(Reassembler::MemoryAllocation(REASSEM_TCP)); + ADD_STAT(Reassembler::MemoryAllocation(REASSEM_UNKNOWN)); + return res; %} diff --git a/src/file_analysis/FileReassembler.cc b/src/file_analysis/FileReassembler.cc index 8b678e5209..ba15086320 100644 --- a/src/file_analysis/FileReassembler.cc +++ b/src/file_analysis/FileReassembler.cc @@ -8,7 +8,7 @@ namespace file_analysis { class File; FileReassembler::FileReassembler(File *f, uint64 starting_offset) - : Reassembler(starting_offset), the_file(f), flushing(false) + : Reassembler(starting_offset, REASSEM_FILE), the_file(f), flushing(false) { } From 88517230b6e5b8239a476096f290040d335e6dea Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 4 Jan 2016 00:57:11 -0500 Subject: [PATCH 02/16] Fix memory usage collection on Mac OS X. - getrusage is broken on Mac OS X, but there is a Mach API available which can collect the same memory usage information. --- bro-config.h.in | 3 +++ src/util.cc | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/bro-config.h.in b/bro-config.h.in index 755a9eee98..0937950604 100644 --- a/bro-config.h.in +++ b/bro-config.h.in @@ -14,6 +14,9 @@ /* We are on a Linux system */ #cmakedefine HAVE_LINUX +/* We are on a Mac OS X (Darwin) system */ +#cmakedefine HAVE_DARWIN + /* Define if you have the `mallinfo' function. */ #cmakedefine HAVE_MALLINFO diff --git a/src/util.cc b/src/util.cc index 6a03859a3c..facbab295f 100644 --- a/src/util.cc +++ b/src/util.cc @@ -14,6 +14,11 @@ # endif #endif +#ifdef HAVE_DARWIN +#include +#include +#endif + #include #include #include @@ -1662,11 +1667,24 @@ void get_memory_usage(unsigned int* total, unsigned int* malloced) #endif +#ifdef HAVE_DARWIN + struct task_basic_info t_info; + mach_msg_type_number_t t_info_count = TASK_BASIC_INFO_COUNT; + + if ( KERN_SUCCESS != task_info(mach_task_self(), + TASK_BASIC_INFO, + (task_info_t)&t_info, + &t_info_count) ) + ret_total = 0; + else + ret_total = t_info.resident_size; +#else struct rusage r; getrusage(RUSAGE_SELF, &r); // In KB. ret_total = r.ru_maxrss * 1024; +#endif if ( total ) *total = ret_total; From 5a4859afe1f59321a354dd9b169d8931d8fb4de7 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 4 Jan 2016 00:59:26 -0500 Subject: [PATCH 03/16] Updating the cmake submodule for the stats updates. --- cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake b/cmake index 843cdf6a91..23773d7107 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 843cdf6a91f06e5407bffbc79a343bff3cf4c81f +Subproject commit 23773d7107e8d51e2b1bb0fd2e2d85fda50df743 From 13cf6e61122099c08aa6a156a629e6f8a6384514 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Tue, 5 Jan 2016 09:26:56 -0500 Subject: [PATCH 04/16] Fixing some small mistakes. --- scripts/base/init-bare.bro | 4 ++-- src/bro.bif | 25 ++++++++++--------------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 337052178d..f49bf89d18 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -475,11 +475,11 @@ type bro_resources: record { blocking_input: count; ##< Blocking input operations. blocking_output: count; ##< Blocking output operations. num_context: count; ##< Number of involuntary context switches. - + num_packets: count; ##< Total number of packets processed to date. num_fragments: count; ##< Current number of fragments pending reassembly. max_fragments: count; ##< Maximum number of concurrently buffered fragments so far. - + num_tcp_conns: count; ##< Current number of TCP connections in memory. max_tcp_conns: count; ##< Maximum number of concurrent TCP connections so far. cumulative_tcp_conns: count; ##< diff --git a/src/bro.bif b/src/bro.bif index 89e132ca24..948fc62684 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1766,34 +1766,29 @@ function resource_usage%(%): bro_resources ADD_STAT(s.num_packets); ADD_STAT(s.num_fragments); ADD_STAT(s.max_fragments); - ADD_STAT(s.num_TCP_conns); ADD_STAT(s.max_TCP_conns); ADD_STAT(s.cumulative_TCP_conns); - ADD_STAT(s.num_UDP_conns); ADD_STAT(s.max_UDP_conns); ADD_STAT(s.cumulative_UDP_conns); - ADD_STAT(s.num_ICMP_conns); ADD_STAT(s.max_ICMP_conns); ADD_STAT(s.cumulative_ICMP_conns); - ADD_STAT(s.num_timers); ADD_STAT(s.max_timers); - - ADD_STAT(s.mem); + ADD_STAT(s.num_events_queued); ADD_STAT(s.num_events_dispatched); - ADD_STAT(Connection::TotalConnections()); - ADD_STAT(Connection::CurrentConnections()); - ADD_STAT(Connection::CurrentExternalConnections()); - ADD_STAT(sessions->CurrentConnections()); - - ADD_STAT(Reassembler::MemoryAllocation(REASSEM_FILE)); - ADD_STAT(Reassembler::MemoryAllocation(REASSEM_FRAG)); - ADD_STAT(Reassembler::MemoryAllocation(REASSEM_TCP)); - ADD_STAT(Reassembler::MemoryAllocation(REASSEM_UNKNOWN)); + res->Assign(n++, new Val(unsigned(Connection::TotalConnections()), TYPE_COUNT)); + res->Assign(n++, new Val(unsigned(Connection::CurrentConnections()), TYPE_COUNT)); + res->Assign(n++, new Val(unsigned(Connection::CurrentExternalConnections()), TYPE_COUNT)); + res->Assign(n++, new Val(unsigned(sessions->CurrentConnections()), TYPE_COUNT)); + + res->Assign(n++, new Val(unsigned(Reassembler::MemoryAllocation(REASSEM_FILE)), TYPE_COUNT)); + res->Assign(n++, new Val(unsigned(Reassembler::MemoryAllocation(REASSEM_FRAG)), TYPE_COUNT)); + res->Assign(n++, new Val(unsigned(Reassembler::MemoryAllocation(REASSEM_TCP)), TYPE_COUNT)); + res->Assign(n++, new Val(unsigned(Reassembler::MemoryAllocation(REASSEM_UNKNOWN)), TYPE_COUNT)); return res; %} From 6aeeb94d760e9860b29eadcc52548721c9a3c630 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Wed, 6 Jan 2016 22:28:57 -0500 Subject: [PATCH 05/16] Slight change to Mach API for collecting memory usage. --- src/util.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/util.cc b/src/util.cc index facbab295f..9a4b4de9f6 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1664,15 +1664,14 @@ void get_memory_usage(unsigned int* total, unsigned int* malloced) if ( malloced ) *malloced = mi.uordblks; - #endif #ifdef HAVE_DARWIN - struct task_basic_info t_info; - mach_msg_type_number_t t_info_count = TASK_BASIC_INFO_COUNT; + struct mach_task_basic_info t_info; + mach_msg_type_number_t t_info_count = MACH_TASK_BASIC_INFO; if ( KERN_SUCCESS != task_info(mach_task_self(), - TASK_BASIC_INFO, + MACH_TASK_BASIC_INFO, (task_info_t)&t_info, &t_info_count) ) ret_total = 0; From 6d836b795648901558df09e3125fa40153f5c670 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 7 Jan 2016 16:20:24 -0500 Subject: [PATCH 06/16] More stats improvements Broke out the stats collection into a bunch of new Bifs in stats.bif. Scripts that use stats collection functions have also been updated. More work to do. --- .../frameworks/packet-filter/netstats.bro | 4 +- scripts/base/init-bare.bro | 123 +++++--- .../base/misc/find-checksum-offloading.bro | 2 +- scripts/policy/misc/capture-loss.bro | 2 +- scripts/policy/misc/stats.bro | 62 ++-- src/CMakeLists.txt | 1 + src/Conn.cc | 6 +- src/Conn.h | 12 +- src/DFA.cc | 17 +- src/DFA.h | 3 +- src/Func.cc | 20 +- src/NFA.cc | 5 - src/NFA.h | 1 - src/NetVar.cc | 1 - src/NetVar.h | 3 - src/Sessions.cc | 4 - src/Sessions.h | 4 - src/Stats.cc | 16 +- src/Stats.h | 8 +- src/analyzer/protocol/tcp/functions.bif | 20 -- src/bro.bif | 177 +---------- src/event.bif | 20 -- src/file_analysis/Manager.h | 9 + src/main.cc | 12 +- src/stats.bif | 293 ++++++++++++++++++ src/util.cc | 4 +- src/util.h | 3 +- 27 files changed, 479 insertions(+), 353 deletions(-) create mode 100644 src/stats.bif diff --git a/scripts/base/frameworks/packet-filter/netstats.bro b/scripts/base/frameworks/packet-filter/netstats.bro index b5ffe24f54..f1757d8d47 100644 --- a/scripts/base/frameworks/packet-filter/netstats.bro +++ b/scripts/base/frameworks/packet-filter/netstats.bro @@ -18,7 +18,7 @@ export { event net_stats_update(last_stat: NetStats) { - local ns = net_stats(); + local ns = get_net_stats(); local new_dropped = ns$pkts_dropped - last_stat$pkts_dropped; if ( new_dropped > 0 ) { @@ -38,5 +38,5 @@ event bro_init() # Since this currently only calculates packet drops, let's skip the stats # collection if reading traces. if ( ! reading_traces() ) - schedule stats_collection_interval { net_stats_update(net_stats()) }; + schedule stats_collection_interval { net_stats_update(get_net_stats()) }; } diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index f49bf89d18..fa9149c674 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -455,30 +455,15 @@ type NetStats: record { bytes_recvd: count &default=0; ##< Bytes received by Bro. }; -## Statistics about Bro's resource consumption. -## -## .. bro:see:: resource_usage -## -## .. note:: All process-level values refer to Bro's main process only, not to -## the child process it spawns for doing communication. -type bro_resources: record { - version: string; ##< Bro version string. - debug: bool; ##< True if compiled with --enable-debug. - start_time: time; ##< Start time of process. - real_time: interval; ##< Elapsed real time since Bro started running. - user_time: interval; ##< User CPU seconds. - system_time: interval; ##< System CPU seconds. - mem: count; ##< Maximum memory consumed, in KB. - minor_faults: count; ##< Page faults not requiring actual I/O. - major_faults: count; ##< Page faults requiring actual I/O. - num_swap: count; ##< Times swapped out. - blocking_input: count; ##< Blocking input operations. - blocking_output: count; ##< Blocking output operations. - num_context: count; ##< Number of involuntary context switches. +type ConnStats: record { + total_conns: count; ##< + current_conns: count; ##< + current_conns_extern: count; ##< + sess_current_conns: count; ##< - num_packets: count; ##< Total number of packets processed to date. - num_fragments: count; ##< Current number of fragments pending reassembly. - max_fragments: count; ##< Maximum number of concurrently buffered fragments so far. + num_packets: count; + num_fragments: count; + max_fragments: count; num_tcp_conns: count; ##< Current number of TCP connections in memory. max_tcp_conns: count; ##< Maximum number of concurrent TCP connections so far. @@ -492,46 +477,96 @@ type bro_resources: record { max_icmp_conns: count; ##< Maximum number of concurrent ICMP connections so far. cumulative_icmp_conns: count; ##< - num_timers: count; ##< Current number of pending timers. - max_timers: count; ##< Maximum number of concurrent timers pending so far. + killed_by_inactivity: count; +}; +## Statistics about Bro's process. +## +## .. bro:see:: get_proc_stats +## +## .. note:: All process-level values refer to Bro's main process only, not to +## the child process it spawns for doing communication. +type ProcStats: record { + debug: bool; ##< True if compiled with --enable-debug. + start_time: time; ##< Start time of process. + real_time: interval; ##< Elapsed real time since Bro started running. + user_time: interval; ##< User CPU seconds. + system_time: interval; ##< System CPU seconds. + mem: count; ##< Maximum memory consumed, in KB. + minor_faults: count; ##< Page faults not requiring actual I/O. + major_faults: count; ##< Page faults requiring actual I/O. + num_swap: count; ##< Times swapped out. + blocking_input: count; ##< Blocking input operations. + blocking_output: count; ##< Blocking output operations. + num_context: count; ##< Number of involuntary context switches. +}; + +type EventStats: record { num_events_queued: count; ##< Total number of events queued so far. num_events_dispatched: count; ##< Total number of events dispatched so far. +}; - total_conns: count; ##< - current_conns: count; ##< - current_conns_extern: count; ##< - sess_current_conns: count; ##< - - reassem_file_size: count; ##< Size of File reassembly tracking. - reassem_frag_size: count; ##< Size of Fragment reassembly tracking. - reassem_tcp_size: count; ##< Size of TCP reassembly tracking. - reassem_unknown_size: count; ##< Size of reassembly tracking for unknown purposes. +## Summary statistics of all regular expression matchers. +## +## .. bro:see:: get_reassembler_stats +type ReassemblerStats: record { + file_size: count; ##< Byte size of File reassembly tracking. + frag_size: count; ##< Byte size of Fragment reassembly tracking. + tcp_size: count; ##< Byte size of TCP reassembly tracking. + unknown_size: count; ##< Byte size of reassembly tracking for unknown purposes. }; ## Summary statistics of all regular expression matchers. ## ## .. bro:see:: get_matcher_stats -type matcher_stats: record { - matchers: count; ##< Number of distinct RE matchers. - dfa_states: count; ##< Number of DFA states across all matchers. - computed: count; ##< Number of computed DFA state transitions. - mem: count; ##< Number of bytes used by DFA states. - hits: count; ##< Number of cache hits. - misses: count; ##< Number of cache misses. - avg_nfa_states: count; ##< Average number of NFA states across all matchers. +type MatcherStats: record { + matchers: count; ##< Number of distinct RE matchers. + dfa_states: count; ##< Number of DFA states across all matchers. + computed: count; ##< Number of computed DFA state transitions. + mem: count; ##< Number of bytes used by DFA states. + hits: count; ##< Number of cache hits. + misses: count; ##< Number of cache misses. + avg_nfa_states: count; ##< Average number of NFA states across all matchers. +}; + +type TimerStats: record { + num_timers: count; ##< Current number of pending timers. + max_timers: count; ##< Maximum number of concurrent timers pending so far. +}; + +type FileAnalysisStats: record { + current: count; + max: count; + cumulative: count; +}; + +type DNSStats: record { + requests: count; + successful: count; + failed: count; + pending: count; + cached_hosts: count; + cached_addresses: count; }; ## Statistics about number of gaps in TCP connections. ## -## .. bro:see:: get_gap_summary -type gap_info: record { +## .. bro:see:: get_gap_stats +type GapStats: record { ack_events: count; ##< How many ack events *could* have had gaps. ack_bytes: count; ##< How many bytes those covered. gap_events: count; ##< How many *did* have gaps. gap_bytes: count; ##< How many bytes were missing in the gaps. }; +type PatternStats: record { + +}; + +type ThreadStats: record { + num_threads: count; +}; + ## Deprecated. ## ## .. todo:: Remove. It's still declared internally but doesn't seem used anywhere diff --git a/scripts/base/misc/find-checksum-offloading.bro b/scripts/base/misc/find-checksum-offloading.bro index fae017fff1..334cf4a2db 100644 --- a/scripts/base/misc/find-checksum-offloading.bro +++ b/scripts/base/misc/find-checksum-offloading.bro @@ -26,7 +26,7 @@ event ChecksumOffloading::check() if ( done ) return; - local pkts_recvd = net_stats()$pkts_recvd; + local pkts_recvd = get_net_stats()$pkts_recvd; local bad_ip_checksum_pct = (pkts_recvd != 0) ? (bad_ip_checksums*1.0 / pkts_recvd*1.0) : 0; local bad_tcp_checksum_pct = (pkts_recvd != 0) ? (bad_tcp_checksums*1.0 / pkts_recvd*1.0) : 0; local bad_udp_checksum_pct = (pkts_recvd != 0) ? (bad_udp_checksums*1.0 / pkts_recvd*1.0) : 0; diff --git a/scripts/policy/misc/capture-loss.bro b/scripts/policy/misc/capture-loss.bro index 28f468a1c8..648e3d6717 100644 --- a/scripts/policy/misc/capture-loss.bro +++ b/scripts/policy/misc/capture-loss.bro @@ -56,7 +56,7 @@ event CaptureLoss::take_measurement(last_ts: time, last_acks: count, last_gaps: } local now = network_time(); - local g = get_gap_summary(); + local g = get_gap_stats(); local acks = g$ack_events - last_acks; local gaps = g$gap_events - last_gaps; local pct_lost = (acks == 0) ? 0.0 : (100 * (1.0 * gaps) / (1.0 * acks)); diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index 484267898c..877d32130b 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -1,6 +1,4 @@ -##! Log memory/packet/lag statistics. Differs from -##! :doc:`/scripts/policy/misc/profiling.bro` in that this -##! is lighter-weight (much less info, and less load to generate). +##! Log memory/packet/lag statistics. @load base/frameworks/notice @@ -10,7 +8,7 @@ export { redef enum Log::ID += { LOG }; ## How often stats are reported. - const stats_report_interval = 5min &redef; + const stats_report_interval = 1sec &redef; type Info: record { ## Timestamp for the measurement. @@ -27,12 +25,19 @@ export { ## interval. events_queued: count &log; + ## TCP connections currently in memory. + active_tcp_conns: count &log; + ## UDP connections currently in memory. + active_udp_conns: count &log; + ## ICMP connections currently in memory. + active_icmp_conns: count &log; + ## TCP connections seen since last stats interval. - tcp_conns: count &log; + tcp_conns: count &log; ## UDP connections seen since last stats interval. - udp_conns: count &log; + udp_conns: count &log; ## ICMP connections seen since last stats interval. - icmp_conns: count &log; + icmp_conns: count &log; ## Current size of TCP data in reassembly. reassem_tcp_size: count &log; @@ -69,11 +74,14 @@ event bro_init() &priority=5 Log::create_stream(Stats::LOG, [$columns=Info, $ev=log_stats, $path="stats"]); } -event check_stats(last_ts: time, last_ns: NetStats, last_res: bro_resources) +event check_stats(last_ts: time, last_ns: NetStats, last_cs: ConnStats, last_ps: ProcStats, last_es: EventStats, last_rs: ReassemblerStats) { local now = current_time(); - local ns = net_stats(); - local res = resource_usage(); + local ns = get_net_stats(); + local cs = get_conn_stats(); + local ps = get_proc_stats(); + local es = get_event_stats(); + local rs = get_reassembler_stats(); if ( bro_is_terminating() ) # No more stats will be written or scheduled when Bro is @@ -82,21 +90,27 @@ event check_stats(last_ts: time, last_ns: NetStats, last_res: bro_resources) local info: Info = [$ts=now, $peer=peer_description, - $mem=res$mem/1000000, - $pkts_proc=res$num_packets - last_res$num_packets, - $events_proc=res$num_events_dispatched - last_res$num_events_dispatched, - $events_queued=res$num_events_queued - last_res$num_events_queued, - $tcp_conns=res$cumulative_tcp_conns - last_res$cumulative_tcp_conns, - $udp_conns=res$cumulative_udp_conns - last_res$cumulative_udp_conns, - $icmp_conns=res$cumulative_icmp_conns - last_res$cumulative_icmp_conns, - $reassem_tcp_size=res$reassem_tcp_size, - $reassem_file_size=res$reassem_file_size, - $reassem_frag_size=res$reassem_frag_size, - $reassem_unknown_size=res$reassem_unknown_size + $mem=ps$mem/1000000, + $pkts_proc=ns$pkts_recvd - last_ns$pkts_recvd, + + $active_tcp_conns=cs$num_tcp_conns, + $tcp_conns=cs$cumulative_tcp_conns - last_cs$cumulative_tcp_conns, + $active_udp_conns=cs$num_udp_conns, + $udp_conns=cs$cumulative_udp_conns - last_cs$cumulative_udp_conns, + $active_icmp_conns=cs$num_icmp_conns, + $icmp_conns=cs$cumulative_icmp_conns - last_cs$cumulative_icmp_conns, + + $reassem_tcp_size=rs$tcp_size, + $reassem_file_size=rs$file_size, + $reassem_frag_size=rs$frag_size, + $reassem_unknown_size=rs$unknown_size, + + $events_proc=es$num_events_dispatched - last_es$num_events_dispatched, + $events_queued=es$num_events_queued - last_es$num_events_queued ]; # Someone's going to have to explain what this is and add a field to the Info record. - # info$util = 100.0*((res$user_time + res$system_time) - (last_res$user_time + last_res$system_time))/(now-last_ts); + # info$util = 100.0*((ps$user_time + ps$system_time) - (last_ps$user_time + last_ps$system_time))/(now-last_ts); if ( reading_live_traffic() ) { @@ -108,10 +122,10 @@ event check_stats(last_ts: time, last_ns: NetStats, last_res: bro_resources) } Log::write(Stats::LOG, info); - schedule stats_report_interval { check_stats(now, ns, res) }; + schedule stats_report_interval { check_stats(now, ns, cs, ps, es, rs) }; } event bro_init() { - schedule stats_report_interval { check_stats(current_time(), net_stats(), resource_usage()) }; + schedule stats_report_interval { check_stats(current_time(), get_net_stats(), get_conn_stats(), get_proc_stats(), get_event_stats(), get_reassembler_stats()) }; } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9a807b3182..7b521125e4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -118,6 +118,7 @@ include(BifCl) set(BIF_SRCS bro.bif + stats.bif event.bif const.bif types.bif diff --git a/src/Conn.cc b/src/Conn.cc index 3f6757d89c..1082230869 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -108,9 +108,9 @@ bool ConnectionTimer::DoUnserialize(UnserialInfo* info) return true; } -unsigned int Connection::total_connections = 0; -unsigned int Connection::current_connections = 0; -unsigned int Connection::external_connections = 0; +uint64 Connection::total_connections = 0; +uint64 Connection::current_connections = 0; +uint64 Connection::external_connections = 0; IMPLEMENT_SERIAL(Connection, SER_CONNECTION); diff --git a/src/Conn.h b/src/Conn.h index 7a4331f91d..ffbc115e6e 100644 --- a/src/Conn.h +++ b/src/Conn.h @@ -220,11 +220,11 @@ public: unsigned int MemoryAllocation() const; unsigned int MemoryAllocationConnVal() const; - static unsigned int TotalConnections() + static uint64 TotalConnections() { return total_connections; } - static unsigned int CurrentConnections() + static uint64 CurrentConnections() { return current_connections; } - static unsigned int CurrentExternalConnections() + static uint64 CurrentExternalConnections() { return external_connections; } // Returns true if the history was already seen, false otherwise. @@ -315,9 +315,9 @@ protected: unsigned int saw_first_orig_packet:1, saw_first_resp_packet:1; // Count number of connections. - static unsigned int total_connections; - static unsigned int current_connections; - static unsigned int external_connections; + static uint64 total_connections; + static uint64 current_connections; + static uint64 external_connections; string history; uint32 hist_seen; diff --git a/src/DFA.cc b/src/DFA.cc index e7b2279ed5..9b8b3e5d31 100644 --- a/src/DFA.cc +++ b/src/DFA.cc @@ -9,6 +9,8 @@ unsigned int DFA_State::transition_counter = 0; +uint64 total_dfa_states = 0; + DFA_State::DFA_State(int arg_state_num, const EquivClass* ec, NFA_state_list* arg_nfa_states, AcceptingSet* arg_accept) @@ -20,6 +22,8 @@ DFA_State::DFA_State(int arg_state_num, const EquivClass* ec, mark = 0; centry = 0; + ++total_dfa_states; + SymPartition(ec); xtions = new DFA_State*[num_sym]; @@ -433,19 +437,6 @@ void DFA_Machine::Dump(FILE* f) start_state->ClearMarks(); } -void DFA_Machine::DumpStats(FILE* f) - { - DFA_State_Cache::Stats stats; - dfa_state_cache->GetStats(&stats); - - fprintf(f, "Computed dfa_states = %d; Classes = %d; Computed trans. = %d; Uncomputed trans. = %d\n", - stats.dfa_states, EC()->NumClasses(), - stats.computed, stats.uncomputed); - - fprintf(f, "DFA cache hits = %d; misses = %d\n", - stats.hits, stats.misses); - } - unsigned int DFA_Machine::MemoryAllocation() const { DFA_State_Cache::Stats s; diff --git a/src/DFA.h b/src/DFA.h index 00cfdc3d39..c329b929d4 100644 --- a/src/DFA.h +++ b/src/DFA.h @@ -19,6 +19,8 @@ class DFA_Machine; class DFA_State; struct CacheEntry; +extern uint64 total_dfa_states; + class DFA_State : public BroObj { public: DFA_State(int state_num, const EquivClass* ec, @@ -132,7 +134,6 @@ public: void Describe(ODesc* d) const; void Dump(FILE* f); - void DumpStats(FILE* f); unsigned int MemoryAllocation() const; diff --git a/src/Func.cc b/src/Func.cc index e1eadb8c9f..ac3cda6dd6 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -628,10 +628,12 @@ void builtin_error(const char* msg, BroObj* arg) } #include "bro.bif.func_h" +#include "stats.bif.func_h" #include "reporter.bif.func_h" #include "strings.bif.func_h" #include "bro.bif.func_def" +#include "stats.bif.func_def" #include "reporter.bif.func_def" #include "strings.bif.func_def" @@ -640,13 +642,23 @@ void builtin_error(const char* msg, BroObj* arg) void init_builtin_funcs() { - bro_resources = internal_type("bro_resources")->AsRecordType(); - net_stats = internal_type("NetStats")->AsRecordType(); - matcher_stats = internal_type("matcher_stats")->AsRecordType(); + ProcStats = internal_type("ProcStats")->AsRecordType(); + NetStats = internal_type("NetStats")->AsRecordType(); + MatcherStats = internal_type("MatcherStats")->AsRecordType(); + ConnStats = internal_type("ConnStats")->AsRecordType(); + ReassemblerStats = internal_type("ReassemblerStats")->AsRecordType(); + DNSStats = internal_type("DNSStats")->AsRecordType(); + GapStats = internal_type("GapStats")->AsRecordType(); + EventStats = internal_type("EventStats")->AsRecordType(); + TimerStats = internal_type("TimerStats")->AsRecordType(); + FileAnalysisStats = internal_type("FileAnalysisStats")->AsRecordType(); + ThreadStats = internal_type("ThreadStats")->AsRecordType(); + PatternStats = internal_type("PatternStats")->AsRecordType(); + var_sizes = internal_type("var_sizes")->AsTableType(); - gap_info = internal_type("gap_info")->AsRecordType(); #include "bro.bif.func_init" +#include "stats.bif.func_init" #include "reporter.bif.func_init" #include "strings.bif.func_init" diff --git a/src/NFA.cc b/src/NFA.cc index def04d79a1..4d18f75226 100644 --- a/src/NFA.cc +++ b/src/NFA.cc @@ -285,11 +285,6 @@ void NFA_Machine::Dump(FILE* f) first_state->ClearMarks(); } -void NFA_Machine::DumpStats(FILE* f) - { - fprintf(f, "highest NFA state ID is %d\n", nfa_state_id); - } - NFA_Machine* make_alternate(NFA_Machine* m1, NFA_Machine* m2) { if ( ! m1 ) diff --git a/src/NFA.h b/src/NFA.h index 9877b8787c..88ce3429c9 100644 --- a/src/NFA.h +++ b/src/NFA.h @@ -105,7 +105,6 @@ public: void Describe(ODesc* d) const; void Dump(FILE* f); - void DumpStats(FILE* f); unsigned int MemoryAllocation() const { return padded_sizeof(*this) + first_state->TotalMemoryAllocation(); } diff --git a/src/NetVar.cc b/src/NetVar.cc index 8a901842fd..457fcae0ce 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -197,7 +197,6 @@ Val* pkt_profile_file; int load_sample_freq; double gap_report_freq; -RecordType* gap_info; int packet_filter_default; diff --git a/src/NetVar.h b/src/NetVar.h index 97018121f9..582abffe65 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -200,9 +200,6 @@ extern Val* pkt_profile_file; extern int load_sample_freq; -extern double gap_report_freq; -extern RecordType* gap_info; - extern int packet_filter_default; extern int sig_max_group_size; diff --git a/src/Sessions.cc b/src/Sessions.cc index 3194985515..aae6712ef2 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -1163,15 +1163,11 @@ void NetSessions::GetStats(SessionStats& s) const s.cumulative_ICMP_conns = icmp_conns.NumCumulativeInserts(); s.num_fragments = fragments.Length(); s.num_packets = num_packets_processed; - s.num_timers = timer_mgr->Size(); - s.num_events_queued = num_events_queued; - s.num_events_dispatched = num_events_dispatched; s.max_TCP_conns = tcp_conns.MaxLength(); s.max_UDP_conns = udp_conns.MaxLength(); s.max_ICMP_conns = icmp_conns.MaxLength(); s.max_fragments = fragments.MaxLength(); - s.max_timers = timer_mgr->PeakSize(); } Connection* NetSessions::NewConn(HashKey* k, double t, const ConnID* id, diff --git a/src/Sessions.h b/src/Sessions.h index e8c53256ff..8da658633c 100644 --- a/src/Sessions.h +++ b/src/Sessions.h @@ -46,10 +46,6 @@ struct SessionStats { int num_fragments; int max_fragments; uint64 num_packets; - int num_timers; - int max_timers; - uint64 num_events_queued; - uint64 num_events_dispatched; }; // Drains and deletes a timer manager if it hasn't seen any advances diff --git a/src/Stats.cc b/src/Stats.cc index 00f603cba7..99e36625b8 100644 --- a/src/Stats.cc +++ b/src/Stats.cc @@ -14,7 +14,7 @@ #include "broker/Manager.h" #endif -int killed_by_inactivity = 0; +uint64 killed_by_inactivity = 0; uint64 tot_ack_events = 0; uint64 tot_ack_bytes = 0; @@ -82,7 +82,7 @@ void ProfileLogger::Log() struct timeval tv_utime = r.ru_utime; struct timeval tv_stime = r.ru_stime; - unsigned int total, malloced; + uint64 total, malloced; get_memory_usage(&total, &malloced); static unsigned int first_total = 0; @@ -110,7 +110,7 @@ void ProfileLogger::Log() file->Write(fmt("\n%.06f ------------------------\n", network_time)); } - file->Write(fmt("%.06f Memory: total=%dK total_adj=%dK malloced: %dK\n", + file->Write(fmt("%.06f Memory: total=%" PRId64 "K total_adj=%" PRId64 "K malloced: %" PRId64 "K\n", network_time, total / 1024, (total - first_total) / 1024, malloced / 1024)); @@ -120,7 +120,7 @@ void ProfileLogger::Log() int conn_mem_use = expensive ? sessions->ConnectionMemoryUsage() : 0; - file->Write(fmt("%.06f Conns: total=%d current=%d/%d ext=%d mem=%dK avg=%.1f table=%dK connvals=%dK\n", + file->Write(fmt("%.06f Conns: total=%" PRIu64 " current=%" PRIu64 "/%" PRIi32 " ext=%" PRIu64 " mem=%" PRIi32 "K avg=%.1f table=%" PRIu32 "K connvals=%" PRIu32 "K\n", network_time, Connection::TotalConnections(), Connection::CurrentConnections(), @@ -161,10 +161,10 @@ void ProfileLogger::Log() )); */ - file->Write(fmt("%.06f Connections expired due to inactivity: %d\n", + file->Write(fmt("%.06f Connections expired due to inactivity: %" PRIu64 "\n", network_time, killed_by_inactivity)); - file->Write(fmt("%.06f Total reassembler data: %" PRIu64"K\n", network_time, + file->Write(fmt("%.06f Total reassembler data: %" PRIu64 "K\n", network_time, Reassembler::TotalMemoryAllocation() / 1024)); // Signature engine. @@ -465,10 +465,10 @@ void PacketProfiler::ProfilePkt(double t, unsigned int bytes) double curr_Rtime = ptimestamp.tv_sec + ptimestamp.tv_usec / 1e6; - unsigned int curr_mem; + uint64 curr_mem; get_memory_usage(&curr_mem, 0); - file->Write(fmt("%.06f %.03f %d %d %.03f %.03f %.03f %d\n", + file->Write(fmt("%.06f %.03f %" PRIu64 " %" PRIu64 " %.03f %.03f %.03f %" PRIu64 "\n", t, time-last_timestamp, pkt_cnt, byte_cnt, curr_Rtime - last_Rtime, curr_Utime - last_Utime, diff --git a/src/Stats.h b/src/Stats.h index 1bcc2e18dc..7fbec8cab6 100644 --- a/src/Stats.h +++ b/src/Stats.h @@ -102,7 +102,7 @@ extern ProfileLogger* segment_logger; extern SampleLogger* sample_logger; // Connection statistics. -extern int killed_by_inactivity; +extern uint64 killed_by_inactivity; // Content gap statistics. extern uint64 tot_ack_events; @@ -127,9 +127,9 @@ protected: double update_freq; double last_Utime, last_Stime, last_Rtime; double last_timestamp, time; - unsigned int last_mem; - unsigned int pkt_cnt; - unsigned int byte_cnt; + uint64 last_mem; + uint64 pkt_cnt; + uint64 byte_cnt; }; #endif diff --git a/src/analyzer/protocol/tcp/functions.bif b/src/analyzer/protocol/tcp/functions.bif index 9fca05329a..75353180c6 100644 --- a/src/analyzer/protocol/tcp/functions.bif +++ b/src/analyzer/protocol/tcp/functions.bif @@ -63,26 +63,6 @@ function get_resp_seq%(cid: conn_id%): count } %} -## Returns statistics about TCP gaps. -## -## Returns: A record with TCP gap statistics. -## -## .. bro:see:: do_profiling -## net_stats -## resource_usage -## dump_rule_stats -## get_matcher_stats -function get_gap_summary%(%): gap_info - %{ - RecordVal* r = new RecordVal(gap_info); - r->Assign(0, new Val(tot_ack_events, TYPE_COUNT)); - r->Assign(1, new Val(tot_ack_bytes, TYPE_COUNT)); - r->Assign(2, new Val(tot_gap_events, TYPE_COUNT)); - r->Assign(3, new Val(tot_gap_bytes, TYPE_COUNT)); - - return r; - %} - ## Associates a file handle with a connection for writing TCP byte stream ## contents. ## diff --git a/src/bro.bif b/src/bro.bif index 948fc62684..ce16695afa 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -26,15 +26,8 @@ using namespace std; -RecordType* net_stats; -RecordType* bro_resources; -RecordType* matcher_stats; TableType* var_sizes; -// This one is extern, since it's used beyond just built-ins, -// and hence it's declared in NetVar.{h,cc}. -extern RecordType* gap_info; - static iosource::PktDumper* addl_pkt_dumper = 0; bro_int_t parse_int(const char*& fmt) @@ -1661,169 +1654,6 @@ function reading_traces%(%): bool return new Val(reading_traces, TYPE_BOOL); %} -## Returns packet capture statistics. Statistics include the number of -## packets *(i)* received by Bro, *(ii)* dropped, and *(iii)* seen on the -## link (not always available). -## -## Returns: A record of packet statistics. -## -## .. bro:see:: do_profiling -## resource_usage -## get_matcher_stats -## dump_rule_stats -## get_gap_summary -function net_stats%(%): NetStats - %{ - unsigned int recv = 0; - unsigned int drop = 0; - unsigned int link = 0; - unsigned int bytes_recv = 0; - - const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); - - for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); - i != pkt_srcs.end(); i++ ) - { - iosource::PktSrc* ps = *i; - - struct iosource::PktSrc::Stats stat; - ps->Statistics(&stat); - recv += stat.received; - drop += stat.dropped; - link += stat.link; - bytes_recv += stat.bytes_received; - } - - RecordVal* ns = new RecordVal(net_stats); - ns->Assign(0, new Val(recv, TYPE_COUNT)); - ns->Assign(1, new Val(drop, TYPE_COUNT)); - ns->Assign(2, new Val(link, TYPE_COUNT)); - ns->Assign(3, new Val(bytes_recv, TYPE_COUNT)); - - return ns; - %} - -## Returns Bro process statistics. Statistics include real/user/sys CPU time, -## memory usage, page faults, number of TCP/UDP/ICMP connections, timers, -## and events queued/dispatched. -## -## Returns: A record with resource usage statistics. -## -## .. bro:see:: do_profiling -## net_stats -## get_matcher_stats -## dump_rule_stats -## get_gap_summary -function resource_usage%(%): bro_resources - %{ - struct rusage r; - - if ( getrusage(RUSAGE_SELF, &r) < 0 ) - reporter->InternalError("getrusage() failed in bro_resource_usage()"); - - double elapsed_time = current_time() - bro_start_time; - - double user_time = - double(r.ru_utime.tv_sec) + double(r.ru_utime.tv_usec) / 1e6; - double system_time = - double(r.ru_stime.tv_sec) + double(r.ru_stime.tv_usec) / 1e6; - - RecordVal* res = new RecordVal(bro_resources); - int n = 0; - - res->Assign(n++, new StringVal(bro_version())); - -#ifdef DEBUG - res->Assign(n++, new Val(1, TYPE_COUNT)); -#else - res->Assign(n++, new Val(0, TYPE_COUNT)); -#endif - - res->Assign(n++, new Val(bro_start_time, TYPE_TIME)); - - res->Assign(n++, new IntervalVal(elapsed_time, Seconds)); - res->Assign(n++, new IntervalVal(user_time, Seconds)); - res->Assign(n++, new IntervalVal(system_time, Seconds)); - - unsigned int total_mem; - get_memory_usage(&total_mem, 0); - res->Assign(n++, new Val(unsigned(total_mem), TYPE_COUNT)); - - res->Assign(n++, new Val(unsigned(r.ru_minflt), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(r.ru_majflt), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(r.ru_nswap), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(r.ru_inblock), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(r.ru_oublock), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(r.ru_nivcsw), TYPE_COUNT)); - - SessionStats s; - if ( sessions ) - sessions->GetStats(s); - -#define ADD_STAT(x) \ - res->Assign(n++, new Val(unsigned(sessions ? x : 0), TYPE_COUNT)); - - ADD_STAT(s.num_packets); - ADD_STAT(s.num_fragments); - ADD_STAT(s.max_fragments); - ADD_STAT(s.num_TCP_conns); - ADD_STAT(s.max_TCP_conns); - ADD_STAT(s.cumulative_TCP_conns); - ADD_STAT(s.num_UDP_conns); - ADD_STAT(s.max_UDP_conns); - ADD_STAT(s.cumulative_UDP_conns); - ADD_STAT(s.num_ICMP_conns); - ADD_STAT(s.max_ICMP_conns); - ADD_STAT(s.cumulative_ICMP_conns); - ADD_STAT(s.num_timers); - ADD_STAT(s.max_timers); - ADD_STAT(s.num_events_queued); - ADD_STAT(s.num_events_dispatched); - - res->Assign(n++, new Val(unsigned(Connection::TotalConnections()), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(Connection::CurrentConnections()), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(Connection::CurrentExternalConnections()), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(sessions->CurrentConnections()), TYPE_COUNT)); - - res->Assign(n++, new Val(unsigned(Reassembler::MemoryAllocation(REASSEM_FILE)), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(Reassembler::MemoryAllocation(REASSEM_FRAG)), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(Reassembler::MemoryAllocation(REASSEM_TCP)), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(Reassembler::MemoryAllocation(REASSEM_UNKNOWN)), TYPE_COUNT)); - - return res; - %} - -## Returns statistics about the regular expression engine. Statistics include -## the number of distinct matchers, DFA states, DFA state transitions, memory -## usage of DFA states, cache hits/misses, and average number of NFA states -## across all matchers. -## -## Returns: A record with matcher statistics. -## -## .. bro:see:: do_profiling -## net_stats -## resource_usage -## dump_rule_stats -## get_gap_summary -function get_matcher_stats%(%): matcher_stats - %{ - RuleMatcher::Stats s; - memset(&s, 0, sizeof(s)); - - if ( rule_matcher ) - rule_matcher->GetStats(&s); - - RecordVal* r = new RecordVal(matcher_stats); - r->Assign(0, new Val(s.matchers, TYPE_COUNT)); - r->Assign(1, new Val(s.dfa_states, TYPE_COUNT)); - r->Assign(2, new Val(s.computed, TYPE_COUNT)); - r->Assign(3, new Val(s.mem, TYPE_COUNT)); - r->Assign(4, new Val(s.hits, TYPE_COUNT)); - r->Assign(5, new Val(s.misses, TYPE_COUNT)); - r->Assign(6, new Val(s.avg_nfa_states, TYPE_COUNT)); - - return r; - %} ## Generates a table of the size of all global variables. The table index is ## the variable name and the value is the variable size in bytes. @@ -1964,8 +1794,7 @@ function record_fields%(rec: any%): record_field_table ## .. bro:see:: net_stats ## resource_usage ## get_matcher_stats -## dump_rule_stats -## get_gap_summary +## get_gap_stats function do_profiling%(%) : any %{ if ( profiling_logger ) @@ -2030,8 +1859,8 @@ function is_local_interface%(ip: addr%) : bool ## .. bro:see:: do_profiling ## resource_usage ## get_matcher_stats -## net_stats -## get_gap_summary +## get_net_stats +## get_gap_stats ## ## .. todo:: The return value should be changed to any or check appropriately. function dump_rule_stats%(f: file%): bool diff --git a/src/event.bif b/src/event.bif index ff6ec059fb..aca1086e66 100644 --- a/src/event.bif +++ b/src/event.bif @@ -366,26 +366,6 @@ event ack_above_hole%(c: connection%); ## the two. event content_gap%(c: connection, is_orig: bool, seq: count, length: count%); -## Summarizes the amount of missing TCP payload at regular intervals. -## Internally, Bro tracks (1) the number of :bro:id:`ack_above_hole` events, -## including the number of bytes missing; and (2) the total number of TCP -## acks seen, with the total volume of bytes that have been acked. This event -## reports these statistics in :bro:id:`gap_report_freq` intervals for the -## purpose of determining packet loss. -## -## dt: The time that has passed since the last ``gap_report`` interval. -## -## info: The gap statistics. -## -## .. bro:see:: content_gap ack_above_hole -## -## .. note:: -## -## Bro comes with a script :doc:`/scripts/policy/misc/capture-loss.bro` that -## uses this event to estimate packet loss and report when a predefined -## threshold is exceeded. -event gap_report%(dt: interval, info: gap_info%); - ## Generated when a protocol analyzer confirms that a connection is indeed ## using that protocol. Bro's dynamic protocol detection heuristically activates ## analyzers as soon as it believes a connection *could* be using a particular diff --git a/src/file_analysis/Manager.h b/src/file_analysis/Manager.h index 93c8e7f613..bcc8ac5dd2 100644 --- a/src/file_analysis/Manager.h +++ b/src/file_analysis/Manager.h @@ -302,6 +302,15 @@ public: */ std::string DetectMIME(const u_char* data, uint64 len) const; + uint64 CurrentFiles() + { return id_map.Length(); } + + uint64 MaxFiles() + { return id_map.MaxLength(); } + + uint64 CumulativeFiles() + { return id_map.NumCumulativeInserts(); } + protected: friend class FileTimer; diff --git a/src/main.cc b/src/main.cc index 73181c82f2..a0615d75da 100644 --- a/src/main.cc +++ b/src/main.cc @@ -1172,8 +1172,8 @@ int main(int argc, char** argv) double time_net_start = current_time(true);; - unsigned int mem_net_start_total; - unsigned int mem_net_start_malloced; + uint64 mem_net_start_total; + uint64 mem_net_start_malloced; if ( time_bro ) { @@ -1181,7 +1181,7 @@ int main(int argc, char** argv) fprintf(stderr, "# initialization %.6f\n", time_net_start - time_start); - fprintf(stderr, "# initialization %uM/%uM\n", + fprintf(stderr, "# initialization %" PRIu64 "M/%" PRIu64 "M\n", mem_net_start_total / 1024 / 1024, mem_net_start_malloced / 1024 / 1024); } @@ -1190,8 +1190,8 @@ int main(int argc, char** argv) double time_net_done = current_time(true);; - unsigned int mem_net_done_total; - unsigned int mem_net_done_malloced; + uint64 mem_net_done_total; + uint64 mem_net_done_malloced; if ( time_bro ) { @@ -1200,7 +1200,7 @@ int main(int argc, char** argv) fprintf(stderr, "# total time %.6f, processing %.6f\n", time_net_done - time_start, time_net_done - time_net_start); - fprintf(stderr, "# total mem %uM/%uM, processing %uM/%uM\n", + fprintf(stderr, "# total mem %" PRId64 "M/%" PRId64 "M, processing %" PRId64 "M/%" PRId64 "M\n", mem_net_done_total / 1024 / 1024, mem_net_done_malloced / 1024 / 1024, (mem_net_done_total - mem_net_start_total) / 1024 / 1024, diff --git a/src/stats.bif b/src/stats.bif new file mode 100644 index 0000000000..d7e812df93 --- /dev/null +++ b/src/stats.bif @@ -0,0 +1,293 @@ + +%%{ // C segment +#include "util.h" +#include "threading/Manager.h" + +RecordType* ProcStats; +RecordType* NetStats; +RecordType* MatcherStats; +RecordType* ReassemblerStats; +RecordType* DNSStats; +RecordType* ConnStats; +RecordType* GapStats; +RecordType* EventStats; +RecordType* ThreadStats; +RecordType* PatternStats; +RecordType* TimerStats; +RecordType* FileAnalysisStats; +%%} + +## Returns packet capture statistics. Statistics include the number of +## packets *(i)* received by Bro, *(ii)* dropped, and *(iii)* seen on the +## link (not always available). +## +## Returns: A record of packet statistics. +## +## .. bro:see:: do_profiling +## get_proc_stats +## get_matcher_stats +## get_gap_stats +function get_net_stats%(%): NetStats + %{ + uint64 recv = 0; + uint64 drop = 0; + uint64 link = 0; + uint64 bytes_recv = 0; + + const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); + + for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); + i != pkt_srcs.end(); i++ ) + { + iosource::PktSrc* ps = *i; + + struct iosource::PktSrc::Stats stat; + ps->Statistics(&stat); + recv += stat.received; + drop += stat.dropped; + link += stat.link; + bytes_recv += stat.bytes_received; + } + + RecordVal* r = new RecordVal(NetStats); + int n = 0; + + r->Assign(n++, new Val(recv, TYPE_COUNT)); + r->Assign(n++, new Val(drop, TYPE_COUNT)); + r->Assign(n++, new Val(link, TYPE_COUNT)); + r->Assign(n++, new Val(bytes_recv, TYPE_COUNT)); + + return r; + %} + +function get_conn_stats%(%): ConnStats + %{ + RecordVal* r = new RecordVal(ConnStats); + int n = 0; + + r->Assign(n++, new Val(Connection::TotalConnections(), TYPE_COUNT)); + r->Assign(n++, new Val(Connection::CurrentConnections(), TYPE_COUNT)); + r->Assign(n++, new Val(Connection::CurrentExternalConnections(), TYPE_COUNT)); + r->Assign(n++, new Val(sessions->CurrentConnections(), TYPE_COUNT)); + + SessionStats s; + if ( sessions ) + sessions->GetStats(s); + +#define ADD_STAT(x) \ + r->Assign(n++, new Val(unsigned(sessions ? x : 0), TYPE_COUNT)); + + ADD_STAT(s.num_packets); + ADD_STAT(s.num_fragments); + ADD_STAT(s.max_fragments); + ADD_STAT(s.num_TCP_conns); + ADD_STAT(s.max_TCP_conns); + ADD_STAT(s.cumulative_TCP_conns); + ADD_STAT(s.num_UDP_conns); + ADD_STAT(s.max_UDP_conns); + ADD_STAT(s.cumulative_UDP_conns); + ADD_STAT(s.num_ICMP_conns); + ADD_STAT(s.max_ICMP_conns); + ADD_STAT(s.cumulative_ICMP_conns); + + r->Assign(n++, new Val(killed_by_inactivity, TYPE_COUNT)); + + return r; + %} + +## Returns Bro process statistics. Statistics include real/user/sys CPU time, +## memory usage, page faults, number of TCP/UDP/ICMP connections, timers, +## and events queued/dispatched. +## +## Returns: A record with resource usage statistics. +## +## .. bro:see:: do_profiling +## get_net_stats +## get_matcher_stats +## get_gap_stats +function get_proc_stats%(%): ProcStats + %{ + struct rusage ru; + if ( getrusage(RUSAGE_SELF, &ru) < 0 ) + reporter->InternalError("getrusage() failed in get_proc_stats()"); + + RecordVal* r = new RecordVal(ProcStats); + int n = 0; + + double elapsed_time = current_time() - bro_start_time; + double user_time = + double(ru.ru_utime.tv_sec) + double(ru.ru_utime.tv_usec) / 1e6; + double system_time = + double(ru.ru_stime.tv_sec) + double(ru.ru_stime.tv_usec) / 1e6; + +#ifdef DEBUG + r->Assign(n++, new Val(1, TYPE_COUNT)); +#else + r->Assign(n++, new Val(0, TYPE_COUNT)); +#endif + + r->Assign(n++, new Val(bro_start_time, TYPE_TIME)); + + r->Assign(n++, new IntervalVal(elapsed_time, Seconds)); + r->Assign(n++, new IntervalVal(user_time, Seconds)); + r->Assign(n++, new IntervalVal(system_time, Seconds)); + + uint64 total_mem; + get_memory_usage(&total_mem, NULL); + r->Assign(n++, new Val(unsigned(total_mem), TYPE_COUNT)); + + r->Assign(n++, new Val(unsigned(ru.ru_minflt), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(ru.ru_majflt), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(ru.ru_nswap), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(ru.ru_inblock), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(ru.ru_oublock), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(ru.ru_nivcsw), TYPE_COUNT)); + + return r; + %} + +function get_event_stats%(%): EventStats + %{ + RecordVal* r = new RecordVal(EventStats); + int n = 0; + + r->Assign(n++, new Val(num_events_queued, TYPE_COUNT)); + r->Assign(n++, new Val(num_events_dispatched, TYPE_COUNT)); + + return r; + %} + +function get_reassembler_stats%(%): ReassemblerStats + %{ + RecordVal* r = new RecordVal(ReassemblerStats); + int n = 0; + + r->Assign(n++, new Val(Reassembler::MemoryAllocation(REASSEM_FILE), TYPE_COUNT)); + r->Assign(n++, new Val(Reassembler::MemoryAllocation(REASSEM_FRAG), TYPE_COUNT)); + r->Assign(n++, new Val(Reassembler::MemoryAllocation(REASSEM_TCP), TYPE_COUNT)); + r->Assign(n++, new Val(Reassembler::MemoryAllocation(REASSEM_UNKNOWN), TYPE_COUNT)); + + return r; + %} + +function get_dns_stats%(%): DNSStats + %{ + RecordVal* r = new RecordVal(DNSStats); + int n = 0; + + DNS_Mgr::Stats dstats; + dns_mgr->GetStats(&dstats); + + r->Assign(n++, new Val(unsigned(dstats.requests), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(dstats.successful), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(dstats.failed), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(dstats.pending), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(dstats.cached_hosts), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(dstats.cached_addresses), TYPE_COUNT)); + + return r; + %} + +function get_pattern_stats%(%): PatternStats + %{ + RecordVal* r = new RecordVal(PatternStats); + int n = 0; + + //DFA_State_Cache::Stats stats; + //dfa_state_cache->GetStats(&stats); + + //fprintf(f, "Computed dfa_states = %d; Classes = %d; Computed trans. = %d; Uncomputed trans. = %d\n", + // stats.dfa_states, EC()->NumClasses(), + // stats.computed, stats.uncomputed); +// + //fprintf(f, "DFA cache hits = %d; misses = %d\n", + // stats.hits, stats.misses); + + return r; + + %} + +function get_timer_stats%(%): TimerStats + %{ + RecordVal* r = new RecordVal(TimerStats); + int n = 0; + + r->Assign(n++, new Val(unsigned(timer_mgr->Size()), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(timer_mgr->PeakSize()), TYPE_COUNT)); + + return r; + %} + +function get_file_analysis_stats%(%): FileAnalysisStats + %{ + RecordVal* r = new RecordVal(FileAnalysisStats); + int n = 0; + + r->Assign(n++, new Val(file_mgr->CurrentFiles(), TYPE_COUNT)); + r->Assign(n++, new Val(file_mgr->MaxFiles(), TYPE_COUNT)); + r->Assign(n++, new Val(file_mgr->CumulativeFiles(), TYPE_COUNT)); + + return r; + %} + +function get_thread_stats%(%): ThreadStats + %{ + RecordVal* r = new RecordVal(ThreadStats); + int n = 0; + + r->Assign(n++, new Val(thread_mgr->NumThreads(), TYPE_COUNT)); + + return r; + %} + +## Returns statistics about TCP gaps. +## +## Returns: A record with TCP gap statistics. +## +## .. bro:see:: do_profiling +## get_net_stats +## get_proc_stats +## get_matcher_stats +function get_gap_stats%(%): GapStats + %{ + RecordVal* r = new RecordVal(GapStats); + int n = 0; + + r->Assign(n++, new Val(tot_ack_events, TYPE_COUNT)); + r->Assign(n++, new Val(tot_ack_bytes, TYPE_COUNT)); + r->Assign(n++, new Val(tot_gap_events, TYPE_COUNT)); + r->Assign(n++, new Val(tot_gap_bytes, TYPE_COUNT)); + + return r; + %} + +## Returns statistics about the regular expression engine. Statistics include +## the number of distinct matchers, DFA states, DFA state transitions, memory +## usage of DFA states, cache hits/misses, and average number of NFA states +## across all matchers. +## +## Returns: A record with matcher statistics. +## +## .. bro:see:: get_net_stats +## get_proc_stats +## get_gap_summary +function get_matcher_stats%(%): MatcherStats + %{ + RecordVal* r = new RecordVal(MatcherStats); + int n = 0; + + RuleMatcher::Stats s; + memset(&s, 0, sizeof(s)); + if ( rule_matcher ) + rule_matcher->GetStats(&s); + + r->Assign(n++, new Val(s.matchers, TYPE_COUNT)); + r->Assign(n++, new Val(s.dfa_states, TYPE_COUNT)); + r->Assign(n++, new Val(s.computed, TYPE_COUNT)); + r->Assign(n++, new Val(s.mem, TYPE_COUNT)); + r->Assign(n++, new Val(s.hits, TYPE_COUNT)); + r->Assign(n++, new Val(s.misses, TYPE_COUNT)); + r->Assign(n++, new Val(s.avg_nfa_states, TYPE_COUNT)); + + return r; + %} diff --git a/src/util.cc b/src/util.cc index 9a4b4de9f6..a6ce473b6c 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1655,9 +1655,9 @@ extern "C" void out_of_memory(const char* where) abort(); } -void get_memory_usage(unsigned int* total, unsigned int* malloced) +void get_memory_usage(uint64* total, uint64* malloced) { - unsigned int ret_total; + uint64 ret_total; #ifdef HAVE_MALLINFO struct mallinfo mi = mallinfo(); diff --git a/src/util.h b/src/util.h index 901bb44d1c..191e5449e1 100644 --- a/src/util.h +++ b/src/util.h @@ -502,8 +502,7 @@ inline int safe_vsnprintf(char* str, size_t size, const char* format, va_list al // Returns total memory allocations and (if available) amount actually // handed out by malloc. -extern void get_memory_usage(unsigned int* total, - unsigned int* malloced); +extern void get_memory_usage(uint64* total, uint64* malloced); // Class to be used as a third argument for STL maps to be able to use // char*'s as keys. Otherwise the pointer values will be compared instead of From 3c71d4ffa8cc063915dd54c461395961368e3866 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Fri, 8 Jan 2016 17:03:16 -0500 Subject: [PATCH 07/16] More stats collection extensions. --- scripts/base/init-bare.bro | 5 +++-- scripts/policy/misc/stats.bro | 32 +++++++++++++++++++++++++------- src/PriorityQueue.cc | 3 ++- src/PriorityQueue.h | 3 +++ src/Timer.h | 3 +++ src/cq.c | 9 +++++++++ src/cq.h | 1 + src/stats.bif | 1 + 8 files changed, 47 insertions(+), 10 deletions(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index fa9149c674..3d870da38f 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -530,8 +530,9 @@ type MatcherStats: record { }; type TimerStats: record { - num_timers: count; ##< Current number of pending timers. - max_timers: count; ##< Maximum number of concurrent timers pending so far. + current: count; ##< Current number of pending timers. + max: count; ##< Maximum number of concurrent timers pending so far. + cumulative: count; }; type FileAnalysisStats: record { diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index 877d32130b..a49d377bae 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -39,6 +39,16 @@ export { ## ICMP connections seen since last stats interval. icmp_conns: count &log; + ## Number of timers scheduled since last stats interval. + timers: count &log; + ## Current number of scheduled timers. + active_timers: count &log; + + ## Number of files seen since last stats interval. + files: count &log; + ## Current number of files actively being seen. + active_files: count &log; + ## Current size of TCP data in reassembly. reassem_tcp_size: count &log; ## Current size of File data in reassembly. @@ -74,14 +84,16 @@ event bro_init() &priority=5 Log::create_stream(Stats::LOG, [$columns=Info, $ev=log_stats, $path="stats"]); } -event check_stats(last_ts: time, last_ns: NetStats, last_cs: ConnStats, last_ps: ProcStats, last_es: EventStats, last_rs: ReassemblerStats) +event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: ProcStats, last_es: EventStats, last_rs: ReassemblerStats, last_ts: TimerStats, last_fs: FileAnalysisStats) { - local now = current_time(); + local now = network_time(); local ns = get_net_stats(); local cs = get_conn_stats(); local ps = get_proc_stats(); local es = get_event_stats(); local rs = get_reassembler_stats(); + local ts = get_timer_stats(); + local fs = get_file_analysis_stats(); if ( bro_is_terminating() ) # No more stats will be written or scheduled when Bro is @@ -90,7 +102,7 @@ event check_stats(last_ts: time, last_ns: NetStats, last_cs: ConnStats, last_ps: local info: Info = [$ts=now, $peer=peer_description, - $mem=ps$mem/1000000, + $mem=ps$mem/1048576, $pkts_proc=ns$pkts_recvd - last_ns$pkts_recvd, $active_tcp_conns=cs$num_tcp_conns, @@ -106,11 +118,17 @@ event check_stats(last_ts: time, last_ns: NetStats, last_cs: ConnStats, last_ps: $reassem_unknown_size=rs$unknown_size, $events_proc=es$num_events_dispatched - last_es$num_events_dispatched, - $events_queued=es$num_events_queued - last_es$num_events_queued + $events_queued=es$num_events_queued - last_es$num_events_queued, + + $timers=ts$cumulative - last_ts$cumulative, + $active_timers=ts$current, + + $files=fs$cumulative - last_fs$cumulative, + $active_files=fs$current ]; # Someone's going to have to explain what this is and add a field to the Info record. - # info$util = 100.0*((ps$user_time + ps$system_time) - (last_ps$user_time + last_ps$system_time))/(now-last_ts); + # info$util = 100.0*((ps$user_time + ps$system_time) - (last_ps$user_time + last_ps$system_time))/(now-then); if ( reading_live_traffic() ) { @@ -122,10 +140,10 @@ event check_stats(last_ts: time, last_ns: NetStats, last_cs: ConnStats, last_ps: } Log::write(Stats::LOG, info); - schedule stats_report_interval { check_stats(now, ns, cs, ps, es, rs) }; + schedule stats_report_interval { check_stats(now, ns, cs, ps, es, rs, ts, fs) }; } event bro_init() { - schedule stats_report_interval { check_stats(current_time(), get_net_stats(), get_conn_stats(), get_proc_stats(), get_event_stats(), get_reassembler_stats()) }; + schedule stats_report_interval { check_stats(network_time(), get_net_stats(), get_conn_stats(), get_proc_stats(), get_event_stats(), get_reassembler_stats(), get_timer_stats(), get_file_analysis_stats()) }; } diff --git a/src/PriorityQueue.cc b/src/PriorityQueue.cc index 75b731142e..4f969c4830 100644 --- a/src/PriorityQueue.cc +++ b/src/PriorityQueue.cc @@ -13,7 +13,7 @@ PriorityQueue::PriorityQueue(int initial_size) { max_heap_size = initial_size; heap = new PQ_Element*[max_heap_size]; - peak_heap_size = heap_size = 0; + peak_heap_size = heap_size = cumulative_num = 0; } PriorityQueue::~PriorityQueue() @@ -62,6 +62,7 @@ int PriorityQueue::Add(PQ_Element* e) BubbleUp(heap_size); + ++cumulative_num; if ( ++heap_size > peak_heap_size ) peak_heap_size = heap_size; diff --git a/src/PriorityQueue.h b/src/PriorityQueue.h index 87e10aa7ac..bb1caad592 100644 --- a/src/PriorityQueue.h +++ b/src/PriorityQueue.h @@ -4,6 +4,7 @@ #define __PriorityQueue__ #include +#include "util.h" class PriorityQueue; @@ -53,6 +54,7 @@ public: int Size() const { return heap_size; } int PeakSize() const { return peak_heap_size; } + uint64 CumulativeNum() const { return cumulative_num; } protected: int Resize(int new_size); @@ -92,6 +94,7 @@ protected: int heap_size; int peak_heap_size; int max_heap_size; + uint64 cumulative_num; }; #endif diff --git a/src/Timer.h b/src/Timer.h index 615c8bf69a..12d849cac2 100644 --- a/src/Timer.h +++ b/src/Timer.h @@ -109,6 +109,7 @@ public: virtual int Size() const = 0; virtual int PeakSize() const = 0; + virtual uint64 CumulativeNum() const = 0; double LastTimestamp() const { return last_timestamp; } // Returns time of last advance in global network time. @@ -148,6 +149,7 @@ public: int Size() const { return q->Size(); } int PeakSize() const { return q->PeakSize(); } + uint64 CumulativeNum() const { return q->CumulativeNum(); } unsigned int MemoryUsage() const; protected: @@ -170,6 +172,7 @@ public: int Size() const { return cq_size(cq); } int PeakSize() const { return cq_max_size(cq); } + uint64 CumulativeNum() const { return cq_cumulative_num(cq); } unsigned int MemoryUsage() const; protected: diff --git a/src/cq.c b/src/cq.c index 8005544400..16153f0a39 100644 --- a/src/cq.c +++ b/src/cq.c @@ -42,6 +42,7 @@ struct cq_handle { int lowmark; /* low bucket threshold */ int nextbucket; /* next bucket to check */ int noresize; /* don't resize while we're resizing */ + uint64_t cumulative_num; /* cumulative entries ever enqueued */ double lastpri; /* last priority */ double ysize; /* length of a year */ double bwidth; /* width of each bucket */ @@ -175,6 +176,7 @@ cq_enqueue(register struct cq_handle *hp, register double pri, } bp->pri = pri; bp->cookie = cookie; + ++hp->cumulative_num; if (++hp->qlen > hp->max_qlen) hp->max_qlen = hp->qlen; #ifdef DEBUG @@ -414,6 +416,13 @@ cq_max_size(struct cq_handle *hp) return hp->max_qlen; } +uint64_t +cq_cumulative_num(struct cq_handle *hp) +{ + return hp->cumulative_num; +} + + /* Return without doing anything if we fail to allocate a new bucket array */ static int cq_resize(register struct cq_handle *hp, register int grow) diff --git a/src/cq.h b/src/cq.h index 540cccde74..c79eefc790 100644 --- a/src/cq.h +++ b/src/cq.h @@ -5,6 +5,7 @@ void *cq_dequeue(struct cq_handle *, double); void *cq_remove(struct cq_handle *, double, void *); int cq_size(struct cq_handle *); int cq_max_size(struct cq_handle *); +uint64_t cq_cumulative_num(struct cq_handle *); unsigned int cq_memory_allocation(void); #ifdef DEBUG void cq_debug(struct cq_handle *, int); diff --git a/src/stats.bif b/src/stats.bif index d7e812df93..3a975145b6 100644 --- a/src/stats.bif +++ b/src/stats.bif @@ -214,6 +214,7 @@ function get_timer_stats%(%): TimerStats r->Assign(n++, new Val(unsigned(timer_mgr->Size()), TYPE_COUNT)); r->Assign(n++, new Val(unsigned(timer_mgr->PeakSize()), TYPE_COUNT)); + r->Assign(n++, new Val(timer_mgr->CumulativeNum(), TYPE_COUNT)); return r; %} From cfdabb901fea7b904e5aaeedc2fc2617efb9de88 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Sat, 9 Jan 2016 01:14:13 -0500 Subject: [PATCH 08/16] Continued stats cleanup and extension. --- scripts/base/init-bare.bro | 22 ++++++------- scripts/policy/misc/stats.bro | 4 +-- src/DFA.cc | 5 +-- src/DFA.h | 7 ++-- src/Func.cc | 1 - src/RuleMatcher.cc | 9 ++---- src/RuleMatcher.h | 6 ++-- src/Stats.cc | 6 ++-- src/stats.bif | 60 +++++++++++++++++++++-------------- 9 files changed, 59 insertions(+), 61 deletions(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 3d870da38f..7b4f2c857f 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -502,8 +502,8 @@ type ProcStats: record { }; type EventStats: record { - num_events_queued: count; ##< Total number of events queued so far. - num_events_dispatched: count; ##< Total number of events dispatched so far. + queued: count; ##< Total number of events queued so far. + dispatched: count; ##< Total number of events dispatched so far. }; ## Summary statistics of all regular expression matchers. @@ -520,13 +520,13 @@ type ReassemblerStats: record { ## ## .. bro:see:: get_matcher_stats type MatcherStats: record { - matchers: count; ##< Number of distinct RE matchers. - dfa_states: count; ##< Number of DFA states across all matchers. - computed: count; ##< Number of computed DFA state transitions. - mem: count; ##< Number of bytes used by DFA states. - hits: count; ##< Number of cache hits. - misses: count; ##< Number of cache misses. - avg_nfa_states: count; ##< Average number of NFA states across all matchers. + matchers: count; ##< Number of distinct RE matchers. + nfa_states: count; ##< Number of NFA states across all matchers. + dfa_states: count; ##< Number of DFA states across all matchers. + computed: count; ##< Number of computed DFA state transitions. + mem: count; ##< Number of bytes used by DFA states. + hits: count; ##< Number of cache hits. + misses: count; ##< Number of cache misses. }; type TimerStats: record { @@ -560,10 +560,6 @@ type GapStats: record { gap_bytes: count; ##< How many bytes were missing in the gaps. }; -type PatternStats: record { - -}; - type ThreadStats: record { num_threads: count; }; diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index a49d377bae..a35ee4a90e 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -117,8 +117,8 @@ event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: Pr $reassem_frag_size=rs$frag_size, $reassem_unknown_size=rs$unknown_size, - $events_proc=es$num_events_dispatched - last_es$num_events_dispatched, - $events_queued=es$num_events_queued - last_es$num_events_queued, + $events_proc=es$dispatched - last_es$dispatched, + $events_queued=es$queued - last_es$queued, $timers=ts$cumulative - last_ts$cumulative, $active_timers=ts$current, diff --git a/src/DFA.cc b/src/DFA.cc index 9b8b3e5d31..5885a9bf3b 100644 --- a/src/DFA.cc +++ b/src/DFA.cc @@ -9,8 +9,6 @@ unsigned int DFA_State::transition_counter = 0; -uint64 total_dfa_states = 0; - DFA_State::DFA_State(int arg_state_num, const EquivClass* ec, NFA_state_list* arg_nfa_states, AcceptingSet* arg_accept) @@ -22,8 +20,6 @@ DFA_State::DFA_State(int arg_state_num, const EquivClass* ec, mark = 0; centry = 0; - ++total_dfa_states; - SymPartition(ec); xtions = new DFA_State*[num_sym]; @@ -350,6 +346,7 @@ DFA_State* DFA_State_Cache::Lookup(const NFA_state_list& nfas, ++misses; return 0; } + ++hits; delete *hash; *hash = 0; diff --git a/src/DFA.h b/src/DFA.h index c329b929d4..a63beca9ac 100644 --- a/src/DFA.h +++ b/src/DFA.h @@ -19,8 +19,6 @@ class DFA_Machine; class DFA_State; struct CacheEntry; -extern uint64 total_dfa_states; - class DFA_State : public BroObj { public: DFA_State(int state_num, const EquivClass* ec, @@ -91,10 +89,9 @@ public: int NumEntries() const { return states.Length(); } struct Stats { - unsigned int dfa_states; - - // Sum over all NFA states per DFA state. + // Sum of all NFA states unsigned int nfa_states; + unsigned int dfa_states; unsigned int computed; unsigned int uncomputed; unsigned int mem; diff --git a/src/Func.cc b/src/Func.cc index ac3cda6dd6..ccb2570f70 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -653,7 +653,6 @@ void init_builtin_funcs() TimerStats = internal_type("TimerStats")->AsRecordType(); FileAnalysisStats = internal_type("FileAnalysisStats")->AsRecordType(); ThreadStats = internal_type("ThreadStats")->AsRecordType(); - PatternStats = internal_type("PatternStats")->AsRecordType(); var_sizes = internal_type("var_sizes")->AsTableType(); diff --git a/src/RuleMatcher.cc b/src/RuleMatcher.cc index f40a5c4349..af4787086d 100644 --- a/src/RuleMatcher.cc +++ b/src/RuleMatcher.cc @@ -1174,7 +1174,7 @@ void RuleMatcher::GetStats(Stats* stats, RuleHdrTest* hdr_test) stats->mem = 0; stats->hits = 0; stats->misses = 0; - stats->avg_nfa_states = 0; + stats->nfa_states = 0; hdr_test = root; } @@ -1195,15 +1195,10 @@ void RuleMatcher::GetStats(Stats* stats, RuleHdrTest* hdr_test) stats->mem += cstats.mem; stats->hits += cstats.hits; stats->misses += cstats.misses; - stats->avg_nfa_states += cstats.nfa_states; + stats->nfa_states += cstats.nfa_states; } } - if ( stats->dfa_states ) - stats->avg_nfa_states /= stats->dfa_states; - else - stats->avg_nfa_states = 0; - for ( RuleHdrTest* h = hdr_test->child; h; h = h->sibling ) GetStats(stats, h); } diff --git a/src/RuleMatcher.h b/src/RuleMatcher.h index 6ffc971db1..b16a1556f9 100644 --- a/src/RuleMatcher.h +++ b/src/RuleMatcher.h @@ -297,6 +297,9 @@ public: struct Stats { unsigned int matchers; // # distinct RE matchers + // NFA states across all matchers. + unsigned int nfa_states; + // # DFA states across all matchers unsigned int dfa_states; unsigned int computed; // # computed DFA state transitions @@ -305,9 +308,6 @@ public: // # cache hits (sampled, multiply by MOVE_TO_FRONT_SAMPLE_SIZE) unsigned int hits; unsigned int misses; // # cache misses - - // Average # NFA states per DFA state. - unsigned int avg_nfa_states; }; Val* BuildRuleStateValue(const Rule* rule, diff --git a/src/Stats.cc b/src/Stats.cc index 99e36625b8..cf364d5747 100644 --- a/src/Stats.cc +++ b/src/Stats.cc @@ -173,9 +173,9 @@ void ProfileLogger::Log() RuleMatcher::Stats stats; rule_matcher->GetStats(&stats); - file->Write(fmt("%06f RuleMatcher: matchers=%d dfa_states=%d ncomputed=%d " - "mem=%dK avg_nfa_states=%d\n", network_time, stats.matchers, - stats.dfa_states, stats.computed, stats.mem / 1024, stats.avg_nfa_states)); + file->Write(fmt("%06f RuleMatcher: matchers=%d nfa_states=%d dfa_states=%d " + "ncomputed=%d mem=%dK\n", network_time, stats.matchers, + stats.nfa_states, stats.dfa_states, stats.computed, stats.mem / 1024)); } file->Write(fmt("%.06f Timers: current=%d max=%d mem=%dK lag=%.2fs\n", diff --git a/src/stats.bif b/src/stats.bif index 3a975145b6..ac8541182f 100644 --- a/src/stats.bif +++ b/src/stats.bif @@ -12,7 +12,6 @@ RecordType* ConnStats; RecordType* GapStats; RecordType* EventStats; RecordType* ThreadStats; -RecordType* PatternStats; RecordType* TimerStats; RecordType* FileAnalysisStats; %%} @@ -157,6 +156,13 @@ function get_event_stats%(%): EventStats return r; %} +## Returns statistics about reassembler usage. +## +## Returns: A record with reassembler statistics. +## +## .. bro:see:: get_net_stats +## get_proc_stats +## get_matcher_stats function get_reassembler_stats%(%): ReassemblerStats %{ RecordVal* r = new RecordVal(ReassemblerStats); @@ -170,6 +176,13 @@ function get_reassembler_stats%(%): ReassemblerStats return r; %} +## Returns statistics about DNS lookup activity. +## +## Returns: A record with DNS lookup statistics. +## +## .. bro:see:: get_net_stats +## get_proc_stats +## get_matcher_stats function get_dns_stats%(%): DNSStats %{ RecordVal* r = new RecordVal(DNSStats); @@ -188,25 +201,13 @@ function get_dns_stats%(%): DNSStats return r; %} -function get_pattern_stats%(%): PatternStats - %{ - RecordVal* r = new RecordVal(PatternStats); - int n = 0; - - //DFA_State_Cache::Stats stats; - //dfa_state_cache->GetStats(&stats); - - //fprintf(f, "Computed dfa_states = %d; Classes = %d; Computed trans. = %d; Uncomputed trans. = %d\n", - // stats.dfa_states, EC()->NumClasses(), - // stats.computed, stats.uncomputed); -// - //fprintf(f, "DFA cache hits = %d; misses = %d\n", - // stats.hits, stats.misses); - - return r; - - %} - +## Returns statistics about timer usage. +## +## Returns: A record with timer usage statistics. +## +## .. bro:see:: get_net_stats +## get_proc_stats +## get_matcher_stats function get_timer_stats%(%): TimerStats %{ RecordVal* r = new RecordVal(TimerStats); @@ -219,6 +220,13 @@ function get_timer_stats%(%): TimerStats return r; %} +## Returns statistics about file analysis. +## +## Returns: A record with file analysis statistics. +## +## .. bro:see:: get_net_stats +## get_proc_stats +## get_matcher_stats function get_file_analysis_stats%(%): FileAnalysisStats %{ RecordVal* r = new RecordVal(FileAnalysisStats); @@ -231,6 +239,13 @@ function get_file_analysis_stats%(%): FileAnalysisStats return r; %} +## Returns statistics about thread usage. +## +## Returns: A record with thread usage statistics. +## +## .. bro:see:: get_net_stats +## get_proc_stats +## get_matcher_stats function get_thread_stats%(%): ThreadStats %{ RecordVal* r = new RecordVal(ThreadStats); @@ -245,8 +260,7 @@ function get_thread_stats%(%): ThreadStats ## ## Returns: A record with TCP gap statistics. ## -## .. bro:see:: do_profiling -## get_net_stats +## .. bro:see:: get_net_stats ## get_proc_stats ## get_matcher_stats function get_gap_stats%(%): GapStats @@ -283,12 +297,12 @@ function get_matcher_stats%(%): MatcherStats rule_matcher->GetStats(&s); r->Assign(n++, new Val(s.matchers, TYPE_COUNT)); + r->Assign(n++, new Val(s.nfa_states, TYPE_COUNT)); r->Assign(n++, new Val(s.dfa_states, TYPE_COUNT)); r->Assign(n++, new Val(s.computed, TYPE_COUNT)); r->Assign(n++, new Val(s.mem, TYPE_COUNT)); r->Assign(n++, new Val(s.hits, TYPE_COUNT)); r->Assign(n++, new Val(s.misses, TYPE_COUNT)); - r->Assign(n++, new Val(s.avg_nfa_states, TYPE_COUNT)); return r; %} From 18a1e6f76b33732c84f54e3e4a07dc99bcce05ee Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 11 Jan 2016 09:25:36 -0500 Subject: [PATCH 09/16] Small stats script tweaks and beginning broker stats. --- scripts/policy/misc/stats.bro | 41 ++++++++++++++++------------------- src/stats.bif | 34 +++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 22 deletions(-) diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index a35ee4a90e..b43326e89d 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -19,6 +19,20 @@ export { mem: count &log; ## Number of packets processed since the last stats interval. pkts_proc: count &log; + ## Number of bytes received since the last stats interval if + ## reading live traffic. + bytes_recv: count &log; + + ## Number of packets dropped since the last stats interval if + ## reading live traffic. + pkts_dropped: count &log &optional; + ## Number of packets seen on the link since the last stats + ## interval if reading live traffic. + pkts_link: count &log &optional; + ## Lag between the wall clock and packet timestamps if reading + ## live traffic. + pkt_lag: interval &log &optional; + ## Number of events processed since the last stats interval. events_proc: count &log; ## Number of events that have been queued since the last stats @@ -57,22 +71,6 @@ export { reassem_frag_size: count &log; ## Current size of unkown data in reassembly (this is only PIA buffer right now). reassem_unknown_size: count &log; - - ## Lag between the wall clock and packet timestamps if reading - ## live traffic. - lag: interval &log &optional; - ## Number of packets received since the last stats interval if - ## reading live traffic. - pkts_recv: count &log &optional; - ## Number of packets dropped since the last stats interval if - ## reading live traffic. - pkts_dropped: count &log &optional; - ## Number of packets seen on the link since the last stats - ## interval if reading live traffic. - pkts_link: count &log &optional; - ## Number of bytes received since the last stats interval if - ## reading live traffic. - bytes_recv: count &log &optional; }; ## Event to catch stats as they are written to the logging stream. @@ -86,7 +84,7 @@ event bro_init() &priority=5 event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: ProcStats, last_es: EventStats, last_rs: ReassemblerStats, last_ts: TimerStats, last_fs: FileAnalysisStats) { - local now = network_time(); + local nettime = network_time(); local ns = get_net_stats(); local cs = get_conn_stats(); local ps = get_proc_stats(); @@ -100,10 +98,11 @@ event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: Pr # shutting down. return; - local info: Info = [$ts=now, + local info: Info = [$ts=nettime, $peer=peer_description, $mem=ps$mem/1048576, $pkts_proc=ns$pkts_recvd - last_ns$pkts_recvd, + $bytes_recv = ns$bytes_recvd - last_ns$bytes_recvd, $active_tcp_conns=cs$num_tcp_conns, $tcp_conns=cs$cumulative_tcp_conns - last_cs$cumulative_tcp_conns, @@ -132,15 +131,13 @@ event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: Pr if ( reading_live_traffic() ) { - info$lag = now - network_time(); - info$pkts_recv = ns$pkts_recvd - last_ns$pkts_recvd; + info$pkt_lag = current_time() - nettime; info$pkts_dropped = ns$pkts_dropped - last_ns$pkts_dropped; info$pkts_link = ns$pkts_link - last_ns$pkts_link; - info$bytes_recv = ns$bytes_recvd - last_ns$bytes_recvd; } Log::write(Stats::LOG, info); - schedule stats_report_interval { check_stats(now, ns, cs, ps, es, rs, ts, fs) }; + schedule stats_report_interval { check_stats(nettime, ns, cs, ps, es, rs, ts, fs) }; } event bro_init() diff --git a/src/stats.bif b/src/stats.bif index ac8541182f..2c5fd6151a 100644 --- a/src/stats.bif +++ b/src/stats.bif @@ -145,6 +145,13 @@ function get_proc_stats%(%): ProcStats return r; %} +## Returns statistics about the event engine. +## +## Returns: A record with event engine statistics. +## +## .. bro:see:: get_net_stats +## get_proc_stats +## get_matcher_stats function get_event_stats%(%): EventStats %{ RecordVal* r = new RecordVal(EventStats); @@ -306,3 +313,30 @@ function get_matcher_stats%(%): MatcherStats return r; %} + +function get_broker_stats%(%): BrokerStats + %{ + RecordVal* r = new RecordVal(CommunicationStats); + int n = 0; + +#ifdef ENABLE_BROKER + auto cs = broker_mgr->ConsumeStatistics(); + + r->Assign(n++, new Val(cs.outgoing_peer_count, TYPE_COUNT)); + r->Assign(n++, new Val(cs.data_store_count, TYPE_COUNT)); + r->Assign(n++, new Val(cs.pending_query_count, TYPE_COUNT)); + r->Assign(n++, new Val(cs.response_count, TYPE_COUNT)); + r->Assign(n++, new Val(cs.outgoing_conn_status_count, TYPE_COUNT)); + r->Assign(n++, new Val(cs.incoming_conn_status_count, TYPE_COUNT)); + r->Assign(n++, new Val(cs.report_count, TYPE_COUNT)); + + //for ( const auto& s : cs.print_count ) + // file->Write(fmt(" %-25s prints dequeued=%zu\n", s.first.data(), s.second)); + //for ( const auto& s : cs.event_count ) + // file->Write(fmt(" %-25s events dequeued=%zu\n", s.first.data(), s.second)); + //for ( const auto& s : cs.log_count ) + // file->Write(fmt(" %-25s logs dequeued=%zu\n", s.first.data(), s.second)); +#endif + + return r; + %} \ No newline at end of file From 16adf2ff5aeeaff4140abf5c960c15c2ccc7e1b0 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 14 Jan 2016 14:05:23 -0500 Subject: [PATCH 10/16] Add DNS stats to the stats.log --- scripts/policy/misc/stats.bro | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index b43326e89d..be84c5f35f 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -63,6 +63,11 @@ export { ## Current number of files actively being seen. active_files: count &log; + ## Number of DNS requests seen since last stats interval. + dns_requests: count &log; + ## Current number of DNS requests awaiting a reply. + active_dns_requests: count &log; + ## Current size of TCP data in reassembly. reassem_tcp_size: count &log; ## Current size of File data in reassembly. @@ -82,7 +87,7 @@ event bro_init() &priority=5 Log::create_stream(Stats::LOG, [$columns=Info, $ev=log_stats, $path="stats"]); } -event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: ProcStats, last_es: EventStats, last_rs: ReassemblerStats, last_ts: TimerStats, last_fs: FileAnalysisStats) +event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: ProcStats, last_es: EventStats, last_rs: ReassemblerStats, last_ts: TimerStats, last_fs: FileAnalysisStats, last_ds: DNSStats) { local nettime = network_time(); local ns = get_net_stats(); @@ -92,6 +97,7 @@ event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: Pr local rs = get_reassembler_stats(); local ts = get_timer_stats(); local fs = get_file_analysis_stats(); + local ds = get_dns_stats(); if ( bro_is_terminating() ) # No more stats will be written or scheduled when Bro is @@ -123,7 +129,10 @@ event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: Pr $active_timers=ts$current, $files=fs$cumulative - last_fs$cumulative, - $active_files=fs$current + $active_files=fs$current, + + $dns_requests=ds$requests - last_ds$requests, + $active_dns_requests=ds$pending ]; # Someone's going to have to explain what this is and add a field to the Info record. @@ -137,10 +146,10 @@ event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: Pr } Log::write(Stats::LOG, info); - schedule stats_report_interval { check_stats(nettime, ns, cs, ps, es, rs, ts, fs) }; + schedule stats_report_interval { check_stats(nettime, ns, cs, ps, es, rs, ts, fs, ds) }; } event bro_init() { - schedule stats_report_interval { check_stats(network_time(), get_net_stats(), get_conn_stats(), get_proc_stats(), get_event_stats(), get_reassembler_stats(), get_timer_stats(), get_file_analysis_stats()) }; + schedule stats_report_interval { check_stats(network_time(), get_net_stats(), get_conn_stats(), get_proc_stats(), get_event_stats(), get_reassembler_stats(), get_timer_stats(), get_file_analysis_stats(), get_dns_stats()) }; } From ee763381b25b0456a01ca40f826fb9c9b9ca9ef8 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 14 Jan 2016 16:17:41 -0500 Subject: [PATCH 11/16] Fixing default stats collection interval to every 5 minutes. --- scripts/policy/misc/stats.bro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index be84c5f35f..d154da05e9 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -8,7 +8,7 @@ export { redef enum Log::ID += { LOG }; ## How often stats are reported. - const stats_report_interval = 1sec &redef; + const stats_report_interval = 5min &redef; type Info: record { ## Timestamp for the measurement. From 6064134119bb119095dff60c6644114571850104 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 14 Jan 2016 16:49:18 -0500 Subject: [PATCH 12/16] Removing Broker stats, it was broken and incomplete. --- src/stats.bif | 58 +++++++++++++++++++++++++-------------------------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/src/stats.bif b/src/stats.bif index 2c5fd6151a..f5c8ee4308 100644 --- a/src/stats.bif +++ b/src/stats.bif @@ -22,8 +22,7 @@ RecordType* FileAnalysisStats; ## ## Returns: A record of packet statistics. ## -## .. bro:see:: do_profiling -## get_proc_stats +## .. bro:see:: get_proc_stats ## get_matcher_stats ## get_gap_stats function get_net_stats%(%): NetStats @@ -100,8 +99,7 @@ function get_conn_stats%(%): ConnStats ## ## Returns: A record with resource usage statistics. ## -## .. bro:see:: do_profiling -## get_net_stats +## .. bro:see:: get_net_stats ## get_matcher_stats ## get_gap_stats function get_proc_stats%(%): ProcStats @@ -314,29 +312,29 @@ function get_matcher_stats%(%): MatcherStats return r; %} -function get_broker_stats%(%): BrokerStats - %{ - RecordVal* r = new RecordVal(CommunicationStats); - int n = 0; - -#ifdef ENABLE_BROKER - auto cs = broker_mgr->ConsumeStatistics(); - - r->Assign(n++, new Val(cs.outgoing_peer_count, TYPE_COUNT)); - r->Assign(n++, new Val(cs.data_store_count, TYPE_COUNT)); - r->Assign(n++, new Val(cs.pending_query_count, TYPE_COUNT)); - r->Assign(n++, new Val(cs.response_count, TYPE_COUNT)); - r->Assign(n++, new Val(cs.outgoing_conn_status_count, TYPE_COUNT)); - r->Assign(n++, new Val(cs.incoming_conn_status_count, TYPE_COUNT)); - r->Assign(n++, new Val(cs.report_count, TYPE_COUNT)); - - //for ( const auto& s : cs.print_count ) - // file->Write(fmt(" %-25s prints dequeued=%zu\n", s.first.data(), s.second)); - //for ( const auto& s : cs.event_count ) - // file->Write(fmt(" %-25s events dequeued=%zu\n", s.first.data(), s.second)); - //for ( const auto& s : cs.log_count ) - // file->Write(fmt(" %-25s logs dequeued=%zu\n", s.first.data(), s.second)); -#endif - - return r; - %} \ No newline at end of file +# function get_broker_stats%(%): BrokerStats +# %{ +# RecordVal* r = new RecordVal(CommunicationStats); +# int n = 0; +# +# #ifdef ENABLE_BROKER +# auto cs = broker_mgr->ConsumeStatistics(); +# +# r->Assign(n++, new Val(cs.outgoing_peer_count, TYPE_COUNT)); +# r->Assign(n++, new Val(cs.data_store_count, TYPE_COUNT)); +# r->Assign(n++, new Val(cs.pending_query_count, TYPE_COUNT)); +# r->Assign(n++, new Val(cs.response_count, TYPE_COUNT)); +# r->Assign(n++, new Val(cs.outgoing_conn_status_count, TYPE_COUNT)); +# r->Assign(n++, new Val(cs.incoming_conn_status_count, TYPE_COUNT)); +# r->Assign(n++, new Val(cs.report_count, TYPE_COUNT)); +# +# //for ( const auto& s : cs.print_count ) +# // file->Write(fmt(" %-25s prints dequeued=%zu\n", s.first.data(), s.second)); +# //for ( const auto& s : cs.event_count ) +# // file->Write(fmt(" %-25s events dequeued=%zu\n", s.first.data(), s.second)); +# //for ( const auto& s : cs.log_count ) +# // file->Write(fmt(" %-25s logs dequeued=%zu\n", s.first.data(), s.second)); +# #endif +# +# return r; +# %} \ No newline at end of file From 53db5d1711e2652596e8660d40789296013f9a0e Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 14 Jan 2016 17:09:55 -0500 Subject: [PATCH 13/16] Removing some references to resource_usage() --- scripts/policy/frameworks/control/controllee.bro | 12 ++++++------ src/bro.bif | 2 -- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/scripts/policy/frameworks/control/controllee.bro b/scripts/policy/frameworks/control/controllee.bro index b4769764f4..6e3b5499b6 100644 --- a/scripts/policy/frameworks/control/controllee.bro +++ b/scripts/policy/frameworks/control/controllee.bro @@ -29,12 +29,12 @@ event Control::peer_status_request() if ( ! peer$connected ) next; - local res = resource_usage(); - status += fmt("%.6f peer=%s host=%s events_in=%s events_out=%s ops_in=%s ops_out=%s bytes_in=? bytes_out=?\n", - network_time(), - peer$peer$descr, peer$host, - res$num_events_queued, res$num_events_dispatched, - res$blocking_input, res$blocking_output); + #local res = resource_usage(); + #status += fmt("%.6f peer=%s host=%s events_in=%s events_out=%s ops_in=%s ops_out=%s bytes_in=? bytes_out=?\n", + # network_time(), + # peer$peer$descr, peer$host, + # res$num_events_queued, res$num_events_dispatched, + # res$blocking_input, res$blocking_output); } event Control::peer_status_response(status); diff --git a/src/bro.bif b/src/bro.bif index ce16695afa..5385a0e22f 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1792,7 +1792,6 @@ function record_fields%(rec: any%): record_field_table ## holds the name of the file. ## ## .. bro:see:: net_stats -## resource_usage ## get_matcher_stats ## get_gap_stats function do_profiling%(%) : any @@ -1857,7 +1856,6 @@ function is_local_interface%(ip: addr%) : bool ## Returns: True (unconditionally). ## ## .. bro:see:: do_profiling -## resource_usage ## get_matcher_stats ## get_net_stats ## get_gap_stats From 41a181d98d7afe06ae47255986fea26bde55cafe Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 14 Jan 2016 21:22:09 -0500 Subject: [PATCH 14/16] Removing more broken functionality due to changed stats apis. --- .../policy/frameworks/control/controllee.bro | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/scripts/policy/frameworks/control/controllee.bro b/scripts/policy/frameworks/control/controllee.bro index 6e3b5499b6..1a62d294b7 100644 --- a/scripts/policy/frameworks/control/controllee.bro +++ b/scripts/policy/frameworks/control/controllee.bro @@ -22,30 +22,30 @@ event Control::id_value_request(id: string) event Control::peer_status_request() { - local status = ""; - for ( p in Communication::nodes ) - { - local peer = Communication::nodes[p]; - if ( ! peer$connected ) - next; - - #local res = resource_usage(); - #status += fmt("%.6f peer=%s host=%s events_in=%s events_out=%s ops_in=%s ops_out=%s bytes_in=? bytes_out=?\n", - # network_time(), - # peer$peer$descr, peer$host, - # res$num_events_queued, res$num_events_dispatched, - # res$blocking_input, res$blocking_output); - } - - event Control::peer_status_response(status); + #local status = ""; + #for ( p in Communication::nodes ) + # { + # local peer = Communication::nodes[p]; + # if ( ! peer$connected ) + # next; + # + # #local res = resource_usage(); + # #status += fmt("%.6f peer=%s host=%s events_in=%s events_out=%s ops_in=%s ops_out=%s bytes_in=? bytes_out=?\n", + # # network_time(), + # # peer$peer$descr, peer$host, + # # res$num_events_queued, res$num_events_dispatched, + # # res$blocking_input, res$blocking_output); + # } + # + #event Control::peer_status_response(status); } event Control::net_stats_request() { - local ns = net_stats(); - local reply = fmt("%.6f recvd=%d dropped=%d link=%d\n", network_time(), - ns$pkts_recvd, ns$pkts_dropped, ns$pkts_link); - event Control::net_stats_response(reply); + #local ns = net_stats(); + #local reply = fmt("%.6f recvd=%d dropped=%d link=%d\n", network_time(), + # ns$pkts_recvd, ns$pkts_dropped, ns$pkts_link); + #event Control::net_stats_response(reply); } event Control::configuration_update_request() From da014e1eca6136ff729eb11aacdf11688bcbb64d Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 2 May 2016 16:20:53 -0400 Subject: [PATCH 15/16] Rename the reporting interval variable for stats. --- scripts/policy/misc/stats.bro | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index d154da05e9..50032f6ec4 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -8,7 +8,7 @@ export { redef enum Log::ID += { LOG }; ## How often stats are reported. - const stats_report_interval = 5min &redef; + const report_interval = 5min &redef; type Info: record { ## Timestamp for the measurement. @@ -146,10 +146,10 @@ event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: Pr } Log::write(Stats::LOG, info); - schedule stats_report_interval { check_stats(nettime, ns, cs, ps, es, rs, ts, fs, ds) }; + schedule report_interval { check_stats(nettime, ns, cs, ps, es, rs, ts, fs, ds) }; } event bro_init() { - schedule stats_report_interval { check_stats(network_time(), get_net_stats(), get_conn_stats(), get_proc_stats(), get_event_stats(), get_reassembler_stats(), get_timer_stats(), get_file_analysis_stats(), get_dns_stats()) }; + schedule report_interval { check_stats(network_time(), get_net_stats(), get_conn_stats(), get_proc_stats(), get_event_stats(), get_reassembler_stats(), get_timer_stats(), get_file_analysis_stats(), get_dns_stats()) }; } From f8f599832832e027b8019554eae2d430f2193251 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 2 May 2016 16:43:08 -0400 Subject: [PATCH 16/16] Fixing tests for stats improvements --- scripts/base/init-bare.bro | 44 ++++++++++++------- .../canonified_loaded_scripts.log | 5 ++- .../canonified_loaded_scripts.log | 5 ++- testing/btest/Baseline/plugins.hooks/output | 26 ++++++----- testing/btest/bifs/net_stats_trace.test | 2 +- testing/btest/bifs/resource_usage.bro | 9 ---- 6 files changed, 50 insertions(+), 41 deletions(-) delete mode 100644 testing/btest/bifs/resource_usage.bro diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index c433aae503..5430d52ba4 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -535,7 +535,7 @@ type ReassemblerStats: record { unknown_size: count; ##< Byte size of reassembly tracking for unknown purposes. }; -## Summary statistics of all regular expression matchers. +## Statistics of all regular expression matchers. ## ## .. bro:see:: get_matcher_stats type MatcherStats: record { @@ -548,37 +548,51 @@ type MatcherStats: record { misses: count; ##< Number of cache misses. }; +## Statistics of timers. +## +## .. bro:see:: get_timer_stats type TimerStats: record { current: count; ##< Current number of pending timers. max: count; ##< Maximum number of concurrent timers pending so far. - cumulative: count; + cumulative: count; ##< Cumulative number of timers scheduled. }; +## Statistics of file analysis. +## +## .. bro:see:: get_file_analysis_stats type FileAnalysisStats: record { - current: count; - max: count; - cumulative: count; + current: count; ##< Current number of files being analyzed. + max: count; ##< Maximum number of concurrent files so far. + cumulative: count; ##< Cumulative number of files analyzed. }; +## Statistics related to Bro's active use of DNS. These numbers are +## about Bro performing DNS queries on it's own, not traffic +## being seen. +## +## .. bro:see:: get_dns_stats type DNSStats: record { - requests: count; - successful: count; - failed: count; - pending: count; - cached_hosts: count; - cached_addresses: count; + requests: count; ##< Number of DNS requests made + successful: count; ##< Number of successful DNS replies. + failed: count; ##< Number of DNS reply failures. + pending: count; ##< Current pending queries. + cached_hosts: count; ##< Number of cached hosts. + cached_addresses: count; ##< Number of cached addresses. }; ## Statistics about number of gaps in TCP connections. ## ## .. bro:see:: get_gap_stats type GapStats: record { - ack_events: count; ##< How many ack events *could* have had gaps. - ack_bytes: count; ##< How many bytes those covered. - gap_events: count; ##< How many *did* have gaps. - gap_bytes: count; ##< How many bytes were missing in the gaps. + ack_events: count; ##< How many ack events *could* have had gaps. + ack_bytes: count; ##< How many bytes those covered. + gap_events: count; ##< How many *did* have gaps. + gap_bytes: count; ##< How many bytes were missing in the gaps. }; +## Statistics about threads. +## +## .. bro:see:: get_thread_stats type ThreadStats: record { num_threads: count; }; diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 703db6ea63..f3fbccdd52 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2016-04-26-18-11-39 +#open 2016-05-02-20-39-26 #fields name #types string scripts/base/init-bare.bro @@ -50,6 +50,7 @@ scripts/base/init-bare.bro scripts/base/utils/patterns.bro scripts/base/frameworks/files/magic/__load__.bro build/scripts/base/bif/__load__.bro + build/scripts/base/bif/stats.bif.bro build/scripts/base/bif/broxygen.bif.bro build/scripts/base/bif/functions.bif.bro build/scripts/base/bif/bloom-filter.bif.bro @@ -132,4 +133,4 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro scripts/policy/misc/loaded-scripts.bro scripts/base/utils/paths.bro -#close 2016-04-26-18-11-39 +#close 2016-05-02-20-39-26 diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index c7a3c03d09..37cfa6ff28 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2016-04-26-18-11-49 +#open 2016-05-02-20-39-35 #fields name #types string scripts/base/init-bare.bro @@ -50,6 +50,7 @@ scripts/base/init-bare.bro scripts/base/utils/patterns.bro scripts/base/frameworks/files/magic/__load__.bro build/scripts/base/bif/__load__.bro + build/scripts/base/bif/stats.bif.bro build/scripts/base/bif/broxygen.bif.bro build/scripts/base/bif/functions.bif.bro build/scripts/base/bif/bloom-filter.bif.bro @@ -305,4 +306,4 @@ scripts/base/init-default.bro scripts/base/misc/find-checksum-offloading.bro scripts/base/misc/find-filtered-trace.bro scripts/policy/misc/loaded-scripts.bro -#close 2016-04-26-18-11-49 +#close 2016-05-02-20-39-35 diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 61099efaf9..186fc55040 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -233,7 +233,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1462221741.258723, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Communication::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Conn::LOG)) -> @@ -354,7 +354,7 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1462221741.258723, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(NetControl::check_plugins, , ()) -> 0.000000 MetaHookPost CallFunction(NetControl::init, , ()) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, , ()) -> @@ -527,6 +527,7 @@ 0.000000 MetaHookPost LoadFile(./sftp) -> -1 0.000000 MetaHookPost LoadFile(./shunt) -> -1 0.000000 MetaHookPost LoadFile(./site) -> -1 +0.000000 MetaHookPost LoadFile(./stats.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./std-dev) -> -1 0.000000 MetaHookPost LoadFile(./store) -> -1 0.000000 MetaHookPost LoadFile(./store.bif.bro) -> -1 @@ -882,7 +883,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1462221741.258723, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Communication::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Conn::LOG)) @@ -1003,7 +1004,7 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1462221741.258723, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(NetControl::check_plugins, , ()) 0.000000 MetaHookPre CallFunction(NetControl::init, , ()) 0.000000 MetaHookPre CallFunction(Notice::want_pp, , ()) @@ -1176,6 +1177,7 @@ 0.000000 MetaHookPre LoadFile(./sftp) 0.000000 MetaHookPre LoadFile(./shunt) 0.000000 MetaHookPre LoadFile(./site) +0.000000 MetaHookPre LoadFile(./stats.bif.bro) 0.000000 MetaHookPre LoadFile(./std-dev) 0.000000 MetaHookPre LoadFile(./store) 0.000000 MetaHookPre LoadFile(./store.bif.bro) @@ -1530,7 +1532,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1462221741.258723, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG) @@ -1651,7 +1653,7 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1462221741.258723, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction NetControl::check_plugins() 0.000000 | HookCallFunction NetControl::init() 0.000000 | HookCallFunction Notice::want_pp() @@ -1699,7 +1701,7 @@ 1362692526.869344 MetaHookPost CallFunction(ChecksumOffloading::check, , ()) -> 1362692526.869344 MetaHookPost CallFunction(NetControl::check_conn, , (141.142.228.5)) -> 1362692526.869344 MetaHookPost CallFunction(filter_change_tracking, , ()) -> -1362692526.869344 MetaHookPost CallFunction(net_stats, , ()) -> +1362692526.869344 MetaHookPost CallFunction(get_net_stats, , ()) -> 1362692526.869344 MetaHookPost CallFunction(new_connection, , ([id=[orig_h=141.142.228.5, orig_p=59856<...>/tcp], orig=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0], resp=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0], start_time=1362692526.869344, duration=0.0, service={}, history=, uid=CXWv6p3arKYeMETxOg, tunnel=, vlan=, inner_vlan=, dpd=, conn=, extract_orig=F, extract_resp=F, thresholds=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=])) -> 1362692526.869344 MetaHookPost DrainEvents() -> 1362692526.869344 MetaHookPost QueueEvent(ChecksumOffloading::check()) -> false @@ -1710,7 +1712,7 @@ 1362692526.869344 MetaHookPre CallFunction(ChecksumOffloading::check, , ()) 1362692526.869344 MetaHookPre CallFunction(NetControl::check_conn, , (141.142.228.5)) 1362692526.869344 MetaHookPre CallFunction(filter_change_tracking, , ()) -1362692526.869344 MetaHookPre CallFunction(net_stats, , ()) +1362692526.869344 MetaHookPre CallFunction(get_net_stats, , ()) 1362692526.869344 MetaHookPre CallFunction(new_connection, , ([id=[orig_h=141.142.228.5, orig_p=59856<...>/tcp], orig=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0], resp=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0], start_time=1362692526.869344, duration=0.0, service={}, history=, uid=CXWv6p3arKYeMETxOg, tunnel=, vlan=, inner_vlan=, dpd=, conn=, extract_orig=F, extract_resp=F, thresholds=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=])) 1362692526.869344 MetaHookPre DrainEvents() 1362692526.869344 MetaHookPre QueueEvent(ChecksumOffloading::check()) @@ -1722,7 +1724,7 @@ 1362692526.869344 | HookCallFunction ChecksumOffloading::check() 1362692526.869344 | HookCallFunction NetControl::check_conn(141.142.228.5) 1362692526.869344 | HookCallFunction filter_change_tracking() -1362692526.869344 | HookCallFunction net_stats() +1362692526.869344 | HookCallFunction get_net_stats() 1362692526.869344 | HookCallFunction new_connection([id=[orig_h=141.142.228.5, orig_p=59856<...>/tcp], orig=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0], resp=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0], start_time=1362692526.869344, duration=0.0, service={}, history=, uid=CXWv6p3arKYeMETxOg, tunnel=, vlan=, inner_vlan=, dpd=, conn=, extract_orig=F, extract_resp=F, thresholds=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=]) 1362692526.869344 | HookDrainEvents 1362692526.869344 | HookQueueEvent ChecksumOffloading::check() @@ -2127,11 +2129,11 @@ 1362692527.080972 MetaHookPost CallFunction(filter_change_tracking, , ()) -> 1362692527.080972 MetaHookPost CallFunction(fmt, , (%s:%d > %s:%d, 141.142.228.5, 59856<...>/tcp)) -> 1362692527.080972 MetaHookPost CallFunction(get_file_handle, , (Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], T)) -> +1362692527.080972 MetaHookPost CallFunction(get_net_stats, , ()) -> 1362692527.080972 MetaHookPost CallFunction(get_port_transport_proto, , (80/tcp)) -> 1362692527.080972 MetaHookPost CallFunction(id_string, , ([orig_h=141.142.228.5, orig_p=59856<...>/tcp])) -> 1362692527.080972 MetaHookPost CallFunction(is_tcp_port, , (59856/tcp)) -> 1362692527.080972 MetaHookPost CallFunction(net_done, , (1362692527.080972)) -> -1362692527.080972 MetaHookPost CallFunction(net_stats, , ()) -> 1362692527.080972 MetaHookPost CallFunction(reading_traces, , ()) -> 1362692527.080972 MetaHookPost CallFunction(set_file_handle, , (Analyzer::ANALYZER_HTTP1362692526.869344T11141.142.228.5:59856 > 192.150.187.43:80)) -> 1362692527.080972 MetaHookPost CallFunction(sub_bytes, , (HTTP, 0, 1)) -> @@ -2157,11 +2159,11 @@ 1362692527.080972 MetaHookPre CallFunction(filter_change_tracking, , ()) 1362692527.080972 MetaHookPre CallFunction(fmt, , (%s:%d > %s:%d, 141.142.228.5, 59856<...>/tcp)) 1362692527.080972 MetaHookPre CallFunction(get_file_handle, , (Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], T)) +1362692527.080972 MetaHookPre CallFunction(get_net_stats, , ()) 1362692527.080972 MetaHookPre CallFunction(get_port_transport_proto, , (80/tcp)) 1362692527.080972 MetaHookPre CallFunction(id_string, , ([orig_h=141.142.228.5, orig_p=59856<...>/tcp])) 1362692527.080972 MetaHookPre CallFunction(is_tcp_port, , (59856/tcp)) 1362692527.080972 MetaHookPre CallFunction(net_done, , (1362692527.080972)) -1362692527.080972 MetaHookPre CallFunction(net_stats, , ()) 1362692527.080972 MetaHookPre CallFunction(reading_traces, , ()) 1362692527.080972 MetaHookPre CallFunction(set_file_handle, , (Analyzer::ANALYZER_HTTP1362692526.869344T11141.142.228.5:59856 > 192.150.187.43:80)) 1362692527.080972 MetaHookPre CallFunction(sub_bytes, , (HTTP, 0, 1)) @@ -2188,11 +2190,11 @@ 1362692527.080972 | HookCallFunction filter_change_tracking() 1362692527.080972 | HookCallFunction fmt(%s:%d > %s:%d, 141.142.228.5, 59856<...>/tcp) 1362692527.080972 | HookCallFunction get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], T) +1362692527.080972 | HookCallFunction get_net_stats() 1362692527.080972 | HookCallFunction get_port_transport_proto(80/tcp) 1362692527.080972 | HookCallFunction id_string([orig_h=141.142.228.5, orig_p=59856<...>/tcp]) 1362692527.080972 | HookCallFunction is_tcp_port(59856/tcp) 1362692527.080972 | HookCallFunction net_done(1362692527.080972) -1362692527.080972 | HookCallFunction net_stats() 1362692527.080972 | HookCallFunction reading_traces() 1362692527.080972 | HookCallFunction set_file_handle(Analyzer::ANALYZER_HTTP1362692526.869344T11141.142.228.5:59856 > 192.150.187.43:80) 1362692527.080972 | HookCallFunction sub_bytes(HTTP, 0, 1) diff --git a/testing/btest/bifs/net_stats_trace.test b/testing/btest/bifs/net_stats_trace.test index fcf3e9ba0d..cd9ee52a27 100644 --- a/testing/btest/bifs/net_stats_trace.test +++ b/testing/btest/bifs/net_stats_trace.test @@ -4,5 +4,5 @@ event bro_done() { - print net_stats(); + print get_net_stats(); } diff --git a/testing/btest/bifs/resource_usage.bro b/testing/btest/bifs/resource_usage.bro deleted file mode 100644 index 5cf3f0f962..0000000000 --- a/testing/btest/bifs/resource_usage.bro +++ /dev/null @@ -1,9 +0,0 @@ -# -# @TEST-EXEC: bro -b %INPUT - -event bro_init() - { - local a = resource_usage(); - if ( a$version != bro_version() ) - exit(1); - }