From c60015af22aa1f155a27f1dabcfe113035f66cf0 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Fri, 17 Jun 2011 09:02:50 -0400 Subject: [PATCH 1/9] Changed netstats (packet loss) handling to script-land. - Removed the net_stats_update event. - Created a net_stats function for building and retrieving the current network statistics. - Removed the internal timer for firing the net_stats_update event along with the global heartbeat_interval variable. - Updated the netstats script to use the new BiF. - Updated the stats script to use the new BiF. --- policy/bro.init | 18 ++++----- policy/netstats.bro | 40 +++++++++---------- policy/stats.bro | 94 ++++++++++----------------------------------- src/Func.cc | 1 + src/NetVar.cc | 6 --- src/NetVar.h | 3 -- src/Sessions.cc | 57 --------------------------- src/Sessions.h | 3 -- src/bro.bif | 26 +++++++++++++ src/event.bif | 1 - 10 files changed, 75 insertions(+), 174 deletions(-) diff --git a/policy/bro.init b/policy/bro.init index e25038e1e7..80cf2adfd4 100644 --- a/policy/bro.init +++ b/policy/bro.init @@ -105,6 +105,14 @@ type SYN_packet: record { SACK_OK: bool; }; +## This record is used for grabbing packet capturing information from +## the core with the :bro:id:`net_stats` BiF. All counts are cumulative. +type NetStats: record { + pkts_recvd: count &default=0; ##< Packets received by Bro. + pkts_dropped: count &default=0; ##< Packets dropped. + pkts_link: count &default=0; ##< Packets seen on the link (not always available). +}; + type bro_resources: record { version: string; # Bro version string debug: bool; # true if compiled with --enable-debug @@ -606,17 +614,7 @@ global discarder_check_udp: function(i: ip_hdr, u: udp_hdr, d: string): bool; global discarder_check_icmp: function(i: ip_hdr, ih: icmp_hdr): bool; # End of definition of access to packet headers, discarders. - -type net_stats: record { - # All counts are cumulative. - pkts_recvd: count; # pkts received by Bro - pkts_dropped: count; # pkts dropped - pkts_link: count; # pkts seen on link (not always available) -}; - - const watchdog_interval = 10 sec &redef; -const heartbeat_interval = 10 sec &redef; # The maximum number of timers to expire after processing each new # packet. The value trades off spreading out the timer expiration load diff --git a/policy/netstats.bro b/policy/netstats.bro index eca27cc9b5..606513bcd9 100644 --- a/policy/netstats.bro +++ b/policy/netstats.bro @@ -6,29 +6,27 @@ redef enum Notice += { DroppedPackets, # Bro reported packets dropped by the packet filter }; -global last_stat: net_stats; -global last_stat_time: time; -global have_stats = F; +const stats_collection_interval = 10secs; -event net_stats_update(t: time, ns: net_stats) +event net_stats_update(last_stat: NetStats) { - if ( have_stats ) + local ns = net_stats(); + local new_dropped = ns$pkts_dropped - last_stat$pkts_dropped; + if ( new_dropped > 0 ) { - local new_dropped = ns$pkts_dropped - last_stat$pkts_dropped; - if ( new_dropped > 0 ) - { - local new_recvd = ns$pkts_recvd - last_stat$pkts_recvd; - local new_link = ns$pkts_link - last_stat$pkts_link; - NOTICE([$note=DroppedPackets, - $msg=fmt("%d packets dropped after filtering, %d received%s", - new_dropped, new_recvd + new_dropped, - new_link != 0 ? - fmt(", %d on link", new_link) : "")]); - } + local new_recvd = ns$pkts_recvd - last_stat$pkts_recvd; + local new_link = ns$pkts_link - last_stat$pkts_link; + NOTICE([$note=DroppedPackets, + $msg=fmt("%d packets dropped after filtering, %d received%s", + new_dropped, new_recvd + new_dropped, + new_link != 0 ? + fmt(", %d on link", new_link) : "")]); } - else - have_stats = T; - - last_stat = ns; - last_stat_time = t; + + schedule stats_collection_interval { net_stats_update(ns) }; } + +event bro_init() + { + schedule stats_collection_interval { net_stats_update(net_stats()) }; + } \ No newline at end of file diff --git a/policy/stats.bro b/policy/stats.bro index b0a35c09a6..13c8548659 100644 --- a/policy/stats.bro +++ b/policy/stats.bro @@ -15,58 +15,47 @@ redef notice_action_filters += { [[ResourceStats, OfflineResourceStats]] = file_notice }; -global last_stats_time = current_time(); -global last_stats_CPU_time = - resource_usage()$user_time + resource_usage()$system_time; - -# Global to store the last net_stats object received. -global last_packet_stat: net_stats; - -# Globals to store the results between reporting intervals -global stat_packets_received = 0; -global stat_packets_dropped = 0; -global stat_packets_link = 0; - -global last_packets_processed = 0; -global last_events_dispatched = 0; -global last_events_queued = 0; - # Interval in which the results are sent as a notice. If this is less # than heartbeat_interval, then it is set to heartbeat_interval, since # some of the reported statistics are only gathered via the heartbeat. global stats_report_interval = 10 sec &redef; -event check_stats() +event check_stats(last_time: time, last_ns: NetStats, last_res: bro_resources) { local now = current_time(); local lag = now - network_time(); - local report_delta = now - last_stats_time; + local report_delta = now - last_time; local res = resource_usage(); - local mem = res$mem; + local ns = net_stats(); + local total_CPU_time = res$user_time + res$system_time; - local CPU_util = (total_CPU_time - last_stats_CPU_time) / report_delta; + local last_CPU_time = last_res$user_time + last_res$system_time; + local CPU_util = ((total_CPU_time - last_CPU_time) / report_delta) * 100.0; + + local pkts_recvd = ns$pkts_recvd - last_ns$pkts_recvd; + local pkts_dropped = ns$pkts_dropped - last_ns$pkts_dropped; + local pkts_link = ns$pkts_link - last_ns$pkts_link; if ( bro_is_terminating() ) # No more stats will be written or scheduled when Bro is # shutting down. return; - local delta_pkts_processed = res$num_packets - last_packets_processed; - local delta_events = res$num_events_dispatched - last_events_dispatched; - local delta_queued = res$num_events_queued - last_events_queued; + local delta_pkts_processed = res$num_packets - last_res$num_packets; + local delta_events = res$num_events_dispatched - last_res$num_events_dispatched; + local delta_queued = res$num_events_queued - last_res$num_events_queued; local stat_msg = fmt("mem=%dMB pkts_proc=%d events_proc=%d events_queued=%d", - mem / 1000000, delta_pkts_processed, + res$mem / 1000000, delta_pkts_processed, delta_events, delta_queued); if ( reading_live_traffic() ) { stat_msg = fmt("%s et=%.2f lag=%fsec util=%.01f%% pkts_rcv=%d pkts_drp=%d pkts_link=%d", - stat_msg, report_delta, lag, CPU_util * 100.0, - stat_packets_received, stat_packets_dropped, - stat_packets_link); + stat_msg, report_delta, lag, CPU_util, + pkts_recvd, pkts_dropped, pkts_link); NOTICE([$note=ResourceStats, $msg=stat_msg]); } @@ -77,57 +66,16 @@ event check_stats() { # Remote communication only. stat_msg = fmt("mem=%dMB events_proc=%d events_queued=%d lag=%fsec util=%.01f%%", - mem / 1000000, delta_events, delta_queued, - lag, CPU_util * 100.0 ); + res$mem / 1000000, delta_events, delta_queued, + lag, CPU_util); NOTICE([$note=ResourceStats, $msg=stat_msg]); } - last_stats_time = now; - last_stats_CPU_time = total_CPU_time; - last_packets_processed = res$num_packets; - last_events_dispatched = res$num_events_dispatched; - last_events_queued = res$num_events_queued; - - stat_packets_received = 0; - stat_packets_dropped = 0; - - schedule stats_report_interval { check_stats() }; - } - -event net_stats_update(t: time, ns: net_stats) - { - if ( ns$pkts_recvd > last_packet_stat$pkts_recvd ) - stat_packets_received += - ns$pkts_recvd - last_packet_stat$pkts_recvd; - - if ( ns$pkts_dropped > last_packet_stat$pkts_dropped ) - stat_packets_dropped += - ns$pkts_dropped - last_packet_stat$pkts_dropped; - - if ( ns$pkts_link > last_packet_stat$pkts_link ) - stat_packets_link += ns$pkts_link - last_packet_stat$pkts_link; - - last_packet_stat = ns; - } - -event start_check_stats() - { - # Can't start reporting data until network_time() is up. - local zero_time: time = 0; - - if ( network_time() > zero_time ) - schedule stats_report_interval { check_stats() }; - else - schedule stats_report_interval { start_check_stats() }; + print "did stats!"; + schedule stats_report_interval { check_stats(now, ns, res) }; } event bro_init() { - last_packet_stat$pkts_recvd = last_packet_stat$pkts_dropped = - last_packet_stat$pkts_link = 0; - - if ( stats_report_interval < heartbeat_interval ) - stats_report_interval = heartbeat_interval; - - schedule stats_report_interval { start_check_stats() }; + schedule stats_report_interval { check_stats(current_time(), net_stats(), resource_usage()) }; } diff --git a/src/Func.cc b/src/Func.cc index e4984d9788..8d588ce67d 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -514,6 +514,7 @@ void init_builtin_funcs() { ftp_port = internal_type("ftp_port")->AsRecordType(); bro_resources = internal_type("bro_resources")->AsRecordType(); + NetStats = internal_type("NetStats")->AsRecordType(); matcher_stats = internal_type("matcher_stats")->AsRecordType(); var_sizes = internal_type("var_sizes")->AsTableType(); gap_info = internal_type("gap_info")->AsRecordType(); diff --git a/src/NetVar.cc b/src/NetVar.cc index a8644fc059..e3afe7bf7e 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -19,10 +19,7 @@ RecordType* signature_state; EnumType* transport_proto; TableType* string_set; -RecordType* net_stats; - int watchdog_interval; -double heartbeat_interval; int max_timer_expires; int max_remote_events_processed; @@ -407,10 +404,7 @@ void init_net_var() ntp_session_timeout = opt_internal_double("ntp_session_timeout"); rpc_timeout = opt_internal_double("rpc_timeout"); - net_stats = internal_type("net_stats")->AsRecordType(); - watchdog_interval = int(opt_internal_double("watchdog_interval")); - heartbeat_interval = opt_internal_double("heartbeat_interval"); max_timer_expires = opt_internal_int("max_timer_expires"); max_remote_events_processed = diff --git a/src/NetVar.h b/src/NetVar.h index 2de1962f4d..1d250c1282 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -22,10 +22,7 @@ extern RecordType* pcap_packet; extern EnumType* transport_proto; extern TableType* string_set; -extern RecordType* net_stats; - extern int watchdog_interval; -extern double heartbeat_interval; extern int max_timer_expires; extern int max_remote_events_processed; diff --git a/src/Sessions.cc b/src/Sessions.cc index 1cbbbb272e..c28bcc01ca 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -44,27 +44,6 @@ enum NetBIOS_Service { NetSessions* sessions; - -class NetworkTimer : public Timer { -public: - NetworkTimer(NetSessions* arg_sess, double arg_t) - : Timer(arg_t, TIMER_NETWORK) - { sess = arg_sess; } - - void Dispatch(double t, int is_expire); - -protected: - NetSessions* sess; -}; - -void NetworkTimer::Dispatch(double t, int is_expire) - { - if ( is_expire ) - return; - - sess->HeartBeat(t); - } - void TimerMgrExpireTimer::Dispatch(double t, int is_expire) { if ( mgr->LastAdvance() + timer_mgr_inactivity_timeout < timer_mgr->Time() ) @@ -106,9 +85,6 @@ NetSessions::NetSessions() udp_conns.SetDeleteFunc(bro_obj_delete_func); fragments.SetDeleteFunc(bro_obj_delete_func); - if ( (reading_live || pseudo_realtime) && net_stats_update ) - timer_mgr->Add(new NetworkTimer(this, 1.0)); - if ( stp_correlate_pair ) stp_manager = new SteppingStoneManager(); else @@ -1085,39 +1061,6 @@ void NetSessions::Drain() ExpireTimerMgrs(); } -void NetSessions::HeartBeat(double t) - { - unsigned int recv = 0; - unsigned int drop = 0; - unsigned int link = 0; - - loop_over_list(pkt_srcs, i) - { - PktSrc* ps = pkt_srcs[i]; - - struct PktSrc::Stats stat; - ps->Statistics(&stat); - recv += stat.received; - drop += stat.dropped; - link += stat.link; - } - - val_list* vl = new val_list; - - vl->append(new Val(t, TYPE_TIME)); - - RecordVal* ns = new RecordVal(net_stats); - ns->Assign(0, new Val(recv, TYPE_COUNT)); - ns->Assign(1, new Val(drop, TYPE_COUNT)); - ns->Assign(2, new Val(link, TYPE_COUNT)); - - vl->append(ns); - - mgr.QueueEvent(net_stats_update, vl); - - timer_mgr->Add(new NetworkTimer(this, t + heartbeat_interval)); - } - void NetSessions::GetStats(SessionStats& s) const { s.num_TCP_conns = tcp_conns.Length(); diff --git a/src/Sessions.h b/src/Sessions.h index a85af005f4..448ecf70dd 100644 --- a/src/Sessions.h +++ b/src/Sessions.h @@ -105,9 +105,6 @@ public: // that are still active. void Drain(); - // Called periodically to generate statistics reports. - void HeartBeat(double t); - void GetStats(SessionStats& s) const; void Weird(const char* name, diff --git a/src/bro.bif b/src/bro.bif index 1daa2e5b68..6766a84122 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -14,6 +14,7 @@ using namespace std; RecordType* ftp_port; +RecordType* NetStats; RecordType* bro_resources; RecordType* matcher_stats; TableType* var_sizes; @@ -1431,6 +1432,31 @@ function bytestring_to_hexstr%(bytestring: string%): string extern const char* bro_version(); %%} +function net_stats%(%): NetStats + %{ + unsigned int recv = 0; + unsigned int drop = 0; + unsigned int link = 0; + + loop_over_list(pkt_srcs, i) + { + PktSrc* ps = pkt_srcs[i]; + + struct PktSrc::Stats stat; + ps->Statistics(&stat); + recv += stat.received; + drop += stat.dropped; + link += stat.link; + } + + RecordVal* ns = new RecordVal(NetStats); + ns->Assign(0, new Val(recv, TYPE_COUNT)); + ns->Assign(1, new Val(drop, TYPE_COUNT)); + ns->Assign(2, new Val(link, TYPE_COUNT)); + + return ns; + %} + function resource_usage%(%): bro_resources %{ struct rusage r; diff --git a/src/event.bif b/src/event.bif index 270f1b0d0b..687b0c92b2 100644 --- a/src/event.bif +++ b/src/event.bif @@ -52,7 +52,6 @@ event icmp_echo_request%(c: connection, icmp: icmp_conn, id: count, seq: count, event icmp_echo_reply%(c: connection, icmp: icmp_conn, id: count, seq: count, payload: string%); event icmp_unreachable%(c: connection, icmp: icmp_conn, code: count, context: icmp_context%); event icmp_time_exceeded%(c: connection, icmp: icmp_conn, code: count, context: icmp_context%); -event net_stats_update%(t: time, ns: net_stats%); event conn_stats%(c: connection, os: endpoint_stats, rs: endpoint_stats%); event conn_weird%(name: string, c: connection%); event conn_weird_addl%(name: string, c: connection, addl: string%); From d4e1f380095fab5ee04d6719f85fa28be29061b3 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Fri, 17 Jun 2011 09:03:45 -0400 Subject: [PATCH 2/9] Removing a stray print statement. --- policy/stats.bro | 1 - 1 file changed, 1 deletion(-) diff --git a/policy/stats.bro b/policy/stats.bro index 13c8548659..117080c9e6 100644 --- a/policy/stats.bro +++ b/policy/stats.bro @@ -71,7 +71,6 @@ event check_stats(last_time: time, last_ns: NetStats, last_res: bro_resources) NOTICE([$note=ResourceStats, $msg=stat_msg]); } - print "did stats!"; schedule stats_report_interval { check_stats(now, ns, res) }; } From 0f6a6ddc282b224c41a59fd3c26bb321af3acd4d Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 20 Jun 2011 10:11:44 -0500 Subject: [PATCH 3/9] Fix baseline for core.conn-id (now core.conn-uid) test. "conn-id.bro" was the name of a script in the policy/ directory that conn.bro expected to @load, but when the unit test was also named "conn-id.bro", it would be loaded twice during the test (once from conn.bro and once as a command line argument to bro). This means two event handlers were registered in error and the baseline output contained duplicate lines that can be removed. --- testing/btest/Baseline/core.conn-id/output.cc | 80 ------------------- .../{core.conn-id => core.conn-uid}/counts | 0 testing/btest/Baseline/core.conn-uid/output | 39 +++++++++ .../output => core.conn-uid/output.cc} | 38 --------- .../output.cc2 | 39 --------- .../btest/core/{conn-id.bro => conn-uid.bro} | 0 6 files changed, 39 insertions(+), 157 deletions(-) delete mode 100644 testing/btest/Baseline/core.conn-id/output.cc rename testing/btest/Baseline/{core.conn-id => core.conn-uid}/counts (100%) create mode 100644 testing/btest/Baseline/core.conn-uid/output rename testing/btest/Baseline/{core.conn-id/output => core.conn-uid/output.cc} (51%) rename testing/btest/Baseline/{core.conn-id => core.conn-uid}/output.cc2 (50%) rename testing/btest/core/{conn-id.bro => conn-uid.bro} (100%) diff --git a/testing/btest/Baseline/core.conn-id/output.cc b/testing/btest/Baseline/core.conn-id/output.cc deleted file mode 100644 index f03a74f541..0000000000 --- a/testing/btest/Baseline/core.conn-id/output.cc +++ /dev/null @@ -1,80 +0,0 @@ -[orig_h=141.142.220.202, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp], UWkUyAuUGXf -[orig_h=141.142.220.202, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp], UWkUyAuUGXf -[orig_h=141.142.220.50, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp], 56gKBmhBBB6 -[orig_h=141.142.220.50, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp], 56gKBmhBBB6 -[orig_h=141.142.220.118, orig_p=35634/tcp, resp_h=208.80.152.2, resp_p=80/tcp], 50da4BEzauh -[orig_h=141.142.220.118, orig_p=35634/tcp, resp_h=208.80.152.2, resp_p=80/tcp], 50da4BEzauh -[orig_h=141.142.220.118, orig_p=48649/tcp, resp_h=208.80.152.118, resp_p=80/tcp], WUjEZFOdSS -[orig_h=141.142.220.118, orig_p=48649/tcp, resp_h=208.80.152.118, resp_p=80/tcp], WUjEZFOdSS -[orig_h=141.142.220.118, orig_p=48649/tcp, resp_h=208.80.152.118, resp_p=80/tcp], WUjEZFOdSS -[orig_h=141.142.220.118, orig_p=48649/tcp, resp_h=208.80.152.118, resp_p=80/tcp], WUjEZFOdSS -[orig_h=141.142.220.118, orig_p=43927/udp, resp_h=141.142.2.2, resp_p=53/udp], ecqdozAET6c -[orig_h=141.142.220.118, orig_p=43927/udp, resp_h=141.142.2.2, resp_p=53/udp], ecqdozAET6c -[orig_h=141.142.220.118, orig_p=37676/udp, resp_h=141.142.2.2, resp_p=53/udp], tdkrEYpj5ja -[orig_h=141.142.220.118, orig_p=37676/udp, resp_h=141.142.2.2, resp_p=53/udp], tdkrEYpj5ja -[orig_h=141.142.220.118, orig_p=40526/udp, resp_h=141.142.2.2, resp_p=53/udp], F5XgctwO3Vl -[orig_h=141.142.220.118, orig_p=40526/udp, resp_h=141.142.2.2, resp_p=53/udp], F5XgctwO3Vl -[orig_h=141.142.220.118, orig_p=49996/tcp, resp_h=208.80.152.3, resp_p=80/tcp], svqqNKN9CFj -[orig_h=141.142.220.118, orig_p=49996/tcp, resp_h=208.80.152.3, resp_p=80/tcp], svqqNKN9CFj -[orig_h=141.142.220.118, orig_p=49997/tcp, resp_h=208.80.152.3, resp_p=80/tcp], UZkBBvjF0r8 -[orig_h=141.142.220.118, orig_p=49997/tcp, resp_h=208.80.152.3, resp_p=80/tcp], UZkBBvjF0r8 -[orig_h=141.142.220.118, orig_p=32902/udp, resp_h=141.142.2.2, resp_p=53/udp], nSEQzFk1LZc -[orig_h=141.142.220.118, orig_p=32902/udp, resp_h=141.142.2.2, resp_p=53/udp], nSEQzFk1LZc -[orig_h=141.142.220.118, orig_p=59816/udp, resp_h=141.142.2.2, resp_p=53/udp], rmXOq6wncn1 -[orig_h=141.142.220.118, orig_p=59816/udp, resp_h=141.142.2.2, resp_p=53/udp], rmXOq6wncn1 -[orig_h=141.142.220.118, orig_p=59714/udp, resp_h=141.142.2.2, resp_p=53/udp], 4YYJTjETe1i -[orig_h=141.142.220.118, orig_p=59714/udp, resp_h=141.142.2.2, resp_p=53/udp], 4YYJTjETe1i -[orig_h=141.142.220.118, orig_p=49998/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OldlyspNIr7 -[orig_h=141.142.220.118, orig_p=49998/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OldlyspNIr7 -[orig_h=141.142.220.118, orig_p=58206/udp, resp_h=141.142.2.2, resp_p=53/udp], R8BqVlcp23e -[orig_h=141.142.220.118, orig_p=58206/udp, resp_h=141.142.2.2, resp_p=53/udp], R8BqVlcp23e -[orig_h=141.142.220.118, orig_p=38911/udp, resp_h=141.142.2.2, resp_p=53/udp], duYdXg7bTa3 -[orig_h=141.142.220.118, orig_p=38911/udp, resp_h=141.142.2.2, resp_p=53/udp], duYdXg7bTa3 -[orig_h=141.142.220.118, orig_p=59746/udp, resp_h=141.142.2.2, resp_p=53/udp], yzqaQTU9DXe -[orig_h=141.142.220.118, orig_p=59746/udp, resp_h=141.142.2.2, resp_p=53/udp], yzqaQTU9DXe -[orig_h=141.142.220.118, orig_p=49999/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OPM7xFSDNw3 -[orig_h=141.142.220.118, orig_p=49999/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OPM7xFSDNw3 -[orig_h=141.142.220.118, orig_p=50000/tcp, resp_h=208.80.152.3, resp_p=80/tcp], j5w2LueK8Ti -[orig_h=141.142.220.118, orig_p=50000/tcp, resp_h=208.80.152.3, resp_p=80/tcp], j5w2LueK8Ti -[orig_h=141.142.220.118, orig_p=45000/udp, resp_h=141.142.2.2, resp_p=53/udp], N6rbUGwigQ7 -[orig_h=141.142.220.118, orig_p=45000/udp, resp_h=141.142.2.2, resp_p=53/udp], N6rbUGwigQ7 -[orig_h=141.142.220.118, orig_p=48479/udp, resp_h=141.142.2.2, resp_p=53/udp], 8b9q7qPtzhd -[orig_h=141.142.220.118, orig_p=48479/udp, resp_h=141.142.2.2, resp_p=53/udp], 8b9q7qPtzhd -[orig_h=141.142.220.118, orig_p=48128/udp, resp_h=141.142.2.2, resp_p=53/udp], KOdlL7sC9z2 -[orig_h=141.142.220.118, orig_p=48128/udp, resp_h=141.142.2.2, resp_p=53/udp], KOdlL7sC9z2 -[orig_h=141.142.220.118, orig_p=50001/tcp, resp_h=208.80.152.3, resp_p=80/tcp], hvOo97vj60k -[orig_h=141.142.220.118, orig_p=50001/tcp, resp_h=208.80.152.3, resp_p=80/tcp], hvOo97vj60k -[orig_h=141.142.220.118, orig_p=56056/udp, resp_h=141.142.2.2, resp_p=53/udp], FHu81uYujA9 -[orig_h=141.142.220.118, orig_p=56056/udp, resp_h=141.142.2.2, resp_p=53/udp], FHu81uYujA9 -[orig_h=141.142.220.118, orig_p=55092/udp, resp_h=141.142.2.2, resp_p=53/udp], 2M1wDTa0C7a -[orig_h=141.142.220.118, orig_p=55092/udp, resp_h=141.142.2.2, resp_p=53/udp], 2M1wDTa0C7a -[orig_h=141.142.220.118, orig_p=35642/tcp, resp_h=208.80.152.2, resp_p=80/tcp], OKiJdtzKWPk -[orig_h=141.142.220.118, orig_p=35642/tcp, resp_h=208.80.152.2, resp_p=80/tcp], OKiJdtzKWPk -[orig_h=141.142.220.118, orig_p=49997/tcp, resp_h=208.80.152.3, resp_p=80/tcp], UZkBBvjF0r8 -[orig_h=141.142.220.118, orig_p=49997/tcp, resp_h=208.80.152.3, resp_p=80/tcp], UZkBBvjF0r8 -[orig_h=141.142.220.118, orig_p=49996/tcp, resp_h=208.80.152.3, resp_p=80/tcp], svqqNKN9CFj -[orig_h=141.142.220.118, orig_p=49996/tcp, resp_h=208.80.152.3, resp_p=80/tcp], svqqNKN9CFj -[orig_h=141.142.220.118, orig_p=49998/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OldlyspNIr7 -[orig_h=141.142.220.118, orig_p=49998/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OldlyspNIr7 -[orig_h=141.142.220.118, orig_p=50000/tcp, resp_h=208.80.152.3, resp_p=80/tcp], j5w2LueK8Ti -[orig_h=141.142.220.118, orig_p=50000/tcp, resp_h=208.80.152.3, resp_p=80/tcp], j5w2LueK8Ti -[orig_h=141.142.220.118, orig_p=49999/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OPM7xFSDNw3 -[orig_h=141.142.220.118, orig_p=49999/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OPM7xFSDNw3 -[orig_h=141.142.220.118, orig_p=50001/tcp, resp_h=208.80.152.3, resp_p=80/tcp], hvOo97vj60k -[orig_h=141.142.220.118, orig_p=50001/tcp, resp_h=208.80.152.3, resp_p=80/tcp], hvOo97vj60k -[orig_h=141.142.220.118, orig_p=35642/tcp, resp_h=208.80.152.2, resp_p=80/tcp], OKiJdtzKWPk -[orig_h=141.142.220.118, orig_p=35642/tcp, resp_h=208.80.152.2, resp_p=80/tcp], OKiJdtzKWPk -[orig_h=141.142.220.235, orig_p=6705/tcp, resp_h=173.192.163.128, resp_p=80/tcp], tpUWfNdSLE -[orig_h=141.142.220.235, orig_p=6705/tcp, resp_h=173.192.163.128, resp_p=80/tcp], tpUWfNdSLE -[orig_h=141.142.220.235, orig_p=6705/tcp, resp_h=173.192.163.128, resp_p=80/tcp], tpUWfNdSLE -[orig_h=141.142.220.235, orig_p=6705/tcp, resp_h=173.192.163.128, resp_p=80/tcp], tpUWfNdSLE -[orig_h=141.142.220.44, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp], ra1C6ZLut4b -[orig_h=141.142.220.44, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp], ra1C6ZLut4b -[orig_h=141.142.220.226, orig_p=137/udp, resp_h=141.142.220.255, resp_p=137/udp], UElDH5b9qA5 -[orig_h=141.142.220.226, orig_p=137/udp, resp_h=141.142.220.255, resp_p=137/udp], UElDH5b9qA5 -[orig_h=141.142.220.226, orig_p=55131/udp, resp_h=224.0.0.252, resp_p=5355/udp], sO3mBXBav1h -[orig_h=141.142.220.226, orig_p=55131/udp, resp_h=224.0.0.252, resp_p=5355/udp], sO3mBXBav1h -[orig_h=141.142.220.226, orig_p=55671/udp, resp_h=224.0.0.252, resp_p=5355/udp], xAQqZE8Wdp4 -[orig_h=141.142.220.226, orig_p=55671/udp, resp_h=224.0.0.252, resp_p=5355/udp], xAQqZE8Wdp4 -[orig_h=141.142.220.238, orig_p=56641/udp, resp_h=141.142.220.255, resp_p=137/udp], zVecVnfOlsf -[orig_h=141.142.220.238, orig_p=56641/udp, resp_h=141.142.220.255, resp_p=137/udp], zVecVnfOlsf diff --git a/testing/btest/Baseline/core.conn-id/counts b/testing/btest/Baseline/core.conn-uid/counts similarity index 100% rename from testing/btest/Baseline/core.conn-id/counts rename to testing/btest/Baseline/core.conn-uid/counts diff --git a/testing/btest/Baseline/core.conn-uid/output b/testing/btest/Baseline/core.conn-uid/output new file mode 100644 index 0000000000..6db116d098 --- /dev/null +++ b/testing/btest/Baseline/core.conn-uid/output @@ -0,0 +1,39 @@ +[orig_h=141.142.220.202, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp], UWkUyAuUGXf +[orig_h=141.142.220.50, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp], 56gKBmhBBB6 +[orig_h=141.142.220.118, orig_p=35634/tcp, resp_h=208.80.152.2, resp_p=80/tcp], 50da4BEzauh +[orig_h=141.142.220.118, orig_p=48649/tcp, resp_h=208.80.152.118, resp_p=80/tcp], WUjEZFOdSS +[orig_h=141.142.220.118, orig_p=48649/tcp, resp_h=208.80.152.118, resp_p=80/tcp], WUjEZFOdSS +[orig_h=141.142.220.118, orig_p=43927/udp, resp_h=141.142.2.2, resp_p=53/udp], ecqdozAET6c +[orig_h=141.142.220.118, orig_p=37676/udp, resp_h=141.142.2.2, resp_p=53/udp], tdkrEYpj5ja +[orig_h=141.142.220.118, orig_p=40526/udp, resp_h=141.142.2.2, resp_p=53/udp], F5XgctwO3Vl +[orig_h=141.142.220.118, orig_p=49996/tcp, resp_h=208.80.152.3, resp_p=80/tcp], svqqNKN9CFj +[orig_h=141.142.220.118, orig_p=49997/tcp, resp_h=208.80.152.3, resp_p=80/tcp], UZkBBvjF0r8 +[orig_h=141.142.220.118, orig_p=32902/udp, resp_h=141.142.2.2, resp_p=53/udp], nSEQzFk1LZc +[orig_h=141.142.220.118, orig_p=59816/udp, resp_h=141.142.2.2, resp_p=53/udp], rmXOq6wncn1 +[orig_h=141.142.220.118, orig_p=59714/udp, resp_h=141.142.2.2, resp_p=53/udp], 4YYJTjETe1i +[orig_h=141.142.220.118, orig_p=49998/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OldlyspNIr7 +[orig_h=141.142.220.118, orig_p=58206/udp, resp_h=141.142.2.2, resp_p=53/udp], R8BqVlcp23e +[orig_h=141.142.220.118, orig_p=38911/udp, resp_h=141.142.2.2, resp_p=53/udp], duYdXg7bTa3 +[orig_h=141.142.220.118, orig_p=59746/udp, resp_h=141.142.2.2, resp_p=53/udp], yzqaQTU9DXe +[orig_h=141.142.220.118, orig_p=49999/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OPM7xFSDNw3 +[orig_h=141.142.220.118, orig_p=50000/tcp, resp_h=208.80.152.3, resp_p=80/tcp], j5w2LueK8Ti +[orig_h=141.142.220.118, orig_p=45000/udp, resp_h=141.142.2.2, resp_p=53/udp], N6rbUGwigQ7 +[orig_h=141.142.220.118, orig_p=48479/udp, resp_h=141.142.2.2, resp_p=53/udp], 8b9q7qPtzhd +[orig_h=141.142.220.118, orig_p=48128/udp, resp_h=141.142.2.2, resp_p=53/udp], KOdlL7sC9z2 +[orig_h=141.142.220.118, orig_p=50001/tcp, resp_h=208.80.152.3, resp_p=80/tcp], hvOo97vj60k +[orig_h=141.142.220.118, orig_p=56056/udp, resp_h=141.142.2.2, resp_p=53/udp], FHu81uYujA9 +[orig_h=141.142.220.118, orig_p=55092/udp, resp_h=141.142.2.2, resp_p=53/udp], 2M1wDTa0C7a +[orig_h=141.142.220.118, orig_p=35642/tcp, resp_h=208.80.152.2, resp_p=80/tcp], OKiJdtzKWPk +[orig_h=141.142.220.118, orig_p=49997/tcp, resp_h=208.80.152.3, resp_p=80/tcp], UZkBBvjF0r8 +[orig_h=141.142.220.118, orig_p=49996/tcp, resp_h=208.80.152.3, resp_p=80/tcp], svqqNKN9CFj +[orig_h=141.142.220.118, orig_p=49998/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OldlyspNIr7 +[orig_h=141.142.220.118, orig_p=50000/tcp, resp_h=208.80.152.3, resp_p=80/tcp], j5w2LueK8Ti +[orig_h=141.142.220.118, orig_p=49999/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OPM7xFSDNw3 +[orig_h=141.142.220.118, orig_p=50001/tcp, resp_h=208.80.152.3, resp_p=80/tcp], hvOo97vj60k +[orig_h=141.142.220.118, orig_p=35642/tcp, resp_h=208.80.152.2, resp_p=80/tcp], OKiJdtzKWPk +[orig_h=141.142.220.235, orig_p=6705/tcp, resp_h=173.192.163.128, resp_p=80/tcp], tpUWfNdSLE +[orig_h=141.142.220.44, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp], ra1C6ZLut4b +[orig_h=141.142.220.226, orig_p=137/udp, resp_h=141.142.220.255, resp_p=137/udp], UElDH5b9qA5 +[orig_h=141.142.220.226, orig_p=55131/udp, resp_h=224.0.0.252, resp_p=5355/udp], sO3mBXBav1h +[orig_h=141.142.220.226, orig_p=55671/udp, resp_h=224.0.0.252, resp_p=5355/udp], xAQqZE8Wdp4 +[orig_h=141.142.220.238, orig_p=56641/udp, resp_h=141.142.220.255, resp_p=137/udp], zVecVnfOlsf diff --git a/testing/btest/Baseline/core.conn-id/output b/testing/btest/Baseline/core.conn-uid/output.cc similarity index 51% rename from testing/btest/Baseline/core.conn-id/output rename to testing/btest/Baseline/core.conn-uid/output.cc index 3f7256278e..f00b065849 100644 --- a/testing/btest/Baseline/core.conn-id/output +++ b/testing/btest/Baseline/core.conn-uid/output.cc @@ -1,78 +1,40 @@ [orig_h=141.142.220.202, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp], UWkUyAuUGXf -[orig_h=141.142.220.202, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp], UWkUyAuUGXf [orig_h=141.142.220.50, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp], 56gKBmhBBB6 -[orig_h=141.142.220.50, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp], 56gKBmhBBB6 -[orig_h=141.142.220.118, orig_p=35634/tcp, resp_h=208.80.152.2, resp_p=80/tcp], 50da4BEzauh [orig_h=141.142.220.118, orig_p=35634/tcp, resp_h=208.80.152.2, resp_p=80/tcp], 50da4BEzauh [orig_h=141.142.220.118, orig_p=48649/tcp, resp_h=208.80.152.118, resp_p=80/tcp], WUjEZFOdSS [orig_h=141.142.220.118, orig_p=48649/tcp, resp_h=208.80.152.118, resp_p=80/tcp], WUjEZFOdSS -[orig_h=141.142.220.118, orig_p=48649/tcp, resp_h=208.80.152.118, resp_p=80/tcp], WUjEZFOdSS -[orig_h=141.142.220.118, orig_p=48649/tcp, resp_h=208.80.152.118, resp_p=80/tcp], WUjEZFOdSS -[orig_h=141.142.220.118, orig_p=43927/udp, resp_h=141.142.2.2, resp_p=53/udp], ecqdozAET6c [orig_h=141.142.220.118, orig_p=43927/udp, resp_h=141.142.2.2, resp_p=53/udp], ecqdozAET6c [orig_h=141.142.220.118, orig_p=37676/udp, resp_h=141.142.2.2, resp_p=53/udp], tdkrEYpj5ja -[orig_h=141.142.220.118, orig_p=37676/udp, resp_h=141.142.2.2, resp_p=53/udp], tdkrEYpj5ja -[orig_h=141.142.220.118, orig_p=40526/udp, resp_h=141.142.2.2, resp_p=53/udp], F5XgctwO3Vl [orig_h=141.142.220.118, orig_p=40526/udp, resp_h=141.142.2.2, resp_p=53/udp], F5XgctwO3Vl [orig_h=141.142.220.118, orig_p=49996/tcp, resp_h=208.80.152.3, resp_p=80/tcp], svqqNKN9CFj -[orig_h=141.142.220.118, orig_p=49996/tcp, resp_h=208.80.152.3, resp_p=80/tcp], svqqNKN9CFj -[orig_h=141.142.220.118, orig_p=49997/tcp, resp_h=208.80.152.3, resp_p=80/tcp], UZkBBvjF0r8 [orig_h=141.142.220.118, orig_p=49997/tcp, resp_h=208.80.152.3, resp_p=80/tcp], UZkBBvjF0r8 [orig_h=141.142.220.118, orig_p=32902/udp, resp_h=141.142.2.2, resp_p=53/udp], nSEQzFk1LZc -[orig_h=141.142.220.118, orig_p=32902/udp, resp_h=141.142.2.2, resp_p=53/udp], nSEQzFk1LZc -[orig_h=141.142.220.118, orig_p=59816/udp, resp_h=141.142.2.2, resp_p=53/udp], rmXOq6wncn1 [orig_h=141.142.220.118, orig_p=59816/udp, resp_h=141.142.2.2, resp_p=53/udp], rmXOq6wncn1 [orig_h=141.142.220.118, orig_p=59714/udp, resp_h=141.142.2.2, resp_p=53/udp], 4YYJTjETe1i -[orig_h=141.142.220.118, orig_p=59714/udp, resp_h=141.142.2.2, resp_p=53/udp], 4YYJTjETe1i -[orig_h=141.142.220.118, orig_p=49998/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OldlyspNIr7 [orig_h=141.142.220.118, orig_p=49998/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OldlyspNIr7 [orig_h=141.142.220.118, orig_p=58206/udp, resp_h=141.142.2.2, resp_p=53/udp], R8BqVlcp23e -[orig_h=141.142.220.118, orig_p=58206/udp, resp_h=141.142.2.2, resp_p=53/udp], R8BqVlcp23e -[orig_h=141.142.220.118, orig_p=38911/udp, resp_h=141.142.2.2, resp_p=53/udp], duYdXg7bTa3 [orig_h=141.142.220.118, orig_p=38911/udp, resp_h=141.142.2.2, resp_p=53/udp], duYdXg7bTa3 [orig_h=141.142.220.118, orig_p=59746/udp, resp_h=141.142.2.2, resp_p=53/udp], yzqaQTU9DXe -[orig_h=141.142.220.118, orig_p=59746/udp, resp_h=141.142.2.2, resp_p=53/udp], yzqaQTU9DXe [orig_h=141.142.220.118, orig_p=49999/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OPM7xFSDNw3 -[orig_h=141.142.220.118, orig_p=49999/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OPM7xFSDNw3 -[orig_h=141.142.220.118, orig_p=50000/tcp, resp_h=208.80.152.3, resp_p=80/tcp], j5w2LueK8Ti [orig_h=141.142.220.118, orig_p=50000/tcp, resp_h=208.80.152.3, resp_p=80/tcp], j5w2LueK8Ti [orig_h=141.142.220.118, orig_p=45000/udp, resp_h=141.142.2.2, resp_p=53/udp], N6rbUGwigQ7 -[orig_h=141.142.220.118, orig_p=45000/udp, resp_h=141.142.2.2, resp_p=53/udp], N6rbUGwigQ7 -[orig_h=141.142.220.118, orig_p=48479/udp, resp_h=141.142.2.2, resp_p=53/udp], 8b9q7qPtzhd [orig_h=141.142.220.118, orig_p=48479/udp, resp_h=141.142.2.2, resp_p=53/udp], 8b9q7qPtzhd [orig_h=141.142.220.118, orig_p=48128/udp, resp_h=141.142.2.2, resp_p=53/udp], KOdlL7sC9z2 -[orig_h=141.142.220.118, orig_p=48128/udp, resp_h=141.142.2.2, resp_p=53/udp], KOdlL7sC9z2 -[orig_h=141.142.220.118, orig_p=50001/tcp, resp_h=208.80.152.3, resp_p=80/tcp], hvOo97vj60k [orig_h=141.142.220.118, orig_p=50001/tcp, resp_h=208.80.152.3, resp_p=80/tcp], hvOo97vj60k [orig_h=141.142.220.118, orig_p=56056/udp, resp_h=141.142.2.2, resp_p=53/udp], FHu81uYujA9 -[orig_h=141.142.220.118, orig_p=56056/udp, resp_h=141.142.2.2, resp_p=53/udp], FHu81uYujA9 -[orig_h=141.142.220.118, orig_p=55092/udp, resp_h=141.142.2.2, resp_p=53/udp], 2M1wDTa0C7a [orig_h=141.142.220.118, orig_p=55092/udp, resp_h=141.142.2.2, resp_p=53/udp], 2M1wDTa0C7a [orig_h=141.142.220.118, orig_p=35642/tcp, resp_h=208.80.152.2, resp_p=80/tcp], OKiJdtzKWPk -[orig_h=141.142.220.118, orig_p=35642/tcp, resp_h=208.80.152.2, resp_p=80/tcp], OKiJdtzKWPk -[orig_h=141.142.220.118, orig_p=49997/tcp, resp_h=208.80.152.3, resp_p=80/tcp], UZkBBvjF0r8 [orig_h=141.142.220.118, orig_p=49997/tcp, resp_h=208.80.152.3, resp_p=80/tcp], UZkBBvjF0r8 [orig_h=141.142.220.118, orig_p=49996/tcp, resp_h=208.80.152.3, resp_p=80/tcp], svqqNKN9CFj -[orig_h=141.142.220.118, orig_p=49996/tcp, resp_h=208.80.152.3, resp_p=80/tcp], svqqNKN9CFj -[orig_h=141.142.220.118, orig_p=49998/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OldlyspNIr7 [orig_h=141.142.220.118, orig_p=49998/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OldlyspNIr7 [orig_h=141.142.220.118, orig_p=50000/tcp, resp_h=208.80.152.3, resp_p=80/tcp], j5w2LueK8Ti -[orig_h=141.142.220.118, orig_p=50000/tcp, resp_h=208.80.152.3, resp_p=80/tcp], j5w2LueK8Ti -[orig_h=141.142.220.118, orig_p=49999/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OPM7xFSDNw3 [orig_h=141.142.220.118, orig_p=49999/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OPM7xFSDNw3 [orig_h=141.142.220.118, orig_p=50001/tcp, resp_h=208.80.152.3, resp_p=80/tcp], hvOo97vj60k -[orig_h=141.142.220.118, orig_p=50001/tcp, resp_h=208.80.152.3, resp_p=80/tcp], hvOo97vj60k -[orig_h=141.142.220.118, orig_p=35642/tcp, resp_h=208.80.152.2, resp_p=80/tcp], OKiJdtzKWPk [orig_h=141.142.220.118, orig_p=35642/tcp, resp_h=208.80.152.2, resp_p=80/tcp], OKiJdtzKWPk [orig_h=141.142.220.235, orig_p=6705/tcp, resp_h=173.192.163.128, resp_p=80/tcp], tpUWfNdSLE [orig_h=141.142.220.235, orig_p=6705/tcp, resp_h=173.192.163.128, resp_p=80/tcp], tpUWfNdSLE [orig_h=141.142.220.44, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp], ra1C6ZLut4b -[orig_h=141.142.220.44, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp], ra1C6ZLut4b -[orig_h=141.142.220.226, orig_p=137/udp, resp_h=141.142.220.255, resp_p=137/udp], UElDH5b9qA5 [orig_h=141.142.220.226, orig_p=137/udp, resp_h=141.142.220.255, resp_p=137/udp], UElDH5b9qA5 [orig_h=141.142.220.226, orig_p=55131/udp, resp_h=224.0.0.252, resp_p=5355/udp], sO3mBXBav1h -[orig_h=141.142.220.226, orig_p=55131/udp, resp_h=224.0.0.252, resp_p=5355/udp], sO3mBXBav1h -[orig_h=141.142.220.226, orig_p=55671/udp, resp_h=224.0.0.252, resp_p=5355/udp], xAQqZE8Wdp4 [orig_h=141.142.220.226, orig_p=55671/udp, resp_h=224.0.0.252, resp_p=5355/udp], xAQqZE8Wdp4 [orig_h=141.142.220.238, orig_p=56641/udp, resp_h=141.142.220.255, resp_p=137/udp], zVecVnfOlsf -[orig_h=141.142.220.238, orig_p=56641/udp, resp_h=141.142.220.255, resp_p=137/udp], zVecVnfOlsf diff --git a/testing/btest/Baseline/core.conn-id/output.cc2 b/testing/btest/Baseline/core.conn-uid/output.cc2 similarity index 50% rename from testing/btest/Baseline/core.conn-id/output.cc2 rename to testing/btest/Baseline/core.conn-uid/output.cc2 index 3f7256278e..6db116d098 100644 --- a/testing/btest/Baseline/core.conn-id/output.cc2 +++ b/testing/btest/Baseline/core.conn-uid/output.cc2 @@ -1,78 +1,39 @@ [orig_h=141.142.220.202, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp], UWkUyAuUGXf -[orig_h=141.142.220.202, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp], UWkUyAuUGXf [orig_h=141.142.220.50, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp], 56gKBmhBBB6 -[orig_h=141.142.220.50, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp], 56gKBmhBBB6 -[orig_h=141.142.220.118, orig_p=35634/tcp, resp_h=208.80.152.2, resp_p=80/tcp], 50da4BEzauh [orig_h=141.142.220.118, orig_p=35634/tcp, resp_h=208.80.152.2, resp_p=80/tcp], 50da4BEzauh [orig_h=141.142.220.118, orig_p=48649/tcp, resp_h=208.80.152.118, resp_p=80/tcp], WUjEZFOdSS [orig_h=141.142.220.118, orig_p=48649/tcp, resp_h=208.80.152.118, resp_p=80/tcp], WUjEZFOdSS -[orig_h=141.142.220.118, orig_p=48649/tcp, resp_h=208.80.152.118, resp_p=80/tcp], WUjEZFOdSS -[orig_h=141.142.220.118, orig_p=48649/tcp, resp_h=208.80.152.118, resp_p=80/tcp], WUjEZFOdSS -[orig_h=141.142.220.118, orig_p=43927/udp, resp_h=141.142.2.2, resp_p=53/udp], ecqdozAET6c [orig_h=141.142.220.118, orig_p=43927/udp, resp_h=141.142.2.2, resp_p=53/udp], ecqdozAET6c [orig_h=141.142.220.118, orig_p=37676/udp, resp_h=141.142.2.2, resp_p=53/udp], tdkrEYpj5ja -[orig_h=141.142.220.118, orig_p=37676/udp, resp_h=141.142.2.2, resp_p=53/udp], tdkrEYpj5ja -[orig_h=141.142.220.118, orig_p=40526/udp, resp_h=141.142.2.2, resp_p=53/udp], F5XgctwO3Vl [orig_h=141.142.220.118, orig_p=40526/udp, resp_h=141.142.2.2, resp_p=53/udp], F5XgctwO3Vl [orig_h=141.142.220.118, orig_p=49996/tcp, resp_h=208.80.152.3, resp_p=80/tcp], svqqNKN9CFj -[orig_h=141.142.220.118, orig_p=49996/tcp, resp_h=208.80.152.3, resp_p=80/tcp], svqqNKN9CFj -[orig_h=141.142.220.118, orig_p=49997/tcp, resp_h=208.80.152.3, resp_p=80/tcp], UZkBBvjF0r8 [orig_h=141.142.220.118, orig_p=49997/tcp, resp_h=208.80.152.3, resp_p=80/tcp], UZkBBvjF0r8 [orig_h=141.142.220.118, orig_p=32902/udp, resp_h=141.142.2.2, resp_p=53/udp], nSEQzFk1LZc -[orig_h=141.142.220.118, orig_p=32902/udp, resp_h=141.142.2.2, resp_p=53/udp], nSEQzFk1LZc -[orig_h=141.142.220.118, orig_p=59816/udp, resp_h=141.142.2.2, resp_p=53/udp], rmXOq6wncn1 [orig_h=141.142.220.118, orig_p=59816/udp, resp_h=141.142.2.2, resp_p=53/udp], rmXOq6wncn1 [orig_h=141.142.220.118, orig_p=59714/udp, resp_h=141.142.2.2, resp_p=53/udp], 4YYJTjETe1i -[orig_h=141.142.220.118, orig_p=59714/udp, resp_h=141.142.2.2, resp_p=53/udp], 4YYJTjETe1i -[orig_h=141.142.220.118, orig_p=49998/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OldlyspNIr7 [orig_h=141.142.220.118, orig_p=49998/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OldlyspNIr7 [orig_h=141.142.220.118, orig_p=58206/udp, resp_h=141.142.2.2, resp_p=53/udp], R8BqVlcp23e -[orig_h=141.142.220.118, orig_p=58206/udp, resp_h=141.142.2.2, resp_p=53/udp], R8BqVlcp23e -[orig_h=141.142.220.118, orig_p=38911/udp, resp_h=141.142.2.2, resp_p=53/udp], duYdXg7bTa3 [orig_h=141.142.220.118, orig_p=38911/udp, resp_h=141.142.2.2, resp_p=53/udp], duYdXg7bTa3 [orig_h=141.142.220.118, orig_p=59746/udp, resp_h=141.142.2.2, resp_p=53/udp], yzqaQTU9DXe -[orig_h=141.142.220.118, orig_p=59746/udp, resp_h=141.142.2.2, resp_p=53/udp], yzqaQTU9DXe [orig_h=141.142.220.118, orig_p=49999/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OPM7xFSDNw3 -[orig_h=141.142.220.118, orig_p=49999/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OPM7xFSDNw3 -[orig_h=141.142.220.118, orig_p=50000/tcp, resp_h=208.80.152.3, resp_p=80/tcp], j5w2LueK8Ti [orig_h=141.142.220.118, orig_p=50000/tcp, resp_h=208.80.152.3, resp_p=80/tcp], j5w2LueK8Ti [orig_h=141.142.220.118, orig_p=45000/udp, resp_h=141.142.2.2, resp_p=53/udp], N6rbUGwigQ7 -[orig_h=141.142.220.118, orig_p=45000/udp, resp_h=141.142.2.2, resp_p=53/udp], N6rbUGwigQ7 -[orig_h=141.142.220.118, orig_p=48479/udp, resp_h=141.142.2.2, resp_p=53/udp], 8b9q7qPtzhd [orig_h=141.142.220.118, orig_p=48479/udp, resp_h=141.142.2.2, resp_p=53/udp], 8b9q7qPtzhd [orig_h=141.142.220.118, orig_p=48128/udp, resp_h=141.142.2.2, resp_p=53/udp], KOdlL7sC9z2 -[orig_h=141.142.220.118, orig_p=48128/udp, resp_h=141.142.2.2, resp_p=53/udp], KOdlL7sC9z2 -[orig_h=141.142.220.118, orig_p=50001/tcp, resp_h=208.80.152.3, resp_p=80/tcp], hvOo97vj60k [orig_h=141.142.220.118, orig_p=50001/tcp, resp_h=208.80.152.3, resp_p=80/tcp], hvOo97vj60k [orig_h=141.142.220.118, orig_p=56056/udp, resp_h=141.142.2.2, resp_p=53/udp], FHu81uYujA9 -[orig_h=141.142.220.118, orig_p=56056/udp, resp_h=141.142.2.2, resp_p=53/udp], FHu81uYujA9 -[orig_h=141.142.220.118, orig_p=55092/udp, resp_h=141.142.2.2, resp_p=53/udp], 2M1wDTa0C7a [orig_h=141.142.220.118, orig_p=55092/udp, resp_h=141.142.2.2, resp_p=53/udp], 2M1wDTa0C7a [orig_h=141.142.220.118, orig_p=35642/tcp, resp_h=208.80.152.2, resp_p=80/tcp], OKiJdtzKWPk -[orig_h=141.142.220.118, orig_p=35642/tcp, resp_h=208.80.152.2, resp_p=80/tcp], OKiJdtzKWPk -[orig_h=141.142.220.118, orig_p=49997/tcp, resp_h=208.80.152.3, resp_p=80/tcp], UZkBBvjF0r8 [orig_h=141.142.220.118, orig_p=49997/tcp, resp_h=208.80.152.3, resp_p=80/tcp], UZkBBvjF0r8 [orig_h=141.142.220.118, orig_p=49996/tcp, resp_h=208.80.152.3, resp_p=80/tcp], svqqNKN9CFj -[orig_h=141.142.220.118, orig_p=49996/tcp, resp_h=208.80.152.3, resp_p=80/tcp], svqqNKN9CFj -[orig_h=141.142.220.118, orig_p=49998/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OldlyspNIr7 [orig_h=141.142.220.118, orig_p=49998/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OldlyspNIr7 [orig_h=141.142.220.118, orig_p=50000/tcp, resp_h=208.80.152.3, resp_p=80/tcp], j5w2LueK8Ti -[orig_h=141.142.220.118, orig_p=50000/tcp, resp_h=208.80.152.3, resp_p=80/tcp], j5w2LueK8Ti -[orig_h=141.142.220.118, orig_p=49999/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OPM7xFSDNw3 [orig_h=141.142.220.118, orig_p=49999/tcp, resp_h=208.80.152.3, resp_p=80/tcp], OPM7xFSDNw3 [orig_h=141.142.220.118, orig_p=50001/tcp, resp_h=208.80.152.3, resp_p=80/tcp], hvOo97vj60k -[orig_h=141.142.220.118, orig_p=50001/tcp, resp_h=208.80.152.3, resp_p=80/tcp], hvOo97vj60k -[orig_h=141.142.220.118, orig_p=35642/tcp, resp_h=208.80.152.2, resp_p=80/tcp], OKiJdtzKWPk [orig_h=141.142.220.118, orig_p=35642/tcp, resp_h=208.80.152.2, resp_p=80/tcp], OKiJdtzKWPk [orig_h=141.142.220.235, orig_p=6705/tcp, resp_h=173.192.163.128, resp_p=80/tcp], tpUWfNdSLE -[orig_h=141.142.220.235, orig_p=6705/tcp, resp_h=173.192.163.128, resp_p=80/tcp], tpUWfNdSLE -[orig_h=141.142.220.44, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp], ra1C6ZLut4b [orig_h=141.142.220.44, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp], ra1C6ZLut4b [orig_h=141.142.220.226, orig_p=137/udp, resp_h=141.142.220.255, resp_p=137/udp], UElDH5b9qA5 -[orig_h=141.142.220.226, orig_p=137/udp, resp_h=141.142.220.255, resp_p=137/udp], UElDH5b9qA5 -[orig_h=141.142.220.226, orig_p=55131/udp, resp_h=224.0.0.252, resp_p=5355/udp], sO3mBXBav1h [orig_h=141.142.220.226, orig_p=55131/udp, resp_h=224.0.0.252, resp_p=5355/udp], sO3mBXBav1h [orig_h=141.142.220.226, orig_p=55671/udp, resp_h=224.0.0.252, resp_p=5355/udp], xAQqZE8Wdp4 -[orig_h=141.142.220.226, orig_p=55671/udp, resp_h=224.0.0.252, resp_p=5355/udp], xAQqZE8Wdp4 -[orig_h=141.142.220.238, orig_p=56641/udp, resp_h=141.142.220.255, resp_p=137/udp], zVecVnfOlsf [orig_h=141.142.220.238, orig_p=56641/udp, resp_h=141.142.220.255, resp_p=137/udp], zVecVnfOlsf diff --git a/testing/btest/core/conn-id.bro b/testing/btest/core/conn-uid.bro similarity index 100% rename from testing/btest/core/conn-id.bro rename to testing/btest/core/conn-uid.bro From 9de6e9170c350ac51b9955ae765991e36417cc02 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 20 Jun 2011 12:10:09 -0500 Subject: [PATCH 4/9] Cleaning up the script loading implementation. This change primarily improves the way Bro detects and prevents the same script from being loaded twice. It now compares inode numbers instead of path names. --- src/scan.l | 72 +++++++++++-------- .../btest/Baseline/core.load-unload/output | 12 ++++ testing/btest/core/load-normalization.bro | 14 ++++ testing/btest/core/load-unload.bro | 7 ++ 4 files changed, 76 insertions(+), 29 deletions(-) create mode 100644 testing/btest/Baseline/core.load-unload/output create mode 100644 testing/btest/core/load-normalization.bro create mode 100644 testing/btest/core/load-unload.bro diff --git a/src/scan.l b/src/scan.l index 8db694c53c..c91de1607c 100644 --- a/src/scan.l +++ b/src/scan.l @@ -8,6 +8,7 @@ #include #include #include +#include #include "input.h" #include "util.h" @@ -55,7 +56,8 @@ char last_tok[128]; error(fmt("read failed with \"%s\"", strerror(errno))); // Files we have already scanned (or are in the process of scanning). -static PList(char) files_scanned; +// They are tracked by inode number +static std::list files_scanned; // reST documents that we've created (or have at least opened so far). std::list docs_generated; @@ -84,6 +86,14 @@ static const char* canon_doc_comment(const char* comment) return ( comment[0] == ' ' ) ? comment + 1 : comment; } +static ino_t get_inode_num(FILE* f, const char* filename) + { + struct stat b; + if ( fstat(fileno(f), &b) ) + internal_error("failed to fstat fd of %s\n", filename); + return b.st_ino; + } + class FileInfo { public: FileInfo(string restore_module = ""); @@ -332,7 +342,19 @@ when return TOK_WHEN; const char* new_file = skip_whitespace(yytext + 7); // All we have to do is pretend we've already scanned it. - files_scanned.append(copy_string(new_file)); + const char* full_filename; + FILE* f = search_for_file(new_file, "bro", &full_filename, true); + if ( f ) + { + ino_t i = get_inode_num(f, full_filename); + fclose(f); + delete [] full_filename; + files_scanned.push_back(i); + } + else + { + internal_error("failed find file associated with @unload %s", new_file); + } } @prefixes{WS}("+"?)={WS}{PREFIX} { @@ -499,33 +521,10 @@ YYLTYPE GetCurrentLocation() static int load_files_with_prefix(const char* orig_file) { - loop_over_list(files_scanned, j) - { - if ( streq(files_scanned[j], orig_file) ) - return 0; - } - - // Be sure to copy "orig_file", since it could be an alias - // for yytext, which is ephemeral and will be zapped - // if we do a yy_switch_to_buffer() below. - char* file = copy_string(orig_file); - // Whether we pushed on a FileInfo that will restore the // current module after the final file has been scanned. bool did_module_restore = false; - files_scanned.append(file); - - // If the file has a .bro extension, add a second version to the list - // of known files which has it stripped. - char* ext = strrchr(file, '.'); - if ( ext && streq(ext, ".bro") ) - { - char* s = copy_string(file); - s[ext - file] = '\0'; - files_scanned.append(s); - } - // Note, we need to loop through the prefixes backwards, since // we push them onto a stack, with the last one we push on the // stack being the first one we will scan. @@ -536,7 +535,7 @@ static int load_files_with_prefix(const char* orig_file) const char* full_filename = ""; FILE* f; - if ( streq(file, "-") ) + if ( streq(orig_file, "-") ) { f = stdin; full_filename = ""; @@ -550,13 +549,13 @@ static int load_files_with_prefix(const char* orig_file) else { - int n = strlen(prefix) + strlen(file) + 2; + int n = strlen(prefix) + strlen(orig_file) + 2; char* new_filename = new char[n]; if ( prefix[0] ) - sprintf(new_filename, "%s.%s", prefix, file); + sprintf(new_filename, "%s.%s", prefix, orig_file); else - strcpy(new_filename, file); + strcpy(new_filename, orig_file); f = search_for_file(new_filename, "bro", &full_filename, true); delete [] new_filename; @@ -564,6 +563,21 @@ static int load_files_with_prefix(const char* orig_file) if ( f ) { + ino_t i = get_inode_num(f, full_filename); + std::list::const_iterator it; + + for ( it = files_scanned.begin(); it != files_scanned.end(); ++it ) + { + if ( *it == i ) + { + fclose(f); + delete [] full_filename; + return 0; + } + } + + files_scanned.push_back(i); + if ( g_policy_debug ) { // Add the filename to the file mapping diff --git a/testing/btest/Baseline/core.load-unload/output b/testing/btest/Baseline/core.load-unload/output new file mode 100644 index 0000000000..5bda7e456e --- /dev/null +++ b/testing/btest/Baseline/core.load-unload/output @@ -0,0 +1,12 @@ +loading /Users/jsiwek/tmp/bro/policy/bro.init + loading /Users/jsiwek/tmp/bro/build/src/const.bif.bro + loading /Users/jsiwek/tmp/bro/build/src/types.bif.bro + loading /Users/jsiwek/tmp/bro/build/src/strings.bif.bro + loading /Users/jsiwek/tmp/bro/build/src/bro.bif.bro + loading /Users/jsiwek/tmp/bro/policy/logging.bro + loading /Users/jsiwek/tmp/bro/build/src/logging.bif.bro + loading /Users/jsiwek/tmp/bro/policy/logging-ascii.bro + loading /Users/jsiwek/tmp/bro/build/src/event.bif.bro + loading /Users/jsiwek/tmp/bro/policy/pcap.bro + loading /Users/jsiwek/tmp/bro/policy/server-ports.bro +loading /Users/jsiwek/tmp/bro/testing/btest/.tmp/core.load-unload/load-unload.bro diff --git a/testing/btest/core/load-normalization.bro b/testing/btest/core/load-normalization.bro new file mode 100644 index 0000000000..ff9ad3fb52 --- /dev/null +++ b/testing/btest/core/load-normalization.bro @@ -0,0 +1,14 @@ +# This tests bro's mechanism to prevent duplicate script loading. +# +# @TEST-EXEC: mkdir -p foo/bar +# @TEST-EXEC: echo "@load bar/test" >loader.bro +# @TEST-EXEC: cp %INPUT foo/bar/test.bro +# @TEST-EXEC: BROPATH=$BROPATH:.:./foo bro -l loader bar/test +# @TEST-EXEC: BROPATH=$BROPATH:.:./foo bro -l loader bar/test.bro +# @TEST-EXEC: BROPATH=$BROPATH:.:./foo bro -l loader foo/bar/test +# @TEST-EXEC: BROPATH=$BROPATH:.:./foo bro -l loader foo/bar/test.bro +# @TEST-EXEC: BROPATH=$BROPATH:.:./foo bro -l loader `pwd`/foo/bar/test.bro + +type Test: enum { + TEST, +}; diff --git a/testing/btest/core/load-unload.bro b/testing/btest/core/load-unload.bro new file mode 100644 index 0000000000..701e415134 --- /dev/null +++ b/testing/btest/core/load-unload.bro @@ -0,0 +1,7 @@ +# This tests the @unload directive +# +# @TEST-EXEC: echo 'print "oops";' >dontloadmebro.bro +# @TEST-EXEC: bro -l %INPUT dontloadmebro >output 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff output + +@unload dontloadmebro From 143427e35e0b6a2c62527bd444423385f3b83bca Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 22 Jun 2011 14:43:23 -0500 Subject: [PATCH 5/9] Raise internal error when failing to read contents of state file with -x option Instead of just exiting w/ code 0 --- src/main.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main.cc b/src/main.cc index 37acbdeaeb..d870efeca2 100644 --- a/src/main.cc +++ b/src/main.cc @@ -888,7 +888,8 @@ int main(int argc, char** argv) UnserialInfo info(&s); info.print = stdout; info.install_uniques = true; - s.Read(&info, bst_file); + if ( ! s.Read(&info, bst_file) ) + internal_error("Failed to read events from %s\n", bst_file); } exit(0); From 6b9d23abb21aaab06f449f871507d2d4084706dd Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sat, 25 Jun 2011 17:41:51 -0700 Subject: [PATCH 6/9] Applying DNS TTL patch from #368. I've restructured/cleaned up the original patch by thomas.other a bit. --- src/DNS_Mgr.cc | 35 +++++++++++++++++++++++++++++------ src/nb_dns.c | 4 ++++ src/nb_dns.h | 1 + 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/src/DNS_Mgr.cc b/src/DNS_Mgr.cc index d179ccec49..3c800e1f30 100644 --- a/src/DNS_Mgr.cc +++ b/src/DNS_Mgr.cc @@ -63,6 +63,7 @@ public: protected: char* host; // if non-nil, this is a host request uint32 addr; + uint32 ttl; int request_pending; }; @@ -82,8 +83,8 @@ int DNS_Mgr_Request::MakeRequest(nb_dns_info* nb_dns) class DNS_Mapping { public: - DNS_Mapping(const char* host, struct hostent* h); - DNS_Mapping(uint32 addr, struct hostent* h); + DNS_Mapping(const char* host, struct hostent* h, uint32 ttl); + DNS_Mapping(uint32 addr, struct hostent* h, uint32 ttl); DNS_Mapping(FILE* f); int NoMapping() const { return no_mapping; } @@ -108,6 +109,9 @@ public: int Failed() const { return failed; } int Valid() const { return ! failed; } + bool Expired() const + { return current_time() > (creation_time + req_ttl); } + protected: friend class DNS_Mgr; @@ -119,6 +123,7 @@ protected: char* req_host; uint32 req_addr; + uint32 req_ttl; int num_names; char** names; @@ -146,21 +151,23 @@ static TableVal* empty_addr_set() return new TableVal(s); } -DNS_Mapping::DNS_Mapping(const char* host, struct hostent* h) +DNS_Mapping::DNS_Mapping(const char* host, struct hostent* h, uint32 ttl) { Init(h); req_host = copy_string(host); req_addr = 0; + req_ttl = ttl; if ( names && ! names[0] ) names[0] = copy_string(host); } -DNS_Mapping::DNS_Mapping(uint32 addr, struct hostent* h) +DNS_Mapping::DNS_Mapping(uint32 addr, struct hostent* h, uint32 ttl) { Init(h); req_addr = addr; req_host = 0; + req_ttl = ttl; } DNS_Mapping::DNS_Mapping(FILE* f) @@ -663,6 +670,7 @@ Val* DNS_Mgr::BuildMappingVal(DNS_Mapping* dm) void DNS_Mgr::AddResult(DNS_Mgr_Request* dr, struct nb_dns_result* r) { struct hostent* h = (r && r->host_errno == 0) ? r->hostent : 0; + u_int32_t ttl = r->ttl; DNS_Mapping* new_dm; DNS_Mapping* prev_dm; @@ -670,7 +678,7 @@ void DNS_Mgr::AddResult(DNS_Mgr_Request* dr, struct nb_dns_result* r) if ( dr->ReqHost() ) { - new_dm = new DNS_Mapping(dr->ReqHost(), h); + new_dm = new DNS_Mapping(dr->ReqHost(), h, ttl); prev_dm = host_mappings.Insert(dr->ReqHost(), new_dm); if ( new_dm->Failed() && prev_dm && prev_dm->Valid() ) @@ -683,7 +691,7 @@ void DNS_Mgr::AddResult(DNS_Mgr_Request* dr, struct nb_dns_result* r) } else { - new_dm = new DNS_Mapping(dr->ReqAddr(), h); + new_dm = new DNS_Mapping(dr->ReqAddr(), h, ttl); uint32 tmp_addr = dr->ReqAddr(); HashKey k(&tmp_addr, 1); prev_dm = addr_mappings.Insert(&k, new_dm); @@ -833,9 +841,16 @@ const char* DNS_Mgr::LookupAddrInCache(dns_mgr_addr_type addr) { HashKey h(&addr, 1); DNS_Mapping* d = dns_mgr->addr_mappings.Lookup(&h); + if ( ! d ) return 0; + if ( d->Expired() ) + { + dns_mgr->addr_mappings.Remove(&h); + return 0; + } + // The escapes in the following strings are to avoid having it // interpreted as a trigraph sequence. return d->names ? d->names[0] : "<\?\?\?>"; @@ -844,9 +859,17 @@ const char* DNS_Mgr::LookupAddrInCache(dns_mgr_addr_type addr) TableVal* DNS_Mgr::LookupNameInCache(string name) { DNS_Mapping* d = dns_mgr->host_mappings.Lookup(name.c_str()); + if ( ! d || ! d->names ) return 0; + if ( d->Expired() ) + { + HashKey h(name.c_str()); + dns_mgr->host_mappings.Remove(&h); + return 0; + } + return d->AddrsSet(); } diff --git a/src/nb_dns.c b/src/nb_dns.c index 5033aadad4..225eb984cf 100644 --- a/src/nb_dns.c +++ b/src/nb_dns.c @@ -438,6 +438,7 @@ nb_dns_activity(struct nb_dns_info *nd, struct nb_dns_result *nr, char *errstr) register char **ap, **hap; register u_int16_t id; register const u_char *rdata; + register u_int32_t rttl; register struct hostent *he; register size_t rdlen; ns_msg handle; @@ -557,6 +558,7 @@ nb_dns_activity(struct nb_dns_info *nd, struct nb_dns_result *nr, char *errstr) rdata = ns_rr_rdata(rr); rdlen = ns_rr_rdlen(rr); + rttl = ns_rr_ttl(rr); switch (atype) { case T_A: @@ -603,10 +605,12 @@ nb_dns_activity(struct nb_dns_info *nd, struct nb_dns_result *nr, char *errstr) /* "Find first satisfactory answer" */ nr->hostent = he; + nr->ttl = rttl; return (1); } } nr->hostent = he; + nr->ttl = rttl; return (1); } diff --git a/src/nb_dns.h b/src/nb_dns.h index 41b5946e48..5787a3fdb0 100644 --- a/src/nb_dns.h +++ b/src/nb_dns.h @@ -11,6 +11,7 @@ struct nb_dns_result { void *cookie; int host_errno; struct hostent *hostent; + uint32_t ttl; }; typedef unsigned int nb_uint32_t; From a7ced3228b21d47410e98b63376f8c2ed865cf88 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sat, 25 Jun 2011 18:10:40 -0700 Subject: [PATCH 7/9] Removing language.rare-events test which is no longer meaningful. It relied on the now removed netstats_update event. --- .../Baseline/language.rare-events/output | 1 - testing/btest/language/rare-events.bro | 37 ------------------- 2 files changed, 38 deletions(-) delete mode 100644 testing/btest/Baseline/language.rare-events/output delete mode 100644 testing/btest/language/rare-events.bro diff --git a/testing/btest/Baseline/language.rare-events/output b/testing/btest/Baseline/language.rare-events/output deleted file mode 100644 index 1de3641284..0000000000 --- a/testing/btest/Baseline/language.rare-events/output +++ /dev/null @@ -1 +0,0 @@ -1106953531.452525 DroppedPackets 2 packets dropped after filtering, 1109 received, 10000 on link diff --git a/testing/btest/language/rare-events.bro b/testing/btest/language/rare-events.bro deleted file mode 100644 index ae7674d406..0000000000 --- a/testing/btest/language/rare-events.bro +++ /dev/null @@ -1,37 +0,0 @@ -# @TEST-EXEC: bro %INPUT >output 2>&1 -# @TEST-EXEC: btest-diff output - -# This is a test script whose job is to generate rarely-seen events -# (i.e., events that test traces might not include) to ensure that they're -# handled properly. - -# This is needed or else the output fails on the warning that -# Drop::restore_dropped_address is never defined. -redef check_for_unused_event_handlers = F; - -@load netstats - -function test_net_stats_update() - { - local t = current_time(); - - local s: net_stats; - s$pkts_recvd = 1234; - s$pkts_dropped = 123; - s$pkts_link = 9999; - - event net_stats_update(t, s); - - local s2: net_stats; - s2$pkts_recvd = 2341; - s2$pkts_dropped = 125; - s2$pkts_link = 19999; - - event net_stats_update(t + 33 sec, s2); - } - -event bro_init() - { - test_net_stats_update(); - } - From 704cc45165420d029bb99143ca4b199816f9f69c Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sat, 25 Jun 2011 18:11:33 -0700 Subject: [PATCH 8/9] Updating submodule(s). --- CHANGES | 539 +++++++++++++++++++++++++++++++++++++++++++++++++++ VERSION | 2 +- aux/broccoli | 2 +- 3 files changed, 541 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 449b5c9eb4..2707a7f905 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,542 @@ +1.6-dev.146 Sat Jun 25 18:12:27 PDT 2011 + +- DNS mapping are now becoming invalid when an entry's TTL expires. + (Thomas Other) + +- Merge remote branch 'origin/topic/jsiwek/script-load-cleanup' + + * origin/topic/jsiwek/script-load-cleanup: + Cleaning up the script loading implementation. + Fix baseline for core.conn-id (now core.conn-uid) test. (Robin Sommer) + +- Cleaning up the script loading implementation. + + This change primarily improves the way Bro detects and prevents + the same script from being loaded twice. It now compares inode + numbers instead of path names. (Jon Siwek) + +- Fix baseline for core.conn-id (now core.conn-uid) test. + + "conn-id.bro" was the name of a script in the policy/ directory that conn.bro + expected to @load, but when the unit test was also named "conn-id.bro", + it would be loaded twice during the test (once from conn.bro and once + as a command line argument to bro). This means two event handlers were + registered in error and the baseline output contained duplicate lines + that can be removed. (Jon Siwek) + +- Merge remote branch 'origin/topic/seth/net-stats-bif' + + * origin/topic/seth/net-stats-bif: + Removing a stray print statement. + Changed netstats (packet loss) handling to script-land. + + Nice idea to pass the old data into a regular scheduled event! + + Conflicts: + src/event.bif (Robin Sommer) + +- Removing a stray print statement. (Seth Hall) + +- Changed netstats (packet loss) handling to script-land. + + - Removed the net_stats_update event. + - Created a net_stats function for building and retrieving the + current network statistics. + - Removed the internal timer for firing the net_stats_update event + along with the global heartbeat_interval variable. + - Updated the netstats script to use the new BiF. + - Updated the stats script to use the new BiF. (Seth Hall) + +- Merge remote branch 'origin/fastpath' + + * origin/fastpath: + Raise internal error when failing to read contents of state file with -x option + + I've changed this to use error() instead of internal_error(). The + latter should only be used for logic errors that indicate a bug in + Bro. In this case, the message flags a problem that's more likely to + be external. (Robin Sommer) + +- Raise internal error when failing to read contents of state file with -x option + + Instead of just exiting w/ code 0 (Jon Siwek) + +- Fixing bug with logging &optional records. + + Closes #476. (Robin Sommer) + +- Merge remote branch 'origin/fastpath' + + * origin/fastpath: + Fix istate.events-ssl test failing because of expired cert. (Robin Sommer) + +- Fix istate.events-ssl test failing because of expired cert. + + Replaced expired certificate w/ one that's valid for ~100 years. (Jon Siwek) + +- Updating submodule(s). (Robin Sommer) + +- Merge remote branch 'origin/fastpath' + + * origin/fastpath: + Change bro doc mode to write out docs immediately after parsing. (Robin Sommer) + +- Merge branch 'master' into fastpath (Jon Siwek) + +- Merge remote branch 'origin/fastpath' (Robin Sommer) + +- Merge remote branch 'origin/topic/gregor/rpc' + + Note, I haven't gone through the script-level code as that will change + soon anyway. (Robin Sommer) + +- Change bro doc mode to write out docs immediately after parsing. + + Originally docs were written right after parsing, but it changed to after + the bro_init event happens when I was experimenting with auto-documenting + logging streams by querying the LogMgr after bro_init. That experiment + dead-ended, and that location is bad for other reasons: the doc framework + may try to access BroObj's that have already been freed. (Jon Siwek) + +- Really, null-terminate full 15-char NetBIOS host names, too. (Jon Siwek) + +- Fixed core.load-pkg test w/ diff canonifier instead (Jon Siwek) + +- Revert "Fix core.load-pkg unit test." + + This reverts commit 80558a994a7ef2040164f79b3992df1ee91bbae7. (Jon Siwek) + +- Fix language.wrong-delete-field test by running through abs path canonifier (Jon Siwek) + +- Fix bifs.unique_id-rnd test failing because of wc output formatting (Jon Siwek) + +- Null-terminate the string created by decode_netbios_name BiF. + + (initially observed through failures of bifs.netbios-functions unit test) (Jon Siwek) + +- Fix core.conn-id test on some platforms. + + The output of some versions of `wc` (e.g. MacOS) seems to indent + their output while others don't, causing the baseline diff to fail. + So pipe to sed to get rid of spaces before diffing. (Jon Siwek) + +- Fix core.load-pkg unit test. + + Removed the test's diff against baseline output that contained absolute + paths so that it will work across systems. Also don't redirect anything + to stderr so that failure information shows up in btest diagnostic output. (Jon Siwek) + +- Small but crucial fix for the new unique_id function. (Seth Hall) + +- A new bif unique_id(prefix) that returns a string that's unique across + Bro instaces with high probablity. + + "prefix" is a string that will be prepended to the returned ID. (Robin Sommer) + +- Merge remote branch 'origin/topic/seth/ssl-binpac' + + * origin/topic/seth/ssl-binpac: + Fixed bug due to vectors now initially indexed on 0. + Finished core support for new SSL analyzer. + SSL analyzer changes with accompanying BiF. + A table_s_of_s type to get around bifcl type limitation. + Regenerated the Mozilla CA bundle without the untrusted server authentication certs. + Complete rewrite to SSL analyzer. + + Conflicts: + src/AnalyzerTags.h + src/CMakeLists.txt + + Notes: + + - Haven't looked at the script-level, postponed to + policy-scripts-new. + + - I renamed X509Extension to X509_extension for consistency. (Robin Sommer) + +- Fixed bug due to vectors now initially indexed on 0. (Seth Hall) + +- Merge remote branch 'origin/master' into topic/seth/ssl-binpac + + Conflicts: + src/bro.bif (Seth Hall) + +- Finished core support for new SSL analyzer. + + - Certificate and certificate chain validation is now done + fully in policy script land. The script to do this will + be written in the new policy scripts branch once this is + merged. + + - Removed hand written SSL analyzer. + + - Rewrote and reworked much of the BinPAC SSL analyzer. (Seth Hall) + +- SSL analyzer changes with accompanying BiF. + + - Full DER certificates are extracted as strings to be used with + corresponding BiFs. + - x509_verify function to verify single certs and/or full certificate chains. (Seth Hall) + +- A table_s_of_s type to get around bifcl type limitation. (Seth Hall) + +- Merge remote branch 'origin/master' into topic/seth/ssl-binpac (Seth Hall) + +- Regenerated the Mozilla CA bundle without the untrusted server authentication certs. + + Certs intended for email protection and code signing have been removed + as well due to the change. (Seth Hall) + +- Complete rewrite to SSL analyzer. + + * I haven't removed handwritten analyzer code yet although it isn't built anymore. + * The ssl.bro script is just an example and doesn't keep any state yet. (Seth Hall) + +- Merge remote branch 'origin/topic/seth/syslog-analyzer' + + Note: I didn't merge anything in policy/*. Seems there was some + unrelated stuff in there, and the ticket says that the policy script + was a dummy for now anyway. + + * origin/topic/seth/syslog-analyzer: + Updates for syslog analyzer to prepare it for merging. + Added the initial syslog analyzer and policy script. + Adding some of the initial scripts that are going to be merged from my script repository. (Robin Sommer) + +- Updates for syslog analyzer to prepare it for merging. + + - Integrated with CMake. + - Analyzer only support syslog over UDP right now. + - Fixed small bug in the analyzer to make it generate events correctly. (Seth Hall) + +- Merge remote branch 'origin/master' into topic/seth/syslog-analyzer + + Conflicts: + src/Analyzer.cc + src/Makefile.am (Seth Hall) + +- Fixing typo in test. (Robin Sommer) + +- @load now supports loading a directory. + + With a directory "foo" somewhere in BROPATH, "@load foo" now checks if + there's a file "foo/__load__.bro". If so, it reads that file in. (If + not, Bro reports the same error as before, complaining that it can't + read a directory). (Robin Sommer) + +- ASCII logger now escapes non-printable characters. + + Closes #450. (Robin Sommer) + +- Updating submodule(s). (Robin Sommer) + +- Updating submodule(s). (Robin Sommer) + +- Revert "Setting the snaplen to 0 to capture the full packet regardless of size." + + This reverts commit fe274c3e64c845a2d587dc30254c439c530cf4a4. (Robin Sommer) + +- Setting the snaplen to 0 to capture the full packet regardless of size. + + In my limited testing this seemed to work fine but we should + make an actual test for this eventually. (Seth Hall) + +- Packaging tweaks and rewrite of 'dist' target. + + - Move binary packaging scripts out of source root into pkg/ subdir + - A consistent CMake version (2.8.4) is now enforced for binary packaging + - Added a 'bindist' target to top Makefile as a convenience + - The 'dist' target has been rewritten to depend on standard system + command/utils rather than CMake and the full dependency chain of Bro, + addressing #398 (but the CMake 'package_source' target is still available + in the generated build/Makefile and can be used if desired) (Jon Siwek) + +- Updating submodule(s). (Robin Sommer) + +- Merge remote branch 'origin/topic/jsiwek/deb-packaging' + + * origin/topic/jsiwek/deb-packaging: + Changes to allow DEB packaging via CPack, addresses #458 (Robin Sommer) + +- Changes to allow DEB packaging via CPack, addresses #458 (Jon Siwek) + +- Merge remote branch 'origin/fastpath' + + * origin/fastpath: + Fix reST markup generated for record redefs. + Fixes for more doc mode corner cases caused by type cloning. + + Jon, I added the line below, please double-check. + + diff --git a/src/Var.cc b/src/Var.cc + index 7880325..00ac734 100644 + --- a/src/Var.cc + +++ b/src/Var.cc + @@ -260,6 +260,7 @@ void add_type(ID* id, BroType* t, attr_list* attr, int /* is_event */) + tnew = new FuncType(t->AsFuncType()->Args(), + t->AsFuncType()->YieldType(), + t->AsFuncType()->IsEvent()); + + break; + default: + SerializationFormat* form = new BinarySerializationFormat(); + form->StartWrite(); (Robin Sommer) + +- Fix reST markup generated for record redefs. + + They should have been using reST roles to xref the original record type + instead of a reST directive to declare a new type. (Jon Siwek) + +- Fixes for more doc mode corner cases caused by type cloning. + + "shallow" copying has to be done for any type that can contain + record types in order to accommodate record redefs that add fields. (Jon Siwek) + +- An extension to the ICMP analyzer to handle redirects. + + The analyzer now raises icmp_redirect() events that come with the + redirection address. + + By Julien Sentier. (Robin Sommer) + +- Updating submodule(s). (Robin Sommer) + +- Merge branch 'fastpath' of ssh://git.bro-ids.org/bro into fastpath (Gregor Maier) + +- Fix CommentedTypeDecl to track whether it's in a record like TypeDecl does. (Jon Siwek) + +- Portability fixes for tests on MacOS. (Robin Sommer) + +- Merge branch 'master' of ssh://git.bro-ids.org/bro (Robin Sommer) + +- Fixing bug with uninitialized counter. (Robin Sommer) + +- Updating submodule(s). (Robin Sommer) + +- Sorting was still not consistent. (Robin Sommer) + +- Test updates. + + Includes splitting up one test which's output now depends on wether + we've compiled with IPv6 support or not. (Robin Sommer) + +- Removing old istate test-suite. (Robin Sommer) + +- Updating submodule(s). (Robin Sommer) + +- A hack to report missing GeoIP support only once. + + This closes #357, but #455 captures the need for a more general + solution. (Robin Sommer) + +- Bugfix: vectors in records were not initalized. + + Closes #421. (Robin Sommer) + +- If IPv6 default is not compiled in, the default BPF filters now + excludes IPv6 packets. (Robin Sommer) + +- New bif bro_has_ipv6() to check whether IPv6 support is compiled in. (Robin Sommer) + +- Bringing connection state history back, which was accidentally deleted + from conn.bro. + + However, this is primarily for the record, conn.bro will be replaced + with a new version soon. (Robin Sommer) + +- Updating btests and a Makefile. + + "make" now runs all the tests. (Robin Sommer) + +- Moving the test-scripts from the old test-suite over to btest. (Robin Sommer) + +- Fix for major bug in POP3 analyzer, which didn't recognize '.' + terminators in multi-line replies if the terminator was bare (no + newline). This caused it to ignore the rest of the session that it's + analyzing. + + Patch from #444 by Vern. (Robin Sommer) + +- Fix compiler warning with gcc-4.4.4 (Gregor Maier) + +- Fix `make doc` CMake 2.8.3 incompatibility. + + CMake 2.8.4 seems to be able to handle add_custom_target() + interdependencies with the DEPENDS arguments, but 2.8.3 does not. + + Using add_dependencies() to create top-level target + dependencies works in both cases. (Jon Siwek) + +- Fixing top-level Makefile target 'doc-clean' (now 'docclean') (Jon Siwek) + +- Merge remote branch 'remotes/origin/topic/jsiwek/doc-framework' + + * remotes/origin/topic/jsiwek/doc-framework: + Adding example documentation for a script's use of logging features. + Adding &log attribute to static attr_names array. + Small typo fix. + Bro doc mode now tracks record redefs that extend its field list. + BroBifDoc was unneeded; now dead code, so removed. + Bro doc mode now only does a "shallow" copy of declared record types + Bro's doc mode now terminates after processing bro_init but before net_run + Fixes related to `make doc` handling of script summary text (##! comments) + Overhaul of "doc" build target for generating policy script documentation. + Add parser error hint when in doc mode about checking ## comment syntax. + Move stuff related to policy script documentation from doc/ to doc/scripts/ + Fixing example.bro's auto-reST generation baseline test. (Robin Sommer) + +- Merge branch 'master' into topic/jsiwek/doc-framework (Jon Siwek) + +- Adding example documentation for a script's use of logging features. (Jon Siwek) + +- Adding &log attribute to static attr_names array. (Jon Siwek) + +- Small typo fix. (Jon Siwek) + +- Bro doc mode now tracks record redefs that extend its field list. (Jon Siwek) + +- BroBifDoc was unneeded; now dead code, so removed. (Jon Siwek) + +- Bro doc mode now only does a "shallow" copy of declared record types + + This is necessary so that the cloned type will be able to see additions + to the original type's list of fields (Jon Siwek) + +- Bro's doc mode now terminates after processing bro_init but before net_run + + Generated script reST documentation is also written out at this time + instead of at the end of lexical scanning. + + The persistence serializer will no longer write out Bro's state to the + .state directory when in doc mode. (Jon Siwek) + +- Merge branch 'master' into topic/jsiwek/doc-framework (Jon Siwek) + +- Merge branch 'master' into topic/jsiwek/doc-framework (Jon Siwek) + +- Fixes related to `make doc` handling of script summary text (##! comments) + + - Summary comments (##!) can now be placed at the beginning of + BiF files (but still outside C segments). An issue was fixed where + these comments would mistakenly be transferred into the generated + .func_def file and cause a compile error. I completely removed writing + any opt_ws value into the .func_def file because it was currently not + writing anything besides whitespace. + + - The generation of reST for the collecting of "groups" of policy + script documentation now happens at build time of `make doc` through the + use of a helper script rather than doing this at configure time so that + changes to summary text will always be reflected in the documentation. (Jon Siwek) + +- Merge branch 'master' into topic/jsiwek/doc-framework (Jon Siwek) + +- Overhaul of "doc" build target for generating policy script documentation. + + It's now all implemented in CMake scripting. + + The generation of reST docs is now a distinct target, "restdoc", while + the target to generate HTML docs, "doc", depends on "restdoc". reST doc + generation supports incremental builds (documentation for a given policy + script is only regenerated when it is out of date), but HTML doc generation + via ``make doc`` is not incremental (Sphinx always starts with fresh input). + + Building the "restdoc" target is now covered by a btest to ensure all + policy scripts are parse-able when Bro is in "doc mode". + + Generated reST docs should now support "@load"ing from subdirectories. e.g. + "@load foo/baz" and "@load bar/baz" will now generate the right xref links. (Jon Siwek) + +- Merge branch 'master' into topic/jsiwek/doc-framework (Jon Siwek) + +- Add parser error hint when in doc mode about checking ## comment syntax. (Jon Siwek) + +- Merge branch 'master' into topic/jsiwek/doc-framework (Jon Siwek) + +- Move stuff related to policy script documentation from doc/ to doc/scripts/ (Jon Siwek) + +- Fixing example.bro's auto-reST generation baseline test. + + Adds a diff canonifier that skips diffing the places where example.bro + may use MutableVal derivatives (e.g. sets/tables), which don't always + generate the same ordering in the reST docs across runs. (Jon Siwek) + +- Merge of Gregor's conn-size branch. + + If 'use_conn_size_analyzer' is true, the event engine tracks number of + packets and raw IP bytes per connection. If report_conn_size_analyzer + is true, these values are included as four new columns into conn.log + + I changed conn.bro so that the value of report_conn_size_analyzer + follows that of use_conn_size_analyzer. For the new conn.log, we + probably want to get rid of report_conn_size_analyzer anyway. (Robin Sommer) + +- Merge remote branch 'origin/fastpath' + + * origin/fastpath: + Fix compile errors possible on some platforms. (Robin Sommer) + +- Fix compile errors possible on some platforms. + + Include in some sources that require it; addresses #430 + + Places where STL's min() template function could get used with + arguments of differing types can fail to deduce the right template type. + These are fixed with some type tweaking of local variables and also + giving an explicit template argument for good measure. (Jon Siwek) + +- Updating submodule(s). (Robin Sommer) + +- Switching vectors from being 1-based to 0-based. + + This is obviously a change that break backwards-compatibility. I hope + I caught all cases where vectors are used ... + + I've completely removed the VECTOR_MIN constant. Turns out that was + already not working: some code pieces were nevertheless hard-coding + the 1-based indexing ... (Robin Sommer) + +- Updating submodule(s). (Robin Sommer) + +- Adding istate tests to default btest configuration. (Robin Sommer) + +- Increasing serialization format version for the recent 64-bit changes. (Robin Sommer) + +- Updating tests. + + The istate tests now all pass except for the SSL one. Still need to + figure out why it fails. (Robin Sommer) + +- Support for (mixed) MPLS and VLAN traffic, and a new default BPF + filter. (Seth Hall and Robin Sommer) + + - Merging in the patch from #264, which provides support for mixed + VLAN and MPLS traffic. + + - Changing Bro's default filter from being built dynamically to being + a static "ip or not ip". To get the old behaviour back (i.e., the + dynamically built filter), redef "all_packets" to false. + + - print-filter.bro now always prints the filter that Bro is actually + using, even if overriden from the command line. (Robin Sommer) + +- Fixing some more format strings. (Robin Sommer) + +- Changing the HTTP's analyzers internals to use 64-bit integers. + (Gregor Maier). + + This is the patch from #326, plus some cleanup. (Robin Sommer) + +- Updating submodule(s). (Robin Sommer) + +- Updating baselines. (Robin Sommer) + +- Fixing bug with deleting still unset record fields of table type. (Robin Sommer) + +- Added the initial syslog analyzer and policy script. (Seth Hall) + +- Adding some of the initial scripts that are going to be merged from + my script repository. (Seth Hall) + + 1.6-dev.99 Fri Apr 22 22:10:03 PDT 2011 - Extending the connection record with a unique identifier. (Robin diff --git a/VERSION b/VERSION index 359249d5d6..fd419c8cef 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.6-dev.99 +1.6-dev.146 diff --git a/aux/broccoli b/aux/broccoli index 8843da57dc..9866a00e78 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 8843da57dc8aee433550727dcbd1199824ca9da4 +Subproject commit 9866a00e78b088315e632960aaa230635bca326a From ed8301a4df1a977fa718e5481d848962c71cbde1 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sat, 25 Jun 2011 20:04:35 -0700 Subject: [PATCH 9/9] Updating CHANGES. --- CHANGES | 565 +++++++++----------------------------------------------- 1 file changed, 86 insertions(+), 479 deletions(-) diff --git a/CHANGES b/CHANGES index 2707a7f905..a1fc3623c2 100644 --- a/CHANGES +++ b/CHANGES @@ -3,539 +3,146 @@ - DNS mapping are now becoming invalid when an entry's TTL expires. (Thomas Other) -- Merge remote branch 'origin/topic/jsiwek/script-load-cleanup' - - * origin/topic/jsiwek/script-load-cleanup: - Cleaning up the script loading implementation. - Fix baseline for core.conn-id (now core.conn-uid) test. (Robin Sommer) +- Reworking how Bro tracks which scripts are already loaded. Rather + than paths, Bro now tracks inode numbers. (Jon Siwek) -- Cleaning up the script loading implementation. - - This change primarily improves the way Bro detects and prevents - the same script from being loaded twice. It now compares inode - numbers instead of path names. (Jon Siwek) +- New BiF netstats() to query packet capture statistics. The netstats + script now uses the new BiF to periocally report packets drops. The + net_stats_update() event and the heartbeat_interval global went + away. (Seth Hall) -- Fix baseline for core.conn-id (now core.conn-uid) test. - - "conn-id.bro" was the name of a script in the policy/ directory that conn.bro - expected to @load, but when the unit test was also named "conn-id.bro", - it would be loaded twice during the test (once from conn.bro and once - as a command line argument to bro). This means two event handlers were - registered in error and the baseline output contained duplicate lines - that can be removed. (Jon Siwek) +- Fixing bug with logging &optional records. Closes #476. (Robin + Sommer) -- Merge remote branch 'origin/topic/seth/net-stats-bif' - - * origin/topic/seth/net-stats-bif: - Removing a stray print statement. - Changed netstats (packet loss) handling to script-land. - - Nice idea to pass the old data into a regular scheduled event! - - Conflicts: - src/event.bif (Robin Sommer) +- Fixing istate.events-ssl test failing because of expired cert. (Jon + Siwek) -- Removing a stray print statement. (Seth Hall) +- A large number of improvements and fixes for Bro's doc mode. (Jon + Siwek) -- Changed netstats (packet loss) handling to script-land. - - - Removed the net_stats_update event. - - Created a net_stats function for building and retrieving the - current network statistics. - - Removed the internal timer for firing the net_stats_update event - along with the global heartbeat_interval variable. - - Updated the netstats script to use the new BiF. - - Updated the stats script to use the new BiF. (Seth Hall) +- Significant updates for RPC and NFS analyzers (Gregor Maier) -- Merge remote branch 'origin/fastpath' - - * origin/fastpath: - Raise internal error when failing to read contents of state file with -x option - - I've changed this to use error() instead of internal_error(). The - latter should only be used for logic errors that indicate a bug in - Bro. In this case, the message flags a problem that's more likely to - be external. (Robin Sommer) + * Unify semantics for UDP and TCP connections. -- Raise internal error when failing to read contents of state file with -x option - - Instead of just exiting w/ code 0 (Jon Siwek) + * RPC can now log to a log file if desired. -- Fixing bug with logging &optional records. - - Closes #476. (Robin Sommer) + * Portmapper can now log general activity to a log file and also log + actual port mappings. -- Merge remote branch 'origin/fastpath' - - * origin/fastpath: - Fix istate.events-ssl test failing because of expired cert. (Robin Sommer) + * NFS analyzer now supports significantly more procedure calls as + as file name tracking and file content extraction. -- Fix istate.events-ssl test failing because of expired cert. - - Replaced expired certificate w/ one that's valid for ~100 years. (Jon Siwek) +- NetBIOS fixes. (Jon Siwek) -- Updating submodule(s). (Robin Sommer) +- A number of unit tests are more robust and portable. (Jon Siwek) -- Merge remote branch 'origin/fastpath' - - * origin/fastpath: - Change bro doc mode to write out docs immediately after parsing. (Robin Sommer) +- A new BiF unique_id() that returns a string that's unique across Bro + instaces with high probablity. (Robin Sommer) -- Merge branch 'master' into fastpath (Jon Siwek) +- Complete rewrite of the BinPAC SSL analyzer. (Seth Hall) -- Merge remote branch 'origin/fastpath' (Robin Sommer) - -- Merge remote branch 'origin/topic/gregor/rpc' - - Note, I haven't gone through the script-level code as that will change - soon anyway. (Robin Sommer) - -- Change bro doc mode to write out docs immediately after parsing. - - Originally docs were written right after parsing, but it changed to after - the bro_init event happens when I was experimenting with auto-documenting - logging streams by querying the LogMgr after bro_init. That experiment - dead-ended, and that location is bad for other reasons: the doc framework - may try to access BroObj's that have already been freed. (Jon Siwek) - -- Really, null-terminate full 15-char NetBIOS host names, too. (Jon Siwek) - -- Fixed core.load-pkg test w/ diff canonifier instead (Jon Siwek) - -- Revert "Fix core.load-pkg unit test." - - This reverts commit 80558a994a7ef2040164f79b3992df1ee91bbae7. (Jon Siwek) - -- Fix language.wrong-delete-field test by running through abs path canonifier (Jon Siwek) - -- Fix bifs.unique_id-rnd test failing because of wc output formatting (Jon Siwek) - -- Null-terminate the string created by decode_netbios_name BiF. - - (initially observed through failures of bifs.netbios-functions unit test) (Jon Siwek) - -- Fix core.conn-id test on some platforms. - - The output of some versions of `wc` (e.g. MacOS) seems to indent - their output while others don't, causing the baseline diff to fail. - So pipe to sed to get rid of spaces before diffing. (Jon Siwek) - -- Fix core.load-pkg unit test. - - Removed the test's diff against baseline output that contained absolute - paths so that it will work across systems. Also don't redirect anything - to stderr so that failure information shows up in btest diagnostic output. (Jon Siwek) - -- Small but crucial fix for the new unique_id function. (Seth Hall) - -- A new bif unique_id(prefix) that returns a string that's unique across - Bro instaces with high probablity. - - "prefix" is a string that will be prepended to the returned ID. (Robin Sommer) - -- Merge remote branch 'origin/topic/seth/ssl-binpac' - - * origin/topic/seth/ssl-binpac: - Fixed bug due to vectors now initially indexed on 0. - Finished core support for new SSL analyzer. - SSL analyzer changes with accompanying BiF. - A table_s_of_s type to get around bifcl type limitation. - Regenerated the Mozilla CA bundle without the untrusted server authentication certs. - Complete rewrite to SSL analyzer. - - Conflicts: - src/AnalyzerTags.h - src/CMakeLists.txt - - Notes: - - - Haven't looked at the script-level, postponed to - policy-scripts-new. - - - I renamed X509Extension to X509_extension for consistency. (Robin Sommer) - -- Fixed bug due to vectors now initially indexed on 0. (Seth Hall) - -- Merge remote branch 'origin/master' into topic/seth/ssl-binpac - - Conflicts: - src/bro.bif (Seth Hall) - -- Finished core support for new SSL analyzer. - - - Certificate and certificate chain validation is now done - fully in policy script land. The script to do this will - be written in the new policy scripts branch once this is - merged. - - - Removed hand written SSL analyzer. - - - Rewrote and reworked much of the BinPAC SSL analyzer. (Seth Hall) - -- SSL analyzer changes with accompanying BiF. - - - Full DER certificates are extracted as strings to be used with + * DER certificates are extracted as strings to be used with corresponding BiFs. - - x509_verify function to verify single certs and/or full certificate chains. (Seth Hall) -- A table_s_of_s type to get around bifcl type limitation. (Seth Hall) + * x509_verify function to verify single certs and/or full + certificate chains. -- Merge remote branch 'origin/master' into topic/seth/ssl-binpac (Seth Hall) + * Removed hand written SSL analyzer. -- Regenerated the Mozilla CA bundle without the untrusted server authentication certs. - - Certs intended for email protection and code signing have been removed - as well due to the change. (Seth Hall) + * The ssl.bro script is just a place-holder for now. New version + will come with the other new scripts. -- Complete rewrite to SSL analyzer. - - * I haven't removed handwritten analyzer code yet although it isn't built anymore. - * The ssl.bro script is just an example and doesn't keep any state yet. (Seth Hall) +- New syslog analyzer. (Seth Hall) -- Merge remote branch 'origin/topic/seth/syslog-analyzer' - - Note: I didn't merge anything in policy/*. Seems there was some - unrelated stuff in there, and the ticket says that the policy script - was a dummy for now anyway. - - * origin/topic/seth/syslog-analyzer: - Updates for syslog analyzer to prepare it for merging. - Added the initial syslog analyzer and policy script. - Adding some of the initial scripts that are going to be merged from my script repository. (Robin Sommer) +- @load now supports loading a directory. With a directory "foo" + somewhere in BROPATH, "@load foo" now checks if there's a file + "foo/__load__.bro". If so, it reads that file in. (Robin Sommer) -- Updates for syslog analyzer to prepare it for merging. - - - Integrated with CMake. - - Analyzer only support syslog over UDP right now. - - Fixed small bug in the analyzer to make it generate events correctly. (Seth Hall) +- ASCII logger now escapes non-printable characters. Closes #450. + (Robin Sommer) -- Merge remote branch 'origin/master' into topic/seth/syslog-analyzer - - Conflicts: - src/Analyzer.cc - src/Makefile.am (Seth Hall) +- Packaging tweaks and rewrite of 'dist' target. (Jon Siwek) -- Fixing typo in test. (Robin Sommer) +- Changes to allow DEB packaging via CPack, addresses #458. (Jon + Siwek) -- @load now supports loading a directory. - - With a directory "foo" somewhere in BROPATH, "@load foo" now checks if - there's a file "foo/__load__.bro". If so, it reads that file in. (If - not, Bro reports the same error as before, complaining that it can't - read a directory). (Robin Sommer) - -- ASCII logger now escapes non-printable characters. - - Closes #450. (Robin Sommer) - -- Updating submodule(s). (Robin Sommer) - -- Updating submodule(s). (Robin Sommer) - -- Revert "Setting the snaplen to 0 to capture the full packet regardless of size." - - This reverts commit fe274c3e64c845a2d587dc30254c439c530cf4a4. (Robin Sommer) - -- Setting the snaplen to 0 to capture the full packet regardless of size. - - In my limited testing this seemed to work fine but we should - make an actual test for this eventually. (Seth Hall) - -- Packaging tweaks and rewrite of 'dist' target. - - - Move binary packaging scripts out of source root into pkg/ subdir - - A consistent CMake version (2.8.4) is now enforced for binary packaging - - Added a 'bindist' target to top Makefile as a convenience - - The 'dist' target has been rewritten to depend on standard system - command/utils rather than CMake and the full dependency chain of Bro, - addressing #398 (but the CMake 'package_source' target is still available - in the generated build/Makefile and can be used if desired) (Jon Siwek) - -- Updating submodule(s). (Robin Sommer) - -- Merge remote branch 'origin/topic/jsiwek/deb-packaging' - - * origin/topic/jsiwek/deb-packaging: - Changes to allow DEB packaging via CPack, addresses #458 (Robin Sommer) - -- Changes to allow DEB packaging via CPack, addresses #458 (Jon Siwek) - -- Merge remote branch 'origin/fastpath' - - * origin/fastpath: - Fix reST markup generated for record redefs. - Fixes for more doc mode corner cases caused by type cloning. - - Jon, I added the line below, please double-check. - - diff --git a/src/Var.cc b/src/Var.cc - index 7880325..00ac734 100644 - --- a/src/Var.cc - +++ b/src/Var.cc - @@ -260,6 +260,7 @@ void add_type(ID* id, BroType* t, attr_list* attr, int /* is_event */) - tnew = new FuncType(t->AsFuncType()->Args(), - t->AsFuncType()->YieldType(), - t->AsFuncType()->IsEvent()); - + break; - default: - SerializationFormat* form = new BinarySerializationFormat(); - form->StartWrite(); (Robin Sommer) - -- Fix reST markup generated for record redefs. - - They should have been using reST roles to xref the original record type - instead of a reST directive to declare a new type. (Jon Siwek) - -- Fixes for more doc mode corner cases caused by type cloning. - - "shallow" copying has to be done for any type that can contain - record types in order to accommodate record redefs that add fields. (Jon Siwek) - -- An extension to the ICMP analyzer to handle redirects. - - The analyzer now raises icmp_redirect() events that come with the - redirection address. - - By Julien Sentier. (Robin Sommer) - -- Updating submodule(s). (Robin Sommer) - -- Merge branch 'fastpath' of ssh://git.bro-ids.org/bro into fastpath (Gregor Maier) - -- Fix CommentedTypeDecl to track whether it's in a record like TypeDecl does. (Jon Siwek) - -- Portability fixes for tests on MacOS. (Robin Sommer) - -- Merge branch 'master' of ssh://git.bro-ids.org/bro (Robin Sommer) - -- Fixing bug with uninitialized counter. (Robin Sommer) - -- Updating submodule(s). (Robin Sommer) - -- Sorting was still not consistent. (Robin Sommer) - -- Test updates. - - Includes splitting up one test which's output now depends on wether - we've compiled with IPv6 support or not. (Robin Sommer) +- An extension to the ICMP analyzer to handle redirects. Julien + Sentier - Removing old istate test-suite. (Robin Sommer) -- Updating submodule(s). (Robin Sommer) +- A hack to report missing GeoIP support only once. This closes #357, + but #455 captures the need for a more general solution. (Robin + Sommer) -- A hack to report missing GeoIP support only once. - - This closes #357, but #455 captures the need for a more general - solution. (Robin Sommer) - -- Bugfix: vectors in records were not initalized. - - Closes #421. (Robin Sommer) +- Bugfix: vectors in records were not initalized. Closes #421. (Robin + Sommer) - If IPv6 default is not compiled in, the default BPF filters now - excludes IPv6 packets. (Robin Sommer) + excludes IPv6 packets. (Robin Sommer) -- New bif bro_has_ipv6() to check whether IPv6 support is compiled in. (Robin Sommer) +- New bif bro_has_ipv6() to check whether IPv6 support is compiled in. + (Robin Sommer) -- Bringing connection state history back, which was accidentally deleted - from conn.bro. - - However, this is primarily for the record, conn.bro will be replaced - with a new version soon. (Robin Sommer) +- Updating btests and a Makefile. "make" now runs all the tests. + (Robin Sommer) -- Updating btests and a Makefile. - - "make" now runs all the tests. (Robin Sommer) - -- Moving the test-scripts from the old test-suite over to btest. (Robin Sommer) +- Moving the test-scripts from the old test-suite over to btest. + (Robin Sommer) - Fix for major bug in POP3 analyzer, which didn't recognize '.' - terminators in multi-line replies if the terminator was bare (no - newline). This caused it to ignore the rest of the session that it's - analyzing. + terminators in multi-line replies if the terminator was bare (no + newline). This caused it to ignore the rest of the session that it's + analyzing. (Vern Paxson) - Patch from #444 by Vern. (Robin Sommer) - - Fix compiler warning with gcc-4.4.4 (Gregor Maier) - -- Fix `make doc` CMake 2.8.3 incompatibility. - - CMake 2.8.4 seems to be able to handle add_custom_target() - interdependencies with the DEPENDS arguments, but 2.8.3 does not. - - Using add_dependencies() to create top-level target - dependencies works in both cases. (Jon Siwek) - -- Fixing top-level Makefile target 'doc-clean' (now 'docclean') (Jon Siwek) - -- Merge remote branch 'remotes/origin/topic/jsiwek/doc-framework' - - * remotes/origin/topic/jsiwek/doc-framework: - Adding example documentation for a script's use of logging features. - Adding &log attribute to static attr_names array. - Small typo fix. - Bro doc mode now tracks record redefs that extend its field list. - BroBifDoc was unneeded; now dead code, so removed. - Bro doc mode now only does a "shallow" copy of declared record types - Bro's doc mode now terminates after processing bro_init but before net_run - Fixes related to `make doc` handling of script summary text (##! comments) - Overhaul of "doc" build target for generating policy script documentation. - Add parser error hint when in doc mode about checking ## comment syntax. - Move stuff related to policy script documentation from doc/ to doc/scripts/ - Fixing example.bro's auto-reST generation baseline test. (Robin Sommer) - -- Merge branch 'master' into topic/jsiwek/doc-framework (Jon Siwek) - -- Adding example documentation for a script's use of logging features. (Jon Siwek) + +- Adding example documentation for a script's use of logging features. + (Jon Siwek) - Adding &log attribute to static attr_names array. (Jon Siwek) + +- Bro can now track packet and byte counts per connection. (Gregor + Maier) -- Small typo fix. (Jon Siwek) - -- Bro doc mode now tracks record redefs that extend its field list. (Jon Siwek) - -- BroBifDoc was unneeded; now dead code, so removed. (Jon Siwek) - -- Bro doc mode now only does a "shallow" copy of declared record types + * If 'use_conn_size_analyzer' is true, the event engine tracks + number of packets and raw IP bytes per connection. If + report_conn_size_analyzer is true, these values are included as + four new columns into conn.log - This is necessary so that the cloned type will be able to see additions - to the original type's list of fields (Jon Siwek) + * I changed conn.bro so that the value of + report_conn_size_analyzer follows that of + use_conn_size_analyzer. For the new conn.log, we probably want + to get rid of report_conn_size_analyzer anyway. -- Bro's doc mode now terminates after processing bro_init but before net_run - - Generated script reST documentation is also written out at this time - instead of at the end of lexical scanning. - - The persistence serializer will no longer write out Bro's state to the - .state directory when in doc mode. (Jon Siwek) +- Fixing numerous compiler warnings and portability issues. (All) -- Merge branch 'master' into topic/jsiwek/doc-framework (Jon Siwek) +- Switching vectors from being 1-based to 0-based. Note that this is a + change that break backwards-compatibility. (Robin Sommer) -- Merge branch 'master' into topic/jsiwek/doc-framework (Jon Siwek) - -- Fixes related to `make doc` handling of script summary text (##! comments) - - - Summary comments (##!) can now be placed at the beginning of - BiF files (but still outside C segments). An issue was fixed where - these comments would mistakenly be transferred into the generated - .func_def file and cause a compile error. I completely removed writing - any opt_ws value into the .func_def file because it was currently not - writing anything besides whitespace. - - - The generation of reST for the collecting of "groups" of policy - script documentation now happens at build time of `make doc` through the - use of a helper script rather than doing this at configure time so that - changes to summary text will always be reflected in the documentation. (Jon Siwek) - -- Merge branch 'master' into topic/jsiwek/doc-framework (Jon Siwek) - -- Overhaul of "doc" build target for generating policy script documentation. - - It's now all implemented in CMake scripting. - - The generation of reST docs is now a distinct target, "restdoc", while - the target to generate HTML docs, "doc", depends on "restdoc". reST doc - generation supports incremental builds (documentation for a given policy - script is only regenerated when it is out of date), but HTML doc generation - via ``make doc`` is not incremental (Sphinx always starts with fresh input). - - Building the "restdoc" target is now covered by a btest to ensure all - policy scripts are parse-able when Bro is in "doc mode". - - Generated reST docs should now support "@load"ing from subdirectories. e.g. - "@load foo/baz" and "@load bar/baz" will now generate the right xref links. (Jon Siwek) - -- Merge branch 'master' into topic/jsiwek/doc-framework (Jon Siwek) - -- Add parser error hint when in doc mode about checking ## comment syntax. (Jon Siwek) - -- Merge branch 'master' into topic/jsiwek/doc-framework (Jon Siwek) - -- Move stuff related to policy script documentation from doc/ to doc/scripts/ (Jon Siwek) - -- Fixing example.bro's auto-reST generation baseline test. - - Adds a diff canonifier that skips diffing the places where example.bro - may use MutableVal derivatives (e.g. sets/tables), which don't always - generate the same ordering in the reST docs across runs. (Jon Siwek) - -- Merge of Gregor's conn-size branch. - - If 'use_conn_size_analyzer' is true, the event engine tracks number of - packets and raw IP bytes per connection. If report_conn_size_analyzer - is true, these values are included as four new columns into conn.log - - I changed conn.bro so that the value of report_conn_size_analyzer - follows that of use_conn_size_analyzer. For the new conn.log, we - probably want to get rid of report_conn_size_analyzer anyway. (Robin Sommer) - -- Merge remote branch 'origin/fastpath' - - * origin/fastpath: - Fix compile errors possible on some platforms. (Robin Sommer) - -- Fix compile errors possible on some platforms. - - Include in some sources that require it; addresses #430 - - Places where STL's min() template function could get used with - arguments of differing types can fail to deduce the right template type. - These are fixed with some type tweaking of local variables and also - giving an explicit template argument for good measure. (Jon Siwek) - -- Updating submodule(s). (Robin Sommer) - -- Switching vectors from being 1-based to 0-based. - - This is obviously a change that break backwards-compatibility. I hope - I caught all cases where vectors are used ... - - I've completely removed the VECTOR_MIN constant. Turns out that was - already not working: some code pieces were nevertheless hard-coding - the 1-based indexing ... (Robin Sommer) - -- Updating submodule(s). (Robin Sommer) - -- Adding istate tests to default btest configuration. (Robin Sommer) - -- Increasing serialization format version for the recent 64-bit changes. (Robin Sommer) - -- Updating tests. - - The istate tests now all pass except for the SSL one. Still need to - figure out why it fails. (Robin Sommer) +- Increasing serialization format version for the recent 64-bit + changes. (Robin Sommer) - Support for (mixed) MPLS and VLAN traffic, and a new default BPF - filter. (Seth Hall and Robin Sommer) + filter. (Seth Hall and Robin Sommer) - Merging in the patch from #264, which provides support for mixed VLAN and MPLS traffic. - - Changing Bro's default filter from being built dynamically to being - a static "ip or not ip". To get the old behaviour back (i.e., the - dynamically built filter), redef "all_packets" to false. + - Changing Bro's default filter from being built dynamically to + being a static "ip or not ip". To get the old behaviour back + (i.e., the dynamically built filter), redef "all_packets" to + false. - - print-filter.bro now always prints the filter that Bro is actually - using, even if overriden from the command line. (Robin Sommer) - -- Fixing some more format strings. (Robin Sommer) + - print-filter.bro now always prints the filter that Bro is + actually using, even if overriden from the command line. (Robin + Sommer) - Changing the HTTP's analyzers internals to use 64-bit integers. - (Gregor Maier). + (Gregor Maier). - This is the patch from #326, plus some cleanup. (Robin Sommer) - -- Updating submodule(s). (Robin Sommer) - -- Updating baselines. (Robin Sommer) - -- Fixing bug with deleting still unset record fields of table type. (Robin Sommer) - -- Added the initial syslog analyzer and policy script. (Seth Hall) - -- Adding some of the initial scripts that are going to be merged from - my script repository. (Seth Hall) - +- Fixing bug with deleting still unset record fields of table type. + (Robin Sommer) 1.6-dev.99 Fri Apr 22 22:10:03 PDT 2011