diff --git a/CHANGES b/CHANGES index b82ec24664..e8f4736a1c 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,14 @@ +2.1-820 | 2013-07-18 12:30:04 -0700 + + * Extending external canonifier to remove fractional values from + capture_loss.log. (Robin Sommer) + + * Canonifying internal order for plugins and their components to + make it deterministic. (Robin Sommer) + + * Small raw reader tweaks that got left our earlier. (Robin Sommer) + 2.1-814 | 2013-07-15 18:18:20 -0700 * Fixing raw reader crash when accessing nonexistant file, and diff --git a/VERSION b/VERSION index d474b25cd4..b45c71a46a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1-814 +2.1-820 diff --git a/aux/btest b/aux/btest index c2e73c9e1e..ce366206e3 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit c2e73c9e1efed6bfdf2d977d716c97773c39492e +Subproject commit ce366206e3407e534a786ad572c342e9f9fef26b diff --git a/scripts/base/protocols/dns/main.bro b/scripts/base/protocols/dns/main.bro index ea3ec016de..bf47519cd8 100644 --- a/scripts/base/protocols/dns/main.bro +++ b/scripts/base/protocols/dns/main.bro @@ -207,6 +207,11 @@ event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string) { if ( ans$answer_type == DNS_ANS ) { + if ( ! c?$dns ) + { + event conn_weird("dns_unmatched_reply", c, ""); + hook set_session(c, msg, F); + } c$dns$AA = msg$AA; c$dns$RA = msg$RA; diff --git a/src/analyzer/Component.h b/src/analyzer/Component.h index f3d91c7f90..9e12ed347e 100644 --- a/src/analyzer/Component.h +++ b/src/analyzer/Component.h @@ -72,7 +72,7 @@ public: * from what's passed to the constructor but upper-cased and * canonified to allow being part of a script-level ID. */ - const char* Name() const { return name; } + virtual const char* Name() const { return name; } /** * Returns a canonocalized version of the analyzer's name. The diff --git a/src/file_analysis/Component.h b/src/file_analysis/Component.h index 8b79436991..3cdc69efdf 100644 --- a/src/file_analysis/Component.h +++ b/src/file_analysis/Component.h @@ -64,7 +64,7 @@ public: * from what's passed to the constructor but upper-cased and * canonified to allow being part of a script-level ID. */ - const char* Name() const { return name; } + virtual const char* Name() const { return name; } /** * Returns a canonocalized version of the analyzer's name. The diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc index 98f1dfcab6..2820923a25 100644 --- a/src/input/readers/Raw.cc +++ b/src/input/readers/Raw.cc @@ -55,7 +55,7 @@ void Raw::DoClose() if ( file != 0 ) CloseInput(); - if ( buf != 0 ) + if ( buf != 0 ) { // we still have output that has not been flushed. Throw away. delete buf; @@ -169,8 +169,8 @@ bool Raw::OpenInput() Error(Fmt("Init: cannot open %s", fname.c_str())); return false; } - } fcntl(fileno(file), F_SETFD, FD_CLOEXEC); + } return true; } @@ -468,7 +468,7 @@ bool Raw::DoUpdate() if ( length == -3 ) return false; - else if ( length == -2 || length == -1 ) + else if ( length == -2 || length == -1 ) // no data ready or eof break; diff --git a/src/plugin/Component.h b/src/plugin/Component.h index 4ac448e466..ad02dc7e4b 100644 --- a/src/plugin/Component.h +++ b/src/plugin/Component.h @@ -45,6 +45,12 @@ public: */ component::Type Type() const; + /** + * Returns a descriptive name for the analyzer. This name must be + * unique across all components of the same type. + */ + virtual const char* Name() const = 0; + /** * Returns a textual representation of the component. The default * version just output the type. Derived version should call the diff --git a/src/plugin/Manager.cc b/src/plugin/Manager.cc index 93ed3f2b97..67f4dea2bd 100644 --- a/src/plugin/Manager.cc +++ b/src/plugin/Manager.cc @@ -30,9 +30,18 @@ bool Manager::LoadPluginsFrom(const std::string& dir) return false; } +static bool plugin_cmp(const Plugin* a, const Plugin* b) + { + return a->Name() < b->Name(); + } + bool Manager::RegisterPlugin(Plugin *plugin) { Manager::PluginsInternal()->push_back(plugin); + + // Sort plugins by name to make sure we have a deterministic order. + PluginsInternal()->sort(plugin_cmp); + return true; } diff --git a/src/plugin/Plugin.cc b/src/plugin/Plugin.cc index 084c49f51e..eaac8a3b25 100644 --- a/src/plugin/Plugin.cc +++ b/src/plugin/Plugin.cc @@ -156,9 +156,18 @@ Plugin::component_list Plugin::Components() const return components; } +static bool component_cmp(const Component* a, const Component* b) + { + return a->Name() < b->Name(); + } + void Plugin::AddComponent(Component* c) { components.push_back(c); + + // Sort components by name to make sure we have a deterministic + // order. + components.sort(component_cmp); } void Plugin::AddBifInitFunction(bif_init_func c) diff --git a/testing/btest/Baseline/core.print-bpf-filters/conn.log b/testing/btest/Baseline/core.print-bpf-filters/conn.log index 745673c027..166286203e 100644 --- a/testing/btest/Baseline/core.print-bpf-filters/conn.log +++ b/testing/btest/Baseline/core.print-bpf-filters/conn.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path conn -#open 2013-07-08-20-05-18 +#open 2013-07-18-00-18-33 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents #types time string addr port addr port enum string interval count count string bool count string count count count count table[string] 1278600802.069419 UWkUyAuUGXf 10.20.80.1 50343 10.0.0.15 80 tcp - 0.004152 9 3429 SF - 0 ShADadfF 7 381 7 3801 (empty) -#close 2013-07-08-20-05-18 +#close 2013-07-18-00-18-33 diff --git a/testing/btest/Baseline/core.print-bpf-filters/output b/testing/btest/Baseline/core.print-bpf-filters/output index 8ccc04b1a7..2f7a1d9386 100644 --- a/testing/btest/Baseline/core.print-bpf-filters/output +++ b/testing/btest/Baseline/core.print-bpf-filters/output @@ -3,38 +3,28 @@ #empty_field (empty) #unset_field - #path packet_filter -#open 2013-07-08-20-05-17 +#open 2013-07-19-02-54-13 #fields ts node filter init success #types time string string bool bool -1373313917.926565 - ip or not ip T T -#close 2013-07-08-20-05-17 +1374202453.158981 - ip or not ip T T +#close 2013-07-19-02-54-13 #separator \x09 #set_separator , #empty_field (empty) #unset_field - #path packet_filter -#open 2013-07-08-20-05-18 +#open 2013-07-19-02-54-13 #fields ts node filter init success #types time string string bool bool -1373313918.205206 - port 42 T T -#close 2013-07-08-20-05-18 +1374202453.437816 - port 42 T T +#close 2013-07-19-02-54-13 #separator \x09 #set_separator , #empty_field (empty) #unset_field - #path packet_filter -#open 2013-07-08-20-05-18 +#open 2013-07-19-02-54-13 #fields ts node filter init success #types time string string bool bool -1373313918.491383 - (vlan) and (ip or not ip) T T -#close 2013-07-08-20-05-18 -#separator \x09 -#set_separator , -#empty_field (empty) -#unset_field - -#path packet_filter -#open 2013-07-08-20-05-18 -#fields ts node filter init success -#types time string string bool bool -1373313918.795264 - ((((((((((((udp and port 3544) or (udp and port 514)) or ((tcp and port 2811) or (tcp and port 21))) or (tcp and port 502)) or ((((tcp and port 6669) or (tcp and port 6666)) or (tcp and port 6668)) or (tcp and port 6667))) or (tcp and port 1080)) or ((udp and port 2152) or (udp and port 2123))) or ((((((((tcp and port 631) or (tcp and port 8888)) or (tcp and port 3128)) or (tcp and port 80)) or (tcp and port 1080)) or (tcp and port 8000)) or (tcp and port 81)) or (tcp and port 8080))) or (udp and port 5072)) or ((tcp and port 25) or (tcp and port 587))) or (((((((((((tcp and port 5223) or (tcp and port 585)) or (tcp and port 614)) or (tcp and port 993)) or (tcp and port 636)) or (tcp and port 989)) or (tcp and port 995)) or (tcp and port 443)) or (tcp and port 563)) or (tcp and port 990)) or (tcp and port 992))) or (((((udp and port 5355) or (tcp and port 53)) or (udp and port 5353)) or (udp and port 137)) or (udp and port 53))) or (tcp and port 22) T T -#close 2013-07-08-20-05-18 +1374202453.715717 - (vlan) and (ip or not ip) T T +#close 2013-07-19-02-54-13 diff --git a/testing/btest/Baseline/core.print-bpf-filters/output2 b/testing/btest/Baseline/core.print-bpf-filters/output2 new file mode 100644 index 0000000000..99ad929fbf --- /dev/null +++ b/testing/btest/Baseline/core.print-bpf-filters/output2 @@ -0,0 +1,43 @@ +2 1080 +1 137 +1 21 +1 2123 +1 2152 +1 22 +1 25 +1 2811 +1 3128 +1 3544 +1 443 +1 502 +1 5072 +1 514 +1 5223 +2 53 +1 5353 +1 5355 +1 563 +1 585 +1 587 +1 614 +1 631 +1 636 +1 6666 +1 6667 +1 6668 +1 6669 +1 80 +1 8000 +1 8080 +1 81 +1 8888 +1 989 +1 990 +1 992 +1 993 +1 995 +40 and +39 or +40 port +31 tcp +9 udp diff --git a/testing/btest/Baseline/scripts.base.protocols.dns.duplicate-reponses/dns.log b/testing/btest/Baseline/scripts.base.protocols.dns.duplicate-reponses/dns.log new file mode 100644 index 0000000000..ca071ee8ef --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.dns.duplicate-reponses/dns.log @@ -0,0 +1,11 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path dns +#open 2013-07-18-13-21-52 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto trans_id query qclass qclass_name qtype qtype_name rcode rcode_name AA TC RD RA Z answers TTLs rejected +#types time string addr port addr port enum count string count string count string count string bool bool bool bool count vector[string] vector[interval] bool +1363716396.798072 UWkUyAuUGXf 55.247.223.174 27285 222.195.43.124 53 udp 21140 www.cmu.edu 1 C_INTERNET 1 A 0 NOERROR T F F F 1 www-cmu.andrew.cmu.edu,www-cmu-2.andrew.cmu.edu,128.2.10.163,www-cmu.andrew.cmu.edu 86400.000000,5.000000,21600.000000,86400.000000 F +1363716396.798374 UWkUyAuUGXf 55.247.223.174 27285 222.195.43.124 53 udp 21140 - - - - - 0 NOERROR T F F F 0 www-cmu-2.andrew.cmu.edu,128.2.10.163 5.000000,21600.000000 F +#close 2013-07-18-13-21-52 diff --git a/testing/btest/Baseline/scripts.base.protocols.dns.duplicate-reponses/weird.log b/testing/btest/Baseline/scripts.base.protocols.dns.duplicate-reponses/weird.log new file mode 100644 index 0000000000..c7de92f894 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.dns.duplicate-reponses/weird.log @@ -0,0 +1,11 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path weird +#open 2013-07-18-13-21-52 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer +#types time string addr port addr port string string bool string +1363716396.798286 UWkUyAuUGXf 55.247.223.174 27285 222.195.43.124 53 DNS_RR_unknown_type - F bro +1363716396.798374 UWkUyAuUGXf 55.247.223.174 27285 222.195.43.124 53 dns_unmatched_reply - F bro +#close 2013-07-18-13-21-52 diff --git a/testing/btest/Traces/dns-two-responses.trace b/testing/btest/Traces/dns-two-responses.trace new file mode 100644 index 0000000000..627b0d2ebe Binary files /dev/null and b/testing/btest/Traces/dns-two-responses.trace differ diff --git a/testing/btest/core/print-bpf-filters.bro b/testing/btest/core/print-bpf-filters.bro index 2c3d761cca..6e4a4d5c30 100644 --- a/testing/btest/core/print-bpf-filters.bro +++ b/testing/btest/core/print-bpf-filters.bro @@ -4,7 +4,12 @@ # @TEST-EXEC: cat packet_filter.log >>output # @TEST-EXEC: bro -r $TRACES/mixed-vlan-mpls.trace PacketFilter::restricted_filter="vlan" >>output # @TEST-EXEC: cat packet_filter.log >>output -# @TEST-EXEC: bro -r $TRACES/empty.trace PacketFilter::enable_auto_protocol_capture_filters=T >>output -# @TEST-EXEC: cat packet_filter.log >>output # @TEST-EXEC: btest-diff output # @TEST-EXEC: btest-diff conn.log +# +# The order in the output of enable_auto_protocol_capture_filters isn't +# stable, for reasons not clear. We canonify it first. +# @TEST-EXEC: bro -r $TRACES/empty.trace PacketFilter::enable_auto_protocol_capture_filters=T +# @TEST-EXEC: cat packet_filter.log | bro-cut filter | sed 's#[()]##g' | tr ' ' '\n' | sort | uniq -c | awk '{print $1, $2}' >output2 +# @TEST-EXEC: btest-diff output2 + diff --git a/testing/btest/scripts/base/protocols/dns/duplicate-reponses.bro b/testing/btest/scripts/base/protocols/dns/duplicate-reponses.bro new file mode 100644 index 0000000000..a16235b9a5 --- /dev/null +++ b/testing/btest/scripts/base/protocols/dns/duplicate-reponses.bro @@ -0,0 +1,5 @@ +# This tests the case where the DNS server responded with zero RRs. +# +# @TEST-EXEC: bro -r $TRACES/dns-two-responses.trace +# @TEST-EXEC: btest-diff dns.log +# @TEST-EXEC: btest-diff weird.log \ No newline at end of file diff --git a/testing/scripts/diff-canonifier-external b/testing/scripts/diff-canonifier-external index f4356154e4..37a51fa72f 100755 --- a/testing/scripts/diff-canonifier-external +++ b/testing/scripts/diff-canonifier-external @@ -2,10 +2,17 @@ # # Default canonifier used with the trace-based tests in testing/external/*. +addl="cat" + +if [ "$1" == "capture_loss.log" ]; then + addl="`dirname $0`/diff-remove-fractions" +fi + `dirname $0`/diff-remove-timestamps \ | `dirname $0`/diff-remove-uids \ | `dirname $0`/diff-remove-file-ids \ | `dirname $0`/diff-remove-x509-names \ | `dirname $0`/diff-canon-notice-policy \ - | `dirname $0`/diff-sort + | `dirname $0`/diff-sort \ + | eval $addl diff --git a/testing/scripts/diff-remove-fractions b/testing/scripts/diff-remove-fractions new file mode 100755 index 0000000000..975157913c --- /dev/null +++ b/testing/scripts/diff-remove-fractions @@ -0,0 +1,6 @@ +#! /usr/bin/env bash +# +# Replace fractions of double value (i.e., 3.14 -> 3.x). + +sed 's/\.[0-9]\{1,\}/.X/g' +