From ea43c154cf94b31f3a5eb51122418deceaa31855 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 27 Jun 2019 16:08:46 -0700 Subject: [PATCH 01/17] Un-deprecate file rotation functions - rotate_file - rotate_file_by_name - calc_next_rotate These still have use-cases even though no longer used for our logging functionality. E.g. rotate_file_by_name may be used to rotate pcap dump files. Also the log_rotate_base_time option was marked deprecated, but still used in the new logging framework. --- NEWS | 5 ----- scripts/base/init-bare.zeek | 4 +--- src/zeek.bif | 6 +++--- testing/btest/Baseline/coverage.bare-mode-errors/errors | 4 ---- 4 files changed, 4 insertions(+), 15 deletions(-) diff --git a/NEWS b/NEWS index 35b5678ef9..af8523e02d 100644 --- a/NEWS +++ b/NEWS @@ -491,11 +491,6 @@ Deprecated Functionality - The ``bro_is_terminating`` and ``bro_version`` function are deprecated and replaced by functions named ``zeek_is_terminating`` and ``zeek_version``. -- The ``rotate_file``, ``rotate_file_by_name`` and ``calc_next_rotate`` functions - were marked as deprecated. These functions were used with the old pre-2.0 logging - framework and are no longer used. They also were marked as deprecated in their - documentation, however the functions themselves did not carry the deprecation marker. - Bro 2.6 ======= diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index 1910dd82fb..23bb5897ef 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -775,8 +775,6 @@ type IPAddrAnonymizationClass: enum { OTHER_ADDR, }; -## Deprecated. -## ## .. zeek:see:: rotate_file rotate_file_by_name type rotate_info: record { old_name: string; ##< Original filename. @@ -4622,7 +4620,7 @@ module GLOBAL; ## BPF filter the user has set via the -f command line options. Empty if none. const cmd_line_bpf_filter = "" &redef; -## Deprecated. +## Base time of log rotations in 24-hour time format (``%H:%M``), e.g. "12:00". const log_rotate_base_time = "0:00" &redef; ## Write profiling info into this file in regular intervals. The easiest way to diff --git a/src/zeek.bif b/src/zeek.bif index 3e2eac3740..be6224d6b7 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -4621,7 +4621,7 @@ function get_file_name%(f: file%): string ## after the rotation, and the time when *f* was opened/closed. ## ## .. zeek:see:: rotate_file_by_name calc_next_rotate -function rotate_file%(f: file%): rotate_info &deprecated +function rotate_file%(f: file%): rotate_info %{ RecordVal* info = f->Rotate(); if ( info ) @@ -4645,7 +4645,7 @@ function rotate_file%(f: file%): rotate_info &deprecated ## after the rotation, and the time when *f* was opened/closed. ## ## .. zeek:see:: rotate_file calc_next_rotate -function rotate_file_by_name%(f: string%): rotate_info &deprecated +function rotate_file_by_name%(f: string%): rotate_info %{ RecordVal* info = new RecordVal(rotate_info); @@ -4699,7 +4699,7 @@ function rotate_file_by_name%(f: string%): rotate_info &deprecated ## Returns: The duration until the next file rotation time. ## ## .. zeek:see:: rotate_file rotate_file_by_name -function calc_next_rotate%(i: interval%) : interval &deprecated +function calc_next_rotate%(i: interval%) : interval %{ const char* base_time = log_rotate_base_time ? log_rotate_base_time->AsString()->CheckString() : 0; diff --git a/testing/btest/Baseline/coverage.bare-mode-errors/errors b/testing/btest/Baseline/coverage.bare-mode-errors/errors index a13c8849a1..e69de29bb2 100644 --- a/testing/btest/Baseline/coverage.bare-mode-errors/errors +++ b/testing/btest/Baseline/coverage.bare-mode-errors/errors @@ -1,4 +0,0 @@ -warning in /Users/johanna/bro/master/scripts/policy/misc/trim-trace-file.zeek, line 25: deprecated (rotate_file_by_name) -warning in /Users/johanna/bro/master/scripts/policy/misc/trim-trace-file.zeek, line 25: deprecated (rotate_file_by_name) -warning in /Users/johanna/bro/master/scripts/policy/misc/trim-trace-file.zeek, line 25: deprecated (rotate_file_by_name) -warning in /Users/johanna/bro/master/testing/btest/../../scripts//policy/misc/trim-trace-file.zeek, line 25: deprecated (rotate_file_by_name) From 88ffe060048409a92817b876d67e8e1c02e6a24c Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 27 Jun 2019 16:17:36 -0700 Subject: [PATCH 02/17] Un-deprecate anonymizer BIFs --- scripts/base/init-bare.zeek | 4 ---- src/event.bif | 2 +- src/zeek.bif | 3 ++- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index 23bb5897ef..e63c08d7df 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -755,8 +755,6 @@ global restrict_filters: table[string] of string &redef; ## :zeek:see:`Pcap::precompile_pcap_filter` and :zeek:see:`Pcap::precompile_pcap_filter`. type PcapFilterID: enum { None }; -## Deprecated. -## ## .. zeek:see:: anonymize_addr type IPAddrAnonymization: enum { KEEP_ORIG_ADDR, @@ -766,8 +764,6 @@ type IPAddrAnonymization: enum { PREFIX_PRESERVING_MD5, }; -## Deprecated. -## ## .. zeek:see:: anonymize_addr type IPAddrAnonymizationClass: enum { ORIG_ADDR, diff --git a/src/event.bif b/src/event.bif index 92f3532ef0..d5017804be 100644 --- a/src/event.bif +++ b/src/event.bif @@ -864,7 +864,7 @@ event kazaa_signature_found%(c: connection%); ## Deprecated. Will be removed. event gaobot_signature_found%(c: connection%); -## Deprecated. Will be removed. +## Shows an IP address anonymization mapping. event anonymization_mapping%(orig: addr, mapped: addr%); ## Deprecated. Will be removed. diff --git a/src/zeek.bif b/src/zeek.bif index be6224d6b7..dac9f09bad 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -5017,7 +5017,8 @@ function match_signatures%(c: connection, pattern_type: int, s: string, # =========================================================================== # -# Deprecated Functions +# Anonymization Functions +# (Not Fully Functional) # # =========================================================================== From b635cc240be6e08420ca0f4a60a960b5bbf9d3d3 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 27 Jun 2019 16:38:32 -0700 Subject: [PATCH 03/17] Remove deprecated/unused "packet" type --- NEWS | 1 + scripts/base/init-bare.zeek | 11 ----------- src/NetVar.cc | 4 ---- src/NetVar.h | 2 -- 4 files changed, 1 insertion(+), 17 deletions(-) diff --git a/NEWS b/NEWS index af8523e02d..3f987ee7aa 100644 --- a/NEWS +++ b/NEWS @@ -410,6 +410,7 @@ Removed Functionality - ``peer_id`` - ``event_peer`` + - ``packet`` - The following configuration options were deprecated in version 2.6 or below and are removed from this release: diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index e63c08d7df..a69f681f1f 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -644,17 +644,6 @@ type ReporterStats: record { weirds_by_type: table[string] of count; }; -## Deprecated. -## -## .. todo:: Remove. It's still declared internally but doesn't seem used anywhere -## else. -type packet: record { - conn: connection; - is_orig: bool; - seq: count; ##< seq=k => it is the kth *packet* of the connection - timestamp: time; -}; - ## Table type used to map variable names to their memory allocation. ## ## .. zeek:see:: global_sizes diff --git a/src/NetVar.cc b/src/NetVar.cc index 6c59f2a5fa..1ab9b6bcb6 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -145,8 +145,6 @@ double table_expire_interval; double table_expire_delay; int table_incremental_step; -RecordType* packet_type; - double connection_status_update_interval; int orig_addr_anonymization, resp_addr_anonymization; @@ -413,8 +411,6 @@ void init_net_var() software = internal_type("software")->AsRecordType(); software_version = internal_type("software_version")->AsRecordType(); - packet_type = internal_type("packet")->AsRecordType(); - orig_addr_anonymization = opt_internal_int("orig_addr_anonymization"); resp_addr_anonymization = opt_internal_int("resp_addr_anonymization"); other_addr_anonymization = opt_internal_int("other_addr_anonymization"); diff --git a/src/NetVar.h b/src/NetVar.h index 3c9d92eb4d..4d91514006 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -148,8 +148,6 @@ extern double table_expire_interval; extern double table_expire_delay; extern int table_incremental_step; -extern RecordType* packet_type; - extern int orig_addr_anonymization, resp_addr_anonymization; extern int other_addr_anonymization; extern TableVal* preserve_orig_addr; From bfd037989ba71fdf95c2dca9f7fee1ec49265ebb Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 27 Jun 2019 16:39:20 -0700 Subject: [PATCH 04/17] Remove deprecated open_log_file and log_file_name functions --- NEWS | 2 ++ scripts/base/init-bare.zeek | 21 ------------------- scripts/policy/misc/profiling.zeek | 12 ++++++++++- testing/btest/core/reporter.zeek | 2 +- testing/btest/language/sizeof.zeek | 2 +- .../scripts/base/frameworks/logging/file.zeek | 2 +- 6 files changed, 16 insertions(+), 25 deletions(-) diff --git a/NEWS b/NEWS index 3f987ee7aa..32eeaa7fe3 100644 --- a/NEWS +++ b/NEWS @@ -379,6 +379,8 @@ Removed Functionality - ``send_state`` - ``checkpoint_state`` - ``rescan_state`` + - ``log_file_name`` + - ``open_log_file`` - The following events were deprecated in version 2.6 or below and are completely removed from this release: diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index a69f681f1f..24f6e24389 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -1775,30 +1775,9 @@ type gtp_delete_pdp_ctx_response_elements: record { @load base/bif/strings.bif @load base/bif/option.bif -## Deprecated. This is superseded by the new logging framework. -global log_file_name: function(tag: string): string &redef; - -## Deprecated. This is superseded by the new logging framework. -global open_log_file: function(tag: string): file &redef; - global done_with_network = F; event net_done(t: time) { done_with_network = T; } -function log_file_name(tag: string): string - { - local suffix = getenv("ZEEK_LOG_SUFFIX"); - - if ( suffix == "" ) - suffix = "log"; - - return fmt("%s.%s", tag, suffix); - } - -function open_log_file(tag: string): file - { - return open(log_file_name(tag)); - } - ## Internal function. function add_interface(iold: string, inew: string): string { diff --git a/scripts/policy/misc/profiling.zeek b/scripts/policy/misc/profiling.zeek index 197d3a3f82..4b6f05f169 100644 --- a/scripts/policy/misc/profiling.zeek +++ b/scripts/policy/misc/profiling.zeek @@ -2,8 +2,18 @@ module Profiling; +function log_suffix(): string + { + local rval = getenv("ZEEK_LOG_SUFFIX"); + + if ( rval == "" ) + return "log"; + + return rval; + } + ## Set the profiling output file. -redef profiling_file = open_log_file("prof"); +redef profiling_file = open(fmt("prof.%s", Profiling::log_suffix())); ## Set the cheap profiling interval. redef profiling_interval = 15 secs; diff --git a/testing/btest/core/reporter.zeek b/testing/btest/core/reporter.zeek index 8591096c2b..8ed75500ad 100644 --- a/testing/btest/core/reporter.zeek +++ b/testing/btest/core/reporter.zeek @@ -32,7 +32,7 @@ event connection_established(c: connection) first = 0; } -global f = open_log_file("logger-test"); +global f = open("logger-test.log"); event reporter_info(t: time, msg: string, location: string) { diff --git a/testing/btest/language/sizeof.zeek b/testing/btest/language/sizeof.zeek index fc510afb70..48c69ddc27 100644 --- a/testing/btest/language/sizeof.zeek +++ b/testing/btest/language/sizeof.zeek @@ -24,7 +24,7 @@ global a6: addr = [::1]; global b: bool = T; global c: count = 10; global d: double = -1.23; -global f: file = open_log_file("sizeof_demo"); +global f: file = open("sizeof_demo.log"); global i: int = -10; global iv: interval = -5sec; global p: port = 80/tcp; diff --git a/testing/btest/scripts/base/frameworks/logging/file.zeek b/testing/btest/scripts/base/frameworks/logging/file.zeek index 6aa07f1699..b584f56fca 100644 --- a/testing/btest/scripts/base/frameworks/logging/file.zeek +++ b/testing/btest/scripts/base/frameworks/logging/file.zeek @@ -13,7 +13,7 @@ export { } &log; } -const foo_log = open_log_file("Foo") &redef; +const foo_log = open("Foo.log") &redef; event zeek_init() { From 2655a653314237f5354fdb644bf642d0d5867f63 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 27 Jun 2019 16:48:36 -0700 Subject: [PATCH 05/17] Remove unused software_version_found events - software_version_found - software_unparsed_version_found - software_parse_error --- NEWS | 5 + scripts/base/init-bare.zeek | 20 ---- src/Conn.cc | 167 ----------------------------- src/Conn.h | 12 --- src/NetVar.cc | 6 -- src/NetVar.h | 3 - src/analyzer/protocol/http/HTTP.cc | 132 ----------------------- src/analyzer/protocol/http/HTTP.h | 1 - src/event.bif | 53 --------- 9 files changed, 5 insertions(+), 394 deletions(-) diff --git a/NEWS b/NEWS index 32eeaa7fe3..2f58fb4070 100644 --- a/NEWS +++ b/NEWS @@ -406,6 +406,9 @@ Removed Functionality - ``remote_log`` - ``finished_send_state`` - ``remote_pong`` + - ``software_version_found`` + - ``software_unparsed_version_found`` + - ``software_parse_error`` - The following types/records were deprecated in version 2.6 or below and are removed from this release: @@ -413,6 +416,8 @@ Removed Functionality - ``peer_id`` - ``event_peer`` - ``packet`` + - ``software`` + - ``software_version`` - The following configuration options were deprecated in version 2.6 or below and are removed from this release: diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index 24f6e24389..8bbf6cfbc8 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -3930,26 +3930,6 @@ type signature_state: record { payload_size: count; ##< Payload size of the first matching packet of current endpoint. }; -# Deprecated. -# -# .. todo:: This type is no longer used. Remove any reference of this from the -# core. -type software_version: record { - major: int; - minor: int; - minor2: int; - addl: string; -}; - -# Deprecated. -# -# .. todo:: This type is no longer used. Remove any reference of this from the -# core. -type software: record { - name: string; - version: software_version; -}; - # Type used to report load samples via :zeek:see:`load_sample`. For now, it's a # set of names (event names, source file names, and perhaps ````), which were seen during the sample. diff --git a/src/Conn.cc b/src/Conn.cc index 31c1b1a191..dc510038f0 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -448,173 +448,6 @@ void Connection::Match(Rule::PatternType type, const u_char* data, int len, bool primary_PIA->Match(type, data, len, is_orig, bol, eol, clear_state); } -Val* Connection::BuildVersionVal(const char* s, int len) - { - Val* name = 0; - Val* major = 0; - Val* minor = 0; - Val* minor2 = 0; - Val* addl = 0; - - const char* last = s + len; - const char* e = s; - - // This is all just a guess... - - // Eat non-alpha-numerical chars. - for ( ; s < last && ! isalnum(*s); ++s ) - ; - - // Leading characters are the program name. - // (first character must not be a digit) - if ( isalpha(*s) ) - { - for ( e = s; e < last && ! is_version_sep(e, last); ++e ) - ; - - if ( s != e ) - name = new StringVal(e - s, s); - } - - // Find first number - that's the major version. - for ( s = e; s < last && ! isdigit(*s); ++s ) - ; - for ( e = s; e < last && isdigit(*e); ++e ) - ; - - if ( s != e ) - major = val_mgr->GetInt(atoi(s)); - - // Find second number seperated only by punctuation chars - - // that's the minor version. - for ( s = e; s < last && ispunct(*s); ++s ) - ; - for ( e = s; e < last && isdigit(*e); ++e ) - ; - - if ( s != e ) - minor = val_mgr->GetInt(atoi(s)); - - // Find second number seperated only by punctuation chars; - - // that's the minor version. - for ( s = e; s < last && ispunct(*s); ++s ) - ; - for ( e = s; e < last && isdigit(*e); ++e ) - ; - - if ( s != e ) - minor2 = val_mgr->GetInt(atoi(s)); - - // Anything after following punctuation and until next white space is - // an additional version string. - for ( s = e; s < last && ispunct(*s); ++s ) - ; - for ( e = s; e < last && ! isspace(*e); ++e ) - ; - - if ( s != e ) - addl = new StringVal(e - s, s); - - // If we do not have a name yet, the next alphanumerical string is it. - if ( ! name ) - { // eat non-alpha-numerical characters - for ( s = e; s < last && ! isalpha(*s); ++s ) - ; - - // Get name. - for ( e = s; e < last && (isalnum(*e) || *e == '_'); ++e ) - ; - - if ( s != e ) - name = new StringVal(e - s, s); - } - - // We need at least a name. - if ( ! name ) - { - Unref(major); - Unref(minor); - Unref(minor2); - Unref(addl); - return 0; - } - - RecordVal* version = new RecordVal(software_version); - version->Assign(0, major ? major : val_mgr->GetInt(-1)); - version->Assign(1, minor ? minor : val_mgr->GetInt(-1)); - version->Assign(2, minor2 ? minor2 : val_mgr->GetInt(-1)); - version->Assign(3, addl ? addl : val_mgr->GetEmptyString()); - - RecordVal* sw = new RecordVal(software); - sw->Assign(0, name); - sw->Assign(1, version); - - return sw; - } - -int Connection::VersionFoundEvent(const IPAddr& addr, const char* s, int len, - analyzer::Analyzer* analyzer) - { - if ( ! software_version_found && ! software_parse_error ) - return 1; - - if ( ! is_printable(s, len) ) - return 0; - - Val* val = BuildVersionVal(s, len); - if ( ! val ) - { - if ( software_parse_error ) - { - ConnectionEventFast(software_parse_error, analyzer, { - BuildConnVal(), - new AddrVal(addr), - new StringVal(len, s), - }); - } - return 0; - } - - if ( software_version_found ) - { - ConnectionEventFast(software_version_found, 0, { - BuildConnVal(), - new AddrVal(addr), - val, - new StringVal(len, s), - }); - } - else - Unref(val); - - return 1; - } - -int Connection::UnparsedVersionFoundEvent(const IPAddr& addr, - const char* full, int len, analyzer::Analyzer* analyzer) - { - // Skip leading white space. - while ( len && isspace(*full) ) - { - --len; - ++full; - } - - if ( ! is_printable(full, len) ) - return 0; - - if ( software_unparsed_version_found ) - { - ConnectionEventFast(software_unparsed_version_found, analyzer, { - BuildConnVal(), - new AddrVal(addr), - new StringVal(len, full), - }); - } - - return 1; - } - void Connection::Event(EventHandlerPtr f, analyzer::Analyzer* analyzer, const char* name) { if ( ! f ) diff --git a/src/Conn.h b/src/Conn.h index bd5ddaae92..1f0215e01f 100644 --- a/src/Conn.h +++ b/src/Conn.h @@ -160,18 +160,6 @@ public: void Match(Rule::PatternType type, const u_char* data, int len, bool is_orig, bool bol, bool eol, bool clear_state); - // Tries really hard to extract a program name and a version. - Val* BuildVersionVal(const char* s, int len); - - // Raises a software_version_found event based on the - // given string (returns false if it's not parseable). - int VersionFoundEvent(const IPAddr& addr, const char* s, int len, - analyzer::Analyzer* analyzer = 0); - - // Raises a software_unparsed_version_found event. - int UnparsedVersionFoundEvent(const IPAddr& addr, - const char* full_descr, int len, analyzer::Analyzer* analyzer); - // If a handler exists for 'f', an event will be generated. If 'name' is // given that event's first argument will be it, and it's second will be // the connection value. If 'name' is null, then the event's first diff --git a/src/NetVar.cc b/src/NetVar.cc index 1ab9b6bcb6..dee874e4be 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -138,9 +138,6 @@ double backdoor_stat_backoff; RecordType* backdoor_endp_stats; -RecordType* software; -RecordType* software_version; - double table_expire_interval; double table_expire_delay; int table_incremental_step; @@ -408,9 +405,6 @@ void init_net_var() backdoor_stat_backoff = opt_internal_double("backdoor_stat_backoff"); backdoor_endp_stats = internal_type("backdoor_endp_stats")->AsRecordType(); - software = internal_type("software")->AsRecordType(); - software_version = internal_type("software_version")->AsRecordType(); - orig_addr_anonymization = opt_internal_int("orig_addr_anonymization"); resp_addr_anonymization = opt_internal_int("resp_addr_anonymization"); other_addr_anonymization = opt_internal_int("other_addr_anonymization"); diff --git a/src/NetVar.h b/src/NetVar.h index 4d91514006..ca85809378 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -141,9 +141,6 @@ extern double backdoor_stat_backoff; extern RecordType* backdoor_endp_stats; -extern RecordType* software; -extern RecordType* software_version; - extern double table_expire_interval; extern double table_expire_delay; extern int table_incremental_step; diff --git a/src/analyzer/protocol/http/HTTP.cc b/src/analyzer/protocol/http/HTTP.cc index 291990119a..1dbdcf9bd8 100644 --- a/src/analyzer/protocol/http/HTTP.cc +++ b/src/analyzer/protocol/http/HTTP.cc @@ -1640,17 +1640,6 @@ int HTTP_Analyzer::ExpectReplyMessageBody() void HTTP_Analyzer::HTTP_Header(int is_orig, mime::MIME_Header* h) { -#if 0 - // ### Only call ParseVersion if we're tracking versions: - if ( istrequal(h->get_name(), "server") ) - ParseVersion(h->get_value(), - (is_orig ? Conn()->OrigAddr() : Conn()->RespAddr()), false); - - else if ( istrequal(h->get_name(), "user-agent") ) - ParseVersion(h->get_value(), - (is_orig ? Conn()->OrigAddr() : Conn()->RespAddr()), true); -#endif - // To be "liberal", we only look at "keep-alive" on the client // side, and if seen assume the connection to be persistent. // This seems fairly safe - at worst, the client does indeed @@ -1702,127 +1691,6 @@ void HTTP_Analyzer::HTTP_Header(int is_orig, mime::MIME_Header* h) } } -void HTTP_Analyzer::ParseVersion(data_chunk_t ver, const IPAddr& host, - bool user_agent) - { - int len = ver.length; - const char* data = ver.data; - - if ( software_unparsed_version_found ) - Conn()->UnparsedVersionFoundEvent(host, data, len, this); - - // The RFC defines: - // - // product = token ["/" product-version] - // product-version = token - // Server = "Server" ":" 1*( product | comment ) - - int offset; - data_chunk_t product, product_version; - int num_version = 0; - - while ( len > 0 ) - { - // Skip white space. - while ( len && mime::is_lws(*data) ) - { - ++data; - --len; - } - - // See if a comment is coming next. For User-Agent, - // we parse it, too. - if ( user_agent && len && *data == '(' ) - { - // Find end of comment. - const char* data_start = data; - const char* eoc = - data + mime::MIME_skip_lws_comments(len, data); - - // Split into parts. - // (This may get confused by nested comments, - // but we ignore this for now.) - const char* eot; - ++data; - while ( 1 ) - { - // Eat spaces. - while ( data < eoc && mime::is_lws(*data) ) - ++data; - - // Find end of token. - for ( eot = data; - eot < eoc && *eot != ';' && *eot != ')'; - ++eot ) - ; - - if ( eot == eoc ) - break; - - // Delete spaces at end of token. - for ( ; eot > data && mime::is_lws(*(eot-1)); --eot ) - ; - - if ( data != eot && software_version_found ) - Conn()->VersionFoundEvent(host, data, eot - data, this); - data = eot + 1; - } - - len -= eoc - data_start; - data = eoc; - continue; - } - - offset = mime::MIME_get_slash_token_pair(len, data, - &product, &product_version); - if ( offset < 0 ) - { - // I guess version detection is best-effort, - // so we do not complain in the final version - if ( num_version == 0 ) - HTTP_Event("bad_HTTP_version", - mime::new_string_val(len, data)); - - // Try to simply skip next token. - offset = mime::MIME_get_token(len, data, &product); - if ( offset < 0 ) - break; - - len -= offset; - data += offset; - } - - else - { - len -= offset; - data += offset; - - int version_len = - product.length + 1 + product_version.length; - - char* version_str = new char[version_len+1]; - char* s = version_str; - - memcpy(s, product.data, product.length); - - s += product.length; - *(s++) = '/'; - - memcpy(s, product_version.data, product_version.length); - - s += product_version.length; - *s = 0; - - if ( software_version_found ) - Conn()->VersionFoundEvent(host, version_str, - version_len, this); - - delete [] version_str; - ++num_version; - } - } - } - void HTTP_Analyzer::HTTP_EntityData(int is_orig, BroString* entity_data) { if ( http_entity_data ) diff --git a/src/analyzer/protocol/http/HTTP.h b/src/analyzer/protocol/http/HTTP.h index 743f9965e7..901f5e86d9 100644 --- a/src/analyzer/protocol/http/HTTP.h +++ b/src/analyzer/protocol/http/HTTP.h @@ -220,7 +220,6 @@ protected: const BroString* UnansweredRequestMethod(); - void ParseVersion(data_chunk_t ver, const IPAddr& host, bool user_agent); int HTTP_ReplyCode(const char* code_str); int ExpectReplyMessageBody(); diff --git a/src/event.bif b/src/event.bif index d5017804be..76a16fb071 100644 --- a/src/event.bif +++ b/src/event.bif @@ -530,59 +530,6 @@ event load_sample%(samples: load_sample_info, CPU: interval, dmem: int%); ## triggering the match will be passed on to the event. event signature_match%(state: signature_state, msg: string, data: string%); -## Generated when a protocol analyzer finds an identification of a software -## used on a system. This is a protocol-independent event that is fed by -## different analyzers. For example, the HTTP analyzer reports user-agent and -## server software by raising this event, assuming it can parse it (if not, -## :zeek:id:`software_parse_error` will be generated instead). -## -## c: The connection. -## -## host: The host running the reported software. -## -## s: A description of the software found. -## -## descr: The raw (unparsed) software identification string as extracted from -## the protocol. -## -## .. zeek:see:: software_parse_error software_unparsed_version_found -event software_version_found%(c: connection, host: addr, - s: software, descr: string%); - -## Generated when a protocol analyzer finds an identification of a software -## used on a system but cannot parse it. This is a protocol-independent event -## that is fed by different analyzers. For example, the HTTP analyzer reports -## user-agent and server software by raising this event if it cannot parse them -## directly (if it can :zeek:id:`software_version_found` will be generated -## instead). -## -## c: The connection. -## -## host: The host running the reported software. -## -## descr: The raw (unparsed) software identification string as extracted from -## the protocol. -## -## .. zeek:see:: software_version_found software_unparsed_version_found -event software_parse_error%(c: connection, host: addr, descr: string%); - -## Generated when a protocol analyzer finds an identification of a software -## used on a system. This is a protocol-independent event that is fed by -## different analyzers. For example, the HTTP analyzer reports user-agent and -## server software by raising this event. Different from -## :zeek:id:`software_version_found` and :zeek:id:`software_parse_error`, this -## event is always raised, independent of whether Zeek can parse the version -## string. -## -## c: The connection. -## -## host: The host running the reported software. -## -## str: The software identification string as extracted from the protocol. -## -## .. zeek:see:: software_parse_error software_version_found -event software_unparsed_version_found%(c: connection, host: addr, str: string%); - ## Generated each time Zeek's internal profiling log is updated. The file is ## defined by :zeek:id:`profiling_file`, and its update frequency by ## :zeek:id:`profiling_interval` and :zeek:id:`expensive_profiling_multiple`. From 5343924eb9a390025e309dbea0b02bbad28cfbaa Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 27 Jun 2019 16:52:05 -0700 Subject: [PATCH 06/17] Remove dead code: dump_used_event_handlers --- scripts/base/init-bare.zeek | 4 ---- src/NetVar.cc | 3 --- src/NetVar.h | 1 - src/main.cc | 12 ------------ 4 files changed, 20 deletions(-) diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index 8bbf6cfbc8..d002334a03 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -4723,10 +4723,6 @@ const time_machine_profiling = F &redef; ## If true, warns about unused event handlers at startup. const check_for_unused_event_handlers = F &redef; -# If true, dumps all invoked event handlers at startup. -# todo::Still used? -# const dump_used_event_handlers = F &redef; - ## Deprecated. const suppress_local_output = F &redef; diff --git a/src/NetVar.cc b/src/NetVar.cc index dee874e4be..d7e614b5fa 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -184,7 +184,6 @@ int dpd_ignore_ports; TableVal* likely_server_ports; int check_for_unused_event_handlers; -int dump_used_event_handlers; int suppress_local_output; @@ -236,8 +235,6 @@ void init_general_global_var() check_for_unused_event_handlers = opt_internal_int("check_for_unused_event_handlers"); - dump_used_event_handlers = - opt_internal_int("dump_used_event_handlers"); suppress_local_output = opt_internal_int("suppress_local_output"); diff --git a/src/NetVar.h b/src/NetVar.h index ca85809378..cb5ff3b599 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -186,7 +186,6 @@ extern int dpd_ignore_ports; extern TableVal* likely_server_ports; extern int check_for_unused_event_handlers; -extern int dump_used_event_handlers; extern int suppress_local_output; diff --git a/src/main.cc b/src/main.cc index 456920fb1b..456ab3f813 100644 --- a/src/main.cc +++ b/src/main.cc @@ -993,18 +993,6 @@ int main(int argc, char** argv) delete dead_handlers; - EventRegistry::string_list* alive_handlers = - event_registry->UsedHandlers(); - - if ( alive_handlers->length() > 0 && dump_used_event_handlers ) - { - reporter->Info("invoked event handlers:"); - for ( int i = 0; i < alive_handlers->length(); ++i ) - reporter->Info("%s", (*alive_handlers)[i]); - } - - delete alive_handlers; - if ( stmts ) { stmt_flow_type flow; From e9fefa65019d5ca097b19b9d35a2d2bb0bac0baa Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 27 Jun 2019 17:37:51 -0700 Subject: [PATCH 07/17] Remove deprecated print_hook event --- NEWS | 3 ++ scripts/base/init-bare.zeek | 3 -- src/File.cc | 1 - src/File.h | 4 -- src/Stmt.cc | 48 ++++++--------------- src/event.bif | 3 -- src/zeek.bif | 18 -------- testing/btest/bifs/enable_raw_output.test | 8 ---- testing/btest/language/raw_output_attr.test | 8 ---- 9 files changed, 15 insertions(+), 81 deletions(-) diff --git a/NEWS b/NEWS index 2f58fb4070..060b25192f 100644 --- a/NEWS +++ b/NEWS @@ -381,6 +381,7 @@ Removed Functionality - ``rescan_state`` - ``log_file_name`` - ``open_log_file`` + - ``disable_print_hook`` - The following events were deprecated in version 2.6 or below and are completely removed from this release: @@ -409,6 +410,7 @@ Removed Functionality - ``software_version_found`` - ``software_unparsed_version_found`` - ``software_parse_error`` + - ``print_hook`` - The following types/records were deprecated in version 2.6 or below and are removed from this release: @@ -437,6 +439,7 @@ Removed Functionality - ``ssl_ca_certificate`` - ``ssl_private_key`` - ``ssl_passphrase`` + - ``suppress_local_output`` - The following constants were used as part of deprecated functionality in version 2.6 or below and are removed from this release: diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index d002334a03..c35525fc23 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -4723,9 +4723,6 @@ const time_machine_profiling = F &redef; ## If true, warns about unused event handlers at startup. const check_for_unused_event_handlers = F &redef; -## Deprecated. -const suppress_local_output = F &redef; - ## Holds the filename of the trace file given with ``-w`` (empty if none). ## ## .. zeek:see:: record_all_packets diff --git a/src/File.cc b/src/File.cc index d1f706514f..28fd8bea9c 100644 --- a/src/File.cc +++ b/src/File.cc @@ -168,7 +168,6 @@ void BroFile::Init() is_open = 0; attrs = 0; buffered = true; - print_hook = true; raw_output = false; t = 0; diff --git a/src/File.h b/src/File.h index 07512d4465..ffd912ab39 100644 --- a/src/File.h +++ b/src/File.h @@ -63,9 +63,6 @@ public: // Get the file with the given name, opening it if it doesn't yet exist. static BroFile* GetFile(const char* name); - void DisablePrintHook() { print_hook = false; } - bool IsPrintHookEnabled() const { return print_hook; } - void EnableRawOutput() { raw_output = true; } bool IsRawOutput() const { return raw_output; } @@ -98,7 +95,6 @@ protected: Attributes* attrs; bool buffered; double open_time; - bool print_hook; bool raw_output; static const int MIN_BUFFER_SIZE = 1024; diff --git a/src/Stmt.cc b/src/Stmt.cc index 58bca4fc5b..913fe31573 100644 --- a/src/Stmt.cc +++ b/src/Stmt.cc @@ -203,49 +203,25 @@ Val* PrintStmt::DoExec(val_list* vals, stmt_flow_type& /* flow */) const ++offset; } - bool ph = print_hook && f->IsPrintHookEnabled(); - desc_style style = f->IsRawOutput() ? RAW_STYLE : STANDARD_STYLE; - if ( ! (suppress_local_output && ph) ) - { - if ( f->IsRawOutput() ) - { - ODesc d(DESC_READABLE); - d.SetFlush(0); - d.SetStyle(style); - - PrintVals(&d, vals, offset); - f->Write(d.Description(), d.Len()); - } - else - { - ODesc d(DESC_READABLE, f); - d.SetFlush(0); - d.SetStyle(style); - - PrintVals(&d, vals, offset); - f->Write("\n", 1); - } - } - - if ( ph ) + if ( f->IsRawOutput() ) { ODesc d(DESC_READABLE); + d.SetFlush(0); d.SetStyle(style); + PrintVals(&d, vals, offset); + f->Write(d.Description(), d.Len()); + } + else + { + ODesc d(DESC_READABLE, f); + d.SetFlush(0); + d.SetStyle(style); - if ( print_hook ) - { - ::Ref(f); - - // Note, this doesn't do remote printing. - mgr.Dispatch( - new Event( - print_hook, - {new Val(f), new StringVal(d.Len(), d.Description())}), - true); - } + PrintVals(&d, vals, offset); + f->Write("\n", 1); } return 0; diff --git a/src/event.bif b/src/event.bif index 76a16fb071..79e54b9d37 100644 --- a/src/event.bif +++ b/src/event.bif @@ -813,6 +813,3 @@ event gaobot_signature_found%(c: connection%); ## Shows an IP address anonymization mapping. event anonymization_mapping%(orig: addr, mapped: addr%); - -## Deprecated. Will be removed. -event print_hook%(f:file, s: string%); diff --git a/src/zeek.bif b/src/zeek.bif index dac9f09bad..005d0be541 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -4723,28 +4723,10 @@ function file_size%(f: string%) : double return new Val(double(s.st_size), TYPE_DOUBLE); %} -## Disables sending :zeek:id:`print_hook` events to remote peers for a given -## file. In a -## distributed setup, communicating Zeek instances generate the event -## :zeek:id:`print_hook` for each print statement and send it to the remote -## side. When disabled for a particular file, these events will not be -## propagated to other peers. -## -## f: The file to disable :zeek:id:`print_hook` events for. -## -## .. zeek:see:: enable_raw_output -function disable_print_hook%(f: file%): any - %{ - f->DisablePrintHook(); - return 0; - %} - ## Prevents escaping of non-ASCII characters when writing to a file. ## This function is equivalent to :zeek:attr:`&raw_output`. ## ## f: The file to disable raw output for. -## -## .. zeek:see:: disable_print_hook function enable_raw_output%(f: file%): any %{ f->EnableRawOutput(); diff --git a/testing/btest/bifs/enable_raw_output.test b/testing/btest/bifs/enable_raw_output.test index c46b6e317f..b685b44069 100644 --- a/testing/btest/bifs/enable_raw_output.test +++ b/testing/btest/bifs/enable_raw_output.test @@ -4,7 +4,6 @@ # @TEST-EXEC: zeek -b %INPUT # @TEST-EXEC: tr '\000' 'X' output # @TEST-EXEC: btest-diff output -# @TEST-EXEC: cmp myfile hookfile event zeek_init() { @@ -14,10 +13,3 @@ event zeek_init() print myfile, "hello\x00world", "hi"; close(myfile); } - -event print_hook(f: file, s: string) - { - local hookfile = open("hookfile"); - write_file(hookfile, s); - close(hookfile); - } diff --git a/testing/btest/language/raw_output_attr.test b/testing/btest/language/raw_output_attr.test index ccf616405e..7f57481807 100644 --- a/testing/btest/language/raw_output_attr.test +++ b/testing/btest/language/raw_output_attr.test @@ -4,7 +4,6 @@ # @TEST-EXEC: zeek -b %INPUT # @TEST-EXEC: tr '\000' 'X' output # @TEST-EXEC: btest-diff output -# @TEST-EXEC: cmp myfile hookfile # first check local variable of file type w/ &raw_output @@ -16,10 +15,3 @@ event zeek_init() print myfile, "hello\x00world", "hi"; close(myfile); } - -event print_hook(f: file, s: string) - { - local hookfile = open("hookfile"); - write_file(hookfile, s); - close(hookfile); - } From a52043363652a01c9cd3b93fef24ceb9cf6adce5 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 27 Jun 2019 17:48:01 -0700 Subject: [PATCH 08/17] Remove deprecated/unused irc_servers option --- NEWS | 1 + scripts/base/init-bare.zeek | 6 ------ src/NetVar.cc | 2 -- src/NetVar.h | 1 - 4 files changed, 1 insertion(+), 9 deletions(-) diff --git a/NEWS b/NEWS index 060b25192f..26695d19eb 100644 --- a/NEWS +++ b/NEWS @@ -440,6 +440,7 @@ Removed Functionality - ``ssl_private_key`` - ``ssl_passphrase`` - ``suppress_local_output`` + - ``irc_servers`` - The following constants were used as part of deprecated functionality in version 2.6 or below and are removed from this release: diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index c35525fc23..efef93d200 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -3855,12 +3855,6 @@ type PE::SectionHeader: record { } module GLOBAL; -## Deprecated. -## -## .. todo:: Remove. It's still declared internally but doesn't seem used anywhere -## else. -global irc_servers : set[addr] &redef; - ## Internal to the stepping stone detector. const stp_delta: interval &redef; diff --git a/src/NetVar.cc b/src/NetVar.cc index d7e614b5fa..922d5ffe4a 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -174,7 +174,6 @@ int sig_max_group_size; TableType* irc_join_list; RecordType* irc_join_info; -TableVal* irc_servers; int dpd_reassemble_first_packets; int dpd_buffer_size; @@ -429,7 +428,6 @@ void init_net_var() irc_join_info = internal_type("irc_join_info")->AsRecordType(); irc_join_list = internal_type("irc_join_list")->AsTableType(); - irc_servers = internal_val("irc_servers")->AsTableVal(); dpd_reassemble_first_packets = opt_internal_int("dpd_reassemble_first_packets"); diff --git a/src/NetVar.h b/src/NetVar.h index cb5ff3b599..f2b3353f56 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -176,7 +176,6 @@ extern int sig_max_group_size; extern TableType* irc_join_list; extern RecordType* irc_join_info; -extern TableVal* irc_servers; extern int dpd_reassemble_first_packets; extern int dpd_buffer_size; From a940cf3fb57c802bb0bc8efc4e5c6bf61d93fa3b Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 27 Jun 2019 18:05:32 -0700 Subject: [PATCH 09/17] Remove InterConn analyzer --- NEWS | 11 + scripts/base/frameworks/analyzer/main.zeek | 1 - scripts/base/init-bare.zeek | 32 -- src/NetVar.cc | 16 - src/NetVar.h | 8 - src/Sessions.cc | 2 - src/analyzer/Manager.cc | 7 - src/analyzer/Manager.h | 1 - src/analyzer/protocol/CMakeLists.txt | 1 - .../protocol/interconn/CMakeLists.txt | 9 - src/analyzer/protocol/interconn/InterConn.cc | 274 ------------------ src/analyzer/protocol/interconn/InterConn.h | 88 ------ src/analyzer/protocol/interconn/Plugin.cc | 25 -- src/analyzer/protocol/interconn/events.bif | 8 - .../canonified_loaded_scripts.log | 1 - .../canonified_loaded_scripts.log | 1 - testing/btest/Baseline/plugins.hooks/output | 23 +- 17 files changed, 18 insertions(+), 490 deletions(-) delete mode 100644 src/analyzer/protocol/interconn/CMakeLists.txt delete mode 100644 src/analyzer/protocol/interconn/InterConn.cc delete mode 100644 src/analyzer/protocol/interconn/InterConn.h delete mode 100644 src/analyzer/protocol/interconn/Plugin.cc delete mode 100644 src/analyzer/protocol/interconn/events.bif diff --git a/NEWS b/NEWS index 26695d19eb..29ea654d75 100644 --- a/NEWS +++ b/NEWS @@ -411,6 +411,8 @@ Removed Functionality - ``software_unparsed_version_found`` - ``software_parse_error`` - ``print_hook`` + - ``interconn_stats`` + - ``interconn_remove_conn`` - The following types/records were deprecated in version 2.6 or below and are removed from this release: @@ -441,6 +443,13 @@ Removed Functionality - ``ssl_passphrase`` - ``suppress_local_output`` - ``irc_servers`` + - ``interconn_min_interarrival`` + - ``interconn_max_interarrival`` + - ``interconn_max_keystroke_pkt_size`` + - ``interconn_default_pkt_size`` + - ``interconn_stat_period`` + - ``interconn_stat_backoff`` + - ``interconn_endp_stats`` - The following constants were used as part of deprecated functionality in version 2.6 or below and are removed from this release: @@ -486,6 +495,8 @@ Removed Functionality This is typically not necessary and it's a problem that is more appropriately addressed at the system configuration level. +- Removed the InterConn analyzer. + Deprecated Functionality ------------------------ diff --git a/scripts/base/frameworks/analyzer/main.zeek b/scripts/base/frameworks/analyzer/main.zeek index 8fd986e497..cfadffb739 100644 --- a/scripts/base/frameworks/analyzer/main.zeek +++ b/scripts/base/frameworks/analyzer/main.zeek @@ -124,7 +124,6 @@ export { ## A set of analyzers to disable by default at startup. The default set ## contains legacy analyzers that are no longer supported. global disabled_analyzers: set[Analyzer::Tag] = { - ANALYZER_INTERCONN, ANALYZER_STEPPINGSTONE, ANALYZER_BACKDOOR, ANALYZER_TCPSTATS, diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index efef93d200..e505698434 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -3864,38 +3864,6 @@ const stp_idle_min: interval &redef; ## Internal to the stepping stone detector. global stp_skip_src: set[addr] &redef; -## Deprecated. -const interconn_min_interarrival: interval &redef; - -## Deprecated. -const interconn_max_interarrival: interval &redef; - -## Deprecated. -const interconn_max_keystroke_pkt_size: count &redef; - -## Deprecated. -const interconn_default_pkt_size: count &redef; - -## Deprecated. -const interconn_stat_period: interval &redef; - -## Deprecated. -const interconn_stat_backoff: double &redef; - -## Deprecated. -type interconn_endp_stats: record { - num_pkts: count; - num_keystrokes_two_in_row: count; - num_normal_interarrivals: count; - num_8k0_pkts: count; - num_8k4_pkts: count; - is_partial: bool; - num_bytes: count; - num_7bit_ascii: count; - num_lines: count; - num_normal_lines: count; -}; - ## Deprecated. const backdoor_stat_period: interval &redef; diff --git a/src/NetVar.cc b/src/NetVar.cc index 922d5ffe4a..5ef7e9b9c5 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -125,14 +125,6 @@ double stp_delta; double stp_idle_min; TableVal* stp_skip_src; -double interconn_min_interarrival; -double interconn_max_interarrival; -int interconn_max_keystroke_pkt_size; -int interconn_default_pkt_size; -double interconn_stat_period; -double interconn_stat_backoff; -RecordType* interconn_endp_stats; - double backdoor_stat_period; double backdoor_stat_backoff; @@ -389,14 +381,6 @@ void init_net_var() stp_idle_min = opt_internal_double("stp_idle_min"); stp_skip_src = internal_val("stp_skip_src")->AsTableVal(); - interconn_min_interarrival = opt_internal_double("interconn_min_interarrival"); - interconn_max_interarrival = opt_internal_double("interconn_max_interarrival"); - interconn_max_keystroke_pkt_size = opt_internal_int("interconn_max_keystroke_pkt_size"); - interconn_default_pkt_size = opt_internal_int("interconn_default_pkt_size"); - interconn_stat_period = opt_internal_double("interconn_stat_period"); - interconn_stat_backoff = opt_internal_double("interconn_stat_backoff"); - interconn_endp_stats = internal_type("interconn_endp_stats")->AsRecordType(); - backdoor_stat_period = opt_internal_double("backdoor_stat_period"); backdoor_stat_backoff = opt_internal_double("backdoor_stat_backoff"); backdoor_endp_stats = internal_type("backdoor_endp_stats")->AsRecordType(); diff --git a/src/NetVar.h b/src/NetVar.h index f2b3353f56..28936b9c61 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -128,14 +128,6 @@ extern double stp_delta; extern double stp_idle_min; extern TableVal* stp_skip_src; -extern double interconn_min_interarrival; -extern double interconn_max_interarrival; -extern int interconn_max_keystroke_pkt_size; -extern int interconn_default_pkt_size; -extern double interconn_stat_period; -extern double interconn_stat_backoff; -extern RecordType* interconn_endp_stats; - extern double backdoor_stat_period; extern double backdoor_stat_backoff; diff --git a/src/Sessions.cc b/src/Sessions.cc index e668815cfb..019082a6f0 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -22,8 +22,6 @@ #include "analyzer/protocol/stepping-stone/events.bif.h" #include "analyzer/protocol/backdoor/BackDoor.h" #include "analyzer/protocol/backdoor/events.bif.h" -#include "analyzer/protocol/interconn/InterConn.h" -#include "analyzer/protocol/interconn/events.bif.h" #include "analyzer/protocol/arp/ARP.h" #include "analyzer/protocol/arp/events.bif.h" #include "Discard.h" diff --git a/src/analyzer/Manager.cc b/src/analyzer/Manager.cc index c7e156b41e..d62749284d 100644 --- a/src/analyzer/Manager.cc +++ b/src/analyzer/Manager.cc @@ -8,7 +8,6 @@ #include "protocol/backdoor/BackDoor.h" #include "protocol/conn-size/ConnSize.h" #include "protocol/icmp/ICMP.h" -#include "protocol/interconn/InterConn.h" #include "protocol/pia/PIA.h" #include "protocol/stepping-stone/SteppingStone.h" #include "protocol/tcp/TCP.h" @@ -89,7 +88,6 @@ void Manager::InitPreScript() // Cache these tags. analyzer_backdoor = GetComponentTag("BACKDOOR"); analyzer_connsize = GetComponentTag("CONNSIZE"); - analyzer_interconn = GetComponentTag("INTERCONN"); analyzer_stepping = GetComponentTag("STEPPINGSTONE"); analyzer_tcpstats = GetComponentTag("TCPSTATS"); } @@ -466,11 +464,6 @@ bool Manager::BuildInitialAnalyzerTree(Connection* conn) // can handle both reassembled and non-reassembled input. tcp->AddChildAnalyzer(new backdoor::BackDoor_Analyzer(conn), false); - if ( IsEnabled(analyzer_interconn) ) - // Add a InterConn analyzer if requested. This analyzer - // can handle both reassembled and non-reassembled input. - tcp->AddChildAnalyzer(new interconn::InterConn_Analyzer(conn), false); - if ( IsEnabled(analyzer_stepping) ) { // Add a SteppingStone analyzer if requested. The port diff --git a/src/analyzer/Manager.h b/src/analyzer/Manager.h index 8f6d982394..1f9fc21090 100644 --- a/src/analyzer/Manager.h +++ b/src/analyzer/Manager.h @@ -357,7 +357,6 @@ private: Tag analyzer_backdoor; Tag analyzer_connsize; - Tag analyzer_interconn; Tag analyzer_stepping; Tag analyzer_tcpstats; diff --git a/src/analyzer/protocol/CMakeLists.txt b/src/analyzer/protocol/CMakeLists.txt index 8ebded627b..906f5e32dc 100644 --- a/src/analyzer/protocol/CMakeLists.txt +++ b/src/analyzer/protocol/CMakeLists.txt @@ -18,7 +18,6 @@ add_subdirectory(http) add_subdirectory(icmp) add_subdirectory(ident) add_subdirectory(imap) -add_subdirectory(interconn) add_subdirectory(irc) add_subdirectory(krb) add_subdirectory(login) diff --git a/src/analyzer/protocol/interconn/CMakeLists.txt b/src/analyzer/protocol/interconn/CMakeLists.txt deleted file mode 100644 index c1cf40da3f..0000000000 --- a/src/analyzer/protocol/interconn/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ - -include(ZeekPlugin) - -include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) - -zeek_plugin_begin(Zeek InterConn) -zeek_plugin_cc(InterConn.cc Plugin.cc) -zeek_plugin_bif(events.bif) -zeek_plugin_end() diff --git a/src/analyzer/protocol/interconn/InterConn.cc b/src/analyzer/protocol/interconn/InterConn.cc deleted file mode 100644 index e9a9378c90..0000000000 --- a/src/analyzer/protocol/interconn/InterConn.cc +++ /dev/null @@ -1,274 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#include "zeek-config.h" - -#include "InterConn.h" -#include "Event.h" -#include "Net.h" -#include "analyzer/protocol/tcp/TCP.h" - -#include "events.bif.h" - -using namespace analyzer::interconn; - -InterConnEndpoint::InterConnEndpoint(tcp::TCP_Endpoint* e) - { - endp = e; - max_top_seq = 0; - num_pkts = num_keystrokes_two_in_a_row = num_normal_interarrivals = - num_8k0_pkts = num_8k4_pkts = num_bytes = num_7bit_ascii = - num_lines = num_normal_lines = 0; - is_partial = keystroke_just_seen = 0; - last_keystroke_time = 0.0; - } - -#define NORMAL_LINE_LENGTH 80 - -int InterConnEndpoint::DataSent(double t, uint64 seq, int len, int caplen, - const u_char* data, const IP_Hdr* /* ip */, - const struct tcphdr* /* tp */) - { - if ( caplen < len ) - len = caplen; - - if ( len <= 0 ) - return 0; - - if ( endp->state == tcp::TCP_ENDPOINT_PARTIAL ) - is_partial = 1; - - uint64 ack = endp->ToRelativeSeqSpace(endp->AckSeq(), endp->AckWraps()); - uint64 top_seq = seq + len; - - if ( top_seq <= ack || top_seq <= max_top_seq ) - // There is no new data in this packet - return 0; - - if ( seq < max_top_seq ) - { // Only consider new data - int64 amount_seen = max_top_seq - seq; - seq += amount_seen; - data += amount_seen; - len -= amount_seen; - } - - if ( max_top_seq && seq > max_top_seq ) - // We've got a pkt above a hole - num_pkts += EstimateGapPacketNum(seq - max_top_seq); - - ++num_pkts; - max_top_seq = top_seq; - - // Count the bytes. - num_bytes += len; - - int last_char = 0; - int offset = 0; // where we consider the latest line to have begun - - for ( int i = 0; i < len; ++i ) - { - unsigned int c = data[i]; - - if ( c == '\n' && last_char == '\r' ) - { - // Compress CRLF to just one line termination. - last_char = c; - continue; - } - - if ( c == '\n' || c == '\r' ) - { - ++num_lines; - if ( i - offset <= NORMAL_LINE_LENGTH ) - ++num_normal_lines; - offset = i; - } - - else if ( c != 0 && c < 128 ) - ++num_7bit_ascii; - - last_char = c; - } - - if ( IsPotentialKeystrokePacket(len) ) - { - if ( keystroke_just_seen ) - { - ++num_keystrokes_two_in_a_row; - - if ( IsNormalKeystrokeInterarrival(t - last_keystroke_time) ) - ++num_normal_interarrivals; - } - else - keystroke_just_seen = 1; - - // Look for packets matching the SSH signature of - // being either 0 or 4 modulo 8. - switch ( len & 7 ) { - case 0: - if ( len >= 16 ) - ++num_8k0_pkts; - break; - - case 4: - ++num_8k4_pkts; - break; - } - - last_keystroke_time = t; - } - else - keystroke_just_seen = 0; - - return 1; - } - -RecordVal* InterConnEndpoint::BuildStats() - { - RecordVal* stats = new RecordVal(interconn_endp_stats); - - stats->Assign(0, val_mgr->GetCount(num_pkts)); - stats->Assign(1, val_mgr->GetCount(num_keystrokes_two_in_a_row)); - stats->Assign(2, val_mgr->GetCount(num_normal_interarrivals)); - stats->Assign(3, val_mgr->GetCount(num_8k0_pkts)); - stats->Assign(4, val_mgr->GetCount(num_8k4_pkts)); - stats->Assign(5, val_mgr->GetBool(is_partial)); - stats->Assign(6, val_mgr->GetCount(num_bytes)); - stats->Assign(7, val_mgr->GetCount(num_7bit_ascii)); - stats->Assign(8, val_mgr->GetCount(num_lines)); - stats->Assign(9, val_mgr->GetCount(num_normal_lines)); - - return stats; - } - -int InterConnEndpoint::EstimateGapPacketNum(int gap) const - { - return (gap + interconn_default_pkt_size - 1) / interconn_default_pkt_size; - } - -int InterConnEndpoint::IsPotentialKeystrokePacket(int len) const - { - return len <= interconn_max_keystroke_pkt_size; - } - -int InterConnEndpoint::IsNormalKeystrokeInterarrival(double t) const - { - return interconn_min_interarrival <= t && t <= interconn_max_interarrival; - } - -InterConn_Analyzer::InterConn_Analyzer(Connection* c) -: tcp::TCP_ApplicationAnalyzer("INTERCONN", c) - { - orig_endp = resp_endp = 0; - orig_stream_pos = resp_stream_pos = 1; - - timeout = backdoor_stat_period; - backoff = backdoor_stat_backoff; - - c->GetTimerMgr()->Add(new InterConnTimer(network_time + timeout, this)); - } - -InterConn_Analyzer::~InterConn_Analyzer() - { - Unref(orig_endp); - Unref(resp_endp); - } - -void InterConn_Analyzer::Init() - { - tcp::TCP_ApplicationAnalyzer::Init(); - - assert(TCP()); - orig_endp = new InterConnEndpoint(TCP()->Orig()); - resp_endp = new InterConnEndpoint(TCP()->Resp()); - } - -void InterConn_Analyzer::DeliverPacket(int len, const u_char* data, - bool is_orig, uint64 seq, const IP_Hdr* ip, int caplen) - { - tcp::TCP_ApplicationAnalyzer::DeliverPacket(len, data, is_orig, - seq, ip, caplen); - - if ( is_orig ) - orig_endp->DataSent(network_time, seq, len, caplen, data, 0, 0); - else - resp_endp->DataSent(network_time, seq, len, caplen, data, 0, 0); - } - -void InterConn_Analyzer::DeliverStream(int len, const u_char* data, bool is_orig) - { - tcp::TCP_ApplicationAnalyzer::DeliverStream(len, data, is_orig); - - if ( is_orig ) - { - orig_endp->DataSent(network_time, orig_stream_pos, len, len, data, 0, 0); - orig_stream_pos += len; - } - - else - { - resp_endp->DataSent(network_time, resp_stream_pos, len, len, data, 0, 0); - resp_stream_pos += len; - } - } - -void InterConn_Analyzer::Done() - { - if ( ! IsFinished() ) - { - if ( ! Conn()->Skipping() ) - StatEvent(); - - RemoveEvent(); - } - - tcp::TCP_ApplicationAnalyzer::Done(); - } - -void InterConn_Analyzer::StatTimer(double t, int is_expire) - { - if ( IsFinished() || Conn()->Skipping() ) - return; - - StatEvent(); - - if ( ! is_expire ) - { - timeout *= backoff; - timer_mgr->Add(new InterConnTimer(t + timeout, this)); - } - } - -void InterConn_Analyzer::StatEvent() - { - if ( interconn_stats ) - Conn()->ConnectionEventFast(interconn_stats, this, { - Conn()->BuildConnVal(), - orig_endp->BuildStats(), - resp_endp->BuildStats(), - }); - } - -void InterConn_Analyzer::RemoveEvent() - { - if ( interconn_remove_conn ) - Conn()->ConnectionEventFast(interconn_remove_conn, this, {Conn()->BuildConnVal()}); - } - -InterConnTimer::InterConnTimer(double t, InterConn_Analyzer* a) -: Timer(t, TIMER_INTERCONN) - { - analyzer = a; - // Make sure connection does not expire. - Ref(a->Conn()); - } - -InterConnTimer::~InterConnTimer() - { - Unref(analyzer->Conn()); - } - -void InterConnTimer::Dispatch(double t, int is_expire) - { - analyzer->StatTimer(t, is_expire); - } diff --git a/src/analyzer/protocol/interconn/InterConn.h b/src/analyzer/protocol/interconn/InterConn.h deleted file mode 100644 index 04d5a2b5c6..0000000000 --- a/src/analyzer/protocol/interconn/InterConn.h +++ /dev/null @@ -1,88 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#ifndef ANALYZER_PROTOCOL_INTERCONN_INTERCONN_H -#define ANALYZER_PROTOCOL_INTERCONN_INTERCONN_H - -#include "analyzer/protocol/tcp/TCP.h" -#include "Timer.h" -#include "NetVar.h" - -namespace analyzer { namespace interconn { - -class InterConnEndpoint : public BroObj { -public: - explicit InterConnEndpoint(tcp::TCP_Endpoint* e); - - int DataSent(double t, uint64 seq, int len, int caplen, const u_char* data, - const IP_Hdr* ip, const struct tcphdr* tp); - - RecordVal* BuildStats(); - -protected: - int EstimateGapPacketNum(int gap) const; - int IsPotentialKeystrokePacket(int len) const; - int IsNormalKeystrokeInterarrival(double t) const; - - tcp::TCP_Endpoint* endp; - double last_keystroke_time; - uint64 max_top_seq; - uint32 num_pkts; - uint32 num_keystrokes_two_in_a_row; - uint32 num_normal_interarrivals; - uint32 num_8k4_pkts; - uint32 num_8k0_pkts; - uint32 num_bytes; - uint32 num_7bit_ascii; - uint32 num_lines; - uint32 num_normal_lines; - int is_partial; - int keystroke_just_seen; -}; - - -class InterConn_Analyzer : public tcp::TCP_ApplicationAnalyzer { -public: - explicit InterConn_Analyzer(Connection* c); - ~InterConn_Analyzer() override; - - void Init() override; - void Done() override; - void StatTimer(double t, int is_expire); - - static analyzer::Analyzer* Instantiate(Connection* conn) - { return new InterConn_Analyzer(conn); } - -protected: - // We support both packet and stream input and can be put in place even - // if the TCP analyzer is not yet reassembling. - void DeliverPacket(int len, const u_char* data, bool is_orig, - uint64 seq, const IP_Hdr* ip, int caplen) override; - void DeliverStream(int len, const u_char* data, bool is_orig) override; - - void StatEvent(); - void RemoveEvent(); - - InterConnEndpoint* orig_endp; - InterConnEndpoint* resp_endp; - - int orig_stream_pos; - int resp_stream_pos; - - double timeout; - double backoff; -}; - -class InterConnTimer : public Timer { -public: - InterConnTimer(double t, InterConn_Analyzer* a); - ~InterConnTimer() override; - - void Dispatch(double t, int is_expire) override; - -protected: - InterConn_Analyzer* analyzer; -}; - -} } // namespace analyzer::* - -#endif diff --git a/src/analyzer/protocol/interconn/Plugin.cc b/src/analyzer/protocol/interconn/Plugin.cc deleted file mode 100644 index bbd1b866ed..0000000000 --- a/src/analyzer/protocol/interconn/Plugin.cc +++ /dev/null @@ -1,25 +0,0 @@ -// See the file in the main distribution directory for copyright. - - -#include "plugin/Plugin.h" - -#include "InterConn.h" - -namespace plugin { -namespace Zeek_InterConn { - -class Plugin : public plugin::Plugin { -public: - plugin::Configuration Configure() - { - AddComponent(new ::analyzer::Component("InterConn", ::analyzer::interconn::InterConn_Analyzer::Instantiate)); - - plugin::Configuration config; - config.name = "Zeek::InterConn"; - config.description = "InterConn analyzer deprecated"; - return config; - } -} plugin; - -} -} diff --git a/src/analyzer/protocol/interconn/events.bif b/src/analyzer/protocol/interconn/events.bif deleted file mode 100644 index 6e859da668..0000000000 --- a/src/analyzer/protocol/interconn/events.bif +++ /dev/null @@ -1,8 +0,0 @@ -# ##### Deprecated events. Proposed for removal. - -## Deprecated. Will be removed. -event interconn_stats%(c: connection, os: interconn_endp_stats, rs: interconn_endp_stats%); - -## Deprecated. Will be removed. -event interconn_remove_conn%(c: connection%); - diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 8fa1ab560a..8a1f6fd603 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -85,7 +85,6 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/plugins/Zeek_ICMP.events.bif.zeek build/scripts/base/bif/plugins/Zeek_Ident.events.bif.zeek build/scripts/base/bif/plugins/Zeek_IMAP.events.bif.zeek - build/scripts/base/bif/plugins/Zeek_InterConn.events.bif.zeek build/scripts/base/bif/plugins/Zeek_IRC.events.bif.zeek build/scripts/base/bif/plugins/Zeek_KRB.events.bif.zeek build/scripts/base/bif/plugins/Zeek_Login.events.bif.zeek diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 0b68a0ce83..361019899c 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -85,7 +85,6 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/plugins/Zeek_ICMP.events.bif.zeek build/scripts/base/bif/plugins/Zeek_Ident.events.bif.zeek build/scripts/base/bif/plugins/Zeek_IMAP.events.bif.zeek - build/scripts/base/bif/plugins/Zeek_InterConn.events.bif.zeek build/scripts/base/bif/plugins/Zeek_IRC.events.bif.zeek build/scripts/base/bif/plugins/Zeek_KRB.events.bif.zeek build/scripts/base/bif/plugins/Zeek_Login.events.bif.zeek diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 37c2f126e3..4d70a144e8 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -1,5 +1,4 @@ 0.000000 MetaHookPost CallFunction(Analyzer::__disable_analyzer, , (Analyzer::ANALYZER_BACKDOOR)) -> -0.000000 MetaHookPost CallFunction(Analyzer::__disable_analyzer, , (Analyzer::ANALYZER_INTERCONN)) -> 0.000000 MetaHookPost CallFunction(Analyzer::__disable_analyzer, , (Analyzer::ANALYZER_STEPPINGSTONE)) -> 0.000000 MetaHookPost CallFunction(Analyzer::__disable_analyzer, , (Analyzer::ANALYZER_TCPSTATS)) -> 0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_AYIYA, 5072/udp)) -> @@ -66,7 +65,6 @@ 0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_XMPP, 5222/tcp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_XMPP, 5269/tcp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_BACKDOOR)) -> -0.000000 MetaHookPost CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_INTERCONN)) -> 0.000000 MetaHookPost CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_STEPPINGSTONE)) -> 0.000000 MetaHookPost CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_TCPSTATS)) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_AYIYA, 5072/udp)) -> @@ -278,7 +276,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1560631035.263667, node=zeek, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1561683801.242144, node=zeek, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Broker::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Config::LOG)) -> @@ -459,7 +457,7 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, , (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1560631035.263667, node=zeek, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1561683801.242144, node=zeek, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(NetControl::check_plugins, , ()) -> 0.000000 MetaHookPost CallFunction(NetControl::init, , ()) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, , ()) -> @@ -605,7 +603,6 @@ 0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_IMAP.events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_IRC.events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_Ident.events.bif.zeek) -> -1 -0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_InterConn.events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_KRB.events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_KRB.types.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_Login.events.bif.zeek) -> -1 @@ -896,7 +893,6 @@ 0.000000 MetaHookPost QueueEvent(filter_change_tracking()) -> false 0.000000 MetaHookPost QueueEvent(zeek_init()) -> false 0.000000 MetaHookPre CallFunction(Analyzer::__disable_analyzer, , (Analyzer::ANALYZER_BACKDOOR)) -0.000000 MetaHookPre CallFunction(Analyzer::__disable_analyzer, , (Analyzer::ANALYZER_INTERCONN)) 0.000000 MetaHookPre CallFunction(Analyzer::__disable_analyzer, , (Analyzer::ANALYZER_STEPPINGSTONE)) 0.000000 MetaHookPre CallFunction(Analyzer::__disable_analyzer, , (Analyzer::ANALYZER_TCPSTATS)) 0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_AYIYA, 5072/udp)) @@ -963,7 +959,6 @@ 0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_XMPP, 5222/tcp)) 0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_XMPP, 5269/tcp)) 0.000000 MetaHookPre CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_BACKDOOR)) -0.000000 MetaHookPre CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_INTERCONN)) 0.000000 MetaHookPre CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_STEPPINGSTONE)) 0.000000 MetaHookPre CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_TCPSTATS)) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_AYIYA, 5072/udp)) @@ -1175,7 +1170,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1560631035.263667, node=zeek, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1561683801.242144, node=zeek, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Broker::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Config::LOG)) @@ -1356,7 +1351,7 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, , (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1560631035.263667, node=zeek, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1561683801.242144, node=zeek, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(NetControl::check_plugins, , ()) 0.000000 MetaHookPre CallFunction(NetControl::init, , ()) 0.000000 MetaHookPre CallFunction(Notice::want_pp, , ()) @@ -1502,7 +1497,6 @@ 0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_IMAP.events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_IRC.events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_Ident.events.bif.zeek) -0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_InterConn.events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_KRB.events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_KRB.types.bif.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_Login.events.bif.zeek) @@ -1793,7 +1787,6 @@ 0.000000 MetaHookPre QueueEvent(filter_change_tracking()) 0.000000 MetaHookPre QueueEvent(zeek_init()) 0.000000 | HookCallFunction Analyzer::__disable_analyzer(Analyzer::ANALYZER_BACKDOOR) -0.000000 | HookCallFunction Analyzer::__disable_analyzer(Analyzer::ANALYZER_INTERCONN) 0.000000 | HookCallFunction Analyzer::__disable_analyzer(Analyzer::ANALYZER_STEPPINGSTONE) 0.000000 | HookCallFunction Analyzer::__disable_analyzer(Analyzer::ANALYZER_TCPSTATS) 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_AYIYA, 5072/udp) @@ -1860,7 +1853,6 @@ 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_XMPP, 5222/tcp) 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_XMPP, 5269/tcp) 0.000000 | HookCallFunction Analyzer::disable_analyzer(Analyzer::ANALYZER_BACKDOOR) -0.000000 | HookCallFunction Analyzer::disable_analyzer(Analyzer::ANALYZER_INTERCONN) 0.000000 | HookCallFunction Analyzer::disable_analyzer(Analyzer::ANALYZER_STEPPINGSTONE) 0.000000 | HookCallFunction Analyzer::disable_analyzer(Analyzer::ANALYZER_TCPSTATS) 0.000000 | HookCallFunction Analyzer::register_for_port(Analyzer::ANALYZER_AYIYA, 5072/udp) @@ -2071,7 +2063,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1560631035.263667, node=zeek, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1561683801.242144, node=zeek, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Broker::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Config::LOG) @@ -2252,7 +2244,7 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1560631035.263667, node=zeek, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1561683801.242144, node=zeek, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction NetControl::check_plugins() 0.000000 | HookCallFunction NetControl::init() 0.000000 | HookCallFunction Notice::want_pp() @@ -2398,7 +2390,6 @@ 0.000000 | HookLoadFile .<...>/Zeek_IMAP.events.bif.zeek 0.000000 | HookLoadFile .<...>/Zeek_IRC.events.bif.zeek 0.000000 | HookLoadFile .<...>/Zeek_Ident.events.bif.zeek -0.000000 | HookLoadFile .<...>/Zeek_InterConn.events.bif.zeek 0.000000 | HookLoadFile .<...>/Zeek_KRB.events.bif.zeek 0.000000 | HookLoadFile .<...>/Zeek_KRB.types.bif.zeek 0.000000 | HookLoadFile .<...>/Zeek_Login.events.bif.zeek @@ -2684,7 +2675,7 @@ 0.000000 | HookLoadFile base<...>/xmpp 0.000000 | HookLoadFile base<...>/zeek.bif.zeek 0.000000 | HookLogInit packet_filter 1/1 {ts (time), node (string), filter (string), init (bool), success (bool)} -0.000000 | HookLogWrite packet_filter [ts=1560631035.263667, node=zeek, filter=ip or not ip, init=T, success=T] +0.000000 | HookLogWrite packet_filter [ts=1561683801.242144, node=zeek, filter=ip or not ip, init=T, success=T] 0.000000 | HookQueueEvent NetControl::init() 0.000000 | HookQueueEvent filter_change_tracking() 0.000000 | HookQueueEvent zeek_init() From 7dc3fca7543e293db2390c841103c3b197f630ec Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 27 Jun 2019 18:22:39 -0700 Subject: [PATCH 10/17] Remove BackDoor analyzer --- NEWS | 20 + scripts/base/frameworks/analyzer/main.zeek | 1 - scripts/base/init-bare.zeek | 18 - src/NetVar.cc | 9 - src/NetVar.h | 5 - src/Sessions.cc | 9 - src/Sessions.h | 1 - src/analyzer/Manager.cc | 7 - src/analyzer/Manager.h | 1 - src/analyzer/protocol/CMakeLists.txt | 1 - src/analyzer/protocol/backdoor/BackDoor.cc | 819 ------------------ src/analyzer/protocol/backdoor/BackDoor.h | 112 --- src/analyzer/protocol/backdoor/CMakeLists.txt | 9 - src/analyzer/protocol/backdoor/Plugin.cc | 25 - src/analyzer/protocol/backdoor/events.bif | 32 - src/analyzer/protocol/gnutella/events.bif | 12 +- src/event.bif | 12 - .../canonified_loaded_scripts.log | 1 - .../canonified_loaded_scripts.log | 1 - testing/btest/Baseline/plugins.hooks/output | 23 +- 20 files changed, 33 insertions(+), 1085 deletions(-) delete mode 100644 src/analyzer/protocol/backdoor/BackDoor.cc delete mode 100644 src/analyzer/protocol/backdoor/BackDoor.h delete mode 100644 src/analyzer/protocol/backdoor/CMakeLists.txt delete mode 100644 src/analyzer/protocol/backdoor/Plugin.cc delete mode 100644 src/analyzer/protocol/backdoor/events.bif diff --git a/NEWS b/NEWS index 29ea654d75..20711ea86d 100644 --- a/NEWS +++ b/NEWS @@ -413,6 +413,21 @@ Removed Functionality - ``print_hook`` - ``interconn_stats`` - ``interconn_remove_conn`` + - ``root_backdoor_signature_found`` + - ``napster_signature_found`` + - ``kazaa_signature_found`` + - ``gaobot_signature_found`` + - ``ftp_signature_found`` + - ``gnutella_signature_found`` + - ``http_signature_found`` + - ``irc_signature_found`` + - ``telnet_signature_found`` + - ``ssh_signature_found`` + - ``rlogin_signature_found`` + - ``smtp_signature_found`` + - ``http_proxy_signature_found`` + - ``backdoor_stats`` + - ``backdoor_remove_conn`` - The following types/records were deprecated in version 2.6 or below and are removed from this release: @@ -450,6 +465,9 @@ Removed Functionality - ``interconn_stat_period`` - ``interconn_stat_backoff`` - ``interconn_endp_stats`` + - ``backdoor_stat_period`` + - ``backdoor_stat_backoff`` + - ``backdoor_endp_stats`` - The following constants were used as part of deprecated functionality in version 2.6 or below and are removed from this release: @@ -497,6 +515,8 @@ Removed Functionality - Removed the InterConn analyzer. +- Removed the BackDoor analyzer. + Deprecated Functionality ------------------------ diff --git a/scripts/base/frameworks/analyzer/main.zeek b/scripts/base/frameworks/analyzer/main.zeek index cfadffb739..3bfd60bfe6 100644 --- a/scripts/base/frameworks/analyzer/main.zeek +++ b/scripts/base/frameworks/analyzer/main.zeek @@ -125,7 +125,6 @@ export { ## contains legacy analyzers that are no longer supported. global disabled_analyzers: set[Analyzer::Tag] = { ANALYZER_STEPPINGSTONE, - ANALYZER_BACKDOOR, ANALYZER_TCPSTATS, } &redef; } diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index e505698434..bfb49579cd 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -3864,24 +3864,6 @@ const stp_idle_min: interval &redef; ## Internal to the stepping stone detector. global stp_skip_src: set[addr] &redef; -## Deprecated. -const backdoor_stat_period: interval &redef; - -## Deprecated. -const backdoor_stat_backoff: double &redef; - -## Deprecated. -type backdoor_endp_stats: record { - is_partial: bool; - num_pkts: count; - num_8k0_pkts: count; - num_8k4_pkts: count; - num_lines: count; - num_normal_lines: count; - num_bytes: count; - num_7bit_ascii: count; -}; - ## Description of a signature match. ## ## .. zeek:see:: signature_match diff --git a/src/NetVar.cc b/src/NetVar.cc index 5ef7e9b9c5..c256cfd496 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -125,11 +125,6 @@ double stp_delta; double stp_idle_min; TableVal* stp_skip_src; -double backdoor_stat_period; -double backdoor_stat_backoff; - -RecordType* backdoor_endp_stats; - double table_expire_interval; double table_expire_delay; int table_incremental_step; @@ -381,10 +376,6 @@ void init_net_var() stp_idle_min = opt_internal_double("stp_idle_min"); stp_skip_src = internal_val("stp_skip_src")->AsTableVal(); - backdoor_stat_period = opt_internal_double("backdoor_stat_period"); - backdoor_stat_backoff = opt_internal_double("backdoor_stat_backoff"); - backdoor_endp_stats = internal_type("backdoor_endp_stats")->AsRecordType(); - orig_addr_anonymization = opt_internal_int("orig_addr_anonymization"); resp_addr_anonymization = opt_internal_int("resp_addr_anonymization"); other_addr_anonymization = opt_internal_int("other_addr_anonymization"); diff --git a/src/NetVar.h b/src/NetVar.h index 28936b9c61..5dc5fcfb79 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -128,11 +128,6 @@ extern double stp_delta; extern double stp_idle_min; extern TableVal* stp_skip_src; -extern double backdoor_stat_period; -extern double backdoor_stat_backoff; - -extern RecordType* backdoor_endp_stats; - extern double table_expire_interval; extern double table_expire_delay; extern int table_incremental_step; diff --git a/src/Sessions.cc b/src/Sessions.cc index 019082a6f0..5f290f5386 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -20,8 +20,6 @@ #include "analyzer/protocol/stepping-stone/SteppingStone.h" #include "analyzer/protocol/stepping-stone/events.bif.h" -#include "analyzer/protocol/backdoor/BackDoor.h" -#include "analyzer/protocol/backdoor/events.bif.h" #include "analyzer/protocol/arp/ARP.h" #include "analyzer/protocol/arp/events.bif.h" #include "Discard.h" @@ -117,13 +115,6 @@ NetSessions::NetSessions() packet_filter = 0; - build_backdoor_analyzer = - backdoor_stats || rlogin_signature_found || - telnet_signature_found || ssh_signature_found || - root_backdoor_signature_found || ftp_signature_found || - napster_signature_found || kazaa_signature_found || - http_signature_found || http_proxy_signature_found; - dump_this_packet = 0; num_packets_processed = 0; diff --git a/src/Sessions.h b/src/Sessions.h index 617ab3e52a..27b0ab474c 100644 --- a/src/Sessions.h +++ b/src/Sessions.h @@ -231,7 +231,6 @@ protected: analyzer::stepping_stone::SteppingStoneManager* stp_manager; Discarder* discarder; PacketFilter* packet_filter; - int build_backdoor_analyzer; int dump_this_packet; // if true, current packet should be recorded uint64 num_packets_processed; PacketProfiler* pkt_profiler; diff --git a/src/analyzer/Manager.cc b/src/analyzer/Manager.cc index d62749284d..da19b7685f 100644 --- a/src/analyzer/Manager.cc +++ b/src/analyzer/Manager.cc @@ -5,7 +5,6 @@ #include "Hash.h" #include "Val.h" -#include "protocol/backdoor/BackDoor.h" #include "protocol/conn-size/ConnSize.h" #include "protocol/icmp/ICMP.h" #include "protocol/pia/PIA.h" @@ -86,7 +85,6 @@ Manager::~Manager() void Manager::InitPreScript() { // Cache these tags. - analyzer_backdoor = GetComponentTag("BACKDOOR"); analyzer_connsize = GetComponentTag("CONNSIZE"); analyzer_stepping = GetComponentTag("STEPPINGSTONE"); analyzer_tcpstats = GetComponentTag("TCPSTATS"); @@ -459,11 +457,6 @@ bool Manager::BuildInitialAnalyzerTree(Connection* conn) if ( reass ) tcp->EnableReassembly(); - if ( IsEnabled(analyzer_backdoor) ) - // Add a BackDoor analyzer if requested. This analyzer - // can handle both reassembled and non-reassembled input. - tcp->AddChildAnalyzer(new backdoor::BackDoor_Analyzer(conn), false); - if ( IsEnabled(analyzer_stepping) ) { // Add a SteppingStone analyzer if requested. The port diff --git a/src/analyzer/Manager.h b/src/analyzer/Manager.h index 1f9fc21090..14bba1d115 100644 --- a/src/analyzer/Manager.h +++ b/src/analyzer/Manager.h @@ -355,7 +355,6 @@ private: analyzer_map_by_port analyzers_by_port_tcp; analyzer_map_by_port analyzers_by_port_udp; - Tag analyzer_backdoor; Tag analyzer_connsize; Tag analyzer_stepping; Tag analyzer_tcpstats; diff --git a/src/analyzer/protocol/CMakeLists.txt b/src/analyzer/protocol/CMakeLists.txt index 906f5e32dc..d5024a2ff1 100644 --- a/src/analyzer/protocol/CMakeLists.txt +++ b/src/analyzer/protocol/CMakeLists.txt @@ -1,7 +1,6 @@ add_subdirectory(arp) add_subdirectory(ayiya) -add_subdirectory(backdoor) add_subdirectory(bittorrent) add_subdirectory(conn-size) add_subdirectory(dce-rpc) diff --git a/src/analyzer/protocol/backdoor/BackDoor.cc b/src/analyzer/protocol/backdoor/BackDoor.cc deleted file mode 100644 index 2e8d47d1d0..0000000000 --- a/src/analyzer/protocol/backdoor/BackDoor.cc +++ /dev/null @@ -1,819 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#include "zeek-config.h" - -#include "BackDoor.h" -#include "Event.h" -#include "Net.h" -#include "analyzer/protocol/tcp/TCP.h" - -#include "events.bif.h" - -using namespace analyzer::backdoor; - -BackDoorEndpoint::BackDoorEndpoint(tcp::TCP_Endpoint* e) - { - endp = e; - is_partial = 0; - max_top_seq = 0; - - rlogin_checking_done = 0; - rlogin_string_separator_pos = 0; - rlogin_num_null = 0; - rlogin_slash_seen = 0; - - num_pkts = num_8k0_pkts = num_8k4_pkts = - num_lines = num_normal_lines = num_bytes = num_7bit_ascii = 0; - } - -#define NORMAL_LINE_LENGTH 80 - -#define TELNET_IAC 255 -#define IS_TELNET_NEGOTIATION_CMD(c) ((c) >= 251 && (c) <= 254) - -#define DEFAULT_MTU 512 - -#define RLOGIN_MAX_SIGNATURE_LENGTH 256 - -void BackDoorEndpoint::FinalCheckForRlogin() - { - if ( ! rlogin_checking_done ) - { - rlogin_checking_done = 1; - - if ( rlogin_num_null > 0 ) - RloginSignatureFound(0); - } - } - -int BackDoorEndpoint::DataSent(double /* t */, uint64 seq, - int len, int caplen, const u_char* data, - const IP_Hdr* /* ip */, - const struct tcphdr* /* tp */) - { - if ( caplen < len ) - len = caplen; - - if ( len <= 0 ) - return 0; - - if ( endp->state == tcp::TCP_ENDPOINT_PARTIAL ) - is_partial = 1; - - uint64 ack = endp->ToRelativeSeqSpace(endp->AckSeq(), endp->AckWraps()); - uint64 top_seq = seq + len; - - if ( top_seq <= ack || top_seq <= max_top_seq ) - // There is no new data in this packet. - return 0; - - if ( rlogin_signature_found ) - CheckForRlogin(seq, len, data); - - if ( telnet_signature_found ) - CheckForTelnet(seq, len, data); - - if ( ssh_signature_found ) - CheckForSSH(seq, len, data); - - if ( ftp_signature_found ) - CheckForFTP(seq, len, data); - - if ( root_backdoor_signature_found ) - CheckForRootBackdoor(seq, len, data); - - if ( napster_signature_found ) - CheckForNapster(seq, len, data); - - if ( gnutella_signature_found ) - CheckForGnutella(seq, len, data); - - if ( kazaa_signature_found ) - CheckForKazaa(seq, len, data); - - if ( http_signature_found || http_proxy_signature_found ) - CheckForHTTP(seq, len, data); - - if ( smtp_signature_found ) - CheckForSMTP(seq, len, data); - - if ( irc_signature_found ) - CheckForIRC(seq, len, data); - - if ( gaobot_signature_found ) - CheckForGaoBot(seq, len, data); - - max_top_seq = top_seq; - - return 1; - } - -RecordVal* BackDoorEndpoint::BuildStats() - { - RecordVal* stats = new RecordVal(backdoor_endp_stats); - - stats->Assign(0, val_mgr->GetBool(is_partial)); - stats->Assign(1, val_mgr->GetCount(num_pkts)); - stats->Assign(2, val_mgr->GetCount(num_8k0_pkts)); - stats->Assign(3, val_mgr->GetCount(num_8k4_pkts)); - stats->Assign(4, val_mgr->GetCount(num_lines)); - stats->Assign(5, val_mgr->GetCount(num_normal_lines)); - stats->Assign(6, val_mgr->GetCount(num_bytes)); - stats->Assign(7, val_mgr->GetCount(num_7bit_ascii)); - - return stats; - } - -void BackDoorEndpoint::CheckForRlogin(uint64 seq, int len, const u_char* data) - { - if ( rlogin_checking_done ) - return; - - // Looking for pattern: - // stringstringstring/string - // where all string's are non-empty 7-bit-ascii string - // - // To avoid having to reassemble, we keep testing each byte until - // one of the following happens: - // - // - A gap in sequence number occurs - // - Four null's have been found - // - The number of bytes we examined reaches RLOGIN_MAX_SIGNATURE_LENGTH - // - An empty or non-7-bit-ascii string is found - // - if ( seq == 1 ) - { // Check if first byte is a NUL. - if ( data[0] == 0 ) - { - rlogin_num_null = 1; - - if ( ! endp->IsOrig() ) - { - RloginSignatureFound(len); - return; - } - - rlogin_string_separator_pos = 1; - - ++seq; // move past the byte - ++data; - --len; - } - else - { - rlogin_checking_done = 1; - return; - } - } - - if ( seq > max_top_seq && max_top_seq != 0 ) - { // A gap! Since we don't reassemble things, stop now. - RloginSignatureFound(0); - return; - } - - if ( seq + len <= max_top_seq ) - return; // nothing new - - if ( seq < max_top_seq ) - { // trim to just the new data - int64 delta = max_top_seq - seq; - seq += delta; - data += delta; - len -= delta; - } - - // Search for rlogin signature. - for ( int i = 0; i < len && rlogin_num_null < 4; ++i ) - { - if ( data[i] == 0 ) - { - if ( i + seq == rlogin_string_separator_pos + 1 ) - { // Empty string found. - rlogin_checking_done = 1; - return; - } - else - { - rlogin_string_separator_pos = i + seq; - ++rlogin_num_null; - } - } - - else if ( data[i] == '/' ) - { - if ( rlogin_num_null == 3 ) - { - if ( i + seq == rlogin_string_separator_pos + 1 ) - { // Empty terminal type. - rlogin_checking_done = 1; - return; - } - - rlogin_string_separator_pos = i + seq; - rlogin_slash_seen = 1; - } - } - - else if ( data[i] >= 128 ) - { // Non-7-bit-ascii - rlogin_checking_done = 1; - return; - } - } - - if ( rlogin_num_null == 4 ) - { - if ( rlogin_slash_seen ) - RloginSignatureFound(0); - else - rlogin_checking_done = 1; - - return; - } - - if ( seq + len > RLOGIN_MAX_SIGNATURE_LENGTH ) - { // We've waited for too long - RloginSignatureFound(0); - return; - } - } - -void BackDoorEndpoint::RloginSignatureFound(int len) - { - if ( rlogin_checking_done ) - return; - - rlogin_checking_done = 1; - - if ( ! rlogin_signature_found ) - return; - - endp->TCP()->ConnectionEventFast(rlogin_signature_found, { - endp->TCP()->BuildConnVal(), - val_mgr->GetBool(endp->IsOrig()), - val_mgr->GetCount(rlogin_num_null), - val_mgr->GetCount(len), - }); - } - -void BackDoorEndpoint::CheckForTelnet(uint64 /* seq */, int len, const u_char* data) - { - if ( len >= 3 && - data[0] == TELNET_IAC && IS_TELNET_NEGOTIATION_CMD(data[1]) ) - { - TelnetSignatureFound(len); - return; - } - - // Note, we do the analysis per-packet rather than on the reassembled - // stream. This is a lot more efficient as then we don't need to - // do stream reassembly; but it's potentially less accurate, and - // subject to evasion. *But*: backdoor detection is inherently - // subject to a wide variety of evasion, so allowing this form - // (which is a pain to exploit) costs little. - - num_bytes += len; - - int last_char = 0; - int offset = 0; // where we consider the latest line to have begun - int option_length = 0; // length of options in a line - - for ( int i = 0; i < len; ++i ) - { - unsigned int c = data[i]; - - if ( c == '\n' && last_char == '\r' ) - { - // Compress CRLF to just one line termination. - last_char = c; - continue; - } - - if ( c == '\n' || c == '\r' ) - { - ++num_lines; - - if ( i - offset - option_length <= NORMAL_LINE_LENGTH ) - ++num_normal_lines; - - option_length = 0; - offset = i; - } - - else if ( c == TELNET_IAC ) - { - ++option_length; - --num_bytes; - - if ( ++i < len ) - { - unsigned int code = data[i]; - if ( code == TELNET_IAC ) - // Escaped IAC. - last_char = code; - - else if ( code >= 251 && code <= 254 ) - { // 3-byte option: ignore next byte - ++i; - option_length += 2; - num_bytes -= 2; - } - - else - // XXX: We don't deal with sub option for simplicity - // although we SHOULD! - { - ++option_length; - --num_bytes; - } - } - continue; - } - - else if ( c != 0 && c < 128 ) - ++num_7bit_ascii; - - last_char = c; - } - } - -void BackDoorEndpoint::TelnetSignatureFound(int len) - { - if ( ! telnet_signature_found ) - return; - - endp->TCP()->ConnectionEventFast(telnet_signature_found, { - endp->TCP()->BuildConnVal(), - val_mgr->GetBool(endp->IsOrig()), - val_mgr->GetCount(len), - }); - } - -void BackDoorEndpoint::CheckForSSH(uint64 seq, int len, const u_char* data) - { - if ( seq == 1 && CheckForString("SSH-", data, len) && len > 4 && - (data[4] == '1' || data[4] == '2') ) - { - SignatureFound(ssh_signature_found, 1); - return; - } - - // Check for length pattern. - - if ( seq < max_top_seq || max_top_seq == 0 ) - // Retransmission involved, or first pkt => size info useless. - return; - - if ( seq > max_top_seq ) - { // Estimate number of packets in the sequence gap - int64 gap = seq - max_top_seq; - if ( gap > 0 ) - num_pkts += uint64((gap + DEFAULT_MTU - 1) / DEFAULT_MTU); - } - - ++num_pkts; - - // According to the spec: - // SSH 1.x pkts have size 8k+4 - // SSH 2.x pkts have size 8k >= 16 (most cipher blocks are 8n) - if ( len <= 127 ) - switch ( len & 7 ) { - case 0: - if ( len >= 16 ) - ++num_8k0_pkts; - break; - - case 4: - ++num_8k4_pkts; - break; - } - else - { // len is likely to be some MTU. - } - } - -void BackDoorEndpoint::CheckForRootBackdoor(uint64 seq, int len, const u_char* data) - { - // Check for root backdoor signature: an initial payload of - // exactly "# ". - if ( seq == 1 && len == 2 && ! endp->IsOrig() && - data[0] == '#' && data[1] == ' ' ) - SignatureFound(root_backdoor_signature_found); - } - -void BackDoorEndpoint::CheckForFTP(uint64 seq, int len, const u_char* data) - { - // Check for FTP signature - // - // Currently, the signatures include: "220 ", "220-" - // - // For a day's worth of LBNL FTP activity (7,229 connections), - // the distribution of the code in the first line returned by - // the server (the lines always began with a code) is: - // - // 220: 6685 - // 421: 535 - // 226: 7 - // 426: 1 - // 200: 1 - // - // The 421's are all "host does not have access" or "timeout" of - // some form, so it's not big deal with we miss them (if that helps - // keep down the false positives). - - if ( seq != 1 || endp->IsOrig() || len < 4 ) - return; - - if ( CheckForString("220", data, len) && - (data[3] == ' ' || data[3] == '-') ) - SignatureFound(ftp_signature_found); - - else if ( CheckForString("421", data, len) && - (data[3] == '-' || data[3] == ' ') ) - SignatureFound(ftp_signature_found); - } - -void BackDoorEndpoint::CheckForNapster(uint64 seq, int len, const u_char* data) - { - // Check for Napster signature "GETfoobar" or "SENDfoobar" where - // "foobar" is the Napster handle associated with the request - // (so pretty much any arbitrary identifier, but sent adjacent - // to the GET or SEND with no intervening whitespace; but also - // sent in a separate packet. - - if ( seq != 1 || ! endp->IsOrig() ) - return; - - if ( len == 3 && CheckForString("GET", data, len) ) - // GETfoobar. - SignatureFound(napster_signature_found); - - else if ( len == 4 && CheckForString("SEND", data, len) ) - // SENDfoobar. - SignatureFound(napster_signature_found); - } - -void BackDoorEndpoint::CheckForSMTP(uint64 seq, int len, const u_char* data) - { - const char* smtp_handshake[] = { "HELO", "EHLO", 0 }; - - if ( seq != 1 ) - return; - - if ( CheckForStrings(smtp_handshake, data, len) ) - SignatureFound(smtp_signature_found); - } - -void BackDoorEndpoint::CheckForIRC(uint64 seq, int len, const u_char* data) - { - if ( seq != 1 || is_partial ) - return; - - const char* irc_indicator[] = { - "ERROR", "INVITE", "ISON", "JOIN", "KICK", "NICK", - "NJOIN", "NOTICE AUTH", "OPER", "PART", "PING", "PONG", - "PRIVMSG", "SQUERY", "SQUIT", "WHO", 0, - }; - - if ( CheckForStrings(irc_indicator, data, len) ) - SignatureFound(irc_signature_found); - } - -void BackDoorEndpoint::CheckForGnutella(uint64 seq, int len, const u_char* data) - { - // After connecting to the server, the connecting client says: - // - // GNUTELLA CONNECT/\n\n - // - // The accepting server responds: - // - // GNUTELLA OK\n\n - // - // We find checking the first 8 bytes suffices, and that will - // also catch variants that use something other than "CONNECT". - - if ( seq == 1 && CheckForString("GNUTELLA ", data, len) ) - SignatureFound(gnutella_signature_found); - } - -void BackDoorEndpoint::CheckForGaoBot(uint64 seq, int len, const u_char* data) - { - if ( seq == 1 && CheckForString("220 Bot Server (Win32)", data, len) ) - SignatureFound(gaobot_signature_found); - } - -void BackDoorEndpoint::CheckForKazaa(uint64 seq, int len, const u_char* data) - { - // *Some*, though not all, KaZaa connections begin with: - // - // GIVE - - if ( seq == 1 && CheckForString("GIVE ", data, len) ) - SignatureFound(kazaa_signature_found); - } - - -int is_http_whitespace(const u_char ch) - { - return ! isprint(ch) || isspace(ch); - } - -int skip_http_whitespace(const u_char* data, int len, int max) - { - int k; - for ( k = 0; k < len; ++k ) - { - if ( ! is_http_whitespace(data[k]) ) - break; - - // Here we do not go beyond CR -- this is OK for - // processing first line of HTTP requests. However, it - // cannot be used to process multiple-line headers. - - if ( data[k] == '\015' || k == max ) - return -1; - } - - return k < len ? k : -1; - } - -int is_absolute_url(const u_char* data, int len) - { - // Look for '://' in the URL. - const char* abs_url_sig = "://"; - const char* abs_url_sig_pos = abs_url_sig; - - // Warning: the following code is NOT general for any signature string, - // but only works for specific strings like "://". - - for ( int pos = 0; pos < len; ++pos ) - { - if ( *abs_url_sig_pos == '\0' ) - return 1; - - if ( data[pos] == *abs_url_sig_pos ) - ++abs_url_sig_pos; - - else - { - if ( is_http_whitespace(data[pos]) ) - return 0; - - abs_url_sig_pos = abs_url_sig; - if ( *abs_url_sig != '\0' && - *abs_url_sig_pos == data[pos] ) - ++abs_url_sig_pos; - } - } - - return *abs_url_sig_pos == '\0'; - } - -void BackDoorEndpoint::CheckForHTTP(uint64 seq, int len, const u_char* data) - { - // According to the RFC, we should look for - // ' SP SP HTTP/ CR LF' - // where: - // - // = GET | HEAD | POST - // - // (i.e., HTTP 1.1 methods are ignored for now) - // = 1.0 | 1.1. - // - // However, this is probably too restrictive to catch 'non-standard' - // requests. Instead, we look for certain methods only in the first - // line of the first packet only. - // - // "The method is case-sensitive." -- RFC 2616 - - const char* http_method[] = { "GET", "HEAD", "POST", 0 }; - - if ( seq != 1 ) - return; // first packet only - - // Pick up the method. - int pos = skip_http_whitespace (data, len, 0); - if ( pos < 0 ) - return; - - int method; - for ( method = 0; http_method[method]; ++method ) - { - const char* s = http_method[method]; - int i; - for ( i = pos; i < len; ++i, ++s ) - if ( data[i] != *s ) - break; - - if ( *s == '\0' ) - { - pos = i; - break; - } - } - - if ( ! http_method[method] ) - return; - - if ( pos >= len || ! is_http_whitespace(data[pos]) ) - return; - - if ( http_signature_found ) - SignatureFound(http_signature_found); - - if ( http_proxy_signature_found ) - { - const u_char* rest = data + pos; - int rest_len = len - pos; - - pos = skip_http_whitespace(rest, rest_len, rest_len); - - if ( pos >= 0 ) - CheckForHTTPProxy(seq, rest_len - pos, rest + pos); - } - } - -void BackDoorEndpoint::CheckForHTTPProxy(uint64 /* seq */, int len, - const u_char* data) - { - // Proxy ONLY accepts absolute URI's: "The absoluteURI form is - // REQUIRED when the request is being made to a proxy." -- RFC 2616 - - if ( is_absolute_url(data, len) ) - SignatureFound(http_proxy_signature_found); - } - - -void BackDoorEndpoint::SignatureFound(EventHandlerPtr e, int do_orig) - { - if ( ! e ) - return; - - if ( do_orig ) - endp->TCP()->ConnectionEventFast(e, - {endp->TCP()->BuildConnVal(), val_mgr->GetBool(endp->IsOrig())}); - - else - endp->TCP()->ConnectionEventFast(e, {endp->TCP()->BuildConnVal()}); - } - - -int BackDoorEndpoint::CheckForStrings(const char** strs, - const u_char* data, int len) - { - for ( ; *strs; ++strs ) - if ( CheckForFullString(*strs, data, len) ) - return 1; - - return 0; - } - -int BackDoorEndpoint::CheckForFullString(const char* str, - const u_char* data, int len) - { - for ( ; len > 0 && *str; --len, ++data, ++str ) - if ( *str != *data ) - return 0; - - // A "full" string means a non-prefix match. - return *str == 0 && (len == 0 || *data == ' ' || *data == '\t'); - } - -int BackDoorEndpoint::CheckForString(const char* str, - const u_char* data, int len) - { - for ( ; len > 0 && *str; --len, ++data, ++str ) - if ( *str != *data ) - return 0; - - return *str == 0; - } - - -BackDoor_Analyzer::BackDoor_Analyzer(Connection* c) -: tcp::TCP_ApplicationAnalyzer("BACKDOOR", c) - { - orig_endp = resp_endp = 0; - - orig_stream_pos = resp_stream_pos = 1; - - timeout = backdoor_stat_period; - backoff = backdoor_stat_backoff; - - c->GetTimerMgr()->Add(new BackDoorTimer(network_time + timeout, this)); - } - -BackDoor_Analyzer::~BackDoor_Analyzer() - { - delete orig_endp; - delete resp_endp; - } - -void BackDoor_Analyzer::Init() - { - tcp::TCP_ApplicationAnalyzer::Init(); - - assert(TCP()); - orig_endp = new BackDoorEndpoint(TCP()->Orig()); - resp_endp = new BackDoorEndpoint(TCP()->Resp()); - } - -void BackDoor_Analyzer::DeliverPacket(int len, const u_char* data, bool is_orig, - uint64 seq, const IP_Hdr* ip, int caplen) - { - Analyzer::DeliverPacket(len, data, is_orig, seq, ip, caplen); - - if ( is_orig ) - orig_endp->DataSent(network_time, seq, len, caplen, data, 0, 0); - else - resp_endp->DataSent(network_time, seq, len, caplen, data, 0, 0); - } - -void BackDoor_Analyzer::DeliverStream(int len, const u_char* data, bool is_orig) - { - Analyzer::DeliverStream(len, data, is_orig); - - if ( is_orig ) - { - orig_endp->DataSent(network_time, orig_stream_pos, - len, len, data, 0, 0); - orig_stream_pos += len; - } - - else - { - resp_endp->DataSent(network_time, resp_stream_pos, - len, len, data, 0, 0); - resp_stream_pos += len; - } - } - -void BackDoor_Analyzer::Done() - { - tcp::TCP_ApplicationAnalyzer::Done(); - - if ( ! IsFinished() ) - { - orig_endp->FinalCheckForRlogin(); - resp_endp->FinalCheckForRlogin(); - - if ( ! TCP()->Skipping() ) - StatEvent(); - - RemoveEvent(); - } - - } - -void BackDoor_Analyzer::StatTimer(double t, int is_expire) - { - if ( IsFinished() || TCP()->Skipping() ) - return; - - StatEvent(); - - if ( ! is_expire ) - { - timeout *= backoff; - timer_mgr->Add(new BackDoorTimer(t + timeout, this)); - } - } - -void BackDoor_Analyzer::StatEvent() - { - if ( ! backdoor_stats ) - return; - - TCP()->ConnectionEventFast(backdoor_stats, { - TCP()->BuildConnVal(), - orig_endp->BuildStats(), - resp_endp->BuildStats(), - }); - } - -void BackDoor_Analyzer::RemoveEvent() - { - if ( ! backdoor_remove_conn ) - return; - - TCP()->ConnectionEventFast(backdoor_remove_conn, {TCP()->BuildConnVal()}); - } - -BackDoorTimer::BackDoorTimer(double t, BackDoor_Analyzer* a) -: Timer(t, TIMER_BACKDOOR) - { - analyzer = a; - // Make sure connection does not expire. - Ref(a->Conn()); - } - -BackDoorTimer::~BackDoorTimer() - { - Unref(analyzer->Conn()); - } - -void BackDoorTimer::Dispatch(double t, int is_expire) - { - analyzer->StatTimer(t, is_expire); - } diff --git a/src/analyzer/protocol/backdoor/BackDoor.h b/src/analyzer/protocol/backdoor/BackDoor.h deleted file mode 100644 index c954d176d5..0000000000 --- a/src/analyzer/protocol/backdoor/BackDoor.h +++ /dev/null @@ -1,112 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#ifndef ANALYZER_PROTOCOL_BACKDOOR_BACKDOOR_H -#define ANALYZER_PROTOCOL_BACKDOOR_BACKDOOR_H - -#include "analyzer/protocol/tcp/TCP.h" -#include "Timer.h" -#include "NetVar.h" -#include "analyzer/protocol/login/Login.h" - -namespace analyzer { namespace backdoor { - -class BackDoorEndpoint { -public: - explicit BackDoorEndpoint(tcp::TCP_Endpoint* e); - - int DataSent(double t, uint64 seq, int len, int caplen, const u_char* data, - const IP_Hdr* ip, const struct tcphdr* tp); - - RecordVal* BuildStats(); - - void FinalCheckForRlogin(); - -protected: - void CheckForRlogin(uint64 seq, int len, const u_char* data); - void RloginSignatureFound(int len); - - void CheckForTelnet(uint64 seq, int len, const u_char* data); - void TelnetSignatureFound(int len); - - void CheckForSSH(uint64 seq, int len, const u_char* data); - void CheckForFTP(uint64 seq, int len, const u_char* data); - void CheckForRootBackdoor(uint64 seq, int len, const u_char* data); - void CheckForNapster(uint64 seq, int len, const u_char* data); - void CheckForGnutella(uint64 seq, int len, const u_char* data); - void CheckForKazaa(uint64 seq, int len, const u_char* data); - void CheckForHTTP(uint64 seq, int len, const u_char* data); - void CheckForHTTPProxy(uint64 seq, int len, const u_char* data); - void CheckForSMTP(uint64 seq, int len, const u_char* data); - void CheckForIRC(uint64 seq, int len, const u_char* data); - void CheckForGaoBot(uint64 seq, int len, const u_char* data); - - void SignatureFound(EventHandlerPtr e, int do_orig = 0); - - int CheckForStrings(const char** strs, const u_char* data, int len); - int CheckForFullString(const char* str, const u_char* data, int len); - int CheckForString(const char* str, const u_char* data, int len); - - tcp::TCP_Endpoint* endp; - int is_partial; - uint64 max_top_seq; - - int rlogin_checking_done; - int rlogin_num_null; - uint64 rlogin_string_separator_pos; - int rlogin_slash_seen; - - uint32 num_pkts; - uint32 num_8k4_pkts; - uint32 num_8k0_pkts; - uint32 num_lines; - uint32 num_normal_lines; - uint32 num_bytes; - uint32 num_7bit_ascii; -}; - -class BackDoor_Analyzer : public tcp::TCP_ApplicationAnalyzer { -public: - explicit BackDoor_Analyzer(Connection* c); - ~BackDoor_Analyzer() override; - - void Init() override; - void Done() override; - void StatTimer(double t, int is_expire); - - static analyzer::Analyzer* Instantiate(Connection* conn) - { return new BackDoor_Analyzer(conn); } - -protected: - // We support both packet and stream input, and can be instantiated - // even if the TCP analyzer is not yet reassembling. - void DeliverPacket(int len, const u_char* data, bool is_orig, - uint64 seq, const IP_Hdr* ip, int caplen) override; - void DeliverStream(int len, const u_char* data, bool is_orig) override; - - void StatEvent(); - void RemoveEvent(); - - BackDoorEndpoint* orig_endp; - BackDoorEndpoint* resp_endp; - - int orig_stream_pos; - int resp_stream_pos; - - double timeout; - double backoff; -}; - -class BackDoorTimer : public Timer { -public: - BackDoorTimer(double t, BackDoor_Analyzer* a); - ~BackDoorTimer() override; - - void Dispatch(double t, int is_expire) override; - -protected: - BackDoor_Analyzer* analyzer; -}; - -} } // namespace analyzer::* - -#endif diff --git a/src/analyzer/protocol/backdoor/CMakeLists.txt b/src/analyzer/protocol/backdoor/CMakeLists.txt deleted file mode 100644 index 66511d3d99..0000000000 --- a/src/analyzer/protocol/backdoor/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ - -include(ZeekPlugin) - -include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) - -zeek_plugin_begin(Zeek BackDoor) -zeek_plugin_cc(BackDoor.cc Plugin.cc) -zeek_plugin_bif(events.bif) -zeek_plugin_end() diff --git a/src/analyzer/protocol/backdoor/Plugin.cc b/src/analyzer/protocol/backdoor/Plugin.cc deleted file mode 100644 index aeec615c50..0000000000 --- a/src/analyzer/protocol/backdoor/Plugin.cc +++ /dev/null @@ -1,25 +0,0 @@ -// See the file in the main distribution directory for copyright. - - -#include "plugin/Plugin.h" - -#include "BackDoor.h" - -namespace plugin { -namespace Zeek_BackDoor { - -class Plugin : public plugin::Plugin { -public: - plugin::Configuration Configure() - { - AddComponent(new ::analyzer::Component("BackDoor", ::analyzer::backdoor::BackDoor_Analyzer::Instantiate)); - - plugin::Configuration config; - config.name = "Zeek::BackDoor"; - config.description = "Backdoor Analyzer deprecated"; - return config; - } -} plugin; - -} -} diff --git a/src/analyzer/protocol/backdoor/events.bif b/src/analyzer/protocol/backdoor/events.bif deleted file mode 100644 index 81676ee43b..0000000000 --- a/src/analyzer/protocol/backdoor/events.bif +++ /dev/null @@ -1,32 +0,0 @@ -## Deprecated. Will be removed. -event backdoor_stats%(c: connection, os: backdoor_endp_stats, rs: backdoor_endp_stats%); - -## Deprecated. Will be removed. -event backdoor_remove_conn%(c: connection%); - -## Deprecated. Will be removed. -event ftp_signature_found%(c: connection%); - -## Deprecated. Will be removed. -event gnutella_signature_found%(c: connection%); - -## Deprecated. Will be removed. -event http_signature_found%(c: connection%); - -## Deprecated. Will be removed. -event irc_signature_found%(c: connection%); - -## Deprecated. Will be removed. -event telnet_signature_found%(c: connection, is_orig: bool, len: count%); - -## Deprecated. Will be removed. -event ssh_signature_found%(c: connection, is_orig: bool%); - -## Deprecated. Will be removed. -event rlogin_signature_found%(c: connection, is_orig: bool, num_null: count, len: count%); - -## Deprecated. Will be removed. -event smtp_signature_found%(c: connection%); - -## Deprecated. Will be removed. -event http_proxy_signature_found%(c: connection%); diff --git a/src/analyzer/protocol/gnutella/events.bif b/src/analyzer/protocol/gnutella/events.bif index 4168646543..8b3ef2a177 100644 --- a/src/analyzer/protocol/gnutella/events.bif +++ b/src/analyzer/protocol/gnutella/events.bif @@ -4,7 +4,7 @@ ## information about the Gnutella protocol. ## ## .. zeek:see:: gnutella_binary_msg gnutella_establish gnutella_http_notify -## gnutella_not_establish gnutella_partial_binary_msg gnutella_signature_found +## gnutella_not_establish gnutella_partial_binary_msg ## ## ## .. todo:: Zeek's current default configuration does not activate the protocol @@ -19,7 +19,7 @@ event gnutella_text_msg%(c: connection, orig: bool, headers: string%); ## information about the Gnutella protocol. ## ## .. zeek:see:: gnutella_establish gnutella_http_notify gnutella_not_establish -## gnutella_partial_binary_msg gnutella_signature_found gnutella_text_msg +## gnutella_partial_binary_msg gnutella_text_msg ## ## .. todo:: Zeek's current default configuration does not activate the protocol ## analyzer that generates this event; the corresponding script has not yet @@ -36,7 +36,7 @@ event gnutella_binary_msg%(c: connection, orig: bool, msg_type: count, ## information about the Gnutella protocol. ## ## .. zeek:see:: gnutella_binary_msg gnutella_establish gnutella_http_notify -## gnutella_not_establish gnutella_signature_found gnutella_text_msg +## gnutella_not_establish gnutella_text_msg ## ## .. todo:: Zeek's current default configuration does not activate the protocol ## analyzer that generates this event; the corresponding script has not yet @@ -51,7 +51,7 @@ event gnutella_partial_binary_msg%(c: connection, orig: bool, ## information about the Gnutella protocol. ## ## .. zeek:see:: gnutella_binary_msg gnutella_http_notify gnutella_not_establish -## gnutella_partial_binary_msg gnutella_signature_found gnutella_text_msg +## gnutella_partial_binary_msg gnutella_text_msg ## ## .. todo:: Zeek's current default configuration does not activate the protocol ## analyzer that generates this event; the corresponding script has not yet @@ -65,7 +65,7 @@ event gnutella_establish%(c: connection%); ## information about the Gnutella protocol. ## ## .. zeek:see:: gnutella_binary_msg gnutella_establish gnutella_http_notify -## gnutella_partial_binary_msg gnutella_signature_found gnutella_text_msg +## gnutella_partial_binary_msg gnutella_text_msg ## ## .. todo:: Zeek's current default configuration does not activate the protocol ## analyzer that generates this event; the corresponding script has not yet @@ -79,7 +79,7 @@ event gnutella_not_establish%(c: connection%); ## information about the Gnutella protocol. ## ## .. zeek:see:: gnutella_binary_msg gnutella_establish gnutella_not_establish -## gnutella_partial_binary_msg gnutella_signature_found gnutella_text_msg +## gnutella_partial_binary_msg gnutella_text_msg ## ## .. todo:: Zeek's current default configuration does not activate the protocol ## analyzer that generates this event; the corresponding script has not yet diff --git a/src/event.bif b/src/event.bif index 79e54b9d37..2adb5532bb 100644 --- a/src/event.bif +++ b/src/event.bif @@ -799,17 +799,5 @@ event dns_mapping_altered%(dm: dns_mapping, old_addrs: addr_set, new_addrs: addr ## params: The event's parameters. event new_event%(name: string, params: call_argument_vector%); -## Deprecated. Will be removed. -event root_backdoor_signature_found%(c: connection%); - -## Deprecated. Will be removed. -event napster_signature_found%(c: connection%); - -## Deprecated. Will be removed. -event kazaa_signature_found%(c: connection%); - -## Deprecated. Will be removed. -event gaobot_signature_found%(c: connection%); - ## Shows an IP address anonymization mapping. event anonymization_mapping%(orig: addr, mapped: addr%); diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 8a1f6fd603..0169c3b838 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -62,7 +62,6 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/top-k.bif.zeek build/scripts/base/bif/plugins/__load__.zeek build/scripts/base/bif/plugins/Zeek_ARP.events.bif.zeek - build/scripts/base/bif/plugins/Zeek_BackDoor.events.bif.zeek build/scripts/base/bif/plugins/Zeek_BitTorrent.events.bif.zeek build/scripts/base/bif/plugins/Zeek_ConnSize.events.bif.zeek build/scripts/base/bif/plugins/Zeek_ConnSize.functions.bif.zeek diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 361019899c..6fbc95e4ec 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -62,7 +62,6 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/top-k.bif.zeek build/scripts/base/bif/plugins/__load__.zeek build/scripts/base/bif/plugins/Zeek_ARP.events.bif.zeek - build/scripts/base/bif/plugins/Zeek_BackDoor.events.bif.zeek build/scripts/base/bif/plugins/Zeek_BitTorrent.events.bif.zeek build/scripts/base/bif/plugins/Zeek_ConnSize.events.bif.zeek build/scripts/base/bif/plugins/Zeek_ConnSize.functions.bif.zeek diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 4d70a144e8..ef7ba59161 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -1,4 +1,3 @@ -0.000000 MetaHookPost CallFunction(Analyzer::__disable_analyzer, , (Analyzer::ANALYZER_BACKDOOR)) -> 0.000000 MetaHookPost CallFunction(Analyzer::__disable_analyzer, , (Analyzer::ANALYZER_STEPPINGSTONE)) -> 0.000000 MetaHookPost CallFunction(Analyzer::__disable_analyzer, , (Analyzer::ANALYZER_TCPSTATS)) -> 0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_AYIYA, 5072/udp)) -> @@ -64,7 +63,6 @@ 0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_VXLAN, 4789/udp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_XMPP, 5222/tcp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_XMPP, 5269/tcp)) -> -0.000000 MetaHookPost CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_BACKDOOR)) -> 0.000000 MetaHookPost CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_STEPPINGSTONE)) -> 0.000000 MetaHookPost CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_TCPSTATS)) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_AYIYA, 5072/udp)) -> @@ -276,7 +274,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1561683801.242144, node=zeek, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1561684839.152939, node=zeek, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Broker::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Config::LOG)) -> @@ -457,7 +455,7 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, , (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1561683801.242144, node=zeek, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1561684839.152939, node=zeek, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(NetControl::check_plugins, , ()) -> 0.000000 MetaHookPost CallFunction(NetControl::init, , ()) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, , ()) -> @@ -572,7 +570,6 @@ 0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_ARP.events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_AsciiReader.ascii.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_AsciiWriter.ascii.bif.zeek) -> -1 -0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_BackDoor.events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_BenchmarkReader.benchmark.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_BinaryReader.binary.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_BitTorrent.events.bif.zeek) -> -1 @@ -892,7 +889,6 @@ 0.000000 MetaHookPost QueueEvent(NetControl::init()) -> false 0.000000 MetaHookPost QueueEvent(filter_change_tracking()) -> false 0.000000 MetaHookPost QueueEvent(zeek_init()) -> false -0.000000 MetaHookPre CallFunction(Analyzer::__disable_analyzer, , (Analyzer::ANALYZER_BACKDOOR)) 0.000000 MetaHookPre CallFunction(Analyzer::__disable_analyzer, , (Analyzer::ANALYZER_STEPPINGSTONE)) 0.000000 MetaHookPre CallFunction(Analyzer::__disable_analyzer, , (Analyzer::ANALYZER_TCPSTATS)) 0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_AYIYA, 5072/udp)) @@ -958,7 +954,6 @@ 0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_VXLAN, 4789/udp)) 0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_XMPP, 5222/tcp)) 0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_XMPP, 5269/tcp)) -0.000000 MetaHookPre CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_BACKDOOR)) 0.000000 MetaHookPre CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_STEPPINGSTONE)) 0.000000 MetaHookPre CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_TCPSTATS)) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_AYIYA, 5072/udp)) @@ -1170,7 +1165,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1561683801.242144, node=zeek, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1561684839.152939, node=zeek, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Broker::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Config::LOG)) @@ -1351,7 +1346,7 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, , (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1561683801.242144, node=zeek, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1561684839.152939, node=zeek, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(NetControl::check_plugins, , ()) 0.000000 MetaHookPre CallFunction(NetControl::init, , ()) 0.000000 MetaHookPre CallFunction(Notice::want_pp, , ()) @@ -1466,7 +1461,6 @@ 0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_ARP.events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_AsciiReader.ascii.bif.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_AsciiWriter.ascii.bif.zeek) -0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_BackDoor.events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_BenchmarkReader.benchmark.bif.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_BinaryReader.binary.bif.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_BitTorrent.events.bif.zeek) @@ -1786,7 +1780,6 @@ 0.000000 MetaHookPre QueueEvent(NetControl::init()) 0.000000 MetaHookPre QueueEvent(filter_change_tracking()) 0.000000 MetaHookPre QueueEvent(zeek_init()) -0.000000 | HookCallFunction Analyzer::__disable_analyzer(Analyzer::ANALYZER_BACKDOOR) 0.000000 | HookCallFunction Analyzer::__disable_analyzer(Analyzer::ANALYZER_STEPPINGSTONE) 0.000000 | HookCallFunction Analyzer::__disable_analyzer(Analyzer::ANALYZER_TCPSTATS) 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_AYIYA, 5072/udp) @@ -1852,7 +1845,6 @@ 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_VXLAN, 4789/udp) 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_XMPP, 5222/tcp) 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_XMPP, 5269/tcp) -0.000000 | HookCallFunction Analyzer::disable_analyzer(Analyzer::ANALYZER_BACKDOOR) 0.000000 | HookCallFunction Analyzer::disable_analyzer(Analyzer::ANALYZER_STEPPINGSTONE) 0.000000 | HookCallFunction Analyzer::disable_analyzer(Analyzer::ANALYZER_TCPSTATS) 0.000000 | HookCallFunction Analyzer::register_for_port(Analyzer::ANALYZER_AYIYA, 5072/udp) @@ -2063,7 +2055,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1561683801.242144, node=zeek, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1561684839.152939, node=zeek, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Broker::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Config::LOG) @@ -2244,7 +2236,7 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1561683801.242144, node=zeek, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1561684839.152939, node=zeek, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction NetControl::check_plugins() 0.000000 | HookCallFunction NetControl::init() 0.000000 | HookCallFunction Notice::want_pp() @@ -2359,7 +2351,6 @@ 0.000000 | HookLoadFile .<...>/Zeek_ARP.events.bif.zeek 0.000000 | HookLoadFile .<...>/Zeek_AsciiReader.ascii.bif.zeek 0.000000 | HookLoadFile .<...>/Zeek_AsciiWriter.ascii.bif.zeek -0.000000 | HookLoadFile .<...>/Zeek_BackDoor.events.bif.zeek 0.000000 | HookLoadFile .<...>/Zeek_BenchmarkReader.benchmark.bif.zeek 0.000000 | HookLoadFile .<...>/Zeek_BinaryReader.binary.bif.zeek 0.000000 | HookLoadFile .<...>/Zeek_BitTorrent.events.bif.zeek @@ -2675,7 +2666,7 @@ 0.000000 | HookLoadFile base<...>/xmpp 0.000000 | HookLoadFile base<...>/zeek.bif.zeek 0.000000 | HookLogInit packet_filter 1/1 {ts (time), node (string), filter (string), init (bool), success (bool)} -0.000000 | HookLogWrite packet_filter [ts=1561683801.242144, node=zeek, filter=ip or not ip, init=T, success=T] +0.000000 | HookLogWrite packet_filter [ts=1561684839.152939, node=zeek, filter=ip or not ip, init=T, success=T] 0.000000 | HookQueueEvent NetControl::init() 0.000000 | HookQueueEvent filter_change_tracking() 0.000000 | HookQueueEvent zeek_init() From 0edc7c6cbbdd282c5577d32b0c619286df51e33d Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 27 Jun 2019 18:30:48 -0700 Subject: [PATCH 11/17] Remove deprecated DNS events - dns_full_request - non_dns_request --- NEWS | 2 + src/analyzer/protocol/dns/DNS.cc | 16 +---- src/analyzer/protocol/dns/events.bif | 92 ++++++++++++---------------- 3 files changed, 43 insertions(+), 67 deletions(-) diff --git a/NEWS b/NEWS index 20711ea86d..0e19ef6ccd 100644 --- a/NEWS +++ b/NEWS @@ -428,6 +428,8 @@ Removed Functionality - ``http_proxy_signature_found`` - ``backdoor_stats`` - ``backdoor_remove_conn`` + - ``dns_full_request`` + - ``non_dns_request`` - The following types/records were deprecated in version 2.6 or below and are removed from this release: diff --git a/src/analyzer/protocol/dns/DNS.cc b/src/analyzer/protocol/dns/DNS.cc index 51a8d1cec3..c07b936f49 100644 --- a/src/analyzer/protocol/dns/DNS.cc +++ b/src/analyzer/protocol/dns/DNS.cc @@ -1758,21 +1758,7 @@ void DNS_Analyzer::DeliverPacket(int len, const u_char* data, bool orig, uint64 seq, const IP_Hdr* ip, int caplen) { tcp::TCP_ApplicationAnalyzer::DeliverPacket(len, data, orig, seq, ip, caplen); - - if ( orig ) - { - if ( ! interp->ParseMessage(data, len, 1) && non_dns_request ) - { - if ( non_dns_request ) - ConnectionEventFast(non_dns_request, { - BuildConnVal(), - new StringVal(len, (const char*) data), - }); - } - } - - else - interp->ParseMessage(data, len, 0); + interp->ParseMessage(data, len, orig); } diff --git a/src/analyzer/protocol/dns/events.bif b/src/analyzer/protocol/dns/events.bif index 7ddbd0c7b3..35e9ffa0fd 100644 --- a/src/analyzer/protocol/dns/events.bif +++ b/src/analyzer/protocol/dns/events.bif @@ -16,9 +16,9 @@ ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply ## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end -## dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_query_reply dns_rejected -## dns_request non_dns_request dns_max_queries dns_session_timeout dns_skip_addl +## dns_request dns_max_queries dns_session_timeout dns_skip_addl ## dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_message%(c: connection, is_orig: bool, msg: dns_msg, len: count%); @@ -43,9 +43,9 @@ event dns_message%(c: connection, is_orig: bool, msg: dns_msg, len: count%); ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply ## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end -## dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply -## dns_rejected non_dns_request dns_max_queries dns_session_timeout dns_skip_addl +## dns_rejected dns_max_queries dns_session_timeout dns_skip_addl ## dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_request%(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count%); @@ -72,9 +72,9 @@ event dns_request%(c: connection, msg: dns_msg, query: string, qtype: count, qcl ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply ## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end -## dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply -## dns_request non_dns_request dns_max_queries dns_session_timeout dns_skip_addl +## dns_request dns_max_queries dns_session_timeout dns_skip_addl ## dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_rejected%(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count%); @@ -98,9 +98,9 @@ event dns_rejected%(c: connection, msg: dns_msg, query: string, qtype: count, qc ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply ## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end -## dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_rejected -## dns_request non_dns_request dns_max_queries dns_session_timeout dns_skip_addl +## dns_request dns_max_queries dns_session_timeout dns_skip_addl ## dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_query_reply%(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count%); @@ -123,10 +123,10 @@ event dns_query_reply%(c: connection, msg: dns_msg, query: string, ## ## .. zeek:see:: dns_AAAA_reply dns_A6_reply dns_CNAME_reply dns_EDNS_addl dns_HINFO_reply ## dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply dns_SRV_reply -## dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_full_request +## dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end ## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply -## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_rejected dns_request dns_max_queries dns_session_timeout ## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_A_reply%(c: connection, msg: dns_msg, ans: dns_answer, a: addr%); @@ -148,10 +148,10 @@ event dns_A_reply%(c: connection, msg: dns_msg, ans: dns_answer, a: addr%); ## ## .. zeek:see:: dns_A_reply dns_A6_reply dns_CNAME_reply dns_EDNS_addl dns_HINFO_reply dns_MX_reply ## dns_NS_reply dns_PTR_reply dns_SOA_reply dns_SRV_reply dns_TSIG_addl -## dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_full_request dns_mapping_altered +## dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_mapping_altered ## dns_mapping_lost_name dns_mapping_new_name dns_mapping_unverified ## dns_mapping_valid dns_message dns_query_reply dns_rejected dns_request -## non_dns_request dns_max_queries dns_session_timeout dns_skip_addl +## dns_max_queries dns_session_timeout dns_skip_addl ## dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_AAAA_reply%(c: connection, msg: dns_msg, ans: dns_answer, a: addr%); @@ -173,10 +173,10 @@ event dns_AAAA_reply%(c: connection, msg: dns_msg, ans: dns_answer, a: addr%); ## ## .. zeek:see:: dns_A_reply dns_AAAA_reply dns_CNAME_reply dns_EDNS_addl dns_HINFO_reply dns_MX_reply ## dns_NS_reply dns_PTR_reply dns_SOA_reply dns_SRV_reply dns_TSIG_addl -## dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_full_request dns_mapping_altered +## dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_mapping_altered ## dns_mapping_lost_name dns_mapping_new_name dns_mapping_unverified ## dns_mapping_valid dns_message dns_query_reply dns_rejected dns_request -## non_dns_request dns_max_queries dns_session_timeout dns_skip_addl +## dns_max_queries dns_session_timeout dns_skip_addl ## dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_A6_reply%(c: connection, msg: dns_msg, ans: dns_answer, a: addr%); @@ -198,10 +198,10 @@ event dns_A6_reply%(c: connection, msg: dns_msg, ans: dns_answer, a: addr%); ## ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_MX_reply dns_PTR_reply dns_SOA_reply dns_SRV_reply -## dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_full_request +## dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end ## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply -## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_rejected dns_request dns_max_queries dns_session_timeout ## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_NS_reply%(c: connection, msg: dns_msg, ans: dns_answer, name: string%); @@ -223,10 +223,10 @@ event dns_NS_reply%(c: connection, msg: dns_msg, ans: dns_answer, name: string%) ## ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_EDNS_addl dns_HINFO_reply dns_MX_reply ## dns_NS_reply dns_PTR_reply dns_SOA_reply dns_SRV_reply dns_TSIG_addl -## dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_full_request dns_mapping_altered +## dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_mapping_altered ## dns_mapping_lost_name dns_mapping_new_name dns_mapping_unverified ## dns_mapping_valid dns_message dns_query_reply dns_rejected dns_request -## non_dns_request dns_max_queries dns_session_timeout dns_skip_addl +## dns_max_queries dns_session_timeout dns_skip_addl ## dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_CNAME_reply%(c: connection, msg: dns_msg, ans: dns_answer, name: string%); @@ -248,10 +248,10 @@ event dns_CNAME_reply%(c: connection, msg: dns_msg, ans: dns_answer, name: strin ## ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_SOA_reply dns_SRV_reply -## dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_full_request +## dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end ## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply -## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_rejected dns_request dns_max_queries dns_session_timeout ## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_PTR_reply%(c: connection, msg: dns_msg, ans: dns_answer, name: string%); @@ -273,10 +273,10 @@ event dns_PTR_reply%(c: connection, msg: dns_msg, ans: dns_answer, name: string% ## ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SRV_reply -## dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_full_request +## dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end ## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply -## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_rejected dns_request dns_max_queries dns_session_timeout ## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_SOA_reply%(c: connection, msg: dns_msg, ans: dns_answer, soa: dns_soa%); @@ -296,10 +296,10 @@ event dns_SOA_reply%(c: connection, msg: dns_msg, ans: dns_answer, soa: dns_soa% ## ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply -## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_end dns_full_request +## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_end ## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply -## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_rejected dns_request dns_max_queries dns_session_timeout ## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_WKS_reply%(c: connection, msg: dns_msg, ans: dns_answer%); @@ -319,10 +319,10 @@ event dns_WKS_reply%(c: connection, msg: dns_msg, ans: dns_answer%); ## ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl dns_MX_reply ## dns_NS_reply dns_PTR_reply dns_SOA_reply dns_SRV_reply dns_TSIG_addl -## dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_full_request dns_mapping_altered +## dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_mapping_altered ## dns_mapping_lost_name dns_mapping_new_name dns_mapping_unverified ## dns_mapping_valid dns_message dns_query_reply dns_rejected dns_request -## non_dns_request dns_max_queries dns_session_timeout dns_skip_addl +## dns_max_queries dns_session_timeout dns_skip_addl ## dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_HINFO_reply%(c: connection, msg: dns_msg, ans: dns_answer%); @@ -346,10 +346,10 @@ event dns_HINFO_reply%(c: connection, msg: dns_msg, ans: dns_answer%); ## ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_NS_reply dns_PTR_reply dns_SOA_reply dns_SRV_reply -## dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_full_request +## dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end ## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply -## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_rejected dns_request dns_max_queries dns_session_timeout ## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_MX_reply%(c: connection, msg: dns_msg, ans: dns_answer, name: string, preference: count%); @@ -371,10 +371,10 @@ event dns_MX_reply%(c: connection, msg: dns_msg, ans: dns_answer, name: string, ## ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply -## dns_SRV_reply dns_TSIG_addl dns_WKS_reply dns_end dns_full_request +## dns_SRV_reply dns_TSIG_addl dns_WKS_reply dns_end ## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply -## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_rejected dns_request dns_max_queries dns_session_timeout ## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_TXT_reply%(c: connection, msg: dns_msg, ans: dns_answer, strs: string_vec%); @@ -396,10 +396,10 @@ event dns_TXT_reply%(c: connection, msg: dns_msg, ans: dns_answer, strs: string_ ## ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply -## dns_SRV_reply dns_TSIG_addl dns_WKS_reply dns_end dns_full_request +## dns_SRV_reply dns_TSIG_addl dns_WKS_reply dns_end ## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply -## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_rejected dns_request dns_max_queries dns_session_timeout ## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_SPF_reply%(c: connection, msg: dns_msg, ans: dns_answer, strs: string_vec%); @@ -450,10 +450,10 @@ event dns_CAA_reply%(c: connection, msg: dns_msg, ans: dns_answer, flags: count, ## ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply -## dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_full_request +## dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end ## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply -## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_rejected dns_request dns_max_queries dns_session_timeout ## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_SRV_reply%(c: connection, msg: dns_msg, ans: dns_answer, target: string, priority: count, weight: count, p: count%); @@ -488,10 +488,10 @@ event dns_unknown_reply%(c: connection, msg: dns_msg, ans: dns_answer%); ## ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_HINFO_reply dns_MX_reply ## dns_NS_reply dns_PTR_reply dns_SOA_reply dns_SRV_reply dns_TSIG_addl -## dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_full_request dns_mapping_altered +## dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_mapping_altered ## dns_mapping_lost_name dns_mapping_new_name dns_mapping_unverified ## dns_mapping_valid dns_message dns_query_reply dns_rejected dns_request -## non_dns_request dns_max_queries dns_session_timeout dns_skip_addl +## dns_max_queries dns_session_timeout dns_skip_addl ## dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_EDNS_addl%(c: connection, msg: dns_msg, ans: dns_edns_additional%); @@ -511,10 +511,10 @@ event dns_EDNS_addl%(c: connection, msg: dns_msg, ans: dns_edns_additional%); ## ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply -## dns_SRV_reply dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_full_request +## dns_SRV_reply dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end ## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply -## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_rejected dns_request dns_max_queries dns_session_timeout ## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_TSIG_addl%(c: connection, msg: dns_msg, ans: dns_tsig_additional%); @@ -600,21 +600,9 @@ event dns_DS%(c: connection, msg: dns_msg, ans: dns_answer, ds: dns_ds_rr%); ## ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply -## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_full_request +## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply ## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply -## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_rejected dns_request dns_max_queries dns_session_timeout ## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_end%(c: connection, msg: dns_msg%); - -## Deprecated. Will be removed. -## -## .. todo:: Unclear what this event is for; it's never raised. We should just -## remove it. -event dns_full_request%(%); - -## msg: The raw DNS payload. -## -## .. note:: This event is deprecated and superseded by Zeek's dynamic protocol -## detection framework. -event non_dns_request%(c: connection, msg: string%); From 7d2d63551d6de2368b8d598a1a3b1382302893dd Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 27 Jun 2019 18:36:27 -0700 Subject: [PATCH 12/17] Improve deprecation warning messages --- src/Val.h | 22 +++++++++++----------- src/event.bif | 6 +++--- src/strings.bif | 2 +- src/zeek.bif | 4 ++-- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/Val.h b/src/Val.h index 43523df26c..2ce61100b5 100644 --- a/src/Val.h +++ b/src/Val.h @@ -86,7 +86,7 @@ typedef union { class Val : public BroObj { public: - ZEEK_DEPRECATED("use val_mgr->GetBool, GetFalse/GetTrue, GetInt, or GetCount instead") + ZEEK_DEPRECATED("Remove in v3.1: use val_mgr->GetBool, GetFalse/GetTrue, GetInt, or GetCount instead") Val(bool b, TypeTag t) { val.int_val = b; @@ -96,7 +96,7 @@ public: #endif } - ZEEK_DEPRECATED("use val_mgr->GetBool, GetFalse/GetTrue, GetInt, or GetCount instead") + ZEEK_DEPRECATED("Remove in v3.1: use val_mgr->GetBool, GetFalse/GetTrue, GetInt, or GetCount instead") Val(int32 i, TypeTag t) { val.int_val = bro_int_t(i); @@ -106,7 +106,7 @@ public: #endif } - ZEEK_DEPRECATED("use val_mgr->GetBool, GetFalse/GetTrue, GetInt, or GetCount instead") + ZEEK_DEPRECATED("Remove in v3.1: use val_mgr->GetBool, GetFalse/GetTrue, GetInt, or GetCount instead") Val(uint32 u, TypeTag t) { val.uint_val = bro_uint_t(u); @@ -116,7 +116,7 @@ public: #endif } - ZEEK_DEPRECATED("use val_mgr->GetBool, GetFalse/GetTrue, GetInt, or GetCount instead") + ZEEK_DEPRECATED("Remove in v3.1: use val_mgr->GetBool, GetFalse/GetTrue, GetInt, or GetCount instead") Val(int64 i, TypeTag t) { val.int_val = i; @@ -126,7 +126,7 @@ public: #endif } - ZEEK_DEPRECATED("use val_mgr->GetBool, GetFalse/GetTrue, GetInt, or GetCount instead") + ZEEK_DEPRECATED("Remove in v3.1: use val_mgr->GetBool, GetFalse/GetTrue, GetInt, or GetCount instead") Val(uint64 u, TypeTag t) { val.uint_val = u; @@ -429,15 +429,15 @@ protected: class PortManager { public: // Port number given in host order. - ZEEK_DEPRECATED("use val_mgr->GetPort() instead") + ZEEK_DEPRECATED("Remove in v3.1: use val_mgr->GetPort() instead") PortVal* Get(uint32 port_num, TransportProto port_type) const; // Host-order port number already masked with port space protocol mask. - ZEEK_DEPRECATED("use val_mgr->GetPort() instead") + ZEEK_DEPRECATED("Remove in v3.1: use val_mgr->GetPort() instead") PortVal* Get(uint32 port_num) const; // Returns a masked port number - ZEEK_DEPRECATED("use PortVal::Mask() instead") + ZEEK_DEPRECATED("Remove in v3.1: use PortVal::Mask() instead") uint32 Mask(uint32 port_num, TransportProto port_type) const; }; @@ -519,11 +519,11 @@ protected: class PortVal : public Val { public: // Port number given in host order. - ZEEK_DEPRECATED("use val_mgr->GetPort() instead") + ZEEK_DEPRECATED("Remove in v3.1: use val_mgr->GetPort() instead") PortVal(uint32 p, TransportProto port_type); // Host-order port number already masked with port space protocol mask. - ZEEK_DEPRECATED("use val_mgr->GetPort() instead") + ZEEK_DEPRECATED("Remove in v3.1: use val_mgr->GetPort() instead") explicit PortVal(uint32 p); Val* SizeVal() const override { return val_mgr->GetInt(val.uint_val); } @@ -990,7 +990,7 @@ protected: class EnumVal : public Val { public: - ZEEK_DEPRECATED("use t->GetVal(i) instead") + ZEEK_DEPRECATED("Remove in v3.1: use t->GetVal(i) instead") EnumVal(int i, EnumType* t) : Val(t) { val.int_val = i; diff --git a/src/event.bif b/src/event.bif index 2adb5532bb..5222545ae5 100644 --- a/src/event.bif +++ b/src/event.bif @@ -49,7 +49,7 @@ event zeek_init%(%); ## Deprecated synonym for :zeek:see:`zeek_init`. -event bro_init%(%) &deprecated; +event bro_init%(%) &deprecated="Remove in v3.1: use zeek_init"; ## Generated at Zeek termination time. The event engine generates this event when ## Zeek is about to terminate, either due to having exhausted reading its input @@ -65,7 +65,7 @@ event bro_init%(%) &deprecated; event zeek_done%(%); ## Deprecated synonym for :zeek:see:`zeek_done`. -event bro_done%(%) &deprecated; +event bro_done%(%) &deprecated="Remove in v3.1: use zeek_done"; ## Generated for every new connection. This event is raised with the first ## packet of a previously unknown connection. Zeek uses a flow-based definition @@ -608,7 +608,7 @@ event reporter_error%(t: time, msg: string, location: string%) &error_handler; event zeek_script_loaded%(path: string, level: count%); ## Deprecated synonym for :zeek:see:`zeek_script_loaded`. -event bro_script_loaded%(path: string, level: count%) &deprecated; +event bro_script_loaded%(path: string, level: count%) &deprecated="Remove in v3.1: use zeek_script_loaded"; ## Generated each time Zeek's script interpreter opens a file. This event is ## triggered only for files opened via :zeek:id:`open`, and in particular not for diff --git a/src/strings.bif b/src/strings.bif index 6c74db77e9..f2661f8cc9 100644 --- a/src/strings.bif +++ b/src/strings.bif @@ -1034,7 +1034,7 @@ function safe_shell_quote%(source: string%): string ## Returns: A shell-escaped version of *source*. ## ## .. zeek:see:: system safe_shell_quote -function str_shell_escape%(source: string%): string &deprecated +function str_shell_escape%(source: string%): string &deprecated="Remove in v3.1: use safe_shell_quote" %{ unsigned j = 0; const u_char* src = source->Bytes(); diff --git a/src/zeek.bif b/src/zeek.bif index 005d0be541..da11a18681 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -1810,7 +1810,7 @@ extern const char* zeek_version(); ## :zeek:see:`zeek_version` instead. ## ## Returns: Zeek's version, e.g., 2.0-beta-47-debug. -function bro_version%(%): string &deprecated +function bro_version%(%): string &deprecated="Remove in v3.1: use zeek_version" %{ return new StringVal(zeek_version()); %} @@ -2137,7 +2137,7 @@ function dump_rule_stats%(f: file%): bool ## Returns: True if Zeek is in the process of shutting down. ## ## .. zeek:see:: terminate -function bro_is_terminating%(%): bool &deprecated +function bro_is_terminating%(%): bool &deprecated="Remove in v3.1: use zeek_is_terminating" %{ return val_mgr->GetBool(terminating); %} From 430f9a92c692884abe9b4eac8e183d8867195a27 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 27 Jun 2019 23:00:42 -0700 Subject: [PATCH 13/17] GH-443: fix uses of timestamp 0 in cluster diagnostic logs For broker.log and cluster.log: there was a race condition. A worker's first IOSource that it processes is potentially Broker if there were no packets available yet and thread scheduling happens to work out such that network connections (inside CAF threads) become established before we enter the main I/O loop. Such peering establishments would generate logs with timestamp 0 as there was not yet any code path taken that would update network_time. For reporter.log: any non-worker (packet-processing) node would just unnecessarily use a timestamp of 0 for their reporter messages. --- src/Reporter.cc | 2 +- src/broker/Manager.cc | 6 ++++++ testing/btest/Baseline/core.reporter/logger-test.log | 12 ++++++------ testing/btest/core/reporter.zeek | 4 ++-- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/Reporter.cc b/src/Reporter.cc index a40ddb9a3d..d44eff556b 100644 --- a/src/Reporter.cc +++ b/src/Reporter.cc @@ -491,7 +491,7 @@ void Reporter::DoLog(const char* prefix, EventHandlerPtr event, FILE* out, val_list vl(vl_size); if ( time ) - vl.append(new Val((bro_start_network_time != 0.0) ? network_time : 0, TYPE_TIME)); + vl.append(new Val(network_time ? network_time : current_time(), TYPE_TIME)); vl.append(new StringVal(buffer)); diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index e4ef3d570c..1f132ac7cc 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -941,6 +941,12 @@ void Manager::Process() if ( had_input ) { + if ( network_time == 0 ) + // If we're getting Broker messages, but still haven't initialized + // network_time, may as well do so now because otherwise the + // broker/cluster logs will end up using timestamp 0. + net_update_time(current_time()); + ++times_processed_without_idle; // The max number of Process calls allowed to happen in a row without diff --git a/testing/btest/Baseline/core.reporter/logger-test.log b/testing/btest/Baseline/core.reporter/logger-test.log index 1dc58b65cd..0297fbe208 100644 --- a/testing/btest/Baseline/core.reporter/logger-test.log +++ b/testing/btest/Baseline/core.reporter/logger-test.log @@ -1,6 +1,6 @@ -reporter_info|init test-info|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.zeek, line 8|0.000000 -reporter_warning|init test-warning|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.zeek, line 9|0.000000 -reporter_error|init test-error|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.zeek, line 10|0.000000 -reporter_info|done test-info|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.zeek, line 15|0.000000 -reporter_warning|done test-warning|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.zeek, line 16|0.000000 -reporter_error|done test-error|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.zeek, line 17|0.000000 +reporter_info|init test-info|/home/jon/pro/zeek/zeek/testing/btest/.tmp/core.reporter/reporter.zeek, line 8|1561700937.882616 +reporter_warning|init test-warning|/home/jon/pro/zeek/zeek/testing/btest/.tmp/core.reporter/reporter.zeek, line 9|1561700937.882616 +reporter_error|init test-error|/home/jon/pro/zeek/zeek/testing/btest/.tmp/core.reporter/reporter.zeek, line 10|1561700937.882616 +reporter_info|done test-info|/home/jon/pro/zeek/zeek/testing/btest/.tmp/core.reporter/reporter.zeek, line 15|1561700937.882616 +reporter_warning|done test-warning|/home/jon/pro/zeek/zeek/testing/btest/.tmp/core.reporter/reporter.zeek, line 16|1561700937.882616 +reporter_error|done test-error|/home/jon/pro/zeek/zeek/testing/btest/.tmp/core.reporter/reporter.zeek, line 17|1561700937.882616 diff --git a/testing/btest/core/reporter.zeek b/testing/btest/core/reporter.zeek index 8591096c2b..a00cd241ab 100644 --- a/testing/btest/core/reporter.zeek +++ b/testing/btest/core/reporter.zeek @@ -1,7 +1,7 @@ # -# @TEST-EXEC: zeek %INPUT >output 2>&1 +# @TEST-EXEC: zeek -b %INPUT >output 2>&1 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff output -# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff logger-test.log +# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps" btest-diff logger-test.log event zeek_init() { From 5b64c35185746f4af30c9311fcb0ca2ff5511ed8 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 28 Jun 2019 16:34:33 -0700 Subject: [PATCH 14/17] Switch default CAF scheduler policy to work sharing It may generally be better for our default use-case, as workers may save a few percent cpu utilization as this policy does not have to use any polling like the stealing policy does. This also helps avoid a potential issue with the implementation of spinlocks used in the work-stealing policy in current CAF versions, where there's some conditions where lock contention causes a thread to spin for long periods without relinquishing the cpu to others. --- scripts/base/frameworks/broker/main.zeek | 25 +++++++++++++++++------- src/broker/Manager.cc | 9 +++++++++ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/scripts/base/frameworks/broker/main.zeek b/scripts/base/frameworks/broker/main.zeek index 2b43c3fd2b..d3fefd8939 100644 --- a/scripts/base/frameworks/broker/main.zeek +++ b/scripts/base/frameworks/broker/main.zeek @@ -73,32 +73,43 @@ export { ## ZEEK_BROKER_MAX_THREADS environment variable overrides this setting. const max_threads = 1 &redef; + ## The CAF scheduling policy to use. Available options are "sharing" and + ## "stealing". The "sharing" policy uses a single, global work queue along + ## with mutex and condition variable used for accessing it, which may be + ## better for cases that don't require much concurrency or need lower power + ## consumption. The "stealing" policy uses multiple work queues protected + ## by spinlocks, which may be better for use-cases that have more + ## concurrency needs. E.g. may be worth testing the "stealing" policy + ## along with dedicating more threads if a lot of data store processing is + ## required. + const scheduler_policy = "sharing" &redef; + ## Interval of time for under-utilized Broker/CAF threads to sleep - ## when in "moderate" mode. + ## when in "moderate" mode. Only used for the "stealing" scheduler policy. const moderate_sleep = 16 msec &redef; ## Interval of time for under-utilized Broker/CAF threads to sleep - ## when in "relaxed" mode. + ## when in "relaxed" mode. Only used for the "stealing" scheduler policy. const relaxed_sleep = 64 msec &redef; ## Number of work-stealing polling attempts for Broker/CAF threads - ## in "aggressive" mode. + ## in "aggressive" mode. Only used for the "stealing" scheduler policy. const aggressive_polls = 5 &redef; ## Number of work-stealing polling attempts for Broker/CAF threads - ## in "moderate" mode. + ## in "moderate" mode. Only used for the "stealing" scheduler policy. const moderate_polls = 5 &redef; ## Frequency of work-stealing polling attempts for Broker/CAF threads - ## in "aggressive" mode. + ## in "aggressive" mode. Only used for the "stealing" scheduler policy. const aggressive_interval = 4 &redef; ## Frequency of work-stealing polling attempts for Broker/CAF threads - ## in "moderate" mode. + ## in "moderate" mode. Only used for the "stealing" scheduler policy. const moderate_interval = 2 &redef; ## Frequency of work-stealing polling attempts for Broker/CAF threads - ## in "relaxed" mode. + ## in "relaxed" mode. Only used for the "stealing" scheduler policy. const relaxed_interval = 1 &redef; ## Forward all received messages to subscribing peers. diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index e4ef3d570c..2b80051ba4 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -178,6 +178,15 @@ void Manager::InitPostScript() BrokerConfig config{std::move(options)}; + auto scheduler_policy = get_option("Broker::scheduler_policy")->AsString()->CheckString(); + + if ( streq(scheduler_policy, "sharing") ) + config.set("scheduler.policy", caf::atom("sharing")); + else if ( streq(scheduler_policy, "stealing") ) + config.set("scheduler.policy", caf::atom("stealing")); + else + reporter->FatalError("Invalid Broker::scheduler_policy: %s", scheduler_policy); + auto max_threads_env = zeekenv("ZEEK_BROKER_MAX_THREADS"); if ( max_threads_env ) From 85cd38a3e1379930d73a8efdf2c79bb6de3e2c96 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Mon, 1 Jul 2019 00:47:08 -0700 Subject: [PATCH 15/17] Update 3rdparty submodule. This updates sqlite to 3.28.0. Fixes GH-448 [nomail] --- src/3rdparty | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/3rdparty b/src/3rdparty index 6e93c5546a..785e581f00 160000 --- a/src/3rdparty +++ b/src/3rdparty @@ -1 +1 @@ -Subproject commit 6e93c5546a4770d513fb57213d7b29e39e12bf4d +Subproject commit 785e581f00a1efae3fca7a62fb15d8756c5aedb1 From 962988e0b4e08f0e142da3ec5508fa929c7ce8ec Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 1 Jul 2019 10:40:48 -0700 Subject: [PATCH 16/17] Updating submodule(s). [nomail] --- aux/binpac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/binpac b/aux/binpac index baabe22a2b..9c116009fd 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit baabe22a2b8a68fac448e862e1c2acc46f89c5fc +Subproject commit 9c116009fdcb0dc8a388c3a7e2b075f019e74be4 From 23a1815e299650de959b1334ced514c2c107fba4 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 1 Jul 2019 12:17:53 -0700 Subject: [PATCH 17/17] Updating submodule(s). [nomail] --- aux/broker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broker b/aux/broker index 3f827567ed..87a678477b 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 3f827567edca20eb0fe9ad071519f305699296ea +Subproject commit 87a678477b4e5c2186ee77c86456f43aff596107