diff --git a/CHANGES b/CHANGES index 261c88494e..fc5d4478a3 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,85 @@ +2.6-558 | 2019-07-01 01:27:50 -0700 + + * GH-443: fix uses of timestamp 0 in cluster diagnostic logs + + For broker.log and cluster.log: there was a race condition. A worker's + first IOSource that it processes is potentially Broker if there were + no packets available yet and thread scheduling happens to work out + such that network connections (inside CAF threads) become established + before we enter the main I/O loop. Such peering establishments would + generate logs with timestamp 0 as there was not yet any code path + taken that would update network_time. + + For reporter.log: any non-worker (packet-processing) node would just + unnecessarily use a timestamp of 0 for their reporter messages. (Jon Siwek, Corelight) + +2.6-556 | 2019-07-01 00:56:13 -0700 + + * Improve deprecation warning messages (Jon Siwek, Corelight) + + * Remove deprecated DNS events + + - dns_full_request + - non_dns_request (Jon Siwek, Corelight) + + * Remove BackDoor analyzer (Jon Siwek, Corelight) + + * Remove InterConn analyzer (Jon Siwek, Corelight) + + * Remove deprecated/unused irc_servers option (Jon Siwek, Corelight) + + * Remove deprecated print_hook event (Jon Siwek, Corelight) + + * Remove dead code: dump_used_event_handlers (Jon Siwek, Corelight) + + * Remove unused software_version_found events + + - software_version_found + - software_unparsed_version_found + - software_parse_error (Jon Siwek, Corelight) + + * Remove deprecated open_log_file and log_file_name functions (Jon Siwek, Corelight) + + * Remove deprecated/unused "packet" type (Jon Siwek, Corelight) + + * Un-deprecate anonymizer BIFs (Jon Siwek, Corelight) + + * Un-deprecate file rotation functions + + - rotate_file + - rotate_file_by_name + - calc_next_rotate + + These still have use-cases even though no longer used for our logging + functionality. E.g. rotate_file_by_name may be used to rotate + pcap dump files. + + Also the log_rotate_base_time option was marked deprecated, but still + used in the new logging framework. (Jon Siwek, Corelight) + + * Switch default CAF scheduler policy to work sharing + + It may generally be better for our default use-case, as workers may + save a few percent cpu utilization as this policy does not have to + use any polling like the stealing policy does. + + This also helps avoid a potential issue with the implementation of + spinlocks used in the work-stealing policy in current CAF versions, + where there's some conditions where lock contention causes a thread + to spin for long periods without relinquishing the cpu to others. (Jon Siwek, Corelight) + + * Update sqlite to 3.28.0. (Johanna Amann, Corelight) + + * GH-320: Improve RFB (VNC) protocol parsing + + Parsing now stops for both client and server if either encounters + any parsing error or invalid state. + + After a complete handshake, server messages are no longer parsed. + Support for that is incomplete and not sure it's that useful anyway + since it mostly contains pixel data. (Jon Siwek, Corelight) + 2.6-536 | 2019-06-28 12:10:55 -0700 * Add Windows Minidump file signature (Alexander Bolshakov) diff --git a/NEWS b/NEWS index 05037e1b12..acd25aff53 100644 --- a/NEWS +++ b/NEWS @@ -388,6 +388,9 @@ Removed Functionality - ``send_state`` - ``checkpoint_state`` - ``rescan_state`` + - ``log_file_name`` + - ``open_log_file`` + - ``disable_print_hook`` - The following events were deprecated in version 2.6 or below and are completely removed from this release: @@ -413,12 +416,38 @@ Removed Functionality - ``remote_log`` - ``finished_send_state`` - ``remote_pong`` + - ``software_version_found`` + - ``software_unparsed_version_found`` + - ``software_parse_error`` + - ``print_hook`` + - ``interconn_stats`` + - ``interconn_remove_conn`` + - ``root_backdoor_signature_found`` + - ``napster_signature_found`` + - ``kazaa_signature_found`` + - ``gaobot_signature_found`` + - ``ftp_signature_found`` + - ``gnutella_signature_found`` + - ``http_signature_found`` + - ``irc_signature_found`` + - ``telnet_signature_found`` + - ``ssh_signature_found`` + - ``rlogin_signature_found`` + - ``smtp_signature_found`` + - ``http_proxy_signature_found`` + - ``backdoor_stats`` + - ``backdoor_remove_conn`` + - ``dns_full_request`` + - ``non_dns_request`` - The following types/records were deprecated in version 2.6 or below and are removed from this release: - ``peer_id`` - ``event_peer`` + - ``packet`` + - ``software`` + - ``software_version`` - The following configuration options were deprecated in version 2.6 or below and are removed from this release: @@ -438,6 +467,18 @@ Removed Functionality - ``ssl_ca_certificate`` - ``ssl_private_key`` - ``ssl_passphrase`` + - ``suppress_local_output`` + - ``irc_servers`` + - ``interconn_min_interarrival`` + - ``interconn_max_interarrival`` + - ``interconn_max_keystroke_pkt_size`` + - ``interconn_default_pkt_size`` + - ``interconn_stat_period`` + - ``interconn_stat_backoff`` + - ``interconn_endp_stats`` + - ``backdoor_stat_period`` + - ``backdoor_stat_backoff`` + - ``backdoor_endp_stats`` - The following constants were used as part of deprecated functionality in version 2.6 or below and are removed from this release: @@ -483,6 +524,10 @@ Removed Functionality This is typically not necessary and it's a problem that is more appropriately addressed at the system configuration level. +- Removed the InterConn analyzer. + +- Removed the BackDoor analyzer. + Deprecated Functionality ------------------------ @@ -500,11 +545,6 @@ Deprecated Functionality - The ``bro_is_terminating`` and ``bro_version`` function are deprecated and replaced by functions named ``zeek_is_terminating`` and ``zeek_version``. -- The ``rotate_file``, ``rotate_file_by_name`` and ``calc_next_rotate`` functions - were marked as deprecated. These functions were used with the old pre-2.0 logging - framework and are no longer used. They also were marked as deprecated in their - documentation, however the functions themselves did not carry the deprecation marker. - Bro 2.6 ======= diff --git a/VERSION b/VERSION index 5afbf50255..8acd744a9c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.6-536 +2.6-558 diff --git a/scripts/base/frameworks/analyzer/main.zeek b/scripts/base/frameworks/analyzer/main.zeek index 8fd986e497..3bfd60bfe6 100644 --- a/scripts/base/frameworks/analyzer/main.zeek +++ b/scripts/base/frameworks/analyzer/main.zeek @@ -124,9 +124,7 @@ export { ## A set of analyzers to disable by default at startup. The default set ## contains legacy analyzers that are no longer supported. global disabled_analyzers: set[Analyzer::Tag] = { - ANALYZER_INTERCONN, ANALYZER_STEPPINGSTONE, - ANALYZER_BACKDOOR, ANALYZER_TCPSTATS, } &redef; } diff --git a/scripts/base/frameworks/broker/main.zeek b/scripts/base/frameworks/broker/main.zeek index 2b43c3fd2b..d3fefd8939 100644 --- a/scripts/base/frameworks/broker/main.zeek +++ b/scripts/base/frameworks/broker/main.zeek @@ -73,32 +73,43 @@ export { ## ZEEK_BROKER_MAX_THREADS environment variable overrides this setting. const max_threads = 1 &redef; + ## The CAF scheduling policy to use. Available options are "sharing" and + ## "stealing". The "sharing" policy uses a single, global work queue along + ## with mutex and condition variable used for accessing it, which may be + ## better for cases that don't require much concurrency or need lower power + ## consumption. The "stealing" policy uses multiple work queues protected + ## by spinlocks, which may be better for use-cases that have more + ## concurrency needs. E.g. may be worth testing the "stealing" policy + ## along with dedicating more threads if a lot of data store processing is + ## required. + const scheduler_policy = "sharing" &redef; + ## Interval of time for under-utilized Broker/CAF threads to sleep - ## when in "moderate" mode. + ## when in "moderate" mode. Only used for the "stealing" scheduler policy. const moderate_sleep = 16 msec &redef; ## Interval of time for under-utilized Broker/CAF threads to sleep - ## when in "relaxed" mode. + ## when in "relaxed" mode. Only used for the "stealing" scheduler policy. const relaxed_sleep = 64 msec &redef; ## Number of work-stealing polling attempts for Broker/CAF threads - ## in "aggressive" mode. + ## in "aggressive" mode. Only used for the "stealing" scheduler policy. const aggressive_polls = 5 &redef; ## Number of work-stealing polling attempts for Broker/CAF threads - ## in "moderate" mode. + ## in "moderate" mode. Only used for the "stealing" scheduler policy. const moderate_polls = 5 &redef; ## Frequency of work-stealing polling attempts for Broker/CAF threads - ## in "aggressive" mode. + ## in "aggressive" mode. Only used for the "stealing" scheduler policy. const aggressive_interval = 4 &redef; ## Frequency of work-stealing polling attempts for Broker/CAF threads - ## in "moderate" mode. + ## in "moderate" mode. Only used for the "stealing" scheduler policy. const moderate_interval = 2 &redef; ## Frequency of work-stealing polling attempts for Broker/CAF threads - ## in "relaxed" mode. + ## in "relaxed" mode. Only used for the "stealing" scheduler policy. const relaxed_interval = 1 &redef; ## Forward all received messages to subscribing peers. diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index 1910dd82fb..bfb49579cd 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -644,17 +644,6 @@ type ReporterStats: record { weirds_by_type: table[string] of count; }; -## Deprecated. -## -## .. todo:: Remove. It's still declared internally but doesn't seem used anywhere -## else. -type packet: record { - conn: connection; - is_orig: bool; - seq: count; ##< seq=k => it is the kth *packet* of the connection - timestamp: time; -}; - ## Table type used to map variable names to their memory allocation. ## ## .. zeek:see:: global_sizes @@ -755,8 +744,6 @@ global restrict_filters: table[string] of string &redef; ## :zeek:see:`Pcap::precompile_pcap_filter` and :zeek:see:`Pcap::precompile_pcap_filter`. type PcapFilterID: enum { None }; -## Deprecated. -## ## .. zeek:see:: anonymize_addr type IPAddrAnonymization: enum { KEEP_ORIG_ADDR, @@ -766,8 +753,6 @@ type IPAddrAnonymization: enum { PREFIX_PRESERVING_MD5, }; -## Deprecated. -## ## .. zeek:see:: anonymize_addr type IPAddrAnonymizationClass: enum { ORIG_ADDR, @@ -775,8 +760,6 @@ type IPAddrAnonymizationClass: enum { OTHER_ADDR, }; -## Deprecated. -## ## .. zeek:see:: rotate_file rotate_file_by_name type rotate_info: record { old_name: string; ##< Original filename. @@ -1792,30 +1775,9 @@ type gtp_delete_pdp_ctx_response_elements: record { @load base/bif/strings.bif @load base/bif/option.bif -## Deprecated. This is superseded by the new logging framework. -global log_file_name: function(tag: string): string &redef; - -## Deprecated. This is superseded by the new logging framework. -global open_log_file: function(tag: string): file &redef; - global done_with_network = F; event net_done(t: time) { done_with_network = T; } -function log_file_name(tag: string): string - { - local suffix = getenv("ZEEK_LOG_SUFFIX"); - - if ( suffix == "" ) - suffix = "log"; - - return fmt("%s.%s", tag, suffix); - } - -function open_log_file(tag: string): file - { - return open(log_file_name(tag)); - } - ## Internal function. function add_interface(iold: string, inew: string): string { @@ -3893,12 +3855,6 @@ type PE::SectionHeader: record { } module GLOBAL; -## Deprecated. -## -## .. todo:: Remove. It's still declared internally but doesn't seem used anywhere -## else. -global irc_servers : set[addr] &redef; - ## Internal to the stepping stone detector. const stp_delta: interval &redef; @@ -3908,56 +3864,6 @@ const stp_idle_min: interval &redef; ## Internal to the stepping stone detector. global stp_skip_src: set[addr] &redef; -## Deprecated. -const interconn_min_interarrival: interval &redef; - -## Deprecated. -const interconn_max_interarrival: interval &redef; - -## Deprecated. -const interconn_max_keystroke_pkt_size: count &redef; - -## Deprecated. -const interconn_default_pkt_size: count &redef; - -## Deprecated. -const interconn_stat_period: interval &redef; - -## Deprecated. -const interconn_stat_backoff: double &redef; - -## Deprecated. -type interconn_endp_stats: record { - num_pkts: count; - num_keystrokes_two_in_row: count; - num_normal_interarrivals: count; - num_8k0_pkts: count; - num_8k4_pkts: count; - is_partial: bool; - num_bytes: count; - num_7bit_ascii: count; - num_lines: count; - num_normal_lines: count; -}; - -## Deprecated. -const backdoor_stat_period: interval &redef; - -## Deprecated. -const backdoor_stat_backoff: double &redef; - -## Deprecated. -type backdoor_endp_stats: record { - is_partial: bool; - num_pkts: count; - num_8k0_pkts: count; - num_8k4_pkts: count; - num_lines: count; - num_normal_lines: count; - num_bytes: count; - num_7bit_ascii: count; -}; - ## Description of a signature match. ## ## .. zeek:see:: signature_match @@ -3968,26 +3874,6 @@ type signature_state: record { payload_size: count; ##< Payload size of the first matching packet of current endpoint. }; -# Deprecated. -# -# .. todo:: This type is no longer used. Remove any reference of this from the -# core. -type software_version: record { - major: int; - minor: int; - minor2: int; - addl: string; -}; - -# Deprecated. -# -# .. todo:: This type is no longer used. Remove any reference of this from the -# core. -type software: record { - name: string; - version: software_version; -}; - # Type used to report load samples via :zeek:see:`load_sample`. For now, it's a # set of names (event names, source file names, and perhaps ````), which were seen during the sample. @@ -4622,7 +4508,7 @@ module GLOBAL; ## BPF filter the user has set via the -f command line options. Empty if none. const cmd_line_bpf_filter = "" &redef; -## Deprecated. +## Base time of log rotations in 24-hour time format (``%H:%M``), e.g. "12:00". const log_rotate_base_time = "0:00" &redef; ## Write profiling info into this file in regular intervals. The easiest way to @@ -4781,13 +4667,6 @@ const time_machine_profiling = F &redef; ## If true, warns about unused event handlers at startup. const check_for_unused_event_handlers = F &redef; -# If true, dumps all invoked event handlers at startup. -# todo::Still used? -# const dump_used_event_handlers = F &redef; - -## Deprecated. -const suppress_local_output = F &redef; - ## Holds the filename of the trace file given with ``-w`` (empty if none). ## ## .. zeek:see:: record_all_packets diff --git a/scripts/policy/misc/profiling.zeek b/scripts/policy/misc/profiling.zeek index 197d3a3f82..4b6f05f169 100644 --- a/scripts/policy/misc/profiling.zeek +++ b/scripts/policy/misc/profiling.zeek @@ -2,8 +2,18 @@ module Profiling; +function log_suffix(): string + { + local rval = getenv("ZEEK_LOG_SUFFIX"); + + if ( rval == "" ) + return "log"; + + return rval; + } + ## Set the profiling output file. -redef profiling_file = open_log_file("prof"); +redef profiling_file = open(fmt("prof.%s", Profiling::log_suffix())); ## Set the cheap profiling interval. redef profiling_interval = 15 secs; diff --git a/src/Conn.cc b/src/Conn.cc index 31c1b1a191..dc510038f0 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -448,173 +448,6 @@ void Connection::Match(Rule::PatternType type, const u_char* data, int len, bool primary_PIA->Match(type, data, len, is_orig, bol, eol, clear_state); } -Val* Connection::BuildVersionVal(const char* s, int len) - { - Val* name = 0; - Val* major = 0; - Val* minor = 0; - Val* minor2 = 0; - Val* addl = 0; - - const char* last = s + len; - const char* e = s; - - // This is all just a guess... - - // Eat non-alpha-numerical chars. - for ( ; s < last && ! isalnum(*s); ++s ) - ; - - // Leading characters are the program name. - // (first character must not be a digit) - if ( isalpha(*s) ) - { - for ( e = s; e < last && ! is_version_sep(e, last); ++e ) - ; - - if ( s != e ) - name = new StringVal(e - s, s); - } - - // Find first number - that's the major version. - for ( s = e; s < last && ! isdigit(*s); ++s ) - ; - for ( e = s; e < last && isdigit(*e); ++e ) - ; - - if ( s != e ) - major = val_mgr->GetInt(atoi(s)); - - // Find second number seperated only by punctuation chars - - // that's the minor version. - for ( s = e; s < last && ispunct(*s); ++s ) - ; - for ( e = s; e < last && isdigit(*e); ++e ) - ; - - if ( s != e ) - minor = val_mgr->GetInt(atoi(s)); - - // Find second number seperated only by punctuation chars; - - // that's the minor version. - for ( s = e; s < last && ispunct(*s); ++s ) - ; - for ( e = s; e < last && isdigit(*e); ++e ) - ; - - if ( s != e ) - minor2 = val_mgr->GetInt(atoi(s)); - - // Anything after following punctuation and until next white space is - // an additional version string. - for ( s = e; s < last && ispunct(*s); ++s ) - ; - for ( e = s; e < last && ! isspace(*e); ++e ) - ; - - if ( s != e ) - addl = new StringVal(e - s, s); - - // If we do not have a name yet, the next alphanumerical string is it. - if ( ! name ) - { // eat non-alpha-numerical characters - for ( s = e; s < last && ! isalpha(*s); ++s ) - ; - - // Get name. - for ( e = s; e < last && (isalnum(*e) || *e == '_'); ++e ) - ; - - if ( s != e ) - name = new StringVal(e - s, s); - } - - // We need at least a name. - if ( ! name ) - { - Unref(major); - Unref(minor); - Unref(minor2); - Unref(addl); - return 0; - } - - RecordVal* version = new RecordVal(software_version); - version->Assign(0, major ? major : val_mgr->GetInt(-1)); - version->Assign(1, minor ? minor : val_mgr->GetInt(-1)); - version->Assign(2, minor2 ? minor2 : val_mgr->GetInt(-1)); - version->Assign(3, addl ? addl : val_mgr->GetEmptyString()); - - RecordVal* sw = new RecordVal(software); - sw->Assign(0, name); - sw->Assign(1, version); - - return sw; - } - -int Connection::VersionFoundEvent(const IPAddr& addr, const char* s, int len, - analyzer::Analyzer* analyzer) - { - if ( ! software_version_found && ! software_parse_error ) - return 1; - - if ( ! is_printable(s, len) ) - return 0; - - Val* val = BuildVersionVal(s, len); - if ( ! val ) - { - if ( software_parse_error ) - { - ConnectionEventFast(software_parse_error, analyzer, { - BuildConnVal(), - new AddrVal(addr), - new StringVal(len, s), - }); - } - return 0; - } - - if ( software_version_found ) - { - ConnectionEventFast(software_version_found, 0, { - BuildConnVal(), - new AddrVal(addr), - val, - new StringVal(len, s), - }); - } - else - Unref(val); - - return 1; - } - -int Connection::UnparsedVersionFoundEvent(const IPAddr& addr, - const char* full, int len, analyzer::Analyzer* analyzer) - { - // Skip leading white space. - while ( len && isspace(*full) ) - { - --len; - ++full; - } - - if ( ! is_printable(full, len) ) - return 0; - - if ( software_unparsed_version_found ) - { - ConnectionEventFast(software_unparsed_version_found, analyzer, { - BuildConnVal(), - new AddrVal(addr), - new StringVal(len, full), - }); - } - - return 1; - } - void Connection::Event(EventHandlerPtr f, analyzer::Analyzer* analyzer, const char* name) { if ( ! f ) diff --git a/src/Conn.h b/src/Conn.h index bd5ddaae92..1f0215e01f 100644 --- a/src/Conn.h +++ b/src/Conn.h @@ -160,18 +160,6 @@ public: void Match(Rule::PatternType type, const u_char* data, int len, bool is_orig, bool bol, bool eol, bool clear_state); - // Tries really hard to extract a program name and a version. - Val* BuildVersionVal(const char* s, int len); - - // Raises a software_version_found event based on the - // given string (returns false if it's not parseable). - int VersionFoundEvent(const IPAddr& addr, const char* s, int len, - analyzer::Analyzer* analyzer = 0); - - // Raises a software_unparsed_version_found event. - int UnparsedVersionFoundEvent(const IPAddr& addr, - const char* full_descr, int len, analyzer::Analyzer* analyzer); - // If a handler exists for 'f', an event will be generated. If 'name' is // given that event's first argument will be it, and it's second will be // the connection value. If 'name' is null, then the event's first diff --git a/src/File.cc b/src/File.cc index d1f706514f..28fd8bea9c 100644 --- a/src/File.cc +++ b/src/File.cc @@ -168,7 +168,6 @@ void BroFile::Init() is_open = 0; attrs = 0; buffered = true; - print_hook = true; raw_output = false; t = 0; diff --git a/src/File.h b/src/File.h index 07512d4465..ffd912ab39 100644 --- a/src/File.h +++ b/src/File.h @@ -63,9 +63,6 @@ public: // Get the file with the given name, opening it if it doesn't yet exist. static BroFile* GetFile(const char* name); - void DisablePrintHook() { print_hook = false; } - bool IsPrintHookEnabled() const { return print_hook; } - void EnableRawOutput() { raw_output = true; } bool IsRawOutput() const { return raw_output; } @@ -98,7 +95,6 @@ protected: Attributes* attrs; bool buffered; double open_time; - bool print_hook; bool raw_output; static const int MIN_BUFFER_SIZE = 1024; diff --git a/src/NetVar.cc b/src/NetVar.cc index 6c59f2a5fa..c256cfd496 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -125,28 +125,10 @@ double stp_delta; double stp_idle_min; TableVal* stp_skip_src; -double interconn_min_interarrival; -double interconn_max_interarrival; -int interconn_max_keystroke_pkt_size; -int interconn_default_pkt_size; -double interconn_stat_period; -double interconn_stat_backoff; -RecordType* interconn_endp_stats; - -double backdoor_stat_period; -double backdoor_stat_backoff; - -RecordType* backdoor_endp_stats; - -RecordType* software; -RecordType* software_version; - double table_expire_interval; double table_expire_delay; int table_incremental_step; -RecordType* packet_type; - double connection_status_update_interval; int orig_addr_anonymization, resp_addr_anonymization; @@ -179,7 +161,6 @@ int sig_max_group_size; TableType* irc_join_list; RecordType* irc_join_info; -TableVal* irc_servers; int dpd_reassemble_first_packets; int dpd_buffer_size; @@ -189,7 +170,6 @@ int dpd_ignore_ports; TableVal* likely_server_ports; int check_for_unused_event_handlers; -int dump_used_event_handlers; int suppress_local_output; @@ -241,8 +221,6 @@ void init_general_global_var() check_for_unused_event_handlers = opt_internal_int("check_for_unused_event_handlers"); - dump_used_event_handlers = - opt_internal_int("dump_used_event_handlers"); suppress_local_output = opt_internal_int("suppress_local_output"); @@ -398,23 +376,6 @@ void init_net_var() stp_idle_min = opt_internal_double("stp_idle_min"); stp_skip_src = internal_val("stp_skip_src")->AsTableVal(); - interconn_min_interarrival = opt_internal_double("interconn_min_interarrival"); - interconn_max_interarrival = opt_internal_double("interconn_max_interarrival"); - interconn_max_keystroke_pkt_size = opt_internal_int("interconn_max_keystroke_pkt_size"); - interconn_default_pkt_size = opt_internal_int("interconn_default_pkt_size"); - interconn_stat_period = opt_internal_double("interconn_stat_period"); - interconn_stat_backoff = opt_internal_double("interconn_stat_backoff"); - interconn_endp_stats = internal_type("interconn_endp_stats")->AsRecordType(); - - backdoor_stat_period = opt_internal_double("backdoor_stat_period"); - backdoor_stat_backoff = opt_internal_double("backdoor_stat_backoff"); - backdoor_endp_stats = internal_type("backdoor_endp_stats")->AsRecordType(); - - software = internal_type("software")->AsRecordType(); - software_version = internal_type("software_version")->AsRecordType(); - - packet_type = internal_type("packet")->AsRecordType(); - orig_addr_anonymization = opt_internal_int("orig_addr_anonymization"); resp_addr_anonymization = opt_internal_int("resp_addr_anonymization"); other_addr_anonymization = opt_internal_int("other_addr_anonymization"); @@ -442,7 +403,6 @@ void init_net_var() irc_join_info = internal_type("irc_join_info")->AsRecordType(); irc_join_list = internal_type("irc_join_list")->AsTableType(); - irc_servers = internal_val("irc_servers")->AsTableVal(); dpd_reassemble_first_packets = opt_internal_int("dpd_reassemble_first_packets"); diff --git a/src/NetVar.h b/src/NetVar.h index 3c9d92eb4d..5dc5fcfb79 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -128,28 +128,10 @@ extern double stp_delta; extern double stp_idle_min; extern TableVal* stp_skip_src; -extern double interconn_min_interarrival; -extern double interconn_max_interarrival; -extern int interconn_max_keystroke_pkt_size; -extern int interconn_default_pkt_size; -extern double interconn_stat_period; -extern double interconn_stat_backoff; -extern RecordType* interconn_endp_stats; - -extern double backdoor_stat_period; -extern double backdoor_stat_backoff; - -extern RecordType* backdoor_endp_stats; - -extern RecordType* software; -extern RecordType* software_version; - extern double table_expire_interval; extern double table_expire_delay; extern int table_incremental_step; -extern RecordType* packet_type; - extern int orig_addr_anonymization, resp_addr_anonymization; extern int other_addr_anonymization; extern TableVal* preserve_orig_addr; @@ -181,7 +163,6 @@ extern int sig_max_group_size; extern TableType* irc_join_list; extern RecordType* irc_join_info; -extern TableVal* irc_servers; extern int dpd_reassemble_first_packets; extern int dpd_buffer_size; @@ -191,7 +172,6 @@ extern int dpd_ignore_ports; extern TableVal* likely_server_ports; extern int check_for_unused_event_handlers; -extern int dump_used_event_handlers; extern int suppress_local_output; diff --git a/src/Reporter.cc b/src/Reporter.cc index a40ddb9a3d..d44eff556b 100644 --- a/src/Reporter.cc +++ b/src/Reporter.cc @@ -491,7 +491,7 @@ void Reporter::DoLog(const char* prefix, EventHandlerPtr event, FILE* out, val_list vl(vl_size); if ( time ) - vl.append(new Val((bro_start_network_time != 0.0) ? network_time : 0, TYPE_TIME)); + vl.append(new Val(network_time ? network_time : current_time(), TYPE_TIME)); vl.append(new StringVal(buffer)); diff --git a/src/Sessions.cc b/src/Sessions.cc index e668815cfb..5f290f5386 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -20,10 +20,6 @@ #include "analyzer/protocol/stepping-stone/SteppingStone.h" #include "analyzer/protocol/stepping-stone/events.bif.h" -#include "analyzer/protocol/backdoor/BackDoor.h" -#include "analyzer/protocol/backdoor/events.bif.h" -#include "analyzer/protocol/interconn/InterConn.h" -#include "analyzer/protocol/interconn/events.bif.h" #include "analyzer/protocol/arp/ARP.h" #include "analyzer/protocol/arp/events.bif.h" #include "Discard.h" @@ -119,13 +115,6 @@ NetSessions::NetSessions() packet_filter = 0; - build_backdoor_analyzer = - backdoor_stats || rlogin_signature_found || - telnet_signature_found || ssh_signature_found || - root_backdoor_signature_found || ftp_signature_found || - napster_signature_found || kazaa_signature_found || - http_signature_found || http_proxy_signature_found; - dump_this_packet = 0; num_packets_processed = 0; diff --git a/src/Sessions.h b/src/Sessions.h index 617ab3e52a..27b0ab474c 100644 --- a/src/Sessions.h +++ b/src/Sessions.h @@ -231,7 +231,6 @@ protected: analyzer::stepping_stone::SteppingStoneManager* stp_manager; Discarder* discarder; PacketFilter* packet_filter; - int build_backdoor_analyzer; int dump_this_packet; // if true, current packet should be recorded uint64 num_packets_processed; PacketProfiler* pkt_profiler; diff --git a/src/Stmt.cc b/src/Stmt.cc index 0a1d22a296..7dfa89d435 100644 --- a/src/Stmt.cc +++ b/src/Stmt.cc @@ -203,49 +203,25 @@ Val* PrintStmt::DoExec(val_list* vals, stmt_flow_type& /* flow */) const ++offset; } - bool ph = print_hook && f->IsPrintHookEnabled(); - desc_style style = f->IsRawOutput() ? RAW_STYLE : STANDARD_STYLE; - if ( ! (suppress_local_output && ph) ) - { - if ( f->IsRawOutput() ) - { - ODesc d(DESC_READABLE); - d.SetFlush(0); - d.SetStyle(style); - - PrintVals(&d, vals, offset); - f->Write(d.Description(), d.Len()); - } - else - { - ODesc d(DESC_READABLE, f); - d.SetFlush(0); - d.SetStyle(style); - - PrintVals(&d, vals, offset); - f->Write("\n", 1); - } - } - - if ( ph ) + if ( f->IsRawOutput() ) { ODesc d(DESC_READABLE); + d.SetFlush(0); d.SetStyle(style); + PrintVals(&d, vals, offset); + f->Write(d.Description(), d.Len()); + } + else + { + ODesc d(DESC_READABLE, f); + d.SetFlush(0); + d.SetStyle(style); - if ( print_hook ) - { - ::Ref(f); - - // Note, this doesn't do remote printing. - mgr.Dispatch( - new Event( - print_hook, - {new Val(f), new StringVal(d.Len(), d.Description())}), - true); - } + PrintVals(&d, vals, offset); + f->Write("\n", 1); } return 0; diff --git a/src/Val.h b/src/Val.h index 881d8d80d6..74039fa62d 100644 --- a/src/Val.h +++ b/src/Val.h @@ -87,7 +87,7 @@ typedef union { class Val : public BroObj { public: - ZEEK_DEPRECATED("use val_mgr->GetBool, GetFalse/GetTrue, GetInt, or GetCount instead") + ZEEK_DEPRECATED("Remove in v3.1: use val_mgr->GetBool, GetFalse/GetTrue, GetInt, or GetCount instead") Val(bool b, TypeTag t) { val.int_val = b; @@ -97,7 +97,7 @@ public: #endif } - ZEEK_DEPRECATED("use val_mgr->GetBool, GetFalse/GetTrue, GetInt, or GetCount instead") + ZEEK_DEPRECATED("Remove in v3.1: use val_mgr->GetBool, GetFalse/GetTrue, GetInt, or GetCount instead") Val(int32 i, TypeTag t) { val.int_val = bro_int_t(i); @@ -107,7 +107,7 @@ public: #endif } - ZEEK_DEPRECATED("use val_mgr->GetBool, GetFalse/GetTrue, GetInt, or GetCount instead") + ZEEK_DEPRECATED("Remove in v3.1: use val_mgr->GetBool, GetFalse/GetTrue, GetInt, or GetCount instead") Val(uint32 u, TypeTag t) { val.uint_val = bro_uint_t(u); @@ -117,7 +117,7 @@ public: #endif } - ZEEK_DEPRECATED("use val_mgr->GetBool, GetFalse/GetTrue, GetInt, or GetCount instead") + ZEEK_DEPRECATED("Remove in v3.1: use val_mgr->GetBool, GetFalse/GetTrue, GetInt, or GetCount instead") Val(int64 i, TypeTag t) { val.int_val = i; @@ -127,7 +127,7 @@ public: #endif } - ZEEK_DEPRECATED("use val_mgr->GetBool, GetFalse/GetTrue, GetInt, or GetCount instead") + ZEEK_DEPRECATED("Remove in v3.1: use val_mgr->GetBool, GetFalse/GetTrue, GetInt, or GetCount instead") Val(uint64 u, TypeTag t) { val.uint_val = u; @@ -430,15 +430,15 @@ protected: class PortManager { public: // Port number given in host order. - ZEEK_DEPRECATED("use val_mgr->GetPort() instead") + ZEEK_DEPRECATED("Remove in v3.1: use val_mgr->GetPort() instead") PortVal* Get(uint32 port_num, TransportProto port_type) const; // Host-order port number already masked with port space protocol mask. - ZEEK_DEPRECATED("use val_mgr->GetPort() instead") + ZEEK_DEPRECATED("Remove in v3.1: use val_mgr->GetPort() instead") PortVal* Get(uint32 port_num) const; // Returns a masked port number - ZEEK_DEPRECATED("use PortVal::Mask() instead") + ZEEK_DEPRECATED("Remove in v3.1: use PortVal::Mask() instead") uint32 Mask(uint32 port_num, TransportProto port_type) const; }; @@ -520,11 +520,11 @@ protected: class PortVal : public Val { public: // Port number given in host order. - ZEEK_DEPRECATED("use val_mgr->GetPort() instead") + ZEEK_DEPRECATED("Remove in v3.1: use val_mgr->GetPort() instead") PortVal(uint32 p, TransportProto port_type); // Host-order port number already masked with port space protocol mask. - ZEEK_DEPRECATED("use val_mgr->GetPort() instead") + ZEEK_DEPRECATED("Remove in v3.1: use val_mgr->GetPort() instead") explicit PortVal(uint32 p); Val* SizeVal() const override { return val_mgr->GetInt(val.uint_val); } @@ -998,7 +998,7 @@ protected: class EnumVal : public Val { public: - ZEEK_DEPRECATED("use t->GetVal(i) instead") + ZEEK_DEPRECATED("Remove in v3.1: use t->GetVal(i) instead") EnumVal(int i, EnumType* t) : Val(t) { val.int_val = i; diff --git a/src/analyzer/Manager.cc b/src/analyzer/Manager.cc index c7e156b41e..da19b7685f 100644 --- a/src/analyzer/Manager.cc +++ b/src/analyzer/Manager.cc @@ -5,10 +5,8 @@ #include "Hash.h" #include "Val.h" -#include "protocol/backdoor/BackDoor.h" #include "protocol/conn-size/ConnSize.h" #include "protocol/icmp/ICMP.h" -#include "protocol/interconn/InterConn.h" #include "protocol/pia/PIA.h" #include "protocol/stepping-stone/SteppingStone.h" #include "protocol/tcp/TCP.h" @@ -87,9 +85,7 @@ Manager::~Manager() void Manager::InitPreScript() { // Cache these tags. - analyzer_backdoor = GetComponentTag("BACKDOOR"); analyzer_connsize = GetComponentTag("CONNSIZE"); - analyzer_interconn = GetComponentTag("INTERCONN"); analyzer_stepping = GetComponentTag("STEPPINGSTONE"); analyzer_tcpstats = GetComponentTag("TCPSTATS"); } @@ -461,16 +457,6 @@ bool Manager::BuildInitialAnalyzerTree(Connection* conn) if ( reass ) tcp->EnableReassembly(); - if ( IsEnabled(analyzer_backdoor) ) - // Add a BackDoor analyzer if requested. This analyzer - // can handle both reassembled and non-reassembled input. - tcp->AddChildAnalyzer(new backdoor::BackDoor_Analyzer(conn), false); - - if ( IsEnabled(analyzer_interconn) ) - // Add a InterConn analyzer if requested. This analyzer - // can handle both reassembled and non-reassembled input. - tcp->AddChildAnalyzer(new interconn::InterConn_Analyzer(conn), false); - if ( IsEnabled(analyzer_stepping) ) { // Add a SteppingStone analyzer if requested. The port diff --git a/src/analyzer/Manager.h b/src/analyzer/Manager.h index 8f6d982394..14bba1d115 100644 --- a/src/analyzer/Manager.h +++ b/src/analyzer/Manager.h @@ -355,9 +355,7 @@ private: analyzer_map_by_port analyzers_by_port_tcp; analyzer_map_by_port analyzers_by_port_udp; - Tag analyzer_backdoor; Tag analyzer_connsize; - Tag analyzer_interconn; Tag analyzer_stepping; Tag analyzer_tcpstats; diff --git a/src/analyzer/protocol/CMakeLists.txt b/src/analyzer/protocol/CMakeLists.txt index 8ebded627b..d5024a2ff1 100644 --- a/src/analyzer/protocol/CMakeLists.txt +++ b/src/analyzer/protocol/CMakeLists.txt @@ -1,7 +1,6 @@ add_subdirectory(arp) add_subdirectory(ayiya) -add_subdirectory(backdoor) add_subdirectory(bittorrent) add_subdirectory(conn-size) add_subdirectory(dce-rpc) @@ -18,7 +17,6 @@ add_subdirectory(http) add_subdirectory(icmp) add_subdirectory(ident) add_subdirectory(imap) -add_subdirectory(interconn) add_subdirectory(irc) add_subdirectory(krb) add_subdirectory(login) diff --git a/src/analyzer/protocol/backdoor/BackDoor.cc b/src/analyzer/protocol/backdoor/BackDoor.cc deleted file mode 100644 index 2e8d47d1d0..0000000000 --- a/src/analyzer/protocol/backdoor/BackDoor.cc +++ /dev/null @@ -1,819 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#include "zeek-config.h" - -#include "BackDoor.h" -#include "Event.h" -#include "Net.h" -#include "analyzer/protocol/tcp/TCP.h" - -#include "events.bif.h" - -using namespace analyzer::backdoor; - -BackDoorEndpoint::BackDoorEndpoint(tcp::TCP_Endpoint* e) - { - endp = e; - is_partial = 0; - max_top_seq = 0; - - rlogin_checking_done = 0; - rlogin_string_separator_pos = 0; - rlogin_num_null = 0; - rlogin_slash_seen = 0; - - num_pkts = num_8k0_pkts = num_8k4_pkts = - num_lines = num_normal_lines = num_bytes = num_7bit_ascii = 0; - } - -#define NORMAL_LINE_LENGTH 80 - -#define TELNET_IAC 255 -#define IS_TELNET_NEGOTIATION_CMD(c) ((c) >= 251 && (c) <= 254) - -#define DEFAULT_MTU 512 - -#define RLOGIN_MAX_SIGNATURE_LENGTH 256 - -void BackDoorEndpoint::FinalCheckForRlogin() - { - if ( ! rlogin_checking_done ) - { - rlogin_checking_done = 1; - - if ( rlogin_num_null > 0 ) - RloginSignatureFound(0); - } - } - -int BackDoorEndpoint::DataSent(double /* t */, uint64 seq, - int len, int caplen, const u_char* data, - const IP_Hdr* /* ip */, - const struct tcphdr* /* tp */) - { - if ( caplen < len ) - len = caplen; - - if ( len <= 0 ) - return 0; - - if ( endp->state == tcp::TCP_ENDPOINT_PARTIAL ) - is_partial = 1; - - uint64 ack = endp->ToRelativeSeqSpace(endp->AckSeq(), endp->AckWraps()); - uint64 top_seq = seq + len; - - if ( top_seq <= ack || top_seq <= max_top_seq ) - // There is no new data in this packet. - return 0; - - if ( rlogin_signature_found ) - CheckForRlogin(seq, len, data); - - if ( telnet_signature_found ) - CheckForTelnet(seq, len, data); - - if ( ssh_signature_found ) - CheckForSSH(seq, len, data); - - if ( ftp_signature_found ) - CheckForFTP(seq, len, data); - - if ( root_backdoor_signature_found ) - CheckForRootBackdoor(seq, len, data); - - if ( napster_signature_found ) - CheckForNapster(seq, len, data); - - if ( gnutella_signature_found ) - CheckForGnutella(seq, len, data); - - if ( kazaa_signature_found ) - CheckForKazaa(seq, len, data); - - if ( http_signature_found || http_proxy_signature_found ) - CheckForHTTP(seq, len, data); - - if ( smtp_signature_found ) - CheckForSMTP(seq, len, data); - - if ( irc_signature_found ) - CheckForIRC(seq, len, data); - - if ( gaobot_signature_found ) - CheckForGaoBot(seq, len, data); - - max_top_seq = top_seq; - - return 1; - } - -RecordVal* BackDoorEndpoint::BuildStats() - { - RecordVal* stats = new RecordVal(backdoor_endp_stats); - - stats->Assign(0, val_mgr->GetBool(is_partial)); - stats->Assign(1, val_mgr->GetCount(num_pkts)); - stats->Assign(2, val_mgr->GetCount(num_8k0_pkts)); - stats->Assign(3, val_mgr->GetCount(num_8k4_pkts)); - stats->Assign(4, val_mgr->GetCount(num_lines)); - stats->Assign(5, val_mgr->GetCount(num_normal_lines)); - stats->Assign(6, val_mgr->GetCount(num_bytes)); - stats->Assign(7, val_mgr->GetCount(num_7bit_ascii)); - - return stats; - } - -void BackDoorEndpoint::CheckForRlogin(uint64 seq, int len, const u_char* data) - { - if ( rlogin_checking_done ) - return; - - // Looking for pattern: - // stringstringstring/string - // where all string's are non-empty 7-bit-ascii string - // - // To avoid having to reassemble, we keep testing each byte until - // one of the following happens: - // - // - A gap in sequence number occurs - // - Four null's have been found - // - The number of bytes we examined reaches RLOGIN_MAX_SIGNATURE_LENGTH - // - An empty or non-7-bit-ascii string is found - // - if ( seq == 1 ) - { // Check if first byte is a NUL. - if ( data[0] == 0 ) - { - rlogin_num_null = 1; - - if ( ! endp->IsOrig() ) - { - RloginSignatureFound(len); - return; - } - - rlogin_string_separator_pos = 1; - - ++seq; // move past the byte - ++data; - --len; - } - else - { - rlogin_checking_done = 1; - return; - } - } - - if ( seq > max_top_seq && max_top_seq != 0 ) - { // A gap! Since we don't reassemble things, stop now. - RloginSignatureFound(0); - return; - } - - if ( seq + len <= max_top_seq ) - return; // nothing new - - if ( seq < max_top_seq ) - { // trim to just the new data - int64 delta = max_top_seq - seq; - seq += delta; - data += delta; - len -= delta; - } - - // Search for rlogin signature. - for ( int i = 0; i < len && rlogin_num_null < 4; ++i ) - { - if ( data[i] == 0 ) - { - if ( i + seq == rlogin_string_separator_pos + 1 ) - { // Empty string found. - rlogin_checking_done = 1; - return; - } - else - { - rlogin_string_separator_pos = i + seq; - ++rlogin_num_null; - } - } - - else if ( data[i] == '/' ) - { - if ( rlogin_num_null == 3 ) - { - if ( i + seq == rlogin_string_separator_pos + 1 ) - { // Empty terminal type. - rlogin_checking_done = 1; - return; - } - - rlogin_string_separator_pos = i + seq; - rlogin_slash_seen = 1; - } - } - - else if ( data[i] >= 128 ) - { // Non-7-bit-ascii - rlogin_checking_done = 1; - return; - } - } - - if ( rlogin_num_null == 4 ) - { - if ( rlogin_slash_seen ) - RloginSignatureFound(0); - else - rlogin_checking_done = 1; - - return; - } - - if ( seq + len > RLOGIN_MAX_SIGNATURE_LENGTH ) - { // We've waited for too long - RloginSignatureFound(0); - return; - } - } - -void BackDoorEndpoint::RloginSignatureFound(int len) - { - if ( rlogin_checking_done ) - return; - - rlogin_checking_done = 1; - - if ( ! rlogin_signature_found ) - return; - - endp->TCP()->ConnectionEventFast(rlogin_signature_found, { - endp->TCP()->BuildConnVal(), - val_mgr->GetBool(endp->IsOrig()), - val_mgr->GetCount(rlogin_num_null), - val_mgr->GetCount(len), - }); - } - -void BackDoorEndpoint::CheckForTelnet(uint64 /* seq */, int len, const u_char* data) - { - if ( len >= 3 && - data[0] == TELNET_IAC && IS_TELNET_NEGOTIATION_CMD(data[1]) ) - { - TelnetSignatureFound(len); - return; - } - - // Note, we do the analysis per-packet rather than on the reassembled - // stream. This is a lot more efficient as then we don't need to - // do stream reassembly; but it's potentially less accurate, and - // subject to evasion. *But*: backdoor detection is inherently - // subject to a wide variety of evasion, so allowing this form - // (which is a pain to exploit) costs little. - - num_bytes += len; - - int last_char = 0; - int offset = 0; // where we consider the latest line to have begun - int option_length = 0; // length of options in a line - - for ( int i = 0; i < len; ++i ) - { - unsigned int c = data[i]; - - if ( c == '\n' && last_char == '\r' ) - { - // Compress CRLF to just one line termination. - last_char = c; - continue; - } - - if ( c == '\n' || c == '\r' ) - { - ++num_lines; - - if ( i - offset - option_length <= NORMAL_LINE_LENGTH ) - ++num_normal_lines; - - option_length = 0; - offset = i; - } - - else if ( c == TELNET_IAC ) - { - ++option_length; - --num_bytes; - - if ( ++i < len ) - { - unsigned int code = data[i]; - if ( code == TELNET_IAC ) - // Escaped IAC. - last_char = code; - - else if ( code >= 251 && code <= 254 ) - { // 3-byte option: ignore next byte - ++i; - option_length += 2; - num_bytes -= 2; - } - - else - // XXX: We don't deal with sub option for simplicity - // although we SHOULD! - { - ++option_length; - --num_bytes; - } - } - continue; - } - - else if ( c != 0 && c < 128 ) - ++num_7bit_ascii; - - last_char = c; - } - } - -void BackDoorEndpoint::TelnetSignatureFound(int len) - { - if ( ! telnet_signature_found ) - return; - - endp->TCP()->ConnectionEventFast(telnet_signature_found, { - endp->TCP()->BuildConnVal(), - val_mgr->GetBool(endp->IsOrig()), - val_mgr->GetCount(len), - }); - } - -void BackDoorEndpoint::CheckForSSH(uint64 seq, int len, const u_char* data) - { - if ( seq == 1 && CheckForString("SSH-", data, len) && len > 4 && - (data[4] == '1' || data[4] == '2') ) - { - SignatureFound(ssh_signature_found, 1); - return; - } - - // Check for length pattern. - - if ( seq < max_top_seq || max_top_seq == 0 ) - // Retransmission involved, or first pkt => size info useless. - return; - - if ( seq > max_top_seq ) - { // Estimate number of packets in the sequence gap - int64 gap = seq - max_top_seq; - if ( gap > 0 ) - num_pkts += uint64((gap + DEFAULT_MTU - 1) / DEFAULT_MTU); - } - - ++num_pkts; - - // According to the spec: - // SSH 1.x pkts have size 8k+4 - // SSH 2.x pkts have size 8k >= 16 (most cipher blocks are 8n) - if ( len <= 127 ) - switch ( len & 7 ) { - case 0: - if ( len >= 16 ) - ++num_8k0_pkts; - break; - - case 4: - ++num_8k4_pkts; - break; - } - else - { // len is likely to be some MTU. - } - } - -void BackDoorEndpoint::CheckForRootBackdoor(uint64 seq, int len, const u_char* data) - { - // Check for root backdoor signature: an initial payload of - // exactly "# ". - if ( seq == 1 && len == 2 && ! endp->IsOrig() && - data[0] == '#' && data[1] == ' ' ) - SignatureFound(root_backdoor_signature_found); - } - -void BackDoorEndpoint::CheckForFTP(uint64 seq, int len, const u_char* data) - { - // Check for FTP signature - // - // Currently, the signatures include: "220 ", "220-" - // - // For a day's worth of LBNL FTP activity (7,229 connections), - // the distribution of the code in the first line returned by - // the server (the lines always began with a code) is: - // - // 220: 6685 - // 421: 535 - // 226: 7 - // 426: 1 - // 200: 1 - // - // The 421's are all "host does not have access" or "timeout" of - // some form, so it's not big deal with we miss them (if that helps - // keep down the false positives). - - if ( seq != 1 || endp->IsOrig() || len < 4 ) - return; - - if ( CheckForString("220", data, len) && - (data[3] == ' ' || data[3] == '-') ) - SignatureFound(ftp_signature_found); - - else if ( CheckForString("421", data, len) && - (data[3] == '-' || data[3] == ' ') ) - SignatureFound(ftp_signature_found); - } - -void BackDoorEndpoint::CheckForNapster(uint64 seq, int len, const u_char* data) - { - // Check for Napster signature "GETfoobar" or "SENDfoobar" where - // "foobar" is the Napster handle associated with the request - // (so pretty much any arbitrary identifier, but sent adjacent - // to the GET or SEND with no intervening whitespace; but also - // sent in a separate packet. - - if ( seq != 1 || ! endp->IsOrig() ) - return; - - if ( len == 3 && CheckForString("GET", data, len) ) - // GETfoobar. - SignatureFound(napster_signature_found); - - else if ( len == 4 && CheckForString("SEND", data, len) ) - // SENDfoobar. - SignatureFound(napster_signature_found); - } - -void BackDoorEndpoint::CheckForSMTP(uint64 seq, int len, const u_char* data) - { - const char* smtp_handshake[] = { "HELO", "EHLO", 0 }; - - if ( seq != 1 ) - return; - - if ( CheckForStrings(smtp_handshake, data, len) ) - SignatureFound(smtp_signature_found); - } - -void BackDoorEndpoint::CheckForIRC(uint64 seq, int len, const u_char* data) - { - if ( seq != 1 || is_partial ) - return; - - const char* irc_indicator[] = { - "ERROR", "INVITE", "ISON", "JOIN", "KICK", "NICK", - "NJOIN", "NOTICE AUTH", "OPER", "PART", "PING", "PONG", - "PRIVMSG", "SQUERY", "SQUIT", "WHO", 0, - }; - - if ( CheckForStrings(irc_indicator, data, len) ) - SignatureFound(irc_signature_found); - } - -void BackDoorEndpoint::CheckForGnutella(uint64 seq, int len, const u_char* data) - { - // After connecting to the server, the connecting client says: - // - // GNUTELLA CONNECT/\n\n - // - // The accepting server responds: - // - // GNUTELLA OK\n\n - // - // We find checking the first 8 bytes suffices, and that will - // also catch variants that use something other than "CONNECT". - - if ( seq == 1 && CheckForString("GNUTELLA ", data, len) ) - SignatureFound(gnutella_signature_found); - } - -void BackDoorEndpoint::CheckForGaoBot(uint64 seq, int len, const u_char* data) - { - if ( seq == 1 && CheckForString("220 Bot Server (Win32)", data, len) ) - SignatureFound(gaobot_signature_found); - } - -void BackDoorEndpoint::CheckForKazaa(uint64 seq, int len, const u_char* data) - { - // *Some*, though not all, KaZaa connections begin with: - // - // GIVE - - if ( seq == 1 && CheckForString("GIVE ", data, len) ) - SignatureFound(kazaa_signature_found); - } - - -int is_http_whitespace(const u_char ch) - { - return ! isprint(ch) || isspace(ch); - } - -int skip_http_whitespace(const u_char* data, int len, int max) - { - int k; - for ( k = 0; k < len; ++k ) - { - if ( ! is_http_whitespace(data[k]) ) - break; - - // Here we do not go beyond CR -- this is OK for - // processing first line of HTTP requests. However, it - // cannot be used to process multiple-line headers. - - if ( data[k] == '\015' || k == max ) - return -1; - } - - return k < len ? k : -1; - } - -int is_absolute_url(const u_char* data, int len) - { - // Look for '://' in the URL. - const char* abs_url_sig = "://"; - const char* abs_url_sig_pos = abs_url_sig; - - // Warning: the following code is NOT general for any signature string, - // but only works for specific strings like "://". - - for ( int pos = 0; pos < len; ++pos ) - { - if ( *abs_url_sig_pos == '\0' ) - return 1; - - if ( data[pos] == *abs_url_sig_pos ) - ++abs_url_sig_pos; - - else - { - if ( is_http_whitespace(data[pos]) ) - return 0; - - abs_url_sig_pos = abs_url_sig; - if ( *abs_url_sig != '\0' && - *abs_url_sig_pos == data[pos] ) - ++abs_url_sig_pos; - } - } - - return *abs_url_sig_pos == '\0'; - } - -void BackDoorEndpoint::CheckForHTTP(uint64 seq, int len, const u_char* data) - { - // According to the RFC, we should look for - // ' SP SP HTTP/ CR LF' - // where: - // - // = GET | HEAD | POST - // - // (i.e., HTTP 1.1 methods are ignored for now) - // = 1.0 | 1.1. - // - // However, this is probably too restrictive to catch 'non-standard' - // requests. Instead, we look for certain methods only in the first - // line of the first packet only. - // - // "The method is case-sensitive." -- RFC 2616 - - const char* http_method[] = { "GET", "HEAD", "POST", 0 }; - - if ( seq != 1 ) - return; // first packet only - - // Pick up the method. - int pos = skip_http_whitespace (data, len, 0); - if ( pos < 0 ) - return; - - int method; - for ( method = 0; http_method[method]; ++method ) - { - const char* s = http_method[method]; - int i; - for ( i = pos; i < len; ++i, ++s ) - if ( data[i] != *s ) - break; - - if ( *s == '\0' ) - { - pos = i; - break; - } - } - - if ( ! http_method[method] ) - return; - - if ( pos >= len || ! is_http_whitespace(data[pos]) ) - return; - - if ( http_signature_found ) - SignatureFound(http_signature_found); - - if ( http_proxy_signature_found ) - { - const u_char* rest = data + pos; - int rest_len = len - pos; - - pos = skip_http_whitespace(rest, rest_len, rest_len); - - if ( pos >= 0 ) - CheckForHTTPProxy(seq, rest_len - pos, rest + pos); - } - } - -void BackDoorEndpoint::CheckForHTTPProxy(uint64 /* seq */, int len, - const u_char* data) - { - // Proxy ONLY accepts absolute URI's: "The absoluteURI form is - // REQUIRED when the request is being made to a proxy." -- RFC 2616 - - if ( is_absolute_url(data, len) ) - SignatureFound(http_proxy_signature_found); - } - - -void BackDoorEndpoint::SignatureFound(EventHandlerPtr e, int do_orig) - { - if ( ! e ) - return; - - if ( do_orig ) - endp->TCP()->ConnectionEventFast(e, - {endp->TCP()->BuildConnVal(), val_mgr->GetBool(endp->IsOrig())}); - - else - endp->TCP()->ConnectionEventFast(e, {endp->TCP()->BuildConnVal()}); - } - - -int BackDoorEndpoint::CheckForStrings(const char** strs, - const u_char* data, int len) - { - for ( ; *strs; ++strs ) - if ( CheckForFullString(*strs, data, len) ) - return 1; - - return 0; - } - -int BackDoorEndpoint::CheckForFullString(const char* str, - const u_char* data, int len) - { - for ( ; len > 0 && *str; --len, ++data, ++str ) - if ( *str != *data ) - return 0; - - // A "full" string means a non-prefix match. - return *str == 0 && (len == 0 || *data == ' ' || *data == '\t'); - } - -int BackDoorEndpoint::CheckForString(const char* str, - const u_char* data, int len) - { - for ( ; len > 0 && *str; --len, ++data, ++str ) - if ( *str != *data ) - return 0; - - return *str == 0; - } - - -BackDoor_Analyzer::BackDoor_Analyzer(Connection* c) -: tcp::TCP_ApplicationAnalyzer("BACKDOOR", c) - { - orig_endp = resp_endp = 0; - - orig_stream_pos = resp_stream_pos = 1; - - timeout = backdoor_stat_period; - backoff = backdoor_stat_backoff; - - c->GetTimerMgr()->Add(new BackDoorTimer(network_time + timeout, this)); - } - -BackDoor_Analyzer::~BackDoor_Analyzer() - { - delete orig_endp; - delete resp_endp; - } - -void BackDoor_Analyzer::Init() - { - tcp::TCP_ApplicationAnalyzer::Init(); - - assert(TCP()); - orig_endp = new BackDoorEndpoint(TCP()->Orig()); - resp_endp = new BackDoorEndpoint(TCP()->Resp()); - } - -void BackDoor_Analyzer::DeliverPacket(int len, const u_char* data, bool is_orig, - uint64 seq, const IP_Hdr* ip, int caplen) - { - Analyzer::DeliverPacket(len, data, is_orig, seq, ip, caplen); - - if ( is_orig ) - orig_endp->DataSent(network_time, seq, len, caplen, data, 0, 0); - else - resp_endp->DataSent(network_time, seq, len, caplen, data, 0, 0); - } - -void BackDoor_Analyzer::DeliverStream(int len, const u_char* data, bool is_orig) - { - Analyzer::DeliverStream(len, data, is_orig); - - if ( is_orig ) - { - orig_endp->DataSent(network_time, orig_stream_pos, - len, len, data, 0, 0); - orig_stream_pos += len; - } - - else - { - resp_endp->DataSent(network_time, resp_stream_pos, - len, len, data, 0, 0); - resp_stream_pos += len; - } - } - -void BackDoor_Analyzer::Done() - { - tcp::TCP_ApplicationAnalyzer::Done(); - - if ( ! IsFinished() ) - { - orig_endp->FinalCheckForRlogin(); - resp_endp->FinalCheckForRlogin(); - - if ( ! TCP()->Skipping() ) - StatEvent(); - - RemoveEvent(); - } - - } - -void BackDoor_Analyzer::StatTimer(double t, int is_expire) - { - if ( IsFinished() || TCP()->Skipping() ) - return; - - StatEvent(); - - if ( ! is_expire ) - { - timeout *= backoff; - timer_mgr->Add(new BackDoorTimer(t + timeout, this)); - } - } - -void BackDoor_Analyzer::StatEvent() - { - if ( ! backdoor_stats ) - return; - - TCP()->ConnectionEventFast(backdoor_stats, { - TCP()->BuildConnVal(), - orig_endp->BuildStats(), - resp_endp->BuildStats(), - }); - } - -void BackDoor_Analyzer::RemoveEvent() - { - if ( ! backdoor_remove_conn ) - return; - - TCP()->ConnectionEventFast(backdoor_remove_conn, {TCP()->BuildConnVal()}); - } - -BackDoorTimer::BackDoorTimer(double t, BackDoor_Analyzer* a) -: Timer(t, TIMER_BACKDOOR) - { - analyzer = a; - // Make sure connection does not expire. - Ref(a->Conn()); - } - -BackDoorTimer::~BackDoorTimer() - { - Unref(analyzer->Conn()); - } - -void BackDoorTimer::Dispatch(double t, int is_expire) - { - analyzer->StatTimer(t, is_expire); - } diff --git a/src/analyzer/protocol/backdoor/BackDoor.h b/src/analyzer/protocol/backdoor/BackDoor.h deleted file mode 100644 index c954d176d5..0000000000 --- a/src/analyzer/protocol/backdoor/BackDoor.h +++ /dev/null @@ -1,112 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#ifndef ANALYZER_PROTOCOL_BACKDOOR_BACKDOOR_H -#define ANALYZER_PROTOCOL_BACKDOOR_BACKDOOR_H - -#include "analyzer/protocol/tcp/TCP.h" -#include "Timer.h" -#include "NetVar.h" -#include "analyzer/protocol/login/Login.h" - -namespace analyzer { namespace backdoor { - -class BackDoorEndpoint { -public: - explicit BackDoorEndpoint(tcp::TCP_Endpoint* e); - - int DataSent(double t, uint64 seq, int len, int caplen, const u_char* data, - const IP_Hdr* ip, const struct tcphdr* tp); - - RecordVal* BuildStats(); - - void FinalCheckForRlogin(); - -protected: - void CheckForRlogin(uint64 seq, int len, const u_char* data); - void RloginSignatureFound(int len); - - void CheckForTelnet(uint64 seq, int len, const u_char* data); - void TelnetSignatureFound(int len); - - void CheckForSSH(uint64 seq, int len, const u_char* data); - void CheckForFTP(uint64 seq, int len, const u_char* data); - void CheckForRootBackdoor(uint64 seq, int len, const u_char* data); - void CheckForNapster(uint64 seq, int len, const u_char* data); - void CheckForGnutella(uint64 seq, int len, const u_char* data); - void CheckForKazaa(uint64 seq, int len, const u_char* data); - void CheckForHTTP(uint64 seq, int len, const u_char* data); - void CheckForHTTPProxy(uint64 seq, int len, const u_char* data); - void CheckForSMTP(uint64 seq, int len, const u_char* data); - void CheckForIRC(uint64 seq, int len, const u_char* data); - void CheckForGaoBot(uint64 seq, int len, const u_char* data); - - void SignatureFound(EventHandlerPtr e, int do_orig = 0); - - int CheckForStrings(const char** strs, const u_char* data, int len); - int CheckForFullString(const char* str, const u_char* data, int len); - int CheckForString(const char* str, const u_char* data, int len); - - tcp::TCP_Endpoint* endp; - int is_partial; - uint64 max_top_seq; - - int rlogin_checking_done; - int rlogin_num_null; - uint64 rlogin_string_separator_pos; - int rlogin_slash_seen; - - uint32 num_pkts; - uint32 num_8k4_pkts; - uint32 num_8k0_pkts; - uint32 num_lines; - uint32 num_normal_lines; - uint32 num_bytes; - uint32 num_7bit_ascii; -}; - -class BackDoor_Analyzer : public tcp::TCP_ApplicationAnalyzer { -public: - explicit BackDoor_Analyzer(Connection* c); - ~BackDoor_Analyzer() override; - - void Init() override; - void Done() override; - void StatTimer(double t, int is_expire); - - static analyzer::Analyzer* Instantiate(Connection* conn) - { return new BackDoor_Analyzer(conn); } - -protected: - // We support both packet and stream input, and can be instantiated - // even if the TCP analyzer is not yet reassembling. - void DeliverPacket(int len, const u_char* data, bool is_orig, - uint64 seq, const IP_Hdr* ip, int caplen) override; - void DeliverStream(int len, const u_char* data, bool is_orig) override; - - void StatEvent(); - void RemoveEvent(); - - BackDoorEndpoint* orig_endp; - BackDoorEndpoint* resp_endp; - - int orig_stream_pos; - int resp_stream_pos; - - double timeout; - double backoff; -}; - -class BackDoorTimer : public Timer { -public: - BackDoorTimer(double t, BackDoor_Analyzer* a); - ~BackDoorTimer() override; - - void Dispatch(double t, int is_expire) override; - -protected: - BackDoor_Analyzer* analyzer; -}; - -} } // namespace analyzer::* - -#endif diff --git a/src/analyzer/protocol/backdoor/CMakeLists.txt b/src/analyzer/protocol/backdoor/CMakeLists.txt deleted file mode 100644 index 66511d3d99..0000000000 --- a/src/analyzer/protocol/backdoor/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ - -include(ZeekPlugin) - -include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) - -zeek_plugin_begin(Zeek BackDoor) -zeek_plugin_cc(BackDoor.cc Plugin.cc) -zeek_plugin_bif(events.bif) -zeek_plugin_end() diff --git a/src/analyzer/protocol/backdoor/Plugin.cc b/src/analyzer/protocol/backdoor/Plugin.cc deleted file mode 100644 index aeec615c50..0000000000 --- a/src/analyzer/protocol/backdoor/Plugin.cc +++ /dev/null @@ -1,25 +0,0 @@ -// See the file in the main distribution directory for copyright. - - -#include "plugin/Plugin.h" - -#include "BackDoor.h" - -namespace plugin { -namespace Zeek_BackDoor { - -class Plugin : public plugin::Plugin { -public: - plugin::Configuration Configure() - { - AddComponent(new ::analyzer::Component("BackDoor", ::analyzer::backdoor::BackDoor_Analyzer::Instantiate)); - - plugin::Configuration config; - config.name = "Zeek::BackDoor"; - config.description = "Backdoor Analyzer deprecated"; - return config; - } -} plugin; - -} -} diff --git a/src/analyzer/protocol/backdoor/events.bif b/src/analyzer/protocol/backdoor/events.bif deleted file mode 100644 index 81676ee43b..0000000000 --- a/src/analyzer/protocol/backdoor/events.bif +++ /dev/null @@ -1,32 +0,0 @@ -## Deprecated. Will be removed. -event backdoor_stats%(c: connection, os: backdoor_endp_stats, rs: backdoor_endp_stats%); - -## Deprecated. Will be removed. -event backdoor_remove_conn%(c: connection%); - -## Deprecated. Will be removed. -event ftp_signature_found%(c: connection%); - -## Deprecated. Will be removed. -event gnutella_signature_found%(c: connection%); - -## Deprecated. Will be removed. -event http_signature_found%(c: connection%); - -## Deprecated. Will be removed. -event irc_signature_found%(c: connection%); - -## Deprecated. Will be removed. -event telnet_signature_found%(c: connection, is_orig: bool, len: count%); - -## Deprecated. Will be removed. -event ssh_signature_found%(c: connection, is_orig: bool%); - -## Deprecated. Will be removed. -event rlogin_signature_found%(c: connection, is_orig: bool, num_null: count, len: count%); - -## Deprecated. Will be removed. -event smtp_signature_found%(c: connection%); - -## Deprecated. Will be removed. -event http_proxy_signature_found%(c: connection%); diff --git a/src/analyzer/protocol/dns/DNS.cc b/src/analyzer/protocol/dns/DNS.cc index 51a8d1cec3..c07b936f49 100644 --- a/src/analyzer/protocol/dns/DNS.cc +++ b/src/analyzer/protocol/dns/DNS.cc @@ -1758,21 +1758,7 @@ void DNS_Analyzer::DeliverPacket(int len, const u_char* data, bool orig, uint64 seq, const IP_Hdr* ip, int caplen) { tcp::TCP_ApplicationAnalyzer::DeliverPacket(len, data, orig, seq, ip, caplen); - - if ( orig ) - { - if ( ! interp->ParseMessage(data, len, 1) && non_dns_request ) - { - if ( non_dns_request ) - ConnectionEventFast(non_dns_request, { - BuildConnVal(), - new StringVal(len, (const char*) data), - }); - } - } - - else - interp->ParseMessage(data, len, 0); + interp->ParseMessage(data, len, orig); } diff --git a/src/analyzer/protocol/dns/events.bif b/src/analyzer/protocol/dns/events.bif index 7ddbd0c7b3..35e9ffa0fd 100644 --- a/src/analyzer/protocol/dns/events.bif +++ b/src/analyzer/protocol/dns/events.bif @@ -16,9 +16,9 @@ ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply ## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end -## dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_query_reply dns_rejected -## dns_request non_dns_request dns_max_queries dns_session_timeout dns_skip_addl +## dns_request dns_max_queries dns_session_timeout dns_skip_addl ## dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_message%(c: connection, is_orig: bool, msg: dns_msg, len: count%); @@ -43,9 +43,9 @@ event dns_message%(c: connection, is_orig: bool, msg: dns_msg, len: count%); ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply ## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end -## dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply -## dns_rejected non_dns_request dns_max_queries dns_session_timeout dns_skip_addl +## dns_rejected dns_max_queries dns_session_timeout dns_skip_addl ## dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_request%(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count%); @@ -72,9 +72,9 @@ event dns_request%(c: connection, msg: dns_msg, query: string, qtype: count, qcl ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply ## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end -## dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply -## dns_request non_dns_request dns_max_queries dns_session_timeout dns_skip_addl +## dns_request dns_max_queries dns_session_timeout dns_skip_addl ## dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_rejected%(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count%); @@ -98,9 +98,9 @@ event dns_rejected%(c: connection, msg: dns_msg, query: string, qtype: count, qc ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply ## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end -## dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name +## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_rejected -## dns_request non_dns_request dns_max_queries dns_session_timeout dns_skip_addl +## dns_request dns_max_queries dns_session_timeout dns_skip_addl ## dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_query_reply%(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count%); @@ -123,10 +123,10 @@ event dns_query_reply%(c: connection, msg: dns_msg, query: string, ## ## .. zeek:see:: dns_AAAA_reply dns_A6_reply dns_CNAME_reply dns_EDNS_addl dns_HINFO_reply ## dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply dns_SRV_reply -## dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_full_request +## dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end ## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply -## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_rejected dns_request dns_max_queries dns_session_timeout ## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_A_reply%(c: connection, msg: dns_msg, ans: dns_answer, a: addr%); @@ -148,10 +148,10 @@ event dns_A_reply%(c: connection, msg: dns_msg, ans: dns_answer, a: addr%); ## ## .. zeek:see:: dns_A_reply dns_A6_reply dns_CNAME_reply dns_EDNS_addl dns_HINFO_reply dns_MX_reply ## dns_NS_reply dns_PTR_reply dns_SOA_reply dns_SRV_reply dns_TSIG_addl -## dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_full_request dns_mapping_altered +## dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_mapping_altered ## dns_mapping_lost_name dns_mapping_new_name dns_mapping_unverified ## dns_mapping_valid dns_message dns_query_reply dns_rejected dns_request -## non_dns_request dns_max_queries dns_session_timeout dns_skip_addl +## dns_max_queries dns_session_timeout dns_skip_addl ## dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_AAAA_reply%(c: connection, msg: dns_msg, ans: dns_answer, a: addr%); @@ -173,10 +173,10 @@ event dns_AAAA_reply%(c: connection, msg: dns_msg, ans: dns_answer, a: addr%); ## ## .. zeek:see:: dns_A_reply dns_AAAA_reply dns_CNAME_reply dns_EDNS_addl dns_HINFO_reply dns_MX_reply ## dns_NS_reply dns_PTR_reply dns_SOA_reply dns_SRV_reply dns_TSIG_addl -## dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_full_request dns_mapping_altered +## dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_mapping_altered ## dns_mapping_lost_name dns_mapping_new_name dns_mapping_unverified ## dns_mapping_valid dns_message dns_query_reply dns_rejected dns_request -## non_dns_request dns_max_queries dns_session_timeout dns_skip_addl +## dns_max_queries dns_session_timeout dns_skip_addl ## dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_A6_reply%(c: connection, msg: dns_msg, ans: dns_answer, a: addr%); @@ -198,10 +198,10 @@ event dns_A6_reply%(c: connection, msg: dns_msg, ans: dns_answer, a: addr%); ## ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_MX_reply dns_PTR_reply dns_SOA_reply dns_SRV_reply -## dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_full_request +## dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end ## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply -## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_rejected dns_request dns_max_queries dns_session_timeout ## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_NS_reply%(c: connection, msg: dns_msg, ans: dns_answer, name: string%); @@ -223,10 +223,10 @@ event dns_NS_reply%(c: connection, msg: dns_msg, ans: dns_answer, name: string%) ## ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_EDNS_addl dns_HINFO_reply dns_MX_reply ## dns_NS_reply dns_PTR_reply dns_SOA_reply dns_SRV_reply dns_TSIG_addl -## dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_full_request dns_mapping_altered +## dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_mapping_altered ## dns_mapping_lost_name dns_mapping_new_name dns_mapping_unverified ## dns_mapping_valid dns_message dns_query_reply dns_rejected dns_request -## non_dns_request dns_max_queries dns_session_timeout dns_skip_addl +## dns_max_queries dns_session_timeout dns_skip_addl ## dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_CNAME_reply%(c: connection, msg: dns_msg, ans: dns_answer, name: string%); @@ -248,10 +248,10 @@ event dns_CNAME_reply%(c: connection, msg: dns_msg, ans: dns_answer, name: strin ## ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_SOA_reply dns_SRV_reply -## dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_full_request +## dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end ## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply -## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_rejected dns_request dns_max_queries dns_session_timeout ## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_PTR_reply%(c: connection, msg: dns_msg, ans: dns_answer, name: string%); @@ -273,10 +273,10 @@ event dns_PTR_reply%(c: connection, msg: dns_msg, ans: dns_answer, name: string% ## ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SRV_reply -## dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_full_request +## dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end ## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply -## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_rejected dns_request dns_max_queries dns_session_timeout ## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_SOA_reply%(c: connection, msg: dns_msg, ans: dns_answer, soa: dns_soa%); @@ -296,10 +296,10 @@ event dns_SOA_reply%(c: connection, msg: dns_msg, ans: dns_answer, soa: dns_soa% ## ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply -## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_end dns_full_request +## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_end ## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply -## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_rejected dns_request dns_max_queries dns_session_timeout ## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_WKS_reply%(c: connection, msg: dns_msg, ans: dns_answer%); @@ -319,10 +319,10 @@ event dns_WKS_reply%(c: connection, msg: dns_msg, ans: dns_answer%); ## ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl dns_MX_reply ## dns_NS_reply dns_PTR_reply dns_SOA_reply dns_SRV_reply dns_TSIG_addl -## dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_full_request dns_mapping_altered +## dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_mapping_altered ## dns_mapping_lost_name dns_mapping_new_name dns_mapping_unverified ## dns_mapping_valid dns_message dns_query_reply dns_rejected dns_request -## non_dns_request dns_max_queries dns_session_timeout dns_skip_addl +## dns_max_queries dns_session_timeout dns_skip_addl ## dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_HINFO_reply%(c: connection, msg: dns_msg, ans: dns_answer%); @@ -346,10 +346,10 @@ event dns_HINFO_reply%(c: connection, msg: dns_msg, ans: dns_answer%); ## ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_NS_reply dns_PTR_reply dns_SOA_reply dns_SRV_reply -## dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_full_request +## dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end ## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply -## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_rejected dns_request dns_max_queries dns_session_timeout ## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_MX_reply%(c: connection, msg: dns_msg, ans: dns_answer, name: string, preference: count%); @@ -371,10 +371,10 @@ event dns_MX_reply%(c: connection, msg: dns_msg, ans: dns_answer, name: string, ## ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply -## dns_SRV_reply dns_TSIG_addl dns_WKS_reply dns_end dns_full_request +## dns_SRV_reply dns_TSIG_addl dns_WKS_reply dns_end ## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply -## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_rejected dns_request dns_max_queries dns_session_timeout ## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_TXT_reply%(c: connection, msg: dns_msg, ans: dns_answer, strs: string_vec%); @@ -396,10 +396,10 @@ event dns_TXT_reply%(c: connection, msg: dns_msg, ans: dns_answer, strs: string_ ## ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply -## dns_SRV_reply dns_TSIG_addl dns_WKS_reply dns_end dns_full_request +## dns_SRV_reply dns_TSIG_addl dns_WKS_reply dns_end ## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply -## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_rejected dns_request dns_max_queries dns_session_timeout ## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_SPF_reply%(c: connection, msg: dns_msg, ans: dns_answer, strs: string_vec%); @@ -450,10 +450,10 @@ event dns_CAA_reply%(c: connection, msg: dns_msg, ans: dns_answer, flags: count, ## ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply -## dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_full_request +## dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end ## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply -## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_rejected dns_request dns_max_queries dns_session_timeout ## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_SRV_reply%(c: connection, msg: dns_msg, ans: dns_answer, target: string, priority: count, weight: count, p: count%); @@ -488,10 +488,10 @@ event dns_unknown_reply%(c: connection, msg: dns_msg, ans: dns_answer%); ## ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_HINFO_reply dns_MX_reply ## dns_NS_reply dns_PTR_reply dns_SOA_reply dns_SRV_reply dns_TSIG_addl -## dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_full_request dns_mapping_altered +## dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_mapping_altered ## dns_mapping_lost_name dns_mapping_new_name dns_mapping_unverified ## dns_mapping_valid dns_message dns_query_reply dns_rejected dns_request -## non_dns_request dns_max_queries dns_session_timeout dns_skip_addl +## dns_max_queries dns_session_timeout dns_skip_addl ## dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_EDNS_addl%(c: connection, msg: dns_msg, ans: dns_edns_additional%); @@ -511,10 +511,10 @@ event dns_EDNS_addl%(c: connection, msg: dns_msg, ans: dns_edns_additional%); ## ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply -## dns_SRV_reply dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end dns_full_request +## dns_SRV_reply dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_end ## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply -## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_rejected dns_request dns_max_queries dns_session_timeout ## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_TSIG_addl%(c: connection, msg: dns_msg, ans: dns_tsig_additional%); @@ -600,21 +600,9 @@ event dns_DS%(c: connection, msg: dns_msg, ans: dns_answer, ds: dns_ds_rr%); ## ## .. zeek:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl ## dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply -## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply dns_full_request +## dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_SPF_reply dns_WKS_reply ## dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name ## dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply -## dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout +## dns_rejected dns_request dns_max_queries dns_session_timeout ## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_end%(c: connection, msg: dns_msg%); - -## Deprecated. Will be removed. -## -## .. todo:: Unclear what this event is for; it's never raised. We should just -## remove it. -event dns_full_request%(%); - -## msg: The raw DNS payload. -## -## .. note:: This event is deprecated and superseded by Zeek's dynamic protocol -## detection framework. -event non_dns_request%(c: connection, msg: string%); diff --git a/src/analyzer/protocol/gnutella/events.bif b/src/analyzer/protocol/gnutella/events.bif index 4168646543..8b3ef2a177 100644 --- a/src/analyzer/protocol/gnutella/events.bif +++ b/src/analyzer/protocol/gnutella/events.bif @@ -4,7 +4,7 @@ ## information about the Gnutella protocol. ## ## .. zeek:see:: gnutella_binary_msg gnutella_establish gnutella_http_notify -## gnutella_not_establish gnutella_partial_binary_msg gnutella_signature_found +## gnutella_not_establish gnutella_partial_binary_msg ## ## ## .. todo:: Zeek's current default configuration does not activate the protocol @@ -19,7 +19,7 @@ event gnutella_text_msg%(c: connection, orig: bool, headers: string%); ## information about the Gnutella protocol. ## ## .. zeek:see:: gnutella_establish gnutella_http_notify gnutella_not_establish -## gnutella_partial_binary_msg gnutella_signature_found gnutella_text_msg +## gnutella_partial_binary_msg gnutella_text_msg ## ## .. todo:: Zeek's current default configuration does not activate the protocol ## analyzer that generates this event; the corresponding script has not yet @@ -36,7 +36,7 @@ event gnutella_binary_msg%(c: connection, orig: bool, msg_type: count, ## information about the Gnutella protocol. ## ## .. zeek:see:: gnutella_binary_msg gnutella_establish gnutella_http_notify -## gnutella_not_establish gnutella_signature_found gnutella_text_msg +## gnutella_not_establish gnutella_text_msg ## ## .. todo:: Zeek's current default configuration does not activate the protocol ## analyzer that generates this event; the corresponding script has not yet @@ -51,7 +51,7 @@ event gnutella_partial_binary_msg%(c: connection, orig: bool, ## information about the Gnutella protocol. ## ## .. zeek:see:: gnutella_binary_msg gnutella_http_notify gnutella_not_establish -## gnutella_partial_binary_msg gnutella_signature_found gnutella_text_msg +## gnutella_partial_binary_msg gnutella_text_msg ## ## .. todo:: Zeek's current default configuration does not activate the protocol ## analyzer that generates this event; the corresponding script has not yet @@ -65,7 +65,7 @@ event gnutella_establish%(c: connection%); ## information about the Gnutella protocol. ## ## .. zeek:see:: gnutella_binary_msg gnutella_establish gnutella_http_notify -## gnutella_partial_binary_msg gnutella_signature_found gnutella_text_msg +## gnutella_partial_binary_msg gnutella_text_msg ## ## .. todo:: Zeek's current default configuration does not activate the protocol ## analyzer that generates this event; the corresponding script has not yet @@ -79,7 +79,7 @@ event gnutella_not_establish%(c: connection%); ## information about the Gnutella protocol. ## ## .. zeek:see:: gnutella_binary_msg gnutella_establish gnutella_not_establish -## gnutella_partial_binary_msg gnutella_signature_found gnutella_text_msg +## gnutella_partial_binary_msg gnutella_text_msg ## ## .. todo:: Zeek's current default configuration does not activate the protocol ## analyzer that generates this event; the corresponding script has not yet diff --git a/src/analyzer/protocol/http/HTTP.cc b/src/analyzer/protocol/http/HTTP.cc index 291990119a..1dbdcf9bd8 100644 --- a/src/analyzer/protocol/http/HTTP.cc +++ b/src/analyzer/protocol/http/HTTP.cc @@ -1640,17 +1640,6 @@ int HTTP_Analyzer::ExpectReplyMessageBody() void HTTP_Analyzer::HTTP_Header(int is_orig, mime::MIME_Header* h) { -#if 0 - // ### Only call ParseVersion if we're tracking versions: - if ( istrequal(h->get_name(), "server") ) - ParseVersion(h->get_value(), - (is_orig ? Conn()->OrigAddr() : Conn()->RespAddr()), false); - - else if ( istrequal(h->get_name(), "user-agent") ) - ParseVersion(h->get_value(), - (is_orig ? Conn()->OrigAddr() : Conn()->RespAddr()), true); -#endif - // To be "liberal", we only look at "keep-alive" on the client // side, and if seen assume the connection to be persistent. // This seems fairly safe - at worst, the client does indeed @@ -1702,127 +1691,6 @@ void HTTP_Analyzer::HTTP_Header(int is_orig, mime::MIME_Header* h) } } -void HTTP_Analyzer::ParseVersion(data_chunk_t ver, const IPAddr& host, - bool user_agent) - { - int len = ver.length; - const char* data = ver.data; - - if ( software_unparsed_version_found ) - Conn()->UnparsedVersionFoundEvent(host, data, len, this); - - // The RFC defines: - // - // product = token ["/" product-version] - // product-version = token - // Server = "Server" ":" 1*( product | comment ) - - int offset; - data_chunk_t product, product_version; - int num_version = 0; - - while ( len > 0 ) - { - // Skip white space. - while ( len && mime::is_lws(*data) ) - { - ++data; - --len; - } - - // See if a comment is coming next. For User-Agent, - // we parse it, too. - if ( user_agent && len && *data == '(' ) - { - // Find end of comment. - const char* data_start = data; - const char* eoc = - data + mime::MIME_skip_lws_comments(len, data); - - // Split into parts. - // (This may get confused by nested comments, - // but we ignore this for now.) - const char* eot; - ++data; - while ( 1 ) - { - // Eat spaces. - while ( data < eoc && mime::is_lws(*data) ) - ++data; - - // Find end of token. - for ( eot = data; - eot < eoc && *eot != ';' && *eot != ')'; - ++eot ) - ; - - if ( eot == eoc ) - break; - - // Delete spaces at end of token. - for ( ; eot > data && mime::is_lws(*(eot-1)); --eot ) - ; - - if ( data != eot && software_version_found ) - Conn()->VersionFoundEvent(host, data, eot - data, this); - data = eot + 1; - } - - len -= eoc - data_start; - data = eoc; - continue; - } - - offset = mime::MIME_get_slash_token_pair(len, data, - &product, &product_version); - if ( offset < 0 ) - { - // I guess version detection is best-effort, - // so we do not complain in the final version - if ( num_version == 0 ) - HTTP_Event("bad_HTTP_version", - mime::new_string_val(len, data)); - - // Try to simply skip next token. - offset = mime::MIME_get_token(len, data, &product); - if ( offset < 0 ) - break; - - len -= offset; - data += offset; - } - - else - { - len -= offset; - data += offset; - - int version_len = - product.length + 1 + product_version.length; - - char* version_str = new char[version_len+1]; - char* s = version_str; - - memcpy(s, product.data, product.length); - - s += product.length; - *(s++) = '/'; - - memcpy(s, product_version.data, product_version.length); - - s += product_version.length; - *s = 0; - - if ( software_version_found ) - Conn()->VersionFoundEvent(host, version_str, - version_len, this); - - delete [] version_str; - ++num_version; - } - } - } - void HTTP_Analyzer::HTTP_EntityData(int is_orig, BroString* entity_data) { if ( http_entity_data ) diff --git a/src/analyzer/protocol/http/HTTP.h b/src/analyzer/protocol/http/HTTP.h index 743f9965e7..901f5e86d9 100644 --- a/src/analyzer/protocol/http/HTTP.h +++ b/src/analyzer/protocol/http/HTTP.h @@ -220,7 +220,6 @@ protected: const BroString* UnansweredRequestMethod(); - void ParseVersion(data_chunk_t ver, const IPAddr& host, bool user_agent); int HTTP_ReplyCode(const char* code_str); int ExpectReplyMessageBody(); diff --git a/src/analyzer/protocol/interconn/CMakeLists.txt b/src/analyzer/protocol/interconn/CMakeLists.txt deleted file mode 100644 index c1cf40da3f..0000000000 --- a/src/analyzer/protocol/interconn/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ - -include(ZeekPlugin) - -include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) - -zeek_plugin_begin(Zeek InterConn) -zeek_plugin_cc(InterConn.cc Plugin.cc) -zeek_plugin_bif(events.bif) -zeek_plugin_end() diff --git a/src/analyzer/protocol/interconn/InterConn.cc b/src/analyzer/protocol/interconn/InterConn.cc deleted file mode 100644 index e9a9378c90..0000000000 --- a/src/analyzer/protocol/interconn/InterConn.cc +++ /dev/null @@ -1,274 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#include "zeek-config.h" - -#include "InterConn.h" -#include "Event.h" -#include "Net.h" -#include "analyzer/protocol/tcp/TCP.h" - -#include "events.bif.h" - -using namespace analyzer::interconn; - -InterConnEndpoint::InterConnEndpoint(tcp::TCP_Endpoint* e) - { - endp = e; - max_top_seq = 0; - num_pkts = num_keystrokes_two_in_a_row = num_normal_interarrivals = - num_8k0_pkts = num_8k4_pkts = num_bytes = num_7bit_ascii = - num_lines = num_normal_lines = 0; - is_partial = keystroke_just_seen = 0; - last_keystroke_time = 0.0; - } - -#define NORMAL_LINE_LENGTH 80 - -int InterConnEndpoint::DataSent(double t, uint64 seq, int len, int caplen, - const u_char* data, const IP_Hdr* /* ip */, - const struct tcphdr* /* tp */) - { - if ( caplen < len ) - len = caplen; - - if ( len <= 0 ) - return 0; - - if ( endp->state == tcp::TCP_ENDPOINT_PARTIAL ) - is_partial = 1; - - uint64 ack = endp->ToRelativeSeqSpace(endp->AckSeq(), endp->AckWraps()); - uint64 top_seq = seq + len; - - if ( top_seq <= ack || top_seq <= max_top_seq ) - // There is no new data in this packet - return 0; - - if ( seq < max_top_seq ) - { // Only consider new data - int64 amount_seen = max_top_seq - seq; - seq += amount_seen; - data += amount_seen; - len -= amount_seen; - } - - if ( max_top_seq && seq > max_top_seq ) - // We've got a pkt above a hole - num_pkts += EstimateGapPacketNum(seq - max_top_seq); - - ++num_pkts; - max_top_seq = top_seq; - - // Count the bytes. - num_bytes += len; - - int last_char = 0; - int offset = 0; // where we consider the latest line to have begun - - for ( int i = 0; i < len; ++i ) - { - unsigned int c = data[i]; - - if ( c == '\n' && last_char == '\r' ) - { - // Compress CRLF to just one line termination. - last_char = c; - continue; - } - - if ( c == '\n' || c == '\r' ) - { - ++num_lines; - if ( i - offset <= NORMAL_LINE_LENGTH ) - ++num_normal_lines; - offset = i; - } - - else if ( c != 0 && c < 128 ) - ++num_7bit_ascii; - - last_char = c; - } - - if ( IsPotentialKeystrokePacket(len) ) - { - if ( keystroke_just_seen ) - { - ++num_keystrokes_two_in_a_row; - - if ( IsNormalKeystrokeInterarrival(t - last_keystroke_time) ) - ++num_normal_interarrivals; - } - else - keystroke_just_seen = 1; - - // Look for packets matching the SSH signature of - // being either 0 or 4 modulo 8. - switch ( len & 7 ) { - case 0: - if ( len >= 16 ) - ++num_8k0_pkts; - break; - - case 4: - ++num_8k4_pkts; - break; - } - - last_keystroke_time = t; - } - else - keystroke_just_seen = 0; - - return 1; - } - -RecordVal* InterConnEndpoint::BuildStats() - { - RecordVal* stats = new RecordVal(interconn_endp_stats); - - stats->Assign(0, val_mgr->GetCount(num_pkts)); - stats->Assign(1, val_mgr->GetCount(num_keystrokes_two_in_a_row)); - stats->Assign(2, val_mgr->GetCount(num_normal_interarrivals)); - stats->Assign(3, val_mgr->GetCount(num_8k0_pkts)); - stats->Assign(4, val_mgr->GetCount(num_8k4_pkts)); - stats->Assign(5, val_mgr->GetBool(is_partial)); - stats->Assign(6, val_mgr->GetCount(num_bytes)); - stats->Assign(7, val_mgr->GetCount(num_7bit_ascii)); - stats->Assign(8, val_mgr->GetCount(num_lines)); - stats->Assign(9, val_mgr->GetCount(num_normal_lines)); - - return stats; - } - -int InterConnEndpoint::EstimateGapPacketNum(int gap) const - { - return (gap + interconn_default_pkt_size - 1) / interconn_default_pkt_size; - } - -int InterConnEndpoint::IsPotentialKeystrokePacket(int len) const - { - return len <= interconn_max_keystroke_pkt_size; - } - -int InterConnEndpoint::IsNormalKeystrokeInterarrival(double t) const - { - return interconn_min_interarrival <= t && t <= interconn_max_interarrival; - } - -InterConn_Analyzer::InterConn_Analyzer(Connection* c) -: tcp::TCP_ApplicationAnalyzer("INTERCONN", c) - { - orig_endp = resp_endp = 0; - orig_stream_pos = resp_stream_pos = 1; - - timeout = backdoor_stat_period; - backoff = backdoor_stat_backoff; - - c->GetTimerMgr()->Add(new InterConnTimer(network_time + timeout, this)); - } - -InterConn_Analyzer::~InterConn_Analyzer() - { - Unref(orig_endp); - Unref(resp_endp); - } - -void InterConn_Analyzer::Init() - { - tcp::TCP_ApplicationAnalyzer::Init(); - - assert(TCP()); - orig_endp = new InterConnEndpoint(TCP()->Orig()); - resp_endp = new InterConnEndpoint(TCP()->Resp()); - } - -void InterConn_Analyzer::DeliverPacket(int len, const u_char* data, - bool is_orig, uint64 seq, const IP_Hdr* ip, int caplen) - { - tcp::TCP_ApplicationAnalyzer::DeliverPacket(len, data, is_orig, - seq, ip, caplen); - - if ( is_orig ) - orig_endp->DataSent(network_time, seq, len, caplen, data, 0, 0); - else - resp_endp->DataSent(network_time, seq, len, caplen, data, 0, 0); - } - -void InterConn_Analyzer::DeliverStream(int len, const u_char* data, bool is_orig) - { - tcp::TCP_ApplicationAnalyzer::DeliverStream(len, data, is_orig); - - if ( is_orig ) - { - orig_endp->DataSent(network_time, orig_stream_pos, len, len, data, 0, 0); - orig_stream_pos += len; - } - - else - { - resp_endp->DataSent(network_time, resp_stream_pos, len, len, data, 0, 0); - resp_stream_pos += len; - } - } - -void InterConn_Analyzer::Done() - { - if ( ! IsFinished() ) - { - if ( ! Conn()->Skipping() ) - StatEvent(); - - RemoveEvent(); - } - - tcp::TCP_ApplicationAnalyzer::Done(); - } - -void InterConn_Analyzer::StatTimer(double t, int is_expire) - { - if ( IsFinished() || Conn()->Skipping() ) - return; - - StatEvent(); - - if ( ! is_expire ) - { - timeout *= backoff; - timer_mgr->Add(new InterConnTimer(t + timeout, this)); - } - } - -void InterConn_Analyzer::StatEvent() - { - if ( interconn_stats ) - Conn()->ConnectionEventFast(interconn_stats, this, { - Conn()->BuildConnVal(), - orig_endp->BuildStats(), - resp_endp->BuildStats(), - }); - } - -void InterConn_Analyzer::RemoveEvent() - { - if ( interconn_remove_conn ) - Conn()->ConnectionEventFast(interconn_remove_conn, this, {Conn()->BuildConnVal()}); - } - -InterConnTimer::InterConnTimer(double t, InterConn_Analyzer* a) -: Timer(t, TIMER_INTERCONN) - { - analyzer = a; - // Make sure connection does not expire. - Ref(a->Conn()); - } - -InterConnTimer::~InterConnTimer() - { - Unref(analyzer->Conn()); - } - -void InterConnTimer::Dispatch(double t, int is_expire) - { - analyzer->StatTimer(t, is_expire); - } diff --git a/src/analyzer/protocol/interconn/InterConn.h b/src/analyzer/protocol/interconn/InterConn.h deleted file mode 100644 index 04d5a2b5c6..0000000000 --- a/src/analyzer/protocol/interconn/InterConn.h +++ /dev/null @@ -1,88 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#ifndef ANALYZER_PROTOCOL_INTERCONN_INTERCONN_H -#define ANALYZER_PROTOCOL_INTERCONN_INTERCONN_H - -#include "analyzer/protocol/tcp/TCP.h" -#include "Timer.h" -#include "NetVar.h" - -namespace analyzer { namespace interconn { - -class InterConnEndpoint : public BroObj { -public: - explicit InterConnEndpoint(tcp::TCP_Endpoint* e); - - int DataSent(double t, uint64 seq, int len, int caplen, const u_char* data, - const IP_Hdr* ip, const struct tcphdr* tp); - - RecordVal* BuildStats(); - -protected: - int EstimateGapPacketNum(int gap) const; - int IsPotentialKeystrokePacket(int len) const; - int IsNormalKeystrokeInterarrival(double t) const; - - tcp::TCP_Endpoint* endp; - double last_keystroke_time; - uint64 max_top_seq; - uint32 num_pkts; - uint32 num_keystrokes_two_in_a_row; - uint32 num_normal_interarrivals; - uint32 num_8k4_pkts; - uint32 num_8k0_pkts; - uint32 num_bytes; - uint32 num_7bit_ascii; - uint32 num_lines; - uint32 num_normal_lines; - int is_partial; - int keystroke_just_seen; -}; - - -class InterConn_Analyzer : public tcp::TCP_ApplicationAnalyzer { -public: - explicit InterConn_Analyzer(Connection* c); - ~InterConn_Analyzer() override; - - void Init() override; - void Done() override; - void StatTimer(double t, int is_expire); - - static analyzer::Analyzer* Instantiate(Connection* conn) - { return new InterConn_Analyzer(conn); } - -protected: - // We support both packet and stream input and can be put in place even - // if the TCP analyzer is not yet reassembling. - void DeliverPacket(int len, const u_char* data, bool is_orig, - uint64 seq, const IP_Hdr* ip, int caplen) override; - void DeliverStream(int len, const u_char* data, bool is_orig) override; - - void StatEvent(); - void RemoveEvent(); - - InterConnEndpoint* orig_endp; - InterConnEndpoint* resp_endp; - - int orig_stream_pos; - int resp_stream_pos; - - double timeout; - double backoff; -}; - -class InterConnTimer : public Timer { -public: - InterConnTimer(double t, InterConn_Analyzer* a); - ~InterConnTimer() override; - - void Dispatch(double t, int is_expire) override; - -protected: - InterConn_Analyzer* analyzer; -}; - -} } // namespace analyzer::* - -#endif diff --git a/src/analyzer/protocol/interconn/Plugin.cc b/src/analyzer/protocol/interconn/Plugin.cc deleted file mode 100644 index bbd1b866ed..0000000000 --- a/src/analyzer/protocol/interconn/Plugin.cc +++ /dev/null @@ -1,25 +0,0 @@ -// See the file in the main distribution directory for copyright. - - -#include "plugin/Plugin.h" - -#include "InterConn.h" - -namespace plugin { -namespace Zeek_InterConn { - -class Plugin : public plugin::Plugin { -public: - plugin::Configuration Configure() - { - AddComponent(new ::analyzer::Component("InterConn", ::analyzer::interconn::InterConn_Analyzer::Instantiate)); - - plugin::Configuration config; - config.name = "Zeek::InterConn"; - config.description = "InterConn analyzer deprecated"; - return config; - } -} plugin; - -} -} diff --git a/src/analyzer/protocol/interconn/events.bif b/src/analyzer/protocol/interconn/events.bif deleted file mode 100644 index 6e859da668..0000000000 --- a/src/analyzer/protocol/interconn/events.bif +++ /dev/null @@ -1,8 +0,0 @@ -# ##### Deprecated events. Proposed for removal. - -## Deprecated. Will be removed. -event interconn_stats%(c: connection, os: interconn_endp_stats, rs: interconn_endp_stats%); - -## Deprecated. Will be removed. -event interconn_remove_conn%(c: connection%); - diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index e4ef3d570c..0a6ae8fa65 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -178,6 +178,15 @@ void Manager::InitPostScript() BrokerConfig config{std::move(options)}; + auto scheduler_policy = get_option("Broker::scheduler_policy")->AsString()->CheckString(); + + if ( streq(scheduler_policy, "sharing") ) + config.set("scheduler.policy", caf::atom("sharing")); + else if ( streq(scheduler_policy, "stealing") ) + config.set("scheduler.policy", caf::atom("stealing")); + else + reporter->FatalError("Invalid Broker::scheduler_policy: %s", scheduler_policy); + auto max_threads_env = zeekenv("ZEEK_BROKER_MAX_THREADS"); if ( max_threads_env ) @@ -941,6 +950,12 @@ void Manager::Process() if ( had_input ) { + if ( network_time == 0 ) + // If we're getting Broker messages, but still haven't initialized + // network_time, may as well do so now because otherwise the + // broker/cluster logs will end up using timestamp 0. + net_update_time(current_time()); + ++times_processed_without_idle; // The max number of Process calls allowed to happen in a row without diff --git a/src/event.bif b/src/event.bif index 92f3532ef0..5222545ae5 100644 --- a/src/event.bif +++ b/src/event.bif @@ -49,7 +49,7 @@ event zeek_init%(%); ## Deprecated synonym for :zeek:see:`zeek_init`. -event bro_init%(%) &deprecated; +event bro_init%(%) &deprecated="Remove in v3.1: use zeek_init"; ## Generated at Zeek termination time. The event engine generates this event when ## Zeek is about to terminate, either due to having exhausted reading its input @@ -65,7 +65,7 @@ event bro_init%(%) &deprecated; event zeek_done%(%); ## Deprecated synonym for :zeek:see:`zeek_done`. -event bro_done%(%) &deprecated; +event bro_done%(%) &deprecated="Remove in v3.1: use zeek_done"; ## Generated for every new connection. This event is raised with the first ## packet of a previously unknown connection. Zeek uses a flow-based definition @@ -530,59 +530,6 @@ event load_sample%(samples: load_sample_info, CPU: interval, dmem: int%); ## triggering the match will be passed on to the event. event signature_match%(state: signature_state, msg: string, data: string%); -## Generated when a protocol analyzer finds an identification of a software -## used on a system. This is a protocol-independent event that is fed by -## different analyzers. For example, the HTTP analyzer reports user-agent and -## server software by raising this event, assuming it can parse it (if not, -## :zeek:id:`software_parse_error` will be generated instead). -## -## c: The connection. -## -## host: The host running the reported software. -## -## s: A description of the software found. -## -## descr: The raw (unparsed) software identification string as extracted from -## the protocol. -## -## .. zeek:see:: software_parse_error software_unparsed_version_found -event software_version_found%(c: connection, host: addr, - s: software, descr: string%); - -## Generated when a protocol analyzer finds an identification of a software -## used on a system but cannot parse it. This is a protocol-independent event -## that is fed by different analyzers. For example, the HTTP analyzer reports -## user-agent and server software by raising this event if it cannot parse them -## directly (if it can :zeek:id:`software_version_found` will be generated -## instead). -## -## c: The connection. -## -## host: The host running the reported software. -## -## descr: The raw (unparsed) software identification string as extracted from -## the protocol. -## -## .. zeek:see:: software_version_found software_unparsed_version_found -event software_parse_error%(c: connection, host: addr, descr: string%); - -## Generated when a protocol analyzer finds an identification of a software -## used on a system. This is a protocol-independent event that is fed by -## different analyzers. For example, the HTTP analyzer reports user-agent and -## server software by raising this event. Different from -## :zeek:id:`software_version_found` and :zeek:id:`software_parse_error`, this -## event is always raised, independent of whether Zeek can parse the version -## string. -## -## c: The connection. -## -## host: The host running the reported software. -## -## str: The software identification string as extracted from the protocol. -## -## .. zeek:see:: software_parse_error software_version_found -event software_unparsed_version_found%(c: connection, host: addr, str: string%); - ## Generated each time Zeek's internal profiling log is updated. The file is ## defined by :zeek:id:`profiling_file`, and its update frequency by ## :zeek:id:`profiling_interval` and :zeek:id:`expensive_profiling_multiple`. @@ -661,7 +608,7 @@ event reporter_error%(t: time, msg: string, location: string%) &error_handler; event zeek_script_loaded%(path: string, level: count%); ## Deprecated synonym for :zeek:see:`zeek_script_loaded`. -event bro_script_loaded%(path: string, level: count%) &deprecated; +event bro_script_loaded%(path: string, level: count%) &deprecated="Remove in v3.1: use zeek_script_loaded"; ## Generated each time Zeek's script interpreter opens a file. This event is ## triggered only for files opened via :zeek:id:`open`, and in particular not for @@ -852,20 +799,5 @@ event dns_mapping_altered%(dm: dns_mapping, old_addrs: addr_set, new_addrs: addr ## params: The event's parameters. event new_event%(name: string, params: call_argument_vector%); -## Deprecated. Will be removed. -event root_backdoor_signature_found%(c: connection%); - -## Deprecated. Will be removed. -event napster_signature_found%(c: connection%); - -## Deprecated. Will be removed. -event kazaa_signature_found%(c: connection%); - -## Deprecated. Will be removed. -event gaobot_signature_found%(c: connection%); - -## Deprecated. Will be removed. +## Shows an IP address anonymization mapping. event anonymization_mapping%(orig: addr, mapped: addr%); - -## Deprecated. Will be removed. -event print_hook%(f:file, s: string%); diff --git a/src/main.cc b/src/main.cc index 456920fb1b..456ab3f813 100644 --- a/src/main.cc +++ b/src/main.cc @@ -993,18 +993,6 @@ int main(int argc, char** argv) delete dead_handlers; - EventRegistry::string_list* alive_handlers = - event_registry->UsedHandlers(); - - if ( alive_handlers->length() > 0 && dump_used_event_handlers ) - { - reporter->Info("invoked event handlers:"); - for ( int i = 0; i < alive_handlers->length(); ++i ) - reporter->Info("%s", (*alive_handlers)[i]); - } - - delete alive_handlers; - if ( stmts ) { stmt_flow_type flow; diff --git a/src/strings.bif b/src/strings.bif index 6c74db77e9..f2661f8cc9 100644 --- a/src/strings.bif +++ b/src/strings.bif @@ -1034,7 +1034,7 @@ function safe_shell_quote%(source: string%): string ## Returns: A shell-escaped version of *source*. ## ## .. zeek:see:: system safe_shell_quote -function str_shell_escape%(source: string%): string &deprecated +function str_shell_escape%(source: string%): string &deprecated="Remove in v3.1: use safe_shell_quote" %{ unsigned j = 0; const u_char* src = source->Bytes(); diff --git a/src/zeek.bif b/src/zeek.bif index 3e2eac3740..da11a18681 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -1810,7 +1810,7 @@ extern const char* zeek_version(); ## :zeek:see:`zeek_version` instead. ## ## Returns: Zeek's version, e.g., 2.0-beta-47-debug. -function bro_version%(%): string &deprecated +function bro_version%(%): string &deprecated="Remove in v3.1: use zeek_version" %{ return new StringVal(zeek_version()); %} @@ -2137,7 +2137,7 @@ function dump_rule_stats%(f: file%): bool ## Returns: True if Zeek is in the process of shutting down. ## ## .. zeek:see:: terminate -function bro_is_terminating%(%): bool &deprecated +function bro_is_terminating%(%): bool &deprecated="Remove in v3.1: use zeek_is_terminating" %{ return val_mgr->GetBool(terminating); %} @@ -4621,7 +4621,7 @@ function get_file_name%(f: file%): string ## after the rotation, and the time when *f* was opened/closed. ## ## .. zeek:see:: rotate_file_by_name calc_next_rotate -function rotate_file%(f: file%): rotate_info &deprecated +function rotate_file%(f: file%): rotate_info %{ RecordVal* info = f->Rotate(); if ( info ) @@ -4645,7 +4645,7 @@ function rotate_file%(f: file%): rotate_info &deprecated ## after the rotation, and the time when *f* was opened/closed. ## ## .. zeek:see:: rotate_file calc_next_rotate -function rotate_file_by_name%(f: string%): rotate_info &deprecated +function rotate_file_by_name%(f: string%): rotate_info %{ RecordVal* info = new RecordVal(rotate_info); @@ -4699,7 +4699,7 @@ function rotate_file_by_name%(f: string%): rotate_info &deprecated ## Returns: The duration until the next file rotation time. ## ## .. zeek:see:: rotate_file rotate_file_by_name -function calc_next_rotate%(i: interval%) : interval &deprecated +function calc_next_rotate%(i: interval%) : interval %{ const char* base_time = log_rotate_base_time ? log_rotate_base_time->AsString()->CheckString() : 0; @@ -4723,28 +4723,10 @@ function file_size%(f: string%) : double return new Val(double(s.st_size), TYPE_DOUBLE); %} -## Disables sending :zeek:id:`print_hook` events to remote peers for a given -## file. In a -## distributed setup, communicating Zeek instances generate the event -## :zeek:id:`print_hook` for each print statement and send it to the remote -## side. When disabled for a particular file, these events will not be -## propagated to other peers. -## -## f: The file to disable :zeek:id:`print_hook` events for. -## -## .. zeek:see:: enable_raw_output -function disable_print_hook%(f: file%): any - %{ - f->DisablePrintHook(); - return 0; - %} - ## Prevents escaping of non-ASCII characters when writing to a file. ## This function is equivalent to :zeek:attr:`&raw_output`. ## ## f: The file to disable raw output for. -## -## .. zeek:see:: disable_print_hook function enable_raw_output%(f: file%): any %{ f->EnableRawOutput(); @@ -5017,7 +4999,8 @@ function match_signatures%(c: connection, pattern_type: int, s: string, # =========================================================================== # -# Deprecated Functions +# Anonymization Functions +# (Not Fully Functional) # # =========================================================================== diff --git a/testing/btest/Baseline/core.reporter/logger-test.log b/testing/btest/Baseline/core.reporter/logger-test.log index 1dc58b65cd..0297fbe208 100644 --- a/testing/btest/Baseline/core.reporter/logger-test.log +++ b/testing/btest/Baseline/core.reporter/logger-test.log @@ -1,6 +1,6 @@ -reporter_info|init test-info|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.zeek, line 8|0.000000 -reporter_warning|init test-warning|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.zeek, line 9|0.000000 -reporter_error|init test-error|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.zeek, line 10|0.000000 -reporter_info|done test-info|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.zeek, line 15|0.000000 -reporter_warning|done test-warning|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.zeek, line 16|0.000000 -reporter_error|done test-error|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.zeek, line 17|0.000000 +reporter_info|init test-info|/home/jon/pro/zeek/zeek/testing/btest/.tmp/core.reporter/reporter.zeek, line 8|1561700937.882616 +reporter_warning|init test-warning|/home/jon/pro/zeek/zeek/testing/btest/.tmp/core.reporter/reporter.zeek, line 9|1561700937.882616 +reporter_error|init test-error|/home/jon/pro/zeek/zeek/testing/btest/.tmp/core.reporter/reporter.zeek, line 10|1561700937.882616 +reporter_info|done test-info|/home/jon/pro/zeek/zeek/testing/btest/.tmp/core.reporter/reporter.zeek, line 15|1561700937.882616 +reporter_warning|done test-warning|/home/jon/pro/zeek/zeek/testing/btest/.tmp/core.reporter/reporter.zeek, line 16|1561700937.882616 +reporter_error|done test-error|/home/jon/pro/zeek/zeek/testing/btest/.tmp/core.reporter/reporter.zeek, line 17|1561700937.882616 diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 8fa1ab560a..0169c3b838 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -62,7 +62,6 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/top-k.bif.zeek build/scripts/base/bif/plugins/__load__.zeek build/scripts/base/bif/plugins/Zeek_ARP.events.bif.zeek - build/scripts/base/bif/plugins/Zeek_BackDoor.events.bif.zeek build/scripts/base/bif/plugins/Zeek_BitTorrent.events.bif.zeek build/scripts/base/bif/plugins/Zeek_ConnSize.events.bif.zeek build/scripts/base/bif/plugins/Zeek_ConnSize.functions.bif.zeek @@ -85,7 +84,6 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/plugins/Zeek_ICMP.events.bif.zeek build/scripts/base/bif/plugins/Zeek_Ident.events.bif.zeek build/scripts/base/bif/plugins/Zeek_IMAP.events.bif.zeek - build/scripts/base/bif/plugins/Zeek_InterConn.events.bif.zeek build/scripts/base/bif/plugins/Zeek_IRC.events.bif.zeek build/scripts/base/bif/plugins/Zeek_KRB.events.bif.zeek build/scripts/base/bif/plugins/Zeek_Login.events.bif.zeek diff --git a/testing/btest/Baseline/coverage.bare-mode-errors/errors b/testing/btest/Baseline/coverage.bare-mode-errors/errors index a13c8849a1..e69de29bb2 100644 --- a/testing/btest/Baseline/coverage.bare-mode-errors/errors +++ b/testing/btest/Baseline/coverage.bare-mode-errors/errors @@ -1,4 +0,0 @@ -warning in /Users/johanna/bro/master/scripts/policy/misc/trim-trace-file.zeek, line 25: deprecated (rotate_file_by_name) -warning in /Users/johanna/bro/master/scripts/policy/misc/trim-trace-file.zeek, line 25: deprecated (rotate_file_by_name) -warning in /Users/johanna/bro/master/scripts/policy/misc/trim-trace-file.zeek, line 25: deprecated (rotate_file_by_name) -warning in /Users/johanna/bro/master/testing/btest/../../scripts//policy/misc/trim-trace-file.zeek, line 25: deprecated (rotate_file_by_name) diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 0b68a0ce83..6fbc95e4ec 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -62,7 +62,6 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/top-k.bif.zeek build/scripts/base/bif/plugins/__load__.zeek build/scripts/base/bif/plugins/Zeek_ARP.events.bif.zeek - build/scripts/base/bif/plugins/Zeek_BackDoor.events.bif.zeek build/scripts/base/bif/plugins/Zeek_BitTorrent.events.bif.zeek build/scripts/base/bif/plugins/Zeek_ConnSize.events.bif.zeek build/scripts/base/bif/plugins/Zeek_ConnSize.functions.bif.zeek @@ -85,7 +84,6 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/plugins/Zeek_ICMP.events.bif.zeek build/scripts/base/bif/plugins/Zeek_Ident.events.bif.zeek build/scripts/base/bif/plugins/Zeek_IMAP.events.bif.zeek - build/scripts/base/bif/plugins/Zeek_InterConn.events.bif.zeek build/scripts/base/bif/plugins/Zeek_IRC.events.bif.zeek build/scripts/base/bif/plugins/Zeek_KRB.events.bif.zeek build/scripts/base/bif/plugins/Zeek_Login.events.bif.zeek diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 2d069efde2..ef7ba59161 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -1,5 +1,3 @@ -0.000000 MetaHookPost CallFunction(Analyzer::__disable_analyzer, , (Analyzer::ANALYZER_BACKDOOR)) -> -0.000000 MetaHookPost CallFunction(Analyzer::__disable_analyzer, , (Analyzer::ANALYZER_INTERCONN)) -> 0.000000 MetaHookPost CallFunction(Analyzer::__disable_analyzer, , (Analyzer::ANALYZER_STEPPINGSTONE)) -> 0.000000 MetaHookPost CallFunction(Analyzer::__disable_analyzer, , (Analyzer::ANALYZER_TCPSTATS)) -> 0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_AYIYA, 5072/udp)) -> @@ -65,8 +63,6 @@ 0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_VXLAN, 4789/udp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_XMPP, 5222/tcp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_XMPP, 5269/tcp)) -> -0.000000 MetaHookPost CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_BACKDOOR)) -> -0.000000 MetaHookPost CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_INTERCONN)) -> 0.000000 MetaHookPost CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_STEPPINGSTONE)) -> 0.000000 MetaHookPost CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_TCPSTATS)) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_AYIYA, 5072/udp)) -> @@ -278,7 +274,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1561670974.090821, node=zeek, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1561684839.152939, node=zeek, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Broker::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Config::LOG)) -> @@ -459,7 +455,7 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, , (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1561670974.090821, node=zeek, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1561684839.152939, node=zeek, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(NetControl::check_plugins, , ()) -> 0.000000 MetaHookPost CallFunction(NetControl::init, , ()) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, , ()) -> @@ -574,7 +570,6 @@ 0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_ARP.events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_AsciiReader.ascii.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_AsciiWriter.ascii.bif.zeek) -> -1 -0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_BackDoor.events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_BenchmarkReader.benchmark.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_BinaryReader.binary.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_BitTorrent.events.bif.zeek) -> -1 @@ -605,7 +600,6 @@ 0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_IMAP.events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_IRC.events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_Ident.events.bif.zeek) -> -1 -0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_InterConn.events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_KRB.events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_KRB.types.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/Zeek_Login.events.bif.zeek) -> -1 @@ -895,8 +889,6 @@ 0.000000 MetaHookPost QueueEvent(NetControl::init()) -> false 0.000000 MetaHookPost QueueEvent(filter_change_tracking()) -> false 0.000000 MetaHookPost QueueEvent(zeek_init()) -> false -0.000000 MetaHookPre CallFunction(Analyzer::__disable_analyzer, , (Analyzer::ANALYZER_BACKDOOR)) -0.000000 MetaHookPre CallFunction(Analyzer::__disable_analyzer, , (Analyzer::ANALYZER_INTERCONN)) 0.000000 MetaHookPre CallFunction(Analyzer::__disable_analyzer, , (Analyzer::ANALYZER_STEPPINGSTONE)) 0.000000 MetaHookPre CallFunction(Analyzer::__disable_analyzer, , (Analyzer::ANALYZER_TCPSTATS)) 0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_AYIYA, 5072/udp)) @@ -962,8 +954,6 @@ 0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_VXLAN, 4789/udp)) 0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_XMPP, 5222/tcp)) 0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_XMPP, 5269/tcp)) -0.000000 MetaHookPre CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_BACKDOOR)) -0.000000 MetaHookPre CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_INTERCONN)) 0.000000 MetaHookPre CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_STEPPINGSTONE)) 0.000000 MetaHookPre CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_TCPSTATS)) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_AYIYA, 5072/udp)) @@ -1175,7 +1165,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1561670974.090821, node=zeek, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1561684839.152939, node=zeek, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Broker::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Config::LOG)) @@ -1356,7 +1346,7 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, , (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1561670974.090821, node=zeek, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1561684839.152939, node=zeek, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(NetControl::check_plugins, , ()) 0.000000 MetaHookPre CallFunction(NetControl::init, , ()) 0.000000 MetaHookPre CallFunction(Notice::want_pp, , ()) @@ -1471,7 +1461,6 @@ 0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_ARP.events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_AsciiReader.ascii.bif.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_AsciiWriter.ascii.bif.zeek) -0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_BackDoor.events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_BenchmarkReader.benchmark.bif.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_BinaryReader.binary.bif.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_BitTorrent.events.bif.zeek) @@ -1502,7 +1491,6 @@ 0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_IMAP.events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_IRC.events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_Ident.events.bif.zeek) -0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_InterConn.events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_KRB.events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_KRB.types.bif.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/Zeek_Login.events.bif.zeek) @@ -1792,8 +1780,6 @@ 0.000000 MetaHookPre QueueEvent(NetControl::init()) 0.000000 MetaHookPre QueueEvent(filter_change_tracking()) 0.000000 MetaHookPre QueueEvent(zeek_init()) -0.000000 | HookCallFunction Analyzer::__disable_analyzer(Analyzer::ANALYZER_BACKDOOR) -0.000000 | HookCallFunction Analyzer::__disable_analyzer(Analyzer::ANALYZER_INTERCONN) 0.000000 | HookCallFunction Analyzer::__disable_analyzer(Analyzer::ANALYZER_STEPPINGSTONE) 0.000000 | HookCallFunction Analyzer::__disable_analyzer(Analyzer::ANALYZER_TCPSTATS) 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_AYIYA, 5072/udp) @@ -1859,8 +1845,6 @@ 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_VXLAN, 4789/udp) 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_XMPP, 5222/tcp) 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_XMPP, 5269/tcp) -0.000000 | HookCallFunction Analyzer::disable_analyzer(Analyzer::ANALYZER_BACKDOOR) -0.000000 | HookCallFunction Analyzer::disable_analyzer(Analyzer::ANALYZER_INTERCONN) 0.000000 | HookCallFunction Analyzer::disable_analyzer(Analyzer::ANALYZER_STEPPINGSTONE) 0.000000 | HookCallFunction Analyzer::disable_analyzer(Analyzer::ANALYZER_TCPSTATS) 0.000000 | HookCallFunction Analyzer::register_for_port(Analyzer::ANALYZER_AYIYA, 5072/udp) @@ -2071,7 +2055,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1561670974.090821, node=zeek, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1561684839.152939, node=zeek, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Broker::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Config::LOG) @@ -2252,7 +2236,7 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1561670974.090821, node=zeek, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1561684839.152939, node=zeek, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction NetControl::check_plugins() 0.000000 | HookCallFunction NetControl::init() 0.000000 | HookCallFunction Notice::want_pp() @@ -2367,7 +2351,6 @@ 0.000000 | HookLoadFile .<...>/Zeek_ARP.events.bif.zeek 0.000000 | HookLoadFile .<...>/Zeek_AsciiReader.ascii.bif.zeek 0.000000 | HookLoadFile .<...>/Zeek_AsciiWriter.ascii.bif.zeek -0.000000 | HookLoadFile .<...>/Zeek_BackDoor.events.bif.zeek 0.000000 | HookLoadFile .<...>/Zeek_BenchmarkReader.benchmark.bif.zeek 0.000000 | HookLoadFile .<...>/Zeek_BinaryReader.binary.bif.zeek 0.000000 | HookLoadFile .<...>/Zeek_BitTorrent.events.bif.zeek @@ -2398,7 +2381,6 @@ 0.000000 | HookLoadFile .<...>/Zeek_IMAP.events.bif.zeek 0.000000 | HookLoadFile .<...>/Zeek_IRC.events.bif.zeek 0.000000 | HookLoadFile .<...>/Zeek_Ident.events.bif.zeek -0.000000 | HookLoadFile .<...>/Zeek_InterConn.events.bif.zeek 0.000000 | HookLoadFile .<...>/Zeek_KRB.events.bif.zeek 0.000000 | HookLoadFile .<...>/Zeek_KRB.types.bif.zeek 0.000000 | HookLoadFile .<...>/Zeek_Login.events.bif.zeek @@ -2684,7 +2666,7 @@ 0.000000 | HookLoadFile base<...>/xmpp 0.000000 | HookLoadFile base<...>/zeek.bif.zeek 0.000000 | HookLogInit packet_filter 1/1 {ts (time), node (string), filter (string), init (bool), success (bool)} -0.000000 | HookLogWrite packet_filter [ts=1561670974.090821, node=zeek, filter=ip or not ip, init=T, success=T] +0.000000 | HookLogWrite packet_filter [ts=1561684839.152939, node=zeek, filter=ip or not ip, init=T, success=T] 0.000000 | HookQueueEvent NetControl::init() 0.000000 | HookQueueEvent filter_change_tracking() 0.000000 | HookQueueEvent zeek_init() diff --git a/testing/btest/bifs/enable_raw_output.test b/testing/btest/bifs/enable_raw_output.test index c46b6e317f..b685b44069 100644 --- a/testing/btest/bifs/enable_raw_output.test +++ b/testing/btest/bifs/enable_raw_output.test @@ -4,7 +4,6 @@ # @TEST-EXEC: zeek -b %INPUT # @TEST-EXEC: tr '\000' 'X' output # @TEST-EXEC: btest-diff output -# @TEST-EXEC: cmp myfile hookfile event zeek_init() { @@ -14,10 +13,3 @@ event zeek_init() print myfile, "hello\x00world", "hi"; close(myfile); } - -event print_hook(f: file, s: string) - { - local hookfile = open("hookfile"); - write_file(hookfile, s); - close(hookfile); - } diff --git a/testing/btest/core/reporter.zeek b/testing/btest/core/reporter.zeek index 8591096c2b..6ab4f18b71 100644 --- a/testing/btest/core/reporter.zeek +++ b/testing/btest/core/reporter.zeek @@ -1,7 +1,7 @@ # -# @TEST-EXEC: zeek %INPUT >output 2>&1 +# @TEST-EXEC: zeek -b %INPUT >output 2>&1 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff output -# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff logger-test.log +# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps" btest-diff logger-test.log event zeek_init() { @@ -32,7 +32,7 @@ event connection_established(c: connection) first = 0; } -global f = open_log_file("logger-test"); +global f = open("logger-test.log"); event reporter_info(t: time, msg: string, location: string) { diff --git a/testing/btest/language/raw_output_attr.test b/testing/btest/language/raw_output_attr.test index ccf616405e..7f57481807 100644 --- a/testing/btest/language/raw_output_attr.test +++ b/testing/btest/language/raw_output_attr.test @@ -4,7 +4,6 @@ # @TEST-EXEC: zeek -b %INPUT # @TEST-EXEC: tr '\000' 'X' output # @TEST-EXEC: btest-diff output -# @TEST-EXEC: cmp myfile hookfile # first check local variable of file type w/ &raw_output @@ -16,10 +15,3 @@ event zeek_init() print myfile, "hello\x00world", "hi"; close(myfile); } - -event print_hook(f: file, s: string) - { - local hookfile = open("hookfile"); - write_file(hookfile, s); - close(hookfile); - } diff --git a/testing/btest/language/sizeof.zeek b/testing/btest/language/sizeof.zeek index fc510afb70..48c69ddc27 100644 --- a/testing/btest/language/sizeof.zeek +++ b/testing/btest/language/sizeof.zeek @@ -24,7 +24,7 @@ global a6: addr = [::1]; global b: bool = T; global c: count = 10; global d: double = -1.23; -global f: file = open_log_file("sizeof_demo"); +global f: file = open("sizeof_demo.log"); global i: int = -10; global iv: interval = -5sec; global p: port = 80/tcp; diff --git a/testing/btest/scripts/base/frameworks/logging/file.zeek b/testing/btest/scripts/base/frameworks/logging/file.zeek index 6aa07f1699..b584f56fca 100644 --- a/testing/btest/scripts/base/frameworks/logging/file.zeek +++ b/testing/btest/scripts/base/frameworks/logging/file.zeek @@ -13,7 +13,7 @@ export { } &log; } -const foo_log = open_log_file("Foo") &redef; +const foo_log = open("Foo.log") &redef; event zeek_init() {