diff --git a/.cirrus.yml b/.cirrus.yml index 7a50417e0e..2604595b92 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -161,6 +161,7 @@ sanitizer_task: << : *CI_TEMPLATE test_fuzzers_script: ./ci/test-fuzzers.sh env: + CXXFLAGS: -DZEEK_DICT_DEBUG ZEEK_CI_CONFIGURE_FLAGS: *SANITIZER_CONFIG ZEEK_TAILORED_UB_CHECKS: 1 UBSAN_OPTIONS: print_stacktrace=1 diff --git a/CHANGES b/CHANGES index 8000736a08..b2bd5da1e9 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,78 @@ +3.3.0-dev.426 | 2020-10-14 10:47:37 -0700 + + * GH-1211: Improve error message for already-defined functions (Jon Siwek, Corelight) + +3.3.0-dev.422 | 2020-10-13 16:26:24 -0700 + + * GH-1208: Use Dictionary validity assertions only during CI (Jon Siwek, Corelight) + +3.3.0-dev.420 | 2020-10-13 15:34:02 -0700 + + * Change "Cluster::Node$p" field to use "&default=0/unknown" (Seth Hall, Corelight) + + This effectively makes defining the listening port optional in the cluster + node layout config, with "0/unknown" meaning the node is not pre-configured + to listen for incoming connections from other cluster nodes. + +3.3.0-dev.414 | 2020-10-13 13:49:05 -0700 + + * Update cmake submodule to pull in fix for building libkqueue (Tim Wojtulewicz, Corelight) + + * GH-1063: Update libkqueue to fix pf_ring-zc failures (Tim Wojtulewicz, Corelight) + +3.3.0-dev.411 | 2020-10-13 13:38:38 -0700 + + * Remove unused LoginConn type and variable in Conn.h (Tim Wojtulewicz, Corelight) + +3.3.0-dev.409 | 2020-10-13 13:32:18 -0700 + + * Make it possible to pass command line options through to scripts. (Seth Hall, Corelight) + + A new ``zeek_script_args`` variable contains a list of arguments passed + to a script. E.g. either when explicitly executing Zeek like + ``zeek -- myscript.zeek -arg1 -arg2``, or when using Zeek to interpret + executable scripts that contain a hashbang line at the top like:: + + #!/usr/local/zeek/bin/zeek -- + +3.3.0-dev.403 | 2020-10-13 10:50:12 -0700 + + * Add new Pcap::findalldevs() BIF (Seth Hall, Corelight) + + * Remove superfluous RuleCondition destructors (Jon Siwek, Corelight) + + * Silence Clang's warning about ignoring GCC's maybe-uninitialized warning (Jon Siwek, Corelight) + + * Add reference to network_time_init from zeek_init docs (Jon Siwek, Corelight) + +3.3.0-dev.390 | 2020-10-12 17:43:15 -0700 + + * Improve documentation for zeek_init event scheduling pitfalls (Jon Siwek, Corelight) + +3.3.0-dev.388 | 2020-10-12 17:02:20 -0700 + + * Add CaptureLoss::Too_Little_Traffic notice (Vlad Grigorescu) + + * Add CaptureLoss::initial_watch_interval for a quick read on cluster health after startup. (Vlad Grigorescu) + + * Improve capture-loss.zeek documentation. (Vlad Grigorescu) + + * Fix whitespace in capture-loss.zek (Vlad Grigorescu) + +3.3.0-dev.381 | 2020-10-12 11:15:29 -0700 + + * GH-779: Add "udp-state" signature condition (Jon Siwek, Corelight) + + It accepts "originator" or "responder" states as a way to enforce that + the signature only matches packets in the associated direction. + The "established" state is rejected as an error since it doesn't + have a useful meaning like it does for the "tcp-state" condition. + + * Rename RuleConditionTCPState::TCPState enum values (Jon Siwek, Corelight) + + * Rename "tcp-state" signature parser tokens to not be TCP-specific (Jon Siwek, Corelight) + 3.3.0-dev.377 | 2020-10-08 12:07:42 -0700 * GH-1200: ignore a maybe-uninitialized warning (Jon Siwek, Corelight) diff --git a/NEWS b/NEWS index 8e183219db..66620f4040 100644 --- a/NEWS +++ b/NEWS @@ -34,6 +34,33 @@ New Functionality See https://docs.zeek.org/en/master/frameworks/logging.html#filter-log-records for more details. +- Added a ``udp-state`` signature condition to enforce matching against + either "originator" or "responder" flow direction of UDP packets. + +- Improvements to capture-loss.zeek: + + - A new option, ``CaptureLoss::initial_watch_interval``. When restarting a + Zeek cluster, one usually wants some immediate feedback as to the health of + the monitoring via capture loss. However, you previously needed to wait a + full ``CaptureLoss::watch_interval``, which defaults to 15 minutes. The + new option specifies the interval for the first-time report. So the new + default behavior provides stats after 1 minute and then after + 15 minutes afterward. + + - A new notice type, ``CaptureLoss::Too_Little_Traffic``. + If a Zeek process sees less than ``CaptureLoss::minimum_acks`` ACKs in a + given interval, this notice gets raised. This can be a useful diagnostic + if, for whatever reason, a Zeek process stops seeing traffic, but + capture-loss.zeek would have previously only reported that "0 gaps and 0 + ACKs is 0% loss". + +- A new ``zeek_script_args`` variable contains a list of arguments passed + to a script. E.g. either when explicitly executing Zeek like + ``zeek -- myscript.zeek -arg1 -arg2``, or when using Zeek to interpret + executable scripts that contain a hashbang line at the top like:: + + #!/usr/local/zeek/bin/zeek -- + Changed Functionality --------------------- @@ -61,6 +88,11 @@ Changed Functionality than the previous one. A detailed explanation of the implementation is here: https://jasonlue.github.io/algo/2019/08/20/clustered-hashing.html +- The ``p`` fields of ``Cluster::Node`` records now use a + ``&default=0/unknown`` attribute with ``0/unknown`` meaning that the node is + not pre-configured to listen for incoming connections from other cluster + nodes. + Removed Functionality --------------------- diff --git a/VERSION b/VERSION index f9fe94395c..a9c83441c6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.3.0-dev.377 +3.3.0-dev.426 diff --git a/auxil/bifcl b/auxil/bifcl index 857261518a..1eaa6aff1d 160000 --- a/auxil/bifcl +++ b/auxil/bifcl @@ -1 +1 @@ -Subproject commit 857261518ad7716af951eaf6894f22ed4e79d00d +Subproject commit 1eaa6aff1d991307b134d85b64e1ab7b68c89c92 diff --git a/auxil/binpac b/auxil/binpac index 84c306afe2..bc719c1565 160000 --- a/auxil/binpac +++ b/auxil/binpac @@ -1 +1 @@ -Subproject commit 84c306afe235c3032ff1ef682fbaf86cf485eb1c +Subproject commit bc719c1565de9454b04a4b9aade14460268bcfbe diff --git a/auxil/broker b/auxil/broker index 830d2e5632..ffac36b4f2 160000 --- a/auxil/broker +++ b/auxil/broker @@ -1 +1 @@ -Subproject commit 830d2e5632c0d268ab085ad32f53b3848a466988 +Subproject commit ffac36b4f2b56a55ae6e298f58d07fe10c13a824 diff --git a/auxil/libkqueue b/auxil/libkqueue index 7636475614..6c1717dea2 160000 --- a/auxil/libkqueue +++ b/auxil/libkqueue @@ -1 +1 @@ -Subproject commit 7636475614b583180229a528c470a45f68285e46 +Subproject commit 6c1717dea2dc34a91d32e07d2cae34b1afa0a84e diff --git a/auxil/zeek-aux b/auxil/zeek-aux index 85b087fc8e..fbb5a21719 160000 --- a/auxil/zeek-aux +++ b/auxil/zeek-aux @@ -1 +1 @@ -Subproject commit 85b087fc8eeeeb2c0e0d241e33e78e39fc099aab +Subproject commit fbb5a21719d4d00244bdd9f0d0a2f8543580a016 diff --git a/auxil/zeekctl b/auxil/zeekctl index 60f06da517..f99e3265c5 160000 --- a/auxil/zeekctl +++ b/auxil/zeekctl @@ -1 +1 @@ -Subproject commit 60f06da51721b043927543fdb11b04877d9eb0f5 +Subproject commit f99e3265c5e7d6c45361b7d8dc03e772f66b0d4b diff --git a/cmake b/cmake index 47935075eb..cf652b8459 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 47935075ebcc4546494d9e0005eff5e9b3a6735d +Subproject commit cf652b845908a15c02e11dca3162f3eecca0a9c5 diff --git a/doc b/doc index 2960bd2ee1..5666bf7d6d 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 2960bd2ee131e1c565da3e02983bbe4b719bc607 +Subproject commit 5666bf7d6d1abad60c83fcdaaf2e43eb92958f77 diff --git a/scripts/base/frameworks/cluster/main.zeek b/scripts/base/frameworks/cluster/main.zeek index 02c63562b6..110f4ad3af 100644 --- a/scripts/base/frameworks/cluster/main.zeek +++ b/scripts/base/frameworks/cluster/main.zeek @@ -162,7 +162,8 @@ export { ## can specify a particular :rfc:`4007` ``zone_id``. zone_id: string &default=""; ## The port that this node will listen on for peer connections. - p: port; + ## A value of ``0/unknown`` means the node is not pre-configured to listen. + p: port &default=0/unknown; ## Identifier for the interface a worker is sniffing. interface: string &optional; ## Name of the manager node this node uses. For workers and proxies. diff --git a/scripts/base/frameworks/cluster/setup-connections.zeek b/scripts/base/frameworks/cluster/setup-connections.zeek index 2abd57b142..55ab8a460a 100644 --- a/scripts/base/frameworks/cluster/setup-connections.zeek +++ b/scripts/base/frameworks/cluster/setup-connections.zeek @@ -89,11 +89,15 @@ event zeek_init() &priority=-10 Broker::subscribe(nodeid_topic(Broker::node_id())); Broker::subscribe(node_topic(node)); - Broker::listen(Broker::default_listen_address, - self$p, - Broker::default_listen_retry); + if ( self$p != 0/unknown ) + { + Broker::listen(Broker::default_listen_address, + self$p, + Broker::default_listen_retry); + + Cluster::log(fmt("listening on %s:%s", Broker::default_listen_address, self$p)); + } - Cluster::log(fmt("listening on %s:%s", Broker::default_listen_address, self$p)); switch ( self$node_type ) { case MANAGER: diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index b7a2b1b80c..635dec2cd2 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -454,6 +454,18 @@ type connection: record { inner_vlan: int &optional; }; +## Arguments given to Zeek from the command line. In order to use this, Zeek +## must use a ``--`` command line argument immediately followed by a script +## file and additional arguments after that. For example:: +## +## zeek --bare-mode -- myscript.zeek -a -b -c +## +## To use Zeek as an executable interpreter, include a line at the top of a script +## like the following and make the script executable:: +## +## #!/usr/local/zeek/bin/zeek -- +const zeek_script_args: vector of string = vector(); + ## Default amount of time a file can be inactive before the file analysis ## gives up and discards any internal state related to the file. option default_file_timeout_interval: interval = 2 mins; @@ -4976,6 +4988,26 @@ export { ## Number of Mbytes to provide as buffer space when capturing from live ## interfaces. const bufsize = 128 &redef; + + ## The definition of a "pcap interface". + type Interface: record { + ## The interface/device name. + name: string; + ## A human-readable description of the device. + description: string &optional; + ## The network addresses associated with the device. + addrs: set[addr]; + ## Whether the device is a loopback interface. E.g. addresses + ## of ``127.0.0.1`` or ``[::1]`` are used by loopback interfaces. + is_loopback: bool; + + ## Whether the device is up. Not set when that info is unavailable. + is_up: bool &optional; + ## Whether the device is running. Not set when that info is unavailable. + is_running: bool &optional; + }; + + type Interfaces: set[Pcap::Interface]; } # end export module DCE_RPC; diff --git a/scripts/policy/misc/capture-loss.zeek b/scripts/policy/misc/capture-loss.zeek index 0b827db544..3f53de3e2a 100644 --- a/scripts/policy/misc/capture-loss.zeek +++ b/scripts/policy/misc/capture-loss.zeek @@ -1,10 +1,10 @@ ##! This script logs evidence regarding the degree to which the packet -##! capture process suffers from measurement loss. -##! The loss could be due to overload on the host or NIC performing -##! the packet capture or it could even be beyond the host. If you are -##! capturing from a switch with a SPAN port, it's very possible that +##! capture process suffers from measurement loss. +##! The loss could be due to overload on the host or NIC performing +##! the packet capture or it could even be beyond the host. If you are +##! capturing from a switch with a SPAN port, it's very possible that ##! the switch itself could be overloaded and dropping packets. -##! Reported loss is computed in terms of the number of "gap events" (ACKs +##! Reported loss is computed in terms of the number of "gap events" (ACKs ##! for a sequence number that's above a gap). @load base/frameworks/notice @@ -13,15 +13,18 @@ module CaptureLoss; export { redef enum Log::ID += { LOG }; - + global log_policy: Log::PolicyHook; redef enum Notice::Type += { ## Report if the detected capture loss exceeds the percentage - ## threshold. - Too_Much_Loss + ## threshold defined in :zeek:id:`CaptureLoss::too_much_loss`. + Too_Much_Loss, + ## Report if the traffic seen by a peer within a given watch + ## interval is less than :zeek:id:`CaptureLoss::minimum_acks`. + Too_Little_Traffic, }; - + type Info: record { ## Timestamp for when the measurement occurred. ts: time &log; @@ -38,25 +41,35 @@ export { ## Percentage of ACKs seen where the data being ACKed wasn't seen. percent_lost: double &log; }; - - ## The interval at which capture loss reports are created. + + ## The interval at which capture loss reports are created in a + ## running cluster (that is, after the first report). option watch_interval = 15mins; - - ## The percentage of missed data that is considered "too much" + + ## For faster feedback on cluster health, the first capture loss + ## report is generated this many minutes after startup. + option initial_watch_interval = 1mins; + + ## The percentage of missed data that is considered "too much" ## when the :zeek:enum:`CaptureLoss::Too_Much_Loss` notice should be ## generated. The value is expressed as a double between 0 and 1 with 1 ## being 100%. option too_much_loss: double = 0.1; + + ## The minimum number of ACKs expected for a single peer in a + ## watch interval. If the number seen is less than this, + ## :zeek:enum:`CaptureLoss::Too_Little_Traffic` is raised. + option minimum_acks: count = 1; } event CaptureLoss::take_measurement(last_ts: time, last_acks: count, last_gaps: count) { if ( last_ts == 0 ) { - schedule watch_interval { CaptureLoss::take_measurement(network_time(), 0, 0) }; + schedule initial_watch_interval { CaptureLoss::take_measurement(network_time(), 0, 0) }; return; } - + local now = network_time(); local g = get_gap_stats(); local acks = g$ack_events - last_acks; @@ -65,13 +78,17 @@ event CaptureLoss::take_measurement(last_ts: time, last_acks: count, last_gaps: local info: Info = [$ts=now, $ts_delta=now-last_ts, $peer=peer_description, - $acks=acks, $gaps=gaps, + $acks=acks, $gaps=gaps, $percent_lost=pct_lost]; - + if ( pct_lost >= too_much_loss*100 ) - NOTICE([$note=Too_Much_Loss, + NOTICE([$note=Too_Much_Loss, $msg=fmt("The capture loss script detected an estimated loss rate above %.3f%%", pct_lost)]); - + + if ( acks < minimum_acks ) + NOTICE([$note=Too_Little_Traffic, + $msg=fmt("Only observed %d TCP ACKs and was expecting at least %d.", acks, minimum_acks)]); + Log::write(LOG, info); schedule watch_interval { CaptureLoss::take_measurement(now, g$ack_events, g$gap_events) }; } @@ -82,5 +99,5 @@ event zeek_init() &priority=5 # We only schedule the event if we are capturing packets. if ( reading_live_traffic() || reading_traces() ) - schedule watch_interval { CaptureLoss::take_measurement(network_time(), 0, 0) }; + schedule initial_watch_interval { CaptureLoss::take_measurement(network_time(), 0, 0) }; } diff --git a/src/Conn.cc b/src/Conn.cc index 048cab06ae..175cc8053a 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -94,8 +94,6 @@ Connection::Connection(NetSessions* s, const detail::ConnIDKey& k, double t, vlan = pkt->vlan; inner_vlan = pkt->inner_vlan; - login_conn = nullptr; - is_active = 1; skip = 0; weird = 0; @@ -610,7 +608,6 @@ unsigned int Connection::MemoryAllocation() const + (timers.MemoryAllocation() - padded_sizeof(timers)) + (conn_val ? conn_val->MemoryAllocation() : 0) + (root_analyzer ? root_analyzer->MemoryAllocation(): 0) - // login_conn is just a casted 'this'. // primary_PIA is already contained in the analyzer tree. ; } diff --git a/src/Conn.h b/src/Conn.h index 9be0afc8fe..5b33b5eeed 100644 --- a/src/Conn.h +++ b/src/Conn.h @@ -24,7 +24,6 @@ ZEEK_FORWARD_DECLARE_NAMESPACED(Connection, zeek); ZEEK_FORWARD_DECLARE_NAMESPACED(ConnectionTimer, zeek::detail); ZEEK_FORWARD_DECLARE_NAMESPACED(NetSessions, zeek); -class LoginConn; ZEEK_FORWARD_DECLARE_NAMESPACED(EncapsulationStack, zeek); ZEEK_FORWARD_DECLARE_NAMESPACED(Specific_RE_Matcher, zeek::detail); @@ -173,8 +172,6 @@ public: void AppendAddl(const char* str); - LoginConn* AsLoginConn() { return login_conn; } - void Match(detail::Rule::PatternType type, const u_char* data, int len, bool is_orig, bool bol, bool eol, bool clear_state); @@ -354,7 +351,6 @@ protected: double start_time, last_time; double inactivity_timeout; RecordValPtr conn_val; - LoginConn* login_conn; // either nil, or this const EncapsulationStack* encapsulation; // tunnels int suppress_event; // suppress certain events to once per conn. diff --git a/src/Dict.cc b/src/Dict.cc index f4144fb464..06aa643ee0 100644 --- a/src/Dict.cc +++ b/src/Dict.cc @@ -17,7 +17,7 @@ #include "3rdparty/doctest.h" -#ifdef DEBUG +#if defined(DEBUG) && defined(ZEEK_DICT_DEBUG) #define ASSERT_VALID(o) o->AssertValid() #else #define ASSERT_VALID(o) diff --git a/src/Options.cc b/src/Options.cc index a9e4f40db7..951c55831b 100644 --- a/src/Options.cc +++ b/src/Options.cc @@ -5,6 +5,7 @@ #include "Options.h" #include +#include #include @@ -186,8 +187,48 @@ Options parse_cmdline(int argc, char** argv) } else { - for ( auto i = 0; i < argc; ++i ) - zeek_args.emplace_back(argv[i]); + if ( argc > 1 ) + { + auto endsWith = [](const std::string& str, const std::string& suffix) + { + return str.size() >= suffix.size() && 0 == str.compare(str.size()-suffix.size(), suffix.size(), suffix); + }; + + auto i = 0; + for ( ; i < argc && ! endsWith(argv[i], "--"); ++i ) + { + zeek_args.emplace_back(argv[i]); + } + + if ( i < argc ) + { + // If a script is invoked with Zeek as the interpreter, the arguments provided + // directly in the interpreter line of the script won't be broken apart in the + // argv on Linux so we split it up here. + if ( endsWith(argv[i], "--") && zeek_args.size() == 1 ) + { + std::istringstream iss(argv[i]); + for ( std::string s; iss >> s; ) + { + if ( ! endsWith(s, "--") ) + { + zeek_args.emplace_back(s); + } + } + } + + // There is an additional increment here to skip over the "--" if it was found. + if ( endsWith(argv[i], "--") ) + ++i; + + // The first argument after the double hyphens in implicitly a script name. + rval.scripts_to_load.emplace_back(argv[i++]); + + // If there are more argument, grab them for script arguments + for ( ; i < argc; ++i ) + rval.script_args.emplace_back(argv[i]); + } + } } constexpr struct option long_opts[] = { diff --git a/src/Options.h b/src/Options.h index d70e37d65c..3f949e675f 100644 --- a/src/Options.h +++ b/src/Options.h @@ -74,6 +74,8 @@ struct Options { std::set plugins_to_load; std::vector scripts_to_load; std::vector script_options_to_set; + + std::vector script_args; }; /** diff --git a/src/RuleCondition.cc b/src/RuleCondition.cc index 977eb9f8e2..789c94cd88 100644 --- a/src/RuleCondition.cc +++ b/src/RuleCondition.cc @@ -32,16 +32,16 @@ bool RuleConditionTCPState::DoMatch(Rule* rule, RuleEndpointState* state, auto* ta = static_cast(root); - if ( tcpstates & STATE_STATELESS ) + if ( tcpstates & RULE_STATE_STATELESS ) return true; - if ( (tcpstates & STATE_ORIG) && ! state->IsOrig() ) + if ( (tcpstates & RULE_STATE_ORIG) && ! state->IsOrig() ) return false; - if ( (tcpstates & STATE_RESP) && state->IsOrig() ) + if ( (tcpstates & RULE_STATE_RESP) && state->IsOrig() ) return false; - if ( (tcpstates & STATE_ESTABLISHED ) && + if ( (tcpstates & RULE_STATE_ESTABLISHED ) && ! (is_established(ta->Orig()) && is_established(ta->Resp()))) return false; @@ -54,6 +54,31 @@ void RuleConditionTCPState::PrintDebug() fprintf(stderr, " RuleConditionTCPState: 0x%x\n", tcpstates); } +bool RuleConditionUDPState::DoMatch(Rule* rule, RuleEndpointState* state, + const u_char* data, int len) + { + analyzer::Analyzer* root = state->GetAnalyzer()->Conn()->GetRootAnalyzer(); + + if ( ! root || ! root->IsAnalyzer("UDP") ) + return false; + + if ( states & RULE_STATE_STATELESS ) + return true; + + if ( (states & RULE_STATE_ORIG) && ! state->IsOrig() ) + return false; + + if ( (states & RULE_STATE_RESP) && state->IsOrig() ) + return false; + + return true; + } + +void RuleConditionUDPState::PrintDebug() + { + fprintf(stderr, " RuleConditionUDPState: 0x%x\n", states); + } + void RuleConditionIPOptions::PrintDebug() { fprintf(stderr, " RuleConditionIPOptions: 0x%x\n", options); diff --git a/src/RuleCondition.h b/src/RuleCondition.h index aa735cf854..2d44816832 100644 --- a/src/RuleCondition.h +++ b/src/RuleCondition.h @@ -22,21 +22,19 @@ public: virtual void PrintDebug() = 0; }; +enum RuleStateKind { + RULE_STATE_ESTABLISHED = 1, + RULE_STATE_ORIG = 2, + RULE_STATE_RESP = 4, + RULE_STATE_STATELESS = 8 +}; + // Implements the "tcp-state" keyword. class RuleConditionTCPState : public RuleCondition { public: - enum TCPState { - STATE_ESTABLISHED = 1, - STATE_ORIG = 2, - STATE_RESP = 4, - STATE_STATELESS = 8 - }; - explicit RuleConditionTCPState(int arg_tcpstates) { tcpstates = arg_tcpstates; } - ~RuleConditionTCPState() override { } - bool DoMatch(Rule* rule, RuleEndpointState* state, const u_char* data, int len) override; @@ -46,6 +44,20 @@ private: int tcpstates; }; +// Implements the "udp-state" keyword. +class RuleConditionUDPState : public RuleCondition { +public: + explicit RuleConditionUDPState(int arg_states) + { states = arg_states; } + + bool DoMatch(Rule* rule, RuleEndpointState* state, const u_char* data, + int len) override; + + void PrintDebug() override; + +private: + int states; +}; // Implements "ip-options". class RuleConditionIPOptions : public RuleCondition { @@ -59,9 +71,6 @@ public: explicit RuleConditionIPOptions(int arg_options) { options = arg_options; } - ~RuleConditionIPOptions() override - { } - bool DoMatch(Rule* rule, RuleEndpointState* state, const u_char* data, int len) override; @@ -75,7 +84,6 @@ private: class RuleConditionSameIP : public RuleCondition { public: RuleConditionSameIP() { } - ~RuleConditionSameIP() override {} bool DoMatch(Rule* rule, RuleEndpointState* state, const u_char* data, int len) override; @@ -91,8 +99,6 @@ public: RuleConditionPayloadSize(uint32_t arg_val, Comp arg_comp) { val = arg_val; comp = arg_comp; } - ~RuleConditionPayloadSize() override {} - bool DoMatch(Rule* rule, RuleEndpointState* state, const u_char* data, int len) override; @@ -107,7 +113,6 @@ private: class RuleConditionEval : public RuleCondition { public: explicit RuleConditionEval(const char* func); - ~RuleConditionEval() override {} bool DoMatch(Rule* rule, RuleEndpointState* state, const u_char* data, int len) override; diff --git a/src/Val.cc b/src/Val.cc index 56f4c15009..fe479a6ad6 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -2192,10 +2192,14 @@ void TableVal::SendToStore(const Val* index, const TableEntryVal* new_entry_val, case ELEMENT_NEW: case ELEMENT_CHANGED: { + #ifndef __clang__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" + #endif broker::optional expiry; + #ifndef __clang__ #pragma GCC diagnostic pop + #endif auto expire_time = GetExpireTime(); if ( expire_time == 0 ) diff --git a/src/Var.cc b/src/Var.cc index fa1db6c3d1..a0adcb8f4f 100644 --- a/src/Var.cc +++ b/src/Var.cc @@ -583,7 +583,7 @@ void begin_func(IDPtr id, const char* module_name, case FUNC_FLAVOR_FUNCTION: if ( ! id->IsRedefinable() ) - id->Error("already defined"); + id->Error("already defined", t.get()); break; default: @@ -594,6 +594,10 @@ void begin_func(IDPtr id, const char* module_name, else id->SetType(t); + if ( IsErrorType(id->GetType()->Tag()) ) + reporter->FatalError("invalid definition of '%s' (see previous errors)", + id->Name()); + const auto& args = t->Params(); const auto& canon_args = id->GetType()->AsFuncType()->Params(); diff --git a/src/event.bif b/src/event.bif index 362cfa42ff..bf82df43f9 100644 --- a/src/event.bif +++ b/src/event.bif @@ -41,10 +41,11 @@ ## ## When a ``zeek_init`` handler executes, Zeek has not yet seen any input ## packets and therefore :zeek:id:`network_time` is not initialized yet. An -## artifact of that is that any timer installed in a ``zeek_init`` handler -## will fire immediately with the first packet. The standard way to work -## around that is to ignore the first time the timer fires and immediately -## reschedule. +## artifact of that is that any timer installed in a ``zeek_init`` handler, +## like with :zeek:keyword:`schedule`, will fire immediately with the first +## packet. The standard way to work around that is to ignore the first time +## the timer fires and immediately reschedule or to instead schedule the +## first event from with the :zeek:see:`network_time_init` event. ## event zeek_init%(%); diff --git a/src/iosource/pcap/pcap.bif b/src/iosource/pcap/pcap.bif index 5655a8fac9..c37416c968 100644 --- a/src/iosource/pcap/pcap.bif +++ b/src/iosource/pcap/pcap.bif @@ -1,10 +1,13 @@ module Pcap; + const snaplen: count; const bufsize: count; %%{ +#include "pcap.h" + #include "iosource/Manager.h" %%} @@ -102,3 +105,57 @@ function error%(%): string return zeek::make_intrusive("no error"); %} + +function findalldevs%(%): Pcap::Interfaces + %{ + pcap_if_t* alldevs; + char errbuf[PCAP_ERRBUF_SIZE]; + + static auto ifaces_type = id::find_type("Pcap::Interfaces"); + auto pcap_interfaces = make_intrusive(ifaces_type); + + int ret = pcap_findalldevs(&alldevs, errbuf); + if ( ret == PCAP_ERROR ) + { + emit_builtin_error(util::fmt("Error calling pcap_findalldevs: %s", errbuf)); + // Return an empty set in case of failure. + return pcap_interfaces; + } + + static auto iface_type = id::find_type("Pcap::Interface"); + for ( auto d = alldevs; d; d = d->next ) + { + auto r = make_intrusive(iface_type); + + r->Assign(0, make_intrusive(d->name)); + if ( d->description ) + r->Assign(1, make_intrusive(d->description)); + + auto addrs = make_intrusive(TYPE_ADDR); + for ( auto addr = d->addresses; addr != NULL; addr = addr->next ) + { + if ( addr->addr->sa_family == AF_INET ) + { + IPAddr a(reinterpret_cast(addr->addr)->sin_addr); + addrs->Append(make_intrusive(a)); + } + else if ( addr->addr->sa_family == AF_INET6 ) + { + IPAddr a(reinterpret_cast(addr->addr)->sin6_addr); + addrs->Append(make_intrusive(a)); + } + } + r->Assign(2, addrs->ToSetVal()); + r->Assign(3, val_mgr->Bool(d->flags & PCAP_IF_LOOPBACK)); +#ifdef PCAP_IF_UP + // These didn't become available until libpcap 1.6.1 + r->Assign(4, val_mgr->Bool(d->flags & PCAP_IF_UP)); + r->Assign(5, val_mgr->Bool(d->flags & PCAP_IF_RUNNING)); +#endif + + pcap_interfaces->Assign(std::move(r), nullptr); + } + + pcap_freealldevs(alldevs); + return pcap_interfaces; + %} diff --git a/src/rule-parse.y b/src/rule-parse.y index eb79250bf5..cb555c0c6d 100644 --- a/src/rule-parse.y +++ b/src/rule-parse.y @@ -58,15 +58,16 @@ static uint8_t ip4_mask_to_len(uint32_t mask) %token TOK_SRC_IP %token TOK_SRC_PORT %token TOK_TCP_STATE +%token TOK_UDP_STATE %token TOK_STRING -%token TOK_TCP_STATE_SYM +%token TOK_STATE_SYM %token TOK_ACTIVE %token TOK_BOOL %token TOK_POLICY_SYMBOL %type TOK_STRING TOK_IDENT TOK_POLICY_SYMBOL TOK_PATTERN pattern string -%type TOK_INT TOK_TCP_STATE_SYM TOK_IP_OPTION_SYM TOK_COMP -%type integer ipoption_list tcpstate_list opt_strength +%type TOK_INT TOK_STATE_SYM TOK_IP_OPTION_SYM TOK_COMP +%type integer ipoption_list state_list opt_strength %type rule %type TOK_BOOL opt_negate %type hdr_expr @@ -246,11 +247,19 @@ rule_attr: (zeek::detail::RuleHdrTest::Comp) $2, $3)); } - | TOK_TCP_STATE tcpstate_list + | TOK_TCP_STATE state_list { current_rule->AddCondition(new zeek::detail::RuleConditionTCPState($2)); } + | TOK_UDP_STATE state_list + { + if ( $2 & zeek::detail::RULE_STATE_ESTABLISHED ) + rules_error("'established' is not a valid 'udp-state'"); + + current_rule->AddCondition(new zeek::detail::RuleConditionUDPState($2)); + } + | TOK_ACTIVE TOK_BOOL { current_rule->SetActiveStatus($2); } ; @@ -382,10 +391,10 @@ ipoption_list: { $$ = $1; } ; -tcpstate_list: - tcpstate_list ',' TOK_TCP_STATE_SYM +state_list: + state_list ',' TOK_STATE_SYM { $$ = $1 | $3; } - | TOK_TCP_STATE_SYM + | TOK_STATE_SYM { $$ = $1; } ; diff --git a/src/rule-scan.l b/src/rule-scan.l index ff4c6b0dfa..8cf698563e 100644 --- a/src/rule-scan.l +++ b/src/rule-scan.l @@ -75,23 +75,23 @@ true { rules_lval.val = true; return TOK_BOOL; } false { rules_lval.val = false; return TOK_BOOL; } established { - rules_lval.val = zeek::detail::RuleConditionTCPState::STATE_ESTABLISHED; - return TOK_TCP_STATE_SYM; + rules_lval.val = zeek::detail::RULE_STATE_ESTABLISHED; + return TOK_STATE_SYM; } originator { - rules_lval.val = zeek::detail::RuleConditionTCPState::STATE_ORIG; - return TOK_TCP_STATE_SYM; + rules_lval.val = zeek::detail::RULE_STATE_ORIG; + return TOK_STATE_SYM; } responder { - rules_lval.val = zeek::detail::RuleConditionTCPState::STATE_RESP; - return TOK_TCP_STATE_SYM; + rules_lval.val = zeek::detail::RULE_STATE_RESP; + return TOK_STATE_SYM; } stateless { - rules_lval.val = zeek::detail::RuleConditionTCPState::STATE_STATELESS; - return TOK_TCP_STATE_SYM; + rules_lval.val = zeek::detail::RULE_STATE_STATELESS; + return TOK_STATE_SYM; } lsrr { @@ -132,6 +132,7 @@ same-ip return TOK_SAME_IP; src-ip return TOK_SRC_IP; src-port return TOK_SRC_PORT; tcp-state return TOK_TCP_STATE; +udp-state return TOK_UDP_STATE; active return TOK_ACTIVE; file-magic { rules_lval.val = zeek::detail::Rule::FILE_MAGIC; return TOK_PATTERN_TYPE; } diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index 600597988d..4d731a507c 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -660,6 +660,16 @@ SetupResult setup(int argc, char** argv, Options* zopts) init_net_var(); run_bif_initializers(); + // Assign the script_args for command line processing in Zeek scripts. + if ( ! options.script_args.empty() ) + { + auto script_args_val = id::find_val("zeek_script_args"); + for ( const string& script_arg : options.script_args ) + { + script_args_val->Assign(script_args_val->Size(), make_intrusive(script_arg)); + } + } + // Must come after plugin activation (and also after hash // initialization). binpac::FlowBuffer::Policy flowbuffer_policy; diff --git a/testing/btest/Baseline/core.script-args/out b/testing/btest/Baseline/core.script-args/out new file mode 100644 index 0000000000..e7be66720c --- /dev/null +++ b/testing/btest/Baseline/core.script-args/out @@ -0,0 +1,2 @@ +[-a, -b, -c] +[-d, -e, -f] diff --git a/testing/btest/Baseline/language.function-already-defined/out b/testing/btest/Baseline/language.function-already-defined/out new file mode 100644 index 0000000000..fcbb88a3e5 --- /dev/null +++ b/testing/btest/Baseline/language.function-already-defined/out @@ -0,0 +1,2 @@ +error in /home/jon/pro/zeek/zeek/testing/btest/.tmp/language.function-already-defined/function-already-defined.zeek, line 4 and /home/jon/pro/zeek/zeek/testing/btest/.tmp/language.function-already-defined/function-already-defined.zeek, line 7: already defined (foo) +fatal error in /home/jon/pro/zeek/zeek/testing/btest/.tmp/language.function-already-defined/function-already-defined.zeek, line 8: invalid definition of 'foo' (see previous errors) diff --git a/testing/btest/Baseline/scripts.policy.misc.capture-loss/capture_loss.log b/testing/btest/Baseline/scripts.policy.misc.capture-loss/capture_loss.log new file mode 100644 index 0000000000..ef5dca2f4b --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.misc.capture-loss/capture_loss.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path capture_loss +#open 2020-10-08-16-33-05 +#fields ts ts_delta peer gaps acks percent_lost +#types time interval string count count double +964953086.310131 0.000000 zeek 0 0 0.0 +#close 2020-10-08-16-33-05 diff --git a/testing/btest/Baseline/scripts.policy.misc.capture-loss/notice.log b/testing/btest/Baseline/scripts.policy.misc.capture-loss/notice.log new file mode 100644 index 0000000000..c5fd517179 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.misc.capture-loss/notice.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path notice +#open 2020-10-12-23-36-17 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc proto note msg sub src dst p n peer_descr actions suppress_for remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude +#types time string addr port addr port string string string enum enum string string addr addr port count string set[enum] interval string string string double double +964953086.310131 - - - - - - - - - CaptureLoss::Too_Little_Traffic Only observed 0 TCP ACKs and was expecting at least 1. - - - - - - Notice::ACTION_LOG 3600.000000 - - - - - +#close 2020-10-12-23-36-17 diff --git a/testing/btest/Baseline/signatures.udp-state/out b/testing/btest/Baseline/signatures.udp-state/out new file mode 100644 index 0000000000..eb4d4bc78a --- /dev/null +++ b/testing/btest/Baseline/signatures.udp-state/out @@ -0,0 +1,10 @@ +signature_match [orig_h=192.168.17.58, orig_p=58755/udp, resp_h=8.8.8.8, resp_p=53/udp] - my_sig_udp_orig +0000 35 5e 01 00 00 01 00 00 00 00 00 00 06 67 6f 6f 5^...... .....goo +0010 67 6c 65 03 63 6f 6d 00 01 01 00 01 gle.com. .... + +signature_match [orig_h=192.168.17.58, orig_p=58755/udp, resp_h=8.8.8.8, resp_p=53/udp] - my_sig_udp_resp +0000 35 5e 81 80 00 01 00 01 00 00 00 00 06 67 6f 6f 5^...... .....goo +0010 67 6c 65 03 63 6f 6d 00 01 01 00 01 c0 0c 01 01 gle.com. ........ +0020 00 01 00 00 54 49 00 13 00 05 69 73 73 75 65 73 ....TI.. ..issues +0030 79 6d 61 6e 74 65 63 2e 63 6f 6d ymantec. com + diff --git a/testing/btest/Baseline/signatures.udp-state/reject b/testing/btest/Baseline/signatures.udp-state/reject new file mode 100644 index 0000000000..b495e79aa4 --- /dev/null +++ b/testing/btest/Baseline/signatures.udp-state/reject @@ -0,0 +1,2 @@ +error: Error in signature (udp-established.sig:5): 'established' is not a valid 'udp-state' + diff --git a/testing/btest/core/script-args.zeek b/testing/btest/core/script-args.zeek new file mode 100644 index 0000000000..2e38f411cc --- /dev/null +++ b/testing/btest/core/script-args.zeek @@ -0,0 +1,14 @@ +# @TEST-EXEC: printf '#!' > test.zeek +# @TEST-EXEC: printf "$BUILD/src/zeek -b --\n" >> test.zeek +# @TEST-EXEC: cat %INPUT >> test.zeek +# @TEST-EXEC: chmod u+x test.zeek + +# @TEST-EXEC: zeek -b -- %INPUT -a -b -c >out +# @TEST-EXEC: $(dirname %INPUT)/test.zeek -d -e -f >>out + +# @TEST-EXEC: btest-diff out + +event zeek_init() + { + print zeek_script_args; + } diff --git a/testing/btest/language/function-already-defined.zeek b/testing/btest/language/function-already-defined.zeek new file mode 100644 index 0000000000..9c609a68ac --- /dev/null +++ b/testing/btest/language/function-already-defined.zeek @@ -0,0 +1,11 @@ +# @TEST-EXEC-FAIL: zeek -b %INPUT >out 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out + +function foo(a: string) + { print a; } + +function foo(a: string) + { } + +event zeek_init() + { foo("hello"); } diff --git a/testing/btest/scripts/policy/misc/capture-loss.zeek b/testing/btest/scripts/policy/misc/capture-loss.zeek new file mode 100644 index 0000000000..d51576a5b4 --- /dev/null +++ b/testing/btest/scripts/policy/misc/capture-loss.zeek @@ -0,0 +1,12 @@ +# @TEST-EXEC: zeek -b -r $TRACES/dns53.pcap %INPUT +# @TEST-EXEC: btest-diff capture_loss.log +# @TEST-EXEC: btest-diff notice.log + +@load misc/capture-loss + +module CaptureLoss; + +event zeek_init() + { + event take_measurement(network_time(), 0, 0); + } diff --git a/testing/btest/signatures/udp-state.zeek b/testing/btest/signatures/udp-state.zeek new file mode 100644 index 0000000000..13cb3d1b8a --- /dev/null +++ b/testing/btest/signatures/udp-state.zeek @@ -0,0 +1,36 @@ +# @TEST-EXEC: zeek -b -s udp-states.sig -r $TRACES/dns-caa.pcap %INPUT >out +# @TEST-EXEC-FAIL: zeek -b -s udp-established.sig -r $TRACES/dns-caa.pcap %INPUT >reject 2>&1 +# @TEST-EXEC: btest-diff out +# @TEST-EXEC: btest-diff reject + +@TEST-START-FILE udp-states.sig +signature my_sig_udp_orig { + ip-proto == udp + payload /.+/ + udp-state originator + event "my_sig_udp_orig" +} + +signature my_sig_udp_resp { + ip-proto == udp + payload /.+/ + udp-state responder + event "my_sig_udp_resp" +} +@TEST-END-FILE + +@TEST-START-FILE udp-established.sig +signature my_sig_udp_est { + ip-proto == udp + payload /.+/ + udp-state established + event "my_sig_udp_est" +} +@TEST-END-FILE + +event signature_match(state: signature_state, msg: string, data: string) + { + print fmt("signature_match %s - %s", state$conn$id, msg); + local s = split_string(hexdump(data), /\n/); + for ( i in s ) print s[i]; + } diff --git a/testing/external/commit-hash.zeek-testing b/testing/external/commit-hash.zeek-testing index 403f9bd9e2..142bb2075d 100644 --- a/testing/external/commit-hash.zeek-testing +++ b/testing/external/commit-hash.zeek-testing @@ -1 +1 @@ -1386fa03e0b84be1491749502d3d3cb9d45a2b95 +e9e9363814c592a4b0557f70bd7d95e3a5573d8f diff --git a/testing/external/commit-hash.zeek-testing-private b/testing/external/commit-hash.zeek-testing-private index 2e89811d14..038bf923a4 100644 --- a/testing/external/commit-hash.zeek-testing-private +++ b/testing/external/commit-hash.zeek-testing-private @@ -1 +1 @@ -e9e3249a9fe5a407ada6de61eeeb4faf1a928ec4 +ca98da7a376b8c6b3fb1c3dc2e415030f6b876bf