diff --git a/.cirrus.yml b/.cirrus.yml index b14fcd08c1..b6dd77e461 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -94,7 +94,7 @@ ubuntu16_task: # image is available. macos_task: osx_instance: - image: mojave-base + image: catalina-base prepare_script: ./ci/macos/prepare.sh << : *CI_TEMPLATE env: diff --git a/CHANGES b/CHANGES index ca3b76006b..231256672c 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,128 @@ +3.1.0-dev.482 | 2020-01-30 19:16:03 -0800 + + * Update NEWS for naming changes. (Robin Sommer, Corelight) + + * Remove test for legacy plugin. (Robin Sommer, Corelight) + + We no longer support creating new plugins using the old naming + scheme. + + * Remove legancy symlinks in aux/. (Robin Sommer, Corelight) + + That's aux/bro-aux and aux/broctl. + + * Add warnings when loading scripts ending in ".bro", or using legacy environment variables. (Robin Sommer, Corelight) + + * Fix missing rename to .zeek-dns-cache. (Robin Sommer, Corelight) + + * No longer symlink local.zeek to local.bro. (Robin Sommer, Corelight) + + * Update notice user agent. (Robin Sommer, Corelight) + + * Remove old_comm_usage_is_ok. (Robin Sommer, Corelight) + + * Remove bro-config.h.in and bro-path-dev.in. (Robin Sommer, Corelight) + + * Change Bro wrapper script to now abort when old executable names are still used. (Robin Sommer, Corelight) + + * Remove APIs that were explicitly deprecated to be removed in 3.1. (Robin Sommer, Corelight) + + Special handling for bro_{init,done,script_loaded} events: if still + used, they cause Zeek to abort at startup. + +3.1.0-dev.469 | 2020-01-30 08:44:15 -0700 + + * Fix leaked FDs in redirecting supervised node stdout/stderr (Jon Siwek, Corelight) + + * Add checks for failed fcntl calls (Jon Siwek, Corelight) + + * Set Pipe file descriptor flags correctly (Jon Siwek, Corelight) + +3.1.0-dev.465 | 2020-01-29 11:45:15 -0800 + + * Scope: fix memory leak by removing duplicate copy_string() call (Max Kellermann) + + Note this is a parse-time memory leak, not a run-time leak. + + * util, nb_dns: fix off-by-one bugs in strncpy() calls (Max Kellermann) + + Fortunately, these bugs had no effect because the following lines + overwrote the last character with a null byte. + + * Type, util: add `constexpr` to static variables (Max Kellermann) + + * Net: remove unused variable (Max Kellermann) + +3.1.0-dev.460 | 2020-01-29 11:16:15 -0800 + + * Skip file analysis for zero-length SSL/TLS data (Jon Siwek, Corelight) + +3.1.0-dev.458 | 2020-01-29 12:53:32 +0000 + + * Add a new supervisor framework that enables Zeek to operate + clusters of processes itself without any external help. (Jon + Siwek, Corelight) + + The Supervisor framework provides an entirely new deployment mode + for Zeek, one that supervises a set of Zeek processes that are + meant to be persistent. A Supervisor automatically revives any + process that dies or exits prematurely and also arranges for an + ordered shutdown of the entire process tree upon its own + termination. This Supervisor mode for Zeek provides the basic + foundation for process configuration/management that could be used + to deploy a Zeek cluster similar to what ZeekControl does, but is + also simpler to integrate as a standard system service. + + This mode is still experimental and will evolve over time. The + command-line argument of ``-j`` toggles Zeek to run in "Supervisor + mode" to allow for creation and management of child processes. If + you're going to test this, please note that you will need some + custom script code to configure the processes you want Zeek to + run. See the documentation for more information: + https://docs.zeek.org/en/stable/frameworks/supervisor.html + +3.1.0-dev.408 | 2020-01-28 17:56:02 -0800 + + * Update Cirrus CI config to use macOS Catalina (Jon Siwek, Corelight) + +3.1.0-dev.406 | 2020-01-28 12:42:25 -0800 + + * util: use getrandom() on Linux if available (Max Kellermann) + + Unlike /dev/urandom, getrandom() doesn't need a file descriptor and + works when there is no /dev. It requires Linux 3.17 and glibc 2.25, + but there is a fallback to the old code. + +3.1.0-dev.403 | 2020-01-24 15:15:04 -0800 + + * Fixed decompose_uri() errors on URIs with empty port component (Frerich Raabe) + +3.1.0-dev.398 | 2020-01-23 18:08:43 -0800 + + * Skip check for outdated docs in Cirrus CI for PRs (Jon Siwek, Corelight) + +3.1.0-dev.397 | 2020-01-23 15:30:33 -0800 + + * Use consistent zeek_init priority for Log::create_stream calls + + Typically in base scripts, Log::create_stream() is called in zeek_init() + handler with &priority=5 such that it will have already been created + in the default zeek_init() &priority=0. (Jon Siwek, Corelight) + +3.1.0-dev.395 | 2020-01-23 13:50:42 +0000 + + * Update Broker submodule. + +3.1.0-dev.394 | 2020-01-23 13:18:54 +0000 + + * Improve creation of SMB transaction data strings. (Jon Siwek, + Corelight) + +3.1.0-dev.392 | 2020-01-23 13:14:28 +0000 + + * Improve FTP word/whitespace handling. (Jon Siwek, Corelight) + 3.1.0-dev.389 | 2020-01-18 10:49:15 +0000 * GHI-595: Convert from nlohmann/json to RapidJSON for performance diff --git a/CMakeLists.txt b/CMakeLists.txt index 956f2cd9b8..a919b8ffe1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -379,11 +379,8 @@ endif () string(TOLOWER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_LOWER) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/zeek-config.h.in ${CMAKE_CURRENT_BINARY_DIR}/zeek-config.h) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/bro-config.h.in - ${CMAKE_CURRENT_BINARY_DIR}/bro-config.h) include_directories(${CMAKE_CURRENT_BINARY_DIR}) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zeek-config.h DESTINATION include/zeek) -install(FILES ${CMAKE_CURRENT_BINARY_DIR}/bro-config.h DESTINATION include/zeek) if ( CAF_ROOT_DIR ) set(ZEEK_CONFIG_CAF_ROOT_DIR ${CAF_ROOT_DIR}) diff --git a/NEWS b/NEWS index 772886070f..a8b147493e 100644 --- a/NEWS +++ b/NEWS @@ -9,6 +9,27 @@ Zeek 3.1.0 New Functionality ----------------- +- Add a new supervisor framework that enables Zeek to operate clusters + of processes itself without any external help. + + The Supervisor framework provides an entirely new deployment mode + for Zeek, one that supervises a set of Zeek processes that are meant + to be persistent. A Supervisor automatically revives any process + that dies or exits prematurely and also arranges for an ordered + shutdown of the entire process tree upon its own termination. This + Supervisor mode for Zeek provides the basic foundation for process + configuration/management that could be used to deploy a Zeek cluster + similar to what ZeekControl does, but is also simpler to integrate + as a standard system service. + + This mode is still experimental and will evolve over time. The + command-line argument of ``-j`` toggles Zeek to run in "Supervisor + mode" to allow for creation and management of child processes. If + you're going to test this, please note that you will need some + custom script code to configure the processes you want Zeek to run. + See the documentation for more information: + https://docs.zeek.org/en/stable/frameworks/supervisor.html + - Add a new option, ``dpd_late_match_stop``, which can be used in conjuction with the option ``dpd_match_only_beginning`` and the new event ``protocol_late_match`` to help annotate the conn.log with a field @@ -29,6 +50,15 @@ New Functionality Changed Functionality --------------------- +- The backwards-compability wrappers & work-arounds introduced in 3.0 + for the "Bro to Zeek rename" have either changed their operation, or in some + cases been removed. Generally, anything that reported a + naming-related warning in 3.0 now aborts with a corresponding error + message. In cases where 3.0 silently continued to accept old names, + 3.1 now reports warnings. Most importantly, that's loading of + scripts with ".bro" endings, which are now flagged and should be + renamed. + - The key type of ``Known::service_store`` has changed to ``Known::AddrPortServTriplet`` and ``Known::services`` is now a table instead of just a set. @@ -67,6 +97,11 @@ Changed Functionality previously used. Output from the formatters remains nearly identical. +- The ``decompose_uri`` function no longer raises an error when parsing + URIs with an empty port number (e.g. ``http://example.org:/``). Instead, + the ``portnum`` component of the returned ``URI`` value is left + uninitialized. + Removed Functionality --------------------- diff --git a/VERSION b/VERSION index 31ca6dd848..ed78f4c49d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.0-dev.389 +3.1.0-dev.482 diff --git a/aux/bro-aux b/aux/bro-aux deleted file mode 120000 index b28967a8e2..0000000000 --- a/aux/bro-aux +++ /dev/null @@ -1 +0,0 @@ -zeek-aux \ No newline at end of file diff --git a/aux/broctl b/aux/broctl deleted file mode 120000 index d17a55b030..0000000000 --- a/aux/broctl +++ /dev/null @@ -1 +0,0 @@ -zeekctl \ No newline at end of file diff --git a/aux/broker b/aux/broker index 2854e3e308..89a43997d6 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 2854e3e30810673fd6b9200321fa4ada364c559b +Subproject commit 89a43997d6715fca4e2242a3b88b3aa1c075a340 diff --git a/aux/netcontrol-connectors b/aux/netcontrol-connectors index 6bb6709e75..ba7b6c81a1 160000 --- a/aux/netcontrol-connectors +++ b/aux/netcontrol-connectors @@ -1 +1 @@ -Subproject commit 6bb6709e755ecd2b930ff4a3ddd68f16d2b52cba +Subproject commit ba7b6c81a17b578fd4723e903c0a8b112b43e1b1 diff --git a/aux/zeek-aux b/aux/zeek-aux index 0790f42014..5b98e96695 160000 --- a/aux/zeek-aux +++ b/aux/zeek-aux @@ -1 +1 @@ -Subproject commit 0790f420148806c1380fc7e0e0a4278c7970753c +Subproject commit 5b98e96695c334b6c2929fb96666d58eb22f9b66 diff --git a/aux/zeekctl b/aux/zeekctl index 6933b86e60..f8941cd0ba 160000 --- a/aux/zeekctl +++ b/aux/zeekctl @@ -1 +1 @@ -Subproject commit 6933b86e60f22f7a39ac1a8adbee4867902ce02e +Subproject commit f8941cd0ba171d9ea2b2e01724597e5e790ffb16 diff --git a/bro-config.h.in b/bro-config.h.in deleted file mode 100644 index 879249be38..0000000000 --- a/bro-config.h.in +++ /dev/null @@ -1,5 +0,0 @@ -#pragma once - -#pragma message("Warning: bro-config.h is deprecated, use zeek-config.h instead") - -#include "zeek-config.h" diff --git a/bro-path-dev.in b/bro-path-dev.in deleted file mode 120000 index 854029fbb8..0000000000 --- a/bro-path-dev.in +++ /dev/null @@ -1 +0,0 @@ -zeek-path-dev.in \ No newline at end of file diff --git a/doc b/doc index 7192dbedf3..566174d004 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 7192dbedf3ca9ce49294057262074f0e888177f3 +Subproject commit 566174d004c14d061fcf7c03e4829f20d46aaad8 diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index e5611aefaa..96b2c897c1 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -7,26 +7,6 @@ install(DIRECTORY ./ DESTINATION ${ZEEK_SCRIPT_INSTALL_PATH} FILES_MATCHING PATTERN "*.fp" ) -if ( NOT BINARY_PACKAGING_MODE ) - # If the user has a local.bro file from a previous installation, prefer to - # symlink local.zeek to it to avoid breaking their custom configuration -- - # because ZeekControl will now prefer to load local.zeek rather than local.bro - # and we're about to install a default version of local.zeek. - - set(_local_bro_dst ${ZEEK_SCRIPT_INSTALL_PATH}/site/local.bro) - set(_local_zeek_dst ${ZEEK_SCRIPT_INSTALL_PATH}/site/local.zeek) - - install(CODE " - if ( \"\$ENV{DESTDIR}\" STREQUAL \"\" ) - if ( EXISTS \"${_local_bro_dst}\" AND NOT EXISTS \"${_local_zeek_dst}\" ) - message(STATUS \"WARNING: installed ${_local_zeek_dst} as symlink to ${_local_bro_dst}\") - execute_process(COMMAND \"${CMAKE_COMMAND}\" -E create_symlink - \"${_local_bro_dst}\" \"${_local_zeek_dst}\") - endif () - endif () - ") -endif () - # Install local script as a config file since it's meant to be modified directly. InstallPackageConfigFile( ${CMAKE_CURRENT_SOURCE_DIR}/site/local.zeek diff --git a/scripts/base/frameworks/cluster/__load__.zeek b/scripts/base/frameworks/cluster/__load__.zeek index 9effaf835a..a04d6744d2 100644 --- a/scripts/base/frameworks/cluster/__load__.zeek +++ b/scripts/base/frameworks/cluster/__load__.zeek @@ -18,7 +18,12 @@ redef Broker::log_topic = Cluster::rr_log_topic; # Loading the cluster framework requires that a script by this name exists # somewhere in the ZEEKPATH. The only thing in the file should be the # cluster definition in the :zeek:id:`Cluster::nodes` variable. + +@if ( ! Supervisor::__init_cluster() ) +# When running a supervised cluster, Cluster::nodes is instead populated +# from the internal C++-layer directly via the above BIF. @load cluster-layout +@endif @if ( Cluster::node in Cluster::nodes ) diff --git a/scripts/base/frameworks/cluster/main.zeek b/scripts/base/frameworks/cluster/main.zeek index 9040c663e1..c8ff1ce159 100644 --- a/scripts/base/frameworks/cluster/main.zeek +++ b/scripts/base/frameworks/cluster/main.zeek @@ -287,7 +287,13 @@ function is_enabled(): bool function local_node_type(): NodeType { - return is_enabled() ? nodes[node]$node_type : NONE; + if ( ! is_enabled() ) + return NONE; + + if ( node !in nodes ) + return NONE; + + return nodes[node]$node_type; } function node_topic(name: string): string diff --git a/scripts/base/frameworks/cluster/nodes/logger.zeek b/scripts/base/frameworks/cluster/nodes/logger.zeek index 6fb5d09208..a56d4a4d66 100644 --- a/scripts/base/frameworks/cluster/nodes/logger.zeek +++ b/scripts/base/frameworks/cluster/nodes/logger.zeek @@ -23,4 +23,7 @@ redef Log::default_rotation_interval = 1 hrs; redef Log::default_mail_alarms_interval = 24 hrs; ## Use the cluster's archive logging script. + +@if ( ! Supervisor::is_supervised() ) redef Log::default_rotation_postprocessor_cmd = "archive-log"; +@endif diff --git a/scripts/base/frameworks/notice/main.zeek b/scripts/base/frameworks/notice/main.zeek index 9e43a9ed50..f3bc6bc8df 100644 --- a/scripts/base/frameworks/notice/main.zeek +++ b/scripts/base/frameworks/notice/main.zeek @@ -402,7 +402,7 @@ function email_headers(subject_desc: string, dest: string): string "From: ", mail_from, "\n", "Subject: ", mail_subject_prefix, " ", subject_desc, "\n", "To: ", dest, "\n", - "User-Agent: Bro-IDS/", zeek_version(), "\n"); + "User-Agent: Zeek/", zeek_version(), "\n"); if ( reply_to != "" ) header_text = string_cat(header_text, "Reply-To: ", reply_to, "\n"); return header_text; diff --git a/scripts/base/frameworks/signatures/main.zeek b/scripts/base/frameworks/signatures/main.zeek index eb2143d4ec..9a301d520a 100644 --- a/scripts/base/frameworks/signatures/main.zeek +++ b/scripts/base/frameworks/signatures/main.zeek @@ -140,7 +140,7 @@ global count_per_orig: table[addr, string] of count global did_sig_log: set[string] &read_expire = 1 hr; -event zeek_init() +event zeek_init() &priority=5 { Log::create_stream(Signatures::LOG, [$columns=Info, $ev=log_signature, $path="signatures"]); } diff --git a/scripts/base/frameworks/supervisor/__load__.zeek b/scripts/base/frameworks/supervisor/__load__.zeek new file mode 100644 index 0000000000..97de26a373 --- /dev/null +++ b/scripts/base/frameworks/supervisor/__load__.zeek @@ -0,0 +1,3 @@ +@load ./api +@load ./control +@load ./main diff --git a/scripts/base/frameworks/supervisor/api.zeek b/scripts/base/frameworks/supervisor/api.zeek new file mode 100644 index 0000000000..7bd967c79e --- /dev/null +++ b/scripts/base/frameworks/supervisor/api.zeek @@ -0,0 +1,120 @@ +##! The Zeek process supervision API. +##! This API was introduced in Zeek 3.1.0 and considered unstable until 4.0.0. +##! That is, it may change in various incompatible ways without warning or +##! deprecation until the stable 4.0.0 release. + +module Supervisor; + +export { + ## The role a supervised-node will play in Zeek's Cluster Framework. + type ClusterRole: enum { + NONE, + LOGGER, + MANAGER, + PROXY, + WORKER, + }; + + ## Describes configuration of a supervised-node within Zeek's Cluster + ## Framework. + type ClusterEndpoint: record { + ## The role a supervised-node will play in Zeek's Cluster Framework. + role: ClusterRole; + ## The host/IP at which the cluster node runs. + host: addr; + ## The TCP port at which the cluster node listens for connections. + p: port; + ## The interface name from which the node will read/analyze packets. + ## Typically used by worker nodes. + interface: string &optional; + }; + + ## Configuration options that influence behavior of a supervised Zeek node. + type NodeConfig: record { + ## The name of the supervised node. These are unique within a given + ## supervised process tree and typically human-readable. + name: string; + ## The interface name from which the node will read/analyze packets. + interface: string &optional; + ## The working directory that the node should use. + directory: string &optional; + ## The filename/path to which the node's stdout will be redirected. + stdout_file: string &optional; + ## The filename/path to which the node's stderr will be redirected. + stderr_file: string &optional; + ## Additional script filenames/paths that the node should load. + scripts: vector of string &default = vector(); + ## A cpu/core number to which the node will try to pin itself. + cpu_affinity: int &optional; + ## The Cluster Layout definition. Each node in the Cluster Framework + ## knows about the full, static cluster topology to which it belongs. + ## Entries use node names for keys. The Supervisor framework will + ## automatically translate this table into the right Cluster Framework + ## configuration when spawning supervised-nodes. E.g. it will + ## populate the both the CLUSTER_NODE environment variable and + ## :zeek:see:`Cluster::nodes` table. + cluster: table[string] of ClusterEndpoint &default=table(); + }; + + ## The current status of a supervised node. + type NodeStatus: record { + ## The desired node configuration. + node: NodeConfig; + ## The current or last known process ID of the node. This may not + ## be initialized if the process has not yet started. + pid: int &optional; + }; + + ## The current status of a set of supervised nodes. + type Status: record { + ## The status of supervised nodes, keyed by node names. + nodes: table[string] of NodeStatus; + }; + + ## Create a new supervised node process. + ## It's an error to call this from a process other than a Supervisor. + ## + ## node: the desired configuration for the new supervised node process. + ## + ## Returns: an empty string on success or description of the error/failure. + global create: function(node: NodeConfig): string; + + ## Retrieve current status of a supervised node process. + ## It's an error to call this from a process other than a Supervisor. + ## + ## node: the name of the node to get the status of or an empty string + ## to mean "all nodes". + ## + ## Returns: the current status of a set of nodes. + global status: function(node: string &default=""): Status; + + ## Restart a supervised node process by destroying (killing) and + ## re-recreating it. + ## It's an error to call this from a process other than a Supervisor. + ## + ## node: the name of the node to restart or an empty string to mean + ## "all nodes". + ## + ## Returns: true on success. + global restart: function(node: string &default=""): bool; + + ## Destroy and remove a supervised node process. + ## It's an error to call this from a process other than a Supervisor. + ## + ## node: the name of the node to destroy or an empty string to mean + ## "all nodes". + ## + ## Returns: true on success. + global destroy: function(node: string &default=""): bool; + + ## Returns: true if this is the Supervisor process. + global is_supervisor: function(): bool; + + ## Returns: true if this is a supervised node process. + global is_supervised: function(): bool; + + ## Returns: the node configuration if this is a supervised node. + ## It's an error to call this function from a process other than + ## a supervised one. + global node: function(): NodeConfig; +} diff --git a/scripts/base/frameworks/supervisor/control.zeek b/scripts/base/frameworks/supervisor/control.zeek new file mode 100644 index 0000000000..fa20a9dba6 --- /dev/null +++ b/scripts/base/frameworks/supervisor/control.zeek @@ -0,0 +1,89 @@ +##! The Zeek process supervision (remote) control API. This defines a Broker topic +##! prefix and events that can be used to control an external Zeek supervisor process. +##! This API was introduced in Zeek 3.1.0 and considered unstable until 4.0.0. +##! That is, it may change in various incompatible ways without warning or +##! deprecation until the stable 4.0.0 release. + +@load ./api + +module SupervisorControl; + +export { + ## The Broker topic prefix to use when subscribing to Supervisor API + ## requests and when publishing Supervisor API responses. If you are + ## publishing Supervisor requests, this is also the prefix string to use + ## for their topic names. + const topic_prefix = "zeek/supervisor" &redef; + + ## Send a request to a remote Supervisor process to create a node. + ## + ## reqid: an arbitrary string that will be directly echoed in the response + ## + ## node: the desired configuration for the new supervised node process. + global SupervisorControl::create_request: event(reqid: string, node: Supervisor::NodeConfig); + + ## Handle a response from a Supervisor process that received + ## :zeek:see:`SupervisorControl::create_request`. + ## + ## reqid: an arbitrary string matching the value in the original request. + ## + ## result: the return value of the remote call to + ## :zeek:see:`Supervisor::create`. + global SupervisorControl::create_response: event(reqid: string, result: string); + + ## Send a request to a remote Supervisor process to retrieve node status. + ## + ## reqid: an arbitrary string that will be directly echoed in the response + ## + ## node: the name of the node to get status of or empty string to mean "all + ## nodes". + global SupervisorControl::status_request: event(reqid: string, node: string); + + ## Handle a response from a Supervisor process that received + ## :zeek:see:`SupervisorControl::status_request`. + ## + ## reqid: an arbitrary string matching the value in the original request. + ## + ## result: the return value of the remote call to + ## :zeek:see:`Supervisor::status`. + global SupervisorControl::status_response: event(reqid: string, result: Supervisor::Status); + + ## Send a request to a remote Supervisor process to restart a node. + ## + ## reqid: an arbitrary string that will be directly echoed in the response + ## + ## node: the name of the node to restart or empty string to mean "all + ## nodes". + global SupervisorControl::restart_request: event(reqid: string, node: string); + + ## Handle a response from a Supervisor process that received + ## :zeek:see:`SupervisorControl::restart_request`. + ## + ## reqid: an arbitrary string matching the value in the original request. + ## + ## result: the return value of the remote call to + ## :zeek:see:`Supervisor::restart`. + global SupervisorControl::restart_response: event(reqid: string, result: bool); + + ## Send a request to a remote Supervisor process to destroy a node. + ## + ## reqid: an arbitrary string that will be directly echoed in the response + ## + ## node: the name of the node to destory or empty string to mean "all + ## nodes". + global SupervisorControl::destroy_request: event(reqid: string, node: string); + + ## Handle a response from a Supervisor process that received + ## :zeek:see:`SupervisorControl::destroy_request`. + ## + ## reqid: an arbitrary string matching the value in the original request. + ## + ## result: the return value of the remote call to + ## :zeek:see:`Supervisor::destroy`. + global SupervisorControl::destroy_response: event(reqid: string, result: bool); + + ## Send a request to a remote Supervisor to stop and shutdown its + ## process tree. There is no response to this message as the Supervisor + ## simply terminates on receipt. + global SupervisorControl::stop_request: event(); +} diff --git a/scripts/base/frameworks/supervisor/main.zeek b/scripts/base/frameworks/supervisor/main.zeek new file mode 100644 index 0000000000..f892907055 --- /dev/null +++ b/scripts/base/frameworks/supervisor/main.zeek @@ -0,0 +1,94 @@ +##! Implements Zeek process supervision API and default behavior for its +##! associated (remote) control events. + +@load ./api +@load ./control +@load base/frameworks/broker + +function Supervisor::status(node: string): Supervisor::Status + { + return Supervisor::__status(node); + } + +function Supervisor::create(node: Supervisor::NodeConfig): string + { + return Supervisor::__create(node); + } + +function Supervisor::destroy(node: string): bool + { + return Supervisor::__destroy(node); + } + +function Supervisor::restart(node: string): bool + { + return Supervisor::__restart(node); + } + +function Supervisor::is_supervisor(): bool + { + return Supervisor::__is_supervisor(); + } + +function Supervisor::is_supervised(): bool + { + return Supervisor::__is_supervised(); + } + +function Supervisor::node(): Supervisor::NodeConfig + { + return Supervisor::__node(); + } + +event zeek_init() &priority=10 + { + Broker::subscribe(SupervisorControl::topic_prefix); + } + +event SupervisorControl::stop_request() + { + if ( ! Supervisor::is_supervisor() ) + return; + + terminate(); + } + +event SupervisorControl::status_request(reqid: string, node: string) + { + if ( ! Supervisor::is_supervisor() ) + return; + + local res = Supervisor::status(node); + local topic = SupervisorControl::topic_prefix + fmt("/status_response/%s", reqid); + Broker::publish(topic, SupervisorControl::status_response, reqid, res); + } + +event SupervisorControl::create_request(reqid: string, node: Supervisor::NodeConfig) + { + if ( ! Supervisor::is_supervisor() ) + return; + + local res = Supervisor::create(node); + local topic = SupervisorControl::topic_prefix + fmt("/create_response/%s", reqid); + Broker::publish(topic, SupervisorControl::create_response, reqid, res); + } + +event SupervisorControl::destroy_request(reqid: string, node: string) + { + if ( ! Supervisor::is_supervisor() ) + return; + + local res = Supervisor::destroy(node); + local topic = SupervisorControl::topic_prefix + fmt("/destroy_response/%s", reqid); + Broker::publish(topic, SupervisorControl::destroy_response, reqid, res); + } + +event SupervisorControl::restart_request(reqid: string, node: string) + { + if ( ! Supervisor::is_supervisor() ) + return; + + local res = Supervisor::restart(node); + local topic = SupervisorControl::topic_prefix + fmt("/restart_response/%s", reqid); + Broker::publish(topic, SupervisorControl::restart_response, reqid, res); + } diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index 5a6f0a47ae..7360a5bd23 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -1835,6 +1835,8 @@ type gtp_delete_pdp_ctx_response_elements: record { @load base/bif/reporter.bif @load base/bif/strings.bif @load base/bif/option.bif +@load base/frameworks/supervisor/api +@load base/bif/supervisor.bif global done_with_network = F; event net_done(t: time) { done_with_network = T; } @@ -5228,10 +5230,3 @@ const global_hash_seed: string = "" &redef; ## files. The larger the value, the more confidence in UID uniqueness. ## The maximum is currently 128 bits. const bits_per_uid: count = 96 &redef; - -## Whether usage of the old communication system is considered an error or -## not. The default Zeek configuration no longer works with the non-Broker -## communication system unless you have manually taken action to initialize -## and set up the old comm. system. Deprecation warnings are still emitted -## when setting this flag, but they will not result in a fatal error. -const old_comm_usage_is_ok: bool = F &redef; diff --git a/scripts/base/init-frameworks-and-bifs.zeek b/scripts/base/init-frameworks-and-bifs.zeek index 19897e7ffb..a667110a87 100644 --- a/scripts/base/init-frameworks-and-bifs.zeek +++ b/scripts/base/init-frameworks-and-bifs.zeek @@ -5,6 +5,7 @@ # the separate file). @load base/frameworks/logging @load base/frameworks/broker +@load base/frameworks/supervisor @load base/frameworks/input @load base/frameworks/analyzer @load base/frameworks/files diff --git a/scripts/base/utils/json.zeek b/scripts/base/utils/json.zeek deleted file mode 100644 index a9662e74a2..0000000000 --- a/scripts/base/utils/json.zeek +++ /dev/null @@ -1,2 +0,0 @@ -## This file is deprecated in favor of to_json in zeek.bif -@deprecated="Remove in 3.1. to_json is now always available as a built-in function." diff --git a/scripts/base/utils/urls.zeek b/scripts/base/utils/urls.zeek index c6ec41cbfc..c8077f5424 100644 --- a/scripts/base/utils/urls.zeek +++ b/scripts/base/utils/urls.zeek @@ -119,7 +119,10 @@ function decompose_uri(uri: string): URI # Parse location and port. parts = split_string1(s, /:/); u$netlocation = parts[0]; - u$portnum = to_count(parts[1]); + if ( parts[1] != "" ) + { + u$portnum = to_count(parts[1]); + } } else { diff --git a/scripts/policy/files/x509/log-ocsp.zeek b/scripts/policy/files/x509/log-ocsp.zeek index 8cc9d5aef3..a677b38055 100644 --- a/scripts/policy/files/x509/log-ocsp.zeek +++ b/scripts/policy/files/x509/log-ocsp.zeek @@ -39,7 +39,7 @@ export { global log_ocsp: event(rec: Info); } -event zeek_init() +event zeek_init() &priority=5 { Log::create_stream(LOG, [$columns=Info, $ev=log_ocsp, $path="ocsp"]); Files::register_for_mime_type(Files::ANALYZER_OCSP_REPLY, "application/ocsp-response"); diff --git a/scripts/policy/protocols/conn/known-hosts.zeek b/scripts/policy/protocols/conn/known-hosts.zeek index 8a3383e1b2..e95f0cec16 100644 --- a/scripts/policy/protocols/conn/known-hosts.zeek +++ b/scripts/policy/protocols/conn/known-hosts.zeek @@ -145,7 +145,7 @@ event Known::host_found(info: HostsInfo) event known_host_add(info); } -event zeek_init() +event zeek_init() &priority=5 { Log::create_stream(Known::HOSTS_LOG, [$columns=HostsInfo, $ev=log_known_hosts, $path="known_hosts"]); } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d11deb9564..4d23220ab6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -107,6 +107,11 @@ set(BIF_SRCS strings.bif reporter.bif option.bif + # Note: the supervisor BIF file is treated like other top-level BIFs + # instead of contained in its own subdirectory CMake logic because + # subdirectory BIFs are treated differently and don't support being called + # *during* parsing (e.g. within an @if directive). + supervisor/supervisor.bif ) foreach (bift ${BIF_SRCS}) @@ -203,6 +208,7 @@ set(MAIN_SRCS net_util.cc util.cc module_util.cc + zeek-affinity.cc Anon.cc Attr.cc Base64.cc @@ -245,6 +251,7 @@ set(MAIN_SRCS NetVar.cc Obj.cc OpaqueVal.cc + Options.cc PacketFilter.cc Pipe.cc PolicyFile.cc @@ -284,6 +291,8 @@ set(MAIN_SRCS modp_numtoa.c siphash24.c + supervisor/Supervisor.cc + threading/BasicThread.cc threading/Formatter.cc threading/Manager.cc diff --git a/src/DNS_Mgr.cc b/src/DNS_Mgr.cc index 0b50700bd1..5599074b0d 100644 --- a/src/DNS_Mgr.cc +++ b/src/DNS_Mgr.cc @@ -467,7 +467,7 @@ void DNS_Mgr::InitPostScript() const char* cache_dir = dir ? dir : "."; cache_name = new char[strlen(cache_dir) + 64]; - sprintf(cache_name, "%s/%s", cache_dir, ".bro-dns-cache"); + sprintf(cache_name, "%s/%s", cache_dir, ".zeek-dns-cache"); LoadCache(fopen(cache_name, "r")); } diff --git a/src/DebugLogger.cc b/src/DebugLogger.cc index 6af7e26e38..fdb57efc61 100644 --- a/src/DebugLogger.cc +++ b/src/DebugLogger.cc @@ -19,7 +19,8 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = { { "threading", 0, false }, { "file_analysis", 0, false }, { "plugins", 0, false }, { "zeekygen", 0, false }, { "pktio", 0, false }, { "broker", 0, false }, - { "scripts", 0, false} + { "scripts", 0, false}, + { "supervisor", 0, false} }; DebugLogger::DebugLogger() diff --git a/src/DebugLogger.h b/src/DebugLogger.h index 4efb6a0421..e0eb57f47b 100644 --- a/src/DebugLogger.h +++ b/src/DebugLogger.h @@ -29,6 +29,7 @@ enum DebugStream { DBG_PKTIO, // Packet sources and dumpers. DBG_BROKER, // Broker communication DBG_SCRIPTS, // Script initialization + DBG_SUPERVISOR, // Process supervisor NUM_DBGS // Has to be last }; diff --git a/src/Flare.cc b/src/Flare.cc index 87dc946955..166917d914 100644 --- a/src/Flare.cc +++ b/src/Flare.cc @@ -13,14 +13,24 @@ Flare::Flare() { } -static void bad_pipe_op(const char* which) +static void bad_pipe_op(const char* which, bool signal_safe) { + if ( signal_safe ) + abort(); + char buf[256]; bro_strerror_r(errno, buf, sizeof(buf)); - reporter->FatalErrorWithCore("unexpected pipe %s failure: %s", which, buf); + + if ( reporter ) + reporter->FatalErrorWithCore("unexpected pipe %s failure: %s", which, buf); + else + { + fprintf(stderr, "unexpected pipe %s failure: %s", which, buf); + abort(); + } } -void Flare::Fire() +void Flare::Fire(bool signal_safe) { char tmp = 0; @@ -42,15 +52,16 @@ void Flare::Fire() // Interrupted: try again. continue; - bad_pipe_op("write"); + bad_pipe_op("write", signal_safe); } // No error, but didn't write a byte: try again. } } -void Flare::Extinguish() +int Flare::Extinguish(bool signal_safe) { + int rval = 0; char tmp[256]; for ( ; ; ) @@ -58,8 +69,11 @@ void Flare::Extinguish() int n = read(pipe.ReadFD(), &tmp, sizeof(tmp)); if ( n >= 0 ) + { + rval += n; // Pipe may not be empty yet: try again. continue; + } if ( errno == EAGAIN ) // Success: pipe is now empty. @@ -69,6 +83,8 @@ void Flare::Extinguish() // Interrupted: try again. continue; - bad_pipe_op("read"); + bad_pipe_op("read", signal_safe); } + + return rval; } diff --git a/src/Flare.h b/src/Flare.h index cc340494aa..ea484b2858 100644 --- a/src/Flare.h +++ b/src/Flare.h @@ -26,13 +26,19 @@ public: /** * Put the object in the "ready" state. + * @param signal_safe whether to skip error-reporting functionality that + * is not async-signal-safe (errors still abort the process regardless) */ - void Fire(); + void Fire(bool signal_safe = false); /** * Take the object out of the "ready" state. + * @param signal_safe whether to skip error-reporting functionality that + * is not async-signal-safe (errors still abort the process regardless) + * @return number of bytes read from the pipe, corresponds to the number + * of times Fire() was called. */ - void Extinguish(); + int Extinguish(bool signal_safe = false); private: Pipe pipe; diff --git a/src/Func.cc b/src/Func.cc index fb90e2e3d2..e0f4958660 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -743,12 +743,14 @@ void builtin_error(const char* msg, BroObj* arg) #include "reporter.bif.func_h" #include "strings.bif.func_h" #include "option.bif.func_h" +#include "supervisor.bif.func_h" #include "zeek.bif.func_def" #include "stats.bif.func_def" #include "reporter.bif.func_def" #include "strings.bif.func_def" #include "option.bif.func_def" +#include "supervisor.bif.func_def" #include "__all__.bif.cc" // Autogenerated for compiling in the bif_target() code. #include "__all__.bif.register.cc" // Autogenerated for compiling in the bif_target() code. @@ -776,6 +778,7 @@ void init_builtin_funcs() #include "reporter.bif.func_init" #include "strings.bif.func_init" #include "option.bif.func_init" +#include "supervisor.bif.func_init" did_builtin_init = true; } diff --git a/src/List.h b/src/List.h index e0046bb55a..2b25dfb6f8 100644 --- a/src/List.h +++ b/src/List.h @@ -154,12 +154,6 @@ public: return max_entries; } - ZEEK_DEPRECATED("Remove in v3.1: Use std::sort instead") - void sort(list_cmp_func cmp_func) - { - qsort(entries, num_entries, sizeof(T), cmp_func); - } - int MemoryAllocation() const { return padded_sizeof(*this) + pad_size(max_entries * sizeof(T)); } @@ -174,7 +168,7 @@ public: ++num_entries; entries[0] = a; } - + void push_back(const T& a) { if ( num_entries == max_entries ) @@ -182,19 +176,13 @@ public: entries[num_entries++] = a; } - + void pop_front() { remove_nth(0); } void pop_back() { remove_nth(num_entries-1); } T& front() { return entries[0]; } T& back() { return entries[num_entries-1]; } - ZEEK_DEPRECATED("Remove in v3.1: Use push_front instead") - void insert(const T& a) // add at head of list - { - push_front(a); - } - // The append method is maintained for historical/compatibility reasons. // (It's commonly used in the event generation API) void append(const T& a) // add to end of list @@ -229,16 +217,6 @@ public: return old_ent; } - ZEEK_DEPRECATED("Remove in v3.1: Use back()/pop_back() instead") - T get() // return and remove ent at end of list - { - assert(num_entries > 0); - return entries[--num_entries]; - } - - ZEEK_DEPRECATED("Remove in v3.1: Use back() instead") - T& last() { return back(); } - // Return 0 if ent is not in the list, ent otherwise. bool is_member(const T& a) const { diff --git a/src/Net.cc b/src/Net.cc index af9806412f..ac38eb846d 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -145,40 +145,40 @@ void net_update_time(double new_network_time) PLUGIN_HOOK_VOID(HOOK_UPDATE_NETWORK_TIME, HookUpdateNetworkTime(new_network_time)); } -void net_init(name_list& interfaces, name_list& readfiles, - const char* writefile, int do_watchdog) +void net_init(const std::vector& interfaces, + const std::vector& pcap_input_files, + const std::optional& pcap_output_file, + bool do_watchdog) { - if ( readfiles.length() > 0 ) + if ( ! pcap_input_files.empty() ) { reading_live = pseudo_realtime > 0.0; reading_traces = 1; - for ( int i = 0; i < readfiles.length(); ++i ) + for ( const auto& pif : pcap_input_files ) { - iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(readfiles[i], false); + iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(pif, false); assert(ps); if ( ! ps->IsOpen() ) reporter->FatalError("problem with trace file %s (%s)", - readfiles[i], - ps->ErrorMsg()); + pif.data(), ps->ErrorMsg()); } } - else if ( interfaces.length() > 0 ) + else if ( ! interfaces.empty() ) { reading_live = 1; reading_traces = 0; - for ( int i = 0; i < interfaces.length(); ++i ) + for ( const auto& iface : interfaces ) { - iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(interfaces[i], true); + iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(iface, true); assert(ps); if ( ! ps->IsOpen() ) reporter->FatalError("problem with interface %s (%s)", - interfaces[i], - ps->ErrorMsg()); + iface.data(), ps->ErrorMsg()); } } @@ -189,8 +189,9 @@ void net_init(name_list& interfaces, name_list& readfiles, // a timer. reading_traces = reading_live = 0; - if ( writefile ) + if ( pcap_output_file ) { + const char* writefile = pcap_output_file->data(); pkt_dumper = iosource_mgr->OpenPktDumper(writefile, false); assert(pkt_dumper); @@ -361,13 +362,11 @@ void net_run() current_dispatched = 0; current_iosrc = 0; - // Should we put the signal handling into an IOSource? - extern void termination_signal(); - if ( signal_val == SIGTERM || signal_val == SIGINT ) // We received a signal while processing the // current packet and its related events. - termination_signal(); + // Should we put the signal handling into an IOSource? + zeek_terminate_loop("received termination signal"); if ( ! reading_traces ) // Check whether we have timers scheduled for @@ -456,8 +455,6 @@ void net_delete() int _processing_suspended = 0; -static double suspend_start = 0; - void net_suspend_processing() { if ( _processing_suspended == 0 ) diff --git a/src/Net.h b/src/Net.h index d14f1346ef..dab0014dd5 100644 --- a/src/Net.h +++ b/src/Net.h @@ -2,6 +2,10 @@ #pragma once +#include +#include +#include + #include "net_util.h" #include "util.h" #include "List.h" @@ -10,8 +14,10 @@ #include "iosource/PktSrc.h" #include "iosource/PktDumper.h" -extern void net_init(name_list& interfaces, name_list& readfiles, - const char* writefile, int do_watchdog); +extern void net_init(const std::vector& interfaces, + const std::vector& pcap_input_files, + const std::optional& pcap_output_file, + bool do_watchdog); extern void net_run(); extern void net_get_final_stats(); extern void net_finish(int drain_events); @@ -20,7 +26,7 @@ extern void net_update_time(double new_network_time); extern void net_packet_dispatch(double t, const Packet* pkt, iosource::PktSrc* src_ps); extern void expire_timers(iosource::PktSrc* src_ps = 0); -extern void termination_signal(); +extern void zeek_terminate_loop(const char* reason); // Functions to temporarily suspend processing of live input (network packets // and remote events/state). Turning this is on is sure to lead to data loss! @@ -76,8 +82,6 @@ extern iosource::IOSource* current_iosrc; extern iosource::PktDumper* pkt_dumper; // where to save packets -extern char* writefile; - // Script file we have already scanned (or are in the process of scanning). // They are identified by inode number. struct ScannedFile { diff --git a/src/NetVar.cc b/src/NetVar.cc index 1ab99170bb..c3f8b34340 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -196,6 +196,7 @@ bro_uint_t bits_per_uid; #include "types.bif.netvar_def" #include "event.bif.netvar_def" #include "reporter.bif.netvar_def" +#include "supervisor.bif.netvar_def" void init_event_handlers() { @@ -240,6 +241,7 @@ void init_net_var() #include "const.bif.netvar_init" #include "types.bif.netvar_init" #include "reporter.bif.netvar_init" +#include "supervisor.bif.netvar_init" conn_id = internal_type("conn_id")->AsRecordType(); endpoint = internal_type("endpoint")->AsRecordType(); diff --git a/src/NetVar.h b/src/NetVar.h index 95eb99d844..a02c742859 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -203,3 +203,4 @@ extern void init_net_var(); #include "types.bif.netvar_h" #include "event.bif.netvar_h" #include "reporter.bif.netvar_h" +#include "supervisor.bif.netvar_h" diff --git a/src/Options.cc b/src/Options.cc new file mode 100644 index 0000000000..6143657967 --- /dev/null +++ b/src/Options.cc @@ -0,0 +1,460 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include + +#include "zeek-config.h" + +#ifdef HAVE_GETOPT_H +#include +#endif + +#include "bsd-getopt-long.h" +#include "logging/writers/ascii/Ascii.h" + +#include "Options.h" + +void zeek::Options::filter_supervisor_options() + { + pcap_filter = {}; + interfaces = {}; + pcap_files = {}; + signature_files = {}; + pcap_output_file = {}; + } + +void zeek::Options::filter_supervised_node_options() + { + auto og = *this; + *this = {}; + + debug_log_streams = og.debug_log_streams; + debug_script_tracing_file = og.debug_script_tracing_file; + script_code_to_exec = og.script_code_to_exec; + script_prefixes = og.script_prefixes; + + signature_re_level = og.signature_re_level; + ignore_checksums = og.ignore_checksums; + use_watchdog = og.use_watchdog; + pseudo_realtime = og.pseudo_realtime; + dns_mode = og.dns_mode; + + bare_mode = og.bare_mode; + perftools_check_leaks = og.perftools_check_leaks; + perftools_profile = og.perftools_profile; + + pcap_filter = og.pcap_filter; + signature_files = og.signature_files; + + // TODO: These are likely to be handled in a node-specific or + // use-case-specific way. e.g. interfaces is already handled for the + // "cluster" use-case, but don't have supervised-pcap-reading + // functionality yet. + /* interfaces = og.interfaces; */ + /* pcap_files = og.pcap_files; */ + + pcap_output_file = og.pcap_output_file; + random_seed_input_file = og.random_seed_input_file; + random_seed_output_file = og.random_seed_output_file; + process_status_file = og.process_status_file; + + plugins_to_load = og.plugins_to_load; + scripts_to_load = og.scripts_to_load; + script_options_to_set = og.script_options_to_set; + } + +bool zeek::fake_dns() + { + return zeekenv("ZEEK_DNS_FAKE"); + } + +extern const char* zeek_version(); + +void zeek::usage(const char* prog, int code) + { + fprintf(stderr, "zeek version %s\n", zeek_version()); + + fprintf(stderr, "usage: %s [options] [file ...]\n", prog); + fprintf(stderr, "usage: %s --test [doctest-options] -- [options] [file ...]\n", prog); + fprintf(stderr, " | Zeek script file, or read stdin\n"); + fprintf(stderr, " -a|--parse-only | exit immediately after parsing scripts\n"); + fprintf(stderr, " -b|--bare-mode | don't load scripts from the base/ directory\n"); + fprintf(stderr, " -d|--debug-script | activate Zeek script debugging\n"); + fprintf(stderr, " -e|--exec | augment loaded scripts by given code\n"); + fprintf(stderr, " -f|--filter | tcpdump filter\n"); + fprintf(stderr, " -h|--help | command line help\n"); + fprintf(stderr, " -i|--iface | read from given interface\n"); + fprintf(stderr, " -p|--prefix | add given prefix to Zeek script file resolution\n"); + fprintf(stderr, " -r|--readfile | read from given tcpdump file\n"); + fprintf(stderr, " -s|--rulefile | read rules from given file\n"); + fprintf(stderr, " -t|--tracefile | activate execution tracing\n"); + fprintf(stderr, " -v|--version | print version and exit\n"); + fprintf(stderr, " -w|--writefile | write to given tcpdump file\n"); +#ifdef DEBUG + fprintf(stderr, " -B|--debug | Enable debugging output for selected streams ('-B help' for help)\n"); +#endif + fprintf(stderr, " -C|--no-checksums | ignore checksums\n"); + fprintf(stderr, " -F|--force-dns | force DNS\n"); + fprintf(stderr, " -G|--load-seeds | load seeds from given file\n"); + fprintf(stderr, " -H|--save-seeds | save seeds to given file\n"); + fprintf(stderr, " -I|--print-id | print out given ID\n"); + fprintf(stderr, " -N|--print-plugins | print available plugins and exit (-NN for verbose)\n"); + fprintf(stderr, " -P|--prime-dns | prime DNS\n"); + fprintf(stderr, " -Q|--time | print execution time summary to stderr\n"); + fprintf(stderr, " -S|--debug-rules | enable rule debugging\n"); + fprintf(stderr, " -T|--re-level | set 'RE_level' for rules\n"); + fprintf(stderr, " -U|--status-file | Record process status in file\n"); + fprintf(stderr, " -W|--watchdog | activate watchdog timer\n"); + fprintf(stderr, " -X|--zeekygen | generate documentation based on config file\n"); + +#ifdef USE_PERFTOOLS_DEBUG + fprintf(stderr, " -m|--mem-leaks | show leaks [perftools]\n"); + fprintf(stderr, " -M|--mem-profile | record heap [perftools]\n"); +#endif + fprintf(stderr, " --pseudo-realtime[=] | enable pseudo-realtime for performance evaluation (default 1)\n"); + fprintf(stderr, " -j|--jobs | enable supervisor mode\n"); + +#ifdef USE_IDMEF + fprintf(stderr, " -n|--idmef-dtd | specify path to IDMEF DTD file\n"); +#endif + + fprintf(stderr, " --test | run unit tests ('--test -h' for help, only when compiling with ENABLE_ZEEK_UNIT_TESTS)\n"); + fprintf(stderr, " $ZEEKPATH | file search path (%s)\n", bro_path().c_str()); + fprintf(stderr, " $ZEEK_PLUGIN_PATH | plugin search path (%s)\n", bro_plugin_path()); + fprintf(stderr, " $ZEEK_PLUGIN_ACTIVATE | plugins to always activate (%s)\n", bro_plugin_activate()); + fprintf(stderr, " $ZEEK_PREFIXES | prefix list (%s)\n", bro_prefixes().c_str()); + fprintf(stderr, " $ZEEK_DNS_FAKE | disable DNS lookups (%s)\n", zeek::fake_dns() ? "on" : "off"); + fprintf(stderr, " $ZEEK_SEED_FILE | file to load seeds from (not set)\n"); + fprintf(stderr, " $ZEEK_LOG_SUFFIX | ASCII log file extension (.%s)\n", logging::writer::Ascii::LogExt().c_str()); + fprintf(stderr, " $ZEEK_PROFILER_FILE | Output file for script execution statistics (not set)\n"); + fprintf(stderr, " $ZEEK_DISABLE_ZEEKYGEN | Disable Zeekygen documentation support (%s)\n", zeekenv("ZEEK_DISABLE_ZEEKYGEN") ? "set" : "not set"); + fprintf(stderr, " $ZEEK_DNS_RESOLVER | IPv4/IPv6 address of DNS resolver to use (%s)\n", zeekenv("ZEEK_DNS_RESOLVER") ? zeekenv("ZEEK_DNS_RESOLVER") : "not set, will use first IPv4 address from /etc/resolv.conf"); + fprintf(stderr, " $ZEEK_DEBUG_LOG_STDERR | Use stderr for debug logs generated via the -B flag"); + + fprintf(stderr, "\n"); + + exit(code); + } + +zeek::Options zeek::parse_cmdline(int argc, char** argv) + { + zeek::Options rval; + + // When running unit tests, the first argument on the command line must be + // --test, followed by doctest options. Optionally, users can use "--" as + // separator to pass Zeek options afterwards: + // + // zeek --test [doctest-options] -- [zeek-options] + + // Just locally filtering out the args for Zeek usage from doctest args. + std::vector zeek_args; + + if ( argc > 1 && strcmp(argv[1], "--test") == 0 ) + { + #ifdef DOCTEST_CONFIG_DISABLE + fprintf(stderr, "ERROR: C++ unit tests are disabled for this build.\n" + " Please re-compile with ENABLE_ZEEK_UNIT_TESTS " + "to run the C++ unit tests.\n"); + usage(argv[0], 1); + #endif + + auto is_separator = [](const char* cstr) + { + return strcmp(cstr, "--") == 0; + }; + auto first = argv; + auto last = argv + argc; + auto separator = std::find_if(first, last, is_separator); + zeek_args.emplace_back(argv[0]); + + if ( separator != last ) + { + auto first_zeek_arg = std::next(separator); + + for ( auto i = first_zeek_arg; i != last; ++i ) + zeek_args.emplace_back(*i); + } + + rval.run_unit_tests = true; + + for ( auto i = 0; i < std::distance(first, separator); ++i ) + rval.doctest_args.emplace_back(argv[i]); + } + else + { + for ( auto i = 0; i < argc; ++i ) + zeek_args.emplace_back(argv[i]); + } + + constexpr struct option long_opts[] = { + {"parse-only", no_argument, 0, 'a'}, + {"bare-mode", no_argument, 0, 'b'}, + {"debug-script", no_argument, 0, 'd'}, + {"exec", required_argument, 0, 'e'}, + {"filter", required_argument, 0, 'f'}, + {"help", no_argument, 0, 'h'}, + {"iface", required_argument, 0, 'i'}, + {"zeekygen", required_argument, 0, 'X'}, + {"prefix", required_argument, 0, 'p'}, + {"readfile", required_argument, 0, 'r'}, + {"rulefile", required_argument, 0, 's'}, + {"tracefile", required_argument, 0, 't'}, + {"writefile", required_argument, 0, 'w'}, + {"version", no_argument, 0, 'v'}, + {"no-checksums", no_argument, 0, 'C'}, + {"force-dns", no_argument, 0, 'F'}, + {"load-seeds", required_argument, 0, 'G'}, + {"save-seeds", required_argument, 0, 'H'}, + {"print-plugins", no_argument, 0, 'N'}, + {"prime-dns", no_argument, 0, 'P'}, + {"time", no_argument, 0, 'Q'}, + {"debug-rules", no_argument, 0, 'S'}, + {"re-level", required_argument, 0, 'T'}, + {"watchdog", no_argument, 0, 'W'}, + {"print-id", required_argument, 0, 'I'}, + {"status-file", required_argument, 0, 'U'}, + +#ifdef DEBUG + {"debug", required_argument, 0, 'B'}, +#endif +#ifdef USE_IDMEF + {"idmef-dtd", required_argument, 0, 'n'}, +#endif +#ifdef USE_PERFTOOLS_DEBUG + {"mem-leaks", no_argument, 0, 'm'}, + {"mem-profile", no_argument, 0, 'M'}, +#endif + + {"pseudo-realtime", optional_argument, 0, 'E'}, + {"jobs", optional_argument, 0, 'j'}, + {"test", no_argument, 0, '#'}, + + {0, 0, 0, 0}, + }; + + char opts[256]; + safe_strncpy(opts, "B:e:f:G:H:I:i:j::n:p:r:s:T:t:U:w:X:CFNPQSWabdhv", + sizeof(opts)); + +#ifdef USE_PERFTOOLS_DEBUG + strncat(opts, "mM", 2); +#endif + + int op; + int long_optsind; + opterr = 0; + + // getopt may permute the array, so need yet another array + auto zargs = std::make_unique(zeek_args.size()); + + for ( auto i = 0u; i < zeek_args.size(); ++i ) + zargs[i] = zeek_args[i].data(); + + while ( (op = getopt_long(zeek_args.size(), zargs.get(), opts, long_opts, &long_optsind)) != EOF ) + switch ( op ) { + case 'a': + rval.parse_only = true; + break; + case 'b': + rval.bare_mode = true; + break; + case 'd': + rval.debug_scripts = true; + break; + case 'e': + rval.script_code_to_exec = optarg; + break; + case 'f': + rval.pcap_filter = optarg; + break; + case 'h': + rval.print_usage = true; + break; + case 'i': + if ( ! rval.pcap_files.empty() ) + { + fprintf(stderr, "Using -i is not allowed when reading pcap files"); + exit(1); + } + rval.interfaces.emplace_back(optarg); + break; + case 'j': + rval.supervisor_mode = true; + if ( optarg ) + { + // TODO: for supervised offline pcap reading, the argument is + // expected to be number of workers like "-j 4" or possibly a + // list of worker/proxy/logger counts like "-j 4,2,1" + } + break; + case 'p': + rval.script_prefixes.emplace_back(optarg); + break; + case 'r': + if ( ! rval.interfaces.empty() ) + { + fprintf(stderr, "Using -r is not allowed when reading a live interface"); + exit(1); + } + rval.pcap_files.emplace_back(optarg); + break; + case 's': + rval.signature_files.emplace_back(optarg); + break; + case 't': + rval.debug_script_tracing_file = optarg; + break; + case 'v': + rval.print_version = true; + break; + case 'w': + rval.pcap_output_file = optarg; + break; + case 'B': + rval.debug_log_streams = optarg; + break; + case 'C': + rval.ignore_checksums = true; + break; + case 'E': + rval.pseudo_realtime = 1.0; + if ( optarg ) + rval.pseudo_realtime = atof(optarg); + break; + case 'F': + if ( rval.dns_mode != DNS_DEFAULT ) + usage(zargs[0], 1); + rval.dns_mode = DNS_FORCE; + break; + case 'G': + rval.random_seed_input_file = optarg; + break; + case 'H': + rval.random_seed_output_file = optarg; + break; + case 'I': + rval.identifier_to_print = optarg; + break; + case 'N': + ++rval.print_plugins; + break; + case 'P': + if ( rval.dns_mode != DNS_DEFAULT ) + usage(zargs[0], 1); + rval.dns_mode = DNS_PRIME; + break; + case 'Q': + rval.print_execution_time = true; + break; + case 'S': + rval.print_signature_debug_info = true; + break; + case 'T': + rval.signature_re_level = atoi(optarg); + break; + case 'U': + rval.process_status_file = optarg; + break; + case 'W': + rval.use_watchdog = true; + break; + case 'X': + rval.zeekygen_config_file = optarg; + break; + +#ifdef USE_PERFTOOLS_DEBUG + case 'm': + rval.perftools_check_leaks = 1; + break; + case 'M': + rval.perftools_profile = 1; + break; +#endif + +#ifdef USE_IDMEF + case 'n': + rval.libidmef_dtd_path = optarg; + break; +#endif + + case '#': + fprintf(stderr, "ERROR: --test only allowed as first argument.\n"); + usage(zargs[0], 1); + break; + + case 0: + // This happens for long options that don't have + // a short-option equivalent. + break; + + case '?': + default: + usage(zargs[0], 1); + break; + } + + // Process remaining arguments. X=Y arguments indicate script + // variable/parameter assignments. X::Y arguments indicate plugins to + // activate/query. The remainder are treated as scripts to load. + while ( optind < static_cast(zeek_args.size()) ) + { + if ( strchr(zargs[optind], '=') ) + rval.script_options_to_set.emplace_back(zargs[optind++]); + else if ( strstr(zargs[optind], "::") ) + rval.plugins_to_load.emplace(zargs[optind++]); + else + rval.scripts_to_load.emplace_back(zargs[optind++]); + } + + auto canonify_script_path = [](std::string* path) + { + if ( path->empty() ) + return; + + *path = normalize_path(*path); + + if ( (*path)[0] == '/' || (*path)[0] == '~' ) + // Absolute path + return; + + if ( (*path)[0] != '.' ) + { + // Look up file in ZEEKPATH + auto res = find_script_file(*path, bro_path()); + + if ( res.empty() ) + { + fprintf(stderr, "failed to locate script: %s\n", path->data()); + exit(1); + } + + *path = res; + + if ( (*path)[0] == '/' || (*path)[0] == '~' ) + // Now an absolute path + return; + } + + // Need to translate relative path to absolute. + char cwd[PATH_MAX]; + + if ( ! getcwd(cwd, sizeof(cwd)) ) + { + fprintf(stderr, "failed to get current directory: %s\n", + strerror(errno)); + exit(1); + } + + *path = std::string(cwd) + "/" + *path; + }; + + if ( rval.supervisor_mode ) + { + // Translate any relative paths supplied to supervisor into absolute + // paths for use by supervised nodes since they have the option to + // operate out of a different working directory. + for ( auto& s : rval.scripts_to_load ) + canonify_script_path(&s); + } + + return rval; + } diff --git a/src/Options.h b/src/Options.h new file mode 100644 index 0000000000..01711481d9 --- /dev/null +++ b/src/Options.h @@ -0,0 +1,97 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include +#include +#include + +#include "DNS_Mgr.h" + +namespace zeek { + +/** + * Options that define general Zeek processing behavior, usually determined + * from command-line arguments. + */ +struct Options { + /** + * Unset options that aren't meant to be used by the supervisor, but may + * make sense for supervised nodes to inherit (as opposed to flagging + * as an error an exiting outright if used in supervisor-mode). + */ + void filter_supervisor_options(); + + /** + * Inherit certain options set in the original supervisor parent process + * and discard the rest. + */ + void filter_supervised_node_options(); + + bool print_version = false; + bool print_usage = false; + bool print_execution_time = false; + bool print_signature_debug_info = false; + int print_plugins = 0; + + std::optional debug_log_streams; + std::optional debug_script_tracing_file; + + std::optional identifier_to_print; + std::optional script_code_to_exec; + std::vector script_prefixes = { "" }; // "" = "no prefix" + + int signature_re_level = 4; + bool ignore_checksums = false; + bool use_watchdog = false; + double pseudo_realtime = 0; + DNS_MgrMode dns_mode = DNS_DEFAULT; + + bool supervisor_mode = false; + bool parse_only = false; + bool bare_mode = false; + bool debug_scripts = false; + bool perftools_check_leaks = false; + bool perftools_profile = false; + + bool run_unit_tests = false; + std::vector doctest_args; + + std::optional pcap_filter; + std::vector interfaces; + std::vector pcap_files; + std::vector signature_files; + + std::optional pcap_output_file; + std::optional random_seed_input_file; + std::optional random_seed_output_file; + std::optional process_status_file; + std::optional zeekygen_config_file; + std::string libidmef_dtd_file = "idmef-message.dtd"; + + std::set plugins_to_load; + std::vector scripts_to_load; + std::vector script_options_to_set; +}; + +/** + * Parse Zeek command-line arguments. + * @param argc argument count (same semantics as arguments to main()) + * @param argv argument strings (same semantics as arguments to main()) + * @return the parsed command-line options + */ +zeek::Options parse_cmdline(int argc, char** argv); + +/** + * Print command-line Zeek usage information and exit. + * @param prog the name/path of the Zeek command-line invocation + * @code the exit code to use + */ +void usage(const char* prog, int code = 1); + +/** + * @return true if zeek is running a "fake" DNS resolver, else false. + */ +bool fake_dns(); + +} // namespace zeek diff --git a/src/Pipe.cc b/src/Pipe.cc index 0fa0eefdd7..5a0916a3dd 100644 --- a/src/Pipe.cc +++ b/src/Pipe.cc @@ -13,24 +13,68 @@ static void pipe_fail(int eno) { char tmp[256]; bro_strerror_r(eno, tmp, sizeof(tmp)); - reporter->FatalError("Pipe failure: %s", tmp); + + if ( reporter ) + reporter->FatalError("Pipe failure: %s", tmp); + else + fprintf(stderr, "Pipe failure: %s", tmp); } -static void set_flags(int fd, int flags) +static int set_flags(int fd, int flags) { + auto rval = fcntl(fd, F_GETFD); + + if ( rval == -1 ) + pipe_fail(errno); + if ( flags ) - if ( fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | flags) == -1 ) + { + rval |= flags; + + if ( fcntl(fd, F_SETFD, rval) == -1 ) pipe_fail(errno); + } + + return rval; } -static void set_status_flags(int fd, int flags) +static int unset_flags(int fd, int flags) { + auto rval = fcntl(fd, F_GETFD); + + if ( rval == -1 ) + pipe_fail(errno); + if ( flags ) - if ( fcntl(fd, F_SETFL, fcntl(fd, F_GETFL) | flags) == -1 ) + { + rval &= ~flags; + + if ( fcntl(fd, F_SETFD, rval) == -1 ) pipe_fail(errno); + } + + return rval; } -static int dup_or_fail(int fd, int flags) +static int set_status_flags(int fd, int flags) + { + auto rval = fcntl(fd, F_GETFL); + + if ( rval == -1 ) + pipe_fail(errno); + + if ( flags ) + { + rval |= flags; + + if ( fcntl(fd, F_SETFL, rval) == -1 ) + pipe_fail(errno); + } + + return rval; + } + +static int dup_or_fail(int fd, int flags, int status_flags) { int rval = dup(fd); @@ -38,22 +82,41 @@ static int dup_or_fail(int fd, int flags) pipe_fail(errno); set_flags(fd, flags); + set_status_flags(fd, status_flags); return rval; } -Pipe::Pipe(int flags0, int flags1, int status_flags0, int status_flags1) +Pipe::Pipe(int flags0, int flags1, int status_flags0, int status_flags1, + int* arg_fds) { - // pipe2 can set flags atomically, but not yet available everywhere. - if ( ::pipe(fds) ) - pipe_fail(errno); + if ( arg_fds ) + { + fds[0] = arg_fds[0]; + fds[1] = arg_fds[1]; + } + else + { + // pipe2 can set flags atomically, but not yet available everywhere. + if ( ::pipe(fds) ) + pipe_fail(errno); + } - flags[0] = flags0; - flags[1] = flags1; + flags[0] = set_flags(fds[0], flags0); + flags[1] = set_flags(fds[1], flags1); + status_flags[0] = set_status_flags(fds[0], status_flags0); + status_flags[1] = set_status_flags(fds[1], status_flags1); + } - set_flags(fds[0], flags[0]); - set_flags(fds[1], flags[1]); - set_status_flags(fds[0], status_flags0); - set_status_flags(fds[1], status_flags1); +void Pipe::SetFlags(int arg_flags) + { + flags[0] = set_flags(fds[0], arg_flags); + flags[1] = set_flags(fds[1], arg_flags); + } + +void Pipe::UnsetFlags(int arg_flags) + { + flags[0] = unset_flags(fds[0], arg_flags); + flags[1] = unset_flags(fds[1], arg_flags); } Pipe::~Pipe() @@ -64,10 +127,12 @@ Pipe::~Pipe() Pipe::Pipe(const Pipe& other) { - fds[0] = dup_or_fail(other.fds[0], other.flags[0]); - fds[1] = dup_or_fail(other.fds[1], other.flags[1]); + fds[0] = dup_or_fail(other.fds[0], other.flags[0], other.status_flags[0]); + fds[1] = dup_or_fail(other.fds[1], other.flags[1], other.status_flags[1]); flags[0] = other.flags[0]; flags[1] = other.flags[1]; + status_flags[0] = other.status_flags[0]; + status_flags[1] = other.status_flags[1]; } Pipe& Pipe::operator=(const Pipe& other) @@ -77,9 +142,17 @@ Pipe& Pipe::operator=(const Pipe& other) close(fds[0]); close(fds[1]); - fds[0] = dup_or_fail(other.fds[0], other.flags[0]); - fds[1] = dup_or_fail(other.fds[1], other.flags[1]); + fds[0] = dup_or_fail(other.fds[0], other.flags[0], other.status_flags[0]); + fds[1] = dup_or_fail(other.fds[1], other.flags[1], other.status_flags[1]); flags[0] = other.flags[0]; flags[1] = other.flags[1]; + status_flags[0] = other.status_flags[0]; + status_flags[1] = other.status_flags[1]; return *this; } + +PipePair::PipePair(int flags, int status_flags, int* fds) + : pipes{Pipe(flags, flags, status_flags, status_flags, fds ? fds + 0 : nullptr), + Pipe(flags, flags, status_flags, status_flags, fds ? fds + 2 : nullptr)} + { + } diff --git a/src/Pipe.h b/src/Pipe.h index eed32bac01..e310c83edc 100644 --- a/src/Pipe.h +++ b/src/Pipe.h @@ -13,9 +13,12 @@ public: * @param flags1 file descriptor flags to set on write end of pipe. * @param status_flags0 descriptor status flags to set on read end of pipe. * @param status_flags1 descriptor status flags to set on write end of pipe. + * @param fds may be supplied to open an existing file descriptors rather + * than create ones from a new pipe. Should point to memory containing + * two consecutive file descriptors, the "read" one and then the "write" one. */ explicit Pipe(int flags0 = 0, int flags1 = 0, int status_flags0 = 0, - int status_flags1 = 0); + int status_flags1 = 0, int* fds = nullptr); /** * Close the pair of file descriptors owned by the object. @@ -45,9 +48,92 @@ public: int WriteFD() const { return fds[1]; } + /** + * Sets the given file descriptor flags for both the read and write end + * of the pipe. + */ + void SetFlags(int flags); + + /** + * Unsets the given file descriptor flags for both the read and write end + * of the pipe. + */ + void UnsetFlags(int flags); + private: int fds[2]; int flags[2]; + int status_flags[2]; +}; + +/** + * A pair of pipes that can be used for bi-directinoal IPC. + */ +class PipePair { +public: + + /** + * Create a pair of pipes + * @param flags file descriptor flags to set on pipes + * @status_flags descriptor status flags to set on pipes + * @fds may be supplied to open existing file descriptors rather + * than create ones from a new pair of pipes. Should point to memory + * containing four consecutive file descriptors, "read" end and "write" end + * of the first pipe followed by the "read" end and "write" end of the + * second pipe. + */ + PipePair(int flags, int status_flags, int* fds = nullptr); + + /** + * @return the pipe used for receiving input + */ + Pipe& In() + { return pipes[swapped]; } + + /** + * @return the pipe used for sending output + */ + Pipe& Out() + { return pipes[!swapped]; } + + /** + * @return the pipe used for receiving input + */ + const Pipe& In() const + { return pipes[swapped]; } + + /** + * @return the pipe used for sending output + */ + const Pipe& Out() const + { return pipes[!swapped]; } + + /** + * @return a file descriptor that may used for receiving messages by + * polling/reading it. + */ + int InFD() const + { return In().ReadFD(); } + + /** + * @return a file descriptor that may be used for sending messages by + * writing to it. + */ + int OutFD() const + { return Out().WriteFD(); } + + /** + * Swaps the meaning of the pipes in the pair. E.g. call this after + * fork()'ing so that the child process uses the right pipe for + * reading/writing. + */ + void Swap() + { swapped = ! swapped; } + +private: + + Pipe pipes[2]; + bool swapped = false; }; } // namespace bro diff --git a/src/RuleMatcher.cc b/src/RuleMatcher.cc index 7200ae2ad7..747b4ce16e 100644 --- a/src/RuleMatcher.cc +++ b/src/RuleMatcher.cc @@ -231,7 +231,7 @@ void RuleMatcher::Delete(RuleHdrTest* node) delete node; } -bool RuleMatcher::ReadFiles(const name_list& files) +bool RuleMatcher::ReadFiles(const std::vector& files) { #ifdef USE_PERFTOOLS_DEBUG HeapLeakChecker::Disabler disabler; @@ -239,18 +239,18 @@ bool RuleMatcher::ReadFiles(const name_list& files) parse_error = false; - for ( int i = 0; i < files.length(); ++i ) + for ( const auto& f : files ) { - rules_in = open_file(find_file(files[i], bro_path(), ".sig")); + rules_in = open_file(find_file(f, bro_path(), ".sig")); if ( ! rules_in ) { - reporter->Error("Can't open signature file %s", files[i]); + reporter->Error("Can't open signature file %s", f.data()); return false; } rules_line_number = 0; - current_rule_file = files[i]; + current_rule_file = f.data(); rules_parse(); fclose(rules_in); } diff --git a/src/RuleMatcher.h b/src/RuleMatcher.h index 21c65d2700..5804fe63d8 100644 --- a/src/RuleMatcher.h +++ b/src/RuleMatcher.h @@ -221,7 +221,7 @@ public: ~RuleMatcher(); // Parse the given files and built up data structures. - bool ReadFiles(const name_list& files); + bool ReadFiles(const std::vector& files); /** * Inititialize a state object for matching file magic signatures. diff --git a/src/Scope.cc b/src/Scope.cc index 662a86fda7..d4a87bc658 100644 --- a/src/Scope.cc +++ b/src/Scope.cc @@ -59,7 +59,7 @@ Scope::~Scope() ID* Scope::GenerateTemporary(const char* name) { - return new ID(copy_string(name), SCOPE_FUNCTION, false); + return new ID(name, SCOPE_FUNCTION, false); } id_list* Scope::GetInits() diff --git a/src/Timer.cc b/src/Timer.cc index 1138deec79..40ac0696f4 100644 --- a/src/Timer.cc +++ b/src/Timer.cc @@ -37,6 +37,7 @@ const char* TimerNames[] = { "TCPConnectionPartialClose", "TCPConnectionResetTimer", "TriggerTimer", + "ParentProcessIDCheck", "TimerMgrExpireTimer", }; diff --git a/src/Timer.h b/src/Timer.h index a251e5a183..7c135932b6 100644 --- a/src/Timer.h +++ b/src/Timer.h @@ -41,6 +41,7 @@ enum TimerType : uint8_t { TIMER_TCP_PARTIAL_CLOSE, TIMER_TCP_RESET, TIMER_TRIGGER, + TIMER_PPID_CHECK, TIMER_TIMERMGR_EXPIRE, }; const int NUM_TIMER_TYPES = int(TIMER_TIMERMGR_EXPIRE) + 1; diff --git a/src/Type.cc b/src/Type.cc index 43f5da6267..6a4139bc8b 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -19,7 +19,7 @@ BroType::TypeAliasMap BroType::type_aliases; // Note: This function must be thread-safe. const char* type_name(TypeTag t) { - static const char* type_names[int(NUM_TYPES)] = { + static constexpr const char* type_names[int(NUM_TYPES)] = { "void", // 0 "bool", // 1 "int", // 2 diff --git a/src/Val.cc b/src/Val.cc index c5948d981c..3d512e6057 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -719,16 +719,6 @@ void IntervalVal::ValDescribe(ODesc* d) const } } -PortVal* PortManager::Get(uint32_t port_num) const - { - return val_mgr->GetPort(port_num); - } - -PortVal* PortManager::Get(uint32_t port_num, TransportProto port_type) const - { - return val_mgr->GetPort(port_num, port_type); - } - uint32_t PortVal::Mask(uint32_t port_num, TransportProto port_type) { // Note, for ICMP one-way connections: @@ -760,25 +750,8 @@ uint32_t PortVal::Mask(uint32_t port_num, TransportProto port_type) return port_num; } -PortVal::PortVal(uint32_t p, TransportProto port_type) : Val(TYPE_PORT) - { - auto port_num = PortVal::Mask(p, port_type); - val.uint_val = static_cast(port_num); - } - -PortVal::PortVal(uint32_t p, bool unused) : Val(TYPE_PORT) - { - val.uint_val = static_cast(p); - } - PortVal::PortVal(uint32_t p) : Val(TYPE_PORT) { - if ( p >= 65536 * NUM_PORT_SPACES ) - { - InternalWarning("bad port number"); - p = 0; - } - val.uint_val = static_cast(p); } @@ -3305,7 +3278,7 @@ ValManager::ValManager() auto port_type = (TransportProto)i; for ( auto j = 0u; j < arr.size(); ++j ) - arr[j] = new PortVal(PortVal::Mask(j, port_type), true); + arr[j] = new PortVal(PortVal::Mask(j, port_type)); } } diff --git a/src/Val.h b/src/Val.h index 9124df2f74..f1ac3c1936 100644 --- a/src/Val.h +++ b/src/Val.h @@ -85,56 +85,6 @@ typedef union { class Val : public BroObj { public: - ZEEK_DEPRECATED("Remove in v3.1: use val_mgr->GetBool, GetFalse/GetTrue, GetInt, or GetCount instead") - Val(bool b, TypeTag t) - { - val.int_val = b; - type = base_type(t); -#ifdef DEBUG - bound_id = 0; -#endif - } - - ZEEK_DEPRECATED("Remove in v3.1: use val_mgr->GetBool, GetFalse/GetTrue, GetInt, or GetCount instead") - Val(int32_t i, TypeTag t) - { - val.int_val = bro_int_t(i); - type = base_type(t); -#ifdef DEBUG - bound_id = 0; -#endif - } - - ZEEK_DEPRECATED("Remove in v3.1: use val_mgr->GetBool, GetFalse/GetTrue, GetInt, or GetCount instead") - Val(uint32_t u, TypeTag t) - { - val.uint_val = bro_uint_t(u); - type = base_type(t); -#ifdef DEBUG - bound_id = 0; -#endif - } - - ZEEK_DEPRECATED("Remove in v3.1: use val_mgr->GetBool, GetFalse/GetTrue, GetInt, or GetCount instead") - Val(int64_t i, TypeTag t) - { - val.int_val = i; - type = base_type(t); -#ifdef DEBUG - bound_id = 0; -#endif - } - - ZEEK_DEPRECATED("Remove in v3.1: use val_mgr->GetBool, GetFalse/GetTrue, GetInt, or GetCount instead") - Val(uint64_t u, TypeTag t) - { - val.uint_val = u; - type = base_type(t); -#ifdef DEBUG - bound_id = 0; -#endif - } - Val(double d, TypeTag t) { val.double_val = d; @@ -429,23 +379,6 @@ protected: }; -class PortManager { -public: - // Port number given in host order. - ZEEK_DEPRECATED("Remove in v3.1: use val_mgr->GetPort() instead") - PortVal* Get(uint32_t port_num, TransportProto port_type) const; - - // Host-order port number already masked with port space protocol mask. - ZEEK_DEPRECATED("Remove in v3.1: use val_mgr->GetPort() instead") - PortVal* Get(uint32_t port_num) const; - - // Returns a masked port number - ZEEK_DEPRECATED("Remove in v3.1: use PortVal::Mask() instead") - uint32_t Mask(uint32_t port_num, TransportProto port_type) const; -}; - -extern PortManager* port_mgr; - // Holds pre-allocated Val objects for those where it's more optimal to // re-use existing ones rather than allocate anew. class ValManager { @@ -521,14 +454,6 @@ protected: class PortVal : public Val { public: - // Port number given in host order. - ZEEK_DEPRECATED("Remove in v3.1: use val_mgr->GetPort() instead") - PortVal(uint32_t p, TransportProto port_type); - - // Host-order port number already masked with port space protocol mask. - ZEEK_DEPRECATED("Remove in v3.1: use val_mgr->GetPort() instead") - explicit PortVal(uint32_t p); - Val* SizeVal() const override { return val_mgr->GetInt(val.uint_val); } // Returns the port number in host order (not including the mask). @@ -559,7 +484,7 @@ protected: friend class Val; friend class ValManager; PortVal() {} - PortVal(uint32_t p, bool unused); + PortVal(uint32_t p); void ValDescribe(ODesc* d) const override; Val* DoClone(CloneState* state) override; @@ -1014,13 +939,6 @@ protected: class EnumVal : public Val { public: - - ZEEK_DEPRECATED("Remove in v3.1: use t->GetVal(i) instead") - EnumVal(int i, EnumType* t) : Val(t) - { - val.int_val = i; - } - Val* SizeVal() const override { return val_mgr->GetInt(val.int_val); } protected: diff --git a/src/analyzer/protocol/ftp/FTP.cc b/src/analyzer/protocol/ftp/FTP.cc index 97ce54c481..da86ad7db4 100644 --- a/src/analyzer/protocol/ftp/FTP.cc +++ b/src/analyzer/protocol/ftp/FTP.cc @@ -83,7 +83,7 @@ void FTP_Analyzer::DeliverStream(int length, const u_char* data, bool orig) StringVal* cmd_str; line = skip_whitespace(line, end_of_line); - get_word(length, line, cmd_len, cmd); + get_word(end_of_line - line, line, cmd_len, cmd); line = skip_whitespace(line + cmd_len, end_of_line); if ( cmd_len == 0 ) diff --git a/src/analyzer/protocol/rfb/events.bif b/src/analyzer/protocol/rfb/events.bif index 44afb5b043..dd790d9a20 100644 --- a/src/analyzer/protocol/rfb/events.bif +++ b/src/analyzer/protocol/rfb/events.bif @@ -1,8 +1,3 @@ -## Generated for RFB event -## -## c: The connection record for the underlying transport-layer session/flow. -event rfb_event%(c: connection%) &deprecated="Remove in v3.1: This event never served a real purpose and will be removed. Please use the other rfb events instead."; - ## Generated for RFB event authentication mechanism selection ## ## c: The connection record for the underlying transport-layer session/flow. diff --git a/src/analyzer/protocol/rfb/rfb-analyzer.pac b/src/analyzer/protocol/rfb/rfb-analyzer.pac index ee0c4657bc..4568271453 100644 --- a/src/analyzer/protocol/rfb/rfb-analyzer.pac +++ b/src/analyzer/protocol/rfb/rfb-analyzer.pac @@ -1,11 +1,4 @@ refine flow RFB_Flow += { - function proc_rfb_message(msg: RFB_PDU): bool - %{ - if ( rfb_event ) - BifEvent::generate_rfb_event(connection()->bro_analyzer(), connection()->bro_analyzer()->Conn()); - return true; - %} - function proc_rfb_version(client: bool, major: bytestring, minor: bytestring) : bool %{ if (client) @@ -371,7 +364,3 @@ refine connection RFB_Conn += { return true; %} }; - -refine typeattr RFB_PDU += &let { - proc: bool = $context.flow.proc_rfb_message(this); -}; diff --git a/src/analyzer/protocol/smb/smb1-com-transaction.pac b/src/analyzer/protocol/smb/smb1-com-transaction.pac index 414066c6ba..0c2f128794 100644 --- a/src/analyzer/protocol/smb/smb1-com-transaction.pac +++ b/src/analyzer/protocol/smb/smb1-com-transaction.pac @@ -4,10 +4,29 @@ enum Trans_subcommands { NT_TRANSACT_CREATE2 = 0x0009, }; +%code{ + StringVal* SMB_Conn::transaction_data_to_val(SMB1_transaction_data* payload) + { + switch ( payload->trans_type() ) { + case SMB_PIPE: + return bytestring_to_val(payload->pipe_data()); + case SMB_UNKNOWN: + return bytestring_to_val(payload->unknown()); + default: + return bytestring_to_val(payload->data()); + } + + assert(false); + return val_mgr->GetEmptyString(); + } +%} + refine connection SMB_Conn += { %member{ map is_file_a_pipe; + + static StringVal* transaction_data_to_val(SMB1_transaction_data* payload); %} function get_is_file_a_pipe(id: uint16): bool @@ -37,32 +56,11 @@ refine connection SMB_Conn += { StringVal* parameters = new StringVal(${val.parameters}.length(), (const char*)${val.parameters}.data()); StringVal* payload_str = nullptr; - SMB1_transaction_data* payload = nullptr; if ( ${val.data_count} > 0 ) - { - payload = ${val.data}; - } - - if ( payload ) - { - switch ( payload->trans_type() ) { - case SMB_PIPE: - payload_str = new StringVal(${val.data_count}, (const char*)${val.data.pipe_data}.data()); - break; - case SMB_UNKNOWN: - payload_str = new StringVal(${val.data_count}, (const char*)${val.data.unknown}.data()); - break; - default: - payload_str = new StringVal(${val.data_count}, (const char*)${val.data.data}.data()); - break; - } - } - - if ( ! payload_str ) - { + payload_str = transaction_data_to_val(${val.data}); + else payload_str = val_mgr->GetEmptyString(); - } BifEvent::generate_smb1_transaction_request(bro_analyzer(), bro_analyzer()->Conn(), @@ -83,32 +81,11 @@ refine connection SMB_Conn += { StringVal* parameters = new StringVal(${val.parameters}.length(), (const char*)${val.parameters}.data()); StringVal* payload_str = nullptr; - SMB1_transaction_data* payload = nullptr; if ( ${val.data_count} > 0 ) - { - payload = ${val.data[0]}; - } - - if ( payload ) - { - switch ( payload->trans_type() ) { - case SMB_PIPE: - payload_str = new StringVal(${val.data_count}, (const char*)${val.data[0].pipe_data}.data()); - break; - case SMB_UNKNOWN: - payload_str = new StringVal(${val.data_count}, (const char*)${val.data[0].unknown}.data()); - break; - default: - payload_str = new StringVal(${val.data_count}, (const char*)${val.data[0].data}.data()); - break; - } - } - - if ( ! payload_str ) - { + payload_str = transaction_data_to_val(${val.data[0]}); + else payload_str = val_mgr->GetEmptyString(); - } BifEvent::generate_smb1_transaction_response(bro_analyzer(), bro_analyzer()->Conn(), diff --git a/src/analyzer/protocol/ssl/proc-certificate.pac b/src/analyzer/protocol/ssl/proc-certificate.pac index 7c57f31744..d7b2802d0a 100644 --- a/src/analyzer/protocol/ssl/proc-certificate.pac +++ b/src/analyzer/protocol/ssl/proc-certificate.pac @@ -16,6 +16,12 @@ { const bytestring& cert = (*certificates)[i]; + if ( cert.length() <= 0 ) + { + reporter->Weird(bro_analyzer()->Conn(), "zero_length_certificate"); + continue; + } + ODesc file_handle; file_handle.Add(common.Description()); file_handle.Add(i); diff --git a/src/analyzer/protocol/ssl/tls-handshake-analyzer.pac b/src/analyzer/protocol/ssl/tls-handshake-analyzer.pac index 5e8e31e0b2..583ae33f30 100644 --- a/src/analyzer/protocol/ssl/tls-handshake-analyzer.pac +++ b/src/analyzer/protocol/ssl/tls-handshake-analyzer.pac @@ -303,7 +303,7 @@ refine connection Handshake_Conn += { common.AddRaw("F"); bro_analyzer()->Conn()->IDString(&common); - if ( status_type == 1 ) // ocsp + if ( status_type == 1 && response.length() > 0 ) // ocsp { ODesc file_handle; file_handle.Add(common.Description()); @@ -323,6 +323,10 @@ refine connection Handshake_Conn += { file_mgr->EndOfFile(file_id); } + else if ( response.length() == 0 ) + { + reporter->Weird(bro_analyzer()->Conn(), "SSL_zero_length_stapled_OCSP_message"); + } return true; %} diff --git a/src/event.bif b/src/event.bif index 9fa5855359..d1ddf29d52 100644 --- a/src/event.bif +++ b/src/event.bif @@ -48,9 +48,6 @@ ## event zeek_init%(%); -## Deprecated synonym for :zeek:see:`zeek_init`. -event bro_init%(%) &deprecated="Remove in v3.1: use zeek_init"; - ## Generated at Zeek termination time. The event engine generates this event when ## Zeek is about to terminate, either due to having exhausted reading its input ## trace file(s), receiving a termination signal, or because Zeek was run without @@ -64,9 +61,6 @@ event bro_init%(%) &deprecated="Remove in v3.1: use zeek_init"; ## is not generated. event zeek_done%(%); -## Deprecated synonym for :zeek:see:`zeek_done`. -event bro_done%(%) &deprecated="Remove in v3.1: use zeek_done"; - ## Generated for every new connection. This event is raised with the first ## packet of a previously unknown connection. Zeek uses a flow-based definition ## of "connection" here that includes not only TCP sessions but also UDP and @@ -666,9 +660,6 @@ event reporter_error%(t: time, msg: string, location: string%) &error_handler; ## recursively for each ``@load``. event zeek_script_loaded%(path: string, level: count%); -## Deprecated synonym for :zeek:see:`zeek_script_loaded`. -event bro_script_loaded%(path: string, level: count%) &deprecated="Remove in v3.1: use zeek_script_loaded"; - ## Generated each time Zeek's script interpreter opens a file. This event is ## triggered only for files opened via :zeek:id:`open`, and in particular not for ## normal log files as created by log writers. diff --git a/src/input.h b/src/input.h index 327423ed9c..70aa75ae28 100644 --- a/src/input.h +++ b/src/input.h @@ -37,8 +37,8 @@ extern int bro_argc; extern char** bro_argv; extern const char* prog; -extern name_list prefixes; // -p flag -extern char* command_line_policy; // -e flag +extern std::vector zeek_script_prefixes; // -p flag +extern const char* command_line_policy; // -e flag extern std::vector params; class Stmt; diff --git a/src/main.cc b/src/main.cc index 45dbbec983..9847aa2df2 100644 --- a/src/main.cc +++ b/src/main.cc @@ -7,10 +7,9 @@ #include #include #include +#include #include -#ifdef HAVE_GETOPT_H -#include -#endif +#include #ifdef USE_IDMEF extern "C" { @@ -21,7 +20,7 @@ extern "C" { #include #include -#include "bsd-getopt-long.h" +#include "Options.h" #include "input.h" #include "DNS_Mgr.h" #include "Frame.h" @@ -43,10 +42,10 @@ extern "C" { #include "Brofiler.h" #include "Traverse.h" +#include "supervisor/Supervisor.h" #include "threading/Manager.h" #include "input/Manager.h" #include "logging/Manager.h" -#include "logging/writers/ascii/Ascii.h" #include "input/readers/raw/Raw.h" #include "analyzer/Manager.h" #include "analyzer/Tag.h" @@ -84,7 +83,6 @@ int perftools_profile = 0; DNS_Mgr* dns_mgr; TimerMgr* timer_mgr; ValManager* val_mgr = 0; -PortManager* port_mgr = 0; logging::Manager* log_mgr = 0; threading::Manager* thread_mgr = 0; input::Manager* input_mgr = 0; @@ -94,10 +92,9 @@ file_analysis::Manager* file_mgr = 0; zeekygen::Manager* zeekygen_mgr = 0; iosource::Manager* iosource_mgr = 0; bro_broker::Manager* broker_mgr = 0; +zeek::Supervisor* zeek::supervisor_mgr = 0; -const char* prog; -char* writefile = 0; -name_list prefixes; +std::vector zeek_script_prefixes; Stmt* stmts; EventHandlerPtr net_done = 0; RuleMatcher* rule_matcher = 0; @@ -107,10 +104,10 @@ ProfileLogger* segment_logger = 0; SampleLogger* sample_logger = 0; int signal_val = 0; extern char version[]; -char* command_line_policy = 0; +const char* command_line_policy = 0; vector params; set requested_plugins; -char* proc_status_file = 0; +const char* proc_status_file = 0; OpaqueType* md5_type = 0; OpaqueType* sha1_type = 0; @@ -145,74 +142,15 @@ const char* zeek_version() #endif } -bool bro_dns_fake() +static std::vector to_cargs(const std::vector& args) { - return zeekenv("ZEEK_DNS_FAKE"); - } + std::vector rval; + rval.reserve(args.size()); -void usage(int code = 1) - { - fprintf(stderr, "zeek version %s\n", zeek_version()); + for ( const auto& arg : args ) + rval.emplace_back(arg.data()); - fprintf(stderr, "usage: %s [options] [file ...]\n", prog); - fprintf(stderr, "usage: %s --test [doctest-options] -- [options] [file ...]\n", prog); - - fprintf(stderr, " | policy file, or read stdin\n"); - fprintf(stderr, " -a|--parse-only | exit immediately after parsing scripts\n"); - fprintf(stderr, " -b|--bare-mode | don't load scripts from the base/ directory\n"); - fprintf(stderr, " -d|--debug-policy | activate policy file debugging\n"); - fprintf(stderr, " -e|--exec | augment loaded policies by given code\n"); - fprintf(stderr, " -f|--filter | tcpdump filter\n"); - fprintf(stderr, " -h|--help | command line help\n"); - fprintf(stderr, " -i|--iface | read from given interface\n"); - fprintf(stderr, " -p|--prefix | add given prefix to policy file resolution\n"); - fprintf(stderr, " -r|--readfile | read from given tcpdump file\n"); - fprintf(stderr, " -s|--rulefile | read rules from given file\n"); - fprintf(stderr, " -t|--tracefile | activate execution tracing\n"); - fprintf(stderr, " -v|--version | print version and exit\n"); - fprintf(stderr, " -w|--writefile | write to given tcpdump file\n"); -#ifdef DEBUG - fprintf(stderr, " -B|--debug | Enable debugging output for selected streams ('-B help' for help)\n"); -#endif - fprintf(stderr, " -C|--no-checksums | ignore checksums\n"); - fprintf(stderr, " -F|--force-dns | force DNS\n"); - fprintf(stderr, " -G|--load-seeds | load seeds from given file\n"); - fprintf(stderr, " -H|--save-seeds | save seeds to given file\n"); - fprintf(stderr, " -I|--print-id | print out given ID\n"); - fprintf(stderr, " -N|--print-plugins | print available plugins and exit (-NN for verbose)\n"); - fprintf(stderr, " -P|--prime-dns | prime DNS\n"); - fprintf(stderr, " -Q|--time | print execution time summary to stderr\n"); - fprintf(stderr, " -S|--debug-rules | enable rule debugging\n"); - fprintf(stderr, " -T|--re-level | set 'RE_level' for rules\n"); - fprintf(stderr, " -U|--status-file | Record process status in file\n"); - fprintf(stderr, " -W|--watchdog | activate watchdog timer\n"); - fprintf(stderr, " -X|--zeekygen | generate documentation based on config file\n"); - -#ifdef USE_PERFTOOLS_DEBUG - fprintf(stderr, " -m|--mem-leaks | show leaks [perftools]\n"); - fprintf(stderr, " -M|--mem-profile | record heap [perftools]\n"); -#endif - fprintf(stderr, " --pseudo-realtime[=] | enable pseudo-realtime for performance evaluation (default 1)\n"); - -#ifdef USE_IDMEF - fprintf(stderr, " -n|--idmef-dtd | specify path to IDMEF DTD file\n"); -#endif - - fprintf(stderr, " --test | run unit tests ('--test -h' for help, only when compiling with ENABLE_ZEEK_UNIT_TESTS)\n"); - fprintf(stderr, " $ZEEKPATH | file search path (%s)\n", bro_path().c_str()); - fprintf(stderr, " $ZEEK_PLUGIN_PATH | plugin search path (%s)\n", bro_plugin_path()); - fprintf(stderr, " $ZEEK_PLUGIN_ACTIVATE | plugins to always activate (%s)\n", bro_plugin_activate()); - fprintf(stderr, " $ZEEK_PREFIXES | prefix list (%s)\n", bro_prefixes().c_str()); - fprintf(stderr, " $ZEEK_DNS_FAKE | disable DNS lookups (%s)\n", bro_dns_fake() ? "on" : "off"); - fprintf(stderr, " $ZEEK_SEED_FILE | file to load seeds from (not set)\n"); - fprintf(stderr, " $ZEEK_LOG_SUFFIX | ASCII log file extension (.%s)\n", logging::writer::Ascii::LogExt().c_str()); - fprintf(stderr, " $ZEEK_PROFILER_FILE | Output file for script execution statistics (not set)\n"); - fprintf(stderr, " $ZEEK_DISABLE_ZEEKYGEN | Disable Zeekygen documentation support (%s)\n", zeekenv("ZEEK_DISABLE_ZEEKYGEN") ? "set" : "not set"); - fprintf(stderr, " $ZEEK_DNS_RESOLVER | IPv4/IPv6 address of DNS resolver to use (%s)\n", zeekenv("ZEEK_DNS_RESOLVER") ? zeekenv("ZEEK_DNS_RESOLVER") : "not set, will use first IPv4 address from /etc/resolv.conf"); - - fprintf(stderr, "\n"); - - exit(code); + return rval; } bool show_plugins(int level) @@ -359,21 +297,21 @@ void terminate_bro() delete analyzer_mgr; delete file_mgr; // broker_mgr is deleted via iosource_mgr + // supervisor is deleted via iosource_mgr delete iosource_mgr; delete log_mgr; delete reporter; delete plugin_mgr; delete val_mgr; - delete port_mgr; reporter = 0; } -void termination_signal() +void zeek_terminate_loop(const char* reason) { - set_processing_status("TERMINATING", "termination_signal"); + set_processing_status("TERMINATING", reason); + reporter->Info("%s", reason); - reporter->Info("received termination signal"); net_get_final_stats(); done_with_network(); net_delete(); @@ -407,327 +345,141 @@ static void bro_new_handler() out_of_memory("new"); } +static std::vector get_script_signature_files() + { + std::vector rval; + + // Parse rule files defined on the script level. + char* script_signature_files = + copy_string(internal_val("signature_files")->AsString()->CheckString()); + + char* tmp = script_signature_files; + char* s; + while ( (s = strsep(&tmp, " \t")) ) + if ( *s ) + rval.emplace_back(s); + + delete [] script_signature_files; + return rval; + } + +static std::string get_exe_path(const std::string& invocation) + { + if ( invocation.empty() ) + return ""; + + if ( invocation[0] == '/' || invocation[0] == '~' ) + // Absolute path + return invocation; + + if ( invocation.find('/') != std::string::npos ) + { + // Relative path + char cwd[PATH_MAX]; + + if ( ! getcwd(cwd, sizeof(cwd)) ) + { + fprintf(stderr, "failed to get current directory: %s\n", + strerror(errno)); + exit(1); + } + + return std::string(cwd) + "/" + invocation; + } + + auto path = getenv("PATH"); + + if ( ! path ) + return ""; + + return find_file(invocation, path); + } + int main(int argc, char** argv) { ZEEK_LSAN_DISABLE(); std::set_new_handler(bro_new_handler); - // When running unit tests, the first argument on the command line must be - // --test, followed by doctest options. Optionally, users can use "--" as - // separator to pass Zeek options afterwards: - // - // zeek --test [doctest-options] -- [zeek-options] + auto zeek_exe_path = get_exe_path(argv[0]); - if ( argc > 1 && strcmp(argv[1], "--test") == 0 ) + if ( zeek_exe_path.empty() ) { - // Deal with CLI arguments (copy Zeek part over to bro_argv). - auto is_separator = [](const char* cstr) - { - return strcmp(cstr, "--") == 0; - }; - auto first = argv; - auto last = argv + argc; - auto separator = std::find_if(first, last, is_separator); + fprintf(stderr, "failed to get path to executable '%s'", argv[0]); + exit(1); + } - if ( separator == last ) - { - bro_argc = 1; - bro_argv = new char*[1]; - bro_argv[0] = copy_string(argv[0]); - } - else - { - auto first_zeek_arg = std::next(separator); - bro_argc = 1 + std::distance(first_zeek_arg, last); - bro_argv = new char*[bro_argc]; - bro_argv[0] = copy_string(argv[0]); - auto bro_argv_iter = std::next(bro_argv); - for ( auto i = first_zeek_arg; i != last; ++i ) - *bro_argv_iter++ = copy_string(*i); - } + bro_argc = argc; + bro_argv = new char* [argc]; -#ifdef DOCTEST_CONFIG_DISABLE - fprintf(stderr, "ERROR: C++ unit tests are disabled for this build.\n" - " Please re-compile with ENABLE_ZEEK_UNIT_TESTS " - "to run the C++ unit tests.\n"); - return EXIT_FAILURE; -#else + for ( int i = 0; i < argc; i++ ) + bro_argv[i] = copy_string(argv[i]); + + auto options = zeek::parse_cmdline(argc, argv); + + if ( options.print_usage ) + zeek::usage(argv[0], 0); + + if ( options.print_version ) + { + fprintf(stdout, "%s version %s\n", argv[0], zeek_version()); + exit(0); + } + + if ( options.run_unit_tests ) + { doctest::Context context; - context.applyCommandLine(std::distance(first, separator), argv); + auto dargs = to_cargs(options.doctest_args); + context.applyCommandLine(dargs.size(), dargs.data()); ZEEK_LSAN_ENABLE(); return context.run(); -#endif } - else - { - bro_argc = argc; - bro_argv = new char* [argc]; - for ( int i = 0; i < argc; i++ ) - bro_argv[i] = copy_string(argv[i]); - } + auto stem_state = zeek::Supervisor::CreateStem(options.supervisor_mode); + + if ( zeek::Supervisor::ThisNode() ) + zeek::Supervisor::ThisNode()->Init(&options); double time_start = current_time(true); brofiler.ReadStats(); - name_list interfaces; - name_list read_files; - name_list rule_files; - char* id_name = 0; + auto dns_type = options.dns_mode; - char* seed_load_file = zeekenv("ZEEK_SEED_FILE"); - char* seed_save_file = 0; - char* user_pcap_filter = 0; - char* debug_streams = 0; - int parse_only = false; - int bare_mode = false; - int do_watchdog = 0; - int override_ignore_checksums = 0; - int rule_debug = 0; - int RE_level = 4; - int print_plugins = 0; - int time_bro = 0; - - static struct option long_opts[] = { - {"parse-only", no_argument, 0, 'a'}, - {"bare-mode", no_argument, 0, 'b'}, - {"debug-policy", no_argument, 0, 'd'}, - {"exec", required_argument, 0, 'e'}, - {"filter", required_argument, 0, 'f'}, - {"help", no_argument, 0, 'h'}, - {"iface", required_argument, 0, 'i'}, - {"zeekygen", required_argument, 0, 'X'}, - {"prefix", required_argument, 0, 'p'}, - {"readfile", required_argument, 0, 'r'}, - {"rulefile", required_argument, 0, 's'}, - {"tracefile", required_argument, 0, 't'}, - {"writefile", required_argument, 0, 'w'}, - {"version", no_argument, 0, 'v'}, - {"no-checksums", no_argument, 0, 'C'}, - {"force-dns", no_argument, 0, 'F'}, - {"load-seeds", required_argument, 0, 'G'}, - {"save-seeds", required_argument, 0, 'H'}, - {"print-plugins", no_argument, 0, 'N'}, - {"prime-dns", no_argument, 0, 'P'}, - {"time", no_argument, 0, 'Q'}, - {"debug-rules", no_argument, 0, 'S'}, - {"re-level", required_argument, 0, 'T'}, - {"watchdog", no_argument, 0, 'W'}, - {"print-id", required_argument, 0, 'I'}, - {"status-file", required_argument, 0, 'U'}, - -#ifdef DEBUG - {"debug", required_argument, 0, 'B'}, -#endif -#ifdef USE_IDMEF - {"idmef-dtd", required_argument, 0, 'n'}, -#endif -#ifdef USE_PERFTOOLS_DEBUG - {"mem-leaks", no_argument, 0, 'm'}, - {"mem-profile", no_argument, 0, 'M'}, -#endif - - {"pseudo-realtime", optional_argument, 0, 'E'}, - {"test", no_argument, 0, '#'}, - - {0, 0, 0, 0}, - }; - - enum DNS_MgrMode dns_type = DNS_DEFAULT; - - dns_type = bro_dns_fake() ? DNS_FAKE : DNS_DEFAULT; + if ( dns_type == DNS_DEFAULT && zeek::fake_dns() ) + dns_type = DNS_FAKE; RETSIGTYPE (*oldhandler)(int); - prog = argv[0]; + zeek_script_prefixes = options.script_prefixes; + auto zeek_prefixes = zeekenv("ZEEK_PREFIXES"); - prefixes.push_back(strdup("")); // "" = "no prefix" + if ( zeek_prefixes ) + tokenize_string(zeek_prefixes, ":", &zeek_script_prefixes); - char* p = zeekenv("ZEEK_PREFIXES"); - - if ( p ) - add_to_name_list(p, ':', prefixes); - - string zeekygen_config; - -#ifdef USE_IDMEF - string libidmef_dtd_path = "idmef-message.dtd"; -#endif - - extern char* optarg; - extern int optind, opterr; - - int long_optsind; - opterr = 0; - - char opts[256]; - safe_strncpy(opts, "B:e:f:G:H:I:i:n:p:r:s:T:t:U:w:X:CFNPQSWabdhv", - sizeof(opts)); + pseudo_realtime = options.pseudo_realtime; #ifdef USE_PERFTOOLS_DEBUG - strncat(opts, "mM", 2); + perftools_leaks = options.perftools_check_leaks; + perftools_profile = options.perftools_profile; #endif - int op; - while ( (op = getopt_long(bro_argc, bro_argv, opts, long_opts, &long_optsind)) != EOF ) - switch ( op ) { - case 'a': - parse_only = true; - break; - - case 'b': - bare_mode = true; - break; - - case 'd': - fprintf(stderr, "Policy file debugging ON.\n"); - g_policy_debug = true; - break; - - case 'e': - command_line_policy = optarg; - break; - - case 'f': - user_pcap_filter = optarg; - break; - - case 'h': - usage(0); - break; - - case 'i': - interfaces.push_back(optarg); - break; - - case 'p': - prefixes.push_back(optarg); - break; - - case 'r': - read_files.push_back(optarg); - break; - - case 's': - rule_files.push_back(optarg); - break; - - case 't': - g_trace_state.SetTraceFile(optarg); - g_trace_state.TraceOn(); - break; - - case 'v': - fprintf(stdout, "%s version %s\n", prog, zeek_version()); - exit(0); - break; - - case 'w': - writefile = optarg; - break; - - case 'B': - debug_streams = optarg; - break; - - case 'C': - override_ignore_checksums = 1; - break; - - case 'E': - pseudo_realtime = 1.0; - if ( optarg ) - pseudo_realtime = atof(optarg); - break; - - case 'F': - if ( dns_type != DNS_DEFAULT ) - usage(1); - dns_type = DNS_FORCE; - break; - - case 'G': - seed_load_file = optarg; - break; - - case 'H': - seed_save_file = optarg; - break; - - case 'I': - id_name = optarg; - break; - - case 'N': - ++print_plugins; - break; - - case 'P': - if ( dns_type != DNS_DEFAULT ) - usage(1); - dns_type = DNS_PRIME; - break; - - case 'Q': - time_bro = 1; - break; - - case 'S': - rule_debug = 1; - break; - - case 'T': - RE_level = atoi(optarg); - break; - - case 'U': - proc_status_file = optarg; - break; - - case 'W': - do_watchdog = 1; - break; - - case 'X': - zeekygen_config = optarg; - break; - -#ifdef USE_PERFTOOLS_DEBUG - case 'm': - perftools_leaks = 1; - break; - - case 'M': - perftools_profile = 1; - break; -#endif - -#ifdef USE_IDMEF - case 'n': - fprintf(stderr, "Using IDMEF XML DTD from %s\n", optarg); - libidmef_dtd_path = optarg; - break; -#endif - - case '#': - fprintf(stderr, "ERROR: --test only allowed as first argument.\n"); - usage(1); - break; - - case 0: - // This happens for long options that don't have - // a short-option equivalent. - break; - - case '?': - default: - usage(1); - break; + if ( options.debug_scripts ) + { + g_policy_debug = options.debug_scripts; + fprintf(stderr, "Zeek script debugging ON.\n"); } - if ( interfaces.length() > 0 && read_files.length() > 0 ) - usage(1); + if ( options.script_code_to_exec ) + command_line_policy = options.script_code_to_exec->data(); + + if ( options.debug_script_tracing_file ) + { + g_trace_state.SetTraceFile(options.debug_script_tracing_file->data()); + g_trace_state.TraceOn(); + } + + if ( options.process_status_file ) + proc_status_file = options.process_status_file->data(); atexit(atexit_handler); set_processing_status("INITIALIZING", "main"); @@ -735,20 +487,38 @@ int main(int argc, char** argv) bro_start_time = current_time(true); val_mgr = new ValManager(); - port_mgr = new PortManager(); reporter = new Reporter(); thread_mgr = new threading::Manager(); plugin_mgr = new plugin::Manager(); #ifdef DEBUG - if ( debug_streams ) + if ( options.debug_log_streams ) { - debug_logger.EnableStreams(debug_streams); - debug_logger.OpenDebugLog("debug"); + debug_logger.EnableStreams(options.debug_log_streams->data()); + + if ( getenv("ZEEK_DEBUG_LOG_STDERR") ) + debug_logger.OpenDebugLog(nullptr); + else + debug_logger.OpenDebugLog("debug"); } #endif - init_random_seed((seed_load_file && *seed_load_file ? seed_load_file : 0) , seed_save_file); + if ( options.supervisor_mode ) + { + zeek::Supervisor::Config cfg = {}; + cfg.zeek_exe_path = zeek_exe_path; + options.filter_supervisor_options(); + zeek::supervisor_mgr = new zeek::Supervisor(std::move(cfg), + std::move(*stem_state)); + } + + const char* seed_load_file = zeekenv("ZEEK_SEED_FILE"); + + if ( options.random_seed_input_file ) + seed_load_file = options.random_seed_input_file->data(); + + init_random_seed((seed_load_file && *seed_load_file ? seed_load_file : 0), + options.random_seed_output_file ? options.random_seed_output_file->data() : 0); // DEBUG_MSG("HMAC key: %s\n", md5_digest_print(shared_hmac_md5_key)); init_hash_function(); @@ -767,9 +537,9 @@ int main(int argc, char** argv) reporter->Error("Failed to initialize sqlite3: %s", sqlite3_errstr(r)); #ifdef USE_IDMEF - char* libidmef_dtd_path_cstr = new char[libidmef_dtd_path.length() + 1]; - safe_strncpy(libidmef_dtd_path_cstr, libidmef_dtd_path.c_str(), - libidmef_dtd_path.length()); + char* libidmef_dtd_path_cstr = new char[options.libidmef_dtd_file.size() + 1]; + safe_strncpy(libidmef_dtd_path_cstr, options.libidmef_dtd_file.data(), + options.libidmef_dtd_file.size()); globalsInit(libidmef_dtd_path_cstr); // Init LIBIDMEF globals createCurrentDoc("1.0"); // Set a global XML document #endif @@ -777,34 +547,34 @@ int main(int argc, char** argv) timer_mgr = new PQ_TimerMgr(""); // timer_mgr = new CQ_TimerMgr(); - zeekygen_mgr = new zeekygen::Manager(zeekygen_config, bro_argv[0]); + auto zeekygen_cfg = options.zeekygen_config_file.value_or(""); + zeekygen_mgr = new zeekygen::Manager(zeekygen_cfg, bro_argv[0]); add_essential_input_file("base/init-bare.zeek"); add_essential_input_file("base/init-frameworks-and-bifs.zeek"); - if ( ! bare_mode ) + if ( ! options.bare_mode ) add_input_file("base/init-default.zeek"); plugin_mgr->SearchDynamicPlugins(bro_plugin_path()); - if ( optind == bro_argc && - read_files.length() == 0 && - interfaces.length() == 0 && - ! id_name && ! command_line_policy && ! print_plugins ) + if ( options.plugins_to_load.empty() && options.scripts_to_load.empty() && + options.script_options_to_set.empty() && + options.pcap_files.size() == 0 && + options.interfaces.size() == 0 && + ! options.identifier_to_print && + ! command_line_policy && ! options.print_plugins && + ! options.supervisor_mode && ! zeek::Supervisor::ThisNode() ) add_input_file("-"); - // Process remaining arguments. X=Y arguments indicate script - // variable/parameter assignments. X::Y arguments indicate plugins to - // activate/query. The remainder are treated as scripts to load. - while ( optind < bro_argc ) - { - if ( strchr(bro_argv[optind], '=') ) - params.push_back(bro_argv[optind++]); - else if ( strstr(bro_argv[optind], "::") ) - requested_plugins.insert(bro_argv[optind++]); - else - add_input_file(bro_argv[optind++]); - } + for ( const auto& script_option : options.script_options_to_set ) + params.push_back(script_option); + + for ( const auto& plugin : options.plugins_to_load ) + requested_plugins.insert(plugin); + + for ( const auto& script : options.scripts_to_load ) + add_input_file(script.data()); push_scope(nullptr, nullptr); @@ -821,7 +591,7 @@ int main(int argc, char** argv) log_mgr = new logging::Manager(); input_mgr = new input::Manager(); file_mgr = new file_analysis::Manager(); - broker_mgr = new bro_broker::Manager(read_files.length() > 0); + broker_mgr = new bro_broker::Manager(! options.pcap_files.empty()); plugin_mgr->InitPreScript(); analyzer_mgr->InitPreScript(); @@ -840,7 +610,7 @@ int main(int argc, char** argv) if ( missing_plugin ) reporter->FatalError("Failed to activate requested dynamic plugin(s)."); - plugin_mgr->ActivateDynamicPlugins(! bare_mode); + plugin_mgr->ActivateDynamicPlugins(! options.bare_mode); init_event_handlers(); @@ -896,9 +666,9 @@ int main(int argc, char** argv) zeekygen_mgr->InitPostScript(); broker_mgr->InitPostScript(); - if ( print_plugins ) + if ( options.print_plugins ) { - bool success = show_plugins(print_plugins); + bool success = show_plugins(options.print_plugins); exit(success ? 0 : 1); } @@ -906,7 +676,7 @@ int main(int argc, char** argv) file_mgr->InitPostScript(); dns_mgr->InitPostScript(); - if ( parse_only ) + if ( options.parse_only ) { int rc = (reporter->Errors() > 0 ? 1 : 0); exit(rc); @@ -925,52 +695,48 @@ int main(int argc, char** argv) reporter->InitOptions(); zeekygen_mgr->GenerateDocs(); - if ( user_pcap_filter ) + if ( options.pcap_filter ) { ID* id = global_scope()->Lookup("cmd_line_bpf_filter"); if ( ! id ) reporter->InternalError("global cmd_line_bpf_filter not defined"); - id->SetVal(new StringVal(user_pcap_filter)); + id->SetVal(new StringVal(*options.pcap_filter)); } - // Parse rule files defined on the script level. - char* script_rule_files = - copy_string(internal_val("signature_files")->AsString()->CheckString()); + auto all_signature_files = options.signature_files; - char* tmp = script_rule_files; - char* s; - while ( (s = strsep(&tmp, " \t")) ) - if ( *s ) - rule_files.push_back(s); + // Append signature files defined in "signature_files" script option + for ( auto&& sf : get_script_signature_files() ) + all_signature_files.emplace_back(std::move(sf)); // Append signature files defined in @load-sigs - for ( size_t i = 0; i < sig_files.size(); ++i ) - rule_files.push_back(copy_string(sig_files[i].c_str())); + for ( const auto& sf : sig_files ) + all_signature_files.emplace_back(sf); - if ( rule_files.length() > 0 ) + if ( ! all_signature_files.empty() ) { - rule_matcher = new RuleMatcher(RE_level); - if ( ! rule_matcher->ReadFiles(rule_files) ) + rule_matcher = new RuleMatcher(options.signature_re_level); + if ( ! rule_matcher->ReadFiles(all_signature_files) ) { delete dns_mgr; exit(1); } - if ( rule_debug ) + if ( options.print_signature_debug_info ) rule_matcher->PrintDebug(); file_mgr->InitMagic(); } - delete [] script_rule_files; - if ( g_policy_debug ) // ### Add support for debug command file. dbg_init_debugger(0); - if ( read_files.length() == 0 && interfaces.length() == 0 ) + auto all_interfaces = options.interfaces; + + if ( options.pcap_files.empty() && options.interfaces.empty() ) { Val* interfaces_val = internal_val("interfaces"); if ( interfaces_val ) @@ -979,14 +745,15 @@ int main(int argc, char** argv) interfaces_val->AsString()->Render(); if ( interfaces_str[0] != '\0' ) - add_to_name_list(interfaces_str, ' ', interfaces); + tokenize_string(interfaces_str, " ", &all_interfaces); delete [] interfaces_str; } } if ( dns_type != DNS_PRIME ) - net_init(interfaces, read_files, writefile, do_watchdog); + net_init(all_interfaces, options.pcap_files, + options.pcap_output_file, options.use_watchdog); net_done = internal_handler("net_done"); @@ -1015,11 +782,11 @@ int main(int argc, char** argv) } // Print the ID. - if ( id_name ) + if ( options.identifier_to_print ) { - ID* id = global_scope()->Lookup(id_name); + ID* id = global_scope()->Lookup(*options.identifier_to_print); if ( ! id ) - reporter->FatalError("No such ID: %s\n", id_name); + reporter->FatalError("No such ID: %s\n", options.identifier_to_print->data()); ODesc desc; desc.SetQuotes(true); @@ -1082,7 +849,7 @@ int main(int argc, char** argv) g_frame_stack.pop_back(); } - if ( override_ignore_checksums ) + if ( options.ignore_checksums ) ignore_checksums = 1; if ( zeek_script_loaded ) @@ -1116,6 +883,9 @@ int main(int argc, char** argv) iosource_mgr->Register(thread_mgr, true); + if ( zeek::supervisor_mgr ) + iosource_mgr->Register(zeek::supervisor_mgr); + if ( iosource_mgr->Size() > 0 || have_pending_timers || BifConst::exit_only_after_terminate ) @@ -1135,12 +905,15 @@ int main(int argc, char** argv) #endif + if ( zeek::Supervisor::ThisNode() ) + timer_mgr->Add(new zeek::ParentProcessCheckTimer(1, 1)); + double time_net_start = current_time(true);; uint64_t mem_net_start_total; uint64_t mem_net_start_malloced; - if ( time_bro ) + if ( options.print_execution_time ) { get_memory_usage(&mem_net_start_total, &mem_net_start_malloced); @@ -1158,7 +931,7 @@ int main(int argc, char** argv) uint64_t mem_net_done_total; uint64_t mem_net_done_malloced; - if ( time_bro ) + if ( options.print_execution_time ) { get_memory_usage(&mem_net_done_total, &mem_net_done_malloced); diff --git a/src/nb_dns.c b/src/nb_dns.c index f8d939b4ab..106740bb61 100644 --- a/src/nb_dns.c +++ b/src/nb_dns.c @@ -411,7 +411,7 @@ _nb_dns_mkquery(register struct nb_dns_info *nd, register const char *name, return (-1); } memset(ne, 0, sizeof(*ne)); - strncpy(ne->name, name, sizeof(ne->name)); + strncpy(ne->name, name, sizeof(ne->name) - 1); ne->name[sizeof(ne->name) - 1] = '\0'; ne->qtype = qtype; ne->atype = atype; diff --git a/src/parse.y b/src/parse.y index ff65c4b184..1df96397ad 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1164,14 +1164,12 @@ func_hdr: } | TOK_EVENT event_id func_params opt_attr { - // Gracefully handle the deprecation of bro_init, bro_done, - // and bro_script_loaded - if ( streq("bro_init", $2->Name()) ) - $2 = global_scope()->Lookup("zeek_init"); - else if ( streq("bro_done", $2->Name()) ) - $2 = global_scope()->Lookup("zeek_done"); - else if ( streq("bro_script_loaded", $2->Name()) ) - $2 = global_scope()->Lookup("zeek_script_loaded"); + const char* name = $2->Name(); + if ( streq("bro_init", name) || streq("bro_done", name) || streq("bro_script_loaded", name) ) + { + auto base = std::string(name).substr(4); + reporter->Error(fmt("event %s() is no longer available, use zeek_%s() instead", name, base.c_str())); + } begin_func($2, current_module.c_str(), FUNC_FLAVOR_EVENT, 0, $3, $4); diff --git a/src/plugin/Manager.cc b/src/plugin/Manager.cc index 58dc52e033..51b85bdbb5 100644 --- a/src/plugin/Manager.cc +++ b/src/plugin/Manager.cc @@ -189,6 +189,7 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_ if ( is_file(init) ) { DBG_LOG(DBG_PLUGINS, " Loading %s", init.c_str()); + warn_if_legacy_script(init); scripts_to_load.push_back(init); break; } @@ -202,6 +203,7 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_ if ( is_file(init) ) { DBG_LOG(DBG_PLUGINS, " Loading %s", init.c_str()); + warn_if_legacy_script(init); scripts_to_load.push_back(init); break; } @@ -214,6 +216,7 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_ if ( is_file(init) ) { DBG_LOG(DBG_PLUGINS, " Loading %s", init.c_str()); + warn_if_legacy_script(init); scripts_to_load.push_back(init); break; } diff --git a/src/scan.l b/src/scan.l index d8792a1470..9549bf9b5d 100644 --- a/src/scan.l +++ b/src/scan.l @@ -435,10 +435,9 @@ when return TOK_WHEN; pref = skip_whitespace(pref + 1); // Skip over '='. if ( ! append ) - while ( prefixes.length() > 1 ) // don't delete "" prefix - delete prefixes.remove_nth(1); + zeek_script_prefixes = { "" }; // don't delete the "" prefix - add_to_name_list(pref, ':', prefixes); + tokenize_string(pref, ":", &zeek_script_prefixes); } @if return TOK_ATIF; @@ -943,14 +942,14 @@ int yywrap() it->prefixes_checked = true; // Prefixes are pushed onto a stack, so iterate backwards. - for ( int i = prefixes.length() - 1; i >= 0; --i ) + for ( int i = zeek_script_prefixes.size() - 1; i >= 0; --i ) { // Don't look at empty prefixes. - if ( ! prefixes[i][0] ) + if ( ! zeek_script_prefixes[i][0] ) continue; string canon = without_bropath_component(it->name); - string flat = flatten_script_name(canon, prefixes[i]); + string flat = flatten_script_name(canon, zeek_script_prefixes[i]); string path = find_relative_script_file(flat); if ( ! path.empty() ) diff --git a/src/strings.bif b/src/strings.bif index 4f589bd078..ae1f8292d1 100644 --- a/src/strings.bif +++ b/src/strings.bif @@ -935,47 +935,6 @@ function safe_shell_quote%(source: string%): string return new StringVal(new BroString(1, dst, j)); %} -## Takes a string and escapes characters that would allow execution of -## commands at the shell level. Must be used before including strings in -## :zeek:id:`system` or similar calls. This function is deprecated, use -## :zeek:see:`safe_shell_quote` as a replacement. The difference is that -## :zeek:see:`safe_shell_quote` automatically returns a value that is -## wrapped in double-quotes, which is required to correctly and fully -## escape any characters that might be interpreted by the shell. -## -## source: The string to escape. -## -## Returns: A shell-escaped version of *source*. -## -## .. zeek:see:: system safe_shell_quote -function str_shell_escape%(source: string%): string &deprecated="Remove in v3.1: use safe_shell_quote" - %{ - unsigned j = 0; - const u_char* src = source->Bytes(); - unsigned n = source->Len(); - byte_vec dst = new u_char[n * 2 + 1]; - - for ( unsigned i = 0; i < n; ++i ) - { - switch ( src[i] ) { - case '`': case '"': case '\\': case '$': - - // case '|': case '&': case ';': case '(': case ')': case '<': - // case '>': case '\'': case '*': case '?': case '[': case ']': - // case '!': case '#': case '{': case '}': - dst[j++] = '\\'; - break; - default: - break; - } - - dst[j++] = src[i]; - } - - dst[j] = '\0'; - return new StringVal(new BroString(1, dst, j)); - %} - ## Finds all occurrences of a pattern in a string. ## ## str: The string to inspect. diff --git a/src/supervisor/Supervisor.cc b/src/supervisor/Supervisor.cc new file mode 100644 index 0000000000..4d41383a85 --- /dev/null +++ b/src/supervisor/Supervisor.cc @@ -0,0 +1,1413 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "Supervisor.h" +#include "Reporter.h" +#include "DebugLogger.h" +#include "Val.h" +#include "Net.h" +#include "NetVar.h" +#include "zeek-config.h" +#include "util.h" +#include "zeek-affinity.h" + +#define RAPIDJSON_HAS_STDSTRING 1 +#include "3rdparty/rapidjson/include/rapidjson/document.h" + +extern "C" { +#include "setsignal.h" +} + +#ifdef DEBUG +#define DBG_STEM(args...) stem->LogDebug(args); +#else +#define DBG_STEM +#endif + +using namespace zeek; + +std::optional Supervisor::supervised_node; + +namespace { +struct Stem { + Stem(Supervisor::StemState stem_state); + + ~Stem(); + + Supervisor::SupervisedNode Run(); + + std::optional Poll(); + + std::optional Revive(); + + void Reap(); + + std::optional Spawn(Supervisor::Node* node); + + int AliveNodeCount() const; + + void KillNodes(int signal); + + void KillNode(Supervisor::Node* node, int signal) const; + + void Destroy(Supervisor::Node* node) const; + + bool Wait(Supervisor::Node* node, int options) const; + + void Shutdown(int exit_code); + + void ReportStatus(const Supervisor::Node& node) const; + + void Log(std::string_view type, const char* format, va_list args) const; + + void LogDebug(const char* format, ...) const __attribute__((format(printf, 2, 3))); + + void LogError(const char* format, ...) const __attribute__((format(printf, 2, 3))); + + pid_t parent_pid; + int last_signal = -1; + std::unique_ptr signal_flare; + std::unique_ptr pipe; + std::map nodes; + std::string msg_buffer; + bool shutting_down = false; +}; +} + +static Stem* stem = nullptr; + +static RETSIGTYPE stem_signal_handler(int signo) + { + stem->last_signal = signo; + + if ( stem->shutting_down ) + return RETSIGVAL; + + stem->signal_flare->Fire(true); + + if ( signo == SIGTERM ) + stem->shutting_down = true; + + return RETSIGVAL; + } + +static RETSIGTYPE supervisor_signal_handler(int signo) + { + supervisor_mgr->ObserveChildSignal(signo); + return RETSIGVAL; + } + +static std::vector extract_messages(std::string* buffer) + { + std::vector rval; + + for ( ; ; ) + { + auto msg_end = buffer->find('\0'); + + if ( msg_end == std::string::npos ) + // Don't have any full messages left + break; + + auto msg = buffer->substr(0, msg_end); + rval.emplace_back(std::move(msg)); + buffer->erase(0, msg_end + 1); + } + + return rval; + } + +static std::string make_create_message(const Supervisor::NodeConfig& node) + { + auto json_str = node.ToJSON(); + return fmt("create %s %s", node.name.data(), json_str.data()); + } + +ParentProcessCheckTimer::ParentProcessCheckTimer(double t, double arg_interval) + : Timer(t, TIMER_PPID_CHECK), interval(arg_interval) + { + } + +void ParentProcessCheckTimer::Dispatch(double t, int is_expire) + { + // Note: only simple + portable way of detecting loss of parent + // process seems to be polling for change in PPID. There's platform + // specific ways if we do end up needing something more responsive + // and/or have to avoid overhead of polling, but maybe not worth + // the additional complexity: + // Linux: prctl(PR_SET_PDEATHSIG, ...) + // FreeBSD: procctl(PROC_PDEATHSIG_CTL) + // Also note the Stem process has its own polling loop with similar logic. + if ( zeek::Supervisor::ThisNode()->parent_pid != getppid() ) + zeek_terminate_loop("supervised node was orphaned"); + + if ( ! is_expire ) + timer_mgr->Add(new ParentProcessCheckTimer(network_time + interval, + interval)); + } + +Supervisor::Supervisor(Supervisor::Config cfg, StemState ss) + : config(std::move(cfg)), stem_pid(ss.pid), stem_pipe(std::move(ss.pipe)) + { + DBG_LOG(DBG_SUPERVISOR, "forked stem process %d", stem_pid); + setsignal(SIGCHLD, supervisor_signal_handler); + SetIdle(true); + + int status; + auto res = waitpid(stem_pid, &status, WNOHANG); + + if ( res == 0 ) + // Good, stem process is alive and the SIGCHLD handler will keep it so. + return; + + if ( res == -1 ) + fprintf(stderr, "Supervisor failed to get status of stem process: %s\n", + strerror(errno)); + else + { + if ( WIFEXITED(status) ) + fprintf(stderr, "Supervisor stem died early with exit code %d\n", + WEXITSTATUS(status)); + else if ( WIFSIGNALED(status) ) + fprintf(stderr, "Supervisor stem died early by signal %d\n", + WTERMSIG(status)); + else + fprintf(stderr, "Supervisor stem died early for unknown reason\n"); + } + + exit(1); + } + +Supervisor::~Supervisor() + { + setsignal(SIGCHLD, SIG_DFL); + + if ( ! stem_pid ) + { + DBG_LOG(DBG_SUPERVISOR, "shutdown, stem process already exited"); + return; + } + + DBG_LOG(DBG_SUPERVISOR, "shutdown, killing stem process %d", stem_pid); + + auto kill_res = kill(stem_pid, SIGTERM); + + if ( kill_res == -1 ) + { + char tmp[256]; + bro_strerror_r(errno, tmp, sizeof(tmp)); + reporter->Error("Failed to send SIGTERM to stem process: %s", tmp); + } + else + { + int status; + auto wait_res = waitpid(stem_pid, &status, 0); + + if ( wait_res == -1 ) + { + char tmp[256]; + bro_strerror_r(errno, tmp, sizeof(tmp)); + reporter->Error("Failed to wait for stem process to exit: %s", tmp); + } + } + + while ( ProcessMessages() != 0 ); + } + +void Supervisor::ObserveChildSignal(int signo) + { + last_signal = signo; + signal_flare.Fire(true); + } + +void Supervisor::ReapStem() + { + if ( ! stem_pid ) + return; + + int status; + auto res = waitpid(stem_pid, &status, WNOHANG); + + if ( res == 0 ) + // Still alive + return; + + if ( res == -1 ) + { + char tmp[256]; + bro_strerror_r(errno, tmp, sizeof(tmp)); + reporter->Error("Supervisor failed to get exit status" + " of stem process: %s", tmp); + return; + } + + stem_pid = 0; + + if ( WIFEXITED(status) ) + { + DBG_LOG(DBG_SUPERVISOR, "stem process exited with status %d", + WEXITSTATUS(status)); + } + else if ( WIFSIGNALED(status) ) + { + DBG_LOG(DBG_SUPERVISOR, "stem process terminated by signal %d", + WTERMSIG(status)); + } + else + reporter->Error("Supervisor failed to get exit status" + " of stem process for unknown reason"); + } + +void Supervisor::HandleChildSignal() + { + if ( last_signal >= 0 ) + { + DBG_LOG(DBG_SUPERVISOR, "Supervisor received signal %d", last_signal); + last_signal = -1; + } + + bool had_child_signal = signal_flare.Extinguish(); + + if ( had_child_signal ) + { + ReapStem(); + + DBG_LOG(DBG_SUPERVISOR, "Supervisor processed child signal %s", + stem_pid ? "(spurious)" : ""); + } + + if ( stem_pid ) + return; + + // Revive the Stem process + auto stem_ppid = getpid(); + stem_pid = fork(); + + if ( stem_pid == -1 ) + { + stem_pid = 0; + char tmp[256]; + bro_strerror_r(errno, tmp, sizeof(tmp)); + reporter->Error("failed to fork Zeek supervisor stem process: %s\n", tmp); + signal_flare.Fire(); + // Sleep to avoid spinning too fast in a revival-fail loop. + sleep(1); + return; + } + + if ( stem_pid == 0 ) + { + // Child stem process needs to exec() + auto stem_env = fmt("%d,%d,%d,%d,%d", stem_ppid, + stem_pipe->In().ReadFD(), stem_pipe->In().WriteFD(), + stem_pipe->Out().ReadFD(), stem_pipe->Out().WriteFD()); + + if ( setenv("ZEEK_STEM", stem_env, true) == -1 ) + { + fprintf(stderr, "setenv(ZEEK_STEM) failed: %s\n", + strerror(errno)); + exit(1); + } + + stem_pipe->In().UnsetFlags(FD_CLOEXEC); + stem_pipe->Out().UnsetFlags(FD_CLOEXEC); + + char** args = new char*[bro_argc + 1]; + args[0] = config.zeek_exe_path.data(); + args[bro_argc] = nullptr; + + for ( auto i = 1; i < bro_argc; ++i ) + args[i] = bro_argv[i]; + + auto res = execv(config.zeek_exe_path.data(), args); + fprintf(stderr, "failed to exec Zeek supervisor stem process: %s\n", + strerror(errno)); + exit(1); + } + + DBG_LOG(DBG_SUPERVISOR, "stem process revived, new pid: %d", stem_pid); + + // Parent supervisor process resends node configurations to recreate + // the desired process hierarchy. + + // Note: there's probably a preferred order in which to create nodes. + // E.g. logger, manager, proxy, worker. However, fully synchronizing + // a startup order like that is slow and complicated: essentially have + // to wait for each process to start up and reach the point just after + // it starts listening (and maybe that never happens for some error case). + for ( const auto& n : nodes ) + { + const auto& node = n.second; + auto msg = make_create_message(node.config); + safe_write(stem_pipe->OutFD(), msg.data(), msg.size() + 1); + } + } + +void Supervisor::GetFds(iosource::FD_Set* read, iosource::FD_Set* write, + iosource::FD_Set* except) + { + read->Insert(signal_flare.FD()); + read->Insert(stem_pipe->InFD()); + } + +double Supervisor::NextTimestamp(double* local_network_time) + { + return timer_mgr->Time(); + } + +void Supervisor::Process() + { + HandleChildSignal(); + ProcessMessages(); + } + +size_t Supervisor::ProcessMessages() + { + char buf[256]; + int bytes_read = read(stem_pipe->InFD(), buf, 256); + + if ( bytes_read > 0 ) + msg_buffer.append(buf, bytes_read); + + auto msgs = extract_messages(&msg_buffer); + + for ( auto& msg : msgs ) + { + DBG_LOG(DBG_SUPERVISOR, "read msg from Stem: %s", msg.data()); + std::vector msg_tokens; + tokenize_string(msg, " ", &msg_tokens); + const auto& type = msg_tokens[0]; + + if ( type == "status" ) + { + const auto& name = msg_tokens[1]; + auto it = nodes.find(name); + + if ( it != nodes.end() ) + it->second.pid = std::stoi(msg_tokens[2]); + } + else if ( type == "debug" ) + { + // Already logged the unparsed message above. + } + else if ( type == "error" ) + { + msg_tokens.erase(msg_tokens.begin()); + auto err_msg = implode_string_vector(msg_tokens, " "); + reporter->Error("%s", err_msg.data()); + } + else + reporter->Error("Supervisor got unknown msg: %s", msg.data()); + } + + return msgs.size(); + } + +Stem::Stem(Supervisor::StemState ss) + : parent_pid(ss.parent_pid), signal_flare(new bro::Flare()), pipe(std::move(ss.pipe)) + { + zeek::set_thread_name("zeek.stem"); + pipe->Swap(); + stem = this; + setsignal(SIGCHLD, stem_signal_handler); + setsignal(SIGTERM, stem_signal_handler); + + // Note: changing the process group here so that SIGINT to the supervisor + // doesn't also get passed to the children. I.e. the supervisor should be + // in charge of initiating orderly shutdown of the process tree. + // Technically calling setpgid() like this is a race-condition (if we get a + // SIGINT in between the fork() and setpgid() calls), but can treat that as + // mostly be harmless since the only affected node in the process tree at + // the point will be this Stem process and the Supervisor *should* do the + // right thing if it also sees SIGINT with the Stem already having exited + // (since that same type of situation with the Stem dying prematurely can + // happen for any arbitrary reason, not just for SIGINT). + auto res = setpgid(0, 0); + + if ( res == -1 ) + LogError("failed to set stem process group: %s", strerror(errno)); + } + +Stem::~Stem() + { + setsignal(SIGCHLD, SIG_DFL); + setsignal(SIGTERM, SIG_DFL); + } + +void Stem::Reap() + { + for ( auto& n : nodes ) + { + auto& node = n.second; + + if ( ! node.pid ) + continue; + + Wait(&node, WNOHANG); + } + } + +bool Stem::Wait(Supervisor::Node* node, int options) const + { + if ( node->pid <= 0 ) + { + DBG_STEM("Stem skip waiting for node '%s' (PID %d) to terminate: already dead", + node->Name().data(), node->pid); + return true; + } + + int status; + auto res = waitpid(node->pid, &status, options); + + if ( res == 0 ) + // It's still alive. + return false; + + if ( res == -1 ) + { + LogError("Stem failed to get node exit status '%s' (PID %d): %s", + node->Name().data(), node->pid, strerror(errno)); + return false; + } + + if ( WIFEXITED(status) ) + { + node->exit_status = WEXITSTATUS(status); + DBG_STEM("node '%s' (PID %d) exited with status %d", + node->Name().data(), node->pid, node->exit_status); + + if ( ! node->killed ) + LogError("Supervised node '%s' (PID %d) exited prematurely with status %d", + node->Name().data(), node->pid, node->exit_status); + } + else if ( WIFSIGNALED(status) ) + { + node->signal_number = WTERMSIG(status); + DBG_STEM("node '%s' (PID %d) terminated by signal %d", + node->Name().data(), node->pid, node->signal_number); + + if ( ! node->killed ) + LogError("Supervised node '%s' (PID %d) terminated prematurely by signal %d", + node->Name().data(), node->pid, node->signal_number); + } + else + LogError("Stem failed to get node exit status '%s' (PID %d)", + node->Name().data(), node->pid); + + node->pid = 0; + return true; + } + +void Stem::KillNode(Supervisor::Node* node, int signal) const + { + if ( node->pid <= 0 ) + { + DBG_STEM("Stem skip killing node '%s' (PID %d): already dead", + node->Name().data(), node->pid); + return; + } + + node->killed = true; + auto kill_res = kill(node->pid, signal); + + if ( kill_res == -1 ) + LogError("Failed to send signal to node '%s' (PID %d): %s", + node->Name().data(), node->pid, strerror(errno)); + } + +void Stem::Destroy(Supervisor::Node* node) const + { + constexpr auto max_term_attempts = 13; + constexpr auto kill_delay = 2; + auto kill_attempts = 0; + + if ( node->pid <= 0 ) + { + DBG_STEM("Stem skip killing/waiting node '%s' (PID %d): already dead", + node->Name().data(), node->pid); + return; + } + + for ( ; ; ) + { + auto sig = kill_attempts++ < max_term_attempts ? SIGTERM : SIGKILL; + KillNode(node, sig); + usleep(10); + + if ( Wait(node, WNOHANG) ) + break; + + DBG_STEM("Stem waiting to destroy node: %s (PID %d)", + node->Name().data(), node->pid); + sleep(kill_delay); + } + } + +std::optional Stem::Revive() + { + constexpr auto attempts_before_delay_increase = 3; + constexpr auto delay_increase_factor = 2; + constexpr auto reset_revival_state_after = 30; + + for ( auto& n : nodes ) + { + auto& node = n.second; + auto now = std::chrono::steady_clock::now(); + auto revival_reset = std::chrono::seconds(reset_revival_state_after); + auto time_since_spawn = now - node.spawn_time; + + if ( node.pid ) + { + if ( time_since_spawn > revival_reset ) + { + node.revival_attempts = 0; + node.revival_delay = 1; + } + + continue; + } + + auto delay = std::chrono::seconds(node.revival_delay); + + if ( time_since_spawn < delay ) + continue; + + ++node.revival_attempts; + + if ( node.revival_attempts % attempts_before_delay_increase == 0 ) + node.revival_delay *= delay_increase_factor; + + auto sn = Spawn(&node); + + if ( sn ) + return sn; + + LogError("Supervised node '%s' (PID %d) revived after premature exit", + node.Name().data(), node.pid); + ReportStatus(node); + } + + return {}; + } + +std::optional Stem::Spawn(Supervisor::Node* node) + { + auto ppid = getpid(); + auto node_pid = fork(); + + if ( node_pid == -1 ) + { + LogError("failed to fork Zeek node '%s': %s", + node->Name().data(), strerror(errno)); + return {}; + } + + if ( node_pid == 0 ) + { + setsignal(SIGCHLD, SIG_DFL); + setsignal(SIGTERM, SIG_DFL); + zeek::set_thread_name(fmt("zeek.%s", node->Name().data())); + Supervisor::SupervisedNode rval; + rval.config = node->config; + rval.parent_pid = ppid; + return rval; + } + + node->pid = node_pid; + node->spawn_time = std::chrono::steady_clock::now(); + DBG_STEM("Stem spawned node: %s (PID %d)", node->Name().data(), node->pid); + return {}; + } + +int Stem::AliveNodeCount() const + { + auto rval = 0; + + for ( const auto& n : nodes ) + if ( n.second.pid ) + ++rval; + + return rval; + } + +void Stem::KillNodes(int signal) + { + for ( auto& n : nodes ) + KillNode(&n.second, signal); + } + +void Stem::Shutdown(int exit_code) + { + DBG_STEM("Stem shutting down with exit code %d", exit_code); + shutting_down = true; + constexpr auto max_term_attempts = 13; + constexpr auto kill_delay = 2; + auto kill_attempts = 0; + + for ( ; ; ) + { + auto sig = kill_attempts++ < max_term_attempts ? SIGTERM : SIGKILL; + + if ( ! nodes.empty() ) + { + KillNodes(sig); + DBG_STEM("Stem killed nodes with signal %d", sig); + usleep(10); + Reap(); + } + + auto nodes_alive = AliveNodeCount(); + + if ( nodes_alive == 0 ) + exit(exit_code); + + DBG_STEM("Stem nodes still alive %d, sleeping for %d seconds", + nodes_alive, kill_delay); + + auto sleep_time_left = kill_delay; + + while ( sleep_time_left > 0 ) + { + sleep_time_left = sleep(sleep_time_left); + + if ( sleep_time_left > 0 ) + { + // Interrupted by signal, so check if children exited + Reap(); + nodes_alive = AliveNodeCount(); + + if ( nodes_alive == 0 ) + exit(exit_code); + } + } + } + } + +void Stem::ReportStatus(const Supervisor::Node& node) const + { + std::string msg = fmt("status %s %d", node.Name().data(), node.pid); + safe_write(pipe->OutFD(), msg.data(), msg.size() + 1); + } + +void Stem::Log(std::string_view type, const char* format, va_list args) const + { + auto raw_msg = fmt(format, args); + + if ( getenv("ZEEK_DEBUG_STEM_STDERR") ) + { + // Useful when debugging a breaking change to the IPC mechanism itself. + fprintf(stderr, "%s\n", raw_msg); + return; + } + + std::string msg{type.data(), type.size()}; + msg += " "; + msg += raw_msg; + safe_write(pipe->OutFD(), msg.data(), msg.size() + 1); + } + +void Stem::LogDebug(const char* format, ...) const + { + va_list args; + va_start(args, format); + Log("debug", format, args); + va_end(args); + } + +void Stem::LogError(const char* format, ...) const + { + va_list args; + va_start(args, format); + Log("error", format, args); + va_end(args); + } + +Supervisor::SupervisedNode Stem::Run() + { + for ( ; ; ) + { + auto new_node = Poll(); + + if ( new_node ) + return *new_node; + } + + // Shouldn't be reached. + assert(false); + return {}; + } + +std::optional Stem::Poll() + { + pollfd fds[2] = { { pipe->InFD(), POLLIN, 0 }, + { signal_flare->FD(), POLLIN, 0} }; + // Note: the poll timeout here is for periodically checking if the parent + // process died (see below). + constexpr auto poll_timeout_ms = 1000; + auto res = poll(fds, 2, poll_timeout_ms); + + if ( res < 0 ) + { + if ( errno != EINTR ) + { + LogError("Stem poll() failed: %s", strerror(errno)); + return {}; + } + } + + if ( last_signal >= 0 ) + { + DBG_STEM("Stem received signal: %d", last_signal); + last_signal = -1; + } + + if ( getppid() != parent_pid ) + { + // Note: only simple + portable way of detecting loss of parent + // process seems to be polling for change in PPID. There's platform + // specific ways if we do end up needing something more responsive + // and/or have to avoid overhead of polling, but maybe not worth + // the additional complexity: + // Linux: prctl(PR_SET_PDEATHSIG, ...) + // FreeBSD: procctl(PROC_PDEATHSIG_CTL) + // Also note the similar polling methodology in ParentProcessCheckTimer. + DBG_STEM("Stem suicide"); + Shutdown(13); + } + + auto new_node = Revive(); + + if ( new_node ) + return new_node; + + if ( res == 0 ) + return {}; + + if ( signal_flare->Extinguish() ) + { + if ( shutting_down ) + Shutdown(0); + + Reap(); + auto new_node = Revive(); + + if ( new_node ) + return new_node; + } + + if ( ! fds[0].revents ) + return {}; + + char buf[256]; + int bytes_read = read(pipe->InFD(), buf, 256); + + if ( bytes_read == 0 ) + { + // EOF, supervisor must have exited + DBG_STEM("Stem EOF"); + Shutdown(14); + } + + if ( bytes_read < 0 ) + { + LogError("Stem read() failed: %s", strerror(errno)); + return {}; + } + + msg_buffer.append(buf, bytes_read); + auto msgs = extract_messages(&msg_buffer); + + for ( auto& msg : msgs ) + { + std::vector msg_tokens; + tokenize_string(std::move(msg), " ", &msg_tokens, 2); + const auto& cmd = msg_tokens[0]; + const auto& node_name = msg_tokens[1]; + + if ( cmd == "create" ) + { + const auto& node_json = msg_tokens[2]; + assert(nodes.find(node_name) == nodes.end()); + auto node_config = Supervisor::NodeConfig::FromJSON(node_json); + auto it = nodes.emplace(node_name, std::move(node_config)).first; + auto& node = it->second; + + auto sn = Spawn(&node); + + if ( sn ) + return sn; + + DBG_STEM("Stem created node: %s (PID %d)", node.Name().data(), node.pid); + ReportStatus(node); + } + else if ( cmd == "destroy" ) + { + auto it = nodes.find(node_name); + auto& node = it->second; + DBG_STEM("Stem destroying node: %s (PID %d)", node_name.data(), node.pid); + Destroy(&node); + nodes.erase(it); + } + else if ( cmd == "restart" ) + { + auto it = nodes.find(node_name); + assert(it != nodes.end()); + auto& node = it->second; + DBG_STEM("Stem restarting node: %s (PID %d)", node_name.data(), node.pid); + Destroy(&node); + + auto sn = Spawn(&node); + + if ( sn ) + return sn; + + ReportStatus(node); + } + else + LogError("Stem got unknown supervisor message: %s", cmd.data()); + } + + return {}; + } + +std::optional Supervisor::CreateStem(bool supervisor_mode) + { + // If the Stem needs to be re-created via fork()/exec(), then the necessary + // state information is communicated via ZEEK_STEM env. var. + auto zeek_stem_env = getenv("ZEEK_STEM"); + + if ( zeek_stem_env ) + { + std::vector zeek_stem_nums; + tokenize_string(zeek_stem_env, ",", &zeek_stem_nums); + + if ( zeek_stem_nums.size() != 5 ) + { + fprintf(stderr, "invalid ZEEK_STEM environment variable value: '%s'\n", + zeek_stem_env); + exit(1); + } + + pid_t stem_ppid = std::stoi(zeek_stem_nums[0]); + int fds[4]; + + for ( auto i = 0; i < 4; ++i ) + fds[i] = std::stoi(zeek_stem_nums[i + 1]); + + StemState ss; + ss.pipe = std::make_unique(FD_CLOEXEC, O_NONBLOCK, fds); + ss.parent_pid = stem_ppid; + zeek::Supervisor::RunStem(std::move(ss)); + return {}; + } + + if ( ! supervisor_mode ) + return {}; + + StemState ss; + ss.pipe = std::make_unique(FD_CLOEXEC, O_NONBLOCK); + ss.parent_pid = getpid(); + ss.pid = fork(); + + if ( ss.pid == -1 ) + { + fprintf(stderr, "failed to fork Zeek supervisor stem process: %s\n", + strerror(errno)); + exit(1); + } + + if ( ss.pid == 0 ) + { + zeek::Supervisor::RunStem(std::move(ss)); + return {}; + } + + return std::optional(std::move(ss)); + } + +Supervisor::SupervisedNode Supervisor::RunStem(StemState stem_state) + { + Stem s(std::move(stem_state)); + supervised_node = s.Run(); + return *supervised_node; + } + +static BifEnum::Supervisor::ClusterRole role_str_to_enum(std::string_view r) + { + if ( r == "Supervisor::LOGGER" ) + return BifEnum::Supervisor::LOGGER; + if ( r == "Supervisor::MANAGER" ) + return BifEnum::Supervisor::MANAGER; + if ( r == "Supervisor::PROXY" ) + return BifEnum::Supervisor::PROXY; + if ( r == "Supervisor::WORKER" ) + return BifEnum::Supervisor::WORKER; + + return BifEnum::Supervisor::NONE; + } + +Supervisor::NodeConfig Supervisor::NodeConfig::FromRecord(const RecordVal* node) + { + Supervisor::NodeConfig rval; + rval.name = node->Lookup("name")->AsString()->CheckString(); + auto iface_val = node->Lookup("interface"); + + if ( iface_val ) + rval.interface = iface_val->AsString()->CheckString(); + + auto directory_val = node->Lookup("directory"); + + if ( directory_val ) + rval.directory = directory_val->AsString()->CheckString(); + + auto stdout_val = node->Lookup("stdout_file"); + + if ( stdout_val ) + rval.stdout_file = stdout_val->AsString()->CheckString(); + + auto stderr_val = node->Lookup("stderr_file"); + + if ( stderr_val ) + rval.stderr_file = stderr_val->AsString()->CheckString(); + + auto affinity_val = node->Lookup("cpu_affinity"); + + if ( affinity_val ) + rval.cpu_affinity = affinity_val->AsInt(); + + auto scripts_val = node->Lookup("scripts")->AsVectorVal(); + + for ( auto i = 0u; i < scripts_val->Size(); ++i ) + { + auto script = scripts_val->Lookup(i)->AsStringVal()->ToStdString(); + rval.scripts.emplace_back(std::move(script)); + } + + auto cluster_table_val = node->Lookup("cluster")->AsTableVal(); + auto cluster_table = cluster_table_val->AsTable(); + auto c = cluster_table->InitForIteration(); + HashKey* k; + TableEntryVal* v; + + while ( (v = cluster_table->NextEntry(k, c)) ) + { + IntrusivePtr key{cluster_table_val->RecoverIndex(k), false}; + delete k; + auto name = key->Index(0)->AsStringVal()->ToStdString(); + auto rv = v->Value()->AsRecordVal(); + + Supervisor::ClusterEndpoint ep; + ep.role = static_cast(rv->Lookup("role")->AsEnum()); + ep.host = rv->Lookup("host")->AsAddr().AsString(); + ep.port = rv->Lookup("p")->AsPortVal()->Port(); + + auto iface = rv->Lookup("interface"); + + if ( iface ) + ep.interface = iface->AsStringVal()->ToStdString(); + + rval.cluster.emplace(name, std::move(ep)); + } + + return rval; + } + +Supervisor::NodeConfig Supervisor::NodeConfig::FromJSON(std::string_view json) + { + Supervisor::NodeConfig rval; + rapidjson::Document j; + j.Parse(json.data(), json.size()); + rval.name = j["name"].GetString(); + + if ( auto it = j.FindMember("interface"); it != j.MemberEnd() ) + rval.interface = it->value.GetString(); + + if ( auto it = j.FindMember("directory"); it != j.MemberEnd() ) + rval.directory = it->value.GetString(); + + if ( auto it = j.FindMember("stdout_file"); it != j.MemberEnd() ) + rval.stdout_file= it->value.GetString(); + + if ( auto it = j.FindMember("stderr_file"); it != j.MemberEnd() ) + rval.stderr_file= it->value.GetString(); + + if ( auto it = j.FindMember("cpu_affinity"); it != j.MemberEnd() ) + rval.cpu_affinity = it->value.GetInt(); + + auto& scripts = j["scripts"]; + + for ( auto it = scripts.Begin(); it != scripts.End(); ++it ) + rval.scripts.emplace_back(it->GetString()); + + auto& cluster = j["cluster"]; + + for ( auto it = cluster.MemberBegin(); it != cluster.MemberEnd(); ++it ) + { + Supervisor::ClusterEndpoint ep; + + auto key = it->name.GetString(); + auto& val = it->value; + + auto& role_str = val["role"]; + ep.role = role_str_to_enum(role_str.GetString()); + + ep.host = val["host"].GetString(); + ep.port = val["p"]["port"].GetInt(); + + if ( auto it = val.FindMember("interface"); it != val.MemberEnd() ) + ep.interface = it->value.GetString(); + + rval.cluster.emplace(key, std::move(ep)); + } + + return rval; + } + +std::string Supervisor::NodeConfig::ToJSON() const + { + auto re = std::make_unique("^_"); + auto node_val = ToRecord(); + IntrusivePtr json_val{node_val->ToJSON(false, re.get()), false}; + auto rval = json_val->ToStdString(); + return rval; + } + +IntrusivePtr Supervisor::NodeConfig::ToRecord() const + { + auto rt = BifType::Record::Supervisor::NodeConfig; + auto rval = make_intrusive(rt); + rval->Assign(rt->FieldOffset("name"), new StringVal(name)); + + if ( interface ) + rval->Assign(rt->FieldOffset("interface"), new StringVal(*interface)); + + if ( directory ) + rval->Assign(rt->FieldOffset("directory"), new StringVal(*directory)); + + if ( stdout_file ) + rval->Assign(rt->FieldOffset("stdout_file"), new StringVal(*stdout_file)); + + if ( stderr_file ) + rval->Assign(rt->FieldOffset("stderr_file"), new StringVal(*stderr_file)); + + if ( cpu_affinity ) + rval->Assign(rt->FieldOffset("cpu_affinity"), val_mgr->GetInt(*cpu_affinity)); + + auto st = BifType::Record::Supervisor::NodeConfig->FieldType("scripts"); + auto scripts_val = new VectorVal(st->AsVectorType()); + rval->Assign(rt->FieldOffset("scripts"), scripts_val); + + for ( const auto& s : scripts ) + scripts_val->Assign(scripts_val->Size(), new StringVal(s)); + + auto tt = BifType::Record::Supervisor::NodeConfig->FieldType("cluster"); + auto cluster_val = new TableVal(tt->AsTableType()); + rval->Assign(rt->FieldOffset("cluster"), cluster_val); + + for ( const auto& e : cluster ) + { + auto& name = e.first; + auto& ep = e.second; + auto key = make_intrusive(name); + auto ept = BifType::Record::Supervisor::ClusterEndpoint; + auto val = make_intrusive(ept); + + val->Assign(ept->FieldOffset("role"), BifType::Enum::Supervisor::ClusterRole->GetVal(ep.role)); + val->Assign(ept->FieldOffset("host"), new AddrVal(ep.host)); + val->Assign(ept->FieldOffset("p"), val_mgr->GetPort(ep.port, TRANSPORT_TCP)); + + if ( ep.interface ) + val->Assign(ept->FieldOffset("interface"), new StringVal(*ep.interface)); + + cluster_val->Assign(key.get(), val.detach()); + } + + return rval; + } + +IntrusivePtr Supervisor::Node::ToRecord() const + { + auto rt = BifType::Record::Supervisor::NodeStatus; + auto rval = make_intrusive(rt); + + rval->Assign(rt->FieldOffset("node"), config.ToRecord().detach()); + + if ( pid ) + rval->Assign(rt->FieldOffset("pid"), val_mgr->GetInt(pid)); + + return rval; + } + + +static Val* supervisor_role_to_cluster_node_type(BifEnum::Supervisor::ClusterRole role) + { + static auto node_type = global_scope()->Lookup("Cluster::NodeType")->AsType()->AsEnumType(); + + switch ( role ) { + case BifEnum::Supervisor::LOGGER: + return node_type->GetVal(node_type->Lookup("Cluster", "LOGGER")); + case BifEnum::Supervisor::MANAGER: + return node_type->GetVal(node_type->Lookup("Cluster", "MANAGER")); + case BifEnum::Supervisor::PROXY: + return node_type->GetVal(node_type->Lookup("Cluster", "PROXY")); + case BifEnum::Supervisor::WORKER: + return node_type->GetVal(node_type->Lookup("Cluster", "WORKER")); + default: + return node_type->GetVal(node_type->Lookup("Cluster", "NONE")); + } + } + +bool Supervisor::SupervisedNode::InitCluster() const + { + if ( config.cluster.empty() ) + return false; + + auto cluster_node_type = global_scope()->Lookup("Cluster::Node")->AsType()->AsRecordType(); + auto cluster_nodes_id = global_scope()->Lookup("Cluster::nodes"); + auto cluster_manager_is_logger_id = global_scope()->Lookup("Cluster::manager_is_logger"); + auto cluster_nodes = cluster_nodes_id->ID_Val()->AsTableVal(); + auto has_logger = false; + std::optional manager_name; + + for ( const auto& e : config.cluster ) + { + if ( e.second.role == BifEnum::Supervisor::MANAGER ) + manager_name = e.first; + else if ( e.second.role == BifEnum::Supervisor::LOGGER ) + has_logger = true; + } + + for ( const auto& e : config.cluster ) + { + const auto& node_name = e.first; + const auto& ep = e.second; + auto key = make_intrusive(node_name); + auto val = make_intrusive(cluster_node_type); + + auto node_type = supervisor_role_to_cluster_node_type(ep.role); + val->Assign(cluster_node_type->FieldOffset("node_type"), node_type); + val->Assign(cluster_node_type->FieldOffset("ip"), new AddrVal(ep.host)); + val->Assign(cluster_node_type->FieldOffset("p"), val_mgr->GetPort(ep.port, TRANSPORT_TCP)); + + if ( ep.interface ) + val->Assign(cluster_node_type->FieldOffset("interface"), + new StringVal(*ep.interface)); + + if ( manager_name && ep.role != BifEnum::Supervisor::MANAGER ) + val->Assign(cluster_node_type->FieldOffset("manager"), + new StringVal(*manager_name)); + + cluster_nodes->Assign(key.get(), val.detach()); + } + + cluster_manager_is_logger_id->SetVal(val_mgr->GetBool(! has_logger)); + return true; + } + +void Supervisor::SupervisedNode::Init(zeek::Options* options) const + { + const auto& node_name = config.name; + + if ( config.directory ) + { + if ( chdir(config.directory->data()) ) + { + fprintf(stderr, "node '%s' failed to chdir to %s: %s\n", + node_name.data(), config.directory->data(), + strerror(errno)); + exit(1); + } + } + + if ( config.stderr_file ) + { + auto fd = open(config.stderr_file->data(), + O_WRONLY | O_CREAT | O_TRUNC | O_APPEND | O_CLOEXEC, + 0600); + + if ( fd == -1 || dup2(fd, STDERR_FILENO) == -1 ) + { + fprintf(stderr, "node '%s' failed to create stderr file %s: %s\n", + node_name.data(), config.stderr_file->data(), + strerror(errno)); + exit(1); + } + + safe_close(fd); + } + + if ( config.stdout_file ) + { + auto fd = open(config.stdout_file->data(), + O_WRONLY | O_CREAT | O_TRUNC | O_APPEND | O_CLOEXEC, + 0600); + + if ( fd == -1 || dup2(fd, STDOUT_FILENO) == -1 ) + { + fprintf(stderr, "node '%s' failed to create stdout file %s: %s\n", + node_name.data(), config.stdout_file->data(), + strerror(errno)); + exit(1); + } + + safe_close(fd); + } + + if ( config.cpu_affinity ) + { + auto res = zeek::set_affinity(*config.cpu_affinity); + + if ( ! res ) + fprintf(stderr, "node '%s' failed to set CPU affinity: %s\n", + node_name.data(), strerror(errno)); + } + + if ( ! config.cluster.empty() ) + { + if ( setenv("CLUSTER_NODE", node_name.data(), true) == -1 ) + { + fprintf(stderr, "node '%s' failed to setenv: %s\n", + node_name.data(), strerror(errno)); + exit(1); + } + } + + options->filter_supervised_node_options(); + + if ( config.interface ) + options->interfaces.emplace_back(*config.interface); + + for ( const auto& s : config.scripts ) + options->scripts_to_load.emplace_back(s); + } + +RecordVal* Supervisor::Status(std::string_view node_name) + { + auto rval = new RecordVal(BifType::Record::Supervisor::Status); + auto tt = BifType::Record::Supervisor::Status->FieldType("nodes"); + auto node_table_val = new TableVal(tt->AsTableType()); + rval->Assign(0, node_table_val); + + if ( node_name.empty() ) + { + for ( const auto& n : nodes ) + { + const auto& name = n.first; + const auto& node = n.second; + auto key = make_intrusive(name); + auto val = node.ToRecord(); + node_table_val->Assign(key.get(), val.detach()); + } + } + else + { + auto it = nodes.find(node_name); + + if ( it == nodes.end() ) + return rval; + + const auto& name = it->first; + const auto& node = it->second; + auto key = make_intrusive(name); + auto val = node.ToRecord(); + node_table_val->Assign(key.get(), val.detach()); + } + + return rval; + } + +std::string Supervisor::Create(const RecordVal* node_val) + { + auto node = Supervisor::NodeConfig::FromRecord(node_val); + return Create(node); + } + +std::string Supervisor::Create(const Supervisor::NodeConfig& node) + { + if ( node.name.empty() ) + return "node names must not be an empty string"; + + if ( node.name.find(' ') != std::string::npos ) + return fmt("node names must not contain spaces: '%s'", + node.name.data()); + + if ( nodes.find(node.name) != nodes.end() ) + return fmt("node with name '%s' already exists", node.name.data()); + + if ( node.directory ) + { + auto res = ensure_intermediate_dirs(node.directory->data()); + + if ( ! res ) + return fmt("failed to create working directory %s\n", + node.directory->data()); + } + + auto msg = make_create_message(node); + safe_write(stem_pipe->OutFD(), msg.data(), msg.size() + 1); + nodes.emplace(node.name, node); + return ""; + } + +bool Supervisor::Destroy(std::string_view node_name) + { + auto send_destroy_msg = [this](std::string_view name) + { + std::stringstream ss; + ss << "destroy " << name; + std::string msg = ss.str(); + safe_write(stem_pipe->OutFD(), msg.data(), msg.size() + 1); + }; + + if ( node_name.empty() ) + { + for ( const auto& n : nodes ) + send_destroy_msg(n.first); + + nodes.clear(); + return true; + } + + auto it = nodes.find(node_name); + + if ( it == nodes.end() ) + return false; + + nodes.erase(it); + send_destroy_msg(node_name); + return true; + } + +bool Supervisor::Restart(std::string_view node_name) + { + auto send_restart_msg = [this](std::string_view name) + { + std::stringstream ss; + ss << "restart " << name; + std::string msg = ss.str(); + safe_write(stem_pipe->OutFD(), msg.data(), msg.size() + 1); + }; + + if ( node_name.empty() ) + { + for ( const auto& n : nodes ) + send_restart_msg(n.first); + + return true; + } + + if ( nodes.find(node_name) == nodes.end() ) + return false; + + send_restart_msg(node_name); + return true; + } diff --git a/src/supervisor/Supervisor.h b/src/supervisor/Supervisor.h new file mode 100644 index 0000000000..32b07e5aff --- /dev/null +++ b/src/supervisor/Supervisor.h @@ -0,0 +1,414 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iosource/IOSource.h" +#include "Timer.h" +#include "Pipe.h" +#include "Flare.h" +#include "NetVar.h" +#include "IntrusivePtr.h" +#include "Options.h" + +namespace zeek { + +/** + * A Supervisor object manages a tree of persistent Zeek processes. If any + * child process dies it will be re-created with its original configuration. + * The Supervisor process itself actually only manages a single child process, + * called the Stem process. That Stem is created via a fork() just after the + * command-line arguments have been parsed. The Stem process is used as the + * baseline image for spawning and supervising further Zeek child nodes since + * it has the purest global state without having to risk an exec() using an + * on-disk binary that's changed in the meantime from the original Supervisor's + * version of the Zeek binary. However, if the Stem process itself dies + * prematurely, the Supervisor will have to fork() and exec() to revive it (and + * then the revived Stem will re-spawn its own children). Any node in the tree + * will self-terminate if it detects its parent has died and that detection is + * done via polling for change in parent process ID. + */ +class Supervisor : public iosource::IOSource { +public: + + /** + * Configuration options that change Supervisor behavior. + */ + struct Config { + /** + * The filesystem path of the Zeek binary/executable. This is used + * if the Stem process ever dies and we need to fork() and exec() to + * re-create it. + */ + std::string zeek_exe_path; + }; + + /** + * Configuration options that influence how a Supervised Zeek node + * integrates into the normal Zeek Cluster Framework. + */ + struct ClusterEndpoint { + /** + * The node's role within the cluster. E.g. manager, logger, worker. + */ + BifEnum::Supervisor::ClusterRole role; + /** + * The host/IP at which the cluster node is listening for connections. + */ + std::string host; + /** + * The TCP port number at which the cluster node listens for connections. + */ + int port; + /** + * The interface name from which the node read/analyze packets. + * Typically used by worker nodes. + */ + std::optional interface; + }; + + /** + * Configuration options that influence behavior of a Supervised Zeek node. + */ + struct NodeConfig { + /** + * Create configuration from script-layer record value. + * @param node_val the script-layer record value to convert. + */ + static NodeConfig FromRecord(const RecordVal* node_val); + + /** + * Create configuration from JSON representation. + * @param json the JSON string to convert. + */ + static NodeConfig FromJSON(std::string_view json); + + /** + * Convert this object into JSON respresentation. + * @return the JSON string representing the node config. + */ + std::string ToJSON() const; + + /** + * Convert his object into script-layer record value. + * @return the script-layer record value representing the node config. + */ + IntrusivePtr ToRecord() const; + + /** + * The name of the supervised Zeek node. These are unique within + * a given supervised process tree and typically human-readable. + */ + std::string name; + /** + * The interface name from which the node should read/analyze packets. + */ + std::optional interface; + /** + * The working directory that should be used by the node. + */ + std::optional directory; + /** + * The filename/path to which the node's stdout will be redirected. + */ + std::optional stdout_file; + /** + * The filename/path to which the node's stderr will be redirected. + */ + std::optional stderr_file; + /** + * A cpu/core number to which the node will try to pin itself. + */ + std::optional cpu_affinity; + /** + * Additional script filename/paths that the node should load. + */ + std::vector scripts; + /** + * The Cluster Layout definition. Each node in the Cluster Framework + * knows about the full, static cluster topology to which it belongs. + * Entries in the map use node names for keys. + */ + std::map cluster; + }; + + /** + * State which defines a Supervised node's understanding of itself. + */ + struct SupervisedNode { + /** + * Initialize the Supervised node within the Zeek Cluster Framework. + * This function populates the "Cluster::nodes" script-layer variable + * that otherwise is expected to be populated by a + * "cluster-layout.zeek" script in other context (e.g. ZeekCtl + * generates that cluster layout). + * @return true if the supervised node is using the Cluster Framework + * else false. + */ + bool InitCluster() const; + + /** + * Initialize the Supervised node. + * @param options the Zeek options to extend/modify as appropriate + * for the node's configuration. + */ + void Init(zeek::Options* options) const; + + /** + * The node's configuration options. + */ + NodeConfig config; + /** + * The process ID of the supervised node's parent process (i.e. the PID + * of the Stem process). + */ + pid_t parent_pid; + }; + + /** + * The state of a supervised node from the Supervisor's perspective. + */ + struct Node { + /** + * Convert the node into script-layer Supervisor::NodeStatus record + * representation. + */ + IntrusivePtr ToRecord() const; + + /** + * @return the name of the node. + */ + const std::string& Name() const + { return config.name; } + + /** + * Create a new node state from a given configuration. + * @param arg_config the configuration to use for the node. + */ + Node(NodeConfig arg_config) : config(std::move(arg_config)) + { } + + /** + * The desired configuration for the node. + */ + NodeConfig config; + /** + * Process ID of the node (positive/non-zero are valid/live PIDs). + */ + pid_t pid = 0; + /** + * Whether the node is voluntarily marked for termination by the + * Supervisor. + */ + bool killed = false; + /** + * The last exit status of the node. + */ + int exit_status = 0; + /** + * The last signal which terminated the node. + */ + int signal_number = 0; + /** + * Number of process revival attempts made after the node first died + * prematurely. + */ + int revival_attempts = 0; + /** + * How many seconds to wait until the next revival attempt for the node. + */ + int revival_delay = 1; + /** + * The time at which the node's process was last spawned. + */ + std::chrono::time_point spawn_time; + }; + + /** + * State used to initalialize the Stem process. + */ + struct StemState { + /** + * Bidirectional pipes that allow the Supervisor and Stem to talk. + */ + std::unique_ptr pipe; + /** + * The Stem's parent process ID (i.e. PID of the Supervisor). + */ + pid_t parent_pid = 0; + /** + * The Stem's process ID. + */ + pid_t pid = 0; + }; + + /** + * Create and run the Stem process if necessary. + * @param supervisor_mode whether Zeek was invoked with the supervisor + * mode specified as command-line argument/option. + * @return state that defines the Stem process if called from the + * Supervisor process. The Stem process itself will not return from this, + * function but a node it spawns via fork() will return from it and + * information about it is available in ThisNode(). + */ + static std::optional CreateStem(bool supervisor_mode); + + /** + * @return the state which describes what a supervised node should know + * about itself if this is a supervised process. If called from a process + * that is not supervised, this returns an "empty" object. + */ + static const std::optional& ThisNode() + { return supervised_node; } + + using NodeMap = std::map>; + + /** + * Create a new Supervisor object. + * @param stem_state information about the Stem process that was already + * created via CreateStem() + */ + Supervisor(Config cfg, StemState stem_state); + + /** + * Destruction also cleanly shuts down the entire supervised process tree. + */ + ~Supervisor(); + + /** + * @return the process ID of the Stem. + */ + pid_t StemPID() const + { return stem_pid; } + + /** + * @return the state of currently supervised processes. The map uses + * node names for keys. + */ + const NodeMap& Nodes() + { return nodes; } + + /** + * Retrieve current status of a supervised node. + * @param node_name the name of the node for which to retrieve status + * or an empty string to mean "all nodes". + * @return script-layer Supervisor::Status record value describing the + * status of a node or set of nodes. + */ + RecordVal* Status(std::string_view node_name); + + /** + * Create a new supervised node. + * @param node the script-layer Supervisor::NodeConfig value that + * describes the desired node configuration + * @return an empty string on success or description of the error/failure + */ + std::string Create(const RecordVal* node); + + /** + * Create a new supervised node. + * @param node the desired node configuration + * @return an empty string on success or description of the error/failure + */ + std::string Create(const Supervisor::NodeConfig& node); + + /** + * Destroys and removes a supervised node. + * @param node_name the name of the node to destroy or an empty string + * to mean "all nodes" + * @return true on success + */ + bool Destroy(std::string_view node_name); + + /** + * Restart a supervised node process (by destroying and re-recreating). + * @param node_name the name of the node to restart or an empty string + * to mean "all nodes" + * @return true on success + */ + bool Restart(std::string_view node_name); + + /** + * Not meant for public use. For use in a signal handler to tell the + * Supervisor a child process (i.e. the Stem) potentially died. + */ + void ObserveChildSignal(int signo); + +private: + + // IOSource interface overrides: + void GetFds(iosource::FD_Set* read, iosource::FD_Set* write, + iosource::FD_Set* except) override; + + double NextTimestamp(double* local_network_time) override; + + void Process() override; + + size_t ProcessMessages(); + + void HandleChildSignal(); + + void ReapStem(); + + const char* Tag() override + { return "zeek::Supervisor"; } + + /** + * Run the Stem process. The Stem process will receive instructions from + * the Supervisor to manipulate the process hierarchy and it's in charge + * of directly monitoring for whether any nodes die premature and need + * to be revived. + * @param pipe bidirectional pipes that allow the Supervisor and Stem + * process to communicate. + * @param pid the Stem's parent process ID (i.e. the PID of the Supervisor) + * @return state which describes what a supervised node should know about + * itself. I.e. this function only returns from a fork()'d child process. + */ + static SupervisedNode RunStem(StemState stem_state); + + static std::optional supervised_node; + + Config config; + pid_t stem_pid; + std::unique_ptr stem_pipe; + int last_signal = -1; + bro::Flare signal_flare; + NodeMap nodes; + std::string msg_buffer; +}; + +/** + * A timer used by supervised processes to periodically check whether their + * parent (supervisor) process has died. If it has died, the supervised + * process self-terminates. + */ +class ParentProcessCheckTimer : public Timer { +public: + + /** + * Create a timer to check for parent process death. + * @param t the time at which to trigger the timer's check. + * @param interval number of seconds to wait before checking again. + */ + ParentProcessCheckTimer(double t, double interval); + +protected: + + void Dispatch(double t, int is_expire) override; + + double interval; +}; + +extern Supervisor* supervisor_mgr; + +} // namespace zeek diff --git a/src/supervisor/supervisor.bif b/src/supervisor/supervisor.bif new file mode 100644 index 0000000000..6bc6db17d6 --- /dev/null +++ b/src/supervisor/supervisor.bif @@ -0,0 +1,112 @@ +##! The BIFs that define the Zeek supervisor control interface. + +%%{ +#include "supervisor/Supervisor.h" +%%} + +module Supervisor; + +enum ClusterRole %{ + NONE, + LOGGER, + MANAGER, + PROXY, + WORKER, +%} + +type Supervisor::ClusterEndpoint: record; +type Supervisor::Status: record; +type Supervisor::NodeConfig: record; +type Supervisor::NodeStatus: record; + +function Supervisor::__status%(node: string%): Supervisor::Status + %{ + if ( ! zeek::supervisor_mgr ) + { + builtin_error("supervisor mode not enabled"); + return new RecordVal(BifType::Record::Supervisor::Status); + } + + return zeek::supervisor_mgr->Status(node->CheckString()); + %} + +function Supervisor::__create%(node: Supervisor::NodeConfig%): string + %{ + if ( ! zeek::supervisor_mgr ) + { + builtin_error("supervisor mode not enabled"); + return new StringVal("supervisor mode not enabled"); + } + + auto rval = zeek::supervisor_mgr->Create(node->AsRecordVal()); + return new StringVal(rval); + %} + +function Supervisor::__destroy%(node: string%): bool + %{ + if ( ! zeek::supervisor_mgr ) + { + builtin_error("supervisor mode not enabled"); + return val_mgr->GetBool(false); + } + + auto rval = zeek::supervisor_mgr->Destroy(node->CheckString()); + return val_mgr->GetBool(rval); + %} + +function Supervisor::__restart%(node: string%): bool + %{ + if ( ! zeek::supervisor_mgr ) + { + builtin_error("supervisor mode not enabled"); + return val_mgr->GetBool(false); + } + + auto rval = zeek::supervisor_mgr->Restart(node->CheckString()); + return val_mgr->GetBool(rval); + %} + +function Supervisor::__init_cluster%(%): bool + %{ + if ( zeek::Supervisor::ThisNode() ) + return val_mgr->GetBool(zeek::Supervisor::ThisNode()->InitCluster()); + + return val_mgr->GetBool(false); + %} + +function Supervisor::__is_supervised%(%): bool + %{ + return val_mgr->GetBool(zeek::Supervisor::ThisNode().has_value()); + %} + +function Supervisor::__node%(%): Supervisor::NodeConfig + %{ + if ( ! zeek::Supervisor::ThisNode() ) + { + builtin_error("not a supervised process"); + auto rt = BifType::Record::Supervisor::NodeConfig; + auto rval = make_intrusive(rt); + rval->Assign(rt->FieldOffset("name"), new StringVal("")); + return rval.detach(); + } + + auto rval = zeek::Supervisor::ThisNode()->config.ToRecord(); + return rval.detach(); + %} + +function Supervisor::__is_supervisor%(%): bool + %{ + return val_mgr->GetBool(zeek::supervisor_mgr != nullptr); + %} + +function Supervisor::__stem_pid%(%): int + %{ + if ( zeek::supervisor_mgr ) + return val_mgr->GetInt(zeek::supervisor_mgr->StemPID()); + + if ( zeek::Supervisor::ThisNode() ) + return val_mgr->GetInt(zeek::Supervisor::ThisNode()->parent_pid); + + builtin_error("supervisor mode not enabled and not a supervised node"); + return val_mgr->GetInt(-1); + %} diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc index f8257f5b49..3374380748 100644 --- a/src/threading/BasicThread.cc +++ b/src/threading/BasicThread.cc @@ -1,18 +1,11 @@ #include +#include #include "zeek-config.h" #include "BasicThread.h" #include "Manager.h" -#include "pthread.h" - -#ifdef HAVE_LINUX -#include -#endif - -#ifdef __FreeBSD__ -#include -#endif +#include "util.h" using namespace threading; @@ -54,18 +47,7 @@ void BasicThread::SetName(const char* arg_name) void BasicThread::SetOSName(const char* arg_name) { static_assert(std::is_same::value, "libstdc++ doesn't use pthread_t"); - -#ifdef HAVE_LINUX - prctl(PR_SET_NAME, arg_name, 0, 0, 0); -#endif - -#ifdef __APPLE__ - pthread_setname_np(arg_name); -#endif - -#ifdef __FreeBSD__ - pthread_set_name_np(thread.native_handle(), arg_name); -#endif + zeek::set_thread_name(arg_name, thread.native_handle()); } const char* BasicThread::Fmt(const char* format, ...) diff --git a/src/util.cc b/src/util.cc index 18a4defde5..a9b1707297 100644 --- a/src/util.cc +++ b/src/util.cc @@ -55,6 +55,43 @@ #include "3rdparty/doctest.h" +#ifdef __linux__ +#if __has_include() +#define HAVE_GETRANDOM +#include +#endif +#endif + +static bool starts_with(const std::string_view& s, const std::string& beginning) + { + if ( beginning.size() > s.size() ) + return false; + + return std::equal(beginning.begin(), beginning.end(), s.begin()); + } + +TEST_CASE("util starts_with") + { + CHECK(starts_with("abcde", "ab") == true); + CHECK(starts_with("abcde", "de") == false); + CHECK(starts_with("abcde", "abcedf") == false); + } + +static bool ends_with(const std::string_view& s, const std::string& ending) + { + if ( ending.size() > s.size() ) + return false; + + return std::equal(ending.rbegin(), ending.rend(), s.rbegin()); + } + +TEST_CASE("util ends_with") + { + CHECK(ends_with("abcde", "de") == true); + CHECK(ends_with("abcde", "fg") == false); + CHECK(ends_with("abcde", "abcedf") == false); + } + TEST_CASE("util extract_ip") { CHECK(extract_ip("[1.2.3.4]") == "1.2.3.4"); @@ -605,7 +642,7 @@ TEST_CASE("util uitoa_n") char* uitoa_n(uint64_t value, char* str, int n, int base, const char* prefix) { - static char dig[] = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + static constexpr char dig[] = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; assert(n); @@ -616,7 +653,7 @@ char* uitoa_n(uint64_t value, char* str, int n, int base, const char* prefix) if ( prefix ) { - strncpy(str, prefix, n); + strncpy(str, prefix, n-1); str[n-1] = '\0'; i += strlen(prefix); } @@ -1035,6 +1072,14 @@ void init_random_seed(const char* read_file, const char* write_file) seeds_done = true; } +#ifdef HAVE_GETRANDOM + if ( ! seeds_done ) + { + ssize_t nbytes = getrandom(buf, sizeof(buf), 0); + seeds_done = nbytes == ssize_t(sizeof(buf)); + } +#endif + if ( ! seeds_done ) { // Gather up some entropy. @@ -1216,7 +1261,7 @@ string bro_prefixes() { string rval; - for ( const auto& prefix : prefixes ) + for ( const auto& prefix : zeek_script_prefixes ) { if ( ! rval.empty() ) rval.append(":"); @@ -1234,6 +1279,15 @@ TEST_CASE("util is_package_loader") const array script_extensions = {".zeek", ".bro"}; +void warn_if_legacy_script(const std::string_view& filename) + { + if ( ends_with(filename, ".bro") ) + { + std::string x(filename); + reporter->Warning("Loading script '%s' with legacy extension, support for '.bro' will be removed in Zeek v4.1", x.c_str()); + } + } + bool is_package_loader(const string& path) { string filename(std::move(SafeBasename(path).result)); @@ -1241,7 +1295,10 @@ bool is_package_loader(const string& path) for ( const string& ext : script_extensions ) { if ( filename == "__load__" + ext ) + { + warn_if_legacy_script(filename); return true; + } } return false; @@ -1279,6 +1336,7 @@ FILE* open_package(string& path, const string& mode) string p = path + ext; if ( can_read(p) ) { + warn_if_legacy_script(path); path.append(ext); return open_file(path, mode); } @@ -1444,17 +1502,22 @@ TEST_CASE("util tokenize_string") } vector* tokenize_string(string input, const string& delim, - vector* rval) + vector* rval, int limit) { if ( ! rval ) rval = new vector(); size_t n; + auto found = 0; while ( (n = input.find(delim)) != string::npos ) { + ++found; rval->push_back(input.substr(0, n)); input.erase(0, n + 1); + + if ( limit && found == limit ) + break; } rval->push_back(input); @@ -1467,6 +1530,25 @@ TEST_CASE("util normalize_path") CHECK(normalize_path("/1/./2/3") == "/1/2/3"); CHECK(normalize_path("/1/2/../3") == "/1/3"); CHECK(normalize_path("1/2/3/") == "1/2/3"); + CHECK(normalize_path("1/2//3///") == "1/2/3"); + CHECK(normalize_path("~/zeek/testing") == "~/zeek/testing"); + CHECK(normalize_path("~jon/zeek/testing") == "~jon/zeek/testing"); + CHECK(normalize_path("~jon/./zeek/testing") == "~jon/zeek/testing"); + CHECK(normalize_path("~/zeek/testing/../././.") == "~/zeek"); + CHECK(normalize_path("./zeek") == "./zeek"); + CHECK(normalize_path("../zeek") == "../zeek"); + CHECK(normalize_path("../zeek/testing/..") == "../zeek"); + CHECK(normalize_path("./zeek/..") == "."); + CHECK(normalize_path("./zeek/../..") == ".."); + CHECK(normalize_path("./zeek/../../..") == "../.."); + CHECK(normalize_path("./..") == ".."); + CHECK(normalize_path("../..") == "../.."); + CHECK(normalize_path("/..") == "/.."); + CHECK(normalize_path("~/..") == "~/.."); + CHECK(normalize_path("/../..") == "/../.."); + CHECK(normalize_path("~/../..") == "~/../.."); + CHECK(normalize_path("zeek/..") == ""); + CHECK(normalize_path("zeek/../..") == ".."); } string normalize_path(const string& path) @@ -1488,10 +1570,30 @@ string normalize_path(const string& path) if ( *it == "." && it != components.begin() ) final_components.pop_back(); - else if ( *it == ".." && final_components[0] != ".." ) + else if ( *it == ".." ) { - final_components.pop_back(); - final_components.pop_back(); + auto cur_idx = final_components.size() - 1; + + if ( cur_idx != 0 ) + { + auto last_idx = cur_idx - 1; + auto& last_component = final_components[last_idx]; + + if ( last_component == "/" || last_component == "~" || + last_component == ".." ) + continue; + + if ( last_component == "." ) + { + last_component = ".."; + final_components.pop_back(); + } + else + { + final_components.pop_back(); + final_components.pop_back(); + } + } } } @@ -1589,21 +1691,6 @@ string find_file(const string& filename, const string& path_set, return string(); } -static bool ends_with(const std::string& s, const std::string& ending) - { - if ( ending.size() > s.size() ) - return false; - - return std::equal(ending.rbegin(), ending.rend(), s.rbegin()); - } - -TEST_CASE("util ends_with") - { - CHECK(ends_with("abcde", "de") == true); - CHECK(ends_with("abcde", "fg") == false); - CHECK(ends_with("abcde", "abcedf") == false); - } - string find_script_file(const string& filename, const string& path_set) { vector paths; @@ -1616,11 +1703,16 @@ string find_script_file(const string& filename, const string& path_set) string f = find_file_in_path(filename, paths[n], ext); if ( ! f.empty() ) + { + warn_if_legacy_script(f); return f; + } } if ( ends_with(filename, ".bro") ) { + warn_if_legacy_script(filename); + // We were looking for a file explicitly ending in .bro and didn't // find it, so fall back to one ending in .zeek, if it exists. auto fallback = string(filename.data(), filename.size() - 4) + ".zeek"; @@ -1749,7 +1841,7 @@ void terminate_processing() } extern const char* proc_status_file; -void _set_processing_status(const char* status) +void set_processing_status(const char* status, const char* reason) { if ( ! proc_status_file ) return; @@ -1776,20 +1868,27 @@ void _set_processing_status(const char* status) return; } - int len = strlen(status); - while ( len ) + auto write_str = [](int fd, const char* s) { - int n = write(fd, status, len); + int len = strlen(s); + while ( len ) + { + int n = write(fd, s, len); - if ( n < 0 && errno != EINTR && errno != EAGAIN ) - // Ignore errors, as they're too difficult to - // safely report here. - break; + if ( n < 0 && errno != EINTR && errno != EAGAIN ) + // Ignore errors, as they're too difficult to + // safely report here. + break; - status += n; - len -= n; - } + s += n; + len -= n; + } + }; + write_str(fd, status); + write_str(fd, " ["); + write_str(fd, reason); + write_str(fd, "]\n"); safe_close(fd); errno = old_errno; @@ -2171,7 +2270,12 @@ char* zeekenv(const char* name) if ( it == legacy_vars.end() ) return rval; - return getenv(it->second); + auto val = getenv(it->second); + + if ( val && starts_with(it->second, "BRO_") ) + reporter->Warning("Using legacy environment variable %s, support will be removed in Zeek v4.1; use %s instead", it->second, name); + + return val; } static string json_escape_byte(char c) @@ -2299,3 +2403,18 @@ string json_escape_utf8(const string& val) return result; } + +void zeek::set_thread_name(const char* name, pthread_t tid) + { +#ifdef HAVE_LINUX + prctl(PR_SET_NAME, name, 0, 0, 0); +#endif + +#ifdef __APPLE__ + pthread_setname_np(name); +#endif + +#ifdef __FreeBSD__ + pthread_set_name_np(tid, name); +#endif + } diff --git a/src/util.h b/src/util.h index 5a5a8c5159..3065652bea 100644 --- a/src/util.h +++ b/src/util.h @@ -60,6 +60,15 @@ extern HeapLeakChecker* heap_checker; #endif #include +#include + +#ifdef HAVE_LINUX +#include +#endif + +#ifdef __FreeBSD__ +#include +#endif ZEEK_DEPRECATED("Remove in v4.1. Use uint64_t instead.") typedef uint64_t uint64; @@ -138,7 +147,7 @@ inline std::string get_escaped_string(const std::string& str, bool escape_all) std::vector* tokenize_string(std::string input, const std::string& delim, - std::vector* rval = 0); + std::vector* rval = 0, int limit = 0); extern char* copy_string(const char* s); extern int streq(const char* s1, const char* s2); @@ -265,6 +274,9 @@ extern std::string bro_prefixes(); extern const std::array script_extensions; +/** Prints a warning if the filename ends in .bro. */ +void warn_if_legacy_script(const std::string_view& filename); + bool is_package_loader(const std::string& path); extern void add_to_bro_path(const std::string& dir); @@ -402,9 +414,7 @@ void terminate_processing(); // Sets the current status of the Bro process to the given string. // If the option --status-file has been set, this is written into // the the corresponding file. Otherwise, the function is a no-op. -#define set_processing_status(status, location) \ - _set_processing_status(status " [" location "]\n"); -void _set_processing_status(const char* status); +void set_processing_status(const char* status, const char* reason); // Current timestamp, from a networking perspective, not a wall-clock // perspective. In particular, if we're reading from a savefile this @@ -505,7 +515,7 @@ inline void* safe_malloc(size_t size) inline char* safe_strncpy(char* dest, const char* src, size_t n) { - char* result = strncpy(dest, src, n); + char* result = strncpy(dest, src, n-1); dest[n-1] = '\0'; return result; } @@ -572,3 +582,14 @@ char* zeekenv(const char* name); * @return the escaped string */ std::string json_escape_utf8(const std::string& val); + +namespace zeek { +/** + * Set the process/thread name. May not be supported on all OSs. + * @param name new name for the process/thread. OS limitations typically + * truncate the name to 15 bytes maximum. + * @param tid handle of thread whose name shall change + */ +void set_thread_name(const char* name, pthread_t tid = pthread_self()); + +} // namespace zeek diff --git a/src/zeek-affinity.cc b/src/zeek-affinity.cc new file mode 100644 index 0000000000..da23495d4b --- /dev/null +++ b/src/zeek-affinity.cc @@ -0,0 +1,57 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +// This is all in its own source file primarily because the Linux +// implementation uses the _GNU_SOURCE feature test macro which must be +// defined before including any header file and lumping this together with +// other util functions makes that requirement less apparent and less +// self-contained. + +#if defined(__linux__) + +#if !defined(_GNU_SOURCE) +#define _GNU_SOURCE +#endif + +#include + +namespace zeek { +bool set_affinity(int core_number) + { + cpu_set_t cpus; + CPU_ZERO(&cpus); + CPU_SET(core_number, &cpus); + auto res = sched_setaffinity(0, sizeof(cpus), &cpus); + return res == 0; + } +} // namespace zeek + +#elif defined(__FreeBSD__) + +#include +#include + +namespace zeek { +bool set_affinity(int core_number) + { + cpuset_t cpus; + CPU_ZERO(&cpus); + CPU_SET(core_number, &cpus); + auto res = cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, -1, + sizeof(cpus), &cpus); + return res == 0; + } +} // namespace zeek + +#else + +#include + +namespace zeek { +bool set_affinity(int core_number) + { + errno = ENOTSUP; + return false; + } +} // namespace zeek + +#endif diff --git a/src/zeek-affinity.h b/src/zeek-affinity.h new file mode 100644 index 0000000000..f8f173634b --- /dev/null +++ b/src/zeek-affinity.h @@ -0,0 +1,17 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +namespace zeek { + +/** + * Set the process affinity to a given CPU. Currently only supported on + * Linux and FreeBSD. + * @param core_number the core to which this process should set its affinity. + * Cores are typically numbered 0..N. + * @return true if the affinity is successfully set and false if not with + * errno additionally being set to indicate the reason. + */ +bool set_affinity(int core_number); + +} // namespace zeek diff --git a/src/zeek.bif b/src/zeek.bif index 2f56ca52c0..c357c3ce5f 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -1820,15 +1820,6 @@ function getpid%(%) : count extern const char* zeek_version(); %%} -## Returns the Zeek version string. This function is deprecated, use -## :zeek:see:`zeek_version` instead. -## -## Returns: Zeek's version, e.g., 2.0-beta-47-debug. -function bro_version%(%): string &deprecated="Remove in v3.1: use zeek_version" - %{ - return new StringVal(zeek_version()); - %} - ## Returns the Zeek version string. ## ## Returns: Zeek's version, e.g., 2.0-beta-47-debug. @@ -2130,17 +2121,6 @@ function dump_rule_stats%(f: file%): bool return val_mgr->GetBool(1); %} -## Checks if Zeek is terminating. This function is deprecated, use -## :zeek:see:`zeek_is_terminating` instead. -## -## Returns: True if Zeek is in the process of shutting down. -## -## .. zeek:see:: terminate -function bro_is_terminating%(%): bool &deprecated="Remove in v3.1: use zeek_is_terminating" - %{ - return val_mgr->GetBool(terminating); - %} - ## Checks if Zeek is terminating. ## ## Returns: True if Zeek is in the process of shutting down. diff --git a/testing/btest/Baseline/bifs.str_shell_escape/out b/testing/btest/Baseline/bifs.str_shell_escape/out deleted file mode 100644 index 1845fefa37..0000000000 --- a/testing/btest/Baseline/bifs.str_shell_escape/out +++ /dev/null @@ -1,4 +0,0 @@ -24 -echo ${TEST} > "my file" -27 -echo \${TEST} > \"my file\" diff --git a/testing/btest/Baseline/broker.ssl_auth_failure/send.send.out b/testing/btest/Baseline/broker.ssl_auth_failure/send.send.out index 057a0f3d8e..4523e6a6c5 100644 --- a/testing/btest/Baseline/broker.ssl_auth_failure/send.send.out +++ b/testing/btest/Baseline/broker.ssl_auth_failure/send.send.out @@ -1 +1 @@ -sender error: code=Broker::PEER_UNAVAILABLE msg=(invalid-node, *, "remote endpoint unavailable") +sender error: code=Broker::PEER_UNAVAILABLE msg=(invalid-node, *, "remote endpoint unavailable: system_error(disconnect_during_handshake)") diff --git a/testing/btest/Baseline/core.check-unused-event-handlers/.stderr b/testing/btest/Baseline/core.check-unused-event-handlers/.stderr index 8c4e4def40..56eb1756c4 100644 --- a/testing/btest/Baseline/core.check-unused-event-handlers/.stderr +++ b/testing/btest/Baseline/core.check-unused-event-handlers/.stderr @@ -1,3 +1,8 @@ -warning in , line 1: event handler never invoked: this_is_never_used warning in , line 1: event handler never invoked: InputConfig::new_value warning in , line 1: event handler never invoked: InputRaw::process_finished +warning in , line 1: event handler never invoked: SupervisorControl::create_request +warning in , line 1: event handler never invoked: SupervisorControl::destroy_request +warning in , line 1: event handler never invoked: SupervisorControl::restart_request +warning in , line 1: event handler never invoked: SupervisorControl::status_request +warning in , line 1: event handler never invoked: SupervisorControl::stop_request +warning in , line 1: event handler never invoked: this_is_never_used diff --git a/testing/btest/Baseline/core.load-explicit-bro-suffix-fallback/out b/testing/btest/Baseline/core.load-explicit-bro-suffix-fallback/out index c67eefbfc1..a0a25698a1 100644 --- a/testing/btest/Baseline/core.load-explicit-bro-suffix-fallback/out +++ b/testing/btest/Baseline/core.load-explicit-bro-suffix-fallback/out @@ -1 +1,3 @@ +warning in /home/robin/bro/master/testing/btest/.tmp/core.load-explicit-bro-suffix-fallback/load-explicit-bro-suffix-fallback.zeek, line 5: Loading script 'foo.bro' with legacy extension, support for '.bro' will be removed in Zeek v4.1 +warning in /home/robin/bro/master/testing/btest/.tmp/core.load-explicit-bro-suffix-fallback/load-explicit-bro-suffix-fallback.zeek, line 5: Loading script 'foo.bro' with legacy extension, support for '.bro' will be removed in Zeek v4.1 loaded foo.zeek diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 350f284c8a..382bb30cb2 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2019-07-29-19-05-26 +#open 2019-10-15-01-48-24 #fields name #types string scripts/base/init-bare.zeek @@ -14,6 +14,8 @@ scripts/base/init-bare.zeek build/scripts/base/bif/reporter.bif.zeek build/scripts/base/bif/strings.bif.zeek build/scripts/base/bif/option.bif.zeek + scripts/base/frameworks/supervisor/api.zeek + build/scripts/base/bif/supervisor.bif.zeek build/scripts/base/bif/plugins/Zeek_SNMP.types.bif.zeek build/scripts/base/bif/plugins/Zeek_KRB.types.bif.zeek build/scripts/base/bif/event.bif.zeek @@ -35,6 +37,9 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/data.bif.zeek build/scripts/base/bif/store.bif.zeek scripts/base/frameworks/broker/log.zeek + scripts/base/frameworks/supervisor/__load__.zeek + scripts/base/frameworks/supervisor/control.zeek + scripts/base/frameworks/supervisor/main.zeek scripts/base/frameworks/input/__load__.zeek scripts/base/frameworks/input/main.zeek build/scripts/base/bif/input.bif.zeek @@ -182,4 +187,4 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/plugins/Zeek_SQLiteWriter.sqlite.bif.zeek scripts/policy/misc/loaded-scripts.zeek scripts/base/utils/paths.zeek -#close 2019-07-29-19-05-26 +#close 2019-10-15-01-48-24 diff --git a/testing/btest/Baseline/coverage.bare-mode-errors/errors b/testing/btest/Baseline/coverage.bare-mode-errors/errors index c87b897c61..e69de29bb2 100644 --- a/testing/btest/Baseline/coverage.bare-mode-errors/errors +++ b/testing/btest/Baseline/coverage.bare-mode-errors/errors @@ -1 +0,0 @@ -warning in /Users/tim/Desktop/projects/zeek/testing/btest/../../scripts//base/utils/json.zeek, line 2: deprecated script loaded from command line arguments ="Remove in 3.1. to_json is now always available as a built-in function." diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 6311a1ef8f..5d9fbfde6d 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2019-08-06-00-02-39 +#open 2019-10-15-01-48-24 #fields name #types string scripts/base/init-bare.zeek @@ -14,6 +14,8 @@ scripts/base/init-bare.zeek build/scripts/base/bif/reporter.bif.zeek build/scripts/base/bif/strings.bif.zeek build/scripts/base/bif/option.bif.zeek + scripts/base/frameworks/supervisor/api.zeek + build/scripts/base/bif/supervisor.bif.zeek build/scripts/base/bif/plugins/Zeek_SNMP.types.bif.zeek build/scripts/base/bif/plugins/Zeek_KRB.types.bif.zeek build/scripts/base/bif/event.bif.zeek @@ -35,6 +37,9 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/data.bif.zeek build/scripts/base/bif/store.bif.zeek scripts/base/frameworks/broker/log.zeek + scripts/base/frameworks/supervisor/__load__.zeek + scripts/base/frameworks/supervisor/control.zeek + scripts/base/frameworks/supervisor/main.zeek scripts/base/frameworks/input/__load__.zeek scripts/base/frameworks/input/main.zeek build/scripts/base/bif/input.bif.zeek @@ -376,4 +381,4 @@ scripts/base/init-default.zeek scripts/base/misc/find-filtered-trace.zeek scripts/base/misc/version.zeek scripts/policy/misc/loaded-scripts.zeek -#close 2019-08-06-00-02-39 +#close 2019-10-15-01-48-25 diff --git a/testing/btest/Baseline/coverage.init-default/missing_loads b/testing/btest/Baseline/coverage.init-default/missing_loads index 87361a686a..893a603972 100644 --- a/testing/btest/Baseline/coverage.init-default/missing_loads +++ b/testing/btest/Baseline/coverage.init-default/missing_loads @@ -8,4 +8,3 @@ -./frameworks/openflow/cluster.zeek -./frameworks/packet-filter/cluster.zeek -./frameworks/sumstats/cluster.zeek --./utils/json.zeek diff --git a/testing/btest/Baseline/language.bro_init/out b/testing/btest/Baseline/language.bro_init/out new file mode 100644 index 0000000000..17fe3f8cc3 --- /dev/null +++ b/testing/btest/Baseline/language.bro_init/out @@ -0,0 +1,2 @@ +error in /Users/jsiwek/pro/zeek/zeek/testing/btest/.tmp/language.bro_init/bro_init.zeek, line 5: event bro_init() is no longer available, use zeek_init() instead +error in /Users/jsiwek/pro/zeek/zeek/testing/btest/.tmp/language.bro_init/bro_init.zeek, line 10: event bro_done() is no longer available, use zeek_done() instead diff --git a/testing/btest/Baseline/language.bro_script_loaded/out b/testing/btest/Baseline/language.bro_script_loaded/out new file mode 100644 index 0000000000..be0c7a6be2 --- /dev/null +++ b/testing/btest/Baseline/language.bro_script_loaded/out @@ -0,0 +1 @@ +error in /Users/jsiwek/pro/zeek/zeek/testing/btest/.tmp/language.bro_script_loaded/bro_script_loaded.zeek, line 5: event bro_script_loaded() is no longer available, use zeek_script_loaded() instead diff --git a/testing/btest/Baseline/language.zeek_init/out b/testing/btest/Baseline/language.zeek_init/out index aa17ec8aa8..ef5d8d137d 100644 --- a/testing/btest/Baseline/language.zeek_init/out +++ b/testing/btest/Baseline/language.zeek_init/out @@ -1,8 +1,6 @@ zeek_init at priority 10! -bro_init at priority 5! zeek_init at priority 0! -bro_init at priority -10! +zeek_init at priority -10! zeek_done at priority 10! -bro_done at priority 5! zeek_done at priority 0! -bro_done at priority -10! +zeek_done at priority -10! diff --git a/testing/btest/Baseline/language.zeek_script_loaded/out b/testing/btest/Baseline/language.zeek_script_loaded/out index cddf509308..a7b6b483c7 100644 --- a/testing/btest/Baseline/language.zeek_script_loaded/out +++ b/testing/btest/Baseline/language.zeek_script_loaded/out @@ -1,4 +1,2 @@ zeek_script_loaded priority 10 -bro_script_loaded priority 5 zeek_script_loaded priority 0 -bro_script_loaded priority -10 diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index face82ab89..dde704c5af 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -157,6 +157,8 @@ 0.000000 MetaHookPost CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_TEREDO, {3544/udp})) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_VXLAN, {4789/udp})) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_XMPP, {5222<...>/tcp})) -> +0.000000 MetaHookPost CallFunction(Broker::__subscribe, , (zeek/supervisor)) -> +0.000000 MetaHookPost CallFunction(Broker::subscribe, , (zeek/supervisor)) -> 0.000000 MetaHookPost CallFunction(Cluster::is_enabled, , ()) -> 0.000000 MetaHookPost CallFunction(Cluster::is_enabled, , ()) -> 0.000000 MetaHookPost CallFunction(Cluster::local_node_type, , ()) -> @@ -274,7 +276,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1573527265.694831, node=zeek, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1579727603.636084, node=zeek, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Broker::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Config::LOG)) -> @@ -455,7 +457,7 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, , (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1573527265.694831, node=zeek, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1579727603.636084, node=zeek, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(NetControl::check_plugins, , ()) -> 0.000000 MetaHookPost CallFunction(NetControl::init, , ()) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, , ()) -> @@ -694,6 +696,7 @@ 0.000000 MetaHookPost LoadFile(0, .<...>/add-geodata.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/addrs.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/analyzer.bif.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, .<...>/api.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/ascii.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/average.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/benchmark.zeek) -> -1 @@ -708,6 +711,7 @@ 0.000000 MetaHookPost LoadFile(0, .<...>/const.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/consts.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/contents.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, .<...>/control.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/ct-list.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/data.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/dcc-send.zeek) -> -1 @@ -767,6 +771,7 @@ 0.000000 MetaHookPost LoadFile(0, .<...>/store.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/strings.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/sum.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, .<...>/supervisor.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/thresholds.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/top-k.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, .<...>/topk.zeek) -> -1 @@ -788,6 +793,7 @@ 0.000000 MetaHookPost LoadFile(0, base<...>/addrs.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/analyzer) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/analyzer.bif.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, base<...>/api.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/bif) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/broker) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/cluster) -> -1 @@ -868,6 +874,8 @@ 0.000000 MetaHookPost LoadFile(0, base<...>/strings.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/strings.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/sumstats) -> -1 +0.000000 MetaHookPost LoadFile(0, base<...>/supervisor) -> -1 +0.000000 MetaHookPost LoadFile(0, base<...>/supervisor.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/syslog) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/thresholds.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/time.zeek) -> -1 @@ -1053,6 +1061,8 @@ 0.000000 MetaHookPre CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_TEREDO, {3544/udp})) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_VXLAN, {4789/udp})) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_XMPP, {5222<...>/tcp})) +0.000000 MetaHookPre CallFunction(Broker::__subscribe, , (zeek/supervisor)) +0.000000 MetaHookPre CallFunction(Broker::subscribe, , (zeek/supervisor)) 0.000000 MetaHookPre CallFunction(Cluster::is_enabled, , ()) 0.000000 MetaHookPre CallFunction(Cluster::is_enabled, , ()) 0.000000 MetaHookPre CallFunction(Cluster::local_node_type, , ()) @@ -1170,7 +1180,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1573527265.694831, node=zeek, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1579727603.636084, node=zeek, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Broker::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Config::LOG)) @@ -1351,7 +1361,7 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, , (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1573527265.694831, node=zeek, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1579727603.636084, node=zeek, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(NetControl::check_plugins, , ()) 0.000000 MetaHookPre CallFunction(NetControl::init, , ()) 0.000000 MetaHookPre CallFunction(Notice::want_pp, , ()) @@ -1590,6 +1600,7 @@ 0.000000 MetaHookPre LoadFile(0, .<...>/add-geodata.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/addrs.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/analyzer.bif.zeek) +0.000000 MetaHookPre LoadFile(0, .<...>/api.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/ascii.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/average.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/benchmark.zeek) @@ -1604,6 +1615,7 @@ 0.000000 MetaHookPre LoadFile(0, .<...>/const.bif.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/consts.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/contents.zeek) +0.000000 MetaHookPre LoadFile(0, .<...>/control.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/ct-list.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/data.bif.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/dcc-send.zeek) @@ -1663,6 +1675,7 @@ 0.000000 MetaHookPre LoadFile(0, .<...>/store.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/strings.bif.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/sum.zeek) +0.000000 MetaHookPre LoadFile(0, .<...>/supervisor.bif.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/thresholds.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/top-k.bif.zeek) 0.000000 MetaHookPre LoadFile(0, .<...>/topk.zeek) @@ -1684,6 +1697,7 @@ 0.000000 MetaHookPre LoadFile(0, base<...>/addrs.zeek) 0.000000 MetaHookPre LoadFile(0, base<...>/analyzer) 0.000000 MetaHookPre LoadFile(0, base<...>/analyzer.bif.zeek) +0.000000 MetaHookPre LoadFile(0, base<...>/api.zeek) 0.000000 MetaHookPre LoadFile(0, base<...>/bif) 0.000000 MetaHookPre LoadFile(0, base<...>/broker) 0.000000 MetaHookPre LoadFile(0, base<...>/cluster) @@ -1764,6 +1778,8 @@ 0.000000 MetaHookPre LoadFile(0, base<...>/strings.bif.zeek) 0.000000 MetaHookPre LoadFile(0, base<...>/strings.zeek) 0.000000 MetaHookPre LoadFile(0, base<...>/sumstats) +0.000000 MetaHookPre LoadFile(0, base<...>/supervisor) +0.000000 MetaHookPre LoadFile(0, base<...>/supervisor.bif.zeek) 0.000000 MetaHookPre LoadFile(0, base<...>/syslog) 0.000000 MetaHookPre LoadFile(0, base<...>/thresholds.zeek) 0.000000 MetaHookPre LoadFile(0, base<...>/time.zeek) @@ -1949,6 +1965,8 @@ 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_TEREDO, {3544/udp}) 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_VXLAN, {4789/udp}) 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_XMPP, {5222<...>/tcp}) +0.000000 | HookCallFunction Broker::__subscribe(zeek/supervisor) +0.000000 | HookCallFunction Broker::subscribe(zeek/supervisor) 0.000000 | HookCallFunction Cluster::is_enabled() 0.000000 | HookCallFunction Cluster::local_node_type() 0.000000 | HookCallFunction Cluster::register_pool([topic=zeek<...>/logger, node_type=Cluster::LOGGER, max_nodes=, exclusive=F]) @@ -2065,7 +2083,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1573527265.694831, node=zeek, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1579727603.636084, node=zeek, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Broker::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Config::LOG) @@ -2246,7 +2264,7 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1573527265.694831, node=zeek, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1579727603.636084, node=zeek, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction NetControl::check_plugins() 0.000000 | HookCallFunction NetControl::init() 0.000000 | HookCallFunction Notice::want_pp() @@ -2485,6 +2503,7 @@ 0.000000 | HookLoadFile .<...>/add-geodata.zeek 0.000000 | HookLoadFile .<...>/addrs.zeek 0.000000 | HookLoadFile .<...>/analyzer.bif.zeek +0.000000 | HookLoadFile .<...>/api.zeek 0.000000 | HookLoadFile .<...>/archive.sig 0.000000 | HookLoadFile .<...>/ascii.zeek 0.000000 | HookLoadFile .<...>/audio.sig @@ -2501,6 +2520,7 @@ 0.000000 | HookLoadFile .<...>/const.bif.zeek 0.000000 | HookLoadFile .<...>/consts.zeek 0.000000 | HookLoadFile .<...>/contents.zeek +0.000000 | HookLoadFile .<...>/control.zeek 0.000000 | HookLoadFile .<...>/ct-list.zeek 0.000000 | HookLoadFile .<...>/data.bif.zeek 0.000000 | HookLoadFile .<...>/dcc-send.zeek @@ -2566,6 +2586,7 @@ 0.000000 | HookLoadFile .<...>/store.zeek 0.000000 | HookLoadFile .<...>/strings.bif.zeek 0.000000 | HookLoadFile .<...>/sum.zeek +0.000000 | HookLoadFile .<...>/supervisor.bif.zeek 0.000000 | HookLoadFile .<...>/thresholds.zeek 0.000000 | HookLoadFile .<...>/top-k.bif.zeek 0.000000 | HookLoadFile .<...>/topk.zeek @@ -2588,6 +2609,7 @@ 0.000000 | HookLoadFile base<...>/addrs.zeek 0.000000 | HookLoadFile base<...>/analyzer 0.000000 | HookLoadFile base<...>/analyzer.bif.zeek +0.000000 | HookLoadFile base<...>/api.zeek 0.000000 | HookLoadFile base<...>/bif 0.000000 | HookLoadFile base<...>/broker 0.000000 | HookLoadFile base<...>/cluster @@ -2668,6 +2690,8 @@ 0.000000 | HookLoadFile base<...>/strings.bif.zeek 0.000000 | HookLoadFile base<...>/strings.zeek 0.000000 | HookLoadFile base<...>/sumstats +0.000000 | HookLoadFile base<...>/supervisor +0.000000 | HookLoadFile base<...>/supervisor.bif.zeek 0.000000 | HookLoadFile base<...>/syslog 0.000000 | HookLoadFile base<...>/thresholds.zeek 0.000000 | HookLoadFile base<...>/time.zeek @@ -2681,7 +2705,7 @@ 0.000000 | HookLoadFile base<...>/xmpp 0.000000 | HookLoadFile base<...>/zeek.bif.zeek 0.000000 | HookLogInit packet_filter 1/1 {ts (time), node (string), filter (string), init (bool), success (bool)} -0.000000 | HookLogWrite packet_filter [ts=1573527265.694831, node=zeek, filter=ip or not ip, init=T, success=T] +0.000000 | HookLogWrite packet_filter [ts=1579727603.636084, node=zeek, filter=ip or not ip, init=T, success=T] 0.000000 | HookQueueEvent NetControl::init() 0.000000 | HookQueueEvent filter_change_tracking() 0.000000 | HookQueueEvent zeek_init() diff --git a/testing/btest/Baseline/plugins.legacy/output b/testing/btest/Baseline/plugins.legacy/output deleted file mode 100644 index 675a884b16..0000000000 --- a/testing/btest/Baseline/plugins.legacy/output +++ /dev/null @@ -1,6 +0,0 @@ -Demo::Foo - A Foo test analyzer (dynamic, version 1.0.0) - [Analyzer] Foo (ANALYZER_FOO, enabled) - [Event] foo_message - -=== -foo_message, [orig_h=::1, orig_p=37927/tcp, resp_h=::1, resp_p=4242/tcp], Hello, Foo!\x0a diff --git a/testing/btest/Baseline/scripts.base.utils.urls/output b/testing/btest/Baseline/scripts.base.utils.urls/output index 2d8f5b2c4d..e10010ab10 100644 --- a/testing/btest/Baseline/scripts.base.utils.urls/output +++ b/testing/btest/Baseline/scripts.base.utils.urls/output @@ -6,6 +6,7 @@ [scheme=http, netlocation=hyphen-example.com, portnum=, path=/index.asp, file_name=index.asp, file_base=index, file_ext=asp, params={ [q] = 123 }] +[scheme=git, netlocation=git.kernel.org, portnum=, path=/pub/scm/linux/, file_name=, file_base=, file_ext=, params=] [scheme=, netlocation=dfasjdfasdfasdf, portnum=, path=/, file_name=, file_base=, file_ext=, params={ }] diff --git a/testing/btest/Baseline/supervisor.config-cluster/zeek.logger-1.stdout b/testing/btest/Baseline/supervisor.config-cluster/zeek.logger-1.stdout new file mode 100644 index 0000000000..6e844e4641 --- /dev/null +++ b/testing/btest/Baseline/supervisor.config-cluster/zeek.logger-1.stdout @@ -0,0 +1,2 @@ +supervised node zeek_init(), logger-1, Cluster::LOGGER +supervised node zeek_done(), logger-1, logger-1 diff --git a/testing/btest/Baseline/supervisor.config-cluster/zeek.manager.stdout b/testing/btest/Baseline/supervisor.config-cluster/zeek.manager.stdout new file mode 100644 index 0000000000..8c9567b50d --- /dev/null +++ b/testing/btest/Baseline/supervisor.config-cluster/zeek.manager.stdout @@ -0,0 +1,2 @@ +supervised node zeek_init(), manager, Cluster::MANAGER +supervised node zeek_done(), manager, manager diff --git a/testing/btest/Baseline/supervisor.config-cluster/zeek.proxy-1.stdout b/testing/btest/Baseline/supervisor.config-cluster/zeek.proxy-1.stdout new file mode 100644 index 0000000000..a11da802ad --- /dev/null +++ b/testing/btest/Baseline/supervisor.config-cluster/zeek.proxy-1.stdout @@ -0,0 +1,2 @@ +supervised node zeek_init(), proxy-1, Cluster::PROXY +supervised node zeek_done(), proxy-1, proxy-1 diff --git a/testing/btest/Baseline/supervisor.config-cluster/zeek.supervisor.out b/testing/btest/Baseline/supervisor.config-cluster/zeek.supervisor.out new file mode 100644 index 0000000000..545693ae5f --- /dev/null +++ b/testing/btest/Baseline/supervisor.config-cluster/zeek.supervisor.out @@ -0,0 +1,3 @@ +supervisor zeek_init() +shutting down +supervisor zeek_done() diff --git a/testing/btest/Baseline/supervisor.config-cluster/zeek.worker-1.stdout b/testing/btest/Baseline/supervisor.config-cluster/zeek.worker-1.stdout new file mode 100644 index 0000000000..a2f6e71d47 --- /dev/null +++ b/testing/btest/Baseline/supervisor.config-cluster/zeek.worker-1.stdout @@ -0,0 +1,2 @@ +supervised node zeek_init(), worker-1, Cluster::WORKER +supervised node zeek_done(), worker-1, worker-1 diff --git a/testing/btest/Baseline/supervisor.config-directory/zeek.qux.node.out b/testing/btest/Baseline/supervisor.config-directory/zeek.qux.node.out new file mode 100644 index 0000000000..101b306cf0 --- /dev/null +++ b/testing/btest/Baseline/supervisor.config-directory/zeek.qux.node.out @@ -0,0 +1,2 @@ +supervised node zeek_init() +supervised node zeek_done() diff --git a/testing/btest/Baseline/supervisor.config-directory/zeek.supervisor.out b/testing/btest/Baseline/supervisor.config-directory/zeek.supervisor.out new file mode 100644 index 0000000000..295c7211d6 --- /dev/null +++ b/testing/btest/Baseline/supervisor.config-directory/zeek.supervisor.out @@ -0,0 +1,3 @@ +supervisor zeek_init() +destroying node +supervisor zeek_done() diff --git a/testing/btest/Baseline/supervisor.config-output-redirect/zeek.qux.grault.stderr b/testing/btest/Baseline/supervisor.config-output-redirect/zeek.qux.grault.stderr new file mode 100644 index 0000000000..30a9fb1fde --- /dev/null +++ b/testing/btest/Baseline/supervisor.config-output-redirect/zeek.qux.grault.stderr @@ -0,0 +1,3 @@ +(stderr) supervised node zeek_init() +received termination signal +(stderr) supervised node zeek_done() diff --git a/testing/btest/Baseline/supervisor.config-output-redirect/zeek.qux.grault.stdout b/testing/btest/Baseline/supervisor.config-output-redirect/zeek.qux.grault.stdout new file mode 100644 index 0000000000..7b21bfad60 --- /dev/null +++ b/testing/btest/Baseline/supervisor.config-output-redirect/zeek.qux.grault.stdout @@ -0,0 +1,2 @@ +(stdout) supervised node zeek_init() +(stdout) supervised node zeek_done() diff --git a/testing/btest/Baseline/supervisor.config-output-redirect/zeek.supervisor.out b/testing/btest/Baseline/supervisor.config-output-redirect/zeek.supervisor.out new file mode 100644 index 0000000000..295c7211d6 --- /dev/null +++ b/testing/btest/Baseline/supervisor.config-output-redirect/zeek.supervisor.out @@ -0,0 +1,3 @@ +supervisor zeek_init() +destroying node +supervisor zeek_done() diff --git a/testing/btest/Baseline/supervisor.config-scripts/zeek.node.out b/testing/btest/Baseline/supervisor.config-scripts/zeek.node.out new file mode 100644 index 0000000000..64685fcf36 --- /dev/null +++ b/testing/btest/Baseline/supervisor.config-scripts/zeek.node.out @@ -0,0 +1,3 @@ +supervised node zeek_init() +supervised node loaded qux.zeek +supervised node zeek_done() diff --git a/testing/btest/Baseline/supervisor.config-scripts/zeek.supervisor.out b/testing/btest/Baseline/supervisor.config-scripts/zeek.supervisor.out new file mode 100644 index 0000000000..295c7211d6 --- /dev/null +++ b/testing/btest/Baseline/supervisor.config-scripts/zeek.supervisor.out @@ -0,0 +1,3 @@ +supervisor zeek_init() +destroying node +supervisor zeek_done() diff --git a/testing/btest/Baseline/supervisor.create/zeek.node.out b/testing/btest/Baseline/supervisor.create/zeek.node.out new file mode 100644 index 0000000000..101b306cf0 --- /dev/null +++ b/testing/btest/Baseline/supervisor.create/zeek.node.out @@ -0,0 +1,2 @@ +supervised node zeek_init() +supervised node zeek_done() diff --git a/testing/btest/Baseline/supervisor.create/zeek.supervisor.out b/testing/btest/Baseline/supervisor.create/zeek.supervisor.out new file mode 100644 index 0000000000..01bdfe9332 --- /dev/null +++ b/testing/btest/Baseline/supervisor.create/zeek.supervisor.out @@ -0,0 +1,2 @@ +supervisor zeek_init() +supervisor zeek_done() diff --git a/testing/btest/Baseline/supervisor.destroy/zeek.node.out b/testing/btest/Baseline/supervisor.destroy/zeek.node.out new file mode 100644 index 0000000000..101b306cf0 --- /dev/null +++ b/testing/btest/Baseline/supervisor.destroy/zeek.node.out @@ -0,0 +1,2 @@ +supervised node zeek_init() +supervised node zeek_done() diff --git a/testing/btest/Baseline/supervisor.destroy/zeek.supervisor.out b/testing/btest/Baseline/supervisor.destroy/zeek.supervisor.out new file mode 100644 index 0000000000..295c7211d6 --- /dev/null +++ b/testing/btest/Baseline/supervisor.destroy/zeek.supervisor.out @@ -0,0 +1,3 @@ +supervisor zeek_init() +destroying node +supervisor zeek_done() diff --git a/testing/btest/Baseline/supervisor.restart/zeek..stdout b/testing/btest/Baseline/supervisor.restart/zeek..stdout new file mode 100644 index 0000000000..15f6aa3c98 --- /dev/null +++ b/testing/btest/Baseline/supervisor.restart/zeek..stdout @@ -0,0 +1,3 @@ +got supervised node status, grault +got supervised node status, grault +got supervised node status, grault diff --git a/testing/btest/Baseline/supervisor.revive-leaf/zeek.node.out b/testing/btest/Baseline/supervisor.revive-leaf/zeek.node.out new file mode 100644 index 0000000000..101b306cf0 --- /dev/null +++ b/testing/btest/Baseline/supervisor.revive-leaf/zeek.node.out @@ -0,0 +1,2 @@ +supervised node zeek_init() +supervised node zeek_done() diff --git a/testing/btest/Baseline/supervisor.revive-leaf/zeek.supervisor.out b/testing/btest/Baseline/supervisor.revive-leaf/zeek.supervisor.out new file mode 100644 index 0000000000..7e3ffd50b4 --- /dev/null +++ b/testing/btest/Baseline/supervisor.revive-leaf/zeek.supervisor.out @@ -0,0 +1,7 @@ +supervisor zeek_init() +supervisor connected to peer +supervisor lost peer +supervisor connected to peer +supervisor lost peer +supervisor connected to peer +supervisor zeek_done() diff --git a/testing/btest/Baseline/supervisor.revive-stem/zeek.node.out b/testing/btest/Baseline/supervisor.revive-stem/zeek.node.out new file mode 100644 index 0000000000..101b306cf0 --- /dev/null +++ b/testing/btest/Baseline/supervisor.revive-stem/zeek.node.out @@ -0,0 +1,2 @@ +supervised node zeek_init() +supervised node zeek_done() diff --git a/testing/btest/Baseline/supervisor.revive-stem/zeek.supervisor.out b/testing/btest/Baseline/supervisor.revive-stem/zeek.supervisor.out new file mode 100644 index 0000000000..7e3ffd50b4 --- /dev/null +++ b/testing/btest/Baseline/supervisor.revive-stem/zeek.supervisor.out @@ -0,0 +1,7 @@ +supervisor zeek_init() +supervisor connected to peer +supervisor lost peer +supervisor connected to peer +supervisor lost peer +supervisor connected to peer +supervisor zeek_done() diff --git a/testing/btest/Baseline/supervisor.status/zeek..stdout b/testing/btest/Baseline/supervisor.status/zeek..stdout new file mode 100644 index 0000000000..59a36a37e0 --- /dev/null +++ b/testing/btest/Baseline/supervisor.status/zeek..stdout @@ -0,0 +1 @@ +got supervised node status, grault diff --git a/testing/btest/bifs/str_shell_escape.zeek b/testing/btest/bifs/str_shell_escape.zeek deleted file mode 100644 index f3f08b0072..0000000000 --- a/testing/btest/bifs/str_shell_escape.zeek +++ /dev/null @@ -1,15 +0,0 @@ -# -# @TEST-EXEC: zeek -b %INPUT >out -# @TEST-EXEC: btest-diff out - -event zeek_init() - { - local a = "echo ${TEST} > \"my file\""; - - print |a|; - print a; - - local b = str_shell_escape(a); - print |b|; - print b; - } diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index ef56fd2afa..258a7b04ae 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -1,5 +1,5 @@ [btest] -TestDirs = doc bifs language core scripts coverage signatures plugins broker +TestDirs = doc bifs language core scripts coverage signatures plugins broker supervisor TmpDir = %(testbase)s/.tmp BaselineDir = %(testbase)s/Baseline IgnoreDirs = .svn CVS .tmp diff --git a/testing/btest/core/load-explicit-bro-suffix-fallback.zeek b/testing/btest/core/load-explicit-bro-suffix-fallback.zeek index d2ce412209..4cf890cb09 100644 --- a/testing/btest/core/load-explicit-bro-suffix-fallback.zeek +++ b/testing/btest/core/load-explicit-bro-suffix-fallback.zeek @@ -1,5 +1,5 @@ -# @TEST-EXEC: zeek -b %INPUT >out -# @TEST-EXEC: btest-diff out +# @TEST-EXEC: zeek -b %INPUT >out 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out # We don't have a foo.bro, but we'll accept foo.zeek. @load foo.bro diff --git a/testing/btest/coverage/sphinx-zeekygen-docs.sh b/testing/btest/coverage/sphinx-zeekygen-docs.sh index b5e3d7262c..20e18c2364 100644 --- a/testing/btest/coverage/sphinx-zeekygen-docs.sh +++ b/testing/btest/coverage/sphinx-zeekygen-docs.sh @@ -8,10 +8,15 @@ # @TEST-EXEC: bash $SCRIPTS/update-zeekygen-docs.sh ./doc # @TEST-EXEC: bash %INPUT +# This test isn't run on Travis or Cirrus CI for pull-requests. Instead, +# the person merging to master will manually update zeek-docs. + +if [ -n "$CIRRUS_PR" ]; then + exit 0 +fi + if [ -n "$TRAVIS_PULL_REQUEST" ]; then if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then - # Don't run this test on Travis for pull-requests, just let someone - # manually update zeek-docs for things when merging to master. exit 0 fi fi diff --git a/testing/btest/language/bro_init.zeek b/testing/btest/language/bro_init.zeek new file mode 100644 index 0000000000..0beda04a7b --- /dev/null +++ b/testing/btest/language/bro_init.zeek @@ -0,0 +1,12 @@ +# @TEST-EXEC-FAIL: zeek -b %INPUT >out 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out + +event bro_init() + { + print "ran bro_init()"; + } + +event bro_done() + { + print "ran bro_done()"; + } diff --git a/testing/btest/language/bro_script_loaded.zeek b/testing/btest/language/bro_script_loaded.zeek new file mode 100644 index 0000000000..fb3d080a31 --- /dev/null +++ b/testing/btest/language/bro_script_loaded.zeek @@ -0,0 +1,8 @@ +# @TEST-EXEC-FAIL: zeek -b %INPUT >out 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out + +event bro_script_loaded(path: string, level: count) &priority=5 + { + if ( /zeek_script_loaded.zeek/ in path ) + print "bro_script_loaded priority 5"; + } diff --git a/testing/btest/language/zeek_init.zeek b/testing/btest/language/zeek_init.zeek index c1ca3ba65c..b2fb37093b 100644 --- a/testing/btest/language/zeek_init.zeek +++ b/testing/btest/language/zeek_init.zeek @@ -1,36 +1,24 @@ -# @TEST-EXEC: zeek -b %INPUT >out +# @TEST-EXEC: zeek -b %INPUT >out 2>&1 # @TEST-EXEC: btest-diff out +event zeek_init() &priority=-10 + { + print "zeek_init at priority -10!"; + } event zeek_init() &priority=10 { print "zeek_init at priority 10!"; } -event bro_init() &priority=5 - { - print "bro_init at priority 5!"; - } - event zeek_init() &priority=0 { print "zeek_init at priority 0!"; } -event bro_init() &priority=-10 +event zeek_done() &priority=-10 { - print "bro_init at priority -10!"; - } - - -event zeek_done() &priority=10 - { - print "zeek_done at priority 10!"; - } - -event bro_done() &priority=5 - { - print "bro_done at priority 5!"; + print "zeek_done at priority -10!"; } event zeek_done() &priority=0 @@ -38,7 +26,7 @@ event zeek_done() &priority=0 print "zeek_done at priority 0!"; } -event bro_done() &priority=-10 +event zeek_done() &priority=10 { - print "bro_done at priority -10!"; + print "zeek_done at priority 10!"; } diff --git a/testing/btest/language/zeek_script_loaded.zeek b/testing/btest/language/zeek_script_loaded.zeek index 9011790e93..0a0004ada3 100644 --- a/testing/btest/language/zeek_script_loaded.zeek +++ b/testing/btest/language/zeek_script_loaded.zeek @@ -1,4 +1,4 @@ -# @TEST-EXEC: zeek -b %INPUT >out +# @TEST-EXEC: zeek -b %INPUT >out 2>&1 # @TEST-EXEC: btest-diff out event zeek_script_loaded(path: string, level: count) &priority=10 @@ -7,20 +7,8 @@ event zeek_script_loaded(path: string, level: count) &priority=10 print "zeek_script_loaded priority 10"; } -event bro_script_loaded(path: string, level: count) &priority=5 - { - if ( /zeek_script_loaded.zeek/ in path ) - print "bro_script_loaded priority 5"; - } - event zeek_script_loaded(path: string, level: count) &priority=0 { if ( /zeek_script_loaded.zeek/ in path ) print "zeek_script_loaded priority 0"; } - -event bro_script_loaded(path: string, level: count) &priority=-10 - { - if ( /zeek_script_loaded.zeek/ in path ) - print "bro_script_loaded priority -10"; - } diff --git a/testing/btest/plugins/legacy-plugin/.btest-ignore b/testing/btest/plugins/legacy-plugin/.btest-ignore deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/testing/btest/plugins/legacy-plugin/CMakeLists.txt b/testing/btest/plugins/legacy-plugin/CMakeLists.txt deleted file mode 100644 index 92e1a90e9d..0000000000 --- a/testing/btest/plugins/legacy-plugin/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ - -project(Zeek-Plugin-Demo-Foo) - -cmake_minimum_required(VERSION 2.6.3) - -if ( NOT BRO_DIST ) - message(FATAL_ERROR "BRO_DIST not set") -endif () - -set(CMAKE_MODULE_PATH ${BRO_DIST}/cmake) - -include(BroPlugin) - -bro_plugin_begin(Demo Foo) -bro_plugin_cc(src/Plugin.cc) -bro_plugin_cc(src/Foo.cc) -bro_plugin_bif(src/events.bif) -bro_plugin_pac(src/foo.pac src/foo-protocol.pac src/foo-analyzer.pac) -bro_plugin_end() diff --git a/testing/btest/plugins/legacy-plugin/scripts/Demo/Foo/base/main.zeek b/testing/btest/plugins/legacy-plugin/scripts/Demo/Foo/base/main.zeek deleted file mode 100644 index 76c63723b7..0000000000 --- a/testing/btest/plugins/legacy-plugin/scripts/Demo/Foo/base/main.zeek +++ /dev/null @@ -1,7 +0,0 @@ - -const ports = { 4242/tcp }; - -event zeek_init() &priority=5 - { - Analyzer::register_for_ports(Analyzer::ANALYZER_FOO, ports); - } diff --git a/testing/btest/plugins/legacy-plugin/scripts/__load__.zeek b/testing/btest/plugins/legacy-plugin/scripts/__load__.zeek deleted file mode 100644 index 330718c604..0000000000 --- a/testing/btest/plugins/legacy-plugin/scripts/__load__.zeek +++ /dev/null @@ -1 +0,0 @@ -@load Demo/Foo/base/main diff --git a/testing/btest/plugins/legacy-plugin/src/Foo.cc b/testing/btest/plugins/legacy-plugin/src/Foo.cc deleted file mode 100644 index be3c52a98b..0000000000 --- a/testing/btest/plugins/legacy-plugin/src/Foo.cc +++ /dev/null @@ -1,59 +0,0 @@ - -#include "Foo.h" -#include "foo_pac.h" -#include "events.bif.h" - -#include - -using namespace plugin::Demo_Foo; - -Foo::Foo(Connection* conn) - : analyzer::tcp::TCP_ApplicationAnalyzer("Foo", conn) - { - interp = new binpac::Foo::Foo_Conn(this); - } - -Foo::~Foo() - { - delete interp; - } - -void Foo::Done() - { - analyzer::tcp::TCP_ApplicationAnalyzer::Done(); - - interp->FlowEOF(true); - interp->FlowEOF(false); - } - -void Foo::EndpointEOF(bool is_orig) - { - analyzer::tcp::TCP_ApplicationAnalyzer::EndpointEOF(is_orig); - interp->FlowEOF(is_orig); - } - -void Foo::DeliverStream(int len, const u_char* data, bool orig) - { - analyzer::tcp::TCP_ApplicationAnalyzer::DeliverStream(len, data, orig); - - assert(TCP()); - - if ( TCP()->IsPartial() ) - // punt on partial. - return; - - try - { - interp->NewData(orig, data, data + len); - } - catch ( const binpac::Exception& e ) - { - ProtocolViolation(fmt("Binpac exception: %s", e.c_msg())); - } - } - -void Foo::Undelivered(uint64 seq, int len, bool orig) - { - analyzer::tcp::TCP_ApplicationAnalyzer::Undelivered(seq, len, orig); - interp->NewGap(orig, len); - } diff --git a/testing/btest/plugins/legacy-plugin/src/Foo.h b/testing/btest/plugins/legacy-plugin/src/Foo.h deleted file mode 100644 index f68aa3814d..0000000000 --- a/testing/btest/plugins/legacy-plugin/src/Foo.h +++ /dev/null @@ -1,29 +0,0 @@ - -#pragma once - -#include "analyzer/protocol/tcp/TCP.h" -#include "analyzer/protocol/pia/PIA.h" - -namespace binpac { namespace Foo { class Foo_Conn; } } - -namespace plugin { -namespace Demo_Foo { - -class Foo : public analyzer::tcp::TCP_ApplicationAnalyzer { -public: - Foo(Connection* conn); - ~Foo(); - - virtual void Done(); - virtual void DeliverStream(int len, const u_char* data, bool orig); - virtual void Undelivered(uint64 seq, int len, bool orig); - virtual void EndpointEOF(bool is_orig); - - static analyzer::Analyzer* Instantiate(Connection* conn) - { return new Foo(conn); } - -protected: - binpac::Foo::Foo_Conn* interp; -}; - -} } diff --git a/testing/btest/plugins/legacy-plugin/src/Plugin.cc b/testing/btest/plugins/legacy-plugin/src/Plugin.cc deleted file mode 100644 index bd2662d67c..0000000000 --- a/testing/btest/plugins/legacy-plugin/src/Plugin.cc +++ /dev/null @@ -1,21 +0,0 @@ - -#include "Plugin.h" - -#include "Foo.h" - -namespace plugin { namespace Demo_Foo { Plugin plugin; } } - -using namespace plugin::Demo_Foo; - -plugin::Configuration Plugin::Configure() - { - AddComponent(new ::analyzer::Component("Foo", plugin::Demo_Foo::Foo::Instantiate)); - - plugin::Configuration config; - config.name = "Demo::Foo"; - config.description = "A Foo test analyzer"; - config.version.major = 1; - config.version.minor = 0; - config.version.patch = 0; - return config; - } diff --git a/testing/btest/plugins/legacy-plugin/src/events.bif b/testing/btest/plugins/legacy-plugin/src/events.bif deleted file mode 100644 index 4603fe4cf6..0000000000 --- a/testing/btest/plugins/legacy-plugin/src/events.bif +++ /dev/null @@ -1,2 +0,0 @@ - -event foo_message%(c: connection, data: string%); diff --git a/testing/btest/plugins/legacy-plugin/src/foo-analyzer.pac b/testing/btest/plugins/legacy-plugin/src/foo-analyzer.pac deleted file mode 100644 index a210a8430c..0000000000 --- a/testing/btest/plugins/legacy-plugin/src/foo-analyzer.pac +++ /dev/null @@ -1,15 +0,0 @@ - -refine connection Foo_Conn += { - - function Foo_data(msg: Foo_Message): bool - %{ - StringVal* data = new StringVal(${msg.data}.length(), (const char*) ${msg.data}.data()); - BifEvent::generate_foo_message(bro_analyzer(), bro_analyzer()->Conn(), data); - return true; - %} - -}; - -refine typeattr Foo_Message += &let { - proc: bool = $context.connection.Foo_data(this); -}; diff --git a/testing/btest/plugins/legacy-plugin/src/foo-protocol.pac b/testing/btest/plugins/legacy-plugin/src/foo-protocol.pac deleted file mode 100644 index 892513c4f0..0000000000 --- a/testing/btest/plugins/legacy-plugin/src/foo-protocol.pac +++ /dev/null @@ -1,4 +0,0 @@ - -type Foo_Message(is_orig: bool) = record { - data: bytestring &restofdata; -}; diff --git a/testing/btest/plugins/legacy-plugin/src/foo.pac b/testing/btest/plugins/legacy-plugin/src/foo.pac deleted file mode 100644 index 826bcc624e..0000000000 --- a/testing/btest/plugins/legacy-plugin/src/foo.pac +++ /dev/null @@ -1,26 +0,0 @@ -%include binpac.pac -%include bro.pac - -%extern{ -#include "Foo.h" - -#include "events.bif.h" -%} - -analyzer Foo withcontext { - connection: Foo_Conn; - flow: Foo_Flow; -}; - -connection Foo_Conn(bro_analyzer: BroAnalyzer) { - upflow = Foo_Flow(true); - downflow = Foo_Flow(false); -}; - -%include foo-protocol.pac - -flow Foo_Flow(is_orig: bool) { - datagram = Foo_Message(is_orig) withcontext(connection, this); -}; - -%include foo-analyzer.pac diff --git a/testing/btest/plugins/legacy.zeek b/testing/btest/plugins/legacy.zeek deleted file mode 100644 index bb663d744b..0000000000 --- a/testing/btest/plugins/legacy.zeek +++ /dev/null @@ -1,14 +0,0 @@ -# Test that legacy Bro plugins still work. -# @TEST-EXEC: ${DIST}/aux/zeek-aux/plugin-support/init-plugin -u . Demo Foo -# @TEST-EXEC: cp -r %DIR/legacy-plugin/* . -# @TEST-EXEC: ./configure --bro-dist=${DIST} && make -# @TEST-EXEC: unset ZEEK_PLUGIN_PATH; BRO_PLUGIN_PATH=`pwd` zeek -NN Demo::Foo >>output -# @TEST-EXEC: echo === >>output -# @TEST-EXEC: unset ZEEK_PLUGIN_PATH; BRO_PLUGIN_PATH=`pwd` zeek -r $TRACES/port4242.trace %INPUT >>output -# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff output - -event foo_message(c: connection, data: string) - { - print "foo_message", c$id, data; - } - diff --git a/testing/btest/scripts/base/utils/urls.test b/testing/btest/scripts/base/utils/urls.test index 666f805edb..c307ee601e 100644 --- a/testing/btest/scripts/base/utils/urls.test +++ b/testing/btest/scripts/base/utils/urls.test @@ -8,6 +8,7 @@ print decompose_uri("https://www.example.com/"); print decompose_uri("http://example.com:99/test//?foo=bar"); print decompose_uri("ftp://1.2.3.4/pub/files/something.exe"); print decompose_uri("http://hyphen-example.com/index.asp?q=123"); +print decompose_uri("git://git.kernel.org:/pub/scm/linux/"); # This is mostly undefined behavior but it doesn't give any # reporter messages at least. diff --git a/testing/btest/supervisor/config-cluster.zeek b/testing/btest/supervisor/config-cluster.zeek new file mode 100644 index 0000000000..6f8d290e0d --- /dev/null +++ b/testing/btest/supervisor/config-cluster.zeek @@ -0,0 +1,89 @@ +# @TEST-PORT: SUPERVISOR_PORT +# @TEST-PORT: MANAGER_PORT +# @TEST-PORT: LOGGER_PORT +# @TEST-PORT: PROXY_PORT +# @TEST-PORT: WORKER_PORT +# @TEST-EXEC: btest-bg-run zeek zeek -j -b %INPUT +# @TEST-EXEC: btest-bg-wait 20 +# @TEST-EXEC: btest-diff zeek/supervisor.out +# @TEST-EXEC: btest-diff zeek/manager/stdout +# @TEST-EXEC: btest-diff zeek/logger-1/stdout +# @TEST-EXEC: btest-diff zeek/worker-1/stdout +# @TEST-EXEC: btest-diff zeek/proxy-1/stdout + +@load base/frameworks/cluster + +# So the supervised node doesn't terminate right away. +redef exit_only_after_terminate=T; + +global supervisor_output_file: file; +global topic = "test-topic"; +global peer_count = 0; + +event shutdown() + { + print supervisor_output_file, "shutting down"; + terminate(); + } + +event zeek_init() + { + if ( Supervisor::is_supervisor() ) + { + Broker::subscribe(topic); + Broker::listen("127.0.0.1", to_port(getenv("SUPERVISOR_PORT"))); + supervisor_output_file = open("supervisor.out"); + print supervisor_output_file, "supervisor zeek_init()"; + + local cluster: table[string] of Supervisor::ClusterEndpoint; + cluster["manager"] = [$role=Supervisor::MANAGER, $host=127.0.0.1, + $p=to_port(getenv("MANAGER_PORT"))]; + cluster["logger-1"] = [$role=Supervisor::LOGGER, $host=127.0.0.1, + $p=to_port(getenv("LOGGER_PORT"))]; + cluster["proxy-1"] = [$role=Supervisor::PROXY, $host=127.0.0.1, + $p=to_port(getenv("PROXY_PORT"))]; + cluster["worker-1"] = [$role=Supervisor::WORKER, $host=127.0.0.1, + $p=to_port(getenv("WORKER_PORT"))]; + + for ( n, ep in cluster ) + { + local sn = Supervisor::NodeConfig($name = n); + sn$cluster = cluster; + sn$directory = n; + sn$stdout_file = "stdout"; + sn$stderr_file = "stderr"; + local res = Supervisor::create(sn); + + if ( res != "" ) + print fmt("failed to create node %s: %s", n, res); + } + } + else + { + Broker::peer("127.0.0.1", to_port(getenv("SUPERVISOR_PORT"))); + print "supervised node zeek_init()", Cluster::node, Cluster::local_node_type(); + } + } + +event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) + { + ++peer_count; + + if ( Supervisor::is_supervised() ) + { + if ( Cluster::node == "manager" && peer_count == 4 ) + Broker::publish(topic, shutdown); + } + } + +event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) + { + } + +event zeek_done() + { + if ( Supervisor::is_supervised() ) + print "supervised node zeek_done()", Cluster::node, Supervisor::node()$name; + else + print supervisor_output_file, "supervisor zeek_done()"; + } diff --git a/testing/btest/supervisor/config-directory.zeek b/testing/btest/supervisor/config-directory.zeek new file mode 100644 index 0000000000..bcf9ebd69f --- /dev/null +++ b/testing/btest/supervisor/config-directory.zeek @@ -0,0 +1,60 @@ +# @TEST-PORT: BROKER_PORT +# @TEST-EXEC: btest-bg-run zeek zeek -j -b %INPUT +# @TEST-EXEC: btest-bg-wait 20 +# @TEST-EXEC: btest-diff zeek/supervisor.out +# @TEST-EXEC: btest-diff zeek/qux/node.out + +# So the supervised node doesn't terminate right away. +redef exit_only_after_terminate=T; + +global supervisor_output_file: file; +global node_output_file: file; +global topic = "test-topic"; + +event do_destroy() + { + print supervisor_output_file, "destroying node"; + Supervisor::destroy("grault"); + } + +event zeek_init() + { + if ( Supervisor::is_supervisor() ) + { + Broker::subscribe(topic); + Broker::listen("127.0.0.1", to_port(getenv("BROKER_PORT"))); + supervisor_output_file = open("supervisor.out"); + print supervisor_output_file, "supervisor zeek_init()"; + local sn = Supervisor::NodeConfig($name="grault", $directory="qux"); + local res = Supervisor::create(sn); + + if ( res != "" ) + print supervisor_output_file, res; + } + else + { + Broker::peer("127.0.0.1", to_port(getenv("BROKER_PORT"))); + node_output_file = open("node.out"); + print node_output_file, "supervised node zeek_init()"; + } + } + +event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) + { + if ( Supervisor::is_supervised() ) + Broker::publish(topic, do_destroy); + } + +event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) + { + # Should only be run by supervisor + terminate(); + } + +event zeek_done() + { + if ( Supervisor::is_supervised() ) + print node_output_file, "supervised node zeek_done()"; + else + print supervisor_output_file, "supervisor zeek_done()"; + } diff --git a/testing/btest/supervisor/config-output-redirect.zeek b/testing/btest/supervisor/config-output-redirect.zeek new file mode 100644 index 0000000000..78a8d3d6ad --- /dev/null +++ b/testing/btest/supervisor/config-output-redirect.zeek @@ -0,0 +1,66 @@ +# @TEST-PORT: BROKER_PORT +# @TEST-EXEC: btest-bg-run zeek zeek -j -b %INPUT +# @TEST-EXEC: btest-bg-wait 20 +# @TEST-EXEC: btest-diff zeek/supervisor.out +# @TEST-EXEC: btest-diff zeek/qux/grault.stdout +# @TEST-EXEC: btest-diff zeek/qux/grault.stderr + +# So the supervised node doesn't terminate right away. +redef exit_only_after_terminate=T; + +global supervisor_output_file: file; +global topic = "test-topic"; +global stderr = open("/dev/stderr"); + +event do_destroy() + { + print supervisor_output_file, "destroying node"; + Supervisor::destroy("grault"); + } + +event zeek_init() + { + if ( Supervisor::is_supervisor() ) + { + Broker::subscribe(topic); + Broker::listen("127.0.0.1", to_port(getenv("BROKER_PORT"))); + supervisor_output_file = open("supervisor.out"); + print supervisor_output_file, "supervisor zeek_init()"; + local sn = Supervisor::NodeConfig($name="grault", $directory="qux", + $stdout_file="grault.stdout", + $stderr_file="grault.stderr"); + local res = Supervisor::create(sn); + + if ( res != "" ) + print supervisor_output_file, res; + } + else + { + Broker::peer("127.0.0.1", to_port(getenv("BROKER_PORT"))); + print "(stdout) supervised node zeek_init()"; + print stderr, "(stderr) supervised node zeek_init()"; + } + } + +event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) + { + if ( Supervisor::is_supervised() ) + Broker::publish(topic, do_destroy); + } + +event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) + { + # Should only be run by supervisor + terminate(); + } + +event zeek_done() + { + if ( Supervisor::is_supervised() ) + { + print "(stdout) supervised node zeek_done()"; + print stderr, "(stderr) supervised node zeek_done()"; + } + else + print supervisor_output_file, "supervisor zeek_done()"; + } diff --git a/testing/btest/supervisor/config-scripts.zeek b/testing/btest/supervisor/config-scripts.zeek new file mode 100644 index 0000000000..beeaa567f5 --- /dev/null +++ b/testing/btest/supervisor/config-scripts.zeek @@ -0,0 +1,70 @@ +# @TEST-PORT: BROKER_PORT +# @TEST-EXEC: btest-bg-run zeek zeek -j -b %INPUT +# @TEST-EXEC: btest-bg-wait 20 +# @TEST-EXEC: btest-diff zeek/supervisor.out +# @TEST-EXEC: btest-diff zeek/node.out + +# So the supervised node doesn't terminate right away. +redef exit_only_after_terminate=T; + +global supervisor_output_file: file; +global node_output_file: file; +global topic = "test-topic"; + +event do_destroy() + { + print supervisor_output_file, "destroying node"; + Supervisor::destroy("grault"); + } + +event zeek_init() + { + if ( Supervisor::is_supervisor() ) + { + Broker::subscribe(topic); + Broker::listen("127.0.0.1", to_port(getenv("BROKER_PORT"))); + supervisor_output_file = open("supervisor.out"); + print supervisor_output_file, "supervisor zeek_init()"; + local sn = Supervisor::NodeConfig($name="grault", + $scripts=vector("../qux.zeek")); + local res = Supervisor::create(sn); + + if ( res != "" ) + print supervisor_output_file, res; + } + else + { + Broker::peer("127.0.0.1", to_port(getenv("BROKER_PORT"))); + node_output_file = open("node.out"); + print node_output_file, "supervised node zeek_init()"; + } + } + +event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) + { + if ( Supervisor::is_supervised() ) + Broker::publish(topic, do_destroy); + } + +event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) + { + # Should only be run by supervisor + terminate(); + } + +event zeek_done() + { + if ( Supervisor::is_supervised() ) + print node_output_file, "supervised node zeek_done()"; + else + print supervisor_output_file, "supervisor zeek_done()"; + } + +@TEST-START-FILE qux.zeek + +event zeek_init() &priority=-10 + { + print node_output_file, "supervised node loaded qux.zeek"; + } + +@TEST-END-FILE diff --git a/testing/btest/supervisor/create.zeek b/testing/btest/supervisor/create.zeek new file mode 100644 index 0000000000..3455d63f12 --- /dev/null +++ b/testing/btest/supervisor/create.zeek @@ -0,0 +1,42 @@ +# @TEST-EXEC: btest-bg-run zeek zeek -j -b %INPUT +# @TEST-EXEC: btest-bg-wait 20 +# @TEST-EXEC: btest-diff zeek/supervisor.out +# @TEST-EXEC: btest-diff zeek/node.out + +# So the supervised node doesn't terminate right away. +redef exit_only_after_terminate=T; + +global supervisor_output_file: file; +global node_output_file: file; + +event zeek_init() + { + local pid_file = "supervisor.pid"; + + if ( Supervisor::is_supervisor() ) + { + supervisor_output_file = open("supervisor.out"); + print supervisor_output_file, "supervisor zeek_init()"; + local f = open(pid_file); + print f, getpid(); + local sn = Supervisor::NodeConfig($name="grault"); + local res = Supervisor::create(sn); + + if ( res != "" ) + print supervisor_output_file, res; + } + else + { + node_output_file = open("node.out"); + print node_output_file, "supervised node zeek_init()"; + system(fmt("kill `cat %s`", pid_file)); + } + } + +event zeek_done() + { + if ( Supervisor::is_supervised() ) + print node_output_file, "supervised node zeek_done()"; + else + print supervisor_output_file, "supervisor zeek_done()"; + } diff --git a/testing/btest/supervisor/destroy.zeek b/testing/btest/supervisor/destroy.zeek new file mode 100644 index 0000000000..233870b5dc --- /dev/null +++ b/testing/btest/supervisor/destroy.zeek @@ -0,0 +1,60 @@ +# @TEST-PORT: BROKER_PORT +# @TEST-EXEC: btest-bg-run zeek zeek -j -b %INPUT +# @TEST-EXEC: btest-bg-wait 20 +# @TEST-EXEC: btest-diff zeek/supervisor.out +# @TEST-EXEC: btest-diff zeek/node.out + +# So the supervised node doesn't terminate right away. +redef exit_only_after_terminate=T; + +global supervisor_output_file: file; +global node_output_file: file; +global topic = "test-topic"; + +event do_destroy() + { + print supervisor_output_file, "destroying node"; + Supervisor::destroy("grault"); + } + +event zeek_init() + { + if ( Supervisor::is_supervisor() ) + { + Broker::subscribe(topic); + Broker::listen("127.0.0.1", to_port(getenv("BROKER_PORT"))); + supervisor_output_file = open("supervisor.out"); + print supervisor_output_file, "supervisor zeek_init()"; + local sn = Supervisor::NodeConfig($name="grault"); + local res = Supervisor::create(sn); + + if ( res != "" ) + print supervisor_output_file, res; + } + else + { + Broker::peer("127.0.0.1", to_port(getenv("BROKER_PORT"))); + node_output_file = open("node.out"); + print node_output_file, "supervised node zeek_init()"; + } + } + +event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) + { + if ( Supervisor::is_supervised() ) + Broker::publish(topic, do_destroy); + } + +event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) + { + # Should only be run by supervisor + terminate(); + } + +event zeek_done() + { + if ( Supervisor::is_supervised() ) + print node_output_file, "supervised node zeek_done()"; + else + print supervisor_output_file, "supervisor zeek_done()"; + } diff --git a/testing/btest/supervisor/restart.zeek b/testing/btest/supervisor/restart.zeek new file mode 100644 index 0000000000..e9ddeed97d --- /dev/null +++ b/testing/btest/supervisor/restart.zeek @@ -0,0 +1,66 @@ +# @TEST-EXEC: btest-bg-run zeek zeek -j -b %INPUT +# @TEST-EXEC: btest-bg-wait 20 +# @TEST-EXEC: btest-diff zeek/.stdout + +# So the supervised node doesn't terminate right away. +redef exit_only_after_terminate=T; + +global node_pid: int = 0; +global status_count = 0; +global check_interval = 0.1sec; + +event check_status(name: string &default="") + { + local s = Supervisor::status(name); + local ns = s$nodes["grault"]; + + if ( ! ns?$pid ) + { + schedule check_interval { check_status() }; + return; + } + + if ( status_count > 0 && node_pid == ns$pid ) + { + schedule check_interval { check_status() }; + return; + } + + print "got supervised node status", ns$node$name; + + node_pid = ns$pid; + ++status_count; + + if ( status_count == 1 ) + { + Supervisor::restart(); + schedule check_interval { check_status() }; + } + else if ( status_count == 2 ) + { + Supervisor::restart("grault"); + schedule check_interval { check_status("grault") }; + } + else + terminate(); + } + +event zeek_init() + { + if ( Supervisor::is_supervisor() ) + { + local sn = Supervisor::NodeConfig($name="grault"); + local res = Supervisor::create(sn); + + if ( res != "" ) + print "failed to create node", res; + + sn$name = "qux"; + res = Supervisor::create(sn); + + if ( res != "" ) + print "failed to create node", res; + + event check_status(); + } + } diff --git a/testing/btest/supervisor/revive-leaf.zeek b/testing/btest/supervisor/revive-leaf.zeek new file mode 100644 index 0000000000..448970283b --- /dev/null +++ b/testing/btest/supervisor/revive-leaf.zeek @@ -0,0 +1,70 @@ +# @TEST-PORT: BROKER_PORT +# @TEST-EXEC: btest-bg-run zeek zeek -j -b %INPUT +# @TEST-EXEC: btest-bg-wait 20 +# @TEST-EXEC: btest-diff zeek/supervisor.out +# @TEST-EXEC: btest-diff zeek/node.out + +# So the supervised node doesn't terminate right away. +redef exit_only_after_terminate=T; + +global supervisor_output_file: file; +global node_output_file: file; +global topic = "test-topic"; +global peers_added = 0; + +event kill_self() + { + system(fmt("kill %s", getpid())); + } + +event zeek_init() + { + if ( Supervisor::is_supervisor() ) + { + Broker::subscribe(topic); + Broker::listen("127.0.0.1", to_port(getenv("BROKER_PORT"))); + supervisor_output_file = open("supervisor.out"); + print supervisor_output_file, "supervisor zeek_init()"; + local sn = Supervisor::NodeConfig($name="grault"); + local res = Supervisor::create(sn); + + if ( res != "" ) + print supervisor_output_file, res; + } + else + { + Broker::subscribe(topic); + Broker::peer("127.0.0.1", to_port(getenv("BROKER_PORT"))); + node_output_file = open("node.out"); + print node_output_file, "supervised node zeek_init()"; + } + } + +event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) + { + ++peers_added; + + if ( Supervisor::is_supervisor() ) + { + print supervisor_output_file, "supervisor connected to peer"; + + if ( peers_added == 3 ) + terminate(); + else + Broker::publish(topic, kill_self); + } + } + +event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) + { + if ( Supervisor::is_supervisor() ) + print supervisor_output_file, "supervisor lost peer"; + } + +event zeek_done() + { + if ( Supervisor::is_supervisor() ) + print supervisor_output_file, "supervisor zeek_done()"; + else + print node_output_file, "supervised node zeek_done()"; + } diff --git a/testing/btest/supervisor/revive-stem.zeek b/testing/btest/supervisor/revive-stem.zeek new file mode 100644 index 0000000000..37dc3e5053 --- /dev/null +++ b/testing/btest/supervisor/revive-stem.zeek @@ -0,0 +1,65 @@ +# @TEST-PORT: BROKER_PORT +# @TEST-EXEC: btest-bg-run zeek zeek -j -b %INPUT +# @TEST-EXEC: btest-bg-wait 20 +# @TEST-EXEC: btest-diff zeek/supervisor.out +# @TEST-EXEC: btest-diff zeek/node.out + +# So the supervised node doesn't terminate right away. +redef exit_only_after_terminate=T; + +global supervisor_output_file: file; +global node_output_file: file; +global topic = "test-topic"; +global peers_added = 0; + +event zeek_init() + { + if ( Supervisor::is_supervisor() ) + { + Broker::subscribe(topic); + Broker::listen("127.0.0.1", to_port(getenv("BROKER_PORT"))); + supervisor_output_file = open("supervisor.out"); + print supervisor_output_file, "supervisor zeek_init()"; + local sn = Supervisor::NodeConfig($name="grault"); + local res = Supervisor::create(sn); + + if ( res != "" ) + print supervisor_output_file, res; + } + else + { + Broker::subscribe(topic); + Broker::peer("127.0.0.1", to_port(getenv("BROKER_PORT"))); + node_output_file = open("node.out"); + print node_output_file, "supervised node zeek_init()"; + } + } + +event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) + { + ++peers_added; + + if ( Supervisor::is_supervisor() ) + { + print supervisor_output_file, "supervisor connected to peer"; + + if ( peers_added == 3 ) + terminate(); + else + system(fmt("kill %s", Supervisor::__stem_pid())); + } + } + +event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) + { + if ( Supervisor::is_supervisor() ) + print supervisor_output_file, "supervisor lost peer"; + } + +event zeek_done() + { + if ( Supervisor::is_supervisor() ) + print supervisor_output_file, "supervisor zeek_done()"; + else + print node_output_file, "supervised node zeek_done()"; + } diff --git a/testing/btest/supervisor/status.zeek b/testing/btest/supervisor/status.zeek new file mode 100644 index 0000000000..e9ce4cdeca --- /dev/null +++ b/testing/btest/supervisor/status.zeek @@ -0,0 +1,34 @@ +# @TEST-EXEC: btest-bg-run zeek zeek -j -b %INPUT +# @TEST-EXEC: btest-bg-wait 20 +# @TEST-EXEC: btest-diff zeek/.stdout + +# So the supervised node doesn't terminate right away. +redef exit_only_after_terminate=T; + +event check_status() + { + local s = Supervisor::status(); + local ns = s$nodes["grault"]; + + if ( ! ns?$pid ) + schedule 0.25sec { check_status() }; + else + { + print "got supervised node status", ns$node$name; + terminate(); + } + } + +event zeek_init() + { + if ( Supervisor::is_supervisor() ) + { + local sn = Supervisor::NodeConfig($name="grault"); + local res = Supervisor::create(sn); + + if ( res != "" ) + print "failed to create node", res; + + event check_status(); + } + } diff --git a/zeek-wrapper.in b/zeek-wrapper.in index 1372c573e7..939424d28b 100755 --- a/zeek-wrapper.in +++ b/zeek-wrapper.in @@ -1,14 +1,13 @@ #! /usr/bin/env bash # -# Wrapper to continue supporting old names of executables. -# This will print a deprecation warning to stderr if (1) stdin/stdout/stderr -# are all connected to a tty, and (2) the environment variable ZEEK_IS_BRO -# is unset. +# Wrapper to continue reporting usage of old names of executables. +# This will print an error to stderr if stdin/stdout/stderr +# are all connected to a tty. It will then abort with an error +# exit code. function deprecated { cat >&2 <