Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-06 16:48:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge remote-tracking branch 'origin/master' into topic/johanna/bloomfilter

Browse source
This commit is contained in:
Johanna Amann 2022-05-03 16:43:36 +01:00 • committed by Johanna Amann
commit 42bc6db359
377 changed files with 11627 additions and 8961 deletions
Download patch file Download diff file Expand all files Collapse all files

65
.cirrus.yml
View file

@ -8,11 +8,13 @@
cpus: &CPUS 4
btest_jobs: &BTEST_JOBS 4
btest_retries: &BTEST_RETRIES 2
memory: &MEMORY 4GB
memory: &MEMORY 8GB
config: &CONFIG --build-type=release --disable-broker-tests --prefix=$CIRRUS_WORKING_DIR/install
static_config: &STATIC_CONFIG --build-type=release --disable-broker-tests --enable-static-broker --enable-static-binpac --prefix=$CIRRUS_WORKING_DIR/install
sanitizer_config: &SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=address,undefined --enable-fuzzers --enable-coverage
asan_sanitizer_config: &ASAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=address --enable-fuzzers --enable-coverage
ubsan_sanitizer_config: &UBSAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=undefined --enable-fuzzers
tsan_sanitizer_config: &TSAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=thread --enable-fuzzers
mobile_ipv6_config: &MOBILE_IPV6_CONFIG --build-type=release --enable-mobile-ipv6 --disable-broker-tests --prefix=$CIRRUS_WORKING_DIR/install
openssl30_config: &OPENSSL30_CONFIG --build-type=release --disable-broker-tests --with-openssl=/opt/openssl --prefix=$CIRRUS_WORKING_DIR/install
@ -108,9 +110,16 @@ fedora34_task:
<< : *RESOURCES_TEMPLATE
<< : *CI_TEMPLATE
centosstream9_task:
container:
# Stream 9 EOL: Around Dec 2027
dockerfile: ci/centos-stream-9/Dockerfile
<< : *RESOURCES_TEMPLATE
<< : *CI_TEMPLATE
centosstream8_task:
container:
# Stream 8 support should be 5 years, so until 2024. but I cannot find a concrete timeline --cpk
# Stream 8 EOL: May 31, 2024
dockerfile: ci/centos-stream-8/Dockerfile
<< : *RESOURCES_TEMPLATE
<< : *CI_TEMPLATE
@ -160,13 +169,6 @@ debian9_32bit_task:
<< : *RESOURCES_TEMPLATE
<< : *CI_TEMPLATE
opensuse_leap_15_2_task:
container:
# Opensuse Leap 15.2 EOL: Dec 2021
dockerfile: ci/opensuse-leap-15.2/Dockerfile
<< : *RESOURCES_TEMPLATE
<< : *CI_TEMPLATE
opensuse_leap_15_3_task:
container:
# Opensuse Leap 15.3 EOL: TBD
@ -174,6 +176,13 @@ opensuse_leap_15_3_task:
<< : *RESOURCES_TEMPLATE
<< : *CI_TEMPLATE
ubuntu21_task:
container:
# Ubuntu 21.10 EOL: July 2022
dockerfile: ci/ubuntu-21.10/Dockerfile
<< : *RESOURCES_TEMPLATE
<< : *CI_TEMPLATE
ubuntu20_task:
container:
# Ubuntu 20.04 EOL: April 2025
@ -261,7 +270,7 @@ openssl30_task:
env:
ZEEK_CI_CONFIGURE_FLAGS: *OPENSSL30_CONFIG
sanitizer_task:
asan_sanitizer_task:
container:
# Just uses a recent/common distro to run memory error/leak checks.
dockerfile: ci/ubuntu-20.04/Dockerfile
@ -270,10 +279,38 @@ sanitizer_task:
memory: 12GB
<< : *CI_TEMPLATE
test_fuzzers_script: ./ci/test-fuzzers.sh
coverage_script: ./ci/upload-coverage.sh
env:
CXXFLAGS: -DZEEK_DICT_DEBUG
ZEEK_CI_CONFIGURE_FLAGS: *SANITIZER_CONFIG
ZEEK_TAILORED_UB_CHECKS: 1
ZEEK_CI_CONFIGURE_FLAGS: *ASAN_SANITIZER_CONFIG
ZEEK_CI_DISABLE_SCRIPT_PROFILING: 1
ASAN_OPTIONS: detect_leaks=1
ubsan_sanitizer_task:
container:
# Just uses a recent/common distro to run undefined behavior checks.
dockerfile: ci/ubuntu-20.04/Dockerfile
cpu: 4
# AddressSanitizer uses a lot more memory than a typical config.
memory: 12GB
<< : *CI_TEMPLATE
test_fuzzers_script: ./ci/test-fuzzers.sh
env:
CXXFLAGS: -DZEEK_DICT_DEBUG
ZEEK_CI_CONFIGURE_FLAGS: *UBSAN_SANITIZER_CONFIG
ZEEK_CI_DISABLE_SCRIPT_PROFILING: 1
ZEEK_TAILORED_UB_CHECKS: 1
UBSAN_OPTIONS: print_stacktrace=1
# tsan_sanitizer_task:
# container:
# # Just uses a recent/common distro to run memory error/leak checks.
# dockerfile: ci/ubuntu-20.04/Dockerfile
# cpu: 4
# # AddressSanitizer uses a lot more memory than a typical config.
# memory: 12GB
# << : *CI_TEMPLATE
# test_fuzzers_script: ./ci/test-fuzzers.sh
# env:
# CXXFLAGS: -DZEEK_DICT_DEBUG
# ZEEK_CI_CONFIGURE_FLAGS: *TSAN_SANITIZER_CONFIG
# ZEEK_CI_DISABLE_SCRIPT_PROFILING: 1

14
.github/workflows/coverity-scan.yml vendored
View file

@ -11,14 +11,8 @@ jobs:
steps:
- uses: actions/checkout@v2
- name: Update Submodules
shell: bash
run: |
auth_header="$(git config --local --get http.https://github.com/.extraheader)"
git submodule sync --recursive
git -c "http.extraheader=$auth_header" -c protocol.version=2 \
submodule update --init --force --recursive --depth=1
with:
submodules: "recursive"
- name: Fetch Dependencies
run: |
@ -45,7 +39,7 @@ jobs:
wget
- name: Install CAF
run: ( cd auxil/broker/caf && ./configure --prefix=`pwd`/build/install-root && cd build && make -j 3 install )
run: cd auxil/broker/caf && ./configure --prefix=`pwd`/build/install-root && cd build && make -j $(nproc) install
- name: Configure
run: ./configure --build-type=debug --with-caf=`pwd`/auxil/broker/caf/build/install-root --disable-broker-tests
@ -65,7 +59,7 @@ jobs:
- name: Build
run: |
export PATH=`pwd`/coverity-tools/bin:$PATH
( cd build && cov-build --dir cov-int make -j 3 )
( cd build && cov-build --dir cov-int make -j $(nproc) )
cat build/cov-int/build-log.txt
- name: Submit

55
.github/workflows/generate-docs.yml vendored
View file

@ -1,27 +1,36 @@
name: Generate Documentation
on:
pull_request:
schedule:
- cron: '0 0 * * *'
defaults:
run:
shell: bash
jobs:
generate:
if: github.repository == 'zeek/zeek'
runs-on: ubuntu-18.04
runs-on: ubuntu-latest
steps:
# We only perform a push if the action was triggered via a schedule
# event, so we only need to authenticate in that case. Use
# unauthenticated access otherwise so this action can e.g., also run from
# clones.
- uses: actions/checkout@v2
if: github.event_name == 'schedule'
with:
submodules: "recursive"
token: ${{ secrets.ZEEK_BOT_TOKEN }}
- uses: actions/checkout@v2
if: github.event_name != 'schedule'
with:
submodules: "recursive"
- name: Sync Submodules
shell: bash
run: |
auth_header="$(git config --local --get http.https://github.com/.extraheader)"
git submodule sync --recursive
git -c "http.extraheader=$auth_header" -c protocol.version=2 \
submodule update --init --force --recursive --depth=1
( cd doc && git checkout master )
- name: Switch doc submodule to master
run: cd doc && git checkout master
- name: Fetch Dependencies
run: |
@ -51,14 +60,12 @@ jobs:
sudo pip3 install -r doc/requirements.txt
- name: Configure
run: ./configure
run: ./configure --disable-broker-tests --disable-cpp-tests
- name: Build
run: |
( cd build && make -j 3 )
run: cd build && make -j $(nproc)
- name: Generate Docs
shell: bash
run: |
git config --global user.name zeek-bot
git config --global user.email info@zeek.org
@ -76,25 +83,31 @@ jobs:
echo "*** Check for Sphinx Warnings ***"
grep -q WARNING make.out && exit 1
rm make.out
echo "*** Pushing zeek-docs Changes ***"
git remote set-url origin "https://zeek-bot:${{ secrets.ZEEK_BOT_TOKEN }}@github.com/zeek/zeek-docs"
- name: Push zeek-docs Changes
if: github.event_name == 'schedule'
run: |
cd doc
git add scripts/ script-reference/
git status
git commit -m "Generate docs" && git push || /bin/true
cd ..
# git commit errors when there's nothing to commit, so guard it
# with a check that detects whether there's anything to commit/push.
git diff-index --quiet HEAD || { git commit -m "Generate docs" && git push; }
- name: Update zeek-docs Submodule
if: github.event_name == 'schedule'
run: |
echo "*** Update zeek/doc Submodule ***"
git config --global user.name zeek-bot
git config --global user.email info@zeek.org
git remote add auth "https://zeek-bot:${{ secrets.ZEEK_BOT_TOKEN }}@github.com/zeek/zeek"
git add doc
git status
git commit -m 'Update doc submodule [nomail] [skip ci]' && git push auth master || /bin/true
# Similar logic here: proceed only if there's a change in the submodule.
git diff-index --quiet HEAD || { git commit -m 'Update doc submodule [nomail] [skip ci]' && git push; }
- name: Send email
if: failure()
# Only send notifications for scheduled runs. Runs from pull requests
# show failures in the Github UI.
if: failure() && github.event_name == 'schedule'
uses: dawidd6/action-send-mail@v3.4.1
with:
server_address: ${{secrets.SMTP_HOST}}

9
.gitmodules vendored
View file

@ -49,3 +49,12 @@
[submodule "auxil/zeek-client"]
path = auxil/zeek-client
url = https://github.com/zeek/zeek-client
[submodule "auxil/gen-zam"]
path = auxil/gen-zam
url = https://github.com/zeek/gen-zam
[submodule "auxil/c-ares"]
path = auxil/c-ares
url = https://github.com/c-ares/c-ares
[submodule "auxil/out_ptr"]
path = auxil/out_ptr
url = https://github.com/soasis/out_ptr.git

474
CHANGES
View file

@ -1,3 +1,477 @@
5.0.0-dev.332 | 2022-04-28 19:52:04 +0000
* Initialize OpenSSL on startup (Dominik Charousset, Corelight)
* Avoid double-initialization of OpenSSL (Dominik Charousset, Corelight)
* Canonify intel.log in read-file-dist-cluster test (Dominik Charousset, Corelight)
* Port Zeek to latest Broker API (Dominik Charousset)
5.0.0-dev.322 | 2022-04-27 21:00:29 +0000
* Disable OpenSSL initialization starting with 1.1.0 (Johanna Amann, Corelight)
Starting with OpenSSL 1.1.0, library initialization is no longer
required - and might even be harmful.
See https://wiki.openssl.org/index.php/Library_Initialization for
details.
5.0.0-dev.319 | 2022-04-27 17:42:42 +0000
* Wrap call to doctest's MESSAGE() method in Reporter in try/catch block (Tim Wojtulewicz, Corelight)
Also check whether doctest is even enabled before trying to use it.
* Pre-initialize c-ares channel object. Fixes Coverity 1488318 (Tim Wojtulewicz, Corelight)
* Ask c-ares for the next timeout instead of passing a fixed value (Tim Wojtulewicz, Corelight)
* Remove obsolete DNS_Mgr::asyncs_timeouts (Tim Wojtulewicz, Corelight)
5.0.0-dev.314 | 2022-04-27 09:43:23 -0700
* Management framework: consistency fixes around event() vs Broker::publish() (Christian Kreibich, Corelight)
5.0.0-dev.312 | 2022-04-26 09:52:34 -0700
* Rework FindCAres.cmake to not use ExternalProject, fixing OBS builds (Tim Wojtulewicz, Corelight)
5.0.0-dev.309 | 2022-04-22 13:11:12 -0700
* Add DNS fuzzing corpus from c-ares (Tim Wojtulewicz, Corelight)
* Set larger UDP buffer to avoid TCP fallback if possible (Tim Wojtulewicz, Corelight)
This commit sets the UDP buffer to a larger size, as well as adds
an EDNS block to the DNS request passing this size. This allows
DNS servers to return larger responses, and in turn allow c-ares
to avoid TCP fallback due to requests failing because of the lack
of buffer size.
* Add new features to IOSource::Manager, used by DNS_Mgr (Tim Wojtulewicz, Corelight)
- iosource_mgr can now track write events to file descriptors as well
as read events. This adds an argument to both RegisterFd() and
UnregisterFd() for setting the mode, defaulting to read.
- IOSources can now implement a ProcessFd() method that allows them to
handle events to single file descriptors instead of of having to
loop through/track sets of them at processing time.
* Add out_ptr, use for c-ares interface calls (Tim Wojtulewicz, Corelight)
* Store all mappings in a single map instead of split by type (Tim Wojtulewicz, Corelight)
This opens up the possibility of storing other request types outside
of T_A, T_PTR and T_TXT without requiring redoing the caching. It
also fixes the caching code in DNS_Mapping, adding a version number
to the start of the cache file so the cache structure can be modified
and old caches invalidated more easily.
* Add merging to DNS_Mgr::AddResult() to support both ipv4 and ipv6 responses simultaneously (Tim Wojtulewicz, Corelight)
* Rework DNS_Mgr API to be more consistent and to support more request types (Tim Wojtulewicz, Corelight)
* Replace nb_dns library with C-Ares (Tim Wojtulewicz, Corelight)
* Add unit testing for DNS_Mgr and related classes (Tim Wojtulewicz, Corelight)
* Update doc gen VM to ubuntu-latest, output cmake version during configure (Tim Wojtulewicz, Corelight)
* Use doctest macro to tie Reporter output to test cases (Tim Wojtulewicz, Corelight)
* Add const versions of dereference operators for DictEntry (Tim Wojtulewicz, Corelight)
* Add DNS fuzzer (Tim Wojtulewicz, Corelight)
5.0.0-dev.288 | 2022-04-22 07:00:56 -0700
* Fix generate-docs action for running on forks. (Benjamin Bannier, Corelight)
The generate-docs action previously always required secrets to run so
that it could possibly perform a push (if run from a schedule), and to
send out an email on failure. Since secrets are unavailable for forks
this meant that this action would always fail for PRs from forks.
In this patch we use an unauthenticated clone unless running from a
schedule. This is fine as for PRs this action would just regenerate the
docs to check for errors, but not to actually update them (no push
performed). We also change the failure notification step to only execute
for scheduled runs.
5.0.0-dev.286 | 2022-04-21 13:34:34 -0700
* Suppress progress dots in zkg's output in Docker package-install check (Christian Kreibich, Corelight)
5.0.0-dev.284 | 2022-04-21 09:17:28 -0700
* Enable vptr undefined behavior check (Tim Wojtulewicz, Corelight)
5.0.0-dev.282 | 2022-04-20 17:17:55 -0700
* Update libkqueue for Coverity and build warning fixes (Tim Wojtulewicz, Corelight)
5.0.0-dev.280 | 2022-04-19 09:42:28 -0700
* Escape special characters in paths before using them as regexes (Tim Wojtulewicz, Corelight)
5.0.0-dev.277 | 2022-04-18 16:38:27 -0700
* Management framework updates (Christian Kreibich, Corelight)
- bump external testsuite
- allow selecting cluster nodes in get_id_value
- minor tweaks to logging component
- bump zeek-client to pull in get-id-value command
- improve handling of node run states
- add get_id_value dispatch
- allow dispatching "actions" on cluster nodes.
- some renaming to avoid the term "data cluster"
- allow agents to communicate with cluster nodes
* Avoid whitespace around function type strings in JSON rendering (Christian Kreibich, Corelight)
* Disable TSan CI task temporarily while we sort out some intermittent test failures (Tim Wojtulewicz, Corelight)
5.0.0-dev.265 | 2022-04-18 12:45:08 -0700
* state-holding fix: track unique identifiers for Func's in CompHash's, not Func's themselves (Vern Paxson, Corelight)
5.0.0-dev.263 | 2022-04-18 09:22:30 -0700
* Add "Reporter" entry to fix plugin hook_name() vs HookType imbalance (Christian Kreibich, Corelight)
The hook_name() list was missing an entry corresponding to HOOK_REPORTER.
Co-authored-by: Peter Cullen <peter.cullen@corelight.com>
5.0.0-dev.259 | 2022-04-14 10:26:29 -0700
* GH-2038: Don't sleep when non-selectable PktSrc has data available (Anthony Coddington)
PktSrc::GetNextTimeout always returned a fixed timeout of 20 microseconds for non-selectable packet sources regardless of whether they have packets available. This adds unnecessary delay every FindReadySources poll_interval when packets are available to be read.
Instead, for non-selectable packet sources, check whether packets are available and return a timeout of 0 to indicate data is available. This is closer to the behaviour of the old capture loop.
This was mitigated somewhat by the fact FindReadySources poll interval defaults to 100 packets, and live sources are added to the ready list regardless of whether they have packets available (unless it is time to force a poll).
5.0.0-dev.257 | 2022-04-14 10:13:28 -0700
* Re-instantiate providing location information to `LoadFile` hooks. (Robin Sommer, Corelight)
#1835 subtly changed the semantics of the `LoadFile` plugin hook to no
longer have the current script location available for signature files
being loaded through `@load-sigs`. This was undocumented behavior, so
it's technically not a regression, but since at least one external
plugin is depending on it, this change restores the old behavior.
5.0.0-dev.255 | 2022-04-14 10:12:49 -0700
* Fix another crash during dictionary iteration. (Robin Sommer, Corelight)
Closes #2017.
* Fix assertions in dictionary that can trigger for benign reasons. (Robin Sommer, Corelight)
These assertions were checking for a situation that I believe can
happen legitimately: a robust iterator pointing to an index that,
after some table resizing, happens to be inside the overflow area and
hence empty. We'll now move it to the end of the table in the case.
* Fix robust iterators when modifying dictionary during iteration. (Robin Sommer, Corelight)
When inserting/deleting elements, we now remove their `DictEntries`
from any robust iterators' bookkeeping. First, we don't need that
information anymore, and second the `DictEntries` contain pointers
that may become invalid.
I don't know how to write a unit test for this unfortunately because
it depends on where exactly things land in the hash table.
Btw, memory mgmt for DictEntries is pretty fragile: They contain
pointers to both memory they own (`key`) and memory they don't own
(`value`). The former type of pointers is shallow-copied on
assignment/copy-construction, meaning that there can be multiple
instances seemingly owning the same memory. That only works because
deletion is manual, and not part of standard destruction. The second
type of pointer has a similar problem, except that it's managed
externally. It's important to not end up with multiple `DictEntries`
pointing to the same value (which is actually what that iterator
bookkeeping did).
Addresses #2032.
5.0.0-dev.250 | 2022-04-14 09:51:23 -0700
* Split asan/ubsan CI builds, add tsan build (Tim Wojtulewicz, Corelight)
5.0.0-dev.248 | 2022-04-14 08:59:34 -0700
* Disable object-size analysis if optimization set to -O0 (Tim Wojtulewicz, Corelight)
5.0.0-dev.246 | 2022-04-14 10:48:19 +0200
* Allow analyzer violations to explicitly set tag. (Robin Sommer, Corelight)
5.0.0-dev.244 | 2022-04-13 10:52:58 -0700
* Add test to ensure enum_to_int's return values are ordered (Yacin Nadji, Corelight)
5.0.0-dev.242 | 2022-04-13 10:51:21 -0700
* Add unit test for other get_word() version (Tim Wojtulewicz, Corelight)
5.0.0-dev.240 | 2022-04-11 12:46:51 -0700
* Mask our signal handlers' triggering signals around thread creation (Christian Kreibich, Corelight)
5.0.0-dev.238 | 2022-04-11 12:40:02 -0700
* GH-2026: Ensure both protocol and analyzer confirmation and violation events can be called (Tim Wojtulewicz, Corelight)
5.0.0-dev.235 | 2022-04-09 00:08:50 +0000
* Update libkqueue to 2.6.0 release [skip ci] [nomail] (Tim Wojtulewicz)
5.0.0-dev.233 | 2022-04-08 11:30:52 -0700
* Bump submodules to pull in InstallSymlink fix (Christian Kreibich, Corelight)
5.0.0-dev.231 | 2022-04-05 18:04:47 -0700
* fix for ill-formed (complex) &default function (Vern Paxson, Corelight)
* type-checking for use of empty table constructors in expressions (Vern Paxson, Corelight)
* catch empty constructors used for type inference (Vern Paxson, Corelight)
suppress repeated error messages
* factoring to make checking of &default attributes externally accessible (Vern Paxson, Corelight)
* bug fix for empty table constructors with &default attributes (plus a typo) (Vern Paxson, Corelight)
5.0.0-dev.222 | 2022-04-05 18:04:15 -0700
* reduce interpreter frames for compiled function bodies (Vern Paxson, Corelight)
5.0.0-dev.219 | 2022-04-05 16:07:48 -0700
* Correct origin documentation of the version field in the HTTP log. (Christian Kreibich, Corelight)
5.0.0-dev.217 | 2022-04-04 13:27:32 -0700
* Move new TLS decryption capabilities up to Zeek 5 in NEWS file (Christian Kreibich, Corelight)
* Update NEWS to reflect recent updates (Christian Kreibich, Corelight)
5.0.0-dev.214 | 2022-04-04 10:52:41 -0700
* fix & btest for ZAM bug with inlined nested loop (Vern Paxson, Corelight)
5.0.0-dev.212 | 2022-04-04 10:51:20 -0700
* GH-2009: Use auto to fix ZIP analyzer failure on some platforms (Tim Wojtulewicz, Corelight)
5.0.0-dev.210 | 2022-03-28 17:04:51 -0700
* Add cmake-time reporting of bifcl, binpac, and gen-zam used for build (Christian Kreibich, Corelight)
* Build Gen-ZAM from a submodule and support use of pre-existing executable (Christian Kreibich, Corelight)
5.0.0-dev.204 | 2022-03-25 15:31:21 -0700
* --event-trace / -E option to generate event trace (Vern Paxson, Corelight)
* hooks to support event tracing (Vern Paxson, Corelight)
* classes providing event-tracing/dumping functionality (Vern Paxson, Corelight)
* provide access to Val internals for event tracing purposes (Vern Paxson, Corelight)
* set_network_time() BiF in support of event replaying (Vern Paxson, Corelight)
5.0.0-dev.195 | 2022-03-24 11:01:28 -0700
* switch variable initialization over to being expression-based (Vern Paxson, Corelight)
* simplification of Val classes now that they don't have to support initialization (Vern Paxson, Corelight)
* rework type inference due to switch from separate initializers to expressions (Vern Paxson, Corelight)
* avoid evaluating calls to determine whether an expression value is ignored (Vern Paxson, Corelight)
* reworking of expressions to unify =/+=/-= with initialization (Vern Paxson, Corelight)
* allow {} expression lists for =/+=/-= RHS (Vern Paxson, Corelight)
5.0.0-dev.177 | 2022-03-23 13:05:51 +0100
* Improve the formatting of the SSL::Info::ssl_history documentation (Johanna Amann, Corelight)
5.0.0-dev.173 | 2022-03-16 15:06:05 -0700
* Fix document generation (Christian Kreibich, Corelight)
5.0.0-dev.169 | 2022-03-10 11:09:37 -0700
* add raw_bytes_to_v6_addr in docs when raw_bytes_to_v4_addr is present (Yacin Nadji, Corelight)
* Zero out bytes by default for consistent return value on error (Yacin Nadji, Corelight)
* Add tests for raw_bytes_to_v6_addr (Yacin Nadji, Corelight)
* Add raw_bytes_to_v6_addr function (Yacin Nadji, Corelight)
5.0.0-dev.164 | 2022-03-08 09:30:37 -0700
* Update 3rdparty submodule for bsd-getopt-long fix (Tim Wojtulewicz)
5.0.0-dev.162 | 2022-03-07 12:36:37 +0100
* Improve error message when receiving unexpected record content via
Broker. (Robin Sommer, Corelight)
5.0.0-dev.160 | 2022-03-02 13:48:07 +0000
* restored record constructor checking for missing-but-mandatory fields. This includea a new btest
as well as a fix to the base-scrpts. (Vern Paxson, Corelight)
5.0.0-dev.156 | 2022-03-02 08:23:50 +0000
* The is_num(), is_alpha(), and is_alnum() BiFs now return F on empty string.
The testcases for these functions, and for is_ascii() were expanded. The documentation of is_ascii()
concerning behavior of an empty string was clarified (Christian Kreibich, Corelight)
5.0.0-dev.151 | 2022-03-02 08:09:28 +0000
* SSL: rudimentary decryption for TLS 1.2 (Florian Wilkens, Johanna Amann)
With this version, we support rudimentary decryption of TLS 1.2 connections, if the key material
of the connection (in our case the pre-master secret) is available. Note that this functionality
only works for TLS 1.2 connections using the TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 cipher suite.
No other combinations are currently supported.
For more information, see the NEWS entry and the TLS Decryption documentation.
5.0.0-dev.121 | 2022-02-24 09:11:03 -0700
* GH-1980: Deprecate and return warning for zeek-config's caf-root option (Tim Wojtulewicz, Corelight)
5.0.0-dev.118 | 2022-02-23 10:51:57 -0700
* GH-1949: Remove unused timer_mgr_inactivity_timeout global (Tim Wojtulewicz, Corelight)
5.0.0-dev.116 | 2022-02-21 18:17:13 -0700
* remove deprecated union and timer types, addressing #1898 (Matthew Luckie)
5.0.0-dev.114 | 2022-02-11 09:30:04 -0800
* Minor modernizations to Github workflows (Christian Kreibich, Corelight)
5.0.0-dev.112 | 2022-02-10 17:56:27 -0800
* Reorg of the cluster controller to new "Management framework" layout (Christian Kreibich, Corelight)
* Bump external cluster testsuite to reflect Management framework reorg (Christian Kreibich, Corelight)
* Bump zeek-client to reflect Management framework reorg (Christian Kreibich, Corelight)
5.0.0-dev.108 | 2022-02-10 10:35:02 -0700
* Fixing a big pile of Coverity issues (Tim Wojtulewicz, Corelight)
5.0.0-dev.106 | 2022-02-09 15:15:21 -0800
* Expand generate-docs Github workflow to test docs build on PRs (Christian Kreibich, Corelight)
5.0.0-dev.104 | 2022-02-09 13:14:04 -0800
* Updates to the cluster controller scripts to fix the docs build (Christian Kreibich, Corelight)
* Bump zeek-client for Broker enum fix/workaround (Christian Kreibich, Corelight)
5.0.0-dev.100 | 2022-02-07 14:18:50 -0800
* Add capture to a Sumstats when-statement to fix deprecation warning (Christian Kreibich, Corelight)
5.0.0-dev.97 | 2022-02-07 16:24:06 +0100
* Update to latest Broker without public CAF dependencies. (Dominik
Charousset, Corelight)
* Fix GCC builds and string output for Broker errors (Dominik
Charousset, Corelight)
5.0.0-dev.94 | 2022-02-07 08:14:47 -0700
* String/StringVal: Replace char*/string constructors with string_view (Tim Wojtulewicz, Corelight)
5.0.0-dev.92 | 2022-02-04 10:33:47 -0700
* fix existing checks for looking to use C++ when it's not available (Vern Paxson, Corelight)
5.0.0-dev.90 | 2022-02-04 10:32:41 -0700
* fixes for ZAM profiling, which didn't get fully integrated originally (Vern Paxson, Corelight)
* minor enhancements for ZAM inlining (Vern Paxson, Corelight)
5.0.0-dev.87 | 2022-02-03 13:17:25 -0800
* Expansion of cluster controller functionality (Christian Kreibich, Corelight)
- Bump external cluster testsuite
- Bump zeek-client for the get-nodes command
- Add ClusterController::API::get_nodes_request/response event pair
- Support optional listening ports for cluster nodes
- Don't auto-publish Supervisor response events in the cluster agent
- Make members of the ClusterController::Types::State enum all-caps
- Be more conservative with triggering request timeout events
- Move redefs of ClusterController::Request::Request to their places of use
- Simplify ClusterController::API::set_configuration_request/response
5.0.0-dev.77 | 2022-02-03 11:20:16 +0000
* Match DPD TLS signature on one-sided connections. (Johanna Amann, Corelight)
This commit changes DPD matching for TLS connections. A one-sided match
is enough to enable DPD now.
This commit also removes DPD for SSLv2 connections. SSLv2 connections do
basically no longer happen in the wild. SSLv2 is also really finnicky to
identify correctly - there is very little data required to match it, and
basically all matches today will be false positives. If DPD for SSLv2 is
still desired, the optional signature in policy/protocols/ssl/dpd-v2.sig
can be loaded.
5.0.0-dev.74 | 2022-02-02 09:46:00 +0100
* GH-1890: Consistently warn about mixing vector and scalar operand
depreciaton (Zeke Medley, Corelight)
5.0.0-dev.72 | 2022-02-02 09:36:30 +0100
* Let TCP-based application analyzers operate without any TCP parent
analyzer. (Robin Sommer, Corelight)
5.0.0-dev.70 | 2022-01-25 13:52:00 -0700
* bug fix for vector slice assignment (Vern Paxson, Corelight)
5.0.0-dev.67 | 2022-01-25 12:25:48 +0000
* updated Bro->Zeek in comments in the source tree (Vern Paxson, Corelight)
5.0.0-dev.65 | 2022-01-24 13:41:25 -0800
* CI updates (Christian Kreibich, Corelight)
- add Ubuntu 21.10
- remove OpenSUSE Leap 15.2 (EOL)
- add CentOS Stream 9
5.0.0-dev.61 | 2022-01-17 10:35:15 +0000
* fix for adding a non-managed type to an empty vector (Vern Paxson, Corelight)

80
CMakeLists.txt
View file

@ -178,7 +178,6 @@ if ( ZEEK_SANITIZERS )
# list(APPEND _check_list "nullability-assign") # Not normally part of "undefined"
# list(APPEND _check_list "nullability-return") # Not normally part of "undefined"
# list(APPEND _check_list "objc-cast") # Not truly UB
list(APPEND _check_list "object-size")
# list(APPEND _check_list "pointer-overflow") # Not implemented in older GCCs
list(APPEND _check_list "return")
list(APPEND _check_list "returns-nonnull-attribute")
@ -188,7 +187,14 @@ if ( ZEEK_SANITIZERS )
list(APPEND _check_list "unreachable")
# list(APPEND _check_list "unsigned-integer-overflow") # Not truly UB
list(APPEND _check_list "vla-bound")
# list(APPEND _check_list "vptr") # TODO: fix associated errors
list(APPEND _check_list "vptr")
# Clang complains if this one is defined and the optimizer is set to -O0. We
# only set that optimization level if NO_OPTIMIZATIONS is passed, so disable
# the option if that's set.
if ( NOT DEFINED ENV{NO_OPTIMIZATIONS} )
list(APPEND _check_list "object-size")
endif ()
string(REPLACE ";" "," _ub_checks "${_check_list}")
set(ZEEK_SANITIZER_UB_CHECKS "${_ub_checks}" CACHE INTERNAL "" FORCE)
@ -294,6 +300,10 @@ if ( NOT BIFCL_EXE_PATH )
add_subdirectory(auxil/bifcl)
endif ()
if ( NOT GEN_ZAM_EXE_PATH )
add_subdirectory(auxil/gen-zam)
endif ()
if (ENABLE_JEMALLOC)
if (${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD")
if (DEFINED JEMALLOC_ROOT_DIR)
@ -331,36 +341,15 @@ if ( PYTHON_VERSION_STRING VERSION_LESS ${ZEEK_PYTHON_MIN} )
message(FATAL_ERROR "Python ${ZEEK_PYTHON_MIN} or greater is required.")
endif ()
if ( CAF_ROOT OR BROKER_ROOT_DIR )
# TODO: drop < 3.12 compatibility check when raising the minimum CMake version
if ( CAF_ROOT AND CMAKE_VERSION VERSION_LESS 3.12 )
find_package(CAF ${CAF_VERSION_MIN_REQUIRED} REQUIRED
COMPONENTS openssl test io core
PATHS "${CAF_ROOT}")
else ()
find_package(CAF ${CAF_VERSION_MIN_REQUIRED} REQUIRED
COMPONENTS openssl test io core)
endif ()
message(STATUS "Using system CAF version ${CAF_VERSION}")
# TODO: drop these legacy variables and simply use the targets consistently
set(CAF_LIBRARIES CAF::core CAF::io CAF::openssl CACHE INTERNAL "")
set(caf_dirs "")
foreach (caf_lib IN LISTS CAF_LIBRARIES ITEMS CAF::test)
get_target_property(dirs ${caf_lib} INTERFACE_INCLUDE_DIRECTORIES)
if ( dirs )
list(APPEND caf_dirs ${dirs})
endif ()
endforeach ()
list(REMOVE_DUPLICATES caf_dirs)
list(GET caf_dirs 0 caf_dir)
set(CAF_INCLUDE_DIRS "${caf_dirs}" CACHE INTERNAL "")
endif ()
add_subdirectory(auxil/paraglob)
set(zeekdeps ${zeekdeps} paraglob)
if ( BROKER_ROOT_DIR )
find_package(Broker REQUIRED)
if ( Broker_ROOT )
find_package(Broker REQUIRED PATHS "${Broker_ROOT}")
set(zeekdeps ${zeekdeps} ${BROKER_LIBRARY})
set(broker_includes ${BROKER_INCLUDE_DIR})
elseif ( BROKER_ROOT_DIR )
find_package(Broker REQUIRED PATHS "${BROKER_ROOT_DIR}")
set(zeekdeps ${zeekdeps} ${BROKER_LIBRARY})
set(broker_includes ${BROKER_INCLUDE_DIR})
else ()
@ -385,11 +374,6 @@ else ()
set(broker_includes ${CMAKE_CURRENT_SOURCE_DIR}/auxil/broker/include ${CMAKE_CURRENT_BINARY_DIR}/auxil/broker/include)
endif ()
# CAF_LIBRARIES and CAF_INCLUDE_DIRS are defined either by calling
# find_package(CAF) or by calling add_subdirectory(auxil/broker). In either case,
# we have to care about CAF here because Broker headers can pull in CAF
# headers.
set(zeekdeps ${zeekdeps} ${CAF_LIBRARIES})
include_directories(BEFORE
${PCAP_INCLUDE_DIR}
${BIND_INCLUDE_DIR}
@ -462,7 +446,7 @@ endif ()
# Any headers that are possibly bundled in the Zeek source-tree and that are supposed
# to have priority over any pre-existing/system-wide headers need to appear early in
# compiler search path.
include_directories(BEFORE ${broker_includes} ${CAF_INCLUDE_DIRS})
include_directories(BEFORE ${broker_includes})
include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/auxil/highwayhash)
include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/auxil/paraglob/include)
include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/auxil/rapidjson/include)
@ -496,6 +480,8 @@ include(CheckNameserCompat)
include(GetArchitecture)
include(RequireCXX17)
include(FindKqueue)
include(FindCAres)
include_directories(BEFORE "auxil/out_ptr/include")
if ( (OPENSSL_VERSION VERSION_EQUAL "1.1.0") OR (OPENSSL_VERSION VERSION_GREATER "1.1.0") )
set(ZEEK_HAVE_OPENSSL_1_1 true CACHE INTERNAL "" FORCE)
@ -522,12 +508,6 @@ execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink
"."
"${CMAKE_CURRENT_BINARY_DIR}/zeek")
if ( CAF_ROOT )
set(ZEEK_CONFIG_CAF_ROOT_DIR ${CAF_ROOT})
else ()
set(ZEEK_CONFIG_CAF_ROOT_DIR ${ZEEK_ROOT_DIR})
endif ()
if ( BinPAC_ROOT_DIR )
set(ZEEK_CONFIG_BINPAC_ROOT_DIR ${BinPAC_ROOT_DIR})
else ()
@ -665,6 +645,21 @@ else ()
set(_install_btest_tools_msg "no pcaps")
endif ()
set(_bifcl_exe_path "included")
if ( BIFCL_EXE_PATH )
set(_bifcl_exe_path ${BIFCL_EXE_PATH})
endif ()
set(_binpac_exe_path "included")
if ( BINPAC_EXE_PATH )
set(_binpac_exe_path ${BINPAC_EXE_PATH})
endif ()
set(_gen_zam_exe_path "included")
if ( GEN_ZAM_EXE_PATH )
set(_gen_zam_exe_path ${GEN_ZAM_EXE_PATH})
endif ()
message(
"\n====================| Zeek Build Summary |===================="
"\n"
@ -686,8 +681,11 @@ message(
"\n"
"\nZeekControl: ${INSTALL_ZEEKCTL}"
"\nAux. Tools: ${INSTALL_AUX_TOOLS}"
"\nBifCL: ${_bifcl_exe_path}"
"\nBinPAC: ${_binpac_exe_path}"
"\nBTest: ${INSTALL_BTEST}"
"\nBTest tooling: ${_install_btest_tools_msg}"
"\nGen-ZAM: ${_gen_zam_exe_path}"
"\nzkg: ${INSTALL_ZKG}"
"\n"
"\nlibmaxminddb: ${USE_GEOIP}"

43
NEWS
View file

@ -12,9 +12,43 @@ Breaking Changes
New Functionality
-----------------
- Zeek now supports generation and replay of event traces via the new
``--event-trace`` / ``-E`` command-line options. For details, see:
https://docs.zeek.org/en/master/quickstart.html#tracing-events
- Zeek now features limited TLS decryption capabilities. This feature is experimental
and only works for TLS 1.2 connections that use the TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
ciphersuite. Furthermore Zeek requires access to the pre-master secret of each TLS
connection. Typically this functionality will be most useful when analyzing trace-files
where the TLS client recorded the key material. For more details and examples how to
use this functionality, see the TLS Decryption documentation at
https://docs.zeek.org/en/master/frameworks/tls-decryption.html
- The new --with-gen-zam configure flag and its corresponding GEN_ZAM_EXE_PATH
cmake variable allow reuse of a previously built Gen-ZAM code generator. This
aids cross-compilation: the Zeek build process normally compiles Gen-ZAM on
the fly, but when cross-compiling will do so for the target platform, breaking
its use on the host platform. Gen-ZAM is similar to binpac and bifcl in this
regard. Like binpac and bifcl, it's now also available as a standalone git
repository and hooked into the Zeek distribution as a submodule.
- Zeek now uses the c-ares (https://c-ares.org) library for performing DNS
requests, replacing an old custom implementation of a DNS resolver. Switching
to this library simplifies the DNS code, adds support for IPv6 lookups, and
adds the ability to support more DNS request types in the future.
Changed Functionality
---------------------
- The behavior of the ``=``, ``+=``, and ``-=`` operators has been expanded and
unified. It now covers ``{ ... }`` initializer lists, supports cross-product
initialization, enables ``+=`` for table, set, vector and pattern values,
similarly allows ``-=`` for table and set values, and supports listing
multiple sets for ``+=`` initialization. For details, see:
https://docs.zeek.org/en/master/script-reference/operators.html#assignment-operators
- The is_num(), is_alpha(), and is_alnum() BiFs now return F for the empty string.
Deprecated Functionality
------------------------
@ -32,6 +66,15 @@ Breaking Changes
changes to return types from a number of methods. With this change, any uses
of the `zeek::*::Tag` types will need to be replaced by `zeek::Tag`.
- The DPD signature for SSL version 2 is no longer enabled by default. SSLv2
is basically extinct nowadays - and the protocol has a relatively high probability
of matching with random traffic and being misidentified. If you want to enable
the SSLv2 dpd signature, you can load the signature from `policy/protocols/ssl/dpd-v2.sig`
The DPD signature for SSL version 3 and up (including TLS 1.0 and above) now matches
for one-sided connections and does not require a reverst match anymore. This prevents
missed handshakes, where the client handshake contains a lot of data.
New Functionality
-----------------

2
VERSION
View file

@ -1 +1 @@
5.0.0-dev.61
5.0.0-dev.332

2
auxil/bifcl

@ -1 +1 @@
Subproject commit eed5effea5661e03b50d0436fecb620a05fb1250
Subproject commit 3b5efa59f137fc5a15ace602d44e176f97bae083

2
auxil/broker

@ -1 +1 @@
Subproject commit 41c524f172aa357422f1ccf7e806b448d5def08e
Subproject commit 9f2f16c1da94b03790bc8b9f69bc2688e97b781f

1
auxil/c-ares Submodule

@ -0,0 +1 @@
Subproject commit 2aa086f822aad5017a6f2061ef656f237a62d0ed

1
auxil/gen-zam Submodule

@ -0,0 +1 @@
Subproject commit f8c8fb36fb07a2c1703c63dd8624cf1329c0c4d0

2
auxil/libkqueue

@ -1 +1 @@
Subproject commit aeaeed21198d6f41d0cf70bda63fe0f424922ac5
Subproject commit 374aeb52020e289e8574f059f87a96010d9a46b9

1
auxil/out_ptr Submodule

@ -0,0 +1 @@
Subproject commit ea379b2f35e28d6ee894e05ad4c26ed60a613d30

2
auxil/zeek-aux

@ -1 +1 @@
Subproject commit 0f120aa00c2b666ed5c430a6bcf1043b82f17e64
Subproject commit e76e84e175463f9989b492b1d119e3d846fe696d

2
auxil/zeek-client

@ -1 +1 @@
Subproject commit 553d897734b6d9abbc2e4467fae89f68a2c7315d
Subproject commit a08d9978ac6ff6481ad1e6b18f0376568c08f8c1

2
auxil/zeekctl

@ -1 +1 @@
Subproject commit e7fd4d552ec7c3e55cb556943bf8b499d2db17e1
Subproject commit 9cab8f5c62577c54126e6c63343e024ad6f441bc

36
ci/centos-stream-9/Dockerfile Normal file
View file

@ -0,0 +1,36 @@
FROM quay.io/centos/centos:stream9
# dnf config-manager isn't available at first, and
# we need it to install the CRB repo below.
RUN dnf -y install 'dnf-command(config-manager)'
# What used to be powertools is now called "CRB".
# We need it for some of the packages installed below.
# https://docs.fedoraproject.org/en-US/epel/
RUN dnf config-manager --set-enabled crb
RUN dnf -y install \
https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm \
https://dl.fedoraproject.org/pub/epel/epel-next-release-latest-9.noarch.rpm
# The --nobest flag is hopefully temporary. Without it we currently hit
# package versioning conflicts around OpenSSL.
RUN dnf -y --nobest install \
bison \
cmake \
diffutils \
flex \
git \
gcc \
gcc-c++ \
libpcap-devel \
make \
openssl-devel \
python3-devel \
python3-pip\
sqlite \
swig \
which \
zlib-devel \
&& dnf clean all && rm -rf /var/cache/dnf
RUN pip3 install junit2html

25
ci/opensuse-leap-15.2/Dockerfile
View file

@ -1,25 +0,0 @@
FROM opensuse/leap:15.2
RUN zypper in -y \
cmake \
make \
gcc \
gcc-c++ \
python3 \
python3-devel \
flex \
bison \
libpcap-devel \
libopenssl-devel \
zlib-devel \
swig \
git \
curl \
python3-pip \
which \
gzip \
tar \
&& rm -rf /var/cache/zypp
RUN pip3 install junit2html

33
ci/ubuntu-21.10/Dockerfile Normal file
View file

@ -0,0 +1,33 @@
FROM ubuntu:21.10
ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles"
RUN apt-get update && apt-get -y install \
git \
cmake \
make \
gcc \
g++ \
flex \
bison \
libpcap-dev \
libssl-dev \
python3 \
python3-dev \
python3-pip\
swig \
zlib1g-dev \
libmaxminddb-dev \
libkrb5-dev \
bsdmainutils \
sqlite3 \
curl \
wget \
unzip \
ruby \
bc \
lcov \
&& rm -rf /var/lib/apt/lists/*
RUN pip3 install junit2html
RUN gem install coveralls-lcov

2
cmake

@ -1 +1 @@
Subproject commit 105f6c9df616a4c2286d5ef38c2b31a718192301
Subproject commit 588e6da051cb82a1cd24400f94b86338957d646a

7
configure vendored
View file

@ -85,6 +85,8 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
(Zeek uses an embedded version by default)
--with-caf=PATH path to C++ Actor Framework install root
(a Broker dependency that is embedded by default)
--with-gen-zam=PATH path to Gen-ZAM code generator
(Zeek uses an embedded version by default)
--with-flex=PATH path to flex executable
--with-libkqueue=PATH path to libkqueue install root
(Zeek uses an embedded version by default)
@ -338,6 +340,9 @@ while [ $# -ne 0 ]; do
--with-flex=*)
append_cache_entry FLEX_EXECUTABLE PATH $optarg
;;
--with-gen-zam=*)
append_cache_entry GEN_ZAM_EXE_PATH PATH $optarg
;;
--with-geoip=*)
append_cache_entry LibMMDB_ROOT_DIR PATH $optarg
;;
@ -437,6 +442,8 @@ echo "Build Directory : $builddir"
echo "Source Directory: $sourcedir"
cd $builddir
echo "Using $(cmake --version | head -1)"
echo
if [ -n "$CMakeGenerator" ]; then
"$CMakeCommand" -G "$CMakeGenerator" $CMakeCacheEntries $sourcedir
else

2
doc

@ -1 +1 @@
Subproject commit 789c4b2f4c6b10193ee39f5be82cc9028eddfc38
Subproject commit a5adc652ed3c7cf179a19c8b373a9443dd1fc5dc

4
docker/btest/docker/structure_tests.sh
View file

@ -17,7 +17,9 @@ docker run --rm "${TEST_TAG}" btest --version | sed 's/^[0-9].*/XXX/g'
docker run --rm "${TEST_TAG}" zkg config
# Check that a plugin can be installed. We pick any plugin with minimal deps here.
docker run --rm "${TEST_TAG}" zkg install --force sethhall/domain-tld | sed 's/(.*)/(XXX)/'
docker run --rm "${TEST_TAG}" zkg install --force sethhall/domain-tld |
sed 's/"\.*$/"/' |
sed 's/(.*)/(XXX)/'
# Check that the Broker Python module loads
docker run --rm "${TEST_TAG}" python3 -c "import broker"

2
scripts/base/frameworks/software/main.zeek
View file

@ -95,7 +95,7 @@ export {
## even if it exposes itself with an alternate name. The
## yielded string is the name that will be logged and generally
## used for everything.
global alternate_names: table[string] of string {
global alternate_names: table[string] of string = {
["Flash Player"] = "Flash",
} &default=function(a: string): string { return a; };

2
scripts/base/frameworks/sumstats/cluster.zeek
View file

@ -481,7 +481,7 @@ function request_key(ss_name: string, key: Key): Result
add dynamic_requests[uid];
event SumStats::cluster_get_result(uid, ss_name, key, F);
return when ( uid in done_with && Cluster::worker_count == done_with[uid] )
return when [uid, ss_name, key] ( uid in done_with && Cluster::worker_count == done_with[uid] )
{
#print "done with request_key";
local result = key_requests[uid];

3
scripts/base/init-bare.zeek
View file

@ -4963,9 +4963,6 @@ const dpd_ignore_ports = F &redef;
## connection if it misses the initial handshake.
const likely_server_ports: set[port] &redef;
## Per-incident timer managers are drained after this amount of inactivity.
const timer_mgr_inactivity_timeout = 1 min &redef;
## If true, output profiling for Time-Machine queries.
const time_machine_profiling = F &redef;

8
scripts/base/protocols/conn/main.zeek
View file

@ -239,10 +239,11 @@ function determine_service(c: connection): string
function set_conn(c: connection, eoc: bool)
{
if ( ! c?$conn )
c$conn = Info();
{
local p = get_port_transport_proto(c$id$resp_p);
c$conn = Info($ts=c$start_time, $uid=c$uid, $proto=p);
}
c$conn$ts=c$start_time;
c$conn$uid=c$uid;
c$conn$id=c$id;
if ( c?$tunnel && |c$tunnel| > 0 )
{
@ -250,7 +251,6 @@ function set_conn(c: connection, eoc: bool)
c$conn$tunnel_parents = set();
add c$conn$tunnel_parents[c$tunnel[|c$tunnel|-1]$uid];
}
c$conn$proto=get_port_transport_proto(c$id$resp_p);
if( |Site::local_nets| > 0 )
{
c$conn$local_orig=Site::is_local_addr(c$id$orig_h);

4
scripts/base/protocols/dhcp/consts.zeek
View file

@ -28,7 +28,7 @@ export {
} &default = function(n: count): string { return fmt("unknown-message-type-%d", n); };
## Option types mapped to their names.
const option_types: table[int] of string = {
const option_types = {
[0] = "Pad",
[1] = "Subnet Mask",
[2] = "Time Offset",
@ -185,5 +185,5 @@ export {
[221] = "Virtual Subnet Selection (VSS) Option",
[252] = "auto-proxy-config",
[255] = "End",
} &default = function(n: int): string { return fmt("unknown-option-type-%d", n); };
} &default = function(n: count): string { return fmt("unknown-option-type-%d", n); };
}

7
scripts/base/protocols/http/main.zeek
View file

@ -43,9 +43,12 @@ export {
uri: string &log &optional;
## Value of the "referer" header. The comment is deliberately
## misspelled like the standard declares, but the name used here
## is "referrer" spelled correctly.
## is "referrer", spelled correctly.
referrer: string &log &optional;
## Value of the version portion of the request.
## Value of the version portion of the reply. If you require
## message-level detail, consider the :zeek:see:`http_request` and
## :zeek:see:`http_reply` events, which report each message's
## version string.
version: string &log &optional;
## Value of the User-Agent header from the client.
user_agent: string &log &optional;

2
scripts/base/protocols/smb/consts.zeek
View file

@ -112,7 +112,7 @@ export {
const rpc_sub_cmds: table[string] of rpc_cmd_table = {
["4b324fc8-1670-01d3-1278-5a47bf6ee188"] = srv_cmds,
["6bffd098-a112-3610-9833-46c3f87e345a"] = wksta_cmds,
} &redef &default=function(i: string):rpc_cmd_table { return table() &default=function(j: string):string { return fmt("unknown-uuid-%s", j); }; };
} &redef &default=function(i: string):rpc_cmd_table { return table() &default=function(j: count):string { return fmt("unknown-uuid-%d", j); }; };
}

16
scripts/base/protocols/ssl/dpd.sig
View file

@ -1,17 +1,17 @@
signature dpd_ssl_server {
signature dpd_tls_server {
ip-proto == tcp
# Server hello.
payload /^((\x15\x03[\x00\x01\x02\x03]....)?\x16\x03[\x00\x01\x02\x03]..\x02...((\x03[\x00\x01\x02\x03\x04])|(\x7F[\x00-\x50]))|...?\x04..\x00\x02).*/
requires-reverse-signature dpd_ssl_client
enable "ssl"
# SSL3 / TLS Server hello.
payload /^(\x15\x03[\x00\x01\x02\x03]....)?\x16\x03[\x00\x01\x02\x03]..\x02...((\x03[\x00\x01\x02\x03\x04])|(\x7F[\x00-\x50])).*/
tcp-state responder
enable "ssl"
}
signature dpd_ssl_client {
signature dpd_tls_client {
ip-proto == tcp
# Client hello.
payload /^(\x16\x03[\x00\x01\x02\x03]..\x01...\x03[\x00\x01\x02\x03]|...?\x01[\x00\x03][\x00\x01\x02\x03\x04]).*/
# SSL3 / TLS Client hello.
payload /^\x16\x03[\x00\x01\x02\x03]..\x01...\x03[\x00\x01\x02\x03].*/
tcp-state originator
enable "ssl"
}
signature dpd_dtls_client {

8
scripts/base/protocols/ssl/main.zeek
View file

@ -71,6 +71,10 @@ export {
## SSL history showing which types of packets we received in which order.
## Letters have the following meaning with client-sent letters being capitalized:
##
## ====== ====================================================
## Letter Meaning
## ====== ====================================================
## H hello_request
## C client_hello
## S server_hello
@ -97,12 +101,14 @@ export {
## J hello_retry_request
## L alert
## Q unknown_content_type
## ====== ====================================================
##
ssl_history: string &log &default="";
};
## The default root CA bundle. By default, the mozilla-ca-list.zeek
## script sets this to Mozilla's root CA list.
const root_certs: table[string] of string = {} &redef;
const root_certs: table[string] of string &redef;
## The record type which contains the field for the Certificate
## Transparency log bundle.

4
scripts/policy/frameworks/cluster/agent/__load__.zeek
View file

@ -1,4 +0,0 @@
##! The entry point for the cluster agent. It runs bootstrap logic for launching
##! the agent process via Zeek's Supervisor.
@load ./boot

271
scripts/policy/frameworks/cluster/agent/main.zeek
View file

@ -1,271 +0,0 @@
##! This is the main "runtime" of a cluster agent. Zeek does not load this
##! directly; rather, the agent's bootstrapping module (in ./boot.zeek)
##! specifies it as the script to run in the node newly created via Zeek's
##! supervisor.
@load base/frameworks/broker
@load policy/frameworks/cluster/controller/config
@load policy/frameworks/cluster/controller/log
@load policy/frameworks/cluster/controller/request
@load ./api
module ClusterAgent::Runtime;
redef ClusterController::role = ClusterController::Types::AGENT;
# The global configuration as passed to us by the controller
global g_config: ClusterController::Types::Configuration;
# A map to make other instance info accessible
global g_instances: table[string] of ClusterController::Types::Instance;
# A map for the nodes we run on this instance, via this agent.
global g_nodes: table[string] of ClusterController::Types::Node;
# The node map employed by the supervisor to describe the cluster
# topology to newly forked nodes. We refresh it when we receive
# new configurations.
global g_data_cluster: table[string] of Supervisor::ClusterEndpoint;
event SupervisorControl::create_response(reqid: string, result: string)
{
local req = ClusterController::Request::lookup(reqid);
if ( ClusterController::Request::is_null(req) )
return;
local name = req$supervisor_state$node;
if ( |result| > 0 )
{
local msg = fmt("failed to create node %s: %s", name, result);
ClusterController::Log::error(msg);
event ClusterAgent::API::notify_error(ClusterAgent::name, msg, name);
}
ClusterController::Request::finish(reqid);
}
event SupervisorControl::destroy_response(reqid: string, result: bool)
{
local req = ClusterController::Request::lookup(reqid);
if ( ClusterController::Request::is_null(req) )
return;
local name = req$supervisor_state$node;
if ( ! result )
{
local msg = fmt("failed to destroy node %s, %s", name, reqid);
ClusterController::Log::error(msg);
event ClusterAgent::API::notify_error(ClusterAgent::name, msg, name);
}
ClusterController::Request::finish(reqid);
}
function supervisor_create(nc: Supervisor::NodeConfig)
{
local req = ClusterController::Request::create();
req$supervisor_state = ClusterController::Request::SupervisorState($node = nc$name);
event SupervisorControl::create_request(req$id, nc);
ClusterController::Log::info(fmt("issued supervisor create for %s, %s", nc$name, req$id));
}
function supervisor_destroy(node: string)
{
local req = ClusterController::Request::create();
req$supervisor_state = ClusterController::Request::SupervisorState($node = node);
event SupervisorControl::destroy_request(req$id, node);
ClusterController::Log::info(fmt("issued supervisor destroy for %s, %s", node, req$id));
}
event ClusterAgent::API::set_configuration_request(reqid: string, config: ClusterController::Types::Configuration)
{
ClusterController::Log::info(fmt("rx ClusterAgent::API::set_configuration_request %s", reqid));
local nodename: string;
local node: ClusterController::Types::Node;
local nc: Supervisor::NodeConfig;
local msg: string;
# Adopt the global configuration provided.
# XXX this can later handle validation and persistence
# XXX should do this transactionally, only set when all else worked
g_config = config;
# Refresh the instances table:
g_instances = table();
for ( inst in config$instances )
g_instances[inst$name] = inst;
# Terminate existing nodes
for ( nodename in g_nodes )
supervisor_destroy(nodename);
g_nodes = table();
# Refresh the data cluster and nodes tables
g_data_cluster = table();
for ( node in config$nodes )
{
if ( node$instance == ClusterAgent::name )
g_nodes[node$name] = node;
local cep = Supervisor::ClusterEndpoint(
$role = node$role,
$host = g_instances[node$instance]$host,
$p = node$p);
if ( node?$interface )
cep$interface = node$interface;
g_data_cluster[node$name] = cep;
}
# Apply the new configuration via the supervisor
for ( nodename in g_nodes )
{
node = g_nodes[nodename];
nc = Supervisor::NodeConfig($name=nodename);
if ( ClusterAgent::cluster_directory != "" )
nc$directory = ClusterAgent::cluster_directory;
if ( node?$interface )
nc$interface = node$interface;
if ( node?$cpu_affinity )
nc$cpu_affinity = node$cpu_affinity;
if ( node?$scripts )
nc$scripts = node$scripts;
if ( node?$env )
nc$env = node$env;
# XXX could use options to enable per-node overrides for
# directory, stdout, stderr, others?
nc$cluster = g_data_cluster;
supervisor_create(nc);
}
# XXX this currently doesn not fail if any of above problems occurred,
# mainly due to the tediousness of handling the supervisor's response
# events asynchonously. The only indication of error will be
# notification events to the controller.
if ( reqid != "" )
{
local res = ClusterController::Types::Result(
$reqid = reqid,
$instance = ClusterAgent::name);
ClusterController::Log::info(fmt("tx ClusterAgent::API::set_configuration_response %s",
ClusterController::Types::result_to_string(res)));
event ClusterAgent::API::set_configuration_response(reqid, res);
}
}
event ClusterAgent::API::agent_welcome_request(reqid: string)
{
ClusterController::Log::info(fmt("rx ClusterAgent::API::agent_welcome_request %s", reqid));
local res = ClusterController::Types::Result(
$reqid = reqid,
$instance = ClusterAgent::name);
ClusterController::Log::info(fmt("tx ClusterAgent::API::agent_welcome_response %s",
ClusterController::Types::result_to_string(res)));
event ClusterAgent::API::agent_welcome_response(reqid, res);
}
event ClusterAgent::API::agent_standby_request(reqid: string)
{
ClusterController::Log::info(fmt("rx ClusterAgent::API::agent_standby_request %s", reqid));
# We shut down any existing cluster nodes via an empty configuration,
# and fall silent. We do not unpeer/disconnect (assuming we earlier
# peered/connected -- otherwise there's nothing we can do here via
# Broker anyway), mainly to keep open the possibility of running
# cluster nodes again later.
event ClusterAgent::API::set_configuration_request("", ClusterController::Types::Configuration());
local res = ClusterController::Types::Result(
$reqid = reqid,
$instance = ClusterAgent::name);
ClusterController::Log::info(fmt("tx ClusterAgent::API::agent_standby_response %s",
ClusterController::Types::result_to_string(res)));
event ClusterAgent::API::agent_standby_response(reqid, res);
}
event Broker::peer_added(peer: Broker::EndpointInfo, msg: string)
{
# This does not (cannot?) immediately verify that the new peer
# is in fact a controller, so we might send this in vain.
# Controllers register the agent upon receipt of the event.
local epi = ClusterAgent::endpoint_info();
event ClusterAgent::API::notify_agent_hello(epi$id,
to_addr(epi$network$address), ClusterAgent::API::version);
}
event zeek_init()
{
local epi = ClusterAgent::endpoint_info();
local agent_topic = ClusterAgent::topic_prefix + "/" + epi$id;
# The agent needs to peer with the supervisor -- this doesn't currently
# happen automatically. The address defaults to Broker's default, which
# relies on ZEEK_DEFAULT_LISTEN_ADDR and so might just be "". Broker
# internally falls back to listening on any; we pick 127.0.0.1.
local supervisor_addr = Broker::default_listen_address;
if ( supervisor_addr == "" )
supervisor_addr = "127.0.0.1";
Broker::peer(supervisor_addr, Broker::default_port, Broker::default_listen_retry);
# Agents need receive communication targeted at it, and any responses
# from the supervisor.
Broker::subscribe(agent_topic);
Broker::subscribe(SupervisorControl::topic_prefix);
# Auto-publish a bunch of events. Glob patterns or module-level
# auto-publish would be helpful here.
Broker::auto_publish(agent_topic, ClusterAgent::API::set_configuration_response);
Broker::auto_publish(agent_topic, ClusterAgent::API::agent_welcome_response);
Broker::auto_publish(agent_topic, ClusterAgent::API::agent_standby_response);
Broker::auto_publish(agent_topic, ClusterAgent::API::notify_agent_hello);
Broker::auto_publish(agent_topic, ClusterAgent::API::notify_change);
Broker::auto_publish(agent_topic, ClusterAgent::API::notify_error);
Broker::auto_publish(agent_topic, ClusterAgent::API::notify_log);
Broker::auto_publish(SupervisorControl::topic_prefix, SupervisorControl::create_request);
Broker::auto_publish(SupervisorControl::topic_prefix, SupervisorControl::create_response);
Broker::auto_publish(SupervisorControl::topic_prefix, SupervisorControl::destroy_request);
Broker::auto_publish(SupervisorControl::topic_prefix, SupervisorControl::destroy_response);
Broker::auto_publish(SupervisorControl::topic_prefix, SupervisorControl::restart_request);
Broker::auto_publish(SupervisorControl::topic_prefix, SupervisorControl::restart_response);
Broker::auto_publish(SupervisorControl::topic_prefix, SupervisorControl::stop_request);
# Establish connectivity with the controller.
if ( ClusterAgent::controller$address != "0.0.0.0" )
{
# We connect to the controller.
Broker::peer(ClusterAgent::controller$address,
ClusterAgent::controller$bound_port,
ClusterController::connect_retry);
}
else
{
# Controller connects to us; listen for it.
Broker::listen(cat(epi$network$address), epi$network$bound_port);
}
ClusterController::Log::info("agent is live");
}

4
scripts/policy/frameworks/cluster/controller/__load__.zeek
View file

@ -1,4 +0,0 @@
##! The entry point for the cluster controller. It runs bootstrap logic for
##! launching the controller process via Zeek's Supervisor.
@load ./boot

36
scripts/policy/frameworks/cluster/controller/boot.zeek
View file

@ -1,36 +0,0 @@
##! The cluster controller's boot logic runs in Zeek's supervisor and instructs
##! it to launch the controller process. The controller's main logic resides in
##! main.zeek, similarly to other frameworks. The new process will execute that
##! script.
##!
##! If the current process is not the Zeek supervisor, this does nothing.
@load ./config
event zeek_init()
{
if ( ! Supervisor::is_supervisor() )
return;
local epi = ClusterController::endpoint_info();
local sn = Supervisor::NodeConfig($name=epi$id, $bare_mode=T,
$scripts=vector("policy/frameworks/cluster/controller/main.zeek"));
if ( ClusterController::directory != "" )
sn$directory = ClusterController::directory;
if ( ClusterController::stdout_file != "" )
sn$stdout_file = ClusterController::stdout_file;
if ( ClusterController::stderr_file != "" )
sn$stderr_file = ClusterController::stderr_file;
# This helps Zeek run controller and agent with a minimal set of scripts.
sn$env["ZEEK_CLUSTER_MGMT_NODE"] = "CONTROLLER";
local res = Supervisor::create(sn);
if ( res != "" )
{
print(fmt("error: supervisor could not create controller node: %s", res));
exit(1);
}
}

110
scripts/policy/frameworks/cluster/controller/config.zeek
View file

@ -1,110 +0,0 @@
##! Configuration settings for the cluster controller.
@load policy/frameworks/cluster/agent/config
module ClusterController;
export {
## The name of this controller. Defaults to the value of the
## ZEEK_CONTROLLER_NAME environment variable. When that is unset and the
## user doesn't redef the value, the implementation defaults to
## "controller-<hostname>".
const name = getenv("ZEEK_CONTROLLER_NAME") &redef;
## The controller's stdout log name. If the string is non-empty, Zeek will
## produce a free-form log (i.e., not one governed by Zeek's logging
## framework) in Zeek's working directory. If left empty, no such log
## results.
##
## Note that the controller also establishes a "proper" Zeek log via the
## :zeek:see:`ClusterController::Log` module.
const stdout_file = "controller.stdout" &redef;
## The controller's stderr log name. Like :zeek:see:`ClusterController::stdout_file`,
## but for the stderr stream.
const stderr_file = "controller.stderr" &redef;
## The network address the controller listens on. By default this uses
## the value of the ZEEK_CONTROLLER_ADDR environment variable, but you
## may also redef to a specific value. When empty, the implementation
## falls back to :zeek:see:`ClusterController::default_address`.
const listen_address = getenv("ZEEK_CONTROLLER_ADDR") &redef;
## The fallback listen address if :zeek:see:`ClusterController::listen_address`
## remains empty. Unless redefined, this uses Broker's own default
## listen address.
const default_address = Broker::default_listen_address &redef;
## The network port the controller listens on. Counterpart to
## :zeek:see:`ClusterController::listen_address`, defaulting to the
## ZEEK_CONTROLLER_PORT environment variable.
const listen_port = getenv("ZEEK_CONTROLLER_PORT") &redef;
## The fallback listen port if :zeek:see:`ClusterController::listen_port`
## remains empty.
const default_port = 2150/tcp &redef;
## The controller's connect retry interval. Defaults to a more
## aggressive value compared to Broker's 30s.
const connect_retry = 1sec &redef;
## The controller's Broker topic. Clients send requests to this topic.
const topic = "zeek/cluster-control/controller" &redef;
## The role of this process in cluster management. Agent and controller
## both redefine this. Used during logging.
const role = ClusterController::Types::NONE &redef;
## The timeout for request state. Such state (see the :zeek:see:`ClusterController::Request`
## module) ties together request and response event pairs. The timeout causes
## its cleanup in the absence of a timely response. It applies both to
## state kept for client requests, as well as state in the agents for
## requests to the supervisor.
const request_timeout = 10sec &redef;
## An optional custom output directory for the controller's stdout and
## stderr logs. Agent and controller currently only log locally, not via
## the data cluster's logger node. (This might change in the future.)
## This means that if both write to the same log file, the output gets
## garbled.
const directory = "" &redef;
## Returns a :zeek:see:`Broker::NetworkInfo` record describing the controller.
global network_info: function(): Broker::NetworkInfo;
## Returns a :zeek:see:`Broker::EndpointInfo` record describing the controller.
global endpoint_info: function(): Broker::EndpointInfo;
}
function network_info(): Broker::NetworkInfo
{
local ni: Broker::NetworkInfo;
if ( ClusterController::listen_address != "" )
ni$address = ClusterController::listen_address;
else if ( ClusterController::default_address != "" )
ni$address = ClusterController::default_address;
else
ni$address = "127.0.0.1";
if ( ClusterController::listen_port != "" )
ni$bound_port = to_port(ClusterController::listen_port);
else
ni$bound_port = ClusterController::default_port;
return ni;
}
function endpoint_info(): Broker::EndpointInfo
{
local epi: Broker::EndpointInfo;
if ( ClusterController::name != "" )
epi$id = ClusterController::name;
else
epi$id = fmt("controller-%s", gethostname());
epi$network = network_info();
return epi;
}

555
scripts/policy/frameworks/cluster/controller/main.zeek
View file

@ -1,555 +0,0 @@
##! This is the main "runtime" of the cluster controller. Zeek does not load
##! this directly; rather, the controller's bootstrapping module (in ./boot.zeek)
##! specifies it as the script to run in the node newly created via Zeek's
##! supervisor.
@load base/frameworks/broker
@load policy/frameworks/cluster/agent/config
@load policy/frameworks/cluster/agent/api
@load ./api
@load ./log
@load ./request
@load ./util
module ClusterController::Runtime;
redef ClusterController::role = ClusterController::Types::CONTROLLER;
global check_instances_ready: function();
global add_instance: function(inst: ClusterController::Types::Instance);
global drop_instance: function(inst: ClusterController::Types::Instance);
global null_config: function(): ClusterController::Types::Configuration;
global is_null_config: function(config: ClusterController::Types::Configuration): bool;
# Checks whether the given instance is one that we know with different
# communication settings: a a different peering direction, a different listening
# port, etc. Used as a predicate to indicate when we need to drop the existing
# one from our internal state.
global is_instance_connectivity_change: function
(inst: ClusterController::Types::Instance): bool;
# The set of agents the controller interacts with to manage to currently
# configured cluster. This may be a subset of all the agents known to the
# controller, as tracked by the g_instances_known set. They key is the instance
# name and should match the $name member of the corresponding instance record.
global g_instances: table[string] of ClusterController::Types::Instance = table();
# The set of instances that have checked in with the controller. This is a
# superset of g_instances, since it covers any agent that has sent us a
# notify_agent_hello event.
global g_instances_known: set[string] = set();
# A corresponding set of instances/agents that we track in order to understand
# when all of the above instances have sent agent_welcome_response events. (An
# alternative would be to use a record that adds a single state bit for each
# instance, and store that above.)
global g_instances_ready: set[string] = set();
# The request ID of the most recent configuration update that's come in from
# a client. We track it here until we know we are ready to communicate with all
# agents required by the update.
global g_config_reqid_pending: string = "";
# The most recent configuration we have successfully deployed. This is also
# the one we send whenever the client requests it.
global g_config_current: ClusterController::Types::Configuration;
function send_config_to_agents(req: ClusterController::Request::Request,
config: ClusterController::Types::Configuration)
{
for ( name in g_instances )
{
if ( name !in g_instances_ready )
next;
local agent_topic = ClusterAgent::topic_prefix + "/" + name;
local areq = ClusterController::Request::create();
areq$parent_id = req$id;
# We track the requests sent off to each agent. As the
# responses come in, we can check them off as completed,
# and once all are, we respond back to the client.
req$set_configuration_state$requests += areq;
# We could also broadcast just once on the agent prefix, but
# explicit request/response pairs for each agent seems cleaner.
ClusterController::Log::info(fmt("tx ClusterAgent::API::set_configuration_request %s to %s", areq$id, name));
Broker::publish(agent_topic, ClusterAgent::API::set_configuration_request, areq$id, config);
}
}
# This is the &on_change handler for the g_instances_ready set, meaning
# it runs whenever a required agent has confirmed it's ready.
function check_instances_ready()
{
local cur_instances: set[string];
for ( inst in g_instances )
add cur_instances[inst];
if ( cur_instances == g_instances_ready )
event ClusterController::API::notify_agents_ready(cur_instances);
}
function add_instance(inst: ClusterController::Types::Instance)
{
g_instances[inst$name] = inst;
if ( inst?$listen_port )
Broker::peer(cat(inst$host), inst$listen_port,
ClusterController::connect_retry);
if ( inst$name in g_instances_known )
{
# The agent has already peered with us. Send welcome to indicate
# it's part of cluster management. Once it responds, we update
# the set of ready instances and proceed as feasible with config
# deployments.
local req = ClusterController::Request::create();
ClusterController::Log::info(fmt("tx ClusterAgent::API::agent_welcome_request to %s", inst$name));
Broker::publish(ClusterAgent::topic_prefix + "/" + inst$name,
ClusterAgent::API::agent_welcome_request, req$id);
}
}
function drop_instance(inst: ClusterController::Types::Instance)
{
if ( inst$name !in g_instances )
return;
# Send the agent a standby so it shuts down its cluster nodes & state
ClusterController::Log::info(fmt("tx ClusterAgent::API::agent_standby_request to %s", inst$name));
Broker::publish(ClusterAgent::topic_prefix + "/" + inst$name,
ClusterAgent::API::agent_standby_request, "");
delete g_instances[inst$name];
if ( inst$name in g_instances_ready )
delete g_instances_ready[inst$name];
# The agent remains in g_instances_known, to track that we're able
# to communicate with it in case it's required again.
ClusterController::Log::info(fmt("dropped instance %s", inst$name));
}
function null_config(): ClusterController::Types::Configuration
{
return ClusterController::Types::Configuration($id="");
}
function is_null_config(config: ClusterController::Types::Configuration): bool
{
return config$id == "";
}
function is_instance_connectivity_change(inst: ClusterController::Types::Instance): bool
{
# If we're not tracking this instance as part of a cluster config, it's
# not a change. (More precisely: we cannot say whether it's changed.)
if ( inst$name !in g_instances )
return F;
# The agent has peered with us and now uses a different host.
# XXX 0.0.0.0 is a workaround until we've resolved how agents that peer
# with us obtain their identity. Broker ID?
if ( inst$host != 0.0.0.0 && inst$host != g_instances[inst$name]$host )
return T;
# The agent has a listening port and the one we know does not, or vice
# versa. I.e., this is a change in the intended peering direction.
if ( inst?$listen_port != g_instances[inst$name]?$listen_port )
return T;
# Both have listening ports, but they differ.
if ( inst?$listen_port && g_instances[inst$name]?$listen_port &&
inst$listen_port != g_instances[inst$name]$listen_port )
return T;
return F;
}
event ClusterController::API::notify_agents_ready(instances: set[string])
{
local insts = ClusterController::Util::set_to_vector(instances);
ClusterController::Log::info(fmt("rx ClusterController::API:notify_agents_ready %s", join_string_vec(insts, ",")));
local req = ClusterController::Request::lookup(g_config_reqid_pending);
# If there's no pending request, when it's no longer available, or it
# doesn't have config state, don't do anything else.
if ( ClusterController::Request::is_null(req) || ! req?$set_configuration_state )
return;
# All instances requested in the pending configuration update are now
# known to us. Send them the config. As they send their response events
# we update the client's request state and eventually send the response
# event to the it.
send_config_to_agents(req, req$set_configuration_state$config);
}
event ClusterAgent::API::notify_agent_hello(instance: string, host: addr, api_version: count)
{
ClusterController::Log::info(fmt("rx ClusterAgent::API::notify_agent_hello %s %s", instance, host));
# When an agent checks in with a mismatching API version, we log the
# fact and drop its state, if any.
if ( api_version != ClusterController::API::version )
{
ClusterController::Log::warning(
fmt("instance %s/%s has checked in with incompatible API version %s",
instance, host, api_version));
if ( instance in g_instances )
drop_instance(g_instances[instance]);
if ( instance in g_instances_known )
delete g_instances_known[instance];
return;
}
add g_instances_known[instance];
if ( instance in g_instances && instance !in g_instances_ready )
{
# We need this instance for our cluster and have full context for
# it from the configuration. Tell agent.
local req = ClusterController::Request::create();
ClusterController::Log::info(fmt("tx ClusterAgent::API::agent_welcome_request to %s", instance));
Broker::publish(ClusterAgent::topic_prefix + "/" + instance,
ClusterAgent::API::agent_welcome_request, req$id);
}
}
event ClusterAgent::API::agent_welcome_response(reqid: string, result: ClusterController::Types::Result)
{
ClusterController::Log::info(fmt("rx ClusterAgent::API::agent_welcome_response %s", reqid));
local req = ClusterController::Request::lookup(reqid);
if ( ClusterController::Request::is_null(req) )
return;
ClusterController::Request::finish(req$id);
# An agent we've been waiting to hear back from is ready for cluster
# work. Double-check we still want it, otherwise drop it.
if ( ! result$success || result$instance !in g_instances )
{
ClusterController::Log::info(fmt(
"tx ClusterAgent::API::agent_standby_request to %s", result$instance));
Broker::publish(ClusterAgent::topic_prefix + "/" + result$instance,
ClusterAgent::API::agent_standby_request, "");
return;
}
add g_instances_ready[result$instance];
ClusterController::Log::info(fmt("instance %s ready", result$instance));
check_instances_ready();
}
event ClusterAgent::API::notify_change(instance: string, n: ClusterController::Types::Node,
old: ClusterController::Types::State,
new: ClusterController::Types::State)
{
# XXX TODO
}
event ClusterAgent::API::notify_error(instance: string, msg: string, node: string)
{
# XXX TODO
}
event ClusterAgent::API::notify_log(instance: string, msg: string, node: string)
{
# XXX TODO
}
event ClusterAgent::API::set_configuration_response(reqid: string, result: ClusterController::Types::Result)
{
ClusterController::Log::info(fmt("rx ClusterAgent::API::set_configuration_response %s", reqid));
# Retrieve state for the request we just got a response to
local areq = ClusterController::Request::lookup(reqid);
if ( ClusterController::Request::is_null(areq) )
return;
# Record the result and mark the request as done. This also
# marks the request as done in the parent-level request, since
# these records are stored by reference.
areq$results[0] = result; # We only have a single result here atm
areq$finished = T;
# Update the original request from the client:
local req = ClusterController::Request::lookup(areq$parent_id);
if ( ClusterController::Request::is_null(req) )
return;
# If there are any requests to the agents still unfinished,
# we're not done yet.
for ( i in req$set_configuration_state$requests )
if ( ! req$set_configuration_state$requests[i]$finished )
return;
# All set_configuration requests to instances are done, so respond
# back to client. We need to compose the result, aggregating
# the results we got from the requests to the agents. In the
# end we have one Result per instance requested in the
# original set_configuration_request.
#
# XXX we can likely generalize result aggregation in the request module.
for ( i in req$set_configuration_state$requests )
{
local r = req$set_configuration_state$requests[i];
local success = T;
local errors: string_vec;
local instance = "";
for ( j in r$results )
{
local res = r$results[j];
instance = res$instance;
if ( res$success )
next;
success = F;
errors += fmt("node %s failed: %s", res$node, res$error);
}
req$results += ClusterController::Types::Result(
$reqid = req$id,
$instance = instance,
$success = success,
$error = join_string_vec(errors, ", ")
);
ClusterController::Request::finish(r$id);
}
# We're now done with the original set_configuration request.
# Adopt the configuration as the current one.
g_config_current = req$set_configuration_state$config;
g_config_reqid_pending = "";
ClusterController::Log::info(fmt("tx ClusterController::API::set_configuration_response %s",
ClusterController::Request::to_string(req)));
event ClusterController::API::set_configuration_response(req$id, req$results);
ClusterController::Request::finish(req$id);
}
event ClusterController::API::set_configuration_request(reqid: string, config: ClusterController::Types::Configuration)
{
ClusterController::Log::info(fmt("rx ClusterController::API::set_configuration_request %s", reqid));
local res: ClusterController::Types::Result;
local req = ClusterController::Request::create(reqid);
req$set_configuration_state = ClusterController::Request::SetConfigurationState($config = config);
# At the moment there can only be one pending request.
if ( g_config_reqid_pending != "" )
{
res = ClusterController::Types::Result($reqid=reqid);
res$success = F;
res$error = fmt("request %s still pending", g_config_reqid_pending);
req$results += res;
ClusterController::Log::info(fmt("tx ClusterController::API::set_configuration_response %s",
ClusterController::Request::to_string(req)));
event ClusterController::API::set_configuration_response(req$id, req$results);
ClusterController::Request::finish(req$id);
return;
}
# XXX validate the configuration:
# - Are node instances among defined instances?
# - Are all names unique?
# - Are any node options understood?
# - Do node types with optional fields have required values?
# ...
# The incoming request is now the pending one. It gets cleared when all
# agents have processed their config updates successfully, or their
# responses time out.
g_config_reqid_pending = req$id;
# Compare the instance configuration to our current one. If it matches,
# we can proceed to deploying the new data cluster topology. If it does
# not, we need to establish connectivity with agents we connect to, or
# wait until all instances that connect to us have done so. Either triggers
# a notify_agents_ready event, upon which we then deploy the data cluster.
# The current & new set of instance names.
local insts_current: set[string];
local insts_new: set[string];
# A set of current instances not contained in the new config.
# Those will need to get dropped.
local insts_to_drop: set[string];
# The opposite: new instances not yet in our current set. Those we will need
# to establish contact with (or they with us).
local insts_to_add: set[string];
# The overlap: instances in both the current and new set. For those we verify
# that we're actually dealign with the same entities, and might need to re-
# connect if not.
local insts_to_keep: set[string];
# Alternative representation of insts_to_add, directly providing the instances.
local insts_to_peer: table[string] of ClusterController::Types::Instance;
# Helpful locals.
local inst_name: string;
local inst: ClusterController::Types::Instance;
for ( inst_name in g_instances )
add insts_current[inst_name];
for ( inst in config$instances )
add insts_new[inst$name];
# Populate TODO lists for instances we need to drop, check, or add.
insts_to_drop = insts_current - insts_new;
insts_to_add = insts_new - insts_current;
insts_to_keep = insts_new & insts_current;
for ( inst in config$instances )
{
if ( inst$name in insts_to_add )
{
insts_to_peer[inst$name] = inst;
next;
}
# Focus on the keepers: check for change in identity/location.
if ( inst$name !in insts_to_keep )
next;
if ( is_instance_connectivity_change(inst) )
{
# The endpoint looks different. We drop the current one
# and need to re-establish connectivity with the new
# one.
add insts_to_drop[inst$name];
add insts_to_add[inst$name];
}
}
# Process our TODO lists. Handle drops first, then additions, in
# case we need to re-establish connectivity with an agent.
for ( inst_name in insts_to_drop )
drop_instance(g_instances[inst_name]);
for ( inst_name in insts_to_peer )
add_instance(insts_to_peer[inst_name]);
# Updates to out instance tables are complete, now check if we're already
# able to send the config to the agents:
check_instances_ready();
}
event ClusterController::API::get_instances_request(reqid: string)
{
ClusterController::Log::info(fmt("rx ClusterController::API::set_instances_request %s", reqid));
local res = ClusterController::Types::Result($reqid = reqid);
local insts: vector of ClusterController::Types::Instance;
for ( i in g_instances )
insts += g_instances[i];
res$data = insts;
ClusterController::Log::info(fmt("tx ClusterController::API::get_instances_response %s", reqid));
event ClusterController::API::get_instances_response(reqid, res);
}
event ClusterController::Request::request_expired(req: ClusterController::Request::Request)
{
# Various handlers for timed-out request state. We use the state members
# to identify how to respond. No need to clean up the request itself,
# since we're getting here via the request module's expiration
# mechanism that handles the cleanup.
local res: ClusterController::Types::Result;
if ( req?$set_configuration_state )
{
# This timeout means we no longer have a pending request.
g_config_reqid_pending = "";
res = ClusterController::Types::Result($reqid=req$id);
res$success = F;
res$error = "request timed out";
req$results += res;
ClusterController::Log::info(fmt("tx ClusterController::API::set_configuration_response %s",
ClusterController::Request::to_string(req)));
event ClusterController::API::set_configuration_response(req$id, req$results);
}
if ( req?$test_state )
{
res = ClusterController::Types::Result($reqid=req$id);
res$success = F;
res$error = "request timed out";
ClusterController::Log::info(fmt("tx ClusterController::API::test_timeout_response %s", req$id));
event ClusterController::API::test_timeout_response(req$id, res);
}
}
event ClusterController::API::test_timeout_request(reqid: string, with_state: bool)
{
ClusterController::Log::info(fmt("rx ClusterController::API::test_timeout_request %s %s", reqid, with_state));
if ( with_state )
{
# This state times out and triggers a timeout response in the
# above request_expired event handler.
local req = ClusterController::Request::create(reqid);
req$test_state = ClusterController::Request::TestState();
}
}
event zeek_init()
{
# Initialize null config at startup. We will replace it once we have
# persistence, and again whenever we complete a client's
# set_configuration request.
g_config_current = null_config();
# The controller always listens -- it needs to be able to respond to the
# Zeek client. This port is also used by the agents if they connect to
# the client. The client doesn't automatically establish or accept
# connectivity to agents: agents are defined and communicated with as
# defined via configurations defined by the client.
local cni = ClusterController::network_info();
Broker::listen(cat(cni$address), cni$bound_port);
Broker::subscribe(ClusterAgent::topic_prefix);
Broker::subscribe(ClusterController::topic);
# Events sent to the client:
Broker::auto_publish(ClusterController::topic,
ClusterController::API::get_instances_response);
Broker::auto_publish(ClusterController::topic,
ClusterController::API::set_configuration_response);
Broker::auto_publish(ClusterController::topic,
ClusterController::API::test_timeout_response);
ClusterController::Log::info("controller is live");
}

11
scripts/policy/frameworks/management/__load__.zeek Normal file
View file

@ -0,0 +1,11 @@
##! This loads Management framework functionality needed by both the controller
##! and agents. Note that there's no notion of loading "the Management
##! framework" -- one always loads "management/controller" or
##! "management/agent". This __load__ script exists only to simplify loading all
##! common functionality.
@load ./config
@load ./log
@load ./request
@load ./types
@load ./util

4
scripts/policy/frameworks/management/agent/__load__.zeek Normal file
View file

@ -0,0 +1,4 @@
##! The entry point for the Management framework's cluster agent. It runs
##! bootstrap logic for launching the agent process via Zeek's Supervisor.
@load ./boot

78
scripts/policy/frameworks/cluster/agent/api.zeek → scripts/policy/frameworks/management/agent/api.zeek
View file

@ -4,16 +4,15 @@
##! "_response", respectively.
@load base/frameworks/supervisor/control
@load policy/frameworks/cluster/controller/types
@load policy/frameworks/management/types
module ClusterAgent::API;
module Management::Agent::API;
export {
## A simple versioning scheme, used to track basic compatibility of
## controller and agent.
const version = 1;
# Agent API events
## The controller sends this event to convey a new cluster configuration
@ -22,14 +21,14 @@ export {
##
## reqid: a request identifier string, echoed in the response event.
##
## config: a :zeek:see:`ClusterController::Types::Configuration` record
## config: a :zeek:see:`Management::Configuration` record
## describing the cluster topology. Note that this contains the full
## topology, not just the part pertaining to this agent. That's because
## the cluster framework requires full cluster visibility to establish
## the needed peerings.
##
global set_configuration_request: event(reqid: string,
config: ClusterController::Types::Configuration);
config: Management::Configuration);
## Response to a set_configuration_request event. The agent sends
## this back to the controller.
@ -39,7 +38,62 @@ export {
## result: the result record.
##
global set_configuration_response: event(reqid: string,
result: ClusterController::Types::Result);
result: Management::Result);
## The controller sends this event to request a list of
## :zeek:see:`Management::NodeStatus` records that capture
## the status of Supervisor-managed nodes running on this instance.
## instances.
##
## reqid: a request identifier string, echoed in the response event.
##
global get_nodes_request: event(reqid: string);
## Response to a get_nodes_request event. The agent sends this back to the
## controller.
##
## reqid: the request identifier used in the request event.
##
## result: a :zeek:see:`Management::Result` record. Its data
## member is a vector of :zeek:see:`Management::NodeStatus`
## records, covering the nodes at this instance. The result may also
## indicate failure, with error messages indicating what went wrong.
##
global get_nodes_response: event(reqid: string, result: Management::Result);
## The controller sends this to every agent to request a dispatch (the
## execution of a pre-implemented activity) to all cluster nodes. This
## is the generic controller-agent "back-end" implementation of explicit
## client-controller "front-end" interactions, including:
##
## - :zeek:see:`Management::Controller::API::get_id_value_request`: two
## arguments, the first being "get_id_value" and the second the name
## of the ID to look up.
##
## reqid: a request identifier string, echoed in the response event.
##
## action: the requested dispatch command, with any arguments.
##
## nodes: a set of cluster node names (e.g. "worker-01") to retrieve
## the values from. An empty set, supplied by default, means
## retrieval from all nodes managed by the agent.
global node_dispatch_request: event(reqid: string, action: vector of string,
nodes: set[string] &default=set());
## Response to a node_dispatch_request event. Each agent sends this back
## to the controller to report the dispatch outcomes on all nodes managed
## by that agent.
##
## reqid: the request identifier used in the request event.
##
## result: a :zeek:type:`vector` of :zeek:see:`Management::Result`
## records. Each record covers one Zeek cluster node managed by this
## agent. Upon success, each :zeek:see:`Management::Result` record's
## data member contains the dispatches' response in a data type
## appropriate for the respective dispatch.
global node_dispatch_response: event(reqid: string, result: Management::ResultVec);
## The controller sends this event to confirm to the agent that it is
@ -58,7 +112,7 @@ export {
## result: the result record.
##
global agent_welcome_response: event(reqid: string,
result: ClusterController::Types::Result);
result: Management::Result);
## The controller sends this event to convey that the agent is not
@ -81,7 +135,7 @@ export {
## result: the result record.
##
global agent_standby_response: event(reqid: string,
result: ClusterController::Types::Result);
result: Management::Result);
# Notification events, agent -> controller
@ -91,7 +145,7 @@ export {
## communicate with. It is a controller-level equivalent of
## `:zeek:see:`Broker::peer_added`.
##
## instance: an instance name, really the agent's name as per :zeek:see:`ClusterAgent::name`.
## instance: an instance name, really the agent's name as per :zeek:see:`Management::Agent::name`.
##
## host: the IP address of the agent. (This may change in the future.)
##
@ -105,9 +159,9 @@ export {
# Report node state changes.
global notify_change: event(instance: string,
n: ClusterController::Types::Node,
old: ClusterController::Types::State,
new: ClusterController::Types::State);
n: Management::Node,
old: Management::State,
new: Management::State);
# Report operational error.
global notify_error: event(instance: string, msg: string, node: string &default="");

18
scripts/policy/frameworks/cluster/agent/boot.zeek → scripts/policy/frameworks/management/agent/boot.zeek
View file

@ -1,5 +1,5 @@
##! The cluster agent boot logic runs in Zeek's supervisor and instructs it to
##! launch an agent process. The agent's main logic resides in main.zeek,
##! launch a Management agent process. The agent's main logic resides in main.zeek,
##! similarly to other frameworks. The new process will execute that script.
##!
##! If the current process is not the Zeek supervisor, this does nothing.
@ -17,16 +17,16 @@ event zeek_init()
if ( ! Supervisor::is_supervisor() )
return;
local epi = ClusterAgent::endpoint_info();
local epi = Management::Agent::endpoint_info();
local sn = Supervisor::NodeConfig($name=epi$id, $bare_mode=T,
$scripts=vector("policy/frameworks/cluster/agent/main.zeek"));
$scripts=vector("policy/frameworks/management/agent/main.zeek"));
if ( ClusterAgent::directory != "" )
sn$directory = ClusterAgent::directory;
if ( ClusterAgent::stdout_file_suffix != "" )
sn$stdout_file = epi$id + "." + ClusterAgent::stdout_file_suffix;
if ( ClusterAgent::stderr_file_suffix != "" )
sn$stderr_file = epi$id + "." + ClusterAgent::stderr_file_suffix;
if ( Management::Agent::directory != "" )
sn$directory = Management::Agent::directory;
if ( Management::Agent::stdout_file_suffix != "" )
sn$stdout_file = epi$id + "." + Management::Agent::stdout_file_suffix;
if ( Management::Agent::stderr_file_suffix != "" )
sn$stderr_file = epi$id + "." + Management::Agent::stderr_file_suffix;
# This helps Zeek run controller and agent with a minimal set of scripts.
sn$env["ZEEK_CLUSTER_MGMT_NODE"] = "AGENT";

67
scripts/policy/frameworks/cluster/agent/config.zeek → scripts/policy/frameworks/management/agent/config.zeek
View file

@ -1,8 +1,9 @@
##! Configuration settings for a cluster agent.
@load policy/frameworks/cluster/controller/types
@load policy/frameworks/management/config
@load policy/frameworks/management/types
module ClusterAgent;
module Management::Agent;
export {
## The name this agent uses to represent the cluster instance it
@ -14,55 +15,49 @@ export {
## Agent stdout log configuration. If the string is non-empty, Zeek will
## produce a free-form log (i.e., not one governed by Zeek's logging
## framework) in Zeek's working directory. The final log's name is
## "<name>.<suffix>", where the name is taken from :zeek:see:`ClusterAgent::name`,
## "<name>.<suffix>", where the name is taken from :zeek:see:`Management::Agent::name`,
## and the suffix is defined by the following variable. If left empty,
## no such log results.
##
## Note that the agent also establishes a "proper" Zeek log via the
## :zeek:see:`ClusterController::Log` module.
## :zeek:see:`Management::Log` module.
const stdout_file_suffix = "agent.stdout" &redef;
## Agent stderr log configuration. Like :zeek:see:`ClusterAgent::stdout_file_suffix`,
## Agent stderr log configuration. Like :zeek:see:`Management::Agent::stdout_file_suffix`,
## but for the stderr stream.
const stderr_file_suffix = "agent.stderr" &redef;
## The network address the agent listens on. This only takes effect if
## the agent isn't configured to connect to the controller (see
## :zeek:see:`ClusterAgent::controller`). By default this uses the value of the
## :zeek:see:`Management::Agent::controller`). By default this uses the value of the
## ZEEK_AGENT_ADDR environment variable, but you may also redef to
## a specific value. When empty, the implementation falls back to
## :zeek:see:`ClusterAgent::default_address`.
## :zeek:see:`Management::default_address`.
const listen_address = getenv("ZEEK_AGENT_ADDR") &redef;
## The fallback listen address if :zeek:see:`ClusterAgent::listen_address`
## remains empty. Unless redefined, this uses Broker's own default listen
## address.
const default_address = Broker::default_listen_address &redef;
## The network port the agent listens on. Counterpart to
## :zeek:see:`ClusterAgent::listen_address`, defaulting to the ZEEK_AGENT_PORT
## :zeek:see:`Management::Agent::listen_address`, defaulting to the ZEEK_AGENT_PORT
## environment variable.
const listen_port = getenv("ZEEK_AGENT_PORT") &redef;
## The fallback listen port if :zeek:see:`ClusterAgent::listen_port` remains empty.
## The fallback listen port if :zeek:see:`Management::Agent::listen_port` remains empty.
const default_port = 2151/tcp &redef;
## The agent's Broker topic prefix. For its own communication, the agent
## suffixes this with "/<name>", based on :zeek:see:`ClusterAgent::name`.
const topic_prefix = "zeek/cluster-control/agent" &redef;
## suffixes this with "/<name>", based on :zeek:see:`Management::Agent::name`.
const topic_prefix = "zeek/management/agent" &redef;
## The network coordinates of the controller. When defined, the agent
## peers with (and connects to) the controller; otherwise the controller
## will peer (and connect to) the agent, listening as defined by
## :zeek:see:`ClusterAgent::listen_address` and :zeek:see:`ClusterAgent::listen_port`.
## :zeek:see:`Management::Agent::listen_address` and :zeek:see:`Management::Agent::listen_port`.
const controller: Broker::NetworkInfo = [
$address="0.0.0.0", $bound_port=0/unknown] &redef;
## An optional custom output directory for the agent's stdout and stderr
## logs. Agent and controller currently only log locally, not via the
## data cluster's logger node. (This might change in the future.) This
## means that if both write to the same log file, the output gets
## garbled.
## An optional custom output directory for stdout/stderr. Agent and
## controller currently only log locally, not via the data cluster's
## logger node. This means that if both write to the same log file,
## output gets garbled.
const directory = "" &redef;
## The working directory for data cluster nodes created by this
@ -71,20 +66,20 @@ export {
## cluster nodes.
const cluster_directory = "" &redef;
## Returns a :zeek:see:`ClusterController::Types::Instance` describing this
## Returns a :zeek:see:`Management::Instance` describing this
## instance (its agent name plus listening address/port, as applicable).
global instance: function(): ClusterController::Types::Instance;
global instance: function(): Management::Instance;
## Returns a :zeek:see:`Broker::EndpointInfo` record for this instance.
## Similar to :zeek:see:`ClusterAgent::instance`, but with slightly different
## Similar to :zeek:see:`Management::Agent::instance`, but with slightly different
## data format.
global endpoint_info: function(): Broker::EndpointInfo;
}
function instance(): ClusterController::Types::Instance
function instance(): Management::Instance
{
local epi = endpoint_info();
return ClusterController::Types::Instance($name=epi$id,
return Management::Instance($name=epi$id,
$host=to_addr(epi$network$address),
$listen_port=epi$network$bound_port);
}
@ -94,22 +89,22 @@ function endpoint_info(): Broker::EndpointInfo
local epi: Broker::EndpointInfo;
local network: Broker::NetworkInfo;
if ( ClusterAgent::name != "" )
epi$id = ClusterAgent::name;
if ( Management::Agent::name != "" )
epi$id = Management::Agent::name;
else
epi$id = fmt("agent-%s", gethostname());
if ( ClusterAgent::listen_address != "" )
network$address = ClusterAgent::listen_address;
else if ( ClusterAgent::default_address != "" )
network$address = ClusterAgent::default_address;
if ( Management::Agent::listen_address != "" )
network$address = Management::Agent::listen_address;
else if ( Management::default_address != "" )
network$address = Management::default_address;
else
network$address = "127.0.0.1";
if ( ClusterAgent::listen_port != "" )
network$bound_port = to_port(ClusterAgent::listen_port);
if ( Management::Agent::listen_port != "" )
network$bound_port = to_port(Management::Agent::listen_port);
else
network$bound_port = ClusterAgent::default_port;
network$bound_port = Management::Agent::default_port;
epi$network = network;

586
scripts/policy/frameworks/management/agent/main.zeek Normal file
View file

@ -0,0 +1,586 @@
##! This is the main "runtime" of a cluster agent. Zeek does not load this
##! directly; rather, the agent's bootstrapping module (in ./boot.zeek)
##! specifies it as the script to run in the node newly created via Zeek's
##! supervisor.
@load base/frameworks/broker
@load policy/frameworks/management
@load policy/frameworks/management/node/api
@load policy/frameworks/management/node/config
@load ./api
@load ./config
module Mangement::Agent::Runtime;
# This export is mainly to appease Zeekygen's need to understand redefs of the
# Request record below. Without it, it fails to establish link targets for the
# tucked-on types.
export {
## Request state specific to the agent's Supervisor interactions.
type SupervisorState: record {
node: string; ##< Name of the node the Supervisor is acting on.
};
## Request state for node dispatches, tracking the requested action
## as well as received responses.
type NodeDispatchState: record {
## The dispatched action. The first string is a command,
## any remaining strings its arguments.
action: vector of string;
## Request state for every node managed by this agent.
requests: set[string] &default=set();
};
}
redef record Management::Request::Request += {
supervisor_state: SupervisorState &optional;
node_dispatch_state: NodeDispatchState &optional;
};
# Tag our logs correctly
redef Management::Log::role = Management::AGENT;
# The global configuration as passed to us by the controller
global g_config: Management::Configuration;
# A map to make other instance info accessible
global g_instances: table[string] of Management::Instance;
# A map for the nodes we run on this instance, via this agent.
global g_nodes: table[string] of Management::Node;
# The complete node map employed by the supervisor to describe the cluster
# topology to newly forked nodes. We refresh it when we receive new
# configurations.
global g_cluster: table[string] of Supervisor::ClusterEndpoint;
function agent_topic(): string
{
local epi = Management::Agent::endpoint_info();
return Management::Agent::topic_prefix + "/" + epi$id;
}
event SupervisorControl::create_response(reqid: string, result: string)
{
local req = Management::Request::lookup(reqid);
if ( Management::Request::is_null(req) )
return;
local name = req$supervisor_state$node;
if ( |result| > 0 )
{
local msg = fmt("failed to create node %s: %s", name, result);
Management::Log::error(msg);
Broker::publish(agent_topic(),
Management::Agent::API::notify_error,
Management::Agent::name, msg, name);
}
Management::Request::finish(reqid);
}
event SupervisorControl::destroy_response(reqid: string, result: bool)
{
local req = Management::Request::lookup(reqid);
if ( Management::Request::is_null(req) )
return;
local name = req$supervisor_state$node;
if ( ! result )
{
local msg = fmt("failed to destroy node %s, %s", name, reqid);
Management::Log::error(msg);
Broker::publish(agent_topic(),
Management::Agent::API::notify_error,
Management::Agent::name, msg, name);
}
Management::Request::finish(reqid);
}
function supervisor_create(nc: Supervisor::NodeConfig)
{
local req = Management::Request::create();
req$supervisor_state = SupervisorState($node = nc$name);
Broker::publish(SupervisorControl::topic_prefix,
SupervisorControl::create_request, req$id, nc);
Management::Log::info(fmt("issued supervisor create for %s, %s", nc$name, req$id));
}
function supervisor_destroy(node: string)
{
local req = Management::Request::create();
req$supervisor_state = SupervisorState($node = node);
Broker::publish(SupervisorControl::topic_prefix,
SupervisorControl::destroy_request, req$id, node);
Management::Log::info(fmt("issued supervisor destroy for %s, %s", node, req$id));
}
event Management::Agent::API::set_configuration_request(reqid: string, config: Management::Configuration)
{
Management::Log::info(fmt("rx Management::Agent::API::set_configuration_request %s", reqid));
local nodename: string;
local node: Management::Node;
local nc: Supervisor::NodeConfig;
local msg: string;
# Adopt the global configuration provided.
# XXX this can later handle validation and persistence
# XXX should do this transactionally, only set when all else worked
g_config = config;
# Refresh the instances table:
g_instances = table();
for ( inst in config$instances )
g_instances[inst$name] = inst;
# Terminate existing nodes
for ( nodename in g_nodes )
supervisor_destroy(nodename);
# Refresh the cluster and nodes tables
g_nodes = table();
g_cluster = table();
for ( node in config$nodes )
{
if ( node$instance == Management::Agent::name )
g_nodes[node$name] = node;
# The cluster and supervisor frameworks require a port for every
# node, using 0/unknown to signify "don't listen". We use
# optional values and map an absent value to 0/unknown.
local p = 0/unknown;
if ( node?$p )
p = node$p;
local cep = Supervisor::ClusterEndpoint(
$role = node$role,
$host = g_instances[node$instance]$host,
$p = p);
if ( node?$interface )
cep$interface = node$interface;
g_cluster[node$name] = cep;
}
# Apply the new configuration via the supervisor
for ( nodename in g_nodes )
{
node = g_nodes[nodename];
node$state = Management::PENDING;
nc = Supervisor::NodeConfig($name=nodename);
if ( Management::Agent::cluster_directory != "" )
nc$directory = Management::Agent::cluster_directory;
if ( node?$interface )
nc$interface = node$interface;
if ( node?$cpu_affinity )
nc$cpu_affinity = node$cpu_affinity;
if ( node?$scripts )
nc$scripts = node$scripts;
if ( node?$env )
nc$env = node$env;
# Always add the policy/management/node scripts to any cluster
# node, since we require it to be able to communicate with the
# node.
nc$scripts[|nc$scripts|] = "policy/frameworks/management/node";
# XXX could use options to enable per-node overrides for
# directory, stdout, stderr, others?
nc$cluster = g_cluster;
supervisor_create(nc);
}
# XXX this currently doesn not fail if any of above problems occurred,
# mainly due to the tediousness of handling the supervisor's response
# events asynchonously. The only indication of error will be
# notification events to the controller.
if ( reqid != "" )
{
local res = Management::Result(
$reqid = reqid,
$instance = Management::Agent::name);
Management::Log::info(fmt("tx Management::Agent::API::set_configuration_response %s",
Management::result_to_string(res)));
Broker::publish(agent_topic(),
Management::Agent::API::set_configuration_response, reqid, res);
}
}
event SupervisorControl::status_response(reqid: string, result: Supervisor::Status)
{
local req = Management::Request::lookup(reqid);
if ( Management::Request::is_null(req) )
return;
Management::Request::finish(reqid);
local res = Management::Result(
$reqid = req$parent_id, $instance = Management::Agent::name);
local node_statuses: Management::NodeStatusVec;
for ( node in result$nodes )
{
local sns = result$nodes[node]; # Supervisor node status
local cns = Management::NodeStatus(
$node=node, $state=Management::PENDING);
# Identify the role of the node. For cluster roles (worker,
# manager, etc) we derive this from the cluster node table. For
# agent and controller, we identify via environment variables
# that the controller framework establishes upon creation (see
# the respective boot.zeek scripts).
if ( node in sns$node$cluster )
{
cns$cluster_role = sns$node$cluster[node]$role;
# For cluster nodes, copy run state from g_nodes, our
# live node status table.
if ( node in g_nodes )
cns$state = g_nodes[node]$state;
# The supervisor's responses use 0/tcp (not 0/unknown)
# when indicating an unused port because its internal
# serialization always assumes TCP.
if ( sns$node$cluster[node]$p != 0/tcp )
cns$p = sns$node$cluster[node]$p;
}
else
{
if ( "ZEEK_CLUSTER_MGMT_NODE" in sns$node$env )
{
local role = sns$node$env["ZEEK_CLUSTER_MGMT_NODE"];
if ( role == "CONTROLLER" )
{
cns$mgmt_role = Management::CONTROLLER;
# Automatically declare the controller in running state
# here -- we'd not have received a request that brought
# us here otherwise.
cns$state = Management::RUNNING;
# The controller always listens, so the Zeek client can connect.
cns$p = Management::Agent::endpoint_info()$network$bound_port;
}
else if ( role == "AGENT" )
{
cns$mgmt_role = Management::AGENT;
# Similarly to above, always declare agent running. We are. :)
cns$state = Management::RUNNING;
# If we have a controller address, the agent connects to it
# and does not listen. See zeek_init() below for similar logic.
if ( Management::Agent::controller$address == "0.0.0.0" )
cns$p = Management::Agent::endpoint_info()$network$bound_port;
}
else
Management::Log::warning(fmt(
"unexpected cluster management node type '%'", role));
}
}
# A PID is available if a supervised node has fully launched.
if ( sns?$pid )
cns$pid = sns$pid;
node_statuses += cns;
}
res$data = node_statuses;
Management::Log::info(fmt("tx Management::Agent::API::get_nodes_response %s",
Management::result_to_string(res)));
Broker::publish(agent_topic(),
Management::Agent::API::get_nodes_response, req$parent_id, res);
}
event Management::Agent::API::get_nodes_request(reqid: string)
{
Management::Log::info(fmt("rx Management::Agent::API::get_nodes_request %s", reqid));
local req = Management::Request::create();
req$parent_id = reqid;
Broker::publish(SupervisorControl::topic_prefix,
SupervisorControl::status_request, req$id, "");
Management::Log::info(fmt("issued supervisor status, %s", req$id));
}
event Management::Node::API::node_dispatch_response(reqid: string, result: Management::Result)
{
local node = "unknown node";
if ( result?$node )
node = result$node;
Management::Log::info(fmt("rx Management::Node::API::node_dispatch_response %s from %s", reqid, node));
# Retrieve state for the request we just got a response to
local nreq = Management::Request::lookup(reqid);
if ( Management::Request::is_null(nreq) )
return;
# Find the original request from the controller
local req = Management::Request::lookup(nreq$parent_id);
if ( Management::Request::is_null(req) )
return;
# Mark the responding node as done. Nodes normally fill their own name
# into the result; we only double-check for resilience. Nodes failing to
# report themselves would eventually lead to request timeout.
if ( result?$node )
{
if ( result$node in req$node_dispatch_state$requests )
delete req$node_dispatch_state$requests[result$node];
else
{
# An unknown or duplicate response -- do nothing.
Management::Log::debug(fmt("response %s not expected, ignoring", reqid));
return;
}
}
# The usual special treatment for Broker values that are of type "any":
# confirm their type here based on the requested dispatch command.
switch req$node_dispatch_state$action[0]
{
case "get_id_value":
if ( result?$data )
result$data = result$data as string;
break;
default:
Management::Log::error(fmt("unexpected dispatch command %s",
req$node_dispatch_state$action[0]));
break;
}
# The result has the reporting node filled in but not the agent/instance
# (which the node doesn't know about), so add it now.
result$instance = Management::Agent::instance()$name;
# Add this result to the overall response
req$results[|req$results|] = result;
# If we still have pending queries out to the agents, do nothing: we'll
# handle this soon, or our request will time out and we respond with
# error.
if ( |req$node_dispatch_state$requests| > 0 )
return;
# Release the agent-nodes request state, since we now have all responses.
Management::Request::finish(nreq$id);
# Send response event back to controller and clean up main request state.
Management::Log::info(fmt("tx Management::Agent::API::node_dispatch_response %s",
Management::Request::to_string(req)));
Broker::publish(agent_topic(),
Management::Agent::API::node_dispatch_response, req$id, req$results);
Management::Request::finish(req$id);
}
event Management::Agent::API::node_dispatch_request(reqid: string, action: vector of string, nodes: set[string])
{
Management::Log::info(fmt("rx Management::Agent::API::node_dispatch_request %s %s %s", reqid, action, nodes));
local node: string;
local cluster_nodes: set[string];
local nodes_final: set[string];
for ( node in g_nodes )
add cluster_nodes[node];
# If this request includes cluster nodes to query, check if this agent
# manages any of those nodes. If it doesn't, respond with an empty
# results vector immediately. Note that any globally unknown nodes
# that the client might have requested already got filtered by the
# controller, so we don't need to worry about them here.
if ( |nodes| > 0 )
{
nodes_final = nodes & cluster_nodes;
if ( |nodes_final| == 0 )
{
Management::Log::info(fmt(
"tx Management::Agent::API::node_dispatch_response %s, no node overlap",
reqid));
Broker::publish(agent_topic(),
Management::Agent::API::node_dispatch_response, reqid, vector());
return;
}
}
else if ( |g_nodes| == 0 )
{
# Special case: the client did not request specific nodes. If
# we aren't running any nodes, respond right away, since there's
# nothing to dispatch to.
Management::Log::info(fmt(
"tx Management::Agent::API::node_dispatch_response %s, no nodes registered",
reqid));
Broker::publish(agent_topic(),
Management::Agent::API::node_dispatch_response, reqid, vector());
return;
}
else
{
# We send to all known nodes.
nodes_final = cluster_nodes;
}
local res: Management::Result;
local req = Management::Request::create(reqid);
req$node_dispatch_state = NodeDispatchState($action=action);
# Build up dispatch state for tracking responses. We only dispatch to
# nodes that are in state RUNNING, as those have confirmed they're ready
# to communicate. For others, establish error state in now.
for ( node in nodes_final )
{
if ( g_nodes[node]$state == Management::RUNNING )
add req$node_dispatch_state$requests[node];
else
{
res = Management::Result($reqid=reqid, $node=node);
res$success = F;
res$error = fmt("cluster node %s not in runnning state", node);
req$results += res;
}
}
# Corner case: nothing is in state RUNNING.
if ( |req$node_dispatch_state$requests| == 0 )
{
Management::Log::info(fmt(
"tx Management::Agent::API::node_dispatch_response %s, no nodes running",
reqid));
Broker::publish(agent_topic(),
Management::Agent::API::node_dispatch_response, reqid, req$results);
Management::Request::finish(req$id);
return;
}
# We use a single request record to track all node responses, and a
# single event that Broker publishes to all nodes. We know when all
# nodes have responded by checking the requests set we built up above.
local nreq = Management::Request::create();
nreq$parent_id = reqid;
Management::Log::info(fmt("tx Management::Node::API::node_dispatch_request %s %s", nreq$id, action));
Broker::publish(Management::Node::node_topic,
Management::Node::API::node_dispatch_request, nreq$id, action, nodes);
}
event Management::Agent::API::agent_welcome_request(reqid: string)
{
Management::Log::info(fmt("rx Management::Agent::API::agent_welcome_request %s", reqid));
local res = Management::Result(
$reqid = reqid,
$instance = Management::Agent::name);
Management::Log::info(fmt("tx Management::Agent::API::agent_welcome_response %s",
Management::result_to_string(res)));
Broker::publish(agent_topic(),
Management::Agent::API::agent_welcome_response, reqid, res);
}
event Management::Agent::API::agent_standby_request(reqid: string)
{
Management::Log::info(fmt("rx Management::Agent::API::agent_standby_request %s", reqid));
# We shut down any existing cluster nodes via an empty configuration,
# and fall silent. We do not unpeer/disconnect (assuming we earlier
# peered/connected -- otherwise there's nothing we can do here via
# Broker anyway), mainly to keep open the possibility of running
# cluster nodes again later.
event Management::Agent::API::set_configuration_request("", Management::Configuration());
local res = Management::Result(
$reqid = reqid,
$instance = Management::Agent::name);
Management::Log::info(fmt("tx Management::Agent::API::agent_standby_response %s",
Management::result_to_string(res)));
Broker::publish(agent_topic(),
Management::Agent::API::agent_standby_response, reqid, res);
}
event Management::Node::API::notify_node_hello(node: string)
{
Management::Log::info(fmt("rx Management::Node::API::notify_node_hello %s", node));
if ( node in g_nodes )
g_nodes[node]$state = Management::RUNNING;
}
event Broker::peer_added(peer: Broker::EndpointInfo, msg: string)
{
# This does not (cannot?) immediately verify that the new peer
# is in fact a controller, so we might send this in vain.
# Controllers register the agent upon receipt of the event.
local epi = Management::Agent::endpoint_info();
Broker::publish(agent_topic(),
Management::Agent::API::notify_agent_hello,
epi$id, to_addr(epi$network$address),
Management::Agent::API::version);
}
# XXX We may want a request timeout event handler here. It's arguably cleaner to
# send supervisor failure events back to the controller than to rely on its
# controller-agent request timeout to kick in.
event zeek_init()
{
local epi = Management::Agent::endpoint_info();
# The agent needs to peer with the supervisor -- this doesn't currently
# happen automatically. The address defaults to Broker's default, which
# relies on ZEEK_DEFAULT_LISTEN_ADDR and so might just be "". Broker
# internally falls back to listening on any; we pick 127.0.0.1.
local supervisor_addr = Broker::default_listen_address;
if ( supervisor_addr == "" )
supervisor_addr = "127.0.0.1";
Broker::peer(supervisor_addr, Broker::default_port, Broker::default_listen_retry);
# Agents need receive communication targeted at it, any responses
# from the supervisor, and any responses from cluster nodes.
Broker::subscribe(agent_topic());
Broker::subscribe(SupervisorControl::topic_prefix);
Broker::subscribe(Management::Node::node_topic);
# Establish connectivity with the controller.
if ( Management::Agent::controller$address != "0.0.0.0" )
{
# We connect to the controller.
Broker::peer(Management::Agent::controller$address,
Management::Agent::controller$bound_port,
Management::connect_retry);
}
# The agent always listens, to allow cluster nodes to peer with it.
# If the controller connects to us, it also uses this port.
Broker::listen(cat(epi$network$address), epi$network$bound_port);
Management::Log::info("agent is live");
}

20
scripts/policy/frameworks/management/config.zeek Normal file
View file

@ -0,0 +1,20 @@
##! Management framework configuration settings common to agent and controller.
##! This does not include config settings that exist in both agent and
##! controller but that they set differently, since setting defaults here would
##! be awkward or pointless (since both node types would overwrite them
##! anyway). For role-specific settings, see management/controller/config.zeek
##! and management/agent/config.zeek.
module Management;
export {
## The fallback listen address if more specific adddresses, such as
## the controller's :zeek:see:`Management::Controller::listen_address`
## remains empty. Unless redefined, this uses Broker's own default
## listen address.
const default_address = Broker::default_listen_address &redef;
## The retry interval for Broker connnects. Defaults to a more
## aggressive value compared to Broker's 30s.
const connect_retry = 1sec &redef;
}

4
scripts/policy/frameworks/management/controller/__load__.zeek Normal file
View file

@ -0,0 +1,4 @@
##! The entry point for the Management framework's cluster controller. It runs
##! bootstrap logic for launching the controller process via Zeek's Supervisor.
@load ./boot

73
scripts/policy/frameworks/cluster/controller/api.zeek → scripts/policy/frameworks/management/controller/api.zeek
View file

@ -3,9 +3,9 @@
##! corresponding response event. Such event pairs share the same name prefix
##! and end in "_request" and "_response", respectively.
@load ./types
@load policy/frameworks/management/types
module ClusterController::API;
module Management::Controller::API;
export {
## A simple versioning scheme, used to track basic compatibility of
@ -26,10 +26,10 @@ export {
## reqid: the request identifier used in the request event.
##
## result: the result record. Its data member is a
## :zeek:see:`ClusterController::Types::Instance` record.
## :zeek:see:`Management::Instance` record.
##
global get_instances_response: event(reqid: string,
result: ClusterController::Types::Result);
result: Management::Result);
## zeek-client sends this event to establish a new cluster configuration,
@ -39,22 +39,75 @@ export {
##
## reqid: a request identifier string, echoed in the response event.
##
## config: a :zeek:see:`ClusterController::Types::Configuration` record
## config: a :zeek:see:`Management::Configuration` record
## specifying the cluster configuration.
##
global set_configuration_request: event(reqid: string,
config: ClusterController::Types::Configuration);
config: Management::Configuration);
## Response to a set_configuration_request event. The controller sends
## this back to the client.
##
## reqid: the request identifier used in the request event.
##
## result: a vector of :zeek:see:`ClusterController::Types::Result` records.
## result: a vector of :zeek:see:`Management::Result` records.
## Each member captures one agent's response.
##
global set_configuration_response: event(reqid: string,
result: ClusterController::Types::ResultVec);
result: Management::ResultVec);
## zeek-client sends this event to request a list of
## :zeek:see:`Management::NodeStatus` records that capture
## the status of Supervisor-managed nodes running on the cluster's
## instances.
##
## reqid: a request identifier string, echoed in the response event.
##
global get_nodes_request: event(reqid: string);
## Response to a get_nodes_request event. The controller sends this
## back to the client.
##
## reqid: the request identifier used in the request event.
##
## result: a :zeek:type:`vector` of :zeek:see:`Management::Result`
## records. Each record covers one cluster instance. Each record's data
## member is a vector of :zeek:see:`Management::NodeStatus`
## records, covering the nodes at that instance. Results may also indicate
## failure, with error messages indicating what went wrong.
global get_nodes_response: event(reqid: string,
result: Management::ResultVec);
## zeek-client sends this event to retrieve the current value of a
## variable in Zeek's global namespace, referenced by the given
## identifier (i.e., variable name). The controller asks all agents
## to retrieve this value from each cluster node, accumulates the
## returned responses, and responds with a get_id_value_response
## event back to the client.
##
## reqid: a request identifier string, echoed in the response event.
##
## id: the name of the variable whose value to retrieve.
##
## nodes: a set of cluster node names (e.g. "worker-01") to retrieve
## the values from. An empty set, supplied by default, means
## retrieval from all current cluster nodes.
global get_id_value_request: event(reqid: string, id: string,
nodes: set[string] &default=set());
## Response to a get_id_value_request event. The controller sends this
## back to the client.
##
## reqid: the request identifier used in the request event.
##
## result: a :zeek:type:`vector` of :zeek:see:`Management::Result`
## records. Each record covers one Zeek cluster node. Each record's
## data field contains a string with the JSON rendering (as produced
## by :zeek:id:`to_json`, including the error strings it potentially
## returns).
global get_id_value_response: event(reqid: string, result: Management::ResultVec);
# Testing events. These don't provide operational value but expose
@ -79,10 +132,10 @@ export {
## reqid: the request identifier used in the request event.
##
global test_timeout_response: event(reqid: string,
result: ClusterController::Types::Result);
result: Management::Result);
# Notification events, agent -> controller
# Notification events
## The controller triggers this event when the operational cluster
## instances align with the ones desired by the cluster

36
scripts/policy/frameworks/management/controller/boot.zeek Normal file
View file

@ -0,0 +1,36 @@
##! The cluster controller's boot logic runs in Zeek's supervisor and instructs
##! it to launch the Management controller process. The controller's main logic
##! resides in main.zeek, similarly to other frameworks. The new process will
##! execute that script.
##!
##! If the current process is not the Zeek supervisor, this does nothing.
@load ./config
event zeek_init()
{
if ( ! Supervisor::is_supervisor() )
return;
local epi = Management::Controller::endpoint_info();
local sn = Supervisor::NodeConfig($name=epi$id, $bare_mode=T,
$scripts=vector("policy/frameworks/management/controller/main.zeek"));
if ( Management::Controller::directory != "" )
sn$directory = Management::Controller::directory;
if ( Management::Controller::stdout_file != "" )
sn$stdout_file = Management::Controller::stdout_file;
if ( Management::Controller::stderr_file != "" )
sn$stderr_file = Management::Controller::stderr_file;
# This helps Zeek run controller and agent with a minimal set of scripts.
sn$env["ZEEK_CLUSTER_MGMT_NODE"] = "CONTROLLER";
local res = Supervisor::create(sn);
if ( res != "" )
{
print(fmt("error: supervisor could not create controller node: %s", res));
exit(1);
}
}

90
scripts/policy/frameworks/management/controller/config.zeek Normal file
View file

@ -0,0 +1,90 @@
##! Configuration settings for the cluster controller.
@load policy/frameworks/management/config
@load policy/frameworks/management/types
module Management::Controller;
export {
## The name of this controller. Defaults to the value of the
## ZEEK_CONTROLLER_NAME environment variable. When that is unset and the
## user doesn't redef the value, the implementation defaults to
## "controller-<hostname>".
const name = getenv("ZEEK_CONTROLLER_NAME") &redef;
## The controller's stdout log name. If the string is non-empty, Zeek will
## produce a free-form log (i.e., not one governed by Zeek's logging
## framework) in Zeek's working directory. If left empty, no such log
## results.
##
## Note that the controller also establishes a "proper" Zeek log via the
## :zeek:see:`Management::Log` module.
const stdout_file = "controller.stdout" &redef;
## The controller's stderr log name. Like :zeek:see:`Management::Controller::stdout_file`,
## but for the stderr stream.
const stderr_file = "controller.stderr" &redef;
## The network address the controller listens on. By default this uses
## the value of the ZEEK_CONTROLLER_ADDR environment variable, but you
## may also redef to a specific value. When empty, the implementation
## falls back to :zeek:see:`Management::default_address`.
const listen_address = getenv("ZEEK_CONTROLLER_ADDR") &redef;
## The network port the controller listens on. Counterpart to
## :zeek:see:`Management::Controller::listen_address`, defaulting to the
## ZEEK_CONTROLLER_PORT environment variable.
const listen_port = getenv("ZEEK_CONTROLLER_PORT") &redef;
## The fallback listen port if :zeek:see:`Management::Controller::listen_port`
## remains empty.
const default_port = 2150/tcp &redef;
## The controller's Broker topic. Clients send requests to this topic.
const topic = "zeek/management/controller" &redef;
## An optional custom output directory for stdout/stderr. Agent and
## controller currently only log locally, not via the data cluster's
## logger node. This means that if both write to the same log file,
## output gets garbled.
const directory = "" &redef;
## Returns a :zeek:see:`Broker::NetworkInfo` record describing the controller.
global network_info: function(): Broker::NetworkInfo;
## Returns a :zeek:see:`Broker::EndpointInfo` record describing the controller.
global endpoint_info: function(): Broker::EndpointInfo;
}
function network_info(): Broker::NetworkInfo
{
local ni: Broker::NetworkInfo;
if ( Management::Controller::listen_address != "" )
ni$address = Management::Controller::listen_address;
else if ( Management::default_address != "" )
ni$address = Management::default_address;
else
ni$address = "127.0.0.1";
if ( Management::Controller::listen_port != "" )
ni$bound_port = to_port(Management::Controller::listen_port);
else
ni$bound_port = Management::Controller::default_port;
return ni;
}
function endpoint_info(): Broker::EndpointInfo
{
local epi: Broker::EndpointInfo;
if ( Management::Controller::name != "" )
epi$id = Management::Controller::name;
else
epi$id = fmt("controller-%s", gethostname());
epi$network = network_info();
return epi;
}

836
scripts/policy/frameworks/management/controller/main.zeek Normal file
View file

@ -0,0 +1,836 @@
##! This is the main "runtime" of the Management framework's controller. Zeek
##! does not load this directly; rather, the controller's bootstrapping module
##! (in ./boot.zeek) specifies it as the script to run in the node newly created
##! by the supervisor.
@load base/frameworks/broker
@load policy/frameworks/management
@load policy/frameworks/management/agent/config # For the agent topic prefix
@load policy/frameworks/management/agent/api
@load ./api
@load ./config
module Management::Controller::Runtime;
# This export is mainly to appease Zeekygen's need to understand redefs of the
# Request record below. Without it, it fails to establish link targets for the
# tucked-on types.
export {
## Request state specific to
## :zeek:see:`Management::Controller::API::set_configuration_request` and
## :zeek:see:`Management::Controller::API::set_configuration_response`.
type SetConfigurationState: record {
## The cluster configuration established with this request
config: Management::Configuration;
## Request state for every controller/agent transaction.
requests: set[string] &default=set();
};
## Request state specific to
## :zeek:see:`Management::Controller::API::get_nodes_request` and
## :zeek:see:`Management::Controller::API::get_nodes_response`.
type GetNodesState: record {
## Request state for every controller/agent transaction.
requests: set[string] &default=set();
};
## Request state for node dispatch requests, to track the requested
## action and received responses. Node dispatches are requests to
## execute pre-implemented actions on every node in the cluster,
## and report their outcomes. See
## :zeek:see:`Management::Agent::API::node_dispatch_request` and
## :zeek:see:`Management::Agent::API::node_dispatch_response` for the
## agent/controller interaction, and
## :zeek:see:`Management::Controller::API::get_id_value_request` and
## :zeek:see:`Management::Controller::API::get_id_value_response`
## for an example of a specific API the controller generalizes into
## a dispatch.
type NodeDispatchState: record {
## The dispatched action. The first string is a command,
## any remaining strings its arguments.
action: vector of string;
## Request state for every controller/agent transaction.
## The set of strings tracks the node names from which
## we still expect responses, before we can respond back
## to the client.
requests: set[string] &default=set();
};
## Dummy state for internal state-keeping test cases.
type TestState: record { };
}
redef record Management::Request::Request += {
set_configuration_state: SetConfigurationState &optional;
get_nodes_state: GetNodesState &optional;
node_dispatch_state: NodeDispatchState &optional;
test_state: TestState &optional;
};
# Tag our logs correctly
redef Management::Log::role = Management::CONTROLLER;
global check_instances_ready: function();
global add_instance: function(inst: Management::Instance);
global drop_instance: function(inst: Management::Instance);
global null_config: function(): Management::Configuration;
global is_null_config: function(config: Management::Configuration): bool;
# Checks whether the given instance is one that we know with different
# communication settings: a a different peering direction, a different listening
# port, etc. Used as a predicate to indicate when we need to drop the existing
# one from our internal state.
global is_instance_connectivity_change: function
(inst: Management::Instance): bool;
# The set of agents the controller interacts with to manage to currently
# configured cluster. This may be a subset of all the agents known to the
# controller, as tracked by the g_instances_known set. They key is the instance
# name and should match the $name member of the corresponding instance record.
global g_instances: table[string] of Management::Instance = table();
# The set of instances that have checked in with the controller. This is a
# superset of g_instances, since it covers any agent that has sent us a
# notify_agent_hello event.
global g_instances_known: set[string] = set();
# A corresponding set of instances/agents that we track in order to understand
# when all of the above instances have sent agent_welcome_response events. (An
# alternative would be to use a record that adds a single state bit for each
# instance, and store that above.)
global g_instances_ready: set[string] = set();
# The request ID of the most recent configuration update that's come in from
# a client. We track it here until we know we are ready to communicate with all
# agents required by the update.
global g_config_reqid_pending: string = "";
# The most recent configuration we have successfully deployed. This is also
# the one we send whenever the client requests it.
global g_config_current: Management::Configuration;
function send_config_to_agents(req: Management::Request::Request, config: Management::Configuration)
{
for ( name in g_instances )
{
if ( name !in g_instances_ready )
next;
local agent_topic = Management::Agent::topic_prefix + "/" + name;
local areq = Management::Request::create();
areq$parent_id = req$id;
# We track the requests sent off to each agent. As the
# responses come in, we delete them. Once the requests
# set is empty, we respond back to the client.
add req$set_configuration_state$requests[areq$id];
# We could also broadcast just once on the agent prefix, but
# explicit request/response pairs for each agent seems cleaner.
Management::Log::info(fmt("tx Management::Agent::API::set_configuration_request %s to %s", areq$id, name));
Broker::publish(agent_topic, Management::Agent::API::set_configuration_request, areq$id, config);
}
}
# This is the &on_change handler for the g_instances_ready set, meaning
# it runs whenever a required agent has confirmed it's ready.
function check_instances_ready()
{
local cur_instances: set[string];
for ( inst in g_instances )
add cur_instances[inst];
if ( cur_instances == g_instances_ready )
event Management::Controller::API::notify_agents_ready(cur_instances);
}
function add_instance(inst: Management::Instance)
{
g_instances[inst$name] = inst;
if ( inst?$listen_port )
Broker::peer(cat(inst$host), inst$listen_port,
Management::connect_retry);
if ( inst$name in g_instances_known )
{
# The agent has already peered with us. Send welcome to indicate
# it's part of cluster management. Once it responds, we update
# the set of ready instances and proceed as feasible with config
# deployments.
local req = Management::Request::create();
Management::Log::info(fmt("tx Management::Agent::API::agent_welcome_request to %s", inst$name));
Broker::publish(Management::Agent::topic_prefix + "/" + inst$name,
Management::Agent::API::agent_welcome_request, req$id);
}
}
function drop_instance(inst: Management::Instance)
{
if ( inst$name !in g_instances )
return;
# Send the agent a standby so it shuts down its cluster nodes & state
Management::Log::info(fmt("tx Management::Agent::API::agent_standby_request to %s", inst$name));
Broker::publish(Management::Agent::topic_prefix + "/" + inst$name,
Management::Agent::API::agent_standby_request, "");
delete g_instances[inst$name];
if ( inst$name in g_instances_ready )
delete g_instances_ready[inst$name];
# The agent remains in g_instances_known, to track that we're able
# to communicate with it in case it's required again.
Management::Log::info(fmt("dropped instance %s", inst$name));
}
function null_config(): Management::Configuration
{
return Management::Configuration($id="");
}
function is_null_config(config: Management::Configuration): bool
{
return config$id == "";
}
function is_instance_connectivity_change(inst: Management::Instance): bool
{
# If we're not tracking this instance as part of a cluster config, it's
# not a change. (More precisely: we cannot say whether it's changed.)
if ( inst$name !in g_instances )
return F;
# The agent has peered with us and now uses a different host.
# XXX 0.0.0.0 is a workaround until we've resolved how agents that peer
# with us obtain their identity. Broker ID?
if ( inst$host != 0.0.0.0 && inst$host != g_instances[inst$name]$host )
return T;
# The agent has a listening port and the one we know does not, or vice
# versa. I.e., this is a change in the intended peering direction.
if ( inst?$listen_port != g_instances[inst$name]?$listen_port )
return T;
# Both have listening ports, but they differ.
if ( inst?$listen_port && g_instances[inst$name]?$listen_port &&
inst$listen_port != g_instances[inst$name]$listen_port )
return T;
return F;
}
function filter_config_nodes_by_name(nodes: set[string]): set[string]
{
local res: set[string];
local cluster_nodes: set[string];
for ( node in g_config_current$nodes )
add cluster_nodes[node$name];
return nodes & cluster_nodes;
}
event Management::Controller::API::notify_agents_ready(instances: set[string])
{
local insts = Management::Util::set_to_vector(instances);
Management::Log::info(fmt("rx Management::Controller::API:notify_agents_ready %s",
join_string_vec(insts, ",")));
local req = Management::Request::lookup(g_config_reqid_pending);
# If there's no pending request, when it's no longer available, or it
# doesn't have config state, don't do anything else.
if ( Management::Request::is_null(req) || ! req?$set_configuration_state )
return;
# All instances requested in the pending configuration update are now
# known to us. Send them the config. As they send their response events
# we update the client's request state and eventually send the response
# event to the it.
send_config_to_agents(req, req$set_configuration_state$config);
}
event Management::Agent::API::notify_agent_hello(instance: string, host: addr, api_version: count)
{
Management::Log::info(fmt("rx Management::Agent::API::notify_agent_hello %s %s", instance, host));
# When an agent checks in with a mismatching API version, we log the
# fact and drop its state, if any.
if ( api_version != Management::Controller::API::version )
{
Management::Log::warning(
fmt("instance %s/%s has checked in with incompatible API version %s",
instance, host, api_version));
if ( instance in g_instances )
drop_instance(g_instances[instance]);
if ( instance in g_instances_known )
delete g_instances_known[instance];
return;
}
add g_instances_known[instance];
if ( instance in g_instances && instance !in g_instances_ready )
{
# We need this instance for our cluster and have full context for
# it from the configuration. Tell agent.
local req = Management::Request::create();
Management::Log::info(fmt("tx Management::Agent::API::agent_welcome_request to %s", instance));
Broker::publish(Management::Agent::topic_prefix + "/" + instance,
Management::Agent::API::agent_welcome_request, req$id);
}
}
event Management::Agent::API::agent_welcome_response(reqid: string, result: Management::Result)
{
Management::Log::info(fmt("rx Management::Agent::API::agent_welcome_response %s", reqid));
local req = Management::Request::lookup(reqid);
if ( Management::Request::is_null(req) )
return;
Management::Request::finish(req$id);
# An agent we've been waiting to hear back from is ready for cluster
# work. Double-check we still want it, otherwise drop it.
if ( ! result$success || result$instance !in g_instances )
{
Management::Log::info(fmt(
"tx Management::Agent::API::agent_standby_request to %s", result$instance));
Broker::publish(Management::Agent::topic_prefix + "/" + result$instance,
Management::Agent::API::agent_standby_request, "");
return;
}
add g_instances_ready[result$instance];
Management::Log::info(fmt("instance %s ready", result$instance));
check_instances_ready();
}
event Management::Agent::API::notify_change(instance: string, n: Management::Node,
old: Management::State, new: Management::State)
{
# XXX TODO
}
event Management::Agent::API::notify_error(instance: string, msg: string, node: string)
{
# XXX TODO
}
event Management::Agent::API::notify_log(instance: string, msg: string, node: string)
{
# XXX TODO
}
event Management::Agent::API::set_configuration_response(reqid: string, result: Management::Result)
{
Management::Log::info(fmt("rx Management::Agent::API::set_configuration_response %s", reqid));
# Retrieve state for the request we just got a response to
local areq = Management::Request::lookup(reqid);
if ( Management::Request::is_null(areq) )
return;
# Release the request, which is now done.
Management::Request::finish(areq$id);
# Find the original request from the client
local req = Management::Request::lookup(areq$parent_id);
if ( Management::Request::is_null(req) )
return;
# Add this result to the overall response
req$results[|req$results|] = result;
# Mark this request as done by removing it from the table of pending
# ones. The following if-check should always be true.
if ( areq$id in req$set_configuration_state$requests )
delete req$set_configuration_state$requests[areq$id];
# If there are any pending requests to the agents, we're
# done: we respond once every agent has responed (or we time out).
if ( |req$set_configuration_state$requests| > 0 )
return;
# All set_configuration requests to instances are done, so adopt the
# client's requested configuration as the new one and respond back to
# client.
g_config_current = req$set_configuration_state$config;
g_config_reqid_pending = "";
Management::Log::info(fmt("tx Management::Controller::API::set_configuration_response %s",
Management::Request::to_string(req)));
Broker::publish(Management::Controller::topic,
Management::Controller::API::set_configuration_response, req$id, req$results);
Management::Request::finish(req$id);
}
event Management::Controller::API::set_configuration_request(reqid: string, config: Management::Configuration)
{
Management::Log::info(fmt("rx Management::Controller::API::set_configuration_request %s", reqid));
local res: Management::Result;
local req = Management::Request::create(reqid);
req$set_configuration_state = SetConfigurationState($config = config);
# At the moment there can only be one pending request.
if ( g_config_reqid_pending != "" )
{
res = Management::Result($reqid=reqid);
res$success = F;
res$error = fmt("request %s still pending", g_config_reqid_pending);
req$results += res;
Management::Log::info(fmt("tx Management::Controller::API::set_configuration_response %s",
Management::Request::to_string(req)));
Broker::publish(Management::Controller::topic,
Management::Controller::API::set_configuration_response, req$id, req$results);
Management::Request::finish(req$id);
return;
}
# XXX validate the configuration:
# - Are node instances among defined instances?
# - Are all names unique?
# - Are any node options understood?
# - Do node types with optional fields have required values?
# ...
# The incoming request is now the pending one. It gets cleared when all
# agents have processed their config updates successfully, or their
# responses time out.
g_config_reqid_pending = req$id;
# Compare the instance configuration to our current one. If it matches,
# we can proceed to deploying the new cluster topology. If it does
# not, we need to establish connectivity with agents we connect to, or
# wait until all instances that connect to us have done so. Either triggers
# a notify_agents_ready event, upon which we then deploy the topology.
# The current & new set of instance names.
local insts_current: set[string];
local insts_new: set[string];
# A set of current instances not contained in the new config.
# Those will need to get dropped.
local insts_to_drop: set[string];
# The opposite: new instances not yet in our current set. Those we will need
# to establish contact with (or they with us).
local insts_to_add: set[string];
# The overlap: instances in both the current and new set. For those we verify
# that we're actually dealign with the same entities, and might need to re-
# connect if not.
local insts_to_keep: set[string];
# Alternative representation of insts_to_add, directly providing the instances.
local insts_to_peer: table[string] of Management::Instance;
# Helpful locals.
local inst_name: string;
local inst: Management::Instance;
for ( inst_name in g_instances )
add insts_current[inst_name];
for ( inst in config$instances )
add insts_new[inst$name];
# Populate TODO lists for instances we need to drop, check, or add.
insts_to_drop = insts_current - insts_new;
insts_to_add = insts_new - insts_current;
insts_to_keep = insts_new & insts_current;
for ( inst in config$instances )
{
if ( inst$name in insts_to_add )
{
insts_to_peer[inst$name] = inst;
next;
}
# Focus on the keepers: check for change in identity/location.
if ( inst$name !in insts_to_keep )
next;
if ( is_instance_connectivity_change(inst) )
{
# The endpoint looks different. We drop the current one
# and need to re-establish connectivity with the new
# one.
add insts_to_drop[inst$name];
add insts_to_add[inst$name];
}
}
# Process our TODO lists. Handle drops first, then additions, in
# case we need to re-establish connectivity with an agent.
for ( inst_name in insts_to_drop )
drop_instance(g_instances[inst_name]);
for ( inst_name in insts_to_peer )
add_instance(insts_to_peer[inst_name]);
# Updates to out instance tables are complete, now check if we're already
# able to send the config to the agents:
check_instances_ready();
}
event Management::Controller::API::get_instances_request(reqid: string)
{
Management::Log::info(fmt("rx Management::Controller::API::set_instances_request %s", reqid));
local res = Management::Result($reqid = reqid);
local insts: vector of Management::Instance;
for ( i in g_instances )
insts += g_instances[i];
res$data = insts;
Management::Log::info(fmt("tx Management::Controller::API::get_instances_response %s", reqid));
Broker::publish(Management::Controller::topic,
Management::Controller::API::get_instances_response, reqid, res);
}
event Management::Agent::API::get_nodes_response(reqid: string, result: Management::Result)
{
Management::Log::info(fmt("rx Management::Agent::API::get_nodes_response %s", reqid));
# Retrieve state for the request we just got a response to
local areq = Management::Request::lookup(reqid);
if ( Management::Request::is_null(areq) )
return;
# Release the request, since this agent is now done.
Management::Request::finish(areq$id);
# Find the original request from the client
local req = Management::Request::lookup(areq$parent_id);
if ( Management::Request::is_null(req) )
return;
# Zeek's ingestion of an any-typed val via Broker yields an opaque
# Broker DataVal. When Zeek forwards this val via another event it stays
# in this opaque form. To avoid forcing recipients to distinguish
# whether the val is of the actual, intended (any-)type or a Broker
# DataVal wrapper, we explicitly cast it back to our intended Zeek
# type. This test case demonstrates: broker.remote_event_vector_any
result$data = result$data as Management::NodeStatusVec;
# Add this result to the overall response
req$results[|req$results|] = result;
# Mark this request as done by removing it from the table of pending
# ones. The following if-check should always be true.
if ( areq$id in req$get_nodes_state$requests )
delete req$get_nodes_state$requests[areq$id];
# If we still have pending queries out to the agents, do nothing: we'll
# handle this soon, or our request will time out and we respond with
# error.
if ( |req$get_nodes_state$requests| > 0 )
return;
Management::Log::info(fmt("tx Management::Controller::API::get_nodes_response %s",
Management::Request::to_string(req)));
Broker::publish(Management::Controller::topic,
Management::Controller::API::get_nodes_response, req$id, req$results);
Management::Request::finish(req$id);
}
event Management::Controller::API::get_nodes_request(reqid: string)
{
Management::Log::info(fmt("rx Management::Controller::API::get_nodes_request %s", reqid));
# Special case: if we have no instances, respond right away.
if ( |g_instances| == 0 )
{
Management::Log::info(fmt("tx Management::Controller::API::get_nodes_response %s", reqid));
local res = Management::Result($reqid=reqid, $success=F,
$error="no instances connected");
Broker::publish(Management::Controller::topic,
Management::Controller::API::get_nodes_response, reqid, vector(res));
return;
}
local req = Management::Request::create(reqid);
req$get_nodes_state = GetNodesState();
for ( name in g_instances )
{
if ( name !in g_instances_ready )
next;
local agent_topic = Management::Agent::topic_prefix + "/" + name;
local areq = Management::Request::create();
areq$parent_id = req$id;
add req$get_nodes_state$requests[areq$id];
Management::Log::info(fmt("tx Management::Agent::API::get_nodes_request %s to %s", areq$id, name));
Broker::publish(agent_topic, Management::Agent::API::get_nodes_request, areq$id);
}
}
event Management::Agent::API::node_dispatch_response(reqid: string, results: Management::ResultVec)
{
Management::Log::info(fmt("rx Management::Agent::API::node_dispatch_response %s", reqid));
# Retrieve state for the request we just got a response to
local areq = Management::Request::lookup(reqid);
if ( Management::Request::is_null(areq) )
return;
# Release the request, since this agent is now done.
Management::Request::finish(areq$id);
# Find the original request from the client
local req = Management::Request::lookup(areq$parent_id);
if ( Management::Request::is_null(req) )
return;
# Add this agent's results to the overall response
for ( i in results )
{
# Same special treatment for Broker values that are of
# type "any": confirm their (known) type here.
switch req$node_dispatch_state$action[0]
{
case "get_id_value":
if ( results[i]?$data )
results[i]$data = results[i]$data as string;
break;
default:
Management::Log::error(fmt("unexpected dispatch command %s",
req$node_dispatch_state$action[0]));
break;
}
req$results[|req$results|] = results[i];
}
# Mark this request as done
if ( areq$id in req$node_dispatch_state$requests )
delete req$node_dispatch_state$requests[areq$id];
# If we still have pending queries out to the agents, do nothing: we'll
# handle this soon, or our request will time out and we respond with
# error.
if ( |req$node_dispatch_state$requests| > 0 )
return;
# Send response event to the client based upon the dispatch type.
switch req$node_dispatch_state$action[0]
{
case "get_id_value":
Management::Log::info(fmt(
"tx Management::Controller::API::get_id_value_response %s",
Management::Request::to_string(req)));
Broker::publish(Management::Controller::topic,
Management::Controller::API::get_id_value_response,
req$id, req$results);
break;
default:
Management::Log::error(fmt("unexpected dispatch command %s",
req$node_dispatch_state$action[0]));
break;
}
Management::Request::finish(req$id);
}
event Management::Controller::API::get_id_value_request(reqid: string, id: string, nodes: set[string])
{
Management::Log::info(fmt("rx Management::Controller::API::get_id_value_request %s %s", reqid, id));
local res: Management::Result;
# Special case: if we have no instances, respond right away.
if ( |g_instances| == 0 )
{
Management::Log::info(fmt("tx Management::Controller::API::get_id_value_response %s", reqid));
res = Management::Result($reqid=reqid, $success=F, $error="no instances connected");
Broker::publish(Management::Controller::topic,
Management::Controller::API::get_id_value_response,
reqid, vector(res));
return;
}
local action = vector("get_id_value", id);
local req = Management::Request::create(reqid);
req$node_dispatch_state = NodeDispatchState($action=action);
local nodes_final: set[string];
local node: string;
# Input sanitization: check for any requested nodes that aren't part of
# the current configuration. We send back error results for those and
# don't propagate them to the agents.
if ( |nodes| > 0 )
{
# Requested nodes that are in the current configuration:
nodes_final = filter_config_nodes_by_name(nodes);
# Requested nodes that are not in current configuration:
local nodes_invalid = nodes - nodes_final;
# Assemble error results for all invalid nodes
for ( node in nodes_invalid )
{
res = Management::Result($reqid=reqid, $node=node);
res$success = F;
res$error = "unknown cluster node";
req$results += res;
}
# If only invalid nodes got requested, we're now done.
if ( |nodes_final| == 0 )
{
Management::Log::info(fmt(
"tx Management::Controller::API::get_id_value_response %s",
Management::Request::to_string(req)));
Broker::publish(Management::Controller::topic,
Management::Controller::API::get_id_value_response,
req$id, req$results);
Management::Request::finish(req$id);
return;
}
}
# Send dispatch requests to all agents, with the final set of nodes
for ( name in g_instances )
{
if ( name !in g_instances_ready )
next;
local agent_topic = Management::Agent::topic_prefix + "/" + name;
local areq = Management::Request::create();
areq$parent_id = req$id;
add req$node_dispatch_state$requests[areq$id];
Management::Log::info(fmt(
"tx Management::Agent::API::node_dispatch_request %s %s to %s",
areq$id, action, name));
Broker::publish(agent_topic,
Management::Agent::API::node_dispatch_request,
areq$id, action, nodes_final);
}
}
event Management::Request::request_expired(req: Management::Request::Request)
{
# Various handlers for timed-out request state. We use the state members
# to identify how to respond. No need to clean up the request itself,
# since we're getting here via the request module's expiration
# mechanism that handles the cleanup.
local res = Management::Result($reqid=req$id,
$success = F,
$error = "request timed out");
if ( req?$set_configuration_state )
{
# This timeout means we no longer have a pending request.
g_config_reqid_pending = "";
req$results += res;
Management::Log::info(fmt("tx Management::Controller::API::set_configuration_response %s",
Management::Request::to_string(req)));
Broker::publish(Management::Controller::topic,
Management::Controller::API::set_configuration_response, req$id, req$results);
}
if ( req?$get_nodes_state )
{
req$results += res;
Management::Log::info(fmt("tx Management::Controller::API::get_nodes_response %s",
Management::Request::to_string(req)));
Broker::publish(Management::Controller::topic,
Management::Controller::API::get_nodes_response, req$id, req$results);
}
if ( req?$node_dispatch_state )
{
req$results += res;
switch req$node_dispatch_state$action[0]
{
case "get_id_value":
Management::Log::info(fmt(
"tx Management::Controller::API::get_id_value_response %s",
Management::Request::to_string(req)));
Broker::publish(Management::Controller::topic,
Management::Controller::API::get_id_value_response,
req$id, req$results);
break;
default:
Management::Log::error(fmt("unexpected dispatch command %s",
req$node_dispatch_state$action[0]));
break;
}
}
if ( req?$test_state )
{
Management::Log::info(fmt("tx Management::Controller::API::test_timeout_response %s", req$id));
Broker::publish(Management::Controller::topic,
Management::Controller::API::test_timeout_response, req$id, res);
}
}
event Management::Controller::API::test_timeout_request(reqid: string, with_state: bool)
{
Management::Log::info(fmt("rx Management::Controller::API::test_timeout_request %s %s", reqid, with_state));
if ( with_state )
{
# This state times out and triggers a timeout response in the
# above request_expired event handler.
local req = Management::Request::create(reqid);
req$test_state = TestState();
}
}
event zeek_init()
{
# Initialize null config at startup. We will replace it once we have
# persistence, and again whenever we complete a client's
# set_configuration request.
g_config_current = null_config();
# The controller always listens -- it needs to be able to respond to the
# Zeek client. This port is also used by the agents if they connect to
# the client. The client doesn't automatically establish or accept
# connectivity to agents: agents are defined and communicated with as
# defined via configurations defined by the client.
local cni = Management::Controller::network_info();
Broker::listen(cat(cni$address), cni$bound_port);
Broker::subscribe(Management::Agent::topic_prefix);
Broker::subscribe(Management::Controller::topic);
Management::Log::info("controller is live");
}

48
scripts/policy/frameworks/cluster/controller/log.zeek → scripts/policy/frameworks/management/log.zeek
View file

@ -1,11 +1,11 @@
##! This module implements straightforward logging abilities for cluster
##! controller and agent. It uses Zeek's logging framework, and works only for
##! nodes managed by the supervisor. In this setting Zeek's logging framework
##! operates locally, i.e., this logging does not involve any logger nodes.
##! This module implements logging abilities for controller and agent. It uses
##! Zeek's logging framework and works only for nodes managed by the
##! supervisor. In this setting Zeek's logging framework operates locally, i.e.,
##! this does not involve logger nodes.
@load ./config
@load ./types
module ClusterController::Log;
module Management::Log;
export {
## The cluster logging stream identifier.
@ -16,10 +16,10 @@ export {
## The controller/agent log supports four different log levels.
type Level: enum {
DEBUG,
INFO,
WARNING,
ERROR,
DEBUG = 10,
INFO = 20,
WARNING = 30,
ERROR = 40,
};
## The record type containing the column fields of the agent/controller log.
@ -30,14 +30,15 @@ export {
node: string;
## Log level of this message, converted from the above Level enum
level: string;
## The role of the node, translated from ClusterController::Types::Role.
## The role of the node, translated from Management::Role.
role: string;
## A message indicating information about cluster controller operation.
message: string;
} &log;
## The log level in use for this node.
global log_level = DEBUG &redef;
## The log level in use for this node. This is the minimum
## log level required to produce output.
global log_level = INFO &redef;
## A debug-level log message writer.
##
@ -63,6 +64,10 @@ export {
## message: the message to log.
##
global error: function(message: string);
## The role of this process in cluster management. Agent and controller
## both redefine this, and we use it during logging.
const role = Management::NONE &redef;
}
# Enum translations to strings. This avoids those enums being reported
@ -75,9 +80,10 @@ global l2s: table[Level] of string = {
[ERROR] = "ERROR",
};
global r2s: table[ClusterController::Types::Role] of string = {
[ClusterController::Types::AGENT] = "AGENT",
[ClusterController::Types::CONTROLLER] = "CONTROLLER",
global r2s: table[Management::Role] of string = {
[Management::AGENT] = "AGENT",
[Management::CONTROLLER] = "CONTROLLER",
[Management::NODE] = "NODE",
};
function debug(message: string)
@ -87,7 +93,7 @@ function debug(message: string)
local node = Supervisor::node();
Log::write(LOG, [$ts=network_time(), $node=node$name, $level=l2s[DEBUG],
$role=r2s[ClusterController::role], $message=message]);
$role=r2s[role], $message=message]);
}
function info(message: string)
@ -97,7 +103,7 @@ function info(message: string)
local node = Supervisor::node();
Log::write(LOG, [$ts=network_time(), $node=node$name, $level=l2s[INFO],
$role=r2s[ClusterController::role], $message=message]);
$role=r2s[role], $message=message]);
}
function warning(message: string)
@ -107,7 +113,7 @@ function warning(message: string)
local node = Supervisor::node();
Log::write(LOG, [$ts=network_time(), $node=node$name, $level=l2s[WARNING],
$role=r2s[ClusterController::role], $message=message]);
$role=r2s[role], $message=message]);
}
function error(message: string)
@ -117,7 +123,7 @@ function error(message: string)
local node = Supervisor::node();
Log::write(LOG, [$ts=network_time(), $node=node$name, $level=l2s[ERROR],
$role=r2s[ClusterController::role], $message=message]);
$role=r2s[role], $message=message]);
}
event zeek_init()
@ -133,5 +139,5 @@ event zeek_init()
local stream = Log::Stream($columns=Info, $path=fmt("cluster-%s", node$name),
$policy=log_policy);
Log::create_stream(ClusterController::Log::LOG, stream);
Log::create_stream(Management::Log::LOG, stream);
}

1
scripts/policy/frameworks/management/node/__load__.zeek Normal file
View file

@ -0,0 +1 @@
@load ./main

48
scripts/policy/frameworks/management/node/api.zeek Normal file
View file

@ -0,0 +1,48 @@
##! The Management event API of cluster nodes. The API consists of request/
##! response event pairs, like elsewhere in the Management, Supervisor, and
##! Control frameworks.
@load policy/frameworks/management/types
module Management::Node::API;
export {
## Management agents send this event to every Zeek cluster node to run a
## "dispatch" -- a particular, pre-implemented action. This is the agent-node
## complement to :zeek:see:`Management::Agent::API::node_dispatch_request`.
##
## reqid: a request identifier string, echoed in the response event.
##
## action: the requested dispatch command, with any arguments.
##
## nodes: the cluster node names this dispatch targets. An empty set,
## supplied by default, means it applies to all nodes. Since nodes
## receive all dispatch requests, they can use any node names provided
## here to filter themselves out of responding.
global node_dispatch_request: event(reqid: string, action: vector of string,
nodes: set[string] &default=set());
## Response to a node_dispatch_request event. The nodes send this back
## to the agent. This is the agent-node equivalent of
## :zeek:see:`Management::Agent::API::node_dispatch_response`.
##
## reqid: the request identifier used in the request event.
##
## result: a :zeek:see:`Management::Result` record covering one Zeek
## cluster node managed by the agent. Upon success, the data field
## contains a value appropriate for the requested dispatch.
global node_dispatch_response: event(reqid: string, result: Management::Result);
# Notification events, node -> agent
## The cluster nodes send this event upon peering as a "check-in" to
## the agent, to indicate the node is now available to communicate
## with. It is an agent-level equivalent of :zeek:see:`Broker::peer_added`,
## and similar to :zeek:see:`Management::Agent::API::notify_agent_hello`
## for agents.
##
## node: the name of the node, as given in :zeek:see:`Cluster::node`.
##
global notify_node_hello: event(node: string);
}

9
scripts/policy/frameworks/management/node/config.zeek Normal file
View file

@ -0,0 +1,9 @@
##! Configuration settings for nodes controlled by the Management framework.
module Management::Node;
export {
## The nodes' Broker topic. Cluster nodes automatically subscribe
## to it, to receive request events from the Management framework.
const node_topic = "zeek/management/node" &redef;
}

110
scripts/policy/frameworks/management/node/main.zeek Normal file
View file

@ -0,0 +1,110 @@
##! This module provides Management framework functionality present in every
##! cluster node, to allowing Management agents to interact with the nodes.
@load base/frameworks/cluster
@load policy/frameworks/management/agent/config
@load policy/frameworks/management/log
@load ./api
@load ./config
module Management::Node;
# Tag our logs correctly
redef Management::Log::role = Management::NODE;
## The type of dispatch callbacks. These implement a particular dispatch action,
## using the provided string vector as arguments, filling results into the
## provided result record.
type DispatchCallback: function(args: vector of string, res: Management::Result);
## Implementation of the "get_id_value" dispatch. Its only argument is the name
## of the ID to look up.
function dispatch_get_id_value(args: vector of string, res: Management::Result)
{
if ( |args| == 0 )
{
res$success = F;
res$error = "get_id_value expects name of global identifier";
return;
}
local val = lookup_ID(args[0]);
# The following lookup_ID() result strings indicate errors:
if ( type_name(val) == "string" )
{
local valstr: string = val;
if ( valstr == "<unknown id>" || valstr == "<no ID value>" )
{
res$success = F;
res$error = valstr[1:-1];
}
}
if ( res$success )
res$data = to_json(val);
}
global g_dispatch_table: table[string] of DispatchCallback = {
["get_id_value"] = dispatch_get_id_value,
};
event Management::Node::API::node_dispatch_request(reqid: string, action: vector of string, nodes: set[string])
{
Management::Log::info(fmt("rx Management::Node::API::node_dispatch_request %s %s %s", reqid, action, nodes));
if ( |nodes| > 0 && Cluster::node !in nodes )
{
Management::Log::debug(fmt(
"dispatch %s not targeting this node (%s !in %s), skipping",
reqid, Cluster::node, nodes));
return;
}
local res = Management::Result($reqid = reqid, $node = Cluster::node);
if ( |action| == 0 )
{
res$success = F;
res$error = "no dispatch arguments provided";
}
else if ( action[0] !in g_dispatch_table )
{
res$success = F;
res$error = fmt("dispatch %s unknown", action[0]);
}
if ( ! res$success )
{
Management::Log::info(fmt("tx Management::Node::API::node_dispatch_response %s",
Management::result_to_string(res)));
Broker::publish(node_topic, Management::Node::API::node_dispatch_response, reqid, res);
return;
}
g_dispatch_table[action[0]](action[1:], res);
Management::Log::info(fmt("tx Management::Node::API::node_dispatch_response %s",
Management::result_to_string(res)));
Broker::publish(node_topic, Management::Node::API::node_dispatch_response, reqid, res);
}
event Broker::peer_added(peer: Broker::EndpointInfo, msg: string)
{
local epi = Management::Agent::endpoint_info();
# If this is the agent peering, notify it that we're ready
if ( peer$network$address == epi$network$address &&
peer$network$bound_port == epi$network$bound_port )
Broker::publish(node_topic, Management::Node::API::notify_node_hello, Cluster::node);
}
event zeek_init()
{
local epi = Management::Agent::endpoint_info();
Broker::peer(epi$network$address, epi$network$bound_port, Management::connect_retry);
Broker::subscribe(node_topic);
}

79
scripts/policy/frameworks/cluster/controller/request.zeek → scripts/policy/frameworks/management/request.zeek
View file

@ -1,14 +1,19 @@
##! This module implements a request state abstraction that both cluster
##! controller and agent use to tie responses to received request events and be
##! able to time-out such requests.
##! This module implements a request state abstraction in the Management
##! framework that both controller and agent use to connect request events to
##! subsequent response ones, and to be able to time out such requests.
@load ./types
@load ./config
@load ./types
module ClusterController::Request;
module Management::Request;
export {
## Request records track each request's state.
## Request records track state associated with a request/response event
## pair. Calls to
## :zeek:see:`Management::Request::create` establish such state
## when an entity sends off a request event, while
## :zeek:see:`Management::Request::finish` clears the state when
## a corresponding response event comes in, or the state times out.
type Request: record {
## Each request has a hopfully unique ID provided by the requester.
id: string;
@ -18,39 +23,22 @@ export {
## received by the client), this specifies that original, "parent"
## request.
parent_id: string &optional;
};
# API-specific state. XXX we may be able to generalize after this has
# settled a bit more. It would also be nice to move request-specific
# state out of this module -- we could for example redef Request in
# main.zeek as needed.
## The results vector builds up the list of results we eventually
## send to the requestor when we have processed the request.
results: Management::ResultVec &default=vector();
# State specific to the set_configuration request/response events
type SetConfigurationState: record {
config: ClusterController::Types::Configuration;
requests: vector of Request &default=vector();
};
# State specific to supervisor interactions
type SupervisorState: record {
node: string;
};
# State for testing events
type TestState: record {
};
# The redef is a workaround so we can use the Request type
# while it is still being defined.
redef record Request += {
results: ClusterController::Types::ResultVec &default=vector();
## An internal flag to track whether a request is complete.
finished: bool &default=F;
set_configuration_state: SetConfigurationState &optional;
supervisor_state: SupervisorState &optional;
test_state: TestState &optional;
};
## The timeout for request state. Such state (see the :zeek:see:`Management::Request`
## module) ties together request and response event pairs. The timeout causes
## its cleanup in the absence of a timely response. It applies both to
## state kept for client requests, as well as state in the agents for
## requests to the supervisor.
const timeout_interval = 10sec &redef;
## A token request that serves as a null/nonexistant request.
global null_req = Request($id="", $finished=T);
@ -61,7 +49,7 @@ export {
global create: function(reqid: string &default=unique_id("")): Request;
## This function looks up the request for a given request ID and returns
## it. When no such request exists, returns ClusterController::Request::null_req.
## it. When no such request exists, returns Management::Request::null_req.
##
## reqid: the ID of the request state to retrieve.
##
@ -76,8 +64,8 @@ export {
global finish: function(reqid: string): bool;
## This event fires when a request times out (as per the
## ClusterController::request_timeout) before it has been finished via
## ClusterController::Request::finish().
## Management::Request::timeout_interval) before it has been finished via
## Management::Request::finish().
##
## req: the request state that is expiring.
##
@ -101,17 +89,20 @@ export {
function requests_expire_func(reqs: table[string] of Request, reqid: string): interval
{
event ClusterController::Request::request_expired(reqs[reqid]);
# No need to flag request expiration when we've already internally marked
# the request as done.
if ( ! reqs[reqid]$finished )
event Management::Request::request_expired(reqs[reqid]);
return 0secs;
}
# This is the global request-tracking table. The table maps from request ID
# strings to corresponding Request records. Entries time out after the
# ClusterController::request_timeout interval. Upon expiration, a
# request_expired event triggers that conveys the request state.
# Management::Request::timeout_interval. Upon expiration, a request_expired
# event triggers that conveys the request state.
global g_requests: table[string] of Request
&create_expire=ClusterController::request_timeout
&expire_func=requests_expire_func;
&create_expire=timeout_interval &expire_func=requests_expire_func;
function create(reqid: string): Request
{
@ -152,7 +143,7 @@ function is_null(request: Request): bool
function to_string(request: Request): string
{
local results: string_vec;
local res: ClusterController::Types::Result;
local res: Management::Result;
local parent_id = "";
if ( request?$parent_id )
@ -161,7 +152,7 @@ function to_string(request: Request): string
for ( idx in request$results )
{
res = request$results[idx];
results[|results|] = ClusterController::Types::result_to_string(res);
results[|results|] = Management::result_to_string(res);
}
return fmt("[request %s%s %s, results: %s]", request$id, parent_id,

57
scripts/policy/frameworks/cluster/controller/types.zeek → scripts/policy/frameworks/management/types.zeek
View file

@ -1,17 +1,18 @@
##! This module holds the basic types needed for the Cluster Controller
##! framework. These are used by both agent and controller, and several
##! have corresponding equals in the zeek-client implementation.
##! This module holds the basic types needed for the Management framework. These
##! are used by both cluster agent and controller, and several have corresponding
##! implementations in zeek-client.
module ClusterController::Types;
module Management;
export {
## Management infrastructure node type. This intentionally does not
## include the data cluster node types (worker, logger, etc) -- those
## include the managed cluster node types (worker, logger, etc) -- those
## continue to be managed by the cluster framework.
type Role: enum {
NONE,
AGENT,
CONTROLLER,
NONE, ##< No active role in cluster management
AGENT, ##< A cluster management agent.
CONTROLLER, ##< The cluster's controller.
NODE, ##< A managed cluster node (worker, manager, etc).
};
## A Zeek-side option with value.
@ -35,22 +36,25 @@ export {
type InstanceVec: vector of Instance;
## State that a Cluster Node can be in. State changes trigger an
## API notification (see notify_change()).
## API notification (see notify_change()). The Pending state corresponds
## to the Supervisor not yet reporting a PID for a node when it has not
## yet fully launched.
type State: enum {
Running, ##< Running and operating normally
Stopped, ##< Explicitly stopped
Failed, ##< Failed to start; and permanently halted
Crashed, ##< Crashed, will be restarted,
Unknown, ##< State not known currently (e.g., because of lost connectivity)
PENDING, ##< Not yet running
RUNNING, ##< Running and operating normally
STOPPED, ##< Explicitly stopped
FAILED, ##< Failed to start; and permanently halted
CRASHED, ##< Crashed, will be restarted,
UNKNOWN, ##< State not known currently (e.g., because of lost connectivity)
};
## Configuration describing a Cluster Node process.
type Node: record {
name: string; ##< Cluster-unique, human-readable node name
instance: string; ##< Name of instance where node is to run
p: port; ##< Port on which this node will listen
role: Supervisor::ClusterRole; ##< Role of the node.
state: State; ##< Desired, or current, run state.
p: port &optional; ##< Port on which this node will listen
scripts: vector of string &optional; ##< Additional Zeek scripts for node
options: set[Option] &optional; ##< Zeek options for node
interface: string &optional; ##< Interface to sniff
@ -61,7 +65,6 @@ export {
## Data structure capturing a cluster's complete configuration.
type Configuration: record {
id: string &default=unique_id(""); ##< Unique identifier for a particular configuration
## The instances in the cluster.
instances: set[Instance] &default=set();
@ -69,6 +72,26 @@ export {
nodes: set[Node] &default=set();
};
## The status of a Supervisor-managed node, as reported to the client in
## a get_nodes_request/get_nodes_response transaction.
type NodeStatus: record {
## Cluster-unique, human-readable node name
node: string;
## Current run state of the node.
state: State;
## Role the node plays in cluster management.
mgmt_role: Role &default=NONE;
## Role the node plays in the data cluster.
cluster_role: Supervisor::ClusterRole &default=Supervisor::NONE;
## Process ID of the node. This is optional because the Supervisor may not have
## a PID when a node is still bootstrapping.
pid: int &optional;
## The node's Broker peering listening port, if any.
p: port &optional;
};
type NodeStatusVec: vector of NodeStatus;
## Return value for request-response API event pairs
type Result: record {
reqid: string; ##< Request ID of operation this result refers to
@ -81,6 +104,8 @@ export {
type ResultVec: vector of Result;
## Given a :zeek:see:`Management::Result` record,
## this function returns a string summarizing it.
global result_to_string: function(res: Result): string;
}

4
scripts/policy/frameworks/cluster/controller/util.zeek → scripts/policy/frameworks/management/util.zeek
View file

@ -1,7 +1,7 @@
##! Utility functions for the cluster controller framework, available to agent
##! Utility functions for the Management framework, available to agent
##! and controller.
module ClusterController::Util;
module Management::Util;
export {
## Renders a set of strings to an alphabetically sorted vector.

111
scripts/policy/protocols/ssl/decryption.zeek Normal file
View file

@ -0,0 +1,111 @@
##! This script allows for the decryption of certain TLS 1.2 connections, if the user is in possession
##! of the private key material for the session. Key material can either be provided via a file (useful
##! for processing trace files) or via sending events via Broker (for live decoding).
##!
##! Please note that this feature is experimental and can change without guarantees to our typical
##! deprecation timeline. Please also note that currently only TLS 1.2 connections that use the
##! TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 cipher suite are supported.
@load base/frameworks/input
@load base/frameworks/notice
@load base/protocols/conn
@load base/protocols/ssl
module SSL;
# Do not disable analyzers after detection - otherwise we will not receive
# encrypted packets.
redef SSL::disable_analyzer_after_detection = F;
export {
## This can be set to a file that contains the session secrets for decryption, when parsing a pcap file.
## Please note that, when using this feature, you probably want to pause processing of data till this
## file has been read.
const keylog_file = getenv("ZEEK_TLS_KEYLOG_FILE") &redef;
## Secrets expire after this time of not being used.
const secret_expiration = 5 mins &redef;
## This event can be triggered, e.g., via Broker to add known keys to the TLS key database.
##
## client_random: client random for which the key is set
##
## keys: key material
global add_keys: event(client_random: string, keys: string);
## This event can be triggered, e.g., via Broker to add known secrets to the TLS secret datbase.
##
## client_random: client random for which the secret is set
##
## secrets: derived TLS secrets material
global add_secret: event(client_random: string, secrets: string);
}
@if ( keylog_file == "" )
# If a keylog file was given via an environment variable, let's disable secret expiration - that does not
# make sense for pcaps.
global secrets: table[string] of string = {} &redef;
global keys: table[string] of string = {} &redef;
@else
global secrets: table[string] of string = {} &read_expire=secret_expiration &redef;
global keys: table[string] of string = {} &read_expire=secret_expiration &redef;
@endif
redef record SSL::Info += {
# Decryption uses client_random as identifier
client_random: string &optional;
};
type SecretsIdx: record {
client_random: string;
};
type SecretsVal: record {
secret: string;
};
const tls_decrypt_stream_name = "tls-keylog-file";
event zeek_init()
{
# listen for secrets
Broker::subscribe("/zeek/tls/decryption");
if ( keylog_file != "" )
{
Input::add_table([$name=tls_decrypt_stream_name, $source=keylog_file, $destination=secrets, $idx=SecretsIdx, $val=SecretsVal, $want_record=F]);
Input::remove(tls_decrypt_stream_name);
}
}
event SSL::add_keys(client_random: string, val: string)
{
SSL::keys[client_random] = val;
}
event SSL::add_secret(client_random: string, val: string)
{
SSL::secrets[client_random] = val;
}
event ssl_client_hello(c: connection, version: count, record_version: count, possible_ts: time, client_random: string, session_id: string, ciphers: index_vec, comp_methods: index_vec)
{
c$ssl$client_random = client_random;
if ( client_random in keys )
set_keys(c, keys[client_random]);
else if ( client_random in secrets )
set_secret(c, secrets[client_random]);
}
event ssl_change_cipher_spec(c: connection, is_orig: bool)
{
if ( c$ssl?$client_random )
{
if ( c$ssl$client_random in keys )
set_keys(c, keys[c$ssl$client_random]);
else if ( c$ssl$client_random in secrets )
set_secret(c, secrets[c$ssl$client_random]);
}
}

17
scripts/policy/protocols/ssl/dpd-v2.sig Normal file
View file

@ -0,0 +1,17 @@
# This signature can be used to enable DPD for SSL version 2.
# Note that SSLv2 is basically unused by now. Due to the structure of the protocol, it also is sometimes
# hard to disambiguate it from random noise - so you will probably always get a few false positives.
signature dpd_ssl_server {
ip-proto == tcp
payload /^...?\x04..\x00\x02.*/
requires-reverse-signature dpd_ssl_client
tcp-state responder
enable "ssl"
}
signature dpd_ssl_client {
ip-proto == tcp
payload /^...?\x01[\x00\x03][\x00\x01\x02\x03\x04].*/
tcp-state originator
}

35
scripts/test-all-policy.zeek
View file

@ -11,20 +11,26 @@
# @load frameworks/control/controllee.zeek
# @load frameworks/control/controller.zeek
@load frameworks/cluster/agent/__load__.zeek
@load frameworks/cluster/agent/api.zeek
@load frameworks/cluster/agent/boot.zeek
@load frameworks/cluster/agent/config.zeek
# @load frameworks/cluster/agent/main.zeek
@load frameworks/cluster/controller/__load__.zeek
@load frameworks/cluster/controller/api.zeek
@load frameworks/cluster/controller/boot.zeek
@load frameworks/cluster/controller/config.zeek
@load frameworks/cluster/controller/log.zeek
# @load frameworks/cluster/controller/main.zeek
@load frameworks/cluster/controller/request.zeek
@load frameworks/cluster/controller/types.zeek
@load frameworks/cluster/controller/util.zeek
@load frameworks/management/agent/__load__.zeek
@load frameworks/management/agent/api.zeek
@load frameworks/management/agent/boot.zeek
@load frameworks/management/agent/config.zeek
# @load frameworks/management/agent/main.zeek
@load frameworks/management/controller/__load__.zeek
@load frameworks/management/controller/api.zeek
@load frameworks/management/controller/boot.zeek
@load frameworks/management/controller/config.zeek
# @load frameworks/management/controller/main.zeek
@load frameworks/management/__load__.zeek
@load frameworks/management/config.zeek
@load frameworks/management/log.zeek
# @load frameworks/management/node/__load__.zeek
@load frameworks/management/node/api.zeek
@load frameworks/management/node/config.zeek
# @load frameworks/management/node/main.zeek
@load frameworks/management/request.zeek
@load frameworks/management/types.zeek
@load frameworks/management/util.zeek
@load frameworks/dpd/detect-protocols.zeek
@load frameworks/dpd/packet-segment-logging.zeek
@load frameworks/intel/do_notice.zeek
@ -116,6 +122,7 @@
@load protocols/ssh/geo-data.zeek
@load protocols/ssh/interesting-hostnames.zeek
@load protocols/ssh/software.zeek
@load protocols/ssl/decryption.zeek
@load protocols/ssl/expiring-certs.zeek
# @load protocols/ssl/extract-certs-pem.zeek
@load protocols/ssl/heartbleed.zeek

7
scripts/zeekygen/__load__.zeek
View file

@ -1,11 +1,14 @@
@load test-all-policy.zeek
# Scripts which are commented out in test-all-policy.zeek.
@load protocols/ssl/decryption.zeek
@load protocols/ssl/notary.zeek
@load frameworks/control/controllee.zeek
@load frameworks/control/controller.zeek
@load frameworks/cluster/agent/main.zeek
@load frameworks/cluster/controller/main.zeek
@load frameworks/management/agent/main.zeek
@load frameworks/management/controller/main.zeek
@load frameworks/management/node/__load__.zeek
@load frameworks/management/node/main.zeek
@load frameworks/files/extract-all-files.zeek
@load policy/misc/dump-events.zeek
@load policy/protocols/conn/speculative-service.zeek

2
src/3rdparty

@ -1 +1 @@
Subproject commit cb626c94f67e0ac0437beba076da1184eb1f8ad7
Subproject commit 6cbb3d65877f80326c047364583f506ce58758ba

224
src/Attr.cc
View file

@ -192,6 +192,9 @@ void Attributes::AddAttr(AttrPtr attr, bool is_redef)
{
auto acceptable_duplicate_attr = [](const AttrPtr& attr, const AttrPtr& existing) -> bool
{
if ( attr == existing )
return true;
AttrTag new_tag = attr->Tag();
if ( new_tag == ATTR_DEPRECATED )
@ -341,117 +344,13 @@ void Attributes::CheckAttr(Attr* a)
case ATTR_DEFAULT:
{
// &default is allowed for global tables, since it's used in initialization
// of table fields. it's not allowed otherwise.
if ( global_var && ! type->IsTable() )
{
Error("&default is not valid for global variables except for tables");
std::string err_msg;
if ( ! check_default_attr(a, type, global_var, in_record, err_msg) &&
! err_msg.empty() )
Error(err_msg.c_str());
break;
}
const auto& atype = a->GetExpr()->GetType();
if ( type->Tag() != TYPE_TABLE || (type->IsSet() && ! in_record) )
{
if ( same_type(atype, type) )
// Ok.
break;
// Record defaults may be promotable.
if ( (type->Tag() == TYPE_RECORD && atype->Tag() == TYPE_RECORD &&
record_promotion_compatible(atype->AsRecordType(), type->AsRecordType())) )
// Ok.
break;
if ( type->Tag() == TYPE_TABLE && type->AsTableType()->IsUnspecifiedTable() )
// Ok.
break;
auto e = check_and_promote_expr(a->GetExpr(), type);
if ( e )
{
a->SetAttrExpr(std::move(e));
// Ok.
break;
}
a->GetExpr()->Error("&default value has inconsistent type", type.get());
return;
}
TableType* tt = type->AsTableType();
const auto& ytype = tt->Yield();
if ( ! in_record )
{
// &default applies to the type itself.
if ( ! same_type(atype, ytype) )
{
// It can still be a default function.
if ( atype->Tag() == TYPE_FUNC )
{
FuncType* f = atype->AsFuncType();
if ( ! f->CheckArgs(tt->GetIndexTypes()) || ! same_type(f->Yield(), ytype) )
Error("&default function type clash");
// Ok.
break;
}
// Table defaults may be promotable.
if ( (ytype->Tag() == TYPE_RECORD && atype->Tag() == TYPE_RECORD &&
record_promotion_compatible(atype->AsRecordType(),
ytype->AsRecordType())) )
// Ok.
break;
auto e = check_and_promote_expr(a->GetExpr(), ytype);
if ( e )
{
a->SetAttrExpr(std::move(e));
// Ok.
break;
}
Error("&default value has inconsistent type 2");
}
// Ok.
break;
}
else
{
// &default applies to record field.
if ( same_type(atype, type) )
// Ok.
break;
if ( (atype->Tag() == TYPE_TABLE && atype->AsTableType()->IsUnspecifiedTable()) )
{
auto e = check_and_promote_expr(a->GetExpr(), type);
if ( e )
{
a->SetAttrExpr(std::move(e));
break;
}
}
// Table defaults may be promotable.
if ( ytype && ytype->Tag() == TYPE_RECORD && atype->Tag() == TYPE_RECORD &&
record_promotion_compatible(atype->AsRecordType(), ytype->AsRecordType()) )
// Ok.
break;
Error("&default value has inconsistent type");
}
}
break;
case ATTR_EXPIRE_READ:
{
if ( Find(ATTR_BROKER_STORE) )
@ -748,4 +647,113 @@ bool Attributes::operator==(const Attributes& other) const
return true;
}
bool check_default_attr(Attr* a, const TypePtr& type, bool global_var, bool in_record,
std::string& err_msg)
{
// &default is allowed for global tables, since it's used in
// initialization of table fields. It's not allowed otherwise.
if ( global_var && ! type->IsTable() )
{
err_msg = "&default is not valid for global variables except for tables";
return false;
}
const auto& atype = a->GetExpr()->GetType();
if ( type->Tag() != TYPE_TABLE || (type->IsSet() && ! in_record) )
{
if ( same_type(atype, type) )
// Ok.
return true;
// Record defaults may be promotable.
if ( (type->Tag() == TYPE_RECORD && atype->Tag() == TYPE_RECORD &&
record_promotion_compatible(atype->AsRecordType(), type->AsRecordType())) )
// Ok.
return true;
if ( type->Tag() == TYPE_TABLE && type->AsTableType()->IsUnspecifiedTable() )
// Ok.
return true;
auto e = check_and_promote_expr(a->GetExpr(), type);
if ( e )
{
a->SetAttrExpr(std::move(e));
// Ok.
return true;
}
a->GetExpr()->Error("&default value has inconsistent type", type.get());
return false;
}
TableType* tt = type->AsTableType();
const auto& ytype = tt->Yield();
if ( ! in_record )
{ // &default applies to the type itself.
if ( same_type(atype, ytype) )
return true;
// It can still be a default function.
if ( atype->Tag() == TYPE_FUNC )
{
FuncType* f = atype->AsFuncType();
if ( ! f->CheckArgs(tt->GetIndexTypes()) || ! same_type(f->Yield(), ytype) )
{
err_msg = "&default function type clash";
return false;
}
// Ok.
return true;
}
// Table defaults may be promotable.
if ( (ytype->Tag() == TYPE_RECORD && atype->Tag() == TYPE_RECORD &&
record_promotion_compatible(atype->AsRecordType(), ytype->AsRecordType())) )
// Ok.
return true;
auto e = check_and_promote_expr(a->GetExpr(), ytype);
if ( e )
{
a->SetAttrExpr(std::move(e));
// Ok.
return true;
}
err_msg = "&default value has inconsistent type";
return false;
}
// &default applies to record field.
if ( same_type(atype, type) )
return true;
if ( (atype->Tag() == TYPE_TABLE && atype->AsTableType()->IsUnspecifiedTable()) )
{
auto e = check_and_promote_expr(a->GetExpr(), type);
if ( e )
{
a->SetAttrExpr(std::move(e));
return true;
}
}
// Table defaults may be promotable.
if ( ytype && ytype->Tag() == TYPE_RECORD && atype->Tag() == TYPE_RECORD &&
record_promotion_compatible(atype->AsRecordType(), ytype->AsRecordType()) )
// Ok.
return true;
err_msg = "&default value has inconsistent type";
return false;
}
}

11
src/Attr.h
View file

@ -139,5 +139,16 @@ protected:
bool global_var;
};
// Checks whether default attribute "a" is compatible with the given type.
// "global_var" specifies whether the attribute is being associated with
// a global variable, and "in_record" whether it's occurring inside of
// a record declaration.
//
// Returns true on compatibility (which might include modifying "a"), false
// on an error. If an error message hasn't been directly generated, then
// it will be returned in err_msg.
extern bool check_default_attr(Attr* a, const TypePtr& type, bool global_var, bool in_record,
std::string& err_msg);
} // namespace detail
} // namespace zeek

56
src/CMakeLists.txt
View file

@ -150,6 +150,15 @@ list(APPEND BINPAC_OUTPUTS "${BINPAC_OUTPUT_CC}")
binpac_target(binpac_zeek-lib.pac)
list(APPEND BINPAC_OUTPUTS "${BINPAC_OUTPUT_CC}")
########################################################################
## Gen-ZAM setup
include(Gen-ZAM)
set(GEN_ZAM_SRC ${CMAKE_CURRENT_SOURCE_DIR}/script_opt/ZAM/Ops.in)
gen_zam_target(${GEN_ZAM_SRC})
########################################################################
## Including subdirectories.
########################################################################
@ -250,36 +259,6 @@ set(_gen_zeek_script_cpp ${CMAKE_CURRENT_BINARY_DIR}/../CPP-gen.cc)
add_custom_command(OUTPUT ${_gen_zeek_script_cpp}
COMMAND ${CMAKE_COMMAND} -E touch ${_gen_zeek_script_cpp})
# define a command that's used to run the ZAM instruction generator;
# building the zeek binary depends on the outputs of this script
add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ZAM-AssignFlavorsDefs.h
${CMAKE_CURRENT_BINARY_DIR}/ZAM-Conds.h
${CMAKE_CURRENT_BINARY_DIR}/ZAM-DirectDefs.h
${CMAKE_CURRENT_BINARY_DIR}/ZAM-EvalDefs.h
${CMAKE_CURRENT_BINARY_DIR}/ZAM-EvalMacros.h
${CMAKE_CURRENT_BINARY_DIR}/ZAM-GenExprsDefsC1.h
${CMAKE_CURRENT_BINARY_DIR}/ZAM-GenExprsDefsC2.h
${CMAKE_CURRENT_BINARY_DIR}/ZAM-GenExprsDefsC3.h
${CMAKE_CURRENT_BINARY_DIR}/ZAM-GenExprsDefsV.h
${CMAKE_CURRENT_BINARY_DIR}/ZAM-GenFieldsDefsC1.h
${CMAKE_CURRENT_BINARY_DIR}/ZAM-GenFieldsDefsC2.h
${CMAKE_CURRENT_BINARY_DIR}/ZAM-GenFieldsDefsV.h
${CMAKE_CURRENT_BINARY_DIR}/ZAM-MethodDecls.h
${CMAKE_CURRENT_BINARY_DIR}/ZAM-MethodDefs.h
${CMAKE_CURRENT_BINARY_DIR}/ZAM-Op1FlavorsDefs.h
${CMAKE_CURRENT_BINARY_DIR}/ZAM-OpSideEffects.h
${CMAKE_CURRENT_BINARY_DIR}/ZAM-OpsDefs.h
${CMAKE_CURRENT_BINARY_DIR}/ZAM-OpsNamesDefs.h
${CMAKE_CURRENT_BINARY_DIR}/ZAM-Vec1EvalDefs.h
${CMAKE_CURRENT_BINARY_DIR}/ZAM-Vec2EvalDefs.h
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/Gen-ZAM
ARGS ${CMAKE_CURRENT_SOURCE_DIR}/script_opt/ZAM/Ops.in
DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/Gen-ZAM
${CMAKE_CURRENT_SOURCE_DIR}/script_opt/ZAM/Ops.in
COMMENT "[sh] Generating ZAM operations"
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
)
set_source_files_properties(3rdparty/nb_dns.c PROPERTIES COMPILE_FLAGS
-fno-strict-aliasing)
@ -307,12 +286,14 @@ set(MAIN_SRCS
Desc.cc
Dict.cc
Discard.cc
DNS_Mapping.cc
DNS_Mgr.cc
EquivClass.cc
Event.cc
EventHandler.cc
EventLauncher.cc
EventRegistry.cc
EventTrace.cc
Expr.cc
File.cc
Flare.cc
@ -445,10 +426,6 @@ set(THIRD_PARTY_SRCS
3rdparty/strsep.c
)
set(GEN_ZAM_SRCS
script_opt/ZAM/Gen-ZAM.cc
)
# Highwayhash. Highwayhash is a bit special since it has architecture dependent code...
set(HH_SRCS
@ -504,6 +481,8 @@ set(zeek_SRCS
${BIF_SRCS}
${BINPAC_AUXSRC}
${BINPAC_OUTPUTS}
${GEN_ZAM_SRC}
${GEN_ZAM_OUTPUT_H}
${TRANSFORMED_BISON_OUTPUTS}
${FLEX_RuleScanner_OUTPUTS}
${FLEX_RuleScanner_INPUT}
@ -522,7 +501,6 @@ set(zeek_SRCS
)
collect_headers(zeek_HEADERS ${zeek_SRCS})
collect_headers(GEN_ZAM_HEADERS ${GEN_ZAM_SRCS})
add_library(zeek_objs OBJECT ${zeek_SRCS})
@ -538,8 +516,6 @@ set_target_properties(zeek PROPERTIES ENABLE_EXPORTS TRUE)
install(TARGETS zeek DESTINATION bin)
add_executable(Gen-ZAM ${GEN_ZAM_SRCS} ${GEN_ZAM_HEADERS})
# Install wrapper script for Bro-to-Zeek renaming.
include(InstallSymlink)
InstallSymlink("${CMAKE_INSTALL_PREFIX}/bin/zeek-wrapper" "${CMAKE_INSTALL_PREFIX}/bin/bro")
@ -600,6 +576,10 @@ install(CODE "
)
")
# Make sure to escape a bunch of special characters in the path before trying to use it as a
# regular expression below.
string(REGEX REPLACE "([][+.*()^])" "\\\\\\1" escaped_path "${CMAKE_CURRENT_SOURCE_DIR}/zeek")
install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/
DESTINATION include/zeek
FILES_MATCHING
@ -607,7 +587,7 @@ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/
PATTERN "*.pac"
PATTERN "3rdparty/*" EXCLUDE
# The "zeek -> ." symlink isn't needed in the install-tree
REGEX "^${CMAKE_CURRENT_SOURCE_DIR}/zeek$" EXCLUDE
REGEX "^${escaped_path}$" EXCLUDE
)
install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/

33
src/CompHash.cc
View file

@ -260,13 +260,16 @@ bool CompositeHash::RecoverOneVal(const HashKey& hk, Type* t, ValPtr* pval, bool
{
uint32_t id;
hk.Read("func", id);
const auto& f = Func::GetFuncPtrByID(id);
if ( ! f )
ASSERT(func_id_to_func != nullptr);
if ( id >= func_id_to_func->size() )
reporter->InternalError("failed to look up unique function id %" PRIu32
" in CompositeHash::RecoverOneVal()",
id);
const auto& f = func_id_to_func->at(id);
*pval = make_intrusive<FuncVal>(f);
const auto& pvt = (*pval)->GetType();
@ -547,7 +550,31 @@ bool CompositeHash::SingleValHash(HashKey& hk, const Val* v, Type* bt, bool type
switch ( v->GetType()->Tag() )
{
case TYPE_FUNC:
hk.Write("func", v->AsFunc()->GetUniqueFuncID());
{
auto f = v->AsFunc();
if ( ! func_to_func_id )
const_cast<CompositeHash*>(this)->BuildFuncMappings();
auto id_mapping = func_to_func_id->find(f);
uint32_t id;
if ( id_mapping == func_to_func_id->end() )
{
// We need the pointer to stick around
// for our lifetime, so we have to get
// a non-const version we can ref.
FuncPtr fptr = {NewRef{}, const_cast<Func*>(f)};
id = func_id_to_func->size();
func_id_to_func->push_back(std::move(fptr));
func_to_func_id->insert_or_assign(f, id);
}
else
id = id_mapping->second;
hk.Write("func", id);
}
break;
case TYPE_PATTERN:

14
src/CompHash.h
View file

@ -4,7 +4,7 @@
#include <memory>
#include "zeek/IntrusivePtr.h"
#include "zeek/Func.h"
#include "zeek/Type.h"
namespace zeek
@ -61,6 +61,18 @@ protected:
bool EnsureTypeReserve(HashKey& hk, const Val* v, Type* bt, bool type_check) const;
// The following are for allowing hashing of function values.
// These can occur, for example, in sets of predicates that get
// iterated over. We use pointers in order to keep storage
// lower for the common case of these not being needed.
std::unique_ptr<std::unordered_map<const Func*, uint32_t>> func_to_func_id;
std::unique_ptr<std::vector<FuncPtr>> func_id_to_func;
void BuildFuncMappings()
{
func_to_func_id = std::make_unique<std::unordered_map<const Func*, uint32_t>>();
func_id_to_func = std::make_unique<std::vector<FuncPtr>>();
}
TypeListPtr type;
bool is_singleton = false; // if just one type in index
};

428
src/DNS_Mapping.cc Normal file
View file

@ -0,0 +1,428 @@
#include "zeek/DNS_Mapping.h"
#include <ares_nameser.h>
#include "zeek/3rdparty/doctest.h"
#include "zeek/DNS_Mgr.h"
#include "zeek/Reporter.h"
namespace zeek::detail
{
DNS_Mapping::DNS_Mapping(std::string host, struct hostent* h, uint32_t ttl, int type)
{
Init(h);
req_host = host;
req_ttl = ttl;
req_type = type;
if ( names.empty() )
names.push_back(std::move(host));
}
DNS_Mapping::DNS_Mapping(const IPAddr& addr, struct hostent* h, uint32_t ttl)
{
Init(h);
req_addr = addr;
req_ttl = ttl;
req_type = T_PTR;
}
DNS_Mapping::DNS_Mapping(FILE* f)
{
Clear();
init_failed = true;
req_ttl = 0;
creation_time = 0;
char buf[512];
if ( ! fgets(buf, sizeof(buf), f) )
{
no_mapping = true;
return;
}
char req_buf[512 + 1], name_buf[512 + 1];
int is_req_host;
int failed_local;
int num_addrs;
if ( sscanf(buf, "%lf %d %512s %d %512s %d %d %" PRIu32, &creation_time, &is_req_host, req_buf,
&failed_local, name_buf, &req_type, &num_addrs, &req_ttl) != 8 )
{
no_mapping = true;
return;
}
failed = static_cast<bool>(failed_local);
if ( is_req_host )
req_host = req_buf;
else
req_addr = IPAddr(req_buf);
names.push_back(name_buf);
for ( int i = 0; i < num_addrs; ++i )
{
if ( ! fgets(buf, sizeof(buf), f) )
return;
char* newline = strchr(buf, '\n');
if ( newline )
*newline = '\0';
addrs.emplace_back(IPAddr(buf));
}
init_failed = false;
}
ListValPtr DNS_Mapping::Addrs()
{
if ( failed )
return nullptr;
if ( ! addrs_val )
{
addrs_val = make_intrusive<ListVal>(TYPE_ADDR);
for ( const auto& addr : addrs )
addrs_val->Append(make_intrusive<AddrVal>(addr));
}
return addrs_val;
}
TableValPtr DNS_Mapping::AddrsSet()
{
auto l = Addrs();
if ( ! l || l->Length() == 0 )
return DNS_Mgr::empty_addr_set();
return l->ToSetVal();
}
StringValPtr DNS_Mapping::Host()
{
if ( failed || names.empty() )
return nullptr;
if ( ! host_val )
host_val = make_intrusive<StringVal>(names[0]);
return host_val;
}
void DNS_Mapping::Init(struct hostent* h)
{
no_mapping = false;
init_failed = false;
creation_time = util::current_time();
host_val = nullptr;
addrs_val = nullptr;
if ( ! h )
{
Clear();
return;
}
if ( h->h_name )
// for now, just use the official name
// TODO: this could easily be expanded to include all of the aliases as well
names.push_back(h->h_name);
if ( h->h_addr_list )
{
for ( int i = 0; h->h_addr_list[i] != NULL; ++i )
{
if ( h->h_addrtype == AF_INET )
addrs.push_back(IPAddr(IPv4, (uint32_t*)h->h_addr_list[i], IPAddr::Network));
else if ( h->h_addrtype == AF_INET6 )
addrs.push_back(IPAddr(IPv6, (uint32_t*)h->h_addr_list[i], IPAddr::Network));
}
}
failed = false;
}
void DNS_Mapping::Clear()
{
names.clear();
host_val = nullptr;
addrs.clear();
addrs_val = nullptr;
no_mapping = false;
req_type = 0;
failed = true;
}
void DNS_Mapping::Save(FILE* f) const
{
fprintf(f, "%.0f %d %s %d %s %d %zu %" PRIu32 "\n", creation_time, ! req_host.empty(),
req_host.empty() ? req_addr.AsString().c_str() : req_host.c_str(), failed,
names.empty() ? "*" : names[0].c_str(), req_type, addrs.size(), req_ttl);
for ( const auto& addr : addrs )
fprintf(f, "%s\n", addr.AsString().c_str());
}
void DNS_Mapping::Merge(const DNS_MappingPtr& other)
{
std::copy(other->names.begin(), other->names.end(), std::back_inserter(names));
std::copy(other->addrs.begin(), other->addrs.end(), std::back_inserter(addrs));
}
// This value needs to be incremented if something changes in the data stored by Save(). This
// allows us to change the structure of the cache without breaking something in DNS_Mgr.
constexpr int FILE_VERSION = 1;
void DNS_Mapping::InitializeCache(FILE* f)
{
fprintf(f, "%d\n", FILE_VERSION);
}
bool DNS_Mapping::ValidateCacheVersion(FILE* f)
{
char buf[512];
if ( ! fgets(buf, sizeof(buf), f) )
return false;
int version;
if ( sscanf(buf, "%d", &version) != 1 )
{
reporter->Warning("Existing DNS cache did not have correct version, ignoring");
return false;
}
return FILE_VERSION == version;
}
//////////////////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////////////////
TEST_CASE("dns_mapping init null hostent")
{
DNS_Mapping mapping("www.apple.com", nullptr, 123, T_A);
CHECK(! mapping.Valid());
CHECK(mapping.Addrs() == nullptr);
CHECK(mapping.AddrsSet()->EqualTo(DNS_Mgr::empty_addr_set()));
CHECK(mapping.Host() == nullptr);
}
TEST_CASE("dns_mapping init host")
{
IPAddr addr("1.2.3.4");
in4_addr in4;
addr.CopyIPv4(&in4);
struct hostent he;
he.h_name = util::copy_string("testing.home");
he.h_aliases = NULL;
he.h_addrtype = AF_INET;
he.h_length = sizeof(in_addr);
std::vector<in_addr*> addrs = {&in4, NULL};
he.h_addr_list = reinterpret_cast<char**>(addrs.data());
DNS_Mapping mapping("testing.home", &he, 123, T_A);
CHECK(mapping.Valid());
CHECK(mapping.ReqAddr() == IPAddr::v6_unspecified);
CHECK(strcmp(mapping.ReqHost(), "testing.home") == 0);
CHECK(mapping.ReqStr() == "testing.home");
auto lva = mapping.Addrs();
REQUIRE(lva != nullptr);
CHECK(lva->Length() == 1);
auto lvae = lva->Idx(0)->AsAddrVal();
REQUIRE(lvae != nullptr);
CHECK(lvae->Get().AsString() == "1.2.3.4");
auto tvas = mapping.AddrsSet();
REQUIRE(tvas != nullptr);
CHECK_FALSE(tvas->EqualTo(DNS_Mgr::empty_addr_set()));
auto svh = mapping.Host();
REQUIRE(svh != nullptr);
CHECK(svh->ToStdString() == "testing.home");
delete[] he.h_name;
}
TEST_CASE("dns_mapping init addr")
{
IPAddr addr("1.2.3.4");
in4_addr in4;
addr.CopyIPv4(&in4);
struct hostent he;
he.h_name = util::copy_string("testing.home");
he.h_aliases = NULL;
he.h_addrtype = AF_INET;
he.h_length = sizeof(in_addr);
std::vector<in_addr*> addrs = {&in4, NULL};
he.h_addr_list = reinterpret_cast<char**>(addrs.data());
DNS_Mapping mapping(addr, &he, 123);
CHECK(mapping.Valid());
CHECK(mapping.ReqAddr() == addr);
CHECK(mapping.ReqHost() == nullptr);
CHECK(mapping.ReqStr() == "1.2.3.4");
auto lva = mapping.Addrs();
REQUIRE(lva != nullptr);
CHECK(lva->Length() == 1);
auto lvae = lva->Idx(0)->AsAddrVal();
REQUIRE(lvae != nullptr);
CHECK(lvae->Get().AsString() == "1.2.3.4");
auto tvas = mapping.AddrsSet();
REQUIRE(tvas != nullptr);
CHECK_FALSE(tvas->EqualTo(DNS_Mgr::empty_addr_set()));
auto svh = mapping.Host();
REQUIRE(svh != nullptr);
CHECK(svh->ToStdString() == "testing.home");
delete[] he.h_name;
}
TEST_CASE("dns_mapping save reload")
{
IPAddr addr("1.2.3.4");
in4_addr in4;
addr.CopyIPv4(&in4);
struct hostent he;
he.h_name = util::copy_string("testing.home");
he.h_aliases = NULL;
he.h_addrtype = AF_INET;
he.h_length = sizeof(in_addr);
std::vector<in_addr*> addrs = {&in4, NULL};
he.h_addr_list = reinterpret_cast<char**>(addrs.data());
// Create a temporary file in memory and fseek to the end of it so we're at
// EOF for the next bit.
char buffer[4096];
memset(buffer, 0, 4096);
FILE* tmpfile = fmemopen(buffer, 4096, "r+");
fseek(tmpfile, 0, SEEK_END);
// Try loading from the file at EOF. This should cause a mapping failure.
DNS_Mapping mapping(tmpfile);
CHECK(mapping.NoMapping());
rewind(tmpfile);
// Try reading from the empty file. This should cause an init failure.
DNS_Mapping mapping2(tmpfile);
CHECK(mapping2.InitFailed());
rewind(tmpfile);
// Save a valid mapping into the file and rewind to the start.
DNS_Mapping mapping3(addr, &he, 123);
mapping3.Save(tmpfile);
rewind(tmpfile);
// Test loading the mapping back out of the file
DNS_Mapping mapping4(tmpfile);
fclose(tmpfile);
CHECK(mapping4.Valid());
CHECK(mapping4.ReqAddr() == addr);
CHECK(mapping4.ReqHost() == nullptr);
CHECK(mapping4.ReqStr() == "1.2.3.4");
auto lva = mapping4.Addrs();
REQUIRE(lva != nullptr);
CHECK(lva->Length() == 1);
auto lvae = lva->Idx(0)->AsAddrVal();
REQUIRE(lvae != nullptr);
CHECK(lvae->Get().AsString() == "1.2.3.4");
auto tvas = mapping4.AddrsSet();
REQUIRE(tvas != nullptr);
CHECK(tvas != DNS_Mgr::empty_addr_set());
auto svh = mapping4.Host();
REQUIRE(svh != nullptr);
CHECK(svh->ToStdString() == "testing.home");
delete[] he.h_name;
}
TEST_CASE("dns_mapping multiple addresses")
{
IPAddr addr("1.2.3.4");
in4_addr in4_1;
addr.CopyIPv4(&in4_1);
IPAddr addr2("5.6.7.8");
in4_addr in4_2;
addr2.CopyIPv4(&in4_2);
struct hostent he;
he.h_name = util::copy_string("testing.home");
he.h_aliases = NULL;
he.h_addrtype = AF_INET;
he.h_length = sizeof(in_addr);
std::vector<in_addr*> addrs = {&in4_1, &in4_2, NULL};
he.h_addr_list = reinterpret_cast<char**>(addrs.data());
DNS_Mapping mapping("testing.home", &he, 123, T_A);
CHECK(mapping.Valid());
auto lva = mapping.Addrs();
REQUIRE(lva != nullptr);
CHECK(lva->Length() == 2);
auto lvae = lva->Idx(0)->AsAddrVal();
REQUIRE(lvae != nullptr);
CHECK(lvae->Get().AsString() == "1.2.3.4");
lvae = lva->Idx(1)->AsAddrVal();
REQUIRE(lvae != nullptr);
CHECK(lvae->Get().AsString() == "5.6.7.8");
delete[] he.h_name;
}
TEST_CASE("dns_mapping ipv6")
{
IPAddr addr("64:ff9b:1::");
in6_addr in6;
addr.CopyIPv6(&in6);
struct hostent he;
he.h_name = util::copy_string("testing.home");
he.h_aliases = NULL;
he.h_addrtype = AF_INET6;
he.h_length = sizeof(in6_addr);
std::vector<in6_addr*> addrs = {&in6, NULL};
he.h_addr_list = reinterpret_cast<char**>(addrs.data());
DNS_Mapping mapping(addr, &he, 123);
CHECK(mapping.Valid());
CHECK(mapping.ReqAddr() == addr);
CHECK(mapping.ReqHost() == nullptr);
CHECK(mapping.ReqStr() == "64:ff9b:1::");
auto lva = mapping.Addrs();
REQUIRE(lva != nullptr);
CHECK(lva->Length() == 1);
auto lvae = lva->Idx(0)->AsAddrVal();
REQUIRE(lvae != nullptr);
CHECK(lvae->Get().AsString() == "64:ff9b:1::");
delete[] he.h_name;
}
} // namespace zeek::detail

86
src/DNS_Mapping.h Normal file
View file

@ -0,0 +1,86 @@
#pragma once
#include <netdb.h>
#include <sys/socket.h>
#include <cstdint>
#include <string>
#include "zeek/IPAddr.h"
#include "zeek/Val.h"
namespace zeek::detail
{
class DNS_Mapping;
using DNS_MappingPtr = std::shared_ptr<DNS_Mapping>;
class DNS_Mapping
{
public:
DNS_Mapping() = delete;
DNS_Mapping(std::string host, struct hostent* h, uint32_t ttl, int type);
DNS_Mapping(const IPAddr& addr, struct hostent* h, uint32_t ttl);
DNS_Mapping(FILE* f);
bool NoMapping() const { return no_mapping; }
bool InitFailed() const { return init_failed; }
~DNS_Mapping() = default;
// Returns nil if this was an address request.
// TODO: fix this an uses of this to just return the empty string
const char* ReqHost() const { return req_host.empty() ? nullptr : req_host.c_str(); }
const IPAddr& ReqAddr() const { return req_addr; }
std::string ReqStr() const { return req_host.empty() ? req_addr.AsString() : req_host; }
int ReqType() const { return req_type; }
ListValPtr Addrs();
TableValPtr AddrsSet(); // addresses returned as a set
StringValPtr Host();
double CreationTime() const { return creation_time; }
uint32_t TTL() const { return req_ttl; }
void Save(FILE* f) const;
bool Failed() const { return failed; }
bool Valid() const { return ! failed; }
bool Expired() const
{
if ( ! req_host.empty() && addrs.empty() )
return false; // nothing to expire
return util::current_time() > (creation_time + req_ttl);
}
void Merge(const DNS_MappingPtr& other);
static void InitializeCache(FILE* f);
static bool ValidateCacheVersion(FILE* f);
protected:
friend class DNS_Mgr;
void Init(struct hostent* h);
void Clear();
std::string req_host;
IPAddr req_addr;
uint32_t req_ttl = 0;
int req_type = 0;
// This class supports multiple names per address, but we only store one of them.
std::vector<std::string> names;
StringValPtr host_val;
std::vector<IPAddr> addrs;
ListValPtr addrs_val;
double creation_time = 0.0;
bool no_mapping = false; // when initializing from a file, immediately hit EOF
bool init_failed = false;
bool failed = false;
};
} // namespace zeek::detail

2455
src/DNS_Mgr.cc
View file

File diff suppressed because it is too large Load diff

384
src/DNS_Mgr.h
View file

@ -2,10 +2,12 @@
#pragma once
#include <netdb.h>
#include <list>
#include <map>
#include <queue>
#include <utility>
#include <variant>
#include "zeek/EventHandler.h"
#include "zeek/IPAddr.h"
@ -13,33 +15,38 @@
#include "zeek/iosource/IOSource.h"
#include "zeek/util.h"
// These are defined in ares headers but we don't want to have to include
// those headers here and create install dependencies on them.
struct ares_channeldata;
typedef struct ares_channeldata* ares_channel;
#ifndef T_PTR
#define T_PTR 12
#endif
#ifndef T_TXT
#define T_TXT 16
#endif
namespace zeek
{
class EventHandler;
class RecordType;
class Val;
class ListVal;
class TableVal;
class StringVal;
template <class T> class IntrusivePtr;
using ValPtr = IntrusivePtr<Val>;
using ListValPtr = IntrusivePtr<ListVal>;
using TableValPtr = IntrusivePtr<TableVal>;
using StringValPtr = IntrusivePtr<StringVal>;
} // namespace zeek
// Defined in nb_dns.h
struct nb_dns_info;
struct nb_dns_result;
namespace zeek::detail
{
class DNS_Mgr_Request;
using DNS_mgr_request_list = PList<DNS_Mgr_Request>;
class DNS_Mapping;
using DNS_MappingPtr = std::shared_ptr<DNS_Mapping>;
class DNS_Request;
enum DNS_MgrMode
{
@ -49,50 +56,144 @@ enum DNS_MgrMode
DNS_FAKE, // don't look up names, just return dummy results
};
// Number of seconds we'll wait for a reply.
#define DNS_TIMEOUT 5
class DNS_Mgr final : public iosource::IOSource
class DNS_Mgr : public iosource::IOSource
{
public:
explicit DNS_Mgr(DNS_MgrMode mode);
~DNS_Mgr() override;
void InitPostScript();
void Flush();
// Looks up the address or addresses of the given host, and returns
// a set of addr.
TableValPtr LookupHost(const char* host);
ValPtr LookupAddr(const IPAddr& addr);
// Define the directory where to store the data.
void SetDir(const char* arg_dir) { dir = util::copy_string(arg_dir); }
void Verify();
void Resolve();
bool Save();
const char* LookupAddrInCache(const IPAddr& addr);
TableValPtr LookupNameInCache(const std::string& name);
const char* LookupTextInCache(const std::string& name);
// Support for async lookups.
/**
* Base class for callback handling for asynchronous lookups.
*/
class LookupCallback
{
public:
LookupCallback() { }
virtual ~LookupCallback() { }
virtual ~LookupCallback() = default;
virtual void Resolved(const char* name){};
virtual void Resolved(TableVal* addrs){};
/**
* Called when an address lookup finishes.
*
* @param name The resulting name from the lookup.
*/
virtual void Resolved(const std::string& name){};
/**
* Called when a name lookup finishes.
*
* @param addrs A table of the resulting addresses from the lookup.
*/
virtual void Resolved(TableValPtr addrs){};
/**
* Generic callback method for all request types.
*
* @param val A Val containing the data from the query.
*/
virtual void Resolved(ValPtr data, int request_type) { }
/**
* Called when a timeout request occurs.
*/
virtual void Timeout() = 0;
};
void AsyncLookupAddr(const IPAddr& host, LookupCallback* callback);
void AsyncLookupName(const std::string& name, LookupCallback* callback);
void AsyncLookupNameText(const std::string& name, LookupCallback* callback);
explicit DNS_Mgr(DNS_MgrMode mode);
~DNS_Mgr() override;
/**
* Finalizes the manager initialization. This should be called only after all
* of the scripts have been parsed at startup.
*/
void InitPostScript();
/**
* Attempts to process one more round of requests and then flushes the
* mapping caches.
*/
void Flush();
/**
* Looks up the address(es) of a given host and returns a set of addresses.
* This is a shorthand method for doing A/AAAA requests. This is a
* synchronous request and will block until the request completes or times
* out.
*
* @param host The hostname to lookup an address for.
* @return A set of addresses for the host.
*/
TableValPtr LookupHost(const std::string& host);
/**
* Looks up the hostname of a given address. This is a shorthand method for
* doing PTR requests. This is a synchronous request and will block until
* the request completes or times out.
*
* @param host The addr to lookup a hostname for.
* @return The hostname for the address.
*/
StringValPtr LookupAddr(const IPAddr& addr);
/**
* Performs a generic request to the DNS server. This is a synchronous
* request and will block until the request completes or times out.
*
* @param name The name or address to make a request for. If this is an
* address it should be in arpa format (x.x.x.x.in-addr.arpa or x-*.ip6.arpa).
* Note that calling LookupAddr for PTR requests does this conversion
* automatically.
* @param request_type The type of request to make. This should be one of
* the type values defined in arpa/nameser.h or ares_nameser.h.
* @return The requested data.
*/
ValPtr Lookup(const std::string& name, int request_type);
/**
* Looks up the address(es) of a given host. This is a shorthand method
* for doing A/AAAA requests. This is an asynchronous request. The
* response will be handled via the provided callback object.
*
* @param host The hostname to lookup an address for.
* @param callback A callback object for handling the response.
*/
void LookupHost(const std::string& host, LookupCallback* callback);
/**
* Looks up the hostname of a given address. This is a shorthand method for
* doing PTR requests. This is an asynchronous request. The response will
* be handled via the provided callback object.
*
* @param host The addr to lookup a hostname for.
* @param callback A callback object for handling the response.
*/
void LookupAddr(const IPAddr& addr, LookupCallback* callback);
/**
* Performs a generic request to the DNS server. This is an asynchronous
* request. The response will be handled via the provided callback
* object.
*
* @param name The name or address to make a request for. If this is an
* address it should be in arpa format (x.x.x.x.in-addr.arpa or x-*.ip6.arpa).
* Note that calling LookupAddr for PTR requests does this conversion
* automatically.
* @param request_type The type of request to make. This should be one of
* the type values defined in arpa/nameser.h or ares_nameser.h.
* @param callback A callback object for handling the response.
*/
void Lookup(const std::string& name, int request_type, LookupCallback* callback);
/**
* Sets the directory where to store DNS data when Save() is called.
*/
void SetDir(const std::string& arg_dir) { dir = arg_dir; }
/**
* Waits for responses to become available or a timeout to occur,
* and handles any responses.
*/
void Resolve();
/**
* Saves the current name and address caches to disk.
*/
bool Save();
struct Stats
{
@ -105,142 +206,137 @@ public:
unsigned long cached_texts;
};
/**
* Returns the current statistics for the DNS_Manager.
*
* @param stats A pointer to a stats object to return the data in.
*/
void GetStats(Stats* stats);
void Terminate();
/**
* Adds a result from a request to the caches. This is public so that the
* callback methods can call it from outside of the DNS_Mgr class.
*
* @param dr The request associated with the result.
* @param h A hostent structure containing the actual result data.
* @param ttl A ttl value contained in the response from the server.
* @param merge A flag for whether these results should be merged into
* an existing mapping. If false, AddResult will attempt to replace the
* existing mapping with the new data and delete the old mapping.
*/
void AddResult(DNS_Request* dr, struct hostent* h, uint32_t ttl, bool merge = false);
/**
* Returns an empty set of addresses, used in various error cases and during
* cache priming.
*/
static TableValPtr empty_addr_set();
/**
* Returns the full path to the file used to store the DNS cache.
*/
std::string CacheFile() const { return cache_name; }
/**
* Used by the c-ares socket call back to register/unregister a socket file descriptor.
*/
void RegisterSocket(int fd, bool read, bool write);
ares_channel& GetChannel() { return channel; }
protected:
friend class LookupCallback;
friend class DNS_Mgr_Request;
friend class DNS_Request;
void Event(EventHandlerPtr e, DNS_Mapping* dm);
void Event(EventHandlerPtr e, DNS_Mapping* dm, ListValPtr l1, ListValPtr l2);
void Event(EventHandlerPtr e, DNS_Mapping* old_dm, DNS_Mapping* new_dm);
ValPtr BuildMappingVal(DNS_Mapping* dm);
void AddResult(DNS_Mgr_Request* dr, struct nb_dns_result* r);
void CompareMappings(DNS_Mapping* prev_dm, DNS_Mapping* new_dm);
ListValPtr AddrListDelta(ListVal* al1, ListVal* al2);
void DumpAddrList(FILE* f, ListVal* al);
using HostMap = std::map<std::string, std::pair<DNS_Mapping*, DNS_Mapping*>>;
using AddrMap = std::map<IPAddr, DNS_Mapping*>;
using TextMap = std::map<std::string, DNS_Mapping*>;
void LoadCache(FILE* f);
void Save(FILE* f, const AddrMap& m);
void Save(FILE* f, const HostMap& m);
// Selects on the fd to see if there is an answer available (timeout
// is secs). Returns 0 on timeout, -1 on EINTR or other error, and 1
// if answer is ready.
int AnswerAvailable(int timeout);
// Issue as many queued async requests as slots are available.
void IssueAsyncRequests();
StringValPtr LookupAddrInCache(const IPAddr& addr, bool cleanup_expired = false,
bool check_failed = false);
TableValPtr LookupNameInCache(const std::string& name, bool cleanup_expired = false,
bool check_failed = false);
StringValPtr LookupOtherInCache(const std::string& name, int request_type,
bool cleanup_expired = false);
// Finish the request if we have a result. If not, time it out if
// requested.
void CheckAsyncAddrRequest(const IPAddr& addr, bool timeout);
void CheckAsyncHostRequest(const char* host, bool timeout);
void CheckAsyncTextRequest(const char* host, bool timeout);
void CheckAsyncHostRequest(const std::string& host, bool timeout);
void CheckAsyncOtherRequest(const std::string& host, bool timeout, int request_type);
void Event(EventHandlerPtr e, const DNS_MappingPtr& dm);
void Event(EventHandlerPtr e, const DNS_MappingPtr& dm, ListValPtr l1, ListValPtr l2);
void Event(EventHandlerPtr e, const DNS_MappingPtr& old_dm, DNS_MappingPtr new_dm);
ValPtr BuildMappingVal(const DNS_MappingPtr& dm);
void CompareMappings(const DNS_MappingPtr& prev_dm, const DNS_MappingPtr& new_dm);
ListValPtr AddrListDelta(ListValPtr al1, ListValPtr al2);
using MappingKey = std::variant<IPAddr, std::pair<int, std::string>>;
using MappingMap = std::map<MappingKey, DNS_MappingPtr>;
void LoadCache(const std::string& path);
void Save(FILE* f, const MappingMap& m);
// Issue as many queued async requests as slots are available.
void IssueAsyncRequests();
// IOSource interface.
void Process() override;
void Process() override { }
void ProcessFd(int fd, int flags) override;
void InitSource() override;
const char* Tag() override { return "DNS_Mgr"; }
double GetNextTimeout() override;
DNS_MgrMode mode;
HostMap host_mappings;
AddrMap addr_mappings;
TextMap text_mappings;
MappingMap all_mappings;
DNS_mgr_request_list requests;
std::string cache_name;
std::string dir; // directory in which cache_name resides
nb_dns_info* nb_dns;
char* cache_name;
char* dir; // directory in which cache_name resides
bool did_init;
int asyncs_pending;
bool did_init = false;
int asyncs_pending = 0;
RecordTypePtr dm_rec;
ares_channel channel{};
using CallbackList = std::list<LookupCallback*>;
struct AsyncRequest
{
double time;
IPAddr host;
std::string name;
double time = 0.0;
IPAddr addr;
std::string host;
CallbackList callbacks;
bool is_txt;
bool processed;
int type = 0;
bool processed = false;
AsyncRequest() : time(0.0), is_txt(false), processed(false) { }
AsyncRequest(std::string host, int request_type) : host(std::move(host)), type(request_type)
{
}
AsyncRequest(const IPAddr& addr) : addr(addr), type(T_PTR) { }
bool IsAddrReq() const { return name.empty(); }
void Resolved(const char* name)
{
for ( CallbackList::iterator i = callbacks.begin(); i != callbacks.end(); ++i )
{
(*i)->Resolved(name);
delete *i;
}
callbacks.clear();
processed = true;
}
void Resolved(TableVal* addrs)
{
for ( CallbackList::iterator i = callbacks.begin(); i != callbacks.end(); ++i )
{
(*i)->Resolved(addrs);
delete *i;
}
callbacks.clear();
processed = true;
}
void Timeout()
{
for ( CallbackList::iterator i = callbacks.begin(); i != callbacks.end(); ++i )
{
(*i)->Timeout();
delete *i;
}
callbacks.clear();
processed = true;
}
void Resolved(const std::string& name);
void Resolved(TableValPtr addrs);
void Timeout();
};
using AsyncRequestAddrMap = std::map<IPAddr, AsyncRequest*>;
AsyncRequestAddrMap asyncs_addrs;
using AsyncRequestNameMap = std::map<std::string, AsyncRequest*>;
AsyncRequestNameMap asyncs_names;
using AsyncRequestTextMap = std::map<std::string, AsyncRequest*>;
AsyncRequestTextMap asyncs_texts;
using QueuedList = std::list<AsyncRequest*>;
QueuedList asyncs_queued;
struct AsyncRequestCompare
{
bool operator()(const AsyncRequest* a, const AsyncRequest* b) { return a->time > b->time; }
};
using TimeoutQueue =
std::priority_queue<AsyncRequest*, std::vector<AsyncRequest*>, AsyncRequestCompare>;
TimeoutQueue asyncs_timeouts;
using AsyncRequestMap = std::map<MappingKey, AsyncRequest*>;
AsyncRequestMap asyncs;
unsigned long num_requests;
unsigned long successful;
unsigned long failed;
using QueuedList = std::list<AsyncRequest*>;
QueuedList asyncs_queued;
unsigned long num_requests = 0;
unsigned long successful = 0;
unsigned long failed = 0;
std::set<int> socket_fds;
std::set<int> write_socket_fds;
};
extern DNS_Mgr* dns_mgr;

2
src/DbgBreakpoint.h
View file

@ -1,4 +1,4 @@
// Structures and methods for implementing breakpoints in the Bro debugger.
// Structures and methods for implementing breakpoints in the Zeek debugger.
#pragma once

2
src/DbgDisplay.h
View file

@ -1,4 +1,4 @@
// Structures and methods for implementing watches in the Bro debugger.
// Structures and methods for implementing watches in the Zeek debugger.
#pragma once

2
src/DbgHelp.cc
View file

@ -1,4 +1,4 @@
// Bro Debugger Help
// Zeek Debugger Help
#include "zeek/zeek-config.h"

2
src/DbgWatch.h
View file

@ -1,4 +1,4 @@
// Structures and methods for implementing watches in the Bro debugger.
// Structures and methods for implementing watches in the Zeek debugger.
#pragma once

2
src/Debug.cc
View file

@ -1,4 +1,4 @@
// Debugging support for Bro policy files.
// Debugging support for Zeek policy files.
#include "zeek/Debug.h"

2
src/Debug.h
View file

@ -1,4 +1,4 @@
// Debugging support for Bro policy files.
// Debugging support for Zeek policy files.
#pragma once

2
src/DebugCmds.cc
View file

@ -1,4 +1,4 @@
// Support routines to help deal with Bro debugging commands and
// Support routines to help deal with Zeek debugging commands and
// implementation of most commands.
#include "zeek/DebugCmds.h"

2
src/DebugCmds.h
View file

@ -1,4 +1,4 @@
// Support routines to help deal with Bro debugging commands and
// Support routines to help deal with Zeek debugging commands and
// implementation of most commands.
#pragma once

47
src/Dict.cc
View file

@ -1299,6 +1299,15 @@ void Dictionary::AdjustOnInsert(IterCookie* c, const detail::DictEntry& entry, i
{
ASSERT(c);
ASSERT_VALID(c);
// Remove any previous instances of this value that we may have recorded as
// their pointers will get invalid. We won't need that knowledge anymore
// anyways, will update with new information below as needed.
c->inserted->erase(std::remove(c->inserted->begin(), c->inserted->end(), entry),
c->inserted->end());
c->visited->erase(std::remove(c->visited->begin(), c->visited->end(), entry),
c->visited->end());
if ( insert_position < c->next )
c->inserted->push_back(entry);
if ( insert_position < c->next && c->next <= last_affected_position )
@ -1314,6 +1323,12 @@ void Dictionary::AdjustOnInsert(IterCookie* c, const detail::DictEntry& entry, i
void Dictionary::AdjustOnInsert(RobustDictIterator* c, const detail::DictEntry& entry,
int insert_position, int last_affected_position)
{
// See note in Dictionary::AdjustOnInsert() above.
c->inserted->erase(std::remove(c->inserted->begin(), c->inserted->end(), entry),
c->inserted->end());
c->visited->erase(std::remove(c->visited->begin(), c->visited->end(), entry),
c->visited->end());
if ( insert_position < c->next )
c->inserted->push_back(entry);
if ( insert_position < c->next && c->next <= last_affected_position )
@ -1442,8 +1457,13 @@ void Dictionary::AdjustOnRemove(IterCookie* c, const detail::DictEntry& entry, i
int last_affected_position)
{
ASSERT_VALID(c);
// See note in Dictionary::AdjustOnInsert() above.
c->inserted->erase(std::remove(c->inserted->begin(), c->inserted->end(), entry),
c->inserted->end());
c->visited->erase(std::remove(c->visited->begin(), c->visited->end(), entry),
c->visited->end());
if ( position < c->next && c->next <= last_affected_position )
{
int moved = HeadOfClusterByPosition(c->next - 1);
@ -1462,8 +1482,12 @@ void Dictionary::AdjustOnRemove(IterCookie* c, const detail::DictEntry& entry, i
void Dictionary::AdjustOnRemove(RobustDictIterator* c, const detail::DictEntry& entry, int position,
int last_affected_position)
{
// See note in Dictionary::AdjustOnInsert() above.
c->inserted->erase(std::remove(c->inserted->begin(), c->inserted->end(), entry),
c->inserted->end());
c->visited->erase(std::remove(c->visited->begin(), c->visited->end(), entry),
c->visited->end());
if ( position < c->next && c->next <= last_affected_position )
{
int moved = HeadOfClusterByPosition(c->next - 1);
@ -1475,6 +1499,14 @@ void Dictionary::AdjustOnRemove(RobustDictIterator* c, const detail::DictEntry&
// if not already the end of the dictionary, adjust next to a valid one.
if ( c->next < Capacity() && table[c->next].Empty() )
c->next = Next(c->next);
if ( c->curr == entry )
{
if ( c->next >= 0 && c->next < Capacity() && ! table[c->next].Empty() )
c->curr = table[c->next];
else
c->curr = detail::DictEntry(nullptr); // -> c == end_robust()
}
}
///////////////////////////////////////////////////////////////////////////////////////////////////
@ -1808,7 +1840,20 @@ detail::DictEntry Dictionary::GetNextRobustIteration(RobustDictIterator* iter)
if ( iter->next < 0 )
iter->next = Next(-1);
ASSERT(iter->next >= Capacity() || ! table[iter->next].Empty());
if ( iter->next < Capacity() && table[iter->next].Empty() )
{
// [Robin] I believe this means that the table has resized in a way
// that we're now inside the overflow area where elements are empty,
// because elsewhere empty slots aren't allowed. Assuming that's right,
// then it means we'll always be at the end of the table now and could
// also just set `next` to capacity. However, just to be sure, we
// instead reuse logic from below to move forward "to a valid position"
// and then double check, through an assertion in debug mode, that it's
// actually the end. If this ever triggered, the above assumption would
// be wrong (but the Next() call would probably still be right).
iter->next = Next(iter->next);
ASSERT(iter->next == Capacity());
}
// Filter out visited keys.
int capacity = Capacity();

2
src/Dict.h
View file

@ -168,7 +168,9 @@ public:
DictIterator& operator=(DictIterator&& that);
reference operator*() { return *curr; }
reference operator*() const { return *curr; }
pointer operator->() { return curr; }
pointer operator->() const { return curr; }
DictIterator& operator++();
DictIterator operator++(int)

2
src/Event.cc
View file

@ -136,7 +136,7 @@ void EventMgr::Drain()
draining = true;
// Past Bro versions drained as long as there events, including when
// Past Zeek versions drained as long as there events, including when
// a handler queued new events during its execution. This could lead
// to endless loops in case a handler kept triggering its own event.
// We now limit this to just a couple of rounds. We do more than

2
src/EventHandler.h
View file

@ -20,7 +20,7 @@ class EventHandler
public:
explicit EventHandler(std::string name);
const char* Name() { return name.data(); }
const char* Name() const { return name.data(); }
const FuncPtr& GetFunc() { return local; }

2
src/EventRegistry.h
View file

@ -1,4 +1,4 @@
// Each event raised/handled by Bro is registered in the EventRegistry.
// Each event raised/handled by Zeek is registered in the EventRegistry.
#pragma once

1079
src/EventTrace.cc Normal file
View file

File diff suppressed because it is too large Load diff

464
src/EventTrace.h Normal file
View file

@ -0,0 +1,464 @@
// Classes for tracing/dumping Zeek events.
#pragma once
#include "zeek/Val.h"
namespace zeek::detail
{
class ValTrace;
class ValTraceMgr;
// Abstract class for capturing a single difference between two script-level
// values. Includes notions of inserting, changing, or deleting a value.
class ValDelta
{
public:
ValDelta(const ValTrace* _vt) : vt(_vt) { }
virtual ~ValDelta() { }
// Return a string that performs the update operation, expressed
// as Zeek scripting. Does not include a terminating semicolon.
virtual std::string Generate(ValTraceMgr* vtm) const;
// Whether the generated string needs the affected value to
// explicitly appear on the left-hand-side. Note that this
// might not be as a simple "LHS = RHS" assignment, but instead
// as "LHS$field = RHS" or "LHS[index] = RHS".
//
// Returns false for generated strings like "delete LHS[index]".
virtual bool NeedsLHS() const { return true; }
const ValTrace* GetValTrace() const { return vt; }
protected:
const ValTrace* vt;
};
using DeltaVector = std::vector<std::unique_ptr<ValDelta>>;
// Tracks the elements of a value as seen at a given point in execution.
// For non-aggregates, this is simply the Val object, but for aggregates
// it is (recursively) each of the sub-elements, in a manner that can then
// be readily compared against future instances.
class ValTrace
{
public:
ValTrace(const ValPtr& v);
~ValTrace();
const ValPtr& GetVal() const { return v; }
const TypePtr& GetType() const { return t; }
const auto& GetElems() const { return elems; }
// Returns true if this trace and the given one represent the
// same underlying value. Can involve subelement-by-subelement
// (recursive) comparisons.
bool operator==(const ValTrace& vt) const;
bool operator!=(const ValTrace& vt) const { return ! ((*this) == vt); }
// Computes the deltas between a previous ValTrace and this one.
// If "prev" is nil then we're creating this value from scratch
// (though if it's an aggregate, we may reuse existing values
// for some of its components).
//
// Returns the accumulated differences in "deltas". If on return
// nothing was added to "deltas" then the two ValTrace's are equivalent
// (no changes between them).
void ComputeDelta(const ValTrace* prev, DeltaVector& deltas) const;
private:
// Methods for tracing different types of aggregate values.
void TraceList(const ListValPtr& lv);
void TraceRecord(const RecordValPtr& rv);
void TraceTable(const TableValPtr& tv);
void TraceVector(const VectorValPtr& vv);
// Predicates for comparing different types of aggregates for equality.
bool SameList(const ValTrace& vt) const;
bool SameRecord(const ValTrace& vt) const;
bool SameTable(const ValTrace& vt) const;
bool SameVector(const ValTrace& vt) const;
// Helper function that knows about the internal vector-of-subelements
// we use for aggregates.
bool SameElems(const ValTrace& vt) const;
// True if this value is a singleton and it's the same value as
// represented in "vt".
bool SameSingleton(const ValTrace& vt) const;
// Add to "deltas" the differences needed to turn a previous instance
// of the given type of aggregate to the current instance.
void ComputeRecordDelta(const ValTrace* prev, DeltaVector& deltas) const;
void ComputeTableDelta(const ValTrace* prev, DeltaVector& deltas) const;
void ComputeVectorDelta(const ValTrace* prev, DeltaVector& deltas) const;
// Holds sub-elements for aggregates.
std::vector<std::shared_ptr<ValTrace>> elems;
// A parallel vector used for the yield values of tables.
std::vector<std::shared_ptr<ValTrace>> elems2;
ValPtr v;
TypePtr t; // v's type, for convenience
};
// Captures the basic notion of a new, non-equivalent value being assigned.
class DeltaReplaceValue : public ValDelta
{
public:
DeltaReplaceValue(const ValTrace* _vt, ValPtr _new_val)
: ValDelta(_vt), new_val(std::move(_new_val))
{
}
std::string Generate(ValTraceMgr* vtm) const override;
private:
ValPtr new_val;
};
// Captures the notion of setting a record field.
class DeltaSetField : public ValDelta
{
public:
DeltaSetField(const ValTrace* _vt, int _field, ValPtr _new_val)
: ValDelta(_vt), field(_field), new_val(std::move(_new_val))
{
}
std::string Generate(ValTraceMgr* vtm) const override;
private:
int field;
ValPtr new_val;
};
// Captures the notion of deleting a record field.
class DeltaRemoveField : public ValDelta
{
public:
DeltaRemoveField(const ValTrace* _vt, int _field) : ValDelta(_vt), field(_field) { }
std::string Generate(ValTraceMgr* vtm) const override;
bool NeedsLHS() const override { return false; }
private:
int field;
};
// Captures the notion of creating a record from scratch.
class DeltaRecordCreate : public ValDelta
{
public:
DeltaRecordCreate(const ValTrace* _vt) : ValDelta(_vt) { }
std::string Generate(ValTraceMgr* vtm) const override;
};
// Captures the notion of adding an element to a set. Use DeltaRemoveTableEntry to
// delete values.
class DeltaSetSetEntry : public ValDelta
{
public:
DeltaSetSetEntry(const ValTrace* _vt, ValPtr _index) : ValDelta(_vt), index(_index) { }
std::string Generate(ValTraceMgr* vtm) const override;
bool NeedsLHS() const override { return false; }
private:
ValPtr index;
};
// Captures the notion of setting a table entry (which includes both changing
// an existing one and adding a new one). Use DeltaRemoveTableEntry to
// delete values.
class DeltaSetTableEntry : public ValDelta
{
public:
DeltaSetTableEntry(const ValTrace* _vt, ValPtr _index, ValPtr _new_val)
: ValDelta(_vt), index(_index), new_val(std::move(_new_val))
{
}
std::string Generate(ValTraceMgr* vtm) const override;
private:
ValPtr index;
ValPtr new_val;
};
// Captures the notion of removing a table/set entry.
class DeltaRemoveTableEntry : public ValDelta
{
public:
DeltaRemoveTableEntry(const ValTrace* _vt, ValPtr _index)
: ValDelta(_vt), index(std::move(_index))
{
}
std::string Generate(ValTraceMgr* vtm) const override;
bool NeedsLHS() const override { return false; }
private:
ValPtr index;
};
// Captures the notion of creating a set from scratch.
class DeltaSetCreate : public ValDelta
{
public:
DeltaSetCreate(const ValTrace* _vt) : ValDelta(_vt) { }
std::string Generate(ValTraceMgr* vtm) const override;
};
// Captures the notion of creating a table from scratch.
class DeltaTableCreate : public ValDelta
{
public:
DeltaTableCreate(const ValTrace* _vt) : ValDelta(_vt) { }
std::string Generate(ValTraceMgr* vtm) const override;
};
// Captures the notion of changing an element of a vector.
class DeltaVectorSet : public ValDelta
{
public:
DeltaVectorSet(const ValTrace* _vt, int _index, ValPtr _elem)
: ValDelta(_vt), index(_index), elem(std::move(_elem))
{
}
std::string Generate(ValTraceMgr* vtm) const override;
private:
int index;
ValPtr elem;
};
// Captures the notion of adding an entry to the end of a vector.
class DeltaVectorAppend : public ValDelta
{
public:
DeltaVectorAppend(const ValTrace* _vt, int _index, ValPtr _elem)
: ValDelta(_vt), index(_index), elem(std::move(_elem))
{
}
std::string Generate(ValTraceMgr* vtm) const override;
private:
int index;
ValPtr elem;
};
// Captures the notion of replacing a vector wholesale.
class DeltaVectorCreate : public ValDelta
{
public:
DeltaVectorCreate(const ValTrace* _vt) : ValDelta(_vt) { }
std::string Generate(ValTraceMgr* vtm) const override;
private:
};
// Manages the changes to (or creation of) a variable used to represent
// a value.
class DeltaGen
{
public:
DeltaGen(ValPtr _val, std::string _rhs, bool _needs_lhs, bool _is_first_def)
: val(std::move(_val)), rhs(std::move(_rhs)), needs_lhs(_needs_lhs),
is_first_def(_is_first_def)
{
}
const ValPtr& GetVal() const { return val; }
const std::string& RHS() const { return rhs; }
bool NeedsLHS() const { return needs_lhs; }
bool IsFirstDef() const { return is_first_def; }
private:
ValPtr val;
// The expression to set the variable to.
std::string rhs;
// Whether that expression needs the variable explicitly provides
// on the lefthand side.
bool needs_lhs;
// Whether this is the first definition of the variable (in which
// case we also need to declare the variable).
bool is_first_def;
};
using DeltaGenVec = std::vector<DeltaGen>;
// Tracks a single event.
class EventTrace
{
public:
// Constructed in terms of the associated script function, "network
// time" when the event occurred, and the position of this event
// within all of those being traced.
EventTrace(const ScriptFunc* _ev, double _nt, int event_num);
// Sets a string representation of the arguments (values) being
// passed to the event.
void SetArgs(std::string _args) { args = std::move(_args); }
// Adds to the trace an update for the given value.
void AddDelta(ValPtr val, std::string rhs, bool needs_lhs, bool is_first_def)
{
auto& d = is_post ? post_deltas : deltas;
d.emplace_back(DeltaGen(val, rhs, needs_lhs, is_first_def));
}
// Initially we analyze events pre-execution. When this flag
// is set, we switch to instead analyzing post-execution. The
// difference allows us to annotate the output with "# from script"
// comments that flag changes created by script execution rather
// than event engine activity.
void SetDoingPost() { is_post = true; }
const char* GetName() const { return name.c_str(); }
// Generates an internal event handler that sets up the values
// associated with the traced event, followed by queueing the traced
// event, and then queueing the successor internal event handler,
// if any.
//
// "predecessor", if non-nil, gives the event that came just before
// this one (used for "# from script" annotations"). "successor",
// if not empty, gives the name of the successor internal event.
void Generate(FILE* f, ValTraceMgr& vtm, const EventTrace* predecessor,
std::string successor) const;
private:
// "dvec" is either just our deltas, or the "post_deltas" of our
// predecessor plus our deltas.
void Generate(FILE* f, ValTraceMgr& vtm, const DeltaGenVec& dvec, std::string successor,
int num_pre = 0) const;
const ScriptFunc* ev;
double nt;
bool is_post = false;
// The deltas needed to construct the values associated with this
// event prior to its execution.
DeltaGenVec deltas;
// The deltas capturing any changes to the original values as induced
// by executing its event handlers.
DeltaGenVec post_deltas;
// The event's name and a string representation of its arguments.
std::string name;
std::string args;
};
// Manages all of the events and associated values seen during the execution.
class ValTraceMgr
{
public:
// Invoked to trace a new event with the associated arguments.
void TraceEventValues(std::shared_ptr<EventTrace> et, const zeek::Args* args);
// Invoked when the current event finishes execution. The arguments
// are again provided, for convenience so we don't have to remember
// them from the previous method.
void FinishCurrentEvent(const zeek::Args* args);
// Returns the name of the script variable associated with the
// given value.
const std::string& ValName(const ValPtr& v);
const std::string& ValName(const ValTrace* vt) { return ValName(vt->GetVal()); }
// Returns true if the script variable associated with the given value
// needs to be global (because it's used across multiple events).
bool IsGlobal(const ValPtr& v) const { return globals.count(v.get()) > 0; }
private:
// Traces the given value, which we may-or-may-not have seen before.
void AddVal(ValPtr v);
// Creates a new value, associating a script variable with it.
void NewVal(ValPtr v);
// Called when the given value is used in an expression that sets
// or updates another value. This lets us track which values are
// used across multiple events, and thus need to be global.
void ValUsed(const ValPtr& v);
// Compares the two value traces to build up deltas capturing
// the difference between the previous one and the current one.
void AssessChange(const ValTrace* vt, const ValTrace* prev_vt);
// Create and track a script variable associated with the given value.
void TrackVar(const Val* vt);
// Maps values to their associated traces.
std::unordered_map<const Val*, std::shared_ptr<ValTrace>> val_map;
// Maps values to the "names" we associated with them. For simple
// values, the name is just a Zeek script constant. For aggregates,
// it's a dedicated script variable.
std::unordered_map<const Val*, std::string> val_names;
int num_vars = 0; // the number of dedicated script variables
// Tracks which values we've processed up through the preceding event.
// Any re-use we then see for the current event (via a ValUsed() call)
// then tells us that the value is used across events, and thus its
// associated script variable needs to be global.
std::unordered_set<const Val*> processed_vals;
// Tracks which values have associated script variables that need
// to be global.
std::unordered_set<const Val*> globals;
// The event we're currently tracing.
std::shared_ptr<EventTrace> curr_ev;
// Hang on to values we're tracking to make sure the pointers don't
// get reused when the main use of the value ends.
std::vector<ValPtr> vals;
};
// Manages tracing of all of the events seen during execution, including
// the final generation of the trace script.
class EventTraceMgr
{
public:
EventTraceMgr(const std::string& trace_file);
~EventTraceMgr();
// Called at the beginning of invoking an event's handlers.
void StartEvent(const ScriptFunc* ev, const zeek::Args* args);
// Called after finishing with invoking an event's handlers.
void EndEvent(const ScriptFunc* ev, const zeek::Args* args);
// Used to track events generated at script-level.
void ScriptEventQueued(const EventHandlerPtr& h);
private:
FILE* f = nullptr;
ValTraceMgr vtm;
// All of the events we've traced so far.
std::vector<std::shared_ptr<EventTrace>> events;
// The names of all of the script events that have been generated.
std::unordered_set<std::string> script_events;
};
// If non-nil then we're doing event tracing.
extern std::unique_ptr<EventTraceMgr> etm;
} // namespace zeek::detail

1128
src/Expr.cc
View file

File diff suppressed because it is too large Load diff

102
src/Expr.h
View file

@ -164,12 +164,6 @@ public:
// or nil if the expression's value isn't fixed.
virtual ValPtr Eval(Frame* f) const = 0;
// Same, but the context is that we are adding an element
// into the given aggregate of the given type. Note that
// return type is void since it's updating an existing
// value, rather than creating a new one.
virtual void EvalIntoAggregate(const TypePtr& t, ValPtr aggr, Frame* f) const;
// Assign to the given value, if appropriate.
virtual void Assign(Frame* f, ValPtr v);
@ -183,15 +177,8 @@ public:
// TypeDecl with a description of the element.
virtual bool IsRecordElement(TypeDecl* td) const;
// Returns a value corresponding to this expression interpreted
// as an initialization, or nil if the expression is inconsistent
// with the given type. If "aggr" is non-nil, then this expression
// is an element of the given aggregate, and it is added to it
// accordingly.
virtual ValPtr InitVal(const TypePtr& t, ValPtr aggr) const;
// True if the expression has no side effects, false otherwise.
virtual bool IsPure() const;
virtual bool IsPure() const { return true; }
// True if the expression is a constant, false otherwise.
bool IsConst() const { return tag == EXPR_CONST; }
@ -467,7 +454,6 @@ public:
ValPtr Eval(Frame* f) const override;
void Assign(Frame* f, ValPtr v) override;
ExprPtr MakeLvalue() override;
bool IsPure() const override;
TraversalCode Traverse(TraversalCallback* cb) const override;
@ -599,6 +585,9 @@ protected:
// Same for when the constants are sets.
virtual ValPtr SetFold(Val* v1, Val* v2) const;
// Same for when the constants are tables.
virtual ValPtr TableFold(Val* v1, Val* v2) const;
// Same for when the constants are addresses or subnets.
virtual ValPtr AddrFold(Val* v1, Val* v2) const;
virtual ValPtr SubNetFold(Val* v1, Val* v2) const;
@ -622,6 +611,20 @@ protected:
void ExprDescribe(ODesc* d) const override;
// Reports on if this BinaryExpr involves a scalar and aggregate
// type (vec, list, table, record).
bool IsScalarAggregateOp() const;
// Warns about deprecated scalar vector operations like
// `[1, 2, 3] == 1` or `["a", "b", "c"] + "a"`.
void CheckScalarAggOp() const;
// For assignment operations (=, +=, -=) checks for a valid
// expression-list on the RHS (op2), potentially transforming
// op2 in the process. Returns true if the list is present
// and type-checks correctly, false otherwise.
bool CheckForRHSList();
ExprPtr op1;
ExprPtr op2;
};
@ -646,7 +649,7 @@ public:
ValPtr Eval(Frame* f) const override;
ValPtr DoSingleEval(Frame* f, Val* v) const;
bool IsPure() const override;
bool IsPure() const override { return false; }
// Optimization-related:
ExprPtr Duplicate() override;
@ -749,21 +752,33 @@ public:
ValPtr Eval(Frame* f) const override;
// Optimization-related:
bool IsPure() const override { return false; }
ExprPtr Duplicate() override;
bool HasReducedOps(Reducer* c) const override { return false; }
bool WillTransform(Reducer* c) const override { return true; }
bool IsReduced(Reducer* c) const override;
ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override;
ExprPtr ReduceToSingleton(Reducer* c, StmtPtr& red_stmt) override;
private:
// Whether this operation is appending a single element to a vector.
bool is_vector_elem_append = false;
};
class RemoveFromExpr final : public BinaryExpr
{
public:
bool IsPure() const override { return false; }
RemoveFromExpr(ExprPtr op1, ExprPtr op2);
ValPtr Eval(Frame* f) const override;
// Optimization-related:
ExprPtr Duplicate() override;
bool HasReducedOps(Reducer* c) const override { return false; }
bool WillTransform(Reducer* c) const override { return true; }
bool IsReduced(Reducer* c) const override;
ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override;
ExprPtr ReduceToSingleton(Reducer* c, StmtPtr& red_stmt) override;
};
class SubExpr final : public BinaryExpr
@ -940,11 +955,9 @@ public:
const AttributesPtr& attrs = nullptr, bool type_check = true);
ValPtr Eval(Frame* f) const override;
void EvalIntoAggregate(const TypePtr& t, ValPtr aggr, Frame* f) const override;
TypePtr InitType() const override;
bool IsRecordElement(TypeDecl* td) const override;
ValPtr InitVal(const TypePtr& t, ValPtr aggr) const override;
bool IsPure() const override;
bool IsPure() const override { return false; }
// Optimization-related:
ExprPtr Duplicate() override;
@ -1153,14 +1166,13 @@ public:
// Optimization-related:
ExprPtr Duplicate() override;
ExprPtr Inline(Inliner* inl) override;
bool HasReducedOps(Reducer* c) const override;
ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override;
StmtPtr ReduceToSingletons(Reducer* c) override;
protected:
ValPtr InitVal(const TypePtr& t, ValPtr aggr) const override;
void ExprDescribe(ODesc* d) const override;
ListExprPtr op;
@ -1173,6 +1185,7 @@ public:
TableConstructorExpr(ListExprPtr constructor_list, std::unique_ptr<std::vector<AttrPtr>> attrs,
TypePtr arg_type = nullptr, AttributesPtr arg_attrs = nullptr);
void SetAttrs(AttributesPtr _attrs) { attrs = std::move(_attrs); }
const AttributesPtr& GetAttrs() const { return attrs; }
ValPtr Eval(Frame* f) const override;
@ -1185,8 +1198,6 @@ public:
StmtPtr ReduceToSingletons(Reducer* c) override;
protected:
ValPtr InitVal(const TypePtr& t, ValPtr aggr) const override;
void ExprDescribe(ODesc* d) const override;
AttributesPtr attrs;
@ -1198,6 +1209,7 @@ public:
SetConstructorExpr(ListExprPtr constructor_list, std::unique_ptr<std::vector<AttrPtr>> attrs,
TypePtr arg_type = nullptr, AttributesPtr arg_attrs = nullptr);
void SetAttrs(AttributesPtr _attrs) { attrs = std::move(_attrs); }
const AttributesPtr& GetAttrs() const { return attrs; }
ValPtr Eval(Frame* f) const override;
@ -1210,8 +1222,6 @@ public:
StmtPtr ReduceToSingletons(Reducer* c) override;
protected:
ValPtr InitVal(const TypePtr& t, ValPtr aggr) const override;
void ExprDescribe(ODesc* d) const override;
AttributesPtr attrs;
@ -1230,8 +1240,6 @@ public:
bool HasReducedOps(Reducer* c) const override;
protected:
ValPtr InitVal(const TypePtr& t, ValPtr aggr) const override;
void ExprDescribe(ODesc* d) const override;
};
@ -1250,12 +1258,10 @@ public:
// (in which case an error is reported).
bool PromoteTo(TypePtr t);
void EvalIntoAggregate(const TypePtr& t, ValPtr aggr, Frame* f) const override;
bool IsRecordElement(TypeDecl* td) const override;
// Optimization-related:
ExprPtr Duplicate() override;
bool WillTransform(Reducer* c) const override { return true; }
ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override;
@ -1292,7 +1298,6 @@ public:
const std::vector<int>& Map() const { return map; }
protected:
ValPtr InitVal(const TypePtr& t, ValPtr aggr) const override;
ValPtr Fold(Val* v) const override;
// For each super-record slot, gives subrecord slot with which to
@ -1305,7 +1310,7 @@ extern RecordValPtr coerce_to_record(RecordTypePtr rt, Val* v, const std::vector
class TableCoerceExpr final : public UnaryExpr
{
public:
TableCoerceExpr(ExprPtr op, TableTypePtr r);
TableCoerceExpr(ExprPtr op, TableTypePtr r, bool type_check = true);
~TableCoerceExpr() override;
// Optimization-related:
@ -1346,7 +1351,7 @@ class ScheduleExpr final : public Expr
public:
ScheduleExpr(ExprPtr when, EventExprPtr event);
bool IsPure() const override;
bool IsPure() const override { return false; }
ValPtr Eval(Frame* f) const override;
@ -1481,7 +1486,6 @@ public:
ValPtr Eval(Frame* f) const override;
TypePtr InitType() const override;
ValPtr InitVal(const TypePtr& t, ValPtr aggr) const override;
ExprPtr MakeLvalue() override;
void Assign(Frame* f, ValPtr v) override;
@ -1497,8 +1501,6 @@ public:
StmtPtr ReduceToSingletons(Reducer* c) override;
protected:
ValPtr AddSetInit(TypePtr t, ValPtr aggr) const;
void ExprDescribe(ODesc* d) const override;
ExprPList exprs;
@ -1616,10 +1618,12 @@ public:
AppendToExpr(ExprPtr op1, ExprPtr op2);
ValPtr Eval(Frame* f) const override;
ExprPtr Duplicate() override;
bool IsPure() const override { return false; }
bool IsReduced(Reducer* c) const override;
ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override;
ExprPtr Duplicate() override;
ExprPtr ReduceToSingleton(Reducer* c, StmtPtr& red_stmt) override;
};
// An internal class for reduced form.
@ -1633,6 +1637,7 @@ public:
ExprPtr Duplicate() override;
bool IsPure() const override { return false; }
bool IsReduced(Reducer* c) const override;
bool HasReducedOps(Reducer* c) const override;
ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override;
@ -1664,6 +1669,7 @@ public:
ExprPtr Duplicate() override;
bool IsPure() const override { return false; }
bool IsReduced(Reducer* c) const override;
bool HasReducedOps(Reducer* c) const override;
ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override;
@ -1763,7 +1769,14 @@ inline Val* Expr::ExprVal() const
}
// Decides whether to return an AssignExpr or a RecordAssignExpr.
ExprPtr get_assign_expr(ExprPtr op1, ExprPtr op2, bool is_init);
extern ExprPtr get_assign_expr(ExprPtr op1, ExprPtr op2, bool is_init);
// Takes a RHS constructor list and returns a version with any embedded
// indices within it (used to concisely represent multiple set/table entries)
// expanded.
//
// Second argument gives the type that the list will expand to, if known.
extern ListExprPtr expand_op(ListExprPtr op, const TypePtr& t);
/**
* Type-check the given expression(s) against the given type(s). Complain
@ -1784,7 +1797,7 @@ extern bool check_and_promote_exprs_to_type(ListExpr* elements, TypePtr type);
// Returns a ListExpr simplified down to a list a values, or nil
// if they couldn't all be reduced.
std::optional<std::vector<ValPtr>> eval_list(Frame* f, const ListExpr* l);
extern std::optional<std::vector<ValPtr>> eval_list(Frame* f, const ListExpr* l);
// Returns true if e1 is "greater" than e2 - here "greater" is just
// a heuristic, used with commutative operators to put them into
@ -1801,5 +1814,16 @@ inline bool is_vector(const ExprPtr& e)
return is_vector(e.get());
}
// True if the given Expr* has a list type
inline bool is_list(Expr* e)
{
return e->GetType()->Tag() == TYPE_LIST;
}
inline bool is_list(const ExprPtr& e)
{
return is_list(e.get());
}
} // namespace detail
} // namespace zeek

22
src/Func.cc
View file

@ -33,6 +33,7 @@
#include "zeek/Debug.h"
#include "zeek/Desc.h"
#include "zeek/Event.h"
#include "zeek/EventTrace.h"
#include "zeek/Expr.h"
#include "zeek/File.h"
#include "zeek/Frame.h"
@ -132,20 +133,6 @@ std::string render_call_stack()
return rval;
}
Func::Func()
{
unique_id = unique_ids.size();
unique_ids.push_back({NewRef{}, this});
}
Func::Func(Kind arg_kind) : kind(arg_kind)
{
unique_id = unique_ids.size();
unique_ids.push_back({NewRef{}, this});
}
Func::~Func() = default;
void Func::AddBody(detail::StmtPtr /* new_body */,
const std::vector<detail::IDPtr>& /* new_inits */, size_t /* new_frame_size */,
int /* priority */)
@ -237,7 +224,6 @@ void Func::CopyStateInto(Func* other) const
other->type = type;
other->name = name;
other->unique_id = unique_id;
}
void Func::CheckPluginResult(bool handled, const ValPtr& hook_result, FunctionFlavor flavor) const
@ -401,6 +387,9 @@ ValPtr ScriptFunc::Invoke(zeek::Args* args, Frame* parent) const
const CallExpr* call_expr = parent ? parent->GetCall() : nullptr;
call_stack.emplace_back(CallInfo{call_expr, this, *args});
if ( etm && Flavor() == FUNC_FLAVOR_EVENT )
etm->StartEvent(this, args);
if ( g_trace_state.DoTrace() )
{
ODesc d;
@ -481,6 +470,9 @@ ValPtr ScriptFunc::Invoke(zeek::Args* args, Frame* parent) const
result = val_mgr->True();
}
else if ( etm && Flavor() == FUNC_FLAVOR_EVENT )
etm->EndEvent(this, args);
// Warn if the function returns something, but we returned from
// the function without an explicit return, or without a value.
else if ( GetType()->Yield() && GetType()->Yield()->Tag() != TYPE_VOID &&

29
src/Func.h
View file

@ -18,16 +18,11 @@
#include "zeek/ZeekArgs.h"
#include "zeek/ZeekList.h"
namespace caf
{
template <class> class expected;
}
namespace broker
{
class data;
using vector = std::vector<data>;
using caf::expected;
template <class> class expected;
}
namespace zeek
@ -66,9 +61,7 @@ public:
BUILTIN_FUNC
};
explicit Func(Kind arg_kind);
~Func() override;
explicit Func(Kind arg_kind) : kind(arg_kind) { }
virtual bool IsPure() const = 0;
FunctionFlavor Flavor() const { return GetType()->Flavor(); }
@ -127,14 +120,8 @@ public:
virtual detail::TraversalCode Traverse(detail::TraversalCallback* cb) const;
uint32_t GetUniqueFuncID() const { return unique_id; }
static const FuncPtr& GetFuncPtrByID(uint32_t id)
{
return id >= unique_ids.size() ? Func::nil : unique_ids[id];
}
protected:
Func();
Func() = default;
// Copies this function's state into other.
void CopyStateInto(Func* other) const;
@ -144,11 +131,9 @@ protected:
std::vector<Body> bodies;
detail::ScopePtr scope;
Kind kind;
uint32_t unique_id;
Kind kind = SCRIPT_FUNC;
FuncTypePtr type;
std::string name;
static inline std::vector<FuncPtr> unique_ids;
};
namespace detail
@ -303,7 +288,7 @@ protected:
virtual void SetCaptures(Frame* f);
private:
size_t frame_size;
size_t frame_size = 0;
// List of the outer IDs used in the function.
IDPList outer_ids;
@ -374,8 +359,8 @@ struct function_ingredients
IDPtr id;
StmtPtr body;
std::vector<IDPtr> inits;
int frame_size;
int priority;
int frame_size = 0;
int priority = 0;
ScopePtr scope;
};

21
src/ID.cc
View file

@ -248,13 +248,9 @@ void ID::UpdateValAttrs()
if ( ! attrs )
return;
if ( val && val->GetType()->Tag() == TYPE_TABLE )
val->AsTableVal()->SetAttrs(attrs);
auto tag = GetType()->Tag();
if ( val && val->GetType()->Tag() == TYPE_FILE )
val->AsFile()->SetAttrs(attrs.get());
if ( GetType()->Tag() == TYPE_FUNC )
if ( tag == TYPE_FUNC )
{
const auto& attr = attrs->Find(ATTR_ERROR_HANDLER);
@ -262,7 +258,7 @@ void ID::UpdateValAttrs()
event_registry->SetErrorHandler(Name());
}
if ( GetType()->Tag() == TYPE_RECORD )
if ( tag == TYPE_RECORD )
{
const auto& attr = attrs->Find(ATTR_LOG);
@ -281,6 +277,17 @@ void ID::UpdateValAttrs()
}
}
}
if ( ! val )
return;
auto vtag = val->GetType()->Tag();
if ( vtag == TYPE_TABLE )
val->AsTableVal()->SetAttrs(attrs);
else if ( vtag == TYPE_FILE )
val->AsFile()->SetAttrs(attrs.get());
}
const AttrPtr& ID::GetAttr(AttrTag t) const

4
src/IPAddr.h
View file

@ -504,7 +504,6 @@ inline void IPAddr::ConvertToThreadingValue(threading::Value::addr_t* v) const
switch ( v->family )
{
case IPv4:
CopyIPv4(&v->in.in4);
return;
@ -512,9 +511,6 @@ inline void IPAddr::ConvertToThreadingValue(threading::Value::addr_t* v) const
case IPv6:
CopyIPv6(&v->in.in6);
return;
// Can't be reached.
abort();
}
}

4
src/NetVar.cc
View file

@ -189,8 +189,6 @@ int dpd_ignore_ports;
int check_for_unused_event_handlers;
double timer_mgr_inactivity_timeout;
int record_all_packets;
bro_uint_t bits_per_uid;
@ -345,8 +343,6 @@ void init_net_var()
dpd_match_only_beginning = id::find_val("dpd_match_only_beginning")->AsBool();
dpd_late_match_stop = id::find_val("dpd_late_match_stop")->AsBool();
dpd_ignore_ports = id::find_val("dpd_ignore_ports")->AsBool();
timer_mgr_inactivity_timeout = id::find_val("timer_mgr_inactivity_timeout")->AsInterval();
}
} // namespace zeek::detail

2
src/NetVar.h
View file

@ -90,8 +90,6 @@ extern int dpd_ignore_ports;
extern int check_for_unused_event_handlers;
extern double timer_mgr_inactivity_timeout;
extern int record_all_packets;
extern bro_uint_t bits_per_uid;

1
src/Obj.h
View file

@ -38,6 +38,7 @@ public:
#define YYLTYPE zeek::detail::yyltype
using yyltype = Location;
YYLTYPE GetCurrentLocation();
void SetCurrentLocation(YYLTYPE currloc);
// Used to mean "no location associated with this object".
inline constexpr Location no_location("<no location>", 0, 0, 0, 0);

59
src/OpaqueVal.cc
View file

@ -33,7 +33,7 @@ inline bool get_vector_idx(const V& v, unsigned int i, D* dst)
if ( i >= v.size() )
return false;
auto x = caf::get_if<S>(&v[i]);
auto x = broker::get_if<S>(&v[i]);
if ( ! x )
return false;
@ -81,12 +81,12 @@ broker::expected<broker::data> OpaqueVal::Serialize() const
OpaqueValPtr OpaqueVal::Unserialize(const broker::data& data)
{
auto v = caf::get_if<broker::vector>(&data);
auto v = broker::get_if<broker::vector>(&data);
if ( ! (v && v->size() == 2) )
return nullptr;
auto type = caf::get_if<std::string>(&(*v)[0]);
auto type = broker::get_if<std::string>(&(*v)[0]);
if ( ! type )
return nullptr;
@ -118,17 +118,17 @@ broker::expected<broker::data> OpaqueVal::SerializeType(const TypePtr& t)
TypePtr OpaqueVal::UnserializeType(const broker::data& data)
{
auto v = caf::get_if<broker::vector>(&data);
auto v = broker::get_if<broker::vector>(&data);
if ( ! (v && v->size() == 2) )
return nullptr;
auto by_name = caf::get_if<bool>(&(*v)[0]);
auto by_name = broker::get_if<bool>(&(*v)[0]);
if ( ! by_name )
return nullptr;
if ( *by_name )
{
auto name = caf::get_if<std::string>(&(*v)[1]);
auto name = broker::get_if<std::string>(&(*v)[1]);
if ( ! name )
return nullptr;
@ -142,7 +142,7 @@ TypePtr OpaqueVal::UnserializeType(const broker::data& data)
return id->GetType();
}
auto tag = caf::get_if<uint64_t>(&(*v)[1]);
auto tag = broker::get_if<uint64_t>(&(*v)[1]);
if ( ! tag )
return nullptr;
@ -215,7 +215,10 @@ HashVal::HashVal(OpaqueTypePtr t) : OpaqueVal(std::move(t))
valid = false;
}
MD5Val::MD5Val() : HashVal(md5_type) { }
MD5Val::MD5Val() : HashVal(md5_type)
{
memset(&ctx, 0, sizeof(ctx));
}
MD5Val::~MD5Val() { }
@ -295,11 +298,11 @@ broker::expected<broker::data> MD5Val::DoSerialize() const
bool MD5Val::DoUnserialize(const broker::data& data)
{
auto d = caf::get_if<broker::vector>(&data);
auto d = broker::get_if<broker::vector>(&data);
if ( ! d )
return false;
auto valid = caf::get_if<bool>(&(*d)[0]);
auto valid = broker::get_if<bool>(&(*d)[0]);
if ( ! valid )
return false;
@ -312,7 +315,7 @@ bool MD5Val::DoUnserialize(const broker::data& data)
if ( (*d).size() != 2 )
return false;
auto s = caf::get_if<std::string>(&(*d)[1]);
auto s = broker::get_if<std::string>(&(*d)[1]);
if ( ! s )
return false;
@ -324,7 +327,10 @@ bool MD5Val::DoUnserialize(const broker::data& data)
return true;
}
SHA1Val::SHA1Val() : HashVal(sha1_type) { }
SHA1Val::SHA1Val() : HashVal(sha1_type)
{
memset(&ctx, 0, sizeof(ctx));
}
SHA1Val::~SHA1Val() { }
@ -385,11 +391,11 @@ broker::expected<broker::data> SHA1Val::DoSerialize() const
bool SHA1Val::DoUnserialize(const broker::data& data)
{
auto d = caf::get_if<broker::vector>(&data);
auto d = broker::get_if<broker::vector>(&data);
if ( ! d )
return false;
auto valid = caf::get_if<bool>(&(*d)[0]);
auto valid = broker::get_if<bool>(&(*d)[0]);
if ( ! valid )
return false;
@ -402,7 +408,7 @@ bool SHA1Val::DoUnserialize(const broker::data& data)
if ( (*d).size() != 2 )
return false;
auto s = caf::get_if<std::string>(&(*d)[1]);
auto s = broker::get_if<std::string>(&(*d)[1]);
if ( ! s )
return false;
@ -414,7 +420,10 @@ bool SHA1Val::DoUnserialize(const broker::data& data)
return true;
}
SHA256Val::SHA256Val() : HashVal(sha256_type) { }
SHA256Val::SHA256Val() : HashVal(sha256_type)
{
memset(&ctx, 0, sizeof(ctx));
}
SHA256Val::~SHA256Val() { }
@ -475,11 +484,11 @@ broker::expected<broker::data> SHA256Val::DoSerialize() const
bool SHA256Val::DoUnserialize(const broker::data& data)
{
auto d = caf::get_if<broker::vector>(&data);
auto d = broker::get_if<broker::vector>(&data);
if ( ! d )
return false;
auto valid = caf::get_if<bool>(&(*d)[0]);
auto valid = broker::get_if<bool>(&(*d)[0]);
if ( ! valid )
return false;
@ -492,7 +501,7 @@ bool SHA256Val::DoUnserialize(const broker::data& data)
if ( (*d).size() != 2 )
return false;
auto s = caf::get_if<std::string>(&(*d)[1]);
auto s = broker::get_if<std::string>(&(*d)[1]);
if ( ! s )
return false;
@ -546,7 +555,7 @@ broker::expected<broker::data> EntropyVal::DoSerialize() const
bool EntropyVal::DoUnserialize(const broker::data& data)
{
auto d = caf::get_if<broker::vector>(&data);
auto d = broker::get_if<broker::vector>(&data);
if ( ! d )
return false;
@ -780,12 +789,12 @@ broker::expected<broker::data> BloomFilterVal::DoSerialize() const
bool BloomFilterVal::DoUnserialize(const broker::data& data)
{
auto v = caf::get_if<broker::vector>(&data);
auto v = broker::get_if<broker::vector>(&data);
if ( ! (v && v->size() == 2) )
return false;
auto no_type = caf::get_if<broker::none>(&(*v)[0]);
auto no_type = broker::get_if<broker::none>(&(*v)[0]);
if ( ! no_type )
{
auto t = UnserializeType((*v)[0]);
@ -874,12 +883,12 @@ broker::expected<broker::data> CardinalityVal::DoSerialize() const
bool CardinalityVal::DoUnserialize(const broker::data& data)
{
auto v = caf::get_if<broker::vector>(&data);
auto v = broker::get_if<broker::vector>(&data);
if ( ! (v && v->size() == 2) )
return false;
auto no_type = caf::get_if<broker::none>(&(*v)[0]);
auto no_type = broker::get_if<broker::none>(&(*v)[0]);
if ( ! no_type )
{
auto t = UnserializeType((*v)[0]);
@ -931,7 +940,7 @@ broker::expected<broker::data> ParaglobVal::DoSerialize() const
bool ParaglobVal::DoUnserialize(const broker::data& data)
{
auto d = caf::get_if<broker::vector>(&data);
auto d = broker::get_if<broker::vector>(&data);
if ( ! d )
return false;

22
src/Options.cc
View file

@ -114,6 +114,8 @@ void usage(const char* prog, int code)
#endif
fprintf(stderr, " -C|--no-checksums | ignore checksums\n");
fprintf(stderr, " -D|--deterministic | initialize random seeds to zero\n");
fprintf(stderr, " -E|--event-trace <file> | generate a replayable event trace to "
"the given file\n");
fprintf(stderr, " -F|--force-dns | force DNS\n");
fprintf(stderr, " -G|--load-seeds <file> | load seeds from given file\n");
fprintf(stderr, " -H|--save-seeds <file> | save seeds to given file\n");
@ -193,7 +195,8 @@ static void print_analysis_help()
fprintf(stderr, " no-ZAM-opt omit low-level ZAM optimization\n");
fprintf(stderr, " optimize-all optimize all scripts, even inlined ones\n");
fprintf(stderr, " optimize-AST optimize the (transformed) AST; implies xform\n");
fprintf(stderr, " profile-ZAM generate to stdout a ZAM execution profile\n");
fprintf(stderr,
" profile-ZAM generate to stdout a ZAM execution profile; implies -O ZAM\n");
fprintf(stderr, " report-recursive report on recursive functions and exit\n");
fprintf(stderr, " xform transform scripts to \"reduced\" form\n");
@ -248,7 +251,7 @@ static void set_analysis_option(const char* opt, Options& opts)
else if ( util::streq(opt, "optimize-AST") )
a_o.activate = a_o.optimize_AST = true;
else if ( util::streq(opt, "profile-ZAM") )
a_o.activate = a_o.profile_ZAM = true;
a_o.activate = a_o.gen_ZAM_code = a_o.profile_ZAM = true;
else if ( util::streq(opt, "report-C++") )
a_o.report_CPP = true;
else if ( util::streq(opt, "report-recursive") )
@ -379,6 +382,7 @@ Options parse_cmdline(int argc, char** argv)
{"no-checksums", no_argument, nullptr, 'C'},
{"force-dns", no_argument, nullptr, 'F'},
{"deterministic", no_argument, nullptr, 'D'},
{"event-trace", required_argument, nullptr, 'E'},
{"load-seeds", required_argument, nullptr, 'G'},
{"save-seeds", required_argument, nullptr, 'H'},
{"print-plugins", no_argument, nullptr, 'N'},
@ -399,7 +403,7 @@ Options parse_cmdline(int argc, char** argv)
{"mem-profile", no_argument, nullptr, 'M'},
#endif
{"pseudo-realtime", optional_argument, nullptr, 'E'},
{"pseudo-realtime", optional_argument, nullptr, '~'},
{"jobs", optional_argument, nullptr, 'j'},
{"test", no_argument, nullptr, '#'},
@ -407,7 +411,7 @@ Options parse_cmdline(int argc, char** argv)
};
char opts[256];
util::safe_strncpy(opts, "B:c:e:f:G:H:I:i:j::n:O:0:o:p:r:s:T:t:U:w:X:CDFMNPQSWabdhmuv",
util::safe_strncpy(opts, "B:c:E:e:f:G:H:I:i:j::n:O:0:o:p:r:s:T:t:U:w:X:CDFMNPQSWabdhmuv",
sizeof(opts));
int op;
@ -522,9 +526,7 @@ Options parse_cmdline(int argc, char** argv)
rval.deterministic_mode = true;
break;
case 'E':
rval.pseudo_realtime = 1.0;
if ( optarg )
rval.pseudo_realtime = atof(optarg);
rval.event_trace_file = optarg;
break;
case 'F':
if ( rval.dns_mode != detail::DNS_DEFAULT )
@ -585,6 +587,12 @@ Options parse_cmdline(int argc, char** argv)
break;
#endif
case '~':
rval.pseudo_realtime = 1.0;
if ( optarg )
rval.pseudo_realtime = atof(optarg);
break;
case '#':
fprintf(stderr, "ERROR: --test only allowed as first argument.\n");
usage(zargs[0], 1);

1
src/Options.h
View file

@ -73,6 +73,7 @@ struct Options
std::optional<std::string> process_status_file;
std::optional<std::string> zeekygen_config_file;
std::optional<std::string> unprocessed_output_file;
std::optional<std::string> event_trace_file;
std::set<std::string> plugins_to_load;
std::vector<std::string> scripts_to_load;

Some files were not shown because too many files have changed in this diff Show more