Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 14:48:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge remote-tracking branch 'origin/master' into topic/johanna/gh-859

Browse source
This commit is contained in:
Johanna Amann 2021-06-29 15:09:56 +01:00
commit e4b2fa50a9
571 changed files with 40145 additions and 11997 deletions
Download patch file Download diff file Expand all files Collapse all files

37
.cirrus.yml
View file

@ -83,7 +83,7 @@ env:
# This is the https endpoint host and port used for benchmarking. It's kept
# encrypted as a security measure to avoid leaking the host's information.
ZEEK_BENCHMARK_HOST: ENCRYPTED[62ecdc93e839800d754d09d9a9070e9cb9b209e7d7dd2472ba38648f786ff272d0e0ea71233d0910025f2c6f3771259c]
ZEEK_BENCHMARK_PORT: ENCRYPTED[fb34ae2d51bac798fc01da052f3772154e17bbe2c1c5615509e82935248e748053fda399a0caf909632b6272cebff9f4]
ZEEK_BENCHMARK_PORT: ENCRYPTED[b97fabf4d6bd5eef107c8469c5cb2c44e0107d89c220f43e7d1e7bdfb32dbdc2620855fee8e5a8d889458d5a6ac3e5c7]
# The repo token used for uploading data to Coveralls.io
ZEEK_COVERALLS_REPO_TOKEN: ENCRYPTED[7ffd1e041f848f02b62f5abc7fda8a5a8a1561fbb2b46d88cefb67c74408ddeef6ea6f3b279c7953ca14ae9b4d050e2d]
@ -91,6 +91,13 @@ env:
# Linux EOL timelines: https://linuxlifecycle.com/
# Fedora (~13 months): https://fedoraproject.org/wiki/Fedora_Release_Life_Cycle
fedora34_task:
container:
# Fedora 34 EOL: Around May 2022
dockerfile: ci/fedora-34/Dockerfile
<< : *RESOURCES_TEMPLATE
<< : *CI_TEMPLATE
fedora33_task:
container:
# Fedora 33 EOL: Around November 2022
@ -98,13 +105,6 @@ fedora33_task:
<< : *RESOURCES_TEMPLATE
<< : *CI_TEMPLATE
fedora32_task:
container:
# Fedora 32 EOL: Around May 2021
dockerfile: ci/fedora-32/Dockerfile
<< : *RESOURCES_TEMPLATE
<< : *CI_TEMPLATE
centosstream8_task:
container:
# Stream 8 support should be 5 years, so until 2024. but I cannot find a concrete timeline --cpk
@ -169,6 +169,13 @@ opensuse_leap_15_2_task:
<< : *RESOURCES_TEMPLATE
<< : *CI_TEMPLATE
opensuse_leap_15_3_task:
container:
# Opensuse Leap 15.3 EOL: TBD
dockerfile: ci/opensuse-leap-15.3/Dockerfile
<< : *RESOURCES_TEMPLATE
<< : *CI_TEMPLATE
ubuntu20_task:
container:
# Ubuntu 20.04 EOL: April 2025
@ -205,19 +212,29 @@ alpine_task:
# We aim to support both the current and previous macOS release.
macos_big_sur_task:
macos_instance:
image: big-sur-base
image: big-sur-xcode-12.5
prepare_script: ./ci/macos/prepare.sh
<< : *CI_TEMPLATE
<< : *MACOS_RESOURCES_TEMPLATE
macos_catalina_task:
macos_instance:
image: catalina-xcode-11.6
image: catalina-xcode
prepare_script: ./ci/macos/prepare.sh
<< : *CI_TEMPLATE
<< : *MACOS_RESOURCES_TEMPLATE
# FreeBSD EOL timelines: https://www.freebsd.org/security/security.html#sup
freebsd13_task:
freebsd_instance:
# FreeBSD 13 EOL: January 31, 2026
image_family: freebsd-13-0
cpu: 8
# Not allowed to request less than 8GB for an 8 CPU FreeBSD VM.
memory: 8GB
prepare_script: ./ci/freebsd/prepare.sh
<< : *CI_TEMPLATE
freebsd12_task:
freebsd_instance:
# FreeBSD 12 EOL: June 30, 2024

8
.gitignore vendored
View file

@ -1,4 +1,7 @@
build
# Ignore anything prefixed with build since people
# tend to name all of their build directories prefixed that way.
build*
tmp
*.gcov
@ -11,3 +14,6 @@ cmake-build-*
# ignore pyenv local settings
.python-version
# clangd
.cache

431
CHANGES
View file

@ -1,3 +1,434 @@
4.1.0-dev.818 | 2021-06-28 13:50:13 -0700
* GH-1216: Enable Mobile IPv6 support by default (Tim Wojtulewicz, Corelight)
This removes the ENABLE_MOBILE_IPV6 #define variable. It also marks the
--enable-mobile-ipv6 configure argument as deprecated.
4.1.0-dev.816 | 2021-06-28 11:08:29 -0700
* GH-572: Mark MemoryAllocation() and related methods deprecated (Tim Wojtulewicz, Corelight)
4.1.0-dev.814 | 2021-06-28 11:06:39 -0700
* Check for -1 return from FieldOffset() in Val::HasField()
Fixes Coverity 1457804 (Tim Wojtulewicz, Corelight)
4.1.0-dev.812 | 2021-06-28 11:02:46 -0700
* whoops overlooked the need to canonicalize filenames (Vern Paxson, Corelight)
* another set of tweaks per review comments (Vern Paxson, Corelight)
* addressed a number of code review comments (Vern Paxson, Corelight)
* baseline updates for merge (Vern Paxson, Corelight)
* Merge remote-tracking branch 'origin/master' into topic/vern/ZAM-prep (Vern Paxson, Corelight)
* support "any" coercions for "-O gen-C++" (Vern Paxson, Corelight)
* better descriptions for named record constructors (Vern Paxson, Corelight)
* test suite baseline updates for "-a opt" optimize-AST alternative (Vern Paxson, Corelight)
* test suite baseline updates for "-a xform" alternative / AST transformation (Vern Paxson, Corelight)
* error propagation fix for AST reduction (Vern Paxson, Corelight)
* updates to "-a inline" test suite alternative baseline (Vern Paxson, Corelight)
* updates for the main test suite baseline (Vern Paxson, Corelight)
* updates to test suite tests for compatibility with upcoming ZAM functionality (Vern Paxson, Corelight)
* "-O compile-all" option to specify compilation of inlined functions (Vern Paxson, Corelight)
* compile inlined functions if they're also used indirectly (Vern Paxson, Corelight)
* provide ZAM-generated code with low-level access to record fields (Vern Paxson, Corelight)
* fix for cloning records with fields of type "any" (Vern Paxson, Corelight)
* direct access for ZAM to VectorVal internal vector (Vern Paxson, Corelight)
* ZVal constructors, accessors & methods in support of ZAM (Vern Paxson, Corelight)
* switch ZVal representation of types from Type objects to TypeVal's (Vern Paxson, Corelight)
* revised error-reporting interface for ZVal's, to accommodate ZAM inner loop (Vern Paxson, Corelight)
* faster construction of records by factoring static decisions into RecordType's (Vern Paxson, Corelight)
* make "switch" internals accessible to ZAM; tidying of same (Vern Paxson, Corelight)
* factor out "cast" functionality to make available to lower-level ZAM access (Vern Paxson, Corelight)
* tidying for check_and_promote_expr (Vern Paxson, Corelight)
* employ explicit conversions to/from "any" and "vector of any" types (Vern Paxson, Corelight)
* more robust treatment of arithmetic coercions (Vern Paxson, Corelight)
* support for constructing VectorVal's directly from underlying ZVal vectors (Vern Paxson, Corelight)
* support for ensuring that a vector can be treated as having a homogeneous type (Vern Paxson, Corelight)
* factoring out of low-level vector indexing to make available to ZAM (Vern Paxson, Corelight)
* minor changes for more robust behavior in the face of errors (Vern Paxson, Corelight)
* gracefully deal with "eval" exceptions that occur during AST reduction (Vern Paxson, Corelight)
* directly construct records of known types, rather than requiring coercion (Vern Paxson, Corelight)
* fixes for treating WhileStmt's "loop_cond_pred_stmt" as a first-class citizen (Vern Paxson, Corelight)
* support for profiling function bodies w/o needing accompanying ScriptFunc object (Vern Paxson, Corelight)
* support for Frame's having call locations even if no associated CallExpr (Vern Paxson, Corelight)
* fix for AST optimization altering top-level body statement (Vern Paxson, Corelight)
* fix for analyzing variable usage inside of table initializers (Vern Paxson, Corelight)
* fix for inlining type-based switch statements (Vern Paxson, Corelight)
* fix for computing |size| of files and subnets (Vern Paxson, Corelight)
* fix for tracking the effects of += operations (Vern Paxson, Corelight)
* Expr method to invert the sense of a relational (Vern Paxson, Corelight)
* Trigger constructor (and factoring) to support lower-level constructions (Vern Paxson, Corelight)
* fixed / removed out-of-date comments, tidied check_and_promote_args() interface (Vern Paxson, Corelight)
* convert scopes to be managed using IntrusivePtr's (Vern Paxson, Corelight)
* various accessors used by ZAM compiler (Vern Paxson, Corelight)
* Update submodule(s) [nomail] (Tim Wojtulewicz, Corelight)
4.1.0-dev.764 | 2021-06-27 10:50:19 -0700
* Remove unnecessary -B arguments from Zeek invocations in testsuite
Now that Zeek no longer silently accepts -B when not compiled in debug
mode, these tests were failing. (Christian Kreibich, Corelight)
* Fix perftools-enabled build (Christian Kreibich, Corelight)
* Minor tweaks to single-character command line option handling
- Use of "-B" now triggers usage output and errors out when Zeek
wasn't built with debugging support.
- Always keep the perftools options (-m, -M) in the optparse string,
for consistency with other flags dependent on configuration. We
still fall through to usage and erroring out as before.
- Minor indentation fix. (Christian Kreibich, Corelight)
4.1.0-dev.760 | 2021-06-27 10:46:01 -0700
* Add btest test case (Luke Cesarz)
* Fix typo (Luke Cesarz)
* Fix segfault with incomplete connection
Add required HasField check before GetFieldAs call (Luke Cesarz)
4.1.0-dev.755 | 2021-06-23 13:53:54 -0700
* Call brew update-reset in ci/macos/prepare.sh
This fixes some issues with the Catalina builds when it prepares the image
and the base Cirrus image has old recipes for Homebrew. The VM then has
to build a bunch of packages it shouldn't need to. (Tim Wojtulewicz, Corelight)
* GH-1368: Use --osx-sysroot for macOS CI builds (Tim Wojtulewicz, Corelight)
4.1.0-dev.750 | 2021-06-21 16:14:03 -0700
* Remove the Stepping Stone analyzer
This commit removes the stepping stone analyzer. It has been deactivated
by default since at least Zeek 2.0, is dysfunctional in cluster settings
and has a bunch of other issued.
Relates to GH-1573 (Johanna Amann, Corelight)
4.1.0-dev.748 | 2021-06-21 15:41:29 -0700
* Drop Fedora 32 from CI, now past EOL (Christian Kreibich, Corelight)
* Remove a double-defined TLS cert from a Broker btest (Christian Kreibich, Corelight)
4.1.0-dev.744 | 2021-06-21 09:17:36 +0200
* Add a new field `email_dest` to NOTICEs, which defines where to
send email to. The email-related NOTICE actions fill this now, and
then emails will be sent to all recorded addresses at the end of
NOTICE processing. This makes email generation more consistent and
extensible. (Vlad Grigorescu)
* Add page and email administrator to mails processed by hostnames
extension. (Vlad Grigorescu)
4.1.0-dev.731 | 2021-06-17 10:40:58 +0100
* Change SSH version field to be `&optional`.
In version 3.3.0-dev.537 we added handling for SSH version 1.99 which
used a SSH version of 0 to indicate weird cases where no version could be
determined.
This patch is a fixup for that patch. We now use an `&optional` version value.
If no SSH version can be eixtracted the version will be unset; additionally a
`conn_weird` event will be raised. See GH-1590. (Benjamin Bannier, Corelight)
4.1.0-dev.727 | 2021-06-14 16:19:34 -0700
* Bump Highwayhash submodule to pull in fix for FreeBSD (Christian Kreibich, Corelight)
4.1.0-dev.725 | 2021-06-11 11:54:30 -0700
* Fixes for the builtin plugin functionality (Seth Hall, Corelight)
4.1.0-dev.722 | 2021-06-10 10:42:57 -0700
* Added --include-plugins configure argument (Seth Hall, Corelight)
4.1.0-dev.720 | 2021-06-10 11:29:19 +0100
* Introduce script-land option LogAscii::logdir that can be used to set
the logging directory. (Henrik Kramselund Jereminsen)
4.1.0-dev.715 | 2021-06-09 09:12:26 -0700
* Fix macOS Big Sur builds on Cirrus
- Upgrade the Big Sur VM to use the Xcode 12.5 version. This has a newer
version of brew installed on it that fixes an issue with an EOL package host
that finally shut down for good recently.
- Use 'brew upgrade' for openssl and cmake, since those are both present on the
base VM. This prevents 'brew install' from printing an error if the package
exists but is out of date. (Tim Wojtulewicz, Corelight)
4.1.0-dev.713 | 2021-06-08 13:54:28 -0700
* Add OpenSUSE Leap 15.3 to testing (Johanna Amann, Corelight)
4.1.0-dev.709 | 2021-06-07 09:41:28 +0200
* Improve assignment operators for IntrusivePtr. (Dominik Charousset, Corelight)
* Fix docs for `ProcStats`: `mem` is in bytes, not KB. (Arne Welzel, Corelight)
4.1.0-dev.704 | 2021-06-04 08:29:18 -0700
* Add deprecated headers for UDP and ICMP analyzers (Tim Wojtulewicz, Corelight)
* Fix handling of IP packets with bogus IP header lengths
Credit to OSS-Fuzz for discovery
https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=34711
(Link to details becomes public 30 days after patch release) (Tim Wojtulewicz, Corelight)
4.1.0-dev.700 | 2021-06-03 09:27:57 -0700
* Make update-traces fail when the curl invocation fails (Christian Kreibich, Corelight)
4.1.0-dev.697 | 2021-06-02 15:08:12 -0700
* Add FreeBSD 13 to CI (Christian Kreibich, Corelight)
* Add Fedora 34 to CI (Christian Kreibich, Corelight)
4.1.0-dev.693 | 2021-06-02 13:22:09 -0700
* Label session adapters in the output of zeek -NN (Tim Wojtulewicz, Corelight)
* Split session adapter code into separate files from the analyzers (Tim Wojtulewicz, Corelight)
* Move adapter-specific code back into the adapter (Tim Wojtulewicz, Corelight)
* Move ICMP counterpart methods outside of ICMPAnalyzer class
These were previously global methods in the old analyzer, and moving them
to be private members of ICMPAnalyzer broke the usage of them by at least
one external plugin. (Tim Wojtulewicz, Corelight)
* Remove obsolete Skipping()/SetSkip() from Connection (Tim Wojtulewicz, Corelight)
* Remove some code from IPBasedAnalyzer and children that was waiting for TCP to be implemented (Tim Wojtulewicz, Corelight)
* Move TCPStateStats object out of session_mgr (Tim Wojtulewicz, Corelight)
* Move analyzer-to-port mapping out of analyzer::Manager into packet analyzers (Tim Wojtulewicz, Corelight)
* Move packet parsing code out of adapter into analyzer (Tim Wojtulewicz, Corelight)
* Move old TCP analyzer into analyzer adapter in packet analysis tree (Tim Wojtulewicz, Corelight)
4.1.0-dev.681 | 2021-06-02 09:57:41 -0700
* Add some extra length checking when parsing mobile ipv6 packets
Credit to OSS-Fuzz for discovery
https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=34263
(Link to details becomes public 30 days after patch release) (Tim Wojtulewicz, Corelight)
4.1.0-dev.679 | 2021-06-01 19:11:40 -0700
* Replace toupper() usages in netbios decoding BIFs (Jon Siwek, Corelight)
This avoids potential for locale-dependent results of toupper() by
instead using a function that simply maps ASCII characters a-z to A-Z.
4.1.0-dev.676 | 2021-06-01 10:19:19 -0700
* Integrate review feedback (Dominik Charousset, Corelight)
* Sync new broker options, fix name inconsistencies (Dominik Charousset, Corelight)
* Integrate new Broker metric exporter parameters (Dominik Charousset, Corelight)
4.1.0-dev.671 | 2021-06-01 09:51:38 -0700
* Update detect-MHR.zeek (Chris C)
Update Virustotal URL to current
option match_sub_url = "https://www.virustotal.com/gui/search/%s"
4.1.0-dev.669 | 2021-06-01 09:39:30 -0700
* GH-839: Fix use of &optional sub-records within table/set indices (Jon Siwek, Corelight)
4.1.0-dev.666 | 2021-05-26 10:51:51 -0700
* Ensure SessionAdapter members are initialized (Tim Wojtulewicz, Corelight)
Fixes Coverity #1453273
4.1.0-dev.665 | 2021-05-26 08:07:26 +0200
* Extend the file analyzer API to set source manually. (Robin
Sommer, Corelight)
4.1.0-dev.661 | 2021-05-24 15:03:54 -0700
* Update Broker submodule for bump of embedded CAF to 0.18.3 (Jon Siwek, Corelight)
4.1.0-dev.660 | 2021-05-24 12:38:44 -0700
* Add type field to session::Key to help avoid collisions in map (Tim Wojtulewicz, Corelight)
* Move bad UDP checksum handling into adapter object (Tim Wojtulewicz, Corelight)
* Rename IPBasedTransportAnalyzer to SessionAdapter (Tim Wojtulewicz, Corelight)
This also also combines the old TransportLayerAnalyzer class into
SessionAdapter, and removes the old class. This requires naming changes
in a few places but no functionality changes.
* Move building session analyzer tree out of analyzer::Manager (Tim Wojtulewicz, Corelight)
* Rework the packet flow through the IP-based analyzers (Tim Wojtulewicz, Corelight)
* Add new UDP packet analyzer, remove old one (Tim Wojtulewicz, Corelight)
* Add new ICMP packet analyzer, remove old one (Tim Wojtulewicz, Corelight)
* Add base class for IP-based packet analyzers (Tim Wojtulewicz, Corelight)
* Move SessionManager::ParseIPPacket to IP analyzer's namespace (Tim Wojtulewicz, Corelight)
* Added skeletons for TCP/UDP/ICMP packet analysis plugins. (Tim Wojtulewicz, Corelight)
This includes integration into the IP plugin and calling of the sessions code from each plugin.
4.1.0-dev.646 | 2021-05-18 11:47:25 -0700
* Omit unneeded decimal points in modp_dtoa2() scientific notation output (Jon Siwek, Corelight)
For example, "1e-13" is now used instead of "1.e-13".
* GH-1244: Change modp_dtoa2() to use scientific notation for small values (Jon Siwek, Corelight)
This fixes problems where printing floating point numbers less than
10^-6 output as "0.0". Such numbers now use using scientific notation
and preserve the value's actual floating point representation.
4.1.0-dev.643 | 2021-05-17 11:57:58 -0700
* GH-1546: Make DictIterator() public, add copy/move operators (Tim Wojtulewicz, Corelight)
4.1.0-dev.641 | 2021-05-17 11:28:11 -0700
* GH-1558: Fix reading `vector of enum` types from config files (Jon Siwek, Corelight)
* GH-1555: Fix reading empty set[enum] values from config files (Jon Siwek, Corelight)
4.1.0-dev.638 | 2021-05-17 13:08:28 +0100
* Manual page updates (Henrik Kramselund Jereminsen)
4.1.0-dev.631 | 2021-05-11 09:26:37 -0700
* Add unit tests to ZeekString.cc (Tim Wojtulewicz)
4.1.0-dev.628 | 2021-05-10 12:44:25 -0700
* Add experimental support for translating Zeek scripts to equivalent C++ (Vern Paxson, Corelight)
The generated C++ can then be compiled directly into the `zeek` binary,
replacing use of the interpreter and producing better runtime performance.
See `src/script_opt/CPP/README.md` for a guide on how to use this feature.
* Add new "-a cpp" btest alternative (Vern Paxson, Corelight)
* Add VectorVal methods to leverage ZVal representation (Vern Paxson, Corelight)
* Fix backtrace BiF to avoid iterator invalidation & support compiled code (Vern Paxson, Corelight)
4.1.0-dev.593 | 2021-05-10 10:17:34 +0100
* Explain zeek-config options in help output (Christian Kreibich, Corelight)
* Sort variables at top of zeek-config alphabetically (Christian Kreibich, Corelight)
* Install Zeek's btest tooling with the distribution
This creates $PREFIX/share/btest in the install tree, with the
following folders:
- scripts/ for the canonifiers
- data/ for random.seed
- data/pcaps for the test pcaps
The pcaps can be skipped by configuring with --disable-btest-pcaps. (Christian Kreibich, Corelight)
4.1.0-dev.587 | 2021-05-05 14:05:51 +0000
* Merge remote-tracking branch 'origin/topic/timw/session-coverity'
* origin/topic/timw/session-coverity:
Minor cleanup in IPAddr.h
Fix a few Coverity warnings from the session manager work (Tim Wojtulewicz)
* Minor cleanup in IPAddr.h (Tim Wojtulewicz, Corelight)
* Fix a few Coverity warnings from the session manager work
- Be explicit about setting the copied flag in session::Key. Coverity seems
confused about when that flag is set if it gets set by default
initialization. This should fix 1452757 and 1452759.
- Explicitly copy the fields in ConnKey instead of using memcpy. Fixes
1452758. (Tim Wojtulewicz, Corelight)
4.1.0-dev.583 | 2021-05-03 18:21:33 -0700

42
CMakeLists.txt
View file

@ -490,7 +490,7 @@ endif()
# Tell the plugin code that we're building as part of the main tree.
set(ZEEK_PLUGIN_INTERNAL_BUILD true CACHE INTERNAL "" FORCE)
set(DEFAULT_ZEEKPATH .:${ZEEK_SCRIPT_INSTALL_PATH}:${ZEEK_SCRIPT_INSTALL_PATH}/policy:${ZEEK_SCRIPT_INSTALL_PATH}/site)
set(DEFAULT_ZEEKPATH .:${ZEEK_SCRIPT_INSTALL_PATH}:${ZEEK_SCRIPT_INSTALL_PATH}/policy:${ZEEK_SCRIPT_INSTALL_PATH}/site:${ZEEK_SCRIPT_INSTALL_PATH}/builtin-plugins)
if ( NOT BINARY_PACKAGING_MODE )
set(ZEEK_DIST ${CMAKE_SOURCE_DIR})
@ -539,6 +539,9 @@ if ( GooglePerftools_INCLUDE_DIR )
set(ZEEK_CONFIG_GooglePerftools_INCLUDE_DIR ${GooglePerftools_INCLUDE_DIR})
endif ()
set(ZEEK_CONFIG_BTEST_TOOLS_DIR ${ZEEK_ROOT_DIR}/share/btest)
install(DIRECTORY DESTINATION ${ZEEK_CONFIG_BTEST_TOOLS_DIR})
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/zeek-config.in
${CMAKE_CURRENT_BINARY_DIR}/zeek-config @ONLY)
install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/zeek-config DESTINATION bin)
@ -570,12 +573,41 @@ if ( INSTALL_ZKG )
DESTINATION ${ZEEK_ZKG_CONFIG_DIR} RENAME config)
endif ()
########################################################################
## Look for external plugins to build in
string(REPLACE "," " " _build_in_plugins "${ZEEK_INCLUDE_PLUGINS}")
separate_arguments(_build_in_plugins)
foreach(plugin_dir ${_build_in_plugins})
if ( NOT IS_ABSOLUTE "${plugin_dir}/CMakeLists.txt" )
message(FATAL_ERROR "Plugins to build in need to be defined with absolute path! ${plugin_dir}")
endif()
if ( NOT EXISTS "${plugin_dir}/CMakeLists.txt" )
message(FATAL_ERROR "No plugin found at ${plugin_dir}!")
endif()
get_filename_component(plugin_name ${plugin_dir} NAME)
# Create a list of plugin directories that will then be added in the src/CMakeLists.txt
list(APPEND BUILTIN_PLUGIN_LIST ${plugin_dir})
message(STATUS " Building in plugin: ${plugin_name} (${plugin_dir})")
if ( "${ZEEK_BUILTIN_PLUGINS}" STREQUAL "" )
set(ZEEK_BUILTIN_PLUGINS ${plugin_name})
else ()
set(ZEEK_BUILTIN_PLUGINS "${ZEEK_BUILTIN_PLUGINS}, ${plugin_name}")
endif ()
endforeach()
########################################################################
## Recurse on sub-directories
add_subdirectory(src)
add_subdirectory(scripts)
add_subdirectory(man)
add_subdirectory(testing)
include(CheckOptionalBuildSources)
@ -608,6 +640,12 @@ if (CMAKE_BUILD_TYPE)
string(TOUPPER ${CMAKE_BUILD_TYPE} BuildType)
endif ()
if ( INSTALL_BTEST_PCAPS )
set(_install_btest_tools_msg "all")
else ()
set(_install_btest_tools_msg "no pcaps")
endif ()
message(
"\n====================| Zeek Build Summary |===================="
"\n"
@ -619,6 +657,7 @@ message(
"\nZeek Script Path: ${ZEEK_SCRIPT_INSTALL_PATH}"
"\nDebug mode: ${ENABLE_DEBUG}"
"\nUnit tests: ${ENABLE_ZEEK_UNIT_TESTS}"
"\nBuiltin Plugins: ${ZEEK_BUILTIN_PLUGINS}"
"\n"
"\nCC: ${CMAKE_C_COMPILER}"
"\nCFLAGS: ${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BuildType}}"
@ -629,6 +668,7 @@ message(
"\nZeekControl: ${INSTALL_ZEEKCTL}"
"\nAux. Tools: ${INSTALL_AUX_TOOLS}"
"\nBTest: ${INSTALL_BTEST}"
"\nBTest tooling: ${_install_btest_tools_msg}"
"\nzkg: ${INSTALL_ZKG}"
"\n"
"\nlibmaxminddb: ${USE_GEOIP}"

43
NEWS
View file

@ -49,15 +49,53 @@ New Functionality
- A Telemetry API was added to assist in gathering arbitrary runtime metrics
and allows potential export to Prometheus.
- Experimental support for translating Zeek scripts to equivalent C++.
The generated C++ can then be compiled directly into the `zeek` binary,
replacing use of the interpreter and producing better runtime performance.
See `src/script_opt/CPP/README.md` for a guide on how to use this feature.
- Support for more generic session management. The NetSessions class has been
renamed to SessionMgr (with the old name marked deprecated). The new
class allows plugins to take advantage of session management similar to how
Connection objects were handled previously, but without the need to be based
on IP-based protocols.
- The ASCII writer gained a new option LogAscii::logdir, which can be used to
change the logging output directory.
- Added a ``--include-plugins`` argument to ``configure``. This argument
takes a semicolon separated list of paths containing plugins that will be
statically built into Zeek.
Changed Functionality
---------------------
- The default IP-based transport protocols (UDP, TCP, and ICMP) have been
moved to the packet analysis framework. This change allows us to move other
analyzers in the future that better align with the packet analysis framework
than they do with session analysis.
- The version field in ssh.log is now optional and will not be set if we cannot
determine the version that was negotiated by the client and server.
- Add a new field `email_dest` to NOTICEs, which defines where to
send email to. The email-related NOTICE actions fill this now, and
then emails will be sent to all recorded addresses at the end of
NOTICE processing. This makes email generation more consistent and
extensible.
- Add page and email administrator to mails processed by hostnames extension.
Removed Functionality
---------------------
- Support for the RocksDB Broker data store was previously broken and unusable,
so all code/options related to it are now removed.
- Support for the ENABLE_MOBILE_IPV6 compiler variable has been removed. Mobile
IPv6 is now enabled by default. The --enable-mobile-ipv6 returns a warning
that it will be removed in v5.1 and no longer has any effect.
Deprecated Functionality
------------------------
@ -79,6 +117,11 @@ Deprecated Functionality
- ``supervisor_rotation_format_func`` is renamed to ``archiver_rotation_format_func``
- The ```MemoryAllocation()``` function implemented by a number of interfaces
is now deprecated. In testing we found that the values returned were mostly
incorrect and weren't useful. The ```val_size``` and ```global_sizes``` BIF
methods have also both been marked deprecated.
Zeek 4.0.0
==========

2
VERSION
View file

@ -1 +1 @@
4.1.0-dev.583
4.1.0-dev.818

2
auxil/broker

@ -1 +1 @@
Subproject commit 6ba5a134b0a6c29997bda652ca3a8b5b4da7cd9a
Subproject commit 3c00df399e05d70b2aefdeb8fef6b47f08d0b289

2
auxil/btest

@ -1 +1 @@
Subproject commit 72b8e3a6a5863715cbbce873cd8e1bf5358086ab
Subproject commit 1be682d0744f201551ada8dc568820c5f91a049c

2
auxil/highwayhash

@ -1 +1 @@
Subproject commit bb8b31c7377e047049466246bb1d2c00db8bc0bc
Subproject commit ea06651bd11387f5aac694c819314536fe7df060

2
auxil/package-manager

@ -1 +1 @@
Subproject commit dedfdf443cebc27e4eeb121072578a9d4216ea7f
Subproject commit bb76430eafab98b93570348103ec0c8edfdba9c0

2
auxil/zeek-archiver

@ -1 +1 @@
Subproject commit 7fe80115aa159119febc623cec0660774bcae50c
Subproject commit d3e55991cbe69f37966207479492edd38d548b1d

2
auxil/zeekctl

@ -1 +1 @@
Subproject commit c383d515ef89bd8c4d9785ed23ef6e1c2aafe6ce
Subproject commit d31885671d74932d951778c029fa74d44cf3e542

6
ci/build.sh
View file

@ -3,6 +3,12 @@
set -e
set -x
# If we're on macOS, use --osx-sysroot to ensure we can find the SDKs from Xcode. This avoids
# some problems with Catalina specifically, but it doesn't break anything on Big Sur either.
if [ "${CIRRUS_OS}" == "darwin" ]; then
export ZEEK_CI_CONFIGURE_FLAGS="${ZEEK_CI_CONFIGURE_FLAGS} --osx-sysroot=$(xcrun --show-sdk-path)"
fi
if [ "${ZEEK_CI_CREATE_ARTIFACT}" != "1" ]; then
./configure ${ZEEK_CI_CONFIGURE_FLAGS}
cd build

2
ci/fedora-32/Dockerfile → ci/fedora-34/Dockerfile
View file

@ -1,4 +1,4 @@
FROM fedora:32
FROM fedora:34
RUN dnf -y install \
bison \

4
ci/macos/prepare.sh
View file

@ -5,4 +5,6 @@ sysctl hw.model hw.machine hw.ncpu hw.physicalcpu hw.logicalcpu
set -e
set -x
brew install cmake swig openssl bison
brew update-reset
brew upgrade cmake openssl
brew install swig bison

25
ci/opensuse-leap-15.3/Dockerfile Normal file
View file

@ -0,0 +1,25 @@
FROM opensuse/leap:15.3
RUN zypper in -y \
cmake \
make \
gcc \
gcc-c++ \
python3 \
python3-devel \
flex \
bison \
libpcap-devel \
libopenssl-devel \
zlib-devel \
swig \
git \
curl \
python3-pip \
which \
gzip \
tar \
&& rm -rf /var/cache/zypp
RUN pip3 install junit2html

2
cmake

@ -1 +1 @@
Subproject commit 74259745dea5ee4889d1ac1f4ebde4e2c59c329a
Subproject commit cce53d15008a26dcb1b7eb534a78f52f9355c676

21
configure vendored
View file

@ -28,6 +28,8 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
--toolchain=PATH path to a CMAKE_TOOLCHAIN_FILE
(useful for cross-compiling)
--sanitizers=LIST comma-separated list of sanitizer names to enable
--include-plugins=PATHS paths containing plugins to build directly into Zeek
(semicolon delimited and quoted when multiple)
Installation Directories:
--prefix=PREFIX installation directory [/usr/local/zeek]
@ -66,6 +68,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
--disable-auxtools don't build or install auxiliary tools
--disable-archiver don't build or install zeek-archiver tool
--disable-btest don't install BTest
--disable-btest-pcaps don't install Zeek's BTest input pcaps
--disable-python don't try to build python bindings for Broker
--disable-broker-tests don't try to build Broker unit tests
--disable-zkg don't install zkg
@ -162,12 +165,15 @@ append_cache_entry ENABLE_JEMALLOC BOOL false
append_cache_entry BUILD_SHARED_LIBS BOOL true
append_cache_entry INSTALL_AUX_TOOLS BOOL true
append_cache_entry INSTALL_BTEST BOOL true
append_cache_entry INSTALL_BTEST_PCAPS BOOL true
append_cache_entry INSTALL_ZEEK_ARCHIVER BOOL true
append_cache_entry INSTALL_ZEEKCTL BOOL true
append_cache_entry INSTALL_ZKG BOOL true
append_cache_entry CPACK_SOURCE_IGNORE_FILES STRING
append_cache_entry ENABLE_MOBILE_IPV6 BOOL false
append_cache_entry ZEEK_SANITIZERS STRING ""
append_cache_entry ZEEK_INCLUDE_PLUGINS STRING ""
has_enable_mobile_ipv6=0
# parse arguments
while [ $# -ne 0 ]; do
@ -206,6 +212,9 @@ while [ $# -ne 0 ]; do
--toolchain=*)
append_cache_entry CMAKE_TOOLCHAIN_FILE PATH $optarg
;;
--include-plugins=*)
append_cache_entry ZEEK_INCLUDE_PLUGINS STRING $optarg
;;
--prefix=*)
prefix=$optarg
append_cache_entry CMAKE_INSTALL_PREFIX PATH $optarg
@ -257,7 +266,7 @@ while [ $# -ne 0 ]; do
append_cache_entry ENABLE_DEBUG BOOL true
;;
--enable-mobile-ipv6)
append_cache_entry ENABLE_MOBILE_IPV6 BOOL true
has_enable_mobile_ipv6=1
;;
--enable-perftools)
append_cache_entry ENABLE_PERFTOOLS BOOL true
@ -293,6 +302,9 @@ while [ $# -ne 0 ]; do
--disable-btest)
append_cache_entry INSTALL_BTEST BOOL false
;;
--disable-btest-pcaps)
append_cache_entry INSTALL_BTEST_PCAPS BOOL false
;;
--disable-python)
append_cache_entry DISABLE_PYTHON_BINDINGS BOOL true
;;
@ -429,3 +441,8 @@ fi
echo "# This is the command used to configure this build" > config.status
echo $command >> config.status
chmod u+x config.status
if [ $has_enable_mobile_ipv6 -eq 1 ]; then
echo
echo "NOTE: The --enable-mobile-ipv6 argument no longer has any effect and will be removed in v5.1."
fi

2
doc

@ -1 +1 @@
Subproject commit e9f80f75548a12cfbc711a4c69d3135e0ad105e3
Subproject commit c6a93225ed2c5644d9caa167e4e36cab26c5d10b

27
man/zeek.8
View file

@ -16,6 +16,8 @@ tasks, including detecting malware by interfacing to external registries,
reporting vulnerable versions of software seen on the network, identifying
popular web applications, detecting SSH brute-forcing, validating SSL
certificate chains, among others.
You must have the necessary permissions to access to the files or interfaces specified.
.SH OPTIONS
.TP
.B <file>
@ -148,6 +150,31 @@ Output file for script execution statistics
.TP
.B ZEEK_DISABLE_ZEEKYGEN
Disable Zeekygen (Broxygen) documentation support
.SH OUTPUT FORMAT
Output is written in multiple files depending on configuration. The default
location is the current directory.
The output written by Zeek can be formatted in multiple ways using the
logging framework.
.PP
The default are files in human-readable (ASCII) format. The data is organized
into columns (tab-delimited). The data can be processed using, e.g., the \fBzeek-cut\fR tool.
.SH EXAMPLES
Read a capture file and generate the default logs:
.br
# zeek -r test-capture.pcap
.PP
When running on live traffic, Zeek is usually started by running \fBzeekctl\fR. To configure
Zeek with an initial configuration, install, and restart:
.br
# zeekctl deploy
Note: the zeekctl configuration may need to be updated before first use. Especially the
network interface used should be the correct one.
.SH SEE ALSO
zeekctl(8) zeek-cut(1)
.SH AUTHOR
.B zeek
was written by The Zeek Project <info@zeek.org>.

1
scripts/base/frameworks/analyzer/main.zeek
View file

@ -124,7 +124,6 @@ export {
## A set of analyzers to disable by default at startup. The default set
## contains legacy analyzers that are no longer supported.
global disabled_analyzers: set[Analyzer::Tag] = {
ANALYZER_STEPPINGSTONE,
ANALYZER_TCPSTATS,
} &redef;
}

75
scripts/base/frameworks/broker/main.zeek
View file

@ -122,6 +122,37 @@ export {
## done reading the pcap.
option peer_counts_as_iosource = T;
## Port for Broker's metric exporter. Setting this to a valid TCP port causes
## Broker to make metrics available to Prometheus scrapers via HTTP. Zeek
## overrides any value provided in zeek_init or earlier at startup if the
## environment variable BROKER_METRICS_PORT is defined.
const metrics_port = 0/unknown &redef;
## Frequency for publishing scraped metrics to the target topic. Zeek
## overrides any value provided in zeek_init or earlier at startup if the
## environment variable BROKER_METRICS_EXPORT_INTERVAL is defined.
option metrics_export_interval = 1 sec;
## Target topic for the metrics. Setting a non-empty string starts the
## periodic publishing of local metrics. Zeek overrides any value provided in
## zeek_init or earlier at startup if the environment variable
## BROKER_METRICS_EXPORT_TOPIC is defined.
option metrics_export_topic = "";
## ID for the metrics exporter. When setting a target topic for the
## exporter, Broker sets this option to the suffix of the new topic *unless*
## the ID is a non-empty string. Since setting a topic starts the periodic
## publishing of events, we recommend setting the ID always first or avoid
## setting it at all if the topic suffix serves as a good-enough ID. Zeek
## overrides any value provided in zeek_init or earlier at startup if the
## environment variable BROKER_METRICS_ENDPOINT_NAME is defined.
option metrics_export_endpoint_name = "";
## Selects prefixes from the local metrics. Only metrics with prefixes
## listed in this variable are included when publishing local metrics.
## Setting an empty vector selects *all* metrics.
option metrics_export_prefixes: vector of string = vector();
## The default topic prefix where logs will be published. The log's stream
## id is appended when writing to a particular stream.
const default_log_topic_prefix = "zeek/logs/" &redef;
@ -385,9 +416,53 @@ event Broker::log_flush() &priority=10
schedule Broker::log_batch_interval { Broker::log_flush() };
}
function update_metrics_export_interval(id: string, val: interval): interval
{
Broker::__set_metrics_export_interval(val);
return val;
}
function update_metrics_export_topic(id: string, val: string): string
{
Broker::__set_metrics_export_topic(val);
return val;
}
function update_metrics_export_endpoint_name(id: string, val: string): string
{
Broker::__set_metrics_export_endpoint_name(val);
return val;
}
function update_metrics_export_prefixes(id: string, filter: vector of string): vector of string
{
Broker::__set_metrics_export_prefixes(filter);
return filter;
}
event zeek_init()
{
schedule Broker::log_batch_interval { Broker::log_flush() };
# interval
update_metrics_export_interval("Broker::metrics_export_interval",
Broker::metrics_export_interval);
Option::set_change_handler("Broker::metrics_export_interval",
update_metrics_export_interval);
# topic
update_metrics_export_topic("Broker::metrics_export_topic",
Broker::metrics_export_topic);
Option::set_change_handler("Broker::metrics_export_topic",
update_metrics_export_topic);
# endpoint name
update_metrics_export_endpoint_name("Broker::metrics_export_endpoint_name",
Broker::metrics_export_endpoint_name);
Option::set_change_handler("Broker::metrics_export_endpoint_name",
update_metrics_export_endpoint_name);
# prefixes
update_metrics_export_prefixes("Broker::metrics_export_prefixes",
Broker::metrics_export_prefixes);
Option::set_change_handler("Broker::metrics_export_prefixes",
update_metrics_export_prefixes);
}
event retry_listen(a: string, p: port, retry: interval)

5
scripts/base/frameworks/logging/writers/ascii.zeek
View file

@ -54,6 +54,11 @@ export {
## This option is also available as a per-filter ``$config`` option.
const gzip_file_extension = "gz" &redef;
## Define the default logging directory. If empty, logs are written
## to the current working directory.
##
const logdir = "" &redef;
## Format of timestamps when writing out JSON. By default, the JSON
## formatter will use double values for timestamps which represent the
## number of seconds from the UNIX epoch.

9
scripts/base/frameworks/notice/actions/email_admin.zeek
View file

@ -18,18 +18,15 @@ export {
};
}
hook notice(n: Notice::Info) &priority=-5
hook notice(n: Notice::Info)
{
if ( |Site::local_admins| > 0 &&
ACTION_EMAIL_ADMIN in n$actions )
{
local email = "";
if ( n?$src && |Site::get_emails(n$src)| > 0 )
email = fmt("%s, %s", email, Site::get_emails(n$src));
add n$email_dest[Site::get_emails(n$src)];
if ( n?$dst && |Site::get_emails(n$dst)| > 0 )
email = fmt("%s, %s", email, Site::get_emails(n$dst));
if ( email != "" )
email_notice_to(n, email, T);
add n$email_dest[Site::get_emails(n$dst)];
}
}

4
scripts/base/frameworks/notice/actions/page.zeek
View file

@ -17,8 +17,8 @@ export {
option mail_page_dest = "";
}
hook notice(n: Notice::Info) &priority=-5
hook notice(n: Notice::Info)
{
if ( ACTION_PAGE in n$actions )
email_notice_to(n, mail_page_dest, F);
add n$email_dest[mail_page_dest];
}

14
scripts/base/frameworks/notice/main.zeek
View file

@ -136,6 +136,9 @@ export {
## The actions which have been applied to this notice.
actions: ActionSet &log &default=ActionSet();
## The email address(es) where to send this notice
email_dest: set[string] &log &default=set();
## By adding chunks of text into this element, other scripts
## can expand on notices that are being emailed. The normal
## way to add text is to extend the vector by handling the
@ -510,10 +513,17 @@ hook Notice::policy(n: Notice::Info) &priority=10
add n$actions[ACTION_LOG];
}
hook Notice::notice(n: Notice::Info) &priority=-5
hook Notice::notice(n: Notice::Info)
{
if ( ACTION_EMAIL in n$actions )
email_notice_to(n, mail_dest, T);
add n$email_dest[mail_dest];
}
hook Notice::notice(n: Notice::Info) &priority=-5
{
for ( dest in n$email_dest )
email_notice_to(n, dest, T);
if ( ACTION_LOG in n$actions )
Log::write(Notice::LOG, n);
if ( ACTION_ALARM in n$actions )

12
scripts/base/init-bare.zeek
View file

@ -635,7 +635,7 @@ type ProcStats: record {
real_time: interval; ##< Elapsed real time since Zeek started running.
user_time: interval; ##< User CPU seconds.
system_time: interval; ##< System CPU seconds.
mem: count; ##< Maximum memory consumed, in KB.
mem: count; ##< Maximum memory consumed, in bytes.
minor_faults: count; ##< Page faults not requiring actual I/O.
major_faults: count; ##< Page faults requiring actual I/O.
num_swap: count; ##< Times swapped out.
@ -1933,6 +1933,7 @@ type gtp_delete_pdp_ctx_response_elements: record {
@load base/frameworks/supervisor/api
@load base/bif/supervisor.bif
@load base/bif/packet_analysis.bif
@load base/bif/CPP-load.bif
## Internal function.
function add_interface(iold: string, inew: string): string
@ -4117,15 +4118,6 @@ type PE::SectionHeader: record {
}
module GLOBAL;
## Internal to the stepping stone detector.
const stp_delta: interval &redef;
## Internal to the stepping stone detector.
const stp_idle_min: interval &redef;
## Internal to the stepping stone detector.
global stp_skip_src: set[addr] &redef;
## Description of a signature match.
##
## .. zeek:see:: signature_match

3
scripts/base/packet-protocols/__load__.zeek
View file

@ -15,3 +15,6 @@
@load base/packet-protocols/gre
@load base/packet-protocols/iptunnel
@load base/packet-protocols/vntag
@load base/packet-protocols/udp
@load base/packet-protocols/tcp
@load base/packet-protocols/icmp

1
scripts/base/packet-protocols/icmp/__load__.zeek Normal file
View file

@ -0,0 +1 @@
@load ./main

5
scripts/base/packet-protocols/icmp/main.zeek Normal file
View file

@ -0,0 +1,5 @@
module PacketAnalyzer::ICMP;
#event zeek_init() &priority=20
# {
# }

20
scripts/base/packet-protocols/ip/main.zeek
View file

@ -1,8 +1,22 @@
module PacketAnalyzer::IP;
const IPPROTO_TCP : count = 6;
const IPPROTO_UDP : count = 17;
const IPPROTO_ICMP : count = 1;
const IPPROTO_ICMP6 : count = 58;
const IPPROTO_IPIP : count = 4;
const IPPROTO_IPV6 : count = 41;
const IPPROTO_GRE : count = 47;
event zeek_init() &priority=20
{
PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, 4, PacketAnalyzer::ANALYZER_IPTUNNEL);
PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, 41, PacketAnalyzer::ANALYZER_IPTUNNEL);
PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, 47, PacketAnalyzer::ANALYZER_GRE);
PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_IPIP, PacketAnalyzer::ANALYZER_IPTUNNEL);
PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_IPV6, PacketAnalyzer::ANALYZER_IPTUNNEL);
PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_GRE, PacketAnalyzer::ANALYZER_GRE);
PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_TCP, PacketAnalyzer::ANALYZER_TCP);
PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_UDP, PacketAnalyzer::ANALYZER_UDP);
PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_ICMP, PacketAnalyzer::ANALYZER_ICMP);
PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_ICMP6, PacketAnalyzer::ANALYZER_ICMP);
}

1
scripts/base/packet-protocols/tcp/__load__.zeek Normal file
View file

@ -0,0 +1 @@
@load ./main

5
scripts/base/packet-protocols/tcp/main.zeek Normal file
View file

@ -0,0 +1,5 @@
module PacketAnalyzer::TCP;
#event zeek_init() &priority=20
# {
# }

1
scripts/base/packet-protocols/udp/__load__.zeek Normal file
View file

@ -0,0 +1 @@
@load ./main

5
scripts/base/packet-protocols/udp/main.zeek Normal file
View file

@ -0,0 +1,5 @@
module PacketAnalyzer::UDP;
#event zeek_init() &priority=20
# {
# }

37
scripts/base/protocols/ssh/main.zeek
View file

@ -20,8 +20,12 @@ export {
uid: string &log;
## The connection's 4-tuple of endpoint addresses/ports.
id: conn_id &log;
## SSH major version (1 or 2)
version: count &log;
## SSH major version (1, 2, or unset). The version can be unset if the
## client and server version strings are unset, malformed or incompatible
## so no common version can be extracted. If no version can be extracted
## even though both client and server versions are set a weird
## will be generated.
version: count &log &optional;
## Authentication result (T=success, F=failure, unset=unknown)
auth_success: bool &log &optional;
## The number of authentication attemps we observed. There's always
@ -155,9 +159,20 @@ function set_session(c: connection)
}
}
function set_version(c: connection, version: string)
function set_version(c: connection)
{
if ( c$ssh?$server && c$ssh?$client && |c$ssh$client| > 4 && |c$ssh$server| > 4 )
# We always either set the version field to a concrete value, or unset it.
delete c$ssh$version;
# If either the client or server string is unset we cannot compute a
# version and return early. We do not raise a weird in this case as we
# might arrive here while having only seen one side of the handshake.
const has_server = c$ssh?$server && |c$ssh$server| > 0;
const has_client = c$ssh?$client && |c$ssh$client| > 0;
if ( ! ( has_server && has_client ) )
return;
if ( |c$ssh$client| > 4 && |c$ssh$server| > 4 )
{
if ( c$ssh$client[4] == "1" && c$ssh$server[4] == "2" )
{
@ -166,7 +181,8 @@ function set_version(c: connection, version: string)
c$ssh$version = 2;
# SSH1 vs SSH2 -> Undefined
else
c$ssh$version = 0;
Reporter::conn_weird("SSH_version_mismatch", c, fmt("%s vs %s", c$ssh$server, c$ssh$client));
return;
}
else if ( c$ssh$client[4] == "2" && c$ssh$server[4] == "1" )
{
@ -175,7 +191,8 @@ function set_version(c: connection, version: string)
c$ssh$version = 2;
else
# SSH2 vs SSH1 -> Undefined
c$ssh$version = 0;
Reporter::conn_weird("SSH_version_mismatch", c, fmt("%s vs %s", c$ssh$server, c$ssh$client));
return;
}
else if ( c$ssh$client[4] == "1" && c$ssh$server[4] == "1" )
{
@ -199,21 +216,25 @@ function set_version(c: connection, version: string)
{
c$ssh$version = 2;
}
return;
}
Reporter::conn_weird("SSH_cannot_determine_version", c, fmt("%s vs %s", c$ssh$server, c$ssh$client));
}
event ssh_server_version(c: connection, version: string)
{
set_session(c);
c$ssh$server = version;
set_version(c, version);
set_version(c);
}
event ssh_client_version(c: connection, version: string)
{
set_session(c);
c$ssh$client = version;
set_version(c, version);
set_version(c);
}
event ssh_auth_attempted(c: connection, authenticated: bool) &priority=5

4
scripts/base/protocols/ssl/main.zeek
View file

@ -141,8 +141,8 @@ export {
## record as it is sent on to the logging framework.
global log_ssl: event(rec: Info);
# Hook that can be used to perform actions right before the log record
# is written.
## Hook that can be used to perform actions right before the log record
## is written.
global ssl_finishing: hook(c: connection);
## SSL finalization hook. Remaining SSL info may get logged when it's called.

2
scripts/policy/frameworks/files/detect-MHR.zeek
View file

@ -26,7 +26,7 @@ export {
## The Match notice has a sub message with a URL where you can get more
## information about the file. The %s will be replaced with the SHA-1
## hash of the file.
option match_sub_url = "https://www.virustotal.com/en/search/?query=%s";
option match_sub_url = "https://www.virustotal.com/gui/search/%s";
## The malware hash registry runs each malware sample through several
## A/V engines. Team Cymru returns a percentage to indicate how

5
scripts/policy/frameworks/notice/extend-email/hostnames.zeek
View file

@ -13,13 +13,14 @@ module Notice;
# reference to the original notice)
global tmp_notice_storage: table[string] of Notice::Info &create_expire=max_email_delay+10secs;
hook notice(n: Notice::Info) &priority=10
# Run after e-mail address is set, but before e-mail is sent.
hook notice(n: Notice::Info) &priority=-1
{
if ( ! n?$src && ! n?$dst )
return;
# This should only be done for notices that are being sent to email.
if ( ACTION_EMAIL !in n$actions )
if ( ! n?$email_dest )
return;
# I'm not recovering gracefully from the when statements because I want

6
src/Attr.cc
View file

@ -359,7 +359,7 @@ void Attributes::CheckAttr(Attr* a)
// Ok.
break;
auto e = check_and_promote_expr(a->GetExpr().get(), type.get());
auto e = check_and_promote_expr(a->GetExpr(), type);
if ( e )
{
@ -399,7 +399,7 @@ void Attributes::CheckAttr(Attr* a)
// Ok.
break;
auto e = check_and_promote_expr(a->GetExpr().get(), ytype.get());
auto e = check_and_promote_expr(a->GetExpr(), ytype);
if ( e )
{
@ -425,7 +425,7 @@ void Attributes::CheckAttr(Attr* a)
if ( (atype->Tag() == TYPE_TABLE && atype->AsTableType()->IsUnspecifiedTable()) )
{
auto e = check_and_promote_expr(a->GetExpr().get(), type.get());
auto e = check_and_promote_expr(a->GetExpr(), type);
if ( e )
{

1
src/CCL.h
View file

@ -26,6 +26,7 @@ public:
void ReplaceSyms(int_list* new_syms)
{ delete syms; syms = new_syms; }
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
unsigned int MemoryAllocation() const;
protected:

54
src/CMakeLists.txt
View file

@ -124,6 +124,9 @@ set(BIF_SRCS
# it's needed before parsing the packet protocol scripts, which happen
# very near to the start of parsing.
packet_analysis/packet_analysis.bif
# The C++ loading BIF is treated like other top-level BIFs to give
# us flexibility regarding when it's called.
script_opt/CPP/CPP-load.bif
)
foreach (bift ${BIF_SRCS})
@ -170,6 +173,32 @@ add_subdirectory(session)
add_subdirectory(fuzzers)
########################################################################
## Build in the discovered external plugins and create the autogenerated scripts.
set(PRELOAD_SCRIPT ${CMAKE_BINARY_DIR}/scripts/builtin-plugins/__preload__.zeek)
file(WRITE ${PRELOAD_SCRIPT} "# Warning, this is an autogenerated file!\n")
set(LOAD_SCRIPT ${CMAKE_BINARY_DIR}/scripts/builtin-plugins/__load__.zeek)
file(WRITE ${LOAD_SCRIPT} "# Warning, this is an autogenerated file!\n")
foreach (plugin_dir ${BUILTIN_PLUGIN_LIST})
get_filename_component(plugin_name ${plugin_dir} NAME)
if(IS_DIRECTORY "${plugin_dir}/cmake")
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${plugin_dir}/cmake")
endif()
# Setup the include path for built source artifacts.
include_directories(AFTER
${plugin_dir}/src
${CMAKE_CURRENT_BINARY_DIR}/builtin-plugins/${plugin_name})
add_subdirectory(${plugin_dir} ${CMAKE_CURRENT_BINARY_DIR}/builtin-plugins/${plugin_name})
endforeach()
install(FILES ${PRELOAD_SCRIPT} DESTINATION ${ZEEK_SCRIPT_INSTALL_PATH}/builtin-plugins/)
install(FILES ${LOAD_SCRIPT} DESTINATION ${ZEEK_SCRIPT_INSTALL_PATH}/builtin-plugins/)
########################################################################
## bro target
@ -217,6 +246,10 @@ add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/DebugCmdConstants.h
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
)
set(_gen_zeek_script_cpp ${CMAKE_CURRENT_BINARY_DIR}/../CPP-gen.cc)
add_custom_command(OUTPUT ${_gen_zeek_script_cpp}
COMMAND ${CMAKE_COMMAND} -E touch ${_gen_zeek_script_cpp})
set_source_files_properties(nb_dns.c PROPERTIES COMPILE_FLAGS
-fno-strict-aliasing)
@ -327,6 +360,27 @@ set(MAIN_SRCS
plugin/Manager.cc
plugin/Plugin.cc
script_opt/CPP/Attrs.cc
script_opt/CPP/Consts.cc
script_opt/CPP/DeclFunc.cc
script_opt/CPP/Driver.cc
script_opt/CPP/Emit.cc
script_opt/CPP/Exprs.cc
script_opt/CPP/Func.cc
script_opt/CPP/GenFunc.cc
script_opt/CPP/HashMgr.cc
script_opt/CPP/Inits.cc
script_opt/CPP/RuntimeInit.cc
script_opt/CPP/RuntimeOps.cc
script_opt/CPP/RuntimeVec.cc
script_opt/CPP/Stmts.cc
script_opt/CPP/Tracker.cc
script_opt/CPP/Types.cc
script_opt/CPP/Util.cc
script_opt/CPP/Vars.cc
${_gen_zeek_script_cpp}
script_opt/DefItem.cc
script_opt/DefSetsMgr.cc
script_opt/Expr.cc

3
src/CompHash.cc
View file

@ -509,6 +509,9 @@ int CompositeHash::SingleTypeKeySize(Type* bt, const Val* v,
case TYPE_RECORD:
{
if ( ! v )
return (optional && ! calc_static_size) ? sz : 0;
const RecordVal* rv = v ? v->AsRecordVal() : nullptr;
RecordType* rt = bt->AsRecordType();
int num_fields = rt->NumFields();

1
src/CompHash.h
View file

@ -30,6 +30,7 @@ public:
// Given a hash key, recover the values used to create it.
ListValPtr RecoverVals(const HashKey& k) const;
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
unsigned int MemoryAllocation() const { return padded_sizeof(*this) + util::pad_size(size); }
protected:

61
src/Conn.cc
View file

@ -19,6 +19,8 @@
#include "zeek/analyzer/Analyzer.h"
#include "zeek/analyzer/Manager.h"
#include "zeek/iosource/IOSource.h"
#include "zeek/packet_analysis/protocol/ip/SessionAdapter.h"
#include "zeek/packet_analysis/protocol/tcp/TCP.h"
namespace zeek {
@ -54,7 +56,6 @@ Connection::Connection(const detail::ConnKey& k, double t,
vlan = pkt->vlan;
inner_vlan = pkt->inner_vlan;
skip = 0;
weird = 0;
suppress_event = 0;
@ -64,7 +65,7 @@ Connection::Connection(const detail::ConnKey& k, double t,
hist_seen = 0;
history = "";
root_analyzer = nullptr;
adapter = nullptr;
primary_PIA = nullptr;
++current_connections;
@ -83,7 +84,7 @@ Connection::~Connection()
if ( conn_val )
conn_val->SetOrigin(nullptr);
delete root_analyzer;
delete adapter;
--current_connections;
}
@ -124,23 +125,23 @@ void Connection::CheckEncapsulation(const std::shared_ptr<EncapsulationStack>& a
void Connection::Done()
{
// TODO: this still doesn't feel like the right place to do this, but it's better
// here than in SessionManager. This really should be down in the TCP analyzer
// somewhere, but it's session-related, so maybe not?
finished = 1;
if ( adapter )
{
if ( ConnTransport() == TRANSPORT_TCP )
{
auto ta = static_cast<analyzer::tcp::TCP_Analyzer*>(GetRootAnalyzer());
auto* ta = static_cast<packet_analysis::TCP::TCPSessionAdapter*>(adapter);
assert(ta->IsAnalyzer("TCP"));
analyzer::tcp::TCP_Endpoint* to = ta->Orig();
analyzer::tcp::TCP_Endpoint* tr = ta->Resp();
session_mgr->tcp_stats.StateLeft(to->state, tr->state);
packet_analysis::TCP::TCPAnalyzer::GetStats().StateLeft(to->state, tr->state);
}
finished = 1;
if ( root_analyzer && ! root_analyzer->IsFinished() )
root_analyzer->Done();
if ( ! adapter->IsFinished() )
adapter->Done();
}
}
void Connection::NextPacket(double t, bool is_orig,
@ -153,14 +154,14 @@ void Connection::NextPacket(double t, bool is_orig,
run_state::current_timestamp = t;
run_state::current_pkt = pkt;
if ( Skipping() )
if ( adapter )
{
if ( adapter->Skipping() )
return;
if ( root_analyzer )
{
record_current_packet = record_packet;
record_current_content = record_content;
root_analyzer->NextPacket(len, data, is_orig, -1, ip, caplen);
adapter->NextPacket(len, data, is_orig, -1, ip, caplen);
record_packet = record_current_packet;
record_content = record_current_content;
}
@ -173,7 +174,7 @@ void Connection::NextPacket(double t, bool is_orig,
bool Connection::IsReuse(double t, const u_char* pkt)
{
return root_analyzer && root_analyzer->IsReuse(t, pkt);
return adapter && adapter->IsReuse(t, pkt);
}
bool Connection::ScaledHistoryEntry(char code, uint32_t& counter,
@ -275,8 +276,8 @@ const RecordValPtr& Connection::GetVal()
}
if ( root_analyzer )
root_analyzer->UpdateConnVal(conn_val.get());
if ( adapter )
adapter->UpdateConnVal(conn_val.get());
conn_val->AssignTime(3, start_time); // ###
conn_val->AssignInterval(4, last_time - start_time);
@ -289,17 +290,17 @@ const RecordValPtr& Connection::GetVal()
analyzer::Analyzer* Connection::FindAnalyzer(analyzer::ID id)
{
return root_analyzer ? root_analyzer->FindChild(id) : nullptr;
return adapter ? adapter->FindChild(id) : nullptr;
}
analyzer::Analyzer* Connection::FindAnalyzer(const analyzer::Tag& tag)
{
return root_analyzer ? root_analyzer->FindChild(tag) : nullptr;
return adapter ? adapter->FindChild(tag) : nullptr;
}
analyzer::Analyzer* Connection::FindAnalyzer(const char* name)
{
return root_analyzer->FindChild(name);
return adapter->FindChild(name);
}
void Connection::AppendAddl(const char* str)
@ -370,8 +371,8 @@ void Connection::FlipRoles()
conn_val = nullptr;
if ( root_analyzer )
root_analyzer->FlipRoles();
if ( adapter )
adapter->FlipRoles();
analyzer_mgr->ApplyScheduledAnalyzers(this);
@ -380,17 +381,23 @@ void Connection::FlipRoles()
unsigned int Connection::MemoryAllocation() const
{
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
return session::Session::MemoryAllocation() + padded_sizeof(*this)
+ (timers.MemoryAllocation() - padded_sizeof(timers))
+ (conn_val ? conn_val->MemoryAllocation() : 0)
+ (root_analyzer ? root_analyzer->MemoryAllocation(): 0)
+ (adapter ? adapter->MemoryAllocation(): 0)
// primary_PIA is already contained in the analyzer tree.
;
#pragma GCC diagnostic pop
}
unsigned int Connection::MemoryAllocationVal() const
{
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
return conn_val ? conn_val->MemoryAllocation() : 0;
#pragma GCC diagnostic pop
}
void Connection::Describe(ODesc* d) const
@ -448,10 +455,10 @@ void Connection::IDString(ODesc* d) const
d->Add(ntohs(resp_port));
}
void Connection::SetRootAnalyzer(analyzer::TransportLayerAnalyzer* analyzer,
void Connection::SetSessionAdapter(packet_analysis::IP::SessionAdapter* aa,
analyzer::pia::PIA* pia)
{
root_analyzer = analyzer;
adapter = aa;
primary_PIA = pia;
}

30
src/Conn.h
View file

@ -41,12 +41,8 @@ class RuleHdrTest;
} // namespace detail
namespace analyzer {
class TransportLayerAnalyzer;
class Analyzer;
} // namespace analyzer
namespace analyzer { class Analyzer; }
namespace packet_analysis::IP { class SessionAdapter; }
enum ConnEventToFlag {
NUL_IN_LINE,
@ -117,7 +113,10 @@ public:
// should be marked invalid.
const detail::ConnKey& Key() const { return key; }
session::detail::Key SessionKey(bool copy) const override
{ return session::detail::Key{&key, sizeof(key), copy}; }
{
return session::detail::Key{
&key, sizeof(key), session::detail::Key::CONNECTION_KEY_TYPE, copy};
}
const IPAddr& OrigAddr() const { return orig_addr; }
const IPAddr& RespAddr() const { return resp_addr; }
@ -144,14 +143,6 @@ public:
return "unknown";
}
// FIXME: Now this is in Analyzer and should eventually be removed here.
//
// If true, skip processing of remainder of connection. Note
// that this does not in itself imply that record_packets is false;
// we might want instead to process the connection off-line.
void SetSkip(bool do_skip) { skip = do_skip ? 1 : 0; }
bool Skipping() const { return skip; }
// Returns true if the packet reflects a reuse of this
// connection (i.e., not a continuation but the beginning of
// a new connection).
@ -196,7 +187,9 @@ public:
// Statistics.
// Just a lower bound.
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
unsigned int MemoryAllocation() const override;
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
unsigned int MemoryAllocationVal() const override;
static uint64_t TotalConnections()
@ -231,8 +224,8 @@ public:
void AddHistory(char code) { history += code; }
// Sets the root of the analyzer tree as well as the primary PIA.
void SetRootAnalyzer(analyzer::TransportLayerAnalyzer* analyzer, analyzer::pia::PIA* pia);
analyzer::TransportLayerAnalyzer* GetRootAnalyzer() { return root_analyzer; }
void SetSessionAdapter(packet_analysis::IP::SessionAdapter* aa, analyzer::pia::PIA* pia);
packet_analysis::IP::SessionAdapter* GetSessionAdapter() { return adapter; }
analyzer::pia::PIA* GetPrimaryPIA() { return primary_PIA; }
// Sets the transport protocol in use.
@ -271,7 +264,6 @@ private:
detail::ConnKey key;
unsigned int skip:1;
unsigned int weird:1;
unsigned int finished:1;
unsigned int saw_first_orig_packet:1, saw_first_resp_packet:1;
@ -279,7 +271,7 @@ private:
uint32_t hist_seen;
std::string history;
analyzer::TransportLayerAnalyzer* root_analyzer;
packet_analysis::IP::SessionAdapter* adapter;
analyzer::pia::PIA* primary_PIA;
UID uid; // Globally unique connection ID.

3
src/DFA.cc
View file

@ -426,10 +426,13 @@ unsigned int DFA_Machine::MemoryAllocation() const
dfa_state_cache->GetStats(&s);
// FIXME: Count *ec?
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
return padded_sizeof(*this)
+ s.mem
+ padded_sizeof(*start_state)
+ nfa->MemoryAllocation();
#pragma GCC diagnostic pop
}
bool DFA_Machine::StateSetToDFA_State(NFA_state_list* state_set,

1
src/DFA.h
View file

@ -123,6 +123,7 @@ public:
void Describe(ODesc* d) const override;
void Dump(FILE* f);
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
unsigned int MemoryAllocation() const;
protected:

3
src/Debug.cc
View file

@ -972,10 +972,7 @@ ValPtr dbg_eval_expr(const char* expr)
const ScriptFunc* func = frame->GetFunction();
if ( func )
{
Ref(func->GetScope());
push_existing_scope(func->GetScope());
}
// ### Possibly push a debugger-local scope?

2
src/DebugCmds.cc
View file

@ -61,7 +61,7 @@ static void lookup_global_symbols_regex(const string& orig_regex, vector<ID*>& m
return;
}
Scope* global = global_scope();
auto global = global_scope();
const auto& syms = global->Vars();
ID* nextid;

91
src/Dict.cc
View file

@ -265,6 +265,11 @@ TEST_CASE("dict new iteration")
count++;
}
PDict<uint32_t>::iterator it;
it = dict.begin();
it = dict.end();
PDict<uint32_t>::iterator it2 = it;
CHECK(count == 2);
delete key;
@ -767,7 +772,10 @@ void Dictionary::DumpKeys() const
if ( binary )
{
char key = char(random() % 26) + 'A';
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
sprintf(key_file, "%d.%d.%zu-%c.key", Length(), max_distance, MemoryAllocation()/Length(), key);
#pragma GCC diagnostic pop
std::ofstream f(key_file, std::ios::binary|std::ios::out|std::ios::trunc);
for ( int idx = 0; idx < Capacity(); idx++ )
if ( ! table[idx].Empty() )
@ -780,7 +788,10 @@ void Dictionary::DumpKeys() const
else
{
char key = char(random() % 26) + 'A';
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
sprintf(key_file, "%d.%d.%zu-%d.ckey",Length(), max_distance, MemoryAllocation()/Length(), key);
#pragma GCC diagnostic pop
std::ofstream f(key_file, std::ios::out|std::ios::trunc);
for ( int idx = 0; idx < Capacity(); idx++ )
if ( ! table[idx].Empty() )
@ -828,10 +839,13 @@ void Dictionary::Dump(int level) const
int distances[DICT_NUM_DISTANCES];
int max_distance = 0;
DistanceStats(max_distance, distances, DICT_NUM_DISTANCES);
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
printf("cap %'7d ent %'7d %'-7d load %.2f max_dist %2d mem %10zu mem/ent %3zu key/ent %3d lg %2d remaps %1d remap_end %4d ",
Capacity(), Length(), MaxLength(), (double)Length()/(table? Capacity() : 1),
max_distance, MemoryAllocation(), (MemoryAllocation())/(Length()?Length():1), key_size / (Length()?Length():1),
log2_buckets, remaps, remap_end);
#pragma GCC diagnostic pop
if ( Length() > 0 )
{
for (int i = 0; i < DICT_NUM_DISTANCES-1; i++)
@ -1557,10 +1571,13 @@ DictIterator::DictIterator(const Dictionary* d, detail::DictEntry* begin, detail
}
DictIterator::~DictIterator()
{
if ( dict )
{
assert(dict->num_iterators > 0);
dict->num_iterators--;
}
}
DictIterator& DictIterator::operator++()
{
@ -1574,6 +1591,80 @@ DictIterator& DictIterator::operator++()
return *this;
}
DictIterator::DictIterator(const DictIterator& that)
{
if ( this == &that )
return;
if ( dict )
{
assert(dict->num_iterators > 0);
dict->num_iterators--;
}
dict = that.dict;
curr = that.curr;
end = that.end;
dict->num_iterators++;
}
DictIterator& DictIterator::operator=(const DictIterator& that)
{
if ( this == &that )
return *this;
if ( dict )
{
assert(dict->num_iterators > 0);
dict->num_iterators--;
}
dict = that.dict;
curr = that.curr;
end = that.end;
dict->num_iterators++;
return *this;
}
DictIterator::DictIterator(DictIterator&& that)
{
if ( this == &that )
return;
if ( dict )
{
assert(dict->num_iterators > 0);
dict->num_iterators--;
}
dict = that.dict;
curr = that.curr;
end = that.end;
that.dict = nullptr;
}
DictIterator& DictIterator::operator=(DictIterator&& that)
{
if ( this == &that )
return *this;
if ( dict )
{
assert(dict->num_iterators > 0);
dict->num_iterators--;
}
dict = that.dict;
curr = that.curr;
end = that.end;
that.dict = nullptr;
return *this;
}

9
src/Dict.h
View file

@ -159,8 +159,14 @@ public:
using difference_type = std::ptrdiff_t;
using iterator_category = std::forward_iterator_tag;
DictIterator() = default;
~DictIterator();
DictIterator(const DictIterator& that);
DictIterator& operator=(const DictIterator& that);
DictIterator(DictIterator&& that);
DictIterator& operator=(DictIterator&& that);
reference operator*() { return *curr; }
pointer operator->() { return curr; }
@ -171,9 +177,9 @@ public:
bool operator!=( const DictIterator& that ) const { return !(*this == that); }
private:
friend class Dictionary;
DictIterator() = default;
DictIterator(const Dictionary* d, detail::DictEntry* begin, detail::DictEntry* end);
Dictionary* dict = nullptr;
@ -331,6 +337,7 @@ public:
// Remove all entries.
void Clear();
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
size_t MemoryAllocation() const;
/// The capacity of the table, Buckets + Overflow Size.

323
src/Expr.cc
View file

@ -55,7 +55,7 @@ const char* expr_name(BroExprTag t)
"inline()",
"[]=", "$=",
"vec+=",
"to_any_coerce", "from_any_coerce",
"to_any_coerce", "from_any_coerce", "from_any_vec_coerce",
"any[]",
"nop",
@ -211,6 +211,11 @@ ExprPtr Expr::MakeLvalue()
return {NewRef{}, this};
}
bool Expr::InvertSense()
{
return false;
}
void Expr::EvalIntoAggregate(const zeek::Type* /* t */, Val* /* aggr */,
Frame* /* f */) const
{
@ -568,11 +573,17 @@ void NameExpr::ExprDescribe(ODesc* d) const
ConstExpr::ConstExpr(ValPtr arg_val)
: Expr(EXPR_CONST), val(std::move(arg_val))
{
if ( val->GetType()->Tag() == TYPE_LIST && val->AsListVal()->Length() == 1 )
if ( val )
{
if ( val->GetType()->Tag() == TYPE_LIST &&
val->AsListVal()->Length() == 1 )
val = val->AsListVal()->Idx(0);
SetType(val->GetType());
}
else
SetError();
}
void ConstExpr::ExprDescribe(ODesc* d) const
{
@ -1386,9 +1397,12 @@ SizeExpr::SizeExpr(ExprPtr arg_op)
if ( IsError() )
return;
if ( op->GetType()->Tag() == TYPE_ANY )
auto& t = op->GetType();
if ( t->Tag() == TYPE_ANY )
SetType(base_type(TYPE_ANY));
else if ( op->GetType()->InternalType() == TYPE_INTERNAL_DOUBLE )
else if ( t->Tag() == TYPE_FILE || t->Tag() == TYPE_SUBNET ||
t->InternalType() == TYPE_INTERNAL_DOUBLE )
SetType(base_type(TYPE_DOUBLE));
else
SetType(base_type(TYPE_COUNT));
@ -2043,6 +2057,12 @@ ValPtr EqExpr::Fold(Val* v1, Val* v2) const
return BinaryExpr::Fold(v1, v2);
}
bool EqExpr::InvertSense()
{
tag = (tag == EXPR_EQ ? EXPR_NE : EXPR_EQ);
return true;
}
RelExpr::RelExpr(BroExprTag arg_tag, ExprPtr arg_op1, ExprPtr arg_op2)
: BinaryExpr(arg_tag, std::move(arg_op1), std::move(arg_op2))
{
@ -2100,6 +2120,19 @@ void RelExpr::Canonicize()
}
}
bool RelExpr::InvertSense()
{
switch ( tag ) {
case EXPR_LT: tag = EXPR_GE; return true;
case EXPR_LE: tag = EXPR_GT; return true;
case EXPR_GE: tag = EXPR_LT; return true;
case EXPR_GT: tag = EXPR_LE; return true;
default:
return false;
}
}
CondExpr::CondExpr(ExprPtr arg_op1, ExprPtr arg_op2, ExprPtr arg_op3)
: Expr(EXPR_COND),
op1(std::move(arg_op1)), op2(std::move(arg_op2)), op3(std::move(arg_op3))
@ -2900,7 +2933,7 @@ ValPtr IndexExpr::Eval(Frame* f) const
{
VectorVal* v_v1 = v1->AsVectorVal();
VectorVal* v_v2 = indv->AsVectorVal();
auto v_result = make_intrusive<VectorVal>(GetType<VectorType>());
auto vt = cast_intrusive<VectorType>(GetType());
// Booleans select each element (or not).
if ( IsBool(v_v2->GetType()->Yield()->Tag()) )
@ -2911,23 +2944,11 @@ ValPtr IndexExpr::Eval(Frame* f) const
return nullptr;
}
for ( unsigned int i = 0; i < v_v2->Size(); ++i )
{
if ( v_v2->BoolAt(i) )
v_result->Assign(v_result->Size() + 1, v_v1->ValAt(i));
}
return vector_bool_select(vt, v_v1, v_v2);
}
else
{ // The elements are indices.
// ### Should handle negative indices here like
// S does, i.e., by excluding those elements.
// Probably only do this if *all* are negative.
v_result->Resize(v_v2->Size());
for ( unsigned int i = 0; i < v_v2->Size(); ++i )
v_result->Assign(i, v_v1->ValAt(v_v2->ValAt(i)->CoerceToInt()));
}
return v_result;
// Elements are indices.
return vector_int_select(vt, v_v1, v_v2);
}
else
return Fold(v1.get(), v2.get());
@ -2954,7 +2975,7 @@ ValPtr IndexExpr::Fold(Val* v1, Val* v2) const
break;
case TYPE_TABLE:
v = v1->AsTableVal()->FindOrDefault({NewRef{}, v2}); // Then, we jump into the TableVal here.
v = v1->AsTableVal()->FindOrDefault({NewRef{}, v2});
break;
case TYPE_STRING:
@ -3029,6 +3050,35 @@ VectorValPtr index_slice(VectorVal* vect, int _first, int _last)
return result;
}
VectorValPtr vector_bool_select(VectorTypePtr vt, const VectorVal* v1,
const VectorVal* v2)
{
auto v_result = make_intrusive<VectorVal>(std::move(vt));
for ( unsigned int i = 0; i < v2->Size(); ++i )
if ( v2->BoolAt(i) )
v_result->Assign(v_result->Size() + 1, v1->ValAt(i));
return v_result;
}
VectorValPtr vector_int_select(VectorTypePtr vt, const VectorVal* v1,
const VectorVal* v2)
{
auto v_result = make_intrusive<VectorVal>(std::move(vt));
// The elements are indices.
//
// ### Should handle negative indices here like S does, i.e.,
// by excluding those elements. Probably only do this if *all*
// are negative.
v_result->Resize(v2->Size());
for ( unsigned int i = 0; i < v2->Size(); ++i )
v_result->Assign(i, v1->ValAt(v2->ValAt(i)->CoerceToInt()));
return v_result;
}
void IndexExpr::Assign(Frame* f, ValPtr v)
{
if ( IsError() )
@ -3191,8 +3241,10 @@ void HasFieldExpr::ExprDescribe(ODesc* d) const
d->Add(field);
}
RecordConstructorExpr::RecordConstructorExpr(ListExprPtr constructor_list)
: Expr(EXPR_RECORD_CONSTRUCTOR), op(std::move(constructor_list))
: Expr(EXPR_RECORD_CONSTRUCTOR), op(std::move(constructor_list)),
map(std::nullopt)
{
if ( IsError() )
return;
@ -3221,8 +3273,45 @@ RecordConstructorExpr::RecordConstructorExpr(ListExprPtr constructor_list)
SetType(make_intrusive<RecordType>(record_types));
}
RecordConstructorExpr::~RecordConstructorExpr()
RecordConstructorExpr::RecordConstructorExpr(RecordTypePtr known_rt,
ListExprPtr constructor_list)
: Expr(EXPR_RECORD_CONSTRUCTOR), op(std::move(constructor_list))
{
if ( IsError() )
return;
SetType(known_rt);
const auto& exprs = op->AsListExpr()->Exprs();
map = std::vector<int>(exprs.length());
int i = 0;
for ( const auto& e : exprs )
{
if ( e->Tag() != EXPR_FIELD_ASSIGN )
{
Error("bad type in record constructor", e);
SetError();
continue;
}
auto field = e->AsFieldAssignExpr();
int index = known_rt->FieldOffset(field->FieldName());
if ( index < 0 )
{
Error("no such field in record", e);
SetError();
continue;
}
auto known_ft = known_rt->GetFieldType(index);
if ( ! field->PromoteTo(known_ft) )
SetError();
(*map)[i++] = index;
}
}
ValPtr RecordConstructorExpr::InitVal(const zeek::Type* t, ValPtr aggr) const
@ -3262,13 +3351,16 @@ ValPtr RecordConstructorExpr::Eval(Frame* f) const
const auto& exprs = op->Exprs();
auto rt = cast_intrusive<RecordType>(type);
if ( exprs.length() != rt->NumFields() )
if ( ! map && exprs.length() != rt->NumFields() )
RuntimeErrorWithCallStack("inconsistency evaluating record constructor");
auto rv = make_intrusive<RecordVal>(std::move(rt));
for ( int i = 0; i < exprs.length(); ++i )
rv->Assign(i, exprs[i]->Eval(f));
{
int ind = map ? (*map)[i] : i;
rv->Assign(ind, exprs[i]->Eval(f));
}
return rv;
}
@ -3279,11 +3371,23 @@ bool RecordConstructorExpr::IsPure() const
}
void RecordConstructorExpr::ExprDescribe(ODesc* d) const
{
auto& tn = type->GetName();
if ( tn.size() > 0 )
{
d->Add(tn);
d->Add("(");
op->Describe(d);
d->Add(")");
}
else
{
d->Add("[");
op->Describe(d);
d->Add("]");
}
}
TraversalCode RecordConstructorExpr::Traverse(TraversalCallback* cb) const
{
@ -3348,9 +3452,9 @@ TableConstructorExpr::TableConstructorExpr(ListExprPtr constructor_list,
if ( expr->Tag() != EXPR_ASSIGN )
continue;
auto idx_expr = expr->AsAssignExpr()->Op1();
auto val_expr = expr->AsAssignExpr()->Op2();
auto yield_type = GetType()->AsTableType()->Yield().get();
auto idx_expr = expr->AsAssignExpr()->GetOp1();
auto val_expr = expr->AsAssignExpr()->GetOp2();
auto yield_type = GetType()->AsTableType()->Yield();
// Promote LHS
assert(idx_expr->Tag() == EXPR_LIST);
@ -3361,17 +3465,14 @@ TableConstructorExpr::TableConstructorExpr(ListExprPtr constructor_list,
loop_over_list(idx_exprs, j)
{
Expr* idx = idx_exprs[j];
ExprPtr idx = {NewRef{}, idx_exprs[j]};
auto promoted_idx = check_and_promote_expr(idx, indices[j].get());
auto promoted_idx = check_and_promote_expr(idx, indices[j]);
if ( promoted_idx )
{
if ( promoted_idx.get() != idx )
{
Unref(idx);
idx_exprs.replace(j, promoted_idx.release());
}
if ( promoted_idx != idx )
Unref(idx_exprs.replace(j, promoted_idx.release()));
continue;
}
@ -3483,7 +3584,7 @@ SetConstructorExpr::SetConstructorExpr(ListExprPtr constructor_list,
if ( indices.size() == 1 )
{
if ( ! check_and_promote_exprs_to_type(op->AsListExpr(),
indices[0].get()) )
indices[0]) )
ExprError("inconsistent type in set constructor");
}
@ -3602,7 +3703,7 @@ VectorConstructorExpr::VectorConstructorExpr(ListExprPtr constructor_list,
}
if ( ! check_and_promote_exprs_to_type(op->AsListExpr(),
type->AsVectorType()->Yield().get()) )
type->AsVectorType()->Yield()) )
ExprError("inconsistent types in vector constructor");
}
@ -3670,6 +3771,12 @@ FieldAssignExpr::FieldAssignExpr(const char* arg_field_name, ExprPtr value)
SetType(op->GetType());
}
bool FieldAssignExpr::PromoteTo(TypePtr t)
{
op = check_and_promote_expr(op, t);
return op != nullptr;
}
void FieldAssignExpr::EvalIntoAggregate(const zeek::Type* t, Val* aggr, Frame* f)
const
{
@ -3707,7 +3814,11 @@ void FieldAssignExpr::ExprDescribe(ODesc* d) const
d->Add("$");
d->Add(FieldName());
d->Add("=");
if ( op )
op->Describe(d);
else
d->Add("<error>");
}
ArithCoerceExpr::ArithCoerceExpr(ExprPtr arg_op, TypeTag t)
@ -3739,49 +3850,36 @@ ArithCoerceExpr::ArithCoerceExpr(ExprPtr arg_op, TypeTag t)
ExprError("bad coercion value");
}
ValPtr ArithCoerceExpr::FoldSingleVal(Val* v, InternalTypeTag t) const
ValPtr ArithCoerceExpr::FoldSingleVal(ValPtr v, const TypePtr& t) const
{
switch ( t ) {
case TYPE_INTERNAL_DOUBLE:
return make_intrusive<DoubleVal>(v->CoerceToDouble());
case TYPE_INTERNAL_INT:
return val_mgr->Int(v->CoerceToInt());
case TYPE_INTERNAL_UNSIGNED:
return val_mgr->Count(v->CoerceToUnsigned());
default:
RuntimeErrorWithCallStack("bad type in CoerceExpr::Fold");
return nullptr;
}
return check_and_promote(v, t.get(), false, location);
}
ValPtr ArithCoerceExpr::Fold(Val* v) const
{
InternalTypeTag t = type->InternalType();
auto t = GetType();
if ( ! is_vector(v) )
{
// Our result type might be vector, in which case this
// invocation is being done per-element rather than on
// the whole vector. Correct the type tag if necessary.
// the whole vector. Correct the type if so.
if ( type->Tag() == TYPE_VECTOR )
t = GetType()->AsVectorType()->Yield()->InternalType();
t = t->AsVectorType()->Yield();
return FoldSingleVal(v, t);
return FoldSingleVal({NewRef{}, v}, t);
}
t = GetType()->AsVectorType()->Yield()->InternalType();
VectorVal* vv = v->AsVectorVal();
auto result = make_intrusive<VectorVal>(GetType<VectorType>());
auto result = make_intrusive<VectorVal>(cast_intrusive<VectorType>(t));
auto yt = t->AsVectorType()->Yield();
for ( unsigned int i = 0; i < vv->Size(); ++i )
{
auto elt = vv->ValAt(i);
if ( elt )
result->Assign(i, FoldSingleVal(elt.get(), t));
result->Assign(i, FoldSingleVal(elt, yt));
else
result->Assign(i, nullptr);
}
@ -4554,9 +4652,9 @@ void LambdaExpr::CheckCaptures()
}
}
Scope* LambdaExpr::GetScope() const
ScopePtr LambdaExpr::GetScope() const
{
return ingredients->scope.get();
return ingredients->scope;
}
ValPtr LambdaExpr::Eval(Frame* f) const
@ -4879,12 +4977,12 @@ ValPtr ListExpr::InitVal(const zeek::Type* t, ValPtr aggr) const
loop_over_list(exprs, i)
{
Expr* e = exprs[i];
ExprPtr e = {NewRef{}, exprs[i]};
const auto& vyt = vec->GetType()->AsVectorType()->Yield();
auto promoted_e = check_and_promote_expr(e, vyt.get());
auto promoted_e = check_and_promote_expr(e, vyt);
if ( promoted_e )
e = promoted_e.get();
e = promoted_e;
if ( ! vec->Assign(i, e->Eval(nullptr)) )
{
@ -5106,34 +5204,38 @@ CastExpr::CastExpr(ExprPtr arg_op, TypePtr t)
ExprError("cast not supported");
}
ValPtr CastExpr::Eval(Frame* f) const
ValPtr CastExpr::Fold(Val* v) const
{
if ( IsError() )
return nullptr;
std::string error;
auto res = cast_value({NewRef{}, v}, GetType(), error);
auto v = op->Eval(f);
if ( ! res )
RuntimeError(error.c_str());
if ( ! v )
return nullptr;
return res;
}
auto nv = cast_value_to_type(v.get(), GetType().get());
ValPtr cast_value(ValPtr v, const TypePtr& t, std::string& error)
{
auto nv = cast_value_to_type(v.get(), t.get());
if ( nv )
return nv;
ODesc d;
d.Add("invalid cast of value with type '");
v->GetType()->Describe(&d);
d.Add("' to type '");
GetType()->Describe(&d);
t->Describe(&d);
d.Add("'");
if ( same_type(v->GetType(), Broker::detail::DataVal::ScriptDataType()) &&
! v->AsRecordVal()->HasField(0) )
d.Add(" (nil $data field)");
RuntimeError(d.Description());
return nullptr; // not reached.
error = d.Description();
return nullptr;
}
void CastExpr::ExprDescribe(ODesc* d) const
@ -5180,34 +5282,42 @@ ExprPtr get_assign_expr(ExprPtr op1, ExprPtr op2, bool is_init)
std::move(op1), std::move(op2), is_init);
}
ExprPtr check_and_promote_expr(Expr* const e, zeek::Type* t)
ExprPtr check_and_promote_expr(ExprPtr e, TypePtr t)
{
const auto& et = e->GetType();
TypeTag e_tag = et->Tag();
TypeTag t_tag = t->Tag();
if ( t->Tag() == TYPE_ANY )
return {NewRef{}, e};
if ( t_tag == TYPE_ANY )
{
if ( e_tag != TYPE_ANY )
return make_intrusive<CoerceToAnyExpr>(e);
return e;
}
if ( e_tag == TYPE_ANY )
return make_intrusive<CoerceFromAnyExpr>(e, t);
if ( EitherArithmetic(t_tag, e_tag) )
{
if ( e_tag == t_tag )
return {NewRef{}, e};
return e;
if ( ! BothArithmetic(t_tag, e_tag) )
{
t->Error("arithmetic mixed with non-arithmetic", e);
t->Error("arithmetic mixed with non-arithmetic", e.get());
return nullptr;
}
TypeTag mt = max_type(t_tag, e_tag);
if ( mt != t_tag )
{
t->Error("over-promotion of arithmetic value", e);
t->Error("over-promotion of arithmetic value", e.get());
return nullptr;
}
return make_intrusive<ArithCoerceExpr>(IntrusivePtr{NewRef{}, e}, t_tag);
return make_intrusive<ArithCoerceExpr>(e, t_tag);
}
if ( t->Tag() == TYPE_RECORD && et->Tag() == TYPE_RECORD )
@ -5216,14 +5326,13 @@ ExprPtr check_and_promote_expr(Expr* const e, zeek::Type* t)
RecordType* et_r = et->AsRecordType();
if ( same_type(t, et) )
return {NewRef{}, e};
return e;
if ( record_promotion_compatible(t_r, et_r) )
return make_intrusive<RecordCoerceExpr>(
IntrusivePtr{NewRef{}, e},
return make_intrusive<RecordCoerceExpr>(e,
IntrusivePtr{NewRef{}, t_r});
t->Error("incompatible record types", e);
t->Error("incompatible record types", e.get());
return nullptr;
}
@ -5232,23 +5341,21 @@ ExprPtr check_and_promote_expr(Expr* const e, zeek::Type* t)
{
if ( t->Tag() == TYPE_TABLE && et->Tag() == TYPE_TABLE &&
et->AsTableType()->IsUnspecifiedTable() )
return make_intrusive<TableCoerceExpr>(
IntrusivePtr{NewRef{}, e},
return make_intrusive<TableCoerceExpr>(e,
IntrusivePtr{NewRef{}, t->AsTableType()});
if ( t->Tag() == TYPE_VECTOR && et->Tag() == TYPE_VECTOR &&
et->AsVectorType()->IsUnspecifiedVector() )
return make_intrusive<VectorCoerceExpr>(
IntrusivePtr{NewRef{}, e},
return make_intrusive<VectorCoerceExpr>(e,
IntrusivePtr{NewRef{}, t->AsVectorType()});
if ( t->Tag() != TYPE_ERROR && et->Tag() != TYPE_ERROR )
t->Error("type clash", e);
t->Error("type clash", e.get());
return nullptr;
}
return {NewRef{}, e};
return e;
}
bool check_and_promote_exprs(ListExpr* const elements, TypeList* types)
@ -5267,8 +5374,8 @@ bool check_and_promote_exprs(ListExpr* const elements, TypeList* types)
loop_over_list(el, i)
{
Expr* e = el[i];
auto promoted_e = check_and_promote_expr(e, tl[i].get());
ExprPtr e = {NewRef{}, el[i]};
auto promoted_e = check_and_promote_expr(e, tl[i]);
if ( ! promoted_e )
{
@ -5276,17 +5383,14 @@ bool check_and_promote_exprs(ListExpr* const elements, TypeList* types)
return false;
}
if ( promoted_e.get() != e )
{
Unref(e);
el.replace(i, promoted_e.release());
}
if ( promoted_e != e )
Unref(el.replace(i, promoted_e.release()));
}
return true;
}
bool check_and_promote_args(ListExpr* const args, RecordType* types)
bool check_and_promote_args(ListExpr* const args, const RecordType* types)
{
ExprPList& el = args->Exprs();
int ntypes = types->NumFields();
@ -5303,7 +5407,7 @@ bool check_and_promote_args(ListExpr* const args, RecordType* types)
// arguments using &default expressions.
for ( int i = ntypes - 1; i >= el.length(); --i )
{
TypeDecl* td = types->FieldDecl(i);
auto td = types->FieldDecl(i);
const auto& def_attr = td->attrs ? td->attrs->Find(ATTR_DEFAULT).get() : nullptr;
if ( ! def_attr )
@ -5340,29 +5444,26 @@ bool check_and_promote_args(ListExpr* const args, RecordType* types)
return rval;
}
bool check_and_promote_exprs_to_type(ListExpr* const elements, zeek::Type* type)
bool check_and_promote_exprs_to_type(ListExpr* const elements, TypePtr t)
{
ExprPList& el = elements->Exprs();
if ( type->Tag() == TYPE_ANY )
if ( t->Tag() == TYPE_ANY )
return true;
loop_over_list(el, i)
{
Expr* e = el[i];
auto promoted_e = check_and_promote_expr(e, type);
ExprPtr e = {NewRef{}, el[i]};
auto promoted_e = check_and_promote_expr(e, t);
if ( ! promoted_e )
{
e->Error("type mismatch", type);
e->Error("type mismatch", t.get());
return false;
}
if ( promoted_e.get() != e )
{
Unref(e);
el.replace(i, promoted_e.release());
}
if ( promoted_e != e )
Unref(el.replace(i, promoted_e.release()));
}
return true;

101
src/Expr.h
View file

@ -27,6 +27,7 @@ class Frame;
class Scope;
struct function_ingredients;
using IDPtr = IntrusivePtr<ID>;
using ScopePtr = IntrusivePtr<Scope>;
enum BroExprTag : int {
EXPR_ANY = -1,
@ -71,7 +72,7 @@ enum BroExprTag : int {
// ASTs produced by parsing .zeek script files.
EXPR_INDEX_ASSIGN, EXPR_FIELD_LHS_ASSIGN,
EXPR_APPEND_TO,
EXPR_TO_ANY_COERCE, EXPR_FROM_ANY_COERCE,
EXPR_TO_ANY_COERCE, EXPR_FROM_ANY_COERCE, EXPR_FROM_ANY_VEC_COERCE,
EXPR_ANY_INDEX,
EXPR_NOP,
@ -94,12 +95,16 @@ class ForExpr;
class HasFieldExpr;
class IndexAssignExpr;
class IndexExpr;
class IsExpr;
class InlineExpr;
class IsExpr;
class LambdaExpr;
class ListExpr;
class NameExpr;
class RecordCoerceExpr;
class RecordConstructorExpr;
class RefExpr;
class SetConstructorExpr;
class TableConstructorExpr;
class Expr;
using CallExprPtr = IntrusivePtr<CallExpr>;
@ -192,6 +197,11 @@ public:
// the current value of expr (this is the default method).
virtual ExprPtr MakeLvalue();
// Invert the sense of the operation. Returns true if the expression
// was invertible (currently only true for relational/equality
// expressions), false otherwise.
virtual bool InvertSense();
// Marks the expression as one requiring (or at least appearing
// with) parentheses. Used for pretty-printing.
void MarkParen() { paren = true; }
@ -215,12 +225,16 @@ public:
ZEEK_EXPR_ACCESSOR_DECLS(HasFieldExpr)
ZEEK_EXPR_ACCESSOR_DECLS(IndexAssignExpr)
ZEEK_EXPR_ACCESSOR_DECLS(IndexExpr)
ZEEK_EXPR_ACCESSOR_DECLS(IsExpr)
ZEEK_EXPR_ACCESSOR_DECLS(InlineExpr)
ZEEK_EXPR_ACCESSOR_DECLS(IsExpr)
ZEEK_EXPR_ACCESSOR_DECLS(LambdaExpr)
ZEEK_EXPR_ACCESSOR_DECLS(ListExpr)
ZEEK_EXPR_ACCESSOR_DECLS(NameExpr)
ZEEK_EXPR_ACCESSOR_DECLS(RecordCoerceExpr)
ZEEK_EXPR_ACCESSOR_DECLS(RecordConstructorExpr)
ZEEK_EXPR_ACCESSOR_DECLS(RefExpr)
ZEEK_EXPR_ACCESSOR_DECLS(SetConstructorExpr)
ZEEK_EXPR_ACCESSOR_DECLS(TableConstructorExpr)
void Describe(ODesc* d) const override final;
@ -782,6 +796,7 @@ public:
ExprPtr Duplicate() override;
bool WillTransform(Reducer* c) const override;
ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override;
bool InvertSense() override;
protected:
ValPtr Fold(Val* v1, Val* v2) const override;
@ -796,6 +811,7 @@ public:
ExprPtr Duplicate() override;
bool WillTransform(Reducer* c) const override;
ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override;
bool InvertSense() override;
};
class CondExpr final : public Expr {
@ -957,6 +973,15 @@ extern VectorValPtr index_slice(VectorVal* vect, int first, int last);
// (exactly) two values.
extern StringValPtr index_string(const String* s, const ListVal* lv);
// Returns a vector indexed by a boolean vector.
extern VectorValPtr vector_bool_select(VectorTypePtr vt, const VectorVal* v1,
const VectorVal* v2);
// Returns a vector indexed by a numeric vector (which specifies the
// indices to select).
extern VectorValPtr vector_int_select(VectorTypePtr vt, const VectorVal* v1,
const VectorVal* v2);
class IndexExprWhen final : public IndexExpr {
public:
static inline std::vector<ValPtr> results = {};
@ -1046,9 +1071,13 @@ protected:
class RecordConstructorExpr final : public Expr {
public:
explicit RecordConstructorExpr(ListExprPtr constructor_list);
~RecordConstructorExpr() override;
ListExpr* Op() const { return op.get(); }
// This form is used to construct records of a known (ultimate) type.
explicit RecordConstructorExpr(RecordTypePtr known_rt,
ListExprPtr constructor_list);
ListExprPtr Op() const { return op; }
const auto& Map() const { return map; }
ValPtr Eval(Frame* f) const override;
@ -1069,6 +1098,7 @@ protected:
void ExprDescribe(ODesc* d) const override;
ListExprPtr op;
std::optional<std::vector<int>> map;
};
class TableConstructorExpr final : public UnaryExpr {
@ -1149,6 +1179,14 @@ public:
const char* FieldName() const { return field_name.c_str(); }
// When these are first constructed, we don't know the type.
// The following method coerces/promotes the assignment expression
// as needed, once we do know the type.
//
// Returns true on success, false if the types were incompatible
// (in which case an error is reported).
bool PromoteTo(TypePtr t);
void EvalIntoAggregate(const zeek::Type* t, Val* aggr, Frame* f) const override;
bool IsRecordElement(TypeDecl* td) const override;
@ -1175,7 +1213,7 @@ public:
ExprPtr Reduce(Reducer* c, StmtPtr& red_stmt) override;
protected:
ValPtr FoldSingleVal(Val* v, InternalTypeTag t) const;
ValPtr FoldSingleVal(ValPtr v, const TypePtr& t) const;
ValPtr Fold(Val* v) const override;
};
@ -1291,6 +1329,7 @@ public:
Expr* Func() const { return func.get(); }
ListExpr* Args() const { return args.get(); }
ListExprPtr ArgsPtr() const { return args; }
bool IsPure() const override;
@ -1332,7 +1371,7 @@ public:
ValPtr Eval(Frame* f) const override;
TraversalCode Traverse(TraversalCallback* cb) const override;
Scope* GetScope() const;
ScopePtr GetScope() const;
// Optimization-related:
ExprPtr Duplicate() override;
@ -1442,10 +1481,14 @@ public:
ExprPtr Duplicate() override;
protected:
ValPtr Eval(Frame* f) const override;
ValPtr Fold(Val* v) const override;
void ExprDescribe(ODesc* d) const override;
};
// Returns the value 'v' cast to type 't'. On an error, returns nil
// and populates "error" with an error message.
extern ValPtr cast_value(ValPtr v, const TypePtr& t, std::string& error);
class IsExpr final : public UnaryExpr {
public:
IsExpr(ExprPtr op, TypePtr t);
@ -1583,6 +1626,20 @@ protected:
ExprPtr Duplicate() override;
};
// ... and for conversion from a "vector of any" type.
class CoerceFromAnyVecExpr : public UnaryExpr {
public:
// to_type is yield type, not VectorType.
CoerceFromAnyVecExpr(ExprPtr op, TypePtr to_type);
// Can't use UnaryExpr's Eval() because it will do folding
// over the individual vector elements.
ValPtr Eval(Frame* f) const override;
protected:
ExprPtr Duplicate() override;
};
// Expression used to explicitly capture [a, b, c, ...] = x assignments.
class AnyIndexExpr : public UnaryExpr {
public:
@ -1635,26 +1692,22 @@ ExprPtr get_assign_expr(
ExprPtr op1,
ExprPtr op2, bool is_init);
// Type-check the given expression(s) against the given type(s). Complain
// if the expression cannot match the given type, returning 0. If it can
// match, promote it as necessary (modifying the ref parameter accordingly)
// and return 1.
//
// The second, third, and fourth forms are for promoting a list of
// expressions (which is updated in place) to either match a list of
// types or a single type.
//
// Note, the type is not "const" because it can be ref'd.
/**
* Returns nullptr if the expression cannot match or a promoted
* expression.
* Type-check the given expression(s) against the given type(s). Complain
* if the expression cannot match the given type, returning nullptr;
* otherwise, returns an expression reflecting the promotion.
*
* The second, third, and fourth forms are for promoting a list of
* expressions (which is updated in place) to either match a list of
* types or a single type.
*
* Note, the type is not "const" because it can be ref'd.
*/
extern ExprPtr check_and_promote_expr(Expr* e, Type* t);
extern ExprPtr check_and_promote_expr(ExprPtr e, TypePtr t);
extern bool check_and_promote_exprs(ListExpr* elements, TypeList* types);
extern bool check_and_promote_args(ListExpr* args, RecordType* types);
extern bool check_and_promote_exprs_to_type(ListExpr* elements, Type* type);
extern bool check_and_promote_args(ListExpr* args, const RecordType* types);
extern bool check_and_promote_exprs_to_type(ListExpr* elements, TypePtr type);
// Returns a ListExpr simplified down to a list a values, or nil
// if they couldn't all be reduced.

1
src/Frag.h
View file

@ -83,6 +83,7 @@ public:
size_t Size() const { return fragments.size(); }
size_t MaxFragments() const { return max_fragments; }
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
uint32_t MemoryAllocation() const;
private:

5
src/Frame.cc
View file

@ -430,7 +430,8 @@ std::pair<bool, FramePtr> Frame::Unserialize(const broker::vector& data,
if ( captures || *has_name == "CopyFrame" )
{
ASSERT(captures && *has_name == "CopyFrame");
if ( captures )
ASSERT(*has_name == "CopyFrame");
auto has_body = broker::get_if<broker::vector>(*where);
if ( ! has_body )
@ -618,7 +619,7 @@ void Frame::CaptureClosure(Frame* c, IDPList arg_outer_ids)
const detail::Location* Frame::GetCallLocation() const
{
return call ? call->GetLocationInfo() : nullptr;
return call ? call->GetLocationInfo() : call_loc;
}
void Frame::SetTrigger(trigger::TriggerPtr arg_trigger)

5
src/Frame.h
View file

@ -262,6 +262,8 @@ public:
void SetCall(const CallExpr* arg_call) { call = arg_call; }
void ClearCall() { call = nullptr; }
const CallExpr* GetCall() const { return call; }
void SetCallLoc(const Location* loc) { call_loc = loc; }
const detail::Location* GetCallLocation() const;
void SetDelayed() { delayed = true; }
@ -389,7 +391,8 @@ private:
Stmt* next_stmt;
trigger::TriggerPtr trigger;
const CallExpr* call;
const CallExpr* call = nullptr;
const Location* call_loc = nullptr; // only needed if call is nil
std::unique_ptr<std::vector<ScriptFunc*>> functions_with_closure_frame_reference;
};

38
src/Func.cc
View file

@ -44,7 +44,7 @@
#include "zeek/File.h"
#include "zeek/Frame.h"
#include "zeek/Var.h"
#include "zeek/analyzer/protocol/login/Login.h"
#include "zeek/analyzer/protocol/tcp/TCP.h"
#include "zeek/session/Manager.h"
#include "zeek/RE.h"
#include "zeek/Event.h"
@ -62,6 +62,7 @@
#include "option.bif.func_h"
#include "supervisor.bif.func_h"
#include "packet_analysis.bif.func_h"
#include "CPP-load.bif.func_h"
#include "zeek.bif.func_def"
#include "stats.bif.func_def"
@ -70,6 +71,7 @@
#include "option.bif.func_def"
#include "supervisor.bif.func_def"
#include "packet_analysis.bif.func_def"
#include "CPP-load.bif.func_def"
extern RETSIGTYPE sig_handler(int signo);
@ -199,8 +201,8 @@ void Func::DescribeDebug(ODesc* d, const Args* args) const
detail::TraversalCode Func::Traverse(detail::TraversalCallback* cb) const
{
// FIXME: Make a fake scope for builtins?
detail::Scope* old_scope = cb->current_scope;
cb->current_scope = scope.get();
auto old_scope = cb->current_scope;
cb->current_scope = scope;
detail::TraversalCode tc = cb->PreFunction(this);
HANDLE_TC_STMT_PRE(tc);
@ -305,11 +307,35 @@ ScriptFunc::ScriptFunc(const IDPtr& arg_id, StmtPtr arg_body,
Body b;
b.stmts = AddInits(std::move(arg_body), aggr_inits);
current_body = b.stmts;
b.priority = priority;
current_priority = b.priority = priority;
bodies.push_back(b);
}
}
ScriptFunc::ScriptFunc(std::string _name, FuncTypePtr ft,
std::vector<StmtPtr> bs, std::vector<int> priorities)
{
name = std::move(_name);
frame_size = ft->ParamList()->GetTypes().size();
type = std::move(ft);
auto n = bs.size();
ASSERT(n == priorities.size());
for ( auto i = 0u; i < n; ++i )
{
Body b;
b.stmts = std::move(bs[i]);
b.priority = priorities[i];
bodies.push_back(b);
}
sort(bodies.begin(), bodies.end());
current_body = bodies[0].stmts;
current_priority = bodies[0].priority;
}
ScriptFunc::~ScriptFunc()
{
if ( ! weak_closure_ref )
@ -542,9 +568,8 @@ void ScriptFunc::AddBody(StmtPtr new_body,
Body b;
b.stmts = new_body;
b.priority = priority;
current_body = new_body;
current_priority = b.priority = priority;
bodies.push_back(b);
sort(bodies.begin(), bodies.end());
@ -558,6 +583,7 @@ void ScriptFunc::ReplaceBody(const StmtPtr& old_body, StmtPtr new_body)
if ( body.stmts.get() == old_body.get() )
{
body.stmts = new_body;
current_priority = body.priority;
found_it = true;
}

14
src/Func.h
View file

@ -12,7 +12,7 @@
#include "zeek/ZeekList.h"
#include "zeek/Stmt.h"
#include "zeek/Obj.h"
#include "zeek/IntrusivePtr.h"
#include "zeek/Scope.h"
#include "zeek/Type.h" /* for function_flavor */
#include "zeek/TraverseTypes.h"
#include "zeek/ZeekArgs.h"
@ -103,7 +103,7 @@ public:
size_t new_frame_size, int priority = 0);
virtual void SetScope(detail::ScopePtr newscope);
virtual detail::Scope* GetScope() const { return scope.get(); }
virtual detail::ScopePtr GetScope() const { return scope; }
const FuncTypePtr& GetType() const
{ return type; }
@ -151,6 +151,10 @@ public:
const std::vector<IDPtr>& inits,
size_t frame_size, int priority);
// For compiled scripts.
ScriptFunc(std::string name, FuncTypePtr ft,
std::vector<StmtPtr> bodies, std::vector<int> priorities);
~ScriptFunc() override;
bool IsPure() const override;
@ -239,6 +243,7 @@ public:
detail::StmtPtr new_body);
StmtPtr CurrentBody() const { return current_body; }
int CurrentPriority() const { return current_priority; }
/**
* Returns the function's frame size.
@ -307,8 +312,11 @@ private:
OffsetMap* captures_offset_mapping = nullptr;
// The most recently added/updated body.
// The most recently added/updated body ...
StmtPtr current_body;
// ... and its priority.
int current_priority;
};
using built_in_func = BifReturnVal (*)(Frame* frame, const Args* args);

1
src/Hash.h
View file

@ -261,6 +261,7 @@ public:
int Size() const { return size; }
hash_t Hash() const { return hash; }
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
unsigned int MemoryAllocation() const { return padded_sizeof(*this) + util::pad_size(size); }
static hash_t HashBytes(const void* bytes, int size);

5
src/ID.cc
View file

@ -285,6 +285,11 @@ const AttrPtr& ID::GetAttr(AttrTag t) const
return attrs ? attrs->Find(t) : Attr::nil;
}
void ID::AddInitExpr(ExprPtr init_expr)
{
init_exprs.emplace_back(std::move(init_expr));
}
bool ID::IsDeprecated() const
{
return GetAttr(ATTR_DEPRECATED) != nullptr;

11
src/ID.h
View file

@ -112,6 +112,10 @@ public:
const AttrPtr& GetAttr(AttrTag t) const;
void AddInitExpr(ExprPtr init_expr);
const std::vector<ExprPtr>& GetInitExprs() const
{ return init_exprs; }
bool IsDeprecated() const;
void MakeDeprecated(ExprPtr deprecation);
@ -156,6 +160,13 @@ protected:
int offset;
ValPtr val;
AttributesPtr attrs;
// Expressions used to initialize the identifier, for use by
// the scripts-to-C++ compiler. We need to track all of them
// because it's possible that a global value gets created using
// one of the earlier instances rather than the last one.
std::vector<ExprPtr> init_exprs;
// contains list of functions that are called when an option changes
std::multimap<int, FuncPtr> option_handlers;

69
src/IP.cc
View file

@ -157,7 +157,6 @@ RecordValPtr IPv6_Hdr::ToVal(VectorValPtr chain) const
}
break;
#ifdef ENABLE_MOBILE_IPV6
case IPPROTO_MOBILITY:
{
static auto ip6_mob_type = id::find_type<RecordType>("ip6_mobility_hdr");
@ -290,7 +289,6 @@ RecordValPtr IPv6_Hdr::ToVal(VectorValPtr chain) const
rv->Assign(5, std::move(msg));
}
break;
#endif //ENABLE_MOBILE_IPV6
default:
break;
@ -445,9 +443,7 @@ static inline bool isIPv6ExtHeader(uint8_t type)
case IPPROTO_FRAGMENT:
case IPPROTO_AH:
case IPPROTO_ESP:
#ifdef ENABLE_MOBILE_IPV6
case IPPROTO_MOBILITY:
#endif
return true;
default:
return false;
@ -457,9 +453,7 @@ static inline bool isIPv6ExtHeader(uint8_t type)
IPv6_Hdr_Chain::~IPv6_Hdr_Chain()
{
for ( size_t i = 0; i < chain.size(); ++i ) delete chain[i];
#ifdef ENABLE_MOBILE_IPV6
delete homeAddr;
#endif
delete finalDst;
}
@ -509,11 +503,9 @@ void IPv6_Hdr_Chain::Init(const struct ip6_hdr* ip6, int total_len,
if ( current_type == IPPROTO_ROUTING )
ProcessRoutingHeader((const struct ip6_rthdr*) hdrs, cur_len);
#ifdef ENABLE_MOBILE_IPV6
// Only Mobile IPv6 has a destination option we care about right now.
if ( current_type == IPPROTO_DSTOPTS )
ProcessDstOpts((const struct ip6_dest*) hdrs, cur_len);
#endif
hdrs += cur_len;
length += cur_len;
@ -521,9 +513,7 @@ void IPv6_Hdr_Chain::Init(const struct ip6_hdr* ip6, int total_len,
} while ( current_type != IPPROTO_FRAGMENT &&
current_type != IPPROTO_ESP &&
#ifdef ENABLE_MOBILE_IPV6
current_type != IPPROTO_MOBILITY &&
#endif
isIPv6ExtHeader(next_type) );
}
@ -540,10 +530,8 @@ bool IPv6_Hdr_Chain::IsFragment() const
IPAddr IPv6_Hdr_Chain::SrcAddr() const
{
#ifdef ENABLE_MOBILE_IPV6
if ( homeAddr )
return IPAddr(*homeAddr);
#endif
if ( chain.empty() )
{
reporter->InternalWarning("empty IPv6 header chain");
@ -595,7 +583,6 @@ void IPv6_Hdr_Chain::ProcessRoutingHeader(const struct ip6_rthdr* r, uint16_t le
}
break;
#ifdef ENABLE_MOBILE_IPV6
case 2: // Defined by Mobile IPv6 RFC 6275.
{
if ( r->ip6r_segleft > 0 )
@ -607,7 +594,6 @@ void IPv6_Hdr_Chain::ProcessRoutingHeader(const struct ip6_rthdr* r, uint16_t le
}
}
break;
#endif
default:
reporter->Weird(SrcAddr(), DstAddr(), "unknown_routing_type",
@ -616,9 +602,14 @@ void IPv6_Hdr_Chain::ProcessRoutingHeader(const struct ip6_rthdr* r, uint16_t le
}
}
#ifdef ENABLE_MOBILE_IPV6
void IPv6_Hdr_Chain::ProcessDstOpts(const struct ip6_dest* d, uint16_t len)
{
// Skip two bytes to get the beginning of the first option structure. These
// two bytes are the protocol for the next header and extension header length,
// already known to exist before calling this method. See header format:
// https://datatracker.ietf.org/doc/html/rfc8200#section-4.6
assert(len >= 2);
const u_char* data = (const u_char*) d;
len -= 2 * sizeof(uint8_t);
data += 2* sizeof(uint8_t);
@ -627,35 +618,45 @@ void IPv6_Hdr_Chain::ProcessDstOpts(const struct ip6_dest* d, uint16_t len)
{
const struct ip6_opt* opt = (const struct ip6_opt*) data;
switch ( opt->ip6o_type ) {
case 201: // Home Address Option, Mobile IPv6 RFC 6275 section 6.3
case 0:
// If option type is zero, it's a Pad0 and can be just a single
// byte in width. Skip over it.
data += sizeof(uint8_t);
len -= sizeof(uint8_t);
break;
default:
{
// Double-check that the len can hold the whole option structure.
// Otherwise we get a buffer-overflow when we check the option_len.
// Also check that it holds everything for the option itself.
if ( len < sizeof(struct ip6_opt) ||
len < sizeof(struct ip6_opt) + opt->ip6o_len )
{
reporter->Weird(SrcAddr(), DstAddr(), "bad_ipv6_dest_opt_len");
len = 0;
break;
}
if ( opt->ip6o_type == 201 ) // Home Address Option, Mobile IPv6 RFC 6275 section 6.3
{
if ( opt->ip6o_len == sizeof(struct in6_addr) )
{
if ( opt->ip6o_len == 16 )
if ( homeAddr )
reporter->Weird(SrcAddr(), DstAddr(), "multiple_home_addr_opts");
else
homeAddr = new IPAddr(*((const in6_addr*)(data + 2)));
homeAddr = new IPAddr(*((const in6_addr*)(data + sizeof(struct ip6_opt))));
}
else
reporter->Weird(SrcAddr(), DstAddr(), "bad_home_addr_len");
}
break;
default:
data += sizeof(struct ip6_opt) + opt->ip6o_len;
len -= sizeof(struct ip6_opt) + opt->ip6o_len;
}
break;
}
if ( opt->ip6o_type == 0 )
{
data += sizeof(uint8_t);
len -= sizeof(uint8_t);
}
else
{
data += 2 * sizeof(uint8_t) + opt->ip6o_len;
len -= 2 * sizeof(uint8_t) + opt->ip6o_len;
}
}
}
#endif
VectorValPtr IPv6_Hdr_Chain::ToVal() const
{
@ -695,11 +696,9 @@ VectorValPtr IPv6_Hdr_Chain::ToVal() const
case IPPROTO_ESP:
ext_hdr->Assign(6, std::move(v));
break;
#ifdef ENABLE_MOBILE_IPV6
case IPPROTO_MOBILITY:
ext_hdr->Assign(7, std::move(v));
break;
#endif
default:
reporter->InternalWarning("IPv6_Hdr_Chain bad header %d", type);
continue;
@ -732,10 +731,8 @@ IPv6_Hdr_Chain* IPv6_Hdr_Chain::Copy(const ip6_hdr* new_hdr) const
IPv6_Hdr_Chain* rval = new IPv6_Hdr_Chain;
rval->length = length;
#ifdef ENABLE_MOBILE_IPV6
if ( homeAddr )
rval->homeAddr = new IPAddr(*homeAddr);
#endif
if ( finalDst )
rval->finalDst = new IPAddr(*finalDst);

16
src/IP.h
View file

@ -26,8 +26,6 @@ using VectorValPtr = IntrusivePtr<VectorVal>;
namespace detail { class FragReassembler; }
#ifdef ENABLE_MOBILE_IPV6
#ifndef IPPROTO_MOBILITY
#define IPPROTO_MOBILITY 135
#endif
@ -40,8 +38,6 @@ struct ip6_mobility {
uint16_t ip6mob_chksum;
};
#endif //ENABLE_MOBILE_IPV6
/**
* Base class for IPv6 header/extensions.
*/
@ -66,9 +62,7 @@ public:
case IPPROTO_ROUTING:
case IPPROTO_FRAGMENT:
case IPPROTO_AH:
#ifdef ENABLE_MOBILE_IPV6
case IPPROTO_MOBILITY:
#endif
((ip6_ext*)data)->ip6e_nxt = next_type;
break;
case IPPROTO_ESP:
@ -93,9 +87,7 @@ public:
case IPPROTO_ROUTING:
case IPPROTO_FRAGMENT:
case IPPROTO_AH:
#ifdef ENABLE_MOBILE_IPV6
case IPPROTO_MOBILITY:
#endif
return ((ip6_ext*)data)->ip6e_nxt;
case IPPROTO_ESP:
default:
@ -114,9 +106,7 @@ public:
case IPPROTO_HOPOPTS:
case IPPROTO_DSTOPTS:
case IPPROTO_ROUTING:
#ifdef ENABLE_MOBILE_IPV6
case IPPROTO_MOBILITY:
#endif
return 8 + 8 * ((ip6_ext*)data)->ip6e_len;
case IPPROTO_FRAGMENT:
return 8;
@ -262,13 +252,11 @@ protected:
*/
void ProcessRoutingHeader(const struct ip6_rthdr* r, uint16_t len);
#ifdef ENABLE_MOBILE_IPV6
/**
* Inspect a Destination Option header's options for things we need to
* remember, such as the Home Address option from Mobile IPv6.
*/
void ProcessDstOpts(const struct ip6_dest* d, uint16_t len);
#endif
std::vector<IPv6_Hdr*> chain;
@ -277,12 +265,10 @@ protected:
*/
uint16_t length = 0;
#ifdef ENABLE_MOBILE_IPV6
/**
* Home Address of the packet's source as defined by Mobile IPv6 (RFC 6275).
*/
IPAddr* homeAddr = nullptr;
#endif
/**
* The final destination address in chain's first Routing header that has
@ -394,7 +380,6 @@ public:
return ((const u_char*) ip6) + ip6_hdrs->TotalLength();
}
#ifdef ENABLE_MOBILE_IPV6
/**
* Returns a pointer to the mobility header of the IP packet, if present,
* else a null pointer.
@ -408,7 +393,6 @@ public:
else
return (const ip6_mobility*)(*ip6_hdrs)[ip6_hdrs->Size()-1]->Data();
}
#endif
/**
* Returns the length of the IP packet's payload (length of packet minus

16
src/IPAddr.cc
View file

@ -44,12 +44,26 @@ ConnKey::ConnKey(const IPAddr& src, const IPAddr& dst, uint16_t src_port,
}
}
detail::ConnKey::ConnKey(const ConnTuple& id)
ConnKey::ConnKey(const ConnTuple& id)
: ConnKey(id.src_addr, id.dst_addr, id.src_port, id.dst_port,
id.proto, id.is_one_way)
{
}
ConnKey& ConnKey::operator=(const ConnKey& rhs)
{
if ( this == &rhs )
return *this;
memcpy(&ip1, &rhs.ip1, sizeof(in6_addr));
memcpy(&ip2, &rhs.ip2, sizeof(in6_addr));
port1 = rhs.port1;
port2 = rhs.port2;
transport = rhs.transport;
return *this;
}
} // namespace detail
IPAddr::IPAddr(const String& s)

30
src/IPAddr.h
View file

@ -4,13 +4,13 @@
#include <netinet/in.h>
#include <arpa/inet.h>
#include <string.h>
#include <cstring>
#include <string>
#include <memory>
#include "zeek/threading/SerialTypes.h"
typedef in_addr in4_addr;
using in4_addr = in_addr;
namespace zeek {
@ -31,10 +31,7 @@ struct ConnKey {
ConnKey(const IPAddr& src, const IPAddr& dst, uint16_t src_port,
uint16_t dst_port, TransportProto t, bool one_way);
ConnKey(const ConnTuple& conn);
ConnKey(const ConnKey& rhs)
{
*this = rhs;
}
ConnKey(const ConnKey& rhs) { *this = rhs; }
bool operator<(const ConnKey& rhs) const { return memcmp(this, &rhs, sizeof(ConnKey)) < 0; }
bool operator<=(const ConnKey& rhs) const { return memcmp(this, &rhs, sizeof(ConnKey)) <= 0; }
@ -43,13 +40,7 @@ struct ConnKey {
bool operator>=(const ConnKey& rhs) const { return memcmp(this, &rhs, sizeof(ConnKey)) >= 0; }
bool operator>(const ConnKey& rhs) const { return memcmp(this, &rhs, sizeof(ConnKey)) > 0; }
ConnKey& operator=(const ConnKey& rhs)
{
if ( this != &rhs )
memcpy(this, &rhs, sizeof(ConnKey));
return *this;
}
ConnKey& operator=(const ConnKey& rhs);
};
using ConnIDKey [[deprecated("Remove in v5.1. Use zeek::detail::ConnKey.")]] = ConnKey;
@ -64,7 +55,7 @@ public:
/**
* Address family.
*/
typedef IPFamily Family;
using Family = IPFamily;
/**
* Byte order.
@ -387,11 +378,13 @@ public:
return ! ( addr1 <= addr2 );
}
/** Converts the address into the type used internally by the
/**
* Converts the address into the type used internally by the
* inter-thread communication.
*/
void ConvertToThreadingValue(threading::Value::addr_t* v) const;
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
unsigned int MemoryAllocation() const { return padded_sizeof(*this); }
/**
@ -607,7 +600,8 @@ public:
*/
uint8_t LengthIPv6() const { return length; }
/** Returns true if the given address is part of the prefix.
/**
* Returns true if the given address is part of the prefix.
*
* @param addr The address to test.
*/
@ -643,7 +637,8 @@ public:
*/
std::unique_ptr<detail::HashKey> MakeHashKey() const;
/** Converts the prefix into the type used internally by the
/**
* Converts the prefix into the type used internally by the
* inter-thread communication.
*/
void ConvertToThreadingValue(threading::Value::subnet_t* v) const
@ -652,6 +647,7 @@ public:
prefix.ConvertToThreadingValue(&v->prefix);
}
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
unsigned int MemoryAllocation() const { return padded_sizeof(*this); }
/**

19
src/IntrusivePtr.h
View file

@ -137,12 +137,29 @@ public:
return std::exchange(ptr_, nullptr);
}
IntrusivePtr& operator=(IntrusivePtr other) noexcept
IntrusivePtr& operator=(const IntrusivePtr& other) noexcept
{
IntrusivePtr tmp{other};
swap(tmp);
return *this;
}
IntrusivePtr& operator=(IntrusivePtr&& other) noexcept
{
swap(other);
return *this;
}
IntrusivePtr& operator=(std::nullptr_t) noexcept
{
if ( ptr_ )
{
Unref(ptr_);
ptr_ = nullptr;
}
return *this;
}
pointer get() const noexcept
{
return ptr_;

1
src/List.h
View file

@ -158,6 +158,7 @@ public:
return max_entries;
}
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
int MemoryAllocation() const
{ return padded_sizeof(*this) + util::pad_size(max_entries * sizeof(T)); }

3
src/NFA.cc
View file

@ -157,9 +157,12 @@ void NFA_State::Dump(FILE* f)
unsigned int NFA_State::TotalMemoryAllocation() const
{
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
return padded_sizeof(*this)
+ xtions.MemoryAllocation() - padded_sizeof(xtions)
+ (epsclosure ? epsclosure->MemoryAllocation() : 0);
#pragma GCC diagnostic pop
}
NFA_Machine::NFA_Machine(NFA_State* first, NFA_State* final)

9
src/NFA.h
View file

@ -62,6 +62,7 @@ public:
void Dump(FILE* f);
// Recursivly count all the reachable states.
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
unsigned int TotalMemoryAllocation() const;
protected:
@ -118,8 +119,14 @@ public:
void Describe(ODesc* d) const override;
void Dump(FILE* f);
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
unsigned int MemoryAllocation() const
{ return padded_sizeof(*this) + first_state->TotalMemoryAllocation(); }
{
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
return padded_sizeof(*this) + first_state->TotalMemoryAllocation();
#pragma GCC diagnostic pop
}
protected:
NFA_State* first_state;

8
src/NetVar.cc
View file

@ -157,9 +157,6 @@ int dns_skip_all_auth;
int dns_skip_all_addl;
int dns_max_queries;
double stp_delta;
double stp_idle_min;
double table_expire_interval;
double table_expire_delay;
int table_incremental_step;
@ -312,11 +309,6 @@ void init_net_var()
dns_skip_all_addl = id::find_val("dns_skip_all_addl")->AsBool();
dns_max_queries = id::find_val("dns_max_queries")->AsCount();
stp_delta = 0.0;
if ( const auto& v = id::find_val("stp_delta") ) stp_delta = v->AsInterval();
stp_idle_min = 0.0;
if ( const auto& v = id::find_val("stp_idle_min") ) stp_delta = v->AsInterval();
orig_addr_anonymization = 0;
if ( const auto& id = id::find("orig_addr_anonymization") )
if ( const auto& v = id->GetVal() )

2
src/NetVar.h
View file

@ -59,8 +59,6 @@ extern int dns_skip_all_auth;
extern int dns_skip_all_addl;
extern int dns_max_queries;
extern double stp_delta;
extern double stp_idle_min;
extern double table_expire_interval;
extern double table_expire_delay;
extern int table_incremental_step;

42
src/Options.cc
View file

@ -158,30 +158,52 @@ static void set_analysis_option(const char* opt, Options& opts)
if ( util::streq(opt, "help") )
{
fprintf(stderr, "--optimize options:\n");
fprintf(stderr, " all equivalent to \"inline\" and \"activate\"\n");
fprintf(stderr, " add-C++ generate private C++ for any missing script bodies\n");
fprintf(stderr, " compile-all *if* compiling, compile all scripts, even inlined ones\n");
fprintf(stderr, " dump-uds dump use-defs to stdout; implies xform\n");
fprintf(stderr, " dump-xform dump transformed scripts to stdout; implies xform\n");
fprintf(stderr, " gen-C++ generate C++ script bodies\n");
fprintf(stderr, " gen-standalone-C++ generate \"standalone\" C++ script bodies\n");
fprintf(stderr, " help print this list\n");
fprintf(stderr, " inline inline function calls\n");
fprintf(stderr, " optimize-AST optimize the (transformed) AST; implies xform\n");
fprintf(stderr, " recursive report on recursive functions and exit\n");
fprintf(stderr, " report-C++ report available C++ script bodies and exit\n");
fprintf(stderr, " update-C++ generate reusable C++ for any missing script bodies\n");
fprintf(stderr, " use-C++ use available C++ script bodies\n");
fprintf(stderr, " xform tranform scripts to \"reduced\" form\n");
exit(0);
}
auto& a_o = opts.analysis_options;
if ( util::streq(opt, "dump-uds") )
if ( util::streq(opt, "add-C++") )
a_o.add_CPP = true;
else if ( util::streq(opt, "compile-all") )
a_o.activate = a_o.compile_all = true;
else if ( util::streq(opt, "dump-uds") )
a_o.activate = a_o.dump_uds = true;
else if ( util::streq(opt, "dump-xform") )
a_o.activate = a_o.dump_xform = true;
else if ( util::streq(opt, "gen-C++") )
a_o.gen_CPP = true;
else if ( util::streq(opt, "gen-standalone-C++") )
a_o.gen_standalone_CPP = true;
else if ( util::streq(opt, "inline") )
a_o.inliner = true;
else if ( util::streq(opt, "recursive") )
a_o.inliner = a_o.report_recursive = true;
else if ( util::streq(opt, "xform") )
a_o.activate = true;
else if ( util::streq(opt, "optimize-AST") )
a_o.activate = a_o.optimize_AST = true;
else if ( util::streq(opt, "recursive") )
a_o.inliner = a_o.report_recursive = true;
else if ( util::streq(opt, "report-C++") )
a_o.report_CPP = true;
else if ( util::streq(opt, "update-C++") )
a_o.update_CPP = true;
else if ( util::streq(opt, "use-C++") )
a_o.use_CPP = true;
else if ( util::streq(opt, "xform") )
a_o.activate = true;
else
{
@ -331,13 +353,9 @@ Options parse_cmdline(int argc, char** argv)
};
char opts[256];
util::safe_strncpy(opts, "B:e:f:G:H:I:i:j::n:O:o:p:r:s:T:t:U:w:X:CDFNPQSWabdhuv",
util::safe_strncpy(opts, "B:e:f:G:H:I:i:j::n:O:o:p:r:s:T:t:U:w:X:CDFMNPQSWabdhmuv",
sizeof(opts));
#ifdef USE_PERFTOOLS_DEBUG
strncat(opts, "mM", 2);
#endif
int op;
int long_optsind;
opterr = 0;
@ -425,9 +443,13 @@ Options parse_cmdline(int argc, char** argv)
case 'w':
rval.pcap_output_file = optarg;
break;
#ifdef DEBUG
case 'B':
rval.debug_log_streams = optarg;
break;
#endif
case 'C':
rval.ignore_checksums = true;
break;

9
src/RE.cc
View file

@ -416,14 +416,20 @@ unsigned int Specific_RE_Matcher::MemoryAllocation() const
{
unsigned int size = 0;
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
for ( int i = 0; i < ccl_list.length(); ++i )
size += ccl_list[i]->MemoryAllocation();
#pragma GCC diagnostic pop
size += util::pad_size(sizeof(CCL*) * ccl_dict.size());
for ( const auto& entry : ccl_dict )
{
size += padded_sizeof(std::string) + util::pad_size(sizeof(std::string::value_type) * entry.first.size());
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
size += entry.second->MemoryAllocation();
#pragma GCC diagnostic pop
}
for ( const auto& entry : defs )
@ -432,6 +438,8 @@ unsigned int Specific_RE_Matcher::MemoryAllocation() const
size += padded_sizeof(std::string) + util::pad_size(sizeof(std::string::value_type) * entry.second.size());
}
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
return size + padded_sizeof(*this)
+ (pattern_text ? util::pad_size(strlen(pattern_text) + 1) : 0)
+ ccl_list.MemoryAllocation() - padded_sizeof(ccl_list)
@ -440,6 +448,7 @@ unsigned int Specific_RE_Matcher::MemoryAllocation() const
+ padded_sizeof(*any_ccl)
+ padded_sizeof(*accepted) // NOLINT(bugprone-sizeof-container)
+ accepted->size() * padded_sizeof(AcceptingSet::key_type);
#pragma GCC diagnostic pop
}
static RE_Matcher* matcher_merge(const RE_Matcher* re1, const RE_Matcher* re2,

5
src/RE.h
View file

@ -115,6 +115,7 @@ public:
void Dump(FILE* f);
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
unsigned int MemoryAllocation() const;
protected:
@ -232,11 +233,15 @@ public:
// the main ("explicit") constructor was used.
const char* OrigText() const { return orig_text.c_str(); }
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
unsigned int MemoryAllocation() const
{
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
return padded_sizeof(*this)
+ (re_anywhere ? re_anywhere->MemoryAllocation() : 0)
+ (re_exact ? re_exact->MemoryAllocation() : 0);
#pragma GCC diagnostic pop
}
protected:

2
src/Reassem.h
View file

@ -282,9 +282,11 @@ public:
void Describe(ODesc* d) const override;
// Sum over all data buffered in some reassembler.
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
static uint64_t TotalMemoryAllocation() { return total_size; }
// Data buffered by type of reassembler.
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
static uint64_t MemoryAllocation(ReassemblerType rtype);
void SetMaxOldBlocks(uint32_t count) { max_old_blocks = count; }

19
src/Reporter.cc
View file

@ -187,6 +187,21 @@ void Reporter::RuntimeError(const detail::Location* location, const char* fmt, .
throw InterpreterException();
}
void Reporter::CPPRuntimeError(const char* fmt, ...)
{
++errors;
va_list ap;
va_start(ap, fmt);
FILE* out = EmitToStderr(errors_to_stderr) ? stderr : nullptr;
DoLog("runtime error in compiled code", reporter_error, out, nullptr, nullptr, true, true, "", fmt, ap);
va_end(ap);
if ( abort_on_scripting_errors )
abort();
throw InterpreterException();
}
void Reporter::InternalError(const char* fmt, ...)
{
va_list ap;
@ -365,6 +380,10 @@ bool Reporter::PermitFlowWeird(const char* name,
bool Reporter::PermitExpiredConnWeird(const char* name, const RecordVal& conn_id)
{
if ( !conn_id.HasField("orig_h") || !conn_id.HasField("resp_h") ||
!conn_id.HasField("orig_p") || !conn_id.HasField("resp_p") )
return false;
auto conn_tuple = std::make_tuple(conn_id.GetFieldAs<AddrVal>("orig_h"),
conn_id.GetFieldAs<AddrVal>("resp_h"),
conn_id.GetFieldAs<PortVal>("orig_p")->Port(),

4
src/Reporter.h
View file

@ -100,6 +100,10 @@ public:
// function will not return but raise an InterpreterException.
[[noreturn]] void RuntimeError(const detail::Location* location, const char* fmt, ...) __attribute__((format(printf, 3, 4)));
// Report a runtime error in executing a compiled script. This
// function will not return but raise an InterpreterException.
[[noreturn]] void CPPRuntimeError(const char* fmt, ...) __attribute__((format(printf, 2, 3)));
// Report a traffic weirdness, i.e., an unexpected protocol situation
// that may lead to incorrectly processing a connnection.
void Weird(const char* name, const char* addl = "", const char* source = ""); // Raises net_weird().

10
src/RuleCondition.cc
View file

@ -25,12 +25,12 @@ namespace zeek::detail {
bool RuleConditionTCPState::DoMatch(Rule* rule, RuleEndpointState* state,
const u_char* data, int len)
{
analyzer::Analyzer* root = state->GetAnalyzer()->Conn()->GetRootAnalyzer();
auto* adapter = state->GetAnalyzer()->Conn()->GetSessionAdapter();
if ( ! root || ! root->IsAnalyzer("TCP") )
if ( ! adapter || ! adapter->IsAnalyzer("TCP") )
return false;
auto* ta = static_cast<analyzer::tcp::TCP_Analyzer*>(root);
auto* ta = static_cast<packet_analysis::TCP::TCPSessionAdapter*>(adapter);
if ( tcpstates & RULE_STATE_STATELESS )
return true;
@ -57,9 +57,9 @@ void RuleConditionTCPState::PrintDebug()
bool RuleConditionUDPState::DoMatch(Rule* rule, RuleEndpointState* state,
const u_char* data, int len)
{
analyzer::Analyzer* root = state->GetAnalyzer()->Conn()->GetRootAnalyzer();
auto* adapter = state->GetAnalyzer()->Conn()->GetSessionAdapter();
if ( ! root || ! root->IsAnalyzer("UDP") )
if ( ! adapter || ! adapter->IsAnalyzer("UDP") )
return false;
if ( states & RULE_STATE_STATELESS )

7
src/RunState.cc
View file

@ -37,7 +37,6 @@ extern "C" {
#include "zeek/plugin/Manager.h"
#include "zeek/broker/Manager.h"
#include "zeek/packet_analysis/Manager.h"
#include "zeek/analyzer/protocol/stepping-stone/SteppingStone.h"
extern "C" {
extern int select(int, fd_set *, fd_set *, fd_set *, struct timeval *);
@ -46,8 +45,6 @@ extern int select(int, fd_set *, fd_set *, fd_set *, struct timeval *);
static double last_watchdog_proc_time = 0.0; // value of above during last watchdog
extern int signal_val;
using namespace zeek::analyzer::stepping_stone;
namespace zeek::run_state {
namespace detail {
@ -197,9 +194,6 @@ void init_run(const std::optional<std::string>& interface,
session_mgr = new session::Manager();
// Initialize the stepping stone manager. We intentionally throw away the result here.
SteppingStoneManager::Get();
if ( do_watchdog )
{
// Set up the watchdog to make sure we don't wedge.
@ -414,7 +408,6 @@ void delete_run()
util::detail::set_processing_status("TERMINATING", "delete_run");
delete session_mgr;
delete SteppingStoneManager::Get();
for ( int i = 0; i < zeek::detail::NUM_ADDR_ANONYMIZATION_METHODS; ++i )
delete zeek::detail::ip_anonymizer[i];

25
src/Scope.cc
View file

@ -12,10 +12,8 @@
namespace zeek::detail {
using scope_list = PList<Scope>;
static scope_list scopes;
static Scope* top_scope;
static std::vector<ScopePtr> scopes;
static ScopePtr top_scope;
Scope::Scope(IDPtr id,
std::unique_ptr<std::vector<AttrPtr>> al)
@ -117,9 +115,9 @@ const IDPtr& lookup_ID(const char* name, const char* curr_module,
bool need_export = check_export && (ID_module != GLOBAL_MODULE_NAME &&
ID_module != curr_module);
for ( int i = scopes.length() - 1; i >= 0; --i )
for ( auto s_i = scopes.rbegin(); s_i != scopes.rend(); ++s_i )
{
const auto& id = scopes[i]->Find(fullname);
const auto& id = (*s_i)->Find(fullname);
if ( id )
{
@ -172,16 +170,15 @@ IDPtr install_ID(const char* name, const char* module_name,
return id;
}
void push_existing_scope(Scope* scope)
void push_existing_scope(ScopePtr scope)
{
top_scope = scope;
scopes.push_back(scope);
}
void push_scope(IDPtr id,
std::unique_ptr<std::vector<AttrPtr>> attrs)
void push_scope(IDPtr id, std::unique_ptr<std::vector<AttrPtr>> attrs)
{
top_scope = new Scope(std::move(id), std::move(attrs));
top_scope = make_intrusive<Scope>(std::move(id), std::move(attrs));
scopes.push_back(top_scope);
}
@ -191,19 +188,19 @@ ScopePtr pop_scope()
reporter->InternalError("scope underflow");
scopes.pop_back();
Scope* old_top = top_scope;
auto old_top = top_scope;
top_scope = scopes.empty() ? nullptr : scopes.back();
return {AdoptRef{}, old_top};
return old_top;
}
Scope* current_scope()
ScopePtr current_scope()
{
return top_scope;
}
Scope* global_scope()
ScopePtr global_scope()
{
return scopes.empty() ? 0 : scopes.front();
}

6
src/Scope.h
View file

@ -96,12 +96,12 @@ extern IDPtr install_ID(
bool is_global, bool is_export);
extern void push_scope(IDPtr id, std::unique_ptr<std::vector<AttrPtr>> attrs);
extern void push_existing_scope(Scope* scope);
extern void push_existing_scope(ScopePtr scope);
// Returns the one popped off.
extern ScopePtr pop_scope();
extern Scope* current_scope();
extern Scope* global_scope();
extern ScopePtr current_scope();
extern ScopePtr global_scope();
// Current module (identified by its name).
extern std::string current_module;

15
src/Stats.cc
View file

@ -15,6 +15,7 @@
#include "zeek/broker/Manager.h"
#include "zeek/input.h"
#include "zeek/Func.h"
#include "zeek/packet_analysis/protocol/tcp/TCP.h"
uint64_t zeek::detail::killed_by_inactivity = 0;
uint64_t& killed_by_inactivity = zeek::detail::killed_by_inactivity;
@ -126,7 +127,10 @@ void ProfileLogger::Log()
run_state::network_time, (utime + stime) - (first_utime + first_stime),
utime - first_utime, stime - first_stime, rtime - first_rtime));
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
int conn_mem_use = expensive ? session_mgr->SessionMemoryUsage() : 0;
#pragma GCC diagnostic pop
double avg_conn_mem_use = 0;
if ( expensive && session_mgr->CurrentSessions() != 0 )
@ -134,6 +138,8 @@ void ProfileLogger::Log()
// TODO: This previously output the number of connections, but now that we're storing sessions
// as well as connections, this might need to be renamed.
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
file->Write(util::fmt("%.06f Conns: total=%" PRIu64 " current=%" PRIu64 "/%" PRIi32 " mem=%" PRIi32 "K avg=%.1f table=%" PRIu32 "K connvals=%" PRIu32 "K\n",
run_state::network_time,
Connection::TotalConnections(),
@ -144,6 +150,7 @@ void ProfileLogger::Log()
expensive ? session_mgr->MemoryAllocation() / 1024 : 0,
expensive ? session_mgr->SessionMemoryUsageVals() / 1024 : 0
));
#pragma GCC diagnostic pop
session::Stats s;
session_mgr->GetStats(s);
@ -155,7 +162,7 @@ void ProfileLogger::Log()
s.num_ICMP_conns, s.max_ICMP_conns
));
session_mgr->tcp_stats.PrintStats(file,
packet_analysis::TCP::TCPAnalyzer::GetStats().PrintStats(file,
util::fmt("%.06f TCP-States:", run_state::network_time));
// Alternatively, if you prefer more compact output...
@ -177,8 +184,11 @@ void ProfileLogger::Log()
file->Write(util::fmt("%.06f Connections expired due to inactivity: %" PRIu64 "\n",
run_state::network_time, killed_by_inactivity));
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
file->Write(util::fmt("%.06f Total reassembler data: %" PRIu64 "K\n", run_state::network_time,
Reassembler::TotalMemoryAllocation() / 1024));
#pragma GCC diagnostic pop
// Signature engine.
if ( expensive && rule_matcher )
@ -273,7 +283,10 @@ void ProfileLogger::Log()
{
const auto& v = id->GetVal();
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
size = v->MemoryAllocation();
#pragma GCC diagnostic pop
mem += size;
bool print = false;

34
src/Stmt.cc
View file

@ -34,6 +34,7 @@ const char* stmt_name(StmtTag t)
"<init>", "fallthrough", "while",
"catch-return",
"check-any-length",
"compiled-C++",
"null",
};
@ -709,7 +710,7 @@ void SwitchStmt::Init()
t->Append(e->GetType());
comp_hash = new CompositeHash(std::move(t));
case_label_value_map.SetDeleteFunc(int_del_func);
case_label_hash_map.SetDeleteFunc(int_del_func);
}
SwitchStmt::SwitchStmt(ExprPtr index, case_list* arg_cases)
@ -854,12 +855,13 @@ bool SwitchStmt::AddCaseLabelValueMapping(const Val* v, int idx)
type_name(e->GetType()->Tag()));
}
int* label_idx = case_label_value_map.Lookup(hk.get());
int* label_idx = case_label_hash_map.Lookup(hk.get());
if ( label_idx )
return false;
case_label_value_map.Insert(hk.get(), new int(idx));
case_label_value_map[v] = idx;
case_label_hash_map.Insert(hk.get(), new int(idx));
return true;
}
@ -883,7 +885,7 @@ std::pair<int, ID*> SwitchStmt::FindCaseLabelMatch(const Val* v) const
ID* label_id = nullptr;
// Find matching expression cases.
if ( case_label_value_map.Length() )
if ( case_label_hash_map.Length() )
{
auto hk = comp_hash->MakeHashKey(*v, true);
@ -896,7 +898,7 @@ std::pair<int, ID*> SwitchStmt::FindCaseLabelMatch(const Val* v) const
return std::make_pair(-1, nullptr);
}
if ( auto i = case_label_value_map.Lookup(hk.get()) )
if ( auto i = case_label_hash_map.Lookup(hk.get()) )
label_idx = *i;
}
@ -1108,7 +1110,10 @@ WhileStmt::~WhileStmt() = default;
bool WhileStmt::IsPure() const
{
return loop_condition->IsPure() && body->IsPure();
if ( loop_condition->IsPure() && body->IsPure() )
return ! loop_cond_pred_stmt || loop_cond_pred_stmt->IsPure();
else
return false;
}
void WhileStmt::StmtDescribe(ODesc* d) const
@ -1118,6 +1123,13 @@ void WhileStmt::StmtDescribe(ODesc* d) const
if ( d->IsReadable() )
d->Add("(");
if ( loop_cond_pred_stmt )
{
d->Add(" {");
loop_cond_pred_stmt->Describe(d);
d->Add("} ");
}
loop_condition->Describe(d);
if ( d->IsReadable() )
@ -1135,6 +1147,12 @@ TraversalCode WhileStmt::Traverse(TraversalCallback* cb) const
TraversalCode tc = cb->PreStmt(this);
HANDLE_TC_STMT_PRE(tc);
if ( loop_cond_pred_stmt )
{
tc = loop_cond_pred_stmt->Traverse(cb);
HANDLE_TC_STMT_PRE(tc);
}
tc = loop_condition->Traverse(cb);
HANDLE_TC_STMT_PRE(tc);
@ -1513,7 +1531,7 @@ TraversalCode FallthroughStmt::Traverse(TraversalCallback* cb) const
ReturnStmt::ReturnStmt(ExprPtr arg_e)
: ExprStmt(STMT_RETURN, std::move(arg_e))
{
Scope* s = current_scope();
auto s = current_scope();
if ( ! s || ! s->GetID() )
{
@ -1547,7 +1565,7 @@ ReturnStmt::ReturnStmt(ExprPtr arg_e)
else
{
auto promoted_e = check_and_promote_expr(e.get(), yt.get());
auto promoted_e = check_and_promote_expr(e, yt);
if ( promoted_e )
e = std::move(promoted_e);

24
src/Stmt.h
View file

@ -16,9 +16,12 @@ class CompositeHash;
class NameExpr;
using NameExprPtr = IntrusivePtr<zeek::detail::NameExpr>;
class ZAMCompiler; // for "friend" declarations
class ExprListStmt : public Stmt {
public:
const ListExpr* ExprList() const { return l.get(); }
const ListExprPtr& ExprListPtr() const { return l; }
TraversalCode Traverse(TraversalCallback* cb) const override;
@ -180,15 +183,24 @@ public:
bool NoFlowAfter(bool ignore_break) const override;
protected:
friend class ZAMCompiler;
int DefaultCaseIndex() const { return default_case_idx; }
const auto& ValueMap() const { return case_label_value_map; }
const std::vector<std::pair<ID*, int>>* TypeMap() const
{ return &case_label_type_list; }
const CompositeHash* CompHash() const { return comp_hash; }
ValPtr DoExec(Frame* f, Val* v, StmtFlowType& flow) override;
bool IsPure() const override;
// Initialize composite hash and case label map.
void Init();
// Adds an entry in case_label_value_map for the given value to associate it
// with the given index in the cases list. If the entry already exists,
// returns false, else returns true.
// Adds entries in case_label_value_map and case_label_hash_map
// for the given value to associate it with the given index in
// the cases list. If the entry already exists, returns false,
// else returns true.
bool AddCaseLabelValueMapping(const Val* v, int idx);
// Adds an entry in case_label_type_map for the given type (w/ ID) to
@ -205,7 +217,8 @@ protected:
case_list* cases;
int default_case_idx;
CompositeHash* comp_hash;
PDict<int> case_label_value_map;
std::unordered_map<const Val*, int> case_label_value_map;
PDict<int> case_label_hash_map;
std::vector<std::pair<ID*, int>> case_label_type_list;
};
@ -523,6 +536,7 @@ public:
const Stmt* Body() const { return s1.get(); }
const Expr* TimeoutExpr() const { return timeout.get(); }
const Stmt* TimeoutBody() const { return s2.get(); }
bool IsReturn() const { return is_return; }
void StmtDescribe(ODesc* d) const override;
@ -597,6 +611,8 @@ class CheckAnyLenStmt : public ExprStmt {
public:
explicit CheckAnyLenStmt(ExprPtr e, int expected_len);
int ExpectedLen() const { return expected_len; }
ValPtr Exec(Frame* f, StmtFlowType& flow) override;
StmtPtr Duplicate() override;

1
src/StmtEnums.h
View file

@ -20,6 +20,7 @@ enum StmtTag {
STMT_WHILE,
STMT_CATCH_RETURN, // for reduced InlineExpr's
STMT_CHECK_ANY_LEN, // internal reduced statement
STMT_CPP, // compiled C++
STMT_NULL
#define NUM_STMTS (int(STMT_NULL) + 1)
};

4
src/Traverse.h
View file

@ -2,6 +2,7 @@
#pragma once
#include "zeek/Scope.h"
#include "zeek/TraverseTypes.h"
namespace zeek {
@ -10,7 +11,6 @@ class Func;
namespace detail {
class Scope;
class Stmt;
class Expr;
class ID;
@ -38,7 +38,7 @@ public:
virtual TraversalCode PreDecl(const ID*) { return TC_CONTINUE; }
virtual TraversalCode PostDecl(const ID*) { return TC_CONTINUE; }
Scope* current_scope;
ScopePtr current_scope;
};
TraversalCode traverse_all(TraversalCallback* cb);

62
src/Trigger.cc
View file

@ -98,15 +98,45 @@ protected:
double time;
};
Trigger::Trigger(Expr* arg_cond, Stmt* arg_body,
Stmt* arg_timeout_stmts,
Expr* arg_timeout, Frame* arg_frame,
bool arg_is_return, const Location* arg_location)
Trigger::Trigger(const Expr* cond, Stmt* body, Stmt* timeout_stmts,
Expr* timeout_expr, Frame* frame,
bool is_return, const Location* location)
{
timeout_value = -1;
if ( timeout_expr )
{
ValPtr timeout_val;
try
{
timeout_val = timeout_expr->Eval(frame);
}
catch ( InterpreterException& )
{ /* Already reported */ }
if ( timeout_val )
timeout_value = timeout_val->AsInterval();
}
Init(cond, body, timeout_stmts, frame, is_return, location);
}
Trigger::Trigger(const Expr* cond, Stmt* body, Stmt* timeout_stmts,
double timeout, Frame* frame,
bool is_return, const Location* location)
{
timeout_value = timeout;
Init(cond, body, timeout_stmts, frame, is_return, location);
}
void Trigger::Init(const Expr* arg_cond, Stmt* arg_body, Stmt* arg_timeout_stmts,
Frame* arg_frame, bool arg_is_return,
const Location* arg_location)
{
cond = arg_cond;
body = arg_body;
timeout_stmts = arg_timeout_stmts;
timeout = arg_timeout;
frame = arg_frame->Clone();
timer = nullptr;
delayed = false;
@ -114,7 +144,6 @@ Trigger::Trigger(Expr* arg_cond, Stmt* arg_body,
attached = nullptr;
is_return = arg_is_return;
location = arg_location;
timeout_value = -1;
DBG_LOG(DBG_NOTIFIERS, "%s: instantiating", Name());
@ -132,23 +161,6 @@ Trigger::Trigger(Expr* arg_cond, Stmt* arg_body,
arg_frame->SetDelayed();
}
ValPtr timeout_val;
if ( arg_timeout )
{
try
{
timeout_val = arg_timeout->Eval(arg_frame);
}
catch ( InterpreterException& )
{ /* Already reported */ }
}
if ( timeout_val )
{
timeout_value = timeout_val->AsInterval();
}
// Make sure we don't get deleted if somebody calls a method like
// Timeout() while evaluating the trigger.
Ref(this);
@ -198,7 +210,7 @@ Trigger::~Trigger()
// point.
}
void Trigger::Init(std::vector<ValPtr> index_expr_results)
void Trigger::ReInit(std::vector<ValPtr> index_expr_results)
{
assert(! disabled);
UnregisterAll();
@ -276,7 +288,7 @@ bool Trigger::Eval()
// Not true. Perhaps next time...
DBG_LOG(DBG_NOTIFIERS, "%s: trigger condition is false", Name());
Unref(f);
Init(std::move(index_expr_results));
ReInit(std::move(index_expr_results));
return false;
}

12
src/Trigger.h
View file

@ -38,7 +38,9 @@ public:
// instantiation. Note that if the condition is already true, the
// statements are executed immediately and the object is deleted
// right away.
Trigger(Expr* cond, Stmt* body, Stmt* timeout_stmts, Expr* timeout,
Trigger(const Expr* cond, Stmt* body, Stmt* timeout_stmts, Expr* timeout,
Frame* f, bool is_return, const Location* loc);
Trigger(const Expr* cond, Stmt* body, Stmt* timeout_stmts, double timeout,
Frame* f, bool is_return, const Location* loc);
~Trigger() override;
@ -95,12 +97,16 @@ private:
friend class TriggerTraversalCallback;
friend class TriggerTimer;
void Init(std::vector<IntrusivePtr<Val>> index_expr_results);
void Init(const Expr* cond, Stmt* body, Stmt* timeout_stmts, Frame* frame,
bool is_return, const Location* location);
void ReInit(std::vector<IntrusivePtr<Val>> index_expr_results);
void Register(ID* id);
void Register(Val* val);
void UnregisterAll();
Expr* cond;
const Expr* cond;
Stmt* body;
Stmt* timeout_stmts;
Expr* timeout;

167
src/Type.cc
View file

@ -239,7 +239,7 @@ int Type::MatchesIndex(detail::ListExpr* const index) const
if ( index->Exprs().length() != 1 && index->Exprs().length() != 2 )
return DOES_NOT_MATCH_INDEX;
if ( check_and_promote_exprs_to_type(index, zeek::base_type(TYPE_INT).get()) )
if ( check_and_promote_exprs_to_type(index, zeek::base_type(TYPE_INT)) )
return MATCHES_INDEX_SCALAR;
}
@ -335,6 +335,8 @@ unsigned int TypeList::MemoryAllocation() const
{
unsigned int size = 0;
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
for ( const auto& t : types )
size += t->MemoryAllocation();
@ -343,6 +345,7 @@ unsigned int TypeList::MemoryAllocation() const
return Type::MemoryAllocation()
+ padded_sizeof(*this) - padded_sizeof(Type)
+ size;
#pragma GCC diagnostic pop
}
int IndexType::MatchesIndex(detail::ListExpr* const index) const
@ -892,6 +895,39 @@ void TypeDecl::DescribeReST(ODesc* d, bool roles_only) const
}
}
// The following tracks how to initialize a given field, for fast execution
// of Create().
class FieldInit {
public:
// The type of initialization for the field.
enum {
R_INIT_NONE, // skip this entry
R_INIT_DIRECT, // look in direct_init for raw value
R_INIT_DIRECT_MANAGED, // same, but managed type
R_INIT_DEF, // look in def_expr for expression
R_INIT_RECORD, // field requires a new record
R_INIT_TABLE, // field requires a new table/set
R_INIT_VECTOR, // field requires a new vector
} init_type;
// For R_INIT_DIRECT/R_INIT_DIRECT_MANAGED:
ZVal direct_init;
detail::ExprPtr def_expr;
TypePtr def_type;
bool def_coerce = false; // whether coercion's required
RecordTypePtr r_type; // for R_INIT_RECORD
TableTypePtr t_type; // for R_INIT_TABLE
detail::AttributesPtr attrs; // attributes for R_INIT_TABLE
VectorTypePtr v_type; // for R_INIT_VECTOR
};
RecordType::RecordType(type_decl_list* arg_types) : Type(TYPE_RECORD)
{
types = arg_types;
@ -928,13 +964,81 @@ RecordType::~RecordType()
delete types;
}
for ( auto fi : field_inits )
delete fi;
}
void RecordType::AddField(unsigned int field, const TypeDecl* td)
{
ASSERT(field == field_inits.size());
ASSERT(field == managed_fields.size());
managed_fields.push_back(ZVal::IsManagedType(td->type));
auto init = new FieldInit();
init->init_type = FieldInit::R_INIT_NONE;
init->attrs = td->attrs;
auto a = init->attrs;
auto type = td->type;
auto def_attr = a ? a->Find(detail::ATTR_DEFAULT) : nullptr;
auto def_expr = def_attr ? def_attr->GetExpr() : nullptr;
if ( def_expr )
{
if ( type->Tag() == TYPE_RECORD &&
def_expr->GetType()->Tag() == TYPE_RECORD &&
! same_type(def_expr->GetType(), type) )
init->def_coerce = true;
if ( def_expr->Tag() == detail::EXPR_CONST )
{
auto v = def_expr->Eval(nullptr);
if ( ZVal::IsManagedType(type) )
init->init_type =
FieldInit::R_INIT_DIRECT_MANAGED;
else
init->init_type = FieldInit::R_INIT_DIRECT;
init->direct_init = ZVal(v, type);
}
else
{
init->init_type = FieldInit::R_INIT_DEF;
init->def_expr = def_expr;
init->def_type = def_expr->GetType();
}
}
else if ( ! (a && a->Find(detail::ATTR_OPTIONAL)) )
{
TypeTag tag = type->Tag();
if ( tag == TYPE_RECORD )
{
init->init_type = FieldInit::R_INIT_RECORD;
init->r_type = cast_intrusive<RecordType>(type);
}
else if ( tag == TYPE_TABLE )
{
init->init_type = FieldInit::R_INIT_TABLE;
init->t_type = cast_intrusive<TableType>(type);
}
else if ( tag == TYPE_VECTOR )
{
init->init_type = FieldInit::R_INIT_VECTOR;
init->v_type = cast_intrusive<VectorType>(type);
}
}
field_inits.push_back(init);
}
bool RecordType::HasField(const char* field) const
@ -1129,6 +1233,67 @@ void RecordType::AddFieldsDirectly(const type_decl_list& others,
num_fields = types->length();
}
void RecordType::Create(std::vector<std::optional<ZVal>>& r) const
{
int n = NumFields();
for ( int i = 0; i < n; ++i )
{
auto& init = field_inits[i];
ZVal r_i;
switch ( init->init_type ) {
case FieldInit::R_INIT_NONE:
r.push_back(std::nullopt);
continue;
case FieldInit::R_INIT_DIRECT:
r_i = init->direct_init;
break;
case FieldInit::R_INIT_DIRECT_MANAGED:
r_i = init->direct_init;
zeek::Ref(r_i.ManagedVal());
break;
case FieldInit::R_INIT_DEF:
{
auto v = init->def_expr->Eval(nullptr);
if ( v )
{
const auto& t = init->def_type;
if ( init->def_coerce )
{
auto rt = cast_intrusive<RecordType>(t);
v = v->AsRecordVal()->CoerceTo(rt);
}
r_i = ZVal(v, t);
}
else
reporter->Error("failed &default in record creation");
}
break;
case FieldInit::R_INIT_RECORD:
r_i = ZVal(new RecordVal(init->r_type));
break;
case FieldInit::R_INIT_TABLE:
r_i = ZVal(new TableVal(init->t_type, init->attrs));
break;
case FieldInit::R_INIT_VECTOR:
r_i = ZVal(new VectorVal(init->v_type));
break;
}
r.push_back(r_i);
}
}
void RecordType::DescribeFields(ODesc* d) const
{
if ( d->IsReadable() )

30
src/Type.h
View file

@ -18,6 +18,7 @@
namespace zeek {
class Val;
union ZVal;
class EnumVal;
class TableVal;
using ValPtr = IntrusivePtr<Val>;
@ -33,7 +34,7 @@ using ListExprPtr = IntrusivePtr<ListExpr>;
} // namespace detail
// BRO types.
// Zeek types.
enum TypeTag {
TYPE_VOID, // 0
TYPE_BOOL, // 1
@ -171,7 +172,7 @@ public:
explicit Type(TypeTag tag, bool base_type = false);
// Performs a shallow clone operation of the Bro type.
// Performs a shallow clone operation of the Zeek type.
// This especially means that especially for tables the types
// are not recursively cloned; altering one type will in this case
// alter one of them.
@ -249,6 +250,7 @@ public:
void Describe(ODesc* d) const override;
virtual void DescribeReST(ODesc* d, bool roles_only = false) const;
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
virtual unsigned MemoryAllocation() const;
void SetName(const std::string& arg_name) { name = arg_name; }
@ -346,6 +348,7 @@ public:
void Describe(ODesc* d) const override;
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
unsigned int MemoryAllocation() const override;
protected:
@ -559,6 +562,11 @@ public:
using type_decl_list = PList<TypeDecl>;
// The following tracks how to initialize a given field. We don't define
// it here because it requires pulling in a bunch of low-level headers that
// would be nice to avoid.
class FieldInit;
class RecordType final : public Type {
public:
explicit RecordType(type_decl_list* types);
@ -636,6 +644,13 @@ public:
void AddFieldsDirectly(const type_decl_list& types,
bool add_log_attr = false);
/**
*
* Populates a new instance of the record with its initial values.
* @param r The record's underlying value vector.
*/
void Create(std::vector<std::optional<ZVal>>& r) const;
void Describe(ODesc* d) const override;
void DescribeReST(ODesc* d, bool roles_only = false) const override;
void DescribeFields(ODesc* d) const;
@ -660,6 +675,10 @@ protected:
void AddField(unsigned int field, const TypeDecl* td);
// Maps each field to how to initialize it. Uses pointers due to
// keeping the FieldInit definition private to Type.cc (see above).
std::vector<FieldInit*> field_inits;
// If we were willing to bound the size of records, then we could
// use std::bitset here instead.
std::vector<bool> managed_fields;
@ -747,12 +766,15 @@ public:
const EnumValPtr& GetEnumVal(bro_int_t i);
// Only for use by C++-generated code. Non-protected because we
// don't know in advance the names of the functions that will
// access it.
void AddNameInternal(const std::string& full_name, bro_int_t val);
protected:
void AddNameInternal(const std::string& module_name,
const char* name, bro_int_t val, bool is_export);
void AddNameInternal(const std::string& full_name, bro_int_t val);
void CheckAndAddName(const std::string& module_name,
const char* name, bro_int_t val, bool is_export,
detail::Expr* deprecation = nullptr,

135
src/Val.cc
View file

@ -84,6 +84,7 @@ CONVERTERS(TYPE_STRING, StringVal*, Val::AsStringVal)
CONVERTERS(TYPE_VECTOR, VectorVal*, Val::AsVectorVal)
CONVERTERS(TYPE_ENUM, EnumVal*, Val::AsEnumVal)
CONVERTERS(TYPE_OPAQUE, OpaqueVal*, Val::AsOpaqueVal)
CONVERTERS(TYPE_TYPE, TypeVal*, Val::AsTypeVal)
ValPtr Val::CloneState::NewClone(Val* src, ValPtr dst)
{
@ -230,8 +231,6 @@ ValPtr Val::SizeVal() const
{
switch ( type->InternalType() ) {
case TYPE_INTERNAL_INT:
// Return abs value. However abs() only works on ints and llabs
// doesn't work on Mac OS X 10.5. So we do it by hand
if ( AsInt() < 0 )
return val_mgr->Count(-AsInt());
else
@ -817,7 +816,10 @@ AddrVal::~AddrVal()
unsigned int AddrVal::MemoryAllocation() const
{
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
return padded_sizeof(*this) + addr_val->MemoryAllocation();
#pragma GCC diagnostic pop
}
ValPtr AddrVal::SizeVal() const
@ -882,7 +884,10 @@ int SubNetVal::Width() const
unsigned int SubNetVal::MemoryAllocation() const
{
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
return padded_sizeof(*this) + subnet_val->MemoryAllocation();
#pragma GCC diagnostic pop
}
ValPtr SubNetVal::SizeVal() const
@ -1010,7 +1015,10 @@ void StringVal::ValDescribe(ODesc* d) const
unsigned int StringVal::MemoryAllocation() const
{
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
return padded_sizeof(*this) + string_val->MemoryAllocation();
#pragma GCC diagnostic pop
}
StringValPtr StringVal::Replace(
@ -1220,7 +1228,10 @@ void PatternVal::ValDescribe(ODesc* d) const
unsigned int PatternVal::MemoryAllocation() const
{
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
return padded_sizeof(*this) + re_val->MemoryAllocation();
#pragma GCC diagnostic pop
}
ValPtr PatternVal::DoClone(CloneState* state)
@ -1333,12 +1344,15 @@ ValPtr ListVal::DoClone(CloneState* state)
unsigned int ListVal::MemoryAllocation() const
{
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
unsigned int size = 0;
for ( const auto& val : vals )
size += val->MemoryAllocation();
size += util::pad_size(vals.capacity() * sizeof(decltype(vals)::value_type));
return size + padded_sizeof(*this) + type->MemoryAllocation();
#pragma GCC diagnostic pop
}
TableEntryVal* TableEntryVal::Clone(Val::CloneState* state)
@ -2771,6 +2785,8 @@ unsigned int TableVal::MemoryAllocation() const
{
unsigned int size = 0;
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
for ( const auto& ve : *table_val )
{
auto* tv = ve.GetValue<TableEntryVal*>();
@ -2781,6 +2797,7 @@ unsigned int TableVal::MemoryAllocation() const
return size + padded_sizeof(*this) + table_val->MemoryAllocation()
+ table_hash->MemoryAllocation();
#pragma GCC diagnostic pop
}
std::unique_ptr<detail::HashKey> TableVal::MakeHashKey(const Val& index) const
@ -2858,62 +2875,18 @@ RecordVal::RecordVal(RecordTypePtr t, bool init_fields)
if ( run_state::is_parsing )
parse_time_records[rt.get()].emplace_back(NewRef{}, this);
if ( ! init_fields )
return;
// Initialize to default values from RecordType (which are nil
// by default).
for ( int i = 0; i < n; ++i )
if ( init_fields )
{
detail::Attributes* a = rt->FieldDecl(i)->attrs.get();
detail::Attr* def_attr = a ? a->Find(detail::ATTR_DEFAULT).get() : nullptr;
ValPtr def;
if ( def_attr )
try
{
def = def_attr->GetExpr()->Eval(nullptr);
rt->Create(*record_val);
}
catch ( InterpreterException& )
catch ( InterpreterException& e )
{
if ( run_state::is_parsing )
parse_time_records[rt.get()].pop_back();
delete record_val;
throw;
}
const auto& type = rt->FieldDecl(i)->type;
if ( def && type->Tag() == TYPE_RECORD &&
def->GetType()->Tag() == TYPE_RECORD &&
! same_type(def->GetType(), type) )
{
auto tmp = def->AsRecordVal()->CoerceTo(cast_intrusive<RecordType>(type));
if ( tmp )
def = std::move(tmp);
}
if ( ! def && ! (a && a->Find(detail::ATTR_OPTIONAL)) )
{
TypeTag tag = type->Tag();
if ( tag == TYPE_RECORD )
def = make_intrusive<RecordVal>(cast_intrusive<RecordType>(type));
else if ( tag == TYPE_TABLE )
def = make_intrusive<TableVal>(IntrusivePtr{NewRef{}, type->AsTableType()},
IntrusivePtr{NewRef{}, a});
else if ( tag == TYPE_VECTOR )
def = make_intrusive<VectorVal>(cast_intrusive<VectorType>(type));
}
if ( def )
record_val->emplace_back(ZVal(def, def->GetType()));
else
record_val->emplace_back(std::nullopt);
}
}
@ -2987,7 +2960,8 @@ void RecordVal::ResizeParseTimeRecords(RecordType* revised_rt)
if ( required_length > current_length )
{
for ( auto i = current_length; i < required_length; ++i )
rv->AppendField(revised_rt->FieldDefault(i));
rv->AppendField(revised_rt->FieldDefault(i),
revised_rt->GetFieldType(i));
}
}
}
@ -3165,7 +3139,7 @@ ValPtr RecordVal::DoClone(CloneState* state)
// record. As we cannot guarantee that it will ber zeroed out at the
// approproate time (as it seems to be guaranteed for the original record)
// we don't touch it.
auto rv = make_intrusive<RecordVal>(GetType<RecordType>(), false);
auto rv = make_intrusive<RecordVal>(rt, false);
rv->origin = nullptr;
state->NewClone(this, rv);
@ -3174,7 +3148,7 @@ ValPtr RecordVal::DoClone(CloneState* state)
{
auto f_i = GetField(i);
auto v = f_i ? f_i->Clone(state) : nullptr;
rv->AppendField(std::move(v));
rv->AppendField(std::move(v), rt->GetFieldType(i));
}
return rv;
@ -3189,7 +3163,10 @@ unsigned int RecordVal::MemoryAllocation() const
{
auto f_i = GetField(i);
if ( f_i )
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
size += f_i->MemoryAllocation();
#pragma GCC diagnostic pop
}
size += util::pad_size(record_val->capacity() * sizeof(ZVal));
@ -3230,9 +3207,15 @@ ValPtr TypeVal::DoClone(CloneState* state)
return {NewRef{}, this};
}
VectorVal::VectorVal(VectorTypePtr t) : Val(t)
VectorVal::VectorVal(VectorTypePtr t) :
VectorVal(t, new vector<std::optional<ZVal>>())
{
vector_val = new vector<std::optional<ZVal>>();
}
VectorVal::VectorVal(VectorTypePtr t, std::vector<std::optional<ZVal>>* vals)
: Val(t)
{
vector_val = vals;
yield_type = t->Yield();
auto y_tag = yield_type->Tag();
@ -3660,6 +3643,50 @@ VectorValPtr VectorVal::Order(Func* cmp_func)
return result_v;
}
bool VectorVal::Concretize(const TypePtr& t)
{
if ( ! any_yield )
// Could do a same_type() call here, but really this case
// shouldn't happen in any case.
return yield_type->Tag() == t->Tag();
if ( ! vector_val )
// Trivially concretized.
return true;
auto n = vector_val->size();
for ( auto i = 0U; i < n; ++i )
{
auto& v = (*vector_val)[i];
if ( ! v )
// Vector hole does not require concretization.
continue;
auto& vt_i = yield_types ? (*yield_types)[i] : yield_type;
if ( vt_i->Tag() == TYPE_ANY )
{ // Do the concretization.
ValPtr any_v = {NewRef{}, v->AsAny()};
auto& vt = any_v->GetType();
if ( vt->Tag() != t->Tag() )
return false;
v = ZVal(any_v, t);
}
else if ( vt_i->Tag() != t->Tag() )
return false;
}
// Require that this vector be treated consistently in the future.
yield_type = t;
managed_yield = ZVal::IsManagedType(yield_type);
delete yield_types;
yield_types = nullptr;
any_yield = false;
return true;
}
unsigned int VectorVal::Resize(unsigned int new_num_elements)
{
unsigned int oldsize = vector_val->size();

88
src/Val.h
View file

@ -45,6 +45,8 @@ class PrefixTable;
class CompositeHash;
class HashKey;
class ZBody;
} // namespace detail
namespace run_state {
@ -110,6 +112,7 @@ public:
virtual ValPtr SizeVal() const;
// Bytes in total value object.
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
virtual unsigned int MemoryAllocation() const;
// Add this value to the given value (if appropriate).
@ -186,6 +189,9 @@ UNDERLYING_ACCESSOR_DECL(TypeVal, zeek::Type*, AsType)
OpaqueVal* AsOpaqueVal();
const OpaqueVal* AsOpaqueVal() const;
TypeVal* AsTypeVal();
const TypeVal* AsTypeVal() const;
void Describe(ODesc* d) const override;
virtual void DescribeReST(ODesc* d) const;
@ -450,10 +456,11 @@ public:
// Returns a masked port number
static uint32_t Mask(uint32_t port_num, TransportProto port_type);
protected:
friend class ValManager;
// Only meant for use by ValManager and compiled-to-C++ script
// functions.
PortVal(uint32_t p);
protected:
void ValDescribe(ODesc* d) const override;
ValPtr DoClone(CloneState* state) override;
@ -480,6 +487,7 @@ public:
const IPAddr& Get() const { return *addr_val; }
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
unsigned int MemoryAllocation() const override;
protected:
@ -509,6 +517,7 @@ public:
const IPPrefix& Get() const { return *subnet_val; }
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
unsigned int MemoryAllocation() const override;
protected:
@ -544,6 +553,7 @@ public:
const String* Get() const { return string_val; }
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
unsigned int MemoryAllocation() const override;
StringValPtr Replace(RE_Matcher* re, const String& repl,
@ -605,6 +615,7 @@ public:
const RE_Matcher* Get() const { return re_val; }
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
unsigned int MemoryAllocation() const override;
protected:
@ -654,6 +665,7 @@ public:
void Describe(ODesc* d) const override;
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
unsigned int MemoryAllocation() const override;
protected:
@ -927,6 +939,7 @@ public:
// the function in the frame allowing it to capture its closure.
void InitDefaultFunc(detail::Frame* f);
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
unsigned int MemoryAllocation() const override;
void ClearTimer(detail::Timer* t)
@ -1173,13 +1186,15 @@ public:
/**
* Appends a value to the record's fields. The caller is responsible
* for ensuring that fields are appended in the correct order and
* with the correct type.
* with the correct type. The type needs to be passed in because
* it's unsafe to take it from v when the field's type is "any" while
* v is a concrete type.
* @param v The value to append.
*/
void AppendField(ValPtr v)
void AppendField(ValPtr v, const TypePtr& t)
{
if ( v )
record_val->emplace_back(ZVal(v, v->GetType()));
record_val->emplace_back(ZVal(v, t));
else
record_val->emplace_back(std::nullopt);
}
@ -1202,6 +1217,18 @@ public:
return (*record_val)[field] ? true : false;
}
/**
* Returns true if the given field is in the record, false if
* it's missing.
* @param field The field name to retrieve.
* @return Whether there's a value for the given field name.
*/
bool HasField(const char *field) const
{
int idx = GetType()->AsRecordType()->FieldOffset(field);
return (idx != -1) && HasField(idx);
}
/**
* Returns the value of a given field index.
* @param field The field index to retrieve.
@ -1380,6 +1407,7 @@ public:
RecordValPtr CoerceTo(RecordTypePtr other,
bool allow_orphaning = false);
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
unsigned int MemoryAllocation() const override;
void DescribeReST(ODesc* d) const override;
@ -1393,6 +1421,23 @@ public:
static void DoneParsing();
protected:
friend class zeek::detail::ZBody;
// For use by low-level ZAM instructions. Caller assumes
// responsibility for memory management. The first version
// allows manipulation of whether the field is present at all.
// The second version ensures that the optional value is present.
std::optional<ZVal>& RawOptField(int field)
{ return (*record_val)[field]; }
ZVal& RawField(int field)
{
auto& f = RawOptField(field);
if ( ! f )
f = ZVal();
return *f;
}
ValPtr DoClone(CloneState* state) override;
void AddedField(int field)
@ -1436,6 +1481,8 @@ protected:
friend class Val;
friend class EnumType;
friend EnumValPtr make_enum__CPP(TypePtr t, int i);
template<class T, class... Ts>
friend IntrusivePtr<T> make_intrusive(Ts&&... args);
@ -1468,6 +1515,8 @@ protected:
class VectorVal final : public Val, public notifier::detail::Modifiable {
public:
explicit VectorVal(VectorTypePtr t);
VectorVal(VectorTypePtr t, std::vector<std::optional<ZVal>>* vals);
~VectorVal() override;
ValPtr SizeVal() const override;
@ -1548,8 +1597,29 @@ public:
*/
VectorValPtr Order(Func* cmp_func = nullptr);
/**
* Ensures that the vector can be used as a "vector of t". In
* general, this is only relevant for objects that are typed as
* "vector of any", making sure that each element is in fact
* of type "t", and is internally represented as such so that
* this object can be used directly without any special-casing.
*
* Returns true if the object is compatible with "vector of t"
* (including if it's not a vector-of-any but instead already a
* vector-of-t), false if not compatible.
* @param t The yield type to concretize to.
* @return True if the object is compatible with vector-of-t, false
* if not.
*/
bool Concretize(const TypePtr& t);
ValPtr ValAt(unsigned int index) const { return At(index); }
bool Has(unsigned int index) const
// Version to use once std::optional implementation is merged.
// { return index < vector_val->size() && vector_val[index]; }
{ return At(index) != nullptr; }
/**
* Returns the given element in a given underlying representation.
* Enables efficient vector access. Caller must ensure that the
@ -1558,8 +1628,12 @@ public:
* @param index The position in the vector of the element to return.
* @return The element's underlying value.
*/
bro_int_t IntAt(unsigned int index) const
{ return (*vector_val)[index]->int_val; }
bro_uint_t CountAt(unsigned int index) const
{ return (*vector_val)[index]->uint_val; }
double DoubleAt(unsigned int index) const
{ return (*vector_val)[index]->double_val; }
const RecordVal* RecordValAt(unsigned int index) const
{ return (*vector_val)[index]->record_val; }
bool BoolAt(unsigned int index) const
@ -1569,6 +1643,10 @@ public:
const String* StringAt(unsigned int index) const
{ return StringValAt(index)->AsString(); }
// Only intended for low-level access by compiled code.
const auto& RawVec() const { return vector_val; }
auto& RawVec() { return vector_val; }
protected:
/**
* Returns the element at a given index or nullptr if it does not exist.

6
src/Var.cc
View file

@ -672,7 +672,7 @@ void begin_func(IDPtr id, const char* module_name,
class OuterIDBindingFinder : public TraversalCallback {
public:
OuterIDBindingFinder(Scope* s)
OuterIDBindingFinder(ScopePtr s)
{
scopes.emplace_back(s);
}
@ -680,7 +680,7 @@ public:
TraversalCode PreExpr(const Expr*) override;
TraversalCode PostExpr(const Expr*) override;
std::vector<Scope*> scopes;
std::vector<ScopePtr> scopes;
std::unordered_set<ID*> outer_id_references;
};
@ -766,7 +766,7 @@ void end_func(StmtPtr body)
ingredients.release();
}
IDPList gather_outer_ids(Scope* scope, Stmt* body)
IDPList gather_outer_ids(ScopePtr scope, StmtPtr body)
{
OuterIDBindingFinder cb(scope);
body->Traverse(&cb);

3
src/Var.h
View file

@ -20,6 +20,7 @@ class Expr;
class Scope;
class Stmt;
using StmtPtr = IntrusivePtr<Stmt>;
using ScopePtr = IntrusivePtr<Scope>;
enum DeclType { VAR_REGULAR, VAR_CONST, VAR_REDEF, VAR_OPTION, };
@ -41,7 +42,7 @@ extern void begin_func(IDPtr id, const char* module_name, FunctionFlavor flavor,
extern void end_func(StmtPtr body);
// Gather all IDs referenced inside a body that aren't part of a given scope.
extern IDPList gather_outer_ids(Scope* scope, Stmt* body);
extern IDPList gather_outer_ids(ScopePtr scope, StmtPtr body);
} // namespace detail
} // namespace zeek

20
src/ZVal.cc
View file

@ -9,7 +9,7 @@
using namespace zeek;
bool ZVal::zval_was_nil = false;
bool* ZVal::zval_was_nil_addr = nullptr;
ZVal::ZVal(ValPtr v, const TypePtr& t)
@ -24,7 +24,7 @@ ZVal::ZVal(ValPtr v, const TypePtr& t)
return;
}
auto vt = v->GetType();
const auto& vt = v->GetType();
if ( vt->Tag() != t->Tag() && t->Tag() != TYPE_ANY )
{
@ -102,7 +102,7 @@ ZVal::ZVal(ValPtr v, const TypePtr& t)
break;
case TYPE_TYPE:
type_val = t->Ref();
type_val = v.release()->AsTypeVal();
break;
case TYPE_ERROR:
@ -222,15 +222,6 @@ ValPtr ZVal::ToVal(const TypePtr& t) const
case TYPE_ENUM:
return t->AsEnumType()->GetEnumVal(int_val);
case TYPE_ANY:
return {NewRef{}, any_val};
case TYPE_TYPE:
{
TypePtr tp = {NewRef{}, type_val};
return make_intrusive<TypeVal>(tp);
}
case TYPE_FUNC:
if ( func_val )
{
@ -258,6 +249,8 @@ ValPtr ZVal::ToVal(const TypePtr& t) const
case TYPE_RECORD: v = record_val; break;
case TYPE_VECTOR: v = vector_val; break;
case TYPE_PATTERN: v = re_val; break;
case TYPE_ANY: v = any_val; break;
case TYPE_TYPE: v = type_val; break;
case TYPE_ERROR:
case TYPE_TIMER:
@ -271,7 +264,8 @@ ValPtr ZVal::ToVal(const TypePtr& t) const
if ( v )
return {NewRef{}, v};
zval_was_nil = true;
if ( zval_was_nil_addr )
*zval_was_nil_addr = true;
return nullptr;
}

50
src/ZVal.h
View file

@ -18,7 +18,7 @@ class RecordVal;
class StringVal;
class SubNetVal;
class TableVal;
class Type;
class TypeVal;
class Val;
class VectorVal;
@ -31,6 +31,7 @@ using RecordValPtr = IntrusivePtr<RecordVal>;
using StringValPtr = IntrusivePtr<StringVal>;
using SubNetValPtr = IntrusivePtr<SubNetVal>;
using TableValPtr = IntrusivePtr<TableVal>;
using TypeValPtr = IntrusivePtr<TypeVal>;
using ValPtr = IntrusivePtr<Val>;
using VectorValPtr = IntrusivePtr<VectorVal>;
@ -69,9 +70,10 @@ union ZVal {
ZVal(OpaqueVal* v) { opaque_val = v; }
ZVal(PatternVal* v) { re_val = v; }
ZVal(TableVal* v) { table_val = v; }
ZVal(TypeVal* v) { type_val = v; }
ZVal(RecordVal* v) { record_val = v; }
ZVal(VectorVal* v) { vector_val = v; }
ZVal(Type* v) { type_val = v; }
ZVal(Val* v) { any_val = v; }
ZVal(StringValPtr v) { string_val = v.release(); }
ZVal(AddrValPtr v) { addr_val = v.release(); }
@ -80,21 +82,14 @@ union ZVal {
ZVal(OpaqueValPtr v) { opaque_val = v.release(); }
ZVal(PatternValPtr v) { re_val = v.release(); }
ZVal(TableValPtr v) { table_val = v.release(); }
ZVal(TypeValPtr v) { type_val = v.release(); }
ZVal(RecordValPtr v) { record_val = v.release(); }
ZVal(VectorValPtr v) { vector_val = v.release(); }
ZVal(TypePtr v) { type_val = v.release(); }
// Convert to a higher-level script value. The caller needs to
// ensure that they're providing the correct type.
ValPtr ToVal(const TypePtr& t) const;
// Whether a ZVal was accessed that was missing (a nil pointer).
// Used to generate run-time error messages.
static bool ZValNilStatus() { return zval_was_nil; }
// Resets the notion of low-level-error-occurred.
static void ClearZValNilStatus() { zval_was_nil = false; }
bro_int_t AsInt() const { return int_val; }
bro_uint_t AsCount() const { return uint_val; }
double AsDouble() const { return double_val; }
@ -110,10 +105,31 @@ union ZVal {
TableVal* AsTable() const { return table_val; }
RecordVal* AsRecord() const { return record_val; }
VectorVal* AsVector() const { return vector_val; }
Type* AsType() const { return type_val; }
TypeVal* AsType() const { return type_val; }
Val* AsAny() const { return any_val; }
Obj* ManagedVal() const { return managed_val; }
void ClearManagedVal() { managed_val = nullptr; }
// The following return references that can be used to
// populate the ZVal. Handy for compiled ZAM code.
bro_int_t& AsIntRef() { return int_val; }
bro_uint_t& AsCountRef() { return uint_val; }
double& AsDoubleRef() { return double_val; }
StringVal*& AsStringRef() { return string_val; }
AddrVal*& AsAddrRef() { return addr_val; }
SubNetVal*& AsSubNetRef() { return subnet_val; }
File*& AsFileRef() { return file_val; }
Func*& AsFuncRef() { return func_val; }
ListVal*& AsListRef() { return list_val; }
OpaqueVal*& AsOpaqueRef() { return opaque_val; }
PatternVal*& AsPatternRef() { return re_val; }
TableVal*& AsTableRef() { return table_val; }
RecordVal*& AsRecordRef() { return record_val; }
VectorVal*& AsVectorRef() { return vector_val; }
TypeVal*& AsTypeRef() { return type_val; }
Val*& AsAnyRef() { return any_val; }
Obj*& ManagedValRef() { return managed_val; }
// True if a given type is one for which we manage the associated
// memory internally.
@ -133,6 +149,14 @@ union ZVal {
DeleteManagedType(v);
}
// Specifies the address of a flag to set if a ZVal is accessed
// that was missing (a nil pointer). Used to generate run-time
// error messages. We use an address-based interface so that
// this flag can be combined with a general-purpose error flag,
// allowing inner loops to only have to test a single flag.
static void SetZValNilStatusAddr(bool* _zval_was_nil_addr)
{ zval_was_nil_addr = _zval_was_nil_addr; }
private:
friend class RecordVal;
friend class VectorVal;
@ -161,7 +185,7 @@ private:
TableVal* table_val;
RecordVal* record_val;
VectorVal* vector_val;
Type* type_val;
TypeVal* type_val;
// Used for "any" values.
Val* any_val;
@ -175,7 +199,7 @@ private:
// because often the caller won't have direct access to the
// particular ZVal that produces the issue, and just wants to
// know whether it occurred at some point.
static bool zval_was_nil;
static bool* zval_was_nil_addr;
};
} // zeek

254
src/ZeekString.cc
View file

@ -6,18 +6,23 @@
#include <ctype.h>
#include <algorithm>
#include <iostream>
#include <sstream> // Needed for unit testing
#include "zeek/Val.h"
#include "zeek/ID.h"
#include "zeek/Reporter.h"
#include "zeek/util.h"
#include "zeek/3rdparty/doctest.h"
#ifdef DEBUG
#define DEBUG_STR(msg) DBG_LOG(zeek::DBG_STRING, msg)
#else
#define DEBUG_STR(msg)
#endif
using namespace std::string_literals;
namespace zeek {
constexpr int String::EXPANDED_STRING;
@ -101,6 +106,19 @@ bool String::operator<(const String &bs) const
return Bstr_cmp(this, &bs) < 0;
}
bool String::operator==(std::string_view s) const
{
if ( static_cast<size_t>(n) != s.size() )
return false;
if ( b == nullptr )
{
return s.size() == 0;
}
return (memcmp(b, s.data(), n) == 0);
}
void String::Adopt(byte_vec bytes, int len)
{
Reset();
@ -169,6 +187,7 @@ const char* String::CheckString() const
{
// Either an embedded NUL, or no final NUL.
char* exp_s = Render();
if ( nulTerm )
reporter->Error("string with embedded NUL: \"%s\"", exp_s);
else
@ -390,8 +409,7 @@ char* String::VecToString(const Vec* vec)
return strdup(result.c_str());
}
bool StringLenCmp::operator()(String * const& bst1,
String * const& bst2)
bool StringLenCmp::operator()(String* const& bst1, String* const& bst2)
{
return _increasing ? (bst1->Len() < bst2->Len()) :
(bst1->Len() > bst2->Len());
@ -496,3 +514,235 @@ void delete_strings(std::vector<const String*>& v)
}
} // namespace zeek
TEST_SUITE_BEGIN("ZeekString");
TEST_CASE("construction")
{
zeek::String s1{};
CHECK_EQ(s1.Len(), 0);
CHECK_EQ(s1.Bytes(), nullptr);
CHECK_EQ(s1, "");
std::string text = "abcdef";
zeek::byte_vec text2 = new u_char[7];
memcpy(text2, text.c_str(), 7);
zeek::String s2{text2, 6, false};
CHECK_EQ(s2.Len(), 6);
zeek::String s3{text2, 6, true};
CHECK_EQ(s3.Len(), 6);
zeek::String s4{"abcdef"};
CHECK_EQ(s4.Len(), 6);
zeek::String s5{std::string("abcdef")};
CHECK_EQ(s5.Len(), 6);
zeek::String s6{s5};
CHECK_EQ(s6.Len(), 6);
zeek::String s7{true, text2, 6};
CHECK_EQ(s7.Len(), 6);
CHECK_EQ(s7.Bytes(), text2);
// Construct a temporary reporter object for the next two tests
zeek::reporter = new zeek::Reporter(false);
zeek::byte_vec text3 = new u_char[7];
memcpy(text3, text.c_str(), 7);
zeek::String s8{false, text3, 6};
CHECK_EQ(std::string(s8.CheckString()), "<string-with-NUL>");
zeek::byte_vec text4 = new u_char[7];
memcpy(text4, text.c_str(), 7);
text4[2] = '\0';
zeek::String s9{false, text4, 6};
CHECK_EQ(std::string(s9.CheckString()), "<string-with-NUL>");
delete zeek::reporter;
zeek::byte_vec text5 = (zeek::byte_vec)malloc(7);
memcpy(text5, text.c_str(), 7);
zeek::String s10{true, text5, 6};
s10.SetUseFreeToDelete(1);
CHECK_EQ(s10.Bytes(), text5);
}
TEST_CASE("set/assignment/comparison")
{
zeek::String s{"abc"};
CHECK_EQ(s, "abc");
s.Set("def");
CHECK_EQ(s, "def");
s.Set(std::string("ghi"));
CHECK_EQ(s, "ghi");
zeek::String s2{"abc"};
s.Set(s2);
CHECK_EQ(s, "abc");
zeek::String s3{"def"};
s = s3;
CHECK_EQ(s, "def");
CHECK_EQ(s, s3);
CHECK(s2 < s3);
s.Set("ghi");
CHECK_FALSE(s < s2);
std::string text = "abcdef";
zeek::byte_vec text2 = new u_char[7];
memcpy(text2, text.c_str(), 7);
s.Adopt(text2, 7);
CHECK_EQ(s, "abcdef");
CHECK_FALSE(s == s2);
// This is a clearly invalid string and we probably shouldn't allow it to be
// constructed, but this test covers one if statement in Bstr_eq.
zeek::String s4(false, nullptr, 3);
CHECK_FALSE(s4 == s2);
zeek::String s5{};
CHECK_LT(s5, s);
CHECK_FALSE(s < s5);
}
TEST_CASE("searching/modification")
{
zeek::String s{"this is a test"};
auto* ss = s.GetSubstring(5, 4);
CHECK_EQ(*ss, "is a");
delete ss;
auto* ss2 = s.GetSubstring(-1, 4);
CHECK_EQ(ss2, nullptr);
ss2 = s.GetSubstring(s.Len() + 5, 4);
CHECK_EQ(ss2, nullptr);
zeek::String s2{"test"};
CHECK_EQ(s.FindSubstring(&s2), 10);
s2.ToUpper();
CHECK_EQ(s2, "TEST");
zeek::String::IdxVec indexes;
zeek::String::Vec* splits = s.Split(indexes);
CHECK_EQ(splits, nullptr);
indexes.insert(indexes.end(), {4, 7, 9, -1, 30});
splits = s.Split(indexes);
CHECK_EQ(splits->size(), 4);
CHECK_EQ(*(splits->at(0)), "this");
CHECK_EQ(*(splits->at(1)), " is");
CHECK_EQ(*(splits->at(2)), " a");
CHECK_EQ(*(splits->at(3)), " test");
zeek::String* s3 = concatenate(*splits);
CHECK_EQ(s.Len(), s3->Len());
CHECK_EQ(s, *s3);
delete s3;
char* temp = zeek::String::VecToString(splits);
CHECK_EQ(std::string(temp), "[this, is, a, test,]");
free(temp);
for ( auto* entry : *splits )
delete entry;
delete splits;
}
TEST_CASE("rendering")
{
zeek::String s1("\\abcd\'\"");
auto* r = s1.Render(zeek::String::ESC_ESC);
CHECK_EQ(std::string(r), "\\\\abcd\'\"");
delete [] r;
r = s1.Render(zeek::String::ESC_QUOT);
CHECK_EQ(std::string(r), "\\abcd\\\'\\\"");
delete [] r;
r = s1.Render(zeek::String::ESC_ESC | zeek::String::ESC_QUOT | zeek::String::ESC_SER);
CHECK_EQ(std::string(r), "10 \\\\abcd\\\'\\\"");
delete [] r;
zeek::byte_vec text = new u_char[6];
text[0] = 3;
text[1] = 4;
text[2] = 5;
text[3] = 6;
text[4] = '\\';
text[5] = '\'';
zeek::String s2(false, text, 6);
r = s2.Render(zeek::String::ESC_HEX);
CHECK_EQ(std::string(r), "\\x03\\x04\\x05\\x06\\\'");
delete [] r;
int test_length = 0;
r = s2.Render(zeek::String::ESC_DOT, &test_length);
CHECK_EQ(std::string(r), "....\\\'");
CHECK_EQ(test_length, 7);
delete [] r;
r = s2.Render(zeek::String::BRO_STRING_LITERAL);
CHECK_EQ(std::string(r), "\\x03\\x04\\x05\\x06\\\\\\\'");
delete [] r;
std::ostringstream os1;
// This uses ESC_HEX, so it should be the same as the test above
os1 << s2;
CHECK_EQ(os1.str(), "\\x03\\x04\\x05\\x06\\\'");
std::ostringstream os2;
s2.Render(os2, zeek::String::ESC_HEX);
CHECK_EQ(os2.str(), "\\x03\\x04\\x05\\x06\\\'");
}
TEST_CASE("read")
{
std::string text1("5 abcde");
std::istringstream iss1(text1);
zeek::String s1{};
s1.Read(iss1);
CHECK_EQ(s1, "abcde");
std::string text2("abcde");
std::istringstream iss2(text2);
zeek::String s2{};
// Setting to something else disables reading the serialization format
s2.Read(iss2, zeek::String::ESC_HEX);
CHECK_EQ(s2, text2);
}
TEST_CASE("misc")
{
std::vector<const zeek::String*> sv = {new zeek::String{}, new zeek::String{}};
CHECK_EQ(sv.size(), 2);
zeek::delete_strings(sv);
CHECK_EQ(sv.size(), 0);
std::vector<zeek::data_chunk_t> dv = {{5, "abcde"}, {6, "fghijk"}};
auto* s = zeek::concatenate(dv);
CHECK_EQ(*s, "abcdefghijk");
delete s;
std::vector<zeek::String*> sv2 = {new zeek::String{"abcde"}, new zeek::String{"fghi"}};
std::sort(sv2.begin(), sv2.end(), zeek::StringLenCmp(true));
CHECK_EQ(*(sv2.front()), "fghi");
CHECK_EQ(*(sv2.back()), "abcde");
std::sort(sv2.begin(), sv2.end(), zeek::StringLenCmp(false));
CHECK_EQ(*(sv2.front()), "abcde");
CHECK_EQ(*(sv2.back()), "fghi");
for ( auto* entry : sv2 )
delete entry;
}
TEST_SUITE_END();

15
src/ZeekString.h
View file

@ -19,6 +19,12 @@ class VectorVal;
typedef u_char* byte_vec;
/**
* A container type for holding blocks of byte data. This can be used for
* character strings, but is not limited to that alone. This class provides
* methods for rendering byte data into character strings, including
* conversions of non-printable characters into other representations.
*/
class String {
public:
typedef std::vector<String*> Vec;
@ -49,6 +55,7 @@ public:
const String& operator=(const String& bs);
bool operator==(const String& bs) const;
bool operator<(const String& bs) const;
bool operator==(std::string_view s) const;
byte_vec Bytes() const { return b; }
int Len() const { return n; }
@ -71,6 +78,13 @@ public:
void SetUseFreeToDelete(int use_it)
{ use_free_to_delete = use_it; }
/**
* Returns a character-string representation of the stored bytes. This
* method doesn't do any extra rendering or character conversions. If
* null characters are found in the middle of the data or if the data
* is missing a closing null character, an error string is returned and
* a error is reported.
*/
const char* CheckString() const;
enum render_style {
@ -116,6 +130,7 @@ public:
// XXX and to_upper; the latter doesn't use String::ToUpper().
void ToUpper();
[[deprecated("Remove in v5.1. MemoryAllocation() is deprecated and will be removed. See GHI-572.")]]
unsigned int MemoryAllocation() const;
// Returns new string containing the substring of this string,

30
src/analyzer/Analyzer.cc
View file

@ -764,6 +764,8 @@ void Analyzer::AppendNewChildren()
unsigned int Analyzer::MemoryAllocation() const
{
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
unsigned int mem = padded_sizeof(*this)
+ (timers.MemoryAllocation() - padded_sizeof(timers));
@ -777,6 +779,7 @@ unsigned int Analyzer::MemoryAllocation() const
mem += a->MemoryAllocation();
return mem;
#pragma GCC diagnostic pop
}
void Analyzer::UpdateConnVal(RecordVal *conn_val)
@ -880,31 +883,4 @@ void SupportAnalyzer::ForwardUndelivered(uint64_t seq, int len, bool is_orig)
}
void TransportLayerAnalyzer::Done()
{
Analyzer::Done();
}
void TransportLayerAnalyzer::SetContentsFile(unsigned int /* direction */,
FilePtr /* f */)
{
reporter->Error("analyzer type does not support writing to a contents file");
}
FilePtr TransportLayerAnalyzer::GetContentsFile(unsigned int /* direction */) const
{
reporter->Error("analyzer type does not support writing to a contents file");
return nullptr;
}
void TransportLayerAnalyzer::PacketContents(const u_char* data, int len)
{
if ( packet_contents && len > 0 )
{
String* cbs = new String(data, len, true);
auto contents = make_intrusive<StringVal>(cbs);
EnqueueConnEvent(packet_contents, ConnVal(), std::move(contents));
}
}
} // namespace zeek::analyzer

Some files were not shown because too many files have changed in this diff Show more