diff --git a/.gitmodules b/.gitmodules index e971a79201..2df489a3b6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -58,9 +58,6 @@ [submodule "auxil/out_ptr"] path = auxil/out_ptr url = https://github.com/soasis/out_ptr.git -[submodule "auxil/spicy-plugin"] - path = auxil/spicy-plugin - url = https://github.com/zeek/spicy-plugin [submodule "auxil/spicy"] path = auxil/spicy/spicy url = https://github.com/zeek/spicy diff --git a/CMakeLists.txt b/CMakeLists.txt index 6a4830114a..89e9522e90 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -497,17 +497,19 @@ file( WRITE ${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev.sh "export ZEEKPATH=`${cmake_binary_dir}/zeek-path-dev`\n" "export ZEEK_PLUGIN_PATH=\"${cmake_binary_dir}/src\":$\{ZEEK_PLUGIN_PATH\}\n" - "export PATH=\"${cmake_binary_dir}\":\"${cmake_binary_dir}/src\":\"${cmake_binary_dir}/auxil/spicy/spicy/bin\":\"${cmake_binary_dir}/src/builtin-plugins/spicy-plugin/bin\":$\{PATH\}\n" + "export PATH=\"${cmake_binary_dir}\":\"${cmake_binary_dir}/src\":\"${cmake_binary_dir}/auxil/spicy/spicy/bin\":\"${cmake_binary_dir}/src/spicy/spicyz\":$\{PATH\}\n" "export SPICY_PATH=`${cmake_binary_dir}/spicy-path`\n" - "export HILTI_CXX_INCLUDE_DIRS=`${cmake_binary_dir}/hilti-cxx-include-dirs`\n") + "export HILTI_CXX_INCLUDE_DIRS=`${cmake_binary_dir}/hilti-cxx-include-dirs`\n" + "export ZEEK_SPICY_LIBRARY_PATH=${cmake_source_dir}/scripts/spicy\n") file( WRITE ${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev.csh "setenv ZEEKPATH `${cmake_binary_dir}/zeek-path-dev`\n" "setenv ZEEK_PLUGIN_PATH \"${cmake_binary_dir}/src\":$\{ZEEK_PLUGIN_PATH\}\n" - "setenv PATH \"${cmake_binary_dir}\":\"${cmake_binary_dir}/src\":\"${cmake_binary_dir}/auxil/spicy/spicy/bin\":\"${cmake_binary_dir}/src/builtin-plugins/spicy-plugin/bin\":$\{PATH\}\n" + "setenv PATH \"${cmake_binary_dir}\":\"${cmake_binary_dir}/src\":\"${cmake_binary_dir}/auxil/spicy/spicy/bin\":\"${cmake_binary_dir}/src/spicy/spicyz\":$\{PATH\}\n" "setenv SPICY_PATH \"`${cmake_binary_dir}/spicy-path`\"\n" - "setenv HILTI_CXX_INCLUDE_DIRS \"`${cmake_binary_dir}/hilti-cxx-include-dirs`\"\n") + "setenv HILTI_CXX_INCLUDE_DIRS \"`${cmake_binary_dir}/hilti-cxx-include-dirs`\"\n" + "setenv ZEEK_SPICY_LIBRARY_PATH \"${cmake_source_dir}/scripts/spicy\"\n") file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1) execute_process( @@ -887,53 +889,28 @@ else () endif () if (NOT DISABLE_SPICY) - set(USE_SPICY_ANALYZERS yes) + if (SPICY_ROOT_DIR) + find_package(Spicy REQUIRED) # will set HAVE_SPICY + spicy_require_version("1.8.0") - if (NOT SPICY_ROOT_DIR) - set(HAVE_SPICY yes) # evaluated by Spicy plugin build + if (NOT SPICY_HAVE_TOOLCHAIN) + message(FATAL_ERROR "Spicy not built with toolchain support") + endif () + spicy_print_summary() + else () + set(HAVE_SPICY yes) add_subdirectory(auxil/spicy) + zeek_add_dependencies(spicy) - # Set variables used by the spicy-plugin build since we are building Spicy - # as part of Zeek so spicy-plugin cannot use `spicy-config` at configure - # time to set these. - set(SPICY_CONFIG "") - set(SPICY_HAVE_TOOLCHAIN "YES") - set(SPICY_INCLUDE_DIRS_RUNTIME - ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy/hilti/runtime/include - ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy/spicy/runtime/include - ${PROJECT_BINARY_DIR}/auxil/spicy/spicy/include) - set(SPICY_INCLUDE_DIRS_TOOLCHAIN - ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy/hilti/toolchain/include - ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy/spicy/toolchain/include) - set(SPICY_LIBRARY spicy) - set(HILTI_LIBRARY_RT hilti-rt) - set(HILTI_LIBRARY_RT_DEBUG hilti-rt-debug) - set(SPICY_LIBRARY_RT spicy-rt) - set(SPICY_LIBRARY_RT_DEBUG spicy-rt-debug) - - # Needed only for logging from CMake configure phase. - get_directory_property(SPICY_VERSION DIRECTORY ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy - DEFINITION SPICY_VERSION) get_directory_property( SPICY_VERSION_NUMBER DIRECTORY ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy DEFINITION SPICY_VERSION_NUMBER) - get_directory_property(SPICY_PREFIX DIRECTORY ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy - DEFINITION CMAKE_INSTALL_PREFIX) - get_directory_property(SPICY_BUILD_MODE DIRECTORY ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy - DEFINITION CMAKE_BUILD_TYPE) - set(SPICYC "") endif () - if (NOT SPICY_PLUGIN_PATH) - set(_spicy_plugin "included") - set(SPICY_PLUGIN_PATH ${CMAKE_SOURCE_DIR}/auxil/spicy-plugin) - endif () - - set(SPICY_PLUGIN_BINARY_PATH ${CMAKE_BINARY_DIR}/src/builtin-plugins/spicy-plugin) - list(APPEND ZEEK_INCLUDE_PLUGINS ${SPICY_PLUGIN_PATH}) + set(USE_SPICY_ANALYZERS yes) else () - set(HAVE_SPICY no) # evaluated by Spicy plugin build + set(HAVE_SPICY no) set(USE_SPICY_ANALYZERS no) endif () @@ -1255,6 +1232,10 @@ add_subdirectory(scripts) add_subdirectory(man) add_subdirectory(testing) +if (NOT DISABLE_SPICY) + zeek_add_dependencies(spicyz) +endif () + include(CheckOptionalBuildSources) checkoptionalbuildsources(auxil/btest BTest INSTALL_BTEST) @@ -1264,26 +1245,6 @@ checkoptionalbuildsources(auxil/zeek-aux Zeek-Aux INSTALL_AUX_TOOLS) checkoptionalbuildsources(auxil/zeek-archiver ZeekArchiver INSTALL_ZEEK_ARCHIVER) checkoptionalbuildsources(auxil/zeek-client ZeekClient INSTALL_ZEEK_CLIENT) -if (NOT DISABLE_SPICY) - # The `zeek` binary implicitly depends on the driver object file built as part - # of `spicy`; make that dependency explicit. - zeek_add_dependencies(spicyz) - - if (NOT SPICY_ROOT_DIR) - # Make sure we build targets of spicy-plugin after the `spicy` target. - add_dependencies(plugin-Zeek-Spicy spicy) - add_dependencies(spicyz spicy) - - # Also install spicy-plugin's CMake files into Zeek's global `cmake/` - # folder. - # - # NOTE: We do not install spicy-plugin's `FindZeek.cmake` since another - # version of this file is already provided by Zeek. - install(FILES auxil/spicy-plugin/cmake/ZeekSpicyAnalyzerSupport.cmake - auxil/spicy-plugin/cmake/FindSpicy.cmake DESTINATION share/zeek/cmake) - endif () -endif () - # Always generate helper scripts referenced in e.g., `zeek-path-dev.*` so the # scripts work in any build configuration. If we do not include Spicy these # files have no actual effect. @@ -1295,6 +1256,8 @@ endif () # we generate shell definitions to support running and using Spicy or # spicy-plugin functionality in the build tree, including JIT'ing directly from # Zeek. +# +# TODO: Do we still need these? configure_file(${CMAKE_SOURCE_DIR}/auxil/spicy/spicy-path.in ${CMAKE_BINARY_DIR}/spicy-path @ONLY) configure_file(${CMAKE_SOURCE_DIR}/auxil/spicy/hilti-cxx-include-dirs.in ${CMAKE_BINARY_DIR}/hilti-cxx-include-dirs @ONLY) @@ -1375,12 +1338,6 @@ elseif (SPICY_ROOT_DIR) set(_spicy "external (${SPICY_ROOT_DIR})") endif () -if (DISABLE_SPICY) - set(_spicy_plugin "disabled") -elseif ("${_spicy_plugin}" STREQUAL "") - set(_spicy_plugin "external (${SPICY_PLUGIN_PATH})") -endif () - if (ZEEK_LEGACY_ANALYZERS) list(JOIN ZEEK_LEGACY_ANALYZERS ", " _legacy_analyzers) set(_legacy_analyzers @@ -1394,7 +1351,7 @@ endif () if (ZEEK_LEGACY_ANALYZERS OR ZEEK_SKIPPED_ANALYZERS) set(_analyzer_warning - "\n\n[Warning] Some analyzers are not available due to lack of built-in Spicy support:${_legacy_analyzers}${_skipped_analyzers}" + "\n\n[Warning] Some analyzers are not available due to lack of Spicy support:${_legacy_analyzers}${_skipped_analyzers}" ) endif () @@ -1412,6 +1369,7 @@ message( "\nScript dir: ${ZEEK_SCRIPT_INSTALL_PATH}" "\nSpool dir: ${ZEEK_SPOOL_DIR}" "\nState dir: ${ZEEK_STATE_DIR}" + "\nSpicy modules dir: ${ZEEK_SPICY_MODULE_PATH}" "\n" "\nDebug mode: ${ENABLE_DEBUG}" "\nUnit tests: ${ENABLE_ZEEK_UNIT_TESTS}" @@ -1433,7 +1391,6 @@ message( "\nGen-ZAM: ${_gen_zam_exe_path}" "\nzkg: ${INSTALL_ZKG}" "\nSpicy: ${_spicy}" - "\nSpicy plugin: ${_spicy_plugin}" "\nSpicy analyzers: ${USE_SPICY_ANALYZERS}" "\nJavaScript: ${ZEEK_HAVE_JAVASCRIPT}" "\n" diff --git a/auxil/spicy-plugin b/auxil/spicy-plugin deleted file mode 160000 index a618f2ce08..0000000000 --- a/auxil/spicy-plugin +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a618f2ce0831c311f9bcff5d020b85fc44345221 diff --git a/auxil/spicy/spicy b/auxil/spicy/spicy index 64de5d0ef3..ec87b43037 160000 --- a/auxil/spicy/spicy +++ b/auxil/spicy/spicy @@ -1 +1 @@ -Subproject commit 64de5d0ef323428452827469f07b0a1da8e65e16 +Subproject commit ec87b43037dba50648cb93be8940a4db23658905 diff --git a/cmake b/cmake index a90d691796..ac0529be3d 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit a90d69179607c5083158f926be6d37f3db18f110 +Subproject commit ac0529be3d1e14eba426c2e516c4a9c9971be40b diff --git a/configure b/configure index 98ce48c08c..25bde692b6 100755 --- a/configure +++ b/configure @@ -104,7 +104,6 @@ Usage: $0 [OPTION]... [VAR=VALUE]... --with-python-inc=PATH path to Python headers --with-python-lib=PATH path to libpython --with-spicy=PATH path to Spicy install root - --with-spicy-plugin=PATH path to Spicy plugin source tree --with-swig=PATH path to SWIG executable Packaging Options (for developers): @@ -378,9 +377,6 @@ while [ $# -ne 0 ]; do --with-spicy=*) append_cache_entry SPICY_ROOT_DIR PATH $optarg ;; - --with-spicy-plugin=*) - append_cache_entry SPICY_PLUGIN_PATH PATH $optarg - ;; --with-swig=*) append_cache_entry SWIG_EXECUTABLE PATH $optarg ;; diff --git a/scripts/base/frameworks/spicy/__load__.zeek b/scripts/base/frameworks/spicy/__load__.zeek new file mode 100644 index 0000000000..5f39cb27df --- /dev/null +++ b/scripts/base/frameworks/spicy/__load__.zeek @@ -0,0 +1 @@ +@load ./main.zeek diff --git a/scripts/base/frameworks/spicy/init-bare.zeek b/scripts/base/frameworks/spicy/init-bare.zeek new file mode 100644 index 0000000000..1dcef1c710 --- /dev/null +++ b/scripts/base/frameworks/spicy/init-bare.zeek @@ -0,0 +1,37 @@ + +module Spicy; + +export { +# doc-options-start + ## Constant for testing if Spicy is available. + const available = T; + + ## Show output of Spicy print statements. + const enable_print = F &redef; + + # Record and display profiling information, if compiled into analyzer. + const enable_profiling = F &redef; + + ## abort() instead of throwing HILTI exceptions. + const abort_on_exceptions = F &redef; + + ## Include backtraces when reporting unhandled exceptions. + const show_backtraces = F &redef; + + ## Maximum depth of recursive file analysis (Spicy analyzers only) + const max_file_depth: count = 5 &redef; +# doc-options-end + +# doc-types-start + ## Result type for `Spicy::resource_usage()`. + type ResourceUsage: record { + user_time : interval; ##< user CPU time of the Zeek process + system_time :interval; ##< system CPU time of the Zeek process + memory_heap : count; ##< memory allocated on the heap by the Zeek process + num_fibers : count; ##< number of fibers currently in use + max_fibers: count; ##< maximum number of fibers ever in use + max_fiber_stack_size: count; ##< maximum fiber stack size ever in use + cached_fibers: count; ##< number of fibers currently cached + }; +# doc-types-end +} diff --git a/scripts/base/frameworks/spicy/init-framework.zeek b/scripts/base/frameworks/spicy/init-framework.zeek new file mode 100644 index 0000000000..886b0f4170 --- /dev/null +++ b/scripts/base/frameworks/spicy/init-framework.zeek @@ -0,0 +1,81 @@ +@load base/misc/version + +# doc-common-start +module Spicy; + +export { +# doc-functions-start + ## Enable a specific Spicy protocol analyzer if not already active. If this + ## analyzer replaces an standard analyzer, that one will automatically be + ## disabled. + ## + ## tag: analyzer to toggle + ## + ## Returns: true if the operation succeeded + global enable_protocol_analyzer: function(tag: Analyzer::Tag) : bool; + + ## Disable a specific Spicy protocol analyzer if not already inactive. If + ## this analyzer replaces an standard analyzer, that one will automatically + ## be re-enabled. + ## + ## tag: analyzer to toggle + ## + ## Returns: true if the operation succeeded + global disable_protocol_analyzer: function(tag: Analyzer::Tag) : bool; + + + ## Enable a specific Spicy file analyzer if not already active. If this + ## analyzer replaces an standard analyzer, that one will automatically be + ## disabled. + ## + ## tag: analyzer to toggle + ## + ## Returns: true if the operation succeeded + global enable_file_analyzer: function(tag: Files::Tag) : bool; + + ## Disable a specific Spicy file analyzer if not already inactive. If + ## this analyzer replaces an standard analyzer, that one will automatically + ## be re-enabled. + ## + ## tag: analyzer to toggle + ## + ## Returns: true if the operation succeeded + global disable_file_analyzer: function(tag: Files::Tag) : bool; + + ## Returns current resource usage as reported by the Spicy runtime system. + global resource_usage: function() : ResourceUsage; +# doc-functions-end +} + +# Marked with &is_used to suppress complaints when there aren't any +# Spicy file analyzers loaded, and hence this event can't be generated. +# The attribute is only supported for Zeek 5.0 and higher. +event spicy_analyzer_for_mime_type(a: Files::Tag, mt: string) &is_used + { + Files::register_for_mime_type(a, mt); + } + +function enable_protocol_analyzer(tag: Analyzer::Tag) : bool + { + return Spicy::__toggle_analyzer(tag, T); + } + +function disable_protocol_analyzer(tag: Analyzer::Tag) : bool + { + return Spicy::__toggle_analyzer(tag, F); + } + +function enable_file_analyzer(tag: Files::Tag) : bool + { + return Spicy::__toggle_analyzer(tag, T); + } + +function disable_file_analyzer(tag: Files::Tag) : bool + { + return Spicy::__toggle_analyzer(tag, F); + } + +function resource_usage() : ResourceUsage + { + return Spicy::__resource_usage(); + } diff --git a/scripts/base/frameworks/spicy/main.zeek b/scripts/base/frameworks/spicy/main.zeek new file mode 100644 index 0000000000..6c7684c47b --- /dev/null +++ b/scripts/base/frameworks/spicy/main.zeek @@ -0,0 +1,15 @@ +@load base/frameworks/notice + +module Spicy; + +export { + redef enum Notice::Type += { Spicy_Max_File_Depth_Exceeded }; +} + +event max_file_depth_exceeded(f: fa_file, args: Files::AnalyzerArgs, limit: count) + { + NOTICE([ + $note=Spicy::Spicy_Max_File_Depth_Exceeded, + $msg=fmt("Maximum file depth exceeded for file %s", f$id) + ]); + } diff --git a/scripts/base/frameworks/spicy/misc/record-spicy-batch.zeek b/scripts/base/frameworks/spicy/misc/record-spicy-batch.zeek new file mode 100644 index 0000000000..3955077d38 --- /dev/null +++ b/scripts/base/frameworks/spicy/misc/record-spicy-batch.zeek @@ -0,0 +1,90 @@ +# Saves all input traffic in Spicy's batch format. + +module SpicyBatch; + +export { + const filename = "batch.dat" &redef; +} + +redef tcp_content_deliver_all_orig=T; +redef tcp_content_deliver_all_resp=T; +redef udp_content_deliver_all_orig=T; +redef udp_content_deliver_all_resp=T; + +global output: file; +global conns: set[conn_id]; +global num_conns = 0; + +function id(c: connection) : string + { + local cid = c$id; + local proto = "???"; + + if ( is_tcp_port(cid$orig_p) ) + proto = "tcp"; + else if ( is_udp_port(cid$orig_p) ) + proto = "udp"; + else if ( is_icmp_port(cid$orig_p) ) + proto = "icmp"; + + return fmt("%s-%d-%s-%d-%s", cid$orig_h, cid$orig_p, cid$resp_h, cid$resp_p, proto); + } + +function begin(c: connection, type_: string) + { + add conns[c$id]; + ++num_conns; + print fmt("tracking %s", c$id); + + local id_ = id(c); + print output, fmt("@begin-conn %s %s %s-orig %s%%orig %s-resp %s%%resp\n", id_, type_, id_, c$id$resp_p, id_, c$id$resp_p); + } + +event zeek_init() + { + output = open(filename); + enable_raw_output(output); + print output, "!spicy-batch v2\n"; + } + +event new_connection_contents(c: connection) + { + begin(c, "stream"); + } + +event tcp_contents(c: connection, is_orig: bool, seq: count, contents: string) + { + print output, fmt("@data %s-%s %d\n", id(c), (is_orig ? "orig" : "resp"), |contents|); + print output, contents; + print output, "\n"; + } + +event content_gap(c: connection, is_orig: bool, seq: count, length: count) + { + print output, fmt("@gap %s-%s %d\n", id(c), (is_orig ? "orig" : "resp"), length); + } + +event udp_contents(c: connection, is_orig: bool, contents: string) + { + if ( c$id !in conns ) + begin(c, "block"); + + print output, fmt("@data %s-%s %d\n", id(c), (is_orig ? "orig" : "resp"), |contents|); + print output, contents; + print output, "\n"; + } + +event connection_state_remove(c: connection) + { + if ( c$id !in conns ) + return; + + print output, fmt("@end-conn %s\n", id(c)); + } + +event zeek_done() + { + close(output); + print fmt("recorded %d session%s total", num_conns, (num_conns > 1 ? "s" : "")); + print fmt("output in %s", filename); + } diff --git a/scripts/base/frameworks/spicy/misc/resource-usage.zeek b/scripts/base/frameworks/spicy/misc/resource-usage.zeek new file mode 100644 index 0000000000..38a2e87f3b --- /dev/null +++ b/scripts/base/frameworks/spicy/misc/resource-usage.zeek @@ -0,0 +1,18 @@ +module Spicy; + +event print_usage() + { + local r = Spicy::resource_usage(); + + print fmt("%.6f Spicy user=%f sys=%f heap=%d current_fibers=%d cached_fibers=%d max_fibers=%d max_stack=%d", + network_time(), r$user_time, r$system_time, r$memory_heap, + r$num_fibers, r$cached_fibers, r$max_fibers, + r$max_fiber_stack_size); + + schedule 1 min { print_usage() }; + } + +event zeek_init() + { + schedule 1 min { print_usage() }; + } diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index b044892ed8..7f5b0da460 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -5702,3 +5702,4 @@ event net_done(t: time) @endif @load base/packet-protocols +@load base/frameworks/spicy/init-bare diff --git a/scripts/base/init-default.zeek b/scripts/base/init-default.zeek index 5a11969e4e..93a29bf323 100644 --- a/scripts/base/init-default.zeek +++ b/scripts/base/init-default.zeek @@ -43,6 +43,7 @@ @load base/frameworks/openflow @load base/frameworks/netcontrol @load base/frameworks/telemetry +@load base/frameworks/spicy @load base/protocols/conn @load base/protocols/dce-rpc diff --git a/scripts/base/init-frameworks-and-bifs.zeek b/scripts/base/init-frameworks-and-bifs.zeek index a346f9543c..a983c1e453 100644 --- a/scripts/base/init-frameworks-and-bifs.zeek +++ b/scripts/base/init-frameworks-and-bifs.zeek @@ -17,6 +17,8 @@ # Load BiFs defined by plugins. @load base/bif/plugins +@load base/frameworks/spicy/init-framework + # This sets up secondary/subdir BIFs such that they can be used by any # further scripts within their global initializations and is intended to be # the last thing done within this script. It's called within @if simply so diff --git a/scripts/spicy/zeek.spicy b/scripts/spicy/zeek.spicy new file mode 100644 index 0000000000..4f4467008d --- /dev/null +++ b/scripts/spicy/zeek.spicy @@ -0,0 +1,157 @@ +# Copyright (c) 2020-2021 by the Zeek Project. See LICENSE for details. + +module zeek; + +# Note: Retain the formatting here, doc/scripts/autogen-spicy-lib is picking up on that. + +%cxx-include = "zeek/spicy/runtime-support.h"; + +## [Deprecated] Triggers a DPD protocol confirmation for the current connection. +## +## This function has been deprecated and will be removed. Use ``spicy::accept_input`` +## instead, which will have the same effect with Zeek. +public function confirm_protocol() : void &cxxname="zeek::spicy::rt::confirm_protocol"; + +## [Deprecated] Triggers a DPD protocol violation for the current connection. +## +## This function has been deprecated and will be removed. Use ``spicy::decline_input`` +## instead, which will have the same effect with Zeek. +public function reject_protocol(reason: string) : void &cxxname="zeek::spicy::rt::reject_protocol"; + +## Reports a "weird" to Zeek. This should be used with similar semantics as in +## Zeek: something quite unexpected happening at the protocol level, which however +## does not prevent us from continuing to process the connection. +## +## id: the name of the weird, which (just like in Zeek) should be a *static* +## string identifying the situation reported (e.g., ``unexpected_command``). +## +## addl: additional information to record along with the weird +public function weird(id: string, addl: string = "") &cxxname="zeek::spicy::rt::weird"; + +## Returns true if we're currently parsing the originator side of a connection. +public function is_orig() : bool &cxxname="zeek::spicy::rt::is_orig"; + +## Returns the current connection's UID. +public function uid() : string &cxxname="zeek::spicy::rt::uid"; + +## Returns the current connection's 4-tuple ID. +public function conn_id() : tuple &cxxname="zeek::spicy::rt::conn_id"; + +## Instructs Zeek to flip the directionality of the current connection. +public function flip_roles() : void &cxxname="zeek::spicy::rt::flip_roles"; + +## Returns the number of packets seen so far on the current side of the current connection. +public function number_packets() : uint64 &cxxname="zeek::spicy::rt::number_packets"; + +## Opaque handle to a protocol analyzer. +public type ProtocolHandle = __library_type("zeek::spicy::rt::ProtocolHandle"); + +## Adds a Zeek-side child protocol analyzer to the current connection. +## +## If the same analyzer was added previously with protocol_handle_get_or_create or +## protocol_begin with same argument, and not closed with protocol_handle_close +## or protocol_end, no new analyzer will be added. +## +## See `protocol_handle_get_or_create` for the error semantics of this function. +## +## analyzer: type of analyzer to instantiate, specified through its Zeek-side +## name (similar to what Zeek's signature action `enable` takes); if not +## specified, Zeek will perform its usual dynamic protocol detection to figure +## out how to parse the data (the latter will work only for TCP protocols, though.) +public function protocol_begin(analyzer: optional = Null) : void &cxxname="zeek::spicy::rt::protocol_begin"; + +## Gets a handle to a Zeek-side child protocol analyzer for the current connection. +## +## If no such child exists it will be added; otherwise a handle to the +## existing child protocol analyzer will be returned. +## +## This function will return an error +## +## - if not called from a protocol analyzer, or +## - the requested child protocol analyzer is unknown, or +## - creation of a child analyzer of the requested type was prevented by a +## previous call of `disable_analyzer` with `prevent=T` +## +## analyzer: type of analyzer to instantiate, specified through its Zeek-side +## name (similar to what Zeek's signature action `enable` takes). +public function protocol_handle_get_or_create(analyzer: string) : ProtocolHandle &cxxname="zeek::spicy::rt::protocol_handle_get_or_create"; + +## Forwards protocol data to all previously instantiated Zeek-side child protocol analyzers. +## +## is_orig: true to feed the data to the child's originator side, false for the responder +## data: chunk of data to forward to child analyzer +## h: optional handle to the child analyzer to forward data into, else forward to all child analyzers +public function protocol_data_in(is_orig: bool, data: bytes, h: optional = Null) : void &cxxname="zeek::spicy::rt::protocol_data_in"; + +## Signals a gap in input data to all previously instantiated Zeek-side child protocol analyzers. +## +## is_orig: true to signal gap to the child's originator side, false for the responder +## offset: start offset of gap in input stream +## len: size of gap +## h: optional handle to the child analyzer signal a gap to, else signal to all child analyzers +public function protocol_gap(is_orig: bool, offset: uint64, len: uint64, h: optional = Null) : void &cxxname="zeek::spicy::rt::protocol_gap"; + +## Signals end-of-data to all previously instantiated Zeek-side child protocol +## analyzers and removes them. +public function protocol_end() : void &cxxname="zeek::spicy::rt::protocol_end"; + +## Signals end-of-data to the given child analyzer and removes it. +## +## The given handle must be live, i.e., it must not have been used in a +## previous protocol_handle_close call, and must not have been live when +## protocol_end was called. If the handle is not live a runtime error will +## be triggered. +## +## handle: handle to the child analyzer to remove +public function protocol_handle_close(handle: ProtocolHandle): void &cxxname="zeek::spicy::rt::protocol_handle_close"; + +## Signals the beginning of a file to Zeek's file analysis, associating it with the current connection. +## Optionally, a mime type can be provided. It will be passed on to Zeek's file analysis framework. +## Returns the Zeek-side file ID of the new file. +public function file_begin(mime_type: optional = Null) : string &cxxname="zeek::spicy::rt::file_begin"; + +## Returns the current file's FUID. +public function fuid() : string &cxxname="zeek::spicy::rt::fuid"; + +## Terminates the currently active Zeek-side session, flushing all state. Any +## subsequent activity will start a new session from scratch. This can only be +## called from inside a protocol analyzer. +public function terminate_session() : void &cxxname="zeek::spicy::rt::terminate_session"; + +## Signals the expected size of a file to Zeek's file analysis. +## +## size: expected size of file +## fid: Zeek-side ID of the file to operate on; if not given, the file started by the most recent file_begin() will be used +public function file_set_size(size: uint64, fid: optional = Null) : void &cxxname="zeek::spicy::rt::file_set_size"; + +## Passes file content on to Zeek's file analysis. +## +## data: chunk of raw data to pass into analysis +## fid: Zeek-side ID of the file to operate on; if not given, the file started by the most recent file_begin() will be used +public function file_data_in(data: bytes, fid: optional = Null) : void &cxxname="zeek::spicy::rt::file_data_in"; + +## Passes file content at a specific offset on to Zeek's file analysis. +## +## data: chunk of raw data to pass into analysis +## offset: position in file where data starts +## fid: Zeek-side ID of the file to operate on; if not given, the file started by the most recent file_begin() will be used +public function file_data_in_at_offset(data: bytes, offset: uint64, fid: optional = Null) : void &cxxname="zeek::spicy::rt::file_data_in_at_offset"; + +## Signals a gap in a file to Zeek's file analysis. +## +## offset: position in file where gap starts +## len: size of gap +## fid: Zeek-side ID of the file to operate on; if not given, the file started by the most recent file_begin() will be used +public function file_gap(offset: uint64, len: uint64, fid: optional = Null) : void &cxxname="zeek::spicy::rt::file_gap"; + +## Signals the end of a file to Zeek's file analysis. +## +## fid: Zeek-side ID of the file to operate on; if not given, the file started by the most recent file_begin() will be used +public function file_end(fid: optional = Null) : void &cxxname="zeek::spicy::rt::file_end"; + +## Inside a packet analyzer, forwards what data remains after parsing the top-level unit +## on to another analyzer. The index specifies the target, per the current dispatcher table. +public function forward_packet(identifier: uint32) : void &cxxname="zeek::spicy::rt::forward_packet"; + +## Gets the network time from Zeek. +public function network_time() : time &cxxname="zeek::spicy::rt::network_time"; diff --git a/scripts/spicy/zeek_file.spicy b/scripts/spicy/zeek_file.spicy new file mode 100644 index 0000000000..71082f9be9 --- /dev/null +++ b/scripts/spicy/zeek_file.spicy @@ -0,0 +1,31 @@ +# Copyright (c) 2020-2021 by the Zeek Project. See LICENSE for details. +# +# TODO: This code would ideally just be a part of zeek.spicy, but that would +# come with compilation overhead currently even if not used, see +# https://github.com/zeek/spicy/issues/301. + +module zeek_file; + +import zeek; + +## Convenience wrapper for passing content into Zeek's file analysis. +## After connecting an instance of this unit type to a sink, all data +## sent to the sink will be passed on to Zeek as a file. +## +## mime_type: MIME type of the file's content, if known; will be passed on to Zeek +## size: Total number of bytes the file contains, if known; will be passed on to Zeek +public type File = unit(mime_type: optional = Null, size: optional = Null) { + on %init { + self.fid = zeek::file_begin(mime_type); + + if ( size ) + zeek::file_set_size(*size, self.fid); + } + + : bytes &chunked &eod { zeek::file_data_in($$, self.fid); } + + on %finally { zeek::file_end(self.fid); } + + ## Zeek-side file ID + var fid: string; +}; diff --git a/scripts/spicy/zeek_rt.hlt b/scripts/spicy/zeek_rt.hlt new file mode 100644 index 0000000000..75386de9e7 --- /dev/null +++ b/scripts/spicy/zeek_rt.hlt @@ -0,0 +1,43 @@ + +module zeek_rt { + +import hilti; + +%cxx-include = "zeek/spicy/runtime-support.h"; + +public type Val = __library_type("::zeek::ValPtr"); +public type BroType = __library_type("::zeek::TypePtr"); +public type EventHandlerPtr = __library_type("::zeek::EventHandlerPtr"); + +type ZeekTypeTag = enum { + Addr, Any, Bool, Count, Double, Enum, Error, File, Func, Int, Interval, List, Opaque, Pattern, Port, Record, String, Subnet, Table, Time, Type, Vector, Void +} &cxxname="::zeek::spicy::rt::ZeekTypeTag"; + +declare public void register_protocol_analyzer(string name, hilti::Protocol protocol, vector ports, string parser_orig, string parser_resp, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_protocol_analyzer" &have_prototype; +declare public void register_file_analyzer(string name, vector mime_types, string parser, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_file_analyzer" &have_prototype; +declare public void register_packet_analyzer(string name, string parser, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_packet_analyzer" &have_prototype; +declare public void register_type(string ns, string id, BroType t) &cxxname="zeek::spicy::rt::register_type" &have_prototype; + +declare public bool have_handler(EventHandlerPtr handler) &cxxname="zeek::spicy::rt::have_handler" &have_prototype; +declare public EventHandlerPtr internal_handler(string event) &cxxname="zeek::spicy::rt::internal_handler" &have_prototype; +declare public void install_handler(string event) &cxxname="zeek::spicy::rt::install_handler" &have_prototype; + +declare public void raise_event(EventHandlerPtr handler, vector args) &cxxname="zeek::spicy::rt::raise_event" &have_prototype; +declare public BroType event_arg_type(EventHandlerPtr handler, uint<64> idx) &cxxname="zeek::spicy::rt::event_arg_type" &have_prototype; +declare public Val to_val(any x, BroType target) &cxxname="zeek::spicy::rt::to_val" &have_prototype; + +type RecordField = tuple; # (ID, type, optional) +declare public BroType create_base_type(ZeekTypeTag tag) &cxxname="zeek::spicy::rt::create_base_type" &have_prototype; +declare public BroType create_enum_type(string ns, string id, vector>> labels) &cxxname="zeek::spicy::rt::create_enum_type" &have_prototype; +declare public BroType create_record_type(string ns, string id, vector fields) &cxxname="zeek::spicy::rt::create_record_type" &have_prototype; +declare public BroType create_table_type(BroType key, optional value = Null) &cxxname="zeek::spicy::rt::create_table_type" &have_prototype; +declare public BroType create_vector_type(BroType elem) &cxxname="zeek::spicy::rt::create_vector_type" &have_prototype; + +declare public Val current_conn() &cxxname="zeek::spicy::rt::current_conn" &have_prototype; +declare public Val current_file() &cxxname="zeek::spicy::rt::current_file" &have_prototype; +declare public Val current_packet() &cxxname="zeek::spicy::rt::current_packet" &have_prototype; +declare public Val current_is_orig() &cxxname="zeek::spicy::rt::current_is_orig" &have_prototype; + +declare public void debug(string msg) &cxxname="zeek::spicy::rt::debug" &have_prototype; + +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e76d0f6703..2c04a778bf 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -172,6 +172,10 @@ add_subdirectory(logging) add_subdirectory(probabilistic) add_subdirectory(session) +if (HAVE_SPICY) + add_subdirectory(spicy) +endif () + # ############################################################################## # Build in the discovered external plugins and create the autogenerated scripts. @@ -512,6 +516,10 @@ add_dependencies(zeek_objs zeek_autogen_files) add_clang_tidy_files(${zeek_SRCS}) zeek_target_link_libraries(zeek_objs) +if (HAVE_SPICY) + target_link_libraries(zeek_objs PRIVATE hilti spicy) +endif () + if (TARGET zeek_exe) target_sources(zeek_exe PRIVATE main.cc ${zeek_HEADERS}) diff --git a/src/DebugLogger.cc b/src/DebugLogger.cc index a7b2a41dfe..4ce647157b 100644 --- a/src/DebugLogger.cc +++ b/src/DebugLogger.cc @@ -22,8 +22,7 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = { {"logging", 0, false}, {"input", 0, false}, {"threading", 0, false}, {"plugins", 0, false}, {"zeekygen", 0, false}, {"pktio", 0, false}, {"broker", 0, false}, {"scripts", 0, false}, {"supervisor", 0, false}, - {"hashkey", 0, false}, -}; + {"hashkey", 0, false}, {"spicy", 0, false}}; DebugLogger::DebugLogger() { diff --git a/src/DebugLogger.h b/src/DebugLogger.h index e8be0a71e2..041e18e407 100644 --- a/src/DebugLogger.h +++ b/src/DebugLogger.h @@ -57,6 +57,7 @@ enum DebugStream DBG_SCRIPTS, // Script initialization DBG_SUPERVISOR, // Process supervisor DBG_HASHKEY, // HashKey buffers + DBG_SPICY, // Spicy functionality NUM_DBGS // Has to be last }; diff --git a/src/plugin/Plugin.h b/src/plugin/Plugin.h index d4f34f28ff..feaf12894f 100644 --- a/src/plugin/Plugin.h +++ b/src/plugin/Plugin.h @@ -809,6 +809,13 @@ public: */ void Describe(ODesc* d) const; + /** + * Registers a component. + * + * @param c The component. The method takes ownership. + */ + void AddComponent(Component* c); + /** * Registers an individual BiF that the plugin defines. The * information is for informational purposes only and will show up in @@ -878,13 +885,6 @@ protected: */ virtual void Done(); - /** - * Registers a component. - * - * @param c The component. The method takes ownership. - */ - void AddComponent(Component* c); - /** * Calls the Initialize() function of all components. */ diff --git a/src/spicy/.clang-format b/src/spicy/.clang-format new file mode 100644 index 0000000000..ec20c65ea9 --- /dev/null +++ b/src/spicy/.clang-format @@ -0,0 +1,132 @@ +--- +Language: Cpp +AccessModifierOffset: -4 +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Right +AlignOperands: true +AlignTrailingComments: true +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: true +AllowShortFunctionsOnASingleLine: true +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: Yes +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: true + IndentBraces: false + SplitEmptyFunction: false + SplitEmptyRecord: false + SplitEmptyNamespace: false +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Custom +BreakBeforeInheritanceComma: false +BreakInheritanceList: BeforeColon +BreakBeforeTernaryOperators: false +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +ColumnLimit: 120 +CommentPragmas: 'NOLINT' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DerivePointerAlignment: false +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IncludeBlocks: Regroup +IncludeCategories: + - Regex: 'hilti/rt.*\.(h|hpp)>' + Priority: 5 + - Regex: 'spicy/rt.*\.(h|hpp)>' + Priority: 6 + - Regex: 'hilti.*\.(h|hpp)>' + Priority: 7 + - Regex: 'spicy.*\.(h|hpp)>' + Priority: 8 + - Regex: '' + Priority: 2 + - Regex: '^<.*\.h>' + Priority: 1 + - Regex: '.*' + Priority: 11 +IncludeIsMainRegex: '$' +IndentCaseLabels: true +IndentPPDirectives: None +IndentWidth: 4 +IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '^BEGIN_' +MacroBlockEnd: '^END_' +MaxEmptyLinesToKeep: 2 +NamespaceIndentation: None +ObjCBinPackProtocolList: Auto +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 500 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 1000 +PointerAlignment: Left +ReflowComments: true +SortIncludes: true +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterTemplateKeyword: false +SpaceAfterLogicalNot: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 1 +SpacesInAngles: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +SpacesInConditionalStatement: true +Standard: Cpp11 +StatementMacros: + - STANDARD_OPERATOR_1 +TabWidth: 4 +UseTab: Never +... diff --git a/src/spicy/CMakeLists.txt b/src/spicy/CMakeLists.txt new file mode 100644 index 0000000000..101c048159 --- /dev/null +++ b/src/spicy/CMakeLists.txt @@ -0,0 +1,29 @@ +# See the file "COPYING" in the main distribution directory for copyright. + +add_subdirectory(spicyz) + +zeek_add_subdir_library( + spicy + INTERNAL_DEPENDENCIES + ${BIF_BUILD_TARGET} + INCLUDE_DIRS + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} + SOURCES + manager.cc + file-analyzer.cc + packet-analyzer.cc + protocol-analyzer.cc + runtime-support.cc + BIFS + spicy.bif) + +target_link_libraries(zeek_spicy_obj PRIVATE hilti spicy) + +set(ZEEK_SPICY_MODULE_PATH "${CMAKE_INSTALL_FULL_LIBDIR}/zeek/spicy" CACHE PATH "") +install(DIRECTORY DESTINATION "${ZEEK_SPICY_MODULE_PATH}") + +set(ZEEK_SPICY_LIBRARY_PATH "${CMAKE_INSTALL_FULL_DATADIR}/zeek/spicy" CACHE PATH "") +install(DIRECTORY "${PROJECT_SOURCE_DIR}/scripts/spicy/" DESTINATION "${ZEEK_SPICY_LIBRARY_PATH}") + +set(ZEEK_SPICY_DATA_PATH "${CMAKE_INSTALL_FULL_DATADIR}/zeek" CACHE PATH "") diff --git a/src/spicy/cookie.h b/src/spicy/cookie.h new file mode 100644 index 0000000000..bc59f09a7b --- /dev/null +++ b/src/spicy/cookie.h @@ -0,0 +1,233 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +/** + * Cookie types that's stored in the HILTI context to provide access to the + * current analyzer. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "zeek/Val.h" +#include "zeek/analyzer/Analyzer.h" +#include "zeek/analyzer/protocol/tcp/TCP.h" +#include "zeek/file_analysis/Analyzer.h" +#include "zeek/packet_analysis/Analyzer.h" + +namespace zeek::spicy::rt { + +namespace cookie { + +/** State representing analysis of one file. */ +struct FileState { + FileState(std::string fid) : fid(std::move(fid)) {} + std::string fid; /**< unique Zeek-side file ID */ + std::optional mime_type; /**< MIME type, if explicitly set */ +}; + +/** + * State stored inside protocol/file analyzer cookies retaining file analysis + * state. + * + * Internally, this maintains a stack of state objects representing individual + * files that are currently in-flight. + */ +class FileStateStack { +public: + /** + * Constructor. + * + * @param analyzer_id unique ID string representing parent connection/file analyzer + */ + FileStateStack(std::string analyzer_id) : _analyzer_id(std::move(analyzer_id)) {} + + /** + * Begins analysis for a new file, pushing a new state object onto the + * stack. + */ + FileState* push(); + + /** Returns true if the stack is currently empty. */ + bool isEmpty() const { return _stack.empty(); } + + /** + * Removes an object from the stack. + * + * @param fid ID of file to remove state for; no-op if not found + */ + void remove(const std::string& fid); + + /** + * Returns a pointer to the state of the most recently pushed file. Must not + * be called on an empty stack. + **/ + const FileState* current() const { + assert(_stack.size()); + return &_stack.back(); + } + + /** + * Returns the state of a given file currently on the stack. + * + * @param fid ID of file to find + * @returns pointer to the file's state, or null if not found + */ + const FileState* find(const std::string& fid) const; + +private: + std::vector _stack; // stack of files in flight + std::string _analyzer_id; // unique ID string of parent analyzer, as passed into ctor + uint64_t _id_counter = 0; // counter incremented for each file added to this stack +}; + +/** State on the current protocol analyzer. */ +struct ProtocolAnalyzer { + analyzer::Analyzer* analyzer = nullptr; /**< current analyzer */ + bool is_orig = false; /**< direction of the connection */ + uint64_t num_packets = 0; /**< number of packets seen so far */ + FileStateStack fstate_orig; /**< file analysis state for originator side */ + FileStateStack fstate_resp; /**< file analysis state for responder side */ + std::shared_ptr fake_tcp; /**< fake TPC analyzer created internally */ +}; + +/** State on the current file analyzer. */ +struct FileAnalyzer { + file_analysis::Analyzer* analyzer = nullptr; /**< current analyzer */ + uint64_t depth = 0; /**< recursive depth of file analysis (Spicy-side file analysis only) */ + FileStateStack fstate; /**< file analysis state for nested files */ +}; + +/** State on the current file analyzer. */ +struct PacketAnalyzer { + packet_analysis::Analyzer* analyzer = nullptr; /**< current analyzer */ + Packet* packet = nullptr; /**< current packet */ + ValPtr packet_val = nullptr; /**< cached "raw_pkt_hdr" val for packet */ + std::optional next_analyzer; +}; + +} // namespace cookie + +/** + * Type of state stored in HILTI's execution context during Spicy processing. + * This is optimized for fast access and small size. + */ +struct Cookie { + // Exactly one of these pointers is non-null at any time. In that way, the + // pointers provide the semantics of a tagged union. Internals are bit + // tricky because the union itself cannot be copied/moved. + cookie::ProtocolAnalyzer* protocol = nullptr; + cookie::FileAnalyzer* file = nullptr; + cookie::PacketAnalyzer* packet = nullptr; + + Cookie(cookie::ProtocolAnalyzer&& c) : data(std::move(c)) { protocol = &data.protocol; } + Cookie(cookie::FileAnalyzer&& c) : data(std::move(c)) { file = &data.file; } + Cookie(cookie::PacketAnalyzer&& c) : data(std::move(c)) { packet = &data.packet; } + Cookie(Cookie&& other) noexcept : data(other.tag(), std::move(other.data)) { _initLike(other); } + ~Cookie() { _delete(); } + + Cookie& operator=(Cookie&& other) noexcept { + if ( this == &other ) + return *this; + + _delete(); + _initLike(other); + + new (&data) Data(tag(), std::move(other.data)); + return *this; + } + + // Cache of values that can be expensive to compute. + struct { + ValPtr conn = nullptr; // valid only for protocol analyzers + ValPtr is_orig = nullptr; // valid only for protocol analyzers + bool confirmed = false; // valid only for protocol analyzers; + } cache; + + enum Tag { Protocol, File, Packet }; + + /** Returns the type of cookie currently stored. */ + Tag tag() const { + if ( protocol ) + return Tag::Protocol; + else if ( file ) + return Tag::File; + else if ( packet ) + return Tag::Packet; + else + throw std::runtime_error("invalid cookie"); + } + +private: + union Data { + cookie::ProtocolAnalyzer protocol; + cookie::FileAnalyzer file; + cookie::PacketAnalyzer packet; + + Data(cookie::ProtocolAnalyzer&& protocol) : protocol(std::move(protocol)) {} + Data(cookie::FileAnalyzer&& file) : file(std::move(file)) {} + Data(cookie::PacketAnalyzer&& packet) : packet(std::move(packet)) {} + Data(Tag tag, Data&& other) { + switch ( tag ) { + case Tag::Protocol: new (&protocol) cookie::ProtocolAnalyzer(std::move(other.protocol)); break; + case Tag::File: new (&file) cookie::FileAnalyzer(std::move(other.file)); break; + case Tag::Packet: new (&packet) cookie::PacketAnalyzer(std::move(other.packet)); break; + } + } + + ~Data() { + // don't delete anything, Cookie is in charge. + } + + Data(const Data& other) = delete; + Data& operator=(const Data& other) = delete; + Data& operator=(Data&& other) = delete; + } data; + + void _delete() { + if ( protocol ) { + data.protocol.~ProtocolAnalyzer(); + protocol = nullptr; + cache.conn = nullptr; + cache.is_orig = nullptr; + cache.confirmed = false; + } + else if ( file ) { + data.file.~FileAnalyzer(); + file = nullptr; + } + else if ( packet ) { + data.packet.~PacketAnalyzer(); + packet = nullptr; + } + } + + void _initLike(const Cookie& other) { + if ( other.protocol ) { + protocol = &data.protocol; + cache.confirmed = other.cache.confirmed; + } + + else if ( other.file ) + file = &data.file; + + else if ( other.packet ) + packet = &data.packet; + } + + Cookie(const Cookie& other) = delete; + Cookie& operator=(const Cookie& other) = delete; + + friend inline void swap(Cookie& lhs, Cookie& rhs) { + Cookie tmp = std::move(lhs); + lhs = std::move(rhs); + rhs = std::move(tmp); + } +}; + +} // namespace zeek::spicy::rt diff --git a/src/spicy/debug.h b/src/spicy/debug.h new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/spicy/empty.cc b/src/spicy/empty.cc new file mode 100644 index 0000000000..b2191cb8b6 --- /dev/null +++ b/src/spicy/empty.cc @@ -0,0 +1,2 @@ + +int main() { return 1; } diff --git a/src/spicy/file-analyzer.cc b/src/spicy/file-analyzer.cc new file mode 100644 index 0000000000..2d3347e796 --- /dev/null +++ b/src/spicy/file-analyzer.cc @@ -0,0 +1,138 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "file-analyzer.h" + +#include + +#include "spicy.bif.h" +#include "zeek/file_analysis/File.h" +#include "zeek/spicy/manager.h" +#include "zeek/spicy/runtime-support.h" + +using namespace zeek; +using namespace zeek::spicy; +using namespace zeek::spicy::rt; + +#ifdef DEBUG +#define STATE_DEBUG_MSG(...) DebugMsg(__VA_ARGS__) +#else +#define STATE_DEBUG_MSG(...) +#endif + +void FileState::debug(const std::string& msg) { spicy::rt::debug(_cookie, msg); } + +static auto create_file_state(FileAnalyzer* analyzer) { + uint64_t depth = 0; + if ( auto current_cookie = static_cast(hilti::rt::context::cookie()) ) { + if ( const auto f = current_cookie->file ) + depth = f->depth + 1; + } + + cookie::FileAnalyzer cookie{.analyzer = analyzer, + .depth = depth, + .fstate = cookie::FileStateStack(analyzer->GetFile()->GetID())}; + return FileState(std::move(cookie)); +} + +FileAnalyzer::FileAnalyzer(RecordValPtr args, file_analysis::File* file) + : file_analysis::Analyzer(std::move(args), file), _state(create_file_state(this)) {} + +FileAnalyzer::~FileAnalyzer() {} + +void FileAnalyzer::Init() {} + +void FileAnalyzer::Done() { Finish(); } + +bool FileAnalyzer::DeliverStream(const u_char* data, uint64_t len) { + file_analysis::Analyzer::DeliverStream(data, len); + + return Process(len, data); +} + +bool FileAnalyzer::Undelivered(uint64_t offset, uint64_t len) { + file_analysis::Analyzer::Undelivered(offset, len); + + STATE_DEBUG_MSG("undelivered data, skipping further originator payload"); + _state.skipRemaining(); + return false; +} + +bool FileAnalyzer::EndOfFile() { + file_analysis::Analyzer::EndOfFile(); + Finish(); + return false; +} + +bool FileAnalyzer::Process(int len, const u_char* data) { + if ( ! _state.hasParser() && ! _state.isSkipping() ) { + auto parser = spicy_mgr->parserForFileAnalyzer(_state.file().analyzer->Tag()); + if ( parser ) + _state.setParser(parser); + else { + STATE_DEBUG_MSG("no unit specified for parsing"); + _state.skipRemaining(); + return false; + } + } + + auto* file = _state.file().analyzer->GetFile(); + + const auto& max_file_depth = BifConst::Spicy::max_file_depth; + + if ( _state.file().depth >= max_file_depth ) { + const auto& file_val = file->ToVal(); + + const auto analyzer_args = _state.file().analyzer->GetArgs(); + + file->FileEvent(Spicy::max_file_depth_exceeded, {file_val, analyzer_args, val_mgr->Count(_state.file().depth)}); + + auto tag = spicy_mgr->tagForFileAnalyzer(_state.file().analyzer->Tag()); +#if ZEEK_VERSION_NUMBER >= 50200 + AnalyzerViolation("maximal file depth exceeded", reinterpret_cast(data), len, tag); +#else + // We don't have an an appropriate way to report this with older Zeeks. +#endif + return false; + } + + try { + hilti::rt::context::CookieSetter _(_state.cookie()); + _state.process(len, reinterpret_cast(data)); + } catch ( const hilti::rt::RuntimeError& e ) { + STATE_DEBUG_MSG(hilti::rt::fmt("error during parsing, triggering analyzer violation: %s", e.what())); + auto tag = spicy_mgr->tagForFileAnalyzer(_state.file().analyzer->Tag()); +#if ZEEK_VERSION_NUMBER >= 50200 + AnalyzerViolation(e.what(), reinterpret_cast(data), len, tag); +#else + // We don't have an an appropriate way to report this with older Zeeks. +#endif + } catch ( const hilti::rt::Exception& e ) { + STATE_DEBUG_MSG(e.what()); + spicy_mgr->analyzerError(_state.file().analyzer, e.description(), + e.location()); // this sets Zeek to skip sending any further input + } + + return true; +} + +void FileAnalyzer::Finish() { + try { + hilti::rt::context::CookieSetter _(_state.cookie()); + _state.finish(); + } catch ( const hilti::rt::RuntimeError& e ) { + STATE_DEBUG_MSG(hilti::rt::fmt("error during parsing, triggering analyzer violation: %s", e.what())); + auto tag = spicy_mgr->tagForFileAnalyzer(_state.file().analyzer->Tag()); +#if ZEEK_VERSION_NUMBER >= 50200 + AnalyzerViolation(e.what(), "", 0, tag); +#else + // We don't have an an appropriate way to report this with older Zeeks. +#endif + } catch ( const hilti::rt::Exception& e ) { + spicy_mgr->analyzerError(_state.file().analyzer, e.description(), + e.location()); // this sets Zeek to skip sending any further input + } +} + +file_analysis::Analyzer* FileAnalyzer::InstantiateAnalyzer(RecordValPtr args, file_analysis::File* file) { + return new FileAnalyzer(std::move(args), file); +} diff --git a/src/spicy/file-analyzer.h b/src/spicy/file-analyzer.h new file mode 100644 index 0000000000..2bb6506a88 --- /dev/null +++ b/src/spicy/file-analyzer.h @@ -0,0 +1,91 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include +#include +#include + +#include + +#include +#include + +#include "zeek/spicy/cookie.h" + +namespace zeek::spicy::rt { + +/** Parsing state for a file. */ +class FileState : public ::spicy::rt::driver::ParsingState { +public: + /** + * Constructor. + * + * @param cookie cookie to associated with the file + */ + FileState(Cookie cookie) : ParsingState(::spicy::rt::driver::ParsingType::Stream), _cookie(std::move(cookie)) {} + + /** Returns the cookie pointer to use with the runtime library during analysis. */ + auto* cookie() { return &_cookie; } + + /** Returns the file-specific cookie state associated with the endpoint. */ + auto& file() { + assert(_cookie.file); + return *_cookie.file; + } + + /** + * Records a debug message pertaining to the specific file. + * + * @param msg message to record + */ + void DebugMsg(const std::string& msg) { debug(msg); } + +protected: + // Overridden from driver::ParsingState. + void debug(const std::string& msg) override; + +private: + Cookie _cookie; +}; + +/** A Spicy file analyzer. */ +class FileAnalyzer : public file_analysis::Analyzer { +public: + FileAnalyzer(RecordValPtr arg_args, file_analysis::File* arg_file); + virtual ~FileAnalyzer(); + + static file_analysis::Analyzer* InstantiateAnalyzer(RecordValPtr args, file_analysis::File* file); + +protected: + // Overridden from Zeek's file analyzer. + void Init() override; + void Done() override; + bool DeliverStream(const u_char* data, uint64_t len) override; + bool Undelivered(uint64_t offset, uint64_t len) override; + bool EndOfFile() override; + + /** + * Feeds a chunk of data into parsing. + * + * @param len number of bytes valid in *data* + * @param data pointer to data + * @return true if processing succeeded, false if an error occurred that + * stopped parsing + */ + bool Process(int len, const u_char* data); + + /** + * Finalizes parsing. After calling this, no more data can be passed into + * Process(). + */ + void Finish(); + + /** Records a debug message. */ + void DebugMsg(const std::string& msg) { _state.DebugMsg(msg); } + +private: + FileState _state; +}; + +} // namespace zeek::spicy::rt diff --git a/src/spicy/manager.cc b/src/spicy/manager.cc new file mode 100644 index 0000000000..267f0cf1ea --- /dev/null +++ b/src/spicy/manager.cc @@ -0,0 +1,920 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/spicy/manager.h" + +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include + +#include "zeek/DebugLogger.h" +#include "zeek/spicy/file-analyzer.h" +#include "zeek/spicy/packet-analyzer.h" +#include "zeek/spicy/protocol-analyzer.h" +#include "zeek/util-config.h" + +using namespace zeek; +using namespace zeek::spicy; + +// Split an potentially scoped ID into namespace and local part. +static std::pair parseID(const std::string& s) { + if ( auto i = s.rfind("::"); i != std::string::npos ) + return std::make_pair(s.substr(0, i), s.substr(i + 2)); + else + return std::make_pair("", s); +} + +Manager::~Manager() {} + +void Manager::registerProtocolAnalyzer(const std::string& name, hilti::rt::Protocol proto, + const hilti::rt::Vector& ports, const std::string& parser_orig, + const std::string& parser_resp, const std::string& replaces, + const std::string& linker_scope) { + SPICY_DEBUG(hilti::rt::fmt("Have Spicy protocol analyzer %s", name)); + + ProtocolAnalyzerInfo info; + info.name_analyzer = name; + info.name_parser_orig = parser_orig; + info.name_parser_resp = parser_resp; + info.name_replaces = replaces; + info.name_zeek = hilti::rt::replace(name, "::", "_"); + info.name_zeekygen = hilti::rt::fmt("", name); + info.protocol = proto; + info.ports = ports; + info.linker_scope = linker_scope; + + // We may have that analyzer already iff it was previously pre-registered + // without a linker scope. We'll then only set the scope now. + if ( auto t = _analyzer_name_to_tag_type.find(info.name_zeek); t != _analyzer_name_to_tag_type.end() ) { + SPICY_DEBUG(hilti::rt::fmt("Updating already registered protocol analyzer %s", name)); + + auto& existing = _protocol_analyzers_by_type.at(t->second); + assert(existing.name_analyzer == name); + existing.linker_scope = info.linker_scope; + + // If the infos don't match now, we have two separate definitions. + if ( info != existing ) + reporter->FatalError("redefinition of protocol analyzer %s", info.name_analyzer.c_str()); + + return; + } + + analyzer::Component::factory_callback factory = nullptr; + +#if SPICY_VERSION_NUMBER >= 10700 + auto proto_ = proto.value(); +#else + auto proto_ = proto; +#endif + + switch ( proto_ ) { + case hilti::rt::Protocol::TCP: factory = spicy::rt::TCP_Analyzer::InstantiateAnalyzer; break; + case hilti::rt::Protocol::UDP: factory = spicy::rt::UDP_Analyzer::InstantiateAnalyzer; break; + default: reporter->Error("unsupported protocol in analyzer"); return; + } + + auto c = new ::zeek::analyzer::Component(info.name_zeek, factory, 0); + AddComponent(c); + + // Hack to prevent Zeekygen from reporting the ID as not having a + // location during the following initialization step. + ::zeek::detail::zeekygen_mgr->Script(info.name_zeekygen); + ::zeek::detail::set_location(makeLocation(info.name_zeekygen)); + + // TODO: Should Zeek do this? It has run component intiialization at + // this point already, so ours won't get initialized anymore. + c->Initialize(); + + trackComponent(c, c->Tag().Type()); // Must come after Initialize(). + + info.type = c->Tag().Type(); + _protocol_analyzers_by_type.resize(info.type + 1); + _protocol_analyzers_by_type[info.type] = info; +} + +void Manager::registerFileAnalyzer(const std::string& name, const hilti::rt::Vector& mime_types, + const std::string& parser, const std::string& replaces, + const std::string& linker_scope) { + SPICY_DEBUG(hilti::rt::fmt("Have Spicy file analyzer %s", name)); + + FileAnalyzerInfo info; + info.name_analyzer = name; + info.name_parser = parser; + info.name_replaces = replaces; + info.name_zeek = hilti::rt::replace(name, "::", "_"); + info.name_zeekygen = hilti::rt::fmt("", name); + info.mime_types = mime_types; + info.linker_scope = linker_scope; + + // We may have that analyzer already iff it was previously pre-registered + // without a linker scope. We'll then only set the scope now. + if ( auto t = _analyzer_name_to_tag_type.find(info.name_zeek); t != _analyzer_name_to_tag_type.end() ) { + SPICY_DEBUG(hilti::rt::fmt("Updating already registered packet analyzer %s", name)); + + auto& existing = _file_analyzers_by_type.at(t->second); + existing.linker_scope = info.linker_scope; + + // If the infos don't match now, we have two separate definitions. + if ( info != existing ) + reporter->FatalError("redefinition of file analyzer %s", info.name_analyzer.c_str()); + + return; + } + + auto c = new ::zeek::file_analysis::Component(info.name_zeek, spicy::rt::FileAnalyzer::InstantiateAnalyzer, 0); + AddComponent(c); + + // Hack to prevent Zeekygen from reporting the ID as not having a + // location during the following initialization step. + ::zeek::detail::zeekygen_mgr->Script(info.name_zeekygen); + ::zeek::detail::set_location(makeLocation(info.name_zeekygen)); + + // TODO: Should Zeek do this? It has run component intiialization at + // this point already, so ours won't get initialized anymore. + c->Initialize(); + + trackComponent(c, c->Tag().Type()); // Must come after Initialize(). + + info.type = c->Tag().Type(); + _file_analyzers_by_type.resize(info.type + 1); + _file_analyzers_by_type[info.type] = info; +} + +void Manager::registerPacketAnalyzer(const std::string& name, const std::string& parser, const std::string& replaces, + const std::string& linker_scope) { + SPICY_DEBUG(hilti::rt::fmt("Have Spicy packet analyzer %s", name)); + + PacketAnalyzerInfo info; + info.name_analyzer = name; + info.name_replaces = replaces; + info.name_parser = parser; + info.name_zeek = hilti::rt::replace(name, "::", "_"); + info.name_zeekygen = hilti::rt::fmt("", info.name_zeek); + info.linker_scope = linker_scope; + + // We may have that analyzer already iff it was previously pre-registered + // without a linker scope. We'll then set the scope now. + if ( auto t = _analyzer_name_to_tag_type.find(info.name_zeek); t != _analyzer_name_to_tag_type.end() ) { + SPICY_DEBUG(hilti::rt::fmt("Updating already registered packet analyzer %s", name)); + + auto& existing = _packet_analyzers_by_type.at(t->second); + assert(existing.name_analyzer == name); + existing.linker_scope = info.linker_scope; + + // If the infos don't match now, we have two separate definitions. + if ( info != existing ) + reporter->FatalError("redefinition of packet analyzer %s", info.name_analyzer.c_str()); + + return; + } + + auto instantiate = [info]() -> packet_analysis::AnalyzerPtr { + return spicy::rt::PacketAnalyzer::Instantiate(info.name_zeek); + }; + + auto c = new ::zeek::packet_analysis::Component(info.name_zeek, instantiate, 0); + AddComponent(c); + + // Hack to prevent Zeekygen from reporting the ID as not having a + // location during the following initialization step. + ::zeek::detail::zeekygen_mgr->Script(info.name_zeekygen); + ::zeek::detail::set_location(makeLocation(info.name_zeekygen)); + + // TODO: Should Zeek do this? It has run component intialization at + // this point already, so ours won't get initialized anymore. + c->Initialize(); + + trackComponent(c, c->Tag().Type()); // Must come after Initialize(). + + info.type = c->Tag().Type(); + _packet_analyzers_by_type.resize(info.type + 1); + _packet_analyzers_by_type[info.type] = info; +} + +void Manager::registerType(const std::string& id, const TypePtr& type) { + auto [ns, local] = parseID(id); + + if ( const auto& old = detail::lookup_ID(local.c_str(), ns.c_str()) ) { + // This is most likely to trigger for IDs that other Spicy modules + // register. If we two Spicy modules need the same type, that's ok as + // long as they match. + if ( ! old->IsType() ) { + reporter->Error("Zeek type registration failed for '%s': ID already exists, but is not a type", id.c_str()); + return; + } + + if ( ! zeek::same_type(type, old->GetType()) ) { + reporter->Error("Zeek type registration failed for '%s': Type already exists, but differs", id.c_str()); + } + + SPICY_DEBUG(hilti::rt::fmt("Not re-registering Zeek type %s: identical type already exists", id)); + return; + } + + SPICY_DEBUG(hilti::rt::fmt("Registering Zeek type %s", id)); + auto zeek_id = detail::install_ID(local.c_str(), ns.c_str(), true, true); + zeek_id->SetType(type); + zeek_id->MakeType(); + AddBifItem(id, ::zeek::plugin::BifItem::TYPE); +} + +TypePtr Manager::findType(const std::string& id) const { + auto [ns, local] = parseID(id); + + auto zid = detail::lookup_ID(local.c_str(), ns.c_str()); + if ( ! zid ) + return nullptr; + + if ( ! zid->IsType() ) + return nullptr; + + return zid->GetType(); +} + +void Manager::registerEvent(const std::string& name) { + // Create a Zeek handler for the event. + event_registry->Register(name); + + // Install the ID into the corresponding namespace and export it. + auto n = ::hilti::rt::split(name, "::"); + std::string mod; + + if ( n.size() > 1 ) + mod = n.front(); + else + mod = detail::GLOBAL_MODULE_NAME; + + if ( auto id = detail::lookup_ID(name.c_str(), mod.c_str(), false, false, false) ) { + // Auto-export IDs that already exist. + id->SetExport(); + _events[name] = id; + } + else + // This installs & exports the ID, but it doesn't set its type yet. + // That will happen as handlers get defined. If there are no handlers, + // we set a dummy type in the plugin's InitPostScript + _events[name] = detail::install_ID(name.c_str(), mod.c_str(), false, true); +} + +const ::spicy::rt::Parser* Manager::parserForProtocolAnalyzer(const Tag& tag, bool is_orig) { + if ( is_orig ) + return _protocol_analyzers_by_type[tag.Type()].parser_orig; + else + return _protocol_analyzers_by_type[tag.Type()].parser_resp; +} + +const ::spicy::rt::Parser* Manager::parserForFileAnalyzer(const Tag& tag) { + return _file_analyzers_by_type[tag.Type()].parser; +} + +const ::spicy::rt::Parser* Manager::parserForPacketAnalyzer(const Tag& tag) { + return _packet_analyzers_by_type[tag.Type()].parser; +} + +Tag Manager::tagForProtocolAnalyzer(const Tag& tag) { + if ( auto r = _protocol_analyzers_by_type[tag.Type()].replaces ) + return r; + else + return tag; +} + +Tag Manager::tagForFileAnalyzer(const Tag& tag) { + if ( auto r = _file_analyzers_by_type[tag.Type()].replaces ) + return r; + else + return tag; +} + +Tag Manager::tagForPacketAnalyzer(const Tag& tag) { + if ( auto r = _packet_analyzers_by_type[tag.Type()].replaces ) + return r; + else + return tag; +} + +bool Manager::toggleProtocolAnalyzer(const Tag& tag, bool enable) { + auto type = tag.Type(); + + if ( type >= _protocol_analyzers_by_type.size() ) + return false; + + const auto& analyzer = _protocol_analyzers_by_type[type]; + + if ( ! analyzer.type ) + // not set -> not ours + return false; + + if ( enable ) { + SPICY_DEBUG(hilti::rt::fmt("Enabling Spicy protocol analyzer %s", analyzer.name_analyzer)); + analyzer_mgr->EnableAnalyzer(tag); + + if ( analyzer.replaces ) { + SPICY_DEBUG(hilti::rt::fmt("Disabling standard protocol analyzer %s", analyzer.name_analyzer)); + analyzer_mgr->DisableAnalyzer(analyzer.replaces); + } + } + else { + SPICY_DEBUG(hilti::rt::fmt("Disabling Spicy protocol analyzer %s", analyzer.name_analyzer)); + analyzer_mgr->DisableAnalyzer(tag); + + if ( analyzer.replaces ) { + SPICY_DEBUG(hilti::rt::fmt("Re-enabling standard protocol analyzer %s", analyzer.name_analyzer)); + analyzer_mgr->EnableAnalyzer(analyzer.replaces); + } + } + + return true; +} + +bool Manager::toggleFileAnalyzer(const Tag& tag, bool enable) { + auto type = tag.Type(); + + if ( type >= _file_analyzers_by_type.size() ) + return false; + + const auto& analyzer = _file_analyzers_by_type[type]; + + if ( ! analyzer.type ) + // not set -> not ours + return false; + + file_analysis::Component* component = file_mgr->Lookup(tag); + file_analysis::Component* component_replaces = analyzer.replaces ? file_mgr->Lookup(analyzer.replaces) : nullptr; + + if ( ! component ) { + // Shouldn't really happen. + reporter->InternalError("failed to lookup file analyzer component"); + return false; + } + + if ( enable ) { + SPICY_DEBUG(hilti::rt::fmt("Enabling Spicy file analyzer %s", analyzer.name_analyzer)); + component->SetEnabled(true); + + if ( component_replaces ) { + SPICY_DEBUG(hilti::rt::fmt("Disabling standard file analyzer %s", analyzer.name_analyzer)); + component_replaces->SetEnabled(false); + } + } + else { + SPICY_DEBUG(hilti::rt::fmt("Disabling Spicy file analyzer %s", analyzer.name_analyzer)); + component->SetEnabled(false); + + if ( component_replaces ) { + SPICY_DEBUG(hilti::rt::fmt("Enabling standard file analyzer %s", analyzer.name_analyzer)); + component_replaces->SetEnabled(true); + } + } + + return true; +} + +bool Manager::togglePacketAnalyzer(const Tag& tag, bool enable) { + auto type = tag.Type(); + + if ( type >= _packet_analyzers_by_type.size() ) + return false; + + const auto& analyzer = _packet_analyzers_by_type[type]; + + if ( ! analyzer.type ) + // not set -> not ours + return false; + + packet_analysis::Component* component = packet_mgr->Lookup(tag); + packet_analysis::Component* component_replaces = + analyzer.replaces ? packet_mgr->Lookup(analyzer.replaces) : nullptr; + + if ( ! component ) { + // Shouldn't really happen. + reporter->InternalError("failed to lookup packet analyzer component"); + return false; + } + + if ( enable ) { + SPICY_DEBUG(hilti::rt::fmt("Enabling Spicy packet analyzer %s", analyzer.name_analyzer)); + component->SetEnabled(true); + + if ( component_replaces ) { + SPICY_DEBUG(hilti::rt::fmt("Disabling standard packet analyzer %s", analyzer.name_analyzer)); + component_replaces->SetEnabled(false); + } + } + else { + SPICY_DEBUG(hilti::rt::fmt("Disabling Spicy packet analyzer %s", analyzer.name_analyzer)); + component->SetEnabled(false); + + if ( component_replaces ) { + SPICY_DEBUG(hilti::rt::fmt("Enabling standard packet analyzer %s", analyzer.name_analyzer)); + component_replaces->SetEnabled(true); + } + } + + return true; +} + +bool Manager::toggleAnalyzer(EnumVal* tag, bool enable) { + if ( tag->GetType() == analyzer_mgr->GetTagType() ) { + if ( auto analyzer = analyzer_mgr->Lookup(tag) ) + return toggleProtocolAnalyzer(analyzer->Tag(), enable); + else + return false; + } + + if ( tag->GetType() == file_mgr->GetTagType() ) { + if ( auto analyzer = file_mgr->Lookup(tag) ) + return toggleFileAnalyzer(analyzer->Tag(), enable); + else + return false; + } + + if ( tag->GetType() == packet_mgr->GetTagType() ) { + if ( auto analyzer = packet_mgr->Lookup(tag) ) + return togglePacketAnalyzer(analyzer->Tag(), enable); + else + return false; + } + + return false; +} + +static std::unique_ptr _makeLocation(const std::string& location) { + static std::set filenames; // see comment below in parse_location + + auto parse_location = [](const auto& s) -> std::unique_ptr { + // This is not so great; In the HILTI runtome we pass locations + // around as string. To pass them to Zeek, we need to unsplit the + // strings into file name and line number. Zeek also won't clean up + // the file names, so we need to track them ourselves. + auto x = hilti::rt::split(s, ":"); + if ( x[0].empty() ) + return nullptr; + + auto loc = std::make_unique(); + loc->filename = filenames.insert(std::string(x[0])).first->c_str(); // we retain ownership + + if ( x.size() >= 2 ) { + auto y = hilti::rt::split(x[1], "-"); + if ( y.size() >= 2 ) { + loc->first_line = std::stoi(std::string(y[0])); + loc->last_line = std::stoi(std::string(y[1])); + } + else if ( y[0].size() ) + loc->first_line = loc->last_line = std::stoi(std::string(y[0])); + } + + return loc; + }; + + if ( location.size() ) + return parse_location(location); + else if ( auto hilti_location = hilti::rt::debug::location() ) + return parse_location(hilti_location); + else + return nullptr; +} + +void Manager::analyzerError(analyzer::Analyzer* a, const std::string& msg, const std::string& location) { + auto zeek_location = _makeLocation(location); + reporter->PushLocation(zeek_location.get()); + reporter->AnalyzerError(a, "%s", msg.c_str()); + reporter->PopLocation(); +} + +void Manager::analyzerError(file_analysis::Analyzer* a, const std::string& msg, const std::string& location) { + auto zeek_location = _makeLocation(location); + reporter->PushLocation(zeek_location.get()); + + // We don't have an reporter error for file analyzers, so we log this as a + // weird instead. + if ( a && a->GetFile() ) + reporter->Weird(a->GetFile(), "file_error", msg.c_str()); + else + reporter->Weird("file_error", msg.c_str()); + + reporter->PopLocation(); + + if ( a ) + a->SetSkip(1); // Imitate what AnalyzerError() does for protocol analyzers. +} + +void Manager::analyzerError(packet_analysis::Analyzer* a, const std::string& msg, const std::string& location) { + auto zeek_location = _makeLocation(location); + reporter->PushLocation(zeek_location.get()); + // We don't have an reporter error for packet analyzers, so we log + // this as a weird instead. + reporter->Weird("packet_error", msg.c_str()); + reporter->PopLocation(); +} + +plugin::Configuration Manager::Configure() { + ::zeek::plugin::Configuration config; + config.name = "Zeek::Spicy"; + config.description = "Support for Spicy parsers (*.hlto)"; + + EnableHook(::zeek::plugin::HOOK_LOAD_FILE); + + return config; +} + +void Manager::InitPreScript() { + SPICY_DEBUG("Beginning pre-script initialization"); + +#if SPICY_VERSION_NUMBER >= 10700 + hilti::rt::executeManualPreInits(); +#endif + + autoDiscoverModules(); + + SPICY_DEBUG("Done with pre-script initialization"); +} + +// Returns a port's Zeek-side transport protocol. +static ::TransportProto transport_protocol(const hilti::rt::Port port) { +#if SPICY_VERSION_NUMBER >= 10700 + auto proto = port.protocol().value(); +#else + auto proto = port.protocol(); +#endif + + switch ( proto ) { + case hilti::rt::Protocol::TCP: return ::TransportProto::TRANSPORT_TCP; + case hilti::rt::Protocol::UDP: return ::TransportProto::TRANSPORT_UDP; + case hilti::rt::Protocol::ICMP: return ::TransportProto::TRANSPORT_ICMP; + default: + reporter->InternalError("unsupported transport protocol in port '%s' for Zeek conversion", + std::string(port).c_str()); + return ::TransportProto::TRANSPORT_UNKNOWN; + } +} + +static void hook_accept_input() { + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( auto x = cookie->protocol ) { + auto tag = spicy_mgr->tagForProtocolAnalyzer(x->analyzer->GetAnalyzerTag()); + SPICY_DEBUG(hilti::rt::fmt("confirming protocol %s", tag.AsString())); + return x->analyzer->AnalyzerConfirmation(tag); + } +} + +static void hook_decline_input(const std::string& reason) { + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( auto x = cookie->protocol ) { + auto tag = spicy_mgr->tagForProtocolAnalyzer(x->analyzer->GetAnalyzerTag()); + SPICY_DEBUG(hilti::rt::fmt("rejecting protocol %s", tag.AsString())); + return x->analyzer->AnalyzerViolation("protocol rejected", nullptr, 0, tag); + } +} + +void Manager::InitPostScript() { + SPICY_DEBUG("Beginning post-script initialization"); + + disableReplacedAnalyzers(); + + // If there's no handler for one of our events, it won't have received a + // type. Give it a dummy event type in that case, so that we don't walk + // around with a nullptr. + for ( const auto& [name, id] : _events ) { + if ( ! id->GetType() ) { + auto args = make_intrusive(new type_decl_list()); + auto et = make_intrusive(std::move(args), base_type(TYPE_VOID), FUNC_FLAVOR_EVENT); + id->SetType(std::move(et)); + } + } + + // Init runtime, which will trigger all initialization code to execute. + SPICY_DEBUG("Initializing Spicy runtime"); + + auto hilti_config = hilti::rt::configuration::get(); + + if ( id::find_const("Spicy::enable_print")->AsBool() ) // NOLINT(clang-analyzer-cplusplus.NewDeleteLeaks) + hilti_config.cout = std::cout; + else + hilti_config.cout.reset(); + + if ( id::find_const("Spicy::enable_profiling")->AsBool() ) +#if SPICY_VERSION_NUMBER >= 10800 + hilti_config.enable_profiling = true; +#else + std::cerr << "Profiling is not supported with this version of Spicy, ignoring " + "'Spicy::enable_profiling'\n"; +#endif + + hilti_config.abort_on_exceptions = id::find_const("Spicy::abort_on_exceptions")->AsBool(); + hilti_config.show_backtraces = id::find_const("Spicy::show_backtraces")->AsBool(); + + hilti::rt::configuration::set(hilti_config); + +#if SPICY_VERSION_NUMBER >= 10700 + auto spicy_config = ::spicy::rt::configuration::get(); + spicy_config.hook_accept_input = hook_accept_input; + spicy_config.hook_decline_input = hook_decline_input; + ::spicy::rt::configuration::set(std::move(spicy_config)); +#endif + + try { + ::hilti::rt::init(); + ::spicy::rt::init(); + } catch ( const hilti::rt::Exception& e ) { + std::cerr << hilti::rt::fmt("uncaught runtime exception %s during initialization: %s", + hilti::rt::demangle(typeid(e).name()), e.what()) + << std::endl; + exit(1); + } catch ( const std::runtime_error& e ) { + std::cerr << hilti::rt::fmt("uncaught C++ exception %s during initialization: %s", + hilti::rt::demangle(typeid(e).name()), e.what()) + << std::endl; + exit(1); + } + + // Fill in the parser information now that we derived from the ASTs. + auto find_parser = [](const std::string& analyzer, const std::string& parser, + const std::string& linker_scope) -> const ::spicy::rt::Parser* { + if ( parser.empty() ) + return nullptr; + + for ( auto p : ::spicy::rt::parsers() ) { + if ( p->name == parser && p->linker_scope == linker_scope ) + return p; + } + + reporter->InternalError("Unknown Spicy parser '%s' requested by analyzer '%s'", parser.c_str(), + analyzer.c_str()); + + return nullptr; // cannot be reached + }; + + for ( auto& p : _protocol_analyzers_by_type ) { + if ( p.type == 0 ) + // vector element not set + continue; + + SPICY_DEBUG(hilti::rt::fmt("Registering %s protocol analyzer %s with Zeek", p.protocol, p.name_analyzer)); + + p.parser_orig = find_parser(p.name_analyzer, p.name_parser_orig, p.linker_scope); + p.parser_resp = find_parser(p.name_analyzer, p.name_parser_resp, p.linker_scope); + + // Register analyzer for its well-known ports. + auto tag = analyzer_mgr->GetAnalyzerTag(p.name_zeek.c_str()); + if ( ! tag ) + reporter->InternalError("cannot get analyzer tag for '%s'", p.name_analyzer.c_str()); + + for ( auto port : p.ports ) { + SPICY_DEBUG(hilti::rt::fmt(" Scheduling analyzer for port %s", port)); + analyzer_mgr->RegisterAnalyzerForPort(tag, transport_protocol(port), port.port()); + } + + if ( p.parser_resp ) { + for ( auto port : p.parser_resp->ports ) { + if ( port.direction != ::spicy::rt::Direction::Both && + port.direction != ::spicy::rt::Direction::Responder ) + continue; + + SPICY_DEBUG(hilti::rt::fmt(" Scheduling analyzer for port %s", port.port)); + analyzer_mgr->RegisterAnalyzerForPort(tag, transport_protocol(port.port), port.port.port()); + } + } + } + + for ( auto& p : _file_analyzers_by_type ) { + if ( p.type == 0 ) + // vector element not set + continue; + + SPICY_DEBUG(hilti::rt::fmt("Registering file analyzer %s with Zeek", p.name_analyzer.c_str())); + + p.parser = find_parser(p.name_analyzer, p.name_parser, p.linker_scope); + + // Register analyzer for its MIME types. + auto tag = file_mgr->GetComponentTag(p.name_zeek.c_str()); + if ( ! tag ) + reporter->InternalError("cannot get analyzer tag for '%s'", p.name_analyzer.c_str()); + + auto register_analyzer_for_mime_type = [&](auto tag, const std::string& mt) { + SPICY_DEBUG(hilti::rt::fmt(" Scheduling analyzer for MIME type %s", mt)); + + // MIME types are registered in scriptland, so we'll raise an + // event that will do it for us through a predefined handler. + zeek::Args vals = Args(); + vals.emplace_back(tag.AsVal()); + vals.emplace_back(make_intrusive(mt)); + EventHandlerPtr handler = event_registry->Register("spicy_analyzer_for_mime_type"); + event_mgr.Enqueue(handler, vals); + }; + + for ( const auto& mt : p.mime_types ) + register_analyzer_for_mime_type(tag, mt); + + if ( p.parser ) { + for ( const auto& mt : p.parser->mime_types ) + register_analyzer_for_mime_type(tag, mt); + } + } + + for ( auto& p : _packet_analyzers_by_type ) { + if ( p.type == 0 ) + // vector element not set + continue; + + SPICY_DEBUG(hilti::rt::fmt("Registering packet analyzer %s with Zeek", p.name_analyzer.c_str())); + p.parser = find_parser(p.name_analyzer, p.name_parser, p.linker_scope); + } + + SPICY_DEBUG("Done with post-script initialization"); +} + +void Manager::Done() { + SPICY_DEBUG("Shutting down Spicy runtime"); + ::spicy::rt::done(); + hilti::rt::done(); +} + +void Manager::loadModule(const hilti::rt::filesystem::path& path) { + try { + // If our auto discovery ends up finding the same module multiple times, + // we ignore subsequent requests. + std::error_code ec; + auto canonical_path = hilti::rt::filesystem::canonical(path, ec); + if ( ec ) + hilti::rt::fatalError(hilti::rt::fmt("could not compute canonical path for %s: %s", path, ec.message())); + + if ( auto [library, inserted] = _libraries.insert({canonical_path, hilti::rt::Library(canonical_path)}); + inserted ) { + SPICY_DEBUG(hilti::rt::fmt("Loading %s", canonical_path.native())); + if ( auto load = library->second.open(); ! load ) + hilti::rt::fatalError( + hilti::rt::fmt("could not open library path %s: %s", canonical_path, load.error())); + } + else { + SPICY_DEBUG(hilti::rt::fmt("Ignoring duplicate loading request for %s", canonical_path.native())); + } +#if SPICY_VERSION_NUMBER >= 10700 + } catch ( const ::hilti::rt::UsageError& e ) { +#else + } catch ( const ::hilti::rt::UserException& e ) { +#endif + hilti::rt::fatalError(e.what()); + } +} + +int Manager::HookLoadFile(const LoadType type, const std::string& file, const std::string& resolved) { + auto ext = hilti::rt::filesystem::path(file).extension(); + + if ( ext == ".hlto" ) { + loadModule(file); + return 1; + } + + if ( ext == ".spicy" || ext == ".evt" || ext == ".hlt" ) + reporter->FatalError("cannot load '%s': analyzers need to be precompiled with 'spicyz' ", file.c_str()); + + return -1; +} + +void Manager::searchModules(const std::string& paths) { + for ( const auto& dir : hilti::rt::split(paths, ":") ) { + auto trimmed_dir = hilti::rt::trim(dir); + if ( trimmed_dir.empty() ) + continue; + + std::error_code ec; + if ( auto is_directory = hilti::rt::filesystem::is_directory(trimmed_dir, ec); ec || ! is_directory ) { + SPICY_DEBUG(hilti::rt::fmt("Module directory %s cannot be read, skipping", trimmed_dir)); + continue; + } + + SPICY_DEBUG(hilti::rt::fmt("Searching %s for *.hlto", trimmed_dir)); + + auto it = hilti::rt::filesystem::recursive_directory_iterator(trimmed_dir, ec); + if ( ! ec ) { + while ( it != hilti::rt::filesystem::recursive_directory_iterator() ) { + if ( it->is_regular_file() && it->path().extension() == ".hlto" ) + loadModule(it->path()); + + if ( it.increment(ec); ec ) { + hilti::rt::warning(hilti::rt::fmt("Error iterating over %s, skipping any remaining files: %s", + trimmed_dir, ec.message())); + break; + } + } + } + else + hilti::rt::warning(hilti::rt::fmt("Cannot iterate over %s, skipping: %s", trimmed_dir, ec.message())); + } +}; + +detail::Location Manager::makeLocation(const std::string& fname) { + auto x = _locations.insert(fname); + return detail::Location(x.first->c_str(), 0, 0, 0, 0); +} + +void Manager::autoDiscoverModules() { + // Always search Zeek's plugin path for modules, that's where zkg puts + // them. + searchModules(util::zeek_plugin_path()); + + if ( auto search_paths = hilti::rt::getenv("ZEEK_SPICY_MODULE_PATH"); search_paths && search_paths->size() ) + // This overrides all other paths. + searchModules(*search_paths); + else + searchModules(ZEEK_SPICY_MODULE_PATH); +} + +void Manager::disableReplacedAnalyzers() { + for ( auto& info : _protocol_analyzers_by_type ) { + if ( info.name_replaces.empty() ) + continue; + + auto replaces = info.name_replaces.c_str(); + + if ( file_mgr->Lookup(replaces) || packet_mgr->Lookup(replaces) ) + reporter->FatalError("cannot replace '%s' analyzer with a protocol analyzer", replaces); + + auto tag = analyzer_mgr->GetAnalyzerTag(replaces); + if ( ! tag ) { + SPICY_DEBUG(hilti::rt::fmt("%s is supposed to replace protocol analyzer %s, but that does not exist", + info.name_analyzer, replaces)); + + continue; + } + + SPICY_DEBUG(hilti::rt::fmt("%s replaces existing protocol analyzer %s", info.name_analyzer, replaces)); + info.replaces = tag; + analyzer_mgr->DisableAnalyzer(tag); + } + + for ( auto& info : _file_analyzers_by_type ) { + if ( info.name_replaces.empty() ) + continue; + + auto replaces = info.name_replaces.c_str(); + + if ( analyzer_mgr->Lookup(replaces) || packet_mgr->Lookup(replaces) ) + reporter->FatalError("cannot replace '%s' analyzer with a file analyzer", replaces); + + auto component = file_mgr->Lookup(replaces); + if ( ! component ) { + SPICY_DEBUG(hilti::rt::fmt("%s is supposed to replace file analyzer %s, but that does not exist", + info.name_analyzer, replaces)); + + continue; + } + + SPICY_DEBUG(hilti::rt::fmt("%s replaces existing file analyzer %s", info.name_analyzer, replaces)); + info.replaces = component->Tag(); + component->SetEnabled(false); + } + + for ( auto& info : _packet_analyzers_by_type ) { + if ( info.name_replaces.empty() ) + continue; + + auto replaces = info.name_replaces.c_str(); + + auto component = packet_mgr->Lookup(replaces); + if ( ! component ) { + SPICY_DEBUG(hilti::rt::fmt("%s is supposed to replace packet analyzer %s, but that does not exist", + info.name_analyzer, replaces)); + + continue; + } + + SPICY_DEBUG(hilti::rt::fmt("%s replaces existing packet analyzer %s", info.name_analyzer, replaces)); + info.replaces = component->Tag(); + component->SetEnabled(false); + } +} + +void Manager::trackComponent(plugin::Component* c, int32_t tag_type) { + auto i = _analyzer_name_to_tag_type.insert({c->Name(), tag_type}); + if ( ! i.second ) + // We enforce on our end that an analyzer name can appear only once + // across all types of analyzers. Makes things easier and avoids + // confusion. + reporter->FatalError("duplicate analyzer name '%s'", c->Name().c_str()); +} diff --git a/src/spicy/manager.h b/src/spicy/manager.h new file mode 100644 index 0000000000..8712d16ee4 --- /dev/null +++ b/src/spicy/manager.h @@ -0,0 +1,413 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "zeek/Scope.h" +#include "zeek/Tag.h" +#include "zeek/plugin/Component.h" +#include "zeek/plugin/Plugin.h" + +// Macro helper to report Spicy debug messages. This forwards to +// to both the Zeek logger and the Spicy runtime logger. +#define SPICY_DEBUG(msg) ::zeek::spicy::log(msg); + +namespace hilti::rt { +class Port; +struct Protocol; +} // namespace hilti::rt + +namespace spicy::rt { +struct Parser; +} + +namespace zeek { + +namespace analyzer { +class Analyzer; +} + +namespace file_analysis { +class Analyzer; +} + +namespace packet_analysis { +class Analyzer; +} + +namespace spicy { + +// Backend to log debug messages. +inline void log(const std::string& msg) { + DBG_LOG(DBG_SPICY, "%s", msg.c_str()); + + if ( hilti::rt::isInitialized() ) + HILTI_RT_DEBUG("zeek", msg); +} + + +/* Top-level entry point for Spicy functionality. */ +class Manager : public zeek::plugin::Plugin { +public: + Manager() {} + virtual ~Manager(); + + /** + * Runtime method to register a protocol analyzer with its Zeek-side + * configuration. This is called at startup by generated Spicy code for + * each protocol analyzer defined in an EVT file. + * + * @param name name of the analyzer as defined in its EVT file + * @param proto analyzer's transport-layer protocol + * @param prts well-known ports for the analyzer; it'll be activated automatically for these + * @param parser_orig name of the Spicy parser for the originator side; must match the name that + * Spicy registers the unit's parser with + * @param parser_resp name of the Spicy parser for the originator side; must match the name that + * Spicy registers the unit's parser with + * @param replaces optional name of existing Zeek analyzer that this one replaces; the Zeek + * analyzer will automatically be disabled + * @param linker_scope scope of current HLTO file, which will restrict visibility of the + * registration + */ + void registerProtocolAnalyzer(const std::string& name, hilti::rt::Protocol proto, + const hilti::rt::Vector& ports, const std::string& parser_orig, + const std::string& parser_resp, const std::string& replaces, + const std::string& linker_scope); + + /** + * Runtime method to register a file analyzer with its Zeek-side + * configuration. This is called at startup by generated Spicy code for + * each file analyzer defined in an EVT file + * + * @param name name of the analyzer as defined in its EVT file + * @param mime_types list of MIME types the analyzer handles; it'll be automatically used for + * all files of matching types + * @param parser name of the Spicy parser for parsing the file; must match the name that Spicy + * registers the unit's parser with + * @param replaces optional name of existing Zeek analyzer that this one replaces; the Zeek + * analyzer will automatically be disabled + * @param linker_scope scope of current HLTO file, which will restrict visibility of the + * registration + */ + void registerFileAnalyzer(const std::string& name, const hilti::rt::Vector& mime_types, + const std::string& parser, const std::string& replaces, const std::string& linker_scope); + + /** + * Runtime method to register a packet analyzer with its Zeek-side + * configuration. This is called at startup by generated Spicy code for + * each packet analyzer defined in an EVT file. + * + * @param name name of the analyzer as defined in its EVT file + * @param parser name of the Spicy parser for parsing the packet; must + * match the name that Spicy registers the unit's + * parser with. + * @param replaces optional name of existing Zeek analyzer that this one + * replaces; the Zeek analyzer will automatically be disabled + * @param linker_scope scope of current HLTO file, which will restrict visibility of the + * registration + */ + void registerPacketAnalyzer(const std::string& name, const std::string& parser, const std::string& replaces, + const std::string& linker_scope); + + /** + * Runtime method to register a Spicy-generated type with Zeek. The type + * must have been encountered during AST traversal already, so that its ID + * is known. The corresponding Zeek type will be created and registered with Zeek. + * + * @param id fully-qualified ID of the type + * @return error if the type could not be registered + */ + hilti::rt::Result registerType(const std::string& id); + + /** + * Runtime method to register an already converted Spicy-generated type + * with Zeek. + * + * @param id fully-qualified ID of the type + * @param type Zeek-side type to register + */ + void registerType(const std::string& id, const TypePtr& type); + + /** + * Looks up a global type by its ID with Zeek. + * + * @param id fully-qualified Zeek-side ID of the type + * @return Zeek-side type, or null if not found + */ + TypePtr findType(const std::string& id) const; + + /** + * Runtime method to register a Spicy-generated event. The installs the ID + * Zeek-side and is called at startup by generated Spicy code for each + * event defined in an EVT file. + * + * @param name fully scoped name of the event + */ + void registerEvent(const std::string& name); + + /** + * Runtime method to retrieve the Spicy parser for a given Zeek protocol analyzer tag. + * + * @param analyzer requested protocol analyzer + * @param is_orig true if requesting the parser parser for a sessions' originator side, false + * for the responder + * @return parser, or null if we don't have one for this tag. The pointer will remain valid for + * the life-time of the process. + */ + const ::spicy::rt::Parser* parserForProtocolAnalyzer(const Tag& tag, bool is_orig); + + /** + * Runtime method to retrieve the Spicy parser for a given Zeek file analyzer tag. + * + * @param analyzer requested file analyzer. + * @return parser, or null if we don't have one for this tag. The pointer will remain valid for + * the life-time of the process. + */ + const ::spicy::rt::Parser* parserForFileAnalyzer(const Tag& tag); + + /** + * Runtime method to retrieve the Spicy parser for a given Zeek packet analyzer tag. + * + * @param analyzer requested packet analyzer. + * @return parser, or null if we don't have one for this tag. The pointer will remain + * valid for the life-time of the process. + */ + const ::spicy::rt::Parser* parserForPacketAnalyzer(const Tag& tag); + + /** + * Runtime method to retrieve the analyzer tag that should be passed to + * script-land when talking about a protocol analyzer. This is normally + * the analyzer's standard tag, but may be replaced with something else + * if the analyzer substitutes for an existing one. + * + * @param tag original tag we query for how to pass it to script-land. + * @return desired tag for passing to script-land. + */ + Tag tagForProtocolAnalyzer(const Tag& tag); + + /** + * Runtime method to retrieve the analyzer tag that should be passed to + * script-land when talking about a file analyzer. This is normally the + * analyzer's standard tag, but may be replaced with something else if + * the analyzer substitutes for an existing one. + * + * @param tag original tag we query for how to pass it to script-land. + * @return desired tag for passing to script-land. + */ + Tag tagForFileAnalyzer(const Tag& tag); + + /** + * Runtime method to retrieve the analyzer tag that should be passed to + * script-land when talking about a packet analyzer. This is normally the + * analyzer's standard tag, but may be replaced with something else if + * the analyzer substitutes for an existing one. + * + * @param tag original tag we query for how to pass it to script-land. + * @return desired tag for passing to script-land. + */ + Tag tagForPacketAnalyzer(const Tag& tag); + + /** + * Explicitly enable/disable a protocol analyzer. By default, all analyzers + * loaded will also be activated. By calling this method, an analyzer can + * toggled. + * + * @param analyzer tag of analyzer + * @param enable true to enable, false to disable + */ + bool toggleProtocolAnalyzer(const Tag& tag, bool enable); + + /** + * Explicitly enable/disable a file analyzer. By default, all analyzers + * loaded will also be activated. By calling this method, an analyzer can + * toggled. + * + * @param analyzer tag of analyzer + * @param enable true to enable, false to disable + */ + bool toggleFileAnalyzer(const Tag& tag, bool enable); + + /** + * Explicitly enable/disable a packet analyzer. By default, all analyzers + * loaded will also be activated. By calling this method, an analyzer can + * toggled. + * + * @note This is currently not supported because Zeek does not provide the + * necessary API. + * + * @param analyzer tag of analyzer + * @param enable true to enable, false to disable + */ + bool togglePacketAnalyzer(const Tag& tag, bool enable); + + /** + * Explicitly enable/disable an analyzer. By default, all analyzers + * loaded will also be activated. By calling this method, an analyzer can + * toggled. + * + * This method is frontend for the versions specific to + * protocol/file/packet analyzers. It takes an enum corresponding to either + * kind and branches out accordingly. + * + * @param analyzer tag of analyzer + * @param enable true to enable, false to disable + */ + bool toggleAnalyzer(EnumVal* tag, bool enable); + + /** Report an error and disable a protocol analyzer's input processing */ + void analyzerError(analyzer::Analyzer* a, const std::string& msg, const std::string& location); + + /** Report an error and disable a file analyzer's input processing */ + void analyzerError(file_analysis::Analyzer* a, const std::string& msg, const std::string& location); + + /** Report an error and disable a packet analyzer's input processing. */ + void analyzerError(packet_analysis::Analyzer* a, const std::string& msg, const std::string& location); + + /** Returns the number of errors recorded by the Zeek reporter. */ + int numberErrors(); + +protected: + // Overriding method from Zeek's plugin API. + zeek::plugin::Configuration Configure() override; + + // Overriding method from Zeek's plugin API. + void InitPreScript() override; + + // Overriding method from Zeek's plugin API. + void InitPostScript() override; + + // Overriding method from Zeek's plugin API. + void Done() override; + + // Overriding method from Zeek's plugin API. + int HookLoadFile(const LoadType type, const std::string& file, const std::string& resolved) override; + +private: + // Load one *.hlto module. + void loadModule(const hilti::rt::filesystem::path& path); + + // Search ZEEK_SPICY_MODULE_PATH for pre-compiled *.hlto modules and load them. + void autoDiscoverModules(); + + // Recursively search pre-compiled *.hlto in colon-separated paths. + void searchModules(const std::string& paths); + + // Return a Zeek location object for the given file name that will stay valid. + detail::Location makeLocation(const std::string& fname); + + // Disable any Zeek-side analyzers that are replaced by one of ours. + void disableReplacedAnalyzers(); + + /** + * Internal callback that records the component-to-tag-type mapping for plugin's analyzer. + * + * @param c component to track + * @param tag_type tag type to associate with component + */ + void trackComponent(plugin::Component* c, int32_t tag_type); + + /** Captures a registered protocol analyzer. */ + struct ProtocolAnalyzerInfo { + // Provided when registering the analyzer. + std::string name_analyzer; + std::string name_parser_orig; + std::string name_parser_resp; + std::string name_replaces; + hilti::rt::Protocol protocol = hilti::rt::Protocol::Undef; + hilti::rt::Vector ports; + std::string linker_scope; + + // Computed and available once the analyzer has been registered. + std::string name_zeek; + std::string name_zeekygen; + Tag::type_t type; + const ::spicy::rt::Parser* parser_orig; + const ::spicy::rt::Parser* parser_resp; + Tag replaces; + + bool operator==(const ProtocolAnalyzerInfo& other) const { + return name_analyzer == other.name_analyzer && name_parser_orig == other.name_parser_orig && + name_parser_resp == other.name_parser_resp && name_replaces == other.name_replaces && + protocol == other.protocol && ports == other.ports && linker_scope == other.linker_scope; + } + + bool operator!=(const ProtocolAnalyzerInfo& other) const { return ! (*this == other); } + }; + + /** Captures a registered file analyzer. */ + struct FileAnalyzerInfo { + // Provided when registering the analyzer. + std::string name_analyzer; + std::string name_parser; + std::string name_replaces; + hilti::rt::Vector mime_types; + std::string linker_scope; + + // Computed and available once the analyzer has been registered. + std::string name_zeek; + std::string name_zeekygen; + Tag::type_t type; + const ::spicy::rt::Parser* parser; + Tag replaces; + + bool operator==(const FileAnalyzerInfo& other) const { + return name_analyzer == other.name_analyzer && name_parser == other.name_parser && + name_replaces == other.name_replaces && mime_types == other.mime_types && + linker_scope == other.linker_scope; + } + + bool operator!=(const FileAnalyzerInfo& other) const { return ! (*this == other); } + }; + + /** Captures a registered file analyzer. */ + struct PacketAnalyzerInfo { + // Provided when registering the analyzer. + std::string name_analyzer; + std::string name_parser; + std::string name_replaces; + std::string linker_scope; + + // Computed and available once the analyzer has been registered. + std::string name_zeek; + std::string name_zeekygen; + Tag::type_t type; + const ::spicy::rt::Parser* parser; + Tag replaces; + + // Compares only the provided attributes, as that's what defines us. + bool operator==(const PacketAnalyzerInfo& other) const { + return name_analyzer == other.name_analyzer && name_parser == other.name_parser && + name_replaces == other.name_replaces && linker_scope == other.linker_scope; + } + + bool operator!=(const PacketAnalyzerInfo& other) const { return ! (*this == other); } + }; + + std::string _spicy_version; + + std::vector _protocol_analyzers_by_type; + std::vector _file_analyzers_by_type; + std::vector _packet_analyzers_by_type; + std::unordered_map _libraries; + std::set _locations; + std::unordered_map _events; + + // Mapping of component names to tag types. We use this to ensure analyzer uniqueness. + std::unordered_map _analyzer_name_to_tag_type; +}; + +} // namespace spicy + +extern spicy::Manager* spicy_mgr; + +} // namespace zeek diff --git a/src/spicy/packet-analyzer.cc b/src/spicy/packet-analyzer.cc new file mode 100644 index 0000000000..f69c301bb4 --- /dev/null +++ b/src/spicy/packet-analyzer.cc @@ -0,0 +1,73 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/spicy/packet-analyzer.h" + +#include "zeek/spicy/manager.h" +#include "zeek/spicy/runtime-support.h" + +using namespace zeek; +using namespace zeek::spicy; +using namespace zeek::spicy::rt; + +#ifdef DEBUG +#define STATE_DEBUG_MSG(...) DebugMsg(__VA_ARGS__) +#else +#define STATE_DEBUG_MSG(...) +#endif + +void PacketState::debug(const std::string& msg) { spicy::rt::debug(_cookie, msg); } + +static auto create_packet_state(PacketAnalyzer* analyzer) { + cookie::PacketAnalyzer cookie; + cookie.analyzer = analyzer; + return PacketState(std::move(cookie)); +} + +PacketAnalyzer::PacketAnalyzer(std::string name) + : packet_analysis::Analyzer(std::move(name)), _state(create_packet_state(this)) {} + +PacketAnalyzer::~PacketAnalyzer() = default; + +bool PacketAnalyzer::AnalyzePacket(size_t len, const uint8_t* data, Packet* packet) { + if ( auto parser = spicy_mgr->parserForPacketAnalyzer(_state.packet().analyzer->GetAnalyzerTag()) ) + _state.setParser(parser); + else + reporter->FatalError("no valid unit specified for parsing"); + + try { + hilti::rt::context::CookieSetter _(_state.cookie()); + _state.packet().next_analyzer.reset(); + _state.packet().packet = packet; + _state.process(len, reinterpret_cast(data)); + auto offset = _state.finish(); + assert(offset); + _state.packet().packet = nullptr; + _state.packet().packet_val = nullptr; + _state.reset(); + auto num_processed = offset->Ref(); + const auto& next_analyzer = _state.packet().next_analyzer; + STATE_DEBUG_MSG(hilti::rt::fmt("processed %" PRIu64 " out of %" PRIu64 " bytes, %s", num_processed, len, + (next_analyzer ? hilti::rt::fmt("next analyzer is 0x%" PRIx32, *next_analyzer) : + std::string("no next analyzer")))); + if ( next_analyzer ) + return ForwardPacket(len - num_processed, data + num_processed, packet, *next_analyzer); + else + return true; + } catch ( const hilti::rt::RuntimeError& e ) { + STATE_DEBUG_MSG(hilti::rt::fmt("error during parsing, triggering analyzer violation: %s", e.what())); + auto tag = _state.packet().analyzer->GetAnalyzerTag(); + + if ( auto* session = packet->session ) + _state.packet().analyzer->AnalyzerViolation(e.what(), session, reinterpret_cast(data), len, + tag); + + _state.reset(); + return false; + } catch ( const hilti::rt::Exception& e ) { + STATE_DEBUG_MSG(e.what()); + spicy_mgr->analyzerError(_state.packet().analyzer, e.description(), + e.location()); // this sets Zeek to skip sending any further input + _state.reset(); + return false; + } +} diff --git a/src/spicy/packet-analyzer.h b/src/spicy/packet-analyzer.h new file mode 100644 index 0000000000..7d68e322db --- /dev/null +++ b/src/spicy/packet-analyzer.h @@ -0,0 +1,75 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include +#include +#include +#include + +#include + +#include +#include + +#include "zeek/spicy/cookie.h" + +namespace zeek::spicy::rt { + +/** Parsing state for a packet. */ +class PacketState : public ::spicy::rt::driver::ParsingState { +public: + /** + * Constructor. + * + * @param cookie cookie to associated with the packet + */ + PacketState(Cookie cookie) : ParsingState(::spicy::rt::driver::ParsingType::Block), _cookie(std::move(cookie)) {} + + /** Returns the cookie pointer to use with the runtime library during analysis. */ + auto* cookie() { return &_cookie; } + + /** Returns the packet-specific cookie state associated with the endpoint. */ + auto& packet() { + assert(_cookie.packet); + return *_cookie.packet; + } + + /** + * Records a debug message pertaining to the specific file. + * + * @param msg message to record + */ + void DebugMsg(const std::string& msg) { debug(msg); } + +protected: + // Overridden from driver::ParsingState. + void debug(const std::string& msg) override; + +private: + Cookie _cookie; +}; + +/** A Spicy file analyzer. */ +class PacketAnalyzer : public packet_analysis::Analyzer { +public: + PacketAnalyzer(std::string name); + virtual ~PacketAnalyzer(); + + /** Records a debug message. */ + void DebugMsg(const std::string& msg) { _state.DebugMsg(msg); } + + static packet_analysis::AnalyzerPtr Instantiate(std::string name) { + name = util::canonify_name(name); + return std::make_shared(name); + } + +protected: + // Overridden from Zeek's packet analyzer. + bool AnalyzePacket(size_t len, const uint8_t* data, Packet* packet) override; + +private: + PacketState _state; +}; + +} // namespace zeek::spicy::rt diff --git a/src/spicy/protocol-analyzer.cc b/src/spicy/protocol-analyzer.cc new file mode 100644 index 0000000000..821356ecf7 --- /dev/null +++ b/src/spicy/protocol-analyzer.cc @@ -0,0 +1,231 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/spicy/protocol-analyzer.h" + +#include "zeek/spicy/manager.h" +#include "zeek/spicy/runtime-support.h" + +using namespace zeek; +using namespace zeek::spicy; +using namespace zeek::spicy::rt; + +#ifdef DEBUG +#define STATE_DEBUG_MSG(...) DebugMsg(__VA_ARGS__) +#else +#define STATE_DEBUG_MSG(...) +#endif + +void EndpointState::debug(const std::string& msg) { spicy::rt::debug(_cookie, msg); } + +static auto create_endpoint(bool is_orig, analyzer::Analyzer* analyzer, ::spicy::rt::driver::ParsingType type) { + cookie::ProtocolAnalyzer cookie{.analyzer = analyzer, + .is_orig = is_orig, + .fstate_orig = cookie::FileStateStack(hilti::rt::fmt("%x.orig", analyzer->GetID())), + .fstate_resp = + cookie::FileStateStack(hilti::rt::fmt("%x.resp", analyzer->GetID()))}; + + // Cannot get parser here yet, analyzer may not have been fully set up. + return EndpointState(std::move(cookie), type); +} + +ProtocolAnalyzer::ProtocolAnalyzer(analyzer::Analyzer* analyzer, ::spicy::rt::driver::ParsingType type) + : _originator(create_endpoint(true, analyzer, type)), _responder(create_endpoint(false, analyzer, type)) {} + +ProtocolAnalyzer::~ProtocolAnalyzer() {} + +void ProtocolAnalyzer::Init() {} + +void ProtocolAnalyzer::Done() { + Finish(true); + Finish(false); +} + +void ProtocolAnalyzer::Process(bool is_orig, int len, const u_char* data) { + auto* endp = is_orig ? &_originator : &_responder; + + if ( endp->protocol().analyzer->Skipping() ) + return; + + if ( ! endp->hasParser() && ! endp->isSkipping() ) { + auto parser = spicy_mgr->parserForProtocolAnalyzer(endp->protocol().analyzer->GetAnalyzerTag(), is_orig); + if ( parser ) { + if ( ! _context ) + _context = parser->createContext(); + + endp->setParser(parser, _context); + } + else { + STATE_DEBUG_MSG(is_orig, "no unit specified for parsing"); + endp->skipRemaining(); + return; + } + } + + try { + hilti::rt::context::CookieSetter _(endp->cookie()); + endp->process(len, reinterpret_cast(data)); + } catch ( const hilti::rt::RuntimeError& e ) { + STATE_DEBUG_MSG(is_orig, hilti::rt::fmt("error during parsing, triggering analyzer violation: %s", e.what())); + auto tag = spicy_mgr->tagForProtocolAnalyzer(endp->protocol().analyzer->GetAnalyzerTag()); + endp->protocol().analyzer->AnalyzerViolation(e.what(), reinterpret_cast(data), len, tag); + originator().skipRemaining(); + responder().skipRemaining(); + endp->protocol().analyzer->SetSkip(true); + } catch ( const hilti::rt::Exception& e ) { + spicy_mgr->analyzerError(endp->protocol().analyzer, e.description(), + e.location()); // this sets Zeek to skip sending any further input + } +} + +void ProtocolAnalyzer::Finish(bool is_orig) { + auto* endp = is_orig ? &_originator : &_responder; + + if ( endp->protocol().analyzer->Skipping() ) + return; + + try { + hilti::rt::context::CookieSetter _(endp->cookie()); + endp->finish(); + } catch ( const hilti::rt::RuntimeError& e ) { + STATE_DEBUG_MSG(is_orig, hilti::rt::fmt("error during parsing, triggering analyzer violation: %s", e.what())); + auto tag = spicy_mgr->tagForProtocolAnalyzer(endp->protocol().analyzer->GetAnalyzerTag()); + endp->protocol().analyzer->AnalyzerViolation(e.what(), nullptr, 0, tag); + endp->skipRemaining(); + } catch ( const hilti::rt::Exception& e ) { + spicy_mgr->analyzerError(endp->protocol().analyzer, e.description(), + e.location()); // this sets Zeek to skip sending any further input + } +} + +cookie::ProtocolAnalyzer& ProtocolAnalyzer::cookie(bool is_orig) { + if ( is_orig ) + return _originator.protocol(); + else + return _responder.protocol(); +} + +void ProtocolAnalyzer::DebugMsg(bool is_orig, const std::string& msg) { + if ( is_orig ) + _originator.DebugMsg(msg); + else + _responder.DebugMsg(msg); +} + +void ProtocolAnalyzer::FlipRoles() { std::swap(_originator, _responder); } + +analyzer::Analyzer* TCP_Analyzer::InstantiateAnalyzer(Connection* conn) { return new TCP_Analyzer(conn); } + +TCP_Analyzer::TCP_Analyzer(Connection* conn) + : ProtocolAnalyzer(this, ::spicy::rt::driver::ParsingType::Stream), analyzer::tcp::TCP_ApplicationAnalyzer(conn) {} + +TCP_Analyzer::~TCP_Analyzer() {} + +void TCP_Analyzer::Init() { + analyzer::tcp::TCP_ApplicationAnalyzer::Init(); + ProtocolAnalyzer::Init(); +} + +void TCP_Analyzer::Done() { + analyzer::tcp::TCP_ApplicationAnalyzer::Done(); + ProtocolAnalyzer::Done(); + + EndOfData(true); + EndOfData(false); +} + +void TCP_Analyzer::DeliverStream(int len, const u_char* data, bool is_orig) { + analyzer::tcp::TCP_ApplicationAnalyzer::DeliverStream(len, data, is_orig); + + Process(is_orig, len, data); + + if ( originator().isFinished() && responder().isFinished() && + (! originator().isSkipping() || ! responder().isSkipping()) ) { + STATE_DEBUG_MSG(is_orig, "both endpoints finished, skipping all further TCP processing"); + originator().skipRemaining(); + responder().skipRemaining(); + + if ( is_orig ) // doesn't really matter which endpoint here. + originator().protocol().analyzer->SetSkip(true); + else + responder().protocol().analyzer->SetSkip(true); + } +} + +void TCP_Analyzer::Undelivered(uint64_t seq, int len, bool is_orig) { + analyzer::tcp::TCP_ApplicationAnalyzer::Undelivered(seq, len, is_orig); + + Process(is_orig, len, nullptr); +} + +void TCP_Analyzer::EndOfData(bool is_orig) { + analyzer::tcp::TCP_ApplicationAnalyzer::EndOfData(is_orig); + + if ( TCP() && TCP()->IsPartial() ) { + STATE_DEBUG_MSG(is_orig, "skipping end-of-data delivery on partial TCP connection"); + return; + } + + Finish(is_orig); +} + +void TCP_Analyzer::FlipRoles() { + analyzer::tcp::TCP_ApplicationAnalyzer::FlipRoles(); + ProtocolAnalyzer::FlipRoles(); +} + +void TCP_Analyzer::EndpointEOF(bool is_orig) { + analyzer::tcp::TCP_ApplicationAnalyzer::EndpointEOF(is_orig); + Finish(is_orig); +} + +void TCP_Analyzer::ConnectionClosed(analyzer::tcp::TCP_Endpoint* endpoint, analyzer::tcp::TCP_Endpoint* peer, + bool gen_event) { + analyzer::tcp::TCP_ApplicationAnalyzer::ConnectionClosed(endpoint, peer, gen_event); +} + +void TCP_Analyzer::ConnectionFinished(bool half_finished) { + analyzer::tcp::TCP_ApplicationAnalyzer::ConnectionFinished(half_finished); +} + +void TCP_Analyzer::ConnectionReset() { analyzer::tcp::TCP_ApplicationAnalyzer::ConnectionReset(); } + +void TCP_Analyzer::PacketWithRST() { analyzer::tcp::TCP_ApplicationAnalyzer::PacketWithRST(); } + +analyzer::Analyzer* UDP_Analyzer::InstantiateAnalyzer(Connection* conn) { return new UDP_Analyzer(conn); } + +UDP_Analyzer::UDP_Analyzer(Connection* conn) + : ProtocolAnalyzer(this, ::spicy::rt::driver::ParsingType::Block), analyzer::Analyzer(conn) {} + +UDP_Analyzer::~UDP_Analyzer() {} + +void UDP_Analyzer::Init() { + analyzer::Analyzer::Init(); + ProtocolAnalyzer::Init(); +} + +void UDP_Analyzer::Done() { + analyzer::Analyzer::Done(); + ProtocolAnalyzer::Done(); +} + +void UDP_Analyzer::DeliverPacket(int len, const u_char* data, bool is_orig, uint64_t seq, const IP_Hdr* ip, + int caplen) { + analyzer::Analyzer::DeliverPacket(len, data, is_orig, seq, ip, caplen); + + ++cookie(is_orig).num_packets; + Process(is_orig, len, data); +} + +void UDP_Analyzer::Undelivered(uint64_t seq, int len, bool is_orig) { + analyzer::Analyzer::Undelivered(seq, len, is_orig); +} + +void UDP_Analyzer::EndOfData(bool is_orig) { + analyzer::Analyzer::EndOfData(is_orig); + Finish(is_orig); +} + +void UDP_Analyzer::FlipRoles() { + analyzer::Analyzer::FlipRoles(); + ProtocolAnalyzer::FlipRoles(); +} diff --git a/src/spicy/protocol-analyzer.h b/src/spicy/protocol-analyzer.h new file mode 100644 index 0000000000..71be171aff --- /dev/null +++ b/src/spicy/protocol-analyzer.h @@ -0,0 +1,166 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include +#include +#include + +#include + +#include +#include + +#include "zeek/spicy/cookie.h" + +namespace zeek::spicy::rt { + +/** Parsing state for one endpoint of the connection. */ +class EndpointState : public ::spicy::rt::driver::ParsingState { +public: + /** + * Constructor. + * + * @param cookie cookie to associated with the endpoint + * @param type type of parsing, depending on whether it's a stream- or + * packet-based protocol + */ + EndpointState(Cookie cookie, ::spicy::rt::driver::ParsingType type) + : ParsingState(type), _cookie(std::move(cookie)) {} + + /** Returns the protocol-specific cookie state associated with the endpoint. */ + auto& protocol() { + assert(_cookie.protocol); + return *_cookie.protocol; + } + + /** Returns the cookie pointer to use with the runtime library during analysis. */ + auto* cookie() { return &_cookie; } + + /** + * Records a debug message pertaining to this specific endpoint. + * + * @param msg message to record + */ + void DebugMsg(const std::string& msg) { debug(msg); } + +protected: + // Overridden from driver::ParsingState. + void debug(const std::string& msg) override; + +private: + Cookie _cookie; +}; + +/** Base clase for Spicy protocol analyzers. */ +class ProtocolAnalyzer { +public: + ProtocolAnalyzer(analyzer::Analyzer* analyzer, ::spicy::rt::driver::ParsingType type); + virtual ~ProtocolAnalyzer(); + + /** Returns the originator-side parsing state. */ + auto& originator() { return _originator; } + + /** Returns the responder-side parsing state. */ + auto& responder() { return _responder; } + +protected: + /** Initialize analyzer. */ + void Init(); + + /** Shutdown analyzer. */ + void Done(); + + /** + * Signal that Zeek has flipped the direction of the connection, meaning + * that originator and responder state need to be swapped. + */ + void FlipRoles(); + + /** + * Feeds a chunk of data into one side's parsing. + * + * @param is_orig true to use originator-side endpoint state, false for responder + * @param len number of bytes valid in *data* + * @param data pointer to data + */ + void Process(bool is_orig, int len, const u_char* data); + + /** + * Finalizes parsing. After calling this, no more data must be passed + * into Process() for the corresponding side. + * + * @param is_orig true to finish originator-side parsing, false for responder + */ + void Finish(bool is_orig); + + /** + * Helper returning the protocol analyzer cookie for the requested side. + * + * @param is_orig tru to return the originator's state, false for the + * responder. + * @return protocol analyzer cookie for the requested side + */ + cookie::ProtocolAnalyzer& cookie(bool is_orig); + + /** + * Records a debug message. This forwards to `DebugMsg()` for the + * corresponding `EndpointState`. + */ + void DebugMsg(bool is_orig, const std::string& msg); + +private: + EndpointState _originator; /**< Originator-side state. */ + EndpointState _responder; /**< Responder-side state. */ + std::optional<::spicy::rt::UnitContext> _context; +}; + +/** + * Spicy analyzer for TCP application-layer protocols. Implements the + * standard Zeek API. + */ +class TCP_Analyzer : public ProtocolAnalyzer, public analyzer::tcp::TCP_ApplicationAnalyzer { +public: + TCP_Analyzer(Connection* conn); + virtual ~TCP_Analyzer(); + + // Overridden from Spicy's Analyzer. + void Init() override; + void Done() override; + void DeliverStream(int len, const u_char* data, bool orig) override; + void Undelivered(uint64_t seq, int len, bool orig) override; + void EndOfData(bool is_orig) override; + void FlipRoles() override; + + // Overridden from Zeek's TCP_ApplicationAnalyzer. + void EndpointEOF(bool is_orig) override; + void ConnectionClosed(analyzer::tcp::TCP_Endpoint* endpoint, analyzer::tcp::TCP_Endpoint* peer, + bool gen_event) override; + void ConnectionFinished(bool half_finished) override; + void ConnectionReset() override; + void PacketWithRST() override; + + static analyzer::Analyzer* InstantiateAnalyzer(Connection* conn); +}; + +/** + * Spicy analyzer for UDP application-layer protocols. Implements the + * standard Zeek API. + */ +class UDP_Analyzer : public ProtocolAnalyzer, public analyzer::Analyzer { +public: + UDP_Analyzer(Connection* conn); + virtual ~UDP_Analyzer(); + + // Overridden from Spicy's Analyzer. + void Init() override; + void Done() override; + void DeliverPacket(int len, const u_char* data, bool orig, uint64_t seq, const IP_Hdr* ip, int caplen) override; + void Undelivered(uint64_t seq, int len, bool orig) override; + void EndOfData(bool is_orig) override; + void FlipRoles() override; + + static analyzer::Analyzer* InstantiateAnalyzer(Connection* conn); +}; + +} // namespace zeek::spicy::rt diff --git a/src/spicy/runtime-support.cc b/src/spicy/runtime-support.cc new file mode 100644 index 0000000000..9efac938ea --- /dev/null +++ b/src/spicy/runtime-support.cc @@ -0,0 +1,842 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/spicy/runtime-support.h" + +#include +#include + +#include +#include +#include +#include + +#include "zeek/Event.h" +#include "zeek/analyzer/Manager.h" +#include "zeek/analyzer/protocol/pia/PIA.h" +#include "zeek/file_analysis/File.h" +#include "zeek/file_analysis/Manager.h" +#include "zeek/spicy/manager.h" + +using namespace zeek; +using namespace zeek::spicy; + +void rt::register_protocol_analyzer(const std::string& name, hilti::rt::Protocol proto, + const hilti::rt::Vector& ports, const std::string& parser_orig, + const std::string& parser_resp, const std::string& replaces, + const std::string& linker_scope) { + auto _ = hilti::rt::profiler::start("zeek/rt/register_protocol_analyzer"); + spicy_mgr->registerProtocolAnalyzer(name, proto, ports, parser_orig, parser_resp, replaces, linker_scope); +} + +void rt::register_file_analyzer(const std::string& name, const hilti::rt::Vector& mime_types, + const std::string& parser, const std::string& replaces, + const std::string& linker_scope) { + auto _ = hilti::rt::profiler::start("zeek/rt/register_file_analyzer"); + spicy_mgr->registerFileAnalyzer(name, mime_types, parser, replaces, linker_scope); +} + +void rt::register_packet_analyzer(const std::string& name, const std::string& parser, const std::string& replaces, + const std::string& linker_scope) { + auto _ = hilti::rt::profiler::start("zeek/rt/register_packet_analyzer"); + spicy_mgr->registerPacketAnalyzer(name, parser, replaces, linker_scope); +} + +void rt::register_type(const std::string& ns, const std::string& id, const TypePtr& type) { + auto _ = hilti::rt::profiler::start("zeek/rt/register_type"); + spicy_mgr->registerType(hilti::rt::fmt("%s::%s", (! ns.empty() ? ns : std::string("GLOBAL")), id), type); +} + +// Helper to look up a global Zeek-side type, enforcing that it's of the expected type. +static TypePtr findType(TypeTag tag, const std::string& ns, const std::string& id) { + auto id_ = hilti::rt::fmt("%s::%s", ns, id); + auto type = spicy_mgr->findType(id_); + + if ( ! type ) + return nullptr; + + if ( type->Tag() != tag ) + reporter->FatalError("ID %s is not of expected type %s", id_.c_str(), type_name(tag)); + + return type; +} + +TypePtr rt::create_base_type(ZeekTypeTag tag) { + auto _ = hilti::rt::profiler::start("zeek/rt/create_base_type"); + TypeTag ztag; + + switch ( tag ) { + case ZeekTypeTag::Addr: ztag = TYPE_ADDR; break; + case ZeekTypeTag::Any: ztag = TYPE_ANY; break; + case ZeekTypeTag::Bool: ztag = TYPE_BOOL; break; + case ZeekTypeTag::Count: ztag = TYPE_COUNT; break; + case ZeekTypeTag::Double: ztag = TYPE_DOUBLE; break; + case ZeekTypeTag::Enum: ztag = TYPE_ENUM; break; + case ZeekTypeTag::Error: ztag = TYPE_ERROR; break; + case ZeekTypeTag::File: ztag = TYPE_FILE; break; + case ZeekTypeTag::Func: ztag = TYPE_FUNC; break; + case ZeekTypeTag::List: ztag = TYPE_LIST; break; + case ZeekTypeTag::Int: ztag = TYPE_INT; break; + case ZeekTypeTag::Interval: ztag = TYPE_INTERVAL; break; + case ZeekTypeTag::Opaque: ztag = TYPE_OPAQUE; break; + case ZeekTypeTag::Pattern: ztag = TYPE_PATTERN; break; + case ZeekTypeTag::Port: ztag = TYPE_PORT; break; + case ZeekTypeTag::Record: ztag = TYPE_RECORD; break; + case ZeekTypeTag::String: ztag = TYPE_STRING; break; + case ZeekTypeTag::Subnet: ztag = TYPE_SUBNET; break; + case ZeekTypeTag::Table: ztag = TYPE_TABLE; break; + case ZeekTypeTag::Time: ztag = TYPE_TIME; break; + case ZeekTypeTag::Type: ztag = TYPE_TYPE; break; + case ZeekTypeTag::Vector: ztag = TYPE_VECTOR; break; + case ZeekTypeTag::Void: ztag = TYPE_VOID; break; + default: hilti::rt::cannot_be_reached(); + } + + return base_type(ztag); +} + +TypePtr rt::create_enum_type( + const std::string& ns, const std::string& id, + const hilti::rt::Vector>>& labels) { + auto _ = hilti::rt::profiler::start("zeek/rt/create_enum_type"); + + if ( auto t = findType(TYPE_ENUM, ns, id) ) + return t; + + auto etype = make_intrusive(ns + "::" + id); + + for ( auto [lid, lval] : labels ) { + auto name = ::hilti::rt::fmt("%s_%s", id, lid); + + if ( lval == -1 ) + // Zeek's enum can't be negative, so swap in max_int for our Undef. + lval = std::numeric_limits<::zeek_int_t>::max(); + + etype->AddName(ns, name.c_str(), lval, true); + } + + return etype; +} + +TypePtr rt::create_record_type(const std::string& ns, const std::string& id, + const hilti::rt::Vector& fields) { + auto _ = hilti::rt::profiler::start("zeek/rt/create_record_type"); + + if ( auto t = findType(TYPE_RECORD, ns, id) ) + return t; + + auto decls = std::make_unique(); + + for ( const auto& [id, type, optional] : fields ) { + auto attrs = make_intrusive(nullptr, true, false); + + if ( optional ) { + auto optional_ = make_intrusive(detail::ATTR_OPTIONAL); + attrs->AddAttr(optional_); + } + + decls->append(new TypeDecl(util::copy_string(id.c_str()), type, std::move(attrs))); + } + + return make_intrusive(decls.release()); +} + +TypePtr rt::create_table_type(TypePtr key, std::optional value) { + auto _ = hilti::rt::profiler::start("zeek/rt/create_table_type"); + auto idx = make_intrusive(); + idx->Append(std::move(key)); + return make_intrusive(std::move(idx), value ? *value : nullptr); +} + +TypePtr rt::create_vector_type(const TypePtr& elem) { + auto _ = hilti::rt::profiler::start("zeek/rt/create_vector_type"); + return make_intrusive(elem); +} + +void rt::install_handler(const std::string& name) { + auto _ = hilti::rt::profiler::start("zeek/rt/install_handler"); + spicy_mgr->registerEvent(name); +} + +EventHandlerPtr rt::internal_handler(const std::string& name) { + auto _ = hilti::rt::profiler::start("zeek/rt/internal_handler"); + auto handler = event_registry->Lookup(name); + + if ( ! handler ) + reporter->InternalError("Spicy event %s was not installed", name.c_str()); + + return handler; +} + +void rt::raise_event(const EventHandlerPtr& handler, const hilti::rt::Vector& args) { + auto _ = hilti::rt::profiler::start("zeek/rt/raise_event"); + + // Caller must have checked already that there's a handler available. + assert(handler); + + const auto& zeek_args = const_cast(handler)->GetType()->ParamList()->GetTypes(); + if ( args.size() != static_cast(zeek_args.size()) ) + throw TypeMismatch(hilti::rt::fmt("expected %" PRIu64 " parameters, but got %zu", + static_cast(zeek_args.size()), args.size())); + + Args vl = Args(); + vl.reserve(args.size()); + for ( const auto& v : args ) { + if ( v ) + vl.emplace_back(v); + else + // Shouldn't happen here, but we have to_vals() that + // (legitimately) return null in certain contexts. + throw InvalidValue("null value encountered after conversion"); + } + + event_mgr.Enqueue(handler, std::move(vl)); +} + +TypePtr rt::event_arg_type(const EventHandlerPtr& handler, const hilti::rt::integer::safe& idx) { + auto _ = hilti::rt::profiler::start("zeek/rt/event_arg_type"); + assert(handler); + + const auto& zeek_args = const_cast(handler)->GetType()->ParamList()->GetTypes(); + if ( idx >= static_cast(zeek_args.size()) ) + throw TypeMismatch(hilti::rt::fmt("more parameters given than the %" PRIu64 " that the Zeek event expects", + static_cast(zeek_args.size()))); + + return zeek_args[idx]; +} + +ValPtr& rt::current_conn() { + auto _ = hilti::rt::profiler::start("zeek/rt/current_conn"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( cookie->cache.conn ) + return cookie->cache.conn; + + if ( auto x = cookie->protocol ) { + cookie->cache.conn = x->analyzer->Conn()->GetVal(); + return cookie->cache.conn; + } + else + throw ValueUnavailable("$conn not available"); +} + +ValPtr& rt::current_is_orig() { + auto _ = hilti::rt::profiler::start("zeek/rt/current_is_orig"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( cookie->cache.is_orig ) + return cookie->cache.is_orig; + + if ( auto x = cookie->protocol ) { + cookie->cache.is_orig = val_mgr->Bool(x->is_orig); + return cookie->cache.is_orig; + } + else + throw ValueUnavailable("$is_orig not available"); +} + +void rt::debug(const std::string& msg) { + auto _ = hilti::rt::profiler::start("zeek/rt/debug"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + rt::debug(*cookie, msg); +} + +void rt::debug(const Cookie& cookie, const std::string& msg) { + auto _ = hilti::rt::profiler::start("zeek/rt/debug"); + std::string name; + std::string id; + + if ( const auto p = cookie.protocol ) { + auto name = p->analyzer->GetAnalyzerName(); + SPICY_DEBUG( + hilti::rt::fmt("[%s/%" PRIu32 "/%s] %s", name, p->analyzer->GetID(), (p->is_orig ? "orig" : "resp"), msg)); + } + else if ( const auto f = cookie.file ) { + auto name = file_mgr->GetComponentName(f->analyzer->Tag()); + SPICY_DEBUG(hilti::rt::fmt("[%s/%" PRIu32 "] %s", name, f->analyzer->GetID(), msg)); + } + else if ( const auto f = cookie.packet ) { + auto name = packet_mgr->GetComponentName(f->analyzer->GetAnalyzerTag()); + SPICY_DEBUG(hilti::rt::fmt("[%s] %s", name, msg)); + } + else + throw ValueUnavailable("neither $conn nor $file nor packet analyzer available for debug logging"); +} + +inline rt::cookie::FileStateStack* _file_state_stack(rt::Cookie* cookie) { + auto _ = hilti::rt::profiler::start("zeek/rt/file_state_stack"); + + if ( auto c = cookie->protocol ) + return c->is_orig ? &c->fstate_orig : &c->fstate_resp; + else if ( auto f = cookie->file ) + return &f->fstate; + else + throw rt::ValueUnavailable("no current connection or file available"); +} + +inline const rt::cookie::FileState* _file_state(rt::Cookie* cookie, std::optional fid) { + auto _ = hilti::rt::profiler::start("zeek/rt/file_state"); + + auto* stack = _file_state_stack(cookie); + if ( fid ) { + if ( auto* fstate = stack->find(*fid) ) + return fstate; + else + throw rt::ValueUnavailable(hilti::rt::fmt("no file analysis currently in flight for file ID %s", fid)); + } + else { + if ( stack->isEmpty() ) + throw rt::ValueUnavailable("no file analysis currently in flight"); + + return stack->current(); + } +} + +ValPtr rt::current_file() { + auto _ = hilti::rt::profiler::start("zeek/rt/current_file"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( auto x = cookie->file ) + return x->analyzer->GetFile()->ToVal(); + else if ( auto* fstate = _file_state(cookie, {}) ) { + if ( auto* f = file_mgr->LookupFile(fstate->fid) ) + return f->ToVal(); + } + + throw ValueUnavailable("$file not available"); +} + +ValPtr rt::current_packet() { + auto _ = hilti::rt::profiler::start("zeek/rt/current_packet"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( auto c = cookie->packet ) { + if ( ! c->packet_val ) + // We cache the built value in case we need it multiple times. + c->packet_val = c->packet->ToRawPktHdrVal(); + + return c->packet_val; + } + else + throw ValueUnavailable("$packet not available"); +} + +hilti::rt::Bool rt::is_orig() { + auto _ = hilti::rt::profiler::start("zeek/rt/is_orig"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( auto x = cookie->protocol ) + return x->is_orig; + else + throw ValueUnavailable("is_orig() not available in current context"); +} + +std::string rt::uid() { + auto _ = hilti::rt::profiler::start("zeek/rt/uid"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( auto c = cookie->protocol ) { + // Retrieve the ConnVal() so that we ensure the UID has been set. + c->analyzer->ConnVal(); + return c->analyzer->Conn()->GetUID().Base62("C"); + } + else + throw ValueUnavailable("uid() not available in current context"); +} + +std::tuple rt::conn_id() { + auto _ = hilti::rt::profiler::start("zeek/rt/conn_id"); + + static auto convert_address = [](const IPAddr& zaddr) -> hilti::rt::Address { + const uint32_t* bytes = nullptr; + if ( auto n = zaddr.GetBytes(&bytes); n == 1 ) + // IPv4 + return hilti::rt::Address(*reinterpret_cast(bytes)); + else if ( n == 4 ) + // IPv6 + return hilti::rt::Address(*reinterpret_cast(bytes)); + else + throw ValueUnavailable("unexpected IP address side from Zeek"); // shouldn't really be able to happen + }; + + static auto convert_port = [](uint32_t port, TransportProto proto) -> hilti::rt::Port { + auto p = ntohs(static_cast(port)); + + switch ( proto ) { + case TransportProto::TRANSPORT_ICMP: return {p, hilti::rt::Protocol::ICMP}; + case TransportProto::TRANSPORT_TCP: return {p, hilti::rt::Protocol::TCP}; + case TransportProto::TRANSPORT_UDP: return {p, hilti::rt::Protocol::UDP}; + case TransportProto::TRANSPORT_UNKNOWN: return {p, hilti::rt::Protocol::Undef}; + } + + hilti::rt::cannot_be_reached(); + }; + + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( auto c = cookie->protocol ) { + const auto* conn = c->analyzer->Conn(); + return std::make_tuple(convert_address(conn->OrigAddr()), convert_port(conn->OrigPort(), conn->ConnTransport()), + convert_address(conn->RespAddr()), + convert_port(conn->RespPort(), conn->ConnTransport())); + } + else + throw ValueUnavailable("conn_id() not available in current context"); +} + +void rt::flip_roles() { + auto _ = hilti::rt::profiler::start("zeek/rt/flip_roles"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + rt::debug(*cookie, "flipping roles"); + + if ( auto x = cookie->protocol ) + x->analyzer->Conn()->FlipRoles(); + else + throw ValueUnavailable("flip_roles() not available in current context"); +} + +hilti::rt::integer::safe rt::number_packets() { + auto _ = hilti::rt::profiler::start("zeek/rt/number_packets"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( auto x = cookie->protocol ) { + return x->num_packets; + } + else + throw ValueUnavailable("number_packets() not available in current context"); +} + +void rt::confirm_protocol() { + auto _ = hilti::rt::profiler::start("zeek/rt/confirm_protocol"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( cookie->cache.confirmed ) + return; + + if ( auto x = cookie->protocol ) { + auto tag = spicy_mgr->tagForProtocolAnalyzer(x->analyzer->GetAnalyzerTag()); + SPICY_DEBUG(hilti::rt::fmt("confirming protocol %s", tag.AsString())); + cookie->cache.confirmed = true; + return x->analyzer->AnalyzerConfirmation(tag); + } + throw ValueUnavailable("no current connection available"); +} + +void rt::reject_protocol(const std::string& reason) { + auto _ = hilti::rt::profiler::start("zeek/rt/reject_protocol"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( auto x = cookie->protocol ) { + auto tag = spicy_mgr->tagForProtocolAnalyzer(x->analyzer->GetAnalyzerTag()); + SPICY_DEBUG(hilti::rt::fmt("rejecting protocol %s", tag.AsString())); + return x->analyzer->AnalyzerViolation("protocol rejected", nullptr, 0, tag); + } + else + throw ValueUnavailable("no current connection available"); +} + +void rt::weird(const std::string& id, const std::string& addl) { + auto _ = hilti::rt::profiler::start("zeek/rt/weird"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( const auto x = cookie->protocol ) + x->analyzer->Weird(id.c_str(), addl.data()); + else if ( const auto x = cookie->file ) + zeek::reporter->Weird(x->analyzer->GetFile(), id.c_str(), addl.data()); + else if ( const auto x = cookie->packet ) { + x->analyzer->Weird(id.c_str(), x->packet, addl.c_str()); + } + else + throw ValueUnavailable("none of $conn, $file, or $packet available for weird reporting"); +} + +void rt::protocol_begin(const std::optional& analyzer) { + auto _ = hilti::rt::profiler::start("zeek/rt/protocol_begin"); + + if ( analyzer ) { + protocol_handle_get_or_create(*analyzer); + return; + } + + // Instantiate a DPD analyzer. + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + auto c = cookie->protocol; + if ( ! c ) + throw ValueUnavailable("no current connection available"); + + // Use a Zeek PIA stream analyzer performing DPD. + auto pia_tcp = std::make_unique(c->analyzer->Conn()); + + // Forward empty payload to trigger lifecycle management in this analyzer tree. + c->analyzer->ForwardStream(0, reinterpret_cast(c->analyzer), true); + c->analyzer->ForwardStream(0, reinterpret_cast(c->analyzer), false); + + // Direct child of this type already exists. We ignore this silently + // because that makes usage nicer if either side of the connection + // might end up creating the analyzer; this way the user doesn't + // need to track what the other side already did. + // + // We inspect the children directly to work around zeek/zeek#2899. + const auto& children = c->analyzer->GetChildren(); + if ( auto it = std::find_if(children.begin(), children.end(), + [&](const auto& it) { + return ! it->Removing() && ! it->IsFinished() && + it->GetAnalyzerTag() == pia_tcp->GetAnalyzerTag(); + }); + it != children.end() ) + return; + + auto child = pia_tcp.release(); + c->analyzer->AddChildAnalyzer(child); + + child->FirstPacket(true, nullptr); + child->FirstPacket(false, nullptr); +} + +rt::ProtocolHandle rt::protocol_handle_get_or_create(const std::string& analyzer) { + auto _ = hilti::rt::profiler::start("zeek/rt/protocol_handle_get_or_create"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + auto c = cookie->protocol; + if ( ! c ) + throw ValueUnavailable("no current connection available"); + + // Forward empty payload to trigger lifecycle management in this analyzer tree. + c->analyzer->ForwardStream(0, reinterpret_cast(c->analyzer), true); + c->analyzer->ForwardStream(0, reinterpret_cast(c->analyzer), false); + + // If the child already exists, do not add it again so this function is idempotent. + // + // We inspect the children directly to work around zeek/zeek#2899. + const auto& children = c->analyzer->GetChildren(); + if ( auto it = std::find_if(children.begin(), children.end(), + [&](const auto& it) { + return ! it->Removing() && ! it->IsFinished() && it->GetAnalyzerName() == analyzer; + }); + it != children.end() ) + return rt::ProtocolHandle((*it)->GetID()); + + auto child = analyzer_mgr->InstantiateAnalyzer(analyzer.c_str(), c->analyzer->Conn()); + if ( ! child ) + throw ZeekError(::hilti::rt::fmt("unknown analyzer '%s' requested", analyzer)); + + // If we had no such child before but cannot add it the analyzer was prevented. + // + // NOTE: We make this a hard error since returning e.g., an empty optional + // here would make it easy to incorrectly use the return value with e.g., + // `protocol_data_in` or `protocol_gap`. + if ( ! c->analyzer->AddChildAnalyzer(child) ) + throw ZeekError(::hilti::rt::fmt("creation of child analyzer %s was prevented", analyzer)); + + if ( c->analyzer->Conn()->ConnTransport() != TRANSPORT_TCP ) { + // Some TCP application analyzer may expect to have access to a TCP + // analyzer. To make that work, we'll create a fake TCP analyzer, + // just so that they have something to access. It won't + // semantically have any "TCP" to analyze obviously. + c->fake_tcp = std::make_shared(c->analyzer->Conn()); + static_cast(c->fake_tcp.get()) + ->Done(); // will never see packets; cast to get around protected inheritance + } + + auto* child_as_tcp = dynamic_cast(child); + if ( ! child_as_tcp ) + throw ZeekError( + ::hilti::rt::fmt("could not add analyzer '%s' to connection; not a TCP-based analyzer", analyzer)); + + if ( c->fake_tcp ) + child_as_tcp->SetTCP(c->fake_tcp.get()); + + return rt::ProtocolHandle(child->GetID()); +} + +void rt::protocol_data_in(const hilti::rt::Bool& is_orig, const hilti::rt::Bytes& data, + const std::optional& h) { + auto _ = hilti::rt::profiler::start("zeek/rt/protocol_data_in"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + auto c = cookie->protocol; + if ( ! c ) + throw ValueUnavailable("no current connection available"); + + auto len = data.size(); + auto* data_ = reinterpret_cast(data.data()); + + if ( h ) { + if ( auto* output_handler = c->analyzer->GetOutputHandler() ) + output_handler->DeliverStream(len, data_, is_orig); + + auto* child = c->analyzer->FindChild(h->id()); + if ( ! child ) + throw ValueUnavailable(hilti::rt::fmt("unknown child analyzer %s", *h)); + + if ( child->IsFinished() || child->Removing() ) + throw ValueUnavailable(hilti::rt::fmt("child analyzer %s no longer exist", *h)); + + child->NextStream(len, data_, is_orig); + } + + else + c->analyzer->ForwardStream(len, data_, is_orig); +} + +void rt::protocol_gap(const hilti::rt::Bool& is_orig, const hilti::rt::integer::safe& offset, + const hilti::rt::integer::safe& len, const std::optional& h) { + auto _ = hilti::rt::profiler::start("zeek/rt/protocol_gap"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + auto c = cookie->protocol; + if ( ! c ) + throw ValueUnavailable("no current connection available"); + + if ( h ) { + if ( auto* output_handler = c->analyzer->GetOutputHandler() ) + output_handler->Undelivered(offset, len, is_orig); + + auto* child = c->analyzer->FindChild(h->id()); + if ( ! child ) + throw ValueUnavailable(hilti::rt::fmt("unknown child analyzer %s", *h)); + + if ( child->IsFinished() || child->Removing() ) + throw ValueUnavailable(hilti::rt::fmt("child analyzer %s no longer exist", *h)); + + child->NextUndelivered(offset, len, is_orig); + } + + else + c->analyzer->ForwardUndelivered(offset, len, is_orig); +} + +void rt::protocol_end() { + auto _ = hilti::rt::profiler::start("zeek/rt/protocol_end"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + auto c = cookie->protocol; + if ( ! c ) + throw ValueUnavailable("no current connection available"); + + for ( const auto& i : c->analyzer->GetChildren() ) + c->analyzer->RemoveChildAnalyzer(i); +} + +void rt::protocol_handle_close(const ProtocolHandle& handle) { + auto _ = hilti::rt::profiler::start("zeek/rt/protocol_handle_close"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + auto c = cookie->protocol; + if ( ! c ) + throw ValueUnavailable("no current connection available"); + + auto child = c->analyzer->FindChild(handle.id()); + if ( ! child ) + throw ValueUnavailable(hilti::rt::fmt("unknown child analyzer %s", handle)); + + if ( child->IsFinished() || child->Removing() ) + throw ValueUnavailable(hilti::rt::fmt("child analyzer %s no longer exist", handle)); + + child->NextEndOfData(true); + child->NextEndOfData(false); + + c->analyzer->RemoveChildAnalyzer(handle.id()); +} + +rt::cookie::FileState* rt::cookie::FileStateStack::push() { + auto _ = hilti::rt::profiler::start("zeek/rt/file-stack-push"); + auto fid = file_mgr->HashHandle(hilti::rt::fmt("%s.%d", _analyzer_id, ++_id_counter)); + _stack.emplace_back(fid); + return &_stack.back(); +} + +const rt::cookie::FileState* rt::cookie::FileStateStack::find(const std::string& fid) const { + auto _ = hilti::rt::profiler::start("zeek/rt/file-stack-find"); + + // Reverse search as the default state would be on top of the stack. + for ( auto i = _stack.rbegin(); i != _stack.rend(); i++ ) { + if ( i->fid == fid ) + return &*i; + } + + return nullptr; +} + +void rt::cookie::FileStateStack::remove(const std::string& fid) { + auto _ = hilti::rt::profiler::start("zeek/rt/file-stack-remove"); + + // Reverse search as the default state would be on top of the stack. + for ( auto i = _stack.rbegin(); i != _stack.rend(); i++ ) { + if ( i->fid == fid ) { + _stack.erase((i + 1).base()); // https://stackoverflow.com/a/1830240 + return; + } + } +} + +static void _data_in(const char* data, uint64_t len, std::optional offset, + const std::optional& fid) { + auto cookie = static_cast(hilti::rt::context::cookie()); + auto* fstate = _file_state(cookie, fid); + auto data_ = reinterpret_cast(data); + auto mime_type = (fstate->mime_type ? *fstate->mime_type : std::string()); + + if ( auto c = cookie->protocol ) { + auto tag = spicy_mgr->tagForProtocolAnalyzer(c->analyzer->GetAnalyzerTag()); + + if ( offset ) + file_mgr->DataIn(data_, len, *offset, tag, c->analyzer->Conn(), c->is_orig, fstate->fid, mime_type); + else + file_mgr->DataIn(data_, len, tag, c->analyzer->Conn(), c->is_orig, fstate->fid, mime_type); + } + else { + if ( offset ) + file_mgr->DataIn(data_, len, *offset, Tag(), nullptr, false, fstate->fid, mime_type); + else + file_mgr->DataIn(data_, len, Tag(), nullptr, false, fstate->fid, mime_type); + } +} + +void rt::terminate_session() { + auto _ = hilti::rt::profiler::start("zeek/rt/terminate_session"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( auto c = cookie->protocol ) { + assert(session_mgr); + session_mgr->Remove(c->analyzer->Conn()); + } + else + throw spicy::rt::ValueUnavailable("terminate_session() not available in the current context"); +} + +std::string rt::fuid() { + auto _ = hilti::rt::profiler::start("zeek/rt/fuid"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( auto f = cookie->file ) { + if ( auto file = f->analyzer->GetFile() ) + return file->GetID(); + } + + throw ValueUnavailable("fuid() not available in current context"); +} + +std::string rt::file_begin(const std::optional& mime_type) { + auto _ = hilti::rt::profiler::start("zeek/rt/file_begin"); + auto cookie = static_cast(hilti::rt::context::cookie()); + auto* fstate = _file_state_stack(cookie)->push(); + fstate->mime_type = mime_type; + + // Feed an empty chunk into the analysis to force creating the file state inside Zeek. + _data_in("", 0, {}, {}); + + auto file = file_mgr->LookupFile(fstate->fid); + assert(file); // passing in empty data ensures that this is now available + + if ( auto f = cookie->file ) { + // We need to initialize some fa_info fields ourselves that would + // normally be inferred from the connection. + + // Set the source to the current file analyzer. + file->SetSource(file_mgr->GetComponentName(f->analyzer->Tag())); + + // There are some fields inside the new fa_info record that we want to + // set, but don't have a Zeek API for. Hence, we need to play some + // tricks: we can get to the fa_info value, but read-only; const_cast + // comes to our rescue. And then we just write directly into the + // record fields. + auto rval = file->ToVal()->AsRecordVal(); + auto current = f->analyzer->GetFile()->ToVal()->AsRecordVal(); + rval->Assign(id::fa_file->FieldOffset("parent_id"), + current->GetField("id")); // set to parent + rval->Assign(id::fa_file->FieldOffset("conns"), + current->GetField("conns")); // copy from parent + rval->Assign(id::fa_file->FieldOffset("is_orig"), + current->GetField("is_orig")); // copy from parent + } + + // Double check everybody agrees on the file ID. + assert(fstate->fid == file->GetID()); + return fstate->fid; +} + +void rt::file_set_size(const hilti::rt::integer::safe& size, const std::optional& fid) { + auto _ = hilti::rt::profiler::start("zeek/rt/file_set_size"); + auto cookie = static_cast(hilti::rt::context::cookie()); + auto* fstate = _file_state(cookie, fid); + + if ( auto c = cookie->protocol ) { + auto tag = spicy_mgr->tagForProtocolAnalyzer(c->analyzer->GetAnalyzerTag()); + file_mgr->SetSize(size, tag, c->analyzer->Conn(), c->is_orig, fstate->fid); + } + else + file_mgr->SetSize(size, Tag(), nullptr, false, fstate->fid); +} + +void rt::file_data_in(const hilti::rt::Bytes& data, const std::optional& fid) { + auto _ = hilti::rt::profiler::start("zeek/rt/file_data_in"); + _data_in(data.data(), data.size(), {}, fid); +} + +void rt::file_data_in_at_offset(const hilti::rt::Bytes& data, const hilti::rt::integer::safe& offset, + const std::optional& fid) { + auto _ = hilti::rt::profiler::start("zeek/rt/file_data_in_at_offset"); + _data_in(data.data(), data.size(), offset, fid); +} + +void rt::file_gap(const hilti::rt::integer::safe& offset, const hilti::rt::integer::safe& len, + const std::optional& fid) { + auto _ = hilti::rt::profiler::start("zeek/rt/file_gap"); + auto cookie = static_cast(hilti::rt::context::cookie()); + auto* fstate = _file_state(cookie, fid); + + if ( auto c = cookie->protocol ) { + auto tag = spicy_mgr->tagForProtocolAnalyzer(c->analyzer->GetAnalyzerTag()); + file_mgr->Gap(offset, len, tag, c->analyzer->Conn(), c->is_orig, fstate->fid); + } + else + file_mgr->Gap(offset, len, Tag(), nullptr, false, fstate->fid); +} + +void rt::file_end(const std::optional& fid) { + auto _ = hilti::rt::profiler::start("zeek/rt/file_end"); + auto cookie = static_cast(hilti::rt::context::cookie()); + auto* fstate = _file_state(cookie, fid); + + file_mgr->EndOfFile(fstate->fid); + _file_state_stack(cookie)->remove(fstate->fid); +} + +void rt::forward_packet(const hilti::rt::integer::safe& identifier) { + auto _ = hilti::rt::profiler::start("zeek/rt/forward_packet"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( auto c = cookie->packet ) + c->next_analyzer = identifier; + else + throw ValueUnavailable("no current packet analyzer available"); +} + +hilti::rt::Time rt::network_time() { + auto _ = hilti::rt::profiler::start("zeek/rt/network_time"); + return hilti::rt::Time(run_state::network_time, hilti::rt::Time::SecondTag()); +} diff --git a/src/spicy/runtime-support.h b/src/spicy/runtime-support.h new file mode 100644 index 0000000000..8bfc38b3f2 --- /dev/null +++ b/src/spicy/runtime-support.h @@ -0,0 +1,889 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +/** + * Functions and types available to generated Spicy/Zeek glue code. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "zeek/Desc.h" +#include "zeek/spicy/cookie.h" + +namespace zeek::spicy::rt { + +// Adapt to rename of exception. +#if SPICY_VERSION_NUMBER >= 10700 +using UsageError = ::hilti::rt::UsageError; +#else +using UsageError = ::hilti::rt::UserException; +#endif + +/** + * Exception thrown by event generation code if the value of an `$...` + * expression isn't available. + */ +class ValueUnavailable : public UsageError { +public: + using UsageError::UsageError; +}; + +/** + * Exception thrown by event generation code if the values can't be converted + * to Zeek. + */ +class InvalidValue : public UsageError { +public: + using UsageError::UsageError; +}; + +/** + * Exception thrown by event generation code if functionality is used + * that the current build does not support. + */ +class Unsupported : public UsageError { +public: + using UsageError::UsageError; +}; + +/** + * Exception thrown by event generation code if there's a type mismatch + * between the Spicy-side value and what the Zeek event expects. + */ +class TypeMismatch : public UsageError { +public: + TypeMismatch(const std::string_view& msg, std::string_view location = "") + : UsageError(hilti::rt::fmt("Event parameter mismatch, %s", msg)) {} + TypeMismatch(const std::string_view& have, TypePtr want, std::string_view location = "") + : TypeMismatch(_fmt(have, want)) {} + +private: + std::string _fmt(const std::string_view& have, TypePtr want) { + ODesc d; + want->Describe(&d); + return hilti::rt::fmt("cannot convert Spicy value of type '%s' to Zeek value of type '%s'", have, + d.Description()); + } +}; + +/** + * Exception thrown by the runtime library when Zeek has flagged a problem. + */ +class ZeekError : public UsageError { +public: + using UsageError::UsageError; +}; + +/** + * Registers a Spicy protocol analyzer with its EVT meta information with the + * plugin's runtime. + */ +void register_protocol_analyzer(const std::string& name, hilti::rt::Protocol proto, + const hilti::rt::Vector& ports, const std::string& parser_orig, + const std::string& parser_resp, const std::string& replaces, + const std::string& linker_scope); + +/** + * Registers a Spicy file analyzer with its EVT meta information with the + * plugin's runtime. + */ +void register_file_analyzer(const std::string& name, const hilti::rt::Vector& mime_types, + const std::string& parser, const std::string& replaces, const std::string& linker_scope); + +/** Reports a Zeek-side "weird". */ +void weird(const std::string& id, const std::string& addl); + +/** + * Registers a Spicy packet analyzer with its EVT meta information with the + * plugin's runtime. + */ +void register_packet_analyzer(const std::string& name, const std::string& parser, const std::string& replaces, + const std::string& linker_scope); + +/** Registers a Spicy-generated type to make it available inside Zeek. */ +void register_type(const std::string& ns, const std::string& id, const TypePtr& type); + +/** Identifies a Zeek-side type. */ +enum class ZeekTypeTag : uint64_t { + Addr, + Any, + Bool, + Count, + Double, + Enum, + Error, + File, + Func, + Int, + Interval, + List, + Opaque, + Pattern, + Port, + Record, + String, + Subnet, + Table, + Time, + Type, + Vector, + Void, +}; + +extern TypePtr create_base_type(ZeekTypeTag tag); + +extern TypePtr create_enum_type( + const std::string& ns, const std::string& id, + const hilti::rt::Vector>>& labels); + +using RecordField = std::tuple; // (ID, type, optional) +extern TypePtr create_record_type(const std::string& ns, const std::string& id, + const hilti::rt::Vector& fields); + +extern TypePtr create_table_type(TypePtr key, std::optional value); +extern TypePtr create_vector_type(const TypePtr& elem); + +/** Returns true if an event has at least one handler defined. */ +inline hilti::rt::Bool have_handler(const EventHandlerPtr& handler) { return static_cast(handler); } + +/** + * Creates a new event handler under the given name. + */ +void install_handler(const std::string& name); + +/** + * Looks up an event handler by name. The handler must have been installed + * before through `install_handler()`. + */ +EventHandlerPtr internal_handler(const std::string& name); + +/** Raises a Zeek event, given the handler and arguments. */ +void raise_event(const EventHandlerPtr& handler, const hilti::rt::Vector& args); + +/** + * Returns the Zeek type of an event's i'th argument. The result's ref count + * is not increased. + */ +TypePtr event_arg_type(const EventHandlerPtr& handler, const hilti::rt::integer::safe& idx); + +/** + * Retrieves the connection ID for the currently processed Zeek connection. + * Assumes that the HILTI context's cookie value has been set accordingly. + * + * @return Zeek value of record type + */ +ValPtr& current_conn(); + +/** + * Retrieves the direction of the currently processed Zeek connection. + * Assumes that the HILTI context's cookie value has been set accordingly. + * + * @return Zeek value of boolean type + */ +ValPtr& current_is_orig(); + +/** + * Logs a string through the Spicy plugin's debug output. + * + * @param cookie refers to the connection or file that the message is associated with + * @param msg message to log + */ +void debug(const Cookie& cookie, const std::string& msg); + +/** + * Logs a string through the Spicy plugin's debug output. This version logs + * the information the currently processed connection or file. + * + * @param msg message to log + */ +void debug(const std::string& msg); + +/** + * Retrieves the fa_file instance for the currently processed Zeek file. + * Assumes that the HILTI context's cookie value has been set accordingly. + * + * @return Zeek value of record type + */ +ValPtr current_file(); + +/** + * Retrieves a `raw_pkt_hdr` instance for the currently processed Zeek packet. + * Assumes that the HILTI context's cookie value has been set accordingly. + * + * @return Zeek value of record type + */ +ValPtr current_packet(); + +/** + * Returns true if we're currently parsing the originator side of a + * connection. + */ +hilti::rt::Bool is_orig(); + +/** + * Returns the current connection's UID. + */ +std::string uid(); + +/** + * Returns the current connection's ID tuple. + */ +std::tuple conn_id(); + +/** Instructs to Zeek to flip the directionality of the current connecction. */ +void flip_roles(); + +/** + * Returns the number of packets seen so far on the current side of the + * current connection. + */ +hilti::rt::integer::safe number_packets(); + +/** + * Triggers a DPD protocol confirmation for the currently processed + * connection. Assumes that the HILTI context's cookie value has been set + * accordingly. + */ +void confirm_protocol(); + +/** + * Triggers a DPD protocol violation for the currently processed connection. + * Assumes that the HILTI context's cookie value has been set accordingly. + * + * @param reason short description of what went wrong + */ +void reject_protocol(const std::string& reason); + +/** + * Opaque handle to a protocol analyzer. + */ +class ProtocolHandle { +public: + ProtocolHandle() {} + explicit ProtocolHandle(uint64_t id) : _id(id) {} + + uint64_t id() const { + if ( ! _id ) + throw ValueUnavailable("uninitialized protocol handle"); + + return *_id; + } + + friend std::string to_string(const ProtocolHandle& h, ::hilti::rt::detail::adl::tag) { + if ( ! h._id ) + return "(uninitialized protocol handle)"; + + return std::to_string(*h._id); + } + + friend std::ostream& operator<<(std::ostream& stream, const ProtocolHandle& h) { + return stream << ::hilti::rt::to_string(h); + } + +private: + std::optional _id; +}; + +/** + * Adds a Zeek-side child protocol analyzer to the current connection. + * + * @param analyzer if given, the Zeek-side name of the analyzer to instantiate; + * if not given, DPD will be used + */ +void protocol_begin(const std::optional& analyzer); + +/** + * Gets a handle to a child analyzer of a given type. If a child of that type + * does not yet exist it will be created. + * + * @param analyzer the Zeek-side name of the analyzer to get (e.g., `HTTP`) + * + * @return a handle to the child analyzer. When done, the handle should be + * closed, either explicitly with protocol_handle_close or implicitly with + * protocol_end. + */ +ProtocolHandle protocol_handle_get_or_create(const std::string& analyzer); + +/** + * Forwards data to all previously instantiated Zeek-side child protocol + * analyzers. + * + * @param is_orig true to feed data to originator side, false for responder + * @param data next chunk of stream data for child analyzer to process + * @param h optional handle to the child analyzer to stream data into + */ +void protocol_data_in(const hilti::rt::Bool& is_orig, const hilti::rt::Bytes& data, + const std::optional& h = {}); + +/** + * Signals a gap in input data to all previously instantiated Zeek-side child + * protocol analyzers. + * + * @param is_orig true to signal gap to originator side, false for responder + * @param offset of the gap inside the protocol stream + * @param length of the gap + * @param h optional handle to the child analyzer to signal a gap to + */ +void protocol_gap(const hilti::rt::Bool& is_orig, const hilti::rt::integer::safe& offset, + const hilti::rt::integer::safe& len, const std::optional& h = {}); + +/** + * Signals EOD to all previously instantiated Zeek-side child protocol + * analyzers and removes them. + */ +void protocol_end(); + +/** + * Closes a protocol handle. + * + * @param handle handle of the protocol analyzer to close. + */ +void protocol_handle_close(const ProtocolHandle& handle); + +/** + * Signals the beginning of a file to Zeek's file analysis, associating it + * with the current connection. + * + * param mime_type optional mime type passed to Zeek + * @returns Zeek-side file ID of the new file + */ +std::string file_begin(const std::optional& mime_type); + +/** + * Returns the current file's FUID. + */ +std::string fuid(); + +/** + * Terminates the currently active Zeek-side session, flushing all state. Any + * subsequent activity will start a new session from scratch. + */ +void terminate_session(); + +/** + * Signals the expected size of a file to Zeek's file analysis. + * + * @param size expected final size of the file + * @param fid ID of the file to operate on; if unset, the most recently begun file is used + */ +void file_set_size(const hilti::rt::integer::safe& size, const std::optional& fid = {}); + +/** + * Passes file content on to Zeek's file analysis. + * + * @param data next chunk of data + * @param fid ID of the file to operate on; if unset, the most recently begun file is used + */ +void file_data_in(const hilti::rt::Bytes& data, const std::optional& fid = {}); + +/** + * Passes file content at a specific offset on to Zeek's file analysis. + * + * @param data next chunk of data + * @param offset file offset of the data geing passed in + * @param fid ID of the file to operate on; if unset, the most recently begun file is used + */ +void file_data_in_at_offset(const hilti::rt::Bytes& data, const hilti::rt::integer::safe& offset, + const std::optional& fid = {}); + +/** + * Signals a gap in a file to Zeek's file analysis. + * + * @param offset of the gap + * @param length of the gap + * @param fid ID of the file to operate on; if unset, the most recently begun file is used + */ +void file_gap(const hilti::rt::integer::safe& offset, const hilti::rt::integer::safe& len, + const std::optional& fid = {}); + +/** + * Signals the end of a file to Zeek's file analysis. + * + * @param fid ID of the file to operate on; if unset, the most recently begun file is used + */ +void file_end(const std::optional& fid = {}); + +/** Specifies the next-layer packet analyzer. */ +void forward_packet(const hilti::rt::integer::safe& identifier); + +/** Gets the network time from Zeek. */ +hilti::rt::Time network_time(); + +// Forward-declare to_val() functions. +template::value>* = nullptr> +ValPtr to_val(const T& t, TypePtr target); +template::value>* = nullptr> +ValPtr to_val(const T& t, TypePtr target); +template::value>* = nullptr> +ValPtr to_val(const T& t, TypePtr target); +template::value>* = nullptr> +ValPtr to_val(const T& t, TypePtr target); +template +ValPtr to_val(const hilti::rt::Map& s, TypePtr target); +template +ValPtr to_val(const hilti::rt::Set& s, TypePtr target); +template +ValPtr to_val(const hilti::rt::Vector& v, TypePtr target); +template +ValPtr to_val(const std::optional& t, TypePtr target); +template +ValPtr to_val(const hilti::rt::DeferredExpression& t, TypePtr target); +template +ValPtr to_val(hilti::rt::integer::safe i, TypePtr target); +template +ValPtr to_val(const hilti::rt::ValueReference& t, TypePtr target); + +inline ValPtr to_val(const hilti::rt::Bool& b, TypePtr target); +inline ValPtr to_val(const hilti::rt::Address& d, TypePtr target); +inline ValPtr to_val(const hilti::rt::Bytes& b, TypePtr target); +inline ValPtr to_val(const hilti::rt::Interval& t, TypePtr target); +inline ValPtr to_val(const hilti::rt::Port& d, TypePtr target); +inline ValPtr to_val(const hilti::rt::Time& t, TypePtr target); +inline ValPtr to_val(const std::string& s, TypePtr target); +inline ValPtr to_val(double r, TypePtr target); + +/** + * Converts a Spicy-side optional value to a Zeek value. This assumes the + * optional is set, and will throw an exception if not. The result is + * returned with ref count +1. + */ +template +inline ValPtr to_val(const std::optional& t, TypePtr target) { + if ( t.has_value() ) + return to_val(hilti::rt::optional::value(t), target); + + return nullptr; +} + +/** + * Converts a Spicy-side DeferredExpression value to a Zeek value. Such + * result values are returned by the ``.?`` operator. If the result is not + * set, this will convert into nullptr (which the tuple-to-record to_val() + * picks up on). + */ +template +inline ValPtr to_val(const hilti::rt::DeferredExpression& t, TypePtr target) { + try { + return to_val(t(), target); + } catch ( const hilti::rt::AttributeNotSet& ) { + return nullptr; + } +} + +/** + * Converts a Spicy-side string to a Zeek value. The result is returned with + * ref count +1. + */ +inline ValPtr to_val(const std::string& s, TypePtr target) { + if ( target->Tag() != TYPE_STRING ) + throw TypeMismatch("string", target); + + return make_intrusive(s); +} + +/** + * Converts a Spicy-side bytes instance to a Zeek value. The result is returned with + * ref count +1. + */ +inline ValPtr to_val(const hilti::rt::Bytes& b, TypePtr target) { + if ( target->Tag() != TYPE_STRING ) + throw TypeMismatch("string", target); + + return make_intrusive(b.str()); +} + +/** + * Converts a Spicy-side integer to a Zeek value. The result is + * returned with ref count +1. + */ +template +inline ValPtr to_val(hilti::rt::integer::safe i, TypePtr target) { + ValPtr v = nullptr; + if constexpr ( std::is_unsigned::value ) { + if ( target->Tag() == TYPE_COUNT ) + return val_mgr->Count(i); + + if ( target->Tag() == TYPE_INT ) + return val_mgr->Int(i); + + throw TypeMismatch("uint64", target); + } + else { + if ( target->Tag() == TYPE_INT ) + return val_mgr->Int(i); + + if ( target->Tag() == TYPE_COUNT ) { + if ( i >= 0 ) + return val_mgr->Count(i); + else + throw TypeMismatch("negative int64", target); + } + + throw TypeMismatch("int64", target); + } +} + +template +ValPtr to_val(const hilti::rt::ValueReference& t, TypePtr target) { + if ( auto* x = t.get() ) + return to_val(*x, target); + + return nullptr; +} + +/** + * Converts a Spicy-side signed bool to a Zeek value. The result is + * returned with ref count +1. + */ +inline ValPtr to_val(const hilti::rt::Bool& b, TypePtr target) { + if ( target->Tag() != TYPE_BOOL ) + throw TypeMismatch("bool", target); + + return val_mgr->Bool(b); +} + +/** + * Converts a Spicy-side real to a Zeek value. The result is returned with + * ref count +1. + */ +inline ValPtr to_val(double r, TypePtr target) { + if ( target->Tag() != TYPE_DOUBLE ) + throw TypeMismatch("double", target); + + return make_intrusive(r); +} + +/** + * Converts a Spicy-side address to a Zeek value. The result is returned with + * ref count +1. + */ +inline ValPtr to_val(const hilti::rt::Address& d, TypePtr target) { + if ( target->Tag() != TYPE_ADDR ) + throw TypeMismatch("addr", target); + + auto in_addr = d.asInAddr(); + if ( auto v4 = std::get_if(&in_addr) ) + return make_intrusive(IPAddr(*v4)); + else { + auto v6 = std::get(in_addr); + return make_intrusive(IPAddr(v6)); + } +} + +/** + * Converts a Spicy-side address to a Zeek value. The result is returned with + * ref count +1. + */ +inline ValPtr to_val(const hilti::rt::Port& p, TypePtr target) { + if ( target->Tag() != TYPE_PORT ) + throw TypeMismatch("port", target); + +#if SPICY_VERSION_NUMBER >= 10700 + auto proto = p.protocol().value(); +#else + auto proto = p.protocol(); +#endif + + switch ( proto ) { + case hilti::rt::Protocol::TCP: return val_mgr->Port(p.port(), ::TransportProto::TRANSPORT_TCP); + + case hilti::rt::Protocol::UDP: return val_mgr->Port(p.port(), ::TransportProto::TRANSPORT_UDP); + + case hilti::rt::Protocol::ICMP: return val_mgr->Port(p.port(), ::TransportProto::TRANSPORT_ICMP); + + default: throw InvalidValue("port value with undefined protocol"); + } +} + +/** + * Converts a Spicy-side time to a Zeek value. The result is returned with + * ref count +1. + */ +inline ValPtr to_val(const hilti::rt::Interval& i, TypePtr target) { + if ( target->Tag() != TYPE_INTERVAL ) + throw TypeMismatch("interval", target); + + return make_intrusive(i.seconds()); +} + +/** + * Converts a Spicy-side time to a Zeek value. The result is returned with + * ref count +1. + */ +inline ValPtr to_val(const hilti::rt::Time& t, TypePtr target) { + if ( target->Tag() != TYPE_TIME ) + throw TypeMismatch("time", target); + + return make_intrusive(t.seconds()); +} + +/** + * Converts a Spicy-side vector to a Zeek value. The result is returned with + * ref count +1. + */ +template +inline ValPtr to_val(const hilti::rt::Vector& v, TypePtr target) { + if ( target->Tag() != TYPE_VECTOR && target->Tag() != TYPE_LIST ) + throw TypeMismatch("expected vector or list", target); + + auto vt = cast_intrusive(target); + auto zv = make_intrusive(vt); + for ( const auto& i : v ) + zv->Assign(zv->Size(), to_val(i, vt->Yield())); + + return zv; +} + +/** + * Converts a Spicy-side map to a Zeek value. The result is returned with + * ref count +1. + */ +template +inline ValPtr to_val(const hilti::rt::Map& m, TypePtr target) { + if constexpr ( hilti::rt::is_tuple::value ) + throw TypeMismatch("internal error: sets with tuples not yet supported in to_val()"); + + if ( target->Tag() != TYPE_TABLE ) + throw TypeMismatch("map", target); + + auto tt = cast_intrusive(target); + if ( tt->IsSet() ) + throw TypeMismatch("map", target); + + if ( tt->GetIndexTypes().size() != 1 ) + throw TypeMismatch("map with non-tuple elements", target); + + auto zv = make_intrusive(tt); + + for ( const auto& i : m ) { + auto k = to_val(i.first, tt->GetIndexTypes()[0]); + auto v = to_val(i.second, tt->Yield()); + zv->Assign(std::move(k), std::move(v)); + } + + return zv; +} // namespace spicy::rt + +/** + * Converts a Spicy-side set to a Zeek value. The result is returned with + * ref count +1. + */ +template +inline ValPtr to_val(const hilti::rt::Set& s, TypePtr target) { + if ( target->Tag() != TYPE_TABLE ) + throw TypeMismatch("set", target); + + auto tt = cast_intrusive(target); + if ( ! tt->IsSet() ) + throw TypeMismatch("set", target); + + auto zv = make_intrusive(tt); + + for ( const auto& i : s ) { + if constexpr ( hilti::rt::is_tuple::value ) + throw TypeMismatch("internal error: sets with tuples not yet supported in to_val()"); + else { + if ( tt->GetIndexTypes().size() != 1 ) + throw TypeMismatch("set with non-tuple elements", target); + + auto idx = to_val(i, tt->GetIndexTypes()[0]); + zv->Assign(std::move(idx), nullptr); + } + } + + return zv; +} + +namespace { +template typename> +struct is_instance_impl : std::false_type {}; + +template typename U, typename... Ts> +struct is_instance_impl, U> : std::true_type {}; +} // namespace + +template typename U> +using is_instance = is_instance_impl, U>; + +template +inline void set_record_field(RecordVal* rval, const IntrusivePtr& rtype, int idx, const T& x) { + using NoConversionNeeded = std::integral_constant< + bool, std::is_same_v || std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v>; + + using IsSignedInteger = std::integral_constant> || + std::is_same_v> || + std::is_same_v> || + std::is_same_v>>; + + using IsUnsignedInteger = std::integral_constant> || + std::is_same_v> || + std::is_same_v> || + std::is_same_v>>; + + if constexpr ( NoConversionNeeded::value ) + rval->Assign(idx, x); + else if constexpr ( IsSignedInteger::value ) +#if ZEEK_VERSION_NUMBER >= 50200 + rval->Assign(idx, static_cast(x.Ref())); +#else + rval->Assign(idx, static_cast(x.Ref())); +#endif + else if constexpr ( IsUnsignedInteger::value ) + rval->Assign(idx, static_cast(x.Ref())); + else if constexpr ( std::is_same_v ) + rval->Assign(idx, x.str()); + else if constexpr ( std::is_same_v ) + rval->Assign(idx, static_cast(x)); + else if constexpr ( std::is_same_v ) + rval->Assign(idx, x); + else if constexpr ( std::is_same_v ) + rval->AssignTime(idx, x.seconds()); + else if constexpr ( std::is_same_v ) + rval->AssignInterval(idx, x.seconds()); + else if constexpr ( std::is_same_v ) { + // "Null" turns into an unset optional record field. + } + else if constexpr ( is_instance::value ) { + if ( x.has_value() ) + set_record_field(rval, rtype, idx, *x); + } + else if constexpr ( is_instance::value ) { + try { + set_record_field(rval, rtype, idx, x()); + } catch ( const hilti::rt::AttributeNotSet& ) { + // leave unset + } + } + else { + ValPtr v = nullptr; + + // This may return a nullptr in cases where the field is to be left unset. + v = to_val(x, rtype->GetFieldType(idx)); + + if ( v ) + rval->Assign(idx, v); + else { + // Field must be &optional or &default. + if ( auto attrs = rtype->FieldDecl(idx)->attrs; + ! attrs || ! (attrs->Find(detail::ATTR_DEFAULT) || attrs->Find(detail::ATTR_OPTIONAL)) ) + throw TypeMismatch(hilti::rt::fmt("missing initialization for field '%s'", rtype->FieldName(idx))); + } + } +} + +/** + * Converts a Spicy-side tuple to a Zeek record value. The result is returned + * with ref count +1. + */ +template::value>*> +inline ValPtr to_val(const T& t, TypePtr target) { + if ( target->Tag() != TYPE_RECORD ) + throw TypeMismatch("tuple", target); + + auto rtype = cast_intrusive(target); + + if ( std::tuple_size::value != rtype->NumFields() ) + throw TypeMismatch("tuple", target); + + auto rval = make_intrusive(rtype); + int idx = 0; + hilti::rt::tuple_for_each(t, [&](const auto& x) { set_record_field(rval.get(), rtype, idx++, x); }); + + return rval; +} + +/** + * Converts Spicy-side struct to a Zeek record value. The result is returned + * with a ref count +1. + */ +template::value>*> +inline ValPtr to_val(const T& t, TypePtr target) { + if ( target->Tag() != TYPE_RECORD ) + throw TypeMismatch("struct", target); + + auto rtype = cast_intrusive(target); + + auto rval = make_intrusive(rtype); + int idx = 0; + + auto num_fields = rtype->NumFields(); + + t.__visit([&](const auto& name, const auto& val) { + if ( idx >= num_fields ) + throw TypeMismatch(hilti::rt::fmt("no matching record field for field '%s'", name)); + + auto field = rtype->GetFieldType(idx); + std::string field_name = rtype->FieldName(idx); + + if ( field_name != name ) + throw TypeMismatch(hilti::rt::fmt("mismatch in field name: expected '%s', found '%s'", name, field_name)); + + set_record_field(rval.get(), rtype, idx++, val); + }); + + // We already check above that all Spicy-side fields are mapped so we + // can only hit this if there are uninitialized Zeek-side fields left. + if ( idx != num_fields ) + throw TypeMismatch(hilti::rt::fmt("missing initialization for field '%s'", rtype->FieldName(idx + 1))); + + return rval; +} + +/** + * Converts a Spicy-side enum to a Zeek record value. The result is returned + * with ref count +1. + */ +template::value>*> +inline ValPtr to_val(const T& t, TypePtr target) { +#if SPICY_VERSION_NUMBER >= 10700 + auto proto = typename T::Value(t.value()); +#else + auto proto = t; +#endif + + return to_val(proto, target); +} + +/** + * Converts a C++ Spicy-side enum to a Zeek record value. The result is returned + * with ref count +1. This specialization is provided for compatibility with ::value>*> +inline ValPtr to_val(const T& t, TypePtr target) { + if ( target->Tag() != TYPE_ENUM ) + throw TypeMismatch("enum", target); + + // We'll usually be getting an int64_t for T, but allow other signed ints + // as well. + static_assert(std::is_signed>{}); + auto it = static_cast(t); + + // Zeek's enum can't be negative, so we swap in max_int for our Undef (-1). + if ( it == std::numeric_limits::max() ) + // can't allow this ... + throw InvalidValue("enum values with value max_int not supported by Zeek integration"); + + zeek_int_t bt = (it >= 0 ? it : std::numeric_limits<::zeek_int_t>::max()); + + return target->AsEnumType()->GetEnumVal(bt); +} + +} // namespace zeek::spicy::rt diff --git a/src/spicy/spicy.bif b/src/spicy/spicy.bif new file mode 100644 index 0000000000..cb2c685a2e --- /dev/null +++ b/src/spicy/spicy.bif @@ -0,0 +1,62 @@ +# See the file "COPYING" in the main distribution directory for copyright. + +module Spicy; + +%%{ + #include "zeek/spicy/manager.h" +%%} + +# Constant for testing if Spicy is available. +const available: bool; + +# Show output of Spicy print statements. +const enable_print: bool; + +# Record and display profiling information. +const enable_profiling: bool; + +# abort() instead of throwing HILTI # exceptions. +const abort_on_exceptions: bool; + +# Include backtraces when reporting unhandled exceptions. +const show_backtraces: bool; + +# Maximum depth of recursive file analysis. +const max_file_depth: count; + +event max_file_depth_exceeded%(f: fa_file, args: Files::AnalyzerArgs, limit: count%); + +function Spicy::__toggle_analyzer%(tag: any, enable: bool%) : bool + %{ + if ( tag->GetType()->Tag() != TYPE_ENUM ) { + zeek::reporter->Warning("Spicy::disable_analyzer() must receive an analyzer tag"); + return val_mgr->Bool(false); + } + + bool result = spicy_mgr->toggleAnalyzer(tag->AsEnumVal(), enable); + if ( ! result ) + zeek::reporter->Warning("could not toggle Spicy analyzer"); + + return val_mgr->Bool(result); + %} + +type ResourceUsage: record; + +function Spicy::__resource_usage%(%) : Spicy::ResourceUsage + %{ + auto ru = hilti::rt::resource_usage(); + + auto r = zeek::make_intrusive(BifType::Record::Spicy::ResourceUsage); + int n = 0; + r->Assign(n++, ru.user_time); + r->Assign(n++, ru.system_time); + r->Assign(n++, ru.memory_heap); + r->Assign(n++, ru.num_fibers); + r->Assign(n++, ru.max_fibers); +#if SPICY_VERSION_NUMBER >= 10800 + r->Assign(n++, ru.max_fiber_stack_size); +#endif + r->Assign(n++, ru.cached_fibers); + + return r; + %} diff --git a/src/spicy/spicyz/CMakeLists.txt b/src/spicy/spicyz/CMakeLists.txt new file mode 100644 index 0000000000..54800a3db8 --- /dev/null +++ b/src/spicy/spicyz/CMakeLists.txt @@ -0,0 +1,10 @@ +# See the file "COPYING" in the main distribution directory for copyright. + +configure_file(config.h.in config.h) + +add_executable(spicyz driver.cc glue-compiler.cc main.cc) +target_compile_options(spicyz PRIVATE "-Wall") +target_include_directories(spicyz PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) +target_link_libraries(spicyz PRIVATE hilti spicy) + +install(TARGETS spicyz DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/src/spicy/spicyz/config.h.in b/src/spicy/spicyz/config.h.in new file mode 100644 index 0000000000..8433b8259b --- /dev/null +++ b/src/spicy/spicyz/config.h.in @@ -0,0 +1,67 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include "zeek/util-config.h" + +#include + +#include + +namespace zeek::spicy::configuration { + +using path = hilti::rt::filesystem::path; + +namespace { + +// This mimics zeek-config to get the Zeek include directories. +static inline void add_path(std::string& old_path, const path& new_path) { + if ( new_path.empty() ) + return; + + if ( ! old_path.empty() ) + old_path += ":"; + + old_path += new_path.native(); +} +} // namespace + +inline const auto InstallBinDir() { return path("${CMAKE_INSTALL_PREFIX}") / "${CMAKE_INSTALL_BINDIR}"; } + +inline const auto LibraryPath() { + path base; + + if ( auto custom_base = getenv("ZEEK_SPICY_LIBRARY_PATH"); custom_base && *custom_base ) + base = path(custom_base); + else + base = path("@ZEEK_SPICY_LIBRARY_PATH@"); + + return hilti::rt::filesystem::weakly_canonical(base); +} + +inline const auto ModulePath() { return path("@ZEEK_SPICY_MODULE_PATH@"); } +inline const auto DataPath() { return path("@ZEEK_SPICY_DATA_PATH@"); } + +inline const auto CxxZeekIncludesDirectories() { + std::string includes; + add_path(includes, path("${CMAKE_INSTALL_PREFIX}") / "${CMAKE_INSTALL_INCLUDEDIR}"); + + // When changing any of the following, also update "zeek-config.in". + add_path(includes, "${ZEEK_CONFIG_PCAP_INCLUDE_DIR}"); + add_path(includes, "${ZEEK_CONFIG_ZLIB_INCLUDE_DIR}"); + add_path(includes, "${ZEEK_CONFIG_OPENSSL_INCLUDE_DIR}"); + add_path(includes, "${ZEEK_CONFIG_LibKrb5_INCLUDE_DIR}"); + add_path(includes, "${ZEEK_CONFIG_GooglePerftools_INCLUDE_DIR}"); + + return includes; +} + +// Version of Spicy that we are compiling against. +#cmakedefine SPICY_VERSION_NUMBER ${SPICY_VERSION_NUMBER} +#cmakedefine ZEEK_VERSION_NUMBER ${ZEEK_VERSION_NUMBER} + +inline const auto SpicyVersion = "${SPICY_VERSION}"; +inline const auto ZeekVersion = "${VERSION}"; +inline const auto InstallPrefix = path("${CMAKE_INSTALL_PREFIX}"); + +} // namespace zeek::spicy::configuration diff --git a/src/spicy/spicyz/driver.cc b/src/spicy/spicyz/driver.cc new file mode 100644 index 0000000000..691ec5cfea --- /dev/null +++ b/src/spicy/spicyz/driver.cc @@ -0,0 +1,301 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "driver.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include +#include +#include + +#include "config.h" +#include "glue-compiler.h" + +using namespace zeek::spicy; +using Driver = ::zeek::spicy::Driver; + +/** + * Visitor to type information from a HILTI AST. This extracts user-visible + * types only, we skip any internal ones. + */ +struct VisitorTypes : public hilti::visitor::PreOrder { + explicit VisitorTypes(Driver* driver, hilti::ID module, hilti::rt::filesystem::path path, bool is_resolved) + : driver(driver), module(std::move(module)), path(std::move(path)), is_resolved(is_resolved) {} + + void operator()(const hilti::declaration::Type& t) { + assert(! t.type().typeID() || *t.type().typeID() == hilti::ID(module, t.id())); // ensure consistent IDs + + if ( module == hilti::ID("hilti") || module == hilti::ID("spicy_rt") || module == hilti::ID("zeek_rt") ) + return; + + types.emplace_back(TypeInfo{ + .id = hilti::ID(module, t.id()), + .type = t.type()._clone().as(), + .linkage = t.linkage(), + .is_resolved = is_resolved, + .module_id = module, + .module_path = path, + .location = t.meta().location(), + }); + } + + Driver* driver; + hilti::ID module; + hilti::rt::filesystem::path path; + bool is_resolved; + std::vector types; +}; + +Driver::Driver(std::unique_ptr glue, const char* argv0, hilti::rt::filesystem::path lib_path, + int zeek_version) + : ::spicy::Driver(""), _glue(std::move(glue)) { + _glue->Init(this, zeek_version); + + ::spicy::Configuration::extendHiltiConfiguration(); + auto options = hiltiOptions(); + + // Note that, different from Spicy's own SPICY_PATH, this extends the + // search path, it doesn't replace it. + if ( auto path = hilti::rt::getenv("ZEEK_SPICY_PATH") ) { + for ( const auto& dir : hilti::rt::split(*path, ":") ) { + if ( dir.size() ) + options.library_paths.emplace_back(dir); + } + } + + try { + lib_path = hilti::rt::filesystem::weakly_canonical(lib_path); + + // We make our search paths relative to the plugin library, so that the + // plugin installation can move around. + options.library_paths.push_back(lib_path); + } catch ( const hilti::rt::filesystem::filesystem_error& e ) { + ::hilti::logger().warning( + hilti::util::fmt("invalid plugin base directory %s: %s", lib_path.native(), e.what())); + } + + for ( const auto& i : hilti::util::split(configuration::CxxZeekIncludesDirectories(), ":") ) { + if ( i.size() ) + options.cxx_include_paths.emplace_back(i); + } + +#ifdef DEBUG + SPICY_DEBUG("Search paths:"); + + auto hilti_options = hiltiOptions(); + for ( const auto& x : hilti_options.library_paths ) { + SPICY_DEBUG(hilti::rt::fmt(" %s", x.native())); + } +#endif + + setCompilerOptions(std::move(options)); + + auto& config = ::spicy::configuration(); + config.preprocessor_constants["HAVE_ZEEK"] = 1; + config.preprocessor_constants["ZEEK_VERSION"] = zeek_version; + +#if SPICY_VERSION_NUMBER >= 10500 + ::hilti::init(); + ::spicy::init(); +#endif +} + +Driver::~Driver() {} + +hilti::Result Driver::loadFile(hilti::rt::filesystem::path file, + const hilti::rt::filesystem::path& relative_to) { + std::error_code ec; + if ( ! relative_to.empty() && file.is_relative() ) { + auto p = relative_to / file; + auto exists = hilti::rt::filesystem::exists(p, ec); + + if ( ec ) + return hilti::rt::result::Error( + hilti::util::fmt("error computing path of %s relative to %s: %s", file, relative_to, ec.message())); + + if ( exists ) + file = p; + } + + auto exists = hilti::rt::filesystem::exists(file, ec); + + if ( ec ) + return hilti::rt::result::Error( + hilti::util::fmt("cannot check whether file %s exists: %s", file, ec.message())); + + if ( ! exists ) { + if ( auto path = hilti::util::findInPaths(file, hiltiOptions().library_paths) ) + file = *path; + else + return hilti::result::Error(hilti::util::fmt("Spicy plugin cannot find file %s", file)); + } + + auto rpath = hilti::util::normalizePath(file); + auto ext = rpath.extension(); + + if ( ext == ".evt" ) { + SPICY_DEBUG(hilti::util::fmt("Loading EVT file %s", rpath)); + if ( _glue->loadEvtFile(rpath) ) + return hilti::Nothing(); + else + return hilti::result::Error(hilti::util::fmt("error loading EVT file %s", rpath)); + } + + if ( ext == ".spicy" ) { + SPICY_DEBUG(hilti::util::fmt("Loading Spicy file %s", rpath)); + if ( auto rc = addInput(rpath); ! rc ) + return rc.error(); + + return hilti::Nothing(); + } + + if ( ext == ".hlt" ) { + SPICY_DEBUG(hilti::util::fmt("Loading HILTI file %s", rpath)); + if ( auto rc = addInput(rpath) ) + return hilti::Nothing(); + else + return rc.error(); + } + + if ( ext == ".hlto" ) { + SPICY_DEBUG(hilti::util::fmt("Loading precompiled HILTI code %s", rpath)); + if ( auto rc = addInput(rpath) ) + return hilti::Nothing(); + else + return rc.error(); + } + + if ( ext == ".cc" || ext == ".cxx" ) { + SPICY_DEBUG(hilti::util::fmt("Loading C++ code %s", rpath)); + if ( auto rc = addInput(rpath) ) + return hilti::Nothing(); + else + return rc.error(); + } + + return hilti::result::Error(hilti::util::fmt("unknown file type passed to Spicy loader: %s", rpath)); +} + +hilti::Result Driver::compile() { + if ( ! hasInputs() ) + return hilti::Nothing(); + + SPICY_DEBUG("Running Spicy driver"); + + if ( auto x = ::spicy::Driver::compile(); ! x ) + return x.error(); + + SPICY_DEBUG("Done with Spicy driver"); + return hilti::Nothing(); +} + +hilti::Result Driver::lookupType(const hilti::ID& id) { + if ( auto x = _types.find(id); x != _types.end() ) + return x->second; + else + return hilti::result::Error(hilti::util::fmt("unknown type '%s'", id)); +} + +std::vector Driver::types() const { + std::vector result; + result.reserve(_types.size()); + + for ( const auto& t : _types ) + result.push_back(t.second); + + return result; +} + +std::vector> Driver::exportedTypes() const { + std::vector> result; + + for ( const auto& [spicy_id, zeek_id, _] : _glue->exportedIDs() ) { + if ( auto t = _types.find(spicy_id); t != _types.end() ) + result.emplace_back(t->second, zeek_id); + else { + hilti::logger().error(hilti::rt::fmt("unknown type '%s' exported", spicy_id)); + continue; + } + } + + // Automatically export public enums for backwards compatibility. + for ( const auto& t : _public_enums ) + result.emplace_back(t, t.id); + + return result; +} + +void Driver::hookNewASTPreCompilation(std::shared_ptr unit) { + if ( unit->extension() != ".spicy" ) + return; + + if ( unit->path().empty() ) + // Ignore modules constructed in memory. + return; + + auto v = VisitorTypes(this, unit->id(), unit->path(), false); + for ( auto i : v.walk(unit->module()) ) + v.dispatch(i); + + for ( const auto& ti : v.types ) { + SPICY_DEBUG(hilti::util::fmt(" Got type '%s' (pre-compile)", ti.id)); + _types[ti.id] = ti; + + if ( auto et = ti.type.tryAs(); et && ti.linkage == hilti::declaration::Linkage::Public ) { + SPICY_DEBUG(" Automatically exporting public enum for backwards compatibility"); + _public_enums.push_back(ti); + } + + hookNewType(ti); + } +} + +void Driver::hookNewASTPostCompilation(std::shared_ptr unit) { + if ( unit->extension() != ".spicy" ) + return; + + if ( unit->path().empty() ) + // Ignore modules constructed in memory. + return; + + auto v = VisitorTypes(this, unit->id(), unit->path(), true); + for ( auto i : v.walk(unit->module()) ) + v.dispatch(i); + + for ( auto&& t : v.types ) { + SPICY_DEBUG(hilti::util::fmt(" Got type '%s' (post-compile)", t.id)); + _types[t.id] = t; + hookNewType(t); + } + + _glue->addSpicyModule(unit->id(), unit->path()); +} + +hilti::Result Driver::hookCompilationFinished(const hilti::Plugin& plugin) { + if ( ! _need_glue ) + return hilti::Nothing(); + + _need_glue = false; + + if ( _glue->compile() ) + return hilti::Nothing(); + else + return hilti::result::Error("glue compilation failed"); +} + +void Driver::hookInitRuntime() { ::spicy::rt::init(); } + +void Driver::hookFinishRuntime() { ::spicy::rt::done(); } diff --git a/src/spicy/spicyz/driver.h b/src/spicy/spicyz/driver.h new file mode 100644 index 0000000000..8dd2422ce2 --- /dev/null +++ b/src/spicy/spicyz/driver.h @@ -0,0 +1,199 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include + +#include + +// Debug stream for compiler messages. +static const ::hilti::logging::DebugStream ZeekPlugin("zeek"); + +// Macro helper to report debug messages. +#define SPICY_DEBUG(msg) HILTI_DEBUG(ZeekPlugin, std::string(msg)); + +namespace zeek::spicy { + +class GlueCompiler; + +struct TypeInfo { + hilti::ID id; /**< fully-qualified name of the type */ + hilti::Type type; /**< the type itself */ + hilti::declaration::Linkage linkage; /**< linkage of of the type's declaration */ + bool is_resolved = false; /**< true if we are far enough in processing that the type has been fully resolved */ + hilti::ID module_id; /**< name of module type is defined in */ + hilti::rt::filesystem::path module_path; /**< path of module that type is defined in */ + hilti::Location location; /**< location of type's declaration */ +}; + +/** Spicy compilation driver. */ +class Driver : public ::spicy::Driver { +public: + /** + * Constructor. + * + * @param argv0 path to current executable, or empty to determine automatically + * @param lib_path Path to library files the Spicy support needs + * @param zeek_version Version number of Zeek we're working with + */ + Driver(std::unique_ptr glue, const char* argv0, hilti::rt::filesystem::path lib_path, + int zeek_version); + + /** Destructor. */ + ~Driver(); + + /** + * Schedules an *.spicy, *.evt, or *.hlt file for loading. Note that it + * won't necessarily load them all immediately, but may queue some for + * later processing. + * + * @param file file to load, which will be searched across all current search paths + * @param relative_to if given, relative paths will be interpreted as relative to this directory + */ + hilti::Result loadFile(hilti::rt::filesystem::path file, + const hilti::rt::filesystem::path& relative_to = {}); + + /** + * After user scripts have been read, compiles and links all resulting + * Spicy code. Note that compiler and driver options must have been set + * before calling this. + * + * Must be called before any packet processing starts. + * + * @return False if an error occurred. It will have been reported already. + */ + hilti::Result compile(); + + /** + * Returns meta information for a type. The Spicy module defining the type + * must have been compiled already for it to be found. + * + * @param id fully qualified name of type to look up + * @return meta data, or an error if the type is not (yet) known + */ + hilti::Result lookupType(const hilti::ID& id); + + /** + * Returns meta information for a type, enforcing it to be a of a certain + * kind. The Spicy module defining the type must have been compiled already + * for it to be found. + * + * @tparam T type to enforce; method will return an error if type is not of this class + * @param id fully qualified name of type to look up + * @return meta data, or an error if the type is not (yet) known + */ + template + hilti::Result lookupType(const hilti::ID& id) { + auto ti = lookupType(id); + if ( ! ti ) + return ti.error(); + + if ( ! ti->type.isA() ) + return hilti::result::Error(hilti::util::fmt("'%s' is not of expected type", id)); + + return ti; + } + + /** + * Returns all types seen so far during processing of Spicy files. + * Depending on where we are at with processing, these may or may not be + * resolved yet (as indicated by their `is_resolved` field). + + * @return list of types + */ + std::vector types() const; + + /** + * Returns all *exported* types seen so far during processing of Spicy + * files, including their desired Zeek-side names. Depending on where we + * are at with processing, these may or may not be resolved yet (as + * indicated by their `is_resolved` field). + * + * @return list of pairs of type and Zeek-side name + */ + std::vector> exportedTypes() const; + + /** Returns true if we're running out of the plugin's build directory. */ + bool usingBuildDirectory() const { return _using_build_directory; } + + /** Returns the glue compiler in use by the driver. */ + const auto* glueCompiler() const { return _glue.get(); } + + /** + * Parses some options command-line style *before* Zeek-side scripts have + * been processed. Most of the option processing happens in + * `parseOptionsPostScript()` instead, except for things that must be in + * place already before script processing. + * + * @param options space-separated string of command line argument to parse + * @return success if all argument could be parsed, or a suitable error message + */ + static hilti::Result parseOptionsPreScript(const std::string& options); + + /** + * Parses options command-line style after Zeek-side scripts have been + * fully procssed. Most of the option processing happens here (vs. in + * `parseOptionsPreScript()`) except for things that must be in place + * already before script processing. + * + * @param options space-separated string of command line argument to parse + * @param driver_options instance of options to update per parsed arguments + * @param compiler_options instance of options to update per parsed arguments + * @return success if all argument could be parsed, or a suitable error message + */ + static hilti::Result parseOptionsPostScript(const std::string& options, + hilti::driver::Options* driver_options, + hilti::Options* compiler_options); + + /** Prints a usage message for options supported by `parseOptions{Pre,Post}Script()`. */ + static void usage(std::ostream& out); + +protected: + /** + * Hook executed for all type declarations encountered in a Spicy module. + * Derived classes may override this to add custom processing. This hooks + * executes twices for each declaration: once before we compile the AST + * (meaning types have not been resolved yet), and once after. The type + * info's `is_resolved` field indicates which of the two we're in. + * + * @param t type's meta information + */ + virtual void hookNewType(const TypeInfo& ti) {} + + /** Overridden from HILTI driver. */ + void hookNewASTPreCompilation(std::shared_ptr unit) override; + + /** Overridden from HILTI driver. */ + void hookNewASTPostCompilation(std::shared_ptr unit) override; + + /** Overridden from HILTI driver. */ + hilti::Result hookCompilationFinished(const hilti::Plugin& plugin) override; + + /** Overridden from HILTI driver. */ + void hookInitRuntime() override; + + /** Overridden from HILTI driver. */ + void hookFinishRuntime() override; + + std::unique_ptr _glue; // glue compiler in use + std::unordered_map _types; // map of Spicy type declarations encountered so far + std::vector _public_enums; // tracks Spicy enum types declared public, for automatic export + bool _using_build_directory = false; // true if we're running out of the plugin's build directory + bool _need_glue = true; // true if glue code has not yet been generated +}; + +} // namespace zeek::spicy diff --git a/src/spicy/spicyz/glue-compiler.cc b/src/spicy/spicyz/glue-compiler.cc new file mode 100644 index 0000000000..efbf894d89 --- /dev/null +++ b/src/spicy/spicyz/glue-compiler.cc @@ -0,0 +1,1352 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "glue-compiler.h" + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include "config.h" + +using namespace zeek::spicy; + +namespace builder = hilti::builder; + +#if SPICY_VERSION_NUMBER >= 10700 +static inline auto _linker_scope() { return hilti::builder::scope(); } +#else +static inline auto _linker_scope() { return hilti::builder::call("hilti::linker_scope", {}); } +#endif + +// Small parsing helpers. + +using ParseError = std::runtime_error; + +static void eat_spaces(const std::string& chunk, size_t* i) { + while ( *i < chunk.size() && isspace(chunk[*i]) ) + ++*i; +} + +static std::string::size_type looking_at(const std::string& chunk, std::string::size_type i, + const std::string_view& token) { + eat_spaces(chunk, &i); + + for ( char j : token ) { + if ( i >= chunk.size() || chunk[i++] != j ) + return 0; + } + + return i; +} + +static void eat_token(const std::string& chunk, std::string::size_type* i, const std::string_view& token) { + eat_spaces(chunk, i); + + auto j = looking_at(chunk, *i, token); + + if ( ! j ) + throw ParseError(hilti::util::fmt("expected token '%s'", token)); + + *i = j; +} + +static bool is_id_char(const std::string& chunk, size_t i) { + char c = chunk[i]; + + if ( isalnum(c) ) + return true; + + if ( strchr("_$%", c) != nullptr ) + return true; + + char prev = (i > 0) ? chunk[i - 1] : '\0'; + char next = (i + 1 < chunk.size()) ? chunk[i + 1] : '\0'; + + if ( c == ':' && next == ':' ) + return true; + + if ( c == ':' && prev == ':' ) + return true; + + return false; +} + +static bool is_path_char(const std::string& chunk, size_t i) { + char c = chunk[i]; + return (! isspace(c)) && c != ';'; +} + +static hilti::ID extract_id(const std::string& chunk, size_t* i) { + eat_spaces(chunk, i); + + size_t j = *i; + + while ( j < chunk.size() && is_id_char(chunk, j) ) + ++j; + + if ( *i == j ) + throw ParseError("expected id"); + + auto id = chunk.substr(*i, j - *i); + *i = j; + return hilti::ID(hilti::util::replace(id, "%", "0x25_")); +} + +static hilti::Type extract_type(const std::string& chunk, size_t* i) { + eat_spaces(chunk, i); + + // We currently only parse Spicy types that can appear in parameters of + // built-in hooks--which are not many. + auto token = extract_id(chunk, i); + + if ( token == hilti::ID("string") ) + return hilti::type::String(); + + if ( token == hilti::ID("uint64") ) + return hilti::type::UnsignedInteger(64); + + throw ParseError("mismatching type"); +} + +static hilti::type::function::Parameter extract_parameter(const std::string& chunk, size_t* i) { + eat_spaces(chunk, i); + + auto id = extract_id(chunk, i); + + if ( ! looking_at(chunk, *i, ":") ) + throw ParseError("expected ':'"); + + eat_token(chunk, i, ":"); + + auto type = extract_type(chunk, i); + return builder::parameter(std::move(id), std::move(type)); +} + +static hilti::rt::filesystem::path extract_path(const std::string& chunk, size_t* i) { + eat_spaces(chunk, i); + + size_t j = *i; + + while ( j < chunk.size() && is_path_char(chunk, j) ) + ++j; + + if ( *i == j ) + throw ParseError("expected path"); + + auto path = chunk.substr(*i, j - *i); + *i = j; + return path; +} + +static int extract_int(const std::string& chunk, size_t* i) { + eat_spaces(chunk, i); + + size_t j = *i; + + if ( j < chunk.size() ) { + if ( chunk[j] == '-' ) { + ++j; + } + if ( chunk[j] == '+' ) + ++j; + } + + while ( j < chunk.size() && isdigit(chunk[j]) ) + ++j; + + if ( *i == j ) + throw ParseError("expected integer"); + + auto x = chunk.substr(*i, j - *i); + *i = j; + + int integer = 0; + hilti::util::atoi_n(x.begin(), x.end(), 10, &integer); + return integer; +} + +static std::string extract_expr(const std::string& chunk, size_t* i) { + eat_spaces(chunk, i); + + int level = 0; + bool done = false; + size_t j = *i; + + while ( j < chunk.size() ) { + switch ( chunk[j] ) { + case '(': + case '[': + case '{': + ++level; + ++j; + continue; + + case ')': + if ( level == 0 ) { + done = true; + break; + } + + // fall-through + + case ']': + case '}': + if ( level == 0 ) + throw ParseError("expected Spicy expression"); + + --level; + ++j; + continue; + + case ',': + if ( level == 0 ) { + done = true; + break; + } + + // fall-through + + default: ++j; + } + + if ( done ) + break; + + if ( *i == j ) + break; + } + + auto expr = hilti::util::trim(chunk.substr(*i, j - *i)); + *i = j; + return expr; +} + +static hilti::rt::Port extract_port(const std::string& chunk, size_t* i) { + eat_spaces(chunk, i); + + std::string s; + size_t j = *i; + + while ( j < chunk.size() && isdigit(chunk[j]) ) + ++j; + + if ( *i == j ) + throw ParseError("cannot parse port specification"); + + hilti::rt::Protocol proto; + uint64_t port = std::numeric_limits::max(); + + s = chunk.substr(*i, j - *i); + hilti::util::atoi_n(s.begin(), s.end(), 10, &port); + + if ( port > 65535 ) + throw ParseError("port outside of valid range"); + + *i = j; + + if ( chunk[*i] != '/' ) + throw ParseError("cannot parse port specification"); + + (*i)++; + + if ( looking_at(chunk, *i, "tcp") ) { + proto = hilti::rt::Protocol::TCP; + eat_token(chunk, i, "tcp"); + } + + else if ( looking_at(chunk, *i, "udp") ) { + proto = hilti::rt::Protocol::UDP; + eat_token(chunk, i, "udp"); + } + + else if ( looking_at(chunk, *i, "icmp") ) { + proto = hilti::rt::Protocol::ICMP; + eat_token(chunk, i, "icmp"); + } + + else + throw ParseError("cannot parse port specification"); + + return {static_cast(port), proto}; +} + +static std::vector extract_ports(const std::string& chunk, size_t* i) { + auto start = extract_port(chunk, i); + auto end = std::optional(); + + if ( looking_at(chunk, *i, "-") ) { + eat_token(chunk, i, "-"); + end = extract_port(chunk, i); + } + + if ( end ) { + if ( start.protocol() != end->protocol() ) + throw ParseError("start and end of port range must have same protocol"); + + if ( start.port() > end->port() ) + throw ParseError("start of port range cannot be after its end"); + } + + std::vector result; + + // Port ranges are a closed interval. + for ( auto port = start.port(); ! end || port <= end->port(); ++port ) { + result.emplace_back(port, start.protocol()); + if ( ! end ) + break; + } + + return result; +} + +void GlueCompiler::Init(Driver* driver, int zeek_version) { + _driver = driver; + _zeek_version = zeek_version; +} + +GlueCompiler::~GlueCompiler() {} + +hilti::Result GlueCompiler::getNextEvtBlock(std::istream& in, int* lineno) const { + std::string chunk; + + // Parser need to track whether we are inside a string or a comment. + enum State { Default, InComment, InString } state = Default; + char prev = '\0'; + + while ( true ) { + char cur; + in.get(cur); + if ( in.eof() ) { + chunk = hilti::util::trim(chunk); + if ( chunk.empty() ) + // Legitimate end of data. + return std::string(); + else + // End of input before semicolon. + return hilti::result::Error("unexpected end of file"); + } + + switch ( state ) { + case Default: + if ( cur == '"' && prev != '\\' ) + state = InString; + + if ( cur == '#' && prev != '\\' ) { + state = InComment; + continue; + } + + if ( cur == '\n' ) + ++*lineno; + + if ( cur == ';' ) { + // End of block found. + chunk = hilti::util::trim(chunk); + if ( chunk.size() ) + return chunk + ';'; + else + return hilti::result::Error("empty block"); + } + + break; + + case InString: + if ( cur == '"' && prev != '\\' ) + state = Default; + + if ( cur == '\n' ) + ++*lineno; + + break; + + case InComment: + if ( cur != '\n' ) + // skip + continue; + + state = Default; + ++*lineno; + } + + chunk += cur; + prev = cur; + } +} + +void GlueCompiler::preprocessEvtFile(hilti::rt::filesystem::path& path, std::istream& in, std::ostream& out) { + hilti::util::SourceCodePreprocessor pp({{"ZEEK_VERSION", *_zeek_version}}); + int lineno = 0; + + std::string line; + while ( std::getline(in, line) ) { + lineno++; + + auto trimmed = hilti::util::trim(line); + _locations.emplace_back(path, lineno); + + if ( hilti::util::startsWith(trimmed, "@") ) { + // Output empty line to keep line numbers the same + out << '\n'; + + auto m = hilti::util::split1(trimmed); + + if ( auto rc = pp.processLine(m.first, m.second); ! rc ) + throw ParseError(rc.error()); + } + + else { + switch ( pp.state() ) { + case hilti::util::SourceCodePreprocessor::State::Include: out << line << '\n'; break; + case hilti::util::SourceCodePreprocessor::State::Skip: + // Output empty line to keep line numbers the same + out << '\n'; + break; + } + } + } + + if ( pp.expectingDirective() ) + throw ParseError("unterminated preprocessor directive"); +} + +bool GlueCompiler::loadEvtFile(hilti::rt::filesystem::path& path) { + assert(_zeek_version); + + std::ifstream in(path); + + if ( ! in ) { + hilti::logger().error(hilti::util::fmt("cannot open %s", path)); + return false; + } + + SPICY_DEBUG(hilti::util::fmt("Loading events from %s", path)); + + std::vector new_events; + + try { + std::stringstream preprocessed; + preprocessEvtFile(path, in, preprocessed); + preprocessed.clear(); + preprocessed.seekg(0); + + int lineno = 1; + + while ( true ) { + _locations.emplace_back(path, lineno); + auto chunk = getNextEvtBlock(preprocessed, &lineno); + if ( ! chunk ) + throw ParseError(chunk.error()); + + if ( chunk->empty() ) + break; // end of input + + _locations.pop_back(); + _locations.emplace_back(path, lineno); + + if ( looking_at(*chunk, 0, "protocol") ) { + auto a = parseProtocolAnalyzer(*chunk); + SPICY_DEBUG(hilti::util::fmt(" Got protocol analyzer definition for %s", a.name)); + _protocol_analyzers.push_back(a); + } + + else if ( looking_at(*chunk, 0, "file") ) { + auto a = parseFileAnalyzer(*chunk); + SPICY_DEBUG(hilti::util::fmt(" Got file analyzer definition for %s", a.name)); + _file_analyzers.push_back(a); + } + + else if ( looking_at(*chunk, 0, "packet") ) { + auto a = parsePacketAnalyzer(*chunk); + SPICY_DEBUG(hilti::util::fmt(" Got packet analyzer definition for %s", a.name)); + _packet_analyzers.push_back(a); + } + + else if ( looking_at(*chunk, 0, "on") ) { + auto ev = parseEvent(*chunk); + ev.file = path; + new_events.push_back(ev); + SPICY_DEBUG(hilti::util::fmt(" Got event definition for %s", ev.name)); + } + + else if ( looking_at(*chunk, 0, "import") ) { + size_t i = 0; + eat_token(*chunk, &i, "import"); + + hilti::ID module = extract_id(*chunk, &i); + std::optional scope; + + if ( looking_at(*chunk, i, "from") ) { + eat_token(*chunk, &i, "from"); + scope = extract_path(*chunk, &i); + SPICY_DEBUG(hilti::util::fmt(" Got module %s to import from scope %s", module, *scope)); + } + else + SPICY_DEBUG(hilti::util::fmt(" Got module %s to import", module)); + + _imports.emplace_back(hilti::ID(module), std::move(scope)); + } + + else if ( looking_at(*chunk, 0, "export") ) { + size_t i = 0; + eat_token(*chunk, &i, "export"); + + hilti::ID spicy_id = extract_id(*chunk, &i); + hilti::ID zeek_id = spicy_id; + + if ( looking_at(*chunk, i, "as") ) { + eat_token(*chunk, &i, "as"); + zeek_id = extract_id(*chunk, &i); + } + + eat_spaces(*chunk, &i); + if ( ! looking_at(*chunk, i, ";") ) + throw ParseError("syntax error in export"); + + _exports.emplace_back(std::move(spicy_id), std::move(zeek_id), _locations.back()); + } + + else + throw ParseError("expected 'import', 'export', '{file,packet,protocol} analyzer', or 'on'"); + + _locations.pop_back(); + } + } catch ( const ParseError& e ) { + if ( *e.what() ) + hilti::logger().error(e.what(), _locations.back()); + + return false; + } + + for ( auto&& ev : new_events ) + _events.push_back(ev); + + return true; +} + +void GlueCompiler::addSpicyModule(const hilti::ID& id, const hilti::rt::filesystem::path& file) { + glue::SpicyModule module; + module.id = id; + module.file = file; + _spicy_modules[id] = std::make_shared(std::move(module)); +} + +glue::ProtocolAnalyzer GlueCompiler::parseProtocolAnalyzer(const std::string& chunk) { + glue::ProtocolAnalyzer a; + a.location = _locations.back(); + + size_t i = 0; + + eat_token(chunk, &i, "protocol"); + eat_token(chunk, &i, "analyzer"); + a.name = extract_id(chunk, &i).str(); + + eat_token(chunk, &i, "over"); + + auto proto = hilti::util::tolower(extract_id(chunk, &i).str()); + + if ( proto == "tcp" ) + a.protocol = hilti::rt::Protocol::TCP; + + else if ( proto == "udp" ) + a.protocol = hilti::rt::Protocol::UDP; + + else if ( proto == "icmp" ) + a.protocol = hilti::rt::Protocol::ICMP; + + else + throw ParseError(hilti::util::fmt("unknown transport protocol '%s'", proto)); + + eat_token(chunk, &i, ":"); + + enum { orig, resp, both } dir; + + while ( true ) { + if ( looking_at(chunk, i, "parse") ) { + eat_token(chunk, &i, "parse"); + + if ( looking_at(chunk, i, "originator") ) { + eat_token(chunk, &i, "originator"); + dir = orig; + } + + else if ( looking_at(chunk, i, "responder") ) { + eat_token(chunk, &i, "responder"); + dir = resp; + } + + else if ( looking_at(chunk, i, "with") ) + dir = both; + + else + throw ParseError("invalid \"parse with ...\" specification"); + + eat_token(chunk, &i, "with"); + auto unit = extract_id(chunk, &i); + + switch ( dir ) { + case orig: a.unit_name_orig = unit; break; + + case resp: a.unit_name_resp = unit; break; + + case both: + a.unit_name_orig = unit; + a.unit_name_resp = unit; + break; + } + } + + else if ( looking_at(chunk, i, "ports") ) { + eat_token(chunk, &i, "ports"); + eat_token(chunk, &i, "{"); + + while ( true ) { + auto ports = extract_ports(chunk, &i); + a.ports.insert(a.ports.end(), ports.begin(), ports.end()); + + if ( looking_at(chunk, i, "}") ) { + eat_token(chunk, &i, "}"); + break; + } + + eat_token(chunk, &i, ","); + } + } + + else if ( looking_at(chunk, i, "port") ) { + eat_token(chunk, &i, "port"); + auto ports = extract_ports(chunk, &i); + a.ports.insert(a.ports.end(), ports.begin(), ports.end()); + } + + else if ( looking_at(chunk, i, "replaces") ) { + eat_token(chunk, &i, "replaces"); + a.replaces = extract_id(chunk, &i); + } + + else + throw ParseError("unexpected token"); + + if ( looking_at(chunk, i, ";") ) + break; // All done. + + eat_token(chunk, &i, ","); + } + + return a; +} + +glue::FileAnalyzer GlueCompiler::parseFileAnalyzer(const std::string& chunk) { + glue::FileAnalyzer a; + a.location = _locations.back(); + + size_t i = 0; + + eat_token(chunk, &i, "file"); + eat_token(chunk, &i, "analyzer"); + a.name = extract_id(chunk, &i).str(); + + eat_token(chunk, &i, ":"); + + while ( true ) { + if ( looking_at(chunk, i, "parse") ) { + eat_token(chunk, &i, "parse"); + eat_token(chunk, &i, "with"); + a.unit_name = extract_id(chunk, &i); + } + + else if ( looking_at(chunk, i, "mime-type") ) { + eat_token(chunk, &i, "mime-type"); + auto mtype = extract_path(chunk, &i); + a.mime_types.push_back(mtype.string()); + } + + else if ( looking_at(chunk, i, "replaces") ) { + eat_token(chunk, &i, "replaces"); + a.replaces = extract_id(chunk, &i); + } + + else + throw ParseError("unexpected token"); + + if ( looking_at(chunk, i, ";") ) + break; // All done. + + eat_token(chunk, &i, ","); + } + + return a; +} + +glue::PacketAnalyzer GlueCompiler::parsePacketAnalyzer(const std::string& chunk) { + glue::PacketAnalyzer a; + a.location = _locations.back(); + + size_t i = 0; + + eat_token(chunk, &i, "packet"); + eat_token(chunk, &i, "analyzer"); + a.name = extract_id(chunk, &i).str(); + + eat_token(chunk, &i, ":"); + + while ( true ) { + if ( looking_at(chunk, i, "parse") ) { + eat_token(chunk, &i, "parse"); + eat_token(chunk, &i, "with"); + a.unit_name = extract_id(chunk, &i); + } + + else if ( looking_at(chunk, i, "replaces") ) { + eat_token(chunk, &i, "replaces"); + a.replaces = extract_id(chunk, &i); + } + + else + throw ParseError("unexpected token"); + + if ( looking_at(chunk, i, ";") ) + break; // All done. + + eat_token(chunk, &i, ","); + } + + return a; +} + +glue::Event GlueCompiler::parseEvent(const std::string& chunk) { + glue::Event ev; + ev.location = _locations.back(); + + // We use a quite negative hook priority here to make sure these run last + // after anything the grammar defines by default. + ev.priority = -1000; + + size_t i = 0; + + eat_token(chunk, &i, "on"); + ev.path = extract_id(chunk, &i); + + if ( looking_at(chunk, i, "(") ) { + eat_token(chunk, &i, "("); + + if ( ! looking_at(chunk, i, ")") ) { + while ( true ) { + auto param = extract_parameter(chunk, &i); + ev.parameters.push_back(std::move(param)); + + if ( looking_at(chunk, i, ")") ) + break; + + eat_token(chunk, &i, ","); + } + } + + eat_token(chunk, &i, ")"); + } + + if ( looking_at(chunk, i, "if") ) { + eat_token(chunk, &i, "if"); + eat_token(chunk, &i, "("); + + ev.condition = extract_expr(chunk, &i); + eat_token(chunk, &i, ")"); + } + + eat_token(chunk, &i, "->"); + eat_token(chunk, &i, "event"); + ev.name = extract_id(chunk, &i); + + eat_token(chunk, &i, "("); + + bool first = true; + size_t j = 0; + + while ( true ) { + j = looking_at(chunk, i, ")"); + + if ( j ) { + i = j; + break; + } + + if ( ! first ) + eat_token(chunk, &i, ","); + + auto expr = extract_expr(chunk, &i); + ev.exprs.push_back(expr); + first = false; + } + + if ( looking_at(chunk, i, "&priority") ) { + eat_token(chunk, &i, "&priority"); + eat_token(chunk, &i, "="); + ev.priority = extract_int(chunk, &i); + } + + eat_token(chunk, &i, ";"); + eat_spaces(chunk, &i); + + if ( i < chunk.size() ) + // This shouldn't actually be possible ... + throw ParseError("unexpected characters at end of line"); + + return ev; +} + +bool GlueCompiler::compile() { + assert(_driver); + + auto init_module = hilti::Module(hilti::ID("spicy_init")); + + auto import_ = hilti::builder::import(hilti::ID("zeek_rt"), ".hlt"); + init_module.add(std::move(import_)); + + import_ = hilti::builder::import(hilti::ID("hilti"), ".hlt"); + init_module.add(std::move(import_)); + + auto preinit_body = hilti::builder::Builder(_driver->context()); + + for ( auto&& [id, m] : _spicy_modules ) + m->spicy_module = hilti::Module(hilti::ID(hilti::util::fmt("spicy_hooks_%s", id))); + + if ( ! PopulateEvents() ) + return false; + + for ( auto& a : _protocol_analyzers ) { + SPICY_DEBUG(hilti::util::fmt("Adding protocol analyzer '%s'", a.name)); + + if ( a.unit_name_orig ) { + if ( auto ui = _driver->lookupType<::spicy::type::Unit>(a.unit_name_orig) ) + a.unit_orig = *ui; + else { + hilti::logger().error(hilti::util::fmt("error with protocol analyzer %s: %s", a.name, ui.error())); + return false; + } + } + + if ( a.unit_name_resp ) { + if ( auto ui = _driver->lookupType<::spicy::type::Unit>(a.unit_name_resp) ) + a.unit_resp = *ui; + else { + hilti::logger().error(hilti::util::fmt("error with protocol analyzer %s: %s", a.name, ui.error())); + return false; + } + } + +#if SPICY_VERSION_NUMBER >= 10700 + auto proto = a.protocol.value(); +#else + auto proto = a.protocol; +#endif + + hilti::ID protocol; + switch ( proto ) { + case hilti::rt::Protocol::TCP: protocol = hilti::ID("hilti::Protocol::TCP"); break; + case hilti::rt::Protocol::UDP: protocol = hilti::ID("hilti::Protocol::UDP"); break; + default: hilti::logger().internalError("unexpected protocol"); + } + + preinit_body.addCall("zeek_rt::register_protocol_analyzer", + {builder::string(a.name), builder::id(protocol), + builder::vector(hilti::util::transform(a.ports, [](auto p) { return builder::port(p); })), + builder::string(a.unit_name_orig), builder::string(a.unit_name_resp), + builder::string(a.replaces), _linker_scope()}); + } + + for ( auto& a : _file_analyzers ) { + SPICY_DEBUG(hilti::util::fmt("Adding file analyzer '%s'", a.name)); + + if ( a.unit_name ) { + if ( auto ui = _driver->lookupType<::spicy::type::Unit>(a.unit_name) ) + a.unit = *ui; + else { + hilti::logger().error(hilti::util::fmt("error with file analyzer %s: %s", a.name, ui.error())); + return false; + } + } + + preinit_body.addCall("zeek_rt::register_file_analyzer", + {builder::string(a.name), + builder::vector( + hilti::util::transform(a.mime_types, [](auto m) { return builder::string(m); })), + builder::string(a.unit_name), builder::string(a.replaces), _linker_scope()}); + } + + for ( auto& a : _packet_analyzers ) { + SPICY_DEBUG(hilti::util::fmt("Adding packet analyzer '%s'", a.name)); + + if ( a.unit_name ) { + if ( auto ui = _driver->lookupType<::spicy::type::Unit>(a.unit_name) ) + a.unit = *ui; + else { + hilti::logger().error(hilti::util::fmt("error with packet analyzer %s: %s", a.name, ui.error())); + return false; + } + } + + preinit_body.addCall("zeek_rt::register_packet_analyzer", + {builder::string(a.name), builder::string(a.unit_name), builder::string(a.replaces), + _linker_scope()}); + } + + // Create the Spicy hooks and accessor functions. + for ( auto&& ev : _events ) { + if ( ! CreateSpicyHook(&ev) ) + return false; + } + + // Register our Zeek events at pre-init time. + for ( auto&& ev : _events ) + preinit_body.addCall("zeek_rt::install_handler", {builder::string(ev.name)}); + + // Create Zeek types for exported Spicy types. + for ( const auto& [tinfo, id] : _driver->exportedTypes() ) { + if ( auto type = createZeekType(tinfo.type, id) ) + preinit_body.addCall("zeek_rt::register_type", + {builder::string(id.namespace_()), builder::string(id.local()), *type}); + else + hilti::logger().error(hilti::util::fmt("cannot export Spicy type '%s': %s", id, type.error()), + tinfo.location); + } + + for ( auto&& [id, m] : _spicy_modules ) { + // Import runtime module. + auto import_ = hilti::builder::import(hilti::ID("zeek_rt"), ".hlt"); + m->spicy_module->add(std::move(import_)); + + // Create a vector of unique parent paths from all EVTs files going into this module. + auto search_dirs = hilti::util::transform(m->evts, [](auto p) { return p.parent_path(); }); + auto search_dirs_vec = std::vector(search_dirs.begin(), search_dirs.end()); + + // Import any dependencies. + for ( const auto& [module, scope] : _imports ) { + auto import_ = hilti::declaration::ImportedModule(module, std::string(".spicy"), scope, search_dirs_vec); + m->spicy_module->add(std::move(import_)); + } + + auto unit = hilti::Unit::fromModule(_driver->context(), *m->spicy_module, ".spicy"); + _driver->addInput(unit); + } + + if ( ! preinit_body.empty() ) { + auto preinit_function = + hilti::builder::function("zeek_preinit", hilti::type::void_, {}, preinit_body.block(), + hilti::type::function::Flavor::Standard, hilti::declaration::Linkage::PreInit); + init_module.add(std::move(preinit_function)); + } + + auto unit = hilti::Unit::fromModule(_driver->context(), init_module, ".hlt"); + _driver->addInput(unit); + return true; +} + +bool GlueCompiler::PopulateEvents() { + for ( auto& ev : _events ) { + if ( ev.unit_type ) + // Already done. + continue; + + TypeInfo uinfo; + + // If we find the path itself, it's referring to a unit type directly; + // then add a "%done" to form the hook name. + if ( auto ui = _driver->lookupType<::spicy::type::Unit>(ev.path) ) { + // TODO: Check that it's a unit type. + uinfo = *ui; + ev.unit = ev.path; + ev.hook = ev.unit + hilti::ID("0x25_done"); + } + + else { + // Strip the last element of the path, the remainder must refer + // to a unit now. + ev.unit = ev.path.namespace_(); + if ( ! ev.unit ) { + hilti::logger().error(hilti::util::fmt("unit type missing in hook '%s'", ev.path)); + return false; + } + + if ( auto ui = _driver->lookupType(ev.unit) ) { + uinfo = *ui; + ev.hook = ev.path; + } + else { + hilti::logger().error(hilti::util::fmt("unknown unit type '%s'", ev.unit)); + return false; + } + } + + ev.unit_type = std::move(uinfo.type.as<::spicy::type::Unit>()); + ev.unit_module_id = uinfo.module_id; + ev.unit_module_path = uinfo.module_path; + + if ( auto i = _spicy_modules.find(uinfo.module_id); i != _spicy_modules.end() ) { + ev.spicy_module = i->second; + i->second->evts.insert(ev.file); + } + else + hilti::logger().internalError( + hilti::util::fmt("module %s not known in Spicy module list", uinfo.module_id)); + + // Create accessor expression for event parameters. + int nr = 0; + + for ( const auto& e : ev.exprs ) { + glue::ExpressionAccessor acc; + acc.nr = ++nr; + acc.expression = e; + acc.location = ev.location; + // acc.dollar_id = util::startsWith(e, "$"); + ev.expression_accessors.push_back(acc); + } + } + + return true; +} + +#include + +#include + +// Helper visitor to wrap expressions using the the TryMember operator into a +// "deferred" expression. +class WrapTryMemberVisitor : public hilti::visitor::PostOrder { +public: + WrapTryMemberVisitor(bool catch_exception) : _catch_exception(catch_exception) {} + + void operator()(const hilti::expression::UnresolvedOperator& n, position_t p) { + if ( n.kind() == hilti::operator_::Kind::TryMember ) + p.node = hilti::expression::Deferred(hilti::Expression(n), _catch_exception); + } + +private: + bool _catch_exception; +}; + +static hilti::Result _parseArgument(const std::string& expression, bool catch_exception, + const hilti::Meta& meta) { + auto expr = spicy::parseExpression(expression, meta); + if ( ! expr ) + return hilti::result::Error(hilti::util::fmt("error parsing event argument expression '%s'", expression)); + + // If the expression uses the ".?" operator, we need to defer evaluation + // so that we can handle potential exceptions at runtime. + auto v = WrapTryMemberVisitor(catch_exception); + auto n = hilti::Node(*expr); + for ( auto i : v.walk(&n) ) + v.dispatch(i); + + return n.as(); +} + +bool GlueCompiler::CreateSpicyHook(glue::Event* ev) { + auto mangled_event_name = + hilti::util::fmt("%s_%p", hilti::util::replace(ev->name.str(), "::", "_"), std::hash()(*ev)); + auto meta = hilti::Meta(ev->location); + + // Find the Spicy module that this event belongs to. + SPICY_DEBUG(hilti::util::fmt("Adding Spicy hook '%s' for event %s", ev->hook, ev->name)); + + auto import_ = hilti::declaration::ImportedModule(ev->unit_module_id, ev->unit_module_path); + ev->spicy_module->spicy_module->add(std::move(import_)); + + // Define Zeek-side event handler. + auto handler_id = hilti::ID(hilti::util::fmt("__zeek_handler_%s", mangled_event_name)); + auto handler = builder::global(handler_id, builder::call("zeek_rt::internal_handler", {builder::string(ev->name)}), + hilti::declaration::Linkage::Private, meta); + ev->spicy_module->spicy_module->add(std::move(handler)); + + // Create the hook body that raises the event. + auto body = hilti::builder::Builder(_driver->context()); + +#if SPICY_VERSION_NUMBER >= 10800 + body.startProfiler(hilti::util::fmt("zeek/event/%s", ev->name)); +#endif + + // If the event comes with a condition, evaluate that first. + if ( ev->condition.size() ) { + auto cond = ::spicy::parseExpression(ev->condition, meta); + if ( ! cond ) { + hilti::logger().error(hilti::util::fmt("error parsing conditional expression '%s'", ev->condition)); + return false; + } + + auto exit_ = body.addIf(builder::not_(*cond), meta); + exit_->addReturn(meta); + } + + // Log event in debug code. Note: We cannot log the Zeek-side version + // (i.e., Vals with their types) because we wouldn't be able to determine + // those for events that don't have a handler (or at least a prototype) + // defined because we use the existing type definition to determine what + // Zeek type to convert an Spicy type into. However, we wouldn't want + // limit logging to events with handlers. + if ( _driver->hiltiOptions().debug ) { + std::vector fmt_args = {builder::string(ev->name)}; + + for ( const auto&& [i, e] : hilti::util::enumerate(ev->expression_accessors) ) { + if ( hilti::util::startsWith(e.expression, "$") ) { + fmt_args.emplace_back(builder::string(e.expression)); + continue; + } + + if ( auto expr = _parseArgument(e.expression, true, meta) ) + fmt_args.emplace_back(std::move(*expr)); + else + // We'll catch and report this below. + fmt_args.emplace_back(builder::string("")); + } + + std::vector fmt_ctrls(fmt_args.size() - 1, "%s"); + auto fmt_str = hilti::util::fmt("-> event %%s(%s)", hilti::util::join(fmt_ctrls, ", ")); + auto msg = builder::modulo(builder::string(fmt_str), builder::tuple(fmt_args)); + auto call = builder::call("zeek_rt::debug", {std::move(msg)}); + body.addExpression(call); + } + + auto handler_expr = builder::id(handler_id); + + if ( _driver->hiltiOptions().cxx_enable_dynamic_globals ) { + // Store reference to handler locally to avoid repeated lookups through globals store. + body.addLocal("handler", builder::id(handler_id), meta); + handler_expr = builder::id("handler"); + } + + // Nothing to do if there's not handler defined. + auto have_handler = builder::call("zeek_rt::have_handler", {handler_expr}, meta); + auto exit_ = body.addIf(builder::not_(have_handler), meta); + exit_->addReturn(meta); + + // Build event's argument vector. + body.addLocal(hilti::ID("args"), hilti::type::Vector(builder::typeByID("zeek_rt::Val"), meta), meta); + body.addMemberCall(builder::id("args"), "reserve", + {builder::integer(static_cast(ev->expression_accessors.size()))}, meta); + + int i = 0; + for ( const auto& e : ev->expression_accessors ) { + hilti::Expression val; + + if ( e.expression == "$conn" ) + val = builder::call("zeek_rt::current_conn", {}, meta); + else if ( e.expression == "$file" ) + val = builder::call("zeek_rt::current_file", {}, meta); + else if ( e.expression == "$packet" ) + val = builder::call("zeek_rt::current_packet", {}, meta); + else if ( e.expression == "$is_orig" ) + val = builder::call("zeek_rt::current_is_orig", {}, meta); + else { + if ( hilti::util::startsWith(e.expression, "$") ) { + hilti::logger().error(hilti::util::fmt("unknown reserved parameter '%s'", e.expression)); + return false; + } + + auto expr = _parseArgument(e.expression, false, meta); + if ( ! expr ) { + hilti::logger().error(expr.error()); + return false; + } + + auto ztype = builder::call("zeek_rt::event_arg_type", {handler_expr, builder::integer(i)}, meta); + val = builder::call("zeek_rt::to_val", {std::move(*expr), ztype}, meta); + } + + body.addMemberCall(builder::id("args"), "push_back", {val}, meta); + i++; + } + + body.addCall("zeek_rt::raise_event", {handler_expr, builder::move(builder::id("args"))}, meta); + + auto attrs = hilti::AttributeSet({hilti::Attribute("&priority", builder::integer(ev->priority))}); + auto unit_hook = ::spicy::Hook(ev->parameters, body.block(), ::spicy::Engine::All, {}, meta); + auto hook_decl = ::spicy::declaration::UnitHook(ev->hook, unit_hook, meta); + ev->spicy_module->spicy_module->add(hilti::Declaration(hook_decl)); + + return true; +} + +namespace { +// Visitor creasting code to instantiate a Zeek type corresponding to a give +// HILTI type. +// +// Note: Any logic changes here must be reflected in the plugin driver's +// corresponding `VisitorZeekType` as well. +struct VisitorZeekType : hilti::visitor::PreOrder, VisitorZeekType> { + VisitorZeekType(const GlueCompiler* gc) : gc(gc) {} + + const GlueCompiler* gc; + + std::set zeek_types; + std::optional id; + + result_t base_type(const char* tag) { return builder::call("zeek_rt::create_base_type", {builder::id(tag)}); } + + result_t createZeekType(const hilti::Type& t, const std::optional& id_ = {}) { + if ( id_ ) + id = id_; + else if ( auto x = t.typeID() ) + id = *x; + else + id = std::nullopt; + + if ( id ) { + // Avoid infinite recursion. + if ( zeek_types.count(*id) ) + return hilti::result::Error("type is self-recursive"); + + zeek_types.insert(*id); + } + + auto x = dispatch(t); + if ( ! x ) + return hilti::result::Error( + hilti::util::fmt("no support for automatic conversion into a Zeek type (%s)", t.typename_())); + + zeek_types.erase(*id); + return *x; + } + + result_t operator()(const hilti::type::Address& t) { return base_type("zeek_rt::ZeekTypeTag::Addr"); } + result_t operator()(const hilti::type::Bool& t) { return base_type("zeek_rt::ZeekTypeTag::Bool"); } + result_t operator()(const hilti::type::Bytes& t) { return base_type("zeek_rt::ZeekTypeTag::String"); } + result_t operator()(const hilti::type::Interval& t) { return base_type("zeek_rt::ZeekTypeTag::Interval"); } + result_t operator()(const hilti::type::Port& t) { return base_type("zeek_rt::ZeekTypeTag::Port"); } + result_t operator()(const hilti::type::Real& t) { return base_type("zeek_rt::ZeekTypeTag::Double"); } + result_t operator()(const hilti::type::SignedInteger& t) { return base_type("zeek_rt::ZeekTypeTag::Int"); } + result_t operator()(const hilti::type::String& t) { return base_type("zeek_rt::ZeekTypeTag::String"); } + result_t operator()(const hilti::type::Time& t) { return base_type("zeek_rt::ZeekTypeTag::Time"); } + result_t operator()(const hilti::type::UnsignedInteger& t) { return base_type("zeek_rt::ZeekTypeTag::Count"); } + + result_t operator()(const hilti::type::Enum& t) { + assert(id); + + auto labels = hilti::rt::transform(t.labels(), [](const auto& l) { + return builder::tuple({builder::string(l.get().id()), builder::integer(l.get().value())}); + }); + + return builder::call("zeek_rt::create_enum_type", {builder::string(id->namespace_()), + builder::string(id->local()), builder::vector(labels)}); + } + + result_t operator()(const hilti::type::Map& t) { + auto key = createZeekType(t.keyType()); + if ( ! key ) + return key.error(); + + auto value = createZeekType(t.valueType()); + if ( ! value ) + return value.error(); + + return builder::call("zeek_rt::create_table_type", {*key, *value}); + } + + result_t operator()(const hilti::type::Optional& t) { return createZeekType(t.dereferencedType()); } + + result_t operator()(const hilti::type::Set& t) { + auto elem = createZeekType(t.elementType()); + if ( ! elem ) + return elem.error(); + + return builder::call("zeek_rt::create_table_type", {*elem, builder::null()}); + } + + result_t operator()(const hilti::type::Struct& t) { + assert(id); + + std::vector fields; + for ( const auto& f : t.fields() ) { + auto ztype = createZeekType(f.type()); + if ( ! ztype ) + return ztype.error(); + + fields.emplace_back(builder::tuple({builder::string(f.id()), *ztype, builder::bool_(f.isOptional())})); + } + + return builder::call("zeek_rt::create_record_type", {builder::string(id->namespace_()), + builder::string(id->local()), builder::vector(fields)}); + } + + result_t operator()(const hilti::type::Tuple& t) { + std::vector fields; + for ( const auto& f : t.elements() ) { + if ( ! f.id() ) + return hilti::result::Error("can only convert tuple types with all-named fields to Zeek"); + + auto ztype = createZeekType(f.type()); + if ( ! ztype ) + return ztype.error(); + + fields.emplace_back(builder::tuple({builder::string(*f.id()), *ztype, builder::bool_(false)})); + } + + return builder::call("zeek_rt::create_record_type", {builder::string(id->namespace_()), + builder::string(id->local()), builder::vector(fields)}); + } + + result_t operator()(const ::spicy::type::Unit& t) { + assert(id); + + std::vector fields; + for ( const auto& f : gc->recordFields(t) ) { + auto ztype = createZeekType(std::get<1>(f)); + if ( ! ztype ) + return ztype.error(); + + fields.emplace_back( + builder::tuple({builder::string(std::get<0>(f)), *ztype, builder::bool_(std::get<2>(f))})); + } + + return builder::call("zeek_rt::create_record_type", {builder::string(id->namespace_()), + builder::string(id->local()), builder::vector(fields)}); + } + + result_t operator()(const hilti::type::Vector& t) { + auto elem = createZeekType(t.elementType()); + if ( ! elem ) + return elem.error(); + + return builder::call("zeek_rt::create_vector_type", {*elem}); + } +}; +} // namespace + +hilti::Result GlueCompiler::createZeekType(const hilti::Type& t, const hilti::ID& id) const { + return VisitorZeekType(this).createZeekType(t, id); +} + +namespace { +struct VisitorUnitFields : hilti::visitor::PreOrder { + // NOTE: Align this logic with struct generation in Spicy's unit builder. + std::vector fields; + + void operator()(const ::spicy::type::unit::item::Field& f, position_t p) { + if ( f.isTransient() || f.parseType().isA() ) + return; + + fields.emplace_back(f.id(), f.itemType(), true); + } + + void operator()(const ::spicy::type::unit::item::Variable& f, const position_t p) { + fields.emplace_back(f.id(), f.itemType(), f.isOptional()); + } + + // TODO: void operator()(const ::spicy::type::unit::item::Switch & f, const position_t p) { +}; +} // namespace + +std::vector GlueCompiler::recordFields(const ::spicy::type::Unit& unit) { + VisitorUnitFields unit_field_converter; + + for ( const auto& i : unit.items() ) + unit_field_converter.dispatch(i); + + return std::move(unit_field_converter.fields); +} diff --git a/src/spicy/spicyz/glue-compiler.h b/src/spicy/spicyz/glue-compiler.h new file mode 100644 index 0000000000..2942fa445e --- /dev/null +++ b/src/spicy/spicyz/glue-compiler.h @@ -0,0 +1,242 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "driver.h" + +namespace spicy::rt { +struct Parser; +} + +namespace zeek::spicy { + +namespace glue { + +/** Representation of a Spicy protocol analyzer, parsed from an EVT file. */ +struct ProtocolAnalyzer { + // Information parsed directly from the *.evt file. + hilti::Location location; /**< Location where the analyzer was defined. */ + hilti::ID name; /**< Name of the analyzer. */ + hilti::rt::Protocol protocol = hilti::rt::Protocol::Undef; /**< The transport layer the analyzer uses. */ + std::vector ports; /**< The ports associated with the analyzer. */ + hilti::ID unit_name_orig; /**< The fully-qualified name of the unit type to parse the originator + side. */ + hilti::ID unit_name_resp; /**< The fully-qualified name of the unit type to parse the originator + side. */ + std::string replaces; /**< Name of another analyzer this one replaces. */ + + // Computed information. + std::optional unit_orig; /**< The type of the unit to parse the originator side. */ + std::optional unit_resp; /**< The type of the unit to parse the originator side. */ +}; + +/** Representation of a Spicy file analyzer, parsed from an EVT file. */ +struct FileAnalyzer { + // Information parsed directly from the *.evt file. + hilti::Location location; /**< Location where the analyzer was defined. */ + hilti::ID name; /**< Name of the analyzer. */ + std::vector mime_types; /**< The mime_types associated with the analyzer. */ + hilti::ID unit_name; /**< The fully-qualified name of the unit type to parse with. */ + std::string replaces; /**< Name of another analyzer this one replaces. */ + + // Computed information. + std::optional unit; /**< The type of the unit to parse the originator side. */ +}; + +/** Representation of a Spicy packet analyzer, parsed from an EVT file. */ +struct PacketAnalyzer { + // Information parsed directly from the *.evt file. + hilti::Location location; /**< Location where the analyzer was defined. */ + hilti::ID name; /**< Name of the analyzer. */ + hilti::ID unit_name; /**< The fully-qualified name of the unit type to parse with. */ + std::string replaces; /**< Name of another analyzer this one replaces. */ + + // Computed information. + std::optional unit; /**< The type of the unit to parse the originator side. */ +}; + +/** + * Representation of an expression computing the value of a parameter passed + * to Spicy-generated events. + */ +struct ExpressionAccessor { + // Information parsed directly from the *.evt file. + int nr; /**< Position of this expression in argument list. */ + std::string expression; /**< The original string representation of the expression. */ + hilti::Location location; /**< Location where the expression was defined. */ +}; + +/** Representation of a compiled Spicy module. */ +struct SpicyModule { + // Provided. + hilti::ID id; /**< Name of the module */ + hilti::rt::filesystem::path file; /**< The path the module was read from. */ + std::set evts; /**< EVT files that refer to this module. */ + + // Generated code. + std::optional spicy_module; /**< the ``BroHooks_*.spicy`` module. */ +}; + +/** Representation of an event parsed from an EVT file. */ +struct Event { + // Information parsed directly from the *.evt file. + hilti::rt::filesystem::path file; /**< The path of the *.evt file we parsed this from. */ + hilti::ID name; /**< The name of the event. */ + hilti::ID path; /**< The hook path as specified in the evt file. */ + std::vector parameters; /**< Event parameters specified in the evt file. */ + std::string condition; /**< Condition that must be true for the event to trigger. */ + std::vector exprs; /**< The argument expressions. */ + int priority; /**< Event/hook priority. */ + hilti::Location location; /**< Location where event is defined. */ + + // Computed information. + hilti::ID hook; /**< The name of the hook triggering the event. */ + hilti::ID unit; /**< The fully qualified name of the unit type. */ + std::optional<::spicy::type::Unit> unit_type; /**< The Spicy type of referenced unit. */ + hilti::ID unit_module_id; /**< The name of the module the referenced unit is defined in. */ + hilti::rt::filesystem::path unit_module_path; /**< The path of the module that the referenced unit is defined in. */ + std::shared_ptr + spicy_module; /**< State for the Spichy module the referenced unit is defined in. */ + + // TODO: The following aren't set yet. + + // Code generation. + std::optional<::spicy::declaration::UnitHook> spicy_hook; /**< The generated Spicy hook. */ + std::optional hilti_raise; /**< The generated HILTI raise() function. */ + std::vector expression_accessors; /**< One HILTI function per expression to access the value. */ +}; + +} // namespace glue + +/** Generates the glue code between Zeek and Spicy based on *.evt files. */ +class GlueCompiler { +public: + /** Constructor. */ + GlueCompiler() {} + + /** Destructor. */ + virtual ~GlueCompiler(); + + /** Parses an `*.evt` file, without generating any code yet. */ + bool loadEvtFile(hilti::rt::filesystem::path& path); + + /** + * Registers a Spicy file to generate glue code for, without generating + * any code yet. + * + * @param id ID of the module + * @param file path the module is loaded from + */ + void addSpicyModule(const hilti::ID& id, const hilti::rt::filesystem::path& file); + + /** + * Generates all glue code based on previously registered `*.evt` and + * Spicy files. + */ + bool compile(); + + /** Returns all IDs that have been exported so far. */ + const auto& exportedIDs() const { return _exports; } + + /** Generates code to convert a HILTI type to a corresponding Zeek type at runtime. */ + hilti::Result createZeekType(const hilti::Type& t, const hilti::ID& id) const; + + using RecordField = std::tuple; /**< (ID, type, optional) */ + + /** + * Helper to retrieve a list of Zeek-side record fields that converting a + * Spicy unit to a Zeek record will yield. + * + * @param unit the unit type to retrieve fields for + * @return list of fields + */ + static std::vector recordFields(const ::spicy::type::Unit& unit); + +protected: + friend class Driver; + + /** Called by driver to initialized a provided glue compiler. */ + void Init(Driver* driver, int zeek_version); + +private: + /** + * Filters input EVT file by applying preprocessor directives. + */ + void preprocessEvtFile(hilti::rt::filesystem::path& path, std::istream& in, std::ostream& out); + + /** + * Extracts the next semicolon-terminated block from an input stream, + * accounting for special EVT constructs like strings and comments. + * + * @param in stream to read from + * @param lineno pointer to integer that will be increased with line breaks + * @return the read block of data, with comments removed, and empty if end of + * data has been reached; error will be set if parsing failed + */ + hilti::Result getNextEvtBlock(std::istream& in, int* lineno) const; + + // Parsers for parts from EVT files. + glue::ProtocolAnalyzer parseProtocolAnalyzer(const std::string& chunk); + glue::FileAnalyzer parseFileAnalyzer(const std::string& chunk); + glue::PacketAnalyzer parsePacketAnalyzer(const std::string& chunk); + glue::Event parseEvent(const std::string& chunk); + + /** Computes the missing pieces for all `Event` instances. */ + bool PopulateEvents(); + + /** + * Create the Spicy hook for an event that triggers a corresponding Zeek + * event. + */ + bool CreateSpicyHook(glue::Event* ev); + + Driver* _driver = nullptr; /**< driver provided to Init() */ + std::optional _zeek_version; /**< Zeek version provided to Init() */ + + std::map> _spicy_modules; + + std::vector>> + _imports; /**< imports from EVT files, with ID and optional scope */ + std::vector> _exports; /**< exports from EVT files */ + std::vector _events; /**< events parsed from EVT files */ + std::vector _protocol_analyzers; /**< protocol analyzers parsed from EVT files */ + std::vector _file_analyzers; /**< file analyzers parsed from EVT files */ + std::vector _packet_analyzers; /**< file analyzers parsed from EVT files */ + std::vector _locations; /**< location stack during parsing EVT files */ +}; +} // namespace zeek::spicy + +namespace std { +template<> +struct hash { + std::size_t operator()(const zeek::spicy::glue::Event& e) { + // We only hash enough information here to unique identify the event. + return hilti::rt::hashCombine(std::hash()(e.file), std::hash()(e.name), + std::hash()(e.path), std::hash()(e.location)); + } +}; +} // namespace std diff --git a/src/spicy/spicyz/main.cc b/src/spicy/spicyz/main.cc new file mode 100644 index 0000000000..3945dfe7d8 --- /dev/null +++ b/src/spicy/spicyz/main.cc @@ -0,0 +1,282 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include + +#include +#include + +#include "config.h" +#include "driver.h" +#include "glue-compiler.h" + +using namespace zeek::spicy; + +constexpr int OPT_CXX_LINK = 1000; + +static struct option long_driver_options[] = {{"abort-on-exceptions", required_argument, nullptr, 'A'}, + {"show-backtraces", required_argument, nullptr, 'B'}, + {"compiler-debug", required_argument, nullptr, 'D'}, + {"cxx-link", required_argument, nullptr, OPT_CXX_LINK}, + {"debug", no_argument, nullptr, 'd'}, + {"debug-addl", required_argument, nullptr, 'X'}, + {"disable-optimizations", no_argument, nullptr, 'g'}, + {"dump-code", no_argument, nullptr, 'C'}, + {"enable-profiling", no_argument, nullptr, 'Z'}, + {"help", no_argument, nullptr, 'h'}, + {"keep-tmps", no_argument, nullptr, 'T'}, + {"library-path", required_argument, nullptr, 'L'}, + {"output", required_argument, nullptr, 'o'}, + {"output-c++", required_argument, nullptr, 'c'}, + {"output-c++-files", no_argument, nullptr, 'x'}, + {"print-module-path", no_argument, nullptr, 'M'}, + {"print-plugin-path", no_argument, nullptr, + 'P'}, // for backwards compatiblity + {"print-prefix-path", no_argument, nullptr, 'p'}, + {"print-zeek-config", no_argument, nullptr, 'z'}, + {"report-times", required_argument, nullptr, 'R'}, + {"print-scripts-path", no_argument, nullptr, 'S'}, + {"skip-validation", no_argument, nullptr, '!'}, + {"version", no_argument, nullptr, 'v'}, + {"version-number", no_argument, nullptr, 'V'}, + {nullptr, 0, nullptr, 0}}; + +static void usage() { + std::cerr << "Usage: spicyz [options] \n" + "\n" + " -c | --output-c++ Output generated C++ code.\n" + " -d | --debug Include debug instrumentation into generated code.\n" + " -g | --disable-optimizations Disable HILTI-side optimizations of the generated " + "code.\n" + " -o | --output-to Path for saving output.\n" + " -v | --version Print version information.\n" + " -x | --output-c++ Output generated C++ code into set of files.\n" + " -z | --print-zeek-config Print path to zeek-config.\n" + " -A | --abort-on-exceptions When executing compiled code, abort() instead of " + "throwing HILTI " + "exceptions.\n" + " -B | --show-backtraces Include backtraces when reporting unhandled " + "exceptions.\n" + " -C | --dump-code Dump all generated code to disk for debugging.\n" + " -D | --compiler-debug Activate compile-time debugging output for given " + "debug streams " + "(comma-separated; 'help' for list).\n" + " -L | --library-path Add path to list of directories to search when " + "importing modules.\n" + " -M | --print-module-path Print the Zeek's search path for compiled Spicy modules.\n" + " -p | --print-prefix-path Print installation prefix path.\n" + " -R | --report-times Report a break-down of compiler's execution time.\n" + " -T | --keep-tmps Do not delete any temporary files created.\n" + " --skip-validation Don't validate ASTs (for debugging only).\n" + " -X | --debug-addl Implies -d and adds selected additional " + "instrumentation." + "(comma-separated; see 'help' for list).\n" + " --cxx-link Link specified static archive or shared library " + "during JIT or to " + " -Z | --enable-profiling Report profiling statistics after execution.\n" + "\n" + "Inputs can be *.spicy, *.evt, *.hlt, .cc/.cxx\n" + "\n"; +} + +using hilti::Nothing; + +static hilti::Result parseOptions(int argc, char** argv, hilti::driver::Options* driver_options, + hilti::Options* compiler_options) { + while ( true ) { + int c = getopt_long(argc, argv, "ABc:Cdgx:X:D:L:Mo:pPRSTvhzZ", long_driver_options, nullptr); + + if ( c == -1 ) + break; + + switch ( c ) { + case 'A': driver_options->abort_on_exceptions = true; break; + + case 'B': driver_options->show_backtraces = true; break; + + case 'c': + driver_options->output_cxx = true; + driver_options->output_cxx_prefix = optarg; + driver_options->execute_code = false; + compiler_options->cxx_namespace_extern = + hilti::util::fmt("hlt_%s", hilti::rt::filesystem::path(optarg).stem().string()); + compiler_options->cxx_namespace_intern = + hilti::util::fmt("__hlt_%s", hilti::rt::filesystem::path(optarg).stem().string()); + break; + + case 'C': { + driver_options->dump_code = true; + break; + } + + case 'd': { + compiler_options->debug = true; + break; + } + + case 'g': { + driver_options->global_optimizations = false; + break; + } + + case 'p': std::cout << configuration::InstallPrefix.native() << std::endl; return Nothing(); + + case 'P': + // For backwards compatibility with older plugins, print + // the path where the `cmake/` folder is located. + std::cout << configuration::DataPath().native() << std::endl; + return Nothing(); + + case 'x': + driver_options->output_cxx = true; + driver_options->output_cxx_prefix = optarg; + driver_options->execute_code = false; + driver_options->include_linker = true; + compiler_options->cxx_namespace_extern = + hilti::util::fmt("hlt_%s", hilti::rt::filesystem::path(optarg).stem().string()); + compiler_options->cxx_namespace_intern = + hilti::util::fmt("__hlt_%s", hilti::rt::filesystem::path(optarg).stem().string()); + break; + + case 'X': { + auto arg = std::string(optarg); + + if ( arg == "help" ) { + std::cerr << "Additional debug instrumentation:\n"; + std::cerr << " flow: log function calls to debug stream \"hilti-flow\"\n"; + std::cerr << " location: track current source code location for error reporting\n"; + std::cerr << " trace: log statements to debug stream \"hilti-trace\"\n"; + std::cerr << "\n"; + exit(0); + } + + compiler_options->debug = true; + + if ( auto r = compiler_options->parseDebugAddl(arg); ! r ) + return hilti::result::Error("nothing to do"); + + break; + } + + case 'D': { + auto arg = std::string(optarg); + + if ( arg == "help" ) { + std::cerr << "Debug streams:\n"; + + for ( const auto& s : hilti::logging::DebugStream::all() ) + std::cerr << " " << s << "\n"; + + std::cerr << "\n"; + return Nothing(); + } + + for ( const auto& s : hilti::util::split(arg, ",") ) { + if ( ! driver_options->logger->debugEnable(s) ) + return hilti::result::Error( + hilti::util::fmt("Unknown debug stream '%s', use 'help' for list", arg)); + } + + break; + } + + case 'L': compiler_options->library_paths.emplace_back(std::string(optarg)); break; + + case 'M': std::cout << configuration::ModulePath().native() << std::endl; return Nothing(); + + case 'o': driver_options->output_path = std::string(optarg); break; + + case 'R': driver_options->report_times = true; break; + + case 'S': std::cout << "" << std::endl; return Nothing(); // No longer needed, but left for compatibility. + + case 'T': + driver_options->keep_tmps = true; + compiler_options->keep_tmps = true; + break; + + case 'v': std::cout << configuration::ZeekVersion << std::endl; return Nothing(); + + case 'V': std::cout << ZEEK_VERSION_NUMBER << std::endl; return Nothing(); + + case 'z': { + if ( auto zcfg = getenv("ZEEK_CONFIG"); zcfg && *zcfg ) + std::cout << zcfg << std::endl; + else + std::cout << configuration::InstallBinDir().native() << std::endl; + + return Nothing(); + } + + case 'Z': +#if SPICY_VERSION_NUMBER >= 10800 + driver_options->enable_profiling = true; + compiler_options->enable_profiling = true; +#else + std::cerr << "Profiling is not supported with this version of Spicy, ignoring '-Z'\n"; +#endif + break; + + case OPT_CXX_LINK: +#if SPICY_VERSION_NUMBER >= 10600 + compiler_options->cxx_link.emplace_back(optarg); +#else + return hilti::result::Error("option '--cxx-link' is only supported for Spicy 1.6 or newer"); +#endif + break; + + case 'h': usage(); return Nothing(); + + case '!': compiler_options->skip_validation = true; break; + + default: usage(); return hilti::result::Error("could not parse options"); + } + } + + while ( optind < argc ) + driver_options->inputs.emplace_back(argv[optind++]); + + if ( driver_options->inputs.empty() ) + return hilti::result::Error("no input file given"); + + if ( driver_options->output_path.empty() && ! driver_options->output_cxx ) + return hilti::result::Error("no output file for object code given, use -o .hlto"); + + return Nothing(); +} + +int main(int argc, char** argv) { + Driver driver(std::make_unique(), "", configuration::LibraryPath(), ZEEK_VERSION_NUMBER); + + hilti::driver::Options driver_options; + driver_options.execute_code = true; + driver_options.include_linker = true; + + auto compiler_options = driver.hiltiOptions(); + + if ( auto rc = parseOptions(argc, argv, &driver_options, &compiler_options); ! rc ) { + hilti::logger().error(rc.error().description()); + return 1; + } + + driver.setDriverOptions(std::move(driver_options)); + driver.setCompilerOptions(std::move(compiler_options)); + driver.initialize(); + + for ( const auto& p : driver.driverOptions().inputs ) { + if ( auto rc = driver.loadFile(p); ! rc ) { + hilti::logger().error(rc.error().description()); + return 1; + } + } + + if ( auto rc = driver.compile(); ! rc ) { + hilti::logger().error(rc.error().description()); + + if ( rc.error().context().size() ) + hilti::logger().error(rc.error().context()); + + return 1; + } + + return 0; +} diff --git a/src/util-config.h.in b/src/util-config.h.in index 6272fba391..689ac2461e 100644 --- a/src/util-config.h.in +++ b/src/util-config.h.in @@ -1,3 +1,7 @@ #define ZEEK_SCRIPT_INSTALL_PATH "@ZEEK_SCRIPT_INSTALL_PATH@" #define BRO_PLUGIN_INSTALL_PATH "@ZEEK_PLUGIN_DIR@" +#define ZEEK_PLUGIN_INSTALL_PATH "@ZEEK_PLUGIN_DIR@" #define DEFAULT_ZEEKPATH "@DEFAULT_ZEEKPATH@" +#define ZEEK_SPICY_MODULE_PATH "@ZEEK_SPICY_MODULE_PATH@" +#define ZEEK_SPICY_LIBRARY_PATH "@ZEEK_SPICY_LIBRARY_PATH@" +#define ZEEK_SPICY_DATA_PATH "@ZEEK_SPICY_DATA_PATH@" diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index 5eb981a2a1..46d5eacef0 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -65,6 +65,9 @@ #include "zeek/plugin/Manager.h" #include "zeek/script_opt/ScriptOpt.h" #include "zeek/session/Manager.h" +#ifdef HAVE_SPICY +#include "zeek/spicy/manager.h" +#endif #include "zeek/supervisor/Supervisor.h" #include "zeek/telemetry/Manager.h" #include "zeek/threading/Manager.h" @@ -191,6 +194,9 @@ zeek::Broker::Manager* zeek::broker_mgr = nullptr; zeek::telemetry::Manager* zeek::telemetry_mgr = nullptr; zeek::Supervisor* zeek::supervisor_mgr = nullptr; zeek::detail::trigger::Manager* zeek::detail::trigger_mgr = nullptr; +#ifdef HAVE_SPICY +zeek::spicy::Manager* zeek::spicy_mgr = nullptr; +#endif std::vector zeek::detail::zeek_script_prefixes; zeek::detail::Stmt* zeek::detail::stmts = nullptr; @@ -437,6 +443,9 @@ static void terminate_zeek() delete session_mgr; delete fragment_mgr; delete telemetry_mgr; +#ifdef HAVE_SPICY + delete spicy_mgr; +#endif // free the global scope pop_scope(); @@ -719,6 +728,9 @@ SetupResult setup(int argc, char** argv, Options* zopts) auto broker_real_time = ! options.pcap_file && ! options.deterministic_mode; broker_mgr = new Broker::Manager(broker_real_time); trigger_mgr = new trigger::Manager(); +#ifdef HAVE_SPICY + spicy_mgr = new spicy::Manager(); // registers as plugin with the plugin manager +#endif plugin_mgr->InitPreScript(); file_mgr->InitPreScript(); diff --git a/testing/CMakeLists.txt b/testing/CMakeLists.txt index 0ed2fa2cf0..f3c12302e4 100644 --- a/testing/CMakeLists.txt +++ b/testing/CMakeLists.txt @@ -8,3 +8,14 @@ install(FILES btest/random.seed DESTINATION ${ZEEK_CONFIG_BTEST_TOOLS_DIR}/data) if (INSTALL_BTEST_PCAPS) install(DIRECTORY btest/Traces/ DESTINATION ${ZEEK_CONFIG_BTEST_TOOLS_DIR}/data/pcaps) endif () + +# The remainder is for backwards-compatability with existing Spicy zkg packages. +install( + CODE "execute_process( \ + COMMAND ${CMAKE_COMMAND} -E create_symlink \ + ${ZEEK_CONFIG_BTEST_TOOLS_DIR}/data \ + ${CMAKE_INSTALL_PREFIX}/share/zeek/tests \ + )") + +install(DIRECTORY scripts/spicy/ DESTINATION ${ZEEK_CONFIG_BTEST_TOOLS_DIR}/data/Scripts + USE_SOURCE_PERMISSIONS) diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 259569eef2..5540e6d45b 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -91,8 +91,8 @@ scripts/base/init-bare.zeek scripts/base/packet-protocols/gtpv1/__load__.zeek scripts/base/packet-protocols/gtpv1/main.zeek build/scripts/base/bif/plugins/Zeek_GTPv1.functions.bif.zeek + scripts/base/frameworks/spicy/init-bare.zeek build/scripts/builtin-plugins/__preload__.zeek - build/scripts/builtin-plugins/Zeek_Spicy/__preload__.zeek scripts/base/init-frameworks-and-bifs.zeek scripts/base/frameworks/logging/__load__.zeek scripts/base/frameworks/logging/main.zeek @@ -148,6 +148,7 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/bloom-filter.bif.zeek build/scripts/base/bif/cardinality-counter.bif.zeek build/scripts/base/bif/top-k.bif.zeek + build/scripts/base/bif/spicy.bif.zeek build/scripts/base/bif/plugins/__load__.zeek build/scripts/base/bif/plugins/Zeek_BitTorrent.events.bif.zeek build/scripts/base/bif/plugins/Zeek_ConnSize.events.bif.zeek @@ -264,16 +265,12 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/plugins/Zeek_AsciiWriter.ascii.bif.zeek build/scripts/base/bif/plugins/Zeek_NoneWriter.none.bif.zeek build/scripts/base/bif/plugins/Zeek_SQLiteWriter.sqlite.bif.zeek - build/scripts/base/bif/plugins/Zeek_Spicy.consts.bif.zeek - build/scripts/base/bif/plugins/Zeek_Spicy.events.bif.zeek - build/scripts/base/bif/plugins/Zeek_Spicy.functions.bif.zeek + scripts/base/frameworks/spicy/init-framework.zeek + scripts/base/misc/version.zeek + scripts/base/frameworks/reporter/__load__.zeek + scripts/base/frameworks/reporter/main.zeek + scripts/base/utils/strings.zeek build/scripts/builtin-plugins/__load__.zeek - build/scripts/builtin-plugins/Zeek_Spicy/__load__.zeek - build/scripts/builtin-plugins/Zeek_Spicy/Zeek/Spicy/bare.zeek - scripts/base/misc/version.zeek - scripts/base/frameworks/reporter/__load__.zeek - scripts/base/frameworks/reporter/main.zeek - scripts/base/utils/strings.zeek scripts/policy/misc/loaded-scripts.zeek scripts/base/utils/paths.zeek #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 47d87c5136..2c597084a7 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -91,8 +91,8 @@ scripts/base/init-bare.zeek scripts/base/packet-protocols/gtpv1/__load__.zeek scripts/base/packet-protocols/gtpv1/main.zeek build/scripts/base/bif/plugins/Zeek_GTPv1.functions.bif.zeek + scripts/base/frameworks/spicy/init-bare.zeek build/scripts/builtin-plugins/__preload__.zeek - build/scripts/builtin-plugins/Zeek_Spicy/__preload__.zeek scripts/base/init-frameworks-and-bifs.zeek scripts/base/frameworks/logging/__load__.zeek scripts/base/frameworks/logging/main.zeek @@ -148,6 +148,7 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/bloom-filter.bif.zeek build/scripts/base/bif/cardinality-counter.bif.zeek build/scripts/base/bif/top-k.bif.zeek + build/scripts/base/bif/spicy.bif.zeek build/scripts/base/bif/plugins/__load__.zeek build/scripts/base/bif/plugins/Zeek_BitTorrent.events.bif.zeek build/scripts/base/bif/plugins/Zeek_ConnSize.events.bif.zeek @@ -264,9 +265,11 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/plugins/Zeek_AsciiWriter.ascii.bif.zeek build/scripts/base/bif/plugins/Zeek_NoneWriter.none.bif.zeek build/scripts/base/bif/plugins/Zeek_SQLiteWriter.sqlite.bif.zeek - build/scripts/base/bif/plugins/Zeek_Spicy.consts.bif.zeek - build/scripts/base/bif/plugins/Zeek_Spicy.events.bif.zeek - build/scripts/base/bif/plugins/Zeek_Spicy.functions.bif.zeek + scripts/base/frameworks/spicy/init-framework.zeek + scripts/base/misc/version.zeek + scripts/base/frameworks/reporter/__load__.zeek + scripts/base/frameworks/reporter/main.zeek + scripts/base/utils/strings.zeek scripts/base/init-default.zeek scripts/base/utils/active-http.zeek scripts/base/utils/exec.zeek @@ -274,8 +277,6 @@ scripts/base/init-default.zeek scripts/base/utils/backtrace.zeek scripts/base/utils/conn-ids.zeek scripts/base/utils/dir.zeek - scripts/base/frameworks/reporter/__load__.zeek - scripts/base/frameworks/reporter/main.zeek scripts/base/utils/paths.zeek scripts/base/utils/directions-and-hosts.zeek scripts/base/utils/email.zeek @@ -283,7 +284,6 @@ scripts/base/init-default.zeek scripts/base/utils/geoip-distance.zeek scripts/base/utils/numbers.zeek scripts/base/utils/queue.zeek - scripts/base/utils/strings.zeek scripts/base/utils/thresholds.zeek scripts/base/utils/time.zeek scripts/base/utils/urls.zeek @@ -347,7 +347,8 @@ scripts/base/init-default.zeek scripts/base/frameworks/netcontrol/non-cluster.zeek scripts/base/frameworks/telemetry/__load__.zeek scripts/base/frameworks/telemetry/main.zeek - scripts/base/misc/version.zeek + scripts/base/frameworks/spicy/__load__.zeek + scripts/base/frameworks/spicy/main.zeek scripts/base/protocols/conn/__load__.zeek scripts/base/protocols/conn/main.zeek scripts/base/protocols/conn/contents.zeek @@ -463,8 +464,5 @@ scripts/base/init-default.zeek scripts/base/misc/find-filtered-trace.zeek build/scripts/base/misc/installation.zeek build/scripts/builtin-plugins/__load__.zeek - build/scripts/builtin-plugins/Zeek_Spicy/__load__.zeek - build/scripts/builtin-plugins/Zeek_Spicy/Zeek/Spicy/bare.zeek - build/scripts/builtin-plugins/Zeek_Spicy/Zeek/Spicy/default.zeek scripts/policy/misc/loaded-scripts.zeek #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/coverage.init-default/missing_loads b/testing/btest/Baseline/coverage.init-default/missing_loads index 33a5c60cfb..6bf8ca440c 100644 --- a/testing/btest/Baseline/coverage.init-default/missing_loads +++ b/testing/btest/Baseline/coverage.init-default/missing_loads @@ -10,6 +10,8 @@ -./frameworks/netcontrol/cluster.zeek -./frameworks/openflow/cluster.zeek -./frameworks/packet-filter/cluster.zeek +-./frameworks/spicy/misc/record-spicy-batch.zeek +-./frameworks/spicy/misc/resource-usage.zeek -./frameworks/sumstats/cluster.zeek -./frameworks/telemetry/cluster.zeek -./init-supervisor.zeek diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 8ab256052c..c1321438b9 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -790,7 +790,6 @@ 0.000000 MetaHookPost CallFunction(Version::parse, ..., ...) -> 0.000000 MetaHookPost CallFunction(__init_primary_bifs, , ()) -> 0.000000 MetaHookPost CallFunction(__init_secondary_bifs, , ()) -> -0.000000 MetaHookPost CallFunction(bare_mode, , ()) -> 0.000000 MetaHookPost CallFunction(cat, ..., ...) -> 0.000000 MetaHookPost CallFunction(current_time, , ()) -> 0.000000 MetaHookPost CallFunction(disable_event_group, , (Analyzer::Logging::include_confirmations)) -> @@ -938,9 +937,6 @@ 0.000000 MetaHookPost LoadFile(0, ./Zeek_SSL.events.bif.zeek, <...>/Zeek_SSL.events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_SSL.functions.bif.zeek, <...>/Zeek_SSL.functions.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_SSL.types.bif.zeek, <...>/Zeek_SSL.types.bif.zeek) -> -1 -0.000000 MetaHookPost LoadFile(0, ./Zeek_Spicy.consts.bif.zeek, <...>/Zeek_Spicy.consts.bif.zeek) -> -1 -0.000000 MetaHookPost LoadFile(0, ./Zeek_Spicy.events.bif.zeek, <...>/Zeek_Spicy.events.bif.zeek) -> -1 -0.000000 MetaHookPost LoadFile(0, ./Zeek_Spicy.functions.bif.zeek, <...>/Zeek_Spicy.functions.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_TCP.events.bif.zeek, <...>/Zeek_TCP.events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_TCP.functions.bif.zeek, <...>/Zeek_TCP.functions.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_TCP.types.bif.zeek, <...>/Zeek_TCP.types.bif.zeek) -> -1 @@ -1024,6 +1020,7 @@ 0.000000 MetaHookPost LoadFile(0, ./smb1-main, <...>/smb1-main.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./smb2-main, <...>/smb2-main.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./spicy-events, <...>/spicy-events.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, ./spicy.bif.zeek, <...>/spicy.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./stats.bif.zeek, <...>/stats.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./std-dev, <...>/std-dev.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./store, <...>/store.zeek) -> -1 @@ -1058,10 +1055,6 @@ 0.000000 MetaHookPost LoadFile(0, <...>/__load__.zeek, <...>/__load__.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, <...>/__preload__.zeek, <...>/__preload__.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, <...>/hooks.zeek, <...>/hooks.zeek) -> -1 -0.000000 MetaHookPost LoadFile(0, Zeek<...>/bare.zeek, <...>/bare.zeek) -> -1 -0.000000 MetaHookPost LoadFile(0, Zeek<...>/default.zeek, <...>/default.zeek) -> -1 -0.000000 MetaHookPost LoadFile(0, Zeek_Spicy/__load__.zeek, <...>/__load__.zeek) -> -1 -0.000000 MetaHookPost LoadFile(0, Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base/bif, <...>/bif) -> -1 0.000000 MetaHookPost LoadFile(0, base/init-default.zeek, <...>/init-default.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base/init-frameworks-and-bifs.zeek, <...>/init-frameworks-and-bifs.zeek) -> -1 @@ -1119,6 +1112,8 @@ 0.000000 MetaHookPost LoadFile(0, base<...>/ieee802_11, <...>/ieee802_11) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/ieee802_11_radio, <...>/ieee802_11_radio) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/imap, <...>/imap) -> -1 +0.000000 MetaHookPost LoadFile(0, base<...>/init-bare, <...>/init-bare.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, base<...>/init-framework, <...>/init-framework.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/input, <...>/input) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/input.bif, <...>/input.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/installation, <...>/installation.zeek) -> -1 @@ -1177,6 +1172,7 @@ 0.000000 MetaHookPost LoadFile(0, base<...>/snmp, <...>/snmp) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/socks, <...>/socks) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/software, <...>/software) -> -1 +0.000000 MetaHookPost LoadFile(0, base<...>/spicy, <...>/spicy) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/ssh, <...>/ssh) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/ssl, <...>/ssl) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/stats.bif, <...>/stats.bif.zeek) -> -1 @@ -1331,9 +1327,6 @@ 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SSL.events.bif.zeek, <...>/Zeek_SSL.events.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SSL.functions.bif.zeek, <...>/Zeek_SSL.functions.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SSL.types.bif.zeek, <...>/Zeek_SSL.types.bif.zeek) -> (-1, ) -0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Spicy.consts.bif.zeek, <...>/Zeek_Spicy.consts.bif.zeek) -> (-1, ) -0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Spicy.events.bif.zeek, <...>/Zeek_Spicy.events.bif.zeek) -> (-1, ) -0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Spicy.functions.bif.zeek, <...>/Zeek_Spicy.functions.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_TCP.events.bif.zeek, <...>/Zeek_TCP.events.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_TCP.functions.bif.zeek, <...>/Zeek_TCP.functions.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_TCP.types.bif.zeek, <...>/Zeek_TCP.types.bif.zeek) -> (-1, ) @@ -1417,6 +1410,7 @@ 0.000000 MetaHookPost LoadFileExtended(0, ./smb1-main, <...>/smb1-main.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./smb2-main, <...>/smb2-main.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./spicy-events, <...>/spicy-events.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./spicy.bif.zeek, <...>/spicy.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./stats.bif.zeek, <...>/stats.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./std-dev, <...>/std-dev.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./store, <...>/store.zeek) -> (-1, ) @@ -1451,10 +1445,6 @@ 0.000000 MetaHookPost LoadFileExtended(0, <...>/__load__.zeek, <...>/__load__.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, <...>/__preload__.zeek, <...>/__preload__.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, <...>/hooks.zeek, <...>/hooks.zeek) -> (-1, ) -0.000000 MetaHookPost LoadFileExtended(0, Zeek<...>/bare.zeek, <...>/bare.zeek) -> (-1, ) -0.000000 MetaHookPost LoadFileExtended(0, Zeek<...>/default.zeek, <...>/default.zeek) -> (-1, ) -0.000000 MetaHookPost LoadFileExtended(0, Zeek_Spicy/__load__.zeek, <...>/__load__.zeek) -> (-1, ) -0.000000 MetaHookPost LoadFileExtended(0, Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base/bif, <...>/bif) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base/init-default.zeek, <...>/init-default.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base/init-frameworks-and-bifs.zeek, <...>/init-frameworks-and-bifs.zeek) -> (-1, ) @@ -1512,6 +1502,8 @@ 0.000000 MetaHookPost LoadFileExtended(0, base<...>/ieee802_11, <...>/ieee802_11) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/ieee802_11_radio, <...>/ieee802_11_radio) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/imap, <...>/imap) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/init-bare, <...>/init-bare.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/init-framework, <...>/init-framework.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/input, <...>/input) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/input.bif, <...>/input.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/installation, <...>/installation.zeek) -> (-1, ) @@ -1570,6 +1562,7 @@ 0.000000 MetaHookPost LoadFileExtended(0, base<...>/snmp, <...>/snmp) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/socks, <...>/socks) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/software, <...>/software) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/spicy, <...>/spicy) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/ssh, <...>/ssh) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/ssl, <...>/ssl) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/stats.bif, <...>/stats.bif.zeek) -> (-1, ) @@ -2410,7 +2403,6 @@ 0.000000 MetaHookPre CallFunction(Version::parse, ..., ...) 0.000000 MetaHookPre CallFunction(__init_primary_bifs, , ()) 0.000000 MetaHookPre CallFunction(__init_secondary_bifs, , ()) -0.000000 MetaHookPre CallFunction(bare_mode, , ()) 0.000000 MetaHookPre CallFunction(cat, ..., ...) 0.000000 MetaHookPre CallFunction(current_time, , ()) 0.000000 MetaHookPre CallFunction(disable_event_group, , (Analyzer::Logging::include_confirmations)) @@ -2558,9 +2550,6 @@ 0.000000 MetaHookPre LoadFile(0, ./Zeek_SSL.events.bif.zeek, <...>/Zeek_SSL.events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_SSL.functions.bif.zeek, <...>/Zeek_SSL.functions.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_SSL.types.bif.zeek, <...>/Zeek_SSL.types.bif.zeek) -0.000000 MetaHookPre LoadFile(0, ./Zeek_Spicy.consts.bif.zeek, <...>/Zeek_Spicy.consts.bif.zeek) -0.000000 MetaHookPre LoadFile(0, ./Zeek_Spicy.events.bif.zeek, <...>/Zeek_Spicy.events.bif.zeek) -0.000000 MetaHookPre LoadFile(0, ./Zeek_Spicy.functions.bif.zeek, <...>/Zeek_Spicy.functions.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_TCP.events.bif.zeek, <...>/Zeek_TCP.events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_TCP.functions.bif.zeek, <...>/Zeek_TCP.functions.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_TCP.types.bif.zeek, <...>/Zeek_TCP.types.bif.zeek) @@ -2644,6 +2633,7 @@ 0.000000 MetaHookPre LoadFile(0, ./smb1-main, <...>/smb1-main.zeek) 0.000000 MetaHookPre LoadFile(0, ./smb2-main, <...>/smb2-main.zeek) 0.000000 MetaHookPre LoadFile(0, ./spicy-events, <...>/spicy-events.zeek) +0.000000 MetaHookPre LoadFile(0, ./spicy.bif.zeek, <...>/spicy.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./stats.bif.zeek, <...>/stats.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./std-dev, <...>/std-dev.zeek) 0.000000 MetaHookPre LoadFile(0, ./store, <...>/store.zeek) @@ -2678,10 +2668,6 @@ 0.000000 MetaHookPre LoadFile(0, <...>/__load__.zeek, <...>/__load__.zeek) 0.000000 MetaHookPre LoadFile(0, <...>/__preload__.zeek, <...>/__preload__.zeek) 0.000000 MetaHookPre LoadFile(0, <...>/hooks.zeek, <...>/hooks.zeek) -0.000000 MetaHookPre LoadFile(0, Zeek<...>/bare.zeek, <...>/bare.zeek) -0.000000 MetaHookPre LoadFile(0, Zeek<...>/default.zeek, <...>/default.zeek) -0.000000 MetaHookPre LoadFile(0, Zeek_Spicy/__load__.zeek, <...>/__load__.zeek) -0.000000 MetaHookPre LoadFile(0, Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek) 0.000000 MetaHookPre LoadFile(0, base/bif, <...>/bif) 0.000000 MetaHookPre LoadFile(0, base/init-default.zeek, <...>/init-default.zeek) 0.000000 MetaHookPre LoadFile(0, base/init-frameworks-and-bifs.zeek, <...>/init-frameworks-and-bifs.zeek) @@ -2739,6 +2725,8 @@ 0.000000 MetaHookPre LoadFile(0, base<...>/ieee802_11, <...>/ieee802_11) 0.000000 MetaHookPre LoadFile(0, base<...>/ieee802_11_radio, <...>/ieee802_11_radio) 0.000000 MetaHookPre LoadFile(0, base<...>/imap, <...>/imap) +0.000000 MetaHookPre LoadFile(0, base<...>/init-bare, <...>/init-bare.zeek) +0.000000 MetaHookPre LoadFile(0, base<...>/init-framework, <...>/init-framework.zeek) 0.000000 MetaHookPre LoadFile(0, base<...>/input, <...>/input) 0.000000 MetaHookPre LoadFile(0, base<...>/input.bif, <...>/input.bif.zeek) 0.000000 MetaHookPre LoadFile(0, base<...>/installation, <...>/installation.zeek) @@ -2797,6 +2785,7 @@ 0.000000 MetaHookPre LoadFile(0, base<...>/snmp, <...>/snmp) 0.000000 MetaHookPre LoadFile(0, base<...>/socks, <...>/socks) 0.000000 MetaHookPre LoadFile(0, base<...>/software, <...>/software) +0.000000 MetaHookPre LoadFile(0, base<...>/spicy, <...>/spicy) 0.000000 MetaHookPre LoadFile(0, base<...>/ssh, <...>/ssh) 0.000000 MetaHookPre LoadFile(0, base<...>/ssl, <...>/ssl) 0.000000 MetaHookPre LoadFile(0, base<...>/stats.bif, <...>/stats.bif.zeek) @@ -2951,9 +2940,6 @@ 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SSL.events.bif.zeek, <...>/Zeek_SSL.events.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SSL.functions.bif.zeek, <...>/Zeek_SSL.functions.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SSL.types.bif.zeek, <...>/Zeek_SSL.types.bif.zeek) -0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Spicy.consts.bif.zeek, <...>/Zeek_Spicy.consts.bif.zeek) -0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Spicy.events.bif.zeek, <...>/Zeek_Spicy.events.bif.zeek) -0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Spicy.functions.bif.zeek, <...>/Zeek_Spicy.functions.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_TCP.events.bif.zeek, <...>/Zeek_TCP.events.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_TCP.functions.bif.zeek, <...>/Zeek_TCP.functions.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_TCP.types.bif.zeek, <...>/Zeek_TCP.types.bif.zeek) @@ -3037,6 +3023,7 @@ 0.000000 MetaHookPre LoadFileExtended(0, ./smb1-main, <...>/smb1-main.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./smb2-main, <...>/smb2-main.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./spicy-events, <...>/spicy-events.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./spicy.bif.zeek, <...>/spicy.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./stats.bif.zeek, <...>/stats.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./std-dev, <...>/std-dev.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./store, <...>/store.zeek) @@ -3071,10 +3058,6 @@ 0.000000 MetaHookPre LoadFileExtended(0, <...>/__load__.zeek, <...>/__load__.zeek) 0.000000 MetaHookPre LoadFileExtended(0, <...>/__preload__.zeek, <...>/__preload__.zeek) 0.000000 MetaHookPre LoadFileExtended(0, <...>/hooks.zeek, <...>/hooks.zeek) -0.000000 MetaHookPre LoadFileExtended(0, Zeek<...>/bare.zeek, <...>/bare.zeek) -0.000000 MetaHookPre LoadFileExtended(0, Zeek<...>/default.zeek, <...>/default.zeek) -0.000000 MetaHookPre LoadFileExtended(0, Zeek_Spicy/__load__.zeek, <...>/__load__.zeek) -0.000000 MetaHookPre LoadFileExtended(0, Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek) 0.000000 MetaHookPre LoadFileExtended(0, base/bif, <...>/bif) 0.000000 MetaHookPre LoadFileExtended(0, base/init-default.zeek, <...>/init-default.zeek) 0.000000 MetaHookPre LoadFileExtended(0, base/init-frameworks-and-bifs.zeek, <...>/init-frameworks-and-bifs.zeek) @@ -3132,6 +3115,8 @@ 0.000000 MetaHookPre LoadFileExtended(0, base<...>/ieee802_11, <...>/ieee802_11) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/ieee802_11_radio, <...>/ieee802_11_radio) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/imap, <...>/imap) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/init-bare, <...>/init-bare.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/init-framework, <...>/init-framework.zeek) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/input, <...>/input) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/input.bif, <...>/input.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/installation, <...>/installation.zeek) @@ -3190,6 +3175,7 @@ 0.000000 MetaHookPre LoadFileExtended(0, base<...>/snmp, <...>/snmp) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/socks, <...>/socks) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/software, <...>/software) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/spicy, <...>/spicy) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/ssh, <...>/ssh) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/ssl, <...>/ssl) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/stats.bif, <...>/stats.bif.zeek) @@ -4029,7 +4015,6 @@ 0.000000 | HookCallFunction Version::parse(...) 0.000000 | HookCallFunction __init_primary_bifs() 0.000000 | HookCallFunction __init_secondary_bifs() -0.000000 | HookCallFunction bare_mode() 0.000000 | HookCallFunction cat(...) 0.000000 | HookCallFunction current_time() 0.000000 | HookCallFunction disable_event_group(Analyzer::Logging::include_confirmations) @@ -4177,9 +4162,6 @@ 0.000000 | HookLoadFile ./Zeek_SSL.events.bif.zeek <...>/Zeek_SSL.events.bif.zeek 0.000000 | HookLoadFile ./Zeek_SSL.functions.bif.zeek <...>/Zeek_SSL.functions.bif.zeek 0.000000 | HookLoadFile ./Zeek_SSL.types.bif.zeek <...>/Zeek_SSL.types.bif.zeek -0.000000 | HookLoadFile ./Zeek_Spicy.consts.bif.zeek <...>/Zeek_Spicy.consts.bif.zeek -0.000000 | HookLoadFile ./Zeek_Spicy.events.bif.zeek <...>/Zeek_Spicy.events.bif.zeek -0.000000 | HookLoadFile ./Zeek_Spicy.functions.bif.zeek <...>/Zeek_Spicy.functions.bif.zeek 0.000000 | HookLoadFile ./Zeek_TCP.events.bif.zeek <...>/Zeek_TCP.events.bif.zeek 0.000000 | HookLoadFile ./Zeek_TCP.functions.bif.zeek <...>/Zeek_TCP.functions.bif.zeek 0.000000 | HookLoadFile ./Zeek_TCP.types.bif.zeek <...>/Zeek_TCP.types.bif.zeek @@ -4274,6 +4256,7 @@ 0.000000 | HookLoadFile ./smb1-main <...>/smb1-main.zeek 0.000000 | HookLoadFile ./smb2-main <...>/smb2-main.zeek 0.000000 | HookLoadFile ./spicy-events <...>/spicy-events.zeek +0.000000 | HookLoadFile ./spicy.bif.zeek <...>/spicy.bif.zeek 0.000000 | HookLoadFile ./stats.bif.zeek <...>/stats.bif.zeek 0.000000 | HookLoadFile ./std-dev <...>/std-dev.zeek 0.000000 | HookLoadFile ./store <...>/store.zeek @@ -4309,10 +4292,6 @@ 0.000000 | HookLoadFile <...>/__load__.zeek <...>/__load__.zeek 0.000000 | HookLoadFile <...>/__preload__.zeek <...>/__preload__.zeek 0.000000 | HookLoadFile <...>/hooks.zeek <...>/hooks.zeek -0.000000 | HookLoadFile Zeek<...>/bare.zeek <...>/bare.zeek -0.000000 | HookLoadFile Zeek<...>/default.zeek <...>/default.zeek -0.000000 | HookLoadFile Zeek_Spicy/__load__.zeek <...>/__load__.zeek -0.000000 | HookLoadFile Zeek_Spicy/__preload__.zeek <...>/__preload__.zeek 0.000000 | HookLoadFile base/bif <...>/bif 0.000000 | HookLoadFile base/init-default.zeek <...>/init-default.zeek 0.000000 | HookLoadFile base/init-frameworks-and-bifs.zeek <...>/init-frameworks-and-bifs.zeek @@ -4370,6 +4349,8 @@ 0.000000 | HookLoadFile base<...>/ieee802_11 <...>/ieee802_11 0.000000 | HookLoadFile base<...>/ieee802_11_radio <...>/ieee802_11_radio 0.000000 | HookLoadFile base<...>/imap <...>/imap +0.000000 | HookLoadFile base<...>/init-bare <...>/init-bare.zeek +0.000000 | HookLoadFile base<...>/init-framework <...>/init-framework.zeek 0.000000 | HookLoadFile base<...>/input <...>/input 0.000000 | HookLoadFile base<...>/input.bif <...>/input.bif.zeek 0.000000 | HookLoadFile base<...>/installation <...>/installation.zeek @@ -4428,6 +4409,7 @@ 0.000000 | HookLoadFile base<...>/snmp <...>/snmp 0.000000 | HookLoadFile base<...>/socks <...>/socks 0.000000 | HookLoadFile base<...>/software <...>/software +0.000000 | HookLoadFile base<...>/spicy <...>/spicy 0.000000 | HookLoadFile base<...>/ssh <...>/ssh 0.000000 | HookLoadFile base<...>/ssl <...>/ssl 0.000000 | HookLoadFile base<...>/stats.bif <...>/stats.bif.zeek @@ -4570,9 +4552,6 @@ 0.000000 | HookLoadFileExtended ./Zeek_SSL.events.bif.zeek <...>/Zeek_SSL.events.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_SSL.functions.bif.zeek <...>/Zeek_SSL.functions.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_SSL.types.bif.zeek <...>/Zeek_SSL.types.bif.zeek -0.000000 | HookLoadFileExtended ./Zeek_Spicy.consts.bif.zeek <...>/Zeek_Spicy.consts.bif.zeek -0.000000 | HookLoadFileExtended ./Zeek_Spicy.events.bif.zeek <...>/Zeek_Spicy.events.bif.zeek -0.000000 | HookLoadFileExtended ./Zeek_Spicy.functions.bif.zeek <...>/Zeek_Spicy.functions.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_TCP.events.bif.zeek <...>/Zeek_TCP.events.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_TCP.functions.bif.zeek <...>/Zeek_TCP.functions.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_TCP.types.bif.zeek <...>/Zeek_TCP.types.bif.zeek @@ -4667,6 +4646,7 @@ 0.000000 | HookLoadFileExtended ./smb1-main <...>/smb1-main.zeek 0.000000 | HookLoadFileExtended ./smb2-main <...>/smb2-main.zeek 0.000000 | HookLoadFileExtended ./spicy-events <...>/spicy-events.zeek +0.000000 | HookLoadFileExtended ./spicy.bif.zeek <...>/spicy.bif.zeek 0.000000 | HookLoadFileExtended ./stats.bif.zeek <...>/stats.bif.zeek 0.000000 | HookLoadFileExtended ./std-dev <...>/std-dev.zeek 0.000000 | HookLoadFileExtended ./store <...>/store.zeek @@ -4702,10 +4682,6 @@ 0.000000 | HookLoadFileExtended <...>/__load__.zeek <...>/__load__.zeek 0.000000 | HookLoadFileExtended <...>/__preload__.zeek <...>/__preload__.zeek 0.000000 | HookLoadFileExtended <...>/hooks.zeek <...>/hooks.zeek -0.000000 | HookLoadFileExtended Zeek<...>/bare.zeek <...>/bare.zeek -0.000000 | HookLoadFileExtended Zeek<...>/default.zeek <...>/default.zeek -0.000000 | HookLoadFileExtended Zeek_Spicy/__load__.zeek <...>/__load__.zeek -0.000000 | HookLoadFileExtended Zeek_Spicy/__preload__.zeek <...>/__preload__.zeek 0.000000 | HookLoadFileExtended base/bif <...>/bif 0.000000 | HookLoadFileExtended base/init-default.zeek <...>/init-default.zeek 0.000000 | HookLoadFileExtended base/init-frameworks-and-bifs.zeek <...>/init-frameworks-and-bifs.zeek @@ -4763,6 +4739,8 @@ 0.000000 | HookLoadFileExtended base<...>/ieee802_11 <...>/ieee802_11 0.000000 | HookLoadFileExtended base<...>/ieee802_11_radio <...>/ieee802_11_radio 0.000000 | HookLoadFileExtended base<...>/imap <...>/imap +0.000000 | HookLoadFileExtended base<...>/init-bare <...>/init-bare.zeek +0.000000 | HookLoadFileExtended base<...>/init-framework <...>/init-framework.zeek 0.000000 | HookLoadFileExtended base<...>/input <...>/input 0.000000 | HookLoadFileExtended base<...>/input.bif <...>/input.bif.zeek 0.000000 | HookLoadFileExtended base<...>/installation <...>/installation.zeek @@ -4821,6 +4799,7 @@ 0.000000 | HookLoadFileExtended base<...>/snmp <...>/snmp 0.000000 | HookLoadFileExtended base<...>/socks <...>/socks 0.000000 | HookLoadFileExtended base<...>/software <...>/software +0.000000 | HookLoadFileExtended base<...>/spicy <...>/spicy 0.000000 | HookLoadFileExtended base<...>/ssh <...>/ssh 0.000000 | HookLoadFileExtended base<...>/ssl <...>/ssl 0.000000 | HookLoadFileExtended base<...>/stats.bif <...>/stats.bif.zeek diff --git a/testing/btest/Baseline/spicy.analyzer-tag/output b/testing/btest/Baseline/spicy.analyzer-tag/output new file mode 100644 index 0000000000..947980697b --- /dev/null +++ b/testing/btest/Baseline/spicy.analyzer-tag/output @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Have analyzer! +tag: Analyzer::ANALYZER_SPICY_SSH +name: SPICY_SSH + +Do not have analyzer! diff --git a/testing/btest/Baseline/spicy.conn-id/output b/testing/btest/Baseline/spicy.conn-id/output new file mode 100644 index 0000000000..bd3e565a86 --- /dev/null +++ b/testing/btest/Baseline/spicy.conn-id/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +(192.150.186.169, 49244/tcp, 131.159.14.23, 22/tcp) +(2001:470:1f11:81f:c999:d94:aa7c:2e3e, 49185/tcp, 2001:470:4867:99::21, 21/tcp) diff --git a/testing/btest/Baseline/spicy.context/output b/testing/btest/Baseline/spicy.context/output new file mode 100644 index 0000000000..a4cf9692d1 --- /dev/null +++ b/testing/btest/Baseline/spicy.context/output @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +(b"2.0", b"1.99") diff --git a/testing/btest/Baseline/spicy.dns/output b/testing/btest/Baseline/spicy.dns/output new file mode 100644 index 0000000000..d53b87e459 --- /dev/null +++ b/testing/btest/Baseline/spicy.dns/output @@ -0,0 +1,3 @@ +[orig_h=192.150.186.169, orig_p=55587/udp, resp_h=192.150.186.8, resp_p=53/udp], [id=29622, opcode=0, rcode=0, QR=F, AA=F, TC=F, RD=T, RA=F, Z=0, num_queries=1, num_answers=0, num_auth=0, num_addl=0], www.heise.de, 1, 1 +[orig_h=192.150.186.169, orig_p=55588/udp, resp_h=192.150.186.8, resp_p=53/udp], [id=15429, opcode=0, rcode=0, QR=F, AA=F, TC=F, RD=T, RA=F, Z=0, num_queries=1, num_answers=0, num_auth=0, num_addl=0], www.google.com, 1, 1 +[orig_h=192.150.186.169, orig_p=55589/udp, resp_h=192.150.186.8, resp_p=53/udp], [id=27360, opcode=0, rcode=0, QR=F, AA=F, TC=F, RD=T, RA=F, Z=0, num_queries=1, num_answers=0, num_auth=0, num_addl=0], www.net.in.tum.de, 1, 1 diff --git a/testing/btest/Baseline/spicy.double-event/output b/testing/btest/Baseline/spicy.double-event/output new file mode 100644 index 0000000000..5468e641f2 --- /dev/null +++ b/testing/btest/Baseline/spicy.double-event/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +1, OpenSSH_3.8.1p1 +1, OpenSSH_3.9p1 +2, OpenSSH_3.8.1p1 +2, OpenSSH_3.9p1 diff --git a/testing/btest/Baseline/spicy.double-types/output b/testing/btest/Baseline/spicy.double-types/output new file mode 100644 index 0000000000..319e263d92 --- /dev/null +++ b/testing/btest/Baseline/spicy.double-types/output @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +dtest_message, dtest::FUNCS_YES +dtest_result, dtest::RESULT_YES_AGAIN +dtest_result_tuple, dtest::FUNCS_YES, dtest::RESULT_YES_AGAIN diff --git a/testing/btest/Baseline/spicy.event-args-fail-2/output b/testing/btest/Baseline/spicy.event-args-fail-2/output new file mode 100644 index 0000000000..bc372c9ae4 --- /dev/null +++ b/testing/btest/Baseline/spicy.event-args-fail-2/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] <...>/event-args-fail.evt:2: signature for hook must be: %error or %error(err: string) +[error] : aborting after errors diff --git a/testing/btest/Baseline/spicy.event-args-fail-3/output b/testing/btest/Baseline/spicy.event-args-fail-3/output new file mode 100644 index 0000000000..bc372c9ae4 --- /dev/null +++ b/testing/btest/Baseline/spicy.event-args-fail-3/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] <...>/event-args-fail.evt:2: signature for hook must be: %error or %error(err: string) +[error] : aborting after errors diff --git a/testing/btest/Baseline/spicy.event-args-fail-4/output b/testing/btest/Baseline/spicy.event-args-fail-4/output new file mode 100644 index 0000000000..24978f354d --- /dev/null +++ b/testing/btest/Baseline/spicy.event-args-fail-4/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] <...>/event-args-fail.evt:2: expected id +[error] error loading EVT file "<...>/event-args-fail.evt" diff --git a/testing/btest/Baseline/spicy.event-args-fail-5/output b/testing/btest/Baseline/spicy.event-args-fail-5/output new file mode 100644 index 0000000000..24978f354d --- /dev/null +++ b/testing/btest/Baseline/spicy.event-args-fail-5/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] <...>/event-args-fail.evt:2: expected id +[error] error loading EVT file "<...>/event-args-fail.evt" diff --git a/testing/btest/Baseline/spicy.event-args-fail-6/output b/testing/btest/Baseline/spicy.event-args-fail-6/output new file mode 100644 index 0000000000..4aa8c30fd9 --- /dev/null +++ b/testing/btest/Baseline/spicy.event-args-fail-6/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] <...>/event-args-fail.evt:2: expected ':' +[error] error loading EVT file "<...>/event-args-fail.evt" diff --git a/testing/btest/Baseline/spicy.event-args-fail/output b/testing/btest/Baseline/spicy.event-args-fail/output new file mode 100644 index 0000000000..8a0a39ce64 --- /dev/null +++ b/testing/btest/Baseline/spicy.event-args-fail/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] <...>/event-args-fail.evt:10: mismatching type +[error] error loading EVT file "<...>/event-args-fail.evt" diff --git a/testing/btest/Baseline/spicy.event-args-mismatch/output b/testing/btest/Baseline/spicy.event-args-mismatch/output new file mode 100644 index 0000000000..3f6f6338db --- /dev/null +++ b/testing/btest/Baseline/spicy.event-args-mismatch/output @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +XXXXXXXXXX.XXXXXX analyzer error in <...>/test.evt, line 6: Event parameter mismatch, cannot convert Spicy value of type 'string' to Zeek value of type 'count' diff --git a/testing/btest/Baseline/spicy.event-args/output b/testing/btest/Baseline/spicy.event-args/output new file mode 100644 index 0000000000..32754ee44b --- /dev/null +++ b/testing/btest/Baseline/spicy.event-args/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Error message: failed to match regular expression (<...>/test.spicy:7:15) +Error message: n/a diff --git a/testing/btest/Baseline/spicy.event-cond/output b/testing/btest/Baseline/spicy.event-cond/output new file mode 100644 index 0000000000..96e69ffb61 --- /dev/null +++ b/testing/btest/Baseline/spicy.event-cond/output @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +1, OpenSSH_3.8.1p1 +1, OpenSSH_3.9p1 +3, OpenSSH_3.9p1 +4, OpenSSH_3.8.1p1 +5, OpenSSH_3.8.1p1 diff --git a/testing/btest/Baseline/spicy.event-user-type/output b/testing/btest/Baseline/spicy.event-user-type/output new file mode 100644 index 0000000000..b2d3b4bb06 --- /dev/null +++ b/testing/btest/Baseline/spicy.event-user-type/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +is_orig=1 y=[n=1, y=12345] +is_orig=0 y=[n=2, y=ABCDE] +is_orig=1 y=[n=3, y=67890] +is_orig=0 y=[n=4, y=FGHIJ] diff --git a/testing/btest/Baseline/spicy.export-enum/output b/testing/btest/Baseline/spicy.export-enum/output new file mode 100644 index 0000000000..a2ba7bf00d --- /dev/null +++ b/testing/btest/Baseline/spicy.export-enum/output @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +TupleEnum::TestEnum_A +TupleEnum::TestEnum_Undef + [Type] TupleEnum::TestEnum diff --git a/testing/btest/Baseline/spicy.export-type-fail/output b/testing/btest/Baseline/spicy.export-type-fail/output new file mode 100644 index 0000000000..0c3723bac2 --- /dev/null +++ b/testing/btest/Baseline/spicy.export-type-fail/output @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] unknown type 'Test::DOES_NOT_EXIST' exported +[error] unknown type 'NOT_SCOPED' exported +[error] <...>/foo.spicy:1:13-5:3: cannot export Spicy type 'Test::X': type is self-recursive +[error] <...>/foo.spicy:9:3-13:3: cannot export Spicy type 'Test::Z': can only convert tuple types with all-named fields to Zeek +[error] : aborting after errors diff --git a/testing/btest/Baseline/spicy.export-types/output b/testing/btest/Baseline/spicy.export-types/output new file mode 100644 index 0000000000..bf6b8458ae --- /dev/null +++ b/testing/btest/Baseline/spicy.export-types/output @@ -0,0 +1,42 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Test::e +Test::s +Test::type_enum +Test::type_enum_A +Test::type_enum_B +Test::type_enum_C +Test::type_enum_Undef +Test::type_record_u, { +[s] = [type_name=string, log=F, value=, default_val=], +[u2] = [type_name=record , log=F, value=, default_val=], +[b] = [type_name=bool, log=F, value=, default_val=] +} +Test::u +Test::u2 +XYZ::type_record_sss, { +[a] = [type_name=addr, log=F, value=, default_val=], +[v] = [type_name=vector of string, log=F, value=, default_val=], +[iv] = [type_name=interval, log=F, value=, default_val=], +[t] = [type_name=time, log=F, value=, default_val=], +[s] = [type_name=set[enum], log=F, value=, default_val=], +[p] = [type_name=port, log=F, value=, default_val=], +[b] = [type_name=string, log=F, value=, default_val=], +[i] = [type_name=int, log=F, value=, default_val=], +[o] = [type_name=string, log=F, value=, default_val=], +[m] = [type_name=table[addr] of string, log=F, value=, default_val=], +[u] = [type_name=record , log=F, value=, default_val=], +[j] = [type_name=count, log=F, value=, default_val=], +[e] = [type_name=enum, log=F, value=, default_val=], +[r] = [type_name=double, log=F, value=, default_val=] +} +type_record_u2, { +[t] = [type_name=record , log=F, value=, default_val=] +} +--- +[a=1.2.3.4, b=bytes, e=Test::type_enum_B, i=-10, iv=5.0 secs, j=10, m={ +[4.3.2.1] = addr1, +[4.3.2.2] = addr2 +}, o=string, p=42/tcp, r=3.14, s={ +Test::type_enum_A, +Test::type_enum_B +}, t=0.0, u=[s=S, b=T, u2=[t=[x=S, y=T]]], v=[1, 2, 3]] diff --git a/testing/btest/Baseline/spicy.file-analysis-data-in-concurrent/output b/testing/btest/Baseline/spicy.file-analysis-data-in-concurrent/output new file mode 100644 index 0000000000..f8c8eb4e6a --- /dev/null +++ b/testing/btest/Baseline/spicy.file-analysis-data-in-concurrent/output @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +123456 +AAABBBCCC +!@#$ diff --git a/testing/btest/Baseline/spicy.file-analysis-data-in/files.log b/testing/btest/Baseline/spicy.file-analysis-data-in/files.log new file mode 100644 index 0000000000..13ef56de71 --- /dev/null +++ b/testing/btest/Baseline/spicy.file-analysis-data-in/files.log @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +dfe2070c79e7ff36a925ffa327ffe3deecf8f9c2 foo-1.txt +dfe2070c79e7ff36a925ffa327ffe3deecf8f9c2 foo-2.txt diff --git a/testing/btest/Baseline/spicy.file-analysis-data-in/output b/testing/btest/Baseline/spicy.file-analysis-data-in/output new file mode 100644 index 0000000000..f041df16ae --- /dev/null +++ b/testing/btest/Baseline/spicy.file-analysis-data-in/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +FL4lmy3f7owPUUoQ8l +FhCRq1oIy2uDU8rog diff --git a/testing/btest/Baseline/spicy.file-analysis-data-in/x509.log b/testing/btest/Baseline/spicy.file-analysis-data-in/x509.log new file mode 100644 index 0000000000..a82109b77b --- /dev/null +++ b/testing/btest/Baseline/spicy.file-analysis-data-in/x509.log @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +01E3B49AA18D8AA981256950B8 CN=GTS CA 1O1,O=Google Trust Services,C=US +01E3B49AA18D8AA981256950B8 CN=GTS CA 1O1,O=Google Trust Services,C=US diff --git a/testing/btest/Baseline/spicy.file-analyzer-nested/files b/testing/btest/Baseline/spicy.file-analyzer-nested/files new file mode 100644 index 0000000000..d4be2a414f --- /dev/null +++ b/testing/btest/Baseline/spicy.file-analyzer-nested/files @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +SPICY_TEXT2 SPICY_TEXT3 - text/plain3 +SPICY_TEXT2 SPICY_TEXT3 - text/plain3 +SPICY_TEXT1 SPICY_TEXT2 - text/plain2 +HTTP SPICY_TEXT1 - text/plain +HTTP (empty) - text/json diff --git a/testing/btest/Baseline/spicy.file-analyzer-nested/output b/testing/btest/Baseline/spicy.file-analyzer-nested/output new file mode 100644 index 0000000000..7ef28af9e7 --- /dev/null +++ b/testing/btest/Baseline/spicy.file-analyzer-nested/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +data2, F2Qpmk14ATv4vFSEsi, from 1:hello world +data3, FyjjRu4ARLzpsPLhNh, from 2a:from 1:hello world +data3, Fz3QLf4Bn4qaQwyUdk, from 2b:from 1:hello world +data1, FcRmxz1fPbKQEgGGUi, hello world diff --git a/testing/btest/Baseline/spicy.file-analyzer-nested/output-max b/testing/btest/Baseline/spicy.file-analyzer-nested/output-max new file mode 100644 index 0000000000..c7a6a03585 --- /dev/null +++ b/testing/btest/Baseline/spicy.file-analyzer-nested/output-max @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +data3, FyjjRu4ARLzpsPLhNh, +data3, Fz3QLf4Bn4qaQwyUdk, +depth warning, FyjjRu4ARLzpsPLhNh, [chunk_event=, stream_event=, extract_filename=, extract_limit=0], 2 +depth warning, Fz3QLf4Bn4qaQwyUdk, [chunk_event=, stream_event=, extract_filename=, extract_limit=0], 2 +data2, F2Qpmk14ATv4vFSEsi, from 1:hello world +data1, FcRmxz1fPbKQEgGGUi, hello world diff --git a/testing/btest/Baseline/spicy.file-analyzer-property/output b/testing/btest/Baseline/spicy.file-analyzer-property/output new file mode 100644 index 0000000000..8a3282b885 --- /dev/null +++ b/testing/btest/Baseline/spicy.file-analyzer-property/output @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +text data, FcRmxz1fPbKQEgGGUi, hello world diff --git a/testing/btest/Baseline/spicy.file-analyzer/output b/testing/btest/Baseline/spicy.file-analyzer/output new file mode 100644 index 0000000000..fbaa94308b --- /dev/null +++ b/testing/btest/Baseline/spicy.file-analyzer/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Files::ANALYZER_SPICY_TEXT +text data, FcRmxz1fPbKQEgGGUi, hello world diff --git a/testing/btest/Baseline/spicy.file-analyzer/weird.log b/testing/btest/Baseline/spicy.file-analyzer/weird.log new file mode 100644 index 0000000000..78c4d901d5 --- /dev/null +++ b/testing/btest/Baseline/spicy.file-analyzer/weird.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path weird +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer source +#types time string addr port addr port string string bool string string +XXXXXXXXXX.XXXXXX - - - - - test_weird FcRmxz1fPbKQEgGGUi F zeek - +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/spicy.file-data-in-at-offset/output b/testing/btest/Baseline/spicy.file-data-in-at-offset/output new file mode 100644 index 0000000000..f041df16ae --- /dev/null +++ b/testing/btest/Baseline/spicy.file-data-in-at-offset/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +FL4lmy3f7owPUUoQ8l +FhCRq1oIy2uDU8rog diff --git a/testing/btest/Baseline/spicy.file-data-in-at-offset/x509.log b/testing/btest/Baseline/spicy.file-data-in-at-offset/x509.log new file mode 100644 index 0000000000..a82109b77b --- /dev/null +++ b/testing/btest/Baseline/spicy.file-data-in-at-offset/x509.log @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +01E3B49AA18D8AA981256950B8 CN=GTS CA 1O1,O=Google Trust Services,C=US +01E3B49AA18D8AA981256950B8 CN=GTS CA 1O1,O=Google Trust Services,C=US diff --git a/testing/btest/Baseline/spicy.file-replaces/output b/testing/btest/Baseline/spicy.file-replaces/output new file mode 100644 index 0000000000..d3b2cbe683 --- /dev/null +++ b/testing/btest/Baseline/spicy.file-replaces/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +FTP_DATA SPICY_PE - application/x-dosexec +FTP_DATA PE - application/x-dosexec diff --git a/testing/btest/Baseline/spicy.gap-recovery/output b/testing/btest/Baseline/spicy.gap-recovery/output new file mode 100644 index 0000000000..e0bff6eeee --- /dev/null +++ b/testing/btest/Baseline/spicy.gap-recovery/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[$responses=[[$marker=b"HTTP", $len=3070], [$marker=b"HTTP", $len=3070], [$marker=b"HTTP", $len=21999], [$marker=b"HTTP", $len=3070], [$marker=b"HTTP", $len=21999], [$marker=b"HTTP", $len=3070], [$marker=b"HTTP", $len=21999], [$marker=b"HTTP", $len=3070], [$marker=b"HTTP", $len=21999]]] +[$requests=[[$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"]]] diff --git a/testing/btest/Baseline/spicy.gap-recovery/output-before-spicy-issue-1303 b/testing/btest/Baseline/spicy.gap-recovery/output-before-spicy-issue-1303 new file mode 100644 index 0000000000..1ea6e7f5fd --- /dev/null +++ b/testing/btest/Baseline/spicy.gap-recovery/output-before-spicy-issue-1303 @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[$responses=[[$marker=b"HTTP", $len=3070], [$marker=b"HTTP", $len=3070], [$marker=b"HTTP", $len=21999], [$marker=b"HTTP", $len=3070], [$marker=b"HTTP", $len=21999], [$marker=b"HTTP", $len=3070], [$marker=b"HTTP", $len=21999]]] +[$requests=[[$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"]]] diff --git a/testing/btest/Baseline/spicy.import-from/output b/testing/btest/Baseline/spicy.import-from/output new file mode 100644 index 0000000000..cbd8dfcf6c --- /dev/null +++ b/testing/btest/Baseline/spicy.import-from/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Foo::x, Foo::y +Foo::x, Foo::y diff --git a/testing/btest/Baseline/spicy.list-conversion/output b/testing/btest/Baseline/spicy.list-conversion/output new file mode 100644 index 0000000000..8cd97f2a2d --- /dev/null +++ b/testing/btest/Baseline/spicy.list-conversion/output @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[orig_h=192.150.186.169, orig_p=49244/tcp, resp_h=131.159.14.23, resp_p=22/tcp] +T +[[a=83, b=84], [a=83, b=84], [a=72, b=73], [a=45, b=46], [a=50, b=51]] +11824 +11599 diff --git a/testing/btest/Baseline/spicy.module-path/output b/testing/btest/Baseline/spicy.module-path/output new file mode 100644 index 0000000000..58fb13eb69 --- /dev/null +++ b/testing/btest/Baseline/spicy.module-path/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Got it +Got it diff --git a/testing/btest/Baseline/spicy.multiple-enum/output b/testing/btest/Baseline/spicy.multiple-enum/output new file mode 100644 index 0000000000..c8c9711862 --- /dev/null +++ b/testing/btest/Baseline/spicy.multiple-enum/output @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +one, dtest::RESULT_B diff --git a/testing/btest/Baseline/spicy.network-time/output b/testing/btest/Baseline/spicy.network-time/output new file mode 100644 index 0000000000..75525822e2 --- /dev/null +++ b/testing/btest/Baseline/spicy.network-time/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +1584014617531080960 +1584014621661943040 +1584014625494254080 +1584014635499969024 diff --git a/testing/btest/Baseline/spicy.optional/output b/testing/btest/Baseline/spicy.optional/output new file mode 100644 index 0000000000..274745b8e9 --- /dev/null +++ b/testing/btest/Baseline/spicy.optional/output @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[x=] diff --git a/testing/btest/Baseline/spicy.packet-analyzer-on-ip/output b/testing/btest/Baseline/spicy.packet-analyzer-on-ip/output new file mode 100644 index 0000000000..8e3eec68ec --- /dev/null +++ b/testing/btest/Baseline/spicy.packet-analyzer-on-ip/output @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +MACs: src=00:d0:b7:1e:be:20 dst=00:10:dc:72:4c:5f +IPs : src=207.158.192.40 dst=10.20.1.31 +raw bytes: 441 diff --git a/testing/btest/Baseline/spicy.packet-analyzer-replaces/output-on b/testing/btest/Baseline/spicy.packet-analyzer-replaces/output-on new file mode 100644 index 0000000000..7da1ca64df --- /dev/null +++ b/testing/btest/Baseline/spicy.packet-analyzer-replaces/output-on @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +My Ethernet:, \x00\x10\xdcrL_\x00\xd0\xb7\x1e\xbe \x08\x00 diff --git a/testing/btest/Baseline/spicy.packet-analyzer-violation/output b/testing/btest/Baseline/spicy.packet-analyzer-violation/output new file mode 100644 index 0000000000..86d53cbba8 --- /dev/null +++ b/testing/btest/Baseline/spicy.packet-analyzer-violation/output @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[spicy] test::Foo diff --git a/testing/btest/Baseline/spicy.packet-analyzer/output b/testing/btest/Baseline/spicy.packet-analyzer/output new file mode 100644 index 0000000000..11985837fa --- /dev/null +++ b/testing/btest/Baseline/spicy.packet-analyzer/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +MACs: src=c8:5b:76:bd:77:ab dst=ff:ff:ff:ff:ff:ff +raw data, I am encapsulating diff --git a/testing/btest/Baseline/spicy.packet-analyzer/weird.log b/testing/btest/Baseline/spicy.packet-analyzer/weird.log new file mode 100644 index 0000000000..df5e5005e0 --- /dev/null +++ b/testing/btest/Baseline/spicy.packet-analyzer/weird.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path weird +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer source +#types time string addr port addr port string string bool string string +XXXXXXXXXX.XXXXXX - - - - - test_weird - F zeek SPICY_RAWLAYER +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/spicy.parse-error/dpd.log b/testing/btest/Baseline/spicy.parse-error/dpd.log new file mode 100644 index 0000000000..8b8724f2b1 --- /dev/null +++ b/testing/btest/Baseline/spicy.parse-error/dpd.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path dpd +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto analyzer failure_reason +#types time string addr port addr port enum string string +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 192.150.186.169 49244 131.159.14.23 22 tcp SPICY_SSH failed to match regular expression (<...>/test.spicy:9:15) [SSH-2.0-OpenSSH_3.8.1p1\x0a] +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/spicy.port-fail-2/output b/testing/btest/Baseline/spicy.port-fail-2/output new file mode 100644 index 0000000000..c4d245f758 --- /dev/null +++ b/testing/btest/Baseline/spicy.port-fail-2/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] <...>/port-fail.evt:3: cannot parse port specification +[error] error loading EVT file "<...>/port-fail.evt" diff --git a/testing/btest/Baseline/spicy.port-fail-3/output b/testing/btest/Baseline/spicy.port-fail-3/output new file mode 100644 index 0000000000..e0371b533d --- /dev/null +++ b/testing/btest/Baseline/spicy.port-fail-3/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] <...>/port-fail.evt:3: start and end of port range must have same protocol +[error] error loading EVT file "<...>/port-fail.evt" diff --git a/testing/btest/Baseline/spicy.port-fail-4/output b/testing/btest/Baseline/spicy.port-fail-4/output new file mode 100644 index 0000000000..04c5deda11 --- /dev/null +++ b/testing/btest/Baseline/spicy.port-fail-4/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] <...>/port-fail.evt:3: start of port range cannot be after its end +[error] error loading EVT file "<...>/port-fail.evt" diff --git a/testing/btest/Baseline/spicy.port-fail-5/output b/testing/btest/Baseline/spicy.port-fail-5/output new file mode 100644 index 0000000000..04c5deda11 --- /dev/null +++ b/testing/btest/Baseline/spicy.port-fail-5/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] <...>/port-fail.evt:3: start of port range cannot be after its end +[error] error loading EVT file "<...>/port-fail.evt" diff --git a/testing/btest/Baseline/spicy.port-fail/output b/testing/btest/Baseline/spicy.port-fail/output new file mode 100644 index 0000000000..24eb09807d --- /dev/null +++ b/testing/btest/Baseline/spicy.port-fail/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] <...>/port-fail.evt:7: port outside of valid range +[error] error loading EVT file "<...>/port-fail.evt" diff --git a/testing/btest/Baseline/spicy.preprocessor-fail-2/output b/testing/btest/Baseline/spicy.preprocessor-fail-2/output new file mode 100644 index 0000000000..ebe79b98ce --- /dev/null +++ b/testing/btest/Baseline/spicy.preprocessor-fail-2/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[debug/zeek] Loading EVT file "<...>/preprocessor-fail.evt" +[debug/zeek] Loading events from "<...>/preprocessor-fail.evt" +[error] <...>/preprocessor-fail.evt:4: unterminated preprocessor directive +[error] error loading EVT file "<...>/preprocessor-fail.evt" diff --git a/testing/btest/Baseline/spicy.preprocessor-fail-3/output b/testing/btest/Baseline/spicy.preprocessor-fail-3/output new file mode 100644 index 0000000000..735d25c6a9 --- /dev/null +++ b/testing/btest/Baseline/spicy.preprocessor-fail-3/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[debug/zeek] Loading EVT file "<...>/preprocessor-fail.evt" +[debug/zeek] Loading events from "<...>/preprocessor-fail.evt" +[error] <...>/preprocessor-fail.evt:2: @else without @if +[error] error loading EVT file "<...>/preprocessor-fail.evt" diff --git a/testing/btest/Baseline/spicy.preprocessor-fail-4/output b/testing/btest/Baseline/spicy.preprocessor-fail-4/output new file mode 100644 index 0000000000..e405d7c984 --- /dev/null +++ b/testing/btest/Baseline/spicy.preprocessor-fail-4/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[debug/zeek] Loading EVT file "<...>/preprocessor-fail.evt" +[debug/zeek] Loading events from "<...>/preprocessor-fail.evt" +[error] <...>/preprocessor-fail.evt:2: @endif without @if +[error] error loading EVT file "<...>/preprocessor-fail.evt" diff --git a/testing/btest/Baseline/spicy.preprocessor-fail-5/output b/testing/btest/Baseline/spicy.preprocessor-fail-5/output new file mode 100644 index 0000000000..56ee863366 --- /dev/null +++ b/testing/btest/Baseline/spicy.preprocessor-fail-5/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[debug/zeek] Loading EVT file "<...>/preprocessor-fail.evt" +[debug/zeek] Loading events from "<...>/preprocessor-fail.evt" +[error] <...>/preprocessor-fail.evt:6: unterminated preprocessor directive +[error] error loading EVT file "<...>/preprocessor-fail.evt" diff --git a/testing/btest/Baseline/spicy.preprocessor-fail-6/output b/testing/btest/Baseline/spicy.preprocessor-fail-6/output new file mode 100644 index 0000000000..240ed695d2 --- /dev/null +++ b/testing/btest/Baseline/spicy.preprocessor-fail-6/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[debug/zeek] Loading EVT file "<...>/preprocessor-fail.evt" +[debug/zeek] Loading events from "<...>/preprocessor-fail.evt" +[error] <...>/preprocessor-fail.evt:2: cannot parse integer value +[error] error loading EVT file "<...>/preprocessor-fail.evt" diff --git a/testing/btest/Baseline/spicy.preprocessor-fail-7/output b/testing/btest/Baseline/spicy.preprocessor-fail-7/output new file mode 100644 index 0000000000..240ed695d2 --- /dev/null +++ b/testing/btest/Baseline/spicy.preprocessor-fail-7/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[debug/zeek] Loading EVT file "<...>/preprocessor-fail.evt" +[debug/zeek] Loading events from "<...>/preprocessor-fail.evt" +[error] <...>/preprocessor-fail.evt:2: cannot parse integer value +[error] error loading EVT file "<...>/preprocessor-fail.evt" diff --git a/testing/btest/Baseline/spicy.preprocessor-fail/output b/testing/btest/Baseline/spicy.preprocessor-fail/output new file mode 100644 index 0000000000..334fba930d --- /dev/null +++ b/testing/btest/Baseline/spicy.preprocessor-fail/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[debug/zeek] Loading EVT file "<...>/preprocessor-fail.evt" +[debug/zeek] Loading events from "<...>/preprocessor-fail.evt" +[error] <...>/preprocessor-fail.evt:7: unterminated preprocessor directive +[error] error loading EVT file "<...>/preprocessor-fail.evt" diff --git a/testing/btest/Baseline/spicy.preprocessor-spicy/output b/testing/btest/Baseline/spicy.preprocessor-spicy/output new file mode 100644 index 0000000000..736578c927 --- /dev/null +++ b/testing/btest/Baseline/spicy.preprocessor-spicy/output @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +== spicyz + ::hilti::rt::print(std::string("have zeek"), ::hilti::rt::Bool(true)); + ::hilti::rt::print(std::string("have spicy version"), ::hilti::rt::Bool(true)); + ::hilti::rt::print(std::string("have spicy version >= 0.4"), ::hilti::rt::Bool(true)); + ::hilti::rt::print(std::string("not have spicy version >= 4"), ::hilti::rt::Bool(true)); + ::hilti::rt::print(std::string("have zeek and zeek version > 1.0"), ::hilti::rt::Bool(true)); diff --git a/testing/btest/Baseline/spicy.preprocessor/output b/testing/btest/Baseline/spicy.preprocessor/output new file mode 100644 index 0000000000..0c77fb42d6 --- /dev/null +++ b/testing/btest/Baseline/spicy.preprocessor/output @@ -0,0 +1,9 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[debug/zeek] Loading EVT file "<...>/preprocessor.evt" +[debug/zeek] Loading events from "<...>/preprocessor.evt" +[debug/zeek] Got protocol analyzer definition for YES_1 +[debug/zeek] Got protocol analyzer definition for YES_2 +[debug/zeek] Got protocol analyzer definition for YES_3 +[debug/zeek] Got protocol analyzer definition for YES_4 +[debug/zeek] Got protocol analyzer definition for YES_5 +[debug/zeek] Got protocol analyzer definition for YES_6 diff --git a/testing/btest/Baseline/spicy.profiling/output b/testing/btest/Baseline/spicy.profiling/output new file mode 100644 index 0000000000..596d18cd05 --- /dev/null +++ b/testing/btest/Baseline/spicy.profiling/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +SSH banner, [orig_h=192.150.186.169, orig_p=49244/tcp, resp_h=131.159.14.23, resp_p=22/tcp], F, 1.99, OpenSSH_3.9p1 +SSH banner, [orig_h=192.150.186.169, orig_p=49244/tcp, resp_h=131.159.14.23, resp_p=22/tcp], T, 2.0, OpenSSH_3.8.1p1 diff --git a/testing/btest/Baseline/spicy.profiling/prof.log b/testing/btest/Baseline/spicy.profiling/prof.log new file mode 100644 index 0000000000..30c7611635 --- /dev/null +++ b/testing/btest/Baseline/spicy.profiling/prof.log @@ -0,0 +1,22 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +# +# Profiling +# +#name count +hilti/func/SSH::Banner::__on_0x25_done 2 +hilti/func/SSH::Banner::__parse_SSH_Banner_stage2 2 +hilti/func/SSH::Banner::__parse_stage1 2 +hilti/func/SSH::Banner::parse1 2 +hilti/func/SSH::__register_SSH_Banner 1 +hilti/total 1 +spicy/prepare/stream/SSH::Banner 2 +spicy/unit/SSH::Banner 2 +spicy/unit/SSH::Banner::dash 2 +spicy/unit/SSH::Banner::magic 2 +spicy/unit/SSH::Banner::software 2 +spicy/unit/SSH::Banner::version 2 +zeek/event/ssh::banner 2 +zeek/rt/current_conn 2 +zeek/rt/current_is_orig 2 +zeek/rt/event_arg_type 4 +zeek/rt/raise_event 2 diff --git a/testing/btest/Baseline/spicy.protocol-analyzer-data-in/http.log b/testing/btest/Baseline/spicy.protocol-analyzer-data-in/http.log new file mode 100644 index 0000000000..ef750de38b --- /dev/null +++ b/testing/btest/Baseline/spicy.protocol-analyzer-data-in/http.log @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path http +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer version user_agent origin request_body_len response_body_len status_code status_msg info_code info_msg tags username password proxied orig_fuids orig_filenames orig_mime_types resp_fuids resp_filenames resp_mime_types +#types time string addr port addr port count string string string string string string string count count count string count string set[enum] string string set[string] vector[string] vector[string] vector[string] vector[string] vector[string] vector[string] +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 192.150.186.169 49244 131.159.14.23 22 1 GET - /etc/passwd1 - 1.0 - - 0 0 200 OK - - (empty) - - - - - - - - - +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 192.150.186.169 49244 131.159.14.23 22 2 GET - /etc/passwd1.1 - 1.0 - - 0 0 200 OK - - (empty) - - - - - - - - - +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 192.150.186.169 49244 131.159.14.23 22 3 GET - /etc/passwd2 - 1.0 - - 0 0 200 OK - - (empty) - - - - - - - - - +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/spicy.protocol-analyzer-explicit-forwarding/output b/testing/btest/Baseline/spicy.protocol-analyzer-explicit-forwarding/output new file mode 100644 index 0000000000..50ec554426 --- /dev/null +++ b/testing/btest/Baseline/spicy.protocol-analyzer-explicit-forwarding/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +ys=only Y +ys=both Y and Z +zs=both Y and Z +ys=only Y after removal of Z diff --git a/testing/btest/Baseline/spicy.protocol-analyzer-tcp-over-udp/ssh.log b/testing/btest/Baseline/spicy.protocol-analyzer-tcp-over-udp/ssh.log new file mode 100644 index 0000000000..7edf099750 --- /dev/null +++ b/testing/btest/Baseline/spicy.protocol-analyzer-tcp-over-udp/ssh.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path ssh +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version auth_success auth_attempts direction client server cipher_alg mac_alg compression_alg kex_alg host_key_alg host_key +#types time string addr port addr port count bool count enum string string string string string string string string +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 ::1 52806 ::1 1234 2 - 0 - SSH-2.0-OpenSSH_42.2 SSH-2.0-OpenSSH_7.4 - - - - - - +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/spicy.replaces-mismatch/output b/testing/btest/Baseline/spicy.replaces-mismatch/output new file mode 100644 index 0000000000..1073ee7fc5 --- /dev/null +++ b/testing/btest/Baseline/spicy.replaces-mismatch/output @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +fatal error: cannot replace 'Ethernet' analyzer with a protocol analyzer diff --git a/testing/btest/Baseline/spicy.replaces/output b/testing/btest/Baseline/spicy.replaces/output new file mode 100644 index 0000000000..924809c5a5 --- /dev/null +++ b/testing/btest/Baseline/spicy.replaces/output @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Analyzer::ANALYZER_SSH, 3 +SSH banner, [orig_h=192.150.186.169, orig_p=49244/tcp, resp_h=131.159.14.23, resp_p=22/tcp], F, 1.99, OpenSSH_3.9p1 +SSH banner, [orig_h=192.150.186.169, orig_p=49244/tcp, resp_h=131.159.14.23, resp_p=22/tcp], T, 2.0, OpenSSH_3.8.1p1 diff --git a/testing/btest/Baseline/spicy.resource-usage/output b/testing/btest/Baseline/spicy.resource-usage/output new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/spicy.resource-usage/output @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/spicy.reuse-parser-across-hltos/output b/testing/btest/Baseline/spicy.reuse-parser-across-hltos/output new file mode 100644 index 0000000000..88a8c40230 --- /dev/null +++ b/testing/btest/Baseline/spicy.reuse-parser-across-hltos/output @@ -0,0 +1,16 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +foo, 1.99 +foo, 2.0 + +bar, 1.99 +bar, 2.0 + +bar, 1.99 +bar, 2.0 +foo, 1.99 +foo, 2.0 + +bar, 1.99 +bar, 2.0 +foo, 1.99 +foo, 2.0 diff --git a/testing/btest/Baseline/spicy.ssh-banner/analyzer.log b/testing/btest/Baseline/spicy.ssh-banner/analyzer.log new file mode 100644 index 0000000000..b60c24d0f9 --- /dev/null +++ b/testing/btest/Baseline/spicy.ssh-banner/analyzer.log @@ -0,0 +1,12 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path analyzer +#open XXXX-XX-XX-XX-XX-XX +#fields ts cause analyzer_kind analyzer_name uid fuid id.orig_h id.orig_p id.resp_h id.resp_p failure_reason failure_data +#types time string string string string string addr port addr port string string +XXXXXXXXXX.XXXXXX violation protocol SPICY_SSH CHhAvVGS1DHFjwGM9 - 141.142.228.5 53595 54.243.55.129 80 protocol rejected - +XXXXXXXXXX.XXXXXX violation protocol SPICY_SSH CHhAvVGS1DHFjwGM9 - 141.142.228.5 53595 54.243.55.129 80 failed to match regular expression (<...>/ssh.spicy:7:15) POST /post HTTP/1.1\x0d\x0aUser-Agent: curl/7. +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/spicy.ssh-banner/output b/testing/btest/Baseline/spicy.ssh-banner/output new file mode 100644 index 0000000000..680dd73714 --- /dev/null +++ b/testing/btest/Baseline/spicy.ssh-banner/output @@ -0,0 +1,10 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +=== confirmation +SSH banner in Foo, [orig_h=192.150.186.169, orig_p=49244/tcp, resp_h=131.159.14.23, resp_p=22/tcp], F, 1.99, OpenSSH_3.9p1 +SSH banner in Foo, [orig_h=192.150.186.169, orig_p=49244/tcp, resp_h=131.159.14.23, resp_p=22/tcp], T, 2.0, OpenSSH_3.8.1p1 +SSH banner, [orig_h=192.150.186.169, orig_p=49244/tcp, resp_h=131.159.14.23, resp_p=22/tcp], F, 1.99, OpenSSH_3.9p1 +SSH banner, [orig_h=192.150.186.169, orig_p=49244/tcp, resp_h=131.159.14.23, resp_p=22/tcp], T, 2.0, OpenSSH_3.8.1p1 +confirm, Analyzer::ANALYZER_SPICY_SSH +=== violation +violation, Analyzer::ANALYZER_SPICY_SSH, failed to match regular expression (<...>/ssh.spicy:7:15) +violation, Analyzer::ANALYZER_SPICY_SSH, protocol rejected diff --git a/testing/btest/Baseline/spicy.ssh-banner/weird.log b/testing/btest/Baseline/spicy.ssh-banner/weird.log new file mode 100644 index 0000000000..7dcdd71aef --- /dev/null +++ b/testing/btest/Baseline/spicy.ssh-banner/weird.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path weird +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer source +#types time string addr port addr port string string bool string string +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 192.150.186.169 49244 131.159.14.23 22 my_weird OpenSSH_3.9p1 F zeek SPICY_SSH +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/spicy.terminate-session/conn.log b/testing/btest/Baseline/spicy.terminate-session/conn.log new file mode 100644 index 0000000000..40b3b79600 --- /dev/null +++ b/testing/btest/Baseline/spicy.terminate-session/conn.log @@ -0,0 +1,12 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open XXXX-XX-XX-XX-XX-XX +#fields uid +#types string +CHhAvVGS1DHFjwGM9 +ClEkJM2Vm5giqnMf4h +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/spicy.toggle-protocol-analyzer/output b/testing/btest/Baseline/spicy.toggle-protocol-analyzer/output new file mode 100644 index 0000000000..0d612ac613 --- /dev/null +++ b/testing/btest/Baseline/spicy.toggle-protocol-analyzer/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +=== +Spicy: SSH banner, [orig_h=192.150.186.169, orig_p=49244/tcp, resp_h=131.159.14.23, resp_p=22/tcp], T, 2.0, OpenSSH_3.8.1p1 +=== +=== diff --git a/testing/btest/Baseline/spicy.tuple-arg/output b/testing/btest/Baseline/spicy.tuple-arg/output new file mode 100644 index 0000000000..3d15a46ac5 --- /dev/null +++ b/testing/btest/Baseline/spicy.tuple-arg/output @@ -0,0 +1,9 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[i=1, s=OpenSSH_3.8.1p1] +[i=2, s=OpenSSH_3.8.1p1] +[i=3, s=] +[i=4, s=] +[zeek] [SPICY_SSH/3/orig] -> event ssh::banner((1, b"OpenSSH_3.8.1p1")) +[zeek] [SPICY_SSH/3/orig] -> event ssh::banner((2, b"OpenSSH_3.8.1p1")) +[zeek] [SPICY_SSH/3/orig] -> event ssh::banner((3, )) +[zeek] [SPICY_SSH/3/orig] -> event ssh::banner((4, Null)) diff --git a/testing/btest/Baseline/spicy.tuple-enum/output b/testing/btest/Baseline/spicy.tuple-enum/output new file mode 100644 index 0000000000..dcc1da9f4f --- /dev/null +++ b/testing/btest/Baseline/spicy.tuple-enum/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[$a=TestEnum::A, $b=83] +[$a=TestEnum::A, $b=83] +[i=TupleEnum::TestEnum_A, j=83] +[i=TupleEnum::TestEnum_A, j=83] diff --git a/testing/btest/Baseline/spicy.tuple-optional/output b/testing/btest/Baseline/spicy.tuple-optional/output new file mode 100644 index 0000000000..568248dce7 --- /dev/null +++ b/testing/btest/Baseline/spicy.tuple-optional/output @@ -0,0 +1,2 @@ +[i=83, j=83, k=, s=H-, ss=] +[i=83, j=83, k=, s=H-, ss=] diff --git a/testing/btest/Baseline/spicy.type-converter/output b/testing/btest/Baseline/spicy.type-converter/output new file mode 100644 index 0000000000..aa5ca62e91 --- /dev/null +++ b/testing/btest/Baseline/spicy.type-converter/output @@ -0,0 +1,28 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[$a=b"SSH-2", $b=11824, $c=11599, $d=3.14, $e=1.2.3.4, $f=2001:db8::1428:57ab, $g=True, $h="MyString", $i=2011-01-19T05:31:50.500000000Z, $j=4.000000s, $r=[$i=11, $s=(not set)], $s={1, 2, 3}, $t=(47, "foo"), $v=[b"A", b"B", b"C"], $l=[b"A", b"B", b"C"], $m={1: "A", 2: "B", 3: "C"}] +[orig_h=192.150.186.169, orig_p=49244/tcp, resp_h=131.159.14.23, resp_p=22/tcp] +T +SSH-2 +11824 +11599 +3.14 +1.2.3.4 +2001:db8::1428:57ab +T +MyString +XXXXXXXXXX.XXXXXX +4.000000, interval +[i=11, s=] +{ +2, +1, +3 +} +[i=47, s=foo] +[A, B, C] +[A, B, C] +{ +[2] = B, +[1] = A, +[3] = C +} diff --git a/testing/btest/Baseline/spicy.udp/output b/testing/btest/Baseline/spicy.udp/output new file mode 100644 index 0000000000..59667ecab1 --- /dev/null +++ b/testing/btest/Baseline/spicy.udp/output @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Analyzer::ANALYZER_SPICY_UDP_TEST +UDP packet, [orig_h=127.0.0.1, orig_p=12345/udp, resp_h=127.0.0.1, resp_p=31337/udp], T, 12345\x0a +UDP packet, [orig_h=127.0.0.1, orig_p=12345/udp, resp_h=127.0.0.1, resp_p=31337/udp], F, ABCDE\x0a +UDP packet, [orig_h=127.0.0.1, orig_p=12345/udp, resp_h=127.0.0.1, resp_p=31337/udp], T, 67890\x0a +UDP packet, [orig_h=127.0.0.1, orig_p=12345/udp, resp_h=127.0.0.1, resp_p=31337/udp], F, FGHIJ\x0a diff --git a/testing/btest/Traces/dns/long-connection.pcap b/testing/btest/Traces/dns/long-connection.pcap new file mode 100644 index 0000000000..c2ff478357 Binary files /dev/null and b/testing/btest/Traces/dns/long-connection.pcap differ diff --git a/testing/btest/Traces/dns/proto255.pcap b/testing/btest/Traces/dns/proto255.pcap new file mode 100644 index 0000000000..a54d7240f4 Binary files /dev/null and b/testing/btest/Traces/dns/proto255.pcap differ diff --git a/testing/btest/Traces/pe/pe-single.trace b/testing/btest/Traces/pe/pe-single.trace new file mode 100644 index 0000000000..0d87463e76 Binary files /dev/null and b/testing/btest/Traces/pe/pe-single.trace differ diff --git a/testing/btest/Traces/spicy/gap-recovery.pcap b/testing/btest/Traces/spicy/gap-recovery.pcap new file mode 100644 index 0000000000..b806213ca9 Binary files /dev/null and b/testing/btest/Traces/spicy/gap-recovery.pcap differ diff --git a/testing/btest/Traces/spicy/packet-analyzer-violation.pcap b/testing/btest/Traces/spicy/packet-analyzer-violation.pcap new file mode 100644 index 0000000000..3aa4bf87c1 Binary files /dev/null and b/testing/btest/Traces/spicy/packet-analyzer-violation.pcap differ diff --git a/testing/btest/Traces/spicy/raw-layer.pcap b/testing/btest/Traces/spicy/raw-layer.pcap new file mode 100644 index 0000000000..5e4ed891dc Binary files /dev/null and b/testing/btest/Traces/spicy/raw-layer.pcap differ diff --git a/testing/btest/Traces/ssh/single-conn.trace b/testing/btest/Traces/ssh/single-conn.trace new file mode 100644 index 0000000000..a05ec98249 Binary files /dev/null and b/testing/btest/Traces/ssh/single-conn.trace differ diff --git a/testing/btest/Traces/ssh/ssh-over-udp.pcap b/testing/btest/Traces/ssh/ssh-over-udp.pcap new file mode 100644 index 0000000000..5a70f6b849 Binary files /dev/null and b/testing/btest/Traces/ssh/ssh-over-udp.pcap differ diff --git a/testing/btest/Traces/udp-packet.pcap b/testing/btest/Traces/udp-packet.pcap new file mode 100644 index 0000000000..5dc0f1fc8a Binary files /dev/null and b/testing/btest/Traces/udp-packet.pcap differ diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index d1714544dc..1247fb44ce 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -18,7 +18,7 @@ ZEEK_PLUGIN_PATH= TZ=UTC LC_ALL=C BTEST_PATH=%(testbase)s/../../auxil/btest -PATH=%(testbase)s/../../%(build_dir)s/src%(pathsep)s%(testbase)s/../scripts%(pathsep)s%(testbase)s/../../auxil/btest%(pathsep)s%(testbase)s/../../%(build_dir)s/auxil/zeek-aux/zeek-cut%(pathsep)s%(testbase)s/../../auxil/btest/sphinx%(pathsep)s%(default_path)s%(pathsep)s/sbin +PATH=%(testbase)s/../../%(build_dir)s/src%(pathsep)s%(testbase)s/../scripts%(pathsep)s%(testbase)s/../../auxil/btest%(pathsep)s%(testbase)s/../../%(build_dir)s/auxil/zeek-aux/zeek-cut%(pathsep)s%(testbase)s/../../%(build_dir)s/auxil/spicy/spicy/bin%(pathsep)s%(testbase)s/../../%(build_dir)s/src/spicy/spicyz%(pathsep)s%(testbase)s/../../auxil/btest/sphinx%(pathsep)s%(default_path)s%(pathsep)s/sbin TRACES=%(testbase)s/Traces FILES=%(testbase)s/Files SCRIPTS=%(testbase)s/../scripts @@ -39,6 +39,8 @@ ZEEK_SUPERVISOR_NO_SIGKILL=1 UBSAN_OPTIONS=print_stacktrace=1 SPICY_PATH=`bash -c %(testbase)s/../../%(build_dir)s/spicy-path` HILTI_CXX_INCLUDE_DIRS=`bash -c %(testbase)s/../../%(build_dir)s/hilti-cxx-include-dirs` +ZEEK_SPICY_MODULE_PATH=/does/not/exist +ZEEK_SPICY_LIBRARY_PATH=%(testbase)s/../../scripts/spicy [environment-AST-dup] # Environment for testing AST duplication functionality, which is diff --git a/testing/btest/spicy/analyzer-tag.zeek b/testing/btest/spicy/analyzer-tag.zeek new file mode 100644 index 0000000000..9b15aabc31 --- /dev/null +++ b/testing/btest/spicy/analyzer-tag.zeek @@ -0,0 +1,33 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o ssh.hlto ssh.spicy ./ssh.evt +# @TEST-EXEC: zeek -b Zeek::Spicy ssh.hlto %INPUT >>output +# @TEST-EXEC: echo >>output +# @TEST-EXEC: zeek -b Zeek::Spicy %INPUT >>output +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Check that we can access ANALYZER_* tags during Zeek-side script parse time. + +# @TEST-START-FILE ssh.spicy +module SSH; +public type Banner = unit {}; +# @TEST-END-FILE + +# @TEST-START-FILE ssh.evt +protocol analyzer spicy::SSH over TCP: + parse with SSH::Banner; +# @TEST-END-FILE + +@ifdef ( Analyzer::ANALYZER_SPICY_SSH ) +event zeek_init() + { + print "Have analyzer!"; + print fmt("tag: %s", Analyzer::get_tag("spicy_SSH")); + print fmt("name: %s", Analyzer::name(Analyzer::ANALYZER_SPICY_SSH)); + } +@else +event zeek_init() + { + print "Do not have analyzer!"; + } +@endif diff --git a/testing/btest/spicy/availability.zeek b/testing/btest/spicy/availability.zeek new file mode 100644 index 0000000000..1aaa9d6f37 --- /dev/null +++ b/testing/btest/spicy/availability.zeek @@ -0,0 +1,15 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: zeek %INPUT | grep -q yes +# @TEST-EXEC: zeek -b Zeek::Spicy %INPUT | grep -q yes +# @TEST-EXEC: if zeek -N Zeek::Spicy | grep -q built-in; then zeek -b %INPUT | grep -q yes; else zeek -b %INPUT | grep -q no; fi +# +# @TEST-DOC: Confirms `Zeek::available` signals correctly whether the Spicy plugin is loaded and active. +# +# Note that bare mode, by default, doesn't activate the plugin. + +@ifdef ( Spicy::available ) + print "yes"; +@else + print "no"; +@endif diff --git a/testing/btest/spicy/conn-id.spicy b/testing/btest/spicy/conn-id.spicy new file mode 100644 index 0000000000..fafaf54b76 --- /dev/null +++ b/testing/btest/spicy/conn-id.spicy @@ -0,0 +1,21 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto %INPUT test.evt +# @TEST-EXEC: zeek -b -r ${TRACES}/ssh/single-conn.trace Zeek::Spicy test.hlto Spicy::enable_print=T >>output +# @TEST-EXEC: zeek -b -r ${TRACES}/ftp/ipv6.trace Zeek::Spicy test.hlto Spicy::enable_print=T >>output +# @TEST-EXEC: btest-diff output + +module Test; + +import zeek; + +public type Foo = unit { + on %init { print zeek::conn_id(); } + x : /./; +}; + +# @TEST-START-FILE test.evt +protocol analyzer spicy::Test over TCP: + port 21/tcp-22/tcp, + parse originator with Test::Foo; +# @TEST-END-FILE diff --git a/testing/btest/spicy/context.spicy b/testing/btest/spicy/context.spicy new file mode 100644 index 0000000000..2095a7f418 --- /dev/null +++ b/testing/btest/spicy/context.spicy @@ -0,0 +1,50 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o x.hlto %INPUT ./ssh.evt +# @TEST-EXEC: zeek -b -r ${TRACES}/ssh/single-conn.trace Zeek::Spicy x.hlto Spicy::enable_print=T >output +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Check that the Zeek plugin passes a (and the same) %context object to both sides of a connection. + +module SSH; + +type Banner = unit { + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; +}; + +type Context = tuple; + +function print_if_we_have_seen_both_sides(ctx: Context&) { + if ( |ctx.orig_version| && |ctx.resp_version| ) + print ctx; +} + +public type Originator = unit { + %context = Context; + + banner: Banner { + self.context().orig_version += self.banner.version; + } + on %done { print_if_we_have_seen_both_sides(self.context()); } +}; + +public type Responder = unit { + %context = Context; + + banner: Banner { + self.context().resp_version += self.banner.version; + } + + on %done { print_if_we_have_seen_both_sides(self.context()); } +}; + + +# @TEST-START-FILE ssh.evt +protocol analyzer spicy::SSH over TCP: + port 22/tcp, + parse originator with SSH::Originator, + parse responder with SSH::Responder; +# @TEST-END-FILE diff --git a/testing/btest/spicy/double-event.zeek b/testing/btest/spicy/double-event.zeek new file mode 100644 index 0000000000..81f759f077 --- /dev/null +++ b/testing/btest/spicy/double-event.zeek @@ -0,0 +1,33 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto ssh.spicy ./ssh-cond.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT | sort >output +# @TEST-EXEC: btest-diff output + +event ssh::banner(i: int, software: string) + { + print i, software; + } + +# @TEST-START-FILE ssh.spicy +module SSH; + +public type Banner = unit { + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; +}; +# @TEST-END-FILE + +# @TEST-START-FILE ssh-cond.evt + +protocol analyzer spicy::SSH over TCP: + parse with SSH::Banner, + port 22/tcp, + replaces SSH; + +on SSH::Banner -> event ssh::banner(1, self.software); +on SSH::Banner -> event ssh::banner(2, self.software); + +# @TEST-END-FILE diff --git a/testing/btest/spicy/double-type-registration.zeek b/testing/btest/spicy/double-type-registration.zeek new file mode 100644 index 0000000000..5e75d564da --- /dev/null +++ b/testing/btest/spicy/double-type-registration.zeek @@ -0,0 +1,34 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -L . -o a.hlto a.spicy +# @TEST-EXEC: spicyz -L . -o b.hlto b.spicy +# @TEST-EXEC: zeek a.hlto b.hlto +# +# @TEST-EXEC: spicyz -o a.hlto a.spicy common.spicy +# @TEST-EXEC: spicyz -o b.hlto b.spicy common.spicy +# @TEST-EXEC: zeek a.hlto b.hlto +# +# @TEST-DOC: Regression test for #177. + +# @TEST-START-FILE common.spicy +module common; + +public type E = enum { + X, + Y, +}; +# @TEST-END-FILE + +# @TEST-START-FILE a.spicy +module a; + +import common; +global x: common::E; +# @TEST-END-FILE + +# @TEST-START-FILE b.spicy +module b; + +import common; +global x: common::E; +# @TEST-END-FILE diff --git a/testing/btest/spicy/double-types.zeek b/testing/btest/spicy/double-types.zeek new file mode 100644 index 0000000000..ce2937f918 --- /dev/null +++ b/testing/btest/spicy/double-types.zeek @@ -0,0 +1,63 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto dtest.spicy ./dtest.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT | sort >output +# @TEST-EXEC: btest-diff output + +event dtest_message(x: dtest::FUNCS) { + print "dtest_message", x; +} + +event dtest_result(y: dtest::RESULT) { + print "dtest_result", y; +} + +type R: record { + x: dtest::FUNCS; + y: dtest::RESULT; +}; + +event dtest_result_tuple(r: R) { + print "dtest_result_tuple", r$x, r$y; +} + +# @TEST-START-FILE dtest.evt + +protocol analyzer spicy::dtest over TCP: + parse originator with dtest::Message, + port 22/tcp; + +on dtest::Message -> event dtest_message(self.func); + +on dtest::Message -> event dtest_result(self.sub.result); + +on dtest::Message -> event dtest_result_tuple(dtest::bro_result(self)); + +# @TEST-END-FILE + +# @TEST-START-FILE dtest.spicy + +module dtest; + +public type FUNCS = enum { + A=0, B=1, C=2, D=3, E=4, F=5, YES=83 +}; + +public type RESULT = enum { + A, B, C, D, E, F, YES_AGAIN=83 +}; + +public type Message = unit { + func: uint8 &convert=FUNCS($$); + sub: SubMessage; +}; + +public type SubMessage = unit { + result: uint8 &convert=RESULT($$); +}; + +public function bro_result(entry: Message) : tuple { + return (entry.func, entry.sub.result); +} + +# @TEST-END-FILE diff --git a/testing/btest/spicy/event-args-fail.evt b/testing/btest/spicy/event-args-fail.evt new file mode 100644 index 0000000000..409f517822 --- /dev/null +++ b/testing/btest/spicy/event-args-fail.evt @@ -0,0 +1,34 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC-FAIL: spicyz -o test.hlto %INPUT test.spicy >output 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff output +# +# @TEST-DOC: In EVT, provide access to hooks arguments + +# @TEST-START-FILE test.spicy +module SSH; +public type Banner = unit {}; +# @TEST-END-FILE + +# Wrong/unknown. parameter type +on SSH::Banner::%error(msg: bytes) -> event Banner::error(msg); + +# @TEST-START-NEXT +# Wrong signature +on SSH::Banner::%error(msg: string, foo: uint64) -> event Banner::error(msg); + +# @TEST-START-NEXT +# Wrong signature +on SSH::Banner::%error(msg: uint64) -> event Banner::error(msg); + +# @TEST-START-NEXT +# Syntax error +on SSH::Banner::%error(msg: string,) -> event Banner::error(msg); + +# @TEST-START-NEXT +# Syntax error +on SSH::Banner::%error(,msg: string) -> event Banner::error(msg); + +# @TEST-START-NEXT +# Syntax error +on SSH::Banner::%error(msg) -> event Banner::error(msg); diff --git a/testing/btest/spicy/event-args-mismatch.zeek b/testing/btest/spicy/event-args-mismatch.zeek new file mode 100644 index 0000000000..eafe479f2d --- /dev/null +++ b/testing/btest/spicy/event-args-mismatch.zeek @@ -0,0 +1,30 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto test.evt test.spicy +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT >output 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff output +# +# @TEST-DOC: Test error reporting when an Zeek-side event parameter type does not match what Spicy sends. + +event Banner::error(i: count) { } + +# @TEST-START-FILE test.spicy +module SSH; + +public type Banner = unit { + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /KAPUTT/; +}; +# @TEST-END-FILE + +# @TEST-START-FILE test.evt + +protocol analyzer spicy::SSH over TCP: + parse originator with SSH::Banner, + port 22/tcp; + +on SSH::Banner::magic -> event Banner::error(self.magic); # Error: string -> count + +# @TEST-END-FILE diff --git a/testing/btest/spicy/event-args.zeek b/testing/btest/spicy/event-args.zeek new file mode 100644 index 0000000000..fddc3d4f6f --- /dev/null +++ b/testing/btest/spicy/event-args.zeek @@ -0,0 +1,33 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto test.evt test.spicy +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT >output +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff output +# +# @TEST-DOC: In EVT, provide access to hooks arguments + +event Banner::error(msg: string) { + print fmt("Error message: %s", msg); +} + +# @TEST-START-FILE test.spicy +module SSH; + +public type Banner = unit { + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /KAPUTT/; +}; +# @TEST-END-FILE + +# @TEST-START-FILE test.evt + +protocol analyzer spicy::SSH over TCP: + parse originator with SSH::Banner, + port 22/tcp; + +on SSH::Banner::%error(msg: string) -> event Banner::error(msg); +on SSH::Banner::%error() -> event Banner::error("n/a"); + +# @TEST-END-FILE diff --git a/testing/btest/spicy/event-cond.zeek b/testing/btest/spicy/event-cond.zeek new file mode 100644 index 0000000000..bbc43298dc --- /dev/null +++ b/testing/btest/spicy/event-cond.zeek @@ -0,0 +1,58 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto ssh.spicy ./ssh-cond.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT | sort >output +# @TEST-EXEC: btest-diff output + +event ssh::banner1(c: connection, is_orig: bool, version: string, software: string) + { + print "1", software; + } + +event ssh::banner2(c: connection, is_orig: bool, version: string, software: string) + { + print "2", software; + } + +event ssh::banner3(c: connection, is_orig: bool, version: string, software: string) + { + print "3", software; + } + +event ssh::banner4(c: connection, is_orig: bool, version: string, software: string) + { + print "4", software; + } + +event ssh::banner5(c: connection, is_orig: bool, version: string, software: string) + { + print "5", software; + } + +# @TEST-START-FILE ssh.spicy +module SSH; + +public type Banner = unit { + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; +}; +# @TEST-END-FILE + +# @TEST-START-FILE ssh-cond.evt + +import zeek; + +protocol analyzer spicy::SSH over TCP: + parse with SSH::Banner, + port 22/tcp, + replaces SSH; + +on SSH::Banner if ( True ) -> event ssh::banner1($conn, $is_orig, self.version, self.software); +on SSH::Banner if ( False )-> event ssh::banner2($conn, $is_orig, self.version, self.software); +on SSH::Banner if ( self.software == b"OpenSSH_3.9p1" )-> event ssh::banner3($conn, $is_orig, self.version, self.software); +on SSH::Banner if ( self.software != b"OpenSSH_3.9p1" )-> event ssh::banner4($conn, $is_orig, self.version, self.software); +on SSH::Banner if ( zeek::is_orig() ) -> event ssh::banner5($conn, $is_orig, self.version, self.software); + +# @TEST-END-FILE diff --git a/testing/btest/spicy/event-user-type b/testing/btest/spicy/event-user-type new file mode 100644 index 0000000000..75f99b4042 --- /dev/null +++ b/testing/btest/spicy/event-user-type @@ -0,0 +1,43 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -d foo.spicy foo.evt -o foo.hlto +# @TEST-EXEC: zeek -Cr ${TRACES}/udp-packet.pcap foo.hlto foo.zeek >output 2>&1 +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Validate that one can pass a user-specified type in an event handler. This is a regression test for #142, plus now a test for "export". + +# @TEST-START-FILE foo.spicy +module foo; + +global n: uint64 = 0; + +public type X = unit { y: Y; }; + +type Y = unit { + var n: uint64; + on %init { + n += 1; + self.n = n; + } + y: bytes &until=b"\x0a"; +}; +# @TEST-END-FILE + +# @TEST-START-FILE foo.evt +protocol analyzer spicy::foo over UDP: + parse with foo::X, + ports { 12345/udp, 31337/udp }; + +import foo; + +export foo::Y; + +on foo::X -> event foo::X($conn, $is_orig, self.y); +# @TEST-END-FILE + +# @TEST-START-FILE foo.zeek +event foo::X(c: connection, is_orig: bool, y: foo::Y) +{ + print fmt("is_orig=%d y=%s", is_orig, y); +} +# @TEST-END-FILE diff --git a/testing/btest/spicy/export-enum.zeek b/testing/btest/spicy/export-enum.zeek new file mode 100644 index 0000000000..5940b1158d --- /dev/null +++ b/testing/btest/spicy/export-enum.zeek @@ -0,0 +1,31 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o x.hlto tupleenum.spicy ./tupleenum.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace x.hlto %INPUT >>output +# @TEST-EXEC: zeek -NN x.hlto | grep TestEnum >>output +# +# @TEST-EXEC: btest-diff output + +event zeek_init() { + local i: TupleEnum::TestEnum; + + i = TupleEnum::TestEnum_A; + print i; + + i = TupleEnum::TestEnum_Undef; + print i; +} + +# @TEST-START-FILE tupleenum.evt + +# @TEST-END-FILE + +# @TEST-START-FILE tupleenum.spicy + +module TupleEnum; + +public type TestEnum = enum { + A = 83, B = 84, C = 85 +}; + +# @TEST-END-FILE diff --git a/testing/btest/spicy/export-type-fail.spicy b/testing/btest/spicy/export-type-fail.spicy new file mode 100644 index 0000000000..5d3ec3a619 --- /dev/null +++ b/testing/btest/spicy/export-type-fail.spicy @@ -0,0 +1,35 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-DOC: Failure cases for `export` +# +# @TEST-EXEC-FAIL: spicyz -d foo.spicy foo.evt -o foo.hlto >output 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff output + +# @TEST-START-FILE foo.spicy +module Test; + +type X = unit { + y: Y; # self-recursive +}; + +type Y = unit { + x: X; +}; + +type Z = unit { + var x: tuple; # names missing +}; + +# @TEST-END-FILE + +# @TEST-START-FILE foo.evt + +export Test::X; +export Test::Z; +export Test::DOES_NOT_EXIST; +export NOT_SCOPED; + +# @TEST-END-FILE + +# @TEST-START-FILE foo.zeek +# @TEST-END-FILE diff --git a/testing/btest/spicy/export-types.zeek b/testing/btest/spicy/export-types.zeek new file mode 100644 index 0000000000..c3001cc00d --- /dev/null +++ b/testing/btest/spicy/export-types.zeek @@ -0,0 +1,103 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o export.hlto export.spicy export.evt >>output +# @TEST-EXEC: zeek export.hlto %INPUT >>output +# +# Zeek 5.0 doesn't include the ID when printing the enum type +# @TEST-EXEC: cat output | sed 's/enum Test::type_enum/enum/g' >output.tmp && mv output.tmp output +# +# Zeek 6.0 includes information on whether a record field is `&optional`. +# @TEST-EXEC: cat output | sed 's/, optional=F//g' >output.tmp && mv output.tmp output +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Test the `export` keyword to automatically create corresponding Zeek types. + +module Test; + +global e: Test::type_enum = Test::type_enum_B; +global u2: type_record_u2 = [$t=[$x="S", $y=T]]; +global u: Test::type_record_u = [$s="S", $b=T, $u2=u2]; +global s: XYZ::type_record_sss = [ + $a=1.2.3.4, + $b="bytes", + $e=e, + $i=-10, + $iv=5secs, + $j=10, + $m=table([4.3.2.1] = "addr1", [4.3.2.2] = "addr2"), + $o="string", + $p=42/tcp, + $r=3.14, + $s=set(Test::type_enum_A, Test::type_enum_B), + $t=network_time(), + $u=u, + $v=vector("1", "2", "3") +]; + +event zeek_init() { + local all_globals: vector of string; + for ( id in global_ids() ) + all_globals[|all_globals|] = id; + + sort(all_globals, strcmp); + + for ( i in all_globals ) { + id = all_globals[i]; + + if ( ! (/((Test|XYZ)::|type_record_u2)/ in id) ) + next; + + if ( /type_record_/ in id ) + print id, record_fields(id); + # else if ( /type_enum$/ in id ) + # print id, enum_names(id); # Not available in 5.0 yet + else + print id; + } + + print "---"; + print s; +} + +# @TEST-START-FILE export.spicy +module Test; + +type type_enum = enum { A, B, C }; + +type type_record_s = struct { + a: addr; + b: bytes; + e: type_enum; + i: int32; + iv: interval; + j: uint8; + m: map; + o: optional; + p: port; + r: real; + s: set; + t: time; + u: type_record_u; + v: vector; +}; + +type type_record_u = unit { + var s: string; + var b: bool; + var u2: type_record_u2; +}; + +type type_record_u2 = unit { + var t: tuple; +}; + +# @TEST-END-FILE + +# @TEST-START-FILE export.evt + +export Test::type_enum; +export Test::type_record_s as XYZ::type_record_sss; +export Test::type_record_u; +export Test::type_record_u2 as type_record_u2; + +# @TEST-END-FILE diff --git a/testing/btest/spicy/file-analysis-data-in-concurrent.zeek b/testing/btest/spicy/file-analysis-data-in-concurrent.zeek new file mode 100644 index 0000000000..dd8abfdd66 --- /dev/null +++ b/testing/btest/spicy/file-analysis-data-in-concurrent.zeek @@ -0,0 +1,60 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto ssh.spicy ./ssh-cond.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT Spicy::enable_print=T >output +# @TEST-EXEC: btest-diff output + +# @TEST-START-FILE ssh.spicy +module SSH; + +import spicy; +import zeek; + +type Context = tuple; + +public type Banner = unit { + %context = Context; + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; +}; + +public type Data = unit { + data: bytes &eod; + on %done { print self.data; } +}; + +on Banner::%done { + local fid1 = zeek::file_begin("foo/bar"); + local fid2 = zeek::file_begin("foo/bar"); + local fid3 = zeek::file_begin("foo/bar"); + zeek::file_data_in(b"12", fid1); + zeek::file_data_in(b"!", fid3); + zeek::file_data_in(b"AAA", fid2); + zeek::file_data_in(b"@", fid3); + zeek::file_data_in(b"34", fid1); + zeek::file_data_in(b"#", fid3); + zeek::file_data_in(b"56", fid1); + zeek::file_data_in(b"BBB", fid2); + zeek::file_data_in(b"$"); # -> fid3 + zeek::file_end(fid1); + zeek::file_data_in(b"CCC", fid2); + zeek::file_end(fid2); + zeek::file_end(fid3); +} +# @TEST-END-FILE + +# @TEST-START-FILE ssh-cond.evt + +import zeek; + +protocol analyzer spicy::SSH over TCP: + parse originator with SSH::Banner, + port 22/tcp, + replaces SSH; + +file analyzer spicy::Text: + parse with SSH::Data, + mime-type foo/bar; +# @TEST-END-FILE diff --git a/testing/btest/spicy/file-analysis-data-in.zeek b/testing/btest/spicy/file-analysis-data-in.zeek new file mode 100644 index 0000000000..d7289455d5 --- /dev/null +++ b/testing/btest/spicy/file-analysis-data-in.zeek @@ -0,0 +1,64 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -d -o test.hlto ssh.spicy ./ssh-cond.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT Spicy::enable_print=T | sort >output +# +# @TEST-EXEC: cat x509.log | grep -v ^# | cut -f 4-5 >x509.log.tmp && mv x509.log.tmp x509.log +# @TEST-EXEC: btest-diff x509.log +# +# @TEST-EXEC: cat files.log | zeek-cut sha1 filename >files.log.tmp && mv files.log.tmp files.log +# @TEST-EXEC: btest-diff files.log +# +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff output + +# @TEST-START-FILE ssh.spicy +module SSH; + +import spicy; +import zeek; + +global file_counter = 0; + +public type Banner = unit { + magic : /SSH-/ { + # This is a bit of cheating. + local d: spicy::Base64Stream; + local dec : bytes = spicy::base64_decode(d, b"MIIESjCCAzKgAwIBAgINAeO0mqGNiqmBJWlQuDANBgkqhkiG9w0BAQsFADBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSMjETMBEGA1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjAeFw0xNzA2MTUwMDAwNDJaFw0yMTEyMTUwMDAwNDJaMEIxCzAJBgNVBAYTAlVTMR4wHAYDVQQKExVHb29nbGUgVHJ1c3QgU2VydmljZXMxEzARBgNVBAMTCkdUUyBDQSAxTzEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDQGM9F1IvN05zkQO9+tN1pIRvJzzyOTHW5DzEZhD2ePCnvUA0Qk28FgICfKqC9EksC4T2fWBYk/jCfC3R3VZMdS/dN4ZKCEPZRrAzDsiKUDzRrmBBJ5wudgzndIMYcLe/RGGFl5yODIKgjEv/SJH/UL+dEaltN11BmsK+eQmMF++AcxGNhr59qM/9il71I2dN8FGfcddwuaej4bXhp0LcQBbjxMcI7JP0aM3T4I+DsaxmKFsbjzaTNC9uzpFlgOIg7rR25xoynUxv8vNmkq7zdPGHXkxWY7oG9j+JkRyBABk7XrJfoucBZEqFJJSPk7XA0LKW0Y3z5oz2D0c1tJKwHAgMBAAGjggEzMIIBLzAOBgNVHQ8BAf8EBAMCAYYwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMBIGA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0OBBYEFJjR+G4Q68+b7GCfGJAboOt9Cf0rMB8GA1UdIwQYMBaAFJviB1dnHB7AagbeWbSaLd/cGYYuMDUGCCsGAQUFBwEBBCkwJzAlBggrBgEFBQcwAYYZaHR0cDovL29jc3AucGtpLmdvb2cvZ3NyMjAyBgNVHR8EKzApMCegJaAjhiFodHRwOi8vY3JsLnBraS5nb29nL2dzcjIvZ3NyMi5jcmwwPwYDVR0gBDgwNjA0BgZngQwBAgIwKjAoBggrBgEFBQcCARYcaHR0cHM6Ly9wa2kuZ29vZy9yZXBvc2l0b3J5LzANBgkqhkiG9w0BAQsFAAOCAQEAGoA+Nnn78y6pRjd9XlQWNa7HTgiZ/r3RNGkmUmYHPQq6Scti9PEajvwRT2iWTHQr02fesqOqBY2ETUwgZQ+lltoNFvhsO9tvBCOIazpswWC9aJ9xju4tWDQH8NVU6YZZ/XteDSGU9YzJqPjY8q3MDxrzmqepBCf5o8mw/wJ4a2G6xzUr6Fb6T8McDO22PLRL6u3M4Tzs3A2M1j6bykJYi8wWIRdAvKLWZu/axBVbzYmqmwkm5zLSDW5nIAJbELCQCZwMH56t2Dvqofxs6BBcCFIZUSpxu6x6td0V7SvJCCosirSmIatj/9dSSVDQibet8q/7UK4v4ZUN80atnZz1yg=="); + dec += spicy::base64_finish(d); + + print self.file_id; + zeek::file_data_in(dec); + } + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; + + var file_id: string = zeek::file_begin("application/x-x509-ca-cert"); + var file_name: string = "foo-%d.txt" % ++file_counter; +}; + +on Banner::%done { zeek::file_end(self.file_id); } + +# @TEST-END-FILE + +# @TEST-START-FILE ssh-cond.evt + +import zeek; + +protocol analyzer spicy::SSH over TCP: + parse with SSH::Banner, + port 22/tcp, + replaces SSH; + +on SSH::Banner::software -> event have_filename($file, self.file_name); + +# @TEST-END-FILE + +# Trigger creation of `files.log`. +@load base/protocols/ssl +redef X509::log_x509_in_files_log = T; + +event have_filename(f: fa_file, filename: string) + { + f$info$filename = filename; + } diff --git a/testing/btest/spicy/file-analyzer-nested.zeek b/testing/btest/spicy/file-analyzer-nested.zeek new file mode 100644 index 0000000000..6c1d22de7c --- /dev/null +++ b/testing/btest/spicy/file-analyzer-nested.zeek @@ -0,0 +1,87 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o text.hlto text.spicy ./text.evt +# @TEST-EXEC: zeek -r ${TRACES}/http/post.trace text.hlto %INPUT Spicy::enable_print=T | sort -k 3 >output +# @TEST-EXEC: btest-diff output +# @TEST-EXEC: cat files.log | zeek-cut source analyzers filename mime_type >files +# @TEST-EXEC: btest-diff files +# +# Check that exceeding max-file-depth leads to aborting and an event. +# @TEST-EXEC: zeek -t /tmp/zeek.trace -r ${TRACES}/http/post.trace text.hlto %INPUT Spicy::max_file_depth=2 | sort -k 3 >output-max +# @TEST-EXEC: cat notice.log | zeek-cut note | grep -q "Spicy_Max_File_Depth_Exceeded" +# @TEST-EXEC: btest-diff output-max + +event text::data1(f: fa_file, data: string) + { + print "data1", f$id, data; + } + +event text::data2(f: fa_file, data: string) + { + print "data2", f$id, data; + } + +event text::data3(f: fa_file, data: string) + { + print "data3", f$id, data; + } + +event Spicy::max_file_depth_exceeded(f: fa_file, args: Files::AnalyzerArgs, limit: count) + { + print "depth warning", f$id, args, limit; + } + +# @TEST-START-FILE text.spicy +module Text; + +import zeek; +import zeek_file; + +# This unit uses the zeek_file::File wrapper to pass data into Zeek's file analysis. +public type Data1 = unit { + on %init { + self.content.connect(new zeek_file::File("text/plain2")); + self.content.write(b"from 1:"); + } + + data: bytes &eod -> self.content; + + sink content; +}; + +# This unit passes data into Zeek's file analysis directly, without the File wrapper. +public type Data2 = unit { + data: bytes &eod { + zeek::file_begin("text/plain3"); + zeek::file_data_in(b"from 2a:" + self.data); + zeek::file_end(); + + zeek::file_begin("text/plain3"); + zeek::file_data_in(b"from 2b:" + self.data); + zeek::file_end(); + } +}; + +public type Data3 = unit { + data: bytes &eod; +}; +# @TEST-END-FILE + +# @TEST-START-FILE text.evt + +file analyzer spicy::Text1: + parse with Text::Data1, + mime-type text/plain; + +file analyzer spicy::Text2: + parse with Text::Data2, + mime-type text/plain2; + +file analyzer spicy::Text3: + parse with Text::Data3, + mime-type text/plain3; + +on Text::Data1 -> event text::data1($file, self.data); +on Text::Data2 -> event text::data2($file, self.data); +on Text::Data3 -> event text::data3($file, self.data); +# @TEST-END-FILE diff --git a/testing/btest/spicy/file-analyzer-property.zeek b/testing/btest/spicy/file-analyzer-property.zeek new file mode 100644 index 0000000000..34299e9117 --- /dev/null +++ b/testing/btest/spicy/file-analyzer-property.zeek @@ -0,0 +1,27 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto text.spicy ./text.evt +# @TEST-EXEC: zeek -r ${TRACES}/http/post.trace test.hlto %INPUT >output +# @TEST-EXEC: btest-diff output + +event text::data(f: fa_file, data: string) + { + print "text data", f$id, data; + } + +# @TEST-START-FILE text.spicy +module Text; + +public type Data = unit { + %mime-type = "text/plain"; + data: bytes &eod; +}; +# @TEST-END-FILE + +# @TEST-START-FILE text.evt + +file analyzer spicy::Text: + parse with Text::Data; + +on Text::Data -> event text::data($file, self.data); +# @TEST-END-FILE diff --git a/testing/btest/spicy/file-analyzer.zeek b/testing/btest/spicy/file-analyzer.zeek new file mode 100644 index 0000000000..99cbccc510 --- /dev/null +++ b/testing/btest/spicy/file-analyzer.zeek @@ -0,0 +1,46 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto text.spicy ./text.evt +# @TEST-EXEC: zeek -r ${TRACES}/http/post.trace test.hlto %INPUT >output +# @TEST-EXEC: btest-diff output +# @TEST-EXEC: btest-diff weird.log + +event zeek_init() + { + # Check we can access the tag. + print Files::ANALYZER_SPICY_TEXT; + } + +event text::data(f: fa_file, data: string) + { + print "text data", f$id, data; + } + +# @TEST-START-FILE text.spicy +module Text; + +import zeek; + +public type Data = unit { + data: bytes &eod; + + on %done { + # File ID isn't stable across platforms, so just check expected length. + assert |zeek::fuid()| == 18; + zeek::weird("test_weird"); + } +}; +# @TEST-END-FILE + +# @TEST-START-FILE text.evt + +file analyzer spicy::Text: + parse with Text::Data, + + # Note that Zeek determines the MIME type not from the Content-Type + # header in the trace, but by content sniffing (i.e., libmagic-style) + mime-type text/plain; + #mime-type application/x-www-form-urlencoded; + +on Text::Data -> event text::data($file, self.data); +# @TEST-END-FILE diff --git a/testing/btest/spicy/file-data-in-at-offset.zeek b/testing/btest/spicy/file-data-in-at-offset.zeek new file mode 100644 index 0000000000..762b7c4f05 --- /dev/null +++ b/testing/btest/spicy/file-data-in-at-offset.zeek @@ -0,0 +1,47 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto ssh.spicy ./ssh-cond.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT Spicy::enable_print=T | sort >output +# +# @TEST-EXEC: cat x509.log | grep -v ^# | cut -f 4-5 >x509.log.tmp && mv x509.log.tmp x509.log +# @TEST-EXEC: btest-diff x509.log +# @TEST-EXEC: btest-diff output + +# @TEST-START-FILE ssh.spicy +module SSH; + +import spicy; +import zeek; + + +public type Banner = unit { + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; +}; + +# this is a bit of cheating... + +on Banner::%done { + local d: spicy::Base64Stream; + local dec : bytes = spicy::base64_decode(d, b"MIIESjCCAzKgAwIBAgINAeO0mqGNiqmBJWlQuDANBgkqhkiG9w0BAQsFADBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSMjETMBEGA1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjAeFw0xNzA2MTUwMDAwNDJaFw0yMTEyMTUwMDAwNDJaMEIxCzAJBgNVBAYTAlVTMR4wHAYDVQQKExVHb29nbGUgVHJ1c3QgU2VydmljZXMxEzARBgNVBAMTCkdUUyBDQSAxTzEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDQGM9F1IvN05zkQO9+tN1pIRvJzzyOTHW5DzEZhD2ePCnvUA0Qk28FgICfKqC9EksC4T2fWBYk/jCfC3R3VZMdS/dN4ZKCEPZRrAzDsiKUDzRrmBBJ5wudgzndIMYcLe/RGGFl5yODIKgjEv/SJH/UL+dEaltN11BmsK+eQmMF++AcxGNhr59qM/9il71I2dN8FGfcddwuaej4bXhp0LcQBbjxMcI7JP0aM3T4I+DsaxmKFsbjzaTNC9uzpFlgOIg7rR25xoynUxv8vNmkq7zdPGHXkxWY7oG9j+JkRyBABk7XrJfoucBZEqFJJSPk7XA0LKW0Y3z5oz2D0c1tJKwHAgMBAAGjggEzMIIBLzAOBgNVHQ8BAf8EBAMCAYYwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMBIGA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0OBBYEFJjR+G4Q68+b7GCfGJAboOt9Cf0rMB8GA1UdIwQYMBaAFJviB1dnHB7AagbeWbSaLd/cGYYuMDUGCCsGAQUFBwEBBCkwJzAlBggrBgEFBQcwAYYZaHR0cDovL29jc3AucGtpLmdvb2cvZ3NyMjAyBgNVHR8EKzApMCegJaAjhiFodHRwOi8vY3JsLnBraS5nb29nL2dzcjIvZ3NyMi5jcmwwPwYDVR0gBDgwNjA0BgZngQwBAgIwKjAoBggrBgEFBQcCARYcaHR0cHM6Ly9wa2kuZ29vZy9yZXBvc2l0b3J5LzANBgkqhkiG9w0BAQsFAAOCAQEAGoA+Nnn78y6pRjd9XlQWNa7HTgiZ/r3RNGkmUmYHPQq6Scti9PEajvwRT2iWTHQr02fesqOqBY2ETUwgZQ+lltoNFvhsO9tvBCOIazpswWC9aJ9xju4tWDQH8NVU6YZZ/XteDSGU9YzJqPjY8q3MDxrzmqepBCf5o8mw/wJ4a2G6xzUr6Fb6T8McDO22PLRL6u3M4Tzs3A2M1j6bykJYi8wWIRdAvKLWZu/axBVbzYmqmwkm5zLSDW5nIAJbELCQCZwMH56t2Dvqofxs6BBcCFIZUSpxu6x6td0V7SvJCCosirSmIatj/9dSSVDQibet8q/7UK4v4ZUN80atnZz1yg=="); + dec += spicy::base64_finish(d); + local id = zeek::file_begin("application/x-x509-ca-cert"); + print id; + zeek::file_data_in_at_offset(dec, 0); + zeek::file_end(); +} + +# @TEST-END-FILE + +# @TEST-START-FILE ssh-cond.evt + +import zeek; + +protocol analyzer spicy::SSH over TCP: + parse with SSH::Banner, + port 22/tcp, + replaces SSH; + +# @TEST-END-FILE diff --git a/testing/btest/spicy/file-replaces.zeek b/testing/btest/spicy/file-replaces.zeek new file mode 100644 index 0000000000..17ebf20af7 --- /dev/null +++ b/testing/btest/spicy/file-replaces.zeek @@ -0,0 +1,117 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o pe.hlto pe.spicy zeek_pe.spicy pe.evt +# @TEST-EXEC: zeek -r ${TRACES}/pe/pe-single.trace pe.hlto %INPUT ENABLE=T +# @TEST-EXEC: cat files.log | zeek-cut source analyzers filename mime_type >>output +# @TEST-EXEC: zeek -r ${TRACES}/pe/pe-single.trace pe.hlto %INPUT ENABLE=F +# @TEST-EXEC: cat files.log | zeek-cut source analyzers filename mime_type >>output +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Test replacing an existing file analyzer, and also toggling the Spicy one on and off + +const ENABLE = T &redef; + +event zeek_init() { + if ( ENABLE ) + Spicy::enable_file_analyzer(Files::ANALYZER_SPICY_PE); + else + Spicy::disable_file_analyzer(Files::ANALYZER_SPICY_PE); +} + +event pe_dos_header(f: fa_file, h: PE::DOSHeader) + { + print "pe_dos_header", h; + } + +# @TEST-START-FILE pe.spicy +module PE; +import spicy; +%byte-order = spicy::ByteOrder::Little; + +public type ImageFile = unit { + %mime-type = "application/x-dosexec"; + + dosHeader: DOS_Header; +}; + +type DOS_Header = unit { + magic: b"MZ"; + bytesInLastPage: uint16; + pagesInFile: uint16; + relocations: uint16; + paragraphsInHeader: uint16; + minExtraParagraphs: uint16; + maxExtraParagraphs: uint16; + initialRelativeSS: uint16; + initialSP: uint16; + checksum: uint16; + initialIP: uint16; + initialRelativeCS: uint16; + relocationTableAddress: uint16; + overlayNumber: uint16; + reserved1: bytes &size=8; + oemID: uint16; + oemInfo: uint16; + reserved2: bytes &size=20; + peHeaderOffset: uint32; +}; +# @TEST-END-FILE + +# @TEST-START-FILE zeek_pe.spicy +module Zeek_PE; +import PE; + +type DOSHeader = tuple< + signature : bytes, + used_bytes_in_last_page : uint64, + file_in_pages : uint64, + num_reloc_items : uint64, + header_in_paragraphs : uint64, + min_extra_paragraphs : uint64, + max_extra_paragraphs : uint64, + init_relative_ss : uint64, + init_sp : uint64, + checksum : uint64, + init_ip : uint64, + init_relative_cs : uint64, + addr_of_reloc_table : uint64, + overlay_num : uint64, + oem_id : uint64, + oem_info : uint64, + addr_of_new_exe_header : uint64 + >; + +public function makeDOSHeader(h: PE::DOS_Header): DOSHeader + { + return ( + b"MZ (Spicy)", + h.bytesInLastPage, + h.pagesInFile, + h.relocations, + h.paragraphsInHeader, + h.minExtraParagraphs, + h.maxExtraParagraphs, + h.initialRelativeSS, + h.initialSP, + h.checksum, + h.initialIP, + h.initialRelativeCS, + h.relocationTableAddress, + h.overlayNumber, + h.oemID, + h.oemInfo, + h.peHeaderOffset, + ); + } +# @TEST-END-FILE + +# @TEST-START-FILE pe.evt +file analyzer spicy::PE: + parse with PE::ImageFile, + replaces PE, + mime-type application/x-dosexec; + +import Zeek_PE; + +on PE::DOS_Header -> event pe_dos_header($file, Zeek_PE::makeDOSHeader(self)); +# @TEST-END-FILE diff --git a/testing/btest/spicy/gap-recovery.zeek b/testing/btest/spicy/gap-recovery.zeek new file mode 100644 index 0000000000..10be2b3584 --- /dev/null +++ b/testing/btest/spicy/gap-recovery.zeek @@ -0,0 +1,47 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o analyzer.hlto analyzer.spicy analyzer.evt +# @TEST-EXEC: zeek -Cr ${TRACES}/spicy/gap-recovery.pcap analyzer.hlto Spicy::enable_print=T >output 2>&1 +# @TEST-EXEC: if spicy-version 10503; then btest-diff output; else OUT=output-before-spicy-issue-1303; mv output "$OUT"; btest-diff "$OUT"; fi +# +# @TEST-DOC: Tests that parsers can resynchronize on gaps. + +# @TEST-START-FILE analyzer.evt +protocol analyzer spicy::HTTP over TCP: + parse originator with test::Requests, + parse responder with test::Responses, + port 9000/tcp, + replaces HTTP; +# @TEST-END-FILE + +# @TEST-START-FILE analyzer.spicy +module test; +public type Requests = unit { + %port = 9000/tcp &originator; + requests: (Request &synchronize)[] foreach { confirm; } + on %done { print self; } +}; + +type Request = unit { + marker: /GET/; + : /[^\r?\n]+\r?\n/; + : (/[^\r?\n]*:[^\r?\n]*\r?\n/)[]; + : /\r?\n/; +}; + +public type Responses = unit { + %port = 9000/tcp &responder; + responses: (Response &synchronize)[] foreach { confirm; } + on %done { print self; } +}; + +type Response = unit { + marker: /HTTP/; + : /[^\r?\n]+\r?\n/; + : /content-length: /; + len: bytes &until=b"\r\n" &convert=cast($$.to_int()); + : (/[^\r?\n]*:[^\r?\n]*\r?\n/)[]; + : /\r?\n/; + : bytes &size=self.len; +}; +# @TEST-END-FILE diff --git a/testing/btest/spicy/import-from.zeek b/testing/btest/spicy/import-from.zeek new file mode 100644 index 0000000000..05ca2c15aa --- /dev/null +++ b/testing/btest/spicy/import-from.zeek @@ -0,0 +1,57 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: mkdir -p a/b/c && mv y.spicy a/b/c +# @TEST-EXEC: spicyz -o test.hlto ssh.spicy ./ssh.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT >output +# @TEST-EXEC: btest-diff output + +event ssh::test(x: string, y: string) + { + print x, y; + } + +# @TEST-START-FILE ssh.spicy +module SSH; + +public type Banner = unit { + %port = 22/tcp; + + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; + + on %done {} +}; +# @TEST-END-FILE + +# @TEST-START-FILE x.spicy + +module X; + +public function x() : string { + return "Foo::x"; +} + +# @TEST-END-FILE + +# @TEST-START-FILE y.spicy + +module Y; + +public function y() : string { + return "Foo::y"; +} + +# @TEST-END-FILE + + +# @TEST-START-FILE ssh.evt +protocol analyzer spicy::SSH over TCP: + parse with SSH::Banner; + +import X; +import Y from a.b.c; + +on SSH::Banner -> event ssh::test(X::x(), Y::y()); +# @TEST-END-FILE diff --git a/testing/btest/spicy/list-conversion.zeek b/testing/btest/spicy/list-conversion.zeek new file mode 100644 index 0000000000..132633297d --- /dev/null +++ b/testing/btest/spicy/list-conversion.zeek @@ -0,0 +1,54 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto listconv.spicy ./listconv.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT >output +# @TEST-EXEC: btest-diff output + +@TEST-START-FILE listconv.spicy + +module listconv; + +public function bro_convert(a: uint8) : tuple { + return (a, a + 1); +} + +public type Test = unit { + a: uint8[5]; + b: int16; + c: uint16; +}; + +@TEST-END-FILE + +@TEST-START-FILE listconv.evt + +protocol analyzer listconv over TCP: + parse originator with listconv::Test, + port 22/tcp; + +on listconv::Test -> event listconv::test($conn, + $is_orig, + [listconv::bro_convert(i) for i in self.a], + self.b, + self.c + ); + +@TEST-END-FILE + +type int_tuple: record { + a: count; + b: count; +}; + +event listconv::test(x: connection, + is_orig: bool, + a: vector of int_tuple, + b: int, + c: count + ) { + print x$id; + print is_orig; + print a; + print b; + print c; +} diff --git a/testing/btest/spicy/module-path.spicy b/testing/btest/spicy/module-path.spicy new file mode 100644 index 0000000000..c857f19b4c --- /dev/null +++ b/testing/btest/spicy/module-path.spicy @@ -0,0 +1,11 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o x.hlto %INPUT +# @TEST-EXEC: zeek -b Zeek::Spicy x.hlto Spicy::enable_print=T >>output +# @TEST-EXEC: mkdir -p y/z && mv x.hlto y/z +# @TEST-EXEC: ZEEK_SPICY_MODULE_PATH=FOO:y:BAR zeek -b Zeek::Spicy Spicy::enable_print=T >>output +# @TEST-EXEC: btest-diff output + +module Test; + +print "Got it"; diff --git a/testing/btest/spicy/multiple-enum.zeek b/testing/btest/spicy/multiple-enum.zeek new file mode 100644 index 0000000000..959f5f3482 --- /dev/null +++ b/testing/btest/spicy/multiple-enum.zeek @@ -0,0 +1,41 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto dtest.spicy ./dtest.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT | sort >output +# @TEST-EXEC: btest-diff output + +event dtest_one(x: dtest::RESULT) { + print "one", x; +} + +event dtest_two(x: dtest::RESULT) { + print "two", x; +} + +# @TEST-START-FILE dtest.evt + +protocol analyzer spicy::dtest over TCP: + parse originator with dtest::Message, + port 22/tcp; + +on dtest::Message if ( self.sswitch == 83 ) + -> event dtest_one(self.result); + +on dtest::Message if ( self.sswitch != 83 ) + -> event dtest_two(self.result); + +# @TEST-END-FILE +# @TEST-START-FILE dtest.spicy + +module dtest; + +public type RESULT = enum { + A, B = 83, C, D, E, F +}; + +public type Message = unit { + sswitch: uint8; + result: uint8 &convert=RESULT($$); +}; + +# @TEST-END-FILE diff --git a/testing/btest/spicy/network-time.spicy b/testing/btest/spicy/network-time.spicy new file mode 100644 index 0000000000..667aff89ee --- /dev/null +++ b/testing/btest/spicy/network-time.spicy @@ -0,0 +1,26 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto %INPUT ./udp-test.evt +# @TEST-EXEC: zeek -Cr ${TRACES}/udp-packet.pcap test.hlto Spicy::enable_print=T >output +# @TEST-EXEC: btest-diff output + +module Test; + +import zeek; + +# Before any processing has started the network time should be zero. +assert(zeek::network_time() == time(0)); + +public type Message = unit { + data: bytes &eod { + # For real traffic we would expect the network time to increase as we progress. + print zeek::network_time().nanoseconds(); + } +}; + +# @TEST-START-FILE udp-test.evt +protocol analyzer spicy::TEST over UDP: + parse with Test::Message, + port 11337/udp-11340/udp, + ports {31337/udp-31340/udp}; +# @TEST-END-FILE diff --git a/testing/btest/spicy/optional.zeek b/testing/btest/spicy/optional.zeek new file mode 100644 index 0000000000..8b848abdbc --- /dev/null +++ b/testing/btest/spicy/optional.zeek @@ -0,0 +1,39 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -d -o foo.hlto foo.spicy foo.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace foo.hlto %INPUT > output 2>&1 +# @TEST-EXEC: btest-diff output + +type R: record { + x: vector of int &optional; +}; + +event foo_result_tuple(r: R) { + print(r); +} + +# @TEST-START-FILE foo.evt + +protocol analyzer spicy::foo over TCP: + parse originator with Foo::Message, + port 22/tcp; + +on Foo::Message -> event foo_result_tuple(Foo::bro_result(self)); + +# @TEST-END-FILE + +# @TEST-START-FILE foo.spicy + +module Foo; + +public type Message = unit { + : uint8; + : uint8; +}; + +public function bro_result(entry: Message) : tuple> { + local y: optional; + return (y, ); +} + +# @TEST-END-FILE diff --git a/testing/btest/spicy/packet-analyzer-on-ip.zeek b/testing/btest/spicy/packet-analyzer-on-ip.zeek new file mode 100644 index 0000000000..b4f65fb02b --- /dev/null +++ b/testing/btest/spicy/packet-analyzer-on-ip.zeek @@ -0,0 +1,37 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto spicy/raw-layer.pcap.spicy spicy/raw-layer.pcap.evt +# @TEST-EXEC: zeek -r ${TRACES}/dns/proto255.pcap test.hlto %INPUT >output +# @TEST-EXEC: btest-diff output + +module PacketAnalyzer::SPICY_RAWLAYER; + +event zeek_init() + { + if ( ! PacketAnalyzer::try_register_packet_analyzer_by_name("IP", 255, "spicy_RawLayer") ) # modified trace to have IP proto 255 + print "cannot register raw analyzer on top of IP"; + } + +event raw::data(p: raw_pkt_hdr, data: string) + { + print fmt("MACs: src=%s dst=%s", p$l2$src, p$l2$dst); + print fmt("IPs : src=%s dst=%s", p$ip$src, p$ip$dst); + print fmt("raw bytes: %d", |data|); + } + +# @TEST-START-FILE spicy/raw-layer.pcap.spicy +module RawLayer; + +import zeek; + +public type Packet = unit { + data: bytes &eod; +}; +# @TEST-END-FILE + +# @TEST-START-FILE spicy/raw-layer.pcap.evt +packet analyzer spicy::RawLayer: + parse with RawLayer::Packet; + +on RawLayer::Packet::data -> event raw::data($packet, self.data); +# @TEST-END-FILE diff --git a/testing/btest/spicy/packet-analyzer-replaces.zeek b/testing/btest/spicy/packet-analyzer-replaces.zeek new file mode 100644 index 0000000000..7f1a2ec906 --- /dev/null +++ b/testing/btest/spicy/packet-analyzer-replaces.zeek @@ -0,0 +1,61 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o my-ethernet.hlto my-ethernet.spicy my-ethernet.evt +# @TEST-EXEC: zeek -r ${TRACES}/dns53.pcap my-ethernet.hlto %INPUT ENABLE=T >output-on +# @TEST-EXEC: zeek -r ${TRACES}/dns53.pcap my-ethernet.hlto %INPUT ENABLE=F >output-off +# @TEST-EXEC: btest-diff output-on + +# +# @TEST-DOC: Check that we can replace Zeek's Ethernet analyzer. +# +# Zeek logs look the same in both cases but we get some additional output +# when our analyzer is running by raising a custom event. + +const ENABLE = T &redef; + +module MyEthernet; + +const DLT_EN10MB : count = 1; + +event zeek_init() &priority=-200 + { + if ( ENABLE ) + Spicy::enable_file_analyzer(PacketAnalyzer::ANALYZER_SPICY_MYETHERNET); + else + Spicy::disable_file_analyzer(PacketAnalyzer::ANALYZER_SPICY_MYETHERNET); +} + +# The priority here needs to be higher than the standard script registering the +# built-in Ethernet analyzer. +event zeek_init() &priority=-100 + { + PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_ROOT, DLT_EN10MB, PacketAnalyzer::ANALYZER_SPICY_MYETHERNET); + PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_SPICY_MYETHERNET, 0x0800, PacketAnalyzer::ANALYZER_IP); + } + +event MyEthernet::data(p: raw_pkt_hdr, data: string) + { + print "My Ethernet:", data; + } + +# @TEST-START-FILE my-ethernet.spicy +module MyEthernet; + +import zeek; + +public type Packet = unit { + ethernet: bytes &size=14; + + on %done { + zeek::forward_packet(0x0800); # in practice, this wouldn't be hardcoded of course; + } +}; +# @TEST-END-FILE + +# @TEST-START-FILE my-ethernet.evt +packet analyzer spicy::MyEthernet: + parse with MyEthernet::Packet, + replaces Ethernet; + +on MyEthernet::Packet -> event MyEthernet::data($packet, self.ethernet); +# @TEST-END-FILE diff --git a/testing/btest/spicy/packet-analyzer-violation.zeek b/testing/btest/spicy/packet-analyzer-violation.zeek new file mode 100644 index 0000000000..46c2ce4af7 --- /dev/null +++ b/testing/btest/spicy/packet-analyzer-violation.zeek @@ -0,0 +1,30 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -d -o zeek_test.hlto analyzer.spicy analyzer.evt +# @TEST-EXEC: HILTI_DEBUG=spicy zeek -Cr ${TRACES}/spicy/packet-analyzer-violation.pcap zeek_test.hlto %INPUT >output 2>&1 +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Checks that packet analyzers correctly report violations. This is a regression test for #132. + +# @TEST-START-FILE analyzer.spicy +module test; +public type Foo = unit { + data: bytes &until=b"\xbe"; +}; +# @TEST-END-FILE + +# @TEST-START-FILE analyzer.evt +import test; + +packet analyzer spicy::TEST: + parse with test::Foo; +# @TEST-END-FILE + +module TEST; + +event zeek_init() +{ + if ( ! PacketAnalyzer::try_register_packet_analyzer_by_name("Ethernet", 0x6666, + "spicy_TEST") ) + print "cannot register spicy analyzer"; +} diff --git a/testing/btest/spicy/packet-analyzer.zeek b/testing/btest/spicy/packet-analyzer.zeek new file mode 100644 index 0000000000..0712750d3f --- /dev/null +++ b/testing/btest/spicy/packet-analyzer.zeek @@ -0,0 +1,45 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto spicy/raw-layer.pcap.spicy spicy/raw-layer.pcap.evt +# @TEST-EXEC: zeek -r ${TRACES}/spicy/raw-layer.pcap test.hlto %INPUT >output +# @TEST-EXEC: btest-diff output +# @TEST-EXEC: btest-diff weird.log + +module PacketAnalyzer::SPICY_RAWLAYER; + +event zeek_init() + { + PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_ETHERNET, 0x88b5, PacketAnalyzer::ANALYZER_SPICY_RAWLAYER); + + if ( ! PacketAnalyzer::try_register_packet_analyzer_by_name("spicy_RawLayer", 0x4950, "IP") ) + print "cannot register IP analyzer"; + } + +event raw::data(p: raw_pkt_hdr, data: string) + { + print fmt("MACs: src=%s dst=%s", p$l2$src, p$l2$dst); + print "raw data", data; + } + +# @TEST-START-FILE spicy/raw-layer.pcap.spicy +module RawLayer; + +import zeek; + +public type Packet = unit { + data: bytes &size=19; + protocol: uint16; + + on %done { + zeek::forward_packet(self.protocol); + zeek::weird("test_weird"); + } +}; +# @TEST-END-FILE + +# @TEST-START-FILE spicy/raw-layer.pcap.evt +packet analyzer spicy::RawLayer: + parse with RawLayer::Packet; + +on RawLayer::Packet::data -> event raw::data($packet, self.data); +# @TEST-END-FILE diff --git a/testing/btest/spicy/parse-error.zeek b/testing/btest/spicy/parse-error.zeek new file mode 100644 index 0000000000..4dbc0184e3 --- /dev/null +++ b/testing/btest/spicy/parse-error.zeek @@ -0,0 +1,36 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto test.evt test.spicy +# @TEST-EXEC: HILTI_DEBUG=zeek zeek -r ${TRACES}/ssh/single-conn.trace misc/dump-events test.hlto %INPUT +# Zeek versions differ in their quoting of the newline character in dpd.log (two slashes vs one). +# @TEST-EXEC: cat dpd.log | sed 's#\\\\#\\#g' >dpd.log.tmp && mv dpd.log.tmp dpd.log +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff dpd.log +# +# @TEST-DOC: Trigger parse error after confirmation, should be recorded in dpd.log + +# @TEST-START-FILE test.spicy +module SSH; + +import zeek; + +public type Banner = unit { + magic : /SSH-/ { zeek::confirm_protocol(); } + version : /[^-]*/; + dash : /-/; + software: /KAPUTT/; +}; +# @TEST-END-FILE + +# @TEST-START-FILE test.evt + +protocol analyzer spicy::SSH over TCP: + parse originator with SSH::Banner, + port 22/tcp + + # With Zeek < 5.0, DPD tracking doesn't work correctly for replaced + # analyzers because the ProtocolViolation() doesn't take a tag. + # + # replaces SSH + ; + +# @TEST-END-FILE diff --git a/testing/btest/spicy/port-fail.evt b/testing/btest/spicy/port-fail.evt new file mode 100644 index 0000000000..e9bae8ac13 --- /dev/null +++ b/testing/btest/spicy/port-fail.evt @@ -0,0 +1,22 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC-FAIL: spicyz %INPUT -o x.hlto >output 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff output + +protocol analyzer spicy::SSH over TCP: + port 123456/udp; + +@TEST-START-NEXT + +protocol analyzer spicy::SSH over TCP: + port -1/udp; + +@TEST-START-NEXT + +protocol analyzer spicy::SSH over TCP: + port 1/udp-2/tcp; + +@TEST-START-NEXT + +protocol analyzer spicy::SSH over TCP: + port 2/udp-1/udp; diff --git a/testing/btest/spicy/preprocessor-fail.evt b/testing/btest/spicy/preprocessor-fail.evt new file mode 100644 index 0000000000..daeb3e0848 --- /dev/null +++ b/testing/btest/spicy/preprocessor-fail.evt @@ -0,0 +1,37 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC-FAIL: spicyz -D zeek -o x.hlto %INPUT 2>output +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff output + +@if ZEEK_VERSION < 90000 + +# @TEST-START-NEXT + +@if ZEEK_VERSION >= 30000 +@else + +# @TEST-START-NEXT + +@else +@endif + +# @TEST-START-NEXT + +@endif + +# @TEST-START-NEXT + +@if ZEEK_VERSION < 90000 +@if ZEEK_VERSION < 90000 +@if ZEEK_VERSION < 90000 +@else +@endif +# @TEST-START-NEXT + +@if ZEEK_VERSION >= thirtythousand +@endif + +# @TEST-START-NEXT + +@if ZEEK_VERSION >= 3.0.0 +@endif diff --git a/testing/btest/spicy/preprocessor-spicy.spicy b/testing/btest/spicy/preprocessor-spicy.spicy new file mode 100644 index 0000000000..0c061df2b0 --- /dev/null +++ b/testing/btest/spicy/preprocessor-spicy.spicy @@ -0,0 +1,61 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: echo == spicyz >>output +# @TEST-EXEC: spicyz -c x %INPUT +# @TEST-EXEC: cat x_Foo.cc | grep print >>output +# +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Test various Spicy preprocessor constructs +# +# Same test is part of Spicy itself as well, with different results. + +module Foo; + +@if HAVE_ZEEK +print "have zeek"; +@else +print "not have zeek"; +@endif + +@if ! HAVE_ZEEK +print "not have zeek"; +@endif + +@if SPICY_VERSION +print "have spicy version"; +@endif + +@if SPICY_VERSION >= 400 +print "have spicy version >= 0.4"; +@endif + +@if SPICY_VERSION >= 40000 +print "have spicy version >= 4"; +@endif + +@if ! SPICY_VERSION >= 40000 +print "not have spicy version >= 4"; +@endif + +@if UNKNOWiN +no valid Spicy syntax here. +@endif + +@if HAVE_ZEEK + @if ZEEK_VERSION > 10000 + print "have zeek and zeek version > 1.0"; + @else + print "have zeek but zeek version < 1.0"; + @endif +@else + print "have not zeek"; +@endif + +@if HAVE_ZEEK + @if ZEEK_VERSION < 10000 + print "have zeek and zeek version < 1.0"; + @endif +@else + print "have not zeek"; +@endif diff --git a/testing/btest/spicy/preprocessor.evt b/testing/btest/spicy/preprocessor.evt new file mode 100644 index 0000000000..e58e7ed135 --- /dev/null +++ b/testing/btest/spicy/preprocessor.evt @@ -0,0 +1,40 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -D zeek -o x.hlto %INPUT 2>output +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff output + +protocol analyzer YES_1 over TCP: parse with X; + +@if ZEEK_VERSION >= 30000 + protocol analyzer YES_2 over TCP: parse with X; +@endif + +@if ZEEK_VERSION < 90000 + protocol analyzer YES_3 over TCP: parse with X; +@endif + +@if ZEEK_VERSION >= 30000 + protocol analyzer YES_4 over TCP: parse with X; +@else + protocol analyzer NO_4 over TCP: parse with X; +@endif + +@if ZEEK_VERSION < 30000 + protocol analyzer NO_5 over TCP: parse with X; +@else + protocol analyzer YES_5 over TCP: parse with X; +@endif + +@if ZEEK_VERSION >= 20000 + @if ZEEK_VERSION >= 90000 + protocol analyzer NO_6 over TCP: parse with X; + @else + protocol analyzer YES_6 over TCP: parse with X; + @endif +@else + @if ZEEK_VERSION < 90000 + protocol analyzer YES_7 over TCP: parse with X; + @else + protocol analyzer NO_7 over TCP: parse with X; + @endif +@endif diff --git a/testing/btest/spicy/profiling.zeek b/testing/btest/spicy/profiling.zeek new file mode 100644 index 0000000000..f34c2e2087 --- /dev/null +++ b/testing/btest/spicy/profiling.zeek @@ -0,0 +1,37 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -Z -o ssh.hlto ssh.spicy ./ssh.evt +# @TEST-EXEC: zeek -b -r ${TRACES}/ssh/single-conn.trace Zeek::Spicy ssh.hlto %INPUT Spicy::enable_profiling=T >output 2>prof.log.raw +# @TEST-EXEC: cat prof.log.raw | awk '{print $1, $2}' | egrep -v 'zeek/rt/debug|zeek/rt/internal_handler' >prof.log +# @TEST-EXEC: btest-diff output +# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff prof.log +# +# @TEST-DOC: Tests that we get profiling information for the Spicy analyzer. + +event ssh::banner(c: connection, is_orig: bool, version: string, software: string) + { + print "SSH banner", c$id, is_orig, version, software; + } + +# @TEST-START-FILE ssh.spicy +module SSH; + +import spicy; + +public type Banner = unit { + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; +}; +# @TEST-END-FILE + +# @TEST-START-FILE ssh.evt +protocol analyzer spicy::SSH over TCP: + # no port, we're using the signature + parse with SSH::Banner, + port 22/tcp, + replaces SSH; + +on SSH::Banner -> event ssh::banner($conn, $is_orig, self.version, self.software); +# @TEST-END-FILE diff --git a/testing/btest/spicy/protocol-analyzer-data-in.zeek b/testing/btest/spicy/protocol-analyzer-data-in.zeek new file mode 100644 index 0000000000..3c12289f77 --- /dev/null +++ b/testing/btest/spicy/protocol-analyzer-data-in.zeek @@ -0,0 +1,57 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto ssh.spicy ./ssh-cond.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT +# @TEST-EXEC: btest-diff http.log + +# @TEST-START-FILE ssh.spicy +module SSH; + +import spicy; +import zeek; + +type Context = tuple; + +public type Banner = unit { + %context = Context; + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; +}; + +on Banner::%done { + zeek::protocol_begin("HTTP"); + + zeek::protocol_data_in(True, b"GET /etc/passwd1 HTTP/1.0\r\n\r\n"); + zeek::protocol_data_in(False, b"HTTP/1.0 200 OK\r\nContent-Length: 0\r\n\r\n"); + + # We can get a handle to the existing HTTP analyzer. + local http1 = zeek::protocol_handle_get_or_create("HTTP"); + zeek::protocol_data_in(True, b"GET /etc/passwd1.1 HTTP/1.0\r\n\r\n", http1); + + # We can get another handle to the existing HTTP analyzer. + local http2 = zeek::protocol_handle_get_or_create("HTTP"); + zeek::protocol_data_in(False, b"HTTP/1.0 200 OK\r\nContent-Length: 0\r\n\r\n", http2); + zeek::protocol_handle_close(http1); + + zeek::protocol_end(); # Noop. + + # Creating a DPD child analyzer creates no handle. + zeek::protocol_begin(); # DPD + zeek::protocol_data_in(True, b"GET /etc/passwd2 HTTP/1.0\r\n\r\n"); + zeek::protocol_data_in(False, b"HTTP/1.0 200 OK\r\nContent-Length: 0\r\n\r\n"); + zeek::protocol_end(); +} +# @TEST-END-FILE + +# @TEST-START-FILE ssh-cond.evt + +import zeek; + +protocol analyzer spicy::SSH over TCP: + parse originator with SSH::Banner, + port 22/tcp, + replaces SSH; + +# @TEST-END-FILE diff --git a/testing/btest/spicy/protocol-analyzer-explicit-forwarding.zeek b/testing/btest/spicy/protocol-analyzer-explicit-forwarding.zeek new file mode 100644 index 0000000000..d12541efd5 --- /dev/null +++ b/testing/btest/spicy/protocol-analyzer-explicit-forwarding.zeek @@ -0,0 +1,48 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -d -o foo.hlto foo.spicy foo.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace foo.hlto %INPUT Spicy::enable_print=T >output +# @TEST-EXEC: btest-diff output + +# @TEST-START-FILE foo.spicy +module foo; +import zeek; + +public type X = unit { + xs: bytes &eod { + local y = zeek::protocol_handle_get_or_create("spicy_Y"); + local z = zeek::protocol_handle_get_or_create("spicy_Z"); + + zeek::protocol_data_in(zeek::is_orig(), b"only Y", y); + zeek::protocol_data_in(zeek::is_orig(), b"both Y and Z"); + + zeek::protocol_handle_close(z); + zeek::protocol_data_in(zeek::is_orig(), b"only Y after removal of Z"); + + zeek::protocol_handle_close(y); + zeek::protocol_data_in(zeek::is_orig(), b"goes nowhere"); + } +}; + +public type Y = unit { + ys: bytes &eod &chunked { print "ys=%s" % $$; } +}; + +public type Z = unit { + zs: bytes &eod &chunked { print "zs=%s" % $$; } +}; +# @TEST-END-FILE + +# @TEST-START-FILE foo.evt +# Analyzer instantiated from Zeek based on the traffic. +protocol analyzer spicy::X over TCP: + parse originator with foo::X, + port 22/tcp, + replaces SSH; + +# Analyzers which will only be instantiated explicitly by us. +protocol analyzer spicy::Y over TCP: + parse originator with foo::Y; +protocol analyzer spicy::Z over TCP: + parse originator with foo::Z; +# @TEST-END-FILE diff --git a/testing/btest/spicy/protocol-analyzer-tcp-over-udp.spicy b/testing/btest/spicy/protocol-analyzer-tcp-over-udp.spicy new file mode 100644 index 0000000000..02ebcd63dd --- /dev/null +++ b/testing/btest/spicy/protocol-analyzer-tcp-over-udp.spicy @@ -0,0 +1,28 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto %INPUT ./foo.evt +# @TEST-EXEC: zeek -Cr ${TRACES}/ssh/ssh-over-udp.pcap test.hlto +# @TEST-EXEC: btest-diff ssh.log +# +# @TEST-DOC: Pass data from inside a UDP analyzer to a Zeek analyzers that works on top of TCP. Regression tests for #92 and also #91. +# + +module Foo; + +import spicy; +import zeek; + +public type Bar = unit { + on %init { zeek::protocol_begin("SSH"); } + data: bytes &eod { zeek::protocol_data_in(zeek::is_orig(), $$); } +}; + +# @TEST-START-FILE foo.evt + +import zeek; + +protocol analyzer spicy::Foo over UDP: + parse with Foo::Bar, + port 1234/udp; + +# @TEST-END-FILE diff --git a/testing/btest/spicy/replaces-mismatch.zeek b/testing/btest/spicy/replaces-mismatch.zeek new file mode 100644 index 0000000000..593767e2a5 --- /dev/null +++ b/testing/btest/spicy/replaces-mismatch.zeek @@ -0,0 +1,25 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o ssh.hlto ssh.spicy ./ssh.evt +# @TEST-EXEC-FAIL: zeek -r ${TRACES}/ssh/single-conn.trace ssh.hlto >output 2>&1 +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Attempt to replace a packet analyzer with a protocol analyzer + +# @TEST-START-FILE ssh.spicy +module SSH; + +import zeek; + +public type Banner = unit { +}; +# @TEST-END-FILE + +# @TEST-START-FILE ssh.evt + +protocol analyzer spicy::SSH over TCP: + parse with SSH::Banner, + port 22/tcp, + replaces Ethernet; # fail + +# @TEST-END-FILE diff --git a/testing/btest/spicy/replaces.zeek b/testing/btest/spicy/replaces.zeek new file mode 100644 index 0000000000..e5105486e2 --- /dev/null +++ b/testing/btest/spicy/replaces.zeek @@ -0,0 +1,45 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: mkdir -p modules +# @TEST-EXEC: spicyz -o modules/ssh.hlto ssh.spicy ./ssh.evt +# @TEST-EXEC: ZEEK_SPICY_MODULE_PATH=$(pwd)/modules zeek -r ${TRACES}/ssh/single-conn.trace %INPUT | sort >output +# @TEST-EXEC: btest-diff output +# +# We use the module search path for loading here as a regression test for #137. +# Note that this that problem only showed up when the Spicy plugin was built +# into Zeek. + +event ssh::banner(c: connection, is_orig: bool, version: string, software: string) + { + print "SSH banner", c$id, is_orig, version, software; + } + +event analyzer_confirmation(c: connection, atype: AllAnalyzers::Tag, aid: count) + { + print atype, aid; + } + +# @TEST-START-FILE ssh.spicy +module SSH; + +import zeek; + +public type Banner = unit { + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; + + on %done { zeek::confirm_protocol(); } +}; +# @TEST-END-FILE + +# @TEST-START-FILE ssh.evt + +protocol analyzer spicy::SSH over TCP: + parse with SSH::Banner, + port 22/tcp, + replaces SSH; + +on SSH::Banner -> event ssh::banner($conn, $is_orig, self.version, self.software); +# @TEST-END-FILE diff --git a/testing/btest/spicy/resource-usage.zeek b/testing/btest/spicy/resource-usage.zeek new file mode 100644 index 0000000000..dcd4991e25 --- /dev/null +++ b/testing/btest/spicy/resource-usage.zeek @@ -0,0 +1,26 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto test.evt test.spicy +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto Zeek/Spicy/misc/resource-usage | sed 's/=[^ ]*/=XXX/g' >output +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Exercise the misc/resource-usage.zeek script. + +# @TEST-START-FILE test.spicy +module SSH; + +public type Banner = unit { + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /KAPUTT/; +}; +# @TEST-END-FILE + +# @TEST-START-FILE test.evt + +protocol analyzer spicy::SSH over TCP: + parse originator with SSH::Banner, + port 22/tcp; + +# @TEST-END-FILE diff --git a/testing/btest/spicy/reuse-parser-across-hltos.zeek b/testing/btest/spicy/reuse-parser-across-hltos.zeek new file mode 100644 index 0000000000..2cac28bf69 --- /dev/null +++ b/testing/btest/spicy/reuse-parser-across-hltos.zeek @@ -0,0 +1,70 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -d -o foo.hlto foo.spicy foo.evt +# @TEST-EXEC: spicyz -d -o bar.hlto bar.spicy bar.evt foo.spicy +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace foo.hlto %INPUT | sort >>output +# @TEST-EXEC: echo >>output +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace bar.hlto %INPUT | sort >>output +# @TEST-EXEC: echo >>output +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace foo.hlto bar.hlto %INPUT | sort >>output +# @TEST-EXEC: echo >>output +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace bar.hlto foo.hlto %INPUT | sort >>output +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Check that HLTOs remain isolated from each other when reusing another's units. +# +# The events triggered should reflect what's being loaded, and not depend on any loading ordering either. + +event foo::test(x: string) + { + print "foo", x; + } + +event bar::test(x: string) + { + print "bar", x; + } + +# @TEST-START-FILE foo.spicy +module Foo; + +public type Banner = unit { + %port = 22/tcp; + + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; +}; +# @TEST-END-FILE +# +# @TEST-START-FILE foo.evt +import Foo; + +protocol analyzer spicy::Foo over TCP: + parse with Foo::Banner; + +on Foo::Banner -> event foo::test(self.version); +# @TEST-END-FILE + +# @TEST-START-FILE bar.spicy +module Bar; + +import Foo; + +public type Banner = unit { + %port = 22/tcp; + x: Foo::Banner; +}; + +# @TEST-END-FILE +# +# @TEST-START-FILE bar.evt +import Foo; +import Bar; + +protocol analyzer spicy::Bar over TCP: + parse with Bar::Banner; + +on Foo::Banner -> event bar::test(self.version); +# @TEST-END-FILE diff --git a/testing/btest/spicy/spicy-dump.spicy b/testing/btest/spicy/spicy-dump.spicy index 5ce7eca37a..149bf87aa6 100644 --- a/testing/btest/spicy/spicy-dump.spicy +++ b/testing/btest/spicy/spicy-dump.spicy @@ -1,8 +1,9 @@ -# @TEST-DOC: Smoke test for a bundled Spicy. +# @TEST-REQUIRES: have-spicy && test -x ${BUILD}/auxil/spicy/spicy/bin/spicy-dump # -# @TEST-REQUIRES: $SCRIPTS/have-spicy # @TEST-EXEC: printf 12345 | ${BUILD}/auxil/spicy/spicy/bin/spicy-dump -d %INPUT >output # @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Smoke test for a bundled Spicy. module test; diff --git a/testing/btest/spicy/spicyz.test b/testing/btest/spicy/spicyz.test index 7abbf30745..f54ec89009 100644 --- a/testing/btest/spicy/spicyz.test +++ b/testing/btest/spicy/spicyz.test @@ -1,10 +1,11 @@ -# @TEST-DOC: Smoke test for a custom ahead-of-time compiled Spicy analyzer hooked into Zeek. +# @TEST-REQUIRES: have-spicy # -# @TEST-REQUIRES: $SCRIPTS/have-spicy -# @TEST-EXEC: ${BUILD}/src/builtin-plugins/spicy-plugin/bin/spicyz test.spicy test.evt -o test.hlto +# @TEST-EXEC: spicyz test.spicy test.evt -o test.hlto # @TEST-EXEC: zeek -NN test.hlto | grep -q ANALYZER_SPICY_TEST # @TEST-EXEC: zeek -r ${TRACES}/http/post.trace test.zeek test.hlto "Spicy::enable_print = T;" >>output 2>&1 # @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Smoke test for a custom ahead-of-time compiled Spicy analyzer hooked into Zeek. # @TEST-START-FILE test.spicy module test; diff --git a/testing/btest/spicy/ssh-banner.zeek b/testing/btest/spicy/ssh-banner.zeek new file mode 100644 index 0000000000..b4af1d682d --- /dev/null +++ b/testing/btest/spicy/ssh-banner.zeek @@ -0,0 +1,75 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o ssh.hlto ssh.spicy ./ssh.evt +# @TEST-EXEC: echo === confirmation >>output +# @TEST-EXEC: zeek -b -r ${TRACES}/ssh/single-conn.trace -s ./ssh.sig Zeek::Spicy base/frameworks/notice/weird ssh.hlto %INPUT ./extern.zeek | sort >>output +# @TEST-EXEC: btest-diff weird.log +# @TEST-EXEC: echo === violation >>output +# Note: The following removes the payload data from the violation log, as that's a recent addition that breaks older version. Can remove later. +# @TEST-EXEC: zeek -r ${TRACES}/http/post.trace -s ./ssh.sig Zeek::Spicy ssh.hlto ./extern.zeek %INPUT | sed 's/ \[POST.*//g' | sort >>output +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-remove-abspath btest-diff output +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff analyzer.log +# @TEST-EXEC: test '!' -f reporter.log + + +event ssh::banner(c: connection, is_orig: bool, version: string, software: string) + { + print "SSH banner", c$id, is_orig, version, software; + } + +event analyzer_confirmation(c: connection, atype: AllAnalyzers::Tag, aid: count) + { + if ( atype == Analyzer::ANALYZER_SPICY_SSH ) + print "confirm", atype; + } + +event analyzer_violation(c: connection, atype: AllAnalyzers::Tag, aid: count, reason: string) + { + if ( atype == Analyzer::ANALYZER_SPICY_SSH ) + print "violation", atype, reason; + } + +# @TEST-START-FILE extern.zeek + +module Foo; + +event ssh::banner(c: connection, is_orig: bool, version: string, software: string) + { + print "SSH banner in Foo", c$id, is_orig, version, software; + } +# @TEST-END-FILE + +# @TEST-START-FILE ssh.spicy +module SSH; + +import spicy; +import zeek; + +public type Banner = unit { + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/ { zeek::weird("my_weird", $$.decode()); } + + on %done { spicy::accept_input(); assert zeek::uid() == "CHhAvVGS1DHFjwGM9"; } + on %error { spicy::decline_input("kaputt"); } +}; +# @TEST-END-FILE + +# @TEST-START-FILE ssh.sig + +signature ssh_server { + ip-proto == tcp + payload /./ + enable "spicy_SSH" + tcp-state responder +} +# @TEST-END-FILE + +# @TEST-START-FILE ssh.evt +protocol analyzer spicy::SSH over TCP: + # no port, we're using the signature + parse with SSH::Banner; + +on SSH::Banner -> event ssh::banner($conn, $is_orig, self.version, self.software); +# @TEST-END-FILE diff --git a/testing/btest/spicy/terminate-session.zeek b/testing/btest/spicy/terminate-session.zeek new file mode 100644 index 0000000000..09c97d19cc --- /dev/null +++ b/testing/btest/spicy/terminate-session.zeek @@ -0,0 +1,41 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto test.spicy test.evt +# @TEST-EXEC: zeek -b -r ${TRACES}/dns/long-connection.pcap Zeek::Spicy test.hlto base/protocols/conn %INPUT +# @TEST-EXEC: cat conn.log | zeek-cut uid -C > conn.log2 && mv conn.log2 conn.log +# @TEST-EXEC: btest-diff conn.log +# +# @TEST-DOC: Validate that `terminate_session` indeed flushes Zeek-side connection state +# +# We expect to see two conn.log entries instead of one. + +redef likely_server_ports += { 53/udp }; # avoid flipping direction after termination +redef udp_inactivity_timeout = 24hrs; # avoid long gaps to trigger removal + +# @TEST-START-FILE test.spicy +module Test; + +import zeek; + +public type Foo = unit { + on %done { + self.context().counter = self.context().counter + 1; + + # close the connection if it is too long + if ( self.context().counter >= 10 ) + zeek::terminate_session(); + } + x : /./; + + %context = Counter; +}; + +type Counter = tuple; + +# @TEST-END-FILE + +# @TEST-START-FILE test.evt +protocol analyzer spicy::Test over UDP: + port 53/udp, + parse originator with Test::Foo; +# @TEST-END-FILE diff --git a/testing/btest/spicy/toggle-protocol-analyzer.zeek b/testing/btest/spicy/toggle-protocol-analyzer.zeek new file mode 100644 index 0000000000..ef0077a473 --- /dev/null +++ b/testing/btest/spicy/toggle-protocol-analyzer.zeek @@ -0,0 +1,46 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o ssh.hlto ssh.spicy ssh.evt +# @TEST-EXEC: echo === >>output +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace ssh.hlto %INPUT ENABLE=T >>output; +# @TEST-EXEC: echo === >>output +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace ssh.hlto %INPUT ENABLE=F >>output; +# @TEST-EXEC: echo === >>output +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Check operation of Spicy::{enable,disable}_protocol_analyzer() + +const ENABLE = T &redef; + +event zeek_init() { + if ( ENABLE ) + Spicy::enable_protocol_analyzer(Analyzer::ANALYZER_SPICY_SSH); + else + Spicy::disable_protocol_analyzer(Analyzer::ANALYZER_SPICY_SSH); +} + +event ssh::banner(c: connection, is_orig: bool, version: string, software: string) + { + print "Spicy: SSH banner", c$id, is_orig, version, software; + } + +# @TEST-START-FILE ssh.spicy +module SSH; + +import zeek; + +public type Banner = unit { + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; +}; +# @TEST-END-FILE + +# @TEST-START-FILE ssh.evt +protocol analyzer spicy::SSH over TCP: + port 22/tcp, + parse originator with SSH::Banner; + +on SSH::Banner -> event ssh::banner($conn, $is_orig, self.version, self.software); +# @TEST-END-FILE diff --git a/testing/btest/spicy/tuple-arg.zeek b/testing/btest/spicy/tuple-arg.zeek new file mode 100644 index 0000000000..4b8a4be8bc --- /dev/null +++ b/testing/btest/spicy/tuple-arg.zeek @@ -0,0 +1,48 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -d -o test.hlto ssh.spicy ./ssh-tuple.evt +# @TEST-EXEC: HILTI_DEBUG=zeek zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT | sort >output +# @TEST-EXEC: grep event .stderr | sort >>output +# @TEST-EXEC: btest-diff output + +type Foo: record { + i: int; + s: string &optional; +}; + +event ssh::banner(f: Foo) + { + print f; + } + +# @TEST-START-FILE ssh.spicy + +module SSH; + +import zeek; + +public type Banner = unit { + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; + + unset_field: uint64 if ( False ); + + on %done { zeek::confirm_protocol(); } +}; +# @TEST-END-FILE + +# @TEST-START-FILE ssh-tuple.evt + +protocol analyzer spicy::SSH over TCP: + parse originator with SSH::Banner, + port 22/tcp, + replaces SSH; + +on SSH::Banner -> event ssh::banner((1, self.software)); +on SSH::Banner -> event ssh::banner((2, self.?software)); +on SSH::Banner -> event ssh::banner((3, self.?unset_field)); +on SSH::Banner -> event ssh::banner((4, Null)); + +# @TEST-END-FILE diff --git a/testing/btest/spicy/tuple-enum.zeek b/testing/btest/spicy/tuple-enum.zeek new file mode 100644 index 0000000000..9918957b40 --- /dev/null +++ b/testing/btest/spicy/tuple-enum.zeek @@ -0,0 +1,42 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto tupleenum.spicy ./tupleenum.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT Spicy::enable_print=T | sort >output +# @TEST-EXEC: btest-diff output + +type Foo: record { + i: TupleEnum::TestEnum; + j: count; +}; + +event enum_message(f: Foo) { + print f; +} + +# @TEST-START-FILE tupleenum.evt + +protocol analyzer TupleEnum over TCP: + parse with TupleEnum::Message, + port 22/tcp, + replaces SSH; + +on TupleEnum::Message -> event enum_message( (self.a, cast(self.b)) ); + +# @TEST-END-FILE + +# @TEST-START-FILE tupleenum.spicy + +module TupleEnum; + +public type TestEnum = enum { + A = 83, B = 84, C = 85 +}; + +public type Message = unit { + a: uint8 &convert=TestEnum($$); + b: uint8; + + on %done { print self; } +}; + +# @TEST-END-FILE diff --git a/testing/btest/spicy/type-converter.zeek b/testing/btest/spicy/type-converter.zeek new file mode 100644 index 0000000000..310470fd7d --- /dev/null +++ b/testing/btest/spicy/type-converter.zeek @@ -0,0 +1,112 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto conv.spicy ./conv.evt +# @TEST-EXEC: ASAN_OPTIONS=detect_leaks=0 zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT Spicy::enable_print=T >output +# @TEST-EXEC: btest-diff output + +@TEST-START-FILE conv.spicy + +module Conv; + +public type Test = unit { + a: bytes &size=5; + b: int16; + c: uint16; + d: bytes &size=1 &convert=3.14; + e: bytes &size=1 &convert=1.2.3.4; + f: bytes &size=1 &convert=[2001:0db8::1428:57ab]; + g: bytes &size=1 &convert=True; + h: bytes &size=1 &convert="MyString"; + i: bytes &size=1 &convert=time(1295415110.5); + j: bytes &size=1 &convert=interval(4.0); + + var r: MyStruct = [$i = 11]; + var s: set = set(1,2,3); + var t: tuple = (47, "foo"); # Tuple conversion will ignore element names. + var v: vector = vector(b"A", b"B", b"C"); + var l: vector = vector(b"A", b"B", b"C"); + var m: map = map(1: "A", 2: "B", 3: "C"); + + on %done { print self; } +}; + +type MyStruct = struct { + i: int64; + s: string &optional; +}; + +@TEST-END-FILE + + +@TEST-START-FILE conv.evt + +protocol analyzer Conv over TCP: + parse originator with Conv::Test, + port 22/tcp; + +on Conv::Test -> event conv::test($conn, + $is_orig, + self.a, + self.b, + self.c, + self.d, + self.e, + self.f, + self.g, + self.h, + self.i, + self.j, + self.r, + self.s, + self.t, + self.v, + self.l, + self.m + ); + +@TEST-END-FILE + +type MyRecord: record { + i: int; + s: string &optional; +}; + +event conv::test(x: connection, + is_orig: bool, + a: string, + b: int, + c: count, + d: double, + e: addr, + f: addr, + g: bool, + h: string, + i: time, + j: interval, + r: MyRecord, + s: set[count], + t: MyRecord, + v: vector of string, + l: vector of string, + m: table[int] of string + ) + { + print x$id; + print is_orig; + print a; + print b; + print c; + print d; + print e; + print f; + print g; + print h; + print i; + print fmt("%f", j), type_name(j); # print as float as interval format differs between versions + print r; + print s; + print t; + print v; + print l; + print m; + } diff --git a/testing/btest/spicy/udp.zeek b/testing/btest/spicy/udp.zeek new file mode 100644 index 0000000000..d9eef7fd86 --- /dev/null +++ b/testing/btest/spicy/udp.zeek @@ -0,0 +1,33 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto udp-test.spicy ./udp-test.evt +# @TEST-EXEC: zeek -Cr ${TRACES}/udp-packet.pcap test.hlto %INPUT >output +# @TEST-EXEC: btest-diff output + +event zeek_init() + { + # Check we can access the tag. + print Analyzer::ANALYZER_SPICY_UDP_TEST; + } + +event udp_test::message(c: connection, is_orig: bool, data: string) + { + print "UDP packet", c$id, is_orig, data; + } + +# @TEST-START-FILE udp-test.spicy +module UDPTest; + +public type Message = unit { + data: bytes &eod; +}; +# @TEST-END-FILE + +# @TEST-START-FILE udp-test.evt +protocol analyzer spicy::UDP_TEST over UDP: + parse with UDPTest::Message, + port 11337/udp-11340/udp, + ports {31337/udp-31340/udp}; + +on UDPTest::Message -> event udp_test::message($conn, $is_orig, self.data); +# @TEST-END-FILE diff --git a/testing/scripts/diff-canonifier-spicy b/testing/scripts/diff-canonifier-spicy new file mode 100755 index 0000000000..3fb4e2c2ba --- /dev/null +++ b/testing/scripts/diff-canonifier-spicy @@ -0,0 +1,6 @@ +#! /usr/bin/env bash +# +# Remove unstable pieces in Zeek logs produced by thge Spicy tests. + +$(dirname $0)/diff-remove-timestamps | + $(dirname $0)/diff-remove-abspath diff --git a/testing/scripts/spicy-version b/testing/scripts/spicy-version new file mode 100755 index 0000000000..d9a685c04e --- /dev/null +++ b/testing/scripts/spicy-version @@ -0,0 +1,13 @@ +#! /bin/sh +# +# Two usages: +# - Without argument, prints out the numerical Spicy version. +# - With a numerical Spicy version in $1, exit with true iff we have at least that version. + +version=$(spicy-config --version-number) + +if [ $# = 0 ]; then + echo "${version}" +else + test "${version}" -ge "$1" +fi diff --git a/testing/scripts/spicy/README b/testing/scripts/spicy/README new file mode 100644 index 0000000000..e531d0bcf6 --- /dev/null +++ b/testing/scripts/spicy/README @@ -0,0 +1,2 @@ +These are scripts kept for backwards compatibility with existing Spicy +packages. diff --git a/testing/scripts/spicy/canonify-zeek-log b/testing/scripts/spicy/canonify-zeek-log new file mode 100755 index 0000000000..c4f22cea32 --- /dev/null +++ b/testing/scripts/spicy/canonify-zeek-log @@ -0,0 +1,17 @@ +#! /usr/bin/env bash +# +# Remove unstable pieces in Zeek logs. + +# Get us "modern" regexps with sed. +if [ $(uname) == "Linux" ]; then + sed="sed -r" +else + sed="sed -E" +fi + +# File IDs changed between Zeek 3.1 and 3.2. + +${sed} 's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g' | + ${sed} 's/F[A-Za-z0-9]{14,17}/XXXXXXXXXXXXXXXXX/g' | + $(dirname $0)/diff-remove-timestamps | + $(dirname $0)/diff-remove-abspath diff --git a/testing/scripts/spicy/canonify-zeek-log-sorted b/testing/scripts/spicy/canonify-zeek-log-sorted new file mode 100755 index 0000000000..a4ff5ddbc9 --- /dev/null +++ b/testing/scripts/spicy/canonify-zeek-log-sorted @@ -0,0 +1,18 @@ +#! /usr/bin/env bash +# +# Remove unstable pieces in Zeek logs. + +# Get us "modern" regexps with sed. +if [ $(uname) == "Linux" ]; then + sed="sed -r" +else + sed="sed -E" +fi + +# File IDs changed between Zeek 3.1 and 3.2. + +${sed} 's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g' | + ${sed} 's/F[A-Za-z0-9]{15,17}/XXXXXXXXXXXXXXXXX/g' | + $(dirname $0)/diff-sort | + $(dirname $0)/diff-remove-timestamps | + $(dirname $0)/diff-remove-abspath diff --git a/testing/scripts/spicy/diff-remove-abspath b/testing/scripts/spicy/diff-remove-abspath new file mode 100755 index 0000000000..e08d64d5e2 --- /dev/null +++ b/testing/scripts/spicy/diff-remove-abspath @@ -0,0 +1,12 @@ +#! /usr/bin/env bash +# +# Replace absolute paths with the basename. + +if [ $(uname) == "Linux" ]; then + sed="sed -r" +else + sed="sed -E" +fi + +$sed 's#/+#/#g' | + $sed 's#/([^ :/]{1,}/){1,}([^ :/]{1,})#<...>/\2#g' diff --git a/testing/scripts/spicy/diff-remove-timestamps b/testing/scripts/spicy/diff-remove-timestamps new file mode 100755 index 0000000000..30ea6d1817 --- /dev/null +++ b/testing/scripts/spicy/diff-remove-timestamps @@ -0,0 +1,12 @@ +#! /usr/bin/env bash +# +# Replace anything which looks like timestamps with XXXs (including the #start/end markers in logs). + +# Get us "modern" regexps with sed. +if [ $(uname) == "Linux" ]; then + sed="sed -r" +else + sed="sed -E" +fi + +$sed -e 's/(^|[^0-9])([0-9]{9,10}\.[0-9]{1,8})/\1XXXXXXXXXX.XXXXXX/g' -e 's/^ *#(open|close).(19|20)..-..-..-..-..-../#\1 XXXX-XX-XX-XX-XX-XX/g' diff --git a/testing/scripts/spicy/diff-sort b/testing/scripts/spicy/diff-sort new file mode 100755 index 0000000000..66e2893c80 --- /dev/null +++ b/testing/scripts/spicy/diff-sort @@ -0,0 +1,19 @@ +#! /usr/bin/env bash +# +# A diff canonifier that sorts all lines but keeps all comments +# at the top. It also adds a note at the beginning as a reminder +# that the output has been sorted. + +if [ "$TMP" == "" ]; then + TMP=/tmp +fi + +tmp=$TMP/$(basename $0).$$.tmp + +cat >$tmp + +echo "### NOTE: This file has been sorted with $(basename $0)." +cat $tmp | grep ^# +cat $tmp | grep -v ^# | sort -s + +rm -f $tmp diff --git a/testing/scripts/spicy/run-zeek b/testing/scripts/spicy/run-zeek new file mode 100755 index 0000000000..db7af1df93 --- /dev/null +++ b/testing/scripts/spicy/run-zeek @@ -0,0 +1,7 @@ +#! /bin/sh +# +# Wrapper around Zeek to set environment. + +base=$(cd $(dirname $0)/.. && pwd) + +LD_PRELOAD=${ZEEK_LD_PRELOAD} DYLD_INSERT_LIBRARIES=${ZEEK_LD_PRELOAD} ASAN_OPTIONS=detect_leaks=0:detect_odr_violation=0 zeek $@ diff --git a/testing/scripts/spicy/spicy-version b/testing/scripts/spicy/spicy-version new file mode 100755 index 0000000000..bd43ce2462 --- /dev/null +++ b/testing/scripts/spicy/spicy-version @@ -0,0 +1,14 @@ +#! /bin/sh +# +# Two usages: +# - Without argument, prints out the numerical Spicy version. +# - With a numerical Spicy version in $1, exit with true iff we have at least that version. + +base=${TEST_BASE-$(cd $(dirname $0)/.. && pwd)} +nversion=$(spicy-config --version-number) + +if [ $# = 0 ]; then + echo "${nversion}" +else + test "${nversion}" -ge "$1" +fi diff --git a/testing/scripts/spicy/zeek-version b/testing/scripts/spicy/zeek-version new file mode 100755 index 0000000000..4d1a283c5b --- /dev/null +++ b/testing/scripts/spicy/zeek-version @@ -0,0 +1,20 @@ +#! /bin/sh +# +# Two usages: +# - Without argument, prints out the numerical Zeek version. +# - With a numerical Zeek version in $1, exit with true iff we have at least that version. + +base=$(cd $(dirname $0)/.. && pwd) + +version=$(zeek-config --version) +major=$(echo ${version} | cut -d . -f 1) +minor=$(echo ${version} | cut -d . -f 2) +patch=$(echo ${version} | cut -d . -f 3) + +nversion=$((${major} * 10000 + ${minor} * 100 + ${patch})) + +if [ $# = 0 ]; then + echo "${nversion}" +else + test "${nversion}" -ge "$1" +fi diff --git a/zeek-config.h.in b/zeek-config.h.in index e79286e3be..11afd6cae1 100644 --- a/zeek-config.h.in +++ b/zeek-config.h.in @@ -298,3 +298,9 @@ #else #define ZEEK_DISABLE_TSAN #endif + +/* compiled with Spicy support */ +#cmakedefine HAVE_SPICY + +// Version of Spicy that we are compiling against. +#define SPICY_VERSION_NUMBER ${SPICY_VERSION_NUMBER} diff --git a/zeek-config.in b/zeek-config.in index 91182796bc..0940f6fe09 100755 --- a/zeek-config.in +++ b/zeek-config.in @@ -32,6 +32,7 @@ add_path() { echo "$1:$2" } +# When changing any of these, also update "src/spicy/spicyz/config.h.in". include_dir=$(add_path "$include_dir" "@ZEEK_CONFIG_PCAP_INCLUDE_DIR@") include_dir=$(add_path "$include_dir" "@ZEEK_CONFIG_ZLIB_INCLUDE_DIR@") include_dir=$(add_path "$include_dir" "@ZEEK_CONFIG_OPENSSL_INCLUDE_DIR@")