Merge remote-tracking branch 'origin/topic/bbannier/bump-spicy'

* origin/topic/bbannier/bump-spicy:
  [Spicy] Add linker scope to debugging output.
  [Spicy] Remove declarations for unimplemented functions.
  [Spicy] Bump `auxil/spicy/` to latest development snapshot.

CHANGES

@@ -1,3 +1,36 @@
+8.0.0-dev.629 | 2025-07-10 09:38:32 +0200
+
+  * Spicy: Add linker scope to debugging output. (Robin Sommer, Corelight)
+
+  * Spicy: Remove declarations for unimplemented functions. (Benjamin Bannier, Corelight)
+
+  * Spicy: Bump `auxil/spicy/` to latest development snapshot. (Benjamin Bannier & Robin Sommer, Corelight)
+
+  * Spicy: Let `zeek::protocol_handle_close()` send a TCP EOF. (Benjamin Bannier, Corelight)
+
+    Zeek's analyzer API makes it hard to determine during analyzer
+    shutdown whether a regular end-of-data has been reached, or if we're
+    aborting in the middle of a session (e.g., because Zeek missed the
+    remaining packets): the corresponding analyzer method, `EndOfData()`
+    gets called in both cases.
+
+    In an earlier change, we had stopped signaling Spicy analyzers a
+    regular finish when that `EndOfData()` method executes, because doing
+    so could trigger a parse error if it wasn't a regular shutdown—-which
+    isn't desired, a user request was to just silently stop processing in
+    this case.
+
+    However, that behavior now seems unfortunate in the case that one
+    deliberately calls `zeek::protocol_handle_close()` to terminate an
+    analyzer: this feels like a regular shutdown that should just
+    immediately happen. We achieve this now in this function by
+    additionally signaling the shutdown at the TCP layer as an "end of
+    file", which, for Spicy analyzers, happens to run the final, orderly
+    tear-down.
+
+    Not exactly great, but ti seems to thread the needle to achieve the
+    desired semantics in both cases.
+
 8.0.0-dev.622 | 2025-07-09 15:10:03 -0700
 
   * ZAM optimizer fix for += / -= set operations (Vern Paxson, Corelight)

VERSION

@@ -1 +1 @@
-8.0.0-dev.622
+8.0.0-dev.629

auxil/spicy

@@ -1 +1 @@
-Subproject commit c26202767742818a0ef0e5538ea7df5ca54620b1
+Subproject commit f9cf2115cf330e7dff61f438ba04c96d9f1a132c

scripts/spicy/zeek_rt.hlt

@@ -18,9 +18,9 @@ type ZeekTypeTag = enum {
 } &cxxname="::zeek::spicy::rt::ZeekTypeTag";
 
 declare public void register_spicy_module_begin(string name, string description) &cxxname="zeek::spicy::rt::register_spicy_module_begin";
-declare public void register_protocol_analyzer(string name, hilti::Protocol protocol, vector<PortRange> ports, string parser_orig, string parser_resp, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_protocol_analyzer" &have_prototype;
+declare public void register_protocol_analyzer(string name, hilti::Protocol protocol, vector<PortRange> ports, string parser_orig, string parser_resp, string replaces, uint<64> linker_scope) &cxxname="zeek::spicy::rt::register_protocol_analyzer" &have_prototype;
-declare public void register_file_analyzer(string name, vector<string> mime_types, string parser, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_file_analyzer" &have_prototype;
+declare public void register_file_analyzer(string name, vector<string> mime_types, string parser, string replaces, uint<64> linker_scope) &cxxname="zeek::spicy::rt::register_file_analyzer" &have_prototype;
-declare public void register_packet_analyzer(string name, string parser, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_packet_analyzer" &have_prototype;
+declare public void register_packet_analyzer(string name, string parser, string replaces, uint<64> linker_scope) &cxxname="zeek::spicy::rt::register_packet_analyzer" &have_prototype;
 declare public void register_type(string ns, string id, BroType t) &cxxname="zeek::spicy::rt::register_type" &have_prototype;
 declare public void register_spicy_module_end() &cxxname="zeek::spicy::rt::register_spicy_module_end";
 

src/spicy/manager.cc

@@ -64,8 +64,9 @@ void Manager::registerSpicyModuleEnd() {
 void Manager::registerProtocolAnalyzer(const std::string& name, hilti::rt::Protocol proto,
                                        const hilti::rt::Vector<::zeek::spicy::rt::PortRange>& ports,
                                        const std::string& parser_orig, const std::string& parser_resp,
-                                       const std::string& replaces, const std::string& linker_scope) {
+                                       const std::string& replaces,
-    SPICY_DEBUG(hilti::rt::fmt("Have Spicy protocol analyzer %s", name));
+                                       const hilti::rt::integer::safe<uint64_t>& linker_scope) {
+    SPICY_DEBUG(hilti::rt::fmt("Have Spicy protocol analyzer %s (scope 0x%" PRIx64 ")", name, linker_scope.Ref()));
 
     ProtocolAnalyzerInfo info;
     info.name_analyzer = name;
@@ -125,8 +126,8 @@ void Manager::registerProtocolAnalyzer(const std::string& name, hilti::rt::Proto
 
 void Manager::registerFileAnalyzer(const std::string& name, const hilti::rt::Vector<std::string>& mime_types,
                                    const std::string& parser, const std::string& replaces,
-                                   const std::string& linker_scope) {
+                                   const hilti::rt::integer::safe<uint64_t>& linker_scope) {
-    SPICY_DEBUG(hilti::rt::fmt("Have Spicy file analyzer %s", name));
+    SPICY_DEBUG(hilti::rt::fmt("Have Spicy file analyzer %s (scope 0x%" PRIx64 ")", name, linker_scope.Ref()));
 
     FileAnalyzerInfo info;
     info.name_analyzer = name;
@@ -170,8 +171,8 @@ void Manager::registerFileAnalyzer(const std::string& name, const hilti::rt::Vec
 }
 
 void Manager::registerPacketAnalyzer(const std::string& name, const std::string& parser, const std::string& replaces,
-                                     const std::string& linker_scope) {
+                                     const hilti::rt::integer::safe<uint64_t>& linker_scope) {
-    SPICY_DEBUG(hilti::rt::fmt("Have Spicy packet analyzer %s", name));
+    SPICY_DEBUG(hilti::rt::fmt("Have Spicy packet analyzer %s (scope 0x%" PRIx64 ")", name, linker_scope.Ref()));
 
     PacketAnalyzerInfo info;
     info.name_analyzer = name;
@@ -651,7 +652,7 @@ void Manager::InitPostScript() {
 
     // Fill in the parser information now that we derived from the ASTs.
     auto find_parser = [](const std::string& analyzer, const std::string& parser,
-                          const std::string& linker_scope) -> const ::spicy::rt::Parser* {
+                          const auto& linker_scope) -> const ::spicy::rt::Parser* {
         if ( parser.empty() )
             return nullptr;
 
@@ -660,8 +661,8 @@ void Manager::InitPostScript() {
                 return p;
         }
 
-        reporter->InternalError("Unknown Spicy parser '%s' requested by analyzer '%s'", parser.c_str(),
+        reporter->InternalError("Unknown Spicy parser '%s' (scope 0x%" PRIx64 ") requested by analyzer '%s'",
-                                analyzer.c_str());
+                                parser.c_str(), linker_scope.Ref(), analyzer.c_str());
 
         return nullptr; // cannot be reached
     };
@@ -671,7 +672,8 @@ void Manager::InitPostScript() {
             // vector element not set
             continue;
 
-        SPICY_DEBUG(hilti::rt::fmt("Registering %s protocol analyzer %s with Zeek", p.protocol, p.name_analyzer));
+        SPICY_DEBUG(hilti::rt::fmt("Registering %s protocol analyzer %s (scope 0x%" PRIx64 ") with Zeek", p.protocol,
+                                   p.name_analyzer, p.linker_scope));
 
         p.parser_orig = find_parser(p.name_analyzer, p.name_parser_orig, p.linker_scope);
         p.parser_resp = find_parser(p.name_analyzer, p.name_parser_resp, p.linker_scope);
@@ -728,7 +730,8 @@ void Manager::InitPostScript() {
             // vector element not set
             continue;
 
-        SPICY_DEBUG(hilti::rt::fmt("Registering file analyzer %s with Zeek", p.name_analyzer.c_str()));
+        SPICY_DEBUG(hilti::rt::fmt("Registering file analyzer %s (scope 0x%" PRIx64 ") with Zeek",
+                                   p.name_analyzer.c_str(), p.linker_scope.Ref()));
 
         p.parser = find_parser(p.name_analyzer, p.name_parser, p.linker_scope);
 
@@ -763,7 +766,8 @@ void Manager::InitPostScript() {
             // vector element not set
             continue;
 
-        SPICY_DEBUG(hilti::rt::fmt("Registering packet analyzer %s with Zeek", p.name_analyzer.c_str()));
+        SPICY_DEBUG(hilti::rt::fmt("Registering packet analyzer %s (scope 0x%" PRIx64 ") with Zeek",
+                                   p.name_analyzer.c_str(), p.linker_scope.Ref()));
         p.parser = find_parser(p.name_analyzer, p.name_parser, p.linker_scope);
     }
 

src/spicy/manager.h

@@ -2,15 +2,16 @@
 
 #pragma once
 
+#include <cstdint>
 #include <memory>
 #include <set>
 #include <string>
-#include <tuple>
 #include <unordered_map>
 #include <vector>
 
 #include <hilti/rt/library.h>
 #include <hilti/rt/logging.h>
+#include <hilti/rt/safe-int.h>
 #include <hilti/rt/types/port.h>
 
 #include "zeek/Scope.h"
@@ -99,7 +100,7 @@ public:
     void registerProtocolAnalyzer(const std::string& name, hilti::rt::Protocol proto,
                                   const hilti::rt::Vector<::zeek::spicy::rt::PortRange>& ports,
                                   const std::string& parser_orig, const std::string& parser_resp,
-                                  const std::string& replaces, const std::string& linker_scope);
+                                  const std::string& replaces, const hilti::rt::integer::safe<uint64_t>& linker_scope);
 
     /**
      * Runtime method to register a file analyzer with its Zeek-side
@@ -117,7 +118,8 @@ public:
      * registration
      */
     void registerFileAnalyzer(const std::string& name, const hilti::rt::Vector<std::string>& mime_types,
-                              const std::string& parser, const std::string& replaces, const std::string& linker_scope);
+                              const std::string& parser, const std::string& replaces,
+                              const hilti::rt::integer::safe<uint64_t>& linker_scope);
 
     /**
      * Runtime method to register a packet analyzer with its Zeek-side
@@ -134,7 +136,7 @@ public:
      * registration
      */
     void registerPacketAnalyzer(const std::string& name, const std::string& parser, const std::string& replaces,
-                                const std::string& linker_scope);
+                                const hilti::rt::integer::safe<uint64_t>& linker_scope);
 
     /**
      * Runtime method to register a Spicy-generated type with Zeek. The type
@@ -345,7 +347,7 @@ private:
         std::string name_replaces;
         hilti::rt::Protocol protocol = hilti::rt::Protocol::Undef;
         std::vector<::zeek::spicy::rt::PortRange> ports; // we keep this sorted
-        std::string linker_scope;
+        hilti::rt::integer::safe<uint64_t> linker_scope;
 
         // Computed and available once the analyzer has been registered.
         std::string name_zeek;
@@ -371,7 +373,7 @@ private:
         std::string name_parser;
         std::string name_replaces;
         hilti::rt::Vector<std::string> mime_types;
-        std::string linker_scope;
+        hilti::rt::integer::safe<uint64_t> linker_scope;
 
         // Computed and available once the analyzer has been registered.
         std::string name_zeek;
@@ -395,7 +397,7 @@ private:
         std::string name_analyzer;
         std::string name_parser;
         std::string name_replaces;
-        std::string linker_scope;
+        hilti::rt::integer::safe<uint64_t> linker_scope;
 
         // Computed and available once the analyzer has been registered.
         std::string name_zeek;

src/spicy/runtime-support.cc

@@ -29,20 +29,21 @@ void rt::register_spicy_module_end() { spicy_mgr->registerSpicyModuleEnd(); }
 void rt::register_protocol_analyzer(const std::string& name, hilti::rt::Protocol proto,
                                     const hilti::rt::Vector<::zeek::spicy::rt::PortRange>& ports,
                                     const std::string& parser_orig, const std::string& parser_resp,
-                                    const std::string& replaces, const std::string& linker_scope) {
+                                    const std::string& replaces,
+                                    const hilti::rt::integer::safe<uint64_t>& linker_scope) {
     auto _ = hilti::rt::profiler::start("zeek/rt/register_protocol_analyzer");
     spicy_mgr->registerProtocolAnalyzer(name, proto, ports, parser_orig, parser_resp, replaces, linker_scope);
 }
 
 void rt::register_file_analyzer(const std::string& name, const hilti::rt::Vector<std::string>& mime_types,
                                 const std::string& parser, const std::string& replaces,
-                                const std::string& linker_scope) {
+                                const hilti::rt::integer::safe<uint64_t>& linker_scope) {
     auto _ = hilti::rt::profiler::start("zeek/rt/register_file_analyzer");
     spicy_mgr->registerFileAnalyzer(name, mime_types, parser, replaces, linker_scope);
 }
 
 void rt::register_packet_analyzer(const std::string& name, const std::string& parser, const std::string& replaces,
-                                  const std::string& linker_scope) {
+                                  const hilti::rt::integer::safe<uint64_t>& linker_scope) {
     auto _ = hilti::rt::profiler::start("zeek/rt/register_packet_analyzer");
     spicy_mgr->registerPacketAnalyzer(name, parser, replaces, linker_scope);
 }

src/spicy/runtime-support.h

@@ -6,6 +6,7 @@
 
 #pragma once
 
+#include <cstdint>
 #include <optional>
 #include <ostream>
 #include <string>
@@ -17,6 +18,7 @@
 #include <hilti/rt/extension-points.h>
 #include <hilti/rt/fmt.h>
 #include <hilti/rt/profiler.h>
+#include <hilti/rt/safe-int.h>
 #include <hilti/rt/type-info.h>
 #include <hilti/rt/types/all.h>
 #include <hilti/rt/util.h>
@@ -111,14 +113,15 @@ void register_spicy_module_begin(const std::string& id, const std::string& descr
 void register_protocol_analyzer(const std::string& id, hilti::rt::Protocol proto,
                                 const hilti::rt::Vector<::zeek::spicy::rt::PortRange>& ports,
                                 const std::string& parser_orig, const std::string& parser_resp,
-                                const std::string& replaces, const std::string& linker_scope);
+                                const std::string& replaces, const hilti::rt::integer::safe<uint64_t>& linker_scope);
 
 /**
  * Registers a Spicy file analyzer with its EVT meta information with the
  * plugin's runtime.
  */
 void register_file_analyzer(const std::string& id, const hilti::rt::Vector<std::string>& mime_types,
-                            const std::string& parser, const std::string& replaces, const std::string& linker_scope);
+                            const std::string& parser, const std::string& replaces,
+                            const hilti::rt::integer::safe<uint64_t>& linker_scope);
 
 /** Reports a Zeek-side "weird". */
 void weird(const std::string& id, const std::string& addl);
@@ -128,7 +131,7 @@ void weird(const std::string& id, const std::string& addl);
  * plugin's runtime.
  */
 void register_packet_analyzer(const std::string& id, const std::string& parser, const std::string& replaces,
-                              const std::string& linker_scope);
+                              const hilti::rt::integer::safe<uint64_t>& linker_scope);
 
 /** Registers a Spicy-generated type to make it available inside Zeek. */
 void register_type(const std::string& ns, const std::string& id, const TypePtr& type);

src/spicy/spicyz/driver.h

@@ -133,32 +133,6 @@ public:
     /** Returns the glue compiler in use by the driver. */
     const auto* glueCompiler() const { return _glue.get(); }
 
-    /**
-     * Parses some options command-line style *before* Zeek-side scripts have
-     * been processed. Most of the option processing happens in
-     * `parseOptionsPostScript()` instead, except for things that must be in
-     * place already before script processing.
-     *
-     * @param options space-separated string of command line argument to parse
-     * @return success if all argument could be parsed, or a suitable error message
-     */
-    static hilti::Result<hilti::Nothing> parseOptionsPreScript(const std::string& options);
-
-    /**
-     * Parses options command-line style after Zeek-side scripts have been
-     * fully processed. Most of the option processing happens here (vs. in
-     * `parseOptionsPreScript()`) except for things that must be in place
-     * already before script processing.
-     *
-     * @param options space-separated string of command line argument to parse
-     * @param driver_options instance of options to update per parsed arguments
-     * @param compiler_options instance of options to update per parsed arguments
-     * @return success if all argument could be parsed, or a suitable error message
-     */
-    static hilti::Result<hilti::Nothing> parseOptionsPostScript(const std::string& options,
-                                                                hilti::driver::Options* driver_options,
-                                                                hilti::Options* compiler_options);
-
     /** Prints a usage message for options supported by `parseOptions{Pre,Post}Script()`. */
     static void usage(std::ostream& out);