Merge remote-tracking branch 'origin/topic/robin/node-rewrite'

* origin/topic/robin/node-rewrite:
  Port Spicy integration to new AST API.

CHANGES

@@ -1,3 +1,7 @@
+7.0.0-dev.34 | 2024-03-04 15:49:19 +0100
+
+  * Port Spicy integration to new AST API. (Robin Sommer, Corelight)
+
 7.0.0-dev.32 | 2024-02-28 17:10:26 +0100
 
   * Bump zeekjs to 0.12.0 (Arne Welzel, Corelight)

VERSION

@@ -1 +1 @@
-7.0.0-dev.32
+7.0.0-dev.34

auxil/spicy

@@ -1 +1 @@
-Subproject commit c39bdb5ed2bff0d390e6eae4ddda83050f047904
+Subproject commit b1c3c4e4eae55ed1924f90c5705d78be9dbfdccc

src/analyzer/protocol/ldap/asn1.spicy

@@ -121,7 +121,7 @@ type ASN1OctetString = unit(len: uint64, constructed: bool) {
 # https://www.obj-sys.com/asn1tutorial/node124.html
 
 type ASN1String = unit(tag: ASN1Tag, len: uint64) {
-  var encoding: hilti::Charset;
+  var encoding: spicy::Charset;
 
   on %init {
     switch ( tag.type_ ) {
@@ -132,7 +132,7 @@ type ASN1String = unit(tag: ASN1Tag, len: uint64) {
       case ASN1Type::PrintableString,
            ASN1Type::GeneralizedTime,
            ASN1Type::UTCTime: {
-        self.encoding = hilti::Charset::ASCII;
+        self.encoding = spicy::Charset::ASCII;
       }
 
       case ASN1Type::UTF8String,
@@ -148,7 +148,7 @@ type ASN1String = unit(tag: ASN1Tag, len: uint64) {
            # BMPString and UniversalString. This *may* not be correct.
            ASN1Type::BMPString,
            ASN1Type::UniversalString: {
-        self.encoding = hilti::Charset::UTF8;
+        self.encoding = spicy::Charset::UTF8;
       }
     }
   }
@@ -216,7 +216,7 @@ public type ASN1Body = unit(head: ASN1Header, recursive: bool) {
     ASN1Type::BitString -> bitstr_value: ASN1BitString(head.len.len, head.tag.constructed);
 
     ASN1Type::OctetString -> str_value: ASN1OctetString(head.len.len, head.tag.constructed)
-                                        &convert=$$.value.decode(hilti::Charset::ASCII);
+                                        &convert=$$.value.decode(spicy::Charset::ASCII);
 
     ASN1Type::ObjectIdentifier -> str_value: ASN1ObjectIdentifier(head.len.len)
                                              &convert=$$.oidstring;

src/analyzer/protocol/ldap/ldap.spicy

@@ -733,7 +733,7 @@ type SearchFilter = unit {
     FilterType::FILTER_EXT     -> FILTER_EXT:     DecodedAttributeValue(FilterType::FILTER_EXT)
                                                   &parse-from=self.filterBytes;
     FilterType::FILTER_PRESENT -> FILTER_PRESENT: ASN1::ASN1OctetString(self.filterLen, False)
-                                                  &convert=$$.value.decode(hilti::Charset::ASCII)
+                                                  &convert=$$.value.decode(spicy::Charset::ASCII)
                                                   &parse-from=self.filterBytes;
   };
 

src/analyzer/protocol/quic/QUIC.spicy

@@ -280,7 +280,7 @@ public type LongHeaderPacket = unit {
 };
 
 # A QUIC Frame.
-public type Frame = unit(header: LongHeaderPacket, from_client: bool, crypto_sink: sink) {
+public type Frame = unit(header: LongHeaderPacket, from_client: bool, inout crypto_sink: sink) {
   frame_type : uint8 &convert=cast<FrameType>($$);
 
   # TODO: add other FrameTypes as well
@@ -447,7 +447,7 @@ type CryptoBuffer = unit() {
 #
 # A UDP datagram contains one or more QUIC packets.
 ##############
-type Packet = unit(from_client: bool, context: ConnectionIDInfo&) {
+type Packet = unit(from_client: bool, inout context: ConnectionIDInfo&) {
   var decrypted_data: bytes;
   var full_packet: bytes;
   var start: iterator<stream>;

src/spicy/spicyz/driver.cc

@@ -16,11 +16,12 @@
 #include <hilti/ast/declarations/type.h>
 #include <hilti/compiler/init.h>
 
-#include <spicy/ast/detail/visitor.h>
 #include <spicy/ast/types/unit.h>
+#include <spicy/ast/visitor.h>
 #include <spicy/autogen/config.h>
 #include <spicy/compiler/init.h>
 
+#include "compiler/plugin.h"
 #include "config.h"
 #include "glue-compiler.h"
 
@@ -31,28 +32,45 @@ using Driver = ::zeek::spicy::Driver;
  * Visitor to type information from a HILTI AST. This extracts user-visible
  * types only, we skip any internal ones.
  */
-struct VisitorTypes : public hilti::visitor::PreOrder<void, VisitorTypes> {
+struct VisitorTypes : public spicy::visitor::PreOrder {
-    explicit VisitorTypes(Driver* driver, hilti::ID module, hilti::rt::filesystem::path path, bool is_resolved)
+    explicit VisitorTypes(Driver* driver, GlueCompiler* glue, bool is_resolved)
-        : driver(driver), module(std::move(module)), path(std::move(path)), is_resolved(is_resolved) {}
+        : driver(driver), glue(glue), is_resolved(is_resolved) {}
 
-    void operator()(const hilti::declaration::Type& t) {
+    void operator()(hilti::declaration::Module* n) final {
-        assert(! t.type().typeID() || *t.type().typeID() == hilti::ID(module, t.id())); // ensure consistent IDs
+        if ( n->uid().in_memory ) {
+            // Ignore modules built by us in memory.
+            module = {};
+            return;
+        }
 
-        if ( module == hilti::ID("hilti") || module == hilti::ID("spicy_rt") || module == hilti::ID("zeek_rt") )
+        module = n->scopeID();
+        path = n->uid().path;
+
+        if ( is_resolved )
+            glue->addSpicyModule(module, path);
+    }
+
+    void operator()(hilti::declaration::Type* n) final {
+        if ( module.empty() || module == hilti::ID("hilti") || module == hilti::ID("spicy_rt") ||
+             module == hilti::ID("zeek_rt") )
             return;
 
+        assert(! n->type()->type()->typeID() ||
+               n->type()->type()->typeID() == hilti::ID(module, n->id())); // ensure consistent IDs
+
         types.emplace_back(TypeInfo{
-            .id = hilti::ID(module, t.id()),
+            .id = hilti::ID(module, n->id()),
-            .type = t.type()._clone().as<hilti::Type>(),
+            .type = n->type(),
-            .linkage = t.linkage(),
+            .linkage = n->linkage(),
             .is_resolved = is_resolved,
             .module_id = module,
             .module_path = path,
-            .location = t.meta().location(),
+            .location = n->meta().location(),
         });
     }
 
     Driver* driver;
+    GlueCompiler* glue;
     hilti::ID module;
     hilti::rt::filesystem::path path;
     bool is_resolved;
@@ -61,8 +79,8 @@ struct VisitorTypes : public hilti::visitor::PreOrder<void, VisitorTypes> {
 
 Driver::Driver(std::unique_ptr<GlueCompiler> glue, const char* argv0, hilti::rt::filesystem::path lib_path,
                int zeek_version)
-    : ::spicy::Driver("<Spicy Plugin for Zeek>"), _glue(std::move(glue)) {
+    : ::spicy::Driver("<Spicy support for Zeek>"), _glue(std::move(glue)) {
-    _glue->Init(this, zeek_version);
+    _glue->init(this, zeek_version);
 
     ::spicy::Configuration::extendHiltiConfiguration();
     auto options = hiltiOptions();
@@ -195,6 +213,8 @@ hilti::Result<hilti::Nothing> Driver::compile() {
 
     SPICY_DEBUG("Running Spicy driver");
 
+    _error = hilti::Nothing();
+
     if ( auto x = ::spicy::Driver::compile(); ! x )
         return x.error();
 
@@ -243,23 +263,16 @@ std::vector<std::pair<TypeInfo, hilti::ID>> Driver::exportedTypes() const {
     return result;
 }
 
-void Driver::hookNewASTPreCompilation(std::shared_ptr<hilti::Unit> unit) {
+void Driver::hookNewASTPreCompilation(const hilti::Plugin& plugin, const std::shared_ptr<hilti::ASTRoot>& root) {
-    if ( unit->extension() != ".spicy" )
+    auto v = VisitorTypes(this, _glue.get(), false);
-        return;
+    hilti::visitor::visit(v, root, ".spicy");
-
-    if ( unit->path().empty() )
-        // Ignore modules constructed in memory.
-        return;
-
-    auto v = VisitorTypes(this, unit->id(), unit->path(), false);
-    for ( auto i : v.walk(unit->module()) )
-        v.dispatch(i);
 
     for ( const auto& ti : v.types ) {
         SPICY_DEBUG(hilti::util::fmt("  Got type '%s' (pre-compile)", ti.id));
         _types[ti.id] = ti;
 
-        if ( auto et = ti.type.tryAs<hilti::type::Enum>(); et && ti.linkage == hilti::declaration::Linkage::Public ) {
+        if ( auto et = ti.type->type()->tryAs<hilti::type::Enum>();
+             et && ti.linkage == hilti::declaration::Linkage::Public ) {
             SPICY_DEBUG("    Automatically exporting public enum for backwards compatibility");
             _public_enums.push_back(ti);
         }
@@ -268,17 +281,17 @@ void Driver::hookNewASTPreCompilation(std::shared_ptr<hilti::Unit> unit) {
     }
 }
 
-void Driver::hookNewASTPostCompilation(std::shared_ptr<hilti::Unit> unit) {
+bool Driver::hookNewASTPostCompilation(const hilti::Plugin& plugin, const std::shared_ptr<hilti::ASTRoot>& root) {
-    if ( unit->extension() != ".spicy" )
+    if ( plugin.component != "Spicy" )
-        return;
+        return false;
 
-    if ( unit->path().empty() )
+    if ( ! _need_glue )
-        // Ignore modules constructed in memory.
+        return false;
-        return;
 
-    auto v = VisitorTypes(this, unit->id(), unit->path(), true);
+    _need_glue = false;
-    for ( auto i : v.walk(unit->module()) )
+
-        v.dispatch(i);
+    auto v = VisitorTypes(this, _glue.get(), true);
+    hilti::visitor::visit(v, root, ".spicy");
 
     for ( auto&& t : v.types ) {
         SPICY_DEBUG(hilti::util::fmt("  Got type '%s' (post-compile)", t.id));
@@ -286,19 +299,14 @@ void Driver::hookNewASTPostCompilation(std::shared_ptr<hilti::Unit> unit) {
         hookNewType(t);
     }
 
-    _glue->addSpicyModule(unit->id(), unit->path());
+    if ( ! _glue->compile() )
+        _error = hilti::result::Error("glue compilation failed");
+
+    return true;
 }
 
-hilti::Result<hilti::Nothing> Driver::hookCompilationFinished(const hilti::Plugin& plugin) {
+hilti::Result<hilti::Nothing> Driver::hookCompilationFinished(const std::shared_ptr<hilti::ASTRoot>& root) {
-    if ( ! _need_glue )
+    return _error;
-        return hilti::Nothing();
-
-    _need_glue = false;
-
-    if ( _glue->compile() )
-        return hilti::Nothing();
-    else
-        return hilti::result::Error("glue compilation failed");
 }
 
 void Driver::hookInitRuntime() { ::spicy::rt::init(); }

src/spicy/spicyz/driver.h

@@ -32,7 +32,7 @@ class GlueCompiler;
 
 struct TypeInfo {
     hilti::ID id;                        /**< fully-qualified name of the type */
-    hilti::Type type;                    /**< the type itself */
+    hilti::QualifiedTypePtr type;        /**< the type itself */
     hilti::declaration::Linkage linkage; /**< linkage of of the type's declaration */
     bool is_resolved = false; /**< true if we are far enough in processing that the type has been fully resolved */
     hilti::ID module_id;      /**< name of module type is defined in */
@@ -102,7 +102,7 @@ public:
         if ( ! ti )
             return ti.error();
 
-        if ( ! ti->type.isA<T>() )
+        if ( ! ti->type->type()->isA<T>() )
             return hilti::result::Error(hilti::util::fmt("'%s' is not of expected type", id));
 
         return ti;
@@ -165,7 +165,7 @@ public:
 protected:
     /**
      * Hook executed for all type declarations encountered in a Spicy module.
-     * Derived classes may override this to add custom processing. This hooks
+     * Derived classes may override this to add custom processing. This hook
      * executes twices for each declaration: once before we compile the AST
      * (meaning types have not been resolved yet), and once after. The type
      * info's `is_resolved` field indicates which of the two we're in.
@@ -175,13 +175,13 @@ protected:
     virtual void hookNewType(const TypeInfo& ti) {}
 
     /** Overridden from HILTI driver. */
-    void hookNewASTPreCompilation(std::shared_ptr<hilti::Unit> unit) override;
+    void hookNewASTPreCompilation(const hilti::Plugin& plugin, const std::shared_ptr<hilti::ASTRoot>& root) override;
 
     /** Overridden from HILTI driver. */
-    void hookNewASTPostCompilation(std::shared_ptr<hilti::Unit> unit) override;
+    bool hookNewASTPostCompilation(const hilti::Plugin& plugin, const std::shared_ptr<hilti::ASTRoot>& root) override;
 
     /** Overridden from HILTI driver. */
-    hilti::Result<hilti::Nothing> hookCompilationFinished(const hilti::Plugin& plugin) override;
+    hilti::Result<hilti::Nothing> hookCompilationFinished(const std::shared_ptr<hilti::ASTRoot>& root) override;
 
     /** Overridden from HILTI driver. */
     void hookInitRuntime() override;
@@ -194,6 +194,7 @@ protected:
     std::vector<TypeInfo> _public_enums;            // tracks Spicy enum types declared public, for automatic export
     bool _using_build_directory = false;            // true if we're running out of the plugin's build directory
     bool _need_glue = true;                         // true if glue code has not yet been generated
+    hilti::Result<hilti::Nothing> _error = hilti::Nothing(); // error encountered during compilation
 };
 
 } // namespace zeek::spicy

src/spicy/spicyz/glue-compiler.h

@@ -18,12 +18,13 @@
 
 #include <hilti/ast/builder/builder.h>
 #include <hilti/ast/declarations/function.h>
+#include <hilti/ast/declarations/module.h>
 #include <hilti/ast/expression.h>
-#include <hilti/ast/module.h>
 #include <hilti/ast/type.h>
 #include <hilti/compiler/context.h>
 #include <hilti/compiler/driver.h>
 
+#include <spicy/ast/builder/builder.h>
 #include <spicy/ast/declarations/unit-hook.h>
 #include <spicy/ast/types/unit.h>
 
@@ -100,7 +101,7 @@ struct SpicyModule {
     std::set<hilti::rt::filesystem::path> evts; /**< EVT files that refer to this module. */
 
     // Generated code.
-    std::optional<hilti::Module> spicy_module; /**< the ``BroHooks_*.spicy`` module. */
+    hilti::ModulePtr spicy_module; /**< the ``BroHooks_*.spicy`` module. */
 };
 
 /** Representation of an event parsed from an EVT file. */
@@ -109,7 +110,7 @@ struct Event {
     hilti::rt::filesystem::path file;             /**< The path of the *.evt file we parsed this from. */
     hilti::ID name;                               /**< The name of the event. */
     hilti::ID path;                               /**< The hook path as specified in the evt file. */
-    std::vector<hilti::type::function::Parameter> parameters; /**< Event parameters specified in the evt file. */
+    hilti::type::function::Parameters parameters; /**< Event parameters specified in the evt file. */
     std::string condition;                        /**< Condition that must be true for the event to trigger. */
     std::vector<std::string> exprs;               /**< The argument expressions. */
     int priority;                                 /**< Event/hook priority. */
@@ -118,7 +119,7 @@ struct Event {
     // Computed information.
     hilti::ID hook;                               /**< The name of the hook triggering the event. */
     hilti::ID unit;                               /**< The fully qualified name of the unit type. */
-    std::optional<::spicy::type::Unit> unit_type; /**< The Spicy type of referenced unit. */
+    ::spicy::type::UnitPtr unit_type;             /**< The Spicy type of referenced unit. */
     hilti::ID unit_module_id;                     /**< The name of the module the referenced unit is defined in. */
     hilti::rt::filesystem::path unit_module_path; /**< The path of the module that the referenced unit is defined in. */
     std::shared_ptr<glue::SpicyModule>
@@ -127,8 +128,8 @@ struct Event {
     // TODO: The following aren't set yet.
 
     // Code generation.
-    std::optional<::spicy::declaration::UnitHook> spicy_hook; /**< The generated Spicy hook. */
+    ::spicy::type::unit::item::UnitHookPtr spicy_hook;         /**< The generated Spicy hook. */
-    std::optional<hilti::declaration::Function> hilti_raise;  /**< The generated HILTI raise() function. */
+    std::shared_ptr<hilti::declaration::Function> hilti_raise; /**< The generated HILTI raise() function. */
     std::vector<ExpressionAccessor> expression_accessors; /**< One HILTI function per expression to access the value. */
 };
 
@@ -162,6 +163,18 @@ public:
     /** Destructor. */
     virtual ~GlueCompiler();
 
+    /**
+     * Returns the AST context in use. Only available once the driver has
+     * initialized the glue compiler.
+     */
+    auto* context() { return _driver->context()->astContext().get(); }
+
+    /**
+     * Returns the AST builder in use. Only available once the driver has
+     * initialized the glue compiler.
+     */
+    auto* builder() { return static_cast<::spicy::Builder*>(_driver->builder()); }
+
     /** Parses an `*.evt` file, without generating any code yet. */
     bool loadEvtFile(hilti::rt::filesystem::path& path);
 
@@ -199,19 +212,19 @@ public:
      */
     ExportedField exportForField(const hilti::ID& zeek_id, const hilti::ID& field_id) const;
 
-    using ZeekTypeCache = std::map<hilti::ID, hilti::Expression>;
+    using ZeekTypeCache = std::map<hilti::ID, hilti::ExpressionPtr>;
 
     /**
      * Generates code to convert a HILTI type to a corresponding Zeek type at
      * runtime. The code is added to the given body.
      */
-    hilti::Result<hilti::Nothing> createZeekType(const hilti::Type& t, const hilti::ID& id,
+    hilti::Result<hilti::Nothing> createZeekType(const hilti::QualifiedTypePtr& t, const hilti::ID& id,
-                                                 hilti::builder::Builder* body, ZeekTypeCache* cache) const;
+                                                 ::spicy::Builder* builder, ZeekTypeCache* cache) const;
 
     /** Return type for `recordField()`. */
     struct RecordField {
         hilti::ID id;                 /**< name of record field */
-        hilti::Type type;  /**< Spicy-side type object */
+        hilti::QualifiedTypePtr type; /**< Spicy-side type object */
         bool is_optional;             /**< true if field is optional */
         bool is_anonymous;            /**< true if field is annymous */
     };
@@ -223,13 +236,13 @@ public:
      * @param unit the unit type to retrieve fields for
      * @return list of fields
      */
-    static std::vector<RecordField> recordFields(const ::spicy::type::Unit& unit);
+    static std::vector<RecordField> recordFields(const ::spicy::type::Unit* unit);
 
 protected:
     friend class Driver;
 
     /** Called by driver to initialized a provided glue compiler. */
-    void Init(Driver* driver, int zeek_version);
+    void init(Driver* driver, int zeek_version);
 
 private:
     /**
@@ -269,8 +282,8 @@ private:
 
     std::map<hilti::ID, std::shared_ptr<glue::SpicyModule>> _spicy_modules;
 
-    std::vector<std::pair<hilti::ID, std::optional<hilti::ID>>>
+    std::vector<std::pair<hilti::ID, hilti::ID>>
-        _imports;                                            /**< imports from EVT files, with ID and optional scope */
+        _imports; /**< imports from EVT files, with ID and scope; latter is optional and can be empty */
     std::map<hilti::ID, glue::Export> _exports;              /**< exports from EVT files */
     std::vector<glue::Event> _events;                        /**< events parsed from EVT files */
     std::vector<glue::ProtocolAnalyzer> _protocol_analyzers; /**< protocol analyzers parsed from EVT files */

src/spicy/spicyz/main.cc

@@ -115,7 +115,7 @@ static hilti::Result<Nothing> parseOptions(int argc, char** argv, hilti::driver:
             }
 
             case 'g': {
-                driver_options->global_optimizations = false;
+                compiler_options->global_optimizations = false;
                 break;
             }
 

testing/btest/Baseline/spicy.event-args-fail-2/output

@@ -1,3 +1,3 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
 [error] <...>/event-args-fail.evt:2: signature for hook must be: %error or %error(err: string)
-[error] <Spicy Plugin for Zeek>: aborting after errors
+[error] <Spicy support for Zeek>: aborting after errors

testing/btest/Baseline/spicy.event-args-fail-3/output

@@ -1,3 +1,3 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
 [error] <...>/event-args-fail.evt:2: signature for hook must be: %error or %error(err: string)
-[error] <Spicy Plugin for Zeek>: aborting after errors
+[error] <Spicy support for Zeek>: aborting after errors

testing/btest/Baseline/spicy.export-type-fail/output

@@ -3,4 +3,4 @@
 [error] unknown type 'Test::DOES_NOT_EXIST' exported
 [error] <...>/foo.spicy:3:1-5:2: cannot export Spicy type 'Test::X': type is self-recursive
 [error] <...>/foo.spicy:11:1-13:2: cannot export Spicy type 'Test::Z': can only convert tuple types with all-named fields to Zeek
-[error] <Spicy Plugin for Zeek>: aborting after errors
+[error] <Spicy support for Zeek>: aborting after errors

testing/btest/Baseline/spicy.export-type-with-fields-fail/output

@@ -1,4 +1,4 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
 [error] <...>/foo.spicy:3:1-5:2: type 'Test::A' does not have field 'does_not_exist'
 [error] <...>/foo.spicy:3:1-5:2: type 'Test::A' does not have field 'does_not_exist'
-[error] <Spicy Plugin for Zeek>: aborting after errors
+[error] <Spicy support for Zeek>: aborting after errors

testing/btest/Baseline/spicy.profiling/prof.log

@@ -3,11 +3,11 @@
 # Profiling
 # 
 #name count
-hilti/func/SSH::Banner::__on_0x25_done 2
 hilti/func/SSH::Banner::__parse_SSH_Banner_stage2 2
 hilti/func/SSH::Banner::__parse_stage1 2
 hilti/func/SSH::Banner::parse1 2
 hilti/func/SSH::__register_SSH_Banner 1
+hilti/func/spicy_hooks_SSH::SSH::Banner::__on_0x25_done 2
 hilti/total 1
 spicy/prepare/stream/SSH::Banner 2
 spicy/unit/SSH::Banner 2

testing/btest/Baseline/spicy.tuple-arg/output

@@ -5,5 +5,5 @@
 [i=4, s=<uninitialized>]
 [zeek] [SPICY_SSH/3/orig] -> event ssh::banner((1, b"OpenSSH_3.8.1p1"))
 [zeek] [SPICY_SSH/3/orig] -> event ssh::banner((2, b"OpenSSH_3.8.1p1"))
-[zeek] [SPICY_SSH/3/orig] -> event ssh::banner((3, <error: n/a>))
+[zeek] [SPICY_SSH/3/orig] -> event ssh::banner((3, b""))
 [zeek] [SPICY_SSH/3/orig] -> event ssh::banner((4, Null))

testing/btest/spicy/tuple-arg.zeek

@@ -27,7 +27,7 @@ public type Banner = unit {
     dash    : /-/;
     software: /[^\r\n]*/;
 
-    unset_field: uint64 if ( False );
+    unset_field: bytes &eod if ( False );
 
     on %done { zeek::confirm_protocol(); }
 };