diff --git a/CHANGES b/CHANGES index 984aec39d9..a7f18f08b6 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,7 @@ +7.0.0-dev.34 | 2024-03-04 15:49:19 +0100 + + * Port Spicy integration to new AST API. (Robin Sommer, Corelight) + 7.0.0-dev.32 | 2024-02-28 17:10:26 +0100 * Bump zeekjs to 0.12.0 (Arne Welzel, Corelight) diff --git a/VERSION b/VERSION index 40958046a9..23255ad34d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -7.0.0-dev.32 +7.0.0-dev.34 diff --git a/auxil/spicy b/auxil/spicy index c39bdb5ed2..b1c3c4e4ea 160000 --- a/auxil/spicy +++ b/auxil/spicy @@ -1 +1 @@ -Subproject commit c39bdb5ed2bff0d390e6eae4ddda83050f047904 +Subproject commit b1c3c4e4eae55ed1924f90c5705d78be9dbfdccc diff --git a/src/analyzer/protocol/ldap/asn1.spicy b/src/analyzer/protocol/ldap/asn1.spicy index 989ad08ebe..809159997e 100644 --- a/src/analyzer/protocol/ldap/asn1.spicy +++ b/src/analyzer/protocol/ldap/asn1.spicy @@ -121,7 +121,7 @@ type ASN1OctetString = unit(len: uint64, constructed: bool) { # https://www.obj-sys.com/asn1tutorial/node124.html type ASN1String = unit(tag: ASN1Tag, len: uint64) { - var encoding: hilti::Charset; + var encoding: spicy::Charset; on %init { switch ( tag.type_ ) { @@ -132,7 +132,7 @@ type ASN1String = unit(tag: ASN1Tag, len: uint64) { case ASN1Type::PrintableString, ASN1Type::GeneralizedTime, ASN1Type::UTCTime: { - self.encoding = hilti::Charset::ASCII; + self.encoding = spicy::Charset::ASCII; } case ASN1Type::UTF8String, @@ -148,7 +148,7 @@ type ASN1String = unit(tag: ASN1Tag, len: uint64) { # BMPString and UniversalString. This *may* not be correct. ASN1Type::BMPString, ASN1Type::UniversalString: { - self.encoding = hilti::Charset::UTF8; + self.encoding = spicy::Charset::UTF8; } } } @@ -216,7 +216,7 @@ public type ASN1Body = unit(head: ASN1Header, recursive: bool) { ASN1Type::BitString -> bitstr_value: ASN1BitString(head.len.len, head.tag.constructed); ASN1Type::OctetString -> str_value: ASN1OctetString(head.len.len, head.tag.constructed) - &convert=$$.value.decode(hilti::Charset::ASCII); + &convert=$$.value.decode(spicy::Charset::ASCII); ASN1Type::ObjectIdentifier -> str_value: ASN1ObjectIdentifier(head.len.len) &convert=$$.oidstring; diff --git a/src/analyzer/protocol/ldap/ldap.spicy b/src/analyzer/protocol/ldap/ldap.spicy index 0831fd4435..7c60d1ec19 100644 --- a/src/analyzer/protocol/ldap/ldap.spicy +++ b/src/analyzer/protocol/ldap/ldap.spicy @@ -733,7 +733,7 @@ type SearchFilter = unit { FilterType::FILTER_EXT -> FILTER_EXT: DecodedAttributeValue(FilterType::FILTER_EXT) &parse-from=self.filterBytes; FilterType::FILTER_PRESENT -> FILTER_PRESENT: ASN1::ASN1OctetString(self.filterLen, False) - &convert=$$.value.decode(hilti::Charset::ASCII) + &convert=$$.value.decode(spicy::Charset::ASCII) &parse-from=self.filterBytes; }; diff --git a/src/analyzer/protocol/quic/QUIC.spicy b/src/analyzer/protocol/quic/QUIC.spicy index 84557c9c19..f9f81c05ed 100644 --- a/src/analyzer/protocol/quic/QUIC.spicy +++ b/src/analyzer/protocol/quic/QUIC.spicy @@ -280,7 +280,7 @@ public type LongHeaderPacket = unit { }; # A QUIC Frame. -public type Frame = unit(header: LongHeaderPacket, from_client: bool, crypto_sink: sink) { +public type Frame = unit(header: LongHeaderPacket, from_client: bool, inout crypto_sink: sink) { frame_type : uint8 &convert=cast($$); # TODO: add other FrameTypes as well @@ -447,7 +447,7 @@ type CryptoBuffer = unit() { # # A UDP datagram contains one or more QUIC packets. ############## -type Packet = unit(from_client: bool, context: ConnectionIDInfo&) { +type Packet = unit(from_client: bool, inout context: ConnectionIDInfo&) { var decrypted_data: bytes; var full_packet: bytes; var start: iterator; diff --git a/src/spicy/spicyz/driver.cc b/src/spicy/spicyz/driver.cc index 5eeaf535d3..a0021678fc 100644 --- a/src/spicy/spicyz/driver.cc +++ b/src/spicy/spicyz/driver.cc @@ -16,11 +16,12 @@ #include #include -#include #include +#include #include #include +#include "compiler/plugin.h" #include "config.h" #include "glue-compiler.h" @@ -31,28 +32,45 @@ using Driver = ::zeek::spicy::Driver; * Visitor to type information from a HILTI AST. This extracts user-visible * types only, we skip any internal ones. */ -struct VisitorTypes : public hilti::visitor::PreOrder { - explicit VisitorTypes(Driver* driver, hilti::ID module, hilti::rt::filesystem::path path, bool is_resolved) - : driver(driver), module(std::move(module)), path(std::move(path)), is_resolved(is_resolved) {} +struct VisitorTypes : public spicy::visitor::PreOrder { + explicit VisitorTypes(Driver* driver, GlueCompiler* glue, bool is_resolved) + : driver(driver), glue(glue), is_resolved(is_resolved) {} - void operator()(const hilti::declaration::Type& t) { - assert(! t.type().typeID() || *t.type().typeID() == hilti::ID(module, t.id())); // ensure consistent IDs + void operator()(hilti::declaration::Module* n) final { + if ( n->uid().in_memory ) { + // Ignore modules built by us in memory. + module = {}; + return; + } - if ( module == hilti::ID("hilti") || module == hilti::ID("spicy_rt") || module == hilti::ID("zeek_rt") ) + module = n->scopeID(); + path = n->uid().path; + + if ( is_resolved ) + glue->addSpicyModule(module, path); + } + + void operator()(hilti::declaration::Type* n) final { + if ( module.empty() || module == hilti::ID("hilti") || module == hilti::ID("spicy_rt") || + module == hilti::ID("zeek_rt") ) return; + assert(! n->type()->type()->typeID() || + n->type()->type()->typeID() == hilti::ID(module, n->id())); // ensure consistent IDs + types.emplace_back(TypeInfo{ - .id = hilti::ID(module, t.id()), - .type = t.type()._clone().as(), - .linkage = t.linkage(), + .id = hilti::ID(module, n->id()), + .type = n->type(), + .linkage = n->linkage(), .is_resolved = is_resolved, .module_id = module, .module_path = path, - .location = t.meta().location(), + .location = n->meta().location(), }); } Driver* driver; + GlueCompiler* glue; hilti::ID module; hilti::rt::filesystem::path path; bool is_resolved; @@ -61,8 +79,8 @@ struct VisitorTypes : public hilti::visitor::PreOrder { Driver::Driver(std::unique_ptr glue, const char* argv0, hilti::rt::filesystem::path lib_path, int zeek_version) - : ::spicy::Driver(""), _glue(std::move(glue)) { - _glue->Init(this, zeek_version); + : ::spicy::Driver(""), _glue(std::move(glue)) { + _glue->init(this, zeek_version); ::spicy::Configuration::extendHiltiConfiguration(); auto options = hiltiOptions(); @@ -195,6 +213,8 @@ hilti::Result Driver::compile() { SPICY_DEBUG("Running Spicy driver"); + _error = hilti::Nothing(); + if ( auto x = ::spicy::Driver::compile(); ! x ) return x.error(); @@ -243,23 +263,16 @@ std::vector> Driver::exportedTypes() const { return result; } -void Driver::hookNewASTPreCompilation(std::shared_ptr unit) { - if ( unit->extension() != ".spicy" ) - return; - - if ( unit->path().empty() ) - // Ignore modules constructed in memory. - return; - - auto v = VisitorTypes(this, unit->id(), unit->path(), false); - for ( auto i : v.walk(unit->module()) ) - v.dispatch(i); +void Driver::hookNewASTPreCompilation(const hilti::Plugin& plugin, const std::shared_ptr& root) { + auto v = VisitorTypes(this, _glue.get(), false); + hilti::visitor::visit(v, root, ".spicy"); for ( const auto& ti : v.types ) { SPICY_DEBUG(hilti::util::fmt(" Got type '%s' (pre-compile)", ti.id)); _types[ti.id] = ti; - if ( auto et = ti.type.tryAs(); et && ti.linkage == hilti::declaration::Linkage::Public ) { + if ( auto et = ti.type->type()->tryAs(); + et && ti.linkage == hilti::declaration::Linkage::Public ) { SPICY_DEBUG(" Automatically exporting public enum for backwards compatibility"); _public_enums.push_back(ti); } @@ -268,17 +281,17 @@ void Driver::hookNewASTPreCompilation(std::shared_ptr unit) { } } -void Driver::hookNewASTPostCompilation(std::shared_ptr unit) { - if ( unit->extension() != ".spicy" ) - return; +bool Driver::hookNewASTPostCompilation(const hilti::Plugin& plugin, const std::shared_ptr& root) { + if ( plugin.component != "Spicy" ) + return false; - if ( unit->path().empty() ) - // Ignore modules constructed in memory. - return; + if ( ! _need_glue ) + return false; - auto v = VisitorTypes(this, unit->id(), unit->path(), true); - for ( auto i : v.walk(unit->module()) ) - v.dispatch(i); + _need_glue = false; + + auto v = VisitorTypes(this, _glue.get(), true); + hilti::visitor::visit(v, root, ".spicy"); for ( auto&& t : v.types ) { SPICY_DEBUG(hilti::util::fmt(" Got type '%s' (post-compile)", t.id)); @@ -286,19 +299,14 @@ void Driver::hookNewASTPostCompilation(std::shared_ptr unit) { hookNewType(t); } - _glue->addSpicyModule(unit->id(), unit->path()); + if ( ! _glue->compile() ) + _error = hilti::result::Error("glue compilation failed"); + + return true; } -hilti::Result Driver::hookCompilationFinished(const hilti::Plugin& plugin) { - if ( ! _need_glue ) - return hilti::Nothing(); - - _need_glue = false; - - if ( _glue->compile() ) - return hilti::Nothing(); - else - return hilti::result::Error("glue compilation failed"); +hilti::Result Driver::hookCompilationFinished(const std::shared_ptr& root) { + return _error; } void Driver::hookInitRuntime() { ::spicy::rt::init(); } diff --git a/src/spicy/spicyz/driver.h b/src/spicy/spicyz/driver.h index 8dd2422ce2..e8a3304008 100644 --- a/src/spicy/spicyz/driver.h +++ b/src/spicy/spicyz/driver.h @@ -32,7 +32,7 @@ class GlueCompiler; struct TypeInfo { hilti::ID id; /**< fully-qualified name of the type */ - hilti::Type type; /**< the type itself */ + hilti::QualifiedTypePtr type; /**< the type itself */ hilti::declaration::Linkage linkage; /**< linkage of of the type's declaration */ bool is_resolved = false; /**< true if we are far enough in processing that the type has been fully resolved */ hilti::ID module_id; /**< name of module type is defined in */ @@ -102,7 +102,7 @@ public: if ( ! ti ) return ti.error(); - if ( ! ti->type.isA() ) + if ( ! ti->type->type()->isA() ) return hilti::result::Error(hilti::util::fmt("'%s' is not of expected type", id)); return ti; @@ -165,7 +165,7 @@ public: protected: /** * Hook executed for all type declarations encountered in a Spicy module. - * Derived classes may override this to add custom processing. This hooks + * Derived classes may override this to add custom processing. This hook * executes twices for each declaration: once before we compile the AST * (meaning types have not been resolved yet), and once after. The type * info's `is_resolved` field indicates which of the two we're in. @@ -175,13 +175,13 @@ protected: virtual void hookNewType(const TypeInfo& ti) {} /** Overridden from HILTI driver. */ - void hookNewASTPreCompilation(std::shared_ptr unit) override; + void hookNewASTPreCompilation(const hilti::Plugin& plugin, const std::shared_ptr& root) override; /** Overridden from HILTI driver. */ - void hookNewASTPostCompilation(std::shared_ptr unit) override; + bool hookNewASTPostCompilation(const hilti::Plugin& plugin, const std::shared_ptr& root) override; /** Overridden from HILTI driver. */ - hilti::Result hookCompilationFinished(const hilti::Plugin& plugin) override; + hilti::Result hookCompilationFinished(const std::shared_ptr& root) override; /** Overridden from HILTI driver. */ void hookInitRuntime() override; @@ -194,6 +194,7 @@ protected: std::vector _public_enums; // tracks Spicy enum types declared public, for automatic export bool _using_build_directory = false; // true if we're running out of the plugin's build directory bool _need_glue = true; // true if glue code has not yet been generated + hilti::Result _error = hilti::Nothing(); // error encountered during compilation }; } // namespace zeek::spicy diff --git a/src/spicy/spicyz/glue-compiler.cc b/src/spicy/spicyz/glue-compiler.cc index f81478f2ad..90532e21f9 100644 --- a/src/spicy/spicyz/glue-compiler.cc +++ b/src/spicy/spicyz/glue-compiler.cc @@ -8,26 +8,23 @@ #include #include -#include #include #include #include -#include -#include +#include +#include "ast/builder/builder.h" +#include "ast/visitor.h" #include "config.h" #include "zeek/spicy/port-range.h" using namespace zeek::spicy; +using Builder = spicy::Builder; -namespace builder = hilti::builder; - -#if SPICY_VERSION_NUMBER >= 10700 -static inline auto _linker_scope() { return hilti::builder::scope(); } -#else -static inline auto _linker_scope() { return hilti::builder::call("hilti::linker_scope", {}); } -#endif +namespace zeek::spicy::logging::debug { +inline const hilti::logging::DebugStream GlueCompiler("zeek-glue"); +} // namespace zeek::spicy::logging::debug // Small parsing helpers. @@ -134,7 +131,7 @@ static std::string extract_string(const std::string& chunk, size_t* i) { return str; } -static hilti::Type extract_type(const std::string& chunk, size_t* i) { +static hilti::UnqualifiedTypePtr extract_type(Builder* builder, const std::string& chunk, size_t* i) { eat_spaces(chunk, i); // We currently only parse Spicy types that can appear in parameters of @@ -142,15 +139,15 @@ static hilti::Type extract_type(const std::string& chunk, size_t* i) { auto token = extract_id(chunk, i); if ( token == hilti::ID("string") ) - return hilti::type::String(); + return builder->typeString(); if ( token == hilti::ID("uint64") ) - return hilti::type::UnsignedInteger(64); + return builder->typeUnsignedInteger(64); throw ParseError("mismatching type"); } -static hilti::type::function::Parameter extract_parameter(const std::string& chunk, size_t* i) { +static hilti::type::function::ParameterPtr extract_parameter(Builder* builder, const std::string& chunk, size_t* i) { eat_spaces(chunk, i); auto id = extract_id(chunk, i); @@ -160,8 +157,8 @@ static hilti::type::function::Parameter extract_parameter(const std::string& chu eat_token(chunk, i, ":"); - auto type = extract_type(chunk, i); - return builder::parameter(std::move(id), std::move(type)); + auto type = extract_type(builder, chunk, i); + return builder->parameter(std::move(id), type); } static hilti::rt::filesystem::path extract_path(const std::string& chunk, size_t* i) { @@ -336,7 +333,7 @@ static ::zeek::spicy::rt::PortRange extract_port_range(const std::string& chunk, return {start, *end}; } -void GlueCompiler::Init(Driver* driver, int zeek_version) { +void GlueCompiler::init(Driver* driver, int zeek_version) { _driver = driver; _zeek_version = zeek_version; } @@ -510,12 +507,12 @@ bool GlueCompiler::loadEvtFile(hilti::rt::filesystem::path& path) { eat_token(*chunk, &i, "import"); hilti::ID module = extract_id(*chunk, &i); - std::optional scope; + hilti::ID scope; if ( looking_at(*chunk, i, "from") ) { eat_token(*chunk, &i, "from"); - scope = extract_path(*chunk, &i); - SPICY_DEBUG(hilti::util::fmt(" Got module %s to import from scope %s", module, *scope)); + scope = hilti::ID(extract_path(*chunk, &i)); + SPICY_DEBUG(hilti::util::fmt(" Got module %s to import from scope %s", module, scope)); } else SPICY_DEBUG(hilti::util::fmt(" Got module %s to import", module)); @@ -608,7 +605,7 @@ GlueCompiler::ExportedField GlueCompiler::exportForField(const hilti::ID& zeek_i bool glue::Export::validate(const TypeInfo& ti) const { - auto utype = ti.type.tryAs<::spicy::type::Unit>(); + auto utype = ti.type->type()->tryAs<::spicy::type::Unit>(); if ( ! utype ) return true; @@ -650,7 +647,7 @@ glue::ProtocolAnalyzer GlueCompiler::parseProtocolAnalyzer(const std::string& ch eat_token(chunk, &i, "protocol"); eat_token(chunk, &i, "analyzer"); - a.name = extract_id(chunk, &i).str(); + a.name = hilti::ID(extract_id(chunk, &i).str()); eat_token(chunk, &i, "over"); @@ -753,7 +750,7 @@ glue::FileAnalyzer GlueCompiler::parseFileAnalyzer(const std::string& chunk) { eat_token(chunk, &i, "file"); eat_token(chunk, &i, "analyzer"); - a.name = extract_id(chunk, &i).str(); + a.name = extract_id(chunk, &i); eat_token(chunk, &i, ":"); @@ -795,7 +792,7 @@ glue::PacketAnalyzer GlueCompiler::parsePacketAnalyzer(const std::string& chunk) eat_token(chunk, &i, "packet"); eat_token(chunk, &i, "analyzer"); - a.name = extract_id(chunk, &i).str(); + a.name = extract_id(chunk, &i); eat_token(chunk, &i, ":"); @@ -841,7 +838,7 @@ glue::Event GlueCompiler::parseEvent(const std::string& chunk) { if ( ! looking_at(chunk, i, ")") ) { while ( true ) { - auto param = extract_parameter(chunk, &i); + auto param = extract_parameter(builder(), chunk, &i); ev.parameters.push_back(std::move(param)); if ( looking_at(chunk, i, ")") ) @@ -970,26 +967,26 @@ glue::Export GlueCompiler::parseExport(const std::string& chunk) { bool GlueCompiler::compile() { assert(_driver); - auto init_module = hilti::Module(hilti::ID("spicy_init")); + auto init_module = context()->newModule(builder(), hilti::ID("spicy_init"), ".spicy"); - auto import_ = hilti::builder::import(hilti::ID("zeek_rt"), ".hlt"); - init_module.add(std::move(import_)); + auto import_ = builder()->import(hilti::ID("zeek_rt"), ".hlt"); + init_module->add(context(), std::move(import_)); - import_ = hilti::builder::import(hilti::ID("hilti"), ".hlt"); - init_module.add(std::move(import_)); + import_ = builder()->import(hilti::ID("hilti"), ".hlt"); + init_module->add(context(), std::move(import_)); - auto preinit_body = hilti::builder::Builder(_driver->context()); + auto preinit_body = Builder(context()); for ( auto&& [id, m] : _spicy_modules ) - m->spicy_module = hilti::Module(hilti::ID(hilti::util::fmt("spicy_hooks_%s", id))); + m->spicy_module = context()->newModule(builder(), hilti::ID(hilti::util::fmt("spicy_hooks_%s", id)), ".spicy"); if ( ! PopulateEvents() ) return false; if ( ! _doc_id.empty() ) { preinit_body.addCall("zeek_rt::register_spicy_module_begin", { - hilti::builder::string_mut(_doc_id), - hilti::builder::string_mut(_doc_description), + builder()->stringMutable(_doc_id), + builder()->stringMutable(_doc_description), }); } @@ -1027,16 +1024,18 @@ bool GlueCompiler::compile() { default: hilti::logger().internalError("unexpected protocol"); } - preinit_body.addCall( - "zeek_rt::register_protocol_analyzer", - {builder::string_mut(a.name), builder::id(protocol), - builder::vector(hilti::util::transform( - a.ports, - [](const auto& p) { - return builder::call("zeek_rt::make_port_range", {builder::port(p.begin), builder::port(p.end)}); - })), - builder::string_mut(a.unit_name_orig), builder::string_mut(a.unit_name_resp), - builder::string_mut(a.replaces), _linker_scope()}); + preinit_body.addCall("zeek_rt::register_protocol_analyzer", + {builder()->stringMutable(a.name.str()), builder()->id(protocol), + builder()->vector( + hilti::util::transform(a.ports, + [this](const auto& p) { + return builder()->call("zeek_rt::make_port_range", + {builder()->port(p.begin), + builder()->port(p.end)}); + })), + builder()->stringMutable(a.unit_name_orig.str()), + builder()->stringMutable(a.unit_name_resp.str()), builder()->stringMutable(a.replaces), + builder()->scope()}); } for ( auto& a : _file_analyzers ) { @@ -1052,10 +1051,15 @@ bool GlueCompiler::compile() { } preinit_body.addCall("zeek_rt::register_file_analyzer", - {builder::string_mut(a.name), - builder::vector( - hilti::util::transform(a.mime_types, [](auto m) { return builder::string_mut(m); })), - builder::string_mut(a.unit_name), builder::string_mut(a.replaces), _linker_scope()}); + {builder()->stringMutable(a.name.str()), + builder()->vector(hilti::util::transform(a.mime_types, + [&](auto m) { + return builder() + ->stringMutable(m) + ->template as(); + })), + builder()->stringMutable(a.unit_name.str()), builder()->stringMutable(a.replaces), + builder()->scope()}); } for ( auto& a : _packet_analyzers ) { @@ -1071,8 +1075,8 @@ bool GlueCompiler::compile() { } preinit_body.addCall("zeek_rt::register_packet_analyzer", - {builder::string_mut(a.name), builder::string_mut(a.unit_name), - builder::string_mut(a.replaces), _linker_scope()}); + {builder()->stringMutable(a.name.str()), builder()->stringMutable(a.unit_name.str()), + builder()->stringMutable(a.replaces), builder()->scope()}); } // Create the Spicy hooks and accessor functions. @@ -1083,7 +1087,7 @@ bool GlueCompiler::compile() { // Register our Zeek events at pre-init time. for ( auto&& ev : _events ) - preinit_body.addCall("zeek_rt::install_handler", {builder::string_mut(ev.name)}); + preinit_body.addCall("zeek_rt::install_handler", {builder()->stringMutable(ev.name.str())}); // Create Zeek types for exported Spicy types. GlueCompiler::ZeekTypeCache cache; @@ -1096,8 +1100,11 @@ bool GlueCompiler::compile() { for ( auto&& [id, m] : _spicy_modules ) { // Import runtime module. - auto import_ = hilti::builder::import(hilti::ID("zeek_rt"), ".hlt"); - m->spicy_module->add(std::move(import_)); + auto import_ = builder()->import(hilti::ID("zeek_rt"), ".hlt"); + m->spicy_module->add(context(), std::move(import_)); + + import_ = builder()->import(hilti::ID("hilti"), ".hlt"); + m->spicy_module->add(context(), std::move(import_)); // Create a vector of unique parent paths from all EVTs files going into this module. auto search_dirs = hilti::util::transform(m->evts, [](auto p) { return p.parent_path(); }); @@ -1105,12 +1112,15 @@ bool GlueCompiler::compile() { // Import any dependencies. for ( const auto& [module, scope] : _imports ) { - auto import_ = hilti::declaration::ImportedModule(module, std::string(".spicy"), scope, search_dirs_vec); - m->spicy_module->add(std::move(import_)); + auto import_ = builder()->declarationImportedModule(module, std::string(".spicy"), scope); + import_->as()->setSearchDirectories(search_dirs_vec); + m->spicy_module->add(context(), std::move(import_)); } - auto unit = hilti::Unit::fromModule(_driver->context(), *m->spicy_module, ".spicy"); - _driver->addInput(unit); + if ( auto rc = _driver->addInput(m->spicy_module->uid()); ! rc ) { + hilti::logger().error(hilti::util::fmt("error adding Spicy unit: %s", rc.error())); + return false; + } } if ( ! _doc_id.empty() ) @@ -1118,13 +1128,18 @@ bool GlueCompiler::compile() { if ( ! preinit_body.empty() ) { auto preinit_function = - hilti::builder::function("zeek_preinit", hilti::type::void_, {}, preinit_body.block(), - hilti::type::function::Flavor::Standard, hilti::declaration::Linkage::PreInit); - init_module.add(std::move(preinit_function)); + builder()->function(hilti::ID("zeek_preinit"), + builder()->qualifiedType(builder()->typeVoid(), hilti::Constness::Const), {}, + preinit_body.block(), hilti::type::function::Flavor::Standard, + hilti::declaration::Linkage::PreInit); + init_module->add(context(), std::move(preinit_function)); + } + + if ( auto rc = _driver->addInput(init_module->uid()); ! rc ) { + hilti::logger().error(hilti::util::fmt("error adding init unit: %s", rc.error())); + return false; } - auto unit = hilti::Unit::fromModule(_driver->context(), init_module, ".hlt"); - _driver->addInput(unit); return true; } @@ -1164,7 +1179,7 @@ bool GlueCompiler::PopulateEvents() { } } - ev.unit_type = uinfo.type.as<::spicy::type::Unit>(); + ev.unit_type = uinfo.type->type()->as<::spicy::type::Unit>(); ev.unit_module_id = uinfo.module_id; ev.unit_module_path = uinfo.module_path; @@ -1194,37 +1209,36 @@ bool GlueCompiler::PopulateEvents() { #include -#include +#include -// Helper visitor to wrap expressions using the the TryMember operator into a +// Helper visitor to wrap expressions using the TryMember operator into a // "deferred" expression. -class WrapTryMemberVisitor : public hilti::visitor::PostOrder { +class WrapTryMemberVisitor : public spicy::visitor::MutatingPostOrder { public: - WrapTryMemberVisitor(bool catch_exception) : _catch_exception(catch_exception) {} + WrapTryMemberVisitor(Builder* builder, bool catch_exception) + : spicy::visitor::MutatingPostOrder(builder, logging::debug::GlueCompiler), _catch_exception(catch_exception) {} - void operator()(const hilti::expression::UnresolvedOperator& n, position_t p) { - if ( n.kind() == hilti::operator_::Kind::TryMember ) - p.node = hilti::expression::Deferred(hilti::Expression(n), _catch_exception); + void operator()(hilti::expression::UnresolvedOperator* n) final { + if ( n->kind() == hilti::operator_::Kind::TryMember ) + replaceNode(n, builder()->expressionDeferred(n->as(), _catch_exception), "wrap .?"); } private: bool _catch_exception; }; -static hilti::Result _parseArgument(const std::string& expression, bool catch_exception, - const hilti::Meta& meta) { - auto expr = spicy::parseExpression(expression, meta); +static hilti::Result parseArgument(Builder* builder, const std::string& expression, + bool catch_exception, const hilti::Meta& meta) { + auto expr = ::spicy::builder::parseExpression(builder, expression, meta); if ( ! expr ) return hilti::result::Error(hilti::util::fmt("error parsing event argument expression '%s'", expression)); // If the expression uses the ".?" operator, we need to defer evaluation // so that we can handle potential exceptions at runtime. - auto v = WrapTryMemberVisitor(catch_exception); - auto n = hilti::Node(*expr); - for ( auto i : v.walk(&n) ) - v.dispatch(i); + auto v = WrapTryMemberVisitor(builder, catch_exception); + spicy::visitor::visit(v, *expr); - return n.as(); + return expr; } bool GlueCompiler::CreateSpicyHook(glue::Event* ev) { @@ -1235,32 +1249,31 @@ bool GlueCompiler::CreateSpicyHook(glue::Event* ev) { // Find the Spicy module that this event belongs to. SPICY_DEBUG(hilti::util::fmt("Adding Spicy hook '%s' for event %s", ev->hook, ev->name)); - auto import_ = hilti::declaration::ImportedModule(ev->unit_module_id, ev->unit_module_path); - ev->spicy_module->spicy_module->add(std::move(import_)); + auto import_ = builder()->declarationImportedModule(ev->unit_module_id, ev->unit_module_path); + ev->spicy_module->spicy_module->add(context(), std::move(import_)); // Define Zeek-side event handler. auto handler_id = hilti::ID(hilti::util::fmt("__zeek_handler_%s", mangled_event_name)); auto handler = - builder::global(handler_id, builder::call("zeek_rt::internal_handler", {builder::string_mut(ev->name)}), - hilti::declaration::Linkage::Private, meta); - ev->spicy_module->spicy_module->add(std::move(handler)); + builder()->global(handler_id, + builder()->call("zeek_rt::internal_handler", {builder()->stringMutable(ev->name.str())}), + hilti::declaration::Linkage::Private, meta); + ev->spicy_module->spicy_module->add(context(), std::move(handler)); // Create the hook body that raises the event. - auto body = hilti::builder::Builder(_driver->context()); + auto body = Builder(context()); -#if SPICY_VERSION_NUMBER >= 10800 body.startProfiler(hilti::util::fmt("zeek/event/%s", ev->name)); -#endif // If the event comes with a condition, evaluate that first. if ( ev->condition.size() ) { - auto cond = ::spicy::parseExpression(ev->condition, meta); + auto cond = ::spicy::builder::parseExpression(&body, ev->condition, meta); if ( ! cond ) { hilti::logger().error(hilti::util::fmt("error parsing conditional expression '%s'", ev->condition)); return false; } - auto exit_ = body.addIf(builder::not_(*cond), meta); + auto exit_ = body.addIf(builder()->not_(*cond), meta); exit_->addReturn(meta); } @@ -1271,84 +1284,91 @@ bool GlueCompiler::CreateSpicyHook(glue::Event* ev) { // Zeek type to convert an Spicy type into. However, we wouldn't want // limit logging to events with handlers. if ( _driver->hiltiOptions().debug ) { - std::vector fmt_args = {builder::string_literal(ev->name.str())}; + hilti::Expressions fmt_args = {builder()->stringLiteral(ev->name.str())}; for ( const auto&& [i, e] : hilti::util::enumerate(ev->expression_accessors) ) { if ( hilti::util::startsWith(e.expression, "$") ) { - fmt_args.emplace_back(builder::string_literal(e.expression)); + fmt_args.emplace_back(builder()->stringLiteral(e.expression)); continue; } - if ( auto expr = _parseArgument(e.expression, true, meta) ) + if ( auto expr = parseArgument(builder(), e.expression, true, meta) ) fmt_args.emplace_back(std::move(*expr)); else // We'll catch and report this below. - fmt_args.emplace_back(builder::string_literal("")); + fmt_args.emplace_back(builder()->stringLiteral("")); } std::vector fmt_ctrls(fmt_args.size() - 1, "%s"); auto fmt_str = hilti::util::fmt("-> event %%s(%s)", hilti::util::join(fmt_ctrls, ", ")); - auto msg = builder::modulo(builder::string_literal(fmt_str), builder::tuple(fmt_args)); - auto call = builder::call("zeek_rt::debug", {std::move(msg)}); + auto msg = builder()->modulo(builder()->stringLiteral(fmt_str), builder()->tuple(fmt_args)); + auto call = builder()->call("zeek_rt::debug", {std::move(msg)}); body.addExpression(call); } - auto handler_expr = builder::id(handler_id); + auto handler_expr = builder()->id(handler_id); if ( _driver->hiltiOptions().cxx_enable_dynamic_globals ) { // Store reference to handler locally to avoid repeated lookups through globals store. - body.addLocal("handler", builder::id(handler_id), meta); - handler_expr = builder::id("handler"); + body.addLocal("handler", builder()->id(handler_id), meta); + handler_expr = builder()->id("handler"); } // Nothing to do if there's not handler defined. - auto have_handler = builder::call("zeek_rt::have_handler", {handler_expr}, meta); - auto exit_ = body.addIf(builder::not_(have_handler), meta); + auto have_handler = builder()->call("zeek_rt::have_handler", {handler_expr}, meta); + auto exit_ = body.addIf(builder()->not_(have_handler), meta); exit_->addReturn(meta); // Build event's argument vector. - body.addLocal(hilti::ID("args"), hilti::type::Vector(builder::typeByID("zeek_rt::Val"), meta), meta); - body.addMemberCall(builder::id("args"), "reserve", - {builder::integer(static_cast(ev->expression_accessors.size()))}, meta); + body.addLocal(hilti::ID("args"), + builder()->qualifiedType(builder()->typeVector(builder()->qualifiedType(builder()->typeName( + "zeek_rt::Val"), + hilti::Constness::Const), + meta), + hilti::Constness::Const), + meta); + + body.addMemberCall(builder()->id("args"), "reserve", + {builder()->integer(static_cast(ev->expression_accessors.size()))}, meta); int i = 0; for ( const auto& e : ev->expression_accessors ) { - hilti::Expression val; + hilti::ExpressionPtr val; if ( e.expression == "$conn" ) - val = builder::call("zeek_rt::current_conn", {}, meta); + val = builder()->call("zeek_rt::current_conn", {}, meta); else if ( e.expression == "$file" ) - val = builder::call("zeek_rt::current_file", {}, meta); + val = builder()->call("zeek_rt::current_file", {}, meta); else if ( e.expression == "$packet" ) - val = builder::call("zeek_rt::current_packet", {}, meta); + val = builder()->call("zeek_rt::current_packet", {}, meta); else if ( e.expression == "$is_orig" ) - val = builder::call("zeek_rt::current_is_orig", {}, meta); + val = builder()->call("zeek_rt::current_is_orig", {}, meta); else { if ( hilti::util::startsWith(e.expression, "$") ) { hilti::logger().error(hilti::util::fmt("unknown reserved parameter '%s'", e.expression)); return false; } - auto expr = _parseArgument(e.expression, false, meta); + auto expr = parseArgument(builder(), e.expression, false, meta); if ( ! expr ) { hilti::logger().error(expr.error()); return false; } - auto ztype = builder::call("zeek_rt::event_arg_type", {handler_expr, builder::integer(i)}, meta); - val = builder::call("zeek_rt::to_val", {std::move(*expr), ztype}, meta); + auto ztype = builder()->call("zeek_rt::event_arg_type", {handler_expr, builder()->integer(i)}, meta); + val = builder()->call("zeek_rt::to_val", {std::move(*expr), ztype}, meta); } - body.addMemberCall(builder::id("args"), "push_back", {val}, meta); + body.addMemberCall(builder()->id("args"), "push_back", {val}, meta); i++; } - body.addCall("zeek_rt::raise_event", {handler_expr, builder::move(builder::id("args"))}, meta); + body.addCall("zeek_rt::raise_event", {handler_expr, builder()->move(builder()->id("args"))}, meta); - auto attrs = hilti::AttributeSet({hilti::Attribute("&priority", builder::integer(ev->priority))}); - auto unit_hook = ::spicy::Hook(ev->parameters, body.block(), ::spicy::Engine::All, {}, meta); - auto hook_decl = ::spicy::declaration::UnitHook(ev->hook, unit_hook, meta); - ev->spicy_module->spicy_module->add(hilti::Declaration(hook_decl)); + auto attrs = builder()->attributeSet({builder()->attribute("&priority", builder()->integer(ev->priority))}); + auto unit_hook = builder()->declarationHook(ev->parameters, body.block(), ::spicy::Engine::All, {}, meta); + auto hook_decl = builder()->declarationUnitHook(ev->hook, unit_hook, meta); + ev->spicy_module->spicy_module->add(context(), hilti::DeclarationPtr(hook_decl)); return true; } @@ -1359,32 +1379,35 @@ namespace { // // Note: Any logic changes here must be reflected in the plugin driver's // corresponding `VisitorZeekType` as well. -struct VisitorZeekType : hilti::visitor::PreOrder, VisitorZeekType> { - VisitorZeekType(const GlueCompiler* gc, hilti::builder::Builder* body, GlueCompiler::ZeekTypeCache* cache) - : gc(gc), body(body), cache(cache) {} +struct VisitorZeekType : spicy::visitor::PreOrder { + VisitorZeekType(const GlueCompiler* gc, Builder* builder, GlueCompiler::ZeekTypeCache* cache) + : gc(gc), builder(builder), cache(cache) {} const GlueCompiler* gc; - hilti::builder::Builder* body; + Builder* builder; GlueCompiler::ZeekTypeCache* cache; + std::vector> results; + std::set zeek_types; - std::vector> ids = {}; + std::vector ids = {}; + + // Record the resulting Zeek type for the currently processed type (or an error). + void result(hilti::Result r) { results.push_back(std::move(r)); } // Returns current ID, if any. - auto id() const { return ids.empty() ? std::nullopt : ids.back(); } + auto id() const { return ids.empty() ? hilti::ID() : ids.back(); } // Returns namespace of top-level ID, if any. - auto namespace_() const { - return (! ids.empty() && ids.front().has_value()) ? ids.front()->namespace_() : hilti::ID(); - } + auto namespace_() const { return (! ids.empty() && ids.front()) ? ids.front().namespace_() : hilti::ID(); } // Returns prefix for a new tmp that includes the given ID in its name. auto tmpName(const std::string& prefix, const hilti::ID& id) const { return hilti::util::fmt("%s_%s", prefix, hilti::util::replace(id, "::", "_")); } - result_t create_record_type(const hilti::ID& ns, const hilti::ID& local, - const std::vector& fields) { + hilti::Result create_record_type(const hilti::ID& ns, const hilti::ID& local, + const hilti::Expressions& fields) { if ( hilti::logger().isEnabled(ZeekPlugin) ) { if ( ! fields.empty() ) { SPICY_DEBUG(hilti::util::fmt("Creating Zeek record type %s::%s with fields:", ns, local)); @@ -1396,52 +1419,64 @@ struct VisitorZeekType : hilti::visitor::PreOrderaddTmp(tmpName("fields", {ns, local}), builder::vector(builder::typeByID("zeek_rt::RecordField"))); + auto tmp = builder->addTmp(tmpName("fields", {ns, local}), + builder->vector(builder->qualifiedType(builder->typeName("zeek_rt::RecordField"), + hilti::Constness::Const))); for ( const auto& f : fields ) - body->addMemberCall(tmp, "push_back", {f}); + builder->addMemberCall(tmp, "push_back", {f}); - return builder::call("zeek_rt::create_record_type", {builder::string_mut(ns), builder::string_mut(local), tmp}); + return builder->call("zeek_rt::create_record_type", + {builder->stringMutable(ns.str()), builder->stringMutable(local.str()), tmp}); } - hilti::Expression create_record_field(const hilti::ID& id, const hilti::Expression& type, bool optional, - bool log) const { - return builder::call("zeek_rt::create_record_field", - {builder::string_mut(id), type, builder::bool_(optional), builder::bool_(log)}); + hilti::ExpressionPtr create_record_field(const hilti::ID& id, const hilti::ExpressionPtr& type, bool optional, + bool log) const { + return builder->call("zeek_rt::create_record_field", + {builder->stringMutable(id.str()), type, builder->bool_(optional), builder->bool_(log)}); } - result_t base_type(const char* tag) { return builder::call("zeek_rt::create_base_type", {builder::id(tag)}); } + hilti::Result base_type(const char* tag) { + return builder->call("zeek_rt::create_base_type", {builder->id(tag)}); + } - result_t createZeekType(const hilti::Type& t, std::optional id = {}) { + hilti::Result createZeekType(const hilti::QualifiedTypePtr& t, hilti::ID id = {}) { if ( ! id ) - id = t.typeID(); // may still be unset + id = t->type()->typeID(); // may still be unset if ( id ) { - if ( auto x = cache->find(*id); x != cache->end() ) + if ( auto x = cache->find(id); x != cache->end() ) return x->second; // Avoid infinite recursion. - if ( zeek_types.count(*id) ) + if ( zeek_types.count(id) ) return hilti::result::Error("type is self-recursive"); - zeek_types.insert(*id); + zeek_types.insert(id); } + auto old_results = results.size(); + ids.push_back(id); - auto x = dispatch(t); + dispatch(t->type()); ids.pop_back(); - if ( ! x ) + if ( results.size() != old_results + 1 ) return hilti::result::Error( - hilti::util::fmt("no support for automatic conversion into a Zeek type (%s)", t.typename_())); + hilti::util::fmt("no support for automatic conversion into a Zeek type (%s)", t->type()->typename_())); + + auto x = results.back(); + results.pop_back(); + + if ( ! x ) + return x; if ( id ) { - zeek_types.erase(*id); + zeek_types.erase(id); if ( *x ) { - auto zt = body->addTmp(tmpName("type", *id), **x); - cache->emplace(*id, zt); + auto zt = builder->addTmp(tmpName("type", id), *x); + cache->emplace(id, zt); return zt; } } @@ -1449,192 +1484,209 @@ struct VisitorZeekType : hilti::visitor::PreOrder fields; - for ( const auto& b : t.bits() ) { - auto ztype = createZeekType(b.itemType()); - if ( ! ztype ) - return ztype.error(); - - fields.emplace_back(create_record_field(b.id(), *ztype, false, false)); + void operator()(hilti::type::Bitfield* t) final { + hilti::Expressions fields; + for ( const auto& b : t->bits() ) { + if ( auto ztype = createZeekType(b->itemType()) ) + fields.emplace_back(create_record_field(b->id(), *ztype, false, false)); + else { + result(ztype.error()); + return; + } } hilti::ID local; hilti::ID ns; if ( auto id_ = id() ) { - local = id_->local(); - ns = id_->namespace_(); + local = id_.local(); + ns = id_.namespace_(); } else { // Invent a (hopefully unique) name for the Zeek-side record type // so that we can handle anonymous tuple types. static uint64_t i = 0; - local = hilti::util:: - fmt("__spicy_bitfield_%u" - "_%" PRIu64, - static_cast(getpid()), ++i); + local = + hilti::ID(hilti::util::fmt("__spicy_bitfield_%u" + "_%" PRIu64, + static_cast(getpid()), ++i)); ns = namespace_(); } - return create_record_type(ns, local, fields); + result(create_record_type(ns, local, fields)); } - result_t operator()(const hilti::type::Bool& t) { return base_type("zeek_rt::ZeekTypeTag::Bool"); } - result_t operator()(const hilti::type::Bytes& t) { return base_type("zeek_rt::ZeekTypeTag::String"); } - result_t operator()(const hilti::type::Interval& t) { return base_type("zeek_rt::ZeekTypeTag::Interval"); } - result_t operator()(const hilti::type::Port& t) { return base_type("zeek_rt::ZeekTypeTag::Port"); } - result_t operator()(const hilti::type::Real& t) { return base_type("zeek_rt::ZeekTypeTag::Double"); } - result_t operator()(const hilti::type::SignedInteger& t) { return base_type("zeek_rt::ZeekTypeTag::Int"); } - result_t operator()(const hilti::type::String& t) { return base_type("zeek_rt::ZeekTypeTag::String"); } - result_t operator()(const hilti::type::Time& t) { return base_type("zeek_rt::ZeekTypeTag::Time"); } - result_t operator()(const hilti::type::UnsignedInteger& t) { return base_type("zeek_rt::ZeekTypeTag::Count"); } + void operator()(hilti::type::Bool* t) final { result(base_type("zeek_rt::ZeekTypeTag::Bool")); } + void operator()(hilti::type::Bytes* t) final { result(base_type("zeek_rt::ZeekTypeTag::String")); } + void operator()(hilti::type::Interval* t) final { result(base_type("zeek_rt::ZeekTypeTag::Interval")); } + void operator()(hilti::type::Port* t) final { result(base_type("zeek_rt::ZeekTypeTag::Port")); } + void operator()(hilti::type::Real* t) final { result(base_type("zeek_rt::ZeekTypeTag::Double")); } + void operator()(hilti::type::SignedInteger* t) final { result(base_type("zeek_rt::ZeekTypeTag::Int")); } + void operator()(hilti::type::String* t) final { result(base_type("zeek_rt::ZeekTypeTag::String")); } + void operator()(hilti::type::Time* t) final { result(base_type("zeek_rt::ZeekTypeTag::Time")); } + void operator()(hilti::type::UnsignedInteger* t) final { result(base_type("zeek_rt::ZeekTypeTag::Count")); } - result_t operator()(const hilti::type::Enum& t) { + void operator()(hilti::type::Enum* t) final { assert(id()); - auto labels = hilti::rt::transform(t.labels(), [](const auto& l) { - return builder::tuple({builder::string_literal(l.get().id().str()), builder::integer(l.get().value())}); + auto labels = hilti::rt::transform(t->labels(), [this](const auto& l) { + return builder->tuple({builder->stringLiteral(l.get()->id().str()), builder->integer(l.get()->value())}); }); - auto tmp = body->addTmp(tmpName("labels", *id()), - hilti::type::Vector( - hilti::type::Tuple({hilti::type::String(), hilti::type::SignedInteger(64)}))); + auto tmp = + builder->addTmp(tmpName("labels", id()), + builder->typeVector( + builder->qualifiedType(builder->typeTuple( + {builder->qualifiedType(builder->typeString(), + hilti::Constness::Const), + builder->qualifiedType(builder->typeSignedInteger(64), + hilti::Constness::Const)}), + hilti::Constness::Const))); - for ( const auto& l : t.labels() ) - body->addMemberCall(tmp, "push_back", - {builder::tuple( - {builder::string_literal(l.get().id().str()), builder::integer(l.get().value())})}); + for ( const auto& l : t->labels() ) + builder->addMemberCall(tmp, "push_back", + {builder->tuple({builder->stringLiteral(l.get()->id().str()), + builder->integer(l.get()->value())})}); - return builder::call("zeek_rt::create_enum_type", - {builder::string_mut(id()->namespace_()), builder::string_mut(id()->local()), tmp}); + result(builder->call("zeek_rt::create_enum_type", {builder->stringMutable(id().namespace_().str()), + builder->stringMutable(id().local().str()), tmp})); } - result_t operator()(const hilti::type::Map& t) { - auto key = createZeekType(t.keyType()); - if ( ! key ) - return key.error(); - - auto value = createZeekType(t.valueType()); - if ( ! value ) - return value.error(); - - return builder::call("zeek_rt::create_table_type", {*key, *value}); - } - - result_t operator()(const hilti::type::Optional& t) { return createZeekType(t.dereferencedType()); } - - result_t operator()(const hilti::type::Set& t) { - auto elem = createZeekType(t.elementType()); - if ( ! elem ) - return elem.error(); - - return builder::call("zeek_rt::create_table_type", {*elem, builder::null()}); - } - - result_t operator()(const hilti::type::Struct& t) { - assert(id()); - - std::vector fields; - for ( const auto& f : t.fields() ) { - auto ztype = createZeekType(f.type()); - if ( ! ztype ) - return ztype.error(); - - fields.emplace_back(create_record_field(f.id(), *ztype, f.isOptional(), false)); + void operator()(hilti::type::Map* t) final { + auto key = createZeekType(t->keyType()); + if ( ! key ) { + result(key.error()); + return; } - return create_record_type(id()->namespace_(), id()->local(), fields); + auto value = createZeekType(t->valueType()); + if ( ! value ) { + result(value.error()); + return; + } + + result(builder->call("zeek_rt::create_table_type", {*key, *value})); } - result_t operator()(const hilti::type::Tuple& t) { - std::vector fields; - for ( const auto& f : t.elements() ) { - if ( ! f.id() ) - return hilti::result::Error("can only convert tuple types with all-named fields to Zeek"); + void operator()(hilti::type::Optional* t) final { result(createZeekType(t->dereferencedType())); } - auto ztype = createZeekType(f.type()); - if ( ! ztype ) - return ztype.error(); + void operator()(hilti::type::Set* t) final { + if ( auto elem = createZeekType(t->elementType()) ) + result(builder->call("zeek_rt::create_table_type", {*elem, builder->null()})); + else + result(elem.error()); + } - fields.emplace_back(create_record_field(*f.id(), *ztype, false, false)); + void operator()(hilti::type::Struct* t) final { + assert(id()); + + hilti::Expressions fields; + for ( const auto& f : t->fields() ) { + if ( auto ztype = createZeekType(f->type()) ) + fields.emplace_back(create_record_field(f->id(), *ztype, f->isOptional(), false)); + else { + result(ztype.error()); + return; + } + } + + result(create_record_type(id().namespace_(), id().local(), fields)); + } + + void operator()(hilti::type::Tuple* t) final { + hilti::Expressions fields; + for ( const auto& f : t->elements() ) { + if ( ! f->id() ) { + result(hilti::result::Error("can only convert tuple types with all-named fields to Zeek")); + return; + } + + auto ztype = createZeekType(f->type()); + if ( ! ztype ) { + result(ztype.error()); + return; + } + + fields.emplace_back(create_record_field(f->id(), *ztype, false, false)); } hilti::ID local; hilti::ID ns; if ( auto id_ = id() ) { - local = id_->local(); - ns = id_->namespace_(); + local = id_.local(); + ns = id_.namespace_(); } else { // Invent a (hopefully unique) name for the Zeek-side record type // so that we can handle anonymous tuple types. static uint64_t i = 0; - local = hilti::util:: - fmt("__spicy_tuple_%u" - "_%" PRIu64, - static_cast(getpid()), ++i); + local = + hilti::ID(hilti::util::fmt("__spicy_tuple_%u" + "_%" PRIu64, + static_cast(getpid()), ++i)); ns = namespace_(); } - return create_record_type(ns, local, fields); + result(create_record_type(ns, local, fields)); } - result_t operator()(const ::spicy::type::Unit& t) { + void operator()(::spicy::type::Unit* t) final { assert(id()); - std::vector fields; + hilti::Expressions fields; for ( const auto& f : gc->recordFields(t) ) { - auto export_ = gc->exportForField(*id(), hilti::ID(f.id)); + auto export_ = gc->exportForField(id(), hilti::ID(f.id)); if ( export_.skip ) continue; // Special-case: Lift up elements of anonymous bitfields. - if ( auto bf = f.type.tryAs(); bf && f.is_anonymous ) { + if ( auto bf = f.type->type()->tryAs(); bf && f.is_anonymous ) { for ( const auto& b : bf->bits() ) { - auto ztype = createZeekType(b.itemType()); - if ( ! ztype ) - return ztype.error(); + auto ztype = createZeekType(b->itemType()); + if ( ! ztype ) { + result(ztype.error()); + return; + } - fields.emplace_back(create_record_field(b.id(), *ztype, f.is_optional, export_.log)); + fields.emplace_back(create_record_field(b->id(), *ztype, f.is_optional, export_.log)); } } else if ( ! f.is_anonymous ) { auto ztype = createZeekType(f.type); - if ( ! ztype ) - return ztype.error(); + if ( ! ztype ) { + result(ztype.error()); + return; + } fields.emplace_back(create_record_field(f.id, *ztype, f.is_optional, export_.log)); } } - return create_record_type(id()->namespace_(), id()->local(), fields); + result(create_record_type(id().namespace_(), id().local(), fields)); } - result_t operator()(const hilti::type::Vector& t) { - auto elem = createZeekType(t.elementType()); - if ( ! elem ) - return elem.error(); - - return builder::call("zeek_rt::create_vector_type", {*elem}); + void operator()(hilti::type::Vector* t) final { + if ( auto elem = createZeekType(t->elementType()) ) + result(builder->call("zeek_rt::create_vector_type", {*elem})); + else + result(elem.error()); } }; } // namespace -hilti::Result GlueCompiler::createZeekType(const hilti::Type& t, const hilti::ID& id, - hilti::builder::Builder* body, - GlueCompiler::ZeekTypeCache* cache) const { - body->addComment(hilti::util::fmt("Creating Zeek type %s", id)); +hilti::Result GlueCompiler::createZeekType(const hilti::QualifiedTypePtr& t, const hilti::ID& id, + Builder* builder, GlueCompiler::ZeekTypeCache* cache) const { + builder->addComment(hilti::util::fmt("Creating Zeek type %s", id)); - auto v = VisitorZeekType(this, body, cache); + auto v = VisitorZeekType(this, builder, cache); if ( auto zt = v.createZeekType(t, id) ) { - body->addCall("zeek_rt::register_type", - {builder::string_mut(id.namespace_()), builder::string_mut(id.local()), *zt}); + builder->addCall("zeek_rt::register_type", {builder->stringMutable(id.namespace_().str()), + builder->stringMutable(id.local().str()), *zt}); return hilti::Nothing(); } else @@ -1642,42 +1694,42 @@ hilti::Result GlueCompiler::createZeekType(const hilti::Type& t, } namespace { -struct VisitorUnitFields : hilti::visitor::PreOrder { +struct VisitorUnitFields : spicy::visitor::PreOrder { // NOTE: Align this logic with struct generation in Spicy's unit builder. std::vector fields; - void operator()(const ::spicy::type::unit::item::Field& f, position_t p) { - if ( (f.isTransient() && ! f.isAnonymous()) || f.parseType().isA() ) + void operator()(::spicy::type::unit::item::Field* n) { + if ( (n->isTransient() && ! n->isAnonymous()) || n->parseType()->type()->isA() ) return; - auto field = GlueCompiler::RecordField{.id = f.id(), - .type = f.itemType(), + auto field = GlueCompiler::RecordField{.id = n->id(), + .type = n->itemType(), .is_optional = true, - .is_anonymous = f.isAnonymous()}; + .is_anonymous = n->isAnonymous()}; fields.emplace_back(std::move(field)); } - void operator()(const ::spicy::type::unit::item::Variable& f, const position_t p) { - auto field = GlueCompiler::RecordField{.id = f.id(), - .type = f.itemType(), - .is_optional = f.isOptional(), + void operator()(::spicy::type::unit::item::Variable* b) { + auto field = GlueCompiler::RecordField{.id = b->id(), + .type = b->itemType(), + .is_optional = b->isOptional(), .is_anonymous = false}; fields.emplace_back(std::move(field)); } - void operator()(const ::spicy::type::unit::item::Switch& f, const position_t p) { - for ( const auto& c : f.cases() ) { - for ( const auto& i : c.items() ) + void operator()(::spicy::type::unit::item::Switch* n) { + for ( const auto& c : n->cases() ) { + for ( const auto& i : c->items() ) dispatch(i); } } }; } // namespace -std::vector GlueCompiler::recordFields(const ::spicy::type::Unit& unit) { +std::vector GlueCompiler::recordFields(const ::spicy::type::Unit* unit) { VisitorUnitFields unit_field_converter; - for ( const auto& i : unit.items() ) + for ( const auto& i : unit->items() ) unit_field_converter.dispatch(i); return std::move(unit_field_converter.fields); diff --git a/src/spicy/spicyz/glue-compiler.h b/src/spicy/spicyz/glue-compiler.h index b6e5e6672c..8b9421e7fe 100644 --- a/src/spicy/spicyz/glue-compiler.h +++ b/src/spicy/spicyz/glue-compiler.h @@ -18,12 +18,13 @@ #include #include +#include #include -#include #include #include #include +#include #include #include @@ -100,25 +101,25 @@ struct SpicyModule { std::set evts; /**< EVT files that refer to this module. */ // Generated code. - std::optional spicy_module; /**< the ``BroHooks_*.spicy`` module. */ + hilti::ModulePtr spicy_module; /**< the ``BroHooks_*.spicy`` module. */ }; /** Representation of an event parsed from an EVT file. */ struct Event { // Information parsed directly from the *.evt file. - hilti::rt::filesystem::path file; /**< The path of the *.evt file we parsed this from. */ - hilti::ID name; /**< The name of the event. */ - hilti::ID path; /**< The hook path as specified in the evt file. */ - std::vector parameters; /**< Event parameters specified in the evt file. */ - std::string condition; /**< Condition that must be true for the event to trigger. */ - std::vector exprs; /**< The argument expressions. */ - int priority; /**< Event/hook priority. */ - hilti::Location location; /**< Location where event is defined. */ + hilti::rt::filesystem::path file; /**< The path of the *.evt file we parsed this from. */ + hilti::ID name; /**< The name of the event. */ + hilti::ID path; /**< The hook path as specified in the evt file. */ + hilti::type::function::Parameters parameters; /**< Event parameters specified in the evt file. */ + std::string condition; /**< Condition that must be true for the event to trigger. */ + std::vector exprs; /**< The argument expressions. */ + int priority; /**< Event/hook priority. */ + hilti::Location location; /**< Location where event is defined. */ // Computed information. hilti::ID hook; /**< The name of the hook triggering the event. */ hilti::ID unit; /**< The fully qualified name of the unit type. */ - std::optional<::spicy::type::Unit> unit_type; /**< The Spicy type of referenced unit. */ + ::spicy::type::UnitPtr unit_type; /**< The Spicy type of referenced unit. */ hilti::ID unit_module_id; /**< The name of the module the referenced unit is defined in. */ hilti::rt::filesystem::path unit_module_path; /**< The path of the module that the referenced unit is defined in. */ std::shared_ptr @@ -127,8 +128,8 @@ struct Event { // TODO: The following aren't set yet. // Code generation. - std::optional<::spicy::declaration::UnitHook> spicy_hook; /**< The generated Spicy hook. */ - std::optional hilti_raise; /**< The generated HILTI raise() function. */ + ::spicy::type::unit::item::UnitHookPtr spicy_hook; /**< The generated Spicy hook. */ + std::shared_ptr hilti_raise; /**< The generated HILTI raise() function. */ std::vector expression_accessors; /**< One HILTI function per expression to access the value. */ }; @@ -162,6 +163,18 @@ public: /** Destructor. */ virtual ~GlueCompiler(); + /** + * Returns the AST context in use. Only available once the driver has + * initialized the glue compiler. + */ + auto* context() { return _driver->context()->astContext().get(); } + + /** + * Returns the AST builder in use. Only available once the driver has + * initialized the glue compiler. + */ + auto* builder() { return static_cast<::spicy::Builder*>(_driver->builder()); } + /** Parses an `*.evt` file, without generating any code yet. */ bool loadEvtFile(hilti::rt::filesystem::path& path); @@ -199,21 +212,21 @@ public: */ ExportedField exportForField(const hilti::ID& zeek_id, const hilti::ID& field_id) const; - using ZeekTypeCache = std::map; + using ZeekTypeCache = std::map; /** * Generates code to convert a HILTI type to a corresponding Zeek type at * runtime. The code is added to the given body. */ - hilti::Result createZeekType(const hilti::Type& t, const hilti::ID& id, - hilti::builder::Builder* body, ZeekTypeCache* cache) const; + hilti::Result createZeekType(const hilti::QualifiedTypePtr& t, const hilti::ID& id, + ::spicy::Builder* builder, ZeekTypeCache* cache) const; /** Return type for `recordField()`. */ struct RecordField { - hilti::ID id; /**< name of record field */ - hilti::Type type; /**< Spicy-side type object */ - bool is_optional; /**< true if field is optional */ - bool is_anonymous; /**< true if field is annymous */ + hilti::ID id; /**< name of record field */ + hilti::QualifiedTypePtr type; /**< Spicy-side type object */ + bool is_optional; /**< true if field is optional */ + bool is_anonymous; /**< true if field is annymous */ }; /** @@ -223,13 +236,13 @@ public: * @param unit the unit type to retrieve fields for * @return list of fields */ - static std::vector recordFields(const ::spicy::type::Unit& unit); + static std::vector recordFields(const ::spicy::type::Unit* unit); protected: friend class Driver; /** Called by driver to initialized a provided glue compiler. */ - void Init(Driver* driver, int zeek_version); + void init(Driver* driver, int zeek_version); private: /** @@ -269,8 +282,8 @@ private: std::map> _spicy_modules; - std::vector>> - _imports; /**< imports from EVT files, with ID and optional scope */ + std::vector> + _imports; /**< imports from EVT files, with ID and scope; latter is optional and can be empty */ std::map _exports; /**< exports from EVT files */ std::vector _events; /**< events parsed from EVT files */ std::vector _protocol_analyzers; /**< protocol analyzers parsed from EVT files */ diff --git a/src/spicy/spicyz/main.cc b/src/spicy/spicyz/main.cc index 38175f4431..7541b53d32 100644 --- a/src/spicy/spicyz/main.cc +++ b/src/spicy/spicyz/main.cc @@ -115,7 +115,7 @@ static hilti::Result parseOptions(int argc, char** argv, hilti::driver: } case 'g': { - driver_options->global_optimizations = false; + compiler_options->global_optimizations = false; break; } diff --git a/testing/btest/Baseline/spicy.event-args-fail-2/output b/testing/btest/Baseline/spicy.event-args-fail-2/output index bc372c9ae4..86459d9d88 100644 --- a/testing/btest/Baseline/spicy.event-args-fail-2/output +++ b/testing/btest/Baseline/spicy.event-args-fail-2/output @@ -1,3 +1,3 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. [error] <...>/event-args-fail.evt:2: signature for hook must be: %error or %error(err: string) -[error] : aborting after errors +[error] : aborting after errors diff --git a/testing/btest/Baseline/spicy.event-args-fail-3/output b/testing/btest/Baseline/spicy.event-args-fail-3/output index bc372c9ae4..86459d9d88 100644 --- a/testing/btest/Baseline/spicy.event-args-fail-3/output +++ b/testing/btest/Baseline/spicy.event-args-fail-3/output @@ -1,3 +1,3 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. [error] <...>/event-args-fail.evt:2: signature for hook must be: %error or %error(err: string) -[error] : aborting after errors +[error] : aborting after errors diff --git a/testing/btest/Baseline/spicy.export-type-fail/output b/testing/btest/Baseline/spicy.export-type-fail/output index 8f28a3feaa..6d09068485 100644 --- a/testing/btest/Baseline/spicy.export-type-fail/output +++ b/testing/btest/Baseline/spicy.export-type-fail/output @@ -3,4 +3,4 @@ [error] unknown type 'Test::DOES_NOT_EXIST' exported [error] <...>/foo.spicy:3:1-5:2: cannot export Spicy type 'Test::X': type is self-recursive [error] <...>/foo.spicy:11:1-13:2: cannot export Spicy type 'Test::Z': can only convert tuple types with all-named fields to Zeek -[error] : aborting after errors +[error] : aborting after errors diff --git a/testing/btest/Baseline/spicy.export-type-with-fields-fail/output b/testing/btest/Baseline/spicy.export-type-with-fields-fail/output index ba576887c5..a4be0cde29 100644 --- a/testing/btest/Baseline/spicy.export-type-with-fields-fail/output +++ b/testing/btest/Baseline/spicy.export-type-with-fields-fail/output @@ -1,4 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. [error] <...>/foo.spicy:3:1-5:2: type 'Test::A' does not have field 'does_not_exist' [error] <...>/foo.spicy:3:1-5:2: type 'Test::A' does not have field 'does_not_exist' -[error] : aborting after errors +[error] : aborting after errors diff --git a/testing/btest/Baseline/spicy.profiling/prof.log b/testing/btest/Baseline/spicy.profiling/prof.log index 30c7611635..14502ac690 100644 --- a/testing/btest/Baseline/spicy.profiling/prof.log +++ b/testing/btest/Baseline/spicy.profiling/prof.log @@ -3,11 +3,11 @@ # Profiling # #name count -hilti/func/SSH::Banner::__on_0x25_done 2 hilti/func/SSH::Banner::__parse_SSH_Banner_stage2 2 hilti/func/SSH::Banner::__parse_stage1 2 hilti/func/SSH::Banner::parse1 2 hilti/func/SSH::__register_SSH_Banner 1 +hilti/func/spicy_hooks_SSH::SSH::Banner::__on_0x25_done 2 hilti/total 1 spicy/prepare/stream/SSH::Banner 2 spicy/unit/SSH::Banner 2 diff --git a/testing/btest/Baseline/spicy.tuple-arg/output b/testing/btest/Baseline/spicy.tuple-arg/output index 3d15a46ac5..20fbee3d84 100644 --- a/testing/btest/Baseline/spicy.tuple-arg/output +++ b/testing/btest/Baseline/spicy.tuple-arg/output @@ -5,5 +5,5 @@ [i=4, s=] [zeek] [SPICY_SSH/3/orig] -> event ssh::banner((1, b"OpenSSH_3.8.1p1")) [zeek] [SPICY_SSH/3/orig] -> event ssh::banner((2, b"OpenSSH_3.8.1p1")) -[zeek] [SPICY_SSH/3/orig] -> event ssh::banner((3, )) +[zeek] [SPICY_SSH/3/orig] -> event ssh::banner((3, b"")) [zeek] [SPICY_SSH/3/orig] -> event ssh::banner((4, Null)) diff --git a/testing/btest/spicy/tuple-arg.zeek b/testing/btest/spicy/tuple-arg.zeek index 4b8a4be8bc..4a50d593ec 100644 --- a/testing/btest/spicy/tuple-arg.zeek +++ b/testing/btest/spicy/tuple-arg.zeek @@ -6,7 +6,7 @@ # @TEST-EXEC: btest-diff output type Foo: record { - i: int; + i: int; s: string &optional; }; @@ -27,7 +27,7 @@ public type Banner = unit { dash : /-/; software: /[^\r\n]*/; - unset_field: uint64 if ( False ); + unset_field: bytes &eod if ( False ); on %done { zeek::confirm_protocol(); } };