Port Spicy integration to new AST API.

This commit is contained in:
Robin Sommer 2024-02-23 10:29:59 +01:00
parent 724daa2792
commit db98dc4193
No known key found for this signature in database
GPG key ID: D8187293B3FFE5D0
16 changed files with 438 additions and 364 deletions

@ -1 +1 @@
Subproject commit c39bdb5ed2bff0d390e6eae4ddda83050f047904
Subproject commit b1c3c4e4eae55ed1924f90c5705d78be9dbfdccc

View file

@ -121,7 +121,7 @@ type ASN1OctetString = unit(len: uint64, constructed: bool) {
# https://www.obj-sys.com/asn1tutorial/node124.html
type ASN1String = unit(tag: ASN1Tag, len: uint64) {
var encoding: hilti::Charset;
var encoding: spicy::Charset;
on %init {
switch ( tag.type_ ) {
@ -132,7 +132,7 @@ type ASN1String = unit(tag: ASN1Tag, len: uint64) {
case ASN1Type::PrintableString,
ASN1Type::GeneralizedTime,
ASN1Type::UTCTime: {
self.encoding = hilti::Charset::ASCII;
self.encoding = spicy::Charset::ASCII;
}
case ASN1Type::UTF8String,
@ -148,7 +148,7 @@ type ASN1String = unit(tag: ASN1Tag, len: uint64) {
# BMPString and UniversalString. This *may* not be correct.
ASN1Type::BMPString,
ASN1Type::UniversalString: {
self.encoding = hilti::Charset::UTF8;
self.encoding = spicy::Charset::UTF8;
}
}
}
@ -216,7 +216,7 @@ public type ASN1Body = unit(head: ASN1Header, recursive: bool) {
ASN1Type::BitString -> bitstr_value: ASN1BitString(head.len.len, head.tag.constructed);
ASN1Type::OctetString -> str_value: ASN1OctetString(head.len.len, head.tag.constructed)
&convert=$$.value.decode(hilti::Charset::ASCII);
&convert=$$.value.decode(spicy::Charset::ASCII);
ASN1Type::ObjectIdentifier -> str_value: ASN1ObjectIdentifier(head.len.len)
&convert=$$.oidstring;

View file

@ -733,7 +733,7 @@ type SearchFilter = unit {
FilterType::FILTER_EXT -> FILTER_EXT: DecodedAttributeValue(FilterType::FILTER_EXT)
&parse-from=self.filterBytes;
FilterType::FILTER_PRESENT -> FILTER_PRESENT: ASN1::ASN1OctetString(self.filterLen, False)
&convert=$$.value.decode(hilti::Charset::ASCII)
&convert=$$.value.decode(spicy::Charset::ASCII)
&parse-from=self.filterBytes;
};

View file

@ -280,7 +280,7 @@ public type LongHeaderPacket = unit {
};
# A QUIC Frame.
public type Frame = unit(header: LongHeaderPacket, from_client: bool, crypto_sink: sink) {
public type Frame = unit(header: LongHeaderPacket, from_client: bool, inout crypto_sink: sink) {
frame_type : uint8 &convert=cast<FrameType>($$);
# TODO: add other FrameTypes as well
@ -447,7 +447,7 @@ type CryptoBuffer = unit() {
#
# A UDP datagram contains one or more QUIC packets.
##############
type Packet = unit(from_client: bool, context: ConnectionIDInfo&) {
type Packet = unit(from_client: bool, inout context: ConnectionIDInfo&) {
var decrypted_data: bytes;
var full_packet: bytes;
var start: iterator<stream>;

View file

@ -16,11 +16,12 @@
#include <hilti/ast/declarations/type.h>
#include <hilti/compiler/init.h>
#include <spicy/ast/detail/visitor.h>
#include <spicy/ast/types/unit.h>
#include <spicy/ast/visitor.h>
#include <spicy/autogen/config.h>
#include <spicy/compiler/init.h>
#include "compiler/plugin.h"
#include "config.h"
#include "glue-compiler.h"
@ -31,28 +32,45 @@ using Driver = ::zeek::spicy::Driver;
* Visitor to type information from a HILTI AST. This extracts user-visible
* types only, we skip any internal ones.
*/
struct VisitorTypes : public hilti::visitor::PreOrder<void, VisitorTypes> {
explicit VisitorTypes(Driver* driver, hilti::ID module, hilti::rt::filesystem::path path, bool is_resolved)
: driver(driver), module(std::move(module)), path(std::move(path)), is_resolved(is_resolved) {}
struct VisitorTypes : public spicy::visitor::PreOrder {
explicit VisitorTypes(Driver* driver, GlueCompiler* glue, bool is_resolved)
: driver(driver), glue(glue), is_resolved(is_resolved) {}
void operator()(const hilti::declaration::Type& t) {
assert(! t.type().typeID() || *t.type().typeID() == hilti::ID(module, t.id())); // ensure consistent IDs
void operator()(hilti::declaration::Module* n) final {
if ( n->uid().in_memory ) {
// Ignore modules built by us in memory.
module = {};
return;
}
if ( module == hilti::ID("hilti") || module == hilti::ID("spicy_rt") || module == hilti::ID("zeek_rt") )
module = n->scopeID();
path = n->uid().path;
if ( is_resolved )
glue->addSpicyModule(module, path);
}
void operator()(hilti::declaration::Type* n) final {
if ( module.empty() || module == hilti::ID("hilti") || module == hilti::ID("spicy_rt") ||
module == hilti::ID("zeek_rt") )
return;
assert(! n->type()->type()->typeID() ||
n->type()->type()->typeID() == hilti::ID(module, n->id())); // ensure consistent IDs
types.emplace_back(TypeInfo{
.id = hilti::ID(module, t.id()),
.type = t.type()._clone().as<hilti::Type>(),
.linkage = t.linkage(),
.id = hilti::ID(module, n->id()),
.type = n->type(),
.linkage = n->linkage(),
.is_resolved = is_resolved,
.module_id = module,
.module_path = path,
.location = t.meta().location(),
.location = n->meta().location(),
});
}
Driver* driver;
GlueCompiler* glue;
hilti::ID module;
hilti::rt::filesystem::path path;
bool is_resolved;
@ -61,8 +79,8 @@ struct VisitorTypes : public hilti::visitor::PreOrder<void, VisitorTypes> {
Driver::Driver(std::unique_ptr<GlueCompiler> glue, const char* argv0, hilti::rt::filesystem::path lib_path,
int zeek_version)
: ::spicy::Driver("<Spicy Plugin for Zeek>"), _glue(std::move(glue)) {
_glue->Init(this, zeek_version);
: ::spicy::Driver("<Spicy support for Zeek>"), _glue(std::move(glue)) {
_glue->init(this, zeek_version);
::spicy::Configuration::extendHiltiConfiguration();
auto options = hiltiOptions();
@ -195,6 +213,8 @@ hilti::Result<hilti::Nothing> Driver::compile() {
SPICY_DEBUG("Running Spicy driver");
_error = hilti::Nothing();
if ( auto x = ::spicy::Driver::compile(); ! x )
return x.error();
@ -243,23 +263,16 @@ std::vector<std::pair<TypeInfo, hilti::ID>> Driver::exportedTypes() const {
return result;
}
void Driver::hookNewASTPreCompilation(std::shared_ptr<hilti::Unit> unit) {
if ( unit->extension() != ".spicy" )
return;
if ( unit->path().empty() )
// Ignore modules constructed in memory.
return;
auto v = VisitorTypes(this, unit->id(), unit->path(), false);
for ( auto i : v.walk(unit->module()) )
v.dispatch(i);
void Driver::hookNewASTPreCompilation(const hilti::Plugin& plugin, const std::shared_ptr<hilti::ASTRoot>& root) {
auto v = VisitorTypes(this, _glue.get(), false);
hilti::visitor::visit(v, root, ".spicy");
for ( const auto& ti : v.types ) {
SPICY_DEBUG(hilti::util::fmt(" Got type '%s' (pre-compile)", ti.id));
_types[ti.id] = ti;
if ( auto et = ti.type.tryAs<hilti::type::Enum>(); et && ti.linkage == hilti::declaration::Linkage::Public ) {
if ( auto et = ti.type->type()->tryAs<hilti::type::Enum>();
et && ti.linkage == hilti::declaration::Linkage::Public ) {
SPICY_DEBUG(" Automatically exporting public enum for backwards compatibility");
_public_enums.push_back(ti);
}
@ -268,17 +281,17 @@ void Driver::hookNewASTPreCompilation(std::shared_ptr<hilti::Unit> unit) {
}
}
void Driver::hookNewASTPostCompilation(std::shared_ptr<hilti::Unit> unit) {
if ( unit->extension() != ".spicy" )
return;
bool Driver::hookNewASTPostCompilation(const hilti::Plugin& plugin, const std::shared_ptr<hilti::ASTRoot>& root) {
if ( plugin.component != "Spicy" )
return false;
if ( unit->path().empty() )
// Ignore modules constructed in memory.
return;
if ( ! _need_glue )
return false;
auto v = VisitorTypes(this, unit->id(), unit->path(), true);
for ( auto i : v.walk(unit->module()) )
v.dispatch(i);
_need_glue = false;
auto v = VisitorTypes(this, _glue.get(), true);
hilti::visitor::visit(v, root, ".spicy");
for ( auto&& t : v.types ) {
SPICY_DEBUG(hilti::util::fmt(" Got type '%s' (post-compile)", t.id));
@ -286,19 +299,14 @@ void Driver::hookNewASTPostCompilation(std::shared_ptr<hilti::Unit> unit) {
hookNewType(t);
}
_glue->addSpicyModule(unit->id(), unit->path());
if ( ! _glue->compile() )
_error = hilti::result::Error("glue compilation failed");
return true;
}
hilti::Result<hilti::Nothing> Driver::hookCompilationFinished(const hilti::Plugin& plugin) {
if ( ! _need_glue )
return hilti::Nothing();
_need_glue = false;
if ( _glue->compile() )
return hilti::Nothing();
else
return hilti::result::Error("glue compilation failed");
hilti::Result<hilti::Nothing> Driver::hookCompilationFinished(const std::shared_ptr<hilti::ASTRoot>& root) {
return _error;
}
void Driver::hookInitRuntime() { ::spicy::rt::init(); }

View file

@ -32,7 +32,7 @@ class GlueCompiler;
struct TypeInfo {
hilti::ID id; /**< fully-qualified name of the type */
hilti::Type type; /**< the type itself */
hilti::QualifiedTypePtr type; /**< the type itself */
hilti::declaration::Linkage linkage; /**< linkage of of the type's declaration */
bool is_resolved = false; /**< true if we are far enough in processing that the type has been fully resolved */
hilti::ID module_id; /**< name of module type is defined in */
@ -102,7 +102,7 @@ public:
if ( ! ti )
return ti.error();
if ( ! ti->type.isA<T>() )
if ( ! ti->type->type()->isA<T>() )
return hilti::result::Error(hilti::util::fmt("'%s' is not of expected type", id));
return ti;
@ -165,7 +165,7 @@ public:
protected:
/**
* Hook executed for all type declarations encountered in a Spicy module.
* Derived classes may override this to add custom processing. This hooks
* Derived classes may override this to add custom processing. This hook
* executes twices for each declaration: once before we compile the AST
* (meaning types have not been resolved yet), and once after. The type
* info's `is_resolved` field indicates which of the two we're in.
@ -175,13 +175,13 @@ protected:
virtual void hookNewType(const TypeInfo& ti) {}
/** Overridden from HILTI driver. */
void hookNewASTPreCompilation(std::shared_ptr<hilti::Unit> unit) override;
void hookNewASTPreCompilation(const hilti::Plugin& plugin, const std::shared_ptr<hilti::ASTRoot>& root) override;
/** Overridden from HILTI driver. */
void hookNewASTPostCompilation(std::shared_ptr<hilti::Unit> unit) override;
bool hookNewASTPostCompilation(const hilti::Plugin& plugin, const std::shared_ptr<hilti::ASTRoot>& root) override;
/** Overridden from HILTI driver. */
hilti::Result<hilti::Nothing> hookCompilationFinished(const hilti::Plugin& plugin) override;
hilti::Result<hilti::Nothing> hookCompilationFinished(const std::shared_ptr<hilti::ASTRoot>& root) override;
/** Overridden from HILTI driver. */
void hookInitRuntime() override;
@ -194,6 +194,7 @@ protected:
std::vector<TypeInfo> _public_enums; // tracks Spicy enum types declared public, for automatic export
bool _using_build_directory = false; // true if we're running out of the plugin's build directory
bool _need_glue = true; // true if glue code has not yet been generated
hilti::Result<hilti::Nothing> _error = hilti::Nothing(); // error encountered during compilation
};
} // namespace zeek::spicy

File diff suppressed because it is too large Load diff

View file

@ -18,12 +18,13 @@
#include <hilti/ast/builder/builder.h>
#include <hilti/ast/declarations/function.h>
#include <hilti/ast/declarations/module.h>
#include <hilti/ast/expression.h>
#include <hilti/ast/module.h>
#include <hilti/ast/type.h>
#include <hilti/compiler/context.h>
#include <hilti/compiler/driver.h>
#include <spicy/ast/builder/builder.h>
#include <spicy/ast/declarations/unit-hook.h>
#include <spicy/ast/types/unit.h>
@ -100,7 +101,7 @@ struct SpicyModule {
std::set<hilti::rt::filesystem::path> evts; /**< EVT files that refer to this module. */
// Generated code.
std::optional<hilti::Module> spicy_module; /**< the ``BroHooks_*.spicy`` module. */
hilti::ModulePtr spicy_module; /**< the ``BroHooks_*.spicy`` module. */
};
/** Representation of an event parsed from an EVT file. */
@ -109,7 +110,7 @@ struct Event {
hilti::rt::filesystem::path file; /**< The path of the *.evt file we parsed this from. */
hilti::ID name; /**< The name of the event. */
hilti::ID path; /**< The hook path as specified in the evt file. */
std::vector<hilti::type::function::Parameter> parameters; /**< Event parameters specified in the evt file. */
hilti::type::function::Parameters parameters; /**< Event parameters specified in the evt file. */
std::string condition; /**< Condition that must be true for the event to trigger. */
std::vector<std::string> exprs; /**< The argument expressions. */
int priority; /**< Event/hook priority. */
@ -118,7 +119,7 @@ struct Event {
// Computed information.
hilti::ID hook; /**< The name of the hook triggering the event. */
hilti::ID unit; /**< The fully qualified name of the unit type. */
std::optional<::spicy::type::Unit> unit_type; /**< The Spicy type of referenced unit. */
::spicy::type::UnitPtr unit_type; /**< The Spicy type of referenced unit. */
hilti::ID unit_module_id; /**< The name of the module the referenced unit is defined in. */
hilti::rt::filesystem::path unit_module_path; /**< The path of the module that the referenced unit is defined in. */
std::shared_ptr<glue::SpicyModule>
@ -127,8 +128,8 @@ struct Event {
// TODO: The following aren't set yet.
// Code generation.
std::optional<::spicy::declaration::UnitHook> spicy_hook; /**< The generated Spicy hook. */
std::optional<hilti::declaration::Function> hilti_raise; /**< The generated HILTI raise() function. */
::spicy::type::unit::item::UnitHookPtr spicy_hook; /**< The generated Spicy hook. */
std::shared_ptr<hilti::declaration::Function> hilti_raise; /**< The generated HILTI raise() function. */
std::vector<ExpressionAccessor> expression_accessors; /**< One HILTI function per expression to access the value. */
};
@ -162,6 +163,18 @@ public:
/** Destructor. */
virtual ~GlueCompiler();
/**
* Returns the AST context in use. Only available once the driver has
* initialized the glue compiler.
*/
auto* context() { return _driver->context()->astContext().get(); }
/**
* Returns the AST builder in use. Only available once the driver has
* initialized the glue compiler.
*/
auto* builder() { return static_cast<::spicy::Builder*>(_driver->builder()); }
/** Parses an `*.evt` file, without generating any code yet. */
bool loadEvtFile(hilti::rt::filesystem::path& path);
@ -199,19 +212,19 @@ public:
*/
ExportedField exportForField(const hilti::ID& zeek_id, const hilti::ID& field_id) const;
using ZeekTypeCache = std::map<hilti::ID, hilti::Expression>;
using ZeekTypeCache = std::map<hilti::ID, hilti::ExpressionPtr>;
/**
* Generates code to convert a HILTI type to a corresponding Zeek type at
* runtime. The code is added to the given body.
*/
hilti::Result<hilti::Nothing> createZeekType(const hilti::Type& t, const hilti::ID& id,
hilti::builder::Builder* body, ZeekTypeCache* cache) const;
hilti::Result<hilti::Nothing> createZeekType(const hilti::QualifiedTypePtr& t, const hilti::ID& id,
::spicy::Builder* builder, ZeekTypeCache* cache) const;
/** Return type for `recordField()`. */
struct RecordField {
hilti::ID id; /**< name of record field */
hilti::Type type; /**< Spicy-side type object */
hilti::QualifiedTypePtr type; /**< Spicy-side type object */
bool is_optional; /**< true if field is optional */
bool is_anonymous; /**< true if field is annymous */
};
@ -223,13 +236,13 @@ public:
* @param unit the unit type to retrieve fields for
* @return list of fields
*/
static std::vector<RecordField> recordFields(const ::spicy::type::Unit& unit);
static std::vector<RecordField> recordFields(const ::spicy::type::Unit* unit);
protected:
friend class Driver;
/** Called by driver to initialized a provided glue compiler. */
void Init(Driver* driver, int zeek_version);
void init(Driver* driver, int zeek_version);
private:
/**
@ -269,8 +282,8 @@ private:
std::map<hilti::ID, std::shared_ptr<glue::SpicyModule>> _spicy_modules;
std::vector<std::pair<hilti::ID, std::optional<hilti::ID>>>
_imports; /**< imports from EVT files, with ID and optional scope */
std::vector<std::pair<hilti::ID, hilti::ID>>
_imports; /**< imports from EVT files, with ID and scope; latter is optional and can be empty */
std::map<hilti::ID, glue::Export> _exports; /**< exports from EVT files */
std::vector<glue::Event> _events; /**< events parsed from EVT files */
std::vector<glue::ProtocolAnalyzer> _protocol_analyzers; /**< protocol analyzers parsed from EVT files */

View file

@ -115,7 +115,7 @@ static hilti::Result<Nothing> parseOptions(int argc, char** argv, hilti::driver:
}
case 'g': {
driver_options->global_optimizations = false;
compiler_options->global_optimizations = false;
break;
}

View file

@ -1,3 +1,3 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
[error] <...>/event-args-fail.evt:2: signature for hook must be: %error or %error(err: string)
[error] <Spicy Plugin for Zeek>: aborting after errors
[error] <Spicy support for Zeek>: aborting after errors

View file

@ -1,3 +1,3 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
[error] <...>/event-args-fail.evt:2: signature for hook must be: %error or %error(err: string)
[error] <Spicy Plugin for Zeek>: aborting after errors
[error] <Spicy support for Zeek>: aborting after errors

View file

@ -3,4 +3,4 @@
[error] unknown type 'Test::DOES_NOT_EXIST' exported
[error] <...>/foo.spicy:3:1-5:2: cannot export Spicy type 'Test::X': type is self-recursive
[error] <...>/foo.spicy:11:1-13:2: cannot export Spicy type 'Test::Z': can only convert tuple types with all-named fields to Zeek
[error] <Spicy Plugin for Zeek>: aborting after errors
[error] <Spicy support for Zeek>: aborting after errors

View file

@ -1,4 +1,4 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
[error] <...>/foo.spicy:3:1-5:2: type 'Test::A' does not have field 'does_not_exist'
[error] <...>/foo.spicy:3:1-5:2: type 'Test::A' does not have field 'does_not_exist'
[error] <Spicy Plugin for Zeek>: aborting after errors
[error] <Spicy support for Zeek>: aborting after errors

View file

@ -3,11 +3,11 @@
# Profiling
#
#name count
hilti/func/SSH::Banner::__on_0x25_done 2
hilti/func/SSH::Banner::__parse_SSH_Banner_stage2 2
hilti/func/SSH::Banner::__parse_stage1 2
hilti/func/SSH::Banner::parse1 2
hilti/func/SSH::__register_SSH_Banner 1
hilti/func/spicy_hooks_SSH::SSH::Banner::__on_0x25_done 2
hilti/total 1
spicy/prepare/stream/SSH::Banner 2
spicy/unit/SSH::Banner 2

View file

@ -5,5 +5,5 @@
[i=4, s=<uninitialized>]
[zeek] [SPICY_SSH/3/orig] -> event ssh::banner((1, b"OpenSSH_3.8.1p1"))
[zeek] [SPICY_SSH/3/orig] -> event ssh::banner((2, b"OpenSSH_3.8.1p1"))
[zeek] [SPICY_SSH/3/orig] -> event ssh::banner((3, <error: n/a>))
[zeek] [SPICY_SSH/3/orig] -> event ssh::banner((3, b""))
[zeek] [SPICY_SSH/3/orig] -> event ssh::banner((4, Null))

View file

@ -27,7 +27,7 @@ public type Banner = unit {
dash : /-/;
software: /[^\r\n]*/;
unset_field: uint64 if ( False );
unset_field: bytes &eod if ( False );
on %done { zeek::confirm_protocol(); }
};