From cdadd934ce4ab9128e7ec51af54c39c4d4961d87 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 11 Aug 2023 12:10:02 +0200 Subject: [PATCH] [Spicy] Extend functionality of `export` in EVT files. We now support selecting which fields of a unit type get exported into the automatically created Zeek record; as well as selecting which fields get a `&log` attribute added automatically to either all fields or to selected fields. Syntax: - To export only selected fields: export Foo::X with { field1, field3 }; - To export all but selected fields: export Foo::X without { field2, field3 }; - To `&log` all fields: export Foo::X &log; - To `&log` only selected fields: export Foo::X with { field1 &log, field3 }; # exports (only) field1 and field3, and marks field1 for logging Syntax is still subject to change. Closes #3218. Closes #3219. --- scripts/spicy/zeek_rt.hlt | 2 +- src/spicy/runtime-support.cc | 9 +- src/spicy/runtime-support.h | 2 +- src/spicy/spicyz/driver.cc | 6 +- src/spicy/spicyz/glue-compiler.cc | 128 +++++++++++++++++- src/spicy/spicyz/glue-compiler.h | 28 ++++ .../spicy.export-type-ambigious-fail/output | 3 + .../Baseline/spicy.export-type-fail/output | 2 +- .../spicy.export-type-with-fields-fail/output | 4 + .../spicy.export-type-with-fields/output | 16 +++ .../spicy/export-type-ambigious-fail.spicy | 25 ++++ .../spicy/export-type-with-fields-fail.spicy | 25 ++++ .../btest/spicy/export-type-with-fields.zeek | 48 +++++++ 13 files changed, 287 insertions(+), 11 deletions(-) create mode 100644 testing/btest/Baseline/spicy.export-type-ambigious-fail/output create mode 100644 testing/btest/Baseline/spicy.export-type-with-fields-fail/output create mode 100644 testing/btest/Baseline/spicy.export-type-with-fields/output create mode 100644 testing/btest/spicy/export-type-ambigious-fail.spicy create mode 100644 testing/btest/spicy/export-type-with-fields-fail.spicy create mode 100644 testing/btest/spicy/export-type-with-fields.zeek diff --git a/scripts/spicy/zeek_rt.hlt b/scripts/spicy/zeek_rt.hlt index 3f5ea23d26..a4e6f874ed 100644 --- a/scripts/spicy/zeek_rt.hlt +++ b/scripts/spicy/zeek_rt.hlt @@ -29,7 +29,7 @@ declare public void raise_event(EventHandlerPtr handler, vector args) &cxxn declare public BroType event_arg_type(EventHandlerPtr handler, uint<64> idx) &cxxname="zeek::spicy::rt::event_arg_type" &have_prototype; declare public Val to_val(any x, BroType target) &cxxname="zeek::spicy::rt::to_val" &have_prototype; -type RecordField = tuple; # (ID, type, optional) +type RecordField = tuple; # (ID, type, optional, log) declare public BroType create_base_type(ZeekTypeTag tag) &cxxname="zeek::spicy::rt::create_base_type" &have_prototype; declare public BroType create_enum_type(string ns, string id, vector>> labels) &cxxname="zeek::spicy::rt::create_enum_type" &have_prototype; declare public BroType create_record_type(string ns, string id, vector fields) &cxxname="zeek::spicy::rt::create_record_type" &have_prototype; diff --git a/src/spicy/runtime-support.cc b/src/spicy/runtime-support.cc index b048e93d74..4be7e14142 100644 --- a/src/spicy/runtime-support.cc +++ b/src/spicy/runtime-support.cc @@ -126,12 +126,17 @@ TypePtr rt::create_record_type(const std::string& ns, const std::string& id, auto decls = std::make_unique(); - for ( const auto& [id, type, optional] : fields ) { + for ( const auto& [id, type, optional, log] : fields ) { auto attrs = make_intrusive(nullptr, true, false); if ( optional ) { auto optional_ = make_intrusive(detail::ATTR_OPTIONAL); - attrs->AddAttr(optional_); + attrs->AddAttr(std::move(optional_)); + } + + if ( log ) { + auto log_ = make_intrusive(detail::ATTR_LOG); + attrs->AddAttr(std::move(log_)); } decls->append(new TypeDecl(util::copy_string(id.c_str()), type, std::move(attrs))); diff --git a/src/spicy/runtime-support.h b/src/spicy/runtime-support.h index db2f9b96ce..bc3c3cf067 100644 --- a/src/spicy/runtime-support.h +++ b/src/spicy/runtime-support.h @@ -151,7 +151,7 @@ extern TypePtr create_enum_type( const std::string& ns, const std::string& id, const hilti::rt::Vector>>& labels); -using RecordField = std::tuple; // (ID, type, optional) +using RecordField = std::tuple; // (ID, type, optional, &log) extern TypePtr create_record_type(const std::string& ns, const std::string& id, const hilti::rt::Vector& fields); diff --git a/src/spicy/spicyz/driver.cc b/src/spicy/spicyz/driver.cc index 6505084ed1..5eeaf535d3 100644 --- a/src/spicy/spicyz/driver.cc +++ b/src/spicy/spicyz/driver.cc @@ -224,8 +224,12 @@ std::vector> Driver::exportedTypes() const { for ( const auto& i : _glue->exportedIDs() ) { const auto& export_ = i.second; - if ( auto t = _types.find(export_.spicy_id); t != _types.end() ) + if ( auto t = _types.find(export_.spicy_id); t != _types.end() ) { + if ( ! export_.validate(t->second) ) + continue; + result.emplace_back(t->second, export_.zeek_id); + } else { hilti::logger().error(hilti::rt::fmt("unknown type '%s' exported", export_.spicy_id)); continue; diff --git a/src/spicy/spicyz/glue-compiler.cc b/src/spicy/spicyz/glue-compiler.cc index 6f0af102ca..a95af83eeb 100644 --- a/src/spicy/spicyz/glue-compiler.cc +++ b/src/spicy/spicyz/glue-compiler.cc @@ -493,6 +493,9 @@ bool GlueCompiler::loadEvtFile(hilti::rt::filesystem::path& path) { else if ( looking_at(*chunk, 0, "export") ) { auto export_ = parseExport(*chunk); + if ( _exports.find(export_.zeek_id) != _exports.end() ) + throw ParseError(hilti::util::fmt("export of '%s' already defined", export_.zeek_id)); + _exports[export_.zeek_id] = export_; } @@ -514,6 +517,70 @@ bool GlueCompiler::loadEvtFile(hilti::rt::filesystem::path& path) { return true; } +std::optional GlueCompiler::exportForZeekID(const hilti::ID& id) const { + if ( auto i = _exports.find(id); i != _exports.end() ) + return i->second; + else + return {}; +} + +GlueCompiler::ExportedField GlueCompiler::exportForField(const hilti::ID& zeek_id, const hilti::ID& field_id) const { + ExportedField field; + + auto export_ = exportForZeekID(zeek_id); + if ( ! export_ ) + // No `export` for this type, return defaults. + return field; + + if ( export_->with.empty() ) { + // Include unless explicitly excluded. + if ( export_->without.find(field_id) != export_->without.end() ) + field.skip = true; + } + else { + // Exclude unless explicitly included. + if ( export_->with.find(field_id) == export_->with.end() ) + field.skip = true; + } + + if ( export_->log_all ) + field.log = true; + + if ( export_->logs.find(field_id) != export_->logs.end() ) + field.log = true; + + return field; +} + + +bool glue::Export::validate(const TypeInfo& ti) const { + auto utype = ti.type.tryAs<::spicy::type::Unit>(); + if ( ! utype ) + return true; + + auto check_field_names = [&](const auto& fields) { + for ( const auto& f : fields ) { + if ( ! utype->itemByName(f) ) { + hilti::logger().error(hilti::rt::fmt("type '%s' does not have field '%s'", ti.id, f), ti.location); + return false; + } + } + + return true; + }; + + if ( ! check_field_names(with) ) + return false; + + if ( ! check_field_names(without) ) + return false; + + if ( ! check_field_names(logs) ) + return false; + + return true; +} + void GlueCompiler::addSpicyModule(const hilti::ID& id, const hilti::rt::filesystem::path& file) { glue::SpicyModule module; module.id = id; @@ -790,13 +857,56 @@ glue::Export GlueCompiler::parseExport(const std::string& chunk) { export_.spicy_id = extract_id(chunk, &i); export_.zeek_id = export_.spicy_id; + export_.location = _locations.back(); if ( looking_at(chunk, i, "as") ) { eat_token(chunk, &i, "as"); export_.zeek_id = extract_id(chunk, &i); } - eat_spaces(chunk, &i); + if ( looking_at(chunk, i, "&log") ) { + eat_token(chunk, &i, "&log"); + export_.log_all = true; + } + + bool expect_fields = false; + bool include_fields; + + if ( looking_at(chunk, i, "without") ) { + eat_token(chunk, &i, "without"); + include_fields = false; + expect_fields = true; + } + else if ( looking_at(chunk, i, "with") ) { + eat_token(chunk, &i, "with"); + include_fields = true; + expect_fields = true; + } + + if ( expect_fields ) { + eat_token(chunk, &i, "{"); + + while ( true ) { + auto field = extract_id(chunk, &i); + if ( include_fields ) + export_.with.insert(field); + else + export_.without.insert(field); + + if ( looking_at(chunk, i, "&log") ) { + eat_token(chunk, &i, "&log"); + export_.logs.insert(field); + } + + if ( looking_at(chunk, i, "}") ) { + eat_token(chunk, &i, "}"); + break; // All done. + } + + eat_token(chunk, &i, ","); + } + } + if ( ! looking_at(chunk, i, ";") ) throw ParseError("syntax error in export"); @@ -1309,7 +1419,8 @@ struct VisitorZeekType : hilti::visitor::PreOrdernamespace_(), id()->local(), fields); @@ -1325,7 +1436,8 @@ struct VisitorZeekType : hilti::visitor::PreOrder fields; for ( const auto& f : gc->recordFields(t) ) { + auto field_id = std::get<0>(f); + auto export_ = gc->exportForField(*id(), hilti::ID(field_id)); + + if ( export_.skip ) + continue; + auto ztype = createZeekType(std::get<1>(f)); if ( ! ztype ) return ztype.error(); - fields.emplace_back( - builder::tuple({builder::string(std::get<0>(f)), *ztype, builder::bool_(std::get<2>(f))})); + fields.emplace_back(builder::tuple({builder::string(std::get<0>(f)), *ztype, builder::bool_(std::get<2>(f)), + builder::bool_(export_.log)})); } return create_record_type(id()->namespace_(), id()->local(), fields); diff --git a/src/spicy/spicyz/glue-compiler.h b/src/spicy/spicyz/glue-compiler.h index 3231d72ed0..a7c05ec542 100644 --- a/src/spicy/spicyz/glue-compiler.h +++ b/src/spicy/spicyz/glue-compiler.h @@ -136,6 +136,18 @@ struct Export { hilti::ID spicy_id; hilti::ID zeek_id; hilti::Location location; + + // Additional information for exported record types. + bool log_all = false; /**< mark all fields for logging in exported record */ + std::set with; /**< fields to include in exported record */ + std::set without; /**< fields to exclude from exported record */ + std::set logs; /**< fields to mark for logging in exported record */ + + /** + * Checks that the information is semantically correct given the provided + * type information. Logs any errors and returns false on failure. + **/ + bool validate(const TypeInfo& ti) const; }; } // namespace glue @@ -170,6 +182,22 @@ public: /** Returns all IDs that have been exported so far. */ const auto& exportedIDs() const { return _exports; } + /** Returns the `export` declaration for a specific type given by the Zeek-side ID, if available. */ + std::optional exportForZeekID(const hilti::ID& id) const; + + /** Provides `export` details for a given record field. */ + struct ExportedField { + bool skip = false; /**< True if field is not to be included in the exported type. */ + bool log = false; /**< True if field is logged. */ + }; + + /** + * Retrieves the `export` details for a given record field. If our EVT + * file doesn't mention the field explicitly, the method returns the + * default behavior. + */ + ExportedField exportForField(const hilti::ID& zeek_id, const hilti::ID& field_id) const; + /** Generates code to convert a HILTI type to a corresponding Zeek type at runtime. */ hilti::Result createZeekType(const hilti::Type& t, const hilti::ID& id) const; diff --git a/testing/btest/Baseline/spicy.export-type-ambigious-fail/output b/testing/btest/Baseline/spicy.export-type-ambigious-fail/output new file mode 100644 index 0000000000..dd33236b0a --- /dev/null +++ b/testing/btest/Baseline/spicy.export-type-ambigious-fail/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] <...>/foo.evt:3: export of 'Test::A' already defined +[error] error loading EVT file "<...>/foo.evt" diff --git a/testing/btest/Baseline/spicy.export-type-fail/output b/testing/btest/Baseline/spicy.export-type-fail/output index 0c3723bac2..ed5e896990 100644 --- a/testing/btest/Baseline/spicy.export-type-fail/output +++ b/testing/btest/Baseline/spicy.export-type-fail/output @@ -1,6 +1,6 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -[error] unknown type 'Test::DOES_NOT_EXIST' exported [error] unknown type 'NOT_SCOPED' exported +[error] unknown type 'Test::DOES_NOT_EXIST' exported [error] <...>/foo.spicy:1:13-5:3: cannot export Spicy type 'Test::X': type is self-recursive [error] <...>/foo.spicy:9:3-13:3: cannot export Spicy type 'Test::Z': can only convert tuple types with all-named fields to Zeek [error] : aborting after errors diff --git a/testing/btest/Baseline/spicy.export-type-with-fields-fail/output b/testing/btest/Baseline/spicy.export-type-with-fields-fail/output new file mode 100644 index 0000000000..9ec5d35297 --- /dev/null +++ b/testing/btest/Baseline/spicy.export-type-with-fields-fail/output @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] <...>/foo.spicy:1:13-5:3: type 'Test::A' does not have field 'does_not_exist' +[error] <...>/foo.spicy:1:13-5:3: type 'Test::A' does not have field 'does_not_exist' +[error] : aborting after errors diff --git a/testing/btest/Baseline/spicy.export-type-with-fields/output b/testing/btest/Baseline/spicy.export-type-with-fields/output new file mode 100644 index 0000000000..40382d9da7 --- /dev/null +++ b/testing/btest/Baseline/spicy.export-type-with-fields/output @@ -0,0 +1,16 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +=== X +name=x log=F +=== X1 +name=x log=F +name=z log=F +name=y log=F +=== X2 +name=x log=T +name=z log=T +name=y log=T +=== X3 +name=x log=F +name=z log=T +=== X4 +name=z log=F diff --git a/testing/btest/spicy/export-type-ambigious-fail.spicy b/testing/btest/spicy/export-type-ambigious-fail.spicy new file mode 100644 index 0000000000..a7cd847df8 --- /dev/null +++ b/testing/btest/spicy/export-type-ambigious-fail.spicy @@ -0,0 +1,25 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-DOC: Fail attempt to export a type multiple times +# +# @TEST-EXEC-FAIL: spicyz -d foo.spicy foo.evt -o foo.hlto >output 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff output + +# @TEST-START-FILE foo.spicy +module Test; + +type A = unit { + x: uint8; +}; + +# @TEST-END-FILE + +# @TEST-START-FILE foo.evt + +export Test::A with { x }; +export Test::A without { x }; + +# @TEST-END-FILE + +# @TEST-START-FILE foo.zeek +# @TEST-END-FILE diff --git a/testing/btest/spicy/export-type-with-fields-fail.spicy b/testing/btest/spicy/export-type-with-fields-fail.spicy new file mode 100644 index 0000000000..07e0b6953f --- /dev/null +++ b/testing/btest/spicy/export-type-with-fields-fail.spicy @@ -0,0 +1,25 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-DOC: Failure cases for `export` with field specifcations. +# +# @TEST-EXEC-FAIL: spicyz -d foo.spicy foo.evt -o foo.hlto >output 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff output + +# @TEST-START-FILE foo.spicy +module Test; + +type A = unit { + x: uint8; +}; + +# @TEST-END-FILE + +# @TEST-START-FILE foo.evt + +export Test::A as Test::A1 with { does_not_exist }; +export Test::A as Test::A2 without { does_not_exist }; + +# @TEST-END-FILE + +# @TEST-START-FILE foo.zeek +# @TEST-END-FILE diff --git a/testing/btest/spicy/export-type-with-fields.zeek b/testing/btest/spicy/export-type-with-fields.zeek new file mode 100644 index 0000000000..30274926c5 --- /dev/null +++ b/testing/btest/spicy/export-type-with-fields.zeek @@ -0,0 +1,48 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -do export.hlto export.spicy export.evt +# @TEST-EXEC: zeek export.hlto %INPUT >>output +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Test type export with specified fields. + +# @TEST-START-FILE export.spicy +module foo; + +public type X = unit { + x: uint8; + y: uint8; + z: uint8; +}; +# @TEST-END-FILE + +# @TEST-START-FILE export.evt +import foo; + +protocol analyzer FOO over TCP: + parse with foo::X, + port 80/tcp; + +export foo::X with { x }; +export foo::X as foo::X1; +export foo::X as foo::X2 &log; +export foo::X as foo::X3 with { x, z &log }; +export foo::X as foo::X4 without { x, y }; + +# @TEST-END-FILE + +function printFields(name: string, t: any) { + print fmt("=== %s", name); + local fields = record_fields(t); + for ( f in fields ) + print fmt("name=%s log=%s", f, fields[f]$log); +} + +event zeek_init() { + printFields("X ", foo::X); + printFields("X1", foo::X1); + printFields("X2", foo::X2); + printFields("X3", foo::X3); + printFields("X4", foo::X4); +} +