diff --git a/CHANGES b/CHANGES index ff1ec03d6d..3226326482 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,36 @@ +6.1.0-dev.310 | 2023-08-22 14:59:28 +0200 + + * GH-3218/GH-3219: Spicy: Extend functionality of `export` in EVT + files. (Robin Sommer, Corelight) + + We now support selecting which fields of a unit type get exported into + the automatically created Zeek record; as well as selecting which + fields get a `&log` attribute added automatically to either all fields + or to selected fields. + + Syntax: + + - To export only selected fields: + + export Foo::X with { field1, field3 }; + + - To export all but selected fields: + + export Foo::X without { field2, field3 }; + + - To `&log` all fields: + + export Foo::X &log; + + - To `&log` only selected fields: + + export Foo::X with { field1 &log, field3 }; # exports (only) field1 and field3, and marks field1 for logging + + * pre-commit: Pin to latest shfmt-py version (Arne Welzel, Corelight) + + This allows users to run shfmt-py with Python > 3.9. Also drop + the explicit Python version for the setup-python action. + 6.1.0-dev.303 | 2023-08-15 17:33:51 +0100 * Raw reader: use posix_spawn instead of fork + exec (Johanna Amann, Corelight) diff --git a/VERSION b/VERSION index 81265df1d1..532e4d73b1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -6.1.0-dev.303 +6.1.0-dev.310 diff --git a/scripts/spicy/zeek_rt.hlt b/scripts/spicy/zeek_rt.hlt index 3f5ea23d26..a238508b34 100644 --- a/scripts/spicy/zeek_rt.hlt +++ b/scripts/spicy/zeek_rt.hlt @@ -9,6 +9,7 @@ public type Val = __library_type("::zeek::ValPtr"); public type BroType = __library_type("::zeek::TypePtr"); public type EventHandlerPtr = __library_type("::zeek::EventHandlerPtr"); public type PortRange = __library_type("::zeek::spicy::rt::PortRange"); +public type RecordField = __library_type("::zeek::spicy::rt::RecordField"); declare public PortRange make_port_range(port begin_, port end_) &cxxname="zeek::spicy::rt::make_port_range" &have_prototype; @@ -29,10 +30,10 @@ declare public void raise_event(EventHandlerPtr handler, vector args) &cxxn declare public BroType event_arg_type(EventHandlerPtr handler, uint<64> idx) &cxxname="zeek::spicy::rt::event_arg_type" &have_prototype; declare public Val to_val(any x, BroType target) &cxxname="zeek::spicy::rt::to_val" &have_prototype; -type RecordField = tuple; # (ID, type, optional) declare public BroType create_base_type(ZeekTypeTag tag) &cxxname="zeek::spicy::rt::create_base_type" &have_prototype; declare public BroType create_enum_type(string ns, string id, vector>> labels) &cxxname="zeek::spicy::rt::create_enum_type" &have_prototype; declare public BroType create_record_type(string ns, string id, vector fields) &cxxname="zeek::spicy::rt::create_record_type" &have_prototype; +declare public RecordField create_record_field(string id, BroType type_, bool is_optional, bool is_log) &cxxname="zeek::spicy::rt::create_record_field" &have_prototype; declare public BroType create_table_type(BroType key, optional value = Null) &cxxname="zeek::spicy::rt::create_table_type" &have_prototype; declare public BroType create_vector_type(BroType elem) &cxxname="zeek::spicy::rt::create_vector_type" &have_prototype; diff --git a/src/spicy/runtime-support.cc b/src/spicy/runtime-support.cc index b048e93d74..8d9fe3127b 100644 --- a/src/spicy/runtime-support.cc +++ b/src/spicy/runtime-support.cc @@ -126,20 +126,30 @@ TypePtr rt::create_record_type(const std::string& ns, const std::string& id, auto decls = std::make_unique(); - for ( const auto& [id, type, optional] : fields ) { + for ( const auto& f : fields ) { auto attrs = make_intrusive(nullptr, true, false); - if ( optional ) { + if ( f.is_optional ) { auto optional_ = make_intrusive(detail::ATTR_OPTIONAL); - attrs->AddAttr(optional_); + attrs->AddAttr(std::move(optional_)); } - decls->append(new TypeDecl(util::copy_string(id.c_str()), type, std::move(attrs))); + if ( f.is_log ) { + auto log_ = make_intrusive(detail::ATTR_LOG); + attrs->AddAttr(std::move(log_)); + } + + decls->append(new TypeDecl(util::copy_string(f.id.c_str()), f.type, std::move(attrs))); } return make_intrusive(decls.release()); } +rt::RecordField rt::create_record_field(const std::string& id, const TypePtr& type, hilti::rt::Bool is_optional, + hilti::rt::Bool is_log) { + return rt::RecordField{id, type, is_optional, is_log}; +} + TypePtr rt::create_table_type(TypePtr key, std::optional value) { auto _ = hilti::rt::profiler::start("zeek/rt/create_table_type"); auto idx = make_intrusive(); diff --git a/src/spicy/runtime-support.h b/src/spicy/runtime-support.h index db2f9b96ce..5bd6178d13 100644 --- a/src/spicy/runtime-support.h +++ b/src/spicy/runtime-support.h @@ -151,9 +151,17 @@ extern TypePtr create_enum_type( const std::string& ns, const std::string& id, const hilti::rt::Vector>>& labels); -using RecordField = std::tuple; // (ID, type, optional) +struct RecordField { + std::string id; /**< name of record field */ + TypePtr type; /**< Spicy-side type object */ + bool is_optional; /**< true if field is optional */ + bool is_log; /**< true if field has `&log` */ +}; + extern TypePtr create_record_type(const std::string& ns, const std::string& id, const hilti::rt::Vector& fields); +extern RecordField create_record_field(const std::string& id, const TypePtr& type, hilti::rt::Bool is_optional, + hilti::rt::Bool is_log); extern TypePtr create_table_type(TypePtr key, std::optional value); extern TypePtr create_vector_type(const TypePtr& elem); diff --git a/src/spicy/spicyz/driver.cc b/src/spicy/spicyz/driver.cc index 691ec5cfea..5eeaf535d3 100644 --- a/src/spicy/spicyz/driver.cc +++ b/src/spicy/spicyz/driver.cc @@ -222,11 +222,16 @@ std::vector Driver::types() const { std::vector> Driver::exportedTypes() const { std::vector> result; - for ( const auto& [spicy_id, zeek_id, _] : _glue->exportedIDs() ) { - if ( auto t = _types.find(spicy_id); t != _types.end() ) - result.emplace_back(t->second, zeek_id); + for ( const auto& i : _glue->exportedIDs() ) { + const auto& export_ = i.second; + if ( auto t = _types.find(export_.spicy_id); t != _types.end() ) { + if ( ! export_.validate(t->second) ) + continue; + + result.emplace_back(t->second, export_.zeek_id); + } else { - hilti::logger().error(hilti::rt::fmt("unknown type '%s' exported", spicy_id)); + hilti::logger().error(hilti::rt::fmt("unknown type '%s' exported", export_.spicy_id)); continue; } } diff --git a/src/spicy/spicyz/glue-compiler.cc b/src/spicy/spicyz/glue-compiler.cc index 375b45aea2..d49e878b9d 100644 --- a/src/spicy/spicyz/glue-compiler.cc +++ b/src/spicy/spicyz/glue-compiler.cc @@ -492,22 +492,11 @@ bool GlueCompiler::loadEvtFile(hilti::rt::filesystem::path& path) { } else if ( looking_at(*chunk, 0, "export") ) { - size_t i = 0; - eat_token(*chunk, &i, "export"); + auto export_ = parseExport(*chunk); + if ( _exports.find(export_.zeek_id) != _exports.end() ) + throw ParseError(hilti::util::fmt("export of '%s' already defined", export_.zeek_id)); - hilti::ID spicy_id = extract_id(*chunk, &i); - hilti::ID zeek_id = spicy_id; - - if ( looking_at(*chunk, i, "as") ) { - eat_token(*chunk, &i, "as"); - zeek_id = extract_id(*chunk, &i); - } - - eat_spaces(*chunk, &i); - if ( ! looking_at(*chunk, i, ";") ) - throw ParseError("syntax error in export"); - - _exports.emplace_back(std::move(spicy_id), std::move(zeek_id), _locations.back()); + _exports[export_.zeek_id] = export_; } else @@ -528,6 +517,70 @@ bool GlueCompiler::loadEvtFile(hilti::rt::filesystem::path& path) { return true; } +std::optional GlueCompiler::exportForZeekID(const hilti::ID& id) const { + if ( auto i = _exports.find(id); i != _exports.end() ) + return i->second; + else + return {}; +} + +GlueCompiler::ExportedField GlueCompiler::exportForField(const hilti::ID& zeek_id, const hilti::ID& field_id) const { + ExportedField field; + + auto export_ = exportForZeekID(zeek_id); + if ( ! export_ ) + // No `export` for this type, return defaults. + return field; + + if ( export_->with.empty() ) { + // Include unless explicitly excluded. + if ( export_->without.find(field_id) != export_->without.end() ) + field.skip = true; + } + else { + // Exclude unless explicitly included. + if ( export_->with.find(field_id) == export_->with.end() ) + field.skip = true; + } + + if ( export_->log_all ) + field.log = true; + + if ( export_->logs.find(field_id) != export_->logs.end() ) + field.log = true; + + return field; +} + + +bool glue::Export::validate(const TypeInfo& ti) const { + auto utype = ti.type.tryAs<::spicy::type::Unit>(); + if ( ! utype ) + return true; + + auto check_field_names = [&](const auto& fields) { + for ( const auto& f : fields ) { + if ( ! utype->itemByName(f) ) { + hilti::logger().error(hilti::rt::fmt("type '%s' does not have field '%s'", ti.id, f), ti.location); + return false; + } + } + + return true; + }; + + if ( ! check_field_names(with) ) + return false; + + if ( ! check_field_names(without) ) + return false; + + if ( ! check_field_names(logs) ) + return false; + + return true; +} + void GlueCompiler::addSpicyModule(const hilti::ID& id, const hilti::rt::filesystem::path& file) { glue::SpicyModule module; module.id = id; @@ -796,6 +849,70 @@ glue::Event GlueCompiler::parseEvent(const std::string& chunk) { return ev; } +glue::Export GlueCompiler::parseExport(const std::string& chunk) { + glue::Export export_; + + size_t i = 0; + eat_token(chunk, &i, "export"); + + export_.spicy_id = extract_id(chunk, &i); + export_.zeek_id = export_.spicy_id; + export_.location = _locations.back(); + + if ( looking_at(chunk, i, "as") ) { + eat_token(chunk, &i, "as"); + export_.zeek_id = extract_id(chunk, &i); + } + + if ( looking_at(chunk, i, "&log") ) { + eat_token(chunk, &i, "&log"); + export_.log_all = true; + } + + bool expect_fields = false; + bool include_fields; + + if ( looking_at(chunk, i, "without") ) { + eat_token(chunk, &i, "without"); + include_fields = false; + expect_fields = true; + } + else if ( looking_at(chunk, i, "with") ) { + eat_token(chunk, &i, "with"); + include_fields = true; + expect_fields = true; + } + + if ( expect_fields ) { + eat_token(chunk, &i, "{"); + + while ( true ) { + auto field = extract_id(chunk, &i); + if ( include_fields ) + export_.with.insert(field); + else + export_.without.insert(field); + + if ( looking_at(chunk, i, "&log") ) { + eat_token(chunk, &i, "&log"); + export_.logs.insert(field); + } + + if ( looking_at(chunk, i, "}") ) { + eat_token(chunk, &i, "}"); + break; // All done. + } + + eat_token(chunk, &i, ","); + } + } + + if ( ! looking_at(chunk, i, ";") ) + throw ParseError("syntax error in export"); + + return export_; +} + bool GlueCompiler::compile() { assert(_driver); @@ -1218,6 +1335,12 @@ struct VisitorZeekType : hilti::visitor::PreOrder& id_ = {}) { @@ -1302,7 +1425,7 @@ struct VisitorZeekType : hilti::visitor::PreOrdernamespace_(), id()->local(), fields); @@ -1318,7 +1441,7 @@ struct VisitorZeekType : hilti::visitor::PreOrder fields; for ( const auto& f : gc->recordFields(t) ) { - auto ztype = createZeekType(std::get<1>(f)); + auto export_ = gc->exportForField(*id(), hilti::ID(f.id)); + + if ( export_.skip ) + continue; + + auto ztype = createZeekType(f.type); if ( ! ztype ) return ztype.error(); - fields.emplace_back( - builder::tuple({builder::string(std::get<0>(f)), *ztype, builder::bool_(std::get<2>(f))})); + fields.emplace_back(create_record_field(f.id, *ztype, f.is_optional, export_.log)); } return create_record_type(id()->namespace_(), id()->local(), fields); @@ -1382,11 +1509,13 @@ struct VisitorUnitFields : hilti::visitor::PreOrder { if ( f.isTransient() || f.parseType().isA() ) return; - fields.emplace_back(f.id(), f.itemType(), true); + auto field = GlueCompiler::RecordField{.id = f.id(), .type = f.itemType(), .is_optional = true}; + fields.emplace_back(std::move(field)); } void operator()(const ::spicy::type::unit::item::Variable& f, const position_t p) { - fields.emplace_back(f.id(), f.itemType(), f.isOptional()); + auto field = GlueCompiler::RecordField{.id = f.id(), .type = f.itemType(), .is_optional = f.isOptional()}; + fields.emplace_back(std::move(field)); } void operator()(const ::spicy::type::unit::item::Switch& f, const position_t p) { diff --git a/src/spicy/spicyz/glue-compiler.h b/src/spicy/spicyz/glue-compiler.h index b6c3631f87..460dfb8117 100644 --- a/src/spicy/spicyz/glue-compiler.h +++ b/src/spicy/spicyz/glue-compiler.h @@ -131,6 +131,25 @@ struct Event { std::vector expression_accessors; /**< One HILTI function per expression to access the value. */ }; +/** Representation of an "export" statement parsed from an EVT file. */ +struct Export { + hilti::ID spicy_id; + hilti::ID zeek_id; + hilti::Location location; + + // Additional information for exported record types. + bool log_all = false; /**< mark all fields for logging in exported record */ + std::set with; /**< fields to include in exported record */ + std::set without; /**< fields to exclude from exported record */ + std::set logs; /**< fields to mark for logging in exported record */ + + /** + * Checks that the information is semantically correct given the provided + * type information. Logs any errors and returns false on failure. + **/ + bool validate(const TypeInfo& ti) const; +}; + } // namespace glue /** Generates the glue code between Zeek and Spicy based on *.evt files. */ @@ -163,10 +182,31 @@ public: /** Returns all IDs that have been exported so far. */ const auto& exportedIDs() const { return _exports; } + /** Returns the `export` declaration for a specific type given by the Zeek-side ID, if available. */ + std::optional exportForZeekID(const hilti::ID& id) const; + + /** Provides `export` details for a given record field. */ + struct ExportedField { + bool skip = false; /**< True if field is not to be included in the exported type. */ + bool log = false; /**< True if field is logged. */ + }; + + /** + * Retrieves the `export` details for a given record field. If our EVT + * file doesn't mention the field explicitly, the method returns the + * default behavior. + */ + ExportedField exportForField(const hilti::ID& zeek_id, const hilti::ID& field_id) const; + /** Generates code to convert a HILTI type to a corresponding Zeek type at runtime. */ hilti::Result createZeekType(const hilti::Type& t, const hilti::ID& id) const; - using RecordField = std::tuple; /**< (ID, type, optional) */ + /** Return type for `recordField()`. */ + struct RecordField { + hilti::ID id; /**< name of record field */ + hilti::Type type; /**< Spicy-side type object */ + bool is_optional; /**< true if field is optional */ + }; /** * Helper to retrieve a list of Zeek-side record fields that converting a @@ -205,6 +245,7 @@ private: glue::FileAnalyzer parseFileAnalyzer(const std::string& chunk); glue::PacketAnalyzer parsePacketAnalyzer(const std::string& chunk); glue::Event parseEvent(const std::string& chunk); + glue::Export parseExport(const std::string& chunk); /** Computes the missing pieces for all `Event` instances. */ bool PopulateEvents(); @@ -221,9 +262,9 @@ private: std::map> _spicy_modules; std::vector>> - _imports; /**< imports from EVT files, with ID and optional scope */ - std::vector> _exports; /**< exports from EVT files */ - std::vector _events; /**< events parsed from EVT files */ + _imports; /**< imports from EVT files, with ID and optional scope */ + std::map _exports; /**< exports from EVT files */ + std::vector _events; /**< events parsed from EVT files */ std::vector _protocol_analyzers; /**< protocol analyzers parsed from EVT files */ std::vector _file_analyzers; /**< file analyzers parsed from EVT files */ std::vector _packet_analyzers; /**< file analyzers parsed from EVT files */ diff --git a/testing/btest/Baseline/spicy.export-type-ambigious-fail/output b/testing/btest/Baseline/spicy.export-type-ambigious-fail/output new file mode 100644 index 0000000000..dd33236b0a --- /dev/null +++ b/testing/btest/Baseline/spicy.export-type-ambigious-fail/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] <...>/foo.evt:3: export of 'Test::A' already defined +[error] error loading EVT file "<...>/foo.evt" diff --git a/testing/btest/Baseline/spicy.export-type-fail/output b/testing/btest/Baseline/spicy.export-type-fail/output index 0c3723bac2..ed5e896990 100644 --- a/testing/btest/Baseline/spicy.export-type-fail/output +++ b/testing/btest/Baseline/spicy.export-type-fail/output @@ -1,6 +1,6 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -[error] unknown type 'Test::DOES_NOT_EXIST' exported [error] unknown type 'NOT_SCOPED' exported +[error] unknown type 'Test::DOES_NOT_EXIST' exported [error] <...>/foo.spicy:1:13-5:3: cannot export Spicy type 'Test::X': type is self-recursive [error] <...>/foo.spicy:9:3-13:3: cannot export Spicy type 'Test::Z': can only convert tuple types with all-named fields to Zeek [error] : aborting after errors diff --git a/testing/btest/Baseline/spicy.export-type-with-fields-fail/output b/testing/btest/Baseline/spicy.export-type-with-fields-fail/output new file mode 100644 index 0000000000..9ec5d35297 --- /dev/null +++ b/testing/btest/Baseline/spicy.export-type-with-fields-fail/output @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] <...>/foo.spicy:1:13-5:3: type 'Test::A' does not have field 'does_not_exist' +[error] <...>/foo.spicy:1:13-5:3: type 'Test::A' does not have field 'does_not_exist' +[error] : aborting after errors diff --git a/testing/btest/Baseline/spicy.export-type-with-fields/output b/testing/btest/Baseline/spicy.export-type-with-fields/output new file mode 100644 index 0000000000..40382d9da7 --- /dev/null +++ b/testing/btest/Baseline/spicy.export-type-with-fields/output @@ -0,0 +1,16 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +=== X +name=x log=F +=== X1 +name=x log=F +name=z log=F +name=y log=F +=== X2 +name=x log=T +name=z log=T +name=y log=T +=== X3 +name=x log=F +name=z log=T +=== X4 +name=z log=F diff --git a/testing/btest/spicy/export-type-ambigious-fail.spicy b/testing/btest/spicy/export-type-ambigious-fail.spicy new file mode 100644 index 0000000000..a7cd847df8 --- /dev/null +++ b/testing/btest/spicy/export-type-ambigious-fail.spicy @@ -0,0 +1,25 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-DOC: Fail attempt to export a type multiple times +# +# @TEST-EXEC-FAIL: spicyz -d foo.spicy foo.evt -o foo.hlto >output 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff output + +# @TEST-START-FILE foo.spicy +module Test; + +type A = unit { + x: uint8; +}; + +# @TEST-END-FILE + +# @TEST-START-FILE foo.evt + +export Test::A with { x }; +export Test::A without { x }; + +# @TEST-END-FILE + +# @TEST-START-FILE foo.zeek +# @TEST-END-FILE diff --git a/testing/btest/spicy/export-type-with-fields-fail.spicy b/testing/btest/spicy/export-type-with-fields-fail.spicy new file mode 100644 index 0000000000..07e0b6953f --- /dev/null +++ b/testing/btest/spicy/export-type-with-fields-fail.spicy @@ -0,0 +1,25 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-DOC: Failure cases for `export` with field specifcations. +# +# @TEST-EXEC-FAIL: spicyz -d foo.spicy foo.evt -o foo.hlto >output 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff output + +# @TEST-START-FILE foo.spicy +module Test; + +type A = unit { + x: uint8; +}; + +# @TEST-END-FILE + +# @TEST-START-FILE foo.evt + +export Test::A as Test::A1 with { does_not_exist }; +export Test::A as Test::A2 without { does_not_exist }; + +# @TEST-END-FILE + +# @TEST-START-FILE foo.zeek +# @TEST-END-FILE diff --git a/testing/btest/spicy/export-type-with-fields.zeek b/testing/btest/spicy/export-type-with-fields.zeek new file mode 100644 index 0000000000..30274926c5 --- /dev/null +++ b/testing/btest/spicy/export-type-with-fields.zeek @@ -0,0 +1,48 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -do export.hlto export.spicy export.evt +# @TEST-EXEC: zeek export.hlto %INPUT >>output +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Test type export with specified fields. + +# @TEST-START-FILE export.spicy +module foo; + +public type X = unit { + x: uint8; + y: uint8; + z: uint8; +}; +# @TEST-END-FILE + +# @TEST-START-FILE export.evt +import foo; + +protocol analyzer FOO over TCP: + parse with foo::X, + port 80/tcp; + +export foo::X with { x }; +export foo::X as foo::X1; +export foo::X as foo::X2 &log; +export foo::X as foo::X3 with { x, z &log }; +export foo::X as foo::X4 without { x, y }; + +# @TEST-END-FILE + +function printFields(name: string, t: any) { + print fmt("=== %s", name); + local fields = record_fields(t); + for ( f in fields ) + print fmt("name=%s log=%s", f, fields[f]$log); +} + +event zeek_init() { + printFields("X ", foo::X); + printFields("X1", foo::X1); + printFields("X2", foo::X2); + printFields("X3", foo::X3); + printFields("X4", foo::X4); +} +