[Spicy] Extend functionality of export in EVT files.

We now support selecting which fields of a unit type get exported into
the automatically created Zeek record; as well as selecting which
fields get a `&log` attribute added automatically to either all fields
or to selected fields.

Syntax:

- To export only selected fields:

    export Foo::X with { field1, field3 };

- To export all but selected fields:

    export Foo::X without { field2, field3 };

- To `&log` all fields:

    export Foo::X &log;

- To `&log` only selected fields:

    export Foo::X with { field1 &log, field3 }; # exports (only) field1 and field3, and marks field1 for logging

Syntax is still subject to change.

Closes #3218.
Closes #3219.

scripts/spicy/zeek_rt.hlt

@@ -29,7 +29,7 @@ declare public void raise_event(EventHandlerPtr handler, vector<Val> args) &cxxn
 declare public BroType event_arg_type(EventHandlerPtr handler, uint<64> idx) &cxxname="zeek::spicy::rt::event_arg_type" &have_prototype;
 declare public Val to_val(any x, BroType target) &cxxname="zeek::spicy::rt::to_val" &have_prototype;
 
-type RecordField = tuple<string, BroType, bool>; # (ID, type, optional)
+type RecordField = tuple<string, BroType, bool, bool>; # (ID, type, optional, log)
 declare public BroType create_base_type(ZeekTypeTag tag) &cxxname="zeek::spicy::rt::create_base_type" &have_prototype;
 declare public BroType create_enum_type(string ns, string id, vector<tuple<string, int<64>>> labels) &cxxname="zeek::spicy::rt::create_enum_type" &have_prototype;
 declare public BroType create_record_type(string ns, string id, vector<RecordField> fields) &cxxname="zeek::spicy::rt::create_record_type" &have_prototype;

src/spicy/runtime-support.cc

@@ -126,12 +126,17 @@ TypePtr rt::create_record_type(const std::string& ns, const std::string& id,
 
     auto decls = std::make_unique<type_decl_list>();
 
-    for ( const auto& [id, type, optional] : fields ) {
+    for ( const auto& [id, type, optional, log] : fields ) {
         auto attrs = make_intrusive<detail::Attributes>(nullptr, true, false);
 
         if ( optional ) {
             auto optional_ = make_intrusive<detail::Attr>(detail::ATTR_OPTIONAL);
-            attrs->AddAttr(optional_);
+            attrs->AddAttr(std::move(optional_));
+        }
+
+        if ( log ) {
+            auto log_ = make_intrusive<detail::Attr>(detail::ATTR_LOG);
+            attrs->AddAttr(std::move(log_));
         }
 
         decls->append(new TypeDecl(util::copy_string(id.c_str()), type, std::move(attrs)));

src/spicy/runtime-support.h

@@ -151,7 +151,7 @@ extern TypePtr create_enum_type(
     const std::string& ns, const std::string& id,
     const hilti::rt::Vector<std::tuple<std::string, hilti::rt::integer::safe<int64_t>>>& labels);
 
-using RecordField = std::tuple<std::string, TypePtr, hilti::rt::Bool>; // (ID, type, optional)
+using RecordField = std::tuple<std::string, TypePtr, hilti::rt::Bool, hilti::rt::Bool>; // (ID, type, optional, &log)
 extern TypePtr create_record_type(const std::string& ns, const std::string& id,
                                   const hilti::rt::Vector<RecordField>& fields);
 

src/spicy/spicyz/driver.cc

@@ -224,8 +224,12 @@ std::vector<std::pair<TypeInfo, hilti::ID>> Driver::exportedTypes() const {
 
     for ( const auto& i : _glue->exportedIDs() ) {
         const auto& export_ = i.second;
-        if ( auto t = _types.find(export_.spicy_id); t != _types.end() )
+        if ( auto t = _types.find(export_.spicy_id); t != _types.end() ) {
+            if ( ! export_.validate(t->second) )
+                continue;
+
             result.emplace_back(t->second, export_.zeek_id);
+        }
         else {
             hilti::logger().error(hilti::rt::fmt("unknown type '%s' exported", export_.spicy_id));
             continue;

src/spicy/spicyz/glue-compiler.cc

@@ -493,6 +493,9 @@ bool GlueCompiler::loadEvtFile(hilti::rt::filesystem::path& path) {
 
             else if ( looking_at(*chunk, 0, "export") ) {
                 auto export_ = parseExport(*chunk);
+                if ( _exports.find(export_.zeek_id) != _exports.end() )
+                    throw ParseError(hilti::util::fmt("export of '%s' already defined", export_.zeek_id));
+
                 _exports[export_.zeek_id] = export_;
             }
 
@@ -514,6 +517,70 @@ bool GlueCompiler::loadEvtFile(hilti::rt::filesystem::path& path) {
     return true;
 }
 
+std::optional<glue::Export> GlueCompiler::exportForZeekID(const hilti::ID& id) const {
+    if ( auto i = _exports.find(id); i != _exports.end() )
+        return i->second;
+    else
+        return {};
+}
+
+GlueCompiler::ExportedField GlueCompiler::exportForField(const hilti::ID& zeek_id, const hilti::ID& field_id) const {
+    ExportedField field;
+
+    auto export_ = exportForZeekID(zeek_id);
+    if ( ! export_ )
+        // No `export` for this type, return defaults.
+        return field;
+
+    if ( export_->with.empty() ) {
+        // Include unless explicitly excluded.
+        if ( export_->without.find(field_id) != export_->without.end() )
+            field.skip = true;
+    }
+    else {
+        // Exclude unless explicitly included.
+        if ( export_->with.find(field_id) == export_->with.end() )
+            field.skip = true;
+    }
+
+    if ( export_->log_all )
+        field.log = true;
+
+    if ( export_->logs.find(field_id) != export_->logs.end() )
+        field.log = true;
+
+    return field;
+}
+
+
+bool glue::Export::validate(const TypeInfo& ti) const {
+    auto utype = ti.type.tryAs<::spicy::type::Unit>();
+    if ( ! utype )
+        return true;
+
+    auto check_field_names = [&](const auto& fields) {
+        for ( const auto& f : fields ) {
+            if ( ! utype->itemByName(f) ) {
+                hilti::logger().error(hilti::rt::fmt("type '%s' does not have field '%s'", ti.id, f), ti.location);
+                return false;
+            }
+        }
+
+        return true;
+    };
+
+    if ( ! check_field_names(with) )
+        return false;
+
+    if ( ! check_field_names(without) )
+        return false;
+
+    if ( ! check_field_names(logs) )
+        return false;
+
+    return true;
+}
+
 void GlueCompiler::addSpicyModule(const hilti::ID& id, const hilti::rt::filesystem::path& file) {
     glue::SpicyModule module;
     module.id = id;
@@ -790,13 +857,56 @@ glue::Export GlueCompiler::parseExport(const std::string& chunk) {
 
     export_.spicy_id = extract_id(chunk, &i);
     export_.zeek_id = export_.spicy_id;
+    export_.location = _locations.back();
 
     if ( looking_at(chunk, i, "as") ) {
         eat_token(chunk, &i, "as");
         export_.zeek_id = extract_id(chunk, &i);
     }
 
-    eat_spaces(chunk, &i);
+    if ( looking_at(chunk, i, "&log") ) {
+        eat_token(chunk, &i, "&log");
+        export_.log_all = true;
+    }
+
+    bool expect_fields = false;
+    bool include_fields;
+
+    if ( looking_at(chunk, i, "without") ) {
+        eat_token(chunk, &i, "without");
+        include_fields = false;
+        expect_fields = true;
+    }
+    else if ( looking_at(chunk, i, "with") ) {
+        eat_token(chunk, &i, "with");
+        include_fields = true;
+        expect_fields = true;
+    }
+
+    if ( expect_fields ) {
+        eat_token(chunk, &i, "{");
+
+        while ( true ) {
+            auto field = extract_id(chunk, &i);
+            if ( include_fields )
+                export_.with.insert(field);
+            else
+                export_.without.insert(field);
+
+            if ( looking_at(chunk, i, "&log") ) {
+                eat_token(chunk, &i, "&log");
+                export_.logs.insert(field);
+            }
+
+            if ( looking_at(chunk, i, "}") ) {
+                eat_token(chunk, &i, "}");
+                break; // All done.
+            }
+
+            eat_token(chunk, &i, ",");
+        }
+    }
+
     if ( ! looking_at(chunk, i, ";") )
         throw ParseError("syntax error in export");
 
@@ -1309,7 +1419,8 @@ struct VisitorZeekType : hilti::visitor::PreOrder<hilti::Result<hilti::Expressio
             if ( ! ztype )
                 return ztype.error();
 
-            fields.emplace_back(builder::tuple({builder::string(f.id()), *ztype, builder::bool_(f.isOptional())}));
+            fields.emplace_back(builder::tuple(
+                {builder::string(f.id()), *ztype, builder::bool_(f.isOptional()), builder::bool_(false)}));
         }
 
         return create_record_type(id()->namespace_(), id()->local(), fields);
@@ -1325,7 +1436,8 @@ struct VisitorZeekType : hilti::visitor::PreOrder<hilti::Result<hilti::Expressio
             if ( ! ztype )
                 return ztype.error();
 
-            fields.emplace_back(builder::tuple({builder::string(*f.id()), *ztype, builder::bool_(false)}));
+            fields.emplace_back(
+                builder::tuple({builder::string(*f.id()), *ztype, builder::bool_(false), builder::bool_(false)}));
         }
 
         hilti::ID local;
@@ -1354,12 +1466,18 @@ struct VisitorZeekType : hilti::visitor::PreOrder<hilti::Result<hilti::Expressio
 
         std::vector<hilti::Expression> fields;
         for ( const auto& f : gc->recordFields(t) ) {
+            auto field_id = std::get<0>(f);
+            auto export_ = gc->exportForField(*id(), hilti::ID(field_id));
+
+            if ( export_.skip )
+                continue;
+
             auto ztype = createZeekType(std::get<1>(f));
             if ( ! ztype )
                 return ztype.error();
 
-            fields.emplace_back(
-                builder::tuple({builder::string(std::get<0>(f)), *ztype, builder::bool_(std::get<2>(f))}));
+            fields.emplace_back(builder::tuple({builder::string(std::get<0>(f)), *ztype, builder::bool_(std::get<2>(f)),
+                                                builder::bool_(export_.log)}));
         }
 
         return create_record_type(id()->namespace_(), id()->local(), fields);

src/spicy/spicyz/glue-compiler.h

@@ -136,6 +136,18 @@ struct Export {
     hilti::ID spicy_id;
     hilti::ID zeek_id;
     hilti::Location location;
+
+    // Additional information for exported record types.
+    bool log_all = false;        /**< mark all fields for logging in exported record */
+    std::set<hilti::ID> with;    /**< fields to include in exported record */
+    std::set<hilti::ID> without; /**<  fields to exclude from exported record */
+    std::set<hilti::ID> logs;    /**< fields to mark for logging in exported record */
+
+    /**
+     * Checks that the information is semantically correct given the provided
+     * type information. Logs any errors and returns false on failure.
+     **/
+    bool validate(const TypeInfo& ti) const;
 };
 
 } // namespace glue
@@ -170,6 +182,22 @@ public:
     /** Returns all IDs that have been exported so far. */
     const auto& exportedIDs() const { return _exports; }
 
+    /** Returns the `export` declaration for a specific type given by the Zeek-side ID, if available. */
+    std::optional<glue::Export> exportForZeekID(const hilti::ID& id) const;
+
+    /** Provides `export` details for a given record field. */
+    struct ExportedField {
+        bool skip = false; /**< True if field is not to be included in the exported type. */
+        bool log = false;  /**< True if field is logged. */
+    };
+
+    /**
+     * Retrieves the  `export` details for a given record field. If our EVT
+     * file doesn't mention the field explicitly, the method returns the
+     * default behavior.
+     */
+    ExportedField exportForField(const hilti::ID& zeek_id, const hilti::ID& field_id) const;
+
     /** Generates code to convert a HILTI type to a corresponding Zeek type at runtime. */
     hilti::Result<hilti::Expression> createZeekType(const hilti::Type& t, const hilti::ID& id) const;
 

testing/btest/Baseline/spicy.export-type-ambigious-fail/output

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+[error] <...>/foo.evt:3: export of 'Test::A' already defined
+[error] error loading EVT file "<...>/foo.evt"

testing/btest/Baseline/spicy.export-type-fail/output

@@ -1,6 +1,6 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-[error] unknown type 'Test::DOES_NOT_EXIST' exported
 [error] unknown type 'NOT_SCOPED' exported
+[error] unknown type 'Test::DOES_NOT_EXIST' exported
 [error] <...>/foo.spicy:1:13-5:3: cannot export Spicy type 'Test::X': type is self-recursive
 [error] <...>/foo.spicy:9:3-13:3: cannot export Spicy type 'Test::Z': can only convert tuple types with all-named fields to Zeek
 [error] <Spicy Plugin for Zeek>: aborting after errors

testing/btest/Baseline/spicy.export-type-with-fields-fail/output

@@ -0,0 +1,4 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+[error] <...>/foo.spicy:1:13-5:3: type 'Test::A' does not have field 'does_not_exist'
+[error] <...>/foo.spicy:1:13-5:3: type 'Test::A' does not have field 'does_not_exist'
+[error] <Spicy Plugin for Zeek>: aborting after errors

testing/btest/Baseline/spicy.export-type-with-fields/output

@@ -0,0 +1,16 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+=== X 
+name=x log=F
+=== X1
+name=x log=F
+name=z log=F
+name=y log=F
+=== X2
+name=x log=T
+name=z log=T
+name=y log=T
+=== X3
+name=x log=F
+name=z log=T
+=== X4
+name=z log=F

testing/btest/spicy/export-type-ambigious-fail.spicy

@@ -0,0 +1,25 @@
+# @TEST-REQUIRES: have-spicy
+#
+# @TEST-DOC: Fail attempt to export a type multiple times
+#
+# @TEST-EXEC-FAIL: spicyz -d foo.spicy foo.evt -o foo.hlto >output 2>&1
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff output
+
+# @TEST-START-FILE foo.spicy
+module Test;
+
+type A = unit {
+    x: uint8;
+};
+
+# @TEST-END-FILE
+
+# @TEST-START-FILE foo.evt
+
+export Test::A with { x };
+export Test::A without { x };
+
+# @TEST-END-FILE
+
+# @TEST-START-FILE foo.zeek
+# @TEST-END-FILE

testing/btest/spicy/export-type-with-fields-fail.spicy

@@ -0,0 +1,25 @@
+# @TEST-REQUIRES: have-spicy
+#
+# @TEST-DOC: Failure cases for `export` with field specifcations.
+#
+# @TEST-EXEC-FAIL: spicyz -d foo.spicy foo.evt -o foo.hlto >output 2>&1
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff output
+
+# @TEST-START-FILE foo.spicy
+module Test;
+
+type A = unit {
+    x: uint8;
+};
+
+# @TEST-END-FILE
+
+# @TEST-START-FILE foo.evt
+
+export Test::A as Test::A1 with { does_not_exist };
+export Test::A as Test::A2 without { does_not_exist };
+
+# @TEST-END-FILE
+
+# @TEST-START-FILE foo.zeek
+# @TEST-END-FILE

testing/btest/spicy/export-type-with-fields.zeek

@@ -0,0 +1,48 @@
+# @TEST-REQUIRES: have-spicy
+#
+# @TEST-EXEC: spicyz -do export.hlto export.spicy export.evt
+# @TEST-EXEC: zeek export.hlto %INPUT >>output
+# @TEST-EXEC: btest-diff output
+#
+# @TEST-DOC: Test type export with specified fields.
+
+# @TEST-START-FILE export.spicy
+module foo;
+
+public type X = unit {
+    x: uint8;
+    y: uint8;
+    z: uint8;
+};
+# @TEST-END-FILE
+
+# @TEST-START-FILE export.evt
+import foo;
+
+protocol analyzer FOO over TCP:
+    parse with foo::X,
+    port 80/tcp;
+
+export foo::X with { x };
+export foo::X as foo::X1;
+export foo::X as foo::X2 &log;
+export foo::X as foo::X3 with { x, z &log };
+export foo::X as foo::X4 without { x, y };
+
+# @TEST-END-FILE
+
+function printFields(name: string, t: any) {
+    print fmt("=== %s", name);
+    local fields = record_fields(t);
+    for ( f in fields )
+	print fmt("name=%s log=%s", f, fields[f]$log);
+}
+
+event zeek_init() {
+    printFields("X ", foo::X);
+    printFields("X1", foo::X1);
+    printFields("X2", foo::X2);
+    printFields("X3", foo::X3);
+    printFields("X4", foo::X4);
+}
+