Spicy: Support additional documentation tags inside EVT files.

So far we had trouble documenting Spicy analyzers through Zeekygen
because they would show up as components belonging to the
`Zeek::Spicy` plugin; whereas traditional analyzers would be their own
plugins and hence documented individually on their own. This commit
teaches Zeekygen to track Spicy analyzers separately inside their own
`Info` instances. This information isn't further used in this commit
yet, but will be merged with the plugin output in a subsequent change
to get the expected joint output.

To pass additional information to Zeekygen, EVT files now also support
two new tags for Zeekygen purposes:

- `%doc-id = ID;` defines the global ID under which everything inside
  the EVT file will be documented by Zeekygen, conceptually comparable
  to plugin names (e.g., `Zeek::Syslog`).

- `%doc-description = "text" provides additional text to go into the
  documentation (comparable to plugin descriptions).

This information is carried through into the HLTO runtime
initialization code, from where it's registered with Zeekygen.

This commit also removes a couple of previous hacks of how Spicy
integrated with Zeekygen which (1) ended up generating broken doc output
for Spicy components, and (2) don't seem to be necessary anymore
anyways.

scripts/spicy/zeek_rt.hlt

@@ -17,10 +17,12 @@ type ZeekTypeTag = enum {
     Addr, Any, Bool, Count, Double, Enum, Error, File, Func, Int, Interval, List, Opaque, Pattern, Port, Record, String, Subnet, Table, Time, Type, Vector, Void
 } &cxxname="::zeek::spicy::rt::ZeekTypeTag";
 
+declare public void register_spicy_module_begin(string name, string description, time mtime) &cxxname="zeek::spicy::rt::register_spicy_module_begin";
 declare public void register_protocol_analyzer(string name, hilti::Protocol protocol, vector<PortRange> ports, string parser_orig, string parser_resp, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_protocol_analyzer" &have_prototype;
 declare public void register_file_analyzer(string name, vector<string> mime_types, string parser, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_file_analyzer" &have_prototype;
 declare public void register_packet_analyzer(string name, string parser, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_packet_analyzer" &have_prototype;
 declare public void register_type(string ns, string id, BroType t) &cxxname="zeek::spicy::rt::register_type" &have_prototype;
+declare public void register_spicy_module_end() &cxxname="zeek::spicy::rt::register_spicy_module_end";
 
 declare public bool have_handler(EventHandlerPtr handler) &cxxname="zeek::spicy::rt::have_handler" &have_prototype;
 declare public EventHandlerPtr internal_handler(string event) &cxxname="zeek::spicy::rt::internal_handler" &have_prototype;

src/spicy/manager.cc

@@ -47,6 +47,20 @@ static std::pair<std::string, std::string> parseID(const std::string& s) {
 
 Manager::~Manager() {}
 
+void Manager::registerSpicyModuleBegin(const std::string& name, const std::string& description, hilti::rt::Time mtime) {
+    assert(! _module_info);
+    _module_info =
+        std::make_unique<zeekygen::detail::SpicyModuleInfo>(name, description, static_cast<time_t>(mtime.seconds()));
+}
+
+void Manager::registerSpicyModuleEnd() {
+    if ( ! _module_info )
+        return;
+
+    detail::zeekygen_mgr->AddSpicyModule(std::move(_module_info));
+    // _module_info now back to null
+}
+
 void Manager::registerProtocolAnalyzer(const std::string& name, hilti::rt::Protocol proto,
                                        const hilti::rt::Vector<::zeek::spicy::rt::PortRange>& ports,
                                        const std::string& parser_orig, const std::string& parser_resp,
@@ -97,10 +111,8 @@ void Manager::registerProtocolAnalyzer(const std::string& name, hilti::rt::Proto
     auto c = new ::zeek::analyzer::Component(info.name_zeek, factory, 0);
     AddComponent(c);
 
-    // Hack to prevent Zeekygen from reporting the ID as not having a
-    // location during the following initialization step.
-    ::zeek::detail::zeekygen_mgr->Script(info.name_zeekygen);
-    ::zeek::detail::set_location(makeLocation(info.name_zeekygen));
+    if ( _module_info )
+        _module_info->AddComponent(c);
 
     // TODO: Should Zeek do this? It has run component intiialization at
     // this point already, so ours won't get initialized anymore.
@@ -145,10 +157,8 @@ void Manager::registerFileAnalyzer(const std::string& name, const hilti::rt::Vec
     auto c = new ::zeek::file_analysis::Component(info.name_zeek, spicy::rt::FileAnalyzer::InstantiateAnalyzer, 0);
     AddComponent(c);
 
-    // Hack to prevent Zeekygen from reporting the ID as not having a
-    // location during the following initialization step.
-    ::zeek::detail::zeekygen_mgr->Script(info.name_zeekygen);
-    ::zeek::detail::set_location(makeLocation(info.name_zeekygen));
+    if ( _module_info )
+        _module_info->AddComponent(c);
 
     // TODO: Should Zeek do this? It has run component intiialization at
     // this point already, so ours won't get initialized anymore.
@@ -196,10 +206,8 @@ void Manager::registerPacketAnalyzer(const std::string& name, const std::string&
     auto c = new ::zeek::packet_analysis::Component(info.name_zeek, instantiate, 0);
     AddComponent(c);
 
-    // Hack to prevent Zeekygen from reporting the ID as not having a
-    // location during the following initialization step.
-    ::zeek::detail::zeekygen_mgr->Script(info.name_zeekygen);
-    ::zeek::detail::set_location(makeLocation(info.name_zeekygen));
+    if ( _module_info )
+        _module_info->AddComponent(c);
 
     // TODO: Should Zeek do this? It has run component initialization at
     // this point already, so ours won't get initialized anymore.
@@ -237,6 +245,9 @@ void Manager::registerType(const std::string& id, const TypePtr& type) {
     zeek_id->SetType(type);
     zeek_id->MakeType();
     AddBifItem(id, ::zeek::plugin::BifItem::TYPE);
+
+    if ( _module_info )
+        _module_info->AddBifItem(id, ::zeek::plugin::BifItem::TYPE);
 }
 
 TypePtr Manager::findType(const std::string& id) const {
@@ -275,6 +286,9 @@ void Manager::registerEvent(const std::string& name) {
         // That will happen as handlers get defined. If there are no handlers,
         // we set a dummy type in the plugin's InitPostScript
         _events[name] = detail::install_ID(name.c_str(), mod.c_str(), false, true);
+
+    if ( _module_info )
+        _module_info->AddBifItem(name, ::zeek::plugin::BifItem::EVENT);
 }
 
 const ::spicy::rt::Parser* Manager::parserForProtocolAnalyzer(const Tag& tag, bool is_orig) {

src/spicy/manager.h

@@ -19,6 +19,7 @@
 #include "zeek/plugin/Plugin.h"
 #include "zeek/spicy/port-range.h"
 #include "zeek/spicy/spicyz/config.h" // include for Spicy version
+#include "zeek/zeekygen/SpicyModuleInfo.h"
 
 // Macro helper to report Spicy debug messages. This forwards to
 // to both the Zeek logger and the Spicy runtime logger.
@@ -64,6 +65,21 @@ public:
     Manager() {}
     virtual ~Manager();
 
+    /**
+     * Runtime method to begin registration of a Spicy EVT module. All
+     * subsequent, other `register*()` methods will be associated with this
+     * module for documentation purposes until a closing
+     * `registerSpicyModuleEnd()` call comes.
+
+     * @param name name of the EVT module that will be used to refer to it,
+     * and to index it, inside the Zeekygen documentation.
+     * @param description textual description in reST that will be shown for
+     * this module inside the Zeekygen documentation
+     * @param mtime timestamp indicating last time of modification of any of
+     * the module's content; used by Zeekygen to trigger rebuilds as necessary
+     */
+    void registerSpicyModuleBegin(const std::string& name, const std::string& description, hilti::rt::Time mtime);
+
     /**
      * Runtime method to register a protocol analyzer with its Zeek-side
      * configuration. This is called at startup by generated Spicy code for
@@ -140,6 +156,12 @@ public:
      */
     void registerType(const std::string& id, const TypePtr& type);
 
+    /**
+     * Runtime method to end registration of a Spicy EVT module. The must
+     * follow a preceding `registerSpicyModuleBegin()`.
+     */
+    void registerSpicyModuleEnd();
+
     /**
      * Looks up a global type by its ID with Zeek.
      *
@@ -397,6 +419,10 @@ private:
 
     std::string _spicy_version;
 
+    // Tracks information relevant for documenting the current EVT module
+    // through Zeekygen.
+    std::unique_ptr<zeekygen::detail::SpicyModuleInfo> _module_info;
+
     std::vector<ProtocolAnalyzerInfo> _protocol_analyzers_by_type;
     std::vector<FileAnalyzerInfo> _file_analyzers_by_type;
     std::vector<PacketAnalyzerInfo> _packet_analyzers_by_type;

src/spicy/runtime-support.cc

@@ -20,6 +20,13 @@
 using namespace zeek;
 using namespace zeek::spicy;
 
+void rt::register_spicy_module_begin(const std::string& name, const std::string& description,
+                                     const hilti::rt::Time& mtime) {
+    spicy_mgr->registerSpicyModuleBegin(name, description, mtime);
+}
+
+void rt::register_spicy_module_end() { spicy_mgr->registerSpicyModuleEnd(); }
+
 void rt::register_protocol_analyzer(const std::string& name, hilti::rt::Protocol proto,
                                     const hilti::rt::Vector<::zeek::spicy::rt::PortRange>& ports,
                                     const std::string& parser_orig, const std::string& parser_resp,

src/spicy/runtime-support.h

@@ -89,6 +89,12 @@ public:
     using UsageError::UsageError;
 };
 
+/**
+ * Begins registration of a Spicy EVT module. All subsequent, other `register_*()`
+ * function call will be associated with this module for documentation purposes.
+ */
+void register_spicy_module_begin(const std::string& name, const std::string& description, const hilti::rt::Time& mtime);
+
 /**
  * Registers a Spicy protocol analyzer with its EVT meta information with the
  * plugin's runtime.
@@ -118,6 +124,13 @@ void register_packet_analyzer(const std::string& name, const std::string& parser
 /** Registers a Spicy-generated type to make it available inside Zeek. */
 void register_type(const std::string& ns, const std::string& id, const TypePtr& type);
 
+/**
+ * Ends registration of a Spicy EVT module. This must follow a preceding
+ * `registerSpicyModuleBegin()`.
+ */
+void register_spicy_module_end();
+
+
 /** Identifies a Zeek-side type. */
 enum class ZeekTypeTag : uint64_t {
     Addr,

src/spicy/spicyz/glue-compiler.cc

@@ -102,6 +102,37 @@ static hilti::ID extract_id(const std::string& chunk, size_t* i) {
     return hilti::ID(hilti::util::replace(id, "%", "0x25_"));
 }
 
+static std::string extract_string(const std::string& chunk, size_t* i) {
+    eat_spaces(chunk, i);
+
+    if ( *i >= chunk.size() || chunk[*i] != '"' )
+        throw ParseError("expected string");
+
+    size_t j = *i + 1;
+
+    std::string str = "";
+    bool in_escape = false;
+    while ( j < chunk.size() - 1 ) {
+        if ( chunk[j] == '"' && ! in_escape )
+            break;
+
+        if ( chunk[j] == '\\' && ! in_escape ) {
+            in_escape = true;
+            ++j;
+            continue;
+        }
+
+        str += chunk[j++];
+        in_escape = false;
+    }
+
+    if ( j >= chunk.size() || chunk[j] != '"' )
+        throw ParseError("string not terminated");
+
+    *i = j + 1;
+    return str;
+}
+
 static hilti::Type extract_type(const std::string& chunk, size_t* i) {
     eat_spaces(chunk, i);
 
@@ -499,8 +530,30 @@ bool GlueCompiler::loadEvtFile(hilti::rt::filesystem::path& path) {
                 _exports[export_.zeek_id] = export_;
             }
 
+            else if ( looking_at(*chunk, 0, "%doc-id") ) {
+                if ( ! _doc_id.empty() )
+                    throw ParseError("multiple %doc-id directives");
+
+                size_t i = 0;
+                eat_token(*chunk, &i, "%doc-id");
+                eat_token(*chunk, &i, "=");
+                _doc_id = extract_id(*chunk, &i);
+                SPICY_DEBUG(hilti::util::fmt("  Got module's documentation name: %s", _doc_id));
+            }
+
+            else if ( looking_at(*chunk, 0, "%doc-description") ) {
+                size_t i = 0;
+                eat_token(*chunk, &i, "%doc-description");
+                eat_token(*chunk, &i, "=");
+                _doc_description = extract_string(*chunk, &i);
+                SPICY_DEBUG(hilti::util::fmt("  Got module's documentation description: %s",
+                                             hilti::util::escapeUTF8(_doc_description)));
+            }
+
             else
-                throw ParseError("expected 'import', 'export', '{file,packet,protocol} analyzer', or 'on'");
+                throw ParseError(
+                    "expected 'import', 'export', '{file,packet,protocol} analyzer', 'on', or '%doc-{id,description}' "
+                    "directive");
 
             _locations.pop_back();
         }
@@ -932,6 +985,12 @@ bool GlueCompiler::compile() {
     if ( ! PopulateEvents() )
         return false;
 
+    if ( ! _doc_id.empty() ) {
+        auto mtime = hilti::expression::Ctor(hilti::ctor::Time(hilti::rt::time::current_time()));
+        preinit_body.addCall("zeek_rt::register_spicy_module_begin",
+                             {hilti::builder::string(_doc_id), hilti::builder::string(_doc_description), mtime});
+    }
+
     for ( auto& a : _protocol_analyzers ) {
         SPICY_DEBUG(hilti::util::fmt("Adding protocol analyzer '%s'", a.name));
 
@@ -1053,6 +1112,9 @@ bool GlueCompiler::compile() {
         _driver->addInput(unit);
     }
 
+    if ( ! _doc_id.empty() )
+        preinit_body.addCall("zeek_rt::register_spicy_module_end", {});
+
     if ( ! preinit_body.empty() ) {
         auto preinit_function =
             hilti::builder::function("zeek_preinit", hilti::type::void_, {}, preinit_body.block(),

src/spicy/spicyz/glue-compiler.h

@@ -270,6 +270,10 @@ private:
     std::vector<glue::FileAnalyzer> _file_analyzers;         /**< file analyzers parsed from EVT files */
     std::vector<glue::PacketAnalyzer> _packet_analyzers;     /**< file analyzers parsed from EVT files */
     std::vector<hilti::Location> _locations;                 /**< location stack during parsing EVT files */
+
+    std::string _doc_id;          /**< name/ID associated with the current EVT module inside Zeekygen documentation */
+    std::string _doc_description; /**< textual description to be included for the current EVT module inside Zeekygen
+                                     documentation */
 };
 } // namespace zeek::spicy
 

src/zeekygen/Manager.h

@@ -13,6 +13,7 @@
 #include "zeek/Reporter.h"
 #include "zeek/util.h"
 #include "zeek/zeekygen/Configuration.h"
+#include "zeek/zeekygen/SpicyModuleInfo.h"
 
 namespace zeek
 	{
@@ -148,6 +149,18 @@ public:
 	void Redef(const zeek::detail::ID* id, const std::string& path,
 	           zeek::detail::InitClass ic = zeek::detail::INIT_NONE);
 
+	/**
+	 * Register a Spicy EVT module.
+	 * @param info the module information
+	 */
+	void AddSpicyModule(std::unique_ptr<SpicyModuleInfo> info)
+		{
+		spicy_modules.map[info->Name()] = info.get();
+		all_info.push_back(info.release()); // switch to manual memory mgmt like all other infos
+		}
+
+	const auto& SpicyModules() const { return spicy_modules.map; }
+
 	/**
 	 * Register Zeekygen script summary content.
 	 * @param path Absolute path to a Zeek script.
@@ -231,6 +244,7 @@ private:
 	InfoMap<PackageInfo> packages;
 	InfoMap<ScriptInfo> scripts;
 	InfoMap<IdentifierInfo> identifiers;
+	InfoMap<SpicyModuleInfo> spicy_modules;
 	std::vector<Info*> all_info;
 	IdentifierInfo* last_identifier_seen;
 	IdentifierInfo* incomplete_type;

src/zeekygen/SpicyModuleInfo.h

@@ -0,0 +1,67 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#pragma once
+
+#include <time.h> // for time_t
+#include <list>
+#include <string>
+#include <vector>
+
+#include "zeek/plugin/Plugin.h"
+#include "zeek/zeekygen/Info.h"
+
+namespace zeek::zeekygen::detail
+	{
+
+/**
+ * Information about a Spicy EVT module.
+ */
+class SpicyModuleInfo : public Info
+	{
+public:
+	/**
+	 * Ctor.
+	 * @param name name of the Spicy EVT module.
+	 * @param description text describing the module further
+	 */
+	explicit SpicyModuleInfo(const std::string& name, const std::string& description, time_t mtime)
+		: name(name), description(description), mtime(mtime)
+		{
+		}
+
+	/** @return textual description of the module */
+	const auto& Description() const { return description; }
+
+	/**
+	 * @return A list of all registered components.
+	 */
+	const auto& Components() const { return components; }
+
+	/**
+	 * @return A list of all registered BiF items.
+	 */
+	const auto& BifItems() const { return bif_items; }
+
+	/** Register a component provided by the EVT module. */
+	void AddComponent(plugin::Component* c) { components.push_back(c); }
+
+	/** Register a BiF item provided by the EVT module. */
+	void AddBifItem(const std::string& id, plugin::BifItem::Type type)
+		{
+		bif_items.emplace_back(id, type);
+		}
+
+private:
+	time_t DoGetModificationTime() const override { return mtime; }
+	std::string DoName() const override { return name; }
+	std::string DoReStructuredText(bool roles_only) const override { return ""; }
+
+	std::string name;
+	std::string description;
+	time_t mtime;
+
+	std::list<plugin::Component*> components;
+	std::list<plugin::BifItem> bif_items;
+	};
+
+	} // namespace zeek::zeekygen::detail

testing/btest/Baseline/spicy.zeekygen/output1

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+[debug/zeek]   Got module's documentation name: Foo::Bar
+[debug/zeek]   Got module's documentation description: Just a "test" analyzer.h

testing/btest/spicy/zeekygen.zeek

@@ -0,0 +1,35 @@
+# @TEST-REQUIRES: have-spicy
+#
+# @TEST-EXEC: spicyz -D zeek -o test.hlto doc.spicy ./doc.evt >output 2>&1
+# @TEST-EXEC: cat output | grep 'module.s documentation' >output1
+# @TEST-EXEC: btest-diff output1
+#
+# @TEST-DOC: Check that Spicy's tells Zeeygen about its analyzers. (TODO: Test only partially implemented right now)
+
+# @TEST-START-FILE doc.spicy
+
+module SSH;
+
+import zeek;
+
+public type Banner = unit {
+    magic   : /SSH-/;
+    version : /[^-]*/;
+    dash    : /-/;
+    software: /[^\r\n]*/;
+};
+# @TEST-END-FILE
+
+# @TEST-START-FILE doc.evt
+
+%doc-id = Foo::Bar;
+%doc-description = "Just a \"test\" analyzer.h";
+
+protocol analyzer spicy::SSH over TCP:
+    parse originator with SSH::Banner,
+    port 22/tcp,
+    replaces SSH;
+
+on SSH::Banner -> event ssh::banner((1, self.software));
+
+# @TEST-END-FILE