From 6f882af7cce1f5f001cb1b3f9e16f441a5f19410 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 18 Sep 2023 11:16:51 +0200 Subject: [PATCH] Spicy: Support additional documentation tags inside EVT files. So far we had trouble documenting Spicy analyzers through Zeekygen because they would show up as components belonging to the `Zeek::Spicy` plugin; whereas traditional analyzers would be their own plugins and hence documented individually on their own. This commit teaches Zeekygen to track Spicy analyzers separately inside their own `Info` instances. This information isn't further used in this commit yet, but will be merged with the plugin output in a subsequent change to get the expected joint output. To pass additional information to Zeekygen, EVT files now also support two new tags for Zeekygen purposes: - `%doc-id = ID;` defines the global ID under which everything inside the EVT file will be documented by Zeekygen, conceptually comparable to plugin names (e.g., `Zeek::Syslog`). - `%doc-description = "text" provides additional text to go into the documentation (comparable to plugin descriptions). This information is carried through into the HLTO runtime initialization code, from where it's registered with Zeekygen. This commit also removes a couple of previous hacks of how Spicy integrated with Zeekygen which (1) ended up generating broken doc output for Spicy components, and (2) don't seem to be necessary anymore anyways. --- scripts/spicy/zeek_rt.hlt | 2 + src/spicy/manager.cc | 38 +++++++---- src/spicy/manager.h | 26 +++++++ src/spicy/runtime-support.cc | 7 ++ src/spicy/runtime-support.h | 13 ++++ src/spicy/spicyz/glue-compiler.cc | 64 +++++++++++++++++- src/spicy/spicyz/glue-compiler.h | 4 ++ src/zeekygen/Manager.h | 14 ++++ src/zeekygen/SpicyModuleInfo.h | 67 +++++++++++++++++++ testing/btest/Baseline/spicy.zeekygen/output1 | 3 + testing/btest/spicy/zeekygen.zeek | 35 ++++++++++ 11 files changed, 260 insertions(+), 13 deletions(-) create mode 100644 src/zeekygen/SpicyModuleInfo.h create mode 100644 testing/btest/Baseline/spicy.zeekygen/output1 create mode 100644 testing/btest/spicy/zeekygen.zeek diff --git a/scripts/spicy/zeek_rt.hlt b/scripts/spicy/zeek_rt.hlt index a238508b34..d90f9c5187 100644 --- a/scripts/spicy/zeek_rt.hlt +++ b/scripts/spicy/zeek_rt.hlt @@ -17,10 +17,12 @@ type ZeekTypeTag = enum { Addr, Any, Bool, Count, Double, Enum, Error, File, Func, Int, Interval, List, Opaque, Pattern, Port, Record, String, Subnet, Table, Time, Type, Vector, Void } &cxxname="::zeek::spicy::rt::ZeekTypeTag"; +declare public void register_spicy_module_begin(string name, string description, time mtime) &cxxname="zeek::spicy::rt::register_spicy_module_begin"; declare public void register_protocol_analyzer(string name, hilti::Protocol protocol, vector ports, string parser_orig, string parser_resp, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_protocol_analyzer" &have_prototype; declare public void register_file_analyzer(string name, vector mime_types, string parser, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_file_analyzer" &have_prototype; declare public void register_packet_analyzer(string name, string parser, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_packet_analyzer" &have_prototype; declare public void register_type(string ns, string id, BroType t) &cxxname="zeek::spicy::rt::register_type" &have_prototype; +declare public void register_spicy_module_end() &cxxname="zeek::spicy::rt::register_spicy_module_end"; declare public bool have_handler(EventHandlerPtr handler) &cxxname="zeek::spicy::rt::have_handler" &have_prototype; declare public EventHandlerPtr internal_handler(string event) &cxxname="zeek::spicy::rt::internal_handler" &have_prototype; diff --git a/src/spicy/manager.cc b/src/spicy/manager.cc index 5968cc6d4a..de27610102 100644 --- a/src/spicy/manager.cc +++ b/src/spicy/manager.cc @@ -47,6 +47,20 @@ static std::pair parseID(const std::string& s) { Manager::~Manager() {} +void Manager::registerSpicyModuleBegin(const std::string& name, const std::string& description, hilti::rt::Time mtime) { + assert(! _module_info); + _module_info = + std::make_unique(name, description, static_cast(mtime.seconds())); +} + +void Manager::registerSpicyModuleEnd() { + if ( ! _module_info ) + return; + + detail::zeekygen_mgr->AddSpicyModule(std::move(_module_info)); + // _module_info now back to null +} + void Manager::registerProtocolAnalyzer(const std::string& name, hilti::rt::Protocol proto, const hilti::rt::Vector<::zeek::spicy::rt::PortRange>& ports, const std::string& parser_orig, const std::string& parser_resp, @@ -97,10 +111,8 @@ void Manager::registerProtocolAnalyzer(const std::string& name, hilti::rt::Proto auto c = new ::zeek::analyzer::Component(info.name_zeek, factory, 0); AddComponent(c); - // Hack to prevent Zeekygen from reporting the ID as not having a - // location during the following initialization step. - ::zeek::detail::zeekygen_mgr->Script(info.name_zeekygen); - ::zeek::detail::set_location(makeLocation(info.name_zeekygen)); + if ( _module_info ) + _module_info->AddComponent(c); // TODO: Should Zeek do this? It has run component intiialization at // this point already, so ours won't get initialized anymore. @@ -145,10 +157,8 @@ void Manager::registerFileAnalyzer(const std::string& name, const hilti::rt::Vec auto c = new ::zeek::file_analysis::Component(info.name_zeek, spicy::rt::FileAnalyzer::InstantiateAnalyzer, 0); AddComponent(c); - // Hack to prevent Zeekygen from reporting the ID as not having a - // location during the following initialization step. - ::zeek::detail::zeekygen_mgr->Script(info.name_zeekygen); - ::zeek::detail::set_location(makeLocation(info.name_zeekygen)); + if ( _module_info ) + _module_info->AddComponent(c); // TODO: Should Zeek do this? It has run component intiialization at // this point already, so ours won't get initialized anymore. @@ -196,10 +206,8 @@ void Manager::registerPacketAnalyzer(const std::string& name, const std::string& auto c = new ::zeek::packet_analysis::Component(info.name_zeek, instantiate, 0); AddComponent(c); - // Hack to prevent Zeekygen from reporting the ID as not having a - // location during the following initialization step. - ::zeek::detail::zeekygen_mgr->Script(info.name_zeekygen); - ::zeek::detail::set_location(makeLocation(info.name_zeekygen)); + if ( _module_info ) + _module_info->AddComponent(c); // TODO: Should Zeek do this? It has run component initialization at // this point already, so ours won't get initialized anymore. @@ -237,6 +245,9 @@ void Manager::registerType(const std::string& id, const TypePtr& type) { zeek_id->SetType(type); zeek_id->MakeType(); AddBifItem(id, ::zeek::plugin::BifItem::TYPE); + + if ( _module_info ) + _module_info->AddBifItem(id, ::zeek::plugin::BifItem::TYPE); } TypePtr Manager::findType(const std::string& id) const { @@ -275,6 +286,9 @@ void Manager::registerEvent(const std::string& name) { // That will happen as handlers get defined. If there are no handlers, // we set a dummy type in the plugin's InitPostScript _events[name] = detail::install_ID(name.c_str(), mod.c_str(), false, true); + + if ( _module_info ) + _module_info->AddBifItem(name, ::zeek::plugin::BifItem::EVENT); } const ::spicy::rt::Parser* Manager::parserForProtocolAnalyzer(const Tag& tag, bool is_orig) { diff --git a/src/spicy/manager.h b/src/spicy/manager.h index 2233d1918e..2c5fac9008 100644 --- a/src/spicy/manager.h +++ b/src/spicy/manager.h @@ -19,6 +19,7 @@ #include "zeek/plugin/Plugin.h" #include "zeek/spicy/port-range.h" #include "zeek/spicy/spicyz/config.h" // include for Spicy version +#include "zeek/zeekygen/SpicyModuleInfo.h" // Macro helper to report Spicy debug messages. This forwards to // to both the Zeek logger and the Spicy runtime logger. @@ -64,6 +65,21 @@ public: Manager() {} virtual ~Manager(); + /** + * Runtime method to begin registration of a Spicy EVT module. All + * subsequent, other `register*()` methods will be associated with this + * module for documentation purposes until a closing + * `registerSpicyModuleEnd()` call comes. + + * @param name name of the EVT module that will be used to refer to it, + * and to index it, inside the Zeekygen documentation. + * @param description textual description in reST that will be shown for + * this module inside the Zeekygen documentation + * @param mtime timestamp indicating last time of modification of any of + * the module's content; used by Zeekygen to trigger rebuilds as necessary + */ + void registerSpicyModuleBegin(const std::string& name, const std::string& description, hilti::rt::Time mtime); + /** * Runtime method to register a protocol analyzer with its Zeek-side * configuration. This is called at startup by generated Spicy code for @@ -140,6 +156,12 @@ public: */ void registerType(const std::string& id, const TypePtr& type); + /** + * Runtime method to end registration of a Spicy EVT module. The must + * follow a preceding `registerSpicyModuleBegin()`. + */ + void registerSpicyModuleEnd(); + /** * Looks up a global type by its ID with Zeek. * @@ -397,6 +419,10 @@ private: std::string _spicy_version; + // Tracks information relevant for documenting the current EVT module + // through Zeekygen. + std::unique_ptr _module_info; + std::vector _protocol_analyzers_by_type; std::vector _file_analyzers_by_type; std::vector _packet_analyzers_by_type; diff --git a/src/spicy/runtime-support.cc b/src/spicy/runtime-support.cc index de942aaf40..d23bb11a40 100644 --- a/src/spicy/runtime-support.cc +++ b/src/spicy/runtime-support.cc @@ -20,6 +20,13 @@ using namespace zeek; using namespace zeek::spicy; +void rt::register_spicy_module_begin(const std::string& name, const std::string& description, + const hilti::rt::Time& mtime) { + spicy_mgr->registerSpicyModuleBegin(name, description, mtime); +} + +void rt::register_spicy_module_end() { spicy_mgr->registerSpicyModuleEnd(); } + void rt::register_protocol_analyzer(const std::string& name, hilti::rt::Protocol proto, const hilti::rt::Vector<::zeek::spicy::rt::PortRange>& ports, const std::string& parser_orig, const std::string& parser_resp, diff --git a/src/spicy/runtime-support.h b/src/spicy/runtime-support.h index 4534eefd44..3abcb6a7d6 100644 --- a/src/spicy/runtime-support.h +++ b/src/spicy/runtime-support.h @@ -89,6 +89,12 @@ public: using UsageError::UsageError; }; +/** + * Begins registration of a Spicy EVT module. All subsequent, other `register_*()` + * function call will be associated with this module for documentation purposes. + */ +void register_spicy_module_begin(const std::string& name, const std::string& description, const hilti::rt::Time& mtime); + /** * Registers a Spicy protocol analyzer with its EVT meta information with the * plugin's runtime. @@ -118,6 +124,13 @@ void register_packet_analyzer(const std::string& name, const std::string& parser /** Registers a Spicy-generated type to make it available inside Zeek. */ void register_type(const std::string& ns, const std::string& id, const TypePtr& type); +/** + * Ends registration of a Spicy EVT module. This must follow a preceding + * `registerSpicyModuleBegin()`. + */ +void register_spicy_module_end(); + + /** Identifies a Zeek-side type. */ enum class ZeekTypeTag : uint64_t { Addr, diff --git a/src/spicy/spicyz/glue-compiler.cc b/src/spicy/spicyz/glue-compiler.cc index eb68bf43ef..598b310587 100644 --- a/src/spicy/spicyz/glue-compiler.cc +++ b/src/spicy/spicyz/glue-compiler.cc @@ -102,6 +102,37 @@ static hilti::ID extract_id(const std::string& chunk, size_t* i) { return hilti::ID(hilti::util::replace(id, "%", "0x25_")); } +static std::string extract_string(const std::string& chunk, size_t* i) { + eat_spaces(chunk, i); + + if ( *i >= chunk.size() || chunk[*i] != '"' ) + throw ParseError("expected string"); + + size_t j = *i + 1; + + std::string str = ""; + bool in_escape = false; + while ( j < chunk.size() - 1 ) { + if ( chunk[j] == '"' && ! in_escape ) + break; + + if ( chunk[j] == '\\' && ! in_escape ) { + in_escape = true; + ++j; + continue; + } + + str += chunk[j++]; + in_escape = false; + } + + if ( j >= chunk.size() || chunk[j] != '"' ) + throw ParseError("string not terminated"); + + *i = j + 1; + return str; +} + static hilti::Type extract_type(const std::string& chunk, size_t* i) { eat_spaces(chunk, i); @@ -499,8 +530,30 @@ bool GlueCompiler::loadEvtFile(hilti::rt::filesystem::path& path) { _exports[export_.zeek_id] = export_; } + else if ( looking_at(*chunk, 0, "%doc-id") ) { + if ( ! _doc_id.empty() ) + throw ParseError("multiple %doc-id directives"); + + size_t i = 0; + eat_token(*chunk, &i, "%doc-id"); + eat_token(*chunk, &i, "="); + _doc_id = extract_id(*chunk, &i); + SPICY_DEBUG(hilti::util::fmt(" Got module's documentation name: %s", _doc_id)); + } + + else if ( looking_at(*chunk, 0, "%doc-description") ) { + size_t i = 0; + eat_token(*chunk, &i, "%doc-description"); + eat_token(*chunk, &i, "="); + _doc_description = extract_string(*chunk, &i); + SPICY_DEBUG(hilti::util::fmt(" Got module's documentation description: %s", + hilti::util::escapeUTF8(_doc_description))); + } + else - throw ParseError("expected 'import', 'export', '{file,packet,protocol} analyzer', or 'on'"); + throw ParseError( + "expected 'import', 'export', '{file,packet,protocol} analyzer', 'on', or '%doc-{id,description}' " + "directive"); _locations.pop_back(); } @@ -932,6 +985,12 @@ bool GlueCompiler::compile() { if ( ! PopulateEvents() ) return false; + if ( ! _doc_id.empty() ) { + auto mtime = hilti::expression::Ctor(hilti::ctor::Time(hilti::rt::time::current_time())); + preinit_body.addCall("zeek_rt::register_spicy_module_begin", + {hilti::builder::string(_doc_id), hilti::builder::string(_doc_description), mtime}); + } + for ( auto& a : _protocol_analyzers ) { SPICY_DEBUG(hilti::util::fmt("Adding protocol analyzer '%s'", a.name)); @@ -1053,6 +1112,9 @@ bool GlueCompiler::compile() { _driver->addInput(unit); } + if ( ! _doc_id.empty() ) + preinit_body.addCall("zeek_rt::register_spicy_module_end", {}); + if ( ! preinit_body.empty() ) { auto preinit_function = hilti::builder::function("zeek_preinit", hilti::type::void_, {}, preinit_body.block(), diff --git a/src/spicy/spicyz/glue-compiler.h b/src/spicy/spicyz/glue-compiler.h index b15be30823..6c7a800a47 100644 --- a/src/spicy/spicyz/glue-compiler.h +++ b/src/spicy/spicyz/glue-compiler.h @@ -270,6 +270,10 @@ private: std::vector _file_analyzers; /**< file analyzers parsed from EVT files */ std::vector _packet_analyzers; /**< file analyzers parsed from EVT files */ std::vector _locations; /**< location stack during parsing EVT files */ + + std::string _doc_id; /**< name/ID associated with the current EVT module inside Zeekygen documentation */ + std::string _doc_description; /**< textual description to be included for the current EVT module inside Zeekygen + documentation */ }; } // namespace zeek::spicy diff --git a/src/zeekygen/Manager.h b/src/zeekygen/Manager.h index 2676816591..adf99f2499 100644 --- a/src/zeekygen/Manager.h +++ b/src/zeekygen/Manager.h @@ -13,6 +13,7 @@ #include "zeek/Reporter.h" #include "zeek/util.h" #include "zeek/zeekygen/Configuration.h" +#include "zeek/zeekygen/SpicyModuleInfo.h" namespace zeek { @@ -148,6 +149,18 @@ public: void Redef(const zeek::detail::ID* id, const std::string& path, zeek::detail::InitClass ic = zeek::detail::INIT_NONE); + /** + * Register a Spicy EVT module. + * @param info the module information + */ + void AddSpicyModule(std::unique_ptr info) + { + spicy_modules.map[info->Name()] = info.get(); + all_info.push_back(info.release()); // switch to manual memory mgmt like all other infos + } + + const auto& SpicyModules() const { return spicy_modules.map; } + /** * Register Zeekygen script summary content. * @param path Absolute path to a Zeek script. @@ -231,6 +244,7 @@ private: InfoMap packages; InfoMap scripts; InfoMap identifiers; + InfoMap spicy_modules; std::vector all_info; IdentifierInfo* last_identifier_seen; IdentifierInfo* incomplete_type; diff --git a/src/zeekygen/SpicyModuleInfo.h b/src/zeekygen/SpicyModuleInfo.h new file mode 100644 index 0000000000..95a4769a94 --- /dev/null +++ b/src/zeekygen/SpicyModuleInfo.h @@ -0,0 +1,67 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include // for time_t +#include +#include +#include + +#include "zeek/plugin/Plugin.h" +#include "zeek/zeekygen/Info.h" + +namespace zeek::zeekygen::detail + { + +/** + * Information about a Spicy EVT module. + */ +class SpicyModuleInfo : public Info + { +public: + /** + * Ctor. + * @param name name of the Spicy EVT module. + * @param description text describing the module further + */ + explicit SpicyModuleInfo(const std::string& name, const std::string& description, time_t mtime) + : name(name), description(description), mtime(mtime) + { + } + + /** @return textual description of the module */ + const auto& Description() const { return description; } + + /** + * @return A list of all registered components. + */ + const auto& Components() const { return components; } + + /** + * @return A list of all registered BiF items. + */ + const auto& BifItems() const { return bif_items; } + + /** Register a component provided by the EVT module. */ + void AddComponent(plugin::Component* c) { components.push_back(c); } + + /** Register a BiF item provided by the EVT module. */ + void AddBifItem(const std::string& id, plugin::BifItem::Type type) + { + bif_items.emplace_back(id, type); + } + +private: + time_t DoGetModificationTime() const override { return mtime; } + std::string DoName() const override { return name; } + std::string DoReStructuredText(bool roles_only) const override { return ""; } + + std::string name; + std::string description; + time_t mtime; + + std::list components; + std::list bif_items; + }; + + } // namespace zeek::zeekygen::detail diff --git a/testing/btest/Baseline/spicy.zeekygen/output1 b/testing/btest/Baseline/spicy.zeekygen/output1 new file mode 100644 index 0000000000..86f7b807f2 --- /dev/null +++ b/testing/btest/Baseline/spicy.zeekygen/output1 @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[debug/zeek] Got module's documentation name: Foo::Bar +[debug/zeek] Got module's documentation description: Just a "test" analyzer.h diff --git a/testing/btest/spicy/zeekygen.zeek b/testing/btest/spicy/zeekygen.zeek new file mode 100644 index 0000000000..9f67ba5c2f --- /dev/null +++ b/testing/btest/spicy/zeekygen.zeek @@ -0,0 +1,35 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -D zeek -o test.hlto doc.spicy ./doc.evt >output 2>&1 +# @TEST-EXEC: cat output | grep 'module.s documentation' >output1 +# @TEST-EXEC: btest-diff output1 +# +# @TEST-DOC: Check that Spicy's tells Zeeygen about its analyzers. (TODO: Test only partially implemented right now) + +# @TEST-START-FILE doc.spicy + +module SSH; + +import zeek; + +public type Banner = unit { + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; +}; +# @TEST-END-FILE + +# @TEST-START-FILE doc.evt + +%doc-id = Foo::Bar; +%doc-description = "Just a \"test\" analyzer.h"; + +protocol analyzer spicy::SSH over TCP: + parse originator with SSH::Banner, + port 22/tcp, + replaces SSH; + +on SSH::Banner -> event ssh::banner((1, self.software)); + +# @TEST-END-FILE