diff --git a/scripts/spicy/zeek_rt.hlt b/scripts/spicy/zeek_rt.hlt index a238508b34..d90f9c5187 100644 --- a/scripts/spicy/zeek_rt.hlt +++ b/scripts/spicy/zeek_rt.hlt @@ -17,10 +17,12 @@ type ZeekTypeTag = enum { Addr, Any, Bool, Count, Double, Enum, Error, File, Func, Int, Interval, List, Opaque, Pattern, Port, Record, String, Subnet, Table, Time, Type, Vector, Void } &cxxname="::zeek::spicy::rt::ZeekTypeTag"; +declare public void register_spicy_module_begin(string name, string description, time mtime) &cxxname="zeek::spicy::rt::register_spicy_module_begin"; declare public void register_protocol_analyzer(string name, hilti::Protocol protocol, vector ports, string parser_orig, string parser_resp, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_protocol_analyzer" &have_prototype; declare public void register_file_analyzer(string name, vector mime_types, string parser, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_file_analyzer" &have_prototype; declare public void register_packet_analyzer(string name, string parser, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_packet_analyzer" &have_prototype; declare public void register_type(string ns, string id, BroType t) &cxxname="zeek::spicy::rt::register_type" &have_prototype; +declare public void register_spicy_module_end() &cxxname="zeek::spicy::rt::register_spicy_module_end"; declare public bool have_handler(EventHandlerPtr handler) &cxxname="zeek::spicy::rt::have_handler" &have_prototype; declare public EventHandlerPtr internal_handler(string event) &cxxname="zeek::spicy::rt::internal_handler" &have_prototype; diff --git a/src/spicy/manager.cc b/src/spicy/manager.cc index 5968cc6d4a..de27610102 100644 --- a/src/spicy/manager.cc +++ b/src/spicy/manager.cc @@ -47,6 +47,20 @@ static std::pair parseID(const std::string& s) { Manager::~Manager() {} +void Manager::registerSpicyModuleBegin(const std::string& name, const std::string& description, hilti::rt::Time mtime) { + assert(! _module_info); + _module_info = + std::make_unique(name, description, static_cast(mtime.seconds())); +} + +void Manager::registerSpicyModuleEnd() { + if ( ! _module_info ) + return; + + detail::zeekygen_mgr->AddSpicyModule(std::move(_module_info)); + // _module_info now back to null +} + void Manager::registerProtocolAnalyzer(const std::string& name, hilti::rt::Protocol proto, const hilti::rt::Vector<::zeek::spicy::rt::PortRange>& ports, const std::string& parser_orig, const std::string& parser_resp, @@ -97,10 +111,8 @@ void Manager::registerProtocolAnalyzer(const std::string& name, hilti::rt::Proto auto c = new ::zeek::analyzer::Component(info.name_zeek, factory, 0); AddComponent(c); - // Hack to prevent Zeekygen from reporting the ID as not having a - // location during the following initialization step. - ::zeek::detail::zeekygen_mgr->Script(info.name_zeekygen); - ::zeek::detail::set_location(makeLocation(info.name_zeekygen)); + if ( _module_info ) + _module_info->AddComponent(c); // TODO: Should Zeek do this? It has run component intiialization at // this point already, so ours won't get initialized anymore. @@ -145,10 +157,8 @@ void Manager::registerFileAnalyzer(const std::string& name, const hilti::rt::Vec auto c = new ::zeek::file_analysis::Component(info.name_zeek, spicy::rt::FileAnalyzer::InstantiateAnalyzer, 0); AddComponent(c); - // Hack to prevent Zeekygen from reporting the ID as not having a - // location during the following initialization step. - ::zeek::detail::zeekygen_mgr->Script(info.name_zeekygen); - ::zeek::detail::set_location(makeLocation(info.name_zeekygen)); + if ( _module_info ) + _module_info->AddComponent(c); // TODO: Should Zeek do this? It has run component intiialization at // this point already, so ours won't get initialized anymore. @@ -196,10 +206,8 @@ void Manager::registerPacketAnalyzer(const std::string& name, const std::string& auto c = new ::zeek::packet_analysis::Component(info.name_zeek, instantiate, 0); AddComponent(c); - // Hack to prevent Zeekygen from reporting the ID as not having a - // location during the following initialization step. - ::zeek::detail::zeekygen_mgr->Script(info.name_zeekygen); - ::zeek::detail::set_location(makeLocation(info.name_zeekygen)); + if ( _module_info ) + _module_info->AddComponent(c); // TODO: Should Zeek do this? It has run component initialization at // this point already, so ours won't get initialized anymore. @@ -237,6 +245,9 @@ void Manager::registerType(const std::string& id, const TypePtr& type) { zeek_id->SetType(type); zeek_id->MakeType(); AddBifItem(id, ::zeek::plugin::BifItem::TYPE); + + if ( _module_info ) + _module_info->AddBifItem(id, ::zeek::plugin::BifItem::TYPE); } TypePtr Manager::findType(const std::string& id) const { @@ -275,6 +286,9 @@ void Manager::registerEvent(const std::string& name) { // That will happen as handlers get defined. If there are no handlers, // we set a dummy type in the plugin's InitPostScript _events[name] = detail::install_ID(name.c_str(), mod.c_str(), false, true); + + if ( _module_info ) + _module_info->AddBifItem(name, ::zeek::plugin::BifItem::EVENT); } const ::spicy::rt::Parser* Manager::parserForProtocolAnalyzer(const Tag& tag, bool is_orig) { diff --git a/src/spicy/manager.h b/src/spicy/manager.h index 2233d1918e..2c5fac9008 100644 --- a/src/spicy/manager.h +++ b/src/spicy/manager.h @@ -19,6 +19,7 @@ #include "zeek/plugin/Plugin.h" #include "zeek/spicy/port-range.h" #include "zeek/spicy/spicyz/config.h" // include for Spicy version +#include "zeek/zeekygen/SpicyModuleInfo.h" // Macro helper to report Spicy debug messages. This forwards to // to both the Zeek logger and the Spicy runtime logger. @@ -64,6 +65,21 @@ public: Manager() {} virtual ~Manager(); + /** + * Runtime method to begin registration of a Spicy EVT module. All + * subsequent, other `register*()` methods will be associated with this + * module for documentation purposes until a closing + * `registerSpicyModuleEnd()` call comes. + + * @param name name of the EVT module that will be used to refer to it, + * and to index it, inside the Zeekygen documentation. + * @param description textual description in reST that will be shown for + * this module inside the Zeekygen documentation + * @param mtime timestamp indicating last time of modification of any of + * the module's content; used by Zeekygen to trigger rebuilds as necessary + */ + void registerSpicyModuleBegin(const std::string& name, const std::string& description, hilti::rt::Time mtime); + /** * Runtime method to register a protocol analyzer with its Zeek-side * configuration. This is called at startup by generated Spicy code for @@ -140,6 +156,12 @@ public: */ void registerType(const std::string& id, const TypePtr& type); + /** + * Runtime method to end registration of a Spicy EVT module. The must + * follow a preceding `registerSpicyModuleBegin()`. + */ + void registerSpicyModuleEnd(); + /** * Looks up a global type by its ID with Zeek. * @@ -397,6 +419,10 @@ private: std::string _spicy_version; + // Tracks information relevant for documenting the current EVT module + // through Zeekygen. + std::unique_ptr _module_info; + std::vector _protocol_analyzers_by_type; std::vector _file_analyzers_by_type; std::vector _packet_analyzers_by_type; diff --git a/src/spicy/runtime-support.cc b/src/spicy/runtime-support.cc index de942aaf40..d23bb11a40 100644 --- a/src/spicy/runtime-support.cc +++ b/src/spicy/runtime-support.cc @@ -20,6 +20,13 @@ using namespace zeek; using namespace zeek::spicy; +void rt::register_spicy_module_begin(const std::string& name, const std::string& description, + const hilti::rt::Time& mtime) { + spicy_mgr->registerSpicyModuleBegin(name, description, mtime); +} + +void rt::register_spicy_module_end() { spicy_mgr->registerSpicyModuleEnd(); } + void rt::register_protocol_analyzer(const std::string& name, hilti::rt::Protocol proto, const hilti::rt::Vector<::zeek::spicy::rt::PortRange>& ports, const std::string& parser_orig, const std::string& parser_resp, diff --git a/src/spicy/runtime-support.h b/src/spicy/runtime-support.h index 4534eefd44..3abcb6a7d6 100644 --- a/src/spicy/runtime-support.h +++ b/src/spicy/runtime-support.h @@ -89,6 +89,12 @@ public: using UsageError::UsageError; }; +/** + * Begins registration of a Spicy EVT module. All subsequent, other `register_*()` + * function call will be associated with this module for documentation purposes. + */ +void register_spicy_module_begin(const std::string& name, const std::string& description, const hilti::rt::Time& mtime); + /** * Registers a Spicy protocol analyzer with its EVT meta information with the * plugin's runtime. @@ -118,6 +124,13 @@ void register_packet_analyzer(const std::string& name, const std::string& parser /** Registers a Spicy-generated type to make it available inside Zeek. */ void register_type(const std::string& ns, const std::string& id, const TypePtr& type); +/** + * Ends registration of a Spicy EVT module. This must follow a preceding + * `registerSpicyModuleBegin()`. + */ +void register_spicy_module_end(); + + /** Identifies a Zeek-side type. */ enum class ZeekTypeTag : uint64_t { Addr, diff --git a/src/spicy/spicyz/glue-compiler.cc b/src/spicy/spicyz/glue-compiler.cc index eb68bf43ef..598b310587 100644 --- a/src/spicy/spicyz/glue-compiler.cc +++ b/src/spicy/spicyz/glue-compiler.cc @@ -102,6 +102,37 @@ static hilti::ID extract_id(const std::string& chunk, size_t* i) { return hilti::ID(hilti::util::replace(id, "%", "0x25_")); } +static std::string extract_string(const std::string& chunk, size_t* i) { + eat_spaces(chunk, i); + + if ( *i >= chunk.size() || chunk[*i] != '"' ) + throw ParseError("expected string"); + + size_t j = *i + 1; + + std::string str = ""; + bool in_escape = false; + while ( j < chunk.size() - 1 ) { + if ( chunk[j] == '"' && ! in_escape ) + break; + + if ( chunk[j] == '\\' && ! in_escape ) { + in_escape = true; + ++j; + continue; + } + + str += chunk[j++]; + in_escape = false; + } + + if ( j >= chunk.size() || chunk[j] != '"' ) + throw ParseError("string not terminated"); + + *i = j + 1; + return str; +} + static hilti::Type extract_type(const std::string& chunk, size_t* i) { eat_spaces(chunk, i); @@ -499,8 +530,30 @@ bool GlueCompiler::loadEvtFile(hilti::rt::filesystem::path& path) { _exports[export_.zeek_id] = export_; } + else if ( looking_at(*chunk, 0, "%doc-id") ) { + if ( ! _doc_id.empty() ) + throw ParseError("multiple %doc-id directives"); + + size_t i = 0; + eat_token(*chunk, &i, "%doc-id"); + eat_token(*chunk, &i, "="); + _doc_id = extract_id(*chunk, &i); + SPICY_DEBUG(hilti::util::fmt(" Got module's documentation name: %s", _doc_id)); + } + + else if ( looking_at(*chunk, 0, "%doc-description") ) { + size_t i = 0; + eat_token(*chunk, &i, "%doc-description"); + eat_token(*chunk, &i, "="); + _doc_description = extract_string(*chunk, &i); + SPICY_DEBUG(hilti::util::fmt(" Got module's documentation description: %s", + hilti::util::escapeUTF8(_doc_description))); + } + else - throw ParseError("expected 'import', 'export', '{file,packet,protocol} analyzer', or 'on'"); + throw ParseError( + "expected 'import', 'export', '{file,packet,protocol} analyzer', 'on', or '%doc-{id,description}' " + "directive"); _locations.pop_back(); } @@ -932,6 +985,12 @@ bool GlueCompiler::compile() { if ( ! PopulateEvents() ) return false; + if ( ! _doc_id.empty() ) { + auto mtime = hilti::expression::Ctor(hilti::ctor::Time(hilti::rt::time::current_time())); + preinit_body.addCall("zeek_rt::register_spicy_module_begin", + {hilti::builder::string(_doc_id), hilti::builder::string(_doc_description), mtime}); + } + for ( auto& a : _protocol_analyzers ) { SPICY_DEBUG(hilti::util::fmt("Adding protocol analyzer '%s'", a.name)); @@ -1053,6 +1112,9 @@ bool GlueCompiler::compile() { _driver->addInput(unit); } + if ( ! _doc_id.empty() ) + preinit_body.addCall("zeek_rt::register_spicy_module_end", {}); + if ( ! preinit_body.empty() ) { auto preinit_function = hilti::builder::function("zeek_preinit", hilti::type::void_, {}, preinit_body.block(), diff --git a/src/spicy/spicyz/glue-compiler.h b/src/spicy/spicyz/glue-compiler.h index b15be30823..6c7a800a47 100644 --- a/src/spicy/spicyz/glue-compiler.h +++ b/src/spicy/spicyz/glue-compiler.h @@ -270,6 +270,10 @@ private: std::vector _file_analyzers; /**< file analyzers parsed from EVT files */ std::vector _packet_analyzers; /**< file analyzers parsed from EVT files */ std::vector _locations; /**< location stack during parsing EVT files */ + + std::string _doc_id; /**< name/ID associated with the current EVT module inside Zeekygen documentation */ + std::string _doc_description; /**< textual description to be included for the current EVT module inside Zeekygen + documentation */ }; } // namespace zeek::spicy diff --git a/src/zeekygen/Manager.h b/src/zeekygen/Manager.h index 2676816591..adf99f2499 100644 --- a/src/zeekygen/Manager.h +++ b/src/zeekygen/Manager.h @@ -13,6 +13,7 @@ #include "zeek/Reporter.h" #include "zeek/util.h" #include "zeek/zeekygen/Configuration.h" +#include "zeek/zeekygen/SpicyModuleInfo.h" namespace zeek { @@ -148,6 +149,18 @@ public: void Redef(const zeek::detail::ID* id, const std::string& path, zeek::detail::InitClass ic = zeek::detail::INIT_NONE); + /** + * Register a Spicy EVT module. + * @param info the module information + */ + void AddSpicyModule(std::unique_ptr info) + { + spicy_modules.map[info->Name()] = info.get(); + all_info.push_back(info.release()); // switch to manual memory mgmt like all other infos + } + + const auto& SpicyModules() const { return spicy_modules.map; } + /** * Register Zeekygen script summary content. * @param path Absolute path to a Zeek script. @@ -231,6 +244,7 @@ private: InfoMap packages; InfoMap scripts; InfoMap identifiers; + InfoMap spicy_modules; std::vector all_info; IdentifierInfo* last_identifier_seen; IdentifierInfo* incomplete_type; diff --git a/src/zeekygen/SpicyModuleInfo.h b/src/zeekygen/SpicyModuleInfo.h new file mode 100644 index 0000000000..95a4769a94 --- /dev/null +++ b/src/zeekygen/SpicyModuleInfo.h @@ -0,0 +1,67 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include // for time_t +#include +#include +#include + +#include "zeek/plugin/Plugin.h" +#include "zeek/zeekygen/Info.h" + +namespace zeek::zeekygen::detail + { + +/** + * Information about a Spicy EVT module. + */ +class SpicyModuleInfo : public Info + { +public: + /** + * Ctor. + * @param name name of the Spicy EVT module. + * @param description text describing the module further + */ + explicit SpicyModuleInfo(const std::string& name, const std::string& description, time_t mtime) + : name(name), description(description), mtime(mtime) + { + } + + /** @return textual description of the module */ + const auto& Description() const { return description; } + + /** + * @return A list of all registered components. + */ + const auto& Components() const { return components; } + + /** + * @return A list of all registered BiF items. + */ + const auto& BifItems() const { return bif_items; } + + /** Register a component provided by the EVT module. */ + void AddComponent(plugin::Component* c) { components.push_back(c); } + + /** Register a BiF item provided by the EVT module. */ + void AddBifItem(const std::string& id, plugin::BifItem::Type type) + { + bif_items.emplace_back(id, type); + } + +private: + time_t DoGetModificationTime() const override { return mtime; } + std::string DoName() const override { return name; } + std::string DoReStructuredText(bool roles_only) const override { return ""; } + + std::string name; + std::string description; + time_t mtime; + + std::list components; + std::list bif_items; + }; + + } // namespace zeek::zeekygen::detail diff --git a/testing/btest/Baseline/spicy.zeekygen/output1 b/testing/btest/Baseline/spicy.zeekygen/output1 new file mode 100644 index 0000000000..86f7b807f2 --- /dev/null +++ b/testing/btest/Baseline/spicy.zeekygen/output1 @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[debug/zeek] Got module's documentation name: Foo::Bar +[debug/zeek] Got module's documentation description: Just a "test" analyzer.h diff --git a/testing/btest/spicy/zeekygen.zeek b/testing/btest/spicy/zeekygen.zeek new file mode 100644 index 0000000000..9f67ba5c2f --- /dev/null +++ b/testing/btest/spicy/zeekygen.zeek @@ -0,0 +1,35 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -D zeek -o test.hlto doc.spicy ./doc.evt >output 2>&1 +# @TEST-EXEC: cat output | grep 'module.s documentation' >output1 +# @TEST-EXEC: btest-diff output1 +# +# @TEST-DOC: Check that Spicy's tells Zeeygen about its analyzers. (TODO: Test only partially implemented right now) + +# @TEST-START-FILE doc.spicy + +module SSH; + +import zeek; + +public type Banner = unit { + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; +}; +# @TEST-END-FILE + +# @TEST-START-FILE doc.evt + +%doc-id = Foo::Bar; +%doc-description = "Just a \"test\" analyzer.h"; + +protocol analyzer spicy::SSH over TCP: + parse originator with SSH::Banner, + port 22/tcp, + replaces SSH; + +on SSH::Banner -> event ssh::banner((1, self.software)); + +# @TEST-END-FILE