From dc735371be6555a3c6f497f8012712eb70980836 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Mon, 29 May 2023 10:56:40 +0200 Subject: [PATCH] Simplify code generated for Spicy analyzer port ranges. We previously would reprent port ranges from EVT files element-wise. This can potentially generate a lot of code (all on a single line though) which some versions of GCC seem to have trouble with, and which also causes JIT overhead. With this patch we switch to directly representing ranges. Single ports are represented as ranges `[start, start]`. Closes #3094. --- scripts/spicy/zeek_rt.hlt | 5 ++++- src/spicy/manager.cc | 23 ++++++++++++++----- src/spicy/manager.h | 9 ++++---- src/spicy/port-range.h | 32 ++++++++++++++++++++++++++ src/spicy/runtime-support.cc | 6 ++--- src/spicy/runtime-support.h | 7 +++--- src/spicy/spicyz/glue-compiler.cc | 37 +++++++++++++++---------------- src/spicy/spicyz/glue-compiler.h | 3 ++- 8 files changed, 85 insertions(+), 37 deletions(-) create mode 100644 src/spicy/port-range.h diff --git a/scripts/spicy/zeek_rt.hlt b/scripts/spicy/zeek_rt.hlt index 75386de9e7..3f5ea23d26 100644 --- a/scripts/spicy/zeek_rt.hlt +++ b/scripts/spicy/zeek_rt.hlt @@ -8,12 +8,15 @@ import hilti; public type Val = __library_type("::zeek::ValPtr"); public type BroType = __library_type("::zeek::TypePtr"); public type EventHandlerPtr = __library_type("::zeek::EventHandlerPtr"); +public type PortRange = __library_type("::zeek::spicy::rt::PortRange"); + +declare public PortRange make_port_range(port begin_, port end_) &cxxname="zeek::spicy::rt::make_port_range" &have_prototype; type ZeekTypeTag = enum { Addr, Any, Bool, Count, Double, Enum, Error, File, Func, Int, Interval, List, Opaque, Pattern, Port, Record, String, Subnet, Table, Time, Type, Vector, Void } &cxxname="::zeek::spicy::rt::ZeekTypeTag"; -declare public void register_protocol_analyzer(string name, hilti::Protocol protocol, vector ports, string parser_orig, string parser_resp, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_protocol_analyzer" &have_prototype; +declare public void register_protocol_analyzer(string name, hilti::Protocol protocol, vector ports, string parser_orig, string parser_resp, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_protocol_analyzer" &have_prototype; declare public void register_file_analyzer(string name, vector mime_types, string parser, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_file_analyzer" &have_prototype; declare public void register_packet_analyzer(string name, string parser, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_packet_analyzer" &have_prototype; declare public void register_type(string ns, string id, BroType t) &cxxname="zeek::spicy::rt::register_type" &have_prototype; diff --git a/src/spicy/manager.cc b/src/spicy/manager.cc index 76a8c949f6..4c36a8c256 100644 --- a/src/spicy/manager.cc +++ b/src/spicy/manager.cc @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -47,9 +48,9 @@ static std::pair parseID(const std::string& s) { Manager::~Manager() {} void Manager::registerProtocolAnalyzer(const std::string& name, hilti::rt::Protocol proto, - const hilti::rt::Vector& ports, const std::string& parser_orig, - const std::string& parser_resp, const std::string& replaces, - const std::string& linker_scope) { + const hilti::rt::Vector<::zeek::spicy::rt::PortRange>& ports, + const std::string& parser_orig, const std::string& parser_resp, + const std::string& replaces, const std::string& linker_scope) { SPICY_DEBUG(hilti::rt::fmt("Have Spicy protocol analyzer %s", name)); ProtocolAnalyzerInfo info; @@ -683,9 +684,19 @@ void Manager::InitPostScript() { if ( ! tag ) reporter->InternalError("cannot get analyzer tag for '%s'", p.name_analyzer.c_str()); - for ( auto port : p.ports ) { - SPICY_DEBUG(hilti::rt::fmt(" Scheduling analyzer for port %s", port)); - analyzer_mgr->RegisterAnalyzerForPort(tag, transport_protocol(port), port.port()); + for ( const auto& ports : p.ports ) { + const auto proto = ports.begin.protocol(); + + // Port ranges are closed intervals. + for ( auto port = ports.begin.port(); port <= ports.end.port(); ++port ) { + const auto port_ = hilti::rt::Port(port, proto); + SPICY_DEBUG(hilti::rt::fmt(" Scheduling analyzer for port %s", port_)); + analyzer_mgr->RegisterAnalyzerForPort(tag, transport_protocol(port_), port); + + // Explicitly prevent overflow. + if ( port == std::numeric_limits::max() ) + break; + } } if ( p.parser_resp ) { diff --git a/src/spicy/manager.h b/src/spicy/manager.h index f583ef023d..2233d1918e 100644 --- a/src/spicy/manager.h +++ b/src/spicy/manager.h @@ -17,6 +17,7 @@ #include "zeek/Tag.h" #include "zeek/plugin/Component.h" #include "zeek/plugin/Plugin.h" +#include "zeek/spicy/port-range.h" #include "zeek/spicy/spicyz/config.h" // include for Spicy version // Macro helper to report Spicy debug messages. This forwards to @@ -81,9 +82,9 @@ public: * registration */ void registerProtocolAnalyzer(const std::string& name, hilti::rt::Protocol proto, - const hilti::rt::Vector& ports, const std::string& parser_orig, - const std::string& parser_resp, const std::string& replaces, - const std::string& linker_scope); + const hilti::rt::Vector<::zeek::spicy::rt::PortRange>& ports, + const std::string& parser_orig, const std::string& parser_resp, + const std::string& replaces, const std::string& linker_scope); /** * Runtime method to register a file analyzer with its Zeek-side @@ -325,7 +326,7 @@ private: std::string name_parser_resp; std::string name_replaces; hilti::rt::Protocol protocol = hilti::rt::Protocol::Undef; - hilti::rt::Vector ports; + hilti::rt::Vector<::zeek::spicy::rt::PortRange> ports; std::string linker_scope; // Computed and available once the analyzer has been registered. diff --git a/src/spicy/port-range.h b/src/spicy/port-range.h new file mode 100644 index 0000000000..bbe0d58c12 --- /dev/null +++ b/src/spicy/port-range.h @@ -0,0 +1,32 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include +#include + +#include + +namespace zeek::spicy::rt { + +/** A closed ranged of ports. */ +struct PortRange { + PortRange() = default; + PortRange(hilti::rt::Port begin_, hilti::rt::Port end_) : begin(begin_), end(end_) { + assert(begin.port() <= end.port()); + assert(begin.protocol() == end.protocol()); + } + + hilti::rt::Port begin; /**< first port in the range */ + hilti::rt::Port end; /**< last port in the range */ +}; + +inline bool operator==(const PortRange& a, const PortRange& b) { + return std::tie(a.begin, a.end) == std::tie(b.begin, b.end); +} + +inline bool operator!=(const PortRange& a, const PortRange& b) { return ! (a == b); } + +inline PortRange make_port_range(hilti::rt::Port begin, hilti::rt::Port end) { return PortRange(begin, end); } + +} // namespace zeek::spicy::rt diff --git a/src/spicy/runtime-support.cc b/src/spicy/runtime-support.cc index 9efac938ea..b048e93d74 100644 --- a/src/spicy/runtime-support.cc +++ b/src/spicy/runtime-support.cc @@ -21,9 +21,9 @@ using namespace zeek; using namespace zeek::spicy; void rt::register_protocol_analyzer(const std::string& name, hilti::rt::Protocol proto, - const hilti::rt::Vector& ports, const std::string& parser_orig, - const std::string& parser_resp, const std::string& replaces, - const std::string& linker_scope) { + const hilti::rt::Vector<::zeek::spicy::rt::PortRange>& ports, + const std::string& parser_orig, const std::string& parser_resp, + const std::string& replaces, const std::string& linker_scope) { auto _ = hilti::rt::profiler::start("zeek/rt/register_protocol_analyzer"); spicy_mgr->registerProtocolAnalyzer(name, proto, ports, parser_orig, parser_resp, replaces, linker_scope); } diff --git a/src/spicy/runtime-support.h b/src/spicy/runtime-support.h index 11fe36bfc3..db2f9b96ce 100644 --- a/src/spicy/runtime-support.h +++ b/src/spicy/runtime-support.h @@ -23,6 +23,7 @@ #include "zeek/Desc.h" #include "zeek/spicy/cookie.h" #include "zeek/spicy/manager.h" +#include "zeek/spicy/port-range.h" namespace zeek::spicy::rt { @@ -93,9 +94,9 @@ public: * plugin's runtime. */ void register_protocol_analyzer(const std::string& name, hilti::rt::Protocol proto, - const hilti::rt::Vector& ports, const std::string& parser_orig, - const std::string& parser_resp, const std::string& replaces, - const std::string& linker_scope); + const hilti::rt::Vector<::zeek::spicy::rt::PortRange>& ports, + const std::string& parser_orig, const std::string& parser_resp, + const std::string& replaces, const std::string& linker_scope); /** * Registers a Spicy file analyzer with its EVT meta information with the diff --git a/src/spicy/spicyz/glue-compiler.cc b/src/spicy/spicyz/glue-compiler.cc index b1fbc28a62..32003549b3 100644 --- a/src/spicy/spicyz/glue-compiler.cc +++ b/src/spicy/spicyz/glue-compiler.cc @@ -16,6 +16,7 @@ #include #include "config.h" +#include "zeek/spicy/port-range.h" using namespace zeek::spicy; @@ -279,7 +280,7 @@ static hilti::rt::Port extract_port(const std::string& chunk, size_t* i) { return {static_cast(port), proto}; } -static std::vector extract_ports(const std::string& chunk, size_t* i) { +static ::zeek::spicy::rt::PortRange extract_port_range(const std::string& chunk, size_t* i) { auto start = extract_port(chunk, i); auto end = std::optional(); @@ -296,16 +297,11 @@ static std::vector extract_ports(const std::string& chunk, size throw ParseError("start of port range cannot be after its end"); } - std::vector result; + if ( ! end ) + // EVT port ranges are a closed interval, but rt are half-closed. + end = hilti::rt::Port(start.port() + 1, start.protocol()); - // Port ranges are a closed interval. - for ( auto port = start.port(); ! end || port <= end->port(); ++port ) { - result.emplace_back(port, start.protocol()); - if ( ! end ) - break; - } - - return result; + return {start, *end}; } void GlueCompiler::Init(Driver* driver, int zeek_version) { @@ -609,8 +605,7 @@ glue::ProtocolAnalyzer GlueCompiler::parseProtocolAnalyzer(const std::string& ch eat_token(chunk, &i, "{"); while ( true ) { - auto ports = extract_ports(chunk, &i); - a.ports.insert(a.ports.end(), ports.begin(), ports.end()); + a.ports.push_back(extract_port_range(chunk, &i)); if ( looking_at(chunk, i, "}") ) { eat_token(chunk, &i, "}"); @@ -623,8 +618,7 @@ glue::ProtocolAnalyzer GlueCompiler::parseProtocolAnalyzer(const std::string& ch else if ( looking_at(chunk, i, "port") ) { eat_token(chunk, &i, "port"); - auto ports = extract_ports(chunk, &i); - a.ports.insert(a.ports.end(), ports.begin(), ports.end()); + a.ports.push_back(extract_port_range(chunk, &i)); } else if ( looking_at(chunk, i, "replaces") ) { @@ -855,11 +849,16 @@ bool GlueCompiler::compile() { default: hilti::logger().internalError("unexpected protocol"); } - preinit_body.addCall("zeek_rt::register_protocol_analyzer", - {builder::string(a.name), builder::id(protocol), - builder::vector(hilti::util::transform(a.ports, [](auto p) { return builder::port(p); })), - builder::string(a.unit_name_orig), builder::string(a.unit_name_resp), - builder::string(a.replaces), _linker_scope()}); + preinit_body.addCall( + "zeek_rt::register_protocol_analyzer", + {builder::string(a.name), builder::id(protocol), + builder::vector(hilti::util::transform( + a.ports, + [](const auto& p) { + return builder::call("zeek_rt::make_port_range", {builder::port(p.begin), builder::port(p.end)}); + })), + builder::string(a.unit_name_orig), builder::string(a.unit_name_resp), builder::string(a.replaces), + _linker_scope()}); } for ( auto& a : _file_analyzers ) { diff --git a/src/spicy/spicyz/glue-compiler.h b/src/spicy/spicyz/glue-compiler.h index 2942fa445e..b6c3631f87 100644 --- a/src/spicy/spicyz/glue-compiler.h +++ b/src/spicy/spicyz/glue-compiler.h @@ -27,6 +27,7 @@ #include #include "driver.h" +#include "zeek/spicy/port-range.h" namespace spicy::rt { struct Parser; @@ -42,7 +43,7 @@ struct ProtocolAnalyzer { hilti::Location location; /**< Location where the analyzer was defined. */ hilti::ID name; /**< Name of the analyzer. */ hilti::rt::Protocol protocol = hilti::rt::Protocol::Undef; /**< The transport layer the analyzer uses. */ - std::vector ports; /**< The ports associated with the analyzer. */ + std::vector<::zeek::spicy::rt::PortRange> ports; /**< The ports associated with the analyzer. */ hilti::ID unit_name_orig; /**< The fully-qualified name of the unit type to parse the originator side. */ hilti::ID unit_name_resp; /**< The fully-qualified name of the unit type to parse the originator