diff --git a/doc b/doc index 425ce7933d..1ca467fe0c 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 425ce7933df90afdd34c7d5695b17d44a13ae8a7 +Subproject commit 1ca467fe0cb524ff375d957475d0319ab546915b diff --git a/scripts/base/frameworks/spicy/init-framework.zeek b/scripts/base/frameworks/spicy/init-framework.zeek index ae3a3b8e65..de6b528ee4 100644 --- a/scripts/base/frameworks/spicy/init-framework.zeek +++ b/scripts/base/frameworks/spicy/init-framework.zeek @@ -47,12 +47,18 @@ export { # Marked with &is_used to suppress complaints when there aren't any # Spicy file analyzers loaded, and hence this event can't be generated. -# The attribute is only supported for Zeek 5.0 and higher. event spicy_analyzer_for_mime_type(a: Files::Tag, mt: string) &is_used { Files::register_for_mime_type(a, mt); } +# Marked with &is_used to suppress complaints when there aren't any +# Spicy protocol analyzers loaded, and hence this event can't be generated. +event spicy_analyzer_for_port(a: Analyzer::Tag, p: port) &is_used + { + Analyzer::register_for_port(a, p); + } + function enable_protocol_analyzer(tag: Analyzer::Tag) : bool { return Spicy::__toggle_analyzer(tag, T); diff --git a/scripts/spicy/zeek_rt.hlt b/scripts/spicy/zeek_rt.hlt index 801c49a9b1..3f4dd28adc 100644 --- a/scripts/spicy/zeek_rt.hlt +++ b/scripts/spicy/zeek_rt.hlt @@ -18,7 +18,7 @@ type ZeekTypeTag = enum { } &cxxname="::zeek::spicy::rt::ZeekTypeTag"; declare public void register_spicy_module_begin(string name, string description) &cxxname="zeek::spicy::rt::register_spicy_module_begin"; -declare public void register_protocol_analyzer(string name, hilti::Protocol protocol, string parser_orig, string parser_resp, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_protocol_analyzer" &have_prototype; +declare public void register_protocol_analyzer(string name, hilti::Protocol protocol, vector ports, string parser_orig, string parser_resp, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_protocol_analyzer" &have_prototype; declare public void register_file_analyzer(string name, vector mime_types, string parser, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_file_analyzer" &have_prototype; declare public void register_packet_analyzer(string name, string parser, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_packet_analyzer" &have_prototype; declare public void register_type(string ns, string id, BroType t) &cxxname="zeek::spicy::rt::register_type" &have_prototype; diff --git a/src/spicy/manager.cc b/src/spicy/manager.cc index 7919380111..423febf1c9 100644 --- a/src/spicy/manager.cc +++ b/src/spicy/manager.cc @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -32,6 +33,7 @@ #include "zeek/spicy/file-analyzer.h" #include "zeek/spicy/packet-analyzer.h" #include "zeek/spicy/protocol-analyzer.h" +#include "zeek/spicy/runtime-support.h" #include "zeek/zeek-config-paths.h" using namespace zeek; @@ -61,6 +63,7 @@ void Manager::registerSpicyModuleEnd() { } void Manager::registerProtocolAnalyzer(const std::string& name, hilti::rt::Protocol proto, + const hilti::rt::Vector<::zeek::spicy::rt::PortRange>& ports, const std::string& parser_orig, const std::string& parser_resp, const std::string& replaces, const std::string& linker_scope) { SPICY_DEBUG(hilti::rt::fmt("Have Spicy protocol analyzer %s", name)); @@ -75,6 +78,11 @@ void Manager::registerProtocolAnalyzer(const std::string& name, hilti::rt::Proto info.protocol = proto; info.linker_scope = linker_scope; + // Store ports in a deterministic order. We can't (easily) sort the + // `hilti::rt::Vector` unfortunately. + std::copy(ports.begin(), ports.end(), std::back_inserter(info.ports)); + std::sort(info.ports.begin(), info.ports.end()); + // We may have that analyzer already iff it was previously pre-registered // without a linker scope. We'll then only set the scope now. if ( auto t = _analyzer_name_to_tag_type.find(info.name_zeek); t != _analyzer_name_to_tag_type.end() ) { @@ -699,6 +707,36 @@ void Manager::InitPostScript() { if ( ! tag ) reporter->InternalError("cannot get analyzer tag for '%s'", p.name_analyzer.c_str()); + auto register_analyzer_for_port = [&](auto tag, const hilti::rt::Port& port_) { + SPICY_DEBUG(hilti::rt::fmt(" Scheduling analyzer for port %s", port_)); + + // Well-known ports are registered in scriptland, so we'll raise an + // event that will do it for us through a predefined handler. + zeek::Args vals = Args(); + vals.emplace_back(tag.AsVal()); + vals.emplace_back(zeek::spicy::rt::to_val(port_, base_type(TYPE_PORT))); + EventHandlerPtr handler = event_registry->Register("spicy_analyzer_for_port"); + event_mgr.Enqueue(handler, vals); + }; + + for ( const auto& ports : p.ports ) { + const auto proto = ports.begin.protocol(); + + // Port ranges are closed intervals. + for ( auto port = ports.begin.port(); port <= ports.end.port(); ++port ) { + const auto port_ = hilti::rt::Port(port, proto); + register_analyzer_for_port(tag, port_); + + // Don't double register in case of single-port ranges. + if ( ports.begin.port() == ports.end.port() ) + break; + + // Explicitly prevent overflow. + if ( port == std::numeric_limits::max() ) + break; + } + } + if ( p.parser_resp ) { for ( auto port : p.parser_resp->ports ) { if ( port.direction != ::spicy::rt::Direction::Both && @@ -706,7 +744,7 @@ void Manager::InitPostScript() { continue; SPICY_DEBUG(hilti::rt::fmt(" Scheduling analyzer for port %s", port.port)); - analyzer_mgr->RegisterAnalyzerForPort(tag, transport_protocol(port.port), port.port.port()); + register_analyzer_for_port(tag, port.port); } } } diff --git a/src/spicy/manager.h b/src/spicy/manager.h index 195ae3adf1..55f47c51fd 100644 --- a/src/spicy/manager.h +++ b/src/spicy/manager.h @@ -85,6 +85,7 @@ public: * * @param name name of the analyzer as defined in its EVT file * @param proto analyzer's transport-layer protocol + * @param ports well-known ports for the analyzer; it'll be activated automatically for these * @param parser_orig name of the Spicy parser for the originator side; must match the name that * Spicy registers the unit's parser with * @param parser_resp name of the Spicy parser for the originator side; must match the name that @@ -94,9 +95,10 @@ public: * @param linker_scope scope of current HLTO file, which will restrict visibility of the * registration */ - void registerProtocolAnalyzer(const std::string& name, hilti::rt::Protocol proto, const std::string& parser_orig, - const std::string& parser_resp, const std::string& replaces, - const std::string& linker_scope); + void registerProtocolAnalyzer(const std::string& name, hilti::rt::Protocol proto, + const hilti::rt::Vector<::zeek::spicy::rt::PortRange>& ports, + const std::string& parser_orig, const std::string& parser_resp, + const std::string& replaces, const std::string& linker_scope); /** * Runtime method to register a file analyzer with its Zeek-side @@ -341,6 +343,7 @@ private: std::string name_parser_resp; std::string name_replaces; hilti::rt::Protocol protocol = hilti::rt::Protocol::Undef; + std::vector<::zeek::spicy::rt::PortRange> ports; // we keep this sorted std::string linker_scope; // Computed and available once the analyzer has been registered. @@ -354,7 +357,7 @@ private: bool operator==(const ProtocolAnalyzerInfo& other) const { return name_analyzer == other.name_analyzer && name_parser_orig == other.name_parser_orig && name_parser_resp == other.name_parser_resp && name_replaces == other.name_replaces && - protocol == other.protocol && linker_scope == other.linker_scope; + protocol == other.protocol && ports == other.ports && linker_scope == other.linker_scope; } bool operator!=(const ProtocolAnalyzerInfo& other) const { return ! (*this == other); } diff --git a/src/spicy/port-range.h b/src/spicy/port-range.h index bbe0d58c12..7e71d433f8 100644 --- a/src/spicy/port-range.h +++ b/src/spicy/port-range.h @@ -19,6 +19,11 @@ struct PortRange { hilti::rt::Port begin; /**< first port in the range */ hilti::rt::Port end; /**< last port in the range */ + + bool operator<(const PortRange& other) const { + // Just get us a deterministic order. + return std::tie(begin, end) < std::tie(other.begin, other.end); + } }; inline bool operator==(const PortRange& a, const PortRange& b) { diff --git a/src/spicy/runtime-support.cc b/src/spicy/runtime-support.cc index 8dbf0c39a7..f5afd37461 100644 --- a/src/spicy/runtime-support.cc +++ b/src/spicy/runtime-support.cc @@ -26,11 +26,12 @@ void rt::register_spicy_module_begin(const std::string& name, const std::string& void rt::register_spicy_module_end() { spicy_mgr->registerSpicyModuleEnd(); } -void rt::register_protocol_analyzer(const std::string& name, hilti::rt::Protocol proto, const std::string& parser_orig, - const std::string& parser_resp, const std::string& replaces, - const std::string& linker_scope) { +void rt::register_protocol_analyzer(const std::string& name, hilti::rt::Protocol proto, + const hilti::rt::Vector<::zeek::spicy::rt::PortRange>& ports, + const std::string& parser_orig, const std::string& parser_resp, + const std::string& replaces, const std::string& linker_scope) { auto _ = hilti::rt::profiler::start("zeek/rt/register_protocol_analyzer"); - spicy_mgr->registerProtocolAnalyzer(name, proto, parser_orig, parser_resp, replaces, linker_scope); + spicy_mgr->registerProtocolAnalyzer(name, proto, ports, parser_orig, parser_resp, replaces, linker_scope); } void rt::register_file_analyzer(const std::string& name, const hilti::rt::Vector& mime_types, diff --git a/src/spicy/runtime-support.h b/src/spicy/runtime-support.h index c0bf9f4631..0397dc86cc 100644 --- a/src/spicy/runtime-support.h +++ b/src/spicy/runtime-support.h @@ -106,9 +106,10 @@ void register_spicy_module_begin(const std::string& id, const std::string& descr * Registers a Spicy protocol analyzer with its EVT meta information with the * plugin's runtime. */ -void register_protocol_analyzer(const std::string& id, hilti::rt::Protocol proto, const std::string& parser_orig, - const std::string& parser_resp, const std::string& replaces, - const std::string& linker_scope); +void register_protocol_analyzer(const std::string& id, hilti::rt::Protocol proto, + const hilti::rt::Vector<::zeek::spicy::rt::PortRange>& ports, + const std::string& parser_orig, const std::string& parser_resp, + const std::string& replaces, const std::string& linker_scope); /** * Registers a Spicy file analyzer with its EVT meta information with the diff --git a/src/spicy/spicyz/glue-compiler.cc b/src/spicy/spicyz/glue-compiler.cc index c34d7e1f6b..2b75a90949 100644 --- a/src/spicy/spicyz/glue-compiler.cc +++ b/src/spicy/spicyz/glue-compiler.cc @@ -260,6 +260,79 @@ static std::string extract_expr(const std::string& chunk, size_t* i) { return expr; } +static hilti::rt::Port extract_port(const std::string& chunk, size_t* i) { + eat_spaces(chunk, i); + + std::string s; + size_t j = *i; + + while ( j < chunk.size() && isdigit(chunk[j]) ) + ++j; + + if ( *i == j ) + throw ParseError("cannot parse port specification"); + + hilti::rt::Protocol proto; + uint64_t port = std::numeric_limits::max(); + + s = chunk.substr(*i, j - *i); + hilti::util::atoi_n(s.begin(), s.end(), 10, &port); + + if ( port > 65535 ) + throw ParseError("port outside of valid range"); + + *i = j; + + if ( chunk[*i] != '/' ) + throw ParseError("cannot parse port specification"); + + (*i)++; + + if ( looking_at(chunk, *i, "tcp") ) { + proto = hilti::rt::Protocol::TCP; + eat_token(chunk, i, "tcp"); + } + + else if ( looking_at(chunk, *i, "udp") ) { + proto = hilti::rt::Protocol::UDP; + eat_token(chunk, i, "udp"); + } + + else if ( looking_at(chunk, *i, "icmp") ) { + proto = hilti::rt::Protocol::ICMP; + eat_token(chunk, i, "icmp"); + } + + else + throw ParseError("cannot parse port specification"); + + return {static_cast(port), proto}; +} + +static ::zeek::spicy::rt::PortRange extract_port_range(const std::string& chunk, size_t* i) { + auto start = extract_port(chunk, i); + auto end = std::optional(); + + if ( looking_at(chunk, *i, "-") ) { + eat_token(chunk, i, "-"); + end = extract_port(chunk, i); + } + + if ( end ) { + if ( start.protocol() != end->protocol() ) + throw ParseError("start and end of port range must have same protocol"); + + if ( start.port() > end->port() ) + throw ParseError("start of port range cannot be after its end"); + } + + if ( ! end ) + // EVT port ranges are a closed. + end = hilti::rt::Port(start.port(), start.protocol()); + + return {start, *end}; +} + void GlueCompiler::init(Driver* driver, int zeek_version) { _driver = driver; _zeek_version = zeek_version; @@ -631,11 +704,25 @@ glue::ProtocolAnalyzer GlueCompiler::parseProtocolAnalyzer(const std::string& ch } } - else if ( looking_at(chunk, i, "ports") || looking_at(chunk, i, "port") ) { - throw ParseError(hilti::rt::fmt( - "Analyzer %s is using the removed 'port' or 'ports' keyword to register " - "well-known ports. Use Analyzer::register_for_ports() in the accompanying Zeek script instead.", - a.name)); + else if ( looking_at(chunk, i, "ports") ) { + eat_token(chunk, &i, "ports"); + eat_token(chunk, &i, "{"); + + while ( true ) { + a.ports.push_back(extract_port_range(chunk, &i)); + + if ( looking_at(chunk, i, "}") ) { + eat_token(chunk, &i, "}"); + break; + } + + eat_token(chunk, &i, ","); + } + } + + else if ( looking_at(chunk, i, "port") ) { + eat_token(chunk, &i, "port"); + a.ports.push_back(extract_port_range(chunk, &i)); } else if ( looking_at(chunk, i, "replaces") ) { @@ -939,6 +1026,13 @@ bool GlueCompiler::compile() { preinit_body.addCall("zeek_rt::register_protocol_analyzer", {builder()->stringMutable(a.name.str()), builder()->id(protocol), + builder()->vector( + hilti::util::transform(a.ports, + [this](const auto& p) -> hilti::Expression* { + return builder()->call("zeek_rt::make_port_range", + {builder()->port(p.begin), + builder()->port(p.end)}); + })), builder()->stringMutable(a.unit_name_orig.str()), builder()->stringMutable(a.unit_name_resp.str()), builder()->stringMutable(a.replaces), builder()->scope()}); diff --git a/src/spicy/spicyz/glue-compiler.h b/src/spicy/spicyz/glue-compiler.h index 22ffcdc332..58e42909f3 100644 --- a/src/spicy/spicyz/glue-compiler.h +++ b/src/spicy/spicyz/glue-compiler.h @@ -45,6 +45,7 @@ struct ProtocolAnalyzer { hilti::Location location; /**< Location where the analyzer was defined. */ hilti::ID name; /**< Name of the analyzer. */ hilti::rt::Protocol protocol = hilti::rt::Protocol::Undef; /**< The transport layer the analyzer uses. */ + std::vector<::zeek::spicy::rt::PortRange> ports; /**< The ports associated with the analyzer. */ hilti::ID unit_name_orig; /**< The fully-qualified name of the unit type to parse the originator side. */ hilti::ID unit_name_resp; /**< The fully-qualified name of the unit type to parse the originator diff --git a/testing/btest/Baseline/spicy.port-deprecated/out.stderr b/testing/btest/Baseline/spicy.port-deprecated/out.stderr deleted file mode 100644 index a033682601..0000000000 --- a/testing/btest/Baseline/spicy.port-deprecated/out.stderr +++ /dev/null @@ -1,2 +0,0 @@ -### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -[warning] <...>/udp-test.evt:4: Remove in v7.1: Analyzer spicy::TEST is using the deprecated 'port' or 'ports' keyword to register well-known ports. Use Analyzer::register_for_ports() in the accompanying Zeek script instead. diff --git a/testing/btest/Baseline/spicy.port-fail/output b/testing/btest/Baseline/spicy.port-fail/output index f572d2e79a..24eb09807d 100644 --- a/testing/btest/Baseline/spicy.port-fail/output +++ b/testing/btest/Baseline/spicy.port-fail/output @@ -1,3 +1,3 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -[error] <...>/port-fail.evt:9: port outside of valid range +[error] <...>/port-fail.evt:7: port outside of valid range [error] error loading EVT file "<...>/port-fail.evt" diff --git a/testing/btest/Baseline/spicy.port/output b/testing/btest/Baseline/spicy.port/output new file mode 100644 index 0000000000..938a6c7b35 --- /dev/null +++ b/testing/btest/Baseline/spicy.port/output @@ -0,0 +1,19 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Analyzer::ANALYZER_SPICY_TEST, 11337/udp +Analyzer::ANALYZER_SPICY_TEST, 11338/udp +Analyzer::ANALYZER_SPICY_TEST, 11339/udp +Analyzer::ANALYZER_SPICY_TEST, 11340/udp +Analyzer::ANALYZER_SPICY_TEST, 31337/udp +Analyzer::ANALYZER_SPICY_TEST, 31338/udp +Analyzer::ANALYZER_SPICY_TEST, 31339/udp +Analyzer::ANALYZER_SPICY_TEST, 31340/udp +{ +31339/udp, +31337/udp, +31338/udp, +11339/udp, +11338/udp, +11340/udp, +31340/udp, +11337/udp +} diff --git a/testing/btest/spicy/event-user-type b/testing/btest/spicy/event-user-type index 14b0883d5f..75f99b4042 100644 --- a/testing/btest/spicy/event-user-type +++ b/testing/btest/spicy/event-user-type @@ -25,7 +25,8 @@ type Y = unit { # @TEST-START-FILE foo.evt protocol analyzer spicy::foo over UDP: - parse with foo::X; + parse with foo::X, + ports { 12345/udp, 31337/udp }; import foo; @@ -35,13 +36,6 @@ on foo::X -> event foo::X($conn, $is_orig, self.y); # @TEST-END-FILE # @TEST-START-FILE foo.zeek -const foo_ports = { 12345/udp, 31337/udp}; - -event zeek_init() -{ - Analyzer::register_for_ports(Analyzer::ANALYZER_SPICY_FOO, foo_ports); -} - event foo::X(c: connection, is_orig: bool, y: foo::Y) { print fmt("is_orig=%d y=%s", is_orig, y); diff --git a/testing/btest/spicy/port-fail.evt b/testing/btest/spicy/port-fail.evt new file mode 100644 index 0000000000..f00efc6210 --- /dev/null +++ b/testing/btest/spicy/port-fail.evt @@ -0,0 +1,22 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC-FAIL: spicyz %INPUT -d -o x.hlto >output 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff output + +protocol analyzer spicy::SSH over TCP: + port 123456/udp; + +@TEST-START-NEXT + +protocol analyzer spicy::SSH over TCP: + port -1/udp; + +@TEST-START-NEXT + +protocol analyzer spicy::SSH over TCP: + port 1/udp-2/tcp; + +@TEST-START-NEXT + +protocol analyzer spicy::SSH over TCP: + port 2/udp-1/udp; diff --git a/testing/btest/spicy/port-range-one-port.zeek b/testing/btest/spicy/port-range-one-port.zeek new file mode 100644 index 0000000000..bdc5219791 --- /dev/null +++ b/testing/btest/spicy/port-range-one-port.zeek @@ -0,0 +1,24 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto udp-test.spicy ./udp-test.evt +# @TEST-EXEC: HILTI_DEBUG=zeek zeek -Cr ${TRACES}/udp-packet.pcap test.hlto %INPUT >out 2>&1 +# @TEST-EXEC: grep -e 'Scheduling analyzer' -e 'error during parsing' < out > out.filtered +# @TEST-EXEC: btest-diff out.filtered + +# @TEST-DOC: Expect a single 'Scheduling analyzer ...' message in the debug output and no parsing errors. There was a bug that 'port 31336/udp' would be wrongly interpreted as a 31336/udp-31337/udp port range. Regression test for #3278. + +# @TEST-START-FILE udp-test.spicy +module UDPTest; + +public type Message = unit { + data: bytes &eod { + assert False: "not reached"; + } +}; +# @TEST-END-FILE + +# @TEST-START-FILE udp-test.evt +protocol analyzer spicy::UDP_TEST over UDP: + parse with UDPTest::Message, + port 31336/udp; +# @TEST-END-FILE diff --git a/testing/btest/spicy/port.zeek b/testing/btest/spicy/port.zeek new file mode 100644 index 0000000000..81d3586c68 --- /dev/null +++ b/testing/btest/spicy/port.zeek @@ -0,0 +1,32 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -d -o test.hlto test.spicy test.evt +# @TEST-EXEC: zeek test.hlto %INPUT >output +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Check that we raise port events for Spicy analyzers, and that the ports get correctly registered. + +event spicy_analyzer_for_port(a: Analyzer::Tag, p: port){ + print a, p; +} + +event zeek_done() { + print Analyzer::ports[Analyzer::ANALYZER_SPICY_TEST]; +} + +# @TEST-START-FILE test.spicy +module Test; + +import zeek; + +public type Message = unit { + data: bytes &eod {} +}; +# @TEST-END-FILE + +# @TEST-START-FILE test.evt +protocol analyzer spicy::Test over UDP: + parse with Test::Message, + port 11337/udp-11340/udp, + ports {31337/udp-31340/udp}; +# @TEST-END-FILE