Merge remote-tracking branch 'origin/topic/robin/gh-3881-spicy-ports'

* origin/topic/robin/gh-3881-spicy-ports:
  Spicy: Register well-known ports through an event handler.
  Revert "Remove deprecated port/ports fields for spicy analyzers"

doc

@@ -1 +1 @@
-Subproject commit 425ce7933df90afdd34c7d5695b17d44a13ae8a7
+Subproject commit 1ca467fe0cb524ff375d957475d0319ab546915b

scripts/base/frameworks/spicy/init-framework.zeek

@@ -47,12 +47,18 @@ export {
 
 # Marked with &is_used to suppress complaints when there aren't any
 # Spicy file analyzers loaded, and hence this event can't be generated.
-# The attribute is only supported for Zeek 5.0 and higher.
 event spicy_analyzer_for_mime_type(a: Files::Tag, mt: string) &is_used
     {
     Files::register_for_mime_type(a, mt);
     }
 
+# Marked with &is_used to suppress complaints when there aren't any
+# Spicy protocol analyzers loaded, and hence this event can't be generated.
+event spicy_analyzer_for_port(a: Analyzer::Tag, p: port) &is_used
+    {
+    Analyzer::register_for_port(a, p);
+    }
+
 function enable_protocol_analyzer(tag: Analyzer::Tag) : bool
     {
     return Spicy::__toggle_analyzer(tag, T);

scripts/spicy/zeek_rt.hlt

@@ -18,7 +18,7 @@ type ZeekTypeTag = enum {
 } &cxxname="::zeek::spicy::rt::ZeekTypeTag";
 
 declare public void register_spicy_module_begin(string name, string description) &cxxname="zeek::spicy::rt::register_spicy_module_begin";
-declare public void register_protocol_analyzer(string name, hilti::Protocol protocol, string parser_orig, string parser_resp, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_protocol_analyzer" &have_prototype;
+declare public void register_protocol_analyzer(string name, hilti::Protocol protocol, vector<PortRange> ports, string parser_orig, string parser_resp, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_protocol_analyzer" &have_prototype;
 declare public void register_file_analyzer(string name, vector<string> mime_types, string parser, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_file_analyzer" &have_prototype;
 declare public void register_packet_analyzer(string name, string parser, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_packet_analyzer" &have_prototype;
 declare public void register_type(string ns, string id, BroType t) &cxxname="zeek::spicy::rt::register_type" &have_prototype;

src/spicy/manager.cc

@@ -6,6 +6,7 @@
 #include <glob.h>
 
 #include <exception>
+#include <iterator>
 #include <limits>
 #include <utility>
 
@@ -32,6 +33,7 @@
 #include "zeek/spicy/file-analyzer.h"
 #include "zeek/spicy/packet-analyzer.h"
 #include "zeek/spicy/protocol-analyzer.h"
+#include "zeek/spicy/runtime-support.h"
 #include "zeek/zeek-config-paths.h"
 
 using namespace zeek;
@@ -61,6 +63,7 @@ void Manager::registerSpicyModuleEnd() {
 }
 
 void Manager::registerProtocolAnalyzer(const std::string& name, hilti::rt::Protocol proto,
+                                       const hilti::rt::Vector<::zeek::spicy::rt::PortRange>& ports,
                                        const std::string& parser_orig, const std::string& parser_resp,
                                        const std::string& replaces, const std::string& linker_scope) {
     SPICY_DEBUG(hilti::rt::fmt("Have Spicy protocol analyzer %s", name));
@@ -75,6 +78,11 @@ void Manager::registerProtocolAnalyzer(const std::string& name, hilti::rt::Proto
     info.protocol = proto;
     info.linker_scope = linker_scope;
 
+    // Store ports in a deterministic order. We can't (easily) sort the
+    // `hilti::rt::Vector` unfortunately.
+    std::copy(ports.begin(), ports.end(), std::back_inserter(info.ports));
+    std::sort(info.ports.begin(), info.ports.end());
+
     // We may have that analyzer already iff it was previously pre-registered
     // without a linker scope. We'll then only set the scope now.
     if ( auto t = _analyzer_name_to_tag_type.find(info.name_zeek); t != _analyzer_name_to_tag_type.end() ) {
@@ -699,6 +707,36 @@ void Manager::InitPostScript() {
         if ( ! tag )
             reporter->InternalError("cannot get analyzer tag for '%s'", p.name_analyzer.c_str());
 
+        auto register_analyzer_for_port = [&](auto tag, const hilti::rt::Port& port_) {
+            SPICY_DEBUG(hilti::rt::fmt("  Scheduling analyzer for port %s", port_));
+
+            // Well-known ports are registered in scriptland, so we'll raise an
+            // event that will do it for us through a predefined handler.
+            zeek::Args vals = Args();
+            vals.emplace_back(tag.AsVal());
+            vals.emplace_back(zeek::spicy::rt::to_val(port_, base_type(TYPE_PORT)));
+            EventHandlerPtr handler = event_registry->Register("spicy_analyzer_for_port");
+            event_mgr.Enqueue(handler, vals);
+        };
+
+        for ( const auto& ports : p.ports ) {
+            const auto proto = ports.begin.protocol();
+
+            // Port ranges are closed intervals.
+            for ( auto port = ports.begin.port(); port <= ports.end.port(); ++port ) {
+                const auto port_ = hilti::rt::Port(port, proto);
+                register_analyzer_for_port(tag, port_);
+
+                // Don't double register in case of single-port ranges.
+                if ( ports.begin.port() == ports.end.port() )
+                    break;
+
+                // Explicitly prevent overflow.
+                if ( port == std::numeric_limits<decltype(port)>::max() )
+                    break;
+            }
+        }
+
         if ( p.parser_resp ) {
             for ( auto port : p.parser_resp->ports ) {
                 if ( port.direction != ::spicy::rt::Direction::Both &&
@@ -706,7 +744,7 @@ void Manager::InitPostScript() {
                     continue;
 
                 SPICY_DEBUG(hilti::rt::fmt("  Scheduling analyzer for port %s", port.port));
-                analyzer_mgr->RegisterAnalyzerForPort(tag, transport_protocol(port.port), port.port.port());
+                register_analyzer_for_port(tag, port.port);
             }
         }
     }

src/spicy/manager.h

@@ -85,6 +85,7 @@ public:
      *
      * @param name name of the analyzer as defined in its EVT file
      * @param proto analyzer's transport-layer protocol
+     * @param ports well-known ports for the analyzer; it'll be activated automatically for these
      * @param parser_orig name of the Spicy parser for the originator side; must match the name that
      * Spicy registers the unit's parser with
      * @param parser_resp name of the Spicy parser for the originator side; must match the name that
@@ -94,9 +95,10 @@ public:
      * @param linker_scope scope of current HLTO file, which will restrict visibility of the
      * registration
      */
-    void registerProtocolAnalyzer(const std::string& name, hilti::rt::Protocol proto, const std::string& parser_orig,
-                                  const std::string& parser_resp, const std::string& replaces,
-                                  const std::string& linker_scope);
+    void registerProtocolAnalyzer(const std::string& name, hilti::rt::Protocol proto,
+                                  const hilti::rt::Vector<::zeek::spicy::rt::PortRange>& ports,
+                                  const std::string& parser_orig, const std::string& parser_resp,
+                                  const std::string& replaces, const std::string& linker_scope);
 
     /**
      * Runtime method to register a file analyzer with its Zeek-side
@@ -341,6 +343,7 @@ private:
         std::string name_parser_resp;
         std::string name_replaces;
         hilti::rt::Protocol protocol = hilti::rt::Protocol::Undef;
+        std::vector<::zeek::spicy::rt::PortRange> ports; // we keep this sorted
         std::string linker_scope;
 
         // Computed and available once the analyzer has been registered.
@@ -354,7 +357,7 @@ private:
         bool operator==(const ProtocolAnalyzerInfo& other) const {
             return name_analyzer == other.name_analyzer && name_parser_orig == other.name_parser_orig &&
                    name_parser_resp == other.name_parser_resp && name_replaces == other.name_replaces &&
-                   protocol == other.protocol && linker_scope == other.linker_scope;
+                   protocol == other.protocol && ports == other.ports && linker_scope == other.linker_scope;
         }
 
         bool operator!=(const ProtocolAnalyzerInfo& other) const { return ! (*this == other); }

src/spicy/port-range.h

@@ -19,6 +19,11 @@ struct PortRange {
 
     hilti::rt::Port begin; /**< first port in the range */
     hilti::rt::Port end;   /**< last port in the range */
+
+    bool operator<(const PortRange& other) const {
+        // Just get us a deterministic order.
+        return std::tie(begin, end) < std::tie(other.begin, other.end);
+    }
 };
 
 inline bool operator==(const PortRange& a, const PortRange& b) {

src/spicy/runtime-support.cc

@@ -26,11 +26,12 @@ void rt::register_spicy_module_begin(const std::string& name, const std::string&
 
 void rt::register_spicy_module_end() { spicy_mgr->registerSpicyModuleEnd(); }
 
-void rt::register_protocol_analyzer(const std::string& name, hilti::rt::Protocol proto, const std::string& parser_orig,
-                                    const std::string& parser_resp, const std::string& replaces,
-                                    const std::string& linker_scope) {
+void rt::register_protocol_analyzer(const std::string& name, hilti::rt::Protocol proto,
+                                    const hilti::rt::Vector<::zeek::spicy::rt::PortRange>& ports,
+                                    const std::string& parser_orig, const std::string& parser_resp,
+                                    const std::string& replaces, const std::string& linker_scope) {
     auto _ = hilti::rt::profiler::start("zeek/rt/register_protocol_analyzer");
-    spicy_mgr->registerProtocolAnalyzer(name, proto, parser_orig, parser_resp, replaces, linker_scope);
+    spicy_mgr->registerProtocolAnalyzer(name, proto, ports, parser_orig, parser_resp, replaces, linker_scope);
 }
 
 void rt::register_file_analyzer(const std::string& name, const hilti::rt::Vector<std::string>& mime_types,

src/spicy/runtime-support.h

@@ -106,9 +106,10 @@ void register_spicy_module_begin(const std::string& id, const std::string& descr
  * Registers a Spicy protocol analyzer with its EVT meta information with the
  * plugin's runtime.
  */
-void register_protocol_analyzer(const std::string& id, hilti::rt::Protocol proto, const std::string& parser_orig,
-                                const std::string& parser_resp, const std::string& replaces,
-                                const std::string& linker_scope);
+void register_protocol_analyzer(const std::string& id, hilti::rt::Protocol proto,
+                                const hilti::rt::Vector<::zeek::spicy::rt::PortRange>& ports,
+                                const std::string& parser_orig, const std::string& parser_resp,
+                                const std::string& replaces, const std::string& linker_scope);
 
 /**
  * Registers a Spicy file analyzer with its EVT meta information with the

src/spicy/spicyz/glue-compiler.cc

@@ -260,6 +260,79 @@ static std::string extract_expr(const std::string& chunk, size_t* i) {
     return expr;
 }
 
+static hilti::rt::Port extract_port(const std::string& chunk, size_t* i) {
+    eat_spaces(chunk, i);
+
+    std::string s;
+    size_t j = *i;
+
+    while ( j < chunk.size() && isdigit(chunk[j]) )
+        ++j;
+
+    if ( *i == j )
+        throw ParseError("cannot parse port specification");
+
+    hilti::rt::Protocol proto;
+    uint64_t port = std::numeric_limits<uint64_t>::max();
+
+    s = chunk.substr(*i, j - *i);
+    hilti::util::atoi_n(s.begin(), s.end(), 10, &port);
+
+    if ( port > 65535 )
+        throw ParseError("port outside of valid range");
+
+    *i = j;
+
+    if ( chunk[*i] != '/' )
+        throw ParseError("cannot parse port specification");
+
+    (*i)++;
+
+    if ( looking_at(chunk, *i, "tcp") ) {
+        proto = hilti::rt::Protocol::TCP;
+        eat_token(chunk, i, "tcp");
+    }
+
+    else if ( looking_at(chunk, *i, "udp") ) {
+        proto = hilti::rt::Protocol::UDP;
+        eat_token(chunk, i, "udp");
+    }
+
+    else if ( looking_at(chunk, *i, "icmp") ) {
+        proto = hilti::rt::Protocol::ICMP;
+        eat_token(chunk, i, "icmp");
+    }
+
+    else
+        throw ParseError("cannot parse port specification");
+
+    return {static_cast<uint16_t>(port), proto};
+}
+
+static ::zeek::spicy::rt::PortRange extract_port_range(const std::string& chunk, size_t* i) {
+    auto start = extract_port(chunk, i);
+    auto end = std::optional<hilti::rt::Port>();
+
+    if ( looking_at(chunk, *i, "-") ) {
+        eat_token(chunk, i, "-");
+        end = extract_port(chunk, i);
+    }
+
+    if ( end ) {
+        if ( start.protocol() != end->protocol() )
+            throw ParseError("start and end of port range must have same protocol");
+
+        if ( start.port() > end->port() )
+            throw ParseError("start of port range cannot be after its end");
+    }
+
+    if ( ! end )
+        // EVT port ranges are a closed.
+        end = hilti::rt::Port(start.port(), start.protocol());
+
+    return {start, *end};
+}
+
 void GlueCompiler::init(Driver* driver, int zeek_version) {
     _driver = driver;
     _zeek_version = zeek_version;
@@ -631,11 +704,25 @@ glue::ProtocolAnalyzer GlueCompiler::parseProtocolAnalyzer(const std::string& ch
             }
         }
 
-        else if ( looking_at(chunk, i, "ports") || looking_at(chunk, i, "port") ) {
-            throw ParseError(hilti::rt::fmt(
-                "Analyzer %s is using the removed 'port' or 'ports' keyword to register "
-                "well-known ports. Use Analyzer::register_for_ports() in the accompanying Zeek script instead.",
-                a.name));
+        else if ( looking_at(chunk, i, "ports") ) {
+            eat_token(chunk, &i, "ports");
+            eat_token(chunk, &i, "{");
+
+            while ( true ) {
+                a.ports.push_back(extract_port_range(chunk, &i));
+
+                if ( looking_at(chunk, i, "}") ) {
+                    eat_token(chunk, &i, "}");
+                    break;
+                }
+
+                eat_token(chunk, &i, ",");
+            }
+        }
+
+        else if ( looking_at(chunk, i, "port") ) {
+            eat_token(chunk, &i, "port");
+            a.ports.push_back(extract_port_range(chunk, &i));
         }
 
         else if ( looking_at(chunk, i, "replaces") ) {
@@ -939,6 +1026,13 @@ bool GlueCompiler::compile() {
 
         preinit_body.addCall("zeek_rt::register_protocol_analyzer",
                              {builder()->stringMutable(a.name.str()), builder()->id(protocol),
+                              builder()->vector(
+                                  hilti::util::transform(a.ports,
+                                                         [this](const auto& p) -> hilti::Expression* {
+                                                             return builder()->call("zeek_rt::make_port_range",
+                                                                                    {builder()->port(p.begin),
+                                                                                     builder()->port(p.end)});
+                                                         })),
                               builder()->stringMutable(a.unit_name_orig.str()),
                               builder()->stringMutable(a.unit_name_resp.str()), builder()->stringMutable(a.replaces),
                               builder()->scope()});

src/spicy/spicyz/glue-compiler.h

@@ -45,6 +45,7 @@ struct ProtocolAnalyzer {
     hilti::Location location;                                  /**< Location where the analyzer was defined. */
     hilti::ID name;                                            /**< Name of the analyzer. */
     hilti::rt::Protocol protocol = hilti::rt::Protocol::Undef; /**< The transport layer the analyzer uses. */
+    std::vector<::zeek::spicy::rt::PortRange> ports;           /**< The ports associated with the analyzer. */
     hilti::ID unit_name_orig; /**< The fully-qualified name of the unit type to parse the originator
                                  side. */
     hilti::ID unit_name_resp; /**< The fully-qualified name of the unit type to parse the originator

testing/btest/Baseline/spicy.port-deprecated/out.stderr

@@ -1,2 +0,0 @@
-### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-[warning] <...>/udp-test.evt:4: Remove in v7.1: Analyzer spicy::TEST is using the deprecated 'port' or 'ports' keyword to register well-known ports. Use Analyzer::register_for_ports() in the accompanying Zeek script instead.

testing/btest/Baseline/spicy.port-fail/output

@@ -1,3 +1,3 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-[error] <...>/port-fail.evt:9: port outside of valid range
+[error] <...>/port-fail.evt:7: port outside of valid range
 [error] error loading EVT file "<...>/port-fail.evt"

testing/btest/Baseline/spicy.port/output

@@ -0,0 +1,19 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Analyzer::ANALYZER_SPICY_TEST, 11337/udp
+Analyzer::ANALYZER_SPICY_TEST, 11338/udp
+Analyzer::ANALYZER_SPICY_TEST, 11339/udp
+Analyzer::ANALYZER_SPICY_TEST, 11340/udp
+Analyzer::ANALYZER_SPICY_TEST, 31337/udp
+Analyzer::ANALYZER_SPICY_TEST, 31338/udp
+Analyzer::ANALYZER_SPICY_TEST, 31339/udp
+Analyzer::ANALYZER_SPICY_TEST, 31340/udp
+{
+31339/udp,
+31337/udp,
+31338/udp,
+11339/udp,
+11338/udp,
+11340/udp,
+31340/udp,
+11337/udp
+}

testing/btest/spicy/event-user-type

@@ -25,7 +25,8 @@ type Y = unit {
 
 # @TEST-START-FILE foo.evt
 protocol analyzer spicy::foo over UDP:
-    parse with foo::X;
+    parse with foo::X,
+    ports { 12345/udp, 31337/udp };
 
 import foo;
 
@@ -35,13 +36,6 @@ on foo::X -> event foo::X($conn, $is_orig, self.y);
 # @TEST-END-FILE
 
 # @TEST-START-FILE foo.zeek
-const foo_ports = { 12345/udp, 31337/udp};
-
-event zeek_init()
-{
-	Analyzer::register_for_ports(Analyzer::ANALYZER_SPICY_FOO, foo_ports);
-}
-
 event foo::X(c: connection, is_orig: bool, y: foo::Y)
 {
 	print fmt("is_orig=%d y=%s", is_orig, y);

testing/btest/spicy/port-fail.evt

@@ -0,0 +1,22 @@
+# @TEST-REQUIRES: have-spicy
+#
+# @TEST-EXEC-FAIL: spicyz %INPUT -d -o x.hlto >output 2>&1
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff output
+
+protocol analyzer spicy::SSH over TCP:
+    port 123456/udp;
+
+@TEST-START-NEXT
+
+protocol analyzer spicy::SSH over TCP:
+    port -1/udp;
+
+@TEST-START-NEXT
+
+protocol analyzer spicy::SSH over TCP:
+    port 1/udp-2/tcp;
+
+@TEST-START-NEXT
+
+protocol analyzer spicy::SSH over TCP:
+    port 2/udp-1/udp;

testing/btest/spicy/port-range-one-port.zeek

@@ -0,0 +1,24 @@
+# @TEST-REQUIRES: have-spicy
+#
+# @TEST-EXEC: spicyz -o test.hlto udp-test.spicy ./udp-test.evt
+# @TEST-EXEC: HILTI_DEBUG=zeek zeek -Cr ${TRACES}/udp-packet.pcap test.hlto %INPUT >out 2>&1
+# @TEST-EXEC: grep -e 'Scheduling analyzer' -e 'error during parsing' < out > out.filtered
+# @TEST-EXEC: btest-diff out.filtered
+
+# @TEST-DOC: Expect a single 'Scheduling analyzer ...' message in the debug output and no parsing errors. There was a bug that 'port 31336/udp' would be wrongly interpreted as a 31336/udp-31337/udp port range. Regression test for #3278.
+
+# @TEST-START-FILE udp-test.spicy
+module UDPTest;
+
+public type Message = unit {
+    data: bytes &eod {
+      assert False: "not reached";
+    }
+};
+# @TEST-END-FILE
+
+# @TEST-START-FILE udp-test.evt
+protocol analyzer spicy::UDP_TEST over UDP:
+    parse with UDPTest::Message,
+    port 31336/udp;
+# @TEST-END-FILE

testing/btest/spicy/port.zeek

@@ -0,0 +1,32 @@
+# @TEST-REQUIRES: have-spicy
+#
+# @TEST-EXEC: spicyz -d -o test.hlto test.spicy test.evt
+# @TEST-EXEC: zeek test.hlto %INPUT >output
+# @TEST-EXEC: btest-diff output
+#
+# @TEST-DOC: Check that we raise port events for Spicy analyzers, and that the ports get correctly registered.
+
+event spicy_analyzer_for_port(a: Analyzer::Tag, p: port){
+    print a, p;
+}
+
+event zeek_done() {
+    print Analyzer::ports[Analyzer::ANALYZER_SPICY_TEST];
+}
+
+# @TEST-START-FILE test.spicy
+module Test;
+
+import zeek;
+
+public type Message = unit {
+    data: bytes &eod {}
+};
+# @TEST-END-FILE
+
+# @TEST-START-FILE test.evt
+protocol analyzer spicy::Test over UDP:
+    parse with Test::Message,
+    port 11337/udp-11340/udp,
+    ports {31337/udp-31340/udp};
+# @TEST-END-FILE