Integrate the Spicy plugin into Zeek proper.

This reflects the `spicy-plugin` code as of `d8c296b81cc2a11`.

In addition to moving the code into Zeek's source tree, this comes
with a couple small functional changes:

- `spicyz` no longer tries to infer if it's running from the build
  directory. Instead `ZEEK_SPICY_LIBRARY` can be set to a custom
  location. `zeek-set-path.sh` does that now.

- ZEEK_CONFIG can be set to change what `spicyz -z` print out. This is
  primarily for backwards compatibility.

Some further notes on specifics:

- We raise the minimum Spicy version to 1.8 (i.e., current `main`
  branch).

- Renamed the `compiler/` subdirectory to `spicyz` to avoid
  include-path conflicts with the Spicy headers.

- In `cmake/`, the corresponding PR brings a new/extended version of
  `FindZeek`, which Spicy analyzer packages need. We also now install
  some of the files that the Spicy plugin used to bring for testing,
  so that existing packages keep working.

- For now, this all remains backwards compatible with the current
  `zkg` analyzer templates so that they work with both external and
  integrated Spicy support. Later, once we don't need to support any
  external Spicy plugin versions anymore, we can clean up the
  templates as well.

- All the plugin's tests have moved into the standard test suite. They
  are skipped if configure with `--disable-spicy`.

This holds off on adapting the new code further to Zeek's coding
conventions, so that it remains easier to maintain it in parallel to
the (now legacy) external plugin. We'll make a pass over the
formatting for (presumable) Zeek 6.1.

scripts/base/frameworks/spicy/__load__.zeek

@@ -0,0 +1 @@
+@load ./main.zeek

scripts/base/frameworks/spicy/init-bare.zeek

@@ -0,0 +1,37 @@
+
+module Spicy;
+
+export {
+# doc-options-start
+    ## Constant for testing if Spicy is available.
+    const available = T;
+
+    ## Show output of Spicy print statements.
+    const enable_print = F &redef;
+
+    # Record and display profiling information, if compiled into analyzer.
+    const enable_profiling = F &redef;
+
+    ## abort() instead of throwing HILTI exceptions.
+    const abort_on_exceptions = F &redef;
+
+    ## Include backtraces when reporting unhandled exceptions.
+    const show_backtraces = F &redef;
+
+    ## Maximum depth of recursive file analysis (Spicy analyzers only)
+    const max_file_depth: count = 5 &redef;
+# doc-options-end
+
+# doc-types-start
+    ## Result type for `Spicy::resource_usage()`.
+    type ResourceUsage: record {
+        user_time : interval;           ##< user CPU time of the Zeek process
+        system_time :interval;          ##< system CPU time of the Zeek process
+        memory_heap : count;            ##< memory allocated on the heap by the Zeek process
+        num_fibers : count;             ##< number of fibers currently in use
+        max_fibers: count;              ##< maximum number of fibers ever in use
+        max_fiber_stack_size: count;    ##< maximum fiber stack size ever in use
+        cached_fibers: count;           ##< number of fibers currently cached
+    };
+# doc-types-end
+}

scripts/base/frameworks/spicy/init-framework.zeek

@@ -0,0 +1,81 @@
+@load base/misc/version
+
+# doc-common-start
+module Spicy;
+
+export {
+# doc-functions-start
+    ## Enable a specific Spicy protocol analyzer if not already active. If this
+    ## analyzer replaces an standard analyzer, that one will automatically be
+    ## disabled.
+    ##
+    ## tag: analyzer to toggle
+    ##
+    ## Returns: true if the operation succeeded
+    global enable_protocol_analyzer: function(tag: Analyzer::Tag) : bool;
+
+    ## Disable a specific Spicy protocol analyzer if not already inactive. If
+    ## this analyzer replaces an standard analyzer, that one will automatically
+    ## be re-enabled.
+    ##
+    ## tag: analyzer to toggle
+    ##
+    ## Returns: true if the operation succeeded
+    global disable_protocol_analyzer: function(tag: Analyzer::Tag) : bool;
+
+
+    ## Enable a specific Spicy file analyzer if not already active. If this
+    ## analyzer replaces an standard analyzer, that one will automatically be
+    ## disabled.
+    ##
+    ## tag: analyzer to toggle
+    ##
+    ## Returns: true if the operation succeeded
+    global enable_file_analyzer: function(tag: Files::Tag) : bool;
+
+    ## Disable a specific Spicy file analyzer if not already inactive. If
+    ## this analyzer replaces an standard analyzer, that one will automatically
+    ## be re-enabled.
+    ##
+    ## tag: analyzer to toggle
+    ##
+    ## Returns: true if the operation succeeded
+    global disable_file_analyzer: function(tag: Files::Tag) : bool;
+
+    ## Returns current resource usage as reported by the Spicy runtime system.
+    global resource_usage: function() : ResourceUsage;
+# doc-functions-end
+}
+
+# Marked with &is_used to suppress complaints when there aren't any
+# Spicy file analyzers loaded, and hence this event can't be generated.
+# The attribute is only supported for Zeek 5.0 and higher.
+event spicy_analyzer_for_mime_type(a: Files::Tag, mt: string) &is_used
+    {
+    Files::register_for_mime_type(a, mt);
+    }
+
+function enable_protocol_analyzer(tag: Analyzer::Tag) : bool
+    {
+    return Spicy::__toggle_analyzer(tag, T);
+    }
+
+function disable_protocol_analyzer(tag: Analyzer::Tag) : bool
+    {
+    return Spicy::__toggle_analyzer(tag, F);
+    }
+
+function enable_file_analyzer(tag: Files::Tag) : bool
+    {
+    return Spicy::__toggle_analyzer(tag, T);
+    }
+
+function disable_file_analyzer(tag: Files::Tag) : bool
+    {
+    return Spicy::__toggle_analyzer(tag, F);
+    }
+
+function resource_usage() : ResourceUsage
+    {
+    return Spicy::__resource_usage();
+    }

scripts/base/frameworks/spicy/main.zeek

@@ -0,0 +1,15 @@
+@load base/frameworks/notice
+
+module Spicy;
+
+export {
+    redef enum Notice::Type += { Spicy_Max_File_Depth_Exceeded };
+}
+
+event max_file_depth_exceeded(f: fa_file, args: Files::AnalyzerArgs, limit: count)
+    {
+    NOTICE([
+            $note=Spicy::Spicy_Max_File_Depth_Exceeded,
+            $msg=fmt("Maximum file depth exceeded for file %s", f$id)
+    ]);
+    }

scripts/base/frameworks/spicy/misc/record-spicy-batch.zeek

@@ -0,0 +1,90 @@
+# Saves all input traffic in Spicy's batch format.
+
+module SpicyBatch;
+
+export {
+    const filename = "batch.dat" &redef;
+}
+
+redef tcp_content_deliver_all_orig=T;
+redef tcp_content_deliver_all_resp=T;
+redef udp_content_deliver_all_orig=T;
+redef udp_content_deliver_all_resp=T;
+
+global output: file;
+global conns: set[conn_id];
+global num_conns = 0;
+
+function id(c: connection) : string
+	{
+	local cid = c$id;
+	local proto = "???";
+
+	if ( is_tcp_port(cid$orig_p) )
+		proto = "tcp";
+	else if ( is_udp_port(cid$orig_p) )
+		proto = "udp";
+	else if ( is_icmp_port(cid$orig_p) )
+		proto = "icmp";
+
+	return fmt("%s-%d-%s-%d-%s", cid$orig_h, cid$orig_p, cid$resp_h, cid$resp_p, proto);
+	}
+
+function begin(c: connection, type_: string)
+	{
+	add conns[c$id];
+	++num_conns;
+	print fmt("tracking %s", c$id);
+
+	local id_ = id(c);
+	print output, fmt("@begin-conn %s %s %s-orig %s%%orig %s-resp %s%%resp\n", id_, type_, id_, c$id$resp_p, id_, c$id$resp_p);
+	}
+
+event zeek_init()
+	{
+	output = open(filename);
+	enable_raw_output(output);
+	print output, "!spicy-batch v2\n";
+	}
+
+event new_connection_contents(c: connection)
+	{
+	begin(c, "stream");
+	}
+
+event tcp_contents(c: connection, is_orig: bool, seq: count, contents: string)
+	{
+	print output, fmt("@data %s-%s %d\n", id(c), (is_orig ? "orig" : "resp"), |contents|);
+	print output, contents;
+	print output, "\n";
+	}
+
+event content_gap(c: connection, is_orig: bool, seq: count, length: count)
+	{
+	print output, fmt("@gap %s-%s %d\n", id(c), (is_orig ? "orig" : "resp"), length);
+	}
+
+event udp_contents(c: connection, is_orig: bool, contents: string)
+	{
+	if ( c$id !in conns )
+		begin(c, "block");
+
+	print output, fmt("@data %s-%s %d\n", id(c), (is_orig ? "orig" : "resp"), |contents|);
+	print output, contents;
+	print output, "\n";
+	}
+
+event connection_state_remove(c: connection)
+	{
+	if ( c$id !in conns )
+		return;
+
+	print output, fmt("@end-conn %s\n", id(c));
+	}
+
+event zeek_done()
+	{
+	close(output);
+	print fmt("recorded %d session%s total", num_conns, (num_conns > 1 ? "s" : ""));
+	print fmt("output in %s", filename);
+	}

scripts/base/frameworks/spicy/misc/resource-usage.zeek

@@ -0,0 +1,18 @@
+module Spicy;
+
+event print_usage()
+	{
+	local r = Spicy::resource_usage();
+
+	print fmt("%.6f Spicy user=%f sys=%f heap=%d current_fibers=%d cached_fibers=%d max_fibers=%d max_stack=%d",
+	    network_time(), r$user_time, r$system_time, r$memory_heap,
+	    r$num_fibers, r$cached_fibers, r$max_fibers,
+	    r$max_fiber_stack_size);
+
+	schedule 1 min { print_usage() };
+	}
+
+event zeek_init()
+	{
+	schedule 1 min { print_usage() };
+	}