From 0040111955d29df7dd9bda040a1d17a4b524a0d1 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 25 Apr 2023 20:52:25 +0200 Subject: [PATCH] Integrate the Spicy plugin into Zeek proper. This reflects the `spicy-plugin` code as of `d8c296b81cc2a11`. In addition to moving the code into Zeek's source tree, this comes with a couple small functional changes: - `spicyz` no longer tries to infer if it's running from the build directory. Instead `ZEEK_SPICY_LIBRARY` can be set to a custom location. `zeek-set-path.sh` does that now. - ZEEK_CONFIG can be set to change what `spicyz -z` print out. This is primarily for backwards compatibility. Some further notes on specifics: - We raise the minimum Spicy version to 1.8 (i.e., current `main` branch). - Renamed the `compiler/` subdirectory to `spicyz` to avoid include-path conflicts with the Spicy headers. - In `cmake/`, the corresponding PR brings a new/extended version of `FindZeek`, which Spicy analyzer packages need. We also now install some of the files that the Spicy plugin used to bring for testing, so that existing packages keep working. - For now, this all remains backwards compatible with the current `zkg` analyzer templates so that they work with both external and integrated Spicy support. Later, once we don't need to support any external Spicy plugin versions anymore, we can clean up the templates as well. - All the plugin's tests have moved into the standard test suite. They are skipped if configure with `--disable-spicy`. This holds off on adapting the new code further to Zeek's coding conventions, so that it remains easier to maintain it in parallel to the (now legacy) external plugin. We'll make a pass over the formatting for (presumable) Zeek 6.1. --- .gitmodules | 3 - CMakeLists.txt | 95 +- auxil/spicy-plugin | 1 - auxil/spicy/spicy | 2 +- cmake | 2 +- configure | 4 - scripts/base/frameworks/spicy/__load__.zeek | 1 + scripts/base/frameworks/spicy/init-bare.zeek | 37 + .../base/frameworks/spicy/init-framework.zeek | 81 + scripts/base/frameworks/spicy/main.zeek | 15 + .../spicy/misc/record-spicy-batch.zeek | 90 ++ .../frameworks/spicy/misc/resource-usage.zeek | 18 + scripts/base/init-bare.zeek | 1 + scripts/base/init-default.zeek | 1 + scripts/base/init-frameworks-and-bifs.zeek | 2 + scripts/spicy/zeek.spicy | 157 ++ scripts/spicy/zeek_file.spicy | 31 + scripts/spicy/zeek_rt.hlt | 43 + src/CMakeLists.txt | 8 + src/DebugLogger.cc | 3 +- src/DebugLogger.h | 1 + src/plugin/Plugin.h | 14 +- src/spicy/.clang-format | 132 ++ src/spicy/CMakeLists.txt | 29 + src/spicy/cookie.h | 233 +++ src/spicy/debug.h | 0 src/spicy/empty.cc | 2 + src/spicy/file-analyzer.cc | 138 ++ src/spicy/file-analyzer.h | 91 ++ src/spicy/manager.cc | 920 +++++++++++ src/spicy/manager.h | 413 +++++ src/spicy/packet-analyzer.cc | 73 + src/spicy/packet-analyzer.h | 75 + src/spicy/protocol-analyzer.cc | 231 +++ src/spicy/protocol-analyzer.h | 166 ++ src/spicy/runtime-support.cc | 842 ++++++++++ src/spicy/runtime-support.h | 889 +++++++++++ src/spicy/spicy.bif | 62 + src/spicy/spicyz/CMakeLists.txt | 10 + src/spicy/spicyz/config.h.in | 67 + src/spicy/spicyz/driver.cc | 301 ++++ src/spicy/spicyz/driver.h | 199 +++ src/spicy/spicyz/glue-compiler.cc | 1352 +++++++++++++++++ src/spicy/spicyz/glue-compiler.h | 242 +++ src/spicy/spicyz/main.cc | 282 ++++ src/util-config.h.in | 4 + src/zeek-setup.cc | 12 + testing/CMakeLists.txt | 11 + .../canonified_loaded_scripts.log | 17 +- .../canonified_loaded_scripts.log | 20 +- .../coverage.init-default/missing_loads | 2 + testing/btest/Baseline/plugins.hooks/output | 69 +- .../btest/Baseline/spicy.analyzer-tag/output | 6 + testing/btest/Baseline/spicy.conn-id/output | 3 + testing/btest/Baseline/spicy.context/output | 2 + testing/btest/Baseline/spicy.dns/output | 3 + .../btest/Baseline/spicy.double-event/output | 5 + .../btest/Baseline/spicy.double-types/output | 4 + .../Baseline/spicy.event-args-fail-2/output | 3 + .../Baseline/spicy.event-args-fail-3/output | 3 + .../Baseline/spicy.event-args-fail-4/output | 3 + .../Baseline/spicy.event-args-fail-5/output | 3 + .../Baseline/spicy.event-args-fail-6/output | 3 + .../Baseline/spicy.event-args-fail/output | 3 + .../Baseline/spicy.event-args-mismatch/output | 2 + .../btest/Baseline/spicy.event-args/output | 3 + .../btest/Baseline/spicy.event-cond/output | 6 + .../Baseline/spicy.event-user-type/output | 5 + .../btest/Baseline/spicy.export-enum/output | 4 + .../Baseline/spicy.export-type-fail/output | 6 + .../btest/Baseline/spicy.export-types/output | 42 + .../output | 4 + .../spicy.file-analysis-data-in/files.log | 3 + .../spicy.file-analysis-data-in/output | 3 + .../spicy.file-analysis-data-in/x509.log | 3 + .../Baseline/spicy.file-analyzer-nested/files | 6 + .../spicy.file-analyzer-nested/output | 5 + .../spicy.file-analyzer-nested/output-max | 7 + .../spicy.file-analyzer-property/output | 2 + .../btest/Baseline/spicy.file-analyzer/output | 3 + .../Baseline/spicy.file-analyzer/weird.log | 11 + .../spicy.file-data-in-at-offset/output | 3 + .../spicy.file-data-in-at-offset/x509.log | 3 + .../btest/Baseline/spicy.file-replaces/output | 3 + .../btest/Baseline/spicy.gap-recovery/output | 3 + .../output-before-spicy-issue-1303 | 3 + .../btest/Baseline/spicy.import-from/output | 3 + .../Baseline/spicy.list-conversion/output | 6 + .../btest/Baseline/spicy.module-path/output | 3 + .../btest/Baseline/spicy.multiple-enum/output | 2 + .../btest/Baseline/spicy.network-time/output | 5 + testing/btest/Baseline/spicy.optional/output | 2 + .../spicy.packet-analyzer-on-ip/output | 4 + .../spicy.packet-analyzer-replaces/output-on | 2 + .../spicy.packet-analyzer-violation/output | 2 + .../Baseline/spicy.packet-analyzer/output | 3 + .../Baseline/spicy.packet-analyzer/weird.log | 11 + .../btest/Baseline/spicy.parse-error/dpd.log | 11 + .../btest/Baseline/spicy.port-fail-2/output | 3 + .../btest/Baseline/spicy.port-fail-3/output | 3 + .../btest/Baseline/spicy.port-fail-4/output | 3 + .../btest/Baseline/spicy.port-fail-5/output | 3 + testing/btest/Baseline/spicy.port-fail/output | 3 + .../Baseline/spicy.preprocessor-fail-2/output | 5 + .../Baseline/spicy.preprocessor-fail-3/output | 5 + .../Baseline/spicy.preprocessor-fail-4/output | 5 + .../Baseline/spicy.preprocessor-fail-5/output | 5 + .../Baseline/spicy.preprocessor-fail-6/output | 5 + .../Baseline/spicy.preprocessor-fail-7/output | 5 + .../Baseline/spicy.preprocessor-fail/output | 5 + .../Baseline/spicy.preprocessor-spicy/output | 7 + .../btest/Baseline/spicy.preprocessor/output | 9 + testing/btest/Baseline/spicy.profiling/output | 3 + .../btest/Baseline/spicy.profiling/prof.log | 22 + .../spicy.protocol-analyzer-data-in/http.log | 13 + .../output | 5 + .../ssh.log | 11 + .../Baseline/spicy.replaces-mismatch/output | 2 + testing/btest/Baseline/spicy.replaces/output | 4 + .../Baseline/spicy.resource-usage/output | 1 + .../spicy.reuse-parser-across-hltos/output | 16 + .../Baseline/spicy.ssh-banner/analyzer.log | 12 + .../btest/Baseline/spicy.ssh-banner/output | 10 + .../btest/Baseline/spicy.ssh-banner/weird.log | 11 + .../Baseline/spicy.terminate-session/conn.log | 12 + .../spicy.toggle-protocol-analyzer/output | 5 + testing/btest/Baseline/spicy.tuple-arg/output | 9 + .../btest/Baseline/spicy.tuple-enum/output | 5 + .../Baseline/spicy.tuple-optional/output | 2 + .../Baseline/spicy.type-converter/output | 28 + testing/btest/Baseline/spicy.udp/output | 6 + testing/btest/Traces/dns/long-connection.pcap | Bin 0 -> 2428 bytes testing/btest/Traces/dns/proto255.pcap | Bin 0 -> 515 bytes testing/btest/Traces/pe/pe-single.trace | Bin 0 -> 54722 bytes testing/btest/Traces/spicy/gap-recovery.pcap | Bin 0 -> 129956 bytes .../spicy/packet-analyzer-violation.pcap | Bin 0 -> 58 bytes testing/btest/Traces/spicy/raw-layer.pcap | Bin 0 -> 161 bytes testing/btest/Traces/ssh/single-conn.trace | Bin 0 -> 19883 bytes testing/btest/Traces/ssh/ssh-over-udp.pcap | Bin 0 -> 201 bytes testing/btest/Traces/udp-packet.pcap | Bin 0 -> 280 bytes testing/btest/btest.cfg | 4 +- testing/btest/spicy/analyzer-tag.zeek | 33 + testing/btest/spicy/availability.zeek | 15 + testing/btest/spicy/conn-id.spicy | 21 + testing/btest/spicy/context.spicy | 50 + testing/btest/spicy/double-event.zeek | 33 + .../btest/spicy/double-type-registration.zeek | 34 + testing/btest/spicy/double-types.zeek | 63 + testing/btest/spicy/event-args-fail.evt | 34 + testing/btest/spicy/event-args-mismatch.zeek | 30 + testing/btest/spicy/event-args.zeek | 33 + testing/btest/spicy/event-cond.zeek | 58 + testing/btest/spicy/event-user-type | 43 + testing/btest/spicy/export-enum.zeek | 31 + testing/btest/spicy/export-type-fail.spicy | 35 + testing/btest/spicy/export-types.zeek | 103 ++ .../file-analysis-data-in-concurrent.zeek | 60 + .../btest/spicy/file-analysis-data-in.zeek | 64 + testing/btest/spicy/file-analyzer-nested.zeek | 87 ++ .../btest/spicy/file-analyzer-property.zeek | 27 + testing/btest/spicy/file-analyzer.zeek | 46 + .../btest/spicy/file-data-in-at-offset.zeek | 47 + testing/btest/spicy/file-replaces.zeek | 117 ++ testing/btest/spicy/gap-recovery.zeek | 47 + testing/btest/spicy/import-from.zeek | 57 + testing/btest/spicy/list-conversion.zeek | 54 + testing/btest/spicy/module-path.spicy | 11 + testing/btest/spicy/multiple-enum.zeek | 41 + testing/btest/spicy/network-time.spicy | 26 + testing/btest/spicy/optional.zeek | 39 + .../btest/spicy/packet-analyzer-on-ip.zeek | 37 + .../btest/spicy/packet-analyzer-replaces.zeek | 61 + .../spicy/packet-analyzer-violation.zeek | 30 + testing/btest/spicy/packet-analyzer.zeek | 45 + testing/btest/spicy/parse-error.zeek | 36 + testing/btest/spicy/port-fail.evt | 22 + testing/btest/spicy/preprocessor-fail.evt | 37 + testing/btest/spicy/preprocessor-spicy.spicy | 61 + testing/btest/spicy/preprocessor.evt | 40 + testing/btest/spicy/profiling.zeek | 37 + .../spicy/protocol-analyzer-data-in.zeek | 57 + ...protocol-analyzer-explicit-forwarding.zeek | 48 + .../protocol-analyzer-tcp-over-udp.spicy | 28 + testing/btest/spicy/replaces-mismatch.zeek | 25 + testing/btest/spicy/replaces.zeek | 45 + testing/btest/spicy/resource-usage.zeek | 26 + .../spicy/reuse-parser-across-hltos.zeek | 70 + testing/btest/spicy/spicy-dump.spicy | 5 +- testing/btest/spicy/spicyz.test | 7 +- testing/btest/spicy/ssh-banner.zeek | 75 + testing/btest/spicy/terminate-session.zeek | 41 + .../btest/spicy/toggle-protocol-analyzer.zeek | 46 + testing/btest/spicy/tuple-arg.zeek | 48 + testing/btest/spicy/tuple-enum.zeek | 42 + testing/btest/spicy/type-converter.zeek | 112 ++ testing/btest/spicy/udp.zeek | 33 + testing/scripts/diff-canonifier-spicy | 6 + testing/scripts/spicy-version | 13 + testing/scripts/spicy/README | 2 + testing/scripts/spicy/canonify-zeek-log | 17 + .../scripts/spicy/canonify-zeek-log-sorted | 18 + testing/scripts/spicy/diff-remove-abspath | 12 + testing/scripts/spicy/diff-remove-timestamps | 12 + testing/scripts/spicy/diff-sort | 19 + testing/scripts/spicy/run-zeek | 7 + testing/scripts/spicy/spicy-version | 14 + testing/scripts/spicy/zeek-version | 20 + zeek-config.h.in | 6 + zeek-config.in | 1 + 209 files changed, 10406 insertions(+), 160 deletions(-) delete mode 160000 auxil/spicy-plugin create mode 100644 scripts/base/frameworks/spicy/__load__.zeek create mode 100644 scripts/base/frameworks/spicy/init-bare.zeek create mode 100644 scripts/base/frameworks/spicy/init-framework.zeek create mode 100644 scripts/base/frameworks/spicy/main.zeek create mode 100644 scripts/base/frameworks/spicy/misc/record-spicy-batch.zeek create mode 100644 scripts/base/frameworks/spicy/misc/resource-usage.zeek create mode 100644 scripts/spicy/zeek.spicy create mode 100644 scripts/spicy/zeek_file.spicy create mode 100644 scripts/spicy/zeek_rt.hlt create mode 100644 src/spicy/.clang-format create mode 100644 src/spicy/CMakeLists.txt create mode 100644 src/spicy/cookie.h create mode 100644 src/spicy/debug.h create mode 100644 src/spicy/empty.cc create mode 100644 src/spicy/file-analyzer.cc create mode 100644 src/spicy/file-analyzer.h create mode 100644 src/spicy/manager.cc create mode 100644 src/spicy/manager.h create mode 100644 src/spicy/packet-analyzer.cc create mode 100644 src/spicy/packet-analyzer.h create mode 100644 src/spicy/protocol-analyzer.cc create mode 100644 src/spicy/protocol-analyzer.h create mode 100644 src/spicy/runtime-support.cc create mode 100644 src/spicy/runtime-support.h create mode 100644 src/spicy/spicy.bif create mode 100644 src/spicy/spicyz/CMakeLists.txt create mode 100644 src/spicy/spicyz/config.h.in create mode 100644 src/spicy/spicyz/driver.cc create mode 100644 src/spicy/spicyz/driver.h create mode 100644 src/spicy/spicyz/glue-compiler.cc create mode 100644 src/spicy/spicyz/glue-compiler.h create mode 100644 src/spicy/spicyz/main.cc create mode 100644 testing/btest/Baseline/spicy.analyzer-tag/output create mode 100644 testing/btest/Baseline/spicy.conn-id/output create mode 100644 testing/btest/Baseline/spicy.context/output create mode 100644 testing/btest/Baseline/spicy.dns/output create mode 100644 testing/btest/Baseline/spicy.double-event/output create mode 100644 testing/btest/Baseline/spicy.double-types/output create mode 100644 testing/btest/Baseline/spicy.event-args-fail-2/output create mode 100644 testing/btest/Baseline/spicy.event-args-fail-3/output create mode 100644 testing/btest/Baseline/spicy.event-args-fail-4/output create mode 100644 testing/btest/Baseline/spicy.event-args-fail-5/output create mode 100644 testing/btest/Baseline/spicy.event-args-fail-6/output create mode 100644 testing/btest/Baseline/spicy.event-args-fail/output create mode 100644 testing/btest/Baseline/spicy.event-args-mismatch/output create mode 100644 testing/btest/Baseline/spicy.event-args/output create mode 100644 testing/btest/Baseline/spicy.event-cond/output create mode 100644 testing/btest/Baseline/spicy.event-user-type/output create mode 100644 testing/btest/Baseline/spicy.export-enum/output create mode 100644 testing/btest/Baseline/spicy.export-type-fail/output create mode 100644 testing/btest/Baseline/spicy.export-types/output create mode 100644 testing/btest/Baseline/spicy.file-analysis-data-in-concurrent/output create mode 100644 testing/btest/Baseline/spicy.file-analysis-data-in/files.log create mode 100644 testing/btest/Baseline/spicy.file-analysis-data-in/output create mode 100644 testing/btest/Baseline/spicy.file-analysis-data-in/x509.log create mode 100644 testing/btest/Baseline/spicy.file-analyzer-nested/files create mode 100644 testing/btest/Baseline/spicy.file-analyzer-nested/output create mode 100644 testing/btest/Baseline/spicy.file-analyzer-nested/output-max create mode 100644 testing/btest/Baseline/spicy.file-analyzer-property/output create mode 100644 testing/btest/Baseline/spicy.file-analyzer/output create mode 100644 testing/btest/Baseline/spicy.file-analyzer/weird.log create mode 100644 testing/btest/Baseline/spicy.file-data-in-at-offset/output create mode 100644 testing/btest/Baseline/spicy.file-data-in-at-offset/x509.log create mode 100644 testing/btest/Baseline/spicy.file-replaces/output create mode 100644 testing/btest/Baseline/spicy.gap-recovery/output create mode 100644 testing/btest/Baseline/spicy.gap-recovery/output-before-spicy-issue-1303 create mode 100644 testing/btest/Baseline/spicy.import-from/output create mode 100644 testing/btest/Baseline/spicy.list-conversion/output create mode 100644 testing/btest/Baseline/spicy.module-path/output create mode 100644 testing/btest/Baseline/spicy.multiple-enum/output create mode 100644 testing/btest/Baseline/spicy.network-time/output create mode 100644 testing/btest/Baseline/spicy.optional/output create mode 100644 testing/btest/Baseline/spicy.packet-analyzer-on-ip/output create mode 100644 testing/btest/Baseline/spicy.packet-analyzer-replaces/output-on create mode 100644 testing/btest/Baseline/spicy.packet-analyzer-violation/output create mode 100644 testing/btest/Baseline/spicy.packet-analyzer/output create mode 100644 testing/btest/Baseline/spicy.packet-analyzer/weird.log create mode 100644 testing/btest/Baseline/spicy.parse-error/dpd.log create mode 100644 testing/btest/Baseline/spicy.port-fail-2/output create mode 100644 testing/btest/Baseline/spicy.port-fail-3/output create mode 100644 testing/btest/Baseline/spicy.port-fail-4/output create mode 100644 testing/btest/Baseline/spicy.port-fail-5/output create mode 100644 testing/btest/Baseline/spicy.port-fail/output create mode 100644 testing/btest/Baseline/spicy.preprocessor-fail-2/output create mode 100644 testing/btest/Baseline/spicy.preprocessor-fail-3/output create mode 100644 testing/btest/Baseline/spicy.preprocessor-fail-4/output create mode 100644 testing/btest/Baseline/spicy.preprocessor-fail-5/output create mode 100644 testing/btest/Baseline/spicy.preprocessor-fail-6/output create mode 100644 testing/btest/Baseline/spicy.preprocessor-fail-7/output create mode 100644 testing/btest/Baseline/spicy.preprocessor-fail/output create mode 100644 testing/btest/Baseline/spicy.preprocessor-spicy/output create mode 100644 testing/btest/Baseline/spicy.preprocessor/output create mode 100644 testing/btest/Baseline/spicy.profiling/output create mode 100644 testing/btest/Baseline/spicy.profiling/prof.log create mode 100644 testing/btest/Baseline/spicy.protocol-analyzer-data-in/http.log create mode 100644 testing/btest/Baseline/spicy.protocol-analyzer-explicit-forwarding/output create mode 100644 testing/btest/Baseline/spicy.protocol-analyzer-tcp-over-udp/ssh.log create mode 100644 testing/btest/Baseline/spicy.replaces-mismatch/output create mode 100644 testing/btest/Baseline/spicy.replaces/output create mode 100644 testing/btest/Baseline/spicy.resource-usage/output create mode 100644 testing/btest/Baseline/spicy.reuse-parser-across-hltos/output create mode 100644 testing/btest/Baseline/spicy.ssh-banner/analyzer.log create mode 100644 testing/btest/Baseline/spicy.ssh-banner/output create mode 100644 testing/btest/Baseline/spicy.ssh-banner/weird.log create mode 100644 testing/btest/Baseline/spicy.terminate-session/conn.log create mode 100644 testing/btest/Baseline/spicy.toggle-protocol-analyzer/output create mode 100644 testing/btest/Baseline/spicy.tuple-arg/output create mode 100644 testing/btest/Baseline/spicy.tuple-enum/output create mode 100644 testing/btest/Baseline/spicy.tuple-optional/output create mode 100644 testing/btest/Baseline/spicy.type-converter/output create mode 100644 testing/btest/Baseline/spicy.udp/output create mode 100644 testing/btest/Traces/dns/long-connection.pcap create mode 100644 testing/btest/Traces/dns/proto255.pcap create mode 100644 testing/btest/Traces/pe/pe-single.trace create mode 100644 testing/btest/Traces/spicy/gap-recovery.pcap create mode 100644 testing/btest/Traces/spicy/packet-analyzer-violation.pcap create mode 100644 testing/btest/Traces/spicy/raw-layer.pcap create mode 100644 testing/btest/Traces/ssh/single-conn.trace create mode 100644 testing/btest/Traces/ssh/ssh-over-udp.pcap create mode 100644 testing/btest/Traces/udp-packet.pcap create mode 100644 testing/btest/spicy/analyzer-tag.zeek create mode 100644 testing/btest/spicy/availability.zeek create mode 100644 testing/btest/spicy/conn-id.spicy create mode 100644 testing/btest/spicy/context.spicy create mode 100644 testing/btest/spicy/double-event.zeek create mode 100644 testing/btest/spicy/double-type-registration.zeek create mode 100644 testing/btest/spicy/double-types.zeek create mode 100644 testing/btest/spicy/event-args-fail.evt create mode 100644 testing/btest/spicy/event-args-mismatch.zeek create mode 100644 testing/btest/spicy/event-args.zeek create mode 100644 testing/btest/spicy/event-cond.zeek create mode 100644 testing/btest/spicy/event-user-type create mode 100644 testing/btest/spicy/export-enum.zeek create mode 100644 testing/btest/spicy/export-type-fail.spicy create mode 100644 testing/btest/spicy/export-types.zeek create mode 100644 testing/btest/spicy/file-analysis-data-in-concurrent.zeek create mode 100644 testing/btest/spicy/file-analysis-data-in.zeek create mode 100644 testing/btest/spicy/file-analyzer-nested.zeek create mode 100644 testing/btest/spicy/file-analyzer-property.zeek create mode 100644 testing/btest/spicy/file-analyzer.zeek create mode 100644 testing/btest/spicy/file-data-in-at-offset.zeek create mode 100644 testing/btest/spicy/file-replaces.zeek create mode 100644 testing/btest/spicy/gap-recovery.zeek create mode 100644 testing/btest/spicy/import-from.zeek create mode 100644 testing/btest/spicy/list-conversion.zeek create mode 100644 testing/btest/spicy/module-path.spicy create mode 100644 testing/btest/spicy/multiple-enum.zeek create mode 100644 testing/btest/spicy/network-time.spicy create mode 100644 testing/btest/spicy/optional.zeek create mode 100644 testing/btest/spicy/packet-analyzer-on-ip.zeek create mode 100644 testing/btest/spicy/packet-analyzer-replaces.zeek create mode 100644 testing/btest/spicy/packet-analyzer-violation.zeek create mode 100644 testing/btest/spicy/packet-analyzer.zeek create mode 100644 testing/btest/spicy/parse-error.zeek create mode 100644 testing/btest/spicy/port-fail.evt create mode 100644 testing/btest/spicy/preprocessor-fail.evt create mode 100644 testing/btest/spicy/preprocessor-spicy.spicy create mode 100644 testing/btest/spicy/preprocessor.evt create mode 100644 testing/btest/spicy/profiling.zeek create mode 100644 testing/btest/spicy/protocol-analyzer-data-in.zeek create mode 100644 testing/btest/spicy/protocol-analyzer-explicit-forwarding.zeek create mode 100644 testing/btest/spicy/protocol-analyzer-tcp-over-udp.spicy create mode 100644 testing/btest/spicy/replaces-mismatch.zeek create mode 100644 testing/btest/spicy/replaces.zeek create mode 100644 testing/btest/spicy/resource-usage.zeek create mode 100644 testing/btest/spicy/reuse-parser-across-hltos.zeek create mode 100644 testing/btest/spicy/ssh-banner.zeek create mode 100644 testing/btest/spicy/terminate-session.zeek create mode 100644 testing/btest/spicy/toggle-protocol-analyzer.zeek create mode 100644 testing/btest/spicy/tuple-arg.zeek create mode 100644 testing/btest/spicy/tuple-enum.zeek create mode 100644 testing/btest/spicy/type-converter.zeek create mode 100644 testing/btest/spicy/udp.zeek create mode 100755 testing/scripts/diff-canonifier-spicy create mode 100755 testing/scripts/spicy-version create mode 100644 testing/scripts/spicy/README create mode 100755 testing/scripts/spicy/canonify-zeek-log create mode 100755 testing/scripts/spicy/canonify-zeek-log-sorted create mode 100755 testing/scripts/spicy/diff-remove-abspath create mode 100755 testing/scripts/spicy/diff-remove-timestamps create mode 100755 testing/scripts/spicy/diff-sort create mode 100755 testing/scripts/spicy/run-zeek create mode 100755 testing/scripts/spicy/spicy-version create mode 100755 testing/scripts/spicy/zeek-version diff --git a/.gitmodules b/.gitmodules index e971a79201..2df489a3b6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -58,9 +58,6 @@ [submodule "auxil/out_ptr"] path = auxil/out_ptr url = https://github.com/soasis/out_ptr.git -[submodule "auxil/spicy-plugin"] - path = auxil/spicy-plugin - url = https://github.com/zeek/spicy-plugin [submodule "auxil/spicy"] path = auxil/spicy/spicy url = https://github.com/zeek/spicy diff --git a/CMakeLists.txt b/CMakeLists.txt index 6a4830114a..89e9522e90 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -497,17 +497,19 @@ file( WRITE ${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev.sh "export ZEEKPATH=`${cmake_binary_dir}/zeek-path-dev`\n" "export ZEEK_PLUGIN_PATH=\"${cmake_binary_dir}/src\":$\{ZEEK_PLUGIN_PATH\}\n" - "export PATH=\"${cmake_binary_dir}\":\"${cmake_binary_dir}/src\":\"${cmake_binary_dir}/auxil/spicy/spicy/bin\":\"${cmake_binary_dir}/src/builtin-plugins/spicy-plugin/bin\":$\{PATH\}\n" + "export PATH=\"${cmake_binary_dir}\":\"${cmake_binary_dir}/src\":\"${cmake_binary_dir}/auxil/spicy/spicy/bin\":\"${cmake_binary_dir}/src/spicy/spicyz\":$\{PATH\}\n" "export SPICY_PATH=`${cmake_binary_dir}/spicy-path`\n" - "export HILTI_CXX_INCLUDE_DIRS=`${cmake_binary_dir}/hilti-cxx-include-dirs`\n") + "export HILTI_CXX_INCLUDE_DIRS=`${cmake_binary_dir}/hilti-cxx-include-dirs`\n" + "export ZEEK_SPICY_LIBRARY_PATH=${cmake_source_dir}/scripts/spicy\n") file( WRITE ${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev.csh "setenv ZEEKPATH `${cmake_binary_dir}/zeek-path-dev`\n" "setenv ZEEK_PLUGIN_PATH \"${cmake_binary_dir}/src\":$\{ZEEK_PLUGIN_PATH\}\n" - "setenv PATH \"${cmake_binary_dir}\":\"${cmake_binary_dir}/src\":\"${cmake_binary_dir}/auxil/spicy/spicy/bin\":\"${cmake_binary_dir}/src/builtin-plugins/spicy-plugin/bin\":$\{PATH\}\n" + "setenv PATH \"${cmake_binary_dir}\":\"${cmake_binary_dir}/src\":\"${cmake_binary_dir}/auxil/spicy/spicy/bin\":\"${cmake_binary_dir}/src/spicy/spicyz\":$\{PATH\}\n" "setenv SPICY_PATH \"`${cmake_binary_dir}/spicy-path`\"\n" - "setenv HILTI_CXX_INCLUDE_DIRS \"`${cmake_binary_dir}/hilti-cxx-include-dirs`\"\n") + "setenv HILTI_CXX_INCLUDE_DIRS \"`${cmake_binary_dir}/hilti-cxx-include-dirs`\"\n" + "setenv ZEEK_SPICY_LIBRARY_PATH \"${cmake_source_dir}/scripts/spicy\"\n") file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1) execute_process( @@ -887,53 +889,28 @@ else () endif () if (NOT DISABLE_SPICY) - set(USE_SPICY_ANALYZERS yes) + if (SPICY_ROOT_DIR) + find_package(Spicy REQUIRED) # will set HAVE_SPICY + spicy_require_version("1.8.0") - if (NOT SPICY_ROOT_DIR) - set(HAVE_SPICY yes) # evaluated by Spicy plugin build + if (NOT SPICY_HAVE_TOOLCHAIN) + message(FATAL_ERROR "Spicy not built with toolchain support") + endif () + spicy_print_summary() + else () + set(HAVE_SPICY yes) add_subdirectory(auxil/spicy) + zeek_add_dependencies(spicy) - # Set variables used by the spicy-plugin build since we are building Spicy - # as part of Zeek so spicy-plugin cannot use `spicy-config` at configure - # time to set these. - set(SPICY_CONFIG "") - set(SPICY_HAVE_TOOLCHAIN "YES") - set(SPICY_INCLUDE_DIRS_RUNTIME - ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy/hilti/runtime/include - ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy/spicy/runtime/include - ${PROJECT_BINARY_DIR}/auxil/spicy/spicy/include) - set(SPICY_INCLUDE_DIRS_TOOLCHAIN - ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy/hilti/toolchain/include - ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy/spicy/toolchain/include) - set(SPICY_LIBRARY spicy) - set(HILTI_LIBRARY_RT hilti-rt) - set(HILTI_LIBRARY_RT_DEBUG hilti-rt-debug) - set(SPICY_LIBRARY_RT spicy-rt) - set(SPICY_LIBRARY_RT_DEBUG spicy-rt-debug) - - # Needed only for logging from CMake configure phase. - get_directory_property(SPICY_VERSION DIRECTORY ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy - DEFINITION SPICY_VERSION) get_directory_property( SPICY_VERSION_NUMBER DIRECTORY ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy DEFINITION SPICY_VERSION_NUMBER) - get_directory_property(SPICY_PREFIX DIRECTORY ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy - DEFINITION CMAKE_INSTALL_PREFIX) - get_directory_property(SPICY_BUILD_MODE DIRECTORY ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy - DEFINITION CMAKE_BUILD_TYPE) - set(SPICYC "") endif () - if (NOT SPICY_PLUGIN_PATH) - set(_spicy_plugin "included") - set(SPICY_PLUGIN_PATH ${CMAKE_SOURCE_DIR}/auxil/spicy-plugin) - endif () - - set(SPICY_PLUGIN_BINARY_PATH ${CMAKE_BINARY_DIR}/src/builtin-plugins/spicy-plugin) - list(APPEND ZEEK_INCLUDE_PLUGINS ${SPICY_PLUGIN_PATH}) + set(USE_SPICY_ANALYZERS yes) else () - set(HAVE_SPICY no) # evaluated by Spicy plugin build + set(HAVE_SPICY no) set(USE_SPICY_ANALYZERS no) endif () @@ -1255,6 +1232,10 @@ add_subdirectory(scripts) add_subdirectory(man) add_subdirectory(testing) +if (NOT DISABLE_SPICY) + zeek_add_dependencies(spicyz) +endif () + include(CheckOptionalBuildSources) checkoptionalbuildsources(auxil/btest BTest INSTALL_BTEST) @@ -1264,26 +1245,6 @@ checkoptionalbuildsources(auxil/zeek-aux Zeek-Aux INSTALL_AUX_TOOLS) checkoptionalbuildsources(auxil/zeek-archiver ZeekArchiver INSTALL_ZEEK_ARCHIVER) checkoptionalbuildsources(auxil/zeek-client ZeekClient INSTALL_ZEEK_CLIENT) -if (NOT DISABLE_SPICY) - # The `zeek` binary implicitly depends on the driver object file built as part - # of `spicy`; make that dependency explicit. - zeek_add_dependencies(spicyz) - - if (NOT SPICY_ROOT_DIR) - # Make sure we build targets of spicy-plugin after the `spicy` target. - add_dependencies(plugin-Zeek-Spicy spicy) - add_dependencies(spicyz spicy) - - # Also install spicy-plugin's CMake files into Zeek's global `cmake/` - # folder. - # - # NOTE: We do not install spicy-plugin's `FindZeek.cmake` since another - # version of this file is already provided by Zeek. - install(FILES auxil/spicy-plugin/cmake/ZeekSpicyAnalyzerSupport.cmake - auxil/spicy-plugin/cmake/FindSpicy.cmake DESTINATION share/zeek/cmake) - endif () -endif () - # Always generate helper scripts referenced in e.g., `zeek-path-dev.*` so the # scripts work in any build configuration. If we do not include Spicy these # files have no actual effect. @@ -1295,6 +1256,8 @@ endif () # we generate shell definitions to support running and using Spicy or # spicy-plugin functionality in the build tree, including JIT'ing directly from # Zeek. +# +# TODO: Do we still need these? configure_file(${CMAKE_SOURCE_DIR}/auxil/spicy/spicy-path.in ${CMAKE_BINARY_DIR}/spicy-path @ONLY) configure_file(${CMAKE_SOURCE_DIR}/auxil/spicy/hilti-cxx-include-dirs.in ${CMAKE_BINARY_DIR}/hilti-cxx-include-dirs @ONLY) @@ -1375,12 +1338,6 @@ elseif (SPICY_ROOT_DIR) set(_spicy "external (${SPICY_ROOT_DIR})") endif () -if (DISABLE_SPICY) - set(_spicy_plugin "disabled") -elseif ("${_spicy_plugin}" STREQUAL "") - set(_spicy_plugin "external (${SPICY_PLUGIN_PATH})") -endif () - if (ZEEK_LEGACY_ANALYZERS) list(JOIN ZEEK_LEGACY_ANALYZERS ", " _legacy_analyzers) set(_legacy_analyzers @@ -1394,7 +1351,7 @@ endif () if (ZEEK_LEGACY_ANALYZERS OR ZEEK_SKIPPED_ANALYZERS) set(_analyzer_warning - "\n\n[Warning] Some analyzers are not available due to lack of built-in Spicy support:${_legacy_analyzers}${_skipped_analyzers}" + "\n\n[Warning] Some analyzers are not available due to lack of Spicy support:${_legacy_analyzers}${_skipped_analyzers}" ) endif () @@ -1412,6 +1369,7 @@ message( "\nScript dir: ${ZEEK_SCRIPT_INSTALL_PATH}" "\nSpool dir: ${ZEEK_SPOOL_DIR}" "\nState dir: ${ZEEK_STATE_DIR}" + "\nSpicy modules dir: ${ZEEK_SPICY_MODULE_PATH}" "\n" "\nDebug mode: ${ENABLE_DEBUG}" "\nUnit tests: ${ENABLE_ZEEK_UNIT_TESTS}" @@ -1433,7 +1391,6 @@ message( "\nGen-ZAM: ${_gen_zam_exe_path}" "\nzkg: ${INSTALL_ZKG}" "\nSpicy: ${_spicy}" - "\nSpicy plugin: ${_spicy_plugin}" "\nSpicy analyzers: ${USE_SPICY_ANALYZERS}" "\nJavaScript: ${ZEEK_HAVE_JAVASCRIPT}" "\n" diff --git a/auxil/spicy-plugin b/auxil/spicy-plugin deleted file mode 160000 index a618f2ce08..0000000000 --- a/auxil/spicy-plugin +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a618f2ce0831c311f9bcff5d020b85fc44345221 diff --git a/auxil/spicy/spicy b/auxil/spicy/spicy index 64de5d0ef3..ec87b43037 160000 --- a/auxil/spicy/spicy +++ b/auxil/spicy/spicy @@ -1 +1 @@ -Subproject commit 64de5d0ef323428452827469f07b0a1da8e65e16 +Subproject commit ec87b43037dba50648cb93be8940a4db23658905 diff --git a/cmake b/cmake index a90d691796..ac0529be3d 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit a90d69179607c5083158f926be6d37f3db18f110 +Subproject commit ac0529be3d1e14eba426c2e516c4a9c9971be40b diff --git a/configure b/configure index 98ce48c08c..25bde692b6 100755 --- a/configure +++ b/configure @@ -104,7 +104,6 @@ Usage: $0 [OPTION]... [VAR=VALUE]... --with-python-inc=PATH path to Python headers --with-python-lib=PATH path to libpython --with-spicy=PATH path to Spicy install root - --with-spicy-plugin=PATH path to Spicy plugin source tree --with-swig=PATH path to SWIG executable Packaging Options (for developers): @@ -378,9 +377,6 @@ while [ $# -ne 0 ]; do --with-spicy=*) append_cache_entry SPICY_ROOT_DIR PATH $optarg ;; - --with-spicy-plugin=*) - append_cache_entry SPICY_PLUGIN_PATH PATH $optarg - ;; --with-swig=*) append_cache_entry SWIG_EXECUTABLE PATH $optarg ;; diff --git a/scripts/base/frameworks/spicy/__load__.zeek b/scripts/base/frameworks/spicy/__load__.zeek new file mode 100644 index 0000000000..5f39cb27df --- /dev/null +++ b/scripts/base/frameworks/spicy/__load__.zeek @@ -0,0 +1 @@ +@load ./main.zeek diff --git a/scripts/base/frameworks/spicy/init-bare.zeek b/scripts/base/frameworks/spicy/init-bare.zeek new file mode 100644 index 0000000000..1dcef1c710 --- /dev/null +++ b/scripts/base/frameworks/spicy/init-bare.zeek @@ -0,0 +1,37 @@ + +module Spicy; + +export { +# doc-options-start + ## Constant for testing if Spicy is available. + const available = T; + + ## Show output of Spicy print statements. + const enable_print = F &redef; + + # Record and display profiling information, if compiled into analyzer. + const enable_profiling = F &redef; + + ## abort() instead of throwing HILTI exceptions. + const abort_on_exceptions = F &redef; + + ## Include backtraces when reporting unhandled exceptions. + const show_backtraces = F &redef; + + ## Maximum depth of recursive file analysis (Spicy analyzers only) + const max_file_depth: count = 5 &redef; +# doc-options-end + +# doc-types-start + ## Result type for `Spicy::resource_usage()`. + type ResourceUsage: record { + user_time : interval; ##< user CPU time of the Zeek process + system_time :interval; ##< system CPU time of the Zeek process + memory_heap : count; ##< memory allocated on the heap by the Zeek process + num_fibers : count; ##< number of fibers currently in use + max_fibers: count; ##< maximum number of fibers ever in use + max_fiber_stack_size: count; ##< maximum fiber stack size ever in use + cached_fibers: count; ##< number of fibers currently cached + }; +# doc-types-end +} diff --git a/scripts/base/frameworks/spicy/init-framework.zeek b/scripts/base/frameworks/spicy/init-framework.zeek new file mode 100644 index 0000000000..886b0f4170 --- /dev/null +++ b/scripts/base/frameworks/spicy/init-framework.zeek @@ -0,0 +1,81 @@ +@load base/misc/version + +# doc-common-start +module Spicy; + +export { +# doc-functions-start + ## Enable a specific Spicy protocol analyzer if not already active. If this + ## analyzer replaces an standard analyzer, that one will automatically be + ## disabled. + ## + ## tag: analyzer to toggle + ## + ## Returns: true if the operation succeeded + global enable_protocol_analyzer: function(tag: Analyzer::Tag) : bool; + + ## Disable a specific Spicy protocol analyzer if not already inactive. If + ## this analyzer replaces an standard analyzer, that one will automatically + ## be re-enabled. + ## + ## tag: analyzer to toggle + ## + ## Returns: true if the operation succeeded + global disable_protocol_analyzer: function(tag: Analyzer::Tag) : bool; + + + ## Enable a specific Spicy file analyzer if not already active. If this + ## analyzer replaces an standard analyzer, that one will automatically be + ## disabled. + ## + ## tag: analyzer to toggle + ## + ## Returns: true if the operation succeeded + global enable_file_analyzer: function(tag: Files::Tag) : bool; + + ## Disable a specific Spicy file analyzer if not already inactive. If + ## this analyzer replaces an standard analyzer, that one will automatically + ## be re-enabled. + ## + ## tag: analyzer to toggle + ## + ## Returns: true if the operation succeeded + global disable_file_analyzer: function(tag: Files::Tag) : bool; + + ## Returns current resource usage as reported by the Spicy runtime system. + global resource_usage: function() : ResourceUsage; +# doc-functions-end +} + +# Marked with &is_used to suppress complaints when there aren't any +# Spicy file analyzers loaded, and hence this event can't be generated. +# The attribute is only supported for Zeek 5.0 and higher. +event spicy_analyzer_for_mime_type(a: Files::Tag, mt: string) &is_used + { + Files::register_for_mime_type(a, mt); + } + +function enable_protocol_analyzer(tag: Analyzer::Tag) : bool + { + return Spicy::__toggle_analyzer(tag, T); + } + +function disable_protocol_analyzer(tag: Analyzer::Tag) : bool + { + return Spicy::__toggle_analyzer(tag, F); + } + +function enable_file_analyzer(tag: Files::Tag) : bool + { + return Spicy::__toggle_analyzer(tag, T); + } + +function disable_file_analyzer(tag: Files::Tag) : bool + { + return Spicy::__toggle_analyzer(tag, F); + } + +function resource_usage() : ResourceUsage + { + return Spicy::__resource_usage(); + } diff --git a/scripts/base/frameworks/spicy/main.zeek b/scripts/base/frameworks/spicy/main.zeek new file mode 100644 index 0000000000..6c7684c47b --- /dev/null +++ b/scripts/base/frameworks/spicy/main.zeek @@ -0,0 +1,15 @@ +@load base/frameworks/notice + +module Spicy; + +export { + redef enum Notice::Type += { Spicy_Max_File_Depth_Exceeded }; +} + +event max_file_depth_exceeded(f: fa_file, args: Files::AnalyzerArgs, limit: count) + { + NOTICE([ + $note=Spicy::Spicy_Max_File_Depth_Exceeded, + $msg=fmt("Maximum file depth exceeded for file %s", f$id) + ]); + } diff --git a/scripts/base/frameworks/spicy/misc/record-spicy-batch.zeek b/scripts/base/frameworks/spicy/misc/record-spicy-batch.zeek new file mode 100644 index 0000000000..3955077d38 --- /dev/null +++ b/scripts/base/frameworks/spicy/misc/record-spicy-batch.zeek @@ -0,0 +1,90 @@ +# Saves all input traffic in Spicy's batch format. + +module SpicyBatch; + +export { + const filename = "batch.dat" &redef; +} + +redef tcp_content_deliver_all_orig=T; +redef tcp_content_deliver_all_resp=T; +redef udp_content_deliver_all_orig=T; +redef udp_content_deliver_all_resp=T; + +global output: file; +global conns: set[conn_id]; +global num_conns = 0; + +function id(c: connection) : string + { + local cid = c$id; + local proto = "???"; + + if ( is_tcp_port(cid$orig_p) ) + proto = "tcp"; + else if ( is_udp_port(cid$orig_p) ) + proto = "udp"; + else if ( is_icmp_port(cid$orig_p) ) + proto = "icmp"; + + return fmt("%s-%d-%s-%d-%s", cid$orig_h, cid$orig_p, cid$resp_h, cid$resp_p, proto); + } + +function begin(c: connection, type_: string) + { + add conns[c$id]; + ++num_conns; + print fmt("tracking %s", c$id); + + local id_ = id(c); + print output, fmt("@begin-conn %s %s %s-orig %s%%orig %s-resp %s%%resp\n", id_, type_, id_, c$id$resp_p, id_, c$id$resp_p); + } + +event zeek_init() + { + output = open(filename); + enable_raw_output(output); + print output, "!spicy-batch v2\n"; + } + +event new_connection_contents(c: connection) + { + begin(c, "stream"); + } + +event tcp_contents(c: connection, is_orig: bool, seq: count, contents: string) + { + print output, fmt("@data %s-%s %d\n", id(c), (is_orig ? "orig" : "resp"), |contents|); + print output, contents; + print output, "\n"; + } + +event content_gap(c: connection, is_orig: bool, seq: count, length: count) + { + print output, fmt("@gap %s-%s %d\n", id(c), (is_orig ? "orig" : "resp"), length); + } + +event udp_contents(c: connection, is_orig: bool, contents: string) + { + if ( c$id !in conns ) + begin(c, "block"); + + print output, fmt("@data %s-%s %d\n", id(c), (is_orig ? "orig" : "resp"), |contents|); + print output, contents; + print output, "\n"; + } + +event connection_state_remove(c: connection) + { + if ( c$id !in conns ) + return; + + print output, fmt("@end-conn %s\n", id(c)); + } + +event zeek_done() + { + close(output); + print fmt("recorded %d session%s total", num_conns, (num_conns > 1 ? "s" : "")); + print fmt("output in %s", filename); + } diff --git a/scripts/base/frameworks/spicy/misc/resource-usage.zeek b/scripts/base/frameworks/spicy/misc/resource-usage.zeek new file mode 100644 index 0000000000..38a2e87f3b --- /dev/null +++ b/scripts/base/frameworks/spicy/misc/resource-usage.zeek @@ -0,0 +1,18 @@ +module Spicy; + +event print_usage() + { + local r = Spicy::resource_usage(); + + print fmt("%.6f Spicy user=%f sys=%f heap=%d current_fibers=%d cached_fibers=%d max_fibers=%d max_stack=%d", + network_time(), r$user_time, r$system_time, r$memory_heap, + r$num_fibers, r$cached_fibers, r$max_fibers, + r$max_fiber_stack_size); + + schedule 1 min { print_usage() }; + } + +event zeek_init() + { + schedule 1 min { print_usage() }; + } diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index b044892ed8..7f5b0da460 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -5702,3 +5702,4 @@ event net_done(t: time) @endif @load base/packet-protocols +@load base/frameworks/spicy/init-bare diff --git a/scripts/base/init-default.zeek b/scripts/base/init-default.zeek index 5a11969e4e..93a29bf323 100644 --- a/scripts/base/init-default.zeek +++ b/scripts/base/init-default.zeek @@ -43,6 +43,7 @@ @load base/frameworks/openflow @load base/frameworks/netcontrol @load base/frameworks/telemetry +@load base/frameworks/spicy @load base/protocols/conn @load base/protocols/dce-rpc diff --git a/scripts/base/init-frameworks-and-bifs.zeek b/scripts/base/init-frameworks-and-bifs.zeek index a346f9543c..a983c1e453 100644 --- a/scripts/base/init-frameworks-and-bifs.zeek +++ b/scripts/base/init-frameworks-and-bifs.zeek @@ -17,6 +17,8 @@ # Load BiFs defined by plugins. @load base/bif/plugins +@load base/frameworks/spicy/init-framework + # This sets up secondary/subdir BIFs such that they can be used by any # further scripts within their global initializations and is intended to be # the last thing done within this script. It's called within @if simply so diff --git a/scripts/spicy/zeek.spicy b/scripts/spicy/zeek.spicy new file mode 100644 index 0000000000..4f4467008d --- /dev/null +++ b/scripts/spicy/zeek.spicy @@ -0,0 +1,157 @@ +# Copyright (c) 2020-2021 by the Zeek Project. See LICENSE for details. + +module zeek; + +# Note: Retain the formatting here, doc/scripts/autogen-spicy-lib is picking up on that. + +%cxx-include = "zeek/spicy/runtime-support.h"; + +## [Deprecated] Triggers a DPD protocol confirmation for the current connection. +## +## This function has been deprecated and will be removed. Use ``spicy::accept_input`` +## instead, which will have the same effect with Zeek. +public function confirm_protocol() : void &cxxname="zeek::spicy::rt::confirm_protocol"; + +## [Deprecated] Triggers a DPD protocol violation for the current connection. +## +## This function has been deprecated and will be removed. Use ``spicy::decline_input`` +## instead, which will have the same effect with Zeek. +public function reject_protocol(reason: string) : void &cxxname="zeek::spicy::rt::reject_protocol"; + +## Reports a "weird" to Zeek. This should be used with similar semantics as in +## Zeek: something quite unexpected happening at the protocol level, which however +## does not prevent us from continuing to process the connection. +## +## id: the name of the weird, which (just like in Zeek) should be a *static* +## string identifying the situation reported (e.g., ``unexpected_command``). +## +## addl: additional information to record along with the weird +public function weird(id: string, addl: string = "") &cxxname="zeek::spicy::rt::weird"; + +## Returns true if we're currently parsing the originator side of a connection. +public function is_orig() : bool &cxxname="zeek::spicy::rt::is_orig"; + +## Returns the current connection's UID. +public function uid() : string &cxxname="zeek::spicy::rt::uid"; + +## Returns the current connection's 4-tuple ID. +public function conn_id() : tuple &cxxname="zeek::spicy::rt::conn_id"; + +## Instructs Zeek to flip the directionality of the current connection. +public function flip_roles() : void &cxxname="zeek::spicy::rt::flip_roles"; + +## Returns the number of packets seen so far on the current side of the current connection. +public function number_packets() : uint64 &cxxname="zeek::spicy::rt::number_packets"; + +## Opaque handle to a protocol analyzer. +public type ProtocolHandle = __library_type("zeek::spicy::rt::ProtocolHandle"); + +## Adds a Zeek-side child protocol analyzer to the current connection. +## +## If the same analyzer was added previously with protocol_handle_get_or_create or +## protocol_begin with same argument, and not closed with protocol_handle_close +## or protocol_end, no new analyzer will be added. +## +## See `protocol_handle_get_or_create` for the error semantics of this function. +## +## analyzer: type of analyzer to instantiate, specified through its Zeek-side +## name (similar to what Zeek's signature action `enable` takes); if not +## specified, Zeek will perform its usual dynamic protocol detection to figure +## out how to parse the data (the latter will work only for TCP protocols, though.) +public function protocol_begin(analyzer: optional = Null) : void &cxxname="zeek::spicy::rt::protocol_begin"; + +## Gets a handle to a Zeek-side child protocol analyzer for the current connection. +## +## If no such child exists it will be added; otherwise a handle to the +## existing child protocol analyzer will be returned. +## +## This function will return an error +## +## - if not called from a protocol analyzer, or +## - the requested child protocol analyzer is unknown, or +## - creation of a child analyzer of the requested type was prevented by a +## previous call of `disable_analyzer` with `prevent=T` +## +## analyzer: type of analyzer to instantiate, specified through its Zeek-side +## name (similar to what Zeek's signature action `enable` takes). +public function protocol_handle_get_or_create(analyzer: string) : ProtocolHandle &cxxname="zeek::spicy::rt::protocol_handle_get_or_create"; + +## Forwards protocol data to all previously instantiated Zeek-side child protocol analyzers. +## +## is_orig: true to feed the data to the child's originator side, false for the responder +## data: chunk of data to forward to child analyzer +## h: optional handle to the child analyzer to forward data into, else forward to all child analyzers +public function protocol_data_in(is_orig: bool, data: bytes, h: optional = Null) : void &cxxname="zeek::spicy::rt::protocol_data_in"; + +## Signals a gap in input data to all previously instantiated Zeek-side child protocol analyzers. +## +## is_orig: true to signal gap to the child's originator side, false for the responder +## offset: start offset of gap in input stream +## len: size of gap +## h: optional handle to the child analyzer signal a gap to, else signal to all child analyzers +public function protocol_gap(is_orig: bool, offset: uint64, len: uint64, h: optional = Null) : void &cxxname="zeek::spicy::rt::protocol_gap"; + +## Signals end-of-data to all previously instantiated Zeek-side child protocol +## analyzers and removes them. +public function protocol_end() : void &cxxname="zeek::spicy::rt::protocol_end"; + +## Signals end-of-data to the given child analyzer and removes it. +## +## The given handle must be live, i.e., it must not have been used in a +## previous protocol_handle_close call, and must not have been live when +## protocol_end was called. If the handle is not live a runtime error will +## be triggered. +## +## handle: handle to the child analyzer to remove +public function protocol_handle_close(handle: ProtocolHandle): void &cxxname="zeek::spicy::rt::protocol_handle_close"; + +## Signals the beginning of a file to Zeek's file analysis, associating it with the current connection. +## Optionally, a mime type can be provided. It will be passed on to Zeek's file analysis framework. +## Returns the Zeek-side file ID of the new file. +public function file_begin(mime_type: optional = Null) : string &cxxname="zeek::spicy::rt::file_begin"; + +## Returns the current file's FUID. +public function fuid() : string &cxxname="zeek::spicy::rt::fuid"; + +## Terminates the currently active Zeek-side session, flushing all state. Any +## subsequent activity will start a new session from scratch. This can only be +## called from inside a protocol analyzer. +public function terminate_session() : void &cxxname="zeek::spicy::rt::terminate_session"; + +## Signals the expected size of a file to Zeek's file analysis. +## +## size: expected size of file +## fid: Zeek-side ID of the file to operate on; if not given, the file started by the most recent file_begin() will be used +public function file_set_size(size: uint64, fid: optional = Null) : void &cxxname="zeek::spicy::rt::file_set_size"; + +## Passes file content on to Zeek's file analysis. +## +## data: chunk of raw data to pass into analysis +## fid: Zeek-side ID of the file to operate on; if not given, the file started by the most recent file_begin() will be used +public function file_data_in(data: bytes, fid: optional = Null) : void &cxxname="zeek::spicy::rt::file_data_in"; + +## Passes file content at a specific offset on to Zeek's file analysis. +## +## data: chunk of raw data to pass into analysis +## offset: position in file where data starts +## fid: Zeek-side ID of the file to operate on; if not given, the file started by the most recent file_begin() will be used +public function file_data_in_at_offset(data: bytes, offset: uint64, fid: optional = Null) : void &cxxname="zeek::spicy::rt::file_data_in_at_offset"; + +## Signals a gap in a file to Zeek's file analysis. +## +## offset: position in file where gap starts +## len: size of gap +## fid: Zeek-side ID of the file to operate on; if not given, the file started by the most recent file_begin() will be used +public function file_gap(offset: uint64, len: uint64, fid: optional = Null) : void &cxxname="zeek::spicy::rt::file_gap"; + +## Signals the end of a file to Zeek's file analysis. +## +## fid: Zeek-side ID of the file to operate on; if not given, the file started by the most recent file_begin() will be used +public function file_end(fid: optional = Null) : void &cxxname="zeek::spicy::rt::file_end"; + +## Inside a packet analyzer, forwards what data remains after parsing the top-level unit +## on to another analyzer. The index specifies the target, per the current dispatcher table. +public function forward_packet(identifier: uint32) : void &cxxname="zeek::spicy::rt::forward_packet"; + +## Gets the network time from Zeek. +public function network_time() : time &cxxname="zeek::spicy::rt::network_time"; diff --git a/scripts/spicy/zeek_file.spicy b/scripts/spicy/zeek_file.spicy new file mode 100644 index 0000000000..71082f9be9 --- /dev/null +++ b/scripts/spicy/zeek_file.spicy @@ -0,0 +1,31 @@ +# Copyright (c) 2020-2021 by the Zeek Project. See LICENSE for details. +# +# TODO: This code would ideally just be a part of zeek.spicy, but that would +# come with compilation overhead currently even if not used, see +# https://github.com/zeek/spicy/issues/301. + +module zeek_file; + +import zeek; + +## Convenience wrapper for passing content into Zeek's file analysis. +## After connecting an instance of this unit type to a sink, all data +## sent to the sink will be passed on to Zeek as a file. +## +## mime_type: MIME type of the file's content, if known; will be passed on to Zeek +## size: Total number of bytes the file contains, if known; will be passed on to Zeek +public type File = unit(mime_type: optional = Null, size: optional = Null) { + on %init { + self.fid = zeek::file_begin(mime_type); + + if ( size ) + zeek::file_set_size(*size, self.fid); + } + + : bytes &chunked &eod { zeek::file_data_in($$, self.fid); } + + on %finally { zeek::file_end(self.fid); } + + ## Zeek-side file ID + var fid: string; +}; diff --git a/scripts/spicy/zeek_rt.hlt b/scripts/spicy/zeek_rt.hlt new file mode 100644 index 0000000000..75386de9e7 --- /dev/null +++ b/scripts/spicy/zeek_rt.hlt @@ -0,0 +1,43 @@ + +module zeek_rt { + +import hilti; + +%cxx-include = "zeek/spicy/runtime-support.h"; + +public type Val = __library_type("::zeek::ValPtr"); +public type BroType = __library_type("::zeek::TypePtr"); +public type EventHandlerPtr = __library_type("::zeek::EventHandlerPtr"); + +type ZeekTypeTag = enum { + Addr, Any, Bool, Count, Double, Enum, Error, File, Func, Int, Interval, List, Opaque, Pattern, Port, Record, String, Subnet, Table, Time, Type, Vector, Void +} &cxxname="::zeek::spicy::rt::ZeekTypeTag"; + +declare public void register_protocol_analyzer(string name, hilti::Protocol protocol, vector ports, string parser_orig, string parser_resp, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_protocol_analyzer" &have_prototype; +declare public void register_file_analyzer(string name, vector mime_types, string parser, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_file_analyzer" &have_prototype; +declare public void register_packet_analyzer(string name, string parser, string replaces, string linker_scope) &cxxname="zeek::spicy::rt::register_packet_analyzer" &have_prototype; +declare public void register_type(string ns, string id, BroType t) &cxxname="zeek::spicy::rt::register_type" &have_prototype; + +declare public bool have_handler(EventHandlerPtr handler) &cxxname="zeek::spicy::rt::have_handler" &have_prototype; +declare public EventHandlerPtr internal_handler(string event) &cxxname="zeek::spicy::rt::internal_handler" &have_prototype; +declare public void install_handler(string event) &cxxname="zeek::spicy::rt::install_handler" &have_prototype; + +declare public void raise_event(EventHandlerPtr handler, vector args) &cxxname="zeek::spicy::rt::raise_event" &have_prototype; +declare public BroType event_arg_type(EventHandlerPtr handler, uint<64> idx) &cxxname="zeek::spicy::rt::event_arg_type" &have_prototype; +declare public Val to_val(any x, BroType target) &cxxname="zeek::spicy::rt::to_val" &have_prototype; + +type RecordField = tuple; # (ID, type, optional) +declare public BroType create_base_type(ZeekTypeTag tag) &cxxname="zeek::spicy::rt::create_base_type" &have_prototype; +declare public BroType create_enum_type(string ns, string id, vector>> labels) &cxxname="zeek::spicy::rt::create_enum_type" &have_prototype; +declare public BroType create_record_type(string ns, string id, vector fields) &cxxname="zeek::spicy::rt::create_record_type" &have_prototype; +declare public BroType create_table_type(BroType key, optional value = Null) &cxxname="zeek::spicy::rt::create_table_type" &have_prototype; +declare public BroType create_vector_type(BroType elem) &cxxname="zeek::spicy::rt::create_vector_type" &have_prototype; + +declare public Val current_conn() &cxxname="zeek::spicy::rt::current_conn" &have_prototype; +declare public Val current_file() &cxxname="zeek::spicy::rt::current_file" &have_prototype; +declare public Val current_packet() &cxxname="zeek::spicy::rt::current_packet" &have_prototype; +declare public Val current_is_orig() &cxxname="zeek::spicy::rt::current_is_orig" &have_prototype; + +declare public void debug(string msg) &cxxname="zeek::spicy::rt::debug" &have_prototype; + +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e76d0f6703..2c04a778bf 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -172,6 +172,10 @@ add_subdirectory(logging) add_subdirectory(probabilistic) add_subdirectory(session) +if (HAVE_SPICY) + add_subdirectory(spicy) +endif () + # ############################################################################## # Build in the discovered external plugins and create the autogenerated scripts. @@ -512,6 +516,10 @@ add_dependencies(zeek_objs zeek_autogen_files) add_clang_tidy_files(${zeek_SRCS}) zeek_target_link_libraries(zeek_objs) +if (HAVE_SPICY) + target_link_libraries(zeek_objs PRIVATE hilti spicy) +endif () + if (TARGET zeek_exe) target_sources(zeek_exe PRIVATE main.cc ${zeek_HEADERS}) diff --git a/src/DebugLogger.cc b/src/DebugLogger.cc index a7b2a41dfe..4ce647157b 100644 --- a/src/DebugLogger.cc +++ b/src/DebugLogger.cc @@ -22,8 +22,7 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = { {"logging", 0, false}, {"input", 0, false}, {"threading", 0, false}, {"plugins", 0, false}, {"zeekygen", 0, false}, {"pktio", 0, false}, {"broker", 0, false}, {"scripts", 0, false}, {"supervisor", 0, false}, - {"hashkey", 0, false}, -}; + {"hashkey", 0, false}, {"spicy", 0, false}}; DebugLogger::DebugLogger() { diff --git a/src/DebugLogger.h b/src/DebugLogger.h index e8be0a71e2..041e18e407 100644 --- a/src/DebugLogger.h +++ b/src/DebugLogger.h @@ -57,6 +57,7 @@ enum DebugStream DBG_SCRIPTS, // Script initialization DBG_SUPERVISOR, // Process supervisor DBG_HASHKEY, // HashKey buffers + DBG_SPICY, // Spicy functionality NUM_DBGS // Has to be last }; diff --git a/src/plugin/Plugin.h b/src/plugin/Plugin.h index d4f34f28ff..feaf12894f 100644 --- a/src/plugin/Plugin.h +++ b/src/plugin/Plugin.h @@ -809,6 +809,13 @@ public: */ void Describe(ODesc* d) const; + /** + * Registers a component. + * + * @param c The component. The method takes ownership. + */ + void AddComponent(Component* c); + /** * Registers an individual BiF that the plugin defines. The * information is for informational purposes only and will show up in @@ -878,13 +885,6 @@ protected: */ virtual void Done(); - /** - * Registers a component. - * - * @param c The component. The method takes ownership. - */ - void AddComponent(Component* c); - /** * Calls the Initialize() function of all components. */ diff --git a/src/spicy/.clang-format b/src/spicy/.clang-format new file mode 100644 index 0000000000..ec20c65ea9 --- /dev/null +++ b/src/spicy/.clang-format @@ -0,0 +1,132 @@ +--- +Language: Cpp +AccessModifierOffset: -4 +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Right +AlignOperands: true +AlignTrailingComments: true +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: true +AllowShortFunctionsOnASingleLine: true +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: Yes +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: true + IndentBraces: false + SplitEmptyFunction: false + SplitEmptyRecord: false + SplitEmptyNamespace: false +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Custom +BreakBeforeInheritanceComma: false +BreakInheritanceList: BeforeColon +BreakBeforeTernaryOperators: false +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +ColumnLimit: 120 +CommentPragmas: 'NOLINT' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DerivePointerAlignment: false +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IncludeBlocks: Regroup +IncludeCategories: + - Regex: 'hilti/rt.*\.(h|hpp)>' + Priority: 5 + - Regex: 'spicy/rt.*\.(h|hpp)>' + Priority: 6 + - Regex: 'hilti.*\.(h|hpp)>' + Priority: 7 + - Regex: 'spicy.*\.(h|hpp)>' + Priority: 8 + - Regex: '' + Priority: 2 + - Regex: '^<.*\.h>' + Priority: 1 + - Regex: '.*' + Priority: 11 +IncludeIsMainRegex: '$' +IndentCaseLabels: true +IndentPPDirectives: None +IndentWidth: 4 +IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '^BEGIN_' +MacroBlockEnd: '^END_' +MaxEmptyLinesToKeep: 2 +NamespaceIndentation: None +ObjCBinPackProtocolList: Auto +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 500 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 1000 +PointerAlignment: Left +ReflowComments: true +SortIncludes: true +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterTemplateKeyword: false +SpaceAfterLogicalNot: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 1 +SpacesInAngles: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +SpacesInConditionalStatement: true +Standard: Cpp11 +StatementMacros: + - STANDARD_OPERATOR_1 +TabWidth: 4 +UseTab: Never +... diff --git a/src/spicy/CMakeLists.txt b/src/spicy/CMakeLists.txt new file mode 100644 index 0000000000..101c048159 --- /dev/null +++ b/src/spicy/CMakeLists.txt @@ -0,0 +1,29 @@ +# See the file "COPYING" in the main distribution directory for copyright. + +add_subdirectory(spicyz) + +zeek_add_subdir_library( + spicy + INTERNAL_DEPENDENCIES + ${BIF_BUILD_TARGET} + INCLUDE_DIRS + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} + SOURCES + manager.cc + file-analyzer.cc + packet-analyzer.cc + protocol-analyzer.cc + runtime-support.cc + BIFS + spicy.bif) + +target_link_libraries(zeek_spicy_obj PRIVATE hilti spicy) + +set(ZEEK_SPICY_MODULE_PATH "${CMAKE_INSTALL_FULL_LIBDIR}/zeek/spicy" CACHE PATH "") +install(DIRECTORY DESTINATION "${ZEEK_SPICY_MODULE_PATH}") + +set(ZEEK_SPICY_LIBRARY_PATH "${CMAKE_INSTALL_FULL_DATADIR}/zeek/spicy" CACHE PATH "") +install(DIRECTORY "${PROJECT_SOURCE_DIR}/scripts/spicy/" DESTINATION "${ZEEK_SPICY_LIBRARY_PATH}") + +set(ZEEK_SPICY_DATA_PATH "${CMAKE_INSTALL_FULL_DATADIR}/zeek" CACHE PATH "") diff --git a/src/spicy/cookie.h b/src/spicy/cookie.h new file mode 100644 index 0000000000..bc59f09a7b --- /dev/null +++ b/src/spicy/cookie.h @@ -0,0 +1,233 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +/** + * Cookie types that's stored in the HILTI context to provide access to the + * current analyzer. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "zeek/Val.h" +#include "zeek/analyzer/Analyzer.h" +#include "zeek/analyzer/protocol/tcp/TCP.h" +#include "zeek/file_analysis/Analyzer.h" +#include "zeek/packet_analysis/Analyzer.h" + +namespace zeek::spicy::rt { + +namespace cookie { + +/** State representing analysis of one file. */ +struct FileState { + FileState(std::string fid) : fid(std::move(fid)) {} + std::string fid; /**< unique Zeek-side file ID */ + std::optional mime_type; /**< MIME type, if explicitly set */ +}; + +/** + * State stored inside protocol/file analyzer cookies retaining file analysis + * state. + * + * Internally, this maintains a stack of state objects representing individual + * files that are currently in-flight. + */ +class FileStateStack { +public: + /** + * Constructor. + * + * @param analyzer_id unique ID string representing parent connection/file analyzer + */ + FileStateStack(std::string analyzer_id) : _analyzer_id(std::move(analyzer_id)) {} + + /** + * Begins analysis for a new file, pushing a new state object onto the + * stack. + */ + FileState* push(); + + /** Returns true if the stack is currently empty. */ + bool isEmpty() const { return _stack.empty(); } + + /** + * Removes an object from the stack. + * + * @param fid ID of file to remove state for; no-op if not found + */ + void remove(const std::string& fid); + + /** + * Returns a pointer to the state of the most recently pushed file. Must not + * be called on an empty stack. + **/ + const FileState* current() const { + assert(_stack.size()); + return &_stack.back(); + } + + /** + * Returns the state of a given file currently on the stack. + * + * @param fid ID of file to find + * @returns pointer to the file's state, or null if not found + */ + const FileState* find(const std::string& fid) const; + +private: + std::vector _stack; // stack of files in flight + std::string _analyzer_id; // unique ID string of parent analyzer, as passed into ctor + uint64_t _id_counter = 0; // counter incremented for each file added to this stack +}; + +/** State on the current protocol analyzer. */ +struct ProtocolAnalyzer { + analyzer::Analyzer* analyzer = nullptr; /**< current analyzer */ + bool is_orig = false; /**< direction of the connection */ + uint64_t num_packets = 0; /**< number of packets seen so far */ + FileStateStack fstate_orig; /**< file analysis state for originator side */ + FileStateStack fstate_resp; /**< file analysis state for responder side */ + std::shared_ptr fake_tcp; /**< fake TPC analyzer created internally */ +}; + +/** State on the current file analyzer. */ +struct FileAnalyzer { + file_analysis::Analyzer* analyzer = nullptr; /**< current analyzer */ + uint64_t depth = 0; /**< recursive depth of file analysis (Spicy-side file analysis only) */ + FileStateStack fstate; /**< file analysis state for nested files */ +}; + +/** State on the current file analyzer. */ +struct PacketAnalyzer { + packet_analysis::Analyzer* analyzer = nullptr; /**< current analyzer */ + Packet* packet = nullptr; /**< current packet */ + ValPtr packet_val = nullptr; /**< cached "raw_pkt_hdr" val for packet */ + std::optional next_analyzer; +}; + +} // namespace cookie + +/** + * Type of state stored in HILTI's execution context during Spicy processing. + * This is optimized for fast access and small size. + */ +struct Cookie { + // Exactly one of these pointers is non-null at any time. In that way, the + // pointers provide the semantics of a tagged union. Internals are bit + // tricky because the union itself cannot be copied/moved. + cookie::ProtocolAnalyzer* protocol = nullptr; + cookie::FileAnalyzer* file = nullptr; + cookie::PacketAnalyzer* packet = nullptr; + + Cookie(cookie::ProtocolAnalyzer&& c) : data(std::move(c)) { protocol = &data.protocol; } + Cookie(cookie::FileAnalyzer&& c) : data(std::move(c)) { file = &data.file; } + Cookie(cookie::PacketAnalyzer&& c) : data(std::move(c)) { packet = &data.packet; } + Cookie(Cookie&& other) noexcept : data(other.tag(), std::move(other.data)) { _initLike(other); } + ~Cookie() { _delete(); } + + Cookie& operator=(Cookie&& other) noexcept { + if ( this == &other ) + return *this; + + _delete(); + _initLike(other); + + new (&data) Data(tag(), std::move(other.data)); + return *this; + } + + // Cache of values that can be expensive to compute. + struct { + ValPtr conn = nullptr; // valid only for protocol analyzers + ValPtr is_orig = nullptr; // valid only for protocol analyzers + bool confirmed = false; // valid only for protocol analyzers; + } cache; + + enum Tag { Protocol, File, Packet }; + + /** Returns the type of cookie currently stored. */ + Tag tag() const { + if ( protocol ) + return Tag::Protocol; + else if ( file ) + return Tag::File; + else if ( packet ) + return Tag::Packet; + else + throw std::runtime_error("invalid cookie"); + } + +private: + union Data { + cookie::ProtocolAnalyzer protocol; + cookie::FileAnalyzer file; + cookie::PacketAnalyzer packet; + + Data(cookie::ProtocolAnalyzer&& protocol) : protocol(std::move(protocol)) {} + Data(cookie::FileAnalyzer&& file) : file(std::move(file)) {} + Data(cookie::PacketAnalyzer&& packet) : packet(std::move(packet)) {} + Data(Tag tag, Data&& other) { + switch ( tag ) { + case Tag::Protocol: new (&protocol) cookie::ProtocolAnalyzer(std::move(other.protocol)); break; + case Tag::File: new (&file) cookie::FileAnalyzer(std::move(other.file)); break; + case Tag::Packet: new (&packet) cookie::PacketAnalyzer(std::move(other.packet)); break; + } + } + + ~Data() { + // don't delete anything, Cookie is in charge. + } + + Data(const Data& other) = delete; + Data& operator=(const Data& other) = delete; + Data& operator=(Data&& other) = delete; + } data; + + void _delete() { + if ( protocol ) { + data.protocol.~ProtocolAnalyzer(); + protocol = nullptr; + cache.conn = nullptr; + cache.is_orig = nullptr; + cache.confirmed = false; + } + else if ( file ) { + data.file.~FileAnalyzer(); + file = nullptr; + } + else if ( packet ) { + data.packet.~PacketAnalyzer(); + packet = nullptr; + } + } + + void _initLike(const Cookie& other) { + if ( other.protocol ) { + protocol = &data.protocol; + cache.confirmed = other.cache.confirmed; + } + + else if ( other.file ) + file = &data.file; + + else if ( other.packet ) + packet = &data.packet; + } + + Cookie(const Cookie& other) = delete; + Cookie& operator=(const Cookie& other) = delete; + + friend inline void swap(Cookie& lhs, Cookie& rhs) { + Cookie tmp = std::move(lhs); + lhs = std::move(rhs); + rhs = std::move(tmp); + } +}; + +} // namespace zeek::spicy::rt diff --git a/src/spicy/debug.h b/src/spicy/debug.h new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/spicy/empty.cc b/src/spicy/empty.cc new file mode 100644 index 0000000000..b2191cb8b6 --- /dev/null +++ b/src/spicy/empty.cc @@ -0,0 +1,2 @@ + +int main() { return 1; } diff --git a/src/spicy/file-analyzer.cc b/src/spicy/file-analyzer.cc new file mode 100644 index 0000000000..2d3347e796 --- /dev/null +++ b/src/spicy/file-analyzer.cc @@ -0,0 +1,138 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "file-analyzer.h" + +#include + +#include "spicy.bif.h" +#include "zeek/file_analysis/File.h" +#include "zeek/spicy/manager.h" +#include "zeek/spicy/runtime-support.h" + +using namespace zeek; +using namespace zeek::spicy; +using namespace zeek::spicy::rt; + +#ifdef DEBUG +#define STATE_DEBUG_MSG(...) DebugMsg(__VA_ARGS__) +#else +#define STATE_DEBUG_MSG(...) +#endif + +void FileState::debug(const std::string& msg) { spicy::rt::debug(_cookie, msg); } + +static auto create_file_state(FileAnalyzer* analyzer) { + uint64_t depth = 0; + if ( auto current_cookie = static_cast(hilti::rt::context::cookie()) ) { + if ( const auto f = current_cookie->file ) + depth = f->depth + 1; + } + + cookie::FileAnalyzer cookie{.analyzer = analyzer, + .depth = depth, + .fstate = cookie::FileStateStack(analyzer->GetFile()->GetID())}; + return FileState(std::move(cookie)); +} + +FileAnalyzer::FileAnalyzer(RecordValPtr args, file_analysis::File* file) + : file_analysis::Analyzer(std::move(args), file), _state(create_file_state(this)) {} + +FileAnalyzer::~FileAnalyzer() {} + +void FileAnalyzer::Init() {} + +void FileAnalyzer::Done() { Finish(); } + +bool FileAnalyzer::DeliverStream(const u_char* data, uint64_t len) { + file_analysis::Analyzer::DeliverStream(data, len); + + return Process(len, data); +} + +bool FileAnalyzer::Undelivered(uint64_t offset, uint64_t len) { + file_analysis::Analyzer::Undelivered(offset, len); + + STATE_DEBUG_MSG("undelivered data, skipping further originator payload"); + _state.skipRemaining(); + return false; +} + +bool FileAnalyzer::EndOfFile() { + file_analysis::Analyzer::EndOfFile(); + Finish(); + return false; +} + +bool FileAnalyzer::Process(int len, const u_char* data) { + if ( ! _state.hasParser() && ! _state.isSkipping() ) { + auto parser = spicy_mgr->parserForFileAnalyzer(_state.file().analyzer->Tag()); + if ( parser ) + _state.setParser(parser); + else { + STATE_DEBUG_MSG("no unit specified for parsing"); + _state.skipRemaining(); + return false; + } + } + + auto* file = _state.file().analyzer->GetFile(); + + const auto& max_file_depth = BifConst::Spicy::max_file_depth; + + if ( _state.file().depth >= max_file_depth ) { + const auto& file_val = file->ToVal(); + + const auto analyzer_args = _state.file().analyzer->GetArgs(); + + file->FileEvent(Spicy::max_file_depth_exceeded, {file_val, analyzer_args, val_mgr->Count(_state.file().depth)}); + + auto tag = spicy_mgr->tagForFileAnalyzer(_state.file().analyzer->Tag()); +#if ZEEK_VERSION_NUMBER >= 50200 + AnalyzerViolation("maximal file depth exceeded", reinterpret_cast(data), len, tag); +#else + // We don't have an an appropriate way to report this with older Zeeks. +#endif + return false; + } + + try { + hilti::rt::context::CookieSetter _(_state.cookie()); + _state.process(len, reinterpret_cast(data)); + } catch ( const hilti::rt::RuntimeError& e ) { + STATE_DEBUG_MSG(hilti::rt::fmt("error during parsing, triggering analyzer violation: %s", e.what())); + auto tag = spicy_mgr->tagForFileAnalyzer(_state.file().analyzer->Tag()); +#if ZEEK_VERSION_NUMBER >= 50200 + AnalyzerViolation(e.what(), reinterpret_cast(data), len, tag); +#else + // We don't have an an appropriate way to report this with older Zeeks. +#endif + } catch ( const hilti::rt::Exception& e ) { + STATE_DEBUG_MSG(e.what()); + spicy_mgr->analyzerError(_state.file().analyzer, e.description(), + e.location()); // this sets Zeek to skip sending any further input + } + + return true; +} + +void FileAnalyzer::Finish() { + try { + hilti::rt::context::CookieSetter _(_state.cookie()); + _state.finish(); + } catch ( const hilti::rt::RuntimeError& e ) { + STATE_DEBUG_MSG(hilti::rt::fmt("error during parsing, triggering analyzer violation: %s", e.what())); + auto tag = spicy_mgr->tagForFileAnalyzer(_state.file().analyzer->Tag()); +#if ZEEK_VERSION_NUMBER >= 50200 + AnalyzerViolation(e.what(), "", 0, tag); +#else + // We don't have an an appropriate way to report this with older Zeeks. +#endif + } catch ( const hilti::rt::Exception& e ) { + spicy_mgr->analyzerError(_state.file().analyzer, e.description(), + e.location()); // this sets Zeek to skip sending any further input + } +} + +file_analysis::Analyzer* FileAnalyzer::InstantiateAnalyzer(RecordValPtr args, file_analysis::File* file) { + return new FileAnalyzer(std::move(args), file); +} diff --git a/src/spicy/file-analyzer.h b/src/spicy/file-analyzer.h new file mode 100644 index 0000000000..2bb6506a88 --- /dev/null +++ b/src/spicy/file-analyzer.h @@ -0,0 +1,91 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include +#include +#include + +#include + +#include +#include + +#include "zeek/spicy/cookie.h" + +namespace zeek::spicy::rt { + +/** Parsing state for a file. */ +class FileState : public ::spicy::rt::driver::ParsingState { +public: + /** + * Constructor. + * + * @param cookie cookie to associated with the file + */ + FileState(Cookie cookie) : ParsingState(::spicy::rt::driver::ParsingType::Stream), _cookie(std::move(cookie)) {} + + /** Returns the cookie pointer to use with the runtime library during analysis. */ + auto* cookie() { return &_cookie; } + + /** Returns the file-specific cookie state associated with the endpoint. */ + auto& file() { + assert(_cookie.file); + return *_cookie.file; + } + + /** + * Records a debug message pertaining to the specific file. + * + * @param msg message to record + */ + void DebugMsg(const std::string& msg) { debug(msg); } + +protected: + // Overridden from driver::ParsingState. + void debug(const std::string& msg) override; + +private: + Cookie _cookie; +}; + +/** A Spicy file analyzer. */ +class FileAnalyzer : public file_analysis::Analyzer { +public: + FileAnalyzer(RecordValPtr arg_args, file_analysis::File* arg_file); + virtual ~FileAnalyzer(); + + static file_analysis::Analyzer* InstantiateAnalyzer(RecordValPtr args, file_analysis::File* file); + +protected: + // Overridden from Zeek's file analyzer. + void Init() override; + void Done() override; + bool DeliverStream(const u_char* data, uint64_t len) override; + bool Undelivered(uint64_t offset, uint64_t len) override; + bool EndOfFile() override; + + /** + * Feeds a chunk of data into parsing. + * + * @param len number of bytes valid in *data* + * @param data pointer to data + * @return true if processing succeeded, false if an error occurred that + * stopped parsing + */ + bool Process(int len, const u_char* data); + + /** + * Finalizes parsing. After calling this, no more data can be passed into + * Process(). + */ + void Finish(); + + /** Records a debug message. */ + void DebugMsg(const std::string& msg) { _state.DebugMsg(msg); } + +private: + FileState _state; +}; + +} // namespace zeek::spicy::rt diff --git a/src/spicy/manager.cc b/src/spicy/manager.cc new file mode 100644 index 0000000000..267f0cf1ea --- /dev/null +++ b/src/spicy/manager.cc @@ -0,0 +1,920 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/spicy/manager.h" + +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include + +#include "zeek/DebugLogger.h" +#include "zeek/spicy/file-analyzer.h" +#include "zeek/spicy/packet-analyzer.h" +#include "zeek/spicy/protocol-analyzer.h" +#include "zeek/util-config.h" + +using namespace zeek; +using namespace zeek::spicy; + +// Split an potentially scoped ID into namespace and local part. +static std::pair parseID(const std::string& s) { + if ( auto i = s.rfind("::"); i != std::string::npos ) + return std::make_pair(s.substr(0, i), s.substr(i + 2)); + else + return std::make_pair("", s); +} + +Manager::~Manager() {} + +void Manager::registerProtocolAnalyzer(const std::string& name, hilti::rt::Protocol proto, + const hilti::rt::Vector& ports, const std::string& parser_orig, + const std::string& parser_resp, const std::string& replaces, + const std::string& linker_scope) { + SPICY_DEBUG(hilti::rt::fmt("Have Spicy protocol analyzer %s", name)); + + ProtocolAnalyzerInfo info; + info.name_analyzer = name; + info.name_parser_orig = parser_orig; + info.name_parser_resp = parser_resp; + info.name_replaces = replaces; + info.name_zeek = hilti::rt::replace(name, "::", "_"); + info.name_zeekygen = hilti::rt::fmt("", name); + info.protocol = proto; + info.ports = ports; + info.linker_scope = linker_scope; + + // We may have that analyzer already iff it was previously pre-registered + // without a linker scope. We'll then only set the scope now. + if ( auto t = _analyzer_name_to_tag_type.find(info.name_zeek); t != _analyzer_name_to_tag_type.end() ) { + SPICY_DEBUG(hilti::rt::fmt("Updating already registered protocol analyzer %s", name)); + + auto& existing = _protocol_analyzers_by_type.at(t->second); + assert(existing.name_analyzer == name); + existing.linker_scope = info.linker_scope; + + // If the infos don't match now, we have two separate definitions. + if ( info != existing ) + reporter->FatalError("redefinition of protocol analyzer %s", info.name_analyzer.c_str()); + + return; + } + + analyzer::Component::factory_callback factory = nullptr; + +#if SPICY_VERSION_NUMBER >= 10700 + auto proto_ = proto.value(); +#else + auto proto_ = proto; +#endif + + switch ( proto_ ) { + case hilti::rt::Protocol::TCP: factory = spicy::rt::TCP_Analyzer::InstantiateAnalyzer; break; + case hilti::rt::Protocol::UDP: factory = spicy::rt::UDP_Analyzer::InstantiateAnalyzer; break; + default: reporter->Error("unsupported protocol in analyzer"); return; + } + + auto c = new ::zeek::analyzer::Component(info.name_zeek, factory, 0); + AddComponent(c); + + // Hack to prevent Zeekygen from reporting the ID as not having a + // location during the following initialization step. + ::zeek::detail::zeekygen_mgr->Script(info.name_zeekygen); + ::zeek::detail::set_location(makeLocation(info.name_zeekygen)); + + // TODO: Should Zeek do this? It has run component intiialization at + // this point already, so ours won't get initialized anymore. + c->Initialize(); + + trackComponent(c, c->Tag().Type()); // Must come after Initialize(). + + info.type = c->Tag().Type(); + _protocol_analyzers_by_type.resize(info.type + 1); + _protocol_analyzers_by_type[info.type] = info; +} + +void Manager::registerFileAnalyzer(const std::string& name, const hilti::rt::Vector& mime_types, + const std::string& parser, const std::string& replaces, + const std::string& linker_scope) { + SPICY_DEBUG(hilti::rt::fmt("Have Spicy file analyzer %s", name)); + + FileAnalyzerInfo info; + info.name_analyzer = name; + info.name_parser = parser; + info.name_replaces = replaces; + info.name_zeek = hilti::rt::replace(name, "::", "_"); + info.name_zeekygen = hilti::rt::fmt("", name); + info.mime_types = mime_types; + info.linker_scope = linker_scope; + + // We may have that analyzer already iff it was previously pre-registered + // without a linker scope. We'll then only set the scope now. + if ( auto t = _analyzer_name_to_tag_type.find(info.name_zeek); t != _analyzer_name_to_tag_type.end() ) { + SPICY_DEBUG(hilti::rt::fmt("Updating already registered packet analyzer %s", name)); + + auto& existing = _file_analyzers_by_type.at(t->second); + existing.linker_scope = info.linker_scope; + + // If the infos don't match now, we have two separate definitions. + if ( info != existing ) + reporter->FatalError("redefinition of file analyzer %s", info.name_analyzer.c_str()); + + return; + } + + auto c = new ::zeek::file_analysis::Component(info.name_zeek, spicy::rt::FileAnalyzer::InstantiateAnalyzer, 0); + AddComponent(c); + + // Hack to prevent Zeekygen from reporting the ID as not having a + // location during the following initialization step. + ::zeek::detail::zeekygen_mgr->Script(info.name_zeekygen); + ::zeek::detail::set_location(makeLocation(info.name_zeekygen)); + + // TODO: Should Zeek do this? It has run component intiialization at + // this point already, so ours won't get initialized anymore. + c->Initialize(); + + trackComponent(c, c->Tag().Type()); // Must come after Initialize(). + + info.type = c->Tag().Type(); + _file_analyzers_by_type.resize(info.type + 1); + _file_analyzers_by_type[info.type] = info; +} + +void Manager::registerPacketAnalyzer(const std::string& name, const std::string& parser, const std::string& replaces, + const std::string& linker_scope) { + SPICY_DEBUG(hilti::rt::fmt("Have Spicy packet analyzer %s", name)); + + PacketAnalyzerInfo info; + info.name_analyzer = name; + info.name_replaces = replaces; + info.name_parser = parser; + info.name_zeek = hilti::rt::replace(name, "::", "_"); + info.name_zeekygen = hilti::rt::fmt("", info.name_zeek); + info.linker_scope = linker_scope; + + // We may have that analyzer already iff it was previously pre-registered + // without a linker scope. We'll then set the scope now. + if ( auto t = _analyzer_name_to_tag_type.find(info.name_zeek); t != _analyzer_name_to_tag_type.end() ) { + SPICY_DEBUG(hilti::rt::fmt("Updating already registered packet analyzer %s", name)); + + auto& existing = _packet_analyzers_by_type.at(t->second); + assert(existing.name_analyzer == name); + existing.linker_scope = info.linker_scope; + + // If the infos don't match now, we have two separate definitions. + if ( info != existing ) + reporter->FatalError("redefinition of packet analyzer %s", info.name_analyzer.c_str()); + + return; + } + + auto instantiate = [info]() -> packet_analysis::AnalyzerPtr { + return spicy::rt::PacketAnalyzer::Instantiate(info.name_zeek); + }; + + auto c = new ::zeek::packet_analysis::Component(info.name_zeek, instantiate, 0); + AddComponent(c); + + // Hack to prevent Zeekygen from reporting the ID as not having a + // location during the following initialization step. + ::zeek::detail::zeekygen_mgr->Script(info.name_zeekygen); + ::zeek::detail::set_location(makeLocation(info.name_zeekygen)); + + // TODO: Should Zeek do this? It has run component intialization at + // this point already, so ours won't get initialized anymore. + c->Initialize(); + + trackComponent(c, c->Tag().Type()); // Must come after Initialize(). + + info.type = c->Tag().Type(); + _packet_analyzers_by_type.resize(info.type + 1); + _packet_analyzers_by_type[info.type] = info; +} + +void Manager::registerType(const std::string& id, const TypePtr& type) { + auto [ns, local] = parseID(id); + + if ( const auto& old = detail::lookup_ID(local.c_str(), ns.c_str()) ) { + // This is most likely to trigger for IDs that other Spicy modules + // register. If we two Spicy modules need the same type, that's ok as + // long as they match. + if ( ! old->IsType() ) { + reporter->Error("Zeek type registration failed for '%s': ID already exists, but is not a type", id.c_str()); + return; + } + + if ( ! zeek::same_type(type, old->GetType()) ) { + reporter->Error("Zeek type registration failed for '%s': Type already exists, but differs", id.c_str()); + } + + SPICY_DEBUG(hilti::rt::fmt("Not re-registering Zeek type %s: identical type already exists", id)); + return; + } + + SPICY_DEBUG(hilti::rt::fmt("Registering Zeek type %s", id)); + auto zeek_id = detail::install_ID(local.c_str(), ns.c_str(), true, true); + zeek_id->SetType(type); + zeek_id->MakeType(); + AddBifItem(id, ::zeek::plugin::BifItem::TYPE); +} + +TypePtr Manager::findType(const std::string& id) const { + auto [ns, local] = parseID(id); + + auto zid = detail::lookup_ID(local.c_str(), ns.c_str()); + if ( ! zid ) + return nullptr; + + if ( ! zid->IsType() ) + return nullptr; + + return zid->GetType(); +} + +void Manager::registerEvent(const std::string& name) { + // Create a Zeek handler for the event. + event_registry->Register(name); + + // Install the ID into the corresponding namespace and export it. + auto n = ::hilti::rt::split(name, "::"); + std::string mod; + + if ( n.size() > 1 ) + mod = n.front(); + else + mod = detail::GLOBAL_MODULE_NAME; + + if ( auto id = detail::lookup_ID(name.c_str(), mod.c_str(), false, false, false) ) { + // Auto-export IDs that already exist. + id->SetExport(); + _events[name] = id; + } + else + // This installs & exports the ID, but it doesn't set its type yet. + // That will happen as handlers get defined. If there are no handlers, + // we set a dummy type in the plugin's InitPostScript + _events[name] = detail::install_ID(name.c_str(), mod.c_str(), false, true); +} + +const ::spicy::rt::Parser* Manager::parserForProtocolAnalyzer(const Tag& tag, bool is_orig) { + if ( is_orig ) + return _protocol_analyzers_by_type[tag.Type()].parser_orig; + else + return _protocol_analyzers_by_type[tag.Type()].parser_resp; +} + +const ::spicy::rt::Parser* Manager::parserForFileAnalyzer(const Tag& tag) { + return _file_analyzers_by_type[tag.Type()].parser; +} + +const ::spicy::rt::Parser* Manager::parserForPacketAnalyzer(const Tag& tag) { + return _packet_analyzers_by_type[tag.Type()].parser; +} + +Tag Manager::tagForProtocolAnalyzer(const Tag& tag) { + if ( auto r = _protocol_analyzers_by_type[tag.Type()].replaces ) + return r; + else + return tag; +} + +Tag Manager::tagForFileAnalyzer(const Tag& tag) { + if ( auto r = _file_analyzers_by_type[tag.Type()].replaces ) + return r; + else + return tag; +} + +Tag Manager::tagForPacketAnalyzer(const Tag& tag) { + if ( auto r = _packet_analyzers_by_type[tag.Type()].replaces ) + return r; + else + return tag; +} + +bool Manager::toggleProtocolAnalyzer(const Tag& tag, bool enable) { + auto type = tag.Type(); + + if ( type >= _protocol_analyzers_by_type.size() ) + return false; + + const auto& analyzer = _protocol_analyzers_by_type[type]; + + if ( ! analyzer.type ) + // not set -> not ours + return false; + + if ( enable ) { + SPICY_DEBUG(hilti::rt::fmt("Enabling Spicy protocol analyzer %s", analyzer.name_analyzer)); + analyzer_mgr->EnableAnalyzer(tag); + + if ( analyzer.replaces ) { + SPICY_DEBUG(hilti::rt::fmt("Disabling standard protocol analyzer %s", analyzer.name_analyzer)); + analyzer_mgr->DisableAnalyzer(analyzer.replaces); + } + } + else { + SPICY_DEBUG(hilti::rt::fmt("Disabling Spicy protocol analyzer %s", analyzer.name_analyzer)); + analyzer_mgr->DisableAnalyzer(tag); + + if ( analyzer.replaces ) { + SPICY_DEBUG(hilti::rt::fmt("Re-enabling standard protocol analyzer %s", analyzer.name_analyzer)); + analyzer_mgr->EnableAnalyzer(analyzer.replaces); + } + } + + return true; +} + +bool Manager::toggleFileAnalyzer(const Tag& tag, bool enable) { + auto type = tag.Type(); + + if ( type >= _file_analyzers_by_type.size() ) + return false; + + const auto& analyzer = _file_analyzers_by_type[type]; + + if ( ! analyzer.type ) + // not set -> not ours + return false; + + file_analysis::Component* component = file_mgr->Lookup(tag); + file_analysis::Component* component_replaces = analyzer.replaces ? file_mgr->Lookup(analyzer.replaces) : nullptr; + + if ( ! component ) { + // Shouldn't really happen. + reporter->InternalError("failed to lookup file analyzer component"); + return false; + } + + if ( enable ) { + SPICY_DEBUG(hilti::rt::fmt("Enabling Spicy file analyzer %s", analyzer.name_analyzer)); + component->SetEnabled(true); + + if ( component_replaces ) { + SPICY_DEBUG(hilti::rt::fmt("Disabling standard file analyzer %s", analyzer.name_analyzer)); + component_replaces->SetEnabled(false); + } + } + else { + SPICY_DEBUG(hilti::rt::fmt("Disabling Spicy file analyzer %s", analyzer.name_analyzer)); + component->SetEnabled(false); + + if ( component_replaces ) { + SPICY_DEBUG(hilti::rt::fmt("Enabling standard file analyzer %s", analyzer.name_analyzer)); + component_replaces->SetEnabled(true); + } + } + + return true; +} + +bool Manager::togglePacketAnalyzer(const Tag& tag, bool enable) { + auto type = tag.Type(); + + if ( type >= _packet_analyzers_by_type.size() ) + return false; + + const auto& analyzer = _packet_analyzers_by_type[type]; + + if ( ! analyzer.type ) + // not set -> not ours + return false; + + packet_analysis::Component* component = packet_mgr->Lookup(tag); + packet_analysis::Component* component_replaces = + analyzer.replaces ? packet_mgr->Lookup(analyzer.replaces) : nullptr; + + if ( ! component ) { + // Shouldn't really happen. + reporter->InternalError("failed to lookup packet analyzer component"); + return false; + } + + if ( enable ) { + SPICY_DEBUG(hilti::rt::fmt("Enabling Spicy packet analyzer %s", analyzer.name_analyzer)); + component->SetEnabled(true); + + if ( component_replaces ) { + SPICY_DEBUG(hilti::rt::fmt("Disabling standard packet analyzer %s", analyzer.name_analyzer)); + component_replaces->SetEnabled(false); + } + } + else { + SPICY_DEBUG(hilti::rt::fmt("Disabling Spicy packet analyzer %s", analyzer.name_analyzer)); + component->SetEnabled(false); + + if ( component_replaces ) { + SPICY_DEBUG(hilti::rt::fmt("Enabling standard packet analyzer %s", analyzer.name_analyzer)); + component_replaces->SetEnabled(true); + } + } + + return true; +} + +bool Manager::toggleAnalyzer(EnumVal* tag, bool enable) { + if ( tag->GetType() == analyzer_mgr->GetTagType() ) { + if ( auto analyzer = analyzer_mgr->Lookup(tag) ) + return toggleProtocolAnalyzer(analyzer->Tag(), enable); + else + return false; + } + + if ( tag->GetType() == file_mgr->GetTagType() ) { + if ( auto analyzer = file_mgr->Lookup(tag) ) + return toggleFileAnalyzer(analyzer->Tag(), enable); + else + return false; + } + + if ( tag->GetType() == packet_mgr->GetTagType() ) { + if ( auto analyzer = packet_mgr->Lookup(tag) ) + return togglePacketAnalyzer(analyzer->Tag(), enable); + else + return false; + } + + return false; +} + +static std::unique_ptr _makeLocation(const std::string& location) { + static std::set filenames; // see comment below in parse_location + + auto parse_location = [](const auto& s) -> std::unique_ptr { + // This is not so great; In the HILTI runtome we pass locations + // around as string. To pass them to Zeek, we need to unsplit the + // strings into file name and line number. Zeek also won't clean up + // the file names, so we need to track them ourselves. + auto x = hilti::rt::split(s, ":"); + if ( x[0].empty() ) + return nullptr; + + auto loc = std::make_unique(); + loc->filename = filenames.insert(std::string(x[0])).first->c_str(); // we retain ownership + + if ( x.size() >= 2 ) { + auto y = hilti::rt::split(x[1], "-"); + if ( y.size() >= 2 ) { + loc->first_line = std::stoi(std::string(y[0])); + loc->last_line = std::stoi(std::string(y[1])); + } + else if ( y[0].size() ) + loc->first_line = loc->last_line = std::stoi(std::string(y[0])); + } + + return loc; + }; + + if ( location.size() ) + return parse_location(location); + else if ( auto hilti_location = hilti::rt::debug::location() ) + return parse_location(hilti_location); + else + return nullptr; +} + +void Manager::analyzerError(analyzer::Analyzer* a, const std::string& msg, const std::string& location) { + auto zeek_location = _makeLocation(location); + reporter->PushLocation(zeek_location.get()); + reporter->AnalyzerError(a, "%s", msg.c_str()); + reporter->PopLocation(); +} + +void Manager::analyzerError(file_analysis::Analyzer* a, const std::string& msg, const std::string& location) { + auto zeek_location = _makeLocation(location); + reporter->PushLocation(zeek_location.get()); + + // We don't have an reporter error for file analyzers, so we log this as a + // weird instead. + if ( a && a->GetFile() ) + reporter->Weird(a->GetFile(), "file_error", msg.c_str()); + else + reporter->Weird("file_error", msg.c_str()); + + reporter->PopLocation(); + + if ( a ) + a->SetSkip(1); // Imitate what AnalyzerError() does for protocol analyzers. +} + +void Manager::analyzerError(packet_analysis::Analyzer* a, const std::string& msg, const std::string& location) { + auto zeek_location = _makeLocation(location); + reporter->PushLocation(zeek_location.get()); + // We don't have an reporter error for packet analyzers, so we log + // this as a weird instead. + reporter->Weird("packet_error", msg.c_str()); + reporter->PopLocation(); +} + +plugin::Configuration Manager::Configure() { + ::zeek::plugin::Configuration config; + config.name = "Zeek::Spicy"; + config.description = "Support for Spicy parsers (*.hlto)"; + + EnableHook(::zeek::plugin::HOOK_LOAD_FILE); + + return config; +} + +void Manager::InitPreScript() { + SPICY_DEBUG("Beginning pre-script initialization"); + +#if SPICY_VERSION_NUMBER >= 10700 + hilti::rt::executeManualPreInits(); +#endif + + autoDiscoverModules(); + + SPICY_DEBUG("Done with pre-script initialization"); +} + +// Returns a port's Zeek-side transport protocol. +static ::TransportProto transport_protocol(const hilti::rt::Port port) { +#if SPICY_VERSION_NUMBER >= 10700 + auto proto = port.protocol().value(); +#else + auto proto = port.protocol(); +#endif + + switch ( proto ) { + case hilti::rt::Protocol::TCP: return ::TransportProto::TRANSPORT_TCP; + case hilti::rt::Protocol::UDP: return ::TransportProto::TRANSPORT_UDP; + case hilti::rt::Protocol::ICMP: return ::TransportProto::TRANSPORT_ICMP; + default: + reporter->InternalError("unsupported transport protocol in port '%s' for Zeek conversion", + std::string(port).c_str()); + return ::TransportProto::TRANSPORT_UNKNOWN; + } +} + +static void hook_accept_input() { + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( auto x = cookie->protocol ) { + auto tag = spicy_mgr->tagForProtocolAnalyzer(x->analyzer->GetAnalyzerTag()); + SPICY_DEBUG(hilti::rt::fmt("confirming protocol %s", tag.AsString())); + return x->analyzer->AnalyzerConfirmation(tag); + } +} + +static void hook_decline_input(const std::string& reason) { + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( auto x = cookie->protocol ) { + auto tag = spicy_mgr->tagForProtocolAnalyzer(x->analyzer->GetAnalyzerTag()); + SPICY_DEBUG(hilti::rt::fmt("rejecting protocol %s", tag.AsString())); + return x->analyzer->AnalyzerViolation("protocol rejected", nullptr, 0, tag); + } +} + +void Manager::InitPostScript() { + SPICY_DEBUG("Beginning post-script initialization"); + + disableReplacedAnalyzers(); + + // If there's no handler for one of our events, it won't have received a + // type. Give it a dummy event type in that case, so that we don't walk + // around with a nullptr. + for ( const auto& [name, id] : _events ) { + if ( ! id->GetType() ) { + auto args = make_intrusive(new type_decl_list()); + auto et = make_intrusive(std::move(args), base_type(TYPE_VOID), FUNC_FLAVOR_EVENT); + id->SetType(std::move(et)); + } + } + + // Init runtime, which will trigger all initialization code to execute. + SPICY_DEBUG("Initializing Spicy runtime"); + + auto hilti_config = hilti::rt::configuration::get(); + + if ( id::find_const("Spicy::enable_print")->AsBool() ) // NOLINT(clang-analyzer-cplusplus.NewDeleteLeaks) + hilti_config.cout = std::cout; + else + hilti_config.cout.reset(); + + if ( id::find_const("Spicy::enable_profiling")->AsBool() ) +#if SPICY_VERSION_NUMBER >= 10800 + hilti_config.enable_profiling = true; +#else + std::cerr << "Profiling is not supported with this version of Spicy, ignoring " + "'Spicy::enable_profiling'\n"; +#endif + + hilti_config.abort_on_exceptions = id::find_const("Spicy::abort_on_exceptions")->AsBool(); + hilti_config.show_backtraces = id::find_const("Spicy::show_backtraces")->AsBool(); + + hilti::rt::configuration::set(hilti_config); + +#if SPICY_VERSION_NUMBER >= 10700 + auto spicy_config = ::spicy::rt::configuration::get(); + spicy_config.hook_accept_input = hook_accept_input; + spicy_config.hook_decline_input = hook_decline_input; + ::spicy::rt::configuration::set(std::move(spicy_config)); +#endif + + try { + ::hilti::rt::init(); + ::spicy::rt::init(); + } catch ( const hilti::rt::Exception& e ) { + std::cerr << hilti::rt::fmt("uncaught runtime exception %s during initialization: %s", + hilti::rt::demangle(typeid(e).name()), e.what()) + << std::endl; + exit(1); + } catch ( const std::runtime_error& e ) { + std::cerr << hilti::rt::fmt("uncaught C++ exception %s during initialization: %s", + hilti::rt::demangle(typeid(e).name()), e.what()) + << std::endl; + exit(1); + } + + // Fill in the parser information now that we derived from the ASTs. + auto find_parser = [](const std::string& analyzer, const std::string& parser, + const std::string& linker_scope) -> const ::spicy::rt::Parser* { + if ( parser.empty() ) + return nullptr; + + for ( auto p : ::spicy::rt::parsers() ) { + if ( p->name == parser && p->linker_scope == linker_scope ) + return p; + } + + reporter->InternalError("Unknown Spicy parser '%s' requested by analyzer '%s'", parser.c_str(), + analyzer.c_str()); + + return nullptr; // cannot be reached + }; + + for ( auto& p : _protocol_analyzers_by_type ) { + if ( p.type == 0 ) + // vector element not set + continue; + + SPICY_DEBUG(hilti::rt::fmt("Registering %s protocol analyzer %s with Zeek", p.protocol, p.name_analyzer)); + + p.parser_orig = find_parser(p.name_analyzer, p.name_parser_orig, p.linker_scope); + p.parser_resp = find_parser(p.name_analyzer, p.name_parser_resp, p.linker_scope); + + // Register analyzer for its well-known ports. + auto tag = analyzer_mgr->GetAnalyzerTag(p.name_zeek.c_str()); + if ( ! tag ) + reporter->InternalError("cannot get analyzer tag for '%s'", p.name_analyzer.c_str()); + + for ( auto port : p.ports ) { + SPICY_DEBUG(hilti::rt::fmt(" Scheduling analyzer for port %s", port)); + analyzer_mgr->RegisterAnalyzerForPort(tag, transport_protocol(port), port.port()); + } + + if ( p.parser_resp ) { + for ( auto port : p.parser_resp->ports ) { + if ( port.direction != ::spicy::rt::Direction::Both && + port.direction != ::spicy::rt::Direction::Responder ) + continue; + + SPICY_DEBUG(hilti::rt::fmt(" Scheduling analyzer for port %s", port.port)); + analyzer_mgr->RegisterAnalyzerForPort(tag, transport_protocol(port.port), port.port.port()); + } + } + } + + for ( auto& p : _file_analyzers_by_type ) { + if ( p.type == 0 ) + // vector element not set + continue; + + SPICY_DEBUG(hilti::rt::fmt("Registering file analyzer %s with Zeek", p.name_analyzer.c_str())); + + p.parser = find_parser(p.name_analyzer, p.name_parser, p.linker_scope); + + // Register analyzer for its MIME types. + auto tag = file_mgr->GetComponentTag(p.name_zeek.c_str()); + if ( ! tag ) + reporter->InternalError("cannot get analyzer tag for '%s'", p.name_analyzer.c_str()); + + auto register_analyzer_for_mime_type = [&](auto tag, const std::string& mt) { + SPICY_DEBUG(hilti::rt::fmt(" Scheduling analyzer for MIME type %s", mt)); + + // MIME types are registered in scriptland, so we'll raise an + // event that will do it for us through a predefined handler. + zeek::Args vals = Args(); + vals.emplace_back(tag.AsVal()); + vals.emplace_back(make_intrusive(mt)); + EventHandlerPtr handler = event_registry->Register("spicy_analyzer_for_mime_type"); + event_mgr.Enqueue(handler, vals); + }; + + for ( const auto& mt : p.mime_types ) + register_analyzer_for_mime_type(tag, mt); + + if ( p.parser ) { + for ( const auto& mt : p.parser->mime_types ) + register_analyzer_for_mime_type(tag, mt); + } + } + + for ( auto& p : _packet_analyzers_by_type ) { + if ( p.type == 0 ) + // vector element not set + continue; + + SPICY_DEBUG(hilti::rt::fmt("Registering packet analyzer %s with Zeek", p.name_analyzer.c_str())); + p.parser = find_parser(p.name_analyzer, p.name_parser, p.linker_scope); + } + + SPICY_DEBUG("Done with post-script initialization"); +} + +void Manager::Done() { + SPICY_DEBUG("Shutting down Spicy runtime"); + ::spicy::rt::done(); + hilti::rt::done(); +} + +void Manager::loadModule(const hilti::rt::filesystem::path& path) { + try { + // If our auto discovery ends up finding the same module multiple times, + // we ignore subsequent requests. + std::error_code ec; + auto canonical_path = hilti::rt::filesystem::canonical(path, ec); + if ( ec ) + hilti::rt::fatalError(hilti::rt::fmt("could not compute canonical path for %s: %s", path, ec.message())); + + if ( auto [library, inserted] = _libraries.insert({canonical_path, hilti::rt::Library(canonical_path)}); + inserted ) { + SPICY_DEBUG(hilti::rt::fmt("Loading %s", canonical_path.native())); + if ( auto load = library->second.open(); ! load ) + hilti::rt::fatalError( + hilti::rt::fmt("could not open library path %s: %s", canonical_path, load.error())); + } + else { + SPICY_DEBUG(hilti::rt::fmt("Ignoring duplicate loading request for %s", canonical_path.native())); + } +#if SPICY_VERSION_NUMBER >= 10700 + } catch ( const ::hilti::rt::UsageError& e ) { +#else + } catch ( const ::hilti::rt::UserException& e ) { +#endif + hilti::rt::fatalError(e.what()); + } +} + +int Manager::HookLoadFile(const LoadType type, const std::string& file, const std::string& resolved) { + auto ext = hilti::rt::filesystem::path(file).extension(); + + if ( ext == ".hlto" ) { + loadModule(file); + return 1; + } + + if ( ext == ".spicy" || ext == ".evt" || ext == ".hlt" ) + reporter->FatalError("cannot load '%s': analyzers need to be precompiled with 'spicyz' ", file.c_str()); + + return -1; +} + +void Manager::searchModules(const std::string& paths) { + for ( const auto& dir : hilti::rt::split(paths, ":") ) { + auto trimmed_dir = hilti::rt::trim(dir); + if ( trimmed_dir.empty() ) + continue; + + std::error_code ec; + if ( auto is_directory = hilti::rt::filesystem::is_directory(trimmed_dir, ec); ec || ! is_directory ) { + SPICY_DEBUG(hilti::rt::fmt("Module directory %s cannot be read, skipping", trimmed_dir)); + continue; + } + + SPICY_DEBUG(hilti::rt::fmt("Searching %s for *.hlto", trimmed_dir)); + + auto it = hilti::rt::filesystem::recursive_directory_iterator(trimmed_dir, ec); + if ( ! ec ) { + while ( it != hilti::rt::filesystem::recursive_directory_iterator() ) { + if ( it->is_regular_file() && it->path().extension() == ".hlto" ) + loadModule(it->path()); + + if ( it.increment(ec); ec ) { + hilti::rt::warning(hilti::rt::fmt("Error iterating over %s, skipping any remaining files: %s", + trimmed_dir, ec.message())); + break; + } + } + } + else + hilti::rt::warning(hilti::rt::fmt("Cannot iterate over %s, skipping: %s", trimmed_dir, ec.message())); + } +}; + +detail::Location Manager::makeLocation(const std::string& fname) { + auto x = _locations.insert(fname); + return detail::Location(x.first->c_str(), 0, 0, 0, 0); +} + +void Manager::autoDiscoverModules() { + // Always search Zeek's plugin path for modules, that's where zkg puts + // them. + searchModules(util::zeek_plugin_path()); + + if ( auto search_paths = hilti::rt::getenv("ZEEK_SPICY_MODULE_PATH"); search_paths && search_paths->size() ) + // This overrides all other paths. + searchModules(*search_paths); + else + searchModules(ZEEK_SPICY_MODULE_PATH); +} + +void Manager::disableReplacedAnalyzers() { + for ( auto& info : _protocol_analyzers_by_type ) { + if ( info.name_replaces.empty() ) + continue; + + auto replaces = info.name_replaces.c_str(); + + if ( file_mgr->Lookup(replaces) || packet_mgr->Lookup(replaces) ) + reporter->FatalError("cannot replace '%s' analyzer with a protocol analyzer", replaces); + + auto tag = analyzer_mgr->GetAnalyzerTag(replaces); + if ( ! tag ) { + SPICY_DEBUG(hilti::rt::fmt("%s is supposed to replace protocol analyzer %s, but that does not exist", + info.name_analyzer, replaces)); + + continue; + } + + SPICY_DEBUG(hilti::rt::fmt("%s replaces existing protocol analyzer %s", info.name_analyzer, replaces)); + info.replaces = tag; + analyzer_mgr->DisableAnalyzer(tag); + } + + for ( auto& info : _file_analyzers_by_type ) { + if ( info.name_replaces.empty() ) + continue; + + auto replaces = info.name_replaces.c_str(); + + if ( analyzer_mgr->Lookup(replaces) || packet_mgr->Lookup(replaces) ) + reporter->FatalError("cannot replace '%s' analyzer with a file analyzer", replaces); + + auto component = file_mgr->Lookup(replaces); + if ( ! component ) { + SPICY_DEBUG(hilti::rt::fmt("%s is supposed to replace file analyzer %s, but that does not exist", + info.name_analyzer, replaces)); + + continue; + } + + SPICY_DEBUG(hilti::rt::fmt("%s replaces existing file analyzer %s", info.name_analyzer, replaces)); + info.replaces = component->Tag(); + component->SetEnabled(false); + } + + for ( auto& info : _packet_analyzers_by_type ) { + if ( info.name_replaces.empty() ) + continue; + + auto replaces = info.name_replaces.c_str(); + + auto component = packet_mgr->Lookup(replaces); + if ( ! component ) { + SPICY_DEBUG(hilti::rt::fmt("%s is supposed to replace packet analyzer %s, but that does not exist", + info.name_analyzer, replaces)); + + continue; + } + + SPICY_DEBUG(hilti::rt::fmt("%s replaces existing packet analyzer %s", info.name_analyzer, replaces)); + info.replaces = component->Tag(); + component->SetEnabled(false); + } +} + +void Manager::trackComponent(plugin::Component* c, int32_t tag_type) { + auto i = _analyzer_name_to_tag_type.insert({c->Name(), tag_type}); + if ( ! i.second ) + // We enforce on our end that an analyzer name can appear only once + // across all types of analyzers. Makes things easier and avoids + // confusion. + reporter->FatalError("duplicate analyzer name '%s'", c->Name().c_str()); +} diff --git a/src/spicy/manager.h b/src/spicy/manager.h new file mode 100644 index 0000000000..8712d16ee4 --- /dev/null +++ b/src/spicy/manager.h @@ -0,0 +1,413 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "zeek/Scope.h" +#include "zeek/Tag.h" +#include "zeek/plugin/Component.h" +#include "zeek/plugin/Plugin.h" + +// Macro helper to report Spicy debug messages. This forwards to +// to both the Zeek logger and the Spicy runtime logger. +#define SPICY_DEBUG(msg) ::zeek::spicy::log(msg); + +namespace hilti::rt { +class Port; +struct Protocol; +} // namespace hilti::rt + +namespace spicy::rt { +struct Parser; +} + +namespace zeek { + +namespace analyzer { +class Analyzer; +} + +namespace file_analysis { +class Analyzer; +} + +namespace packet_analysis { +class Analyzer; +} + +namespace spicy { + +// Backend to log debug messages. +inline void log(const std::string& msg) { + DBG_LOG(DBG_SPICY, "%s", msg.c_str()); + + if ( hilti::rt::isInitialized() ) + HILTI_RT_DEBUG("zeek", msg); +} + + +/* Top-level entry point for Spicy functionality. */ +class Manager : public zeek::plugin::Plugin { +public: + Manager() {} + virtual ~Manager(); + + /** + * Runtime method to register a protocol analyzer with its Zeek-side + * configuration. This is called at startup by generated Spicy code for + * each protocol analyzer defined in an EVT file. + * + * @param name name of the analyzer as defined in its EVT file + * @param proto analyzer's transport-layer protocol + * @param prts well-known ports for the analyzer; it'll be activated automatically for these + * @param parser_orig name of the Spicy parser for the originator side; must match the name that + * Spicy registers the unit's parser with + * @param parser_resp name of the Spicy parser for the originator side; must match the name that + * Spicy registers the unit's parser with + * @param replaces optional name of existing Zeek analyzer that this one replaces; the Zeek + * analyzer will automatically be disabled + * @param linker_scope scope of current HLTO file, which will restrict visibility of the + * registration + */ + void registerProtocolAnalyzer(const std::string& name, hilti::rt::Protocol proto, + const hilti::rt::Vector& ports, const std::string& parser_orig, + const std::string& parser_resp, const std::string& replaces, + const std::string& linker_scope); + + /** + * Runtime method to register a file analyzer with its Zeek-side + * configuration. This is called at startup by generated Spicy code for + * each file analyzer defined in an EVT file + * + * @param name name of the analyzer as defined in its EVT file + * @param mime_types list of MIME types the analyzer handles; it'll be automatically used for + * all files of matching types + * @param parser name of the Spicy parser for parsing the file; must match the name that Spicy + * registers the unit's parser with + * @param replaces optional name of existing Zeek analyzer that this one replaces; the Zeek + * analyzer will automatically be disabled + * @param linker_scope scope of current HLTO file, which will restrict visibility of the + * registration + */ + void registerFileAnalyzer(const std::string& name, const hilti::rt::Vector& mime_types, + const std::string& parser, const std::string& replaces, const std::string& linker_scope); + + /** + * Runtime method to register a packet analyzer with its Zeek-side + * configuration. This is called at startup by generated Spicy code for + * each packet analyzer defined in an EVT file. + * + * @param name name of the analyzer as defined in its EVT file + * @param parser name of the Spicy parser for parsing the packet; must + * match the name that Spicy registers the unit's + * parser with. + * @param replaces optional name of existing Zeek analyzer that this one + * replaces; the Zeek analyzer will automatically be disabled + * @param linker_scope scope of current HLTO file, which will restrict visibility of the + * registration + */ + void registerPacketAnalyzer(const std::string& name, const std::string& parser, const std::string& replaces, + const std::string& linker_scope); + + /** + * Runtime method to register a Spicy-generated type with Zeek. The type + * must have been encountered during AST traversal already, so that its ID + * is known. The corresponding Zeek type will be created and registered with Zeek. + * + * @param id fully-qualified ID of the type + * @return error if the type could not be registered + */ + hilti::rt::Result registerType(const std::string& id); + + /** + * Runtime method to register an already converted Spicy-generated type + * with Zeek. + * + * @param id fully-qualified ID of the type + * @param type Zeek-side type to register + */ + void registerType(const std::string& id, const TypePtr& type); + + /** + * Looks up a global type by its ID with Zeek. + * + * @param id fully-qualified Zeek-side ID of the type + * @return Zeek-side type, or null if not found + */ + TypePtr findType(const std::string& id) const; + + /** + * Runtime method to register a Spicy-generated event. The installs the ID + * Zeek-side and is called at startup by generated Spicy code for each + * event defined in an EVT file. + * + * @param name fully scoped name of the event + */ + void registerEvent(const std::string& name); + + /** + * Runtime method to retrieve the Spicy parser for a given Zeek protocol analyzer tag. + * + * @param analyzer requested protocol analyzer + * @param is_orig true if requesting the parser parser for a sessions' originator side, false + * for the responder + * @return parser, or null if we don't have one for this tag. The pointer will remain valid for + * the life-time of the process. + */ + const ::spicy::rt::Parser* parserForProtocolAnalyzer(const Tag& tag, bool is_orig); + + /** + * Runtime method to retrieve the Spicy parser for a given Zeek file analyzer tag. + * + * @param analyzer requested file analyzer. + * @return parser, or null if we don't have one for this tag. The pointer will remain valid for + * the life-time of the process. + */ + const ::spicy::rt::Parser* parserForFileAnalyzer(const Tag& tag); + + /** + * Runtime method to retrieve the Spicy parser for a given Zeek packet analyzer tag. + * + * @param analyzer requested packet analyzer. + * @return parser, or null if we don't have one for this tag. The pointer will remain + * valid for the life-time of the process. + */ + const ::spicy::rt::Parser* parserForPacketAnalyzer(const Tag& tag); + + /** + * Runtime method to retrieve the analyzer tag that should be passed to + * script-land when talking about a protocol analyzer. This is normally + * the analyzer's standard tag, but may be replaced with something else + * if the analyzer substitutes for an existing one. + * + * @param tag original tag we query for how to pass it to script-land. + * @return desired tag for passing to script-land. + */ + Tag tagForProtocolAnalyzer(const Tag& tag); + + /** + * Runtime method to retrieve the analyzer tag that should be passed to + * script-land when talking about a file analyzer. This is normally the + * analyzer's standard tag, but may be replaced with something else if + * the analyzer substitutes for an existing one. + * + * @param tag original tag we query for how to pass it to script-land. + * @return desired tag for passing to script-land. + */ + Tag tagForFileAnalyzer(const Tag& tag); + + /** + * Runtime method to retrieve the analyzer tag that should be passed to + * script-land when talking about a packet analyzer. This is normally the + * analyzer's standard tag, but may be replaced with something else if + * the analyzer substitutes for an existing one. + * + * @param tag original tag we query for how to pass it to script-land. + * @return desired tag for passing to script-land. + */ + Tag tagForPacketAnalyzer(const Tag& tag); + + /** + * Explicitly enable/disable a protocol analyzer. By default, all analyzers + * loaded will also be activated. By calling this method, an analyzer can + * toggled. + * + * @param analyzer tag of analyzer + * @param enable true to enable, false to disable + */ + bool toggleProtocolAnalyzer(const Tag& tag, bool enable); + + /** + * Explicitly enable/disable a file analyzer. By default, all analyzers + * loaded will also be activated. By calling this method, an analyzer can + * toggled. + * + * @param analyzer tag of analyzer + * @param enable true to enable, false to disable + */ + bool toggleFileAnalyzer(const Tag& tag, bool enable); + + /** + * Explicitly enable/disable a packet analyzer. By default, all analyzers + * loaded will also be activated. By calling this method, an analyzer can + * toggled. + * + * @note This is currently not supported because Zeek does not provide the + * necessary API. + * + * @param analyzer tag of analyzer + * @param enable true to enable, false to disable + */ + bool togglePacketAnalyzer(const Tag& tag, bool enable); + + /** + * Explicitly enable/disable an analyzer. By default, all analyzers + * loaded will also be activated. By calling this method, an analyzer can + * toggled. + * + * This method is frontend for the versions specific to + * protocol/file/packet analyzers. It takes an enum corresponding to either + * kind and branches out accordingly. + * + * @param analyzer tag of analyzer + * @param enable true to enable, false to disable + */ + bool toggleAnalyzer(EnumVal* tag, bool enable); + + /** Report an error and disable a protocol analyzer's input processing */ + void analyzerError(analyzer::Analyzer* a, const std::string& msg, const std::string& location); + + /** Report an error and disable a file analyzer's input processing */ + void analyzerError(file_analysis::Analyzer* a, const std::string& msg, const std::string& location); + + /** Report an error and disable a packet analyzer's input processing. */ + void analyzerError(packet_analysis::Analyzer* a, const std::string& msg, const std::string& location); + + /** Returns the number of errors recorded by the Zeek reporter. */ + int numberErrors(); + +protected: + // Overriding method from Zeek's plugin API. + zeek::plugin::Configuration Configure() override; + + // Overriding method from Zeek's plugin API. + void InitPreScript() override; + + // Overriding method from Zeek's plugin API. + void InitPostScript() override; + + // Overriding method from Zeek's plugin API. + void Done() override; + + // Overriding method from Zeek's plugin API. + int HookLoadFile(const LoadType type, const std::string& file, const std::string& resolved) override; + +private: + // Load one *.hlto module. + void loadModule(const hilti::rt::filesystem::path& path); + + // Search ZEEK_SPICY_MODULE_PATH for pre-compiled *.hlto modules and load them. + void autoDiscoverModules(); + + // Recursively search pre-compiled *.hlto in colon-separated paths. + void searchModules(const std::string& paths); + + // Return a Zeek location object for the given file name that will stay valid. + detail::Location makeLocation(const std::string& fname); + + // Disable any Zeek-side analyzers that are replaced by one of ours. + void disableReplacedAnalyzers(); + + /** + * Internal callback that records the component-to-tag-type mapping for plugin's analyzer. + * + * @param c component to track + * @param tag_type tag type to associate with component + */ + void trackComponent(plugin::Component* c, int32_t tag_type); + + /** Captures a registered protocol analyzer. */ + struct ProtocolAnalyzerInfo { + // Provided when registering the analyzer. + std::string name_analyzer; + std::string name_parser_orig; + std::string name_parser_resp; + std::string name_replaces; + hilti::rt::Protocol protocol = hilti::rt::Protocol::Undef; + hilti::rt::Vector ports; + std::string linker_scope; + + // Computed and available once the analyzer has been registered. + std::string name_zeek; + std::string name_zeekygen; + Tag::type_t type; + const ::spicy::rt::Parser* parser_orig; + const ::spicy::rt::Parser* parser_resp; + Tag replaces; + + bool operator==(const ProtocolAnalyzerInfo& other) const { + return name_analyzer == other.name_analyzer && name_parser_orig == other.name_parser_orig && + name_parser_resp == other.name_parser_resp && name_replaces == other.name_replaces && + protocol == other.protocol && ports == other.ports && linker_scope == other.linker_scope; + } + + bool operator!=(const ProtocolAnalyzerInfo& other) const { return ! (*this == other); } + }; + + /** Captures a registered file analyzer. */ + struct FileAnalyzerInfo { + // Provided when registering the analyzer. + std::string name_analyzer; + std::string name_parser; + std::string name_replaces; + hilti::rt::Vector mime_types; + std::string linker_scope; + + // Computed and available once the analyzer has been registered. + std::string name_zeek; + std::string name_zeekygen; + Tag::type_t type; + const ::spicy::rt::Parser* parser; + Tag replaces; + + bool operator==(const FileAnalyzerInfo& other) const { + return name_analyzer == other.name_analyzer && name_parser == other.name_parser && + name_replaces == other.name_replaces && mime_types == other.mime_types && + linker_scope == other.linker_scope; + } + + bool operator!=(const FileAnalyzerInfo& other) const { return ! (*this == other); } + }; + + /** Captures a registered file analyzer. */ + struct PacketAnalyzerInfo { + // Provided when registering the analyzer. + std::string name_analyzer; + std::string name_parser; + std::string name_replaces; + std::string linker_scope; + + // Computed and available once the analyzer has been registered. + std::string name_zeek; + std::string name_zeekygen; + Tag::type_t type; + const ::spicy::rt::Parser* parser; + Tag replaces; + + // Compares only the provided attributes, as that's what defines us. + bool operator==(const PacketAnalyzerInfo& other) const { + return name_analyzer == other.name_analyzer && name_parser == other.name_parser && + name_replaces == other.name_replaces && linker_scope == other.linker_scope; + } + + bool operator!=(const PacketAnalyzerInfo& other) const { return ! (*this == other); } + }; + + std::string _spicy_version; + + std::vector _protocol_analyzers_by_type; + std::vector _file_analyzers_by_type; + std::vector _packet_analyzers_by_type; + std::unordered_map _libraries; + std::set _locations; + std::unordered_map _events; + + // Mapping of component names to tag types. We use this to ensure analyzer uniqueness. + std::unordered_map _analyzer_name_to_tag_type; +}; + +} // namespace spicy + +extern spicy::Manager* spicy_mgr; + +} // namespace zeek diff --git a/src/spicy/packet-analyzer.cc b/src/spicy/packet-analyzer.cc new file mode 100644 index 0000000000..f69c301bb4 --- /dev/null +++ b/src/spicy/packet-analyzer.cc @@ -0,0 +1,73 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/spicy/packet-analyzer.h" + +#include "zeek/spicy/manager.h" +#include "zeek/spicy/runtime-support.h" + +using namespace zeek; +using namespace zeek::spicy; +using namespace zeek::spicy::rt; + +#ifdef DEBUG +#define STATE_DEBUG_MSG(...) DebugMsg(__VA_ARGS__) +#else +#define STATE_DEBUG_MSG(...) +#endif + +void PacketState::debug(const std::string& msg) { spicy::rt::debug(_cookie, msg); } + +static auto create_packet_state(PacketAnalyzer* analyzer) { + cookie::PacketAnalyzer cookie; + cookie.analyzer = analyzer; + return PacketState(std::move(cookie)); +} + +PacketAnalyzer::PacketAnalyzer(std::string name) + : packet_analysis::Analyzer(std::move(name)), _state(create_packet_state(this)) {} + +PacketAnalyzer::~PacketAnalyzer() = default; + +bool PacketAnalyzer::AnalyzePacket(size_t len, const uint8_t* data, Packet* packet) { + if ( auto parser = spicy_mgr->parserForPacketAnalyzer(_state.packet().analyzer->GetAnalyzerTag()) ) + _state.setParser(parser); + else + reporter->FatalError("no valid unit specified for parsing"); + + try { + hilti::rt::context::CookieSetter _(_state.cookie()); + _state.packet().next_analyzer.reset(); + _state.packet().packet = packet; + _state.process(len, reinterpret_cast(data)); + auto offset = _state.finish(); + assert(offset); + _state.packet().packet = nullptr; + _state.packet().packet_val = nullptr; + _state.reset(); + auto num_processed = offset->Ref(); + const auto& next_analyzer = _state.packet().next_analyzer; + STATE_DEBUG_MSG(hilti::rt::fmt("processed %" PRIu64 " out of %" PRIu64 " bytes, %s", num_processed, len, + (next_analyzer ? hilti::rt::fmt("next analyzer is 0x%" PRIx32, *next_analyzer) : + std::string("no next analyzer")))); + if ( next_analyzer ) + return ForwardPacket(len - num_processed, data + num_processed, packet, *next_analyzer); + else + return true; + } catch ( const hilti::rt::RuntimeError& e ) { + STATE_DEBUG_MSG(hilti::rt::fmt("error during parsing, triggering analyzer violation: %s", e.what())); + auto tag = _state.packet().analyzer->GetAnalyzerTag(); + + if ( auto* session = packet->session ) + _state.packet().analyzer->AnalyzerViolation(e.what(), session, reinterpret_cast(data), len, + tag); + + _state.reset(); + return false; + } catch ( const hilti::rt::Exception& e ) { + STATE_DEBUG_MSG(e.what()); + spicy_mgr->analyzerError(_state.packet().analyzer, e.description(), + e.location()); // this sets Zeek to skip sending any further input + _state.reset(); + return false; + } +} diff --git a/src/spicy/packet-analyzer.h b/src/spicy/packet-analyzer.h new file mode 100644 index 0000000000..7d68e322db --- /dev/null +++ b/src/spicy/packet-analyzer.h @@ -0,0 +1,75 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include +#include +#include +#include + +#include + +#include +#include + +#include "zeek/spicy/cookie.h" + +namespace zeek::spicy::rt { + +/** Parsing state for a packet. */ +class PacketState : public ::spicy::rt::driver::ParsingState { +public: + /** + * Constructor. + * + * @param cookie cookie to associated with the packet + */ + PacketState(Cookie cookie) : ParsingState(::spicy::rt::driver::ParsingType::Block), _cookie(std::move(cookie)) {} + + /** Returns the cookie pointer to use with the runtime library during analysis. */ + auto* cookie() { return &_cookie; } + + /** Returns the packet-specific cookie state associated with the endpoint. */ + auto& packet() { + assert(_cookie.packet); + return *_cookie.packet; + } + + /** + * Records a debug message pertaining to the specific file. + * + * @param msg message to record + */ + void DebugMsg(const std::string& msg) { debug(msg); } + +protected: + // Overridden from driver::ParsingState. + void debug(const std::string& msg) override; + +private: + Cookie _cookie; +}; + +/** A Spicy file analyzer. */ +class PacketAnalyzer : public packet_analysis::Analyzer { +public: + PacketAnalyzer(std::string name); + virtual ~PacketAnalyzer(); + + /** Records a debug message. */ + void DebugMsg(const std::string& msg) { _state.DebugMsg(msg); } + + static packet_analysis::AnalyzerPtr Instantiate(std::string name) { + name = util::canonify_name(name); + return std::make_shared(name); + } + +protected: + // Overridden from Zeek's packet analyzer. + bool AnalyzePacket(size_t len, const uint8_t* data, Packet* packet) override; + +private: + PacketState _state; +}; + +} // namespace zeek::spicy::rt diff --git a/src/spicy/protocol-analyzer.cc b/src/spicy/protocol-analyzer.cc new file mode 100644 index 0000000000..821356ecf7 --- /dev/null +++ b/src/spicy/protocol-analyzer.cc @@ -0,0 +1,231 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/spicy/protocol-analyzer.h" + +#include "zeek/spicy/manager.h" +#include "zeek/spicy/runtime-support.h" + +using namespace zeek; +using namespace zeek::spicy; +using namespace zeek::spicy::rt; + +#ifdef DEBUG +#define STATE_DEBUG_MSG(...) DebugMsg(__VA_ARGS__) +#else +#define STATE_DEBUG_MSG(...) +#endif + +void EndpointState::debug(const std::string& msg) { spicy::rt::debug(_cookie, msg); } + +static auto create_endpoint(bool is_orig, analyzer::Analyzer* analyzer, ::spicy::rt::driver::ParsingType type) { + cookie::ProtocolAnalyzer cookie{.analyzer = analyzer, + .is_orig = is_orig, + .fstate_orig = cookie::FileStateStack(hilti::rt::fmt("%x.orig", analyzer->GetID())), + .fstate_resp = + cookie::FileStateStack(hilti::rt::fmt("%x.resp", analyzer->GetID()))}; + + // Cannot get parser here yet, analyzer may not have been fully set up. + return EndpointState(std::move(cookie), type); +} + +ProtocolAnalyzer::ProtocolAnalyzer(analyzer::Analyzer* analyzer, ::spicy::rt::driver::ParsingType type) + : _originator(create_endpoint(true, analyzer, type)), _responder(create_endpoint(false, analyzer, type)) {} + +ProtocolAnalyzer::~ProtocolAnalyzer() {} + +void ProtocolAnalyzer::Init() {} + +void ProtocolAnalyzer::Done() { + Finish(true); + Finish(false); +} + +void ProtocolAnalyzer::Process(bool is_orig, int len, const u_char* data) { + auto* endp = is_orig ? &_originator : &_responder; + + if ( endp->protocol().analyzer->Skipping() ) + return; + + if ( ! endp->hasParser() && ! endp->isSkipping() ) { + auto parser = spicy_mgr->parserForProtocolAnalyzer(endp->protocol().analyzer->GetAnalyzerTag(), is_orig); + if ( parser ) { + if ( ! _context ) + _context = parser->createContext(); + + endp->setParser(parser, _context); + } + else { + STATE_DEBUG_MSG(is_orig, "no unit specified for parsing"); + endp->skipRemaining(); + return; + } + } + + try { + hilti::rt::context::CookieSetter _(endp->cookie()); + endp->process(len, reinterpret_cast(data)); + } catch ( const hilti::rt::RuntimeError& e ) { + STATE_DEBUG_MSG(is_orig, hilti::rt::fmt("error during parsing, triggering analyzer violation: %s", e.what())); + auto tag = spicy_mgr->tagForProtocolAnalyzer(endp->protocol().analyzer->GetAnalyzerTag()); + endp->protocol().analyzer->AnalyzerViolation(e.what(), reinterpret_cast(data), len, tag); + originator().skipRemaining(); + responder().skipRemaining(); + endp->protocol().analyzer->SetSkip(true); + } catch ( const hilti::rt::Exception& e ) { + spicy_mgr->analyzerError(endp->protocol().analyzer, e.description(), + e.location()); // this sets Zeek to skip sending any further input + } +} + +void ProtocolAnalyzer::Finish(bool is_orig) { + auto* endp = is_orig ? &_originator : &_responder; + + if ( endp->protocol().analyzer->Skipping() ) + return; + + try { + hilti::rt::context::CookieSetter _(endp->cookie()); + endp->finish(); + } catch ( const hilti::rt::RuntimeError& e ) { + STATE_DEBUG_MSG(is_orig, hilti::rt::fmt("error during parsing, triggering analyzer violation: %s", e.what())); + auto tag = spicy_mgr->tagForProtocolAnalyzer(endp->protocol().analyzer->GetAnalyzerTag()); + endp->protocol().analyzer->AnalyzerViolation(e.what(), nullptr, 0, tag); + endp->skipRemaining(); + } catch ( const hilti::rt::Exception& e ) { + spicy_mgr->analyzerError(endp->protocol().analyzer, e.description(), + e.location()); // this sets Zeek to skip sending any further input + } +} + +cookie::ProtocolAnalyzer& ProtocolAnalyzer::cookie(bool is_orig) { + if ( is_orig ) + return _originator.protocol(); + else + return _responder.protocol(); +} + +void ProtocolAnalyzer::DebugMsg(bool is_orig, const std::string& msg) { + if ( is_orig ) + _originator.DebugMsg(msg); + else + _responder.DebugMsg(msg); +} + +void ProtocolAnalyzer::FlipRoles() { std::swap(_originator, _responder); } + +analyzer::Analyzer* TCP_Analyzer::InstantiateAnalyzer(Connection* conn) { return new TCP_Analyzer(conn); } + +TCP_Analyzer::TCP_Analyzer(Connection* conn) + : ProtocolAnalyzer(this, ::spicy::rt::driver::ParsingType::Stream), analyzer::tcp::TCP_ApplicationAnalyzer(conn) {} + +TCP_Analyzer::~TCP_Analyzer() {} + +void TCP_Analyzer::Init() { + analyzer::tcp::TCP_ApplicationAnalyzer::Init(); + ProtocolAnalyzer::Init(); +} + +void TCP_Analyzer::Done() { + analyzer::tcp::TCP_ApplicationAnalyzer::Done(); + ProtocolAnalyzer::Done(); + + EndOfData(true); + EndOfData(false); +} + +void TCP_Analyzer::DeliverStream(int len, const u_char* data, bool is_orig) { + analyzer::tcp::TCP_ApplicationAnalyzer::DeliverStream(len, data, is_orig); + + Process(is_orig, len, data); + + if ( originator().isFinished() && responder().isFinished() && + (! originator().isSkipping() || ! responder().isSkipping()) ) { + STATE_DEBUG_MSG(is_orig, "both endpoints finished, skipping all further TCP processing"); + originator().skipRemaining(); + responder().skipRemaining(); + + if ( is_orig ) // doesn't really matter which endpoint here. + originator().protocol().analyzer->SetSkip(true); + else + responder().protocol().analyzer->SetSkip(true); + } +} + +void TCP_Analyzer::Undelivered(uint64_t seq, int len, bool is_orig) { + analyzer::tcp::TCP_ApplicationAnalyzer::Undelivered(seq, len, is_orig); + + Process(is_orig, len, nullptr); +} + +void TCP_Analyzer::EndOfData(bool is_orig) { + analyzer::tcp::TCP_ApplicationAnalyzer::EndOfData(is_orig); + + if ( TCP() && TCP()->IsPartial() ) { + STATE_DEBUG_MSG(is_orig, "skipping end-of-data delivery on partial TCP connection"); + return; + } + + Finish(is_orig); +} + +void TCP_Analyzer::FlipRoles() { + analyzer::tcp::TCP_ApplicationAnalyzer::FlipRoles(); + ProtocolAnalyzer::FlipRoles(); +} + +void TCP_Analyzer::EndpointEOF(bool is_orig) { + analyzer::tcp::TCP_ApplicationAnalyzer::EndpointEOF(is_orig); + Finish(is_orig); +} + +void TCP_Analyzer::ConnectionClosed(analyzer::tcp::TCP_Endpoint* endpoint, analyzer::tcp::TCP_Endpoint* peer, + bool gen_event) { + analyzer::tcp::TCP_ApplicationAnalyzer::ConnectionClosed(endpoint, peer, gen_event); +} + +void TCP_Analyzer::ConnectionFinished(bool half_finished) { + analyzer::tcp::TCP_ApplicationAnalyzer::ConnectionFinished(half_finished); +} + +void TCP_Analyzer::ConnectionReset() { analyzer::tcp::TCP_ApplicationAnalyzer::ConnectionReset(); } + +void TCP_Analyzer::PacketWithRST() { analyzer::tcp::TCP_ApplicationAnalyzer::PacketWithRST(); } + +analyzer::Analyzer* UDP_Analyzer::InstantiateAnalyzer(Connection* conn) { return new UDP_Analyzer(conn); } + +UDP_Analyzer::UDP_Analyzer(Connection* conn) + : ProtocolAnalyzer(this, ::spicy::rt::driver::ParsingType::Block), analyzer::Analyzer(conn) {} + +UDP_Analyzer::~UDP_Analyzer() {} + +void UDP_Analyzer::Init() { + analyzer::Analyzer::Init(); + ProtocolAnalyzer::Init(); +} + +void UDP_Analyzer::Done() { + analyzer::Analyzer::Done(); + ProtocolAnalyzer::Done(); +} + +void UDP_Analyzer::DeliverPacket(int len, const u_char* data, bool is_orig, uint64_t seq, const IP_Hdr* ip, + int caplen) { + analyzer::Analyzer::DeliverPacket(len, data, is_orig, seq, ip, caplen); + + ++cookie(is_orig).num_packets; + Process(is_orig, len, data); +} + +void UDP_Analyzer::Undelivered(uint64_t seq, int len, bool is_orig) { + analyzer::Analyzer::Undelivered(seq, len, is_orig); +} + +void UDP_Analyzer::EndOfData(bool is_orig) { + analyzer::Analyzer::EndOfData(is_orig); + Finish(is_orig); +} + +void UDP_Analyzer::FlipRoles() { + analyzer::Analyzer::FlipRoles(); + ProtocolAnalyzer::FlipRoles(); +} diff --git a/src/spicy/protocol-analyzer.h b/src/spicy/protocol-analyzer.h new file mode 100644 index 0000000000..71be171aff --- /dev/null +++ b/src/spicy/protocol-analyzer.h @@ -0,0 +1,166 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include +#include +#include + +#include + +#include +#include + +#include "zeek/spicy/cookie.h" + +namespace zeek::spicy::rt { + +/** Parsing state for one endpoint of the connection. */ +class EndpointState : public ::spicy::rt::driver::ParsingState { +public: + /** + * Constructor. + * + * @param cookie cookie to associated with the endpoint + * @param type type of parsing, depending on whether it's a stream- or + * packet-based protocol + */ + EndpointState(Cookie cookie, ::spicy::rt::driver::ParsingType type) + : ParsingState(type), _cookie(std::move(cookie)) {} + + /** Returns the protocol-specific cookie state associated with the endpoint. */ + auto& protocol() { + assert(_cookie.protocol); + return *_cookie.protocol; + } + + /** Returns the cookie pointer to use with the runtime library during analysis. */ + auto* cookie() { return &_cookie; } + + /** + * Records a debug message pertaining to this specific endpoint. + * + * @param msg message to record + */ + void DebugMsg(const std::string& msg) { debug(msg); } + +protected: + // Overridden from driver::ParsingState. + void debug(const std::string& msg) override; + +private: + Cookie _cookie; +}; + +/** Base clase for Spicy protocol analyzers. */ +class ProtocolAnalyzer { +public: + ProtocolAnalyzer(analyzer::Analyzer* analyzer, ::spicy::rt::driver::ParsingType type); + virtual ~ProtocolAnalyzer(); + + /** Returns the originator-side parsing state. */ + auto& originator() { return _originator; } + + /** Returns the responder-side parsing state. */ + auto& responder() { return _responder; } + +protected: + /** Initialize analyzer. */ + void Init(); + + /** Shutdown analyzer. */ + void Done(); + + /** + * Signal that Zeek has flipped the direction of the connection, meaning + * that originator and responder state need to be swapped. + */ + void FlipRoles(); + + /** + * Feeds a chunk of data into one side's parsing. + * + * @param is_orig true to use originator-side endpoint state, false for responder + * @param len number of bytes valid in *data* + * @param data pointer to data + */ + void Process(bool is_orig, int len, const u_char* data); + + /** + * Finalizes parsing. After calling this, no more data must be passed + * into Process() for the corresponding side. + * + * @param is_orig true to finish originator-side parsing, false for responder + */ + void Finish(bool is_orig); + + /** + * Helper returning the protocol analyzer cookie for the requested side. + * + * @param is_orig tru to return the originator's state, false for the + * responder. + * @return protocol analyzer cookie for the requested side + */ + cookie::ProtocolAnalyzer& cookie(bool is_orig); + + /** + * Records a debug message. This forwards to `DebugMsg()` for the + * corresponding `EndpointState`. + */ + void DebugMsg(bool is_orig, const std::string& msg); + +private: + EndpointState _originator; /**< Originator-side state. */ + EndpointState _responder; /**< Responder-side state. */ + std::optional<::spicy::rt::UnitContext> _context; +}; + +/** + * Spicy analyzer for TCP application-layer protocols. Implements the + * standard Zeek API. + */ +class TCP_Analyzer : public ProtocolAnalyzer, public analyzer::tcp::TCP_ApplicationAnalyzer { +public: + TCP_Analyzer(Connection* conn); + virtual ~TCP_Analyzer(); + + // Overridden from Spicy's Analyzer. + void Init() override; + void Done() override; + void DeliverStream(int len, const u_char* data, bool orig) override; + void Undelivered(uint64_t seq, int len, bool orig) override; + void EndOfData(bool is_orig) override; + void FlipRoles() override; + + // Overridden from Zeek's TCP_ApplicationAnalyzer. + void EndpointEOF(bool is_orig) override; + void ConnectionClosed(analyzer::tcp::TCP_Endpoint* endpoint, analyzer::tcp::TCP_Endpoint* peer, + bool gen_event) override; + void ConnectionFinished(bool half_finished) override; + void ConnectionReset() override; + void PacketWithRST() override; + + static analyzer::Analyzer* InstantiateAnalyzer(Connection* conn); +}; + +/** + * Spicy analyzer for UDP application-layer protocols. Implements the + * standard Zeek API. + */ +class UDP_Analyzer : public ProtocolAnalyzer, public analyzer::Analyzer { +public: + UDP_Analyzer(Connection* conn); + virtual ~UDP_Analyzer(); + + // Overridden from Spicy's Analyzer. + void Init() override; + void Done() override; + void DeliverPacket(int len, const u_char* data, bool orig, uint64_t seq, const IP_Hdr* ip, int caplen) override; + void Undelivered(uint64_t seq, int len, bool orig) override; + void EndOfData(bool is_orig) override; + void FlipRoles() override; + + static analyzer::Analyzer* InstantiateAnalyzer(Connection* conn); +}; + +} // namespace zeek::spicy::rt diff --git a/src/spicy/runtime-support.cc b/src/spicy/runtime-support.cc new file mode 100644 index 0000000000..9efac938ea --- /dev/null +++ b/src/spicy/runtime-support.cc @@ -0,0 +1,842 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/spicy/runtime-support.h" + +#include +#include + +#include +#include +#include +#include + +#include "zeek/Event.h" +#include "zeek/analyzer/Manager.h" +#include "zeek/analyzer/protocol/pia/PIA.h" +#include "zeek/file_analysis/File.h" +#include "zeek/file_analysis/Manager.h" +#include "zeek/spicy/manager.h" + +using namespace zeek; +using namespace zeek::spicy; + +void rt::register_protocol_analyzer(const std::string& name, hilti::rt::Protocol proto, + const hilti::rt::Vector& ports, const std::string& parser_orig, + const std::string& parser_resp, const std::string& replaces, + const std::string& linker_scope) { + auto _ = hilti::rt::profiler::start("zeek/rt/register_protocol_analyzer"); + spicy_mgr->registerProtocolAnalyzer(name, proto, ports, parser_orig, parser_resp, replaces, linker_scope); +} + +void rt::register_file_analyzer(const std::string& name, const hilti::rt::Vector& mime_types, + const std::string& parser, const std::string& replaces, + const std::string& linker_scope) { + auto _ = hilti::rt::profiler::start("zeek/rt/register_file_analyzer"); + spicy_mgr->registerFileAnalyzer(name, mime_types, parser, replaces, linker_scope); +} + +void rt::register_packet_analyzer(const std::string& name, const std::string& parser, const std::string& replaces, + const std::string& linker_scope) { + auto _ = hilti::rt::profiler::start("zeek/rt/register_packet_analyzer"); + spicy_mgr->registerPacketAnalyzer(name, parser, replaces, linker_scope); +} + +void rt::register_type(const std::string& ns, const std::string& id, const TypePtr& type) { + auto _ = hilti::rt::profiler::start("zeek/rt/register_type"); + spicy_mgr->registerType(hilti::rt::fmt("%s::%s", (! ns.empty() ? ns : std::string("GLOBAL")), id), type); +} + +// Helper to look up a global Zeek-side type, enforcing that it's of the expected type. +static TypePtr findType(TypeTag tag, const std::string& ns, const std::string& id) { + auto id_ = hilti::rt::fmt("%s::%s", ns, id); + auto type = spicy_mgr->findType(id_); + + if ( ! type ) + return nullptr; + + if ( type->Tag() != tag ) + reporter->FatalError("ID %s is not of expected type %s", id_.c_str(), type_name(tag)); + + return type; +} + +TypePtr rt::create_base_type(ZeekTypeTag tag) { + auto _ = hilti::rt::profiler::start("zeek/rt/create_base_type"); + TypeTag ztag; + + switch ( tag ) { + case ZeekTypeTag::Addr: ztag = TYPE_ADDR; break; + case ZeekTypeTag::Any: ztag = TYPE_ANY; break; + case ZeekTypeTag::Bool: ztag = TYPE_BOOL; break; + case ZeekTypeTag::Count: ztag = TYPE_COUNT; break; + case ZeekTypeTag::Double: ztag = TYPE_DOUBLE; break; + case ZeekTypeTag::Enum: ztag = TYPE_ENUM; break; + case ZeekTypeTag::Error: ztag = TYPE_ERROR; break; + case ZeekTypeTag::File: ztag = TYPE_FILE; break; + case ZeekTypeTag::Func: ztag = TYPE_FUNC; break; + case ZeekTypeTag::List: ztag = TYPE_LIST; break; + case ZeekTypeTag::Int: ztag = TYPE_INT; break; + case ZeekTypeTag::Interval: ztag = TYPE_INTERVAL; break; + case ZeekTypeTag::Opaque: ztag = TYPE_OPAQUE; break; + case ZeekTypeTag::Pattern: ztag = TYPE_PATTERN; break; + case ZeekTypeTag::Port: ztag = TYPE_PORT; break; + case ZeekTypeTag::Record: ztag = TYPE_RECORD; break; + case ZeekTypeTag::String: ztag = TYPE_STRING; break; + case ZeekTypeTag::Subnet: ztag = TYPE_SUBNET; break; + case ZeekTypeTag::Table: ztag = TYPE_TABLE; break; + case ZeekTypeTag::Time: ztag = TYPE_TIME; break; + case ZeekTypeTag::Type: ztag = TYPE_TYPE; break; + case ZeekTypeTag::Vector: ztag = TYPE_VECTOR; break; + case ZeekTypeTag::Void: ztag = TYPE_VOID; break; + default: hilti::rt::cannot_be_reached(); + } + + return base_type(ztag); +} + +TypePtr rt::create_enum_type( + const std::string& ns, const std::string& id, + const hilti::rt::Vector>>& labels) { + auto _ = hilti::rt::profiler::start("zeek/rt/create_enum_type"); + + if ( auto t = findType(TYPE_ENUM, ns, id) ) + return t; + + auto etype = make_intrusive(ns + "::" + id); + + for ( auto [lid, lval] : labels ) { + auto name = ::hilti::rt::fmt("%s_%s", id, lid); + + if ( lval == -1 ) + // Zeek's enum can't be negative, so swap in max_int for our Undef. + lval = std::numeric_limits<::zeek_int_t>::max(); + + etype->AddName(ns, name.c_str(), lval, true); + } + + return etype; +} + +TypePtr rt::create_record_type(const std::string& ns, const std::string& id, + const hilti::rt::Vector& fields) { + auto _ = hilti::rt::profiler::start("zeek/rt/create_record_type"); + + if ( auto t = findType(TYPE_RECORD, ns, id) ) + return t; + + auto decls = std::make_unique(); + + for ( const auto& [id, type, optional] : fields ) { + auto attrs = make_intrusive(nullptr, true, false); + + if ( optional ) { + auto optional_ = make_intrusive(detail::ATTR_OPTIONAL); + attrs->AddAttr(optional_); + } + + decls->append(new TypeDecl(util::copy_string(id.c_str()), type, std::move(attrs))); + } + + return make_intrusive(decls.release()); +} + +TypePtr rt::create_table_type(TypePtr key, std::optional value) { + auto _ = hilti::rt::profiler::start("zeek/rt/create_table_type"); + auto idx = make_intrusive(); + idx->Append(std::move(key)); + return make_intrusive(std::move(idx), value ? *value : nullptr); +} + +TypePtr rt::create_vector_type(const TypePtr& elem) { + auto _ = hilti::rt::profiler::start("zeek/rt/create_vector_type"); + return make_intrusive(elem); +} + +void rt::install_handler(const std::string& name) { + auto _ = hilti::rt::profiler::start("zeek/rt/install_handler"); + spicy_mgr->registerEvent(name); +} + +EventHandlerPtr rt::internal_handler(const std::string& name) { + auto _ = hilti::rt::profiler::start("zeek/rt/internal_handler"); + auto handler = event_registry->Lookup(name); + + if ( ! handler ) + reporter->InternalError("Spicy event %s was not installed", name.c_str()); + + return handler; +} + +void rt::raise_event(const EventHandlerPtr& handler, const hilti::rt::Vector& args) { + auto _ = hilti::rt::profiler::start("zeek/rt/raise_event"); + + // Caller must have checked already that there's a handler available. + assert(handler); + + const auto& zeek_args = const_cast(handler)->GetType()->ParamList()->GetTypes(); + if ( args.size() != static_cast(zeek_args.size()) ) + throw TypeMismatch(hilti::rt::fmt("expected %" PRIu64 " parameters, but got %zu", + static_cast(zeek_args.size()), args.size())); + + Args vl = Args(); + vl.reserve(args.size()); + for ( const auto& v : args ) { + if ( v ) + vl.emplace_back(v); + else + // Shouldn't happen here, but we have to_vals() that + // (legitimately) return null in certain contexts. + throw InvalidValue("null value encountered after conversion"); + } + + event_mgr.Enqueue(handler, std::move(vl)); +} + +TypePtr rt::event_arg_type(const EventHandlerPtr& handler, const hilti::rt::integer::safe& idx) { + auto _ = hilti::rt::profiler::start("zeek/rt/event_arg_type"); + assert(handler); + + const auto& zeek_args = const_cast(handler)->GetType()->ParamList()->GetTypes(); + if ( idx >= static_cast(zeek_args.size()) ) + throw TypeMismatch(hilti::rt::fmt("more parameters given than the %" PRIu64 " that the Zeek event expects", + static_cast(zeek_args.size()))); + + return zeek_args[idx]; +} + +ValPtr& rt::current_conn() { + auto _ = hilti::rt::profiler::start("zeek/rt/current_conn"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( cookie->cache.conn ) + return cookie->cache.conn; + + if ( auto x = cookie->protocol ) { + cookie->cache.conn = x->analyzer->Conn()->GetVal(); + return cookie->cache.conn; + } + else + throw ValueUnavailable("$conn not available"); +} + +ValPtr& rt::current_is_orig() { + auto _ = hilti::rt::profiler::start("zeek/rt/current_is_orig"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( cookie->cache.is_orig ) + return cookie->cache.is_orig; + + if ( auto x = cookie->protocol ) { + cookie->cache.is_orig = val_mgr->Bool(x->is_orig); + return cookie->cache.is_orig; + } + else + throw ValueUnavailable("$is_orig not available"); +} + +void rt::debug(const std::string& msg) { + auto _ = hilti::rt::profiler::start("zeek/rt/debug"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + rt::debug(*cookie, msg); +} + +void rt::debug(const Cookie& cookie, const std::string& msg) { + auto _ = hilti::rt::profiler::start("zeek/rt/debug"); + std::string name; + std::string id; + + if ( const auto p = cookie.protocol ) { + auto name = p->analyzer->GetAnalyzerName(); + SPICY_DEBUG( + hilti::rt::fmt("[%s/%" PRIu32 "/%s] %s", name, p->analyzer->GetID(), (p->is_orig ? "orig" : "resp"), msg)); + } + else if ( const auto f = cookie.file ) { + auto name = file_mgr->GetComponentName(f->analyzer->Tag()); + SPICY_DEBUG(hilti::rt::fmt("[%s/%" PRIu32 "] %s", name, f->analyzer->GetID(), msg)); + } + else if ( const auto f = cookie.packet ) { + auto name = packet_mgr->GetComponentName(f->analyzer->GetAnalyzerTag()); + SPICY_DEBUG(hilti::rt::fmt("[%s] %s", name, msg)); + } + else + throw ValueUnavailable("neither $conn nor $file nor packet analyzer available for debug logging"); +} + +inline rt::cookie::FileStateStack* _file_state_stack(rt::Cookie* cookie) { + auto _ = hilti::rt::profiler::start("zeek/rt/file_state_stack"); + + if ( auto c = cookie->protocol ) + return c->is_orig ? &c->fstate_orig : &c->fstate_resp; + else if ( auto f = cookie->file ) + return &f->fstate; + else + throw rt::ValueUnavailable("no current connection or file available"); +} + +inline const rt::cookie::FileState* _file_state(rt::Cookie* cookie, std::optional fid) { + auto _ = hilti::rt::profiler::start("zeek/rt/file_state"); + + auto* stack = _file_state_stack(cookie); + if ( fid ) { + if ( auto* fstate = stack->find(*fid) ) + return fstate; + else + throw rt::ValueUnavailable(hilti::rt::fmt("no file analysis currently in flight for file ID %s", fid)); + } + else { + if ( stack->isEmpty() ) + throw rt::ValueUnavailable("no file analysis currently in flight"); + + return stack->current(); + } +} + +ValPtr rt::current_file() { + auto _ = hilti::rt::profiler::start("zeek/rt/current_file"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( auto x = cookie->file ) + return x->analyzer->GetFile()->ToVal(); + else if ( auto* fstate = _file_state(cookie, {}) ) { + if ( auto* f = file_mgr->LookupFile(fstate->fid) ) + return f->ToVal(); + } + + throw ValueUnavailable("$file not available"); +} + +ValPtr rt::current_packet() { + auto _ = hilti::rt::profiler::start("zeek/rt/current_packet"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( auto c = cookie->packet ) { + if ( ! c->packet_val ) + // We cache the built value in case we need it multiple times. + c->packet_val = c->packet->ToRawPktHdrVal(); + + return c->packet_val; + } + else + throw ValueUnavailable("$packet not available"); +} + +hilti::rt::Bool rt::is_orig() { + auto _ = hilti::rt::profiler::start("zeek/rt/is_orig"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( auto x = cookie->protocol ) + return x->is_orig; + else + throw ValueUnavailable("is_orig() not available in current context"); +} + +std::string rt::uid() { + auto _ = hilti::rt::profiler::start("zeek/rt/uid"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( auto c = cookie->protocol ) { + // Retrieve the ConnVal() so that we ensure the UID has been set. + c->analyzer->ConnVal(); + return c->analyzer->Conn()->GetUID().Base62("C"); + } + else + throw ValueUnavailable("uid() not available in current context"); +} + +std::tuple rt::conn_id() { + auto _ = hilti::rt::profiler::start("zeek/rt/conn_id"); + + static auto convert_address = [](const IPAddr& zaddr) -> hilti::rt::Address { + const uint32_t* bytes = nullptr; + if ( auto n = zaddr.GetBytes(&bytes); n == 1 ) + // IPv4 + return hilti::rt::Address(*reinterpret_cast(bytes)); + else if ( n == 4 ) + // IPv6 + return hilti::rt::Address(*reinterpret_cast(bytes)); + else + throw ValueUnavailable("unexpected IP address side from Zeek"); // shouldn't really be able to happen + }; + + static auto convert_port = [](uint32_t port, TransportProto proto) -> hilti::rt::Port { + auto p = ntohs(static_cast(port)); + + switch ( proto ) { + case TransportProto::TRANSPORT_ICMP: return {p, hilti::rt::Protocol::ICMP}; + case TransportProto::TRANSPORT_TCP: return {p, hilti::rt::Protocol::TCP}; + case TransportProto::TRANSPORT_UDP: return {p, hilti::rt::Protocol::UDP}; + case TransportProto::TRANSPORT_UNKNOWN: return {p, hilti::rt::Protocol::Undef}; + } + + hilti::rt::cannot_be_reached(); + }; + + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( auto c = cookie->protocol ) { + const auto* conn = c->analyzer->Conn(); + return std::make_tuple(convert_address(conn->OrigAddr()), convert_port(conn->OrigPort(), conn->ConnTransport()), + convert_address(conn->RespAddr()), + convert_port(conn->RespPort(), conn->ConnTransport())); + } + else + throw ValueUnavailable("conn_id() not available in current context"); +} + +void rt::flip_roles() { + auto _ = hilti::rt::profiler::start("zeek/rt/flip_roles"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + rt::debug(*cookie, "flipping roles"); + + if ( auto x = cookie->protocol ) + x->analyzer->Conn()->FlipRoles(); + else + throw ValueUnavailable("flip_roles() not available in current context"); +} + +hilti::rt::integer::safe rt::number_packets() { + auto _ = hilti::rt::profiler::start("zeek/rt/number_packets"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( auto x = cookie->protocol ) { + return x->num_packets; + } + else + throw ValueUnavailable("number_packets() not available in current context"); +} + +void rt::confirm_protocol() { + auto _ = hilti::rt::profiler::start("zeek/rt/confirm_protocol"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( cookie->cache.confirmed ) + return; + + if ( auto x = cookie->protocol ) { + auto tag = spicy_mgr->tagForProtocolAnalyzer(x->analyzer->GetAnalyzerTag()); + SPICY_DEBUG(hilti::rt::fmt("confirming protocol %s", tag.AsString())); + cookie->cache.confirmed = true; + return x->analyzer->AnalyzerConfirmation(tag); + } + throw ValueUnavailable("no current connection available"); +} + +void rt::reject_protocol(const std::string& reason) { + auto _ = hilti::rt::profiler::start("zeek/rt/reject_protocol"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( auto x = cookie->protocol ) { + auto tag = spicy_mgr->tagForProtocolAnalyzer(x->analyzer->GetAnalyzerTag()); + SPICY_DEBUG(hilti::rt::fmt("rejecting protocol %s", tag.AsString())); + return x->analyzer->AnalyzerViolation("protocol rejected", nullptr, 0, tag); + } + else + throw ValueUnavailable("no current connection available"); +} + +void rt::weird(const std::string& id, const std::string& addl) { + auto _ = hilti::rt::profiler::start("zeek/rt/weird"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( const auto x = cookie->protocol ) + x->analyzer->Weird(id.c_str(), addl.data()); + else if ( const auto x = cookie->file ) + zeek::reporter->Weird(x->analyzer->GetFile(), id.c_str(), addl.data()); + else if ( const auto x = cookie->packet ) { + x->analyzer->Weird(id.c_str(), x->packet, addl.c_str()); + } + else + throw ValueUnavailable("none of $conn, $file, or $packet available for weird reporting"); +} + +void rt::protocol_begin(const std::optional& analyzer) { + auto _ = hilti::rt::profiler::start("zeek/rt/protocol_begin"); + + if ( analyzer ) { + protocol_handle_get_or_create(*analyzer); + return; + } + + // Instantiate a DPD analyzer. + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + auto c = cookie->protocol; + if ( ! c ) + throw ValueUnavailable("no current connection available"); + + // Use a Zeek PIA stream analyzer performing DPD. + auto pia_tcp = std::make_unique(c->analyzer->Conn()); + + // Forward empty payload to trigger lifecycle management in this analyzer tree. + c->analyzer->ForwardStream(0, reinterpret_cast(c->analyzer), true); + c->analyzer->ForwardStream(0, reinterpret_cast(c->analyzer), false); + + // Direct child of this type already exists. We ignore this silently + // because that makes usage nicer if either side of the connection + // might end up creating the analyzer; this way the user doesn't + // need to track what the other side already did. + // + // We inspect the children directly to work around zeek/zeek#2899. + const auto& children = c->analyzer->GetChildren(); + if ( auto it = std::find_if(children.begin(), children.end(), + [&](const auto& it) { + return ! it->Removing() && ! it->IsFinished() && + it->GetAnalyzerTag() == pia_tcp->GetAnalyzerTag(); + }); + it != children.end() ) + return; + + auto child = pia_tcp.release(); + c->analyzer->AddChildAnalyzer(child); + + child->FirstPacket(true, nullptr); + child->FirstPacket(false, nullptr); +} + +rt::ProtocolHandle rt::protocol_handle_get_or_create(const std::string& analyzer) { + auto _ = hilti::rt::profiler::start("zeek/rt/protocol_handle_get_or_create"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + auto c = cookie->protocol; + if ( ! c ) + throw ValueUnavailable("no current connection available"); + + // Forward empty payload to trigger lifecycle management in this analyzer tree. + c->analyzer->ForwardStream(0, reinterpret_cast(c->analyzer), true); + c->analyzer->ForwardStream(0, reinterpret_cast(c->analyzer), false); + + // If the child already exists, do not add it again so this function is idempotent. + // + // We inspect the children directly to work around zeek/zeek#2899. + const auto& children = c->analyzer->GetChildren(); + if ( auto it = std::find_if(children.begin(), children.end(), + [&](const auto& it) { + return ! it->Removing() && ! it->IsFinished() && it->GetAnalyzerName() == analyzer; + }); + it != children.end() ) + return rt::ProtocolHandle((*it)->GetID()); + + auto child = analyzer_mgr->InstantiateAnalyzer(analyzer.c_str(), c->analyzer->Conn()); + if ( ! child ) + throw ZeekError(::hilti::rt::fmt("unknown analyzer '%s' requested", analyzer)); + + // If we had no such child before but cannot add it the analyzer was prevented. + // + // NOTE: We make this a hard error since returning e.g., an empty optional + // here would make it easy to incorrectly use the return value with e.g., + // `protocol_data_in` or `protocol_gap`. + if ( ! c->analyzer->AddChildAnalyzer(child) ) + throw ZeekError(::hilti::rt::fmt("creation of child analyzer %s was prevented", analyzer)); + + if ( c->analyzer->Conn()->ConnTransport() != TRANSPORT_TCP ) { + // Some TCP application analyzer may expect to have access to a TCP + // analyzer. To make that work, we'll create a fake TCP analyzer, + // just so that they have something to access. It won't + // semantically have any "TCP" to analyze obviously. + c->fake_tcp = std::make_shared(c->analyzer->Conn()); + static_cast(c->fake_tcp.get()) + ->Done(); // will never see packets; cast to get around protected inheritance + } + + auto* child_as_tcp = dynamic_cast(child); + if ( ! child_as_tcp ) + throw ZeekError( + ::hilti::rt::fmt("could not add analyzer '%s' to connection; not a TCP-based analyzer", analyzer)); + + if ( c->fake_tcp ) + child_as_tcp->SetTCP(c->fake_tcp.get()); + + return rt::ProtocolHandle(child->GetID()); +} + +void rt::protocol_data_in(const hilti::rt::Bool& is_orig, const hilti::rt::Bytes& data, + const std::optional& h) { + auto _ = hilti::rt::profiler::start("zeek/rt/protocol_data_in"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + auto c = cookie->protocol; + if ( ! c ) + throw ValueUnavailable("no current connection available"); + + auto len = data.size(); + auto* data_ = reinterpret_cast(data.data()); + + if ( h ) { + if ( auto* output_handler = c->analyzer->GetOutputHandler() ) + output_handler->DeliverStream(len, data_, is_orig); + + auto* child = c->analyzer->FindChild(h->id()); + if ( ! child ) + throw ValueUnavailable(hilti::rt::fmt("unknown child analyzer %s", *h)); + + if ( child->IsFinished() || child->Removing() ) + throw ValueUnavailable(hilti::rt::fmt("child analyzer %s no longer exist", *h)); + + child->NextStream(len, data_, is_orig); + } + + else + c->analyzer->ForwardStream(len, data_, is_orig); +} + +void rt::protocol_gap(const hilti::rt::Bool& is_orig, const hilti::rt::integer::safe& offset, + const hilti::rt::integer::safe& len, const std::optional& h) { + auto _ = hilti::rt::profiler::start("zeek/rt/protocol_gap"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + auto c = cookie->protocol; + if ( ! c ) + throw ValueUnavailable("no current connection available"); + + if ( h ) { + if ( auto* output_handler = c->analyzer->GetOutputHandler() ) + output_handler->Undelivered(offset, len, is_orig); + + auto* child = c->analyzer->FindChild(h->id()); + if ( ! child ) + throw ValueUnavailable(hilti::rt::fmt("unknown child analyzer %s", *h)); + + if ( child->IsFinished() || child->Removing() ) + throw ValueUnavailable(hilti::rt::fmt("child analyzer %s no longer exist", *h)); + + child->NextUndelivered(offset, len, is_orig); + } + + else + c->analyzer->ForwardUndelivered(offset, len, is_orig); +} + +void rt::protocol_end() { + auto _ = hilti::rt::profiler::start("zeek/rt/protocol_end"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + auto c = cookie->protocol; + if ( ! c ) + throw ValueUnavailable("no current connection available"); + + for ( const auto& i : c->analyzer->GetChildren() ) + c->analyzer->RemoveChildAnalyzer(i); +} + +void rt::protocol_handle_close(const ProtocolHandle& handle) { + auto _ = hilti::rt::profiler::start("zeek/rt/protocol_handle_close"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + auto c = cookie->protocol; + if ( ! c ) + throw ValueUnavailable("no current connection available"); + + auto child = c->analyzer->FindChild(handle.id()); + if ( ! child ) + throw ValueUnavailable(hilti::rt::fmt("unknown child analyzer %s", handle)); + + if ( child->IsFinished() || child->Removing() ) + throw ValueUnavailable(hilti::rt::fmt("child analyzer %s no longer exist", handle)); + + child->NextEndOfData(true); + child->NextEndOfData(false); + + c->analyzer->RemoveChildAnalyzer(handle.id()); +} + +rt::cookie::FileState* rt::cookie::FileStateStack::push() { + auto _ = hilti::rt::profiler::start("zeek/rt/file-stack-push"); + auto fid = file_mgr->HashHandle(hilti::rt::fmt("%s.%d", _analyzer_id, ++_id_counter)); + _stack.emplace_back(fid); + return &_stack.back(); +} + +const rt::cookie::FileState* rt::cookie::FileStateStack::find(const std::string& fid) const { + auto _ = hilti::rt::profiler::start("zeek/rt/file-stack-find"); + + // Reverse search as the default state would be on top of the stack. + for ( auto i = _stack.rbegin(); i != _stack.rend(); i++ ) { + if ( i->fid == fid ) + return &*i; + } + + return nullptr; +} + +void rt::cookie::FileStateStack::remove(const std::string& fid) { + auto _ = hilti::rt::profiler::start("zeek/rt/file-stack-remove"); + + // Reverse search as the default state would be on top of the stack. + for ( auto i = _stack.rbegin(); i != _stack.rend(); i++ ) { + if ( i->fid == fid ) { + _stack.erase((i + 1).base()); // https://stackoverflow.com/a/1830240 + return; + } + } +} + +static void _data_in(const char* data, uint64_t len, std::optional offset, + const std::optional& fid) { + auto cookie = static_cast(hilti::rt::context::cookie()); + auto* fstate = _file_state(cookie, fid); + auto data_ = reinterpret_cast(data); + auto mime_type = (fstate->mime_type ? *fstate->mime_type : std::string()); + + if ( auto c = cookie->protocol ) { + auto tag = spicy_mgr->tagForProtocolAnalyzer(c->analyzer->GetAnalyzerTag()); + + if ( offset ) + file_mgr->DataIn(data_, len, *offset, tag, c->analyzer->Conn(), c->is_orig, fstate->fid, mime_type); + else + file_mgr->DataIn(data_, len, tag, c->analyzer->Conn(), c->is_orig, fstate->fid, mime_type); + } + else { + if ( offset ) + file_mgr->DataIn(data_, len, *offset, Tag(), nullptr, false, fstate->fid, mime_type); + else + file_mgr->DataIn(data_, len, Tag(), nullptr, false, fstate->fid, mime_type); + } +} + +void rt::terminate_session() { + auto _ = hilti::rt::profiler::start("zeek/rt/terminate_session"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( auto c = cookie->protocol ) { + assert(session_mgr); + session_mgr->Remove(c->analyzer->Conn()); + } + else + throw spicy::rt::ValueUnavailable("terminate_session() not available in the current context"); +} + +std::string rt::fuid() { + auto _ = hilti::rt::profiler::start("zeek/rt/fuid"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( auto f = cookie->file ) { + if ( auto file = f->analyzer->GetFile() ) + return file->GetID(); + } + + throw ValueUnavailable("fuid() not available in current context"); +} + +std::string rt::file_begin(const std::optional& mime_type) { + auto _ = hilti::rt::profiler::start("zeek/rt/file_begin"); + auto cookie = static_cast(hilti::rt::context::cookie()); + auto* fstate = _file_state_stack(cookie)->push(); + fstate->mime_type = mime_type; + + // Feed an empty chunk into the analysis to force creating the file state inside Zeek. + _data_in("", 0, {}, {}); + + auto file = file_mgr->LookupFile(fstate->fid); + assert(file); // passing in empty data ensures that this is now available + + if ( auto f = cookie->file ) { + // We need to initialize some fa_info fields ourselves that would + // normally be inferred from the connection. + + // Set the source to the current file analyzer. + file->SetSource(file_mgr->GetComponentName(f->analyzer->Tag())); + + // There are some fields inside the new fa_info record that we want to + // set, but don't have a Zeek API for. Hence, we need to play some + // tricks: we can get to the fa_info value, but read-only; const_cast + // comes to our rescue. And then we just write directly into the + // record fields. + auto rval = file->ToVal()->AsRecordVal(); + auto current = f->analyzer->GetFile()->ToVal()->AsRecordVal(); + rval->Assign(id::fa_file->FieldOffset("parent_id"), + current->GetField("id")); // set to parent + rval->Assign(id::fa_file->FieldOffset("conns"), + current->GetField("conns")); // copy from parent + rval->Assign(id::fa_file->FieldOffset("is_orig"), + current->GetField("is_orig")); // copy from parent + } + + // Double check everybody agrees on the file ID. + assert(fstate->fid == file->GetID()); + return fstate->fid; +} + +void rt::file_set_size(const hilti::rt::integer::safe& size, const std::optional& fid) { + auto _ = hilti::rt::profiler::start("zeek/rt/file_set_size"); + auto cookie = static_cast(hilti::rt::context::cookie()); + auto* fstate = _file_state(cookie, fid); + + if ( auto c = cookie->protocol ) { + auto tag = spicy_mgr->tagForProtocolAnalyzer(c->analyzer->GetAnalyzerTag()); + file_mgr->SetSize(size, tag, c->analyzer->Conn(), c->is_orig, fstate->fid); + } + else + file_mgr->SetSize(size, Tag(), nullptr, false, fstate->fid); +} + +void rt::file_data_in(const hilti::rt::Bytes& data, const std::optional& fid) { + auto _ = hilti::rt::profiler::start("zeek/rt/file_data_in"); + _data_in(data.data(), data.size(), {}, fid); +} + +void rt::file_data_in_at_offset(const hilti::rt::Bytes& data, const hilti::rt::integer::safe& offset, + const std::optional& fid) { + auto _ = hilti::rt::profiler::start("zeek/rt/file_data_in_at_offset"); + _data_in(data.data(), data.size(), offset, fid); +} + +void rt::file_gap(const hilti::rt::integer::safe& offset, const hilti::rt::integer::safe& len, + const std::optional& fid) { + auto _ = hilti::rt::profiler::start("zeek/rt/file_gap"); + auto cookie = static_cast(hilti::rt::context::cookie()); + auto* fstate = _file_state(cookie, fid); + + if ( auto c = cookie->protocol ) { + auto tag = spicy_mgr->tagForProtocolAnalyzer(c->analyzer->GetAnalyzerTag()); + file_mgr->Gap(offset, len, tag, c->analyzer->Conn(), c->is_orig, fstate->fid); + } + else + file_mgr->Gap(offset, len, Tag(), nullptr, false, fstate->fid); +} + +void rt::file_end(const std::optional& fid) { + auto _ = hilti::rt::profiler::start("zeek/rt/file_end"); + auto cookie = static_cast(hilti::rt::context::cookie()); + auto* fstate = _file_state(cookie, fid); + + file_mgr->EndOfFile(fstate->fid); + _file_state_stack(cookie)->remove(fstate->fid); +} + +void rt::forward_packet(const hilti::rt::integer::safe& identifier) { + auto _ = hilti::rt::profiler::start("zeek/rt/forward_packet"); + auto cookie = static_cast(hilti::rt::context::cookie()); + assert(cookie); + + if ( auto c = cookie->packet ) + c->next_analyzer = identifier; + else + throw ValueUnavailable("no current packet analyzer available"); +} + +hilti::rt::Time rt::network_time() { + auto _ = hilti::rt::profiler::start("zeek/rt/network_time"); + return hilti::rt::Time(run_state::network_time, hilti::rt::Time::SecondTag()); +} diff --git a/src/spicy/runtime-support.h b/src/spicy/runtime-support.h new file mode 100644 index 0000000000..8bfc38b3f2 --- /dev/null +++ b/src/spicy/runtime-support.h @@ -0,0 +1,889 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +/** + * Functions and types available to generated Spicy/Zeek glue code. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "zeek/Desc.h" +#include "zeek/spicy/cookie.h" + +namespace zeek::spicy::rt { + +// Adapt to rename of exception. +#if SPICY_VERSION_NUMBER >= 10700 +using UsageError = ::hilti::rt::UsageError; +#else +using UsageError = ::hilti::rt::UserException; +#endif + +/** + * Exception thrown by event generation code if the value of an `$...` + * expression isn't available. + */ +class ValueUnavailable : public UsageError { +public: + using UsageError::UsageError; +}; + +/** + * Exception thrown by event generation code if the values can't be converted + * to Zeek. + */ +class InvalidValue : public UsageError { +public: + using UsageError::UsageError; +}; + +/** + * Exception thrown by event generation code if functionality is used + * that the current build does not support. + */ +class Unsupported : public UsageError { +public: + using UsageError::UsageError; +}; + +/** + * Exception thrown by event generation code if there's a type mismatch + * between the Spicy-side value and what the Zeek event expects. + */ +class TypeMismatch : public UsageError { +public: + TypeMismatch(const std::string_view& msg, std::string_view location = "") + : UsageError(hilti::rt::fmt("Event parameter mismatch, %s", msg)) {} + TypeMismatch(const std::string_view& have, TypePtr want, std::string_view location = "") + : TypeMismatch(_fmt(have, want)) {} + +private: + std::string _fmt(const std::string_view& have, TypePtr want) { + ODesc d; + want->Describe(&d); + return hilti::rt::fmt("cannot convert Spicy value of type '%s' to Zeek value of type '%s'", have, + d.Description()); + } +}; + +/** + * Exception thrown by the runtime library when Zeek has flagged a problem. + */ +class ZeekError : public UsageError { +public: + using UsageError::UsageError; +}; + +/** + * Registers a Spicy protocol analyzer with its EVT meta information with the + * plugin's runtime. + */ +void register_protocol_analyzer(const std::string& name, hilti::rt::Protocol proto, + const hilti::rt::Vector& ports, const std::string& parser_orig, + const std::string& parser_resp, const std::string& replaces, + const std::string& linker_scope); + +/** + * Registers a Spicy file analyzer with its EVT meta information with the + * plugin's runtime. + */ +void register_file_analyzer(const std::string& name, const hilti::rt::Vector& mime_types, + const std::string& parser, const std::string& replaces, const std::string& linker_scope); + +/** Reports a Zeek-side "weird". */ +void weird(const std::string& id, const std::string& addl); + +/** + * Registers a Spicy packet analyzer with its EVT meta information with the + * plugin's runtime. + */ +void register_packet_analyzer(const std::string& name, const std::string& parser, const std::string& replaces, + const std::string& linker_scope); + +/** Registers a Spicy-generated type to make it available inside Zeek. */ +void register_type(const std::string& ns, const std::string& id, const TypePtr& type); + +/** Identifies a Zeek-side type. */ +enum class ZeekTypeTag : uint64_t { + Addr, + Any, + Bool, + Count, + Double, + Enum, + Error, + File, + Func, + Int, + Interval, + List, + Opaque, + Pattern, + Port, + Record, + String, + Subnet, + Table, + Time, + Type, + Vector, + Void, +}; + +extern TypePtr create_base_type(ZeekTypeTag tag); + +extern TypePtr create_enum_type( + const std::string& ns, const std::string& id, + const hilti::rt::Vector>>& labels); + +using RecordField = std::tuple; // (ID, type, optional) +extern TypePtr create_record_type(const std::string& ns, const std::string& id, + const hilti::rt::Vector& fields); + +extern TypePtr create_table_type(TypePtr key, std::optional value); +extern TypePtr create_vector_type(const TypePtr& elem); + +/** Returns true if an event has at least one handler defined. */ +inline hilti::rt::Bool have_handler(const EventHandlerPtr& handler) { return static_cast(handler); } + +/** + * Creates a new event handler under the given name. + */ +void install_handler(const std::string& name); + +/** + * Looks up an event handler by name. The handler must have been installed + * before through `install_handler()`. + */ +EventHandlerPtr internal_handler(const std::string& name); + +/** Raises a Zeek event, given the handler and arguments. */ +void raise_event(const EventHandlerPtr& handler, const hilti::rt::Vector& args); + +/** + * Returns the Zeek type of an event's i'th argument. The result's ref count + * is not increased. + */ +TypePtr event_arg_type(const EventHandlerPtr& handler, const hilti::rt::integer::safe& idx); + +/** + * Retrieves the connection ID for the currently processed Zeek connection. + * Assumes that the HILTI context's cookie value has been set accordingly. + * + * @return Zeek value of record type + */ +ValPtr& current_conn(); + +/** + * Retrieves the direction of the currently processed Zeek connection. + * Assumes that the HILTI context's cookie value has been set accordingly. + * + * @return Zeek value of boolean type + */ +ValPtr& current_is_orig(); + +/** + * Logs a string through the Spicy plugin's debug output. + * + * @param cookie refers to the connection or file that the message is associated with + * @param msg message to log + */ +void debug(const Cookie& cookie, const std::string& msg); + +/** + * Logs a string through the Spicy plugin's debug output. This version logs + * the information the currently processed connection or file. + * + * @param msg message to log + */ +void debug(const std::string& msg); + +/** + * Retrieves the fa_file instance for the currently processed Zeek file. + * Assumes that the HILTI context's cookie value has been set accordingly. + * + * @return Zeek value of record type + */ +ValPtr current_file(); + +/** + * Retrieves a `raw_pkt_hdr` instance for the currently processed Zeek packet. + * Assumes that the HILTI context's cookie value has been set accordingly. + * + * @return Zeek value of record type + */ +ValPtr current_packet(); + +/** + * Returns true if we're currently parsing the originator side of a + * connection. + */ +hilti::rt::Bool is_orig(); + +/** + * Returns the current connection's UID. + */ +std::string uid(); + +/** + * Returns the current connection's ID tuple. + */ +std::tuple conn_id(); + +/** Instructs to Zeek to flip the directionality of the current connecction. */ +void flip_roles(); + +/** + * Returns the number of packets seen so far on the current side of the + * current connection. + */ +hilti::rt::integer::safe number_packets(); + +/** + * Triggers a DPD protocol confirmation for the currently processed + * connection. Assumes that the HILTI context's cookie value has been set + * accordingly. + */ +void confirm_protocol(); + +/** + * Triggers a DPD protocol violation for the currently processed connection. + * Assumes that the HILTI context's cookie value has been set accordingly. + * + * @param reason short description of what went wrong + */ +void reject_protocol(const std::string& reason); + +/** + * Opaque handle to a protocol analyzer. + */ +class ProtocolHandle { +public: + ProtocolHandle() {} + explicit ProtocolHandle(uint64_t id) : _id(id) {} + + uint64_t id() const { + if ( ! _id ) + throw ValueUnavailable("uninitialized protocol handle"); + + return *_id; + } + + friend std::string to_string(const ProtocolHandle& h, ::hilti::rt::detail::adl::tag) { + if ( ! h._id ) + return "(uninitialized protocol handle)"; + + return std::to_string(*h._id); + } + + friend std::ostream& operator<<(std::ostream& stream, const ProtocolHandle& h) { + return stream << ::hilti::rt::to_string(h); + } + +private: + std::optional _id; +}; + +/** + * Adds a Zeek-side child protocol analyzer to the current connection. + * + * @param analyzer if given, the Zeek-side name of the analyzer to instantiate; + * if not given, DPD will be used + */ +void protocol_begin(const std::optional& analyzer); + +/** + * Gets a handle to a child analyzer of a given type. If a child of that type + * does not yet exist it will be created. + * + * @param analyzer the Zeek-side name of the analyzer to get (e.g., `HTTP`) + * + * @return a handle to the child analyzer. When done, the handle should be + * closed, either explicitly with protocol_handle_close or implicitly with + * protocol_end. + */ +ProtocolHandle protocol_handle_get_or_create(const std::string& analyzer); + +/** + * Forwards data to all previously instantiated Zeek-side child protocol + * analyzers. + * + * @param is_orig true to feed data to originator side, false for responder + * @param data next chunk of stream data for child analyzer to process + * @param h optional handle to the child analyzer to stream data into + */ +void protocol_data_in(const hilti::rt::Bool& is_orig, const hilti::rt::Bytes& data, + const std::optional& h = {}); + +/** + * Signals a gap in input data to all previously instantiated Zeek-side child + * protocol analyzers. + * + * @param is_orig true to signal gap to originator side, false for responder + * @param offset of the gap inside the protocol stream + * @param length of the gap + * @param h optional handle to the child analyzer to signal a gap to + */ +void protocol_gap(const hilti::rt::Bool& is_orig, const hilti::rt::integer::safe& offset, + const hilti::rt::integer::safe& len, const std::optional& h = {}); + +/** + * Signals EOD to all previously instantiated Zeek-side child protocol + * analyzers and removes them. + */ +void protocol_end(); + +/** + * Closes a protocol handle. + * + * @param handle handle of the protocol analyzer to close. + */ +void protocol_handle_close(const ProtocolHandle& handle); + +/** + * Signals the beginning of a file to Zeek's file analysis, associating it + * with the current connection. + * + * param mime_type optional mime type passed to Zeek + * @returns Zeek-side file ID of the new file + */ +std::string file_begin(const std::optional& mime_type); + +/** + * Returns the current file's FUID. + */ +std::string fuid(); + +/** + * Terminates the currently active Zeek-side session, flushing all state. Any + * subsequent activity will start a new session from scratch. + */ +void terminate_session(); + +/** + * Signals the expected size of a file to Zeek's file analysis. + * + * @param size expected final size of the file + * @param fid ID of the file to operate on; if unset, the most recently begun file is used + */ +void file_set_size(const hilti::rt::integer::safe& size, const std::optional& fid = {}); + +/** + * Passes file content on to Zeek's file analysis. + * + * @param data next chunk of data + * @param fid ID of the file to operate on; if unset, the most recently begun file is used + */ +void file_data_in(const hilti::rt::Bytes& data, const std::optional& fid = {}); + +/** + * Passes file content at a specific offset on to Zeek's file analysis. + * + * @param data next chunk of data + * @param offset file offset of the data geing passed in + * @param fid ID of the file to operate on; if unset, the most recently begun file is used + */ +void file_data_in_at_offset(const hilti::rt::Bytes& data, const hilti::rt::integer::safe& offset, + const std::optional& fid = {}); + +/** + * Signals a gap in a file to Zeek's file analysis. + * + * @param offset of the gap + * @param length of the gap + * @param fid ID of the file to operate on; if unset, the most recently begun file is used + */ +void file_gap(const hilti::rt::integer::safe& offset, const hilti::rt::integer::safe& len, + const std::optional& fid = {}); + +/** + * Signals the end of a file to Zeek's file analysis. + * + * @param fid ID of the file to operate on; if unset, the most recently begun file is used + */ +void file_end(const std::optional& fid = {}); + +/** Specifies the next-layer packet analyzer. */ +void forward_packet(const hilti::rt::integer::safe& identifier); + +/** Gets the network time from Zeek. */ +hilti::rt::Time network_time(); + +// Forward-declare to_val() functions. +template::value>* = nullptr> +ValPtr to_val(const T& t, TypePtr target); +template::value>* = nullptr> +ValPtr to_val(const T& t, TypePtr target); +template::value>* = nullptr> +ValPtr to_val(const T& t, TypePtr target); +template::value>* = nullptr> +ValPtr to_val(const T& t, TypePtr target); +template +ValPtr to_val(const hilti::rt::Map& s, TypePtr target); +template +ValPtr to_val(const hilti::rt::Set& s, TypePtr target); +template +ValPtr to_val(const hilti::rt::Vector& v, TypePtr target); +template +ValPtr to_val(const std::optional& t, TypePtr target); +template +ValPtr to_val(const hilti::rt::DeferredExpression& t, TypePtr target); +template +ValPtr to_val(hilti::rt::integer::safe i, TypePtr target); +template +ValPtr to_val(const hilti::rt::ValueReference& t, TypePtr target); + +inline ValPtr to_val(const hilti::rt::Bool& b, TypePtr target); +inline ValPtr to_val(const hilti::rt::Address& d, TypePtr target); +inline ValPtr to_val(const hilti::rt::Bytes& b, TypePtr target); +inline ValPtr to_val(const hilti::rt::Interval& t, TypePtr target); +inline ValPtr to_val(const hilti::rt::Port& d, TypePtr target); +inline ValPtr to_val(const hilti::rt::Time& t, TypePtr target); +inline ValPtr to_val(const std::string& s, TypePtr target); +inline ValPtr to_val(double r, TypePtr target); + +/** + * Converts a Spicy-side optional value to a Zeek value. This assumes the + * optional is set, and will throw an exception if not. The result is + * returned with ref count +1. + */ +template +inline ValPtr to_val(const std::optional& t, TypePtr target) { + if ( t.has_value() ) + return to_val(hilti::rt::optional::value(t), target); + + return nullptr; +} + +/** + * Converts a Spicy-side DeferredExpression value to a Zeek value. Such + * result values are returned by the ``.?`` operator. If the result is not + * set, this will convert into nullptr (which the tuple-to-record to_val() + * picks up on). + */ +template +inline ValPtr to_val(const hilti::rt::DeferredExpression& t, TypePtr target) { + try { + return to_val(t(), target); + } catch ( const hilti::rt::AttributeNotSet& ) { + return nullptr; + } +} + +/** + * Converts a Spicy-side string to a Zeek value. The result is returned with + * ref count +1. + */ +inline ValPtr to_val(const std::string& s, TypePtr target) { + if ( target->Tag() != TYPE_STRING ) + throw TypeMismatch("string", target); + + return make_intrusive(s); +} + +/** + * Converts a Spicy-side bytes instance to a Zeek value. The result is returned with + * ref count +1. + */ +inline ValPtr to_val(const hilti::rt::Bytes& b, TypePtr target) { + if ( target->Tag() != TYPE_STRING ) + throw TypeMismatch("string", target); + + return make_intrusive(b.str()); +} + +/** + * Converts a Spicy-side integer to a Zeek value. The result is + * returned with ref count +1. + */ +template +inline ValPtr to_val(hilti::rt::integer::safe i, TypePtr target) { + ValPtr v = nullptr; + if constexpr ( std::is_unsigned::value ) { + if ( target->Tag() == TYPE_COUNT ) + return val_mgr->Count(i); + + if ( target->Tag() == TYPE_INT ) + return val_mgr->Int(i); + + throw TypeMismatch("uint64", target); + } + else { + if ( target->Tag() == TYPE_INT ) + return val_mgr->Int(i); + + if ( target->Tag() == TYPE_COUNT ) { + if ( i >= 0 ) + return val_mgr->Count(i); + else + throw TypeMismatch("negative int64", target); + } + + throw TypeMismatch("int64", target); + } +} + +template +ValPtr to_val(const hilti::rt::ValueReference& t, TypePtr target) { + if ( auto* x = t.get() ) + return to_val(*x, target); + + return nullptr; +} + +/** + * Converts a Spicy-side signed bool to a Zeek value. The result is + * returned with ref count +1. + */ +inline ValPtr to_val(const hilti::rt::Bool& b, TypePtr target) { + if ( target->Tag() != TYPE_BOOL ) + throw TypeMismatch("bool", target); + + return val_mgr->Bool(b); +} + +/** + * Converts a Spicy-side real to a Zeek value. The result is returned with + * ref count +1. + */ +inline ValPtr to_val(double r, TypePtr target) { + if ( target->Tag() != TYPE_DOUBLE ) + throw TypeMismatch("double", target); + + return make_intrusive(r); +} + +/** + * Converts a Spicy-side address to a Zeek value. The result is returned with + * ref count +1. + */ +inline ValPtr to_val(const hilti::rt::Address& d, TypePtr target) { + if ( target->Tag() != TYPE_ADDR ) + throw TypeMismatch("addr", target); + + auto in_addr = d.asInAddr(); + if ( auto v4 = std::get_if(&in_addr) ) + return make_intrusive(IPAddr(*v4)); + else { + auto v6 = std::get(in_addr); + return make_intrusive(IPAddr(v6)); + } +} + +/** + * Converts a Spicy-side address to a Zeek value. The result is returned with + * ref count +1. + */ +inline ValPtr to_val(const hilti::rt::Port& p, TypePtr target) { + if ( target->Tag() != TYPE_PORT ) + throw TypeMismatch("port", target); + +#if SPICY_VERSION_NUMBER >= 10700 + auto proto = p.protocol().value(); +#else + auto proto = p.protocol(); +#endif + + switch ( proto ) { + case hilti::rt::Protocol::TCP: return val_mgr->Port(p.port(), ::TransportProto::TRANSPORT_TCP); + + case hilti::rt::Protocol::UDP: return val_mgr->Port(p.port(), ::TransportProto::TRANSPORT_UDP); + + case hilti::rt::Protocol::ICMP: return val_mgr->Port(p.port(), ::TransportProto::TRANSPORT_ICMP); + + default: throw InvalidValue("port value with undefined protocol"); + } +} + +/** + * Converts a Spicy-side time to a Zeek value. The result is returned with + * ref count +1. + */ +inline ValPtr to_val(const hilti::rt::Interval& i, TypePtr target) { + if ( target->Tag() != TYPE_INTERVAL ) + throw TypeMismatch("interval", target); + + return make_intrusive(i.seconds()); +} + +/** + * Converts a Spicy-side time to a Zeek value. The result is returned with + * ref count +1. + */ +inline ValPtr to_val(const hilti::rt::Time& t, TypePtr target) { + if ( target->Tag() != TYPE_TIME ) + throw TypeMismatch("time", target); + + return make_intrusive(t.seconds()); +} + +/** + * Converts a Spicy-side vector to a Zeek value. The result is returned with + * ref count +1. + */ +template +inline ValPtr to_val(const hilti::rt::Vector& v, TypePtr target) { + if ( target->Tag() != TYPE_VECTOR && target->Tag() != TYPE_LIST ) + throw TypeMismatch("expected vector or list", target); + + auto vt = cast_intrusive(target); + auto zv = make_intrusive(vt); + for ( const auto& i : v ) + zv->Assign(zv->Size(), to_val(i, vt->Yield())); + + return zv; +} + +/** + * Converts a Spicy-side map to a Zeek value. The result is returned with + * ref count +1. + */ +template +inline ValPtr to_val(const hilti::rt::Map& m, TypePtr target) { + if constexpr ( hilti::rt::is_tuple::value ) + throw TypeMismatch("internal error: sets with tuples not yet supported in to_val()"); + + if ( target->Tag() != TYPE_TABLE ) + throw TypeMismatch("map", target); + + auto tt = cast_intrusive(target); + if ( tt->IsSet() ) + throw TypeMismatch("map", target); + + if ( tt->GetIndexTypes().size() != 1 ) + throw TypeMismatch("map with non-tuple elements", target); + + auto zv = make_intrusive(tt); + + for ( const auto& i : m ) { + auto k = to_val(i.first, tt->GetIndexTypes()[0]); + auto v = to_val(i.second, tt->Yield()); + zv->Assign(std::move(k), std::move(v)); + } + + return zv; +} // namespace spicy::rt + +/** + * Converts a Spicy-side set to a Zeek value. The result is returned with + * ref count +1. + */ +template +inline ValPtr to_val(const hilti::rt::Set& s, TypePtr target) { + if ( target->Tag() != TYPE_TABLE ) + throw TypeMismatch("set", target); + + auto tt = cast_intrusive(target); + if ( ! tt->IsSet() ) + throw TypeMismatch("set", target); + + auto zv = make_intrusive(tt); + + for ( const auto& i : s ) { + if constexpr ( hilti::rt::is_tuple::value ) + throw TypeMismatch("internal error: sets with tuples not yet supported in to_val()"); + else { + if ( tt->GetIndexTypes().size() != 1 ) + throw TypeMismatch("set with non-tuple elements", target); + + auto idx = to_val(i, tt->GetIndexTypes()[0]); + zv->Assign(std::move(idx), nullptr); + } + } + + return zv; +} + +namespace { +template typename> +struct is_instance_impl : std::false_type {}; + +template typename U, typename... Ts> +struct is_instance_impl, U> : std::true_type {}; +} // namespace + +template typename U> +using is_instance = is_instance_impl, U>; + +template +inline void set_record_field(RecordVal* rval, const IntrusivePtr& rtype, int idx, const T& x) { + using NoConversionNeeded = std::integral_constant< + bool, std::is_same_v || std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v>; + + using IsSignedInteger = std::integral_constant> || + std::is_same_v> || + std::is_same_v> || + std::is_same_v>>; + + using IsUnsignedInteger = std::integral_constant> || + std::is_same_v> || + std::is_same_v> || + std::is_same_v>>; + + if constexpr ( NoConversionNeeded::value ) + rval->Assign(idx, x); + else if constexpr ( IsSignedInteger::value ) +#if ZEEK_VERSION_NUMBER >= 50200 + rval->Assign(idx, static_cast(x.Ref())); +#else + rval->Assign(idx, static_cast(x.Ref())); +#endif + else if constexpr ( IsUnsignedInteger::value ) + rval->Assign(idx, static_cast(x.Ref())); + else if constexpr ( std::is_same_v ) + rval->Assign(idx, x.str()); + else if constexpr ( std::is_same_v ) + rval->Assign(idx, static_cast(x)); + else if constexpr ( std::is_same_v ) + rval->Assign(idx, x); + else if constexpr ( std::is_same_v ) + rval->AssignTime(idx, x.seconds()); + else if constexpr ( std::is_same_v ) + rval->AssignInterval(idx, x.seconds()); + else if constexpr ( std::is_same_v ) { + // "Null" turns into an unset optional record field. + } + else if constexpr ( is_instance::value ) { + if ( x.has_value() ) + set_record_field(rval, rtype, idx, *x); + } + else if constexpr ( is_instance::value ) { + try { + set_record_field(rval, rtype, idx, x()); + } catch ( const hilti::rt::AttributeNotSet& ) { + // leave unset + } + } + else { + ValPtr v = nullptr; + + // This may return a nullptr in cases where the field is to be left unset. + v = to_val(x, rtype->GetFieldType(idx)); + + if ( v ) + rval->Assign(idx, v); + else { + // Field must be &optional or &default. + if ( auto attrs = rtype->FieldDecl(idx)->attrs; + ! attrs || ! (attrs->Find(detail::ATTR_DEFAULT) || attrs->Find(detail::ATTR_OPTIONAL)) ) + throw TypeMismatch(hilti::rt::fmt("missing initialization for field '%s'", rtype->FieldName(idx))); + } + } +} + +/** + * Converts a Spicy-side tuple to a Zeek record value. The result is returned + * with ref count +1. + */ +template::value>*> +inline ValPtr to_val(const T& t, TypePtr target) { + if ( target->Tag() != TYPE_RECORD ) + throw TypeMismatch("tuple", target); + + auto rtype = cast_intrusive(target); + + if ( std::tuple_size::value != rtype->NumFields() ) + throw TypeMismatch("tuple", target); + + auto rval = make_intrusive(rtype); + int idx = 0; + hilti::rt::tuple_for_each(t, [&](const auto& x) { set_record_field(rval.get(), rtype, idx++, x); }); + + return rval; +} + +/** + * Converts Spicy-side struct to a Zeek record value. The result is returned + * with a ref count +1. + */ +template::value>*> +inline ValPtr to_val(const T& t, TypePtr target) { + if ( target->Tag() != TYPE_RECORD ) + throw TypeMismatch("struct", target); + + auto rtype = cast_intrusive(target); + + auto rval = make_intrusive(rtype); + int idx = 0; + + auto num_fields = rtype->NumFields(); + + t.__visit([&](const auto& name, const auto& val) { + if ( idx >= num_fields ) + throw TypeMismatch(hilti::rt::fmt("no matching record field for field '%s'", name)); + + auto field = rtype->GetFieldType(idx); + std::string field_name = rtype->FieldName(idx); + + if ( field_name != name ) + throw TypeMismatch(hilti::rt::fmt("mismatch in field name: expected '%s', found '%s'", name, field_name)); + + set_record_field(rval.get(), rtype, idx++, val); + }); + + // We already check above that all Spicy-side fields are mapped so we + // can only hit this if there are uninitialized Zeek-side fields left. + if ( idx != num_fields ) + throw TypeMismatch(hilti::rt::fmt("missing initialization for field '%s'", rtype->FieldName(idx + 1))); + + return rval; +} + +/** + * Converts a Spicy-side enum to a Zeek record value. The result is returned + * with ref count +1. + */ +template::value>*> +inline ValPtr to_val(const T& t, TypePtr target) { +#if SPICY_VERSION_NUMBER >= 10700 + auto proto = typename T::Value(t.value()); +#else + auto proto = t; +#endif + + return to_val(proto, target); +} + +/** + * Converts a C++ Spicy-side enum to a Zeek record value. The result is returned + * with ref count +1. This specialization is provided for compatibility with ::value>*> +inline ValPtr to_val(const T& t, TypePtr target) { + if ( target->Tag() != TYPE_ENUM ) + throw TypeMismatch("enum", target); + + // We'll usually be getting an int64_t for T, but allow other signed ints + // as well. + static_assert(std::is_signed>{}); + auto it = static_cast(t); + + // Zeek's enum can't be negative, so we swap in max_int for our Undef (-1). + if ( it == std::numeric_limits::max() ) + // can't allow this ... + throw InvalidValue("enum values with value max_int not supported by Zeek integration"); + + zeek_int_t bt = (it >= 0 ? it : std::numeric_limits<::zeek_int_t>::max()); + + return target->AsEnumType()->GetEnumVal(bt); +} + +} // namespace zeek::spicy::rt diff --git a/src/spicy/spicy.bif b/src/spicy/spicy.bif new file mode 100644 index 0000000000..cb2c685a2e --- /dev/null +++ b/src/spicy/spicy.bif @@ -0,0 +1,62 @@ +# See the file "COPYING" in the main distribution directory for copyright. + +module Spicy; + +%%{ + #include "zeek/spicy/manager.h" +%%} + +# Constant for testing if Spicy is available. +const available: bool; + +# Show output of Spicy print statements. +const enable_print: bool; + +# Record and display profiling information. +const enable_profiling: bool; + +# abort() instead of throwing HILTI # exceptions. +const abort_on_exceptions: bool; + +# Include backtraces when reporting unhandled exceptions. +const show_backtraces: bool; + +# Maximum depth of recursive file analysis. +const max_file_depth: count; + +event max_file_depth_exceeded%(f: fa_file, args: Files::AnalyzerArgs, limit: count%); + +function Spicy::__toggle_analyzer%(tag: any, enable: bool%) : bool + %{ + if ( tag->GetType()->Tag() != TYPE_ENUM ) { + zeek::reporter->Warning("Spicy::disable_analyzer() must receive an analyzer tag"); + return val_mgr->Bool(false); + } + + bool result = spicy_mgr->toggleAnalyzer(tag->AsEnumVal(), enable); + if ( ! result ) + zeek::reporter->Warning("could not toggle Spicy analyzer"); + + return val_mgr->Bool(result); + %} + +type ResourceUsage: record; + +function Spicy::__resource_usage%(%) : Spicy::ResourceUsage + %{ + auto ru = hilti::rt::resource_usage(); + + auto r = zeek::make_intrusive(BifType::Record::Spicy::ResourceUsage); + int n = 0; + r->Assign(n++, ru.user_time); + r->Assign(n++, ru.system_time); + r->Assign(n++, ru.memory_heap); + r->Assign(n++, ru.num_fibers); + r->Assign(n++, ru.max_fibers); +#if SPICY_VERSION_NUMBER >= 10800 + r->Assign(n++, ru.max_fiber_stack_size); +#endif + r->Assign(n++, ru.cached_fibers); + + return r; + %} diff --git a/src/spicy/spicyz/CMakeLists.txt b/src/spicy/spicyz/CMakeLists.txt new file mode 100644 index 0000000000..54800a3db8 --- /dev/null +++ b/src/spicy/spicyz/CMakeLists.txt @@ -0,0 +1,10 @@ +# See the file "COPYING" in the main distribution directory for copyright. + +configure_file(config.h.in config.h) + +add_executable(spicyz driver.cc glue-compiler.cc main.cc) +target_compile_options(spicyz PRIVATE "-Wall") +target_include_directories(spicyz PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) +target_link_libraries(spicyz PRIVATE hilti spicy) + +install(TARGETS spicyz DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/src/spicy/spicyz/config.h.in b/src/spicy/spicyz/config.h.in new file mode 100644 index 0000000000..8433b8259b --- /dev/null +++ b/src/spicy/spicyz/config.h.in @@ -0,0 +1,67 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include "zeek/util-config.h" + +#include + +#include + +namespace zeek::spicy::configuration { + +using path = hilti::rt::filesystem::path; + +namespace { + +// This mimics zeek-config to get the Zeek include directories. +static inline void add_path(std::string& old_path, const path& new_path) { + if ( new_path.empty() ) + return; + + if ( ! old_path.empty() ) + old_path += ":"; + + old_path += new_path.native(); +} +} // namespace + +inline const auto InstallBinDir() { return path("${CMAKE_INSTALL_PREFIX}") / "${CMAKE_INSTALL_BINDIR}"; } + +inline const auto LibraryPath() { + path base; + + if ( auto custom_base = getenv("ZEEK_SPICY_LIBRARY_PATH"); custom_base && *custom_base ) + base = path(custom_base); + else + base = path("@ZEEK_SPICY_LIBRARY_PATH@"); + + return hilti::rt::filesystem::weakly_canonical(base); +} + +inline const auto ModulePath() { return path("@ZEEK_SPICY_MODULE_PATH@"); } +inline const auto DataPath() { return path("@ZEEK_SPICY_DATA_PATH@"); } + +inline const auto CxxZeekIncludesDirectories() { + std::string includes; + add_path(includes, path("${CMAKE_INSTALL_PREFIX}") / "${CMAKE_INSTALL_INCLUDEDIR}"); + + // When changing any of the following, also update "zeek-config.in". + add_path(includes, "${ZEEK_CONFIG_PCAP_INCLUDE_DIR}"); + add_path(includes, "${ZEEK_CONFIG_ZLIB_INCLUDE_DIR}"); + add_path(includes, "${ZEEK_CONFIG_OPENSSL_INCLUDE_DIR}"); + add_path(includes, "${ZEEK_CONFIG_LibKrb5_INCLUDE_DIR}"); + add_path(includes, "${ZEEK_CONFIG_GooglePerftools_INCLUDE_DIR}"); + + return includes; +} + +// Version of Spicy that we are compiling against. +#cmakedefine SPICY_VERSION_NUMBER ${SPICY_VERSION_NUMBER} +#cmakedefine ZEEK_VERSION_NUMBER ${ZEEK_VERSION_NUMBER} + +inline const auto SpicyVersion = "${SPICY_VERSION}"; +inline const auto ZeekVersion = "${VERSION}"; +inline const auto InstallPrefix = path("${CMAKE_INSTALL_PREFIX}"); + +} // namespace zeek::spicy::configuration diff --git a/src/spicy/spicyz/driver.cc b/src/spicy/spicyz/driver.cc new file mode 100644 index 0000000000..691ec5cfea --- /dev/null +++ b/src/spicy/spicyz/driver.cc @@ -0,0 +1,301 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "driver.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include +#include +#include + +#include "config.h" +#include "glue-compiler.h" + +using namespace zeek::spicy; +using Driver = ::zeek::spicy::Driver; + +/** + * Visitor to type information from a HILTI AST. This extracts user-visible + * types only, we skip any internal ones. + */ +struct VisitorTypes : public hilti::visitor::PreOrder { + explicit VisitorTypes(Driver* driver, hilti::ID module, hilti::rt::filesystem::path path, bool is_resolved) + : driver(driver), module(std::move(module)), path(std::move(path)), is_resolved(is_resolved) {} + + void operator()(const hilti::declaration::Type& t) { + assert(! t.type().typeID() || *t.type().typeID() == hilti::ID(module, t.id())); // ensure consistent IDs + + if ( module == hilti::ID("hilti") || module == hilti::ID("spicy_rt") || module == hilti::ID("zeek_rt") ) + return; + + types.emplace_back(TypeInfo{ + .id = hilti::ID(module, t.id()), + .type = t.type()._clone().as(), + .linkage = t.linkage(), + .is_resolved = is_resolved, + .module_id = module, + .module_path = path, + .location = t.meta().location(), + }); + } + + Driver* driver; + hilti::ID module; + hilti::rt::filesystem::path path; + bool is_resolved; + std::vector types; +}; + +Driver::Driver(std::unique_ptr glue, const char* argv0, hilti::rt::filesystem::path lib_path, + int zeek_version) + : ::spicy::Driver(""), _glue(std::move(glue)) { + _glue->Init(this, zeek_version); + + ::spicy::Configuration::extendHiltiConfiguration(); + auto options = hiltiOptions(); + + // Note that, different from Spicy's own SPICY_PATH, this extends the + // search path, it doesn't replace it. + if ( auto path = hilti::rt::getenv("ZEEK_SPICY_PATH") ) { + for ( const auto& dir : hilti::rt::split(*path, ":") ) { + if ( dir.size() ) + options.library_paths.emplace_back(dir); + } + } + + try { + lib_path = hilti::rt::filesystem::weakly_canonical(lib_path); + + // We make our search paths relative to the plugin library, so that the + // plugin installation can move around. + options.library_paths.push_back(lib_path); + } catch ( const hilti::rt::filesystem::filesystem_error& e ) { + ::hilti::logger().warning( + hilti::util::fmt("invalid plugin base directory %s: %s", lib_path.native(), e.what())); + } + + for ( const auto& i : hilti::util::split(configuration::CxxZeekIncludesDirectories(), ":") ) { + if ( i.size() ) + options.cxx_include_paths.emplace_back(i); + } + +#ifdef DEBUG + SPICY_DEBUG("Search paths:"); + + auto hilti_options = hiltiOptions(); + for ( const auto& x : hilti_options.library_paths ) { + SPICY_DEBUG(hilti::rt::fmt(" %s", x.native())); + } +#endif + + setCompilerOptions(std::move(options)); + + auto& config = ::spicy::configuration(); + config.preprocessor_constants["HAVE_ZEEK"] = 1; + config.preprocessor_constants["ZEEK_VERSION"] = zeek_version; + +#if SPICY_VERSION_NUMBER >= 10500 + ::hilti::init(); + ::spicy::init(); +#endif +} + +Driver::~Driver() {} + +hilti::Result Driver::loadFile(hilti::rt::filesystem::path file, + const hilti::rt::filesystem::path& relative_to) { + std::error_code ec; + if ( ! relative_to.empty() && file.is_relative() ) { + auto p = relative_to / file; + auto exists = hilti::rt::filesystem::exists(p, ec); + + if ( ec ) + return hilti::rt::result::Error( + hilti::util::fmt("error computing path of %s relative to %s: %s", file, relative_to, ec.message())); + + if ( exists ) + file = p; + } + + auto exists = hilti::rt::filesystem::exists(file, ec); + + if ( ec ) + return hilti::rt::result::Error( + hilti::util::fmt("cannot check whether file %s exists: %s", file, ec.message())); + + if ( ! exists ) { + if ( auto path = hilti::util::findInPaths(file, hiltiOptions().library_paths) ) + file = *path; + else + return hilti::result::Error(hilti::util::fmt("Spicy plugin cannot find file %s", file)); + } + + auto rpath = hilti::util::normalizePath(file); + auto ext = rpath.extension(); + + if ( ext == ".evt" ) { + SPICY_DEBUG(hilti::util::fmt("Loading EVT file %s", rpath)); + if ( _glue->loadEvtFile(rpath) ) + return hilti::Nothing(); + else + return hilti::result::Error(hilti::util::fmt("error loading EVT file %s", rpath)); + } + + if ( ext == ".spicy" ) { + SPICY_DEBUG(hilti::util::fmt("Loading Spicy file %s", rpath)); + if ( auto rc = addInput(rpath); ! rc ) + return rc.error(); + + return hilti::Nothing(); + } + + if ( ext == ".hlt" ) { + SPICY_DEBUG(hilti::util::fmt("Loading HILTI file %s", rpath)); + if ( auto rc = addInput(rpath) ) + return hilti::Nothing(); + else + return rc.error(); + } + + if ( ext == ".hlto" ) { + SPICY_DEBUG(hilti::util::fmt("Loading precompiled HILTI code %s", rpath)); + if ( auto rc = addInput(rpath) ) + return hilti::Nothing(); + else + return rc.error(); + } + + if ( ext == ".cc" || ext == ".cxx" ) { + SPICY_DEBUG(hilti::util::fmt("Loading C++ code %s", rpath)); + if ( auto rc = addInput(rpath) ) + return hilti::Nothing(); + else + return rc.error(); + } + + return hilti::result::Error(hilti::util::fmt("unknown file type passed to Spicy loader: %s", rpath)); +} + +hilti::Result Driver::compile() { + if ( ! hasInputs() ) + return hilti::Nothing(); + + SPICY_DEBUG("Running Spicy driver"); + + if ( auto x = ::spicy::Driver::compile(); ! x ) + return x.error(); + + SPICY_DEBUG("Done with Spicy driver"); + return hilti::Nothing(); +} + +hilti::Result Driver::lookupType(const hilti::ID& id) { + if ( auto x = _types.find(id); x != _types.end() ) + return x->second; + else + return hilti::result::Error(hilti::util::fmt("unknown type '%s'", id)); +} + +std::vector Driver::types() const { + std::vector result; + result.reserve(_types.size()); + + for ( const auto& t : _types ) + result.push_back(t.second); + + return result; +} + +std::vector> Driver::exportedTypes() const { + std::vector> result; + + for ( const auto& [spicy_id, zeek_id, _] : _glue->exportedIDs() ) { + if ( auto t = _types.find(spicy_id); t != _types.end() ) + result.emplace_back(t->second, zeek_id); + else { + hilti::logger().error(hilti::rt::fmt("unknown type '%s' exported", spicy_id)); + continue; + } + } + + // Automatically export public enums for backwards compatibility. + for ( const auto& t : _public_enums ) + result.emplace_back(t, t.id); + + return result; +} + +void Driver::hookNewASTPreCompilation(std::shared_ptr unit) { + if ( unit->extension() != ".spicy" ) + return; + + if ( unit->path().empty() ) + // Ignore modules constructed in memory. + return; + + auto v = VisitorTypes(this, unit->id(), unit->path(), false); + for ( auto i : v.walk(unit->module()) ) + v.dispatch(i); + + for ( const auto& ti : v.types ) { + SPICY_DEBUG(hilti::util::fmt(" Got type '%s' (pre-compile)", ti.id)); + _types[ti.id] = ti; + + if ( auto et = ti.type.tryAs(); et && ti.linkage == hilti::declaration::Linkage::Public ) { + SPICY_DEBUG(" Automatically exporting public enum for backwards compatibility"); + _public_enums.push_back(ti); + } + + hookNewType(ti); + } +} + +void Driver::hookNewASTPostCompilation(std::shared_ptr unit) { + if ( unit->extension() != ".spicy" ) + return; + + if ( unit->path().empty() ) + // Ignore modules constructed in memory. + return; + + auto v = VisitorTypes(this, unit->id(), unit->path(), true); + for ( auto i : v.walk(unit->module()) ) + v.dispatch(i); + + for ( auto&& t : v.types ) { + SPICY_DEBUG(hilti::util::fmt(" Got type '%s' (post-compile)", t.id)); + _types[t.id] = t; + hookNewType(t); + } + + _glue->addSpicyModule(unit->id(), unit->path()); +} + +hilti::Result Driver::hookCompilationFinished(const hilti::Plugin& plugin) { + if ( ! _need_glue ) + return hilti::Nothing(); + + _need_glue = false; + + if ( _glue->compile() ) + return hilti::Nothing(); + else + return hilti::result::Error("glue compilation failed"); +} + +void Driver::hookInitRuntime() { ::spicy::rt::init(); } + +void Driver::hookFinishRuntime() { ::spicy::rt::done(); } diff --git a/src/spicy/spicyz/driver.h b/src/spicy/spicyz/driver.h new file mode 100644 index 0000000000..8dd2422ce2 --- /dev/null +++ b/src/spicy/spicyz/driver.h @@ -0,0 +1,199 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include + +#include + +// Debug stream for compiler messages. +static const ::hilti::logging::DebugStream ZeekPlugin("zeek"); + +// Macro helper to report debug messages. +#define SPICY_DEBUG(msg) HILTI_DEBUG(ZeekPlugin, std::string(msg)); + +namespace zeek::spicy { + +class GlueCompiler; + +struct TypeInfo { + hilti::ID id; /**< fully-qualified name of the type */ + hilti::Type type; /**< the type itself */ + hilti::declaration::Linkage linkage; /**< linkage of of the type's declaration */ + bool is_resolved = false; /**< true if we are far enough in processing that the type has been fully resolved */ + hilti::ID module_id; /**< name of module type is defined in */ + hilti::rt::filesystem::path module_path; /**< path of module that type is defined in */ + hilti::Location location; /**< location of type's declaration */ +}; + +/** Spicy compilation driver. */ +class Driver : public ::spicy::Driver { +public: + /** + * Constructor. + * + * @param argv0 path to current executable, or empty to determine automatically + * @param lib_path Path to library files the Spicy support needs + * @param zeek_version Version number of Zeek we're working with + */ + Driver(std::unique_ptr glue, const char* argv0, hilti::rt::filesystem::path lib_path, + int zeek_version); + + /** Destructor. */ + ~Driver(); + + /** + * Schedules an *.spicy, *.evt, or *.hlt file for loading. Note that it + * won't necessarily load them all immediately, but may queue some for + * later processing. + * + * @param file file to load, which will be searched across all current search paths + * @param relative_to if given, relative paths will be interpreted as relative to this directory + */ + hilti::Result loadFile(hilti::rt::filesystem::path file, + const hilti::rt::filesystem::path& relative_to = {}); + + /** + * After user scripts have been read, compiles and links all resulting + * Spicy code. Note that compiler and driver options must have been set + * before calling this. + * + * Must be called before any packet processing starts. + * + * @return False if an error occurred. It will have been reported already. + */ + hilti::Result compile(); + + /** + * Returns meta information for a type. The Spicy module defining the type + * must have been compiled already for it to be found. + * + * @param id fully qualified name of type to look up + * @return meta data, or an error if the type is not (yet) known + */ + hilti::Result lookupType(const hilti::ID& id); + + /** + * Returns meta information for a type, enforcing it to be a of a certain + * kind. The Spicy module defining the type must have been compiled already + * for it to be found. + * + * @tparam T type to enforce; method will return an error if type is not of this class + * @param id fully qualified name of type to look up + * @return meta data, or an error if the type is not (yet) known + */ + template + hilti::Result lookupType(const hilti::ID& id) { + auto ti = lookupType(id); + if ( ! ti ) + return ti.error(); + + if ( ! ti->type.isA() ) + return hilti::result::Error(hilti::util::fmt("'%s' is not of expected type", id)); + + return ti; + } + + /** + * Returns all types seen so far during processing of Spicy files. + * Depending on where we are at with processing, these may or may not be + * resolved yet (as indicated by their `is_resolved` field). + + * @return list of types + */ + std::vector types() const; + + /** + * Returns all *exported* types seen so far during processing of Spicy + * files, including their desired Zeek-side names. Depending on where we + * are at with processing, these may or may not be resolved yet (as + * indicated by their `is_resolved` field). + * + * @return list of pairs of type and Zeek-side name + */ + std::vector> exportedTypes() const; + + /** Returns true if we're running out of the plugin's build directory. */ + bool usingBuildDirectory() const { return _using_build_directory; } + + /** Returns the glue compiler in use by the driver. */ + const auto* glueCompiler() const { return _glue.get(); } + + /** + * Parses some options command-line style *before* Zeek-side scripts have + * been processed. Most of the option processing happens in + * `parseOptionsPostScript()` instead, except for things that must be in + * place already before script processing. + * + * @param options space-separated string of command line argument to parse + * @return success if all argument could be parsed, or a suitable error message + */ + static hilti::Result parseOptionsPreScript(const std::string& options); + + /** + * Parses options command-line style after Zeek-side scripts have been + * fully procssed. Most of the option processing happens here (vs. in + * `parseOptionsPreScript()`) except for things that must be in place + * already before script processing. + * + * @param options space-separated string of command line argument to parse + * @param driver_options instance of options to update per parsed arguments + * @param compiler_options instance of options to update per parsed arguments + * @return success if all argument could be parsed, or a suitable error message + */ + static hilti::Result parseOptionsPostScript(const std::string& options, + hilti::driver::Options* driver_options, + hilti::Options* compiler_options); + + /** Prints a usage message for options supported by `parseOptions{Pre,Post}Script()`. */ + static void usage(std::ostream& out); + +protected: + /** + * Hook executed for all type declarations encountered in a Spicy module. + * Derived classes may override this to add custom processing. This hooks + * executes twices for each declaration: once before we compile the AST + * (meaning types have not been resolved yet), and once after. The type + * info's `is_resolved` field indicates which of the two we're in. + * + * @param t type's meta information + */ + virtual void hookNewType(const TypeInfo& ti) {} + + /** Overridden from HILTI driver. */ + void hookNewASTPreCompilation(std::shared_ptr unit) override; + + /** Overridden from HILTI driver. */ + void hookNewASTPostCompilation(std::shared_ptr unit) override; + + /** Overridden from HILTI driver. */ + hilti::Result hookCompilationFinished(const hilti::Plugin& plugin) override; + + /** Overridden from HILTI driver. */ + void hookInitRuntime() override; + + /** Overridden from HILTI driver. */ + void hookFinishRuntime() override; + + std::unique_ptr _glue; // glue compiler in use + std::unordered_map _types; // map of Spicy type declarations encountered so far + std::vector _public_enums; // tracks Spicy enum types declared public, for automatic export + bool _using_build_directory = false; // true if we're running out of the plugin's build directory + bool _need_glue = true; // true if glue code has not yet been generated +}; + +} // namespace zeek::spicy diff --git a/src/spicy/spicyz/glue-compiler.cc b/src/spicy/spicyz/glue-compiler.cc new file mode 100644 index 0000000000..efbf894d89 --- /dev/null +++ b/src/spicy/spicyz/glue-compiler.cc @@ -0,0 +1,1352 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "glue-compiler.h" + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include "config.h" + +using namespace zeek::spicy; + +namespace builder = hilti::builder; + +#if SPICY_VERSION_NUMBER >= 10700 +static inline auto _linker_scope() { return hilti::builder::scope(); } +#else +static inline auto _linker_scope() { return hilti::builder::call("hilti::linker_scope", {}); } +#endif + +// Small parsing helpers. + +using ParseError = std::runtime_error; + +static void eat_spaces(const std::string& chunk, size_t* i) { + while ( *i < chunk.size() && isspace(chunk[*i]) ) + ++*i; +} + +static std::string::size_type looking_at(const std::string& chunk, std::string::size_type i, + const std::string_view& token) { + eat_spaces(chunk, &i); + + for ( char j : token ) { + if ( i >= chunk.size() || chunk[i++] != j ) + return 0; + } + + return i; +} + +static void eat_token(const std::string& chunk, std::string::size_type* i, const std::string_view& token) { + eat_spaces(chunk, i); + + auto j = looking_at(chunk, *i, token); + + if ( ! j ) + throw ParseError(hilti::util::fmt("expected token '%s'", token)); + + *i = j; +} + +static bool is_id_char(const std::string& chunk, size_t i) { + char c = chunk[i]; + + if ( isalnum(c) ) + return true; + + if ( strchr("_$%", c) != nullptr ) + return true; + + char prev = (i > 0) ? chunk[i - 1] : '\0'; + char next = (i + 1 < chunk.size()) ? chunk[i + 1] : '\0'; + + if ( c == ':' && next == ':' ) + return true; + + if ( c == ':' && prev == ':' ) + return true; + + return false; +} + +static bool is_path_char(const std::string& chunk, size_t i) { + char c = chunk[i]; + return (! isspace(c)) && c != ';'; +} + +static hilti::ID extract_id(const std::string& chunk, size_t* i) { + eat_spaces(chunk, i); + + size_t j = *i; + + while ( j < chunk.size() && is_id_char(chunk, j) ) + ++j; + + if ( *i == j ) + throw ParseError("expected id"); + + auto id = chunk.substr(*i, j - *i); + *i = j; + return hilti::ID(hilti::util::replace(id, "%", "0x25_")); +} + +static hilti::Type extract_type(const std::string& chunk, size_t* i) { + eat_spaces(chunk, i); + + // We currently only parse Spicy types that can appear in parameters of + // built-in hooks--which are not many. + auto token = extract_id(chunk, i); + + if ( token == hilti::ID("string") ) + return hilti::type::String(); + + if ( token == hilti::ID("uint64") ) + return hilti::type::UnsignedInteger(64); + + throw ParseError("mismatching type"); +} + +static hilti::type::function::Parameter extract_parameter(const std::string& chunk, size_t* i) { + eat_spaces(chunk, i); + + auto id = extract_id(chunk, i); + + if ( ! looking_at(chunk, *i, ":") ) + throw ParseError("expected ':'"); + + eat_token(chunk, i, ":"); + + auto type = extract_type(chunk, i); + return builder::parameter(std::move(id), std::move(type)); +} + +static hilti::rt::filesystem::path extract_path(const std::string& chunk, size_t* i) { + eat_spaces(chunk, i); + + size_t j = *i; + + while ( j < chunk.size() && is_path_char(chunk, j) ) + ++j; + + if ( *i == j ) + throw ParseError("expected path"); + + auto path = chunk.substr(*i, j - *i); + *i = j; + return path; +} + +static int extract_int(const std::string& chunk, size_t* i) { + eat_spaces(chunk, i); + + size_t j = *i; + + if ( j < chunk.size() ) { + if ( chunk[j] == '-' ) { + ++j; + } + if ( chunk[j] == '+' ) + ++j; + } + + while ( j < chunk.size() && isdigit(chunk[j]) ) + ++j; + + if ( *i == j ) + throw ParseError("expected integer"); + + auto x = chunk.substr(*i, j - *i); + *i = j; + + int integer = 0; + hilti::util::atoi_n(x.begin(), x.end(), 10, &integer); + return integer; +} + +static std::string extract_expr(const std::string& chunk, size_t* i) { + eat_spaces(chunk, i); + + int level = 0; + bool done = false; + size_t j = *i; + + while ( j < chunk.size() ) { + switch ( chunk[j] ) { + case '(': + case '[': + case '{': + ++level; + ++j; + continue; + + case ')': + if ( level == 0 ) { + done = true; + break; + } + + // fall-through + + case ']': + case '}': + if ( level == 0 ) + throw ParseError("expected Spicy expression"); + + --level; + ++j; + continue; + + case ',': + if ( level == 0 ) { + done = true; + break; + } + + // fall-through + + default: ++j; + } + + if ( done ) + break; + + if ( *i == j ) + break; + } + + auto expr = hilti::util::trim(chunk.substr(*i, j - *i)); + *i = j; + return expr; +} + +static hilti::rt::Port extract_port(const std::string& chunk, size_t* i) { + eat_spaces(chunk, i); + + std::string s; + size_t j = *i; + + while ( j < chunk.size() && isdigit(chunk[j]) ) + ++j; + + if ( *i == j ) + throw ParseError("cannot parse port specification"); + + hilti::rt::Protocol proto; + uint64_t port = std::numeric_limits::max(); + + s = chunk.substr(*i, j - *i); + hilti::util::atoi_n(s.begin(), s.end(), 10, &port); + + if ( port > 65535 ) + throw ParseError("port outside of valid range"); + + *i = j; + + if ( chunk[*i] != '/' ) + throw ParseError("cannot parse port specification"); + + (*i)++; + + if ( looking_at(chunk, *i, "tcp") ) { + proto = hilti::rt::Protocol::TCP; + eat_token(chunk, i, "tcp"); + } + + else if ( looking_at(chunk, *i, "udp") ) { + proto = hilti::rt::Protocol::UDP; + eat_token(chunk, i, "udp"); + } + + else if ( looking_at(chunk, *i, "icmp") ) { + proto = hilti::rt::Protocol::ICMP; + eat_token(chunk, i, "icmp"); + } + + else + throw ParseError("cannot parse port specification"); + + return {static_cast(port), proto}; +} + +static std::vector extract_ports(const std::string& chunk, size_t* i) { + auto start = extract_port(chunk, i); + auto end = std::optional(); + + if ( looking_at(chunk, *i, "-") ) { + eat_token(chunk, i, "-"); + end = extract_port(chunk, i); + } + + if ( end ) { + if ( start.protocol() != end->protocol() ) + throw ParseError("start and end of port range must have same protocol"); + + if ( start.port() > end->port() ) + throw ParseError("start of port range cannot be after its end"); + } + + std::vector result; + + // Port ranges are a closed interval. + for ( auto port = start.port(); ! end || port <= end->port(); ++port ) { + result.emplace_back(port, start.protocol()); + if ( ! end ) + break; + } + + return result; +} + +void GlueCompiler::Init(Driver* driver, int zeek_version) { + _driver = driver; + _zeek_version = zeek_version; +} + +GlueCompiler::~GlueCompiler() {} + +hilti::Result GlueCompiler::getNextEvtBlock(std::istream& in, int* lineno) const { + std::string chunk; + + // Parser need to track whether we are inside a string or a comment. + enum State { Default, InComment, InString } state = Default; + char prev = '\0'; + + while ( true ) { + char cur; + in.get(cur); + if ( in.eof() ) { + chunk = hilti::util::trim(chunk); + if ( chunk.empty() ) + // Legitimate end of data. + return std::string(); + else + // End of input before semicolon. + return hilti::result::Error("unexpected end of file"); + } + + switch ( state ) { + case Default: + if ( cur == '"' && prev != '\\' ) + state = InString; + + if ( cur == '#' && prev != '\\' ) { + state = InComment; + continue; + } + + if ( cur == '\n' ) + ++*lineno; + + if ( cur == ';' ) { + // End of block found. + chunk = hilti::util::trim(chunk); + if ( chunk.size() ) + return chunk + ';'; + else + return hilti::result::Error("empty block"); + } + + break; + + case InString: + if ( cur == '"' && prev != '\\' ) + state = Default; + + if ( cur == '\n' ) + ++*lineno; + + break; + + case InComment: + if ( cur != '\n' ) + // skip + continue; + + state = Default; + ++*lineno; + } + + chunk += cur; + prev = cur; + } +} + +void GlueCompiler::preprocessEvtFile(hilti::rt::filesystem::path& path, std::istream& in, std::ostream& out) { + hilti::util::SourceCodePreprocessor pp({{"ZEEK_VERSION", *_zeek_version}}); + int lineno = 0; + + std::string line; + while ( std::getline(in, line) ) { + lineno++; + + auto trimmed = hilti::util::trim(line); + _locations.emplace_back(path, lineno); + + if ( hilti::util::startsWith(trimmed, "@") ) { + // Output empty line to keep line numbers the same + out << '\n'; + + auto m = hilti::util::split1(trimmed); + + if ( auto rc = pp.processLine(m.first, m.second); ! rc ) + throw ParseError(rc.error()); + } + + else { + switch ( pp.state() ) { + case hilti::util::SourceCodePreprocessor::State::Include: out << line << '\n'; break; + case hilti::util::SourceCodePreprocessor::State::Skip: + // Output empty line to keep line numbers the same + out << '\n'; + break; + } + } + } + + if ( pp.expectingDirective() ) + throw ParseError("unterminated preprocessor directive"); +} + +bool GlueCompiler::loadEvtFile(hilti::rt::filesystem::path& path) { + assert(_zeek_version); + + std::ifstream in(path); + + if ( ! in ) { + hilti::logger().error(hilti::util::fmt("cannot open %s", path)); + return false; + } + + SPICY_DEBUG(hilti::util::fmt("Loading events from %s", path)); + + std::vector new_events; + + try { + std::stringstream preprocessed; + preprocessEvtFile(path, in, preprocessed); + preprocessed.clear(); + preprocessed.seekg(0); + + int lineno = 1; + + while ( true ) { + _locations.emplace_back(path, lineno); + auto chunk = getNextEvtBlock(preprocessed, &lineno); + if ( ! chunk ) + throw ParseError(chunk.error()); + + if ( chunk->empty() ) + break; // end of input + + _locations.pop_back(); + _locations.emplace_back(path, lineno); + + if ( looking_at(*chunk, 0, "protocol") ) { + auto a = parseProtocolAnalyzer(*chunk); + SPICY_DEBUG(hilti::util::fmt(" Got protocol analyzer definition for %s", a.name)); + _protocol_analyzers.push_back(a); + } + + else if ( looking_at(*chunk, 0, "file") ) { + auto a = parseFileAnalyzer(*chunk); + SPICY_DEBUG(hilti::util::fmt(" Got file analyzer definition for %s", a.name)); + _file_analyzers.push_back(a); + } + + else if ( looking_at(*chunk, 0, "packet") ) { + auto a = parsePacketAnalyzer(*chunk); + SPICY_DEBUG(hilti::util::fmt(" Got packet analyzer definition for %s", a.name)); + _packet_analyzers.push_back(a); + } + + else if ( looking_at(*chunk, 0, "on") ) { + auto ev = parseEvent(*chunk); + ev.file = path; + new_events.push_back(ev); + SPICY_DEBUG(hilti::util::fmt(" Got event definition for %s", ev.name)); + } + + else if ( looking_at(*chunk, 0, "import") ) { + size_t i = 0; + eat_token(*chunk, &i, "import"); + + hilti::ID module = extract_id(*chunk, &i); + std::optional scope; + + if ( looking_at(*chunk, i, "from") ) { + eat_token(*chunk, &i, "from"); + scope = extract_path(*chunk, &i); + SPICY_DEBUG(hilti::util::fmt(" Got module %s to import from scope %s", module, *scope)); + } + else + SPICY_DEBUG(hilti::util::fmt(" Got module %s to import", module)); + + _imports.emplace_back(hilti::ID(module), std::move(scope)); + } + + else if ( looking_at(*chunk, 0, "export") ) { + size_t i = 0; + eat_token(*chunk, &i, "export"); + + hilti::ID spicy_id = extract_id(*chunk, &i); + hilti::ID zeek_id = spicy_id; + + if ( looking_at(*chunk, i, "as") ) { + eat_token(*chunk, &i, "as"); + zeek_id = extract_id(*chunk, &i); + } + + eat_spaces(*chunk, &i); + if ( ! looking_at(*chunk, i, ";") ) + throw ParseError("syntax error in export"); + + _exports.emplace_back(std::move(spicy_id), std::move(zeek_id), _locations.back()); + } + + else + throw ParseError("expected 'import', 'export', '{file,packet,protocol} analyzer', or 'on'"); + + _locations.pop_back(); + } + } catch ( const ParseError& e ) { + if ( *e.what() ) + hilti::logger().error(e.what(), _locations.back()); + + return false; + } + + for ( auto&& ev : new_events ) + _events.push_back(ev); + + return true; +} + +void GlueCompiler::addSpicyModule(const hilti::ID& id, const hilti::rt::filesystem::path& file) { + glue::SpicyModule module; + module.id = id; + module.file = file; + _spicy_modules[id] = std::make_shared(std::move(module)); +} + +glue::ProtocolAnalyzer GlueCompiler::parseProtocolAnalyzer(const std::string& chunk) { + glue::ProtocolAnalyzer a; + a.location = _locations.back(); + + size_t i = 0; + + eat_token(chunk, &i, "protocol"); + eat_token(chunk, &i, "analyzer"); + a.name = extract_id(chunk, &i).str(); + + eat_token(chunk, &i, "over"); + + auto proto = hilti::util::tolower(extract_id(chunk, &i).str()); + + if ( proto == "tcp" ) + a.protocol = hilti::rt::Protocol::TCP; + + else if ( proto == "udp" ) + a.protocol = hilti::rt::Protocol::UDP; + + else if ( proto == "icmp" ) + a.protocol = hilti::rt::Protocol::ICMP; + + else + throw ParseError(hilti::util::fmt("unknown transport protocol '%s'", proto)); + + eat_token(chunk, &i, ":"); + + enum { orig, resp, both } dir; + + while ( true ) { + if ( looking_at(chunk, i, "parse") ) { + eat_token(chunk, &i, "parse"); + + if ( looking_at(chunk, i, "originator") ) { + eat_token(chunk, &i, "originator"); + dir = orig; + } + + else if ( looking_at(chunk, i, "responder") ) { + eat_token(chunk, &i, "responder"); + dir = resp; + } + + else if ( looking_at(chunk, i, "with") ) + dir = both; + + else + throw ParseError("invalid \"parse with ...\" specification"); + + eat_token(chunk, &i, "with"); + auto unit = extract_id(chunk, &i); + + switch ( dir ) { + case orig: a.unit_name_orig = unit; break; + + case resp: a.unit_name_resp = unit; break; + + case both: + a.unit_name_orig = unit; + a.unit_name_resp = unit; + break; + } + } + + else if ( looking_at(chunk, i, "ports") ) { + eat_token(chunk, &i, "ports"); + eat_token(chunk, &i, "{"); + + while ( true ) { + auto ports = extract_ports(chunk, &i); + a.ports.insert(a.ports.end(), ports.begin(), ports.end()); + + if ( looking_at(chunk, i, "}") ) { + eat_token(chunk, &i, "}"); + break; + } + + eat_token(chunk, &i, ","); + } + } + + else if ( looking_at(chunk, i, "port") ) { + eat_token(chunk, &i, "port"); + auto ports = extract_ports(chunk, &i); + a.ports.insert(a.ports.end(), ports.begin(), ports.end()); + } + + else if ( looking_at(chunk, i, "replaces") ) { + eat_token(chunk, &i, "replaces"); + a.replaces = extract_id(chunk, &i); + } + + else + throw ParseError("unexpected token"); + + if ( looking_at(chunk, i, ";") ) + break; // All done. + + eat_token(chunk, &i, ","); + } + + return a; +} + +glue::FileAnalyzer GlueCompiler::parseFileAnalyzer(const std::string& chunk) { + glue::FileAnalyzer a; + a.location = _locations.back(); + + size_t i = 0; + + eat_token(chunk, &i, "file"); + eat_token(chunk, &i, "analyzer"); + a.name = extract_id(chunk, &i).str(); + + eat_token(chunk, &i, ":"); + + while ( true ) { + if ( looking_at(chunk, i, "parse") ) { + eat_token(chunk, &i, "parse"); + eat_token(chunk, &i, "with"); + a.unit_name = extract_id(chunk, &i); + } + + else if ( looking_at(chunk, i, "mime-type") ) { + eat_token(chunk, &i, "mime-type"); + auto mtype = extract_path(chunk, &i); + a.mime_types.push_back(mtype.string()); + } + + else if ( looking_at(chunk, i, "replaces") ) { + eat_token(chunk, &i, "replaces"); + a.replaces = extract_id(chunk, &i); + } + + else + throw ParseError("unexpected token"); + + if ( looking_at(chunk, i, ";") ) + break; // All done. + + eat_token(chunk, &i, ","); + } + + return a; +} + +glue::PacketAnalyzer GlueCompiler::parsePacketAnalyzer(const std::string& chunk) { + glue::PacketAnalyzer a; + a.location = _locations.back(); + + size_t i = 0; + + eat_token(chunk, &i, "packet"); + eat_token(chunk, &i, "analyzer"); + a.name = extract_id(chunk, &i).str(); + + eat_token(chunk, &i, ":"); + + while ( true ) { + if ( looking_at(chunk, i, "parse") ) { + eat_token(chunk, &i, "parse"); + eat_token(chunk, &i, "with"); + a.unit_name = extract_id(chunk, &i); + } + + else if ( looking_at(chunk, i, "replaces") ) { + eat_token(chunk, &i, "replaces"); + a.replaces = extract_id(chunk, &i); + } + + else + throw ParseError("unexpected token"); + + if ( looking_at(chunk, i, ";") ) + break; // All done. + + eat_token(chunk, &i, ","); + } + + return a; +} + +glue::Event GlueCompiler::parseEvent(const std::string& chunk) { + glue::Event ev; + ev.location = _locations.back(); + + // We use a quite negative hook priority here to make sure these run last + // after anything the grammar defines by default. + ev.priority = -1000; + + size_t i = 0; + + eat_token(chunk, &i, "on"); + ev.path = extract_id(chunk, &i); + + if ( looking_at(chunk, i, "(") ) { + eat_token(chunk, &i, "("); + + if ( ! looking_at(chunk, i, ")") ) { + while ( true ) { + auto param = extract_parameter(chunk, &i); + ev.parameters.push_back(std::move(param)); + + if ( looking_at(chunk, i, ")") ) + break; + + eat_token(chunk, &i, ","); + } + } + + eat_token(chunk, &i, ")"); + } + + if ( looking_at(chunk, i, "if") ) { + eat_token(chunk, &i, "if"); + eat_token(chunk, &i, "("); + + ev.condition = extract_expr(chunk, &i); + eat_token(chunk, &i, ")"); + } + + eat_token(chunk, &i, "->"); + eat_token(chunk, &i, "event"); + ev.name = extract_id(chunk, &i); + + eat_token(chunk, &i, "("); + + bool first = true; + size_t j = 0; + + while ( true ) { + j = looking_at(chunk, i, ")"); + + if ( j ) { + i = j; + break; + } + + if ( ! first ) + eat_token(chunk, &i, ","); + + auto expr = extract_expr(chunk, &i); + ev.exprs.push_back(expr); + first = false; + } + + if ( looking_at(chunk, i, "&priority") ) { + eat_token(chunk, &i, "&priority"); + eat_token(chunk, &i, "="); + ev.priority = extract_int(chunk, &i); + } + + eat_token(chunk, &i, ";"); + eat_spaces(chunk, &i); + + if ( i < chunk.size() ) + // This shouldn't actually be possible ... + throw ParseError("unexpected characters at end of line"); + + return ev; +} + +bool GlueCompiler::compile() { + assert(_driver); + + auto init_module = hilti::Module(hilti::ID("spicy_init")); + + auto import_ = hilti::builder::import(hilti::ID("zeek_rt"), ".hlt"); + init_module.add(std::move(import_)); + + import_ = hilti::builder::import(hilti::ID("hilti"), ".hlt"); + init_module.add(std::move(import_)); + + auto preinit_body = hilti::builder::Builder(_driver->context()); + + for ( auto&& [id, m] : _spicy_modules ) + m->spicy_module = hilti::Module(hilti::ID(hilti::util::fmt("spicy_hooks_%s", id))); + + if ( ! PopulateEvents() ) + return false; + + for ( auto& a : _protocol_analyzers ) { + SPICY_DEBUG(hilti::util::fmt("Adding protocol analyzer '%s'", a.name)); + + if ( a.unit_name_orig ) { + if ( auto ui = _driver->lookupType<::spicy::type::Unit>(a.unit_name_orig) ) + a.unit_orig = *ui; + else { + hilti::logger().error(hilti::util::fmt("error with protocol analyzer %s: %s", a.name, ui.error())); + return false; + } + } + + if ( a.unit_name_resp ) { + if ( auto ui = _driver->lookupType<::spicy::type::Unit>(a.unit_name_resp) ) + a.unit_resp = *ui; + else { + hilti::logger().error(hilti::util::fmt("error with protocol analyzer %s: %s", a.name, ui.error())); + return false; + } + } + +#if SPICY_VERSION_NUMBER >= 10700 + auto proto = a.protocol.value(); +#else + auto proto = a.protocol; +#endif + + hilti::ID protocol; + switch ( proto ) { + case hilti::rt::Protocol::TCP: protocol = hilti::ID("hilti::Protocol::TCP"); break; + case hilti::rt::Protocol::UDP: protocol = hilti::ID("hilti::Protocol::UDP"); break; + default: hilti::logger().internalError("unexpected protocol"); + } + + preinit_body.addCall("zeek_rt::register_protocol_analyzer", + {builder::string(a.name), builder::id(protocol), + builder::vector(hilti::util::transform(a.ports, [](auto p) { return builder::port(p); })), + builder::string(a.unit_name_orig), builder::string(a.unit_name_resp), + builder::string(a.replaces), _linker_scope()}); + } + + for ( auto& a : _file_analyzers ) { + SPICY_DEBUG(hilti::util::fmt("Adding file analyzer '%s'", a.name)); + + if ( a.unit_name ) { + if ( auto ui = _driver->lookupType<::spicy::type::Unit>(a.unit_name) ) + a.unit = *ui; + else { + hilti::logger().error(hilti::util::fmt("error with file analyzer %s: %s", a.name, ui.error())); + return false; + } + } + + preinit_body.addCall("zeek_rt::register_file_analyzer", + {builder::string(a.name), + builder::vector( + hilti::util::transform(a.mime_types, [](auto m) { return builder::string(m); })), + builder::string(a.unit_name), builder::string(a.replaces), _linker_scope()}); + } + + for ( auto& a : _packet_analyzers ) { + SPICY_DEBUG(hilti::util::fmt("Adding packet analyzer '%s'", a.name)); + + if ( a.unit_name ) { + if ( auto ui = _driver->lookupType<::spicy::type::Unit>(a.unit_name) ) + a.unit = *ui; + else { + hilti::logger().error(hilti::util::fmt("error with packet analyzer %s: %s", a.name, ui.error())); + return false; + } + } + + preinit_body.addCall("zeek_rt::register_packet_analyzer", + {builder::string(a.name), builder::string(a.unit_name), builder::string(a.replaces), + _linker_scope()}); + } + + // Create the Spicy hooks and accessor functions. + for ( auto&& ev : _events ) { + if ( ! CreateSpicyHook(&ev) ) + return false; + } + + // Register our Zeek events at pre-init time. + for ( auto&& ev : _events ) + preinit_body.addCall("zeek_rt::install_handler", {builder::string(ev.name)}); + + // Create Zeek types for exported Spicy types. + for ( const auto& [tinfo, id] : _driver->exportedTypes() ) { + if ( auto type = createZeekType(tinfo.type, id) ) + preinit_body.addCall("zeek_rt::register_type", + {builder::string(id.namespace_()), builder::string(id.local()), *type}); + else + hilti::logger().error(hilti::util::fmt("cannot export Spicy type '%s': %s", id, type.error()), + tinfo.location); + } + + for ( auto&& [id, m] : _spicy_modules ) { + // Import runtime module. + auto import_ = hilti::builder::import(hilti::ID("zeek_rt"), ".hlt"); + m->spicy_module->add(std::move(import_)); + + // Create a vector of unique parent paths from all EVTs files going into this module. + auto search_dirs = hilti::util::transform(m->evts, [](auto p) { return p.parent_path(); }); + auto search_dirs_vec = std::vector(search_dirs.begin(), search_dirs.end()); + + // Import any dependencies. + for ( const auto& [module, scope] : _imports ) { + auto import_ = hilti::declaration::ImportedModule(module, std::string(".spicy"), scope, search_dirs_vec); + m->spicy_module->add(std::move(import_)); + } + + auto unit = hilti::Unit::fromModule(_driver->context(), *m->spicy_module, ".spicy"); + _driver->addInput(unit); + } + + if ( ! preinit_body.empty() ) { + auto preinit_function = + hilti::builder::function("zeek_preinit", hilti::type::void_, {}, preinit_body.block(), + hilti::type::function::Flavor::Standard, hilti::declaration::Linkage::PreInit); + init_module.add(std::move(preinit_function)); + } + + auto unit = hilti::Unit::fromModule(_driver->context(), init_module, ".hlt"); + _driver->addInput(unit); + return true; +} + +bool GlueCompiler::PopulateEvents() { + for ( auto& ev : _events ) { + if ( ev.unit_type ) + // Already done. + continue; + + TypeInfo uinfo; + + // If we find the path itself, it's referring to a unit type directly; + // then add a "%done" to form the hook name. + if ( auto ui = _driver->lookupType<::spicy::type::Unit>(ev.path) ) { + // TODO: Check that it's a unit type. + uinfo = *ui; + ev.unit = ev.path; + ev.hook = ev.unit + hilti::ID("0x25_done"); + } + + else { + // Strip the last element of the path, the remainder must refer + // to a unit now. + ev.unit = ev.path.namespace_(); + if ( ! ev.unit ) { + hilti::logger().error(hilti::util::fmt("unit type missing in hook '%s'", ev.path)); + return false; + } + + if ( auto ui = _driver->lookupType(ev.unit) ) { + uinfo = *ui; + ev.hook = ev.path; + } + else { + hilti::logger().error(hilti::util::fmt("unknown unit type '%s'", ev.unit)); + return false; + } + } + + ev.unit_type = std::move(uinfo.type.as<::spicy::type::Unit>()); + ev.unit_module_id = uinfo.module_id; + ev.unit_module_path = uinfo.module_path; + + if ( auto i = _spicy_modules.find(uinfo.module_id); i != _spicy_modules.end() ) { + ev.spicy_module = i->second; + i->second->evts.insert(ev.file); + } + else + hilti::logger().internalError( + hilti::util::fmt("module %s not known in Spicy module list", uinfo.module_id)); + + // Create accessor expression for event parameters. + int nr = 0; + + for ( const auto& e : ev.exprs ) { + glue::ExpressionAccessor acc; + acc.nr = ++nr; + acc.expression = e; + acc.location = ev.location; + // acc.dollar_id = util::startsWith(e, "$"); + ev.expression_accessors.push_back(acc); + } + } + + return true; +} + +#include + +#include + +// Helper visitor to wrap expressions using the the TryMember operator into a +// "deferred" expression. +class WrapTryMemberVisitor : public hilti::visitor::PostOrder { +public: + WrapTryMemberVisitor(bool catch_exception) : _catch_exception(catch_exception) {} + + void operator()(const hilti::expression::UnresolvedOperator& n, position_t p) { + if ( n.kind() == hilti::operator_::Kind::TryMember ) + p.node = hilti::expression::Deferred(hilti::Expression(n), _catch_exception); + } + +private: + bool _catch_exception; +}; + +static hilti::Result _parseArgument(const std::string& expression, bool catch_exception, + const hilti::Meta& meta) { + auto expr = spicy::parseExpression(expression, meta); + if ( ! expr ) + return hilti::result::Error(hilti::util::fmt("error parsing event argument expression '%s'", expression)); + + // If the expression uses the ".?" operator, we need to defer evaluation + // so that we can handle potential exceptions at runtime. + auto v = WrapTryMemberVisitor(catch_exception); + auto n = hilti::Node(*expr); + for ( auto i : v.walk(&n) ) + v.dispatch(i); + + return n.as(); +} + +bool GlueCompiler::CreateSpicyHook(glue::Event* ev) { + auto mangled_event_name = + hilti::util::fmt("%s_%p", hilti::util::replace(ev->name.str(), "::", "_"), std::hash()(*ev)); + auto meta = hilti::Meta(ev->location); + + // Find the Spicy module that this event belongs to. + SPICY_DEBUG(hilti::util::fmt("Adding Spicy hook '%s' for event %s", ev->hook, ev->name)); + + auto import_ = hilti::declaration::ImportedModule(ev->unit_module_id, ev->unit_module_path); + ev->spicy_module->spicy_module->add(std::move(import_)); + + // Define Zeek-side event handler. + auto handler_id = hilti::ID(hilti::util::fmt("__zeek_handler_%s", mangled_event_name)); + auto handler = builder::global(handler_id, builder::call("zeek_rt::internal_handler", {builder::string(ev->name)}), + hilti::declaration::Linkage::Private, meta); + ev->spicy_module->spicy_module->add(std::move(handler)); + + // Create the hook body that raises the event. + auto body = hilti::builder::Builder(_driver->context()); + +#if SPICY_VERSION_NUMBER >= 10800 + body.startProfiler(hilti::util::fmt("zeek/event/%s", ev->name)); +#endif + + // If the event comes with a condition, evaluate that first. + if ( ev->condition.size() ) { + auto cond = ::spicy::parseExpression(ev->condition, meta); + if ( ! cond ) { + hilti::logger().error(hilti::util::fmt("error parsing conditional expression '%s'", ev->condition)); + return false; + } + + auto exit_ = body.addIf(builder::not_(*cond), meta); + exit_->addReturn(meta); + } + + // Log event in debug code. Note: We cannot log the Zeek-side version + // (i.e., Vals with their types) because we wouldn't be able to determine + // those for events that don't have a handler (or at least a prototype) + // defined because we use the existing type definition to determine what + // Zeek type to convert an Spicy type into. However, we wouldn't want + // limit logging to events with handlers. + if ( _driver->hiltiOptions().debug ) { + std::vector fmt_args = {builder::string(ev->name)}; + + for ( const auto&& [i, e] : hilti::util::enumerate(ev->expression_accessors) ) { + if ( hilti::util::startsWith(e.expression, "$") ) { + fmt_args.emplace_back(builder::string(e.expression)); + continue; + } + + if ( auto expr = _parseArgument(e.expression, true, meta) ) + fmt_args.emplace_back(std::move(*expr)); + else + // We'll catch and report this below. + fmt_args.emplace_back(builder::string("")); + } + + std::vector fmt_ctrls(fmt_args.size() - 1, "%s"); + auto fmt_str = hilti::util::fmt("-> event %%s(%s)", hilti::util::join(fmt_ctrls, ", ")); + auto msg = builder::modulo(builder::string(fmt_str), builder::tuple(fmt_args)); + auto call = builder::call("zeek_rt::debug", {std::move(msg)}); + body.addExpression(call); + } + + auto handler_expr = builder::id(handler_id); + + if ( _driver->hiltiOptions().cxx_enable_dynamic_globals ) { + // Store reference to handler locally to avoid repeated lookups through globals store. + body.addLocal("handler", builder::id(handler_id), meta); + handler_expr = builder::id("handler"); + } + + // Nothing to do if there's not handler defined. + auto have_handler = builder::call("zeek_rt::have_handler", {handler_expr}, meta); + auto exit_ = body.addIf(builder::not_(have_handler), meta); + exit_->addReturn(meta); + + // Build event's argument vector. + body.addLocal(hilti::ID("args"), hilti::type::Vector(builder::typeByID("zeek_rt::Val"), meta), meta); + body.addMemberCall(builder::id("args"), "reserve", + {builder::integer(static_cast(ev->expression_accessors.size()))}, meta); + + int i = 0; + for ( const auto& e : ev->expression_accessors ) { + hilti::Expression val; + + if ( e.expression == "$conn" ) + val = builder::call("zeek_rt::current_conn", {}, meta); + else if ( e.expression == "$file" ) + val = builder::call("zeek_rt::current_file", {}, meta); + else if ( e.expression == "$packet" ) + val = builder::call("zeek_rt::current_packet", {}, meta); + else if ( e.expression == "$is_orig" ) + val = builder::call("zeek_rt::current_is_orig", {}, meta); + else { + if ( hilti::util::startsWith(e.expression, "$") ) { + hilti::logger().error(hilti::util::fmt("unknown reserved parameter '%s'", e.expression)); + return false; + } + + auto expr = _parseArgument(e.expression, false, meta); + if ( ! expr ) { + hilti::logger().error(expr.error()); + return false; + } + + auto ztype = builder::call("zeek_rt::event_arg_type", {handler_expr, builder::integer(i)}, meta); + val = builder::call("zeek_rt::to_val", {std::move(*expr), ztype}, meta); + } + + body.addMemberCall(builder::id("args"), "push_back", {val}, meta); + i++; + } + + body.addCall("zeek_rt::raise_event", {handler_expr, builder::move(builder::id("args"))}, meta); + + auto attrs = hilti::AttributeSet({hilti::Attribute("&priority", builder::integer(ev->priority))}); + auto unit_hook = ::spicy::Hook(ev->parameters, body.block(), ::spicy::Engine::All, {}, meta); + auto hook_decl = ::spicy::declaration::UnitHook(ev->hook, unit_hook, meta); + ev->spicy_module->spicy_module->add(hilti::Declaration(hook_decl)); + + return true; +} + +namespace { +// Visitor creasting code to instantiate a Zeek type corresponding to a give +// HILTI type. +// +// Note: Any logic changes here must be reflected in the plugin driver's +// corresponding `VisitorZeekType` as well. +struct VisitorZeekType : hilti::visitor::PreOrder, VisitorZeekType> { + VisitorZeekType(const GlueCompiler* gc) : gc(gc) {} + + const GlueCompiler* gc; + + std::set zeek_types; + std::optional id; + + result_t base_type(const char* tag) { return builder::call("zeek_rt::create_base_type", {builder::id(tag)}); } + + result_t createZeekType(const hilti::Type& t, const std::optional& id_ = {}) { + if ( id_ ) + id = id_; + else if ( auto x = t.typeID() ) + id = *x; + else + id = std::nullopt; + + if ( id ) { + // Avoid infinite recursion. + if ( zeek_types.count(*id) ) + return hilti::result::Error("type is self-recursive"); + + zeek_types.insert(*id); + } + + auto x = dispatch(t); + if ( ! x ) + return hilti::result::Error( + hilti::util::fmt("no support for automatic conversion into a Zeek type (%s)", t.typename_())); + + zeek_types.erase(*id); + return *x; + } + + result_t operator()(const hilti::type::Address& t) { return base_type("zeek_rt::ZeekTypeTag::Addr"); } + result_t operator()(const hilti::type::Bool& t) { return base_type("zeek_rt::ZeekTypeTag::Bool"); } + result_t operator()(const hilti::type::Bytes& t) { return base_type("zeek_rt::ZeekTypeTag::String"); } + result_t operator()(const hilti::type::Interval& t) { return base_type("zeek_rt::ZeekTypeTag::Interval"); } + result_t operator()(const hilti::type::Port& t) { return base_type("zeek_rt::ZeekTypeTag::Port"); } + result_t operator()(const hilti::type::Real& t) { return base_type("zeek_rt::ZeekTypeTag::Double"); } + result_t operator()(const hilti::type::SignedInteger& t) { return base_type("zeek_rt::ZeekTypeTag::Int"); } + result_t operator()(const hilti::type::String& t) { return base_type("zeek_rt::ZeekTypeTag::String"); } + result_t operator()(const hilti::type::Time& t) { return base_type("zeek_rt::ZeekTypeTag::Time"); } + result_t operator()(const hilti::type::UnsignedInteger& t) { return base_type("zeek_rt::ZeekTypeTag::Count"); } + + result_t operator()(const hilti::type::Enum& t) { + assert(id); + + auto labels = hilti::rt::transform(t.labels(), [](const auto& l) { + return builder::tuple({builder::string(l.get().id()), builder::integer(l.get().value())}); + }); + + return builder::call("zeek_rt::create_enum_type", {builder::string(id->namespace_()), + builder::string(id->local()), builder::vector(labels)}); + } + + result_t operator()(const hilti::type::Map& t) { + auto key = createZeekType(t.keyType()); + if ( ! key ) + return key.error(); + + auto value = createZeekType(t.valueType()); + if ( ! value ) + return value.error(); + + return builder::call("zeek_rt::create_table_type", {*key, *value}); + } + + result_t operator()(const hilti::type::Optional& t) { return createZeekType(t.dereferencedType()); } + + result_t operator()(const hilti::type::Set& t) { + auto elem = createZeekType(t.elementType()); + if ( ! elem ) + return elem.error(); + + return builder::call("zeek_rt::create_table_type", {*elem, builder::null()}); + } + + result_t operator()(const hilti::type::Struct& t) { + assert(id); + + std::vector fields; + for ( const auto& f : t.fields() ) { + auto ztype = createZeekType(f.type()); + if ( ! ztype ) + return ztype.error(); + + fields.emplace_back(builder::tuple({builder::string(f.id()), *ztype, builder::bool_(f.isOptional())})); + } + + return builder::call("zeek_rt::create_record_type", {builder::string(id->namespace_()), + builder::string(id->local()), builder::vector(fields)}); + } + + result_t operator()(const hilti::type::Tuple& t) { + std::vector fields; + for ( const auto& f : t.elements() ) { + if ( ! f.id() ) + return hilti::result::Error("can only convert tuple types with all-named fields to Zeek"); + + auto ztype = createZeekType(f.type()); + if ( ! ztype ) + return ztype.error(); + + fields.emplace_back(builder::tuple({builder::string(*f.id()), *ztype, builder::bool_(false)})); + } + + return builder::call("zeek_rt::create_record_type", {builder::string(id->namespace_()), + builder::string(id->local()), builder::vector(fields)}); + } + + result_t operator()(const ::spicy::type::Unit& t) { + assert(id); + + std::vector fields; + for ( const auto& f : gc->recordFields(t) ) { + auto ztype = createZeekType(std::get<1>(f)); + if ( ! ztype ) + return ztype.error(); + + fields.emplace_back( + builder::tuple({builder::string(std::get<0>(f)), *ztype, builder::bool_(std::get<2>(f))})); + } + + return builder::call("zeek_rt::create_record_type", {builder::string(id->namespace_()), + builder::string(id->local()), builder::vector(fields)}); + } + + result_t operator()(const hilti::type::Vector& t) { + auto elem = createZeekType(t.elementType()); + if ( ! elem ) + return elem.error(); + + return builder::call("zeek_rt::create_vector_type", {*elem}); + } +}; +} // namespace + +hilti::Result GlueCompiler::createZeekType(const hilti::Type& t, const hilti::ID& id) const { + return VisitorZeekType(this).createZeekType(t, id); +} + +namespace { +struct VisitorUnitFields : hilti::visitor::PreOrder { + // NOTE: Align this logic with struct generation in Spicy's unit builder. + std::vector fields; + + void operator()(const ::spicy::type::unit::item::Field& f, position_t p) { + if ( f.isTransient() || f.parseType().isA() ) + return; + + fields.emplace_back(f.id(), f.itemType(), true); + } + + void operator()(const ::spicy::type::unit::item::Variable& f, const position_t p) { + fields.emplace_back(f.id(), f.itemType(), f.isOptional()); + } + + // TODO: void operator()(const ::spicy::type::unit::item::Switch & f, const position_t p) { +}; +} // namespace + +std::vector GlueCompiler::recordFields(const ::spicy::type::Unit& unit) { + VisitorUnitFields unit_field_converter; + + for ( const auto& i : unit.items() ) + unit_field_converter.dispatch(i); + + return std::move(unit_field_converter.fields); +} diff --git a/src/spicy/spicyz/glue-compiler.h b/src/spicy/spicyz/glue-compiler.h new file mode 100644 index 0000000000..2942fa445e --- /dev/null +++ b/src/spicy/spicyz/glue-compiler.h @@ -0,0 +1,242 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "driver.h" + +namespace spicy::rt { +struct Parser; +} + +namespace zeek::spicy { + +namespace glue { + +/** Representation of a Spicy protocol analyzer, parsed from an EVT file. */ +struct ProtocolAnalyzer { + // Information parsed directly from the *.evt file. + hilti::Location location; /**< Location where the analyzer was defined. */ + hilti::ID name; /**< Name of the analyzer. */ + hilti::rt::Protocol protocol = hilti::rt::Protocol::Undef; /**< The transport layer the analyzer uses. */ + std::vector ports; /**< The ports associated with the analyzer. */ + hilti::ID unit_name_orig; /**< The fully-qualified name of the unit type to parse the originator + side. */ + hilti::ID unit_name_resp; /**< The fully-qualified name of the unit type to parse the originator + side. */ + std::string replaces; /**< Name of another analyzer this one replaces. */ + + // Computed information. + std::optional unit_orig; /**< The type of the unit to parse the originator side. */ + std::optional unit_resp; /**< The type of the unit to parse the originator side. */ +}; + +/** Representation of a Spicy file analyzer, parsed from an EVT file. */ +struct FileAnalyzer { + // Information parsed directly from the *.evt file. + hilti::Location location; /**< Location where the analyzer was defined. */ + hilti::ID name; /**< Name of the analyzer. */ + std::vector mime_types; /**< The mime_types associated with the analyzer. */ + hilti::ID unit_name; /**< The fully-qualified name of the unit type to parse with. */ + std::string replaces; /**< Name of another analyzer this one replaces. */ + + // Computed information. + std::optional unit; /**< The type of the unit to parse the originator side. */ +}; + +/** Representation of a Spicy packet analyzer, parsed from an EVT file. */ +struct PacketAnalyzer { + // Information parsed directly from the *.evt file. + hilti::Location location; /**< Location where the analyzer was defined. */ + hilti::ID name; /**< Name of the analyzer. */ + hilti::ID unit_name; /**< The fully-qualified name of the unit type to parse with. */ + std::string replaces; /**< Name of another analyzer this one replaces. */ + + // Computed information. + std::optional unit; /**< The type of the unit to parse the originator side. */ +}; + +/** + * Representation of an expression computing the value of a parameter passed + * to Spicy-generated events. + */ +struct ExpressionAccessor { + // Information parsed directly from the *.evt file. + int nr; /**< Position of this expression in argument list. */ + std::string expression; /**< The original string representation of the expression. */ + hilti::Location location; /**< Location where the expression was defined. */ +}; + +/** Representation of a compiled Spicy module. */ +struct SpicyModule { + // Provided. + hilti::ID id; /**< Name of the module */ + hilti::rt::filesystem::path file; /**< The path the module was read from. */ + std::set evts; /**< EVT files that refer to this module. */ + + // Generated code. + std::optional spicy_module; /**< the ``BroHooks_*.spicy`` module. */ +}; + +/** Representation of an event parsed from an EVT file. */ +struct Event { + // Information parsed directly from the *.evt file. + hilti::rt::filesystem::path file; /**< The path of the *.evt file we parsed this from. */ + hilti::ID name; /**< The name of the event. */ + hilti::ID path; /**< The hook path as specified in the evt file. */ + std::vector parameters; /**< Event parameters specified in the evt file. */ + std::string condition; /**< Condition that must be true for the event to trigger. */ + std::vector exprs; /**< The argument expressions. */ + int priority; /**< Event/hook priority. */ + hilti::Location location; /**< Location where event is defined. */ + + // Computed information. + hilti::ID hook; /**< The name of the hook triggering the event. */ + hilti::ID unit; /**< The fully qualified name of the unit type. */ + std::optional<::spicy::type::Unit> unit_type; /**< The Spicy type of referenced unit. */ + hilti::ID unit_module_id; /**< The name of the module the referenced unit is defined in. */ + hilti::rt::filesystem::path unit_module_path; /**< The path of the module that the referenced unit is defined in. */ + std::shared_ptr + spicy_module; /**< State for the Spichy module the referenced unit is defined in. */ + + // TODO: The following aren't set yet. + + // Code generation. + std::optional<::spicy::declaration::UnitHook> spicy_hook; /**< The generated Spicy hook. */ + std::optional hilti_raise; /**< The generated HILTI raise() function. */ + std::vector expression_accessors; /**< One HILTI function per expression to access the value. */ +}; + +} // namespace glue + +/** Generates the glue code between Zeek and Spicy based on *.evt files. */ +class GlueCompiler { +public: + /** Constructor. */ + GlueCompiler() {} + + /** Destructor. */ + virtual ~GlueCompiler(); + + /** Parses an `*.evt` file, without generating any code yet. */ + bool loadEvtFile(hilti::rt::filesystem::path& path); + + /** + * Registers a Spicy file to generate glue code for, without generating + * any code yet. + * + * @param id ID of the module + * @param file path the module is loaded from + */ + void addSpicyModule(const hilti::ID& id, const hilti::rt::filesystem::path& file); + + /** + * Generates all glue code based on previously registered `*.evt` and + * Spicy files. + */ + bool compile(); + + /** Returns all IDs that have been exported so far. */ + const auto& exportedIDs() const { return _exports; } + + /** Generates code to convert a HILTI type to a corresponding Zeek type at runtime. */ + hilti::Result createZeekType(const hilti::Type& t, const hilti::ID& id) const; + + using RecordField = std::tuple; /**< (ID, type, optional) */ + + /** + * Helper to retrieve a list of Zeek-side record fields that converting a + * Spicy unit to a Zeek record will yield. + * + * @param unit the unit type to retrieve fields for + * @return list of fields + */ + static std::vector recordFields(const ::spicy::type::Unit& unit); + +protected: + friend class Driver; + + /** Called by driver to initialized a provided glue compiler. */ + void Init(Driver* driver, int zeek_version); + +private: + /** + * Filters input EVT file by applying preprocessor directives. + */ + void preprocessEvtFile(hilti::rt::filesystem::path& path, std::istream& in, std::ostream& out); + + /** + * Extracts the next semicolon-terminated block from an input stream, + * accounting for special EVT constructs like strings and comments. + * + * @param in stream to read from + * @param lineno pointer to integer that will be increased with line breaks + * @return the read block of data, with comments removed, and empty if end of + * data has been reached; error will be set if parsing failed + */ + hilti::Result getNextEvtBlock(std::istream& in, int* lineno) const; + + // Parsers for parts from EVT files. + glue::ProtocolAnalyzer parseProtocolAnalyzer(const std::string& chunk); + glue::FileAnalyzer parseFileAnalyzer(const std::string& chunk); + glue::PacketAnalyzer parsePacketAnalyzer(const std::string& chunk); + glue::Event parseEvent(const std::string& chunk); + + /** Computes the missing pieces for all `Event` instances. */ + bool PopulateEvents(); + + /** + * Create the Spicy hook for an event that triggers a corresponding Zeek + * event. + */ + bool CreateSpicyHook(glue::Event* ev); + + Driver* _driver = nullptr; /**< driver provided to Init() */ + std::optional _zeek_version; /**< Zeek version provided to Init() */ + + std::map> _spicy_modules; + + std::vector>> + _imports; /**< imports from EVT files, with ID and optional scope */ + std::vector> _exports; /**< exports from EVT files */ + std::vector _events; /**< events parsed from EVT files */ + std::vector _protocol_analyzers; /**< protocol analyzers parsed from EVT files */ + std::vector _file_analyzers; /**< file analyzers parsed from EVT files */ + std::vector _packet_analyzers; /**< file analyzers parsed from EVT files */ + std::vector _locations; /**< location stack during parsing EVT files */ +}; +} // namespace zeek::spicy + +namespace std { +template<> +struct hash { + std::size_t operator()(const zeek::spicy::glue::Event& e) { + // We only hash enough information here to unique identify the event. + return hilti::rt::hashCombine(std::hash()(e.file), std::hash()(e.name), + std::hash()(e.path), std::hash()(e.location)); + } +}; +} // namespace std diff --git a/src/spicy/spicyz/main.cc b/src/spicy/spicyz/main.cc new file mode 100644 index 0000000000..3945dfe7d8 --- /dev/null +++ b/src/spicy/spicyz/main.cc @@ -0,0 +1,282 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include + +#include +#include + +#include "config.h" +#include "driver.h" +#include "glue-compiler.h" + +using namespace zeek::spicy; + +constexpr int OPT_CXX_LINK = 1000; + +static struct option long_driver_options[] = {{"abort-on-exceptions", required_argument, nullptr, 'A'}, + {"show-backtraces", required_argument, nullptr, 'B'}, + {"compiler-debug", required_argument, nullptr, 'D'}, + {"cxx-link", required_argument, nullptr, OPT_CXX_LINK}, + {"debug", no_argument, nullptr, 'd'}, + {"debug-addl", required_argument, nullptr, 'X'}, + {"disable-optimizations", no_argument, nullptr, 'g'}, + {"dump-code", no_argument, nullptr, 'C'}, + {"enable-profiling", no_argument, nullptr, 'Z'}, + {"help", no_argument, nullptr, 'h'}, + {"keep-tmps", no_argument, nullptr, 'T'}, + {"library-path", required_argument, nullptr, 'L'}, + {"output", required_argument, nullptr, 'o'}, + {"output-c++", required_argument, nullptr, 'c'}, + {"output-c++-files", no_argument, nullptr, 'x'}, + {"print-module-path", no_argument, nullptr, 'M'}, + {"print-plugin-path", no_argument, nullptr, + 'P'}, // for backwards compatiblity + {"print-prefix-path", no_argument, nullptr, 'p'}, + {"print-zeek-config", no_argument, nullptr, 'z'}, + {"report-times", required_argument, nullptr, 'R'}, + {"print-scripts-path", no_argument, nullptr, 'S'}, + {"skip-validation", no_argument, nullptr, '!'}, + {"version", no_argument, nullptr, 'v'}, + {"version-number", no_argument, nullptr, 'V'}, + {nullptr, 0, nullptr, 0}}; + +static void usage() { + std::cerr << "Usage: spicyz [options] \n" + "\n" + " -c | --output-c++ Output generated C++ code.\n" + " -d | --debug Include debug instrumentation into generated code.\n" + " -g | --disable-optimizations Disable HILTI-side optimizations of the generated " + "code.\n" + " -o | --output-to Path for saving output.\n" + " -v | --version Print version information.\n" + " -x | --output-c++ Output generated C++ code into set of files.\n" + " -z | --print-zeek-config Print path to zeek-config.\n" + " -A | --abort-on-exceptions When executing compiled code, abort() instead of " + "throwing HILTI " + "exceptions.\n" + " -B | --show-backtraces Include backtraces when reporting unhandled " + "exceptions.\n" + " -C | --dump-code Dump all generated code to disk for debugging.\n" + " -D | --compiler-debug Activate compile-time debugging output for given " + "debug streams " + "(comma-separated; 'help' for list).\n" + " -L | --library-path Add path to list of directories to search when " + "importing modules.\n" + " -M | --print-module-path Print the Zeek's search path for compiled Spicy modules.\n" + " -p | --print-prefix-path Print installation prefix path.\n" + " -R | --report-times Report a break-down of compiler's execution time.\n" + " -T | --keep-tmps Do not delete any temporary files created.\n" + " --skip-validation Don't validate ASTs (for debugging only).\n" + " -X | --debug-addl Implies -d and adds selected additional " + "instrumentation." + "(comma-separated; see 'help' for list).\n" + " --cxx-link Link specified static archive or shared library " + "during JIT or to " + " -Z | --enable-profiling Report profiling statistics after execution.\n" + "\n" + "Inputs can be *.spicy, *.evt, *.hlt, .cc/.cxx\n" + "\n"; +} + +using hilti::Nothing; + +static hilti::Result parseOptions(int argc, char** argv, hilti::driver::Options* driver_options, + hilti::Options* compiler_options) { + while ( true ) { + int c = getopt_long(argc, argv, "ABc:Cdgx:X:D:L:Mo:pPRSTvhzZ", long_driver_options, nullptr); + + if ( c == -1 ) + break; + + switch ( c ) { + case 'A': driver_options->abort_on_exceptions = true; break; + + case 'B': driver_options->show_backtraces = true; break; + + case 'c': + driver_options->output_cxx = true; + driver_options->output_cxx_prefix = optarg; + driver_options->execute_code = false; + compiler_options->cxx_namespace_extern = + hilti::util::fmt("hlt_%s", hilti::rt::filesystem::path(optarg).stem().string()); + compiler_options->cxx_namespace_intern = + hilti::util::fmt("__hlt_%s", hilti::rt::filesystem::path(optarg).stem().string()); + break; + + case 'C': { + driver_options->dump_code = true; + break; + } + + case 'd': { + compiler_options->debug = true; + break; + } + + case 'g': { + driver_options->global_optimizations = false; + break; + } + + case 'p': std::cout << configuration::InstallPrefix.native() << std::endl; return Nothing(); + + case 'P': + // For backwards compatibility with older plugins, print + // the path where the `cmake/` folder is located. + std::cout << configuration::DataPath().native() << std::endl; + return Nothing(); + + case 'x': + driver_options->output_cxx = true; + driver_options->output_cxx_prefix = optarg; + driver_options->execute_code = false; + driver_options->include_linker = true; + compiler_options->cxx_namespace_extern = + hilti::util::fmt("hlt_%s", hilti::rt::filesystem::path(optarg).stem().string()); + compiler_options->cxx_namespace_intern = + hilti::util::fmt("__hlt_%s", hilti::rt::filesystem::path(optarg).stem().string()); + break; + + case 'X': { + auto arg = std::string(optarg); + + if ( arg == "help" ) { + std::cerr << "Additional debug instrumentation:\n"; + std::cerr << " flow: log function calls to debug stream \"hilti-flow\"\n"; + std::cerr << " location: track current source code location for error reporting\n"; + std::cerr << " trace: log statements to debug stream \"hilti-trace\"\n"; + std::cerr << "\n"; + exit(0); + } + + compiler_options->debug = true; + + if ( auto r = compiler_options->parseDebugAddl(arg); ! r ) + return hilti::result::Error("nothing to do"); + + break; + } + + case 'D': { + auto arg = std::string(optarg); + + if ( arg == "help" ) { + std::cerr << "Debug streams:\n"; + + for ( const auto& s : hilti::logging::DebugStream::all() ) + std::cerr << " " << s << "\n"; + + std::cerr << "\n"; + return Nothing(); + } + + for ( const auto& s : hilti::util::split(arg, ",") ) { + if ( ! driver_options->logger->debugEnable(s) ) + return hilti::result::Error( + hilti::util::fmt("Unknown debug stream '%s', use 'help' for list", arg)); + } + + break; + } + + case 'L': compiler_options->library_paths.emplace_back(std::string(optarg)); break; + + case 'M': std::cout << configuration::ModulePath().native() << std::endl; return Nothing(); + + case 'o': driver_options->output_path = std::string(optarg); break; + + case 'R': driver_options->report_times = true; break; + + case 'S': std::cout << "" << std::endl; return Nothing(); // No longer needed, but left for compatibility. + + case 'T': + driver_options->keep_tmps = true; + compiler_options->keep_tmps = true; + break; + + case 'v': std::cout << configuration::ZeekVersion << std::endl; return Nothing(); + + case 'V': std::cout << ZEEK_VERSION_NUMBER << std::endl; return Nothing(); + + case 'z': { + if ( auto zcfg = getenv("ZEEK_CONFIG"); zcfg && *zcfg ) + std::cout << zcfg << std::endl; + else + std::cout << configuration::InstallBinDir().native() << std::endl; + + return Nothing(); + } + + case 'Z': +#if SPICY_VERSION_NUMBER >= 10800 + driver_options->enable_profiling = true; + compiler_options->enable_profiling = true; +#else + std::cerr << "Profiling is not supported with this version of Spicy, ignoring '-Z'\n"; +#endif + break; + + case OPT_CXX_LINK: +#if SPICY_VERSION_NUMBER >= 10600 + compiler_options->cxx_link.emplace_back(optarg); +#else + return hilti::result::Error("option '--cxx-link' is only supported for Spicy 1.6 or newer"); +#endif + break; + + case 'h': usage(); return Nothing(); + + case '!': compiler_options->skip_validation = true; break; + + default: usage(); return hilti::result::Error("could not parse options"); + } + } + + while ( optind < argc ) + driver_options->inputs.emplace_back(argv[optind++]); + + if ( driver_options->inputs.empty() ) + return hilti::result::Error("no input file given"); + + if ( driver_options->output_path.empty() && ! driver_options->output_cxx ) + return hilti::result::Error("no output file for object code given, use -o .hlto"); + + return Nothing(); +} + +int main(int argc, char** argv) { + Driver driver(std::make_unique(), "", configuration::LibraryPath(), ZEEK_VERSION_NUMBER); + + hilti::driver::Options driver_options; + driver_options.execute_code = true; + driver_options.include_linker = true; + + auto compiler_options = driver.hiltiOptions(); + + if ( auto rc = parseOptions(argc, argv, &driver_options, &compiler_options); ! rc ) { + hilti::logger().error(rc.error().description()); + return 1; + } + + driver.setDriverOptions(std::move(driver_options)); + driver.setCompilerOptions(std::move(compiler_options)); + driver.initialize(); + + for ( const auto& p : driver.driverOptions().inputs ) { + if ( auto rc = driver.loadFile(p); ! rc ) { + hilti::logger().error(rc.error().description()); + return 1; + } + } + + if ( auto rc = driver.compile(); ! rc ) { + hilti::logger().error(rc.error().description()); + + if ( rc.error().context().size() ) + hilti::logger().error(rc.error().context()); + + return 1; + } + + return 0; +} diff --git a/src/util-config.h.in b/src/util-config.h.in index 6272fba391..689ac2461e 100644 --- a/src/util-config.h.in +++ b/src/util-config.h.in @@ -1,3 +1,7 @@ #define ZEEK_SCRIPT_INSTALL_PATH "@ZEEK_SCRIPT_INSTALL_PATH@" #define BRO_PLUGIN_INSTALL_PATH "@ZEEK_PLUGIN_DIR@" +#define ZEEK_PLUGIN_INSTALL_PATH "@ZEEK_PLUGIN_DIR@" #define DEFAULT_ZEEKPATH "@DEFAULT_ZEEKPATH@" +#define ZEEK_SPICY_MODULE_PATH "@ZEEK_SPICY_MODULE_PATH@" +#define ZEEK_SPICY_LIBRARY_PATH "@ZEEK_SPICY_LIBRARY_PATH@" +#define ZEEK_SPICY_DATA_PATH "@ZEEK_SPICY_DATA_PATH@" diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index 5eb981a2a1..46d5eacef0 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -65,6 +65,9 @@ #include "zeek/plugin/Manager.h" #include "zeek/script_opt/ScriptOpt.h" #include "zeek/session/Manager.h" +#ifdef HAVE_SPICY +#include "zeek/spicy/manager.h" +#endif #include "zeek/supervisor/Supervisor.h" #include "zeek/telemetry/Manager.h" #include "zeek/threading/Manager.h" @@ -191,6 +194,9 @@ zeek::Broker::Manager* zeek::broker_mgr = nullptr; zeek::telemetry::Manager* zeek::telemetry_mgr = nullptr; zeek::Supervisor* zeek::supervisor_mgr = nullptr; zeek::detail::trigger::Manager* zeek::detail::trigger_mgr = nullptr; +#ifdef HAVE_SPICY +zeek::spicy::Manager* zeek::spicy_mgr = nullptr; +#endif std::vector zeek::detail::zeek_script_prefixes; zeek::detail::Stmt* zeek::detail::stmts = nullptr; @@ -437,6 +443,9 @@ static void terminate_zeek() delete session_mgr; delete fragment_mgr; delete telemetry_mgr; +#ifdef HAVE_SPICY + delete spicy_mgr; +#endif // free the global scope pop_scope(); @@ -719,6 +728,9 @@ SetupResult setup(int argc, char** argv, Options* zopts) auto broker_real_time = ! options.pcap_file && ! options.deterministic_mode; broker_mgr = new Broker::Manager(broker_real_time); trigger_mgr = new trigger::Manager(); +#ifdef HAVE_SPICY + spicy_mgr = new spicy::Manager(); // registers as plugin with the plugin manager +#endif plugin_mgr->InitPreScript(); file_mgr->InitPreScript(); diff --git a/testing/CMakeLists.txt b/testing/CMakeLists.txt index 0ed2fa2cf0..f3c12302e4 100644 --- a/testing/CMakeLists.txt +++ b/testing/CMakeLists.txt @@ -8,3 +8,14 @@ install(FILES btest/random.seed DESTINATION ${ZEEK_CONFIG_BTEST_TOOLS_DIR}/data) if (INSTALL_BTEST_PCAPS) install(DIRECTORY btest/Traces/ DESTINATION ${ZEEK_CONFIG_BTEST_TOOLS_DIR}/data/pcaps) endif () + +# The remainder is for backwards-compatability with existing Spicy zkg packages. +install( + CODE "execute_process( \ + COMMAND ${CMAKE_COMMAND} -E create_symlink \ + ${ZEEK_CONFIG_BTEST_TOOLS_DIR}/data \ + ${CMAKE_INSTALL_PREFIX}/share/zeek/tests \ + )") + +install(DIRECTORY scripts/spicy/ DESTINATION ${ZEEK_CONFIG_BTEST_TOOLS_DIR}/data/Scripts + USE_SOURCE_PERMISSIONS) diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 259569eef2..5540e6d45b 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -91,8 +91,8 @@ scripts/base/init-bare.zeek scripts/base/packet-protocols/gtpv1/__load__.zeek scripts/base/packet-protocols/gtpv1/main.zeek build/scripts/base/bif/plugins/Zeek_GTPv1.functions.bif.zeek + scripts/base/frameworks/spicy/init-bare.zeek build/scripts/builtin-plugins/__preload__.zeek - build/scripts/builtin-plugins/Zeek_Spicy/__preload__.zeek scripts/base/init-frameworks-and-bifs.zeek scripts/base/frameworks/logging/__load__.zeek scripts/base/frameworks/logging/main.zeek @@ -148,6 +148,7 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/bloom-filter.bif.zeek build/scripts/base/bif/cardinality-counter.bif.zeek build/scripts/base/bif/top-k.bif.zeek + build/scripts/base/bif/spicy.bif.zeek build/scripts/base/bif/plugins/__load__.zeek build/scripts/base/bif/plugins/Zeek_BitTorrent.events.bif.zeek build/scripts/base/bif/plugins/Zeek_ConnSize.events.bif.zeek @@ -264,16 +265,12 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/plugins/Zeek_AsciiWriter.ascii.bif.zeek build/scripts/base/bif/plugins/Zeek_NoneWriter.none.bif.zeek build/scripts/base/bif/plugins/Zeek_SQLiteWriter.sqlite.bif.zeek - build/scripts/base/bif/plugins/Zeek_Spicy.consts.bif.zeek - build/scripts/base/bif/plugins/Zeek_Spicy.events.bif.zeek - build/scripts/base/bif/plugins/Zeek_Spicy.functions.bif.zeek + scripts/base/frameworks/spicy/init-framework.zeek + scripts/base/misc/version.zeek + scripts/base/frameworks/reporter/__load__.zeek + scripts/base/frameworks/reporter/main.zeek + scripts/base/utils/strings.zeek build/scripts/builtin-plugins/__load__.zeek - build/scripts/builtin-plugins/Zeek_Spicy/__load__.zeek - build/scripts/builtin-plugins/Zeek_Spicy/Zeek/Spicy/bare.zeek - scripts/base/misc/version.zeek - scripts/base/frameworks/reporter/__load__.zeek - scripts/base/frameworks/reporter/main.zeek - scripts/base/utils/strings.zeek scripts/policy/misc/loaded-scripts.zeek scripts/base/utils/paths.zeek #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 47d87c5136..2c597084a7 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -91,8 +91,8 @@ scripts/base/init-bare.zeek scripts/base/packet-protocols/gtpv1/__load__.zeek scripts/base/packet-protocols/gtpv1/main.zeek build/scripts/base/bif/plugins/Zeek_GTPv1.functions.bif.zeek + scripts/base/frameworks/spicy/init-bare.zeek build/scripts/builtin-plugins/__preload__.zeek - build/scripts/builtin-plugins/Zeek_Spicy/__preload__.zeek scripts/base/init-frameworks-and-bifs.zeek scripts/base/frameworks/logging/__load__.zeek scripts/base/frameworks/logging/main.zeek @@ -148,6 +148,7 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/bloom-filter.bif.zeek build/scripts/base/bif/cardinality-counter.bif.zeek build/scripts/base/bif/top-k.bif.zeek + build/scripts/base/bif/spicy.bif.zeek build/scripts/base/bif/plugins/__load__.zeek build/scripts/base/bif/plugins/Zeek_BitTorrent.events.bif.zeek build/scripts/base/bif/plugins/Zeek_ConnSize.events.bif.zeek @@ -264,9 +265,11 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/plugins/Zeek_AsciiWriter.ascii.bif.zeek build/scripts/base/bif/plugins/Zeek_NoneWriter.none.bif.zeek build/scripts/base/bif/plugins/Zeek_SQLiteWriter.sqlite.bif.zeek - build/scripts/base/bif/plugins/Zeek_Spicy.consts.bif.zeek - build/scripts/base/bif/plugins/Zeek_Spicy.events.bif.zeek - build/scripts/base/bif/plugins/Zeek_Spicy.functions.bif.zeek + scripts/base/frameworks/spicy/init-framework.zeek + scripts/base/misc/version.zeek + scripts/base/frameworks/reporter/__load__.zeek + scripts/base/frameworks/reporter/main.zeek + scripts/base/utils/strings.zeek scripts/base/init-default.zeek scripts/base/utils/active-http.zeek scripts/base/utils/exec.zeek @@ -274,8 +277,6 @@ scripts/base/init-default.zeek scripts/base/utils/backtrace.zeek scripts/base/utils/conn-ids.zeek scripts/base/utils/dir.zeek - scripts/base/frameworks/reporter/__load__.zeek - scripts/base/frameworks/reporter/main.zeek scripts/base/utils/paths.zeek scripts/base/utils/directions-and-hosts.zeek scripts/base/utils/email.zeek @@ -283,7 +284,6 @@ scripts/base/init-default.zeek scripts/base/utils/geoip-distance.zeek scripts/base/utils/numbers.zeek scripts/base/utils/queue.zeek - scripts/base/utils/strings.zeek scripts/base/utils/thresholds.zeek scripts/base/utils/time.zeek scripts/base/utils/urls.zeek @@ -347,7 +347,8 @@ scripts/base/init-default.zeek scripts/base/frameworks/netcontrol/non-cluster.zeek scripts/base/frameworks/telemetry/__load__.zeek scripts/base/frameworks/telemetry/main.zeek - scripts/base/misc/version.zeek + scripts/base/frameworks/spicy/__load__.zeek + scripts/base/frameworks/spicy/main.zeek scripts/base/protocols/conn/__load__.zeek scripts/base/protocols/conn/main.zeek scripts/base/protocols/conn/contents.zeek @@ -463,8 +464,5 @@ scripts/base/init-default.zeek scripts/base/misc/find-filtered-trace.zeek build/scripts/base/misc/installation.zeek build/scripts/builtin-plugins/__load__.zeek - build/scripts/builtin-plugins/Zeek_Spicy/__load__.zeek - build/scripts/builtin-plugins/Zeek_Spicy/Zeek/Spicy/bare.zeek - build/scripts/builtin-plugins/Zeek_Spicy/Zeek/Spicy/default.zeek scripts/policy/misc/loaded-scripts.zeek #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/coverage.init-default/missing_loads b/testing/btest/Baseline/coverage.init-default/missing_loads index 33a5c60cfb..6bf8ca440c 100644 --- a/testing/btest/Baseline/coverage.init-default/missing_loads +++ b/testing/btest/Baseline/coverage.init-default/missing_loads @@ -10,6 +10,8 @@ -./frameworks/netcontrol/cluster.zeek -./frameworks/openflow/cluster.zeek -./frameworks/packet-filter/cluster.zeek +-./frameworks/spicy/misc/record-spicy-batch.zeek +-./frameworks/spicy/misc/resource-usage.zeek -./frameworks/sumstats/cluster.zeek -./frameworks/telemetry/cluster.zeek -./init-supervisor.zeek diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 8ab256052c..c1321438b9 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -790,7 +790,6 @@ 0.000000 MetaHookPost CallFunction(Version::parse, ..., ...) -> 0.000000 MetaHookPost CallFunction(__init_primary_bifs, , ()) -> 0.000000 MetaHookPost CallFunction(__init_secondary_bifs, , ()) -> -0.000000 MetaHookPost CallFunction(bare_mode, , ()) -> 0.000000 MetaHookPost CallFunction(cat, ..., ...) -> 0.000000 MetaHookPost CallFunction(current_time, , ()) -> 0.000000 MetaHookPost CallFunction(disable_event_group, , (Analyzer::Logging::include_confirmations)) -> @@ -938,9 +937,6 @@ 0.000000 MetaHookPost LoadFile(0, ./Zeek_SSL.events.bif.zeek, <...>/Zeek_SSL.events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_SSL.functions.bif.zeek, <...>/Zeek_SSL.functions.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_SSL.types.bif.zeek, <...>/Zeek_SSL.types.bif.zeek) -> -1 -0.000000 MetaHookPost LoadFile(0, ./Zeek_Spicy.consts.bif.zeek, <...>/Zeek_Spicy.consts.bif.zeek) -> -1 -0.000000 MetaHookPost LoadFile(0, ./Zeek_Spicy.events.bif.zeek, <...>/Zeek_Spicy.events.bif.zeek) -> -1 -0.000000 MetaHookPost LoadFile(0, ./Zeek_Spicy.functions.bif.zeek, <...>/Zeek_Spicy.functions.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_TCP.events.bif.zeek, <...>/Zeek_TCP.events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_TCP.functions.bif.zeek, <...>/Zeek_TCP.functions.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_TCP.types.bif.zeek, <...>/Zeek_TCP.types.bif.zeek) -> -1 @@ -1024,6 +1020,7 @@ 0.000000 MetaHookPost LoadFile(0, ./smb1-main, <...>/smb1-main.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./smb2-main, <...>/smb2-main.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./spicy-events, <...>/spicy-events.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, ./spicy.bif.zeek, <...>/spicy.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./stats.bif.zeek, <...>/stats.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./std-dev, <...>/std-dev.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./store, <...>/store.zeek) -> -1 @@ -1058,10 +1055,6 @@ 0.000000 MetaHookPost LoadFile(0, <...>/__load__.zeek, <...>/__load__.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, <...>/__preload__.zeek, <...>/__preload__.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, <...>/hooks.zeek, <...>/hooks.zeek) -> -1 -0.000000 MetaHookPost LoadFile(0, Zeek<...>/bare.zeek, <...>/bare.zeek) -> -1 -0.000000 MetaHookPost LoadFile(0, Zeek<...>/default.zeek, <...>/default.zeek) -> -1 -0.000000 MetaHookPost LoadFile(0, Zeek_Spicy/__load__.zeek, <...>/__load__.zeek) -> -1 -0.000000 MetaHookPost LoadFile(0, Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base/bif, <...>/bif) -> -1 0.000000 MetaHookPost LoadFile(0, base/init-default.zeek, <...>/init-default.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base/init-frameworks-and-bifs.zeek, <...>/init-frameworks-and-bifs.zeek) -> -1 @@ -1119,6 +1112,8 @@ 0.000000 MetaHookPost LoadFile(0, base<...>/ieee802_11, <...>/ieee802_11) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/ieee802_11_radio, <...>/ieee802_11_radio) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/imap, <...>/imap) -> -1 +0.000000 MetaHookPost LoadFile(0, base<...>/init-bare, <...>/init-bare.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, base<...>/init-framework, <...>/init-framework.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/input, <...>/input) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/input.bif, <...>/input.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/installation, <...>/installation.zeek) -> -1 @@ -1177,6 +1172,7 @@ 0.000000 MetaHookPost LoadFile(0, base<...>/snmp, <...>/snmp) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/socks, <...>/socks) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/software, <...>/software) -> -1 +0.000000 MetaHookPost LoadFile(0, base<...>/spicy, <...>/spicy) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/ssh, <...>/ssh) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/ssl, <...>/ssl) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/stats.bif, <...>/stats.bif.zeek) -> -1 @@ -1331,9 +1327,6 @@ 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SSL.events.bif.zeek, <...>/Zeek_SSL.events.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SSL.functions.bif.zeek, <...>/Zeek_SSL.functions.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SSL.types.bif.zeek, <...>/Zeek_SSL.types.bif.zeek) -> (-1, ) -0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Spicy.consts.bif.zeek, <...>/Zeek_Spicy.consts.bif.zeek) -> (-1, ) -0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Spicy.events.bif.zeek, <...>/Zeek_Spicy.events.bif.zeek) -> (-1, ) -0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Spicy.functions.bif.zeek, <...>/Zeek_Spicy.functions.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_TCP.events.bif.zeek, <...>/Zeek_TCP.events.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_TCP.functions.bif.zeek, <...>/Zeek_TCP.functions.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_TCP.types.bif.zeek, <...>/Zeek_TCP.types.bif.zeek) -> (-1, ) @@ -1417,6 +1410,7 @@ 0.000000 MetaHookPost LoadFileExtended(0, ./smb1-main, <...>/smb1-main.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./smb2-main, <...>/smb2-main.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./spicy-events, <...>/spicy-events.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./spicy.bif.zeek, <...>/spicy.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./stats.bif.zeek, <...>/stats.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./std-dev, <...>/std-dev.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./store, <...>/store.zeek) -> (-1, ) @@ -1451,10 +1445,6 @@ 0.000000 MetaHookPost LoadFileExtended(0, <...>/__load__.zeek, <...>/__load__.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, <...>/__preload__.zeek, <...>/__preload__.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, <...>/hooks.zeek, <...>/hooks.zeek) -> (-1, ) -0.000000 MetaHookPost LoadFileExtended(0, Zeek<...>/bare.zeek, <...>/bare.zeek) -> (-1, ) -0.000000 MetaHookPost LoadFileExtended(0, Zeek<...>/default.zeek, <...>/default.zeek) -> (-1, ) -0.000000 MetaHookPost LoadFileExtended(0, Zeek_Spicy/__load__.zeek, <...>/__load__.zeek) -> (-1, ) -0.000000 MetaHookPost LoadFileExtended(0, Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base/bif, <...>/bif) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base/init-default.zeek, <...>/init-default.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base/init-frameworks-and-bifs.zeek, <...>/init-frameworks-and-bifs.zeek) -> (-1, ) @@ -1512,6 +1502,8 @@ 0.000000 MetaHookPost LoadFileExtended(0, base<...>/ieee802_11, <...>/ieee802_11) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/ieee802_11_radio, <...>/ieee802_11_radio) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/imap, <...>/imap) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/init-bare, <...>/init-bare.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/init-framework, <...>/init-framework.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/input, <...>/input) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/input.bif, <...>/input.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/installation, <...>/installation.zeek) -> (-1, ) @@ -1570,6 +1562,7 @@ 0.000000 MetaHookPost LoadFileExtended(0, base<...>/snmp, <...>/snmp) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/socks, <...>/socks) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/software, <...>/software) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base<...>/spicy, <...>/spicy) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/ssh, <...>/ssh) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/ssl, <...>/ssl) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/stats.bif, <...>/stats.bif.zeek) -> (-1, ) @@ -2410,7 +2403,6 @@ 0.000000 MetaHookPre CallFunction(Version::parse, ..., ...) 0.000000 MetaHookPre CallFunction(__init_primary_bifs, , ()) 0.000000 MetaHookPre CallFunction(__init_secondary_bifs, , ()) -0.000000 MetaHookPre CallFunction(bare_mode, , ()) 0.000000 MetaHookPre CallFunction(cat, ..., ...) 0.000000 MetaHookPre CallFunction(current_time, , ()) 0.000000 MetaHookPre CallFunction(disable_event_group, , (Analyzer::Logging::include_confirmations)) @@ -2558,9 +2550,6 @@ 0.000000 MetaHookPre LoadFile(0, ./Zeek_SSL.events.bif.zeek, <...>/Zeek_SSL.events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_SSL.functions.bif.zeek, <...>/Zeek_SSL.functions.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_SSL.types.bif.zeek, <...>/Zeek_SSL.types.bif.zeek) -0.000000 MetaHookPre LoadFile(0, ./Zeek_Spicy.consts.bif.zeek, <...>/Zeek_Spicy.consts.bif.zeek) -0.000000 MetaHookPre LoadFile(0, ./Zeek_Spicy.events.bif.zeek, <...>/Zeek_Spicy.events.bif.zeek) -0.000000 MetaHookPre LoadFile(0, ./Zeek_Spicy.functions.bif.zeek, <...>/Zeek_Spicy.functions.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_TCP.events.bif.zeek, <...>/Zeek_TCP.events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_TCP.functions.bif.zeek, <...>/Zeek_TCP.functions.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_TCP.types.bif.zeek, <...>/Zeek_TCP.types.bif.zeek) @@ -2644,6 +2633,7 @@ 0.000000 MetaHookPre LoadFile(0, ./smb1-main, <...>/smb1-main.zeek) 0.000000 MetaHookPre LoadFile(0, ./smb2-main, <...>/smb2-main.zeek) 0.000000 MetaHookPre LoadFile(0, ./spicy-events, <...>/spicy-events.zeek) +0.000000 MetaHookPre LoadFile(0, ./spicy.bif.zeek, <...>/spicy.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./stats.bif.zeek, <...>/stats.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./std-dev, <...>/std-dev.zeek) 0.000000 MetaHookPre LoadFile(0, ./store, <...>/store.zeek) @@ -2678,10 +2668,6 @@ 0.000000 MetaHookPre LoadFile(0, <...>/__load__.zeek, <...>/__load__.zeek) 0.000000 MetaHookPre LoadFile(0, <...>/__preload__.zeek, <...>/__preload__.zeek) 0.000000 MetaHookPre LoadFile(0, <...>/hooks.zeek, <...>/hooks.zeek) -0.000000 MetaHookPre LoadFile(0, Zeek<...>/bare.zeek, <...>/bare.zeek) -0.000000 MetaHookPre LoadFile(0, Zeek<...>/default.zeek, <...>/default.zeek) -0.000000 MetaHookPre LoadFile(0, Zeek_Spicy/__load__.zeek, <...>/__load__.zeek) -0.000000 MetaHookPre LoadFile(0, Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek) 0.000000 MetaHookPre LoadFile(0, base/bif, <...>/bif) 0.000000 MetaHookPre LoadFile(0, base/init-default.zeek, <...>/init-default.zeek) 0.000000 MetaHookPre LoadFile(0, base/init-frameworks-and-bifs.zeek, <...>/init-frameworks-and-bifs.zeek) @@ -2739,6 +2725,8 @@ 0.000000 MetaHookPre LoadFile(0, base<...>/ieee802_11, <...>/ieee802_11) 0.000000 MetaHookPre LoadFile(0, base<...>/ieee802_11_radio, <...>/ieee802_11_radio) 0.000000 MetaHookPre LoadFile(0, base<...>/imap, <...>/imap) +0.000000 MetaHookPre LoadFile(0, base<...>/init-bare, <...>/init-bare.zeek) +0.000000 MetaHookPre LoadFile(0, base<...>/init-framework, <...>/init-framework.zeek) 0.000000 MetaHookPre LoadFile(0, base<...>/input, <...>/input) 0.000000 MetaHookPre LoadFile(0, base<...>/input.bif, <...>/input.bif.zeek) 0.000000 MetaHookPre LoadFile(0, base<...>/installation, <...>/installation.zeek) @@ -2797,6 +2785,7 @@ 0.000000 MetaHookPre LoadFile(0, base<...>/snmp, <...>/snmp) 0.000000 MetaHookPre LoadFile(0, base<...>/socks, <...>/socks) 0.000000 MetaHookPre LoadFile(0, base<...>/software, <...>/software) +0.000000 MetaHookPre LoadFile(0, base<...>/spicy, <...>/spicy) 0.000000 MetaHookPre LoadFile(0, base<...>/ssh, <...>/ssh) 0.000000 MetaHookPre LoadFile(0, base<...>/ssl, <...>/ssl) 0.000000 MetaHookPre LoadFile(0, base<...>/stats.bif, <...>/stats.bif.zeek) @@ -2951,9 +2940,6 @@ 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SSL.events.bif.zeek, <...>/Zeek_SSL.events.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SSL.functions.bif.zeek, <...>/Zeek_SSL.functions.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SSL.types.bif.zeek, <...>/Zeek_SSL.types.bif.zeek) -0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Spicy.consts.bif.zeek, <...>/Zeek_Spicy.consts.bif.zeek) -0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Spicy.events.bif.zeek, <...>/Zeek_Spicy.events.bif.zeek) -0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Spicy.functions.bif.zeek, <...>/Zeek_Spicy.functions.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_TCP.events.bif.zeek, <...>/Zeek_TCP.events.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_TCP.functions.bif.zeek, <...>/Zeek_TCP.functions.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_TCP.types.bif.zeek, <...>/Zeek_TCP.types.bif.zeek) @@ -3037,6 +3023,7 @@ 0.000000 MetaHookPre LoadFileExtended(0, ./smb1-main, <...>/smb1-main.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./smb2-main, <...>/smb2-main.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./spicy-events, <...>/spicy-events.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./spicy.bif.zeek, <...>/spicy.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./stats.bif.zeek, <...>/stats.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./std-dev, <...>/std-dev.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./store, <...>/store.zeek) @@ -3071,10 +3058,6 @@ 0.000000 MetaHookPre LoadFileExtended(0, <...>/__load__.zeek, <...>/__load__.zeek) 0.000000 MetaHookPre LoadFileExtended(0, <...>/__preload__.zeek, <...>/__preload__.zeek) 0.000000 MetaHookPre LoadFileExtended(0, <...>/hooks.zeek, <...>/hooks.zeek) -0.000000 MetaHookPre LoadFileExtended(0, Zeek<...>/bare.zeek, <...>/bare.zeek) -0.000000 MetaHookPre LoadFileExtended(0, Zeek<...>/default.zeek, <...>/default.zeek) -0.000000 MetaHookPre LoadFileExtended(0, Zeek_Spicy/__load__.zeek, <...>/__load__.zeek) -0.000000 MetaHookPre LoadFileExtended(0, Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek) 0.000000 MetaHookPre LoadFileExtended(0, base/bif, <...>/bif) 0.000000 MetaHookPre LoadFileExtended(0, base/init-default.zeek, <...>/init-default.zeek) 0.000000 MetaHookPre LoadFileExtended(0, base/init-frameworks-and-bifs.zeek, <...>/init-frameworks-and-bifs.zeek) @@ -3132,6 +3115,8 @@ 0.000000 MetaHookPre LoadFileExtended(0, base<...>/ieee802_11, <...>/ieee802_11) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/ieee802_11_radio, <...>/ieee802_11_radio) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/imap, <...>/imap) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/init-bare, <...>/init-bare.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/init-framework, <...>/init-framework.zeek) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/input, <...>/input) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/input.bif, <...>/input.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/installation, <...>/installation.zeek) @@ -3190,6 +3175,7 @@ 0.000000 MetaHookPre LoadFileExtended(0, base<...>/snmp, <...>/snmp) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/socks, <...>/socks) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/software, <...>/software) +0.000000 MetaHookPre LoadFileExtended(0, base<...>/spicy, <...>/spicy) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/ssh, <...>/ssh) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/ssl, <...>/ssl) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/stats.bif, <...>/stats.bif.zeek) @@ -4029,7 +4015,6 @@ 0.000000 | HookCallFunction Version::parse(...) 0.000000 | HookCallFunction __init_primary_bifs() 0.000000 | HookCallFunction __init_secondary_bifs() -0.000000 | HookCallFunction bare_mode() 0.000000 | HookCallFunction cat(...) 0.000000 | HookCallFunction current_time() 0.000000 | HookCallFunction disable_event_group(Analyzer::Logging::include_confirmations) @@ -4177,9 +4162,6 @@ 0.000000 | HookLoadFile ./Zeek_SSL.events.bif.zeek <...>/Zeek_SSL.events.bif.zeek 0.000000 | HookLoadFile ./Zeek_SSL.functions.bif.zeek <...>/Zeek_SSL.functions.bif.zeek 0.000000 | HookLoadFile ./Zeek_SSL.types.bif.zeek <...>/Zeek_SSL.types.bif.zeek -0.000000 | HookLoadFile ./Zeek_Spicy.consts.bif.zeek <...>/Zeek_Spicy.consts.bif.zeek -0.000000 | HookLoadFile ./Zeek_Spicy.events.bif.zeek <...>/Zeek_Spicy.events.bif.zeek -0.000000 | HookLoadFile ./Zeek_Spicy.functions.bif.zeek <...>/Zeek_Spicy.functions.bif.zeek 0.000000 | HookLoadFile ./Zeek_TCP.events.bif.zeek <...>/Zeek_TCP.events.bif.zeek 0.000000 | HookLoadFile ./Zeek_TCP.functions.bif.zeek <...>/Zeek_TCP.functions.bif.zeek 0.000000 | HookLoadFile ./Zeek_TCP.types.bif.zeek <...>/Zeek_TCP.types.bif.zeek @@ -4274,6 +4256,7 @@ 0.000000 | HookLoadFile ./smb1-main <...>/smb1-main.zeek 0.000000 | HookLoadFile ./smb2-main <...>/smb2-main.zeek 0.000000 | HookLoadFile ./spicy-events <...>/spicy-events.zeek +0.000000 | HookLoadFile ./spicy.bif.zeek <...>/spicy.bif.zeek 0.000000 | HookLoadFile ./stats.bif.zeek <...>/stats.bif.zeek 0.000000 | HookLoadFile ./std-dev <...>/std-dev.zeek 0.000000 | HookLoadFile ./store <...>/store.zeek @@ -4309,10 +4292,6 @@ 0.000000 | HookLoadFile <...>/__load__.zeek <...>/__load__.zeek 0.000000 | HookLoadFile <...>/__preload__.zeek <...>/__preload__.zeek 0.000000 | HookLoadFile <...>/hooks.zeek <...>/hooks.zeek -0.000000 | HookLoadFile Zeek<...>/bare.zeek <...>/bare.zeek -0.000000 | HookLoadFile Zeek<...>/default.zeek <...>/default.zeek -0.000000 | HookLoadFile Zeek_Spicy/__load__.zeek <...>/__load__.zeek -0.000000 | HookLoadFile Zeek_Spicy/__preload__.zeek <...>/__preload__.zeek 0.000000 | HookLoadFile base/bif <...>/bif 0.000000 | HookLoadFile base/init-default.zeek <...>/init-default.zeek 0.000000 | HookLoadFile base/init-frameworks-and-bifs.zeek <...>/init-frameworks-and-bifs.zeek @@ -4370,6 +4349,8 @@ 0.000000 | HookLoadFile base<...>/ieee802_11 <...>/ieee802_11 0.000000 | HookLoadFile base<...>/ieee802_11_radio <...>/ieee802_11_radio 0.000000 | HookLoadFile base<...>/imap <...>/imap +0.000000 | HookLoadFile base<...>/init-bare <...>/init-bare.zeek +0.000000 | HookLoadFile base<...>/init-framework <...>/init-framework.zeek 0.000000 | HookLoadFile base<...>/input <...>/input 0.000000 | HookLoadFile base<...>/input.bif <...>/input.bif.zeek 0.000000 | HookLoadFile base<...>/installation <...>/installation.zeek @@ -4428,6 +4409,7 @@ 0.000000 | HookLoadFile base<...>/snmp <...>/snmp 0.000000 | HookLoadFile base<...>/socks <...>/socks 0.000000 | HookLoadFile base<...>/software <...>/software +0.000000 | HookLoadFile base<...>/spicy <...>/spicy 0.000000 | HookLoadFile base<...>/ssh <...>/ssh 0.000000 | HookLoadFile base<...>/ssl <...>/ssl 0.000000 | HookLoadFile base<...>/stats.bif <...>/stats.bif.zeek @@ -4570,9 +4552,6 @@ 0.000000 | HookLoadFileExtended ./Zeek_SSL.events.bif.zeek <...>/Zeek_SSL.events.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_SSL.functions.bif.zeek <...>/Zeek_SSL.functions.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_SSL.types.bif.zeek <...>/Zeek_SSL.types.bif.zeek -0.000000 | HookLoadFileExtended ./Zeek_Spicy.consts.bif.zeek <...>/Zeek_Spicy.consts.bif.zeek -0.000000 | HookLoadFileExtended ./Zeek_Spicy.events.bif.zeek <...>/Zeek_Spicy.events.bif.zeek -0.000000 | HookLoadFileExtended ./Zeek_Spicy.functions.bif.zeek <...>/Zeek_Spicy.functions.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_TCP.events.bif.zeek <...>/Zeek_TCP.events.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_TCP.functions.bif.zeek <...>/Zeek_TCP.functions.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_TCP.types.bif.zeek <...>/Zeek_TCP.types.bif.zeek @@ -4667,6 +4646,7 @@ 0.000000 | HookLoadFileExtended ./smb1-main <...>/smb1-main.zeek 0.000000 | HookLoadFileExtended ./smb2-main <...>/smb2-main.zeek 0.000000 | HookLoadFileExtended ./spicy-events <...>/spicy-events.zeek +0.000000 | HookLoadFileExtended ./spicy.bif.zeek <...>/spicy.bif.zeek 0.000000 | HookLoadFileExtended ./stats.bif.zeek <...>/stats.bif.zeek 0.000000 | HookLoadFileExtended ./std-dev <...>/std-dev.zeek 0.000000 | HookLoadFileExtended ./store <...>/store.zeek @@ -4702,10 +4682,6 @@ 0.000000 | HookLoadFileExtended <...>/__load__.zeek <...>/__load__.zeek 0.000000 | HookLoadFileExtended <...>/__preload__.zeek <...>/__preload__.zeek 0.000000 | HookLoadFileExtended <...>/hooks.zeek <...>/hooks.zeek -0.000000 | HookLoadFileExtended Zeek<...>/bare.zeek <...>/bare.zeek -0.000000 | HookLoadFileExtended Zeek<...>/default.zeek <...>/default.zeek -0.000000 | HookLoadFileExtended Zeek_Spicy/__load__.zeek <...>/__load__.zeek -0.000000 | HookLoadFileExtended Zeek_Spicy/__preload__.zeek <...>/__preload__.zeek 0.000000 | HookLoadFileExtended base/bif <...>/bif 0.000000 | HookLoadFileExtended base/init-default.zeek <...>/init-default.zeek 0.000000 | HookLoadFileExtended base/init-frameworks-and-bifs.zeek <...>/init-frameworks-and-bifs.zeek @@ -4763,6 +4739,8 @@ 0.000000 | HookLoadFileExtended base<...>/ieee802_11 <...>/ieee802_11 0.000000 | HookLoadFileExtended base<...>/ieee802_11_radio <...>/ieee802_11_radio 0.000000 | HookLoadFileExtended base<...>/imap <...>/imap +0.000000 | HookLoadFileExtended base<...>/init-bare <...>/init-bare.zeek +0.000000 | HookLoadFileExtended base<...>/init-framework <...>/init-framework.zeek 0.000000 | HookLoadFileExtended base<...>/input <...>/input 0.000000 | HookLoadFileExtended base<...>/input.bif <...>/input.bif.zeek 0.000000 | HookLoadFileExtended base<...>/installation <...>/installation.zeek @@ -4821,6 +4799,7 @@ 0.000000 | HookLoadFileExtended base<...>/snmp <...>/snmp 0.000000 | HookLoadFileExtended base<...>/socks <...>/socks 0.000000 | HookLoadFileExtended base<...>/software <...>/software +0.000000 | HookLoadFileExtended base<...>/spicy <...>/spicy 0.000000 | HookLoadFileExtended base<...>/ssh <...>/ssh 0.000000 | HookLoadFileExtended base<...>/ssl <...>/ssl 0.000000 | HookLoadFileExtended base<...>/stats.bif <...>/stats.bif.zeek diff --git a/testing/btest/Baseline/spicy.analyzer-tag/output b/testing/btest/Baseline/spicy.analyzer-tag/output new file mode 100644 index 0000000000..947980697b --- /dev/null +++ b/testing/btest/Baseline/spicy.analyzer-tag/output @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Have analyzer! +tag: Analyzer::ANALYZER_SPICY_SSH +name: SPICY_SSH + +Do not have analyzer! diff --git a/testing/btest/Baseline/spicy.conn-id/output b/testing/btest/Baseline/spicy.conn-id/output new file mode 100644 index 0000000000..bd3e565a86 --- /dev/null +++ b/testing/btest/Baseline/spicy.conn-id/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +(192.150.186.169, 49244/tcp, 131.159.14.23, 22/tcp) +(2001:470:1f11:81f:c999:d94:aa7c:2e3e, 49185/tcp, 2001:470:4867:99::21, 21/tcp) diff --git a/testing/btest/Baseline/spicy.context/output b/testing/btest/Baseline/spicy.context/output new file mode 100644 index 0000000000..a4cf9692d1 --- /dev/null +++ b/testing/btest/Baseline/spicy.context/output @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +(b"2.0", b"1.99") diff --git a/testing/btest/Baseline/spicy.dns/output b/testing/btest/Baseline/spicy.dns/output new file mode 100644 index 0000000000..d53b87e459 --- /dev/null +++ b/testing/btest/Baseline/spicy.dns/output @@ -0,0 +1,3 @@ +[orig_h=192.150.186.169, orig_p=55587/udp, resp_h=192.150.186.8, resp_p=53/udp], [id=29622, opcode=0, rcode=0, QR=F, AA=F, TC=F, RD=T, RA=F, Z=0, num_queries=1, num_answers=0, num_auth=0, num_addl=0], www.heise.de, 1, 1 +[orig_h=192.150.186.169, orig_p=55588/udp, resp_h=192.150.186.8, resp_p=53/udp], [id=15429, opcode=0, rcode=0, QR=F, AA=F, TC=F, RD=T, RA=F, Z=0, num_queries=1, num_answers=0, num_auth=0, num_addl=0], www.google.com, 1, 1 +[orig_h=192.150.186.169, orig_p=55589/udp, resp_h=192.150.186.8, resp_p=53/udp], [id=27360, opcode=0, rcode=0, QR=F, AA=F, TC=F, RD=T, RA=F, Z=0, num_queries=1, num_answers=0, num_auth=0, num_addl=0], www.net.in.tum.de, 1, 1 diff --git a/testing/btest/Baseline/spicy.double-event/output b/testing/btest/Baseline/spicy.double-event/output new file mode 100644 index 0000000000..5468e641f2 --- /dev/null +++ b/testing/btest/Baseline/spicy.double-event/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +1, OpenSSH_3.8.1p1 +1, OpenSSH_3.9p1 +2, OpenSSH_3.8.1p1 +2, OpenSSH_3.9p1 diff --git a/testing/btest/Baseline/spicy.double-types/output b/testing/btest/Baseline/spicy.double-types/output new file mode 100644 index 0000000000..319e263d92 --- /dev/null +++ b/testing/btest/Baseline/spicy.double-types/output @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +dtest_message, dtest::FUNCS_YES +dtest_result, dtest::RESULT_YES_AGAIN +dtest_result_tuple, dtest::FUNCS_YES, dtest::RESULT_YES_AGAIN diff --git a/testing/btest/Baseline/spicy.event-args-fail-2/output b/testing/btest/Baseline/spicy.event-args-fail-2/output new file mode 100644 index 0000000000..bc372c9ae4 --- /dev/null +++ b/testing/btest/Baseline/spicy.event-args-fail-2/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] <...>/event-args-fail.evt:2: signature for hook must be: %error or %error(err: string) +[error] : aborting after errors diff --git a/testing/btest/Baseline/spicy.event-args-fail-3/output b/testing/btest/Baseline/spicy.event-args-fail-3/output new file mode 100644 index 0000000000..bc372c9ae4 --- /dev/null +++ b/testing/btest/Baseline/spicy.event-args-fail-3/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] <...>/event-args-fail.evt:2: signature for hook must be: %error or %error(err: string) +[error] : aborting after errors diff --git a/testing/btest/Baseline/spicy.event-args-fail-4/output b/testing/btest/Baseline/spicy.event-args-fail-4/output new file mode 100644 index 0000000000..24978f354d --- /dev/null +++ b/testing/btest/Baseline/spicy.event-args-fail-4/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] <...>/event-args-fail.evt:2: expected id +[error] error loading EVT file "<...>/event-args-fail.evt" diff --git a/testing/btest/Baseline/spicy.event-args-fail-5/output b/testing/btest/Baseline/spicy.event-args-fail-5/output new file mode 100644 index 0000000000..24978f354d --- /dev/null +++ b/testing/btest/Baseline/spicy.event-args-fail-5/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] <...>/event-args-fail.evt:2: expected id +[error] error loading EVT file "<...>/event-args-fail.evt" diff --git a/testing/btest/Baseline/spicy.event-args-fail-6/output b/testing/btest/Baseline/spicy.event-args-fail-6/output new file mode 100644 index 0000000000..4aa8c30fd9 --- /dev/null +++ b/testing/btest/Baseline/spicy.event-args-fail-6/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] <...>/event-args-fail.evt:2: expected ':' +[error] error loading EVT file "<...>/event-args-fail.evt" diff --git a/testing/btest/Baseline/spicy.event-args-fail/output b/testing/btest/Baseline/spicy.event-args-fail/output new file mode 100644 index 0000000000..8a0a39ce64 --- /dev/null +++ b/testing/btest/Baseline/spicy.event-args-fail/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] <...>/event-args-fail.evt:10: mismatching type +[error] error loading EVT file "<...>/event-args-fail.evt" diff --git a/testing/btest/Baseline/spicy.event-args-mismatch/output b/testing/btest/Baseline/spicy.event-args-mismatch/output new file mode 100644 index 0000000000..3f6f6338db --- /dev/null +++ b/testing/btest/Baseline/spicy.event-args-mismatch/output @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +XXXXXXXXXX.XXXXXX analyzer error in <...>/test.evt, line 6: Event parameter mismatch, cannot convert Spicy value of type 'string' to Zeek value of type 'count' diff --git a/testing/btest/Baseline/spicy.event-args/output b/testing/btest/Baseline/spicy.event-args/output new file mode 100644 index 0000000000..32754ee44b --- /dev/null +++ b/testing/btest/Baseline/spicy.event-args/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Error message: failed to match regular expression (<...>/test.spicy:7:15) +Error message: n/a diff --git a/testing/btest/Baseline/spicy.event-cond/output b/testing/btest/Baseline/spicy.event-cond/output new file mode 100644 index 0000000000..96e69ffb61 --- /dev/null +++ b/testing/btest/Baseline/spicy.event-cond/output @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +1, OpenSSH_3.8.1p1 +1, OpenSSH_3.9p1 +3, OpenSSH_3.9p1 +4, OpenSSH_3.8.1p1 +5, OpenSSH_3.8.1p1 diff --git a/testing/btest/Baseline/spicy.event-user-type/output b/testing/btest/Baseline/spicy.event-user-type/output new file mode 100644 index 0000000000..b2d3b4bb06 --- /dev/null +++ b/testing/btest/Baseline/spicy.event-user-type/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +is_orig=1 y=[n=1, y=12345] +is_orig=0 y=[n=2, y=ABCDE] +is_orig=1 y=[n=3, y=67890] +is_orig=0 y=[n=4, y=FGHIJ] diff --git a/testing/btest/Baseline/spicy.export-enum/output b/testing/btest/Baseline/spicy.export-enum/output new file mode 100644 index 0000000000..a2ba7bf00d --- /dev/null +++ b/testing/btest/Baseline/spicy.export-enum/output @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +TupleEnum::TestEnum_A +TupleEnum::TestEnum_Undef + [Type] TupleEnum::TestEnum diff --git a/testing/btest/Baseline/spicy.export-type-fail/output b/testing/btest/Baseline/spicy.export-type-fail/output new file mode 100644 index 0000000000..0c3723bac2 --- /dev/null +++ b/testing/btest/Baseline/spicy.export-type-fail/output @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] unknown type 'Test::DOES_NOT_EXIST' exported +[error] unknown type 'NOT_SCOPED' exported +[error] <...>/foo.spicy:1:13-5:3: cannot export Spicy type 'Test::X': type is self-recursive +[error] <...>/foo.spicy:9:3-13:3: cannot export Spicy type 'Test::Z': can only convert tuple types with all-named fields to Zeek +[error] : aborting after errors diff --git a/testing/btest/Baseline/spicy.export-types/output b/testing/btest/Baseline/spicy.export-types/output new file mode 100644 index 0000000000..bf6b8458ae --- /dev/null +++ b/testing/btest/Baseline/spicy.export-types/output @@ -0,0 +1,42 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Test::e +Test::s +Test::type_enum +Test::type_enum_A +Test::type_enum_B +Test::type_enum_C +Test::type_enum_Undef +Test::type_record_u, { +[s] = [type_name=string, log=F, value=, default_val=], +[u2] = [type_name=record , log=F, value=, default_val=], +[b] = [type_name=bool, log=F, value=, default_val=] +} +Test::u +Test::u2 +XYZ::type_record_sss, { +[a] = [type_name=addr, log=F, value=, default_val=], +[v] = [type_name=vector of string, log=F, value=, default_val=], +[iv] = [type_name=interval, log=F, value=, default_val=], +[t] = [type_name=time, log=F, value=, default_val=], +[s] = [type_name=set[enum], log=F, value=, default_val=], +[p] = [type_name=port, log=F, value=, default_val=], +[b] = [type_name=string, log=F, value=, default_val=], +[i] = [type_name=int, log=F, value=, default_val=], +[o] = [type_name=string, log=F, value=, default_val=], +[m] = [type_name=table[addr] of string, log=F, value=, default_val=], +[u] = [type_name=record , log=F, value=, default_val=], +[j] = [type_name=count, log=F, value=, default_val=], +[e] = [type_name=enum, log=F, value=, default_val=], +[r] = [type_name=double, log=F, value=, default_val=] +} +type_record_u2, { +[t] = [type_name=record , log=F, value=, default_val=] +} +--- +[a=1.2.3.4, b=bytes, e=Test::type_enum_B, i=-10, iv=5.0 secs, j=10, m={ +[4.3.2.1] = addr1, +[4.3.2.2] = addr2 +}, o=string, p=42/tcp, r=3.14, s={ +Test::type_enum_A, +Test::type_enum_B +}, t=0.0, u=[s=S, b=T, u2=[t=[x=S, y=T]]], v=[1, 2, 3]] diff --git a/testing/btest/Baseline/spicy.file-analysis-data-in-concurrent/output b/testing/btest/Baseline/spicy.file-analysis-data-in-concurrent/output new file mode 100644 index 0000000000..f8c8eb4e6a --- /dev/null +++ b/testing/btest/Baseline/spicy.file-analysis-data-in-concurrent/output @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +123456 +AAABBBCCC +!@#$ diff --git a/testing/btest/Baseline/spicy.file-analysis-data-in/files.log b/testing/btest/Baseline/spicy.file-analysis-data-in/files.log new file mode 100644 index 0000000000..13ef56de71 --- /dev/null +++ b/testing/btest/Baseline/spicy.file-analysis-data-in/files.log @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +dfe2070c79e7ff36a925ffa327ffe3deecf8f9c2 foo-1.txt +dfe2070c79e7ff36a925ffa327ffe3deecf8f9c2 foo-2.txt diff --git a/testing/btest/Baseline/spicy.file-analysis-data-in/output b/testing/btest/Baseline/spicy.file-analysis-data-in/output new file mode 100644 index 0000000000..f041df16ae --- /dev/null +++ b/testing/btest/Baseline/spicy.file-analysis-data-in/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +FL4lmy3f7owPUUoQ8l +FhCRq1oIy2uDU8rog diff --git a/testing/btest/Baseline/spicy.file-analysis-data-in/x509.log b/testing/btest/Baseline/spicy.file-analysis-data-in/x509.log new file mode 100644 index 0000000000..a82109b77b --- /dev/null +++ b/testing/btest/Baseline/spicy.file-analysis-data-in/x509.log @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +01E3B49AA18D8AA981256950B8 CN=GTS CA 1O1,O=Google Trust Services,C=US +01E3B49AA18D8AA981256950B8 CN=GTS CA 1O1,O=Google Trust Services,C=US diff --git a/testing/btest/Baseline/spicy.file-analyzer-nested/files b/testing/btest/Baseline/spicy.file-analyzer-nested/files new file mode 100644 index 0000000000..d4be2a414f --- /dev/null +++ b/testing/btest/Baseline/spicy.file-analyzer-nested/files @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +SPICY_TEXT2 SPICY_TEXT3 - text/plain3 +SPICY_TEXT2 SPICY_TEXT3 - text/plain3 +SPICY_TEXT1 SPICY_TEXT2 - text/plain2 +HTTP SPICY_TEXT1 - text/plain +HTTP (empty) - text/json diff --git a/testing/btest/Baseline/spicy.file-analyzer-nested/output b/testing/btest/Baseline/spicy.file-analyzer-nested/output new file mode 100644 index 0000000000..7ef28af9e7 --- /dev/null +++ b/testing/btest/Baseline/spicy.file-analyzer-nested/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +data2, F2Qpmk14ATv4vFSEsi, from 1:hello world +data3, FyjjRu4ARLzpsPLhNh, from 2a:from 1:hello world +data3, Fz3QLf4Bn4qaQwyUdk, from 2b:from 1:hello world +data1, FcRmxz1fPbKQEgGGUi, hello world diff --git a/testing/btest/Baseline/spicy.file-analyzer-nested/output-max b/testing/btest/Baseline/spicy.file-analyzer-nested/output-max new file mode 100644 index 0000000000..c7a6a03585 --- /dev/null +++ b/testing/btest/Baseline/spicy.file-analyzer-nested/output-max @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +data3, FyjjRu4ARLzpsPLhNh, +data3, Fz3QLf4Bn4qaQwyUdk, +depth warning, FyjjRu4ARLzpsPLhNh, [chunk_event=, stream_event=, extract_filename=, extract_limit=0], 2 +depth warning, Fz3QLf4Bn4qaQwyUdk, [chunk_event=, stream_event=, extract_filename=, extract_limit=0], 2 +data2, F2Qpmk14ATv4vFSEsi, from 1:hello world +data1, FcRmxz1fPbKQEgGGUi, hello world diff --git a/testing/btest/Baseline/spicy.file-analyzer-property/output b/testing/btest/Baseline/spicy.file-analyzer-property/output new file mode 100644 index 0000000000..8a3282b885 --- /dev/null +++ b/testing/btest/Baseline/spicy.file-analyzer-property/output @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +text data, FcRmxz1fPbKQEgGGUi, hello world diff --git a/testing/btest/Baseline/spicy.file-analyzer/output b/testing/btest/Baseline/spicy.file-analyzer/output new file mode 100644 index 0000000000..fbaa94308b --- /dev/null +++ b/testing/btest/Baseline/spicy.file-analyzer/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Files::ANALYZER_SPICY_TEXT +text data, FcRmxz1fPbKQEgGGUi, hello world diff --git a/testing/btest/Baseline/spicy.file-analyzer/weird.log b/testing/btest/Baseline/spicy.file-analyzer/weird.log new file mode 100644 index 0000000000..78c4d901d5 --- /dev/null +++ b/testing/btest/Baseline/spicy.file-analyzer/weird.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path weird +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer source +#types time string addr port addr port string string bool string string +XXXXXXXXXX.XXXXXX - - - - - test_weird FcRmxz1fPbKQEgGGUi F zeek - +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/spicy.file-data-in-at-offset/output b/testing/btest/Baseline/spicy.file-data-in-at-offset/output new file mode 100644 index 0000000000..f041df16ae --- /dev/null +++ b/testing/btest/Baseline/spicy.file-data-in-at-offset/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +FL4lmy3f7owPUUoQ8l +FhCRq1oIy2uDU8rog diff --git a/testing/btest/Baseline/spicy.file-data-in-at-offset/x509.log b/testing/btest/Baseline/spicy.file-data-in-at-offset/x509.log new file mode 100644 index 0000000000..a82109b77b --- /dev/null +++ b/testing/btest/Baseline/spicy.file-data-in-at-offset/x509.log @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +01E3B49AA18D8AA981256950B8 CN=GTS CA 1O1,O=Google Trust Services,C=US +01E3B49AA18D8AA981256950B8 CN=GTS CA 1O1,O=Google Trust Services,C=US diff --git a/testing/btest/Baseline/spicy.file-replaces/output b/testing/btest/Baseline/spicy.file-replaces/output new file mode 100644 index 0000000000..d3b2cbe683 --- /dev/null +++ b/testing/btest/Baseline/spicy.file-replaces/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +FTP_DATA SPICY_PE - application/x-dosexec +FTP_DATA PE - application/x-dosexec diff --git a/testing/btest/Baseline/spicy.gap-recovery/output b/testing/btest/Baseline/spicy.gap-recovery/output new file mode 100644 index 0000000000..e0bff6eeee --- /dev/null +++ b/testing/btest/Baseline/spicy.gap-recovery/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[$responses=[[$marker=b"HTTP", $len=3070], [$marker=b"HTTP", $len=3070], [$marker=b"HTTP", $len=21999], [$marker=b"HTTP", $len=3070], [$marker=b"HTTP", $len=21999], [$marker=b"HTTP", $len=3070], [$marker=b"HTTP", $len=21999], [$marker=b"HTTP", $len=3070], [$marker=b"HTTP", $len=21999]]] +[$requests=[[$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"]]] diff --git a/testing/btest/Baseline/spicy.gap-recovery/output-before-spicy-issue-1303 b/testing/btest/Baseline/spicy.gap-recovery/output-before-spicy-issue-1303 new file mode 100644 index 0000000000..1ea6e7f5fd --- /dev/null +++ b/testing/btest/Baseline/spicy.gap-recovery/output-before-spicy-issue-1303 @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[$responses=[[$marker=b"HTTP", $len=3070], [$marker=b"HTTP", $len=3070], [$marker=b"HTTP", $len=21999], [$marker=b"HTTP", $len=3070], [$marker=b"HTTP", $len=21999], [$marker=b"HTTP", $len=3070], [$marker=b"HTTP", $len=21999]]] +[$requests=[[$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"], [$marker=b"GET"]]] diff --git a/testing/btest/Baseline/spicy.import-from/output b/testing/btest/Baseline/spicy.import-from/output new file mode 100644 index 0000000000..cbd8dfcf6c --- /dev/null +++ b/testing/btest/Baseline/spicy.import-from/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Foo::x, Foo::y +Foo::x, Foo::y diff --git a/testing/btest/Baseline/spicy.list-conversion/output b/testing/btest/Baseline/spicy.list-conversion/output new file mode 100644 index 0000000000..8cd97f2a2d --- /dev/null +++ b/testing/btest/Baseline/spicy.list-conversion/output @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[orig_h=192.150.186.169, orig_p=49244/tcp, resp_h=131.159.14.23, resp_p=22/tcp] +T +[[a=83, b=84], [a=83, b=84], [a=72, b=73], [a=45, b=46], [a=50, b=51]] +11824 +11599 diff --git a/testing/btest/Baseline/spicy.module-path/output b/testing/btest/Baseline/spicy.module-path/output new file mode 100644 index 0000000000..58fb13eb69 --- /dev/null +++ b/testing/btest/Baseline/spicy.module-path/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Got it +Got it diff --git a/testing/btest/Baseline/spicy.multiple-enum/output b/testing/btest/Baseline/spicy.multiple-enum/output new file mode 100644 index 0000000000..c8c9711862 --- /dev/null +++ b/testing/btest/Baseline/spicy.multiple-enum/output @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +one, dtest::RESULT_B diff --git a/testing/btest/Baseline/spicy.network-time/output b/testing/btest/Baseline/spicy.network-time/output new file mode 100644 index 0000000000..75525822e2 --- /dev/null +++ b/testing/btest/Baseline/spicy.network-time/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +1584014617531080960 +1584014621661943040 +1584014625494254080 +1584014635499969024 diff --git a/testing/btest/Baseline/spicy.optional/output b/testing/btest/Baseline/spicy.optional/output new file mode 100644 index 0000000000..274745b8e9 --- /dev/null +++ b/testing/btest/Baseline/spicy.optional/output @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[x=] diff --git a/testing/btest/Baseline/spicy.packet-analyzer-on-ip/output b/testing/btest/Baseline/spicy.packet-analyzer-on-ip/output new file mode 100644 index 0000000000..8e3eec68ec --- /dev/null +++ b/testing/btest/Baseline/spicy.packet-analyzer-on-ip/output @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +MACs: src=00:d0:b7:1e:be:20 dst=00:10:dc:72:4c:5f +IPs : src=207.158.192.40 dst=10.20.1.31 +raw bytes: 441 diff --git a/testing/btest/Baseline/spicy.packet-analyzer-replaces/output-on b/testing/btest/Baseline/spicy.packet-analyzer-replaces/output-on new file mode 100644 index 0000000000..7da1ca64df --- /dev/null +++ b/testing/btest/Baseline/spicy.packet-analyzer-replaces/output-on @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +My Ethernet:, \x00\x10\xdcrL_\x00\xd0\xb7\x1e\xbe \x08\x00 diff --git a/testing/btest/Baseline/spicy.packet-analyzer-violation/output b/testing/btest/Baseline/spicy.packet-analyzer-violation/output new file mode 100644 index 0000000000..86d53cbba8 --- /dev/null +++ b/testing/btest/Baseline/spicy.packet-analyzer-violation/output @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[spicy] test::Foo diff --git a/testing/btest/Baseline/spicy.packet-analyzer/output b/testing/btest/Baseline/spicy.packet-analyzer/output new file mode 100644 index 0000000000..11985837fa --- /dev/null +++ b/testing/btest/Baseline/spicy.packet-analyzer/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +MACs: src=c8:5b:76:bd:77:ab dst=ff:ff:ff:ff:ff:ff +raw data, I am encapsulating diff --git a/testing/btest/Baseline/spicy.packet-analyzer/weird.log b/testing/btest/Baseline/spicy.packet-analyzer/weird.log new file mode 100644 index 0000000000..df5e5005e0 --- /dev/null +++ b/testing/btest/Baseline/spicy.packet-analyzer/weird.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path weird +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer source +#types time string addr port addr port string string bool string string +XXXXXXXXXX.XXXXXX - - - - - test_weird - F zeek SPICY_RAWLAYER +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/spicy.parse-error/dpd.log b/testing/btest/Baseline/spicy.parse-error/dpd.log new file mode 100644 index 0000000000..8b8724f2b1 --- /dev/null +++ b/testing/btest/Baseline/spicy.parse-error/dpd.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path dpd +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto analyzer failure_reason +#types time string addr port addr port enum string string +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 192.150.186.169 49244 131.159.14.23 22 tcp SPICY_SSH failed to match regular expression (<...>/test.spicy:9:15) [SSH-2.0-OpenSSH_3.8.1p1\x0a] +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/spicy.port-fail-2/output b/testing/btest/Baseline/spicy.port-fail-2/output new file mode 100644 index 0000000000..c4d245f758 --- /dev/null +++ b/testing/btest/Baseline/spicy.port-fail-2/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] <...>/port-fail.evt:3: cannot parse port specification +[error] error loading EVT file "<...>/port-fail.evt" diff --git a/testing/btest/Baseline/spicy.port-fail-3/output b/testing/btest/Baseline/spicy.port-fail-3/output new file mode 100644 index 0000000000..e0371b533d --- /dev/null +++ b/testing/btest/Baseline/spicy.port-fail-3/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] <...>/port-fail.evt:3: start and end of port range must have same protocol +[error] error loading EVT file "<...>/port-fail.evt" diff --git a/testing/btest/Baseline/spicy.port-fail-4/output b/testing/btest/Baseline/spicy.port-fail-4/output new file mode 100644 index 0000000000..04c5deda11 --- /dev/null +++ b/testing/btest/Baseline/spicy.port-fail-4/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] <...>/port-fail.evt:3: start of port range cannot be after its end +[error] error loading EVT file "<...>/port-fail.evt" diff --git a/testing/btest/Baseline/spicy.port-fail-5/output b/testing/btest/Baseline/spicy.port-fail-5/output new file mode 100644 index 0000000000..04c5deda11 --- /dev/null +++ b/testing/btest/Baseline/spicy.port-fail-5/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] <...>/port-fail.evt:3: start of port range cannot be after its end +[error] error loading EVT file "<...>/port-fail.evt" diff --git a/testing/btest/Baseline/spicy.port-fail/output b/testing/btest/Baseline/spicy.port-fail/output new file mode 100644 index 0000000000..24eb09807d --- /dev/null +++ b/testing/btest/Baseline/spicy.port-fail/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[error] <...>/port-fail.evt:7: port outside of valid range +[error] error loading EVT file "<...>/port-fail.evt" diff --git a/testing/btest/Baseline/spicy.preprocessor-fail-2/output b/testing/btest/Baseline/spicy.preprocessor-fail-2/output new file mode 100644 index 0000000000..ebe79b98ce --- /dev/null +++ b/testing/btest/Baseline/spicy.preprocessor-fail-2/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[debug/zeek] Loading EVT file "<...>/preprocessor-fail.evt" +[debug/zeek] Loading events from "<...>/preprocessor-fail.evt" +[error] <...>/preprocessor-fail.evt:4: unterminated preprocessor directive +[error] error loading EVT file "<...>/preprocessor-fail.evt" diff --git a/testing/btest/Baseline/spicy.preprocessor-fail-3/output b/testing/btest/Baseline/spicy.preprocessor-fail-3/output new file mode 100644 index 0000000000..735d25c6a9 --- /dev/null +++ b/testing/btest/Baseline/spicy.preprocessor-fail-3/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[debug/zeek] Loading EVT file "<...>/preprocessor-fail.evt" +[debug/zeek] Loading events from "<...>/preprocessor-fail.evt" +[error] <...>/preprocessor-fail.evt:2: @else without @if +[error] error loading EVT file "<...>/preprocessor-fail.evt" diff --git a/testing/btest/Baseline/spicy.preprocessor-fail-4/output b/testing/btest/Baseline/spicy.preprocessor-fail-4/output new file mode 100644 index 0000000000..e405d7c984 --- /dev/null +++ b/testing/btest/Baseline/spicy.preprocessor-fail-4/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[debug/zeek] Loading EVT file "<...>/preprocessor-fail.evt" +[debug/zeek] Loading events from "<...>/preprocessor-fail.evt" +[error] <...>/preprocessor-fail.evt:2: @endif without @if +[error] error loading EVT file "<...>/preprocessor-fail.evt" diff --git a/testing/btest/Baseline/spicy.preprocessor-fail-5/output b/testing/btest/Baseline/spicy.preprocessor-fail-5/output new file mode 100644 index 0000000000..56ee863366 --- /dev/null +++ b/testing/btest/Baseline/spicy.preprocessor-fail-5/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[debug/zeek] Loading EVT file "<...>/preprocessor-fail.evt" +[debug/zeek] Loading events from "<...>/preprocessor-fail.evt" +[error] <...>/preprocessor-fail.evt:6: unterminated preprocessor directive +[error] error loading EVT file "<...>/preprocessor-fail.evt" diff --git a/testing/btest/Baseline/spicy.preprocessor-fail-6/output b/testing/btest/Baseline/spicy.preprocessor-fail-6/output new file mode 100644 index 0000000000..240ed695d2 --- /dev/null +++ b/testing/btest/Baseline/spicy.preprocessor-fail-6/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[debug/zeek] Loading EVT file "<...>/preprocessor-fail.evt" +[debug/zeek] Loading events from "<...>/preprocessor-fail.evt" +[error] <...>/preprocessor-fail.evt:2: cannot parse integer value +[error] error loading EVT file "<...>/preprocessor-fail.evt" diff --git a/testing/btest/Baseline/spicy.preprocessor-fail-7/output b/testing/btest/Baseline/spicy.preprocessor-fail-7/output new file mode 100644 index 0000000000..240ed695d2 --- /dev/null +++ b/testing/btest/Baseline/spicy.preprocessor-fail-7/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[debug/zeek] Loading EVT file "<...>/preprocessor-fail.evt" +[debug/zeek] Loading events from "<...>/preprocessor-fail.evt" +[error] <...>/preprocessor-fail.evt:2: cannot parse integer value +[error] error loading EVT file "<...>/preprocessor-fail.evt" diff --git a/testing/btest/Baseline/spicy.preprocessor-fail/output b/testing/btest/Baseline/spicy.preprocessor-fail/output new file mode 100644 index 0000000000..334fba930d --- /dev/null +++ b/testing/btest/Baseline/spicy.preprocessor-fail/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[debug/zeek] Loading EVT file "<...>/preprocessor-fail.evt" +[debug/zeek] Loading events from "<...>/preprocessor-fail.evt" +[error] <...>/preprocessor-fail.evt:7: unterminated preprocessor directive +[error] error loading EVT file "<...>/preprocessor-fail.evt" diff --git a/testing/btest/Baseline/spicy.preprocessor-spicy/output b/testing/btest/Baseline/spicy.preprocessor-spicy/output new file mode 100644 index 0000000000..736578c927 --- /dev/null +++ b/testing/btest/Baseline/spicy.preprocessor-spicy/output @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +== spicyz + ::hilti::rt::print(std::string("have zeek"), ::hilti::rt::Bool(true)); + ::hilti::rt::print(std::string("have spicy version"), ::hilti::rt::Bool(true)); + ::hilti::rt::print(std::string("have spicy version >= 0.4"), ::hilti::rt::Bool(true)); + ::hilti::rt::print(std::string("not have spicy version >= 4"), ::hilti::rt::Bool(true)); + ::hilti::rt::print(std::string("have zeek and zeek version > 1.0"), ::hilti::rt::Bool(true)); diff --git a/testing/btest/Baseline/spicy.preprocessor/output b/testing/btest/Baseline/spicy.preprocessor/output new file mode 100644 index 0000000000..0c77fb42d6 --- /dev/null +++ b/testing/btest/Baseline/spicy.preprocessor/output @@ -0,0 +1,9 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[debug/zeek] Loading EVT file "<...>/preprocessor.evt" +[debug/zeek] Loading events from "<...>/preprocessor.evt" +[debug/zeek] Got protocol analyzer definition for YES_1 +[debug/zeek] Got protocol analyzer definition for YES_2 +[debug/zeek] Got protocol analyzer definition for YES_3 +[debug/zeek] Got protocol analyzer definition for YES_4 +[debug/zeek] Got protocol analyzer definition for YES_5 +[debug/zeek] Got protocol analyzer definition for YES_6 diff --git a/testing/btest/Baseline/spicy.profiling/output b/testing/btest/Baseline/spicy.profiling/output new file mode 100644 index 0000000000..596d18cd05 --- /dev/null +++ b/testing/btest/Baseline/spicy.profiling/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +SSH banner, [orig_h=192.150.186.169, orig_p=49244/tcp, resp_h=131.159.14.23, resp_p=22/tcp], F, 1.99, OpenSSH_3.9p1 +SSH banner, [orig_h=192.150.186.169, orig_p=49244/tcp, resp_h=131.159.14.23, resp_p=22/tcp], T, 2.0, OpenSSH_3.8.1p1 diff --git a/testing/btest/Baseline/spicy.profiling/prof.log b/testing/btest/Baseline/spicy.profiling/prof.log new file mode 100644 index 0000000000..30c7611635 --- /dev/null +++ b/testing/btest/Baseline/spicy.profiling/prof.log @@ -0,0 +1,22 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +# +# Profiling +# +#name count +hilti/func/SSH::Banner::__on_0x25_done 2 +hilti/func/SSH::Banner::__parse_SSH_Banner_stage2 2 +hilti/func/SSH::Banner::__parse_stage1 2 +hilti/func/SSH::Banner::parse1 2 +hilti/func/SSH::__register_SSH_Banner 1 +hilti/total 1 +spicy/prepare/stream/SSH::Banner 2 +spicy/unit/SSH::Banner 2 +spicy/unit/SSH::Banner::dash 2 +spicy/unit/SSH::Banner::magic 2 +spicy/unit/SSH::Banner::software 2 +spicy/unit/SSH::Banner::version 2 +zeek/event/ssh::banner 2 +zeek/rt/current_conn 2 +zeek/rt/current_is_orig 2 +zeek/rt/event_arg_type 4 +zeek/rt/raise_event 2 diff --git a/testing/btest/Baseline/spicy.protocol-analyzer-data-in/http.log b/testing/btest/Baseline/spicy.protocol-analyzer-data-in/http.log new file mode 100644 index 0000000000..ef750de38b --- /dev/null +++ b/testing/btest/Baseline/spicy.protocol-analyzer-data-in/http.log @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path http +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer version user_agent origin request_body_len response_body_len status_code status_msg info_code info_msg tags username password proxied orig_fuids orig_filenames orig_mime_types resp_fuids resp_filenames resp_mime_types +#types time string addr port addr port count string string string string string string string count count count string count string set[enum] string string set[string] vector[string] vector[string] vector[string] vector[string] vector[string] vector[string] +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 192.150.186.169 49244 131.159.14.23 22 1 GET - /etc/passwd1 - 1.0 - - 0 0 200 OK - - (empty) - - - - - - - - - +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 192.150.186.169 49244 131.159.14.23 22 2 GET - /etc/passwd1.1 - 1.0 - - 0 0 200 OK - - (empty) - - - - - - - - - +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 192.150.186.169 49244 131.159.14.23 22 3 GET - /etc/passwd2 - 1.0 - - 0 0 200 OK - - (empty) - - - - - - - - - +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/spicy.protocol-analyzer-explicit-forwarding/output b/testing/btest/Baseline/spicy.protocol-analyzer-explicit-forwarding/output new file mode 100644 index 0000000000..50ec554426 --- /dev/null +++ b/testing/btest/Baseline/spicy.protocol-analyzer-explicit-forwarding/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +ys=only Y +ys=both Y and Z +zs=both Y and Z +ys=only Y after removal of Z diff --git a/testing/btest/Baseline/spicy.protocol-analyzer-tcp-over-udp/ssh.log b/testing/btest/Baseline/spicy.protocol-analyzer-tcp-over-udp/ssh.log new file mode 100644 index 0000000000..7edf099750 --- /dev/null +++ b/testing/btest/Baseline/spicy.protocol-analyzer-tcp-over-udp/ssh.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path ssh +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version auth_success auth_attempts direction client server cipher_alg mac_alg compression_alg kex_alg host_key_alg host_key +#types time string addr port addr port count bool count enum string string string string string string string string +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 ::1 52806 ::1 1234 2 - 0 - SSH-2.0-OpenSSH_42.2 SSH-2.0-OpenSSH_7.4 - - - - - - +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/spicy.replaces-mismatch/output b/testing/btest/Baseline/spicy.replaces-mismatch/output new file mode 100644 index 0000000000..1073ee7fc5 --- /dev/null +++ b/testing/btest/Baseline/spicy.replaces-mismatch/output @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +fatal error: cannot replace 'Ethernet' analyzer with a protocol analyzer diff --git a/testing/btest/Baseline/spicy.replaces/output b/testing/btest/Baseline/spicy.replaces/output new file mode 100644 index 0000000000..924809c5a5 --- /dev/null +++ b/testing/btest/Baseline/spicy.replaces/output @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Analyzer::ANALYZER_SSH, 3 +SSH banner, [orig_h=192.150.186.169, orig_p=49244/tcp, resp_h=131.159.14.23, resp_p=22/tcp], F, 1.99, OpenSSH_3.9p1 +SSH banner, [orig_h=192.150.186.169, orig_p=49244/tcp, resp_h=131.159.14.23, resp_p=22/tcp], T, 2.0, OpenSSH_3.8.1p1 diff --git a/testing/btest/Baseline/spicy.resource-usage/output b/testing/btest/Baseline/spicy.resource-usage/output new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/spicy.resource-usage/output @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/spicy.reuse-parser-across-hltos/output b/testing/btest/Baseline/spicy.reuse-parser-across-hltos/output new file mode 100644 index 0000000000..88a8c40230 --- /dev/null +++ b/testing/btest/Baseline/spicy.reuse-parser-across-hltos/output @@ -0,0 +1,16 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +foo, 1.99 +foo, 2.0 + +bar, 1.99 +bar, 2.0 + +bar, 1.99 +bar, 2.0 +foo, 1.99 +foo, 2.0 + +bar, 1.99 +bar, 2.0 +foo, 1.99 +foo, 2.0 diff --git a/testing/btest/Baseline/spicy.ssh-banner/analyzer.log b/testing/btest/Baseline/spicy.ssh-banner/analyzer.log new file mode 100644 index 0000000000..b60c24d0f9 --- /dev/null +++ b/testing/btest/Baseline/spicy.ssh-banner/analyzer.log @@ -0,0 +1,12 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path analyzer +#open XXXX-XX-XX-XX-XX-XX +#fields ts cause analyzer_kind analyzer_name uid fuid id.orig_h id.orig_p id.resp_h id.resp_p failure_reason failure_data +#types time string string string string string addr port addr port string string +XXXXXXXXXX.XXXXXX violation protocol SPICY_SSH CHhAvVGS1DHFjwGM9 - 141.142.228.5 53595 54.243.55.129 80 protocol rejected - +XXXXXXXXXX.XXXXXX violation protocol SPICY_SSH CHhAvVGS1DHFjwGM9 - 141.142.228.5 53595 54.243.55.129 80 failed to match regular expression (<...>/ssh.spicy:7:15) POST /post HTTP/1.1\x0d\x0aUser-Agent: curl/7. +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/spicy.ssh-banner/output b/testing/btest/Baseline/spicy.ssh-banner/output new file mode 100644 index 0000000000..680dd73714 --- /dev/null +++ b/testing/btest/Baseline/spicy.ssh-banner/output @@ -0,0 +1,10 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +=== confirmation +SSH banner in Foo, [orig_h=192.150.186.169, orig_p=49244/tcp, resp_h=131.159.14.23, resp_p=22/tcp], F, 1.99, OpenSSH_3.9p1 +SSH banner in Foo, [orig_h=192.150.186.169, orig_p=49244/tcp, resp_h=131.159.14.23, resp_p=22/tcp], T, 2.0, OpenSSH_3.8.1p1 +SSH banner, [orig_h=192.150.186.169, orig_p=49244/tcp, resp_h=131.159.14.23, resp_p=22/tcp], F, 1.99, OpenSSH_3.9p1 +SSH banner, [orig_h=192.150.186.169, orig_p=49244/tcp, resp_h=131.159.14.23, resp_p=22/tcp], T, 2.0, OpenSSH_3.8.1p1 +confirm, Analyzer::ANALYZER_SPICY_SSH +=== violation +violation, Analyzer::ANALYZER_SPICY_SSH, failed to match regular expression (<...>/ssh.spicy:7:15) +violation, Analyzer::ANALYZER_SPICY_SSH, protocol rejected diff --git a/testing/btest/Baseline/spicy.ssh-banner/weird.log b/testing/btest/Baseline/spicy.ssh-banner/weird.log new file mode 100644 index 0000000000..7dcdd71aef --- /dev/null +++ b/testing/btest/Baseline/spicy.ssh-banner/weird.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path weird +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer source +#types time string addr port addr port string string bool string string +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 192.150.186.169 49244 131.159.14.23 22 my_weird OpenSSH_3.9p1 F zeek SPICY_SSH +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/spicy.terminate-session/conn.log b/testing/btest/Baseline/spicy.terminate-session/conn.log new file mode 100644 index 0000000000..40b3b79600 --- /dev/null +++ b/testing/btest/Baseline/spicy.terminate-session/conn.log @@ -0,0 +1,12 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open XXXX-XX-XX-XX-XX-XX +#fields uid +#types string +CHhAvVGS1DHFjwGM9 +ClEkJM2Vm5giqnMf4h +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/spicy.toggle-protocol-analyzer/output b/testing/btest/Baseline/spicy.toggle-protocol-analyzer/output new file mode 100644 index 0000000000..0d612ac613 --- /dev/null +++ b/testing/btest/Baseline/spicy.toggle-protocol-analyzer/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +=== +Spicy: SSH banner, [orig_h=192.150.186.169, orig_p=49244/tcp, resp_h=131.159.14.23, resp_p=22/tcp], T, 2.0, OpenSSH_3.8.1p1 +=== +=== diff --git a/testing/btest/Baseline/spicy.tuple-arg/output b/testing/btest/Baseline/spicy.tuple-arg/output new file mode 100644 index 0000000000..3d15a46ac5 --- /dev/null +++ b/testing/btest/Baseline/spicy.tuple-arg/output @@ -0,0 +1,9 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[i=1, s=OpenSSH_3.8.1p1] +[i=2, s=OpenSSH_3.8.1p1] +[i=3, s=] +[i=4, s=] +[zeek] [SPICY_SSH/3/orig] -> event ssh::banner((1, b"OpenSSH_3.8.1p1")) +[zeek] [SPICY_SSH/3/orig] -> event ssh::banner((2, b"OpenSSH_3.8.1p1")) +[zeek] [SPICY_SSH/3/orig] -> event ssh::banner((3, )) +[zeek] [SPICY_SSH/3/orig] -> event ssh::banner((4, Null)) diff --git a/testing/btest/Baseline/spicy.tuple-enum/output b/testing/btest/Baseline/spicy.tuple-enum/output new file mode 100644 index 0000000000..dcc1da9f4f --- /dev/null +++ b/testing/btest/Baseline/spicy.tuple-enum/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[$a=TestEnum::A, $b=83] +[$a=TestEnum::A, $b=83] +[i=TupleEnum::TestEnum_A, j=83] +[i=TupleEnum::TestEnum_A, j=83] diff --git a/testing/btest/Baseline/spicy.tuple-optional/output b/testing/btest/Baseline/spicy.tuple-optional/output new file mode 100644 index 0000000000..568248dce7 --- /dev/null +++ b/testing/btest/Baseline/spicy.tuple-optional/output @@ -0,0 +1,2 @@ +[i=83, j=83, k=, s=H-, ss=] +[i=83, j=83, k=, s=H-, ss=] diff --git a/testing/btest/Baseline/spicy.type-converter/output b/testing/btest/Baseline/spicy.type-converter/output new file mode 100644 index 0000000000..aa5ca62e91 --- /dev/null +++ b/testing/btest/Baseline/spicy.type-converter/output @@ -0,0 +1,28 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +[$a=b"SSH-2", $b=11824, $c=11599, $d=3.14, $e=1.2.3.4, $f=2001:db8::1428:57ab, $g=True, $h="MyString", $i=2011-01-19T05:31:50.500000000Z, $j=4.000000s, $r=[$i=11, $s=(not set)], $s={1, 2, 3}, $t=(47, "foo"), $v=[b"A", b"B", b"C"], $l=[b"A", b"B", b"C"], $m={1: "A", 2: "B", 3: "C"}] +[orig_h=192.150.186.169, orig_p=49244/tcp, resp_h=131.159.14.23, resp_p=22/tcp] +T +SSH-2 +11824 +11599 +3.14 +1.2.3.4 +2001:db8::1428:57ab +T +MyString +XXXXXXXXXX.XXXXXX +4.000000, interval +[i=11, s=] +{ +2, +1, +3 +} +[i=47, s=foo] +[A, B, C] +[A, B, C] +{ +[2] = B, +[1] = A, +[3] = C +} diff --git a/testing/btest/Baseline/spicy.udp/output b/testing/btest/Baseline/spicy.udp/output new file mode 100644 index 0000000000..59667ecab1 --- /dev/null +++ b/testing/btest/Baseline/spicy.udp/output @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Analyzer::ANALYZER_SPICY_UDP_TEST +UDP packet, [orig_h=127.0.0.1, orig_p=12345/udp, resp_h=127.0.0.1, resp_p=31337/udp], T, 12345\x0a +UDP packet, [orig_h=127.0.0.1, orig_p=12345/udp, resp_h=127.0.0.1, resp_p=31337/udp], F, ABCDE\x0a +UDP packet, [orig_h=127.0.0.1, orig_p=12345/udp, resp_h=127.0.0.1, resp_p=31337/udp], T, 67890\x0a +UDP packet, [orig_h=127.0.0.1, orig_p=12345/udp, resp_h=127.0.0.1, resp_p=31337/udp], F, FGHIJ\x0a diff --git a/testing/btest/Traces/dns/long-connection.pcap b/testing/btest/Traces/dns/long-connection.pcap new file mode 100644 index 0000000000000000000000000000000000000000..c2ff4783575bd3bbd6ac884088a2247f195c5ab7 GIT binary patch literal 2428 zcmb7`Ye-XJ7{{Np9d~rDQmD*J*@qS?E%G)zp)!{hna~9iD=3y*YqX2G$%+VP5fo@e z6iPv&-J|G67f|w89}1e}hgE*)LlG5yQo30X+Vh-Ews&W&I`E9W?>Xo9{QvLsocEm2 zlRLdC;9>fhngZZ(aNk~Mo_$GxRrsF*q@I$R4tP87w(dqMtOTe4Fo7x4wpzM!Efr&S zAQ#GE(Yad_9=x9rcxdb%kKJigH+ftDvw(Ym)Xj5Kpa{=Xgn3~$QMtTeKR_U}>oO6E z4CR4bF7?&d2~T?fsTzXDQzad-0aOK`g@P2X%hy~c>6lrqJX@Y{)|ha%IN@wb+-x{o zDb5z50@Bb3{OFiIM>0jnrq7qNrH1I(yfZCZaz@9aV`8>+Fm|lZmbzodV$LA0-0`RU zf7hda&zfG8MQcq<7h8|{z!A0Qqb5p~>l>{HR0M~q9^Lz5>j8_NkFV*K>yeQ96se96 zJZ!?V6pG$i9qFA4sxXn|-YLy`e6~NTRb9NSj1$$xVu{LV6dZ1Y#cFNgEiKI!03C3Y zr0Ek6?Z~HyZzb>!=gO@#&Pd;W@ZQ;eSvqC{_)8M(Axs6_20bj)8I1-MV-O7}Ol`Nh z(|mrP&1u8lR=aIJn7OOY+~(Ip9bzdIt}7$1S1_&%ZdzI;)FR74L4LnqgSZ`DtJ>3I zr+JRZ$BgN)1)&rQtMxB_k=4^8W3}RZR;v_aH31p@QP*;&G2nyUk97H%=|Gy0#Dk}v zp8D@USPx|-Gi1O8%-0+6wg z<9`AVg6eJi)a1r9ySHN2TfdX}NnDeQ!);=6KSVB_6q5J}ts_4tUbB8izO;$|=%-fc z$BJ1`E>BNVpTvGT9i4aM{KW1AxPC-lvOgWRVQDB7^;sXnEgby}F@7p%40Ya(^AktL z`lMieK(!B2sk=F+l;Or~ml{%_9urY0o|Wo|j~b@WimJ44`dA;FliAbI39$#eV%YI} zTJ=j&b|NrNiUy9p>64w19{D*`vb&I7xj+qT@d^rs?y87xGo!okh-Dz3)h*ceT3pT3 zuSS$j=&#o`QFOfQs#iTE`&b_%yHI#Hkk86ip!*P=aYzT)ka8$g*F;oTGpY@>1&dvY zRSRy9&+c&V0($h$BYq?hMpFt!>(snnDl4DGC^rmVSnN7{h+2aVYK>{-41NglOXg?+ zUm9{K)MGVG^d+*2QLi_L(iz*@Z382 lUS4k%-9)MxY-Bg0?||@bgpE%Z;xQw#+sfYv!S08Re*o11XY2p~ literal 0 HcmV?d00001 diff --git a/testing/btest/Traces/dns/proto255.pcap b/testing/btest/Traces/dns/proto255.pcap new file mode 100644 index 0000000000000000000000000000000000000000..a54d7240f4c95220428623dd2ee5431db27468e0 GIT binary patch literal 515 zcmca|c+)~A1{MYw`2U}Qff300$J}CByqksLHjo3t3<7tGeBv1{Y?s@oz`@|kzRTM zY6^2+Y6-{+#sfScl7WG_g@NT5{td>nfyn%`Qx{)G=C_$Px1;e7AoCkCgr*?# z&&=4>jm)?AReXfqVN6gbu`%Zr8y;A5U*!hwP1Vv+*iftv`v1LyK(2VNui;DGvz;DZ9{AU~MT^kxlEzr<9{ XgPLGIjV#TSgQ(G0a+S)dVQ(KFu)co%{CkaWw+}`i`?(=`&{~O)poIR|) z*1OkUd+p)moUZw3laLWJmXFEAFaq5CbS*S7f0!o|gKG**K=-ocOAlRY@Oj~T38Q3~ zi0=LoOoYp8o>f}}JrLw~K4pAYW)4oJ%-e(}lkNh;2*vHT3q+!J0)fPR{=xxkC`~Nn zsh5HJ6x>tb>GE}O)JLNNQ2%5QQKlLg-#edP58f|A4GT7Q9I-~Xm!~ckO5EKe|2dGT zqb674d*74onem`Xp%vd@ig`qN)forA7l69BL?-i{vK0vM{2nFBM6u={?U^JHqtJ@i zsGMO$G5V?luVUi0#E&Tu`0*4$Eew1f4GB@m`xg~Xnh~2W4-HZTg$!u#PPHug$Qj?B zv#FM?b#3tN$MKzR<@GJiTC;|F94N@PrdZl@hGzZ1`epa@a8(28PG&Ipd; zxi%>R>yGc+Qw$9amd9qRQw!Dd!hCgFR>o4fCQ~iXPc1B5oL7(@#7i;Y(-tXO;S|=# zKy|Kt#{s7TXv`=e?=vODiqmZi@{2!f&rF9D6k1V?+?q!eJ$JQ9(NE@+QeaOpHZmbW zK6vp$wI+gR*lmQDGuTIRAl-KWX%t{z;Q;#xDCF&*0lT71$jk<>6k5SPHD?q@@y9kP z0`?_7R@kQ##gI^id|KYZtXz4aHZ4tESeT*B2G&&3uERnr>#dLpYgkMb9o^%A%nD!~ z2durLezzbq@DE@;ohUY)7BW#_n?fs!qf)05#YAG=)=u#h{bb(j2HR6in4K_#=d|57 zEu6N3BrGSKgXFFQByj+_+yUgc`3ndBL7esm#Y8Ykp%ul+Iiop>Keb5_Apa`2g3M72 z2_7m>jESBlpRqJwJxV@p1kd(>b1iJQf+MWweX3;LNC!B|0dP71?ilrj1&%;B031hb ze|=fVOa{voT2UOi^%j8aEu! zNC&=^WK}nQW{XCkN666O@?JxF@uZV~c80;FWkmYhPaHU|1O_=A2HUM1ds;Bq_lwXv zT(!brbe904xIxk;MPN|$kv+v3vtyO=Xr9*-piHACg;p>m+&&o|LSSruSOIMqF;8P9Gz&Uw?D!ig}5aoG*^DMyW z5M{86(%u5j{cFy8;G5n=(WhIR6#Zl!UWTW@kGO#xMOeEe9-7J5g^U)kP-q3s!klp& zH2WQ(c^1&TLeRYXxE(a@LqdkjmAM*q0U;BM5g@BbEmr{-dH>*%A%lX4jTjU>D3mS~ zgMvo}4=}@Z;+8YGHcTQW{76l0&C4r*Yaif}Mxn2ph4Zq&wf;_v;HNZ2W6h#Ny{K z43q{BRmh{%1!{R(UXHqu8#&Xn3e;(uJZMyQR-pz+no;s=c7{^AOC(OGhH(MPiKa^)F$1-7if(1Z*d85|;CuvDWi zv;g{XCue|8C?nPiw>gN9*-*_=0O~$@u~mFu3!t4JciMF(a>CLK$X3{;7Ga;cV->~v@=VQer5t#J(t6gW< z&_N-=!!0d6Gs_uTDW4M$GZ#C^wht{`1GHQxce2XX7ic9CS{Ps_QJv;Re9yw;hnH0dgk@xjxJ7 zkn8vFbR8R3%j*+fM>f&d#uv;x8V8Qey2TW zGN&cq*vopr!y@Z5!>zK;gqHMo)^m~%I^%tij%up>unpdOgLmp68&qImwXHYxha#Op5GQ*7xuoenweIZ+%YyB#p;4NShH_OCM!X2c~h8TD6tPS~(P z!NCek&oLc!hSt|P#KSKK9c23sEu9FoM3XS9H4hQt7PJO`DRQ#9^-O=Fs9>xV_Xg$u zt<#hx=mOm8)?-q~)vdsAN~6^+YKw4CpMrY|t@=!>)vci1J9{NR?HP?4mbS6FMYx62 z>K0KhIPMIIWn6C@)#8rwd@lv{F+jrit7a<_qb*1@pAa!~!6$`Q!>?*hH$tLc3ljV~ zACIk{OjOs!o+4*Ml2nUR3)HhPU}WblEDTaFQS+uG?xcu;=5nDGEV`U%PLs8S+S z=@zNd&0Bm`W>p&f@0gO-9Lf8Kg$^C&U{O-R;MH~v7aSOrv^(#;h#1_^)xw~;-NsP6 z(VTziJ$nXKbJj2-h$}{*mDOGNrcVU-*ZCjF%6RqI?Zs?&`?BXQ!L%=X&dAIvl;;=Z zEi6dQk*B5R=H_YS3)J!gZLS;(Efc3Fm?wGd-2=>|XTE&-3-89~Ua|z{!Ot&s#C@lG zCts3r-#5+oB`lF8%mR9{4u>dBU9U8}2(kD=m{~I!X|$WSD6J zF>}~`aFiwO5+fCO2wdHnSi~sIu`+y9c%1Gd_-ot~PV}*SGiz{z_+y6|4oB{W%26cu zZT+H1W^)F^pmQ-y3953T3-f0U?z>uU80Prn4CCK&+v@6!+Ps=%NkRT&mg_dYgEZr}#9vc4n-Y&MA0hDn7Xmb{W;nO}PehRA?_zKHtADas{!>`! zClA9iH2wdD-y>-sA@B%+M+p4yLx3#*B3S<4;Br!M^!deCytB#jzag`?{Iv$o@?%o+ z)yt18|Np*rf5h=41Rfzk2#^JQ(z)-;t*&k8Y;OUNSyw&o>+LPzqW+n;9*2Jq=f=)N9iY#eq%fY*v{%P#sir(?exW08v5xzEdyU( zMZbksjr_C;mknOJr5dlIAL!nQd+E9=z4D5_xOtJlBHXAYY&YqzM^$aGWp7o z&E)HezYT*~_KAyDyuQ| zkx+aeKZ+m2A*FRO^@+f3Cc&4Ooip94?wpw&RkO%5 zlSu{l*f?Nnl~-QLh+s;Kxv#=rI(0L{DzC6id6mY)biBbev?_u)BOY|hE8>Q4jO$U1 zXvl~LZTEr3MxF+J{UsW+h=!?;oYpp!Njr?17!N_ z0GRTbqvrt;Z7(bx$zEnO|w?87&A4tXX#Wzu;t||iBHIJj_pIk#^IDWQ) z=cJjTRhgSnAfvWC7NU)1KBZMI!{_n1N%;LelW#W4vA1*wB;}3T9wy&Rx@BS$GM7<% zXpjr7z6kw?^lgzKAu|*4nKLNy7ic9rCLQrfi$)q$2Nnt9Oy5Dl%d3i_7cq;TG^~q5 z@px$efmq`l5$TUlS(Qd^Fc2qk#&=MjN+zOYJeBk*uUdStp?&6PqJH-QRV~gOL^sPQ ztI79q)CEpc=CeQXa#|qcfGd4+vp%`WEMgN1hTqp0-_{1RV@OKV zckCI2CRPOWo<#&Jjg3pnd=Vv$%ecyYEOO5@thgRbb{ z%WwnLY>LfX2FjrIroz2p60XwAaSs2I-#qg5U&kl zbXw$R8s@Xb&E+P|Of%lp5Ww4zcq^X0#kCw-i0ga@B}qn=kkru+>ez)<-I|I1N@=8r zzWE$4vIeiD%q;}Xek7n?w)pl;a~qQyp#By$D3617U{o`61UuoVpwuH-r9{^xEcVYl z4c3fLfNW+JU8V9B=nk2ZYqZ|1(vyu6#~JH54#lBW$u&u=){{*V&t%7}0dI>$nN29h z^%&@&R+hldml)@P$lR3Bs`<$^>?ASr+(w>Q>Q$7-i=8Af4glbM4xWSh`xXwq#}!0O z0ShS4bt=vR$*5*Sl5p31Teoh#^8>5&IdQf62VwYey-HTlkyZM^th>}Q^a-Jwbq|CI zGmD-wB>ibEAz|;GxH*|MNq5SsuL>)lXVM~=iVo79vjm|hD`nLUV%-gqu1ch<7VCZx zRsSINIAzWg_a`!$kSX>-b%TrU2PrCUNqj&FCfyI?rdzU`wCfv0fSCO@ru?KPvAj{! zt}<58APhMv-8rKArm*r2uIet*os4wnN!=w;^(8TTullCgqqe-UvV=>F6lmI`Ch1=B zI=XLYw`&YUtB4dHj1-Lli=4zM2L;SrO}jvqOfT6cu*FO11M?GV(vR5q3dYXRy0R*5 zT4h)_kRHHD_a3e&r5u$N0FTK2}iNKs<@M`=}*lyUDE?i zu&&9Z{UK$3NY&irn(#zf;qB@lU39fB9%pnW$+E+`n-cbGw&s+PRe6rDnCNb5*ojt7 znXi{^6QF>_zH%c9X{JgNyU8D1#O)SK7$)#|yrIMou1lL@+Aeqje}vhW|=Cy+MIG{gu9k4l$G zfcI|Tuc37fB11GIYD`Kn`ECTnELnOOf&r{j2B~1vXR_ZL{XvT3%%nLDDasFPgn9v6 zi>{hngM#b%wn>>EN;siPUjNxdPlM72_E!=YA8R}#<7fciZU&WvjceA`i{lqXWu~J+ zGAH7XaZ-~BotQXAdEGS$utaArKMd9tGi#OPxiWff^NJ2!MPwvcZ8OI|IAuGa=dvo9 zVYD!`F8pqh8(aMuRD%iJ&lXGAVWNBNA@)Rdz3}(9*dw~jqTe@jWkkoz>+~J8y)j0B z7OV1M0@=d}RMaEQ;6w4jzEp~4C^Ek`zYj63# z<5xYf)4BP5d&?hhoWixMtacJq1%BNmI74v%VbHiH1qX4us}gEdY1zj}(Ong40(4hh zH1I&lC1Q`GFIM`5L{}TdtmvlcGu^4EeACrkLG$q-u(p$QBe|VF&d&#{O8b93w zx8>fIi9XR49V#aXLQNPa#1r(%en1#*(vFn5=87wim;B&L?PF-eX?hT7xECU?7apddtVQ8n<$}d zGvi~79zo`8&1|x)!V9W6q5QBuS*95V*(yCs?yk@bf*BjBl4%3VP0QpfTy(_}w<5nv zm0wJSSLIYe*@5t$Dg=p)+W-ATIG(FKJMF+#H7yH2rrBa5(rdZx@;WBH_%{5FC zz;}3s>V%;Z!%#g2dz2Ih_kkE8Hej(k;?`o5%nThAyE0_3EUpFhA2P{k0vUuM&v%p| zY+OY6ExoF_SO|$s676(@cn8EFcZI9+3{_c2b3CqxshStb<6!>Y=W7msP~6_&^BMb9 z;Mbv5?7iRL<4nFx@0X$v=9S6d!;y=}^o6svpeF&ngKbP=c)ZK2AYEmmC)lg#V3>_@ z;m(Z49DBbS8zZnIBlq89C^pLPkv`(LV4H4t@iMLpjA?lLu?i}eew@`_6r~cW2aD@X5{qaRvbWH=1OImgiH5;Oi z>+ZTO{tEH99FOaw+YO(E9yWOD4b!5IQ}hxzV>kO#3GLYr{)?_D~b@v*#9j z8$9<-Ff60&BA#H;^^))aR(V?+65h0|gCX&DyaDqH?wnYebIXsR8#|po_Ll#`P|Hr| zMCgR=PN(r>c$!e;@Z#p>BCN@wVeD~4StT6GNvOOXi|!gX)7aaApTwpK zwc_1GSf6~0J%xJe-+{FSOC1d6LP^FOoLk<)Pa@CHM^p@~PO8_SNMlWeri1RseWgbk zIZ?Y-KcCF@(TMs4d?OYXwLS`$mBP`(v>i&G5`<|zbo+(6qw@M-M2`;DdLc4ID?wy^ zFTM=M$$B4LHrB{AUb-J|A~R7IBT9EvR(}`LfHKsm{sswlRECUbSB-+h)GEMW2;hLJ zvZ+?QtuqJcb12H?SKW?-m5NU?{)s`Lt^9lgq?oI!wW-QlbpJz&O1Y{6P_(gDZ065e z$ncOtwAeDK{wBC@td%$xn)Q%E9$cZ_RH#PoSg7|y3IRgYRL&KWITpHcuWdLm>qeni z6tahJuWf$m>MFDrtt@Gq0XYw$KUWc)LQ7Lwp;&2KmREnSsc^g(bzK4b_I1TSB)>10 ze>?Iww%JZyA5sp^G3r^ym1_n0Pj}k}0yBLSn#>hy)wtUpQpk%d)R79cZl|XoQiukE zN%a@NJaq|P&8^r?dPpHxuFw{)P=sTl4i70rhJR9hAr-1^rAFuB-M7_2a1b1Y#-fmY zE4}}a{Nx8us`o;E>bq?=)RPY>N)Fnj`cvSZI#if_eWPs)NmqXWb+c+9->F{X3QEic zX|<=)x#fS7v;5xnmj8zfR?C0Z-tt=;MP$#)R$Es1u9`}$v{r#5ciP4n!zlzG#1(3# z0_z@92^c7o=g*$|~R75)`y=?jrj#GTSnI zlb?nhaOTR7kvz&~U6W*4SN4#>=SvzPuqMd)&RS16&w3Y3I+=;wVWZcwmeFGNZZ zeZl`b#5Ks-u^H?vF|CDFi8GM5!3(!kkRqsTar5}_@0Py-iOZ`nG1PQ0sE&nKYuw@9 zn`aB$nu2MjJc6l4cnV=7f=>W;0D46*ClEXlx*_!H8^OHRFM?^;KZ21UWFhV}!jC<% zriAYZgk>n#2l=v)?{$Q^h#QJ9Z$JdY`1#>K1p(v72>lq5<%3xIi2d5Ri1F9e&rjm# z=Y|+}KMy}YPcD&5Z|@}*i=|#NzYcyKy?rEpzMYWh>DPJ9ub}TN>_R$)w(>kI?Q7(x z@V&VlO$IOrD82S98ZaR#v~i|hTc+2Zsa3wXCM|+FsC*VnHQV*t3cYrX$#?tXXhrya zGJW|nxS|FFd4C2>2vk=v(T$DMCmz5;G^v7d@@=H7Qy>>6NBTsgUb{!Hy=C~EV)R-p z10Rs?^((Jq55aT1-$&yn-8)MVfl1VFdgXz@YE2rOxJPun6kW-*LZ5hA`plT_@GPZR zrWPI2PZsNHC6jayBzhG#1FtO$Wvj8gD-Lah@~E&@McY^s_;Y8lT$Oi&`q*i#dWANE z0c1Dxkhq5||7z!!zksv+9qlc@vD{+$m;G*U`OE91WwqE^GAR#)R?*6nvP7?ZzgBr3 zVwftFcPf-0vWZx7@nKaEze}WEb%2{S8K&~HqSs)cFe_3_;bui!qJcJ_QFnhJ@=D8M zfURLFO^iSkFf)q}aH~41^W3y(D%PaWx6tNu7F}+50~1-*0fV^2FmVE=LR`h>#9wsx z1Eghdfi5@&XZ87DwNiDb?69;v36}?zcjWadTO$wiv5NyBZ|)ht?s@%&^IquwK-X6&Z`pBN8((Z2+5mrMyn#$$B2;vmIy{hznMN zPCh}e^wB3DXuz(-6|8Ewu%0KzH@LBq@nisyl-P_V71jGv!w#t5owK+Eb7F4x0(JxI zH}JNN?N0jIo)MC4*cM`A#g)Pe<+Heog6eFq&!>ECNfStdxZ{sy?V0ABQ;)YL&pER^ z3SOR$kcVr7OVqgB1cnpI5(n-P{U_!EylhgE4T+Gc{uheng-mQFIaTNDPa&>U)l`C} zuCJx^%*0DMJe#i zGl=m@)pVy?j?O79`y5#szmTEDG(OP5Ojdjf>m2OFQ(`PqG&ela_=Q;07YSZS7*8u8 ziOr!^4KsM{L%l6=5shC+G<%R$%4JP#ew7IAZ|L8`Bnq1&y%7mh*v!VlF!+FuKJi$2 z6_!_|&k;w5emZ&L9)AK%&9BX$DZg*#pd1%d~T-^`WTLi>P+# zzAEBguRPSSprv-inj=+>U&u9mEWM5dOKg_zJ>C$~lGz+1x6PzHgxCfdmx);C6f^M< zt2~4q05`CwJk-LQ1iZ1T^W}#tlD&(`MA7u4p@T^YqlDOES45G2!m8GwJv*?~vBqk& zGa1ChX80A*7zt@i9#R7?P;2))sPtwUsw1mI949UD9<9cO)mHc7?p64*~Bfn1OBD=r2VKz zT+%DoXx(d-TOiE}B(dcrC{|#_)C~N9shz?qFO`gl(e~EePpws6I+hl}5cZc2;ez@G zx(T^_^<#k|pXGohOBF6G$vu!vD^SWyurRnrM`u-=0}tu$FVq_XP_4#IcYm_9{5{ap z-NU$cUb^~uT!%xWEAfry6j1wVg=}@;ep2=G#qDTdJddrNQ~JcJ6<~_%u;DdY7s{=1 zXH`{(i2?z-KC9XuepFia0xAwaC0)(pyHYTEg+6(sKKT`W;(BRW7NX1+TDrFiwXUY| zG4Ob0q@WCol3}=+5LvIl7tz+bv(>1Y_D9cGs=P74m1!XZ7UK=$P1yVP!%e*Lg$A;8 z=k?`-a4V`Nb99%uS;6z?$||cOIOa;iRaGlsRjMkLcgm{ts$Ci}O#ZG)<%eb157t!+ zal4`skgzfMk(<#XZq2UZhSdms*O zbTFdHp3>EKU|J1PH>>NZn$vgAR$ms2AkQ9Fc?%=1u1Z$xD#ylS?PM{W2*%F{hvz$|AQk~X*TK^(yF)23^(9&{R4#YUg!_BHT zLUz$nQFVPYXLZ-t-@!LGD#$f*Za9;cv8WqE8mrtx_T^Q*Kp!Wr{}>YrvZ?;&Os`WW z^ls#Uo2Qr6KhI^5XoheZU>L+FSQSS>FR9nkbOd)+^NV7Lp3jw`eHG40@NHG=7%FG+ z8Ng5rKfS3ahBBM#T6l3mqFr4)i#IWSB8wZ!SLY2DGt4|@fScn;=I(8|q=ZkX+!q9o zw5%sun6rDl?3IaoDrEI9A%k6geDn%Lw>^4&2lULNSARI1<=AoLt)TtLf>`FCM%cSKc;EIqA=ZB%rRGah@^DxF&b^}9u3%d@9Cy+>E zoo#uvy>usBr$c1a zhA5_1VJzW)3dj2Fajl_5COuhmqcz3-py=dzTaMxehqL#;f5VMVID7v>gmrY9MSSw8 zePD;06%syiEfRUaeXKfih)*iAl;g+$#6$32_>_56$n2v#_-4>u>ZdJwMpg;4%FSsw z8F`rVO1q)7#ZT1CU{w|98y(>j@Y8s!Q}B=qU?LNnHDh6`<6@0zoHfIyvtC&tEz_Yc zjFb%noE1d|tz_3Ou!AekuUIySO}xaOO;|*W9oslx6vu`%?t#in;We=N%l-AqyhC~$ z^HM{sU`?fjj-cEP#3=(lp^nym+Vna9jb6*SdH9hZXT6roa{)7(mTq7VHSNWs!;xzu8$KD^g*S>{j;|GL+5d4udKYw5LIVh1AhuwKi1zBdF8;__XZPdtId z*80nl-f1aktG`^zdZd_jvH`9Lx8#B5E4@tf1dOu$kqpGZ-fD*O+C-M0opu5f(PKDj zJieZsBs7AfV-LGl?@ByTVX^J7L9++5@vZ*>ox-LCTM`Z zkm3Nl@(@n1BUV(A{9);36Q+>xHpy3ow?^H%WSLv(NhV=!_G&cE| zNAhuPM>^z(YNTb$P>=4OM7tPHB&&QDGlHgAwz>~31;iT4sT`Jwjw2jHcu{v*VBGXQ z`9ppUOKPJnd;#sy-|`teI*^ z+@`A%>N|rPK(noSn)Dy^`Qe0wMJV2(^I}t11apRh1u>uHH#<+?6fQGQ{1izA7Hi zp1V_5{i9ffB{fg1n(9@1{u2MTtrJMhC*pMA9y|W0fSmRt%$<@wNq;yY9Q7xtbz5IJ z*QNv<9ZY>14+^xSFoi;Q!Ca=Zqee=f^@ImY=S%KlA;C^F>(3#EpT(7*gXhSqwg8al za-o3{JbP`bBjN0z;-hR>1WPtr&N&6zc8&W5nkd#l3*P^Rmj8IRP088hEqCfj1-F}3 zFuN@kbTg|UYx+y==dk3RiK_oi{e!3E34G%WS|g^1#)BO5IBzH||B#TU^?LK#JwU;# zgXjn?PQFLuTQW{y!rrY9JR5#)xsW|qUblR_s75b+h>kn1&wqBNO`Jjx1EdYr(Y>L{ zAWewjd=1xW7qx|$s~NHX3SJ|?!oP`E+x_+CcWuf-%i4+SF@%WD(NVxlI3Sn!O8H4l zsHM-9142?@_X4^ep4Td<v7xbMsqPLjFyQUGeR| zs{ztyYv7=hY1)QwN5Se>Q{v&}E-?EWv45GkomIUuC0d`jU9Wn@pnCRUc7#_er6-M+ zTnlhZHCMlBTdm;QaQI92y4m1Qov$_iZBt(J5Xy<$Tl$}LZ&$wmee$2PSQ7B@-_is3 zoo*Yd#O>0(?gG`N`qRy*Df#@Tr1jxvS18L)Xd|J*5p4}VwDQOgX$}TzO zj@XYS7Cp3hv9$MYooKd4XKBP~1f z1A*bt#a`o!)Ne&%t+I@|YeNq>A?PoDt?1|QnOWr;B)i2L|3r4|Cb7y2+}j=tN%X|9 zt2|RA3*T>uGHHZ3Jgt=+6bGa2%gPUH@wkXa%-(7c>Au2T=?V(_7}ja9$c=a#GS;*8 zYh;5Xm;i)R2!j!hAXFo~HYkGG5EQ{&#&`UX2<9xpwBQKlZwQ+Zt|09mf^t{{6EXs) zPew*C+ehJHP2{_RupMb<5c=S{J`7J5A>Zxc5zH{;orE|U@_&x}U2xrZOa#O9>?!Ek zv!}3UPm#E1JC~lW63Yj%{Kw7R!?S%)ubw@nJ$uR!)uCs{o;|%SsXcr8_#(ZNZ|9zV zJ-c-6Chh6p{jr`sd-RlZr^RtX1kZ_Fpa&g%@bn(5^v9WESzNr=0RY66#+FdTvmlEILu(6^?w1B?Z!O$vpJcv9az!SM8C+bhs zaZDZtlfQFJ`fyCjtW0{EnUwHMa;$MoO2DLND-P!qS1q3eGk<=7riDM$4DR-ao@|Ik zZ4c+?5J2-;i$7qULyVa50%DRPNuU`feA{v|*S>!KDGdHmI6V6zVLL%_jFaG`;jBHL zpz8~js%18(<2)7p!P2gIWfy9Q#a#iO>*S7Zh?p>)i5V^#SWg^mZcd_nf(3>ipSVYq9+O z2QqfE2%_sK(OfGaEiKF;EThM7iX}5M_o!)~*~A!k9KiF$V){9X1CW@WFOrqhF)XeB z-ZjX>9t3S}oyEG6&Yo$Qn;9QFA=((tTX8J@xo872Z>8bM!VHkxJC7>}``*$_csTG( zLja8n=5zn?Gn0(7X?HUDHmk)GMn2e+#Rj=aAl<7p#e&UDIj9&rfgW2dLpq*7nu4cg zLkv^#{62Tm5fz|nSFDd7s$8svF1PkxD!v zcMHUpu`cCR9}6_|V~y=GaRNc8nl$qmAT_S9A#)`TsVl3ibs+-U5n!Ps+ZkIA=0l7; z;mm84J%`Vp6jEnRPMMGPC#|eZHsQHibD@@4rOEaQY_5I$A^0L6UcAscR8QL)&6*Wx zv%O%UgsnBGu6P`b%g`s^z%xD>waROcF~(|WdZqFPvTqS+Ezfy{|Bhw9#4GWkja=qp zOXe4CGHcFZNAx%7GxkB*#M>$QSUfu0G+#eUte+*(N69$uk=$}_)B2F6`MD%K2S`s} zOEOp&HU>|B%QE!JMyMF)EP%b=1au~WHa?QC;r14|_2wA(jpJ66R93ZgE@)u>Sk@it z#3p%Wuy_nyrcb<*p;w(Fw zk0Z4HUAa(DNf{bkq&ua0)wR$%ngg0;8pm;BmZ_btR@|@(tZ=6nARLgP`jce91!xdN z&{Ude6C03-w&5)B+!m5hJSl1U1#}UsFe&r#v@=OXy_(aPFnZY79(ElSMP~8L zEUveuTjq|tEkbD-O|vp|#f<`K`5l74_!=7{F{B8>&ldGyW5fyO2k3PC??k+gEc!`e zu&19dDXXF<`c$6!QS5xNzRS|+avTb~%wgS$o#e^JKo=6vwQjxhIE$(;K8CY57%Lvb9l~kx`ny z)W)i7dJo-A9BQsiyjFo}rFfcG?!1CeE(sSQ^1Wx2^zW zEBfX19GO*T;Re=HJE2ROmDh^co$tGupdy5tuC`r#hQ1g_i_Nf633JmvgG+RAE5()} zjqd|~59tFR3rJmD?0dl|5aT&5tI-NW>Y(H77_!XxO_(I}qu=yv6!`HLv+=g6j_QMp z8(Yg{&Qc|1)JAyDN-{?A>~Wp4Rr>5C^?ogfvXjH*2@nQ32EbMWVgbvp;*Pjsw z=$H8D^JOs7dYq1(C4LT=+Wq(e@rUvm>k~N=nSQOPH20tw#tvLf{bs zj}Z88M}RE9-nr#}nzQ_F_Ljd}j>Ynazw`gL<^L#?2Yv5C{3sW~w+I&zZXh%x;AcP? zKZKpQ9*ys52y+m!5uQbO4Pi6FAcXgk=QO_0Bm9hT1EC2)n8h&e2tEk@2z?P02%`|j zBTPd`LYR-R2tk9e9H9bX4Z>>(n-R7l>_#|%a1!A>!WD#T2(}-~TWK&|=H4zX&tMn| z^uc`8l+ppeK#E|wck76!>JTjVDqP1QEHS^-$OHt;g;U%N)OrJt3F-l&9*XRwF_em$Z*0ST(cv~`+#@m5&Qx||5 z-rbsqR}9KI(tQf$c&RH^^9l-b^H7%REm&ZwD_xBm=v{<)T1~!IW9EzJgJ@8anG6ap zM!5oYDp$Zl9F?I8cP1;B%7eo~dVOJ^ba{H7x-eHG&q>7_Ww{rC${+Tz6w1p$KD?=D z0srDzUe1_2E77c6ycH4jsDgC7Nia>L=EPxmTo|TLdP<+{^dP)B3a=d2stYxvFSJynOTdT$j-^l%YU+3IEa|fZwJ}WEs3Ca^hAhY{ zNF6*fR55focR3_GYr&904ZWk5zJhpmm=@Z6{o_!+C64Eo;`--mv$F>_znSwTvag7wpFp4*#_5yh6Npmu~=fk~M3TTx+!qd@_kjm}+yg()bQ#Ex@-A z$|R`KeGAl2Qb!uouTNpWLAcA$UTVIE`j6=otsNvU)TC;(g=2Wr(gwDiEqZ8$sIHAN zX)rSyzA5=ekmoH}gbvT;<$2?*kS%-e{CnJq;D6y$f_gp9}}N2F(Fo}QW?Fk zMX@JGwtp_;)&+`B8#OO)!Mv7!$6prZ6y{}U<}J`>Wv9<8P-m-QyyvCn=NHbi*o}D( z?~uz+hat839P4OvW4<(+#}~RS!I&p`oMA56#2rK&gSfUn|1XWfN0^kf6!nrU4Rep- zzo#-Ov(oZ4ld`h$MsDUyq;tv4%R>Hp8IzKdpOTW2m7Aqe=N2*hnUvI=!i9WcCKss- z)ta1C8e|q0GUEhXMv%&3-auNR8p@2DRA_8L0mFQT(&>w|g_@L%^sFL=S;ddGx)ppJcSfAe~M;lKGJ_BuG7@{niTZN^lWuOD8t|#!xQ3%2M48RXERTr1itff zIp%i>Qp{I%kSi}&Q;?UPk&lsAFj=j^8^^WT>L`v(B*RQq#>Xh9g@$lsn3cj*D5i#a zTbPlLF+h{SOn_v0P_);D85!ByLg>qD!VIqaGt7I!3>wIpo8XQd1BQ7^m_Z$t*)Ggj zTmazU1qscBm`)-LD4IO1Sv)4vWaS{O9mS+&0v~U>!heR@D9iw`g^WAJfNCD%yy!YN zEhis)zgdNa`Kf7Y=6fNHW_cK&-X|_V?GGWC;iATi7ULb^IpBI6t_tb05Wb%akaJ8Q~vM7(OB|| zbIV`MS$Ze;xrL{R$d<1g69f-y`siFPo2mp>>wnhs0yUqr@gNE(MGm+!BXKYy28qZA7BK z2p8s0m5yO@g_b)?3&Twklfb0luab$!RWvi5iNSp|{!YR*eLff8d_eUH8CPb3yPNf! zcG~G9#y`s%5g&t}w?#by(9IhV;`ut~$cbNK_O>qp7}vit?~ z%XbdD&^HFtF7)d?c*}456vM~P$C6TRED_m{C1a8;V@XEpvDrPb63vfROtxi&>F+p6~Ec|x7z>3F2 zQ-Fx0O= z2t(Ngj20=`H~};ma)ly(C^H;1LKrvDAZ|5W`CQ15Z6lc@7sTiyrZ>t=L;i(aZbBs= z9Far|@nT|ZbaP1A$H|#@!ACG^ABo;D7&ViBJOXjfup7_eKHW^4G>iDN=oa1aE#Sy! z5V%`{Kp(#CVnHJh@@s*;#zsQ>wv#h6kSh;$VCxQ_C=O%$vW`gq6`Uu6mH?8HhFG|7 z=@|?YfIh{b?gHew$%+c@ixl9j2^cnC);wUXiTr*AVn`a>%*Oc8q+Ir#-za` zdeNbV-1Jo1mL1b;sk*S&xUue|Qws~#ISaCv%27VIa7-_4LGGx+v`lqQYT@9Vth9nW zItMTqJHw+=3v+^sf_vfcQ*KrUj!z`<$D;Vk#=2u(WF;1zj$;;CnxzzpkNGSVmorEK z*39uZK9i;{EX*s2#I`H8XVWy=0jrIrYaGau$qr^gm#72$Z*LUo}{vRw}45|pQdTGP};>TEMc$iaWIDRNA& z)IuCE%6mdxU=iPvF4ty7rV*TDdS#?$7piU2c|`b(qlY-*X!H;}5k?QOf&on#Vo?=3 z9ri!9|#HbME%1umc-bfWwlI{9yQq`1Rf#qKL7!;8G_4Be>1I&^A;KV9zJc1Wt{*o zda+$67^mn8v4lcKv9fD-7q@|{qE_8$FK`uZT-ntNQSL&4AlO6U=F%?EOC-d*4-}a$ z?tv}>v0&wJp+LNGj$)P~)Fwu%=pz!~Ep~r@ti+!g&HbB>DSII|5mj?j^RW#6DSFz_ z6#IncW_PO|eq+lwzfD!A6l#AQZ|*^!e^yQ|doA z++Oob-yPr1`9{&+%CVpwcuU||6(x#YyoHGg!JQSpF1T^`_QK3BDsN&4BY+Vx7;K}{>S&N}<$_}>RtX)hYyt6kmw zwd`quo-b`DP<@5AE1`E;Df;USKC`zjpVJ;G=MZ0#c z0)e={qOT&ra<34q^2^j{@<$CBl9yJPAB3rORv`oo!VF#E2}HVk3j`*y5Njx*bk{>X zNHI{+e`DW`0jqkM3#Juh+m#!lDJ&eEmKub3MJFoMhe&uT+%1)gBnmHzmr}e~By_1# zJVy5&#L$V&-ii*vQozcofXj@8$l$>twkSiDL@NaXCiaV|hXP8zztTQn|HZB!ZW3;K zd*xtHop0%I_my6Qw$16By5mf{2NiX{u3h_aQrC#L8=l`gV&pdMFwLk}re7Jp`&Ips zb>q8l`?PkWVdkZK-=vGj4~-F=2>pBKL#nO)YJa=DYMO9_O1I=-bHb&DOD@k`*!g|b5`Uu)Ro+H>5*NH^CV zYX5*eQKu%BG;I3%uyDK2*E4E6y;(Bv?$WWz1;rJeHw>Ea_ag~k6_05gIBM7Q7n!$m z+;z!!=O^6u`*@}_zvliQ|5&%ttMkPJok|`*EDg+ECH}SH_L6=l+I?{&Z0`I|qeFfR zMejR+FxcGtChzOA?t4T0qzQNGa-!M?Z+3u8QOer%ggKGHz6x)GSmNzD6N@kEf?WB8 z)O>ZYkD?<*yLx*})D|pA&BYo+w%XDj?%pmlGE*08)WO{q-6)%fw+|m7pP(+#WM!Bi zlnL&k=+3uaCv!4&U@RB${ny-WL(x;iTDmRWhFZGqf5cX-6!fx1g}UoXffTk?;4UPM zdiTgjn~!}d|EGJg{{3gPx33T zPF!2^+v2yE=Z)U?!avFKyE?c0<1uw!b*8cKi3we4g^uqTnFW ztEJvsCdGTl+qf zd_?9u`i-)yGpgNl4?Or{%7sr|JG|U$S>4!vCmvgJ?Tz5Wzy92-OWlEwCr@~-Ys$vQ z)?AtY$Bkvb7QeSZ@Z29NkBi59C2d*vZ6eYVOsA(FF8r2F0b zI5SED7g$NoC_&dZcTse<^ca!1CyWwQcf{b}!9zoBHHu6VEdP`z?p~d>f1-b|e(9@$ zFYR0TzTjK`XovC;T4^Wf{mmtq1=zS}S6sSiF$jr@J!SHB!SJO9FGf#qX&yuRb?<(a0v z+rL=;$Jd@4ZoT{<@Kktww{9c)-5EDUu~NJbp}*NOM6TU5XXKLazm8UCjOad1yWfFh zMT(7Ka5gRvR}2Lp=<`7nvaxn250%Gi7i4Fp$*~lflUlIU-Ui4cwVKR4TE67Q=rG0b z;9)JDooBUBji})9;I9Blb#8K3n|yz2{eS@!bE!jq&l%UOeR=<=y9Mb=p&RBqNar88I%p-^|IY-;OBZ$s{ijT@0aka*n{q)2#bLU?v*a)Q7bHj>bN1jL>OfDXypmy_kd{vZSRg+TS z+F3+ChQ(VD3Oe~aYdu9UQRyK@5e3|mjlEWJjg7W$<(TBBrG|3dpGGHi|4s@YOD*t1 z%r25hWbd885Oi>HPKYx6IZrgCouJ)-NtA8 z*sT-HB8snoqLW`-YRKsQZy6z3=*W zk*;XS+!J!PV#LMs<4)NYS z_Qo6TJ87B3+kFC_TgOxDxml^f9ThT)boF+N#eShYGFy{p8BX1OJo$|{^XzY|Vl-v- z@E$rbYhjiqHCrA%QEsurgXJpA9-=$}8;Ofk3)J!?dOn_iK)Wc!Qk92~n<};73AF|m z+ftRtzAC4)x9@&CeC&3`pM}C#1K;^MdjG|A*TPrV4IgsxiL`4ME^fNHOD?MVbD8{i zU2V+KyWdW3T$KFsl{W{xz3_{W5d%+`&Rm?iyURS^`)f~?M|Q6f-~MUQ=kK;(6*w(+ z+>J=j41<^FhW^1H2X+6r5pN7j-TmA2!JBx`X{BHV zjOWvsjidW{bJH-pnUiaYB7%~7h(}@;Jal8|#^6;$%+An+gbAUTIVr*hx3YuQB+eNA z_gLXJJSq`>iMwUqnWb3x7w4VhETt5}RiUkNAHl8C45p+sb%CM>VbWXdr|4AD zwmow+=wcaWsx2!g-Nc-;iIo_WLoeoLo&!YvJH4`dpyraQ?#T7~buW5%{pO>sUtOF% zHEj0BFDk$L$9sQYlAf6MRhK_ie~fDQ+uewnFLbVb_hPqS?mjPl{9Vnmew)2!y}5dI zxBMU4UtQpGUC>-7j_^O||J2<CBC5OWGs1x8%zGQA4`&lTE-INRr|3ddG~W&-q~4wqVT)=Lsv73_O94{ZSq$i zZuv@{vU=Y2?D?YvF^8)gPyTaS@|UWn=f9nC=$FX!Gd?>9d+gG*YyM$nZif`tr=z#r zj(Ao2^W8&HbI#4sDIfcD&dAsFhV{80E5AG{`sCMBn~!dFU-tU>`4RG=E?spchJMVZ zH=EbR-*!87zG_bAVI{BqxOe*3uPuABF2BpvZo@}?DLP-4Dpb5W@y#CU=$$JTPn-Ai z(GL|X+ljC?)xaBP_f*f0F5CiPH-`pQtazCDnweUd3FD#3%MF%s9s$gkYrHx=CoeZW z7~?5fDR*z*7K1b)uOJ^kH$pq^!2=ZiC|%@jW3V_Ul3Yv6G+7>>m#5*p<6#Oo!@vDT>>JUnmqhTpnX3VgQ9z86Q-{W$ZVi` zgRA9vuSlET@12rAVA$rvtA6?LHTltmx5|GRv$y!Q2LZ~^^F_YbrGI}ieVRI`G*kJG z#SHd_8SKH0EbtfI73iKWVz`csVwj7YdBM}yRZOV&u-r-vmrm$o%D!7)KmyOr%j|q zoFTVgZxyYCoPsNpJ`Pu|(0V_!JY~YgNvn@me!g>Z4G;7a%H8O79=C^lDuDW^h z+|$p^>~K5e*=1jjyD|QpxaX8RhbB}^$(wh?r+Mm$H>Q1lIx+W{aBus!E1p~BwtI7* z&yCw|-D97OS-9=!$}QimPP{X`;nvuSn^!JX{^_3^lo}bO-M(#5_x@+1rQiA93MlL| z`dEeM#s1UOGak(U;MuEP{xK--qrZ)s{Kx2$p@PmoL_hQGj4Ai0zUR7UU-8nf_YGy{ zeYWB2=9T(xBR9sqyz^q!$F8P{FHNip8`x#udpB-{UmFm!!~2Wj<;%6-Z2z{sOYV-7 z*}b1!c=gvOSN}6cv(@wNwSCiO-J8GWZ#g56uBsTv`fuqJ^38*sPk(Lt`096S8;4C^ zQq^yuZ*Atcx4&qf9&>2nIJ|x1jb9%-oEr7ZJEG>`LeE>1Djp=AiypAS{ zAiEfFwdnGY2@TgSEbAB+pg-3?@q?eVF~NZg({^t8?!7{H(N|wwojtC$VW9GZEeqxj zVO|~i-u9Hj8p*Yi4$n>YdTyZigv)DP79@UV+CApocgOUVesH9C0qdKnr?I39EWeZ8 z#tO3h8-468|AGsgn)oo z8%rw{U94LdZlYjM8`nS#!{6d4cm#W4mAvKNSJAtrpg0)QsixIu7j!hN>l^;noPC}b zo(s>t@$CTvzB?SzycU4{rZe(+1e zw%@%PZj5^T`;B>jKQr7r`>iMHg4p0hX7AkVar(#a_xbqzJ5v@G?Fw*7`mo~VSNgeL z{O#hdxijDXZ2I75x(`2g&Fn+NPC)<*>$2wFv4Gyi)K4q85@)Ec}n`^R%(M ze_rru(cH1}`lUvLF7@Ew(dgCfUi&<0J1xvAEG(s^HV{BMRNO15RSZiylIJl!& zw)Xdq=iUlB{p_68Wrx@8@u)e{r|;sEr}yN9_8GBb zTo3WK`fr5A!+clFdt>&<=yz+MowxFRVbt&Gm~X!CvFz!e4t_lD=&}39VkT&-hkEWk z`k`>>u9TcTr*f`j8%`^3Z76?o?XO)QER$UP=eD?QUE908{l=HoH~sX6Y1<7E@9mYD z-dn3P?{rcR_jl?oE_;W1_$WhbZ&B8rRvw-n+}c}|JEfQ{Ac|8%TT|?oL;hzz&hnZ4 z&j^eE+1n|_y90^hUivLa2amR=FY&kthshl&_H|0p^eR#OQDrdWB++3aP`Jd%3enbs7i(L2a~9M|7UG18(v z&pD+SiT7GXxD?TTx`X!2G!T!W^{puu-g3s{tH+6A_w?2jleu503u@7x@fIF0;zbtY z(JLwN=XvYlGFi>RvxVsCBKW9IUq*C;hYeDx-(}AQH(wb zirh2HjPIRKuLmF4yfSm}Ej$KNunND;N!mmFVJ0{xBxIO;MnP(BA^lb&yvqD+{4!)v zdv~%ROK&^leX^cv%6qR(-aE*cfHSRmFSDWOz`Fzw2{1(QlR*~V`F96p;fx~pEACL1Q6Lnny)-%<#B?g#Xf8jaleEY1$s4QgD!+VbnQOiEi{Ri|poh+XHSJsolP+v6Fp5 zNXT&f>?t;9Y1<(ddoJ|TjrfH#wY-1u$dEz7!$u4W9uyifX!!6!BSs7`H|U6FXJ9QU zCKkqi;viZhz`71)e4u>YD%wN~SbGFwJj8>a{6lC3*2FcnM6qvjn-l?-sYSh&L@{1D zBVMk-Zg6lAe#DfAt%DG+OaXBUtzdgPXBr3FCI{HAugnZ`A=t`r5YDXbv!U)v90rk3 zH~%o}_~@9(_}Oy$3g#?e&TS`l(z7T3 zN{r=AuopeZ<$CZJVF2?_WsA+71kq=cF8jVMcGk01oT#LEHa+``FpH?=JFKE6L)1ym zdiJV!o$>yA6xCGI&<5|*z`IE)v?{_P@SaS(|F+c`#XT`Z@!n5uQiQF5EwqejR*Da7 zD7I?&upEwJ^v@1xOamIOB^{Uo-A+3+77>rHeBg}7C$+Nb~k literal 0 HcmV?d00001 diff --git a/testing/btest/Traces/spicy/gap-recovery.pcap b/testing/btest/Traces/spicy/gap-recovery.pcap new file mode 100644 index 0000000000000000000000000000000000000000..b806213ca96c7f28351ed9dffd016d8dbf363d69 GIT binary patch literal 129956 zcmeHw3z#KWao$K;3p_@S&?m!S`-_`dK(lD?ojdP`c9vahwIGlHvr-Hs`G&st_MN%f z`xxEZGrKE6kZ}--^I>C$m+hFC$7dVI1W0V`4GzR6*v{wAiSYy5U@(#KBiInfug~B_ zr0Tq?>hw8%N6RDd(YxB!PM_-PI_K0m|Ea3(s^9v+zkl=9Q%9!gm;U$Lp7zw{Cw~6O z)b~RJ{<#|d+%`4!>Zz$yQ>U(-ntJ)v)K&QBLmxhJ!}5{;^~aZ<@XpZ#GdKS0N4^U# zUiH}NdtUlr^PZQ!;=7-C%{5bZ4ZrUAFFg6zZ_AFq@t41T`VV{Gx%#~~-TcDweU~3| z^pX7^{Hwoz@q4fPlkcDTlin}?{$GCc6aLGm-}Ac9zH$HUAG_gi{_ZpX`Gdc5?p4pe z=P|$d)SrIbbKW+0XBANuGmKr;2Nt3UtwN8m|U zA9=>_UUkhi*Xj#pa|FVq*Vn|{`CoNp$*VJq$J;y*VUjmCA;caO7C?etDI zfVMk__jP-{bbh7YNapTLTiroE9Pi$e+&;*&UIG`A=bcMlm@G8v3oExIql3+jje6r) z^31GtemK9_SZKh13(48;DC-O_&ST?s)2CalY!7(n*~NT*H}Ch3r+a(7ZY#~Z!@>MT zymIqJ|7yQ?%e`BT`o{5YKi$dZ(}Qlu{PRNA+&ljC`KPP)b<4hXM>^Qq2hz-5q*JDxO0*G->!?p>P+jL@ya!60j)V4KPL zEZeK4z3xGV*N5l3zMme<;8Ep2Zs*Vfw$X=`J-lQkDx zFtB?&qqLpXZXb-Z*8V7~y&$`HKO5&`^{8`Mt9CZaTf4P0aBVYb4_o_vn1t)5spZaL z8y-4<-tT~@yyaXshnCi8I3Cx=@NeDr%;(>#cwaY-s`z$Lp-C!Ds`#-rP{ltCsq{7f zXL{W~^w}zlngS^(u20Qug zX0p^+16{FK^GkcsJ*P3ZB(2?aG|uv^{k&6yNpHh&Y$oUS2gj4fD(K}1_%1Fciwm2J z%bP2!$usY~>$>UdrcXTi%=2!&>qXDMEkP|idBXgirR|d^`dOYjwzG=wthu_I=X*7n ztM0+p>#HZ` zch#sghwV#AtCx<)TeGl2(*0h(o$qG-YpNL~d!F^@Zc?o` z9lK>Z%A?y!<}{BdZzY5MUN3pt(+pF%aFZF#n~o*-M_HiwC}>^H%N^ZuUQgR?^YHDP zlAAR@kOC<~zg`xX4@P-^50}#zxxg^>hNEQM!YNOhmtY3=dI$qMsue~Xrgqc?a;+rM zCj#5yKo2)g(-^ie!()(LF#U%qQ_5&3G|H^AKTvzJk@QxS@thKI4p-fy9J^80A0A|N zr8#qQT#qNGN2Byo9hJ>%I#%y=dwDjRJJ>pTP#@>%C?9`kH{YF`RkOQ2d+b=58sMy; zDxkaxOuSuOvy+6!8>Nu`qNa?KQX^|^2C16ZbE8w)ix)tY@3AxEveb;?nw_){m@OV z(OgfScJKZ$zs207e>eu4EbfR~v*#{!0f%u5IivTsWUklU+0Ey1b33Lbn>`t7J29W0 zJTc$v;{Gk=v)kpP6VTojm++k3hP8-JI{$iHGTN*k)!4b);AJQ(UqaYp$MH@@ibB{%|lnpLQReOy;JzOm}Loo=oy8$uz$Y=a+f4 zWajPau^Pvq_QZHE9VigahkGFKJpK#^p9M>+RDL#TcSl>Zy&S+<-Es0nv)wHm4e*-b zFb@ZW1+`=yqv5dEOp$?>Do^2f1cwZLj>vZ{9c~}yDfW{0Bkbe{BRKP%eE9W0Z3VWA z`Wa(5Ol={Lr=|V9TZW@{Hrj$^PHPw15vY&^j#UY?Wdl6tCyI9VS5D!U$;@y$Lsi_Z!Q^y!=1zBM{ z)yXkOlmcaN|JV8eXKa0%Y^JUA_FQ^YDd)ql`botNl-d$?hC{$u=pnmykd4M*kjJkUuY+tJ%+HPbqFoMpQRu&o5bB_hHqhh1c9~=YX|w)gYiD2iq^E!lbeY*eXuG@~ z`TY6!eZ$n}u9{MRv<-CC-}*L?;4?FG`SYK0Hqd*Kk21&UME;d!n777bXH|nCtc~_B zu~q$otdoth(I!}D`QGOIyfjVbuh8r!^I03#$}Hlxv$3j&V4{r1yg?h)W7sfi!%=sq zI|y0Te-D~4Nu#ti{G_K`e(2+$1&#WBrctN~??sLJrPqAJ)R9x*)Pg@+qu?J?r-TR4 z=d<-ULZjY0(W(a9Z(}29R>OQB+KfJBtnXy@?z_&`)@N^-#`UEvmJV2XwQ(Po?XL2z zqyw~TyXiP<t(o?cl)s8-I48do84YFSJsy@iotQwXgn2f+RgCr zWq*dVD=upFw0=5*S=me)NTwd$N&DU2rOl*<$N8)_zBC5A?0E9+XikH>>YV!dEC_i# znLU^74723!+h>m_FBmq5`4Fx>E9)I(IqYr8bF=*nTHFP+1)wX(!FCydj|Do=aZkNq zBwT7JNtDsFnSi-CSF2%{x78JOXvS@CXEfX&v@HQ$Q3QKK@ZTth6ByUvWH`w1F|gOw z3gb|BK(9t)53E2nstffMFlrL~>y~L4uet(xMRNgAu+{`S`urw1y5N7{(du!cl%~=4 zsyXbb$rIgz0|Pk5hNDZ{IJB;V_%yv~D?VzzVg7euk^qNcFUQUpL2IzlS{-B2Ph z!i3h>Pw@2p)Ir64H&jU!d^(0Z%iR!#uPzRvv<4b*9qo^C^ zpF(t{bfs(E+{G5&wB-alY#;WA&CRp}i`xu^^5kwNvk$-OEwi}3+&b(Il-f4|*w14_ zQ=Q3S57@hyjEB9h+Gh0H0Y@;dmWJCM%w@n~!~(jmdDOtO-rWaJdZWImwmrAYz$Li2 zVE00>m6Vijvuv1sl8I@zi^9I3b^|~!rBlf1B=pVj3vb&BpuE0-KH|N5o*a79)`BL9 z!N7IXneawR&|$a8v2tA)7cDg_WDz zw*6c|l$kJ`S361gZ@tMYD2I-pI-gzYjNn8yPR6^#3yQyT<=zjZ37iMOxtPO_t#A}w zN#KO>nii1j%V1Tz2{7wH4=6-dz((8zAt6|$@kVr9j8LfKYEB)?!XS9cYe38@umOGX z5Urtl0i(?Idq)s@%0knRU87l>JgG*NEXQttN6wfL){-gbnW6{*lo|CNIFxJZkX+MY z7l$!Qc+CUcTR1J=N$pBRcl>^*}Pu3j)oQ^L#R(an3EBoX`Lm2){9V0wp~ z$im!K%(N2W`FC?L%;&VoE);W&Uv&OCan*RC#5LnBJBES9#EpXCZCs>D3V~6KIjBL} zHn#-tux;`V^WfkewoTsQIuN{vTIF5VV;a24H6HeqE7oxeL5Bl$eZz9ndrfo|0R_0& zaD^DyQoEp1RRy>DlxvT^K&`xGg&QoCc}u{;rdECQtJL6_JKP6cO!xC)coR`i*rpY# z`=Or;lZt^gjBXt_qA0hGx};CEeA%roD*K{Q*Z;uhv0gRm>ucKd@HAUXfJ@b3jj;%g zg@6cx%+w*lSv&Ow^*A-`MqFh8qPT5?qHfo)(5*Pu;0WA@Gl4zMyK!dyRb$RO2Y?Ib zg;*EmImu_ZTUUNPoOd&2tHuNq z(6l`oUWm;|F zOi=*7=c@yU){wRuTp}x|M&VSajI3HS%P(Z`mI$uydYY?8h^;>*wMCL$fJrcai*z`* z+d|8M9DU8C9NwALp81MHf$GGDlC2w3Sv>OE^(L0#nmId)UMP;fboKrayf0a;JLu@+ zmK!vO{oo-_VQ+xHEjWTP#LY_a6rl=*rNq#*E;{^nx)-%q%dGV!Gv0RRvOfp+qEfE|IHU4mA6|kn9L;%p zsqQw`qN?DP?dCq*IA9tBYsZ>d&P1|)wdU|*ZM>U;??l;1$vPZjYs>K0(RK9;Zxyr3 zA0bZ8%yx1%f^|muprTHx=(F6dw1pf(NOycPP$kvIY$n{JVf%q{6#aF@ zfp!l-`Enf#Q@wIa$wJel{Rbp0I-RDSi%T)(rwY4!(2n2`lg*2Ly5g-p5ZnbObX3u( zR~k?S%B6u#fkbnALvcQ>ht7_i&>tK#OOWWZS#jJgf#pCHO37LG?QZC8v(_~rpT?Xi z#L+@){F!dPy*1+BLH%@hc+ba!{3gW2^4hSYEG`Yrawl~Di~9Z^Iy$3TFj?Kyd|D@l zWv9=xY?p;ooY{PgAa+l(>LwPQ7OH@HmV0M7>MM!eBv~%@DC;~x#(*{q@y2aU!?3S7 zPlDX7SbcD?Lg>9wi4kQG-annxhwwUuhQ)9dIB0ie=EAjtT9;B`D#apgLQDk@%D6-) zZ7E9Q;2pV|n+V4{zR$Jc1|LZX{J7`GPNE*3q@BV`hix;Fnv3%Du^BXo32!^L<`9mO zT7nv(7P5uUU?YwpzJx0S3irXurJY^eOcwR=4If&gpa)6qAX`k6lF^$TWw5R20F-Bb zg=D}HR=#DnFhLx(gs4qgjo4!ry3#fwyyI-vT}h}$KWT5X(;d;ap`OfEFr1U@Mkso# zZf0}gt?z=&$_O!A)7+xooWe&h<6TEL!u_C?9Hf@*NSx|t?QWXP^;3we;G3|U$=YgT z@1o{wZx!C#Mdt&CkxUXV>cvP++{>dkhPbd%7wXIU45>KgZRI#&*4DzYV|dsNTvk0} zxN6sr7R^6LGeYD0#@!c}Y?6r8Y)w>b(Bnq7yp|Eo{KU7)UFWqMQk}Kvn%%P1WyM8f ztD}H(Uwy@zx+s+6t%3|V1;IhlcwccIFx*gsc$qPRqCGN9ZF~xEE9Hy3vp>vs z*c2~1j5}Aeb){&}a6rjc8&1i>3bC}NZ15<9WnvK2Qo;r;qm0eUAvY^r5l)l_hnfwE zu?1v@(Z=V_KkMicem>_i6IaVTf_3Bl^IrKDu0Z=JHDfguTC?ZnqaF1N6^qBgH+0wsIb8%UR1vB;sAb z;q;j^>$k0H4%$xS2MF0qT&Czk&X?#?@qzy z)y{e+)ohC1BP3|zZ%q*J2qDEH{?-&;yck+}Z~POn=(d<;6T~kb@b%FdpSrTxUN?gK zeCec(glqufgv6JTqkKgXfG3wVOyu~*NTV`xenjN>iS9`M5(u`ofxq=!STB`#Sva!A z4DhKER3rwDCsRB@rM1f&9YJAFheww;me-fmd9?;k&0zS^ds2johJ{I0FhPJ9@0&0Y z*m)F1U&P;f`~;b^Bv3A<@P?>hEg&zpZAmh@keR}H2W^yF{aRNq7}aWpBQ#3m-yh{3 zbRpt1*cR+L1wH=kzz8ka!9}w#^A~cUX?+8Kt0VZA2ap`12}E*NLy)wzzSLSaJBS=J zc=@nnv@-%pjiq#>b2vz9EUm9w(jp{f>BdUO%&ynUASqoTDj$+oSKAwz;ShkNrIn?P zWr0|tubKL#6+RA0D~lV;nOPQs>TtXs9VD$R;KNqTE*1o(<;L>LijgM_O3Tfah1Do1 zrOoDI(}0?f31Um+(-S9jyHJ*;Yj!q$a=D4}MeBhb5jwtnjdOg6${Hqe{CKxGDLvX- z_+cW)kIUx;J%G_h)@-(01yJIbB~0+mmQSRhlCrWTE2tbO@h%Gr2TH5y+M+#CMBoNT zg*_=kMZ>~q4FZS-!+8m`k)Xtvupo<#lSG@Sl1MTs8P`wg{edw!1Sqkddp;F!3}c|g zx10bdG5!*ivgPIGvRT?bC}nB01$qd^)|UsAj6o$J$-d!mzcPiy+$@6rE!thy#l~^G z*5S8lhyYS&V{vt@goJ@fv$eF5uIc6IK@L1CUCkC34fuJjjK^}Txv^wW#RsItjplmW z%s>c`noFrQ_;j<>JnXA-Q-GnUOsGN_8KhiWS%W}%0Je&}?0|A!9w23_?WQ#_O%q1Z zjj!xznyoK%vL*FOVyzYgq$Oy!or(YwJMtE-Bo5*}a0neyve?t02OLD`cA(a3x_UT3 zV#DqWKs&ADt2ENg5eL5rAGt5*h7iP@#CaLKx%=H)h-qWNc^&d2(i&J{S{Osh9|3_ za)5*)TBQ>d4v@IYM4-ldQiLivEN1tQE*OC{5|H>37G!*!_>z!F6IEzQ7M1Ku07)nW zT3Au;Q9xo_P7#nee;<%mS61zYs=lgrCtcLS4cNs1kZj;=4-yN(XV&mV9DIHUS&1tKL>xl)6uE#4bV4u5`Cp*(AWVuD05OU@A*&#WQS1px!Z1e8 zc%&%;ne3U5BtnBc4VY|QQ49*7?eiAfuN;mzsoNRl4Pmr`%~r}BCUZRF#3izFo`lnw&es0IuZv)LaL^qa}Xy7K+rglBfA9!jsRHUc@a7qCKsPV zt_D}c(vtn;D26!!J_0!bCgm2;StdUI*|V=4-uJ-uPkf1=Tj1@$=MP^0*h40dG<;^B z_!3GU`9mMY90gVK$oki6@<>#nzXBDSq>4H{#qhq%4}IdVAZcUH+Gby`95ZE2M}64QKoxh>I7 z_p(KLKTaM!3o(pL5TpSk?d!|wsB`q!97p(Z?ne6E@O`a>p< zBtDcpQfbs96RqkmD0!qaqpReRRr1JCFvfhWt6!E-Z*HTbwD_&3`53oK9%;XmU8s^r zMpBVh$s8x0Mj~ygR}=lXsYCq z#X)dM4RzqLN*)QH|GRWVEdHxgXUkBnYbG^w$qG%<7?;(;jkA}!ffjC+Jkq3cwC~r5 zkHVdQPFyu!C~?iaNLnS2RIi!}L>20*-$u}?E7O3ecVI~|)q7Y~@<@2eObn*~jxLO( zOaWLWkMuHkvUDc!o>eU0WAPO%t)ptZUA_^<3tz~JF$=t`CAi^BXv(zXZ(x}+3H>NS*PLU#XHu zLbfj4daLA-R&A=}ks_0o$`w^5kA%uU{`HQYDXc^+=p4Y%`~cMcS;AN4jcMRq{wEEv`=GRr1JwQYDYniMwl6@<_^1S|yL% ze)Jx}DtRQ9p{kNc`sGCMD=L;tuk7Q!vNJTWp%1&t2-(+IRr1KPPxT@n>cJPP;i7Ud z1N8ZtM@5zkPPh9E&Z7!^qUTWrKF~Yd=Xp^orVc0<+F+AD&2#OAbo8i%Rq{xvQ6oNT zq_0Ea`&}%!={_~b6F5n6FGh{Qn}Gb9!f-xAKxR2pv1F8vaiSa@(g(xYXHc3UcfMs1CSUgJsK^K$U}+9 z@#CaLK;l^<86fe?aX_k)M;_%vz_Xm-B)f7Ul!V1uT%i>EeCA7Xb%_bRl5EB-xnq?) zGF1Kfeq}q0DXfx5!mh)!uvN(;WkFIH9-d{9tqyfLd_SRC6WOWzZ?f7 zo;gua5iWXaEU28@4i&vh9w`*M1d#Z)P$iFqs;6JvH_*L!QOw3gGs6cmEe$j{-a-0w|Lz3c$KroI} zat%qsa7y78q$$EFm0iR3QaF{<7AJLkB$ZwZ@s^vtM4dx8EX56|>l0#VC_r927|?mcKXER|bP(90HC_(H?_AxnTuh zP;ikjNbxqDUmpfdxwfxZ804kg>>_p^Y%1o4O>*v>I~kio8E|k3xq@ynA_iU*l|$kp z2vV)7Ks)0@W!;d@2SF5x3f{Xx6lo8i+1s_a#F{e zf+%>L{`p|$Q(~8XAuPD_%jfc4HA&}W=;Vt90);2;@}Dv%UsYo z5R?m3&^QpJSQtJp0&a!pMd;)(xus)}retvmY?J1c91uas$4&M_Vq1v{Ny4IpZaXP= zq$({}Fe!zXvlIY9Ig9u*2?^`WVYy)iKu~azAPBagDG-M;dLU>^#C^qrATJ+>^3&je zN>U6Eo5b9yQ-La&re-E=2~rWc7!d<6DkF!)MG&Mya)EZnhf2vIoezRkOb%NEAV?7v zA}HzkVWNV>$bjWx`S~E|f;7l60D`=*9Fhb@Nu zmXo@jt(>>mqCF*?YzfL6I1&lHmokURoV;=35?MJYoJA0%;2;hv13}p!0ZDA; zuIL;x5OgFC%$^iP&^Zu98XY?<@Nyt16)FO5h37@+BEw{mretvmY?C-5m%Sk5<0co6 z#I_O@l7uA^UKt3=St^2{Y>!!jP?FNrz*O^Cc<)kj@j=iQrNetY5VU3KGzg+c`&+=H zJt?e#{}uAsIe1-}Kv54(Uzj(KB9^L;9L6 zhlDEhR8XNws;Gl2zj5aBL!W#%ByRp7sUoV-o6aMjGtYUs2|VsvHt4`Gfzg${~3JVt%a3Ayws&dc#)QJC_f^2~m|pO82XBNO^W^RSs!9 z-kQy^sPwi8r$8N=G||9%PkNA!Tchq?zPZ1LMr?7s%uQ7}q_qD97xWnbj|l~RkR<62 z5}1~3(C!X)5@iL?o_sDeoS09c&=BTDEYur9lhG?Mv?o;;Cb=2IMA`+rQ9iYun2 zi(a7*Vb(04qxo*+AMH|Kz*RzVhQg2j=)&LpIPq``}{^Sq{nYnfa!- zQaPkIeGl4FRXL=u_Hsz5Q9lEkFiE4dRsHdwyZq1}kHM;bAJZt*gzrHw43!$|1q-rb50Zb9D`~y>BKfdl&J8 zeuzC}wW=JF$$lEurm7rLRSqfZH+P^Y|bKltO;pk>-UaY z!?sReOG-)#&7!b}ttn(UcKbUDc3mQ6BdpPTQ+fzLf!C`^0+jiw*B$4zaek?n)pE$P zyO|7zgG^O;LA9XrMaaf0Rm9`MhZSA9Q~(aEa!92MQ@;tIl?aVml|}uF`!Q-F=GX9U zXC37MO^=7NDTMB@?^($^>?_EjJ8YZ0gTCDf-9xSNE?XKZ*csQDQts#rnMM>{Ps2-Y zFV?Ys4Ma~tRSu~SP`kRgV!)QA;cnE|*X-r091@;Iut*DfzY^sdgTL#_zs5Jv6-kFH z9IFNm@?Ch3w%cpZ!3%D&3P#0u-?WaJXBK(kn{Y8^fw$xYH+1f)#Z%ntBQFSw+pRFW zLvZ6o`1r5;83t83q^ca!eQ+FEs4waRzS?u-_Khp8Du=}0`-ZhsFp*sG%HhS@csGT2 zTH)jezt`aqTU&;|j;^a;c&pgJamm~73F+NQ^$2(3)Q80+p@S{5Vep_P59XhkR|B&Nk|Kw3m^Qu4D>$orE$- zuJkgr?uBB(%xfzs#yatA==RjA91^nTcZP#`z|$*#hZadOGVq6RtpeQ9+jXRVC$q@V=>}j6~72 z0+eSdOv1i{+-NRrtedJR#jhzFjrG;$62e5`TT9MkZ8>YHuZ)XdQl36@X8pER%|Sak z=zsuUa*1B5lF)?o(9HW9agmf!30{83G z$fkjXA>~sgsK|0r18)<%dEZcK?E(rulY)=?GLk;4FHH{M63#`bm^zSD=BzE23-6n0 zugW3eo811q!%L@9rTCuafB#sOLjoiTy|KdJq$-ExSD)Y-6@a9w98y&diB(l-t}LwD z(-^PhCqYSlr>##Z0&*NEQI;|fwj3x?;Mc$ktP-3(pP~yBwK#3BzxkReP ztfiFZ&Jl&GPw-(dk2GWgF8LBI|2!Y|zfgu6*7M@0`Mlrw`2&kmxa>5Pq7CM&2uh5< z1SLw(R`?`;-6UrNuj`lSy$R1sG6t0}lI$A}_bV$zoWCgjnr9>a}1 zGN@y1Bucav%rL{#tVQz6P?EJ+b{T%;E#{V?JYFz;QR4@vCtF|Ws1!tHgl-3-9u$_U*KRK{0qphCqwL?lATmy@G>K^G&JHB99A#YpMVpqjCV5|QJ_ zNsDef%3vnkcKou02p8A}DY*U$DiXt!RZuxVLJ_NSNGx}sk6Y2DtO1A5$&&hIM>7OS zLLm^aqTHi^HNH%6m7!R(s&aQl=M6fUvh*5o#01N zCwkzdxD!$ud>aifxHAMro{(Td>>Q{sZ$IJpY8J& z+ppBlI4SW~cc4cLd56)eptWH#$5VEY6PL&uCUZQ!j+I2DupcaIC&i?&K|m5d(j}#r z6Da7U)Gk_7;O8Jt4uIelu)2^>#G-pWOPwoB+2%PJly|LwfKZ z{&SUEU}olhR1Qh!1khb{3$R_4Ly}`2uqmZZF#wE1P`Qfp*5HDu=XTM?46pRXHR(JRH*m*hp0iles-TPAt?{A|&TYWR9mK zE?SDR>Jfue*&v~f!zsmya4`uw5(nW;84z~B1f7FZIRJvj!6`-6@OcrKEIcnlN5dqf z3&E+BpTnsKgA`pu0R>@v+zeUBp(zJN63Vl*0GDzb5=9GHk+PSHLOw%`Q_Pa2Sew#` zl4u*YOBkeh8@Bl{Sd~Lk-gY&80lJlo&&|)k&9VvRRMfMm2R{ri72AF#4JI%VHF*s@+s01 zY&UFN5ue~$f;O_K%ORmhI|GmQq-?o5|MDM?&px>FjSpOZ*6+SOM1xd z687CkQA5XWvhgB1=kDF_jsPbTE#fh~J}ze=(U8 zLsw{sOJ}?`k8aq^O>xOyoDG`$qk$)uyc(CueBAWtWHL9!Wx8`X7@kkNo=oy8$uz&; zbCq8pU92b=!=UyA*8EW*tSZjf8bG@XPCF6qXCd{{Xlu5Y130TY0QlS8!qEV)84mMs zKv)nD&4Xu>3o(pL59S6qJRQw^}H-^nxzHQ^=5=V9P;3sLKRzEShom&>B@?acsvOcd&-!P<6g{5Ip38QIS#tO7v&WMc44cDz2-lvK^$xNe zO1&k|&Gs{BaTks!r$^nicRU%VgK=TJilm!K+8hsi`#Dt0fdX%; z+M9aG#n!CC%eq`pqw!Q#7UI{4e0Cg3<3?+!1h9E!@7d%yD34)@YlH@{>G^Sn&m zu4t0PH7y|5mqCpQl=384t_QAm5i)7*kH(nqYj4=K*>XjSVi75(fHZYnvb;HYx)Q+U zCa%gM!7Dsiz6b}qDu<-MGa~Jwa#5*KOB^SRgV)+w3o6X(g|k0sXCu%)wQ_`hMi!=i zVM0IWOO)0pSV71{cM@52$T--7ci1+0hjD)e@33w14(DJD-b1bOF6%K3-sBpqa!75h zO3F)T%k+%kAIELw9#gm`$#ty?o)TVHL8{6j1?n8(yCi&PT$N>x&>0GYT$MwjS|-Ia zjQ_fy(bTFOQdJJADu)F5{_*@(LxRdBb{{h}fvR#yP$NsfJB=O<>2j#bAyws&vh+Oo z|9Z-aR#$JNCwrTn?g&a(4?C)EwbB_@cM!@(2CLGt*V)zSz&KVR0G1E?n~AO_k9(AT z3f5#b&zM`(pKXOJQR+Kt=$AvMhheN7fXiyVvxHl$Qq9_GWACEoYj5S{6#eo>b@zIU zdI}@0${`_?y8)@nAu%OeZ8#-^m!D8QMUl>h`qG*zNvvJVHZ2@uuuKesTH0`>@<7Xz zB(9|%;Y4YW86NJ9(8jl}bsgs$UEyCMkD&J%rFEZ)#H#k95-*ypE}E(izNrg;QAtUx z!X)fFNJ>)|`kJyThm>~@vh8qEy2zK4RXL=6K*Isc*bm<|a*YgI+xm!KLQ2*EelzSc zqsS};MCAB!(&9Iglw(eOA;~X`s26u0ROOIhVqJ>3lFT@~{+Im7Q4A+n5=kb$g%lHd zC5gV7?gaATu-4c&kdze8d?y)|j~y6|YA>psXN9jHZ8|srz%LIVxgd~0BzH9gNlWWX ztz}aQPWD)R^of8P#vAB*W67p_3zINNqGWF&NTTFPg;vIc61^2a?xz$W!B6`s+mme; zea+M_-PibM{gf}sG6{avALHeH($6*C_wd8mR^^bYa!8a7mD0C?>R08E43Gq>Pq4C4 z5^V%1@%=`UiEV)|(elspVgCzNpWs_);nRFd;uilfpKm$E&+<732}+cntq4k0Iixo1 zuatPL06EY~*_{fl43H@4S`m;a<(dTxp@Ecn$u>*Pb9u9Dvw<4Hgx4k!uvO%x@q}^Q6)@#yqLa?jKz+0%>I47(N7oOk~@s z$|0#@vtR~pEUG-k1wg9GA!P_ilmx8+InYWe&#kSGCK5s)bTnS}wdpeYNIZ5C^S zG631;ARw{YBym8B^6~-6B|V#%@+{)Eqnu5X)18$kJ1gRW7(L1aRz~PHW=eEcBo`Yf zAF4wD5+_IbitoRGDsfrEL{82aX;enekBA&UE?*IlD1(`B+wsd1BE&}P(wdc7NL&EH z%K;Kn*aAL+!T}Ox$SVUR{-jBO#A!o-6lf1sG3QG-3P^lO7@|-*Yd-UmEV?aRiG{>_ zR&?9()|lIlQkNMkDk`6}kSJ|g0g!CcG8Ub!${|4(CH-nYgByx7snl$<)m&?ugW3u(3KjrsvJ_E7jt2% zg5{IY&yK?sf;y?F9K|pvz$YOmz~Raveb;kUZh@JZcTzbdofE*T13|GaCQ{L{C>NMS z#FFJB2?aEfb5W^GY0Qf#qzM|}QSyu!gvv1wxEa-B?5z1PT9reB4K@7A2F1tJ1O|v0 z2vUR$iHk|F$!8sCXM8A1hID=sZ1V14tG3aE{!*lj2!u{itc(akwjjB#3v`F&=h=f& z+f5^Xjf2O`@S(vZ*i|_syOO!FCUiimTA0l3>2YE*0MY3tHWAK~$Q(~eT(lHr)g$J- zlnv5;6V{@O6QQ06IuZw^RwjrYFhNHaW4f`@ajEVFjk6i6a!A9pZM_j<`Fm4+WiTja zsc1)fvxtR`NZ(yyM7d!FU{E|o!XU-lP@lXc*y~j}Bv-B^gp>@tC@QDMv0zbBTrSYg z_)ug{t?x^Hv8pLHhpizHY%ZlsPG(IYwbsSwh(ibjDMDAEEAsOmX!v;`NKrcE7yv;P zM9P-hcGC)iU{wyu4i85n+7w{xr(hfobePNmR-9NU9tGrlo2!bvuSfD#BKOY2LkOnyhK(H!@)ME#3+RfdrDd#SXZm<RY{R2*mT%4O=bhR*B$`)Ock_I&mOW4 zEF^#*XJ-Sr<@VdQvi1(9`d2q$%Koj{gKl<092zPYHc|C2n=Wfz9dc3Yl}<_@+_9@NKqI?Bi2+0A$7 zX7x189y{LxNP-`Bqg9sVavwd$JFw=5aPU$mSdMg^l{6sl%nKahbmnDxpA^I_RVAvJJM& zBpXPZ<9~kh*$3Zt2bklp_HCe_0zQB6_^CsdLo$43Zg@47Lwehb(3YyoA$_%%LqbjX z3~0h6jnZX|-gd|3hd%vju&RHQX%uS0i;&NE+@MP#VM(O&a!AC7${{I@deKCy`U@(D zeQ>|sk?nMw-5yjzz_6{CWPdlqT9lAN(biPJMXjEu6Rw%!3F^_EwBPMPIfWY5 z#m(UDI4pm3Jo$Dsr|(Q#=hV+e|uUrU;c@TPb=Sveb|%{6@g7-n>HGwpygM9sEwbZ#cI55MXyv$&tTQZn5EbSSHN z6=h&&-@BMVU7fDN2$MzARM&Ay2F+z#X~$jHfw$TyZFl#_00niO&(kG^giCNyr{9I$ z$)+--@(zhj8tTNf>1XfRALrf9B~{5CT)dkJHq-#U)INutROuYm>pfQ@2x=39Z}>nJ zItlj0NU^8y1J|8`LY*bi=_V)rKMiaQh>m3uu$e1MD|@1#ZTQ?>!p{5`+!jCem)E% zgZ~1UR$2XARkJ+Ft8z#Ngd6o(#V2`+brWX>aJP7I|+3AT8K z@n82dnp%}Zs>&f%<&bO{5%#K{KKX~=-GsGMl|!n^A!*YUbwQc1+5=IQLxS3byl80= zisFeh^qaYAc}d^Rgsg%Z%<7ggJby6E=QevO)iu$tfS7Zjt$tw3iP)Ms_d6r5rV9?6 zjp0!`F)TZ6JF~NCqi$UfMi5&r#TGIWy*6+C((AsaoBDc!=@z87!*a1lS?2*t^loMt z7P27e{j9M033Aseh0s}$lo-W{HVg`xlGO9aKA!A#3Jr_lDn`fd%FMjBb5WH;QsEwW z6|yylF`uNSykcr9!hW#(YF!8^U4&XbIs$=jj@NWN1J?=p<~rSrIuIqM$@`#KKWMnO znXF(D??OB(Bm-CirhE`B%s^=&X5s0QV2N)NHtHK@C0~Fy-)kyHQwY_lH`1$eNNO~h zqqi!Dgxcx`q$-ERsu@+~kha|$d&kt7LwhQ>&3kF$nsMPBtDCn{xAi-%w(^lEK4CAa z#i&0a+Zhd^+&g^ZM7bZ}14Ib@yN|ahBT+QB4y7pzld$g~DNSAIYf4H|S7_xtT#CA4 zPMyHJz&l5i3ME|Tv~!FgGQ z&)7^&kYM@%N=Y|4AzgVoI+#4~M9dFgqQp>~(0v9+spyJ)DOT!fp3j#EiLV+kJ;Px` z8hCPz3==tiF;aT8CnM)aM2;Vy&*C?dyzUW86TmNv?-|7maK#f;BnHaL6i-mOZzFk^ z1%*B0AEnRg%f5@`Pnz^qB&Rhec(Gh~--L-EKPsdYaqdIMvIKFCs9-I4N+OL4A(@!a z(wsP;=&D_cT)02VJ@ySGADcif9JV0*?;mYKxWdXoI9Wr0{iKPbhUy*mU+lrPDnIvg*9JI2xyXMq4D@zib;6T8Jg ziC57n=15JO&BbO@Z}L7Sh|%NI6DM?=bC#uRIzQO|2TTQ%jSg>$pv1>Xq7BWzx8F!Iv6RJxUI{3%o_jtOf41`L5b3{6+x*ghg6kAO5ug`Z4Zzr(ONLW3{SHb0VGPY76l|$9i|9K9K_=| z151apj};Myt)kuGK{f*N)@r(XI6!h)?uvYAr=<;_k#lSy5jX zDhNo7R~aC=oQY+C#1vj4O;kxFnFJ&@_u2{)bFzfLY%PzVmBe}$03_BLvyv#kBg_|q zR#84PvJ#NM8m`JAjj~og99?2>u`}zYDu+}6B$r((2uM8TSp<+MXHyK2C_7ciZC6&d zNUUGV`BVfBB6K@Y8DFu1sso-%wOr&&gpMyK=Kw+?AR4wdmNu5ke0z>xjFcV?{9+F! zBFB%D76FMemz!F6IBvPCN_L7*_8lN;r=N1C?L69iy=Vb{Cz-LU0GF?ISLk%O{I=$-A$9ZsvJ^P4vC_5;mlptlqy!ch#9xjj8jEP+WuWa!aE=6FtmQwo|XO`wyRY>?2#;gsS;$UxAM zI0$dbfUr^vItQn600fPLQ;MqL^CB==cwU5#hDk^lf>S9!hf@y*DY}LN3c~of8M4r8 zA97G6p;-RjRF_8CP_&?1Q;b8vu_@YPmL$d6lui_#n}UmkL5jEGy*><9<&Yc>gpiUs zL922|s6XO;M-K!kN{5sI5ae+>Bng5bMe2|wY*R`Tpe$O4JPsMJD<^e38%69wH^?p8 z#YD$qK6bTLk5D5#3@KE=o|>5uvIxE&o?i%q67rJslFmhFu+3q zt!$53f?uX`Na)dSgGYN(wp^Wm`O}|1`{3u^^1$_vU-Gl%z61EI|LT#c@1L4de{{Co zsaH=;otipz?bOuEr>3sLKSyr((1$NQ;hm%9BY*LX2WD=3?IYg>7q5Ek^gSFPwkA2_#XOiFiw$67f{=sXW{73J8Xzd>zoW1d%{N~Dk%vyd|69-g?KR(W(~0lv|NN&u z`m-PSowxUQ{z0en@a#YS-amQ`3ix{<;G~Y4nfO)c>>Ht@KR_Kt0pIcbcLAU6+T_gK zI*;<_-lAe1kLP)mfA+gEk23Q4mxtiPH0-TX!;sI)WA5ReVR42(?w|NVuKX=%Ie@ID0qvgHxA literal 0 HcmV?d00001 diff --git a/testing/btest/Traces/spicy/raw-layer.pcap b/testing/btest/Traces/spicy/raw-layer.pcap new file mode 100644 index 0000000000000000000000000000000000000000..5e4ed891dca385beeec0a58c85689b3790b843ed GIT binary patch literal 161 zcmca|c+)~A1{MYw`2U}Qff2|FzvUmFQN+oh1!RNpe=s-^UADJ;H3x$$1A`1mnFHg? z9c#p{*&JZ-yTZZn9jF(gv+p4%gFTQ9!pJ&1wt6Zg<|?G-B_|dXm*ymvWagzScm^OD L_YG>?G?;M!%da;u+n~)4lqGm$~;fapeBKYA<*ozDG zJqQFA9-$lt4h{kXgN(}4*8UcJ4@7>92M19DjzRl{`3wISG6W(8VMmny2t@_ZA%x9_ zOGqT*0QyoUE@9?-q5mKQsLe_p-XJn?71$>1a3v6Z20*_F0E6}~-uxz1P~JsXcrO80 zkO;)SxY$bt6!^vW9f%Gh3Zo%Hi~umu{>4|h9~CEI7rTZ&HI)9lztGnK>F>7vlm2H> z5YZAH5uy(;2kl>isF=_|5##=fNPKbe)%Y?ZNS~LJ`ycFuIe7s7;FZD#ks)%xML_$P zQ2hNkP@(Ft2&DrEbrFb-kiQNj94I{?)D9@r%-tt%K-6^rH47F3wX6JL<4lA={#}As z_+J14)d>Xn(|cXwzXK$XArihf{Ba$%YtX9tBhSa<8rF#bUvUdtD=S+|4jW4c2S-yU z4r^Cu4;KzgZ*v<{Cu>U%HycwP_P>3E2l^HOfaT_9!{O>?$`1avaB~Cx6lQAa#=|Sj zVQyy5&Szoi20buyaQ3pY1wI4*)!fw0{r4wKUCphWJzUwrFNyF%-v$nO1q7jouC{hg z7N(XC;E9xjkDIBxvn!{Atp}%@CG;EMW!(S!tg9>!6V51>y1CBQqDlB0IG7DJ3IdOI~*c{zfJ%o;27}ne+?ML$=S)0-Pgg^ z47kdr6X5*v3(VM0W5|$?K*oZ0wGLp*`*FcKa2ES(9RLmE)2lyqunX1!kg*TY@a>Mz zdddr)BmT@Z7)5(?nhhY|{aZTzf5}+>|Ci1G{}tB%TGIWm0bfesODC68`9EhG>^y{E zRxblteG~W_Xjl6%gx^FMI*!C&?!ywsOGsqH;K;Orcc~BWf{0Qg=+})Gc+l~p_Hvz& z4WH;p7zbh_jxGS>@vrfsvmdMz1c}Xg5cL&+dd+wz0_FMOuRMYAqG9?E&!T3f&VImG zd_-CZbdm&4l&Ap%U*MIFAXwM&$A|K-`3rw9kpFztZ~k4O0RJMZ_NY}5)f^iO0+4{( z)$Rf5?k9kvQvVf|7;!X!{10jbhzbD3AyWZAgCLVXMA5=UT-Py4UM7FY>gB2qE0@mu+2I;OHRt zB_j4*=*?5cS7@3kCJv7J$w0EBCOa2w#a~RKxKJU`9h|Hn7U4-NVWf!45P;BdLAZNL zw-e|GkbnU@BJ}0A?D-7zHU;qZnyeCqMl1G8w7iy($Q{9~TJ8L2w7McdM0Zv!$N+E+ z(EgRD?IvPSMA*L~0`dR+?J^>O@f497_!)@9z^Yg&T5^Hlg_BRlm;1iB#q1NchwB>0 z>QQU*idc}XlAAygDQ0_G7QAuG0RzerHy0Yb;3r}Vh+1LOnZRahl;@X2ijUJD%uBA_W2&@)YBDhIA|c$`F`(6dC|2 z9x$Z;Ok#jT7&_ob5Tt_>A5O+FvN6Y@3G4pCwu!A#%^=CPw8EAJdXajCfWtG|0hp=M zq&UsNHMu>(w6&k&@Unbyw?FR`+=_pEX!|9A?}qEaT9$9`M-e*}^6rR`SvTjU-2~~{ zn9$>e_SUR&T<$J6{ZEJaPBpoOtbCae_GZk;`L>dvwvx{B$n)yN0LIBkjxEyHz;%#9f0p)Fcg*OpY34r&D`G4~6@vwibY#~AuY&0~J z^k9|$i;Az9*yfv(ufiuJxnKF(sOg#;alR{vJWFf5J?6BbVeAbuz{Eqn&M#u9Ulo7x zOCI!Vy5zTCy=-P(ASy0^>It|6+SNV)vFIm(qCWa7D$oa1axbI$1Uo8j)km*; z8Xs*)!9|TEKNvkb`B{%|H_#?V<&!0n%DUah;S{!hs;~31)HjNu!ujGJ$if(4aZLx8 zgsK~LMO}G7U1$ei(EF#ly=)-jF##$h7;p)+f5pq)L<)+i`d36CUVPz~5modOU`y)z z-;z|wZ)?PdHAiIBqdAW@=9Pwbb6PrkV(orP^~+ef*z-si;Vs62O>q@yG%kG`&#}8I zA7VrMvIAr!frnV&xhG`(u0%jlhvOP zPgya8<-LXE0?*@oC zh>Zcs0{jN;Uy+D2xd}yN`zs<43A*#kh@+LaI!;#%R$O>r46&ePFDA>TE(zta;2`wg zJ{SGT?mjVS|27>>+GPVFuhe?u@uJRFK*4>bv(=P>ofmzS++WaRW06T-pu%!hzkV({ zql~P@DD;`V-tcn+9k)iV$~-H9-cARPdEZH2{$U7Hbek*LHtLsxJ?S zREa)!uT!@fucFKw;)LH;+e19%!^^4J5u^p9awEK&5UjC&?- zM$v;TNHc%Ulfh{GOE-R#TTtDeT+t0!greL4Bk{fNpStzsfrtjcKzL0gWTA-6e?KNv)@?|`= zD~n%?F>U74md-2MndPtuu2D|I%7_opsx-!tryG1Is~g_=KHF_kj_y_WmVRWzj{{^} z6aaT!&5%PQVR|JJ8z(wa@?a$H%lwW+zXceHgZ%Zd8Wqu48c9WS`eO%KU4MeGguv5} zF2T)rNMAi&OzB=5SVLRG(vY)w=B3SXuf6*AhQe(w!i*+o*o|+L+nZs6q^f$M>@l&e znwFVG=g<5eQGqOR1Fu~!l!NLTc3D>`pnFk4N6A^$Kf1neR_eC^5$S?}Q4$syB>}q{ zd)Qkh@*rZD=U)+lNO>$@MudBeySIEBZAdUYrdeEal4#`C@Hq2J6bynmWQ{48q7mxt z!-?tOGSp{xKBClj{Je`~@MB$7-|F+Gdi#w{e>(2lc&kTB=;ou1j9*#j5m4U6~y@&=7vzSX!=ec9&qHoR8M9p!O^`la^U=YGSd$Zu_ib{jroSNv+f6&g5rqrWasYVsQ_@O6`J;X6`!jEk0-7J8ZCuwyn0kz+J@a zCGsv}P*m|UJ?)*6b)|K%u#me4n?!5m%pywB*eYH<5SpS%siK7PFnT{^ZHJCDW?s6` zb+k1%DX_0YG(>8>d`9EZ3u8NuCJ~q>0~bFBrj3U%#m?uY@4oqPzlbUbu7w+AG5vaP zbeC=`M5DW9q+_Le>eC7?Q3=n&z6&y{lNP$)ZT^`UPBy~$cenZ%F;_F*;qaYvhi8|ecq5@_~ zH}3>uBO`y+{asmKWPr+gpWQ)24x%a?BSIK~oCWRb7zc~rPYI$r_x%->7#Z0v^AD;# zh&nP^%|9{A|7|72l}nlfH-iEmz56TQoKfc-)?{T^XZTY{lcOw7L){|n&mf#6%?~?N z?0XLsqvG89&X@#=F&;oBXQBcr9`rDKMn5SSlF@BCAyVCxD3Kv5XtL)lPg;D3_t9#* zAjC5HBBE%oBPqoMSzb-*%NPf)CI&9O-aw$tL?Xny;t|{m=d0=>SiWIeyoq#xwQvqA2*&CIOfUVX{k{ew9eYLz;G%N zmr9l`jbII}#NkbOQc4Q#RjS@|n4r{{OwYU%7k*2iCK2&_WMWz&f^!&(Dn+drmwZ`> z`}CL|DQ54W<(upn{fZ$UW2$&wl%XDPSXZe!Ra>*l! zu8Bhf4V3wI!vZp7ewJ)yl>mVsJt<(;WCB@p6G#})u68?MRY(EaKObN2b~o=VA<cE9HpKlYlRxzQGM25a&sBJRl&r` zo5G+0=PQ0N!79B+w965xNW@B$Avs%S=H1dO=V-x?#^(1?x ziHaGi6}(27E9pC{+s~c6Otc`ZVlOIESf@2TewtGA+uP_AZxZDqhjt3WzDLN# zL`M|dvES3j+!C}TPeHeA(Zoc$icj#5U=|Wr~g;eVzkS(GdvVgfV6GUp?c*rH7 zKc-Er_PKf(p{X(Fdk4)lMz^gC?{}EkyQ=)=tU(peJF*QqxiN$MF(_Y^oB7F5+G3LR zMLl;#tWR>)yPeZvp)R62U_I(xrq2V}kguu{W{ZeByDJ#<#f9ztD9=99lTMaEC+t<{ z&Ek~|T5E}-+xnWqiBB=|HuTlilxQh7^$#mL-5DNp!A@smmQYPNR(`H(V20KDOgU+G z;A6sUsWzCfjiDA>zij-O757O*VOqFF1dSA#+5H^<;^mc3kdKzf?x=0_$E|G~xT*rh z^5pnkW@Nbem_7^AuetM|j}Oh4Fqyq7|7hPM|1FZ40ygAqzF|++r}r%r0X3=JHkgqU z11Q(^TPkS3ExOzpX(&!~ZY2g{qX_o{+hM=fDF@)d&iE4JJ`}ZJ3IzfrE7Y#`TZnc) zH5B#PUr~usgq{A}y$v9Ms2n}0eYAZG9nXZ zVpl0?64Mh3NyR?Yg+~%>NWO`YjR^AfnD;x(+&@r!leU$pk0D5^gAMJ*?EDe2$9rco z3h`ME9|5JVWdZ@nh(Q_Sy8B`psIm{PC<}~_w?+a1W#>o##A^{$*8Jmai-nV_!UGze z5Oa}-X{s4J7`U76b^fU$;(k=QC0@AX$@WbhxH8%$(MjL80J01L1yf_Al! z0i975s>_wtl3OEy($M?jQPe-Wlr}33EP{xGM;H)TpeurQ6%l6NL<`y(Yp*E%3lUJd zX67>DfC}3WHUB$)?dd#9m zhK((G^S;uQWji?X2QdVB8Nqm?c^w%xvCsHBN7360d1zSVG_INwDTJMraaoi-T1SJ^ ztwu-pzK4KKFxVrcE&Q0GFo{s%MUZ9(?B}&4v@u`zW6{TElupZ{>LG5tFMmqDU@Y+X zM5c=0tq>iUZTBub+*&BAZE6xFY|LP^w+L^Sw!}Eu@a2T5)p1F+*3rEumh3J|FQj@O zd1ch2jwz5;R6g)@ng4QAXt|Aj6Xr|?7LWO3*J3yAv^Ym_!$~cV%bOQ$NO`2_7-UPE zYKdewh^S@TtSV+WW~cd{q1(n}zv+o9#jY!@bjr4L>kHkB_@=0Idq?6f&!B9am5=>~ zZU(|@iun)oB@Ay!T|^9x^^Iuq%qP!J9b#*i(!7lV>n<|y1UvrheUEnG$9L!MP+Lyu z?u*=hC)=XNsSi{sFDUe%H_;E&=m#@aV&?hXF%%v*R1WWA|9T-|@SrfsiAgLCC5DjzT-J|AO1N`Yhm z?Mg;qLvl>C!HnqsaXBO8z-o2`_r`ax2I2GUwposa~+Sw zMvAV3s5jA=@K(eN+hejgbc5lKZ@fnquFi1Z-4HSBMm2o8vg4`-1N)H&tRM?< zfW`mN0kDA%pabmyHkYF#j|4;~4V)VdJ@{w64&j4{hxx$DKOC^~57^Z)0}j5Q9`ws; z?22D$P_+QRW`Rd={^J(`=$F+gr#ozRPux6z((`#d?E6Ahf|`RuL621_NngQFVCJNj z1>VH;Z*4)(QyFSFS8VL0`-bL9^MnzdSG|6evFOK4#CzStx1*m@wuG`HU4u`i;bNx; zktw968}i`b{ZZ^AnfcckYhnN86Zx#ze3@O29?} zx3pJdRqV~Of=JTO_>f_dmWr4VTkU7l}`&q#2fUnI#xJU4o10R4*$>9+4unvYRHHS9|V3i1_&c#c!WNr)4th zQ#zs0i}Rg|>3n9v1v8EDat%>DX>IzgkgaI`GkIy{r8ye%%!$gHM}evKa}VD7*bz=^ zw%vWSYlDIGOq4$VHQgeD&wS#2WRu&Rs2jL(C>OZ4X?OTDA5VKHMfdyQe@yAm7TC^2 ztv0Vs8Wt7k33&D3K#%CwM+UL?B*?bA$@_^aA34Yb<-6Y`-jQ#bX>(?@5IBJ7v7$*o zLVw%hL%S7{sL7zi^JL%DNcJw$z1yOai>>n#&$NGV>topD()gO-ZYbDfqnP2kAa9Y? zw#~B0VW@obFMH7YI!B$52+a}=txQH=5k11pE?f_Xh^nq1z1oS<7dFi9Q{}+WX~GQ+ zs$d=hmrmyPl34`Y0DOPL1xNSKCNl5cz5ObN(4LAaQfJ#(YTov)l5$9lP_9Pg%}R;Z z`1Wv9%h{jf^;3$S_J;EfMA0=`gN8yjuuACPTw8a#(7Ln!ciquLN8T@mm+FoH6F_9_ zL%MD)ivikC^e)#O{jV+s)SZ{z@4A~30_%=WJ96-G2!GrYwRh#_9gNwTZ<>We`|ork zJDN7&X?_tJKJZ9ogPcoJxgKlXjqlrTym*F#(0^xbpYKO&IIFpZ;ab{#v-meQuZ?dU z)Ca#ClRtX^WoZYnyr$piLH)M;i{A}kzvi@KJf(@}S(T5`P z{S^@yUtqrdL97801M4|%6*J(<-o*Lokj<+!R5Ev9@z!jvBlLAT%8U*hdv^fCUR2Nl z*8H8Or?fU{{{g z`<;Caa#WT4GHv4mZ(nLtVikWr{)ksTbHiq*$X*cj>mwh>oo>@iC58IjKC=P~Me*)eEV`lVJU_ZW{} z3Y9e7Qp+3HD12u6k`>m+LhnKtHCxyMgDg&p9Q;&_mD;f?#++dWBnc_R6c-)hWX9l; z@_d@9q(^9*)O&{K~!tRS6w`YIVlb%=zvh)U6UgNhB)Nhl2`^^Bo3u=YE>^C-m z$gh3vz5@%?Z}-c7GXVD-^46f=Y3;w`IY$9xNo~z^YH~-S`n#li?>XU8m<Wl;OsGk+;m(~@(z|IHtOD*J&UpGL%dTv-yu#zU}NVmqV zXzzFE+3Zirrjsz8_T1h?I4JC0FgzxDT|KGQcK@N;KmW_11SmVNqo`AiN6g)(?ET{bL6t7@7FCU1qwKVkzv&_UUG-2A z60~elM;M)BGnx2Tnlb)%+aLqC>1&l`hbnvjin3sr`~)hS`Sp*o;UJ6RY7K+<$zWur zuFUtZ6meb`R-lQdT!=SdpwLy#+6j5JysJ}W$jOmp^vm0rTQ7`NCu=AHS=0h7uBjRe zs1A3p=y0@z#7F_o7aF54=@1Sg-Zw+OE@L>Lez{!si;)8G>j?CVknOi$3ml+dkADtP zWsdP7)W{+@v8QYknds<`woR;AO1#S~HHkqgr7R)Ms3PPndlgrTBj?HN&U&I#1hNj3+VcP5A!5ecsUEIgX%tloCI=1O7Me@Yb z(M_0cTWdJrD|#^26@G9!e~Oxt;yXzARpw0XOG2AnMh#u`iRjiBCO@RfoX;K~tqP`W zN`sc}PC3cj*~-P3RAaV%7ku&^sq~D+Q~y(1D5ZR`e-8Xur{_DB;aeTl^2ROWnIBn$VRlP5^6t4< z??XnT<2}|P3m+`jM~%D>BUnOoa2t5iNWPGD7dP~{E6Aa+kP{TDDSBsrkbA(sFF_J- zSo3^x7=#2tesyc3Abo17W(-2FDEy8y~Ve3(A zgU|&cC5O4>PA;6_efJ^RVn7;M-~XhkX$b(KXpj#_I?+~SV_G0D-L?i#o&{H zo8dGGg051dcc|YbrK(S3b~fiSYhd+@tqNh4!yU?L4CQCIIs~X+WJ&3WC|lkO5!`X# ze~M~rAU1W;B=~(X57tLG!7*BwixE9Y_zu++-A_el6jhbpH8UKp#Hi2J^=No7bvltu z&y;R&2TN|D6W(~a1Zt&2pAH z)sgZc8%c$XhWRX3hI(grVfH?H5ICpRX&!!YS?wr}z%WK0d_inAu+bPq>DF&fIIZD4a zq0{or+u^Kl5e`v?%;ld4dJC-e&=gPfEKuj9iOg>1aHlJQ~|UPPus|kODt;BQc@VIa zFpHHp96w~nkMJ5Hens=#ZzcYb;QNd=e)i8eBH53L6$HjJys_Z7ImV_lh4ODo-W`3N z`Q!b{u@C0N#Tz{4i5@R~!$y81Y&Ozhq6+bYrO8O|kry;>NR%z-&-2XN6_v@h^iEjI zMrGfpCvtVUzb5Q+}`ot;i)*vohF+K}N%O9tLl^ejw44@9+o zBw85!#DyLsB)B_5ACfaV16;uI#QFN}&jaoL`JBZdyj;-!+#XalOYDX|;|l|IYOJ(~IX{ z-!49F7_h^AxKk}8oQ5WdvEzX!k;07AUqXx;BehNd+j5knu0y7YvU3B4lv2it{Wb+t z#n!$Q|EIA;ezmdk4@Y|4Tu7N3ELr94*!Y{@F=vBU5(kWqdE<>(>pUo=aha%`MG#y? zpL}1AUs-)fful;A;KtHPV;TRfGF9;7dyeIg(NAY> zljUDWM=~{+>|} zS8SBIZ)*VOUgwV;zui8f-3vA~MuO^n?1$%01@ca)F5K_s~B)QG<)OL`=B6OQg@ zKGZjFsd@EWrcR(N40G+VEl#qG_^g8`g&WeX2NC2(2YpgZvMfk92Y{VSoO`+`3?vBf z!TABs*?Uga;!Tp~4O09rI@%pRRo@$L9qxToRV3CLlRS53v|gw${l50S$*@n?LhOhn z^|N)C%xd9b)~T4_+xBes!e_^eoA-wc9`frb-hF%kJHjr|yJ0&uIve+~ogG8LR0ww2id$ek<~ z5?|mMBvC%5D+{7wXF6U`EOX_0UTnbd2mD;Oq>1pQwtkCN^e%;o&~rw`EGT3M>4i)+ z6Q2`*m4U*kVTf{m!h#(=dJo)AkR0iD0`bB-T|_tbWtoqqG#tXOy!ptheaIN~hi^1T z5QF3LpRsEsoXL<>}xr?Ix$_Lj;R~ZY-rIgfSnsro9oauVM%0rh|vk6$z zMyF-3Yg-`<)#cW!^GEMv*cN!(*`8w$8WJf7&C8+VV0k|!3X0#rZWDNqdLTtle?IS> zxA)n1WUp6G%cH%ye5P_{AI-*qp;1sw*DG))M>=DLg0#7LeDxhk$#m>ob_7;b;I3I1 z=}x%iI(5X(d>TBAt@2S)w+_5ITc`gZ!sM9$>O-00d6lgk(oi^llj_v>D~Cnh=Y+a7 zI4tYr-rB1$rlCC>rq2*^9j4uTP42f74#igRCU4yt;QAnj8=&3y$kYmzTGR<9EUuOd zze4|QG%MKs=YZ~i&HafD^ln1u-**#yMuFJJ->Pu`++Ej$^M4RAlmg?ry9r+Cc(QkS zJmE9?J)W$5{5_s5nt^u{+k9fvw-Hpbb@x4QE#Bh|Db+5*u4bw|AC$VIBEkD)ZYy~# zVwWPFk2qL#LqD`ytA2=Odv-nq~Kx>t07;+i};;-PTlA=5X2C!&7U^KMX6w zhx0QP5vwoREKIMdR+M=9lW3&L(kRFKpu;it0Qc=E|1OWzjeD8q9pPz_Mn$h~j2pCb zwy#@v8sp9E6}#P*IaD|InLlM|Iku}rZHfA>fX~N^LUMcz^9C}0$|8}IFkWGnMNRTAs9M?t`>UXRtpAA@yTnAiKH^F8vXp%#_SGzKs0{IPWt zjal|5U)7`2fafG3{4RVyt)fl^c&MMfM(j`i8mf$3?YKTwaB50jz0K*XjI?TnWVQTp z@=Tmhq;7CVLs#A6fDC0s)$4;0x}rpp_U?s-^Lg0Z@E$}T~PQxH7Sri?-p{9 zw@GoXFjvOqedxcgQRK$FFeyvF;dFi$uUeJK0EraC_VFl$KYHi3FbyBr9<1cDGo*pc>+x3yf~a?F8fY}Hig!Mr^|95 z9(pk)xcX_a+&iaZ_x&l9sb1R7n>suy)D_%PIV_%gGH=GHOV#V#D4#YLQrZA>>qLg% z`0Q~EKBIt!o#n$%VOnob^;F}08pgkjuYKme7w@(ozusE?-j8mbaoI&Q`*EF+o`ZN8 zA$Q3kaS$!e!n5L;6$TRq1=%t@Eb1!4u28Z9ma>-n@m;puj@F1z^OM>w-_$&y%rN@o zLjC2-PaLe;^{EIJr1tPt&w(<%+qEn_N$bag{%DV66!|{UU@=8uYGcv|FS(7xCO|DY@eC@SICv(HOHjPV;&JWtY+MM+ivThZhhOy1|Y91v4 znuhL4%8^bQgUEpd&C~}ET43Yk@5ghFBo%)qu9K}*3)Yt(@7E<{{Vu(2Ia zt$F`AZ<1P(MWsrGP%^4i#9$mX67FpeU;h1ZH;*~_u4e`qIh9WD$yCDI?(R#at@^ut zKJF2ULy|+L3e+U4chx>1hol$LyoiJT@cfeTa%aevTpmKDMK! z-)pb0*3B6cnh_G6$$?nSd`t$MxY}|RuAh}pB~i)|ZB$cp^-EvnE9Tp=w+DVG;w|XtA}7}7mTOxoh+ z+~-3&<~$y(1k>6xPaN^ra$cF*iMVEOOE_wg*L@|oUBYp1=$Sld8N*&lOvWl7{@V2P zr=0$C#)yCdvYYm+OOa~l8toAd%XfFINR-w`8yE2389xfR*5`-JHhjhsm!z8+`A z#f`&R$#@(ghhrl00Tki=i(P^_ti-k|>~klKk(JT)Nwq7z*x@Q+ zHSQ1BZfvz9>6ho-C1s3r;d}a#_edW8*+81_<7iT_`$q%apA!}WwX17Mu;l%GVE1<} zz1;mp5Pq)`PJFrC{UcKsR(EqOw z)EDC(tuo&;7o5kPQ!&CPWc8-f_H7r+8;om>g}LmOiyAERHWT~dDQE0P*N!_iR1DZ( zI=l~Fwv1h%{bWNqAzC1fb$Ezd`H}!$`7Hc-ppDt;H%wRD`Nb%8ggdl(O)Rx7&%V>f zqrJb&{zYpoAobj)zyYOz%<&z=iJ3gn?1#q2hlTWykMB$6r#Gt!Yg4HS+{oG(-f&^H zH%Vv?;A_G`$1x(J$t|_o5QSuI7NapYO`DF=?6VZO-B`biT68Ofiq`?U0>ih;g8NR8R*rtF|vaL~ivNgQ17 z;gVzA!w7qcjuH`-Pa+@e!?L*Zx!5#UpIBQxjL3yqit6UG|*64CE z*>Pf_BX%|yP0p-sK*`i-g+4rzKrWtRw5Ll|?Is4Z-}zym((|kXT?6P&$_bD?*UUz2 zp|cU~f6qolf!(3A%tD!e&PEo^aDaSde|_z~n>`ef_OFP*?oh$wKZp&@N{ePewTvGz z3R!k7*-!|c+YQ-|ocV;b&pc`^-&Wv|fwHy!3hZ^23ZuXYeVw-xQDnpi&k3?H0a#q4tOHaBjmtVHFfAd8L3f98)&Hr( zay*C#*MbNECTmc;TIrDTegUXo#FzaN19pdg`4x!zyV92nK)*Qjf|FyPVvher2*6fV zEmo^qtlAIAd0lIfYb_$Z9bLd$)F5hjf|B$A0^_4anjVnL$$r$$X0=_sq)i2^BP1@bHjH{~4h zaRu6>%uUGJy@%4g-X%Q-y>bG)x<)5Qs7~sC(J2jFUC;UN|M~&>vRnWnzMn&cm;pM0 z_OG2^Z4)ObBG+FLfeaM;?-SKf#Pw8Tm>2i8Vs7g%4GW1=w`b4I<~$;%S03gJoy&6c z^8&A^1^h?RH9j~)uc&tAit<3L(*kh-ACfP{8tQ{b;kD1q3qoU^cqP_p&{zXI1WB>K zW4&Sr#=7mv9Xyby@AarcP}GFKpelgR;v@mv>A#-E`Gs1NB99EB_Po88t1uKb{;#Mw r0P6kAsB?L_1R&}WfO?IlE>KOc{1>x=0-$L+kQe9Kmn=~Kc1bQg`aS$X`b{fg`l~VxEJ6Amot#}&2rXOa%nYo3f0hcyVKjT7t`hAyx^c$$a^uz3TbNBG{;sO9Zx+?hq literal 0 HcmV?d00001 diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index d1714544dc..1247fb44ce 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -18,7 +18,7 @@ ZEEK_PLUGIN_PATH= TZ=UTC LC_ALL=C BTEST_PATH=%(testbase)s/../../auxil/btest -PATH=%(testbase)s/../../%(build_dir)s/src%(pathsep)s%(testbase)s/../scripts%(pathsep)s%(testbase)s/../../auxil/btest%(pathsep)s%(testbase)s/../../%(build_dir)s/auxil/zeek-aux/zeek-cut%(pathsep)s%(testbase)s/../../auxil/btest/sphinx%(pathsep)s%(default_path)s%(pathsep)s/sbin +PATH=%(testbase)s/../../%(build_dir)s/src%(pathsep)s%(testbase)s/../scripts%(pathsep)s%(testbase)s/../../auxil/btest%(pathsep)s%(testbase)s/../../%(build_dir)s/auxil/zeek-aux/zeek-cut%(pathsep)s%(testbase)s/../../%(build_dir)s/auxil/spicy/spicy/bin%(pathsep)s%(testbase)s/../../%(build_dir)s/src/spicy/spicyz%(pathsep)s%(testbase)s/../../auxil/btest/sphinx%(pathsep)s%(default_path)s%(pathsep)s/sbin TRACES=%(testbase)s/Traces FILES=%(testbase)s/Files SCRIPTS=%(testbase)s/../scripts @@ -39,6 +39,8 @@ ZEEK_SUPERVISOR_NO_SIGKILL=1 UBSAN_OPTIONS=print_stacktrace=1 SPICY_PATH=`bash -c %(testbase)s/../../%(build_dir)s/spicy-path` HILTI_CXX_INCLUDE_DIRS=`bash -c %(testbase)s/../../%(build_dir)s/hilti-cxx-include-dirs` +ZEEK_SPICY_MODULE_PATH=/does/not/exist +ZEEK_SPICY_LIBRARY_PATH=%(testbase)s/../../scripts/spicy [environment-AST-dup] # Environment for testing AST duplication functionality, which is diff --git a/testing/btest/spicy/analyzer-tag.zeek b/testing/btest/spicy/analyzer-tag.zeek new file mode 100644 index 0000000000..9b15aabc31 --- /dev/null +++ b/testing/btest/spicy/analyzer-tag.zeek @@ -0,0 +1,33 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o ssh.hlto ssh.spicy ./ssh.evt +# @TEST-EXEC: zeek -b Zeek::Spicy ssh.hlto %INPUT >>output +# @TEST-EXEC: echo >>output +# @TEST-EXEC: zeek -b Zeek::Spicy %INPUT >>output +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Check that we can access ANALYZER_* tags during Zeek-side script parse time. + +# @TEST-START-FILE ssh.spicy +module SSH; +public type Banner = unit {}; +# @TEST-END-FILE + +# @TEST-START-FILE ssh.evt +protocol analyzer spicy::SSH over TCP: + parse with SSH::Banner; +# @TEST-END-FILE + +@ifdef ( Analyzer::ANALYZER_SPICY_SSH ) +event zeek_init() + { + print "Have analyzer!"; + print fmt("tag: %s", Analyzer::get_tag("spicy_SSH")); + print fmt("name: %s", Analyzer::name(Analyzer::ANALYZER_SPICY_SSH)); + } +@else +event zeek_init() + { + print "Do not have analyzer!"; + } +@endif diff --git a/testing/btest/spicy/availability.zeek b/testing/btest/spicy/availability.zeek new file mode 100644 index 0000000000..1aaa9d6f37 --- /dev/null +++ b/testing/btest/spicy/availability.zeek @@ -0,0 +1,15 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: zeek %INPUT | grep -q yes +# @TEST-EXEC: zeek -b Zeek::Spicy %INPUT | grep -q yes +# @TEST-EXEC: if zeek -N Zeek::Spicy | grep -q built-in; then zeek -b %INPUT | grep -q yes; else zeek -b %INPUT | grep -q no; fi +# +# @TEST-DOC: Confirms `Zeek::available` signals correctly whether the Spicy plugin is loaded and active. +# +# Note that bare mode, by default, doesn't activate the plugin. + +@ifdef ( Spicy::available ) + print "yes"; +@else + print "no"; +@endif diff --git a/testing/btest/spicy/conn-id.spicy b/testing/btest/spicy/conn-id.spicy new file mode 100644 index 0000000000..fafaf54b76 --- /dev/null +++ b/testing/btest/spicy/conn-id.spicy @@ -0,0 +1,21 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto %INPUT test.evt +# @TEST-EXEC: zeek -b -r ${TRACES}/ssh/single-conn.trace Zeek::Spicy test.hlto Spicy::enable_print=T >>output +# @TEST-EXEC: zeek -b -r ${TRACES}/ftp/ipv6.trace Zeek::Spicy test.hlto Spicy::enable_print=T >>output +# @TEST-EXEC: btest-diff output + +module Test; + +import zeek; + +public type Foo = unit { + on %init { print zeek::conn_id(); } + x : /./; +}; + +# @TEST-START-FILE test.evt +protocol analyzer spicy::Test over TCP: + port 21/tcp-22/tcp, + parse originator with Test::Foo; +# @TEST-END-FILE diff --git a/testing/btest/spicy/context.spicy b/testing/btest/spicy/context.spicy new file mode 100644 index 0000000000..2095a7f418 --- /dev/null +++ b/testing/btest/spicy/context.spicy @@ -0,0 +1,50 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o x.hlto %INPUT ./ssh.evt +# @TEST-EXEC: zeek -b -r ${TRACES}/ssh/single-conn.trace Zeek::Spicy x.hlto Spicy::enable_print=T >output +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Check that the Zeek plugin passes a (and the same) %context object to both sides of a connection. + +module SSH; + +type Banner = unit { + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; +}; + +type Context = tuple; + +function print_if_we_have_seen_both_sides(ctx: Context&) { + if ( |ctx.orig_version| && |ctx.resp_version| ) + print ctx; +} + +public type Originator = unit { + %context = Context; + + banner: Banner { + self.context().orig_version += self.banner.version; + } + on %done { print_if_we_have_seen_both_sides(self.context()); } +}; + +public type Responder = unit { + %context = Context; + + banner: Banner { + self.context().resp_version += self.banner.version; + } + + on %done { print_if_we_have_seen_both_sides(self.context()); } +}; + + +# @TEST-START-FILE ssh.evt +protocol analyzer spicy::SSH over TCP: + port 22/tcp, + parse originator with SSH::Originator, + parse responder with SSH::Responder; +# @TEST-END-FILE diff --git a/testing/btest/spicy/double-event.zeek b/testing/btest/spicy/double-event.zeek new file mode 100644 index 0000000000..81f759f077 --- /dev/null +++ b/testing/btest/spicy/double-event.zeek @@ -0,0 +1,33 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto ssh.spicy ./ssh-cond.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT | sort >output +# @TEST-EXEC: btest-diff output + +event ssh::banner(i: int, software: string) + { + print i, software; + } + +# @TEST-START-FILE ssh.spicy +module SSH; + +public type Banner = unit { + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; +}; +# @TEST-END-FILE + +# @TEST-START-FILE ssh-cond.evt + +protocol analyzer spicy::SSH over TCP: + parse with SSH::Banner, + port 22/tcp, + replaces SSH; + +on SSH::Banner -> event ssh::banner(1, self.software); +on SSH::Banner -> event ssh::banner(2, self.software); + +# @TEST-END-FILE diff --git a/testing/btest/spicy/double-type-registration.zeek b/testing/btest/spicy/double-type-registration.zeek new file mode 100644 index 0000000000..5e75d564da --- /dev/null +++ b/testing/btest/spicy/double-type-registration.zeek @@ -0,0 +1,34 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -L . -o a.hlto a.spicy +# @TEST-EXEC: spicyz -L . -o b.hlto b.spicy +# @TEST-EXEC: zeek a.hlto b.hlto +# +# @TEST-EXEC: spicyz -o a.hlto a.spicy common.spicy +# @TEST-EXEC: spicyz -o b.hlto b.spicy common.spicy +# @TEST-EXEC: zeek a.hlto b.hlto +# +# @TEST-DOC: Regression test for #177. + +# @TEST-START-FILE common.spicy +module common; + +public type E = enum { + X, + Y, +}; +# @TEST-END-FILE + +# @TEST-START-FILE a.spicy +module a; + +import common; +global x: common::E; +# @TEST-END-FILE + +# @TEST-START-FILE b.spicy +module b; + +import common; +global x: common::E; +# @TEST-END-FILE diff --git a/testing/btest/spicy/double-types.zeek b/testing/btest/spicy/double-types.zeek new file mode 100644 index 0000000000..ce2937f918 --- /dev/null +++ b/testing/btest/spicy/double-types.zeek @@ -0,0 +1,63 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto dtest.spicy ./dtest.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT | sort >output +# @TEST-EXEC: btest-diff output + +event dtest_message(x: dtest::FUNCS) { + print "dtest_message", x; +} + +event dtest_result(y: dtest::RESULT) { + print "dtest_result", y; +} + +type R: record { + x: dtest::FUNCS; + y: dtest::RESULT; +}; + +event dtest_result_tuple(r: R) { + print "dtest_result_tuple", r$x, r$y; +} + +# @TEST-START-FILE dtest.evt + +protocol analyzer spicy::dtest over TCP: + parse originator with dtest::Message, + port 22/tcp; + +on dtest::Message -> event dtest_message(self.func); + +on dtest::Message -> event dtest_result(self.sub.result); + +on dtest::Message -> event dtest_result_tuple(dtest::bro_result(self)); + +# @TEST-END-FILE + +# @TEST-START-FILE dtest.spicy + +module dtest; + +public type FUNCS = enum { + A=0, B=1, C=2, D=3, E=4, F=5, YES=83 +}; + +public type RESULT = enum { + A, B, C, D, E, F, YES_AGAIN=83 +}; + +public type Message = unit { + func: uint8 &convert=FUNCS($$); + sub: SubMessage; +}; + +public type SubMessage = unit { + result: uint8 &convert=RESULT($$); +}; + +public function bro_result(entry: Message) : tuple { + return (entry.func, entry.sub.result); +} + +# @TEST-END-FILE diff --git a/testing/btest/spicy/event-args-fail.evt b/testing/btest/spicy/event-args-fail.evt new file mode 100644 index 0000000000..409f517822 --- /dev/null +++ b/testing/btest/spicy/event-args-fail.evt @@ -0,0 +1,34 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC-FAIL: spicyz -o test.hlto %INPUT test.spicy >output 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff output +# +# @TEST-DOC: In EVT, provide access to hooks arguments + +# @TEST-START-FILE test.spicy +module SSH; +public type Banner = unit {}; +# @TEST-END-FILE + +# Wrong/unknown. parameter type +on SSH::Banner::%error(msg: bytes) -> event Banner::error(msg); + +# @TEST-START-NEXT +# Wrong signature +on SSH::Banner::%error(msg: string, foo: uint64) -> event Banner::error(msg); + +# @TEST-START-NEXT +# Wrong signature +on SSH::Banner::%error(msg: uint64) -> event Banner::error(msg); + +# @TEST-START-NEXT +# Syntax error +on SSH::Banner::%error(msg: string,) -> event Banner::error(msg); + +# @TEST-START-NEXT +# Syntax error +on SSH::Banner::%error(,msg: string) -> event Banner::error(msg); + +# @TEST-START-NEXT +# Syntax error +on SSH::Banner::%error(msg) -> event Banner::error(msg); diff --git a/testing/btest/spicy/event-args-mismatch.zeek b/testing/btest/spicy/event-args-mismatch.zeek new file mode 100644 index 0000000000..eafe479f2d --- /dev/null +++ b/testing/btest/spicy/event-args-mismatch.zeek @@ -0,0 +1,30 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto test.evt test.spicy +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT >output 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff output +# +# @TEST-DOC: Test error reporting when an Zeek-side event parameter type does not match what Spicy sends. + +event Banner::error(i: count) { } + +# @TEST-START-FILE test.spicy +module SSH; + +public type Banner = unit { + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /KAPUTT/; +}; +# @TEST-END-FILE + +# @TEST-START-FILE test.evt + +protocol analyzer spicy::SSH over TCP: + parse originator with SSH::Banner, + port 22/tcp; + +on SSH::Banner::magic -> event Banner::error(self.magic); # Error: string -> count + +# @TEST-END-FILE diff --git a/testing/btest/spicy/event-args.zeek b/testing/btest/spicy/event-args.zeek new file mode 100644 index 0000000000..fddc3d4f6f --- /dev/null +++ b/testing/btest/spicy/event-args.zeek @@ -0,0 +1,33 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto test.evt test.spicy +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT >output +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff output +# +# @TEST-DOC: In EVT, provide access to hooks arguments + +event Banner::error(msg: string) { + print fmt("Error message: %s", msg); +} + +# @TEST-START-FILE test.spicy +module SSH; + +public type Banner = unit { + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /KAPUTT/; +}; +# @TEST-END-FILE + +# @TEST-START-FILE test.evt + +protocol analyzer spicy::SSH over TCP: + parse originator with SSH::Banner, + port 22/tcp; + +on SSH::Banner::%error(msg: string) -> event Banner::error(msg); +on SSH::Banner::%error() -> event Banner::error("n/a"); + +# @TEST-END-FILE diff --git a/testing/btest/spicy/event-cond.zeek b/testing/btest/spicy/event-cond.zeek new file mode 100644 index 0000000000..bbc43298dc --- /dev/null +++ b/testing/btest/spicy/event-cond.zeek @@ -0,0 +1,58 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto ssh.spicy ./ssh-cond.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT | sort >output +# @TEST-EXEC: btest-diff output + +event ssh::banner1(c: connection, is_orig: bool, version: string, software: string) + { + print "1", software; + } + +event ssh::banner2(c: connection, is_orig: bool, version: string, software: string) + { + print "2", software; + } + +event ssh::banner3(c: connection, is_orig: bool, version: string, software: string) + { + print "3", software; + } + +event ssh::banner4(c: connection, is_orig: bool, version: string, software: string) + { + print "4", software; + } + +event ssh::banner5(c: connection, is_orig: bool, version: string, software: string) + { + print "5", software; + } + +# @TEST-START-FILE ssh.spicy +module SSH; + +public type Banner = unit { + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; +}; +# @TEST-END-FILE + +# @TEST-START-FILE ssh-cond.evt + +import zeek; + +protocol analyzer spicy::SSH over TCP: + parse with SSH::Banner, + port 22/tcp, + replaces SSH; + +on SSH::Banner if ( True ) -> event ssh::banner1($conn, $is_orig, self.version, self.software); +on SSH::Banner if ( False )-> event ssh::banner2($conn, $is_orig, self.version, self.software); +on SSH::Banner if ( self.software == b"OpenSSH_3.9p1" )-> event ssh::banner3($conn, $is_orig, self.version, self.software); +on SSH::Banner if ( self.software != b"OpenSSH_3.9p1" )-> event ssh::banner4($conn, $is_orig, self.version, self.software); +on SSH::Banner if ( zeek::is_orig() ) -> event ssh::banner5($conn, $is_orig, self.version, self.software); + +# @TEST-END-FILE diff --git a/testing/btest/spicy/event-user-type b/testing/btest/spicy/event-user-type new file mode 100644 index 0000000000..75f99b4042 --- /dev/null +++ b/testing/btest/spicy/event-user-type @@ -0,0 +1,43 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -d foo.spicy foo.evt -o foo.hlto +# @TEST-EXEC: zeek -Cr ${TRACES}/udp-packet.pcap foo.hlto foo.zeek >output 2>&1 +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Validate that one can pass a user-specified type in an event handler. This is a regression test for #142, plus now a test for "export". + +# @TEST-START-FILE foo.spicy +module foo; + +global n: uint64 = 0; + +public type X = unit { y: Y; }; + +type Y = unit { + var n: uint64; + on %init { + n += 1; + self.n = n; + } + y: bytes &until=b"\x0a"; +}; +# @TEST-END-FILE + +# @TEST-START-FILE foo.evt +protocol analyzer spicy::foo over UDP: + parse with foo::X, + ports { 12345/udp, 31337/udp }; + +import foo; + +export foo::Y; + +on foo::X -> event foo::X($conn, $is_orig, self.y); +# @TEST-END-FILE + +# @TEST-START-FILE foo.zeek +event foo::X(c: connection, is_orig: bool, y: foo::Y) +{ + print fmt("is_orig=%d y=%s", is_orig, y); +} +# @TEST-END-FILE diff --git a/testing/btest/spicy/export-enum.zeek b/testing/btest/spicy/export-enum.zeek new file mode 100644 index 0000000000..5940b1158d --- /dev/null +++ b/testing/btest/spicy/export-enum.zeek @@ -0,0 +1,31 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o x.hlto tupleenum.spicy ./tupleenum.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace x.hlto %INPUT >>output +# @TEST-EXEC: zeek -NN x.hlto | grep TestEnum >>output +# +# @TEST-EXEC: btest-diff output + +event zeek_init() { + local i: TupleEnum::TestEnum; + + i = TupleEnum::TestEnum_A; + print i; + + i = TupleEnum::TestEnum_Undef; + print i; +} + +# @TEST-START-FILE tupleenum.evt + +# @TEST-END-FILE + +# @TEST-START-FILE tupleenum.spicy + +module TupleEnum; + +public type TestEnum = enum { + A = 83, B = 84, C = 85 +}; + +# @TEST-END-FILE diff --git a/testing/btest/spicy/export-type-fail.spicy b/testing/btest/spicy/export-type-fail.spicy new file mode 100644 index 0000000000..5d3ec3a619 --- /dev/null +++ b/testing/btest/spicy/export-type-fail.spicy @@ -0,0 +1,35 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-DOC: Failure cases for `export` +# +# @TEST-EXEC-FAIL: spicyz -d foo.spicy foo.evt -o foo.hlto >output 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff output + +# @TEST-START-FILE foo.spicy +module Test; + +type X = unit { + y: Y; # self-recursive +}; + +type Y = unit { + x: X; +}; + +type Z = unit { + var x: tuple; # names missing +}; + +# @TEST-END-FILE + +# @TEST-START-FILE foo.evt + +export Test::X; +export Test::Z; +export Test::DOES_NOT_EXIST; +export NOT_SCOPED; + +# @TEST-END-FILE + +# @TEST-START-FILE foo.zeek +# @TEST-END-FILE diff --git a/testing/btest/spicy/export-types.zeek b/testing/btest/spicy/export-types.zeek new file mode 100644 index 0000000000..c3001cc00d --- /dev/null +++ b/testing/btest/spicy/export-types.zeek @@ -0,0 +1,103 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o export.hlto export.spicy export.evt >>output +# @TEST-EXEC: zeek export.hlto %INPUT >>output +# +# Zeek 5.0 doesn't include the ID when printing the enum type +# @TEST-EXEC: cat output | sed 's/enum Test::type_enum/enum/g' >output.tmp && mv output.tmp output +# +# Zeek 6.0 includes information on whether a record field is `&optional`. +# @TEST-EXEC: cat output | sed 's/, optional=F//g' >output.tmp && mv output.tmp output +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Test the `export` keyword to automatically create corresponding Zeek types. + +module Test; + +global e: Test::type_enum = Test::type_enum_B; +global u2: type_record_u2 = [$t=[$x="S", $y=T]]; +global u: Test::type_record_u = [$s="S", $b=T, $u2=u2]; +global s: XYZ::type_record_sss = [ + $a=1.2.3.4, + $b="bytes", + $e=e, + $i=-10, + $iv=5secs, + $j=10, + $m=table([4.3.2.1] = "addr1", [4.3.2.2] = "addr2"), + $o="string", + $p=42/tcp, + $r=3.14, + $s=set(Test::type_enum_A, Test::type_enum_B), + $t=network_time(), + $u=u, + $v=vector("1", "2", "3") +]; + +event zeek_init() { + local all_globals: vector of string; + for ( id in global_ids() ) + all_globals[|all_globals|] = id; + + sort(all_globals, strcmp); + + for ( i in all_globals ) { + id = all_globals[i]; + + if ( ! (/((Test|XYZ)::|type_record_u2)/ in id) ) + next; + + if ( /type_record_/ in id ) + print id, record_fields(id); + # else if ( /type_enum$/ in id ) + # print id, enum_names(id); # Not available in 5.0 yet + else + print id; + } + + print "---"; + print s; +} + +# @TEST-START-FILE export.spicy +module Test; + +type type_enum = enum { A, B, C }; + +type type_record_s = struct { + a: addr; + b: bytes; + e: type_enum; + i: int32; + iv: interval; + j: uint8; + m: map; + o: optional; + p: port; + r: real; + s: set; + t: time; + u: type_record_u; + v: vector; +}; + +type type_record_u = unit { + var s: string; + var b: bool; + var u2: type_record_u2; +}; + +type type_record_u2 = unit { + var t: tuple; +}; + +# @TEST-END-FILE + +# @TEST-START-FILE export.evt + +export Test::type_enum; +export Test::type_record_s as XYZ::type_record_sss; +export Test::type_record_u; +export Test::type_record_u2 as type_record_u2; + +# @TEST-END-FILE diff --git a/testing/btest/spicy/file-analysis-data-in-concurrent.zeek b/testing/btest/spicy/file-analysis-data-in-concurrent.zeek new file mode 100644 index 0000000000..dd8abfdd66 --- /dev/null +++ b/testing/btest/spicy/file-analysis-data-in-concurrent.zeek @@ -0,0 +1,60 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto ssh.spicy ./ssh-cond.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT Spicy::enable_print=T >output +# @TEST-EXEC: btest-diff output + +# @TEST-START-FILE ssh.spicy +module SSH; + +import spicy; +import zeek; + +type Context = tuple; + +public type Banner = unit { + %context = Context; + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; +}; + +public type Data = unit { + data: bytes &eod; + on %done { print self.data; } +}; + +on Banner::%done { + local fid1 = zeek::file_begin("foo/bar"); + local fid2 = zeek::file_begin("foo/bar"); + local fid3 = zeek::file_begin("foo/bar"); + zeek::file_data_in(b"12", fid1); + zeek::file_data_in(b"!", fid3); + zeek::file_data_in(b"AAA", fid2); + zeek::file_data_in(b"@", fid3); + zeek::file_data_in(b"34", fid1); + zeek::file_data_in(b"#", fid3); + zeek::file_data_in(b"56", fid1); + zeek::file_data_in(b"BBB", fid2); + zeek::file_data_in(b"$"); # -> fid3 + zeek::file_end(fid1); + zeek::file_data_in(b"CCC", fid2); + zeek::file_end(fid2); + zeek::file_end(fid3); +} +# @TEST-END-FILE + +# @TEST-START-FILE ssh-cond.evt + +import zeek; + +protocol analyzer spicy::SSH over TCP: + parse originator with SSH::Banner, + port 22/tcp, + replaces SSH; + +file analyzer spicy::Text: + parse with SSH::Data, + mime-type foo/bar; +# @TEST-END-FILE diff --git a/testing/btest/spicy/file-analysis-data-in.zeek b/testing/btest/spicy/file-analysis-data-in.zeek new file mode 100644 index 0000000000..d7289455d5 --- /dev/null +++ b/testing/btest/spicy/file-analysis-data-in.zeek @@ -0,0 +1,64 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -d -o test.hlto ssh.spicy ./ssh-cond.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT Spicy::enable_print=T | sort >output +# +# @TEST-EXEC: cat x509.log | grep -v ^# | cut -f 4-5 >x509.log.tmp && mv x509.log.tmp x509.log +# @TEST-EXEC: btest-diff x509.log +# +# @TEST-EXEC: cat files.log | zeek-cut sha1 filename >files.log.tmp && mv files.log.tmp files.log +# @TEST-EXEC: btest-diff files.log +# +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff output + +# @TEST-START-FILE ssh.spicy +module SSH; + +import spicy; +import zeek; + +global file_counter = 0; + +public type Banner = unit { + magic : /SSH-/ { + # This is a bit of cheating. + local d: spicy::Base64Stream; + local dec : bytes = spicy::base64_decode(d, b"MIIESjCCAzKgAwIBAgINAeO0mqGNiqmBJWlQuDANBgkqhkiG9w0BAQsFADBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSMjETMBEGA1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjAeFw0xNzA2MTUwMDAwNDJaFw0yMTEyMTUwMDAwNDJaMEIxCzAJBgNVBAYTAlVTMR4wHAYDVQQKExVHb29nbGUgVHJ1c3QgU2VydmljZXMxEzARBgNVBAMTCkdUUyBDQSAxTzEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDQGM9F1IvN05zkQO9+tN1pIRvJzzyOTHW5DzEZhD2ePCnvUA0Qk28FgICfKqC9EksC4T2fWBYk/jCfC3R3VZMdS/dN4ZKCEPZRrAzDsiKUDzRrmBBJ5wudgzndIMYcLe/RGGFl5yODIKgjEv/SJH/UL+dEaltN11BmsK+eQmMF++AcxGNhr59qM/9il71I2dN8FGfcddwuaej4bXhp0LcQBbjxMcI7JP0aM3T4I+DsaxmKFsbjzaTNC9uzpFlgOIg7rR25xoynUxv8vNmkq7zdPGHXkxWY7oG9j+JkRyBABk7XrJfoucBZEqFJJSPk7XA0LKW0Y3z5oz2D0c1tJKwHAgMBAAGjggEzMIIBLzAOBgNVHQ8BAf8EBAMCAYYwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMBIGA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0OBBYEFJjR+G4Q68+b7GCfGJAboOt9Cf0rMB8GA1UdIwQYMBaAFJviB1dnHB7AagbeWbSaLd/cGYYuMDUGCCsGAQUFBwEBBCkwJzAlBggrBgEFBQcwAYYZaHR0cDovL29jc3AucGtpLmdvb2cvZ3NyMjAyBgNVHR8EKzApMCegJaAjhiFodHRwOi8vY3JsLnBraS5nb29nL2dzcjIvZ3NyMi5jcmwwPwYDVR0gBDgwNjA0BgZngQwBAgIwKjAoBggrBgEFBQcCARYcaHR0cHM6Ly9wa2kuZ29vZy9yZXBvc2l0b3J5LzANBgkqhkiG9w0BAQsFAAOCAQEAGoA+Nnn78y6pRjd9XlQWNa7HTgiZ/r3RNGkmUmYHPQq6Scti9PEajvwRT2iWTHQr02fesqOqBY2ETUwgZQ+lltoNFvhsO9tvBCOIazpswWC9aJ9xju4tWDQH8NVU6YZZ/XteDSGU9YzJqPjY8q3MDxrzmqepBCf5o8mw/wJ4a2G6xzUr6Fb6T8McDO22PLRL6u3M4Tzs3A2M1j6bykJYi8wWIRdAvKLWZu/axBVbzYmqmwkm5zLSDW5nIAJbELCQCZwMH56t2Dvqofxs6BBcCFIZUSpxu6x6td0V7SvJCCosirSmIatj/9dSSVDQibet8q/7UK4v4ZUN80atnZz1yg=="); + dec += spicy::base64_finish(d); + + print self.file_id; + zeek::file_data_in(dec); + } + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; + + var file_id: string = zeek::file_begin("application/x-x509-ca-cert"); + var file_name: string = "foo-%d.txt" % ++file_counter; +}; + +on Banner::%done { zeek::file_end(self.file_id); } + +# @TEST-END-FILE + +# @TEST-START-FILE ssh-cond.evt + +import zeek; + +protocol analyzer spicy::SSH over TCP: + parse with SSH::Banner, + port 22/tcp, + replaces SSH; + +on SSH::Banner::software -> event have_filename($file, self.file_name); + +# @TEST-END-FILE + +# Trigger creation of `files.log`. +@load base/protocols/ssl +redef X509::log_x509_in_files_log = T; + +event have_filename(f: fa_file, filename: string) + { + f$info$filename = filename; + } diff --git a/testing/btest/spicy/file-analyzer-nested.zeek b/testing/btest/spicy/file-analyzer-nested.zeek new file mode 100644 index 0000000000..6c1d22de7c --- /dev/null +++ b/testing/btest/spicy/file-analyzer-nested.zeek @@ -0,0 +1,87 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o text.hlto text.spicy ./text.evt +# @TEST-EXEC: zeek -r ${TRACES}/http/post.trace text.hlto %INPUT Spicy::enable_print=T | sort -k 3 >output +# @TEST-EXEC: btest-diff output +# @TEST-EXEC: cat files.log | zeek-cut source analyzers filename mime_type >files +# @TEST-EXEC: btest-diff files +# +# Check that exceeding max-file-depth leads to aborting and an event. +# @TEST-EXEC: zeek -t /tmp/zeek.trace -r ${TRACES}/http/post.trace text.hlto %INPUT Spicy::max_file_depth=2 | sort -k 3 >output-max +# @TEST-EXEC: cat notice.log | zeek-cut note | grep -q "Spicy_Max_File_Depth_Exceeded" +# @TEST-EXEC: btest-diff output-max + +event text::data1(f: fa_file, data: string) + { + print "data1", f$id, data; + } + +event text::data2(f: fa_file, data: string) + { + print "data2", f$id, data; + } + +event text::data3(f: fa_file, data: string) + { + print "data3", f$id, data; + } + +event Spicy::max_file_depth_exceeded(f: fa_file, args: Files::AnalyzerArgs, limit: count) + { + print "depth warning", f$id, args, limit; + } + +# @TEST-START-FILE text.spicy +module Text; + +import zeek; +import zeek_file; + +# This unit uses the zeek_file::File wrapper to pass data into Zeek's file analysis. +public type Data1 = unit { + on %init { + self.content.connect(new zeek_file::File("text/plain2")); + self.content.write(b"from 1:"); + } + + data: bytes &eod -> self.content; + + sink content; +}; + +# This unit passes data into Zeek's file analysis directly, without the File wrapper. +public type Data2 = unit { + data: bytes &eod { + zeek::file_begin("text/plain3"); + zeek::file_data_in(b"from 2a:" + self.data); + zeek::file_end(); + + zeek::file_begin("text/plain3"); + zeek::file_data_in(b"from 2b:" + self.data); + zeek::file_end(); + } +}; + +public type Data3 = unit { + data: bytes &eod; +}; +# @TEST-END-FILE + +# @TEST-START-FILE text.evt + +file analyzer spicy::Text1: + parse with Text::Data1, + mime-type text/plain; + +file analyzer spicy::Text2: + parse with Text::Data2, + mime-type text/plain2; + +file analyzer spicy::Text3: + parse with Text::Data3, + mime-type text/plain3; + +on Text::Data1 -> event text::data1($file, self.data); +on Text::Data2 -> event text::data2($file, self.data); +on Text::Data3 -> event text::data3($file, self.data); +# @TEST-END-FILE diff --git a/testing/btest/spicy/file-analyzer-property.zeek b/testing/btest/spicy/file-analyzer-property.zeek new file mode 100644 index 0000000000..34299e9117 --- /dev/null +++ b/testing/btest/spicy/file-analyzer-property.zeek @@ -0,0 +1,27 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto text.spicy ./text.evt +# @TEST-EXEC: zeek -r ${TRACES}/http/post.trace test.hlto %INPUT >output +# @TEST-EXEC: btest-diff output + +event text::data(f: fa_file, data: string) + { + print "text data", f$id, data; + } + +# @TEST-START-FILE text.spicy +module Text; + +public type Data = unit { + %mime-type = "text/plain"; + data: bytes &eod; +}; +# @TEST-END-FILE + +# @TEST-START-FILE text.evt + +file analyzer spicy::Text: + parse with Text::Data; + +on Text::Data -> event text::data($file, self.data); +# @TEST-END-FILE diff --git a/testing/btest/spicy/file-analyzer.zeek b/testing/btest/spicy/file-analyzer.zeek new file mode 100644 index 0000000000..99cbccc510 --- /dev/null +++ b/testing/btest/spicy/file-analyzer.zeek @@ -0,0 +1,46 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto text.spicy ./text.evt +# @TEST-EXEC: zeek -r ${TRACES}/http/post.trace test.hlto %INPUT >output +# @TEST-EXEC: btest-diff output +# @TEST-EXEC: btest-diff weird.log + +event zeek_init() + { + # Check we can access the tag. + print Files::ANALYZER_SPICY_TEXT; + } + +event text::data(f: fa_file, data: string) + { + print "text data", f$id, data; + } + +# @TEST-START-FILE text.spicy +module Text; + +import zeek; + +public type Data = unit { + data: bytes &eod; + + on %done { + # File ID isn't stable across platforms, so just check expected length. + assert |zeek::fuid()| == 18; + zeek::weird("test_weird"); + } +}; +# @TEST-END-FILE + +# @TEST-START-FILE text.evt + +file analyzer spicy::Text: + parse with Text::Data, + + # Note that Zeek determines the MIME type not from the Content-Type + # header in the trace, but by content sniffing (i.e., libmagic-style) + mime-type text/plain; + #mime-type application/x-www-form-urlencoded; + +on Text::Data -> event text::data($file, self.data); +# @TEST-END-FILE diff --git a/testing/btest/spicy/file-data-in-at-offset.zeek b/testing/btest/spicy/file-data-in-at-offset.zeek new file mode 100644 index 0000000000..762b7c4f05 --- /dev/null +++ b/testing/btest/spicy/file-data-in-at-offset.zeek @@ -0,0 +1,47 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto ssh.spicy ./ssh-cond.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT Spicy::enable_print=T | sort >output +# +# @TEST-EXEC: cat x509.log | grep -v ^# | cut -f 4-5 >x509.log.tmp && mv x509.log.tmp x509.log +# @TEST-EXEC: btest-diff x509.log +# @TEST-EXEC: btest-diff output + +# @TEST-START-FILE ssh.spicy +module SSH; + +import spicy; +import zeek; + + +public type Banner = unit { + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; +}; + +# this is a bit of cheating... + +on Banner::%done { + local d: spicy::Base64Stream; + local dec : bytes = spicy::base64_decode(d, b"MIIESjCCAzKgAwIBAgINAeO0mqGNiqmBJWlQuDANBgkqhkiG9w0BAQsFADBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSMjETMBEGA1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjAeFw0xNzA2MTUwMDAwNDJaFw0yMTEyMTUwMDAwNDJaMEIxCzAJBgNVBAYTAlVTMR4wHAYDVQQKExVHb29nbGUgVHJ1c3QgU2VydmljZXMxEzARBgNVBAMTCkdUUyBDQSAxTzEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDQGM9F1IvN05zkQO9+tN1pIRvJzzyOTHW5DzEZhD2ePCnvUA0Qk28FgICfKqC9EksC4T2fWBYk/jCfC3R3VZMdS/dN4ZKCEPZRrAzDsiKUDzRrmBBJ5wudgzndIMYcLe/RGGFl5yODIKgjEv/SJH/UL+dEaltN11BmsK+eQmMF++AcxGNhr59qM/9il71I2dN8FGfcddwuaej4bXhp0LcQBbjxMcI7JP0aM3T4I+DsaxmKFsbjzaTNC9uzpFlgOIg7rR25xoynUxv8vNmkq7zdPGHXkxWY7oG9j+JkRyBABk7XrJfoucBZEqFJJSPk7XA0LKW0Y3z5oz2D0c1tJKwHAgMBAAGjggEzMIIBLzAOBgNVHQ8BAf8EBAMCAYYwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMBIGA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0OBBYEFJjR+G4Q68+b7GCfGJAboOt9Cf0rMB8GA1UdIwQYMBaAFJviB1dnHB7AagbeWbSaLd/cGYYuMDUGCCsGAQUFBwEBBCkwJzAlBggrBgEFBQcwAYYZaHR0cDovL29jc3AucGtpLmdvb2cvZ3NyMjAyBgNVHR8EKzApMCegJaAjhiFodHRwOi8vY3JsLnBraS5nb29nL2dzcjIvZ3NyMi5jcmwwPwYDVR0gBDgwNjA0BgZngQwBAgIwKjAoBggrBgEFBQcCARYcaHR0cHM6Ly9wa2kuZ29vZy9yZXBvc2l0b3J5LzANBgkqhkiG9w0BAQsFAAOCAQEAGoA+Nnn78y6pRjd9XlQWNa7HTgiZ/r3RNGkmUmYHPQq6Scti9PEajvwRT2iWTHQr02fesqOqBY2ETUwgZQ+lltoNFvhsO9tvBCOIazpswWC9aJ9xju4tWDQH8NVU6YZZ/XteDSGU9YzJqPjY8q3MDxrzmqepBCf5o8mw/wJ4a2G6xzUr6Fb6T8McDO22PLRL6u3M4Tzs3A2M1j6bykJYi8wWIRdAvKLWZu/axBVbzYmqmwkm5zLSDW5nIAJbELCQCZwMH56t2Dvqofxs6BBcCFIZUSpxu6x6td0V7SvJCCosirSmIatj/9dSSVDQibet8q/7UK4v4ZUN80atnZz1yg=="); + dec += spicy::base64_finish(d); + local id = zeek::file_begin("application/x-x509-ca-cert"); + print id; + zeek::file_data_in_at_offset(dec, 0); + zeek::file_end(); +} + +# @TEST-END-FILE + +# @TEST-START-FILE ssh-cond.evt + +import zeek; + +protocol analyzer spicy::SSH over TCP: + parse with SSH::Banner, + port 22/tcp, + replaces SSH; + +# @TEST-END-FILE diff --git a/testing/btest/spicy/file-replaces.zeek b/testing/btest/spicy/file-replaces.zeek new file mode 100644 index 0000000000..17ebf20af7 --- /dev/null +++ b/testing/btest/spicy/file-replaces.zeek @@ -0,0 +1,117 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o pe.hlto pe.spicy zeek_pe.spicy pe.evt +# @TEST-EXEC: zeek -r ${TRACES}/pe/pe-single.trace pe.hlto %INPUT ENABLE=T +# @TEST-EXEC: cat files.log | zeek-cut source analyzers filename mime_type >>output +# @TEST-EXEC: zeek -r ${TRACES}/pe/pe-single.trace pe.hlto %INPUT ENABLE=F +# @TEST-EXEC: cat files.log | zeek-cut source analyzers filename mime_type >>output +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Test replacing an existing file analyzer, and also toggling the Spicy one on and off + +const ENABLE = T &redef; + +event zeek_init() { + if ( ENABLE ) + Spicy::enable_file_analyzer(Files::ANALYZER_SPICY_PE); + else + Spicy::disable_file_analyzer(Files::ANALYZER_SPICY_PE); +} + +event pe_dos_header(f: fa_file, h: PE::DOSHeader) + { + print "pe_dos_header", h; + } + +# @TEST-START-FILE pe.spicy +module PE; +import spicy; +%byte-order = spicy::ByteOrder::Little; + +public type ImageFile = unit { + %mime-type = "application/x-dosexec"; + + dosHeader: DOS_Header; +}; + +type DOS_Header = unit { + magic: b"MZ"; + bytesInLastPage: uint16; + pagesInFile: uint16; + relocations: uint16; + paragraphsInHeader: uint16; + minExtraParagraphs: uint16; + maxExtraParagraphs: uint16; + initialRelativeSS: uint16; + initialSP: uint16; + checksum: uint16; + initialIP: uint16; + initialRelativeCS: uint16; + relocationTableAddress: uint16; + overlayNumber: uint16; + reserved1: bytes &size=8; + oemID: uint16; + oemInfo: uint16; + reserved2: bytes &size=20; + peHeaderOffset: uint32; +}; +# @TEST-END-FILE + +# @TEST-START-FILE zeek_pe.spicy +module Zeek_PE; +import PE; + +type DOSHeader = tuple< + signature : bytes, + used_bytes_in_last_page : uint64, + file_in_pages : uint64, + num_reloc_items : uint64, + header_in_paragraphs : uint64, + min_extra_paragraphs : uint64, + max_extra_paragraphs : uint64, + init_relative_ss : uint64, + init_sp : uint64, + checksum : uint64, + init_ip : uint64, + init_relative_cs : uint64, + addr_of_reloc_table : uint64, + overlay_num : uint64, + oem_id : uint64, + oem_info : uint64, + addr_of_new_exe_header : uint64 + >; + +public function makeDOSHeader(h: PE::DOS_Header): DOSHeader + { + return ( + b"MZ (Spicy)", + h.bytesInLastPage, + h.pagesInFile, + h.relocations, + h.paragraphsInHeader, + h.minExtraParagraphs, + h.maxExtraParagraphs, + h.initialRelativeSS, + h.initialSP, + h.checksum, + h.initialIP, + h.initialRelativeCS, + h.relocationTableAddress, + h.overlayNumber, + h.oemID, + h.oemInfo, + h.peHeaderOffset, + ); + } +# @TEST-END-FILE + +# @TEST-START-FILE pe.evt +file analyzer spicy::PE: + parse with PE::ImageFile, + replaces PE, + mime-type application/x-dosexec; + +import Zeek_PE; + +on PE::DOS_Header -> event pe_dos_header($file, Zeek_PE::makeDOSHeader(self)); +# @TEST-END-FILE diff --git a/testing/btest/spicy/gap-recovery.zeek b/testing/btest/spicy/gap-recovery.zeek new file mode 100644 index 0000000000..10be2b3584 --- /dev/null +++ b/testing/btest/spicy/gap-recovery.zeek @@ -0,0 +1,47 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o analyzer.hlto analyzer.spicy analyzer.evt +# @TEST-EXEC: zeek -Cr ${TRACES}/spicy/gap-recovery.pcap analyzer.hlto Spicy::enable_print=T >output 2>&1 +# @TEST-EXEC: if spicy-version 10503; then btest-diff output; else OUT=output-before-spicy-issue-1303; mv output "$OUT"; btest-diff "$OUT"; fi +# +# @TEST-DOC: Tests that parsers can resynchronize on gaps. + +# @TEST-START-FILE analyzer.evt +protocol analyzer spicy::HTTP over TCP: + parse originator with test::Requests, + parse responder with test::Responses, + port 9000/tcp, + replaces HTTP; +# @TEST-END-FILE + +# @TEST-START-FILE analyzer.spicy +module test; +public type Requests = unit { + %port = 9000/tcp &originator; + requests: (Request &synchronize)[] foreach { confirm; } + on %done { print self; } +}; + +type Request = unit { + marker: /GET/; + : /[^\r?\n]+\r?\n/; + : (/[^\r?\n]*:[^\r?\n]*\r?\n/)[]; + : /\r?\n/; +}; + +public type Responses = unit { + %port = 9000/tcp &responder; + responses: (Response &synchronize)[] foreach { confirm; } + on %done { print self; } +}; + +type Response = unit { + marker: /HTTP/; + : /[^\r?\n]+\r?\n/; + : /content-length: /; + len: bytes &until=b"\r\n" &convert=cast($$.to_int()); + : (/[^\r?\n]*:[^\r?\n]*\r?\n/)[]; + : /\r?\n/; + : bytes &size=self.len; +}; +# @TEST-END-FILE diff --git a/testing/btest/spicy/import-from.zeek b/testing/btest/spicy/import-from.zeek new file mode 100644 index 0000000000..05ca2c15aa --- /dev/null +++ b/testing/btest/spicy/import-from.zeek @@ -0,0 +1,57 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: mkdir -p a/b/c && mv y.spicy a/b/c +# @TEST-EXEC: spicyz -o test.hlto ssh.spicy ./ssh.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT >output +# @TEST-EXEC: btest-diff output + +event ssh::test(x: string, y: string) + { + print x, y; + } + +# @TEST-START-FILE ssh.spicy +module SSH; + +public type Banner = unit { + %port = 22/tcp; + + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; + + on %done {} +}; +# @TEST-END-FILE + +# @TEST-START-FILE x.spicy + +module X; + +public function x() : string { + return "Foo::x"; +} + +# @TEST-END-FILE + +# @TEST-START-FILE y.spicy + +module Y; + +public function y() : string { + return "Foo::y"; +} + +# @TEST-END-FILE + + +# @TEST-START-FILE ssh.evt +protocol analyzer spicy::SSH over TCP: + parse with SSH::Banner; + +import X; +import Y from a.b.c; + +on SSH::Banner -> event ssh::test(X::x(), Y::y()); +# @TEST-END-FILE diff --git a/testing/btest/spicy/list-conversion.zeek b/testing/btest/spicy/list-conversion.zeek new file mode 100644 index 0000000000..132633297d --- /dev/null +++ b/testing/btest/spicy/list-conversion.zeek @@ -0,0 +1,54 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto listconv.spicy ./listconv.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT >output +# @TEST-EXEC: btest-diff output + +@TEST-START-FILE listconv.spicy + +module listconv; + +public function bro_convert(a: uint8) : tuple { + return (a, a + 1); +} + +public type Test = unit { + a: uint8[5]; + b: int16; + c: uint16; +}; + +@TEST-END-FILE + +@TEST-START-FILE listconv.evt + +protocol analyzer listconv over TCP: + parse originator with listconv::Test, + port 22/tcp; + +on listconv::Test -> event listconv::test($conn, + $is_orig, + [listconv::bro_convert(i) for i in self.a], + self.b, + self.c + ); + +@TEST-END-FILE + +type int_tuple: record { + a: count; + b: count; +}; + +event listconv::test(x: connection, + is_orig: bool, + a: vector of int_tuple, + b: int, + c: count + ) { + print x$id; + print is_orig; + print a; + print b; + print c; +} diff --git a/testing/btest/spicy/module-path.spicy b/testing/btest/spicy/module-path.spicy new file mode 100644 index 0000000000..c857f19b4c --- /dev/null +++ b/testing/btest/spicy/module-path.spicy @@ -0,0 +1,11 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o x.hlto %INPUT +# @TEST-EXEC: zeek -b Zeek::Spicy x.hlto Spicy::enable_print=T >>output +# @TEST-EXEC: mkdir -p y/z && mv x.hlto y/z +# @TEST-EXEC: ZEEK_SPICY_MODULE_PATH=FOO:y:BAR zeek -b Zeek::Spicy Spicy::enable_print=T >>output +# @TEST-EXEC: btest-diff output + +module Test; + +print "Got it"; diff --git a/testing/btest/spicy/multiple-enum.zeek b/testing/btest/spicy/multiple-enum.zeek new file mode 100644 index 0000000000..959f5f3482 --- /dev/null +++ b/testing/btest/spicy/multiple-enum.zeek @@ -0,0 +1,41 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto dtest.spicy ./dtest.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT | sort >output +# @TEST-EXEC: btest-diff output + +event dtest_one(x: dtest::RESULT) { + print "one", x; +} + +event dtest_two(x: dtest::RESULT) { + print "two", x; +} + +# @TEST-START-FILE dtest.evt + +protocol analyzer spicy::dtest over TCP: + parse originator with dtest::Message, + port 22/tcp; + +on dtest::Message if ( self.sswitch == 83 ) + -> event dtest_one(self.result); + +on dtest::Message if ( self.sswitch != 83 ) + -> event dtest_two(self.result); + +# @TEST-END-FILE +# @TEST-START-FILE dtest.spicy + +module dtest; + +public type RESULT = enum { + A, B = 83, C, D, E, F +}; + +public type Message = unit { + sswitch: uint8; + result: uint8 &convert=RESULT($$); +}; + +# @TEST-END-FILE diff --git a/testing/btest/spicy/network-time.spicy b/testing/btest/spicy/network-time.spicy new file mode 100644 index 0000000000..667aff89ee --- /dev/null +++ b/testing/btest/spicy/network-time.spicy @@ -0,0 +1,26 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto %INPUT ./udp-test.evt +# @TEST-EXEC: zeek -Cr ${TRACES}/udp-packet.pcap test.hlto Spicy::enable_print=T >output +# @TEST-EXEC: btest-diff output + +module Test; + +import zeek; + +# Before any processing has started the network time should be zero. +assert(zeek::network_time() == time(0)); + +public type Message = unit { + data: bytes &eod { + # For real traffic we would expect the network time to increase as we progress. + print zeek::network_time().nanoseconds(); + } +}; + +# @TEST-START-FILE udp-test.evt +protocol analyzer spicy::TEST over UDP: + parse with Test::Message, + port 11337/udp-11340/udp, + ports {31337/udp-31340/udp}; +# @TEST-END-FILE diff --git a/testing/btest/spicy/optional.zeek b/testing/btest/spicy/optional.zeek new file mode 100644 index 0000000000..8b848abdbc --- /dev/null +++ b/testing/btest/spicy/optional.zeek @@ -0,0 +1,39 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -d -o foo.hlto foo.spicy foo.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace foo.hlto %INPUT > output 2>&1 +# @TEST-EXEC: btest-diff output + +type R: record { + x: vector of int &optional; +}; + +event foo_result_tuple(r: R) { + print(r); +} + +# @TEST-START-FILE foo.evt + +protocol analyzer spicy::foo over TCP: + parse originator with Foo::Message, + port 22/tcp; + +on Foo::Message -> event foo_result_tuple(Foo::bro_result(self)); + +# @TEST-END-FILE + +# @TEST-START-FILE foo.spicy + +module Foo; + +public type Message = unit { + : uint8; + : uint8; +}; + +public function bro_result(entry: Message) : tuple> { + local y: optional; + return (y, ); +} + +# @TEST-END-FILE diff --git a/testing/btest/spicy/packet-analyzer-on-ip.zeek b/testing/btest/spicy/packet-analyzer-on-ip.zeek new file mode 100644 index 0000000000..b4f65fb02b --- /dev/null +++ b/testing/btest/spicy/packet-analyzer-on-ip.zeek @@ -0,0 +1,37 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto spicy/raw-layer.pcap.spicy spicy/raw-layer.pcap.evt +# @TEST-EXEC: zeek -r ${TRACES}/dns/proto255.pcap test.hlto %INPUT >output +# @TEST-EXEC: btest-diff output + +module PacketAnalyzer::SPICY_RAWLAYER; + +event zeek_init() + { + if ( ! PacketAnalyzer::try_register_packet_analyzer_by_name("IP", 255, "spicy_RawLayer") ) # modified trace to have IP proto 255 + print "cannot register raw analyzer on top of IP"; + } + +event raw::data(p: raw_pkt_hdr, data: string) + { + print fmt("MACs: src=%s dst=%s", p$l2$src, p$l2$dst); + print fmt("IPs : src=%s dst=%s", p$ip$src, p$ip$dst); + print fmt("raw bytes: %d", |data|); + } + +# @TEST-START-FILE spicy/raw-layer.pcap.spicy +module RawLayer; + +import zeek; + +public type Packet = unit { + data: bytes &eod; +}; +# @TEST-END-FILE + +# @TEST-START-FILE spicy/raw-layer.pcap.evt +packet analyzer spicy::RawLayer: + parse with RawLayer::Packet; + +on RawLayer::Packet::data -> event raw::data($packet, self.data); +# @TEST-END-FILE diff --git a/testing/btest/spicy/packet-analyzer-replaces.zeek b/testing/btest/spicy/packet-analyzer-replaces.zeek new file mode 100644 index 0000000000..7f1a2ec906 --- /dev/null +++ b/testing/btest/spicy/packet-analyzer-replaces.zeek @@ -0,0 +1,61 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o my-ethernet.hlto my-ethernet.spicy my-ethernet.evt +# @TEST-EXEC: zeek -r ${TRACES}/dns53.pcap my-ethernet.hlto %INPUT ENABLE=T >output-on +# @TEST-EXEC: zeek -r ${TRACES}/dns53.pcap my-ethernet.hlto %INPUT ENABLE=F >output-off +# @TEST-EXEC: btest-diff output-on + +# +# @TEST-DOC: Check that we can replace Zeek's Ethernet analyzer. +# +# Zeek logs look the same in both cases but we get some additional output +# when our analyzer is running by raising a custom event. + +const ENABLE = T &redef; + +module MyEthernet; + +const DLT_EN10MB : count = 1; + +event zeek_init() &priority=-200 + { + if ( ENABLE ) + Spicy::enable_file_analyzer(PacketAnalyzer::ANALYZER_SPICY_MYETHERNET); + else + Spicy::disable_file_analyzer(PacketAnalyzer::ANALYZER_SPICY_MYETHERNET); +} + +# The priority here needs to be higher than the standard script registering the +# built-in Ethernet analyzer. +event zeek_init() &priority=-100 + { + PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_ROOT, DLT_EN10MB, PacketAnalyzer::ANALYZER_SPICY_MYETHERNET); + PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_SPICY_MYETHERNET, 0x0800, PacketAnalyzer::ANALYZER_IP); + } + +event MyEthernet::data(p: raw_pkt_hdr, data: string) + { + print "My Ethernet:", data; + } + +# @TEST-START-FILE my-ethernet.spicy +module MyEthernet; + +import zeek; + +public type Packet = unit { + ethernet: bytes &size=14; + + on %done { + zeek::forward_packet(0x0800); # in practice, this wouldn't be hardcoded of course; + } +}; +# @TEST-END-FILE + +# @TEST-START-FILE my-ethernet.evt +packet analyzer spicy::MyEthernet: + parse with MyEthernet::Packet, + replaces Ethernet; + +on MyEthernet::Packet -> event MyEthernet::data($packet, self.ethernet); +# @TEST-END-FILE diff --git a/testing/btest/spicy/packet-analyzer-violation.zeek b/testing/btest/spicy/packet-analyzer-violation.zeek new file mode 100644 index 0000000000..46c2ce4af7 --- /dev/null +++ b/testing/btest/spicy/packet-analyzer-violation.zeek @@ -0,0 +1,30 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -d -o zeek_test.hlto analyzer.spicy analyzer.evt +# @TEST-EXEC: HILTI_DEBUG=spicy zeek -Cr ${TRACES}/spicy/packet-analyzer-violation.pcap zeek_test.hlto %INPUT >output 2>&1 +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Checks that packet analyzers correctly report violations. This is a regression test for #132. + +# @TEST-START-FILE analyzer.spicy +module test; +public type Foo = unit { + data: bytes &until=b"\xbe"; +}; +# @TEST-END-FILE + +# @TEST-START-FILE analyzer.evt +import test; + +packet analyzer spicy::TEST: + parse with test::Foo; +# @TEST-END-FILE + +module TEST; + +event zeek_init() +{ + if ( ! PacketAnalyzer::try_register_packet_analyzer_by_name("Ethernet", 0x6666, + "spicy_TEST") ) + print "cannot register spicy analyzer"; +} diff --git a/testing/btest/spicy/packet-analyzer.zeek b/testing/btest/spicy/packet-analyzer.zeek new file mode 100644 index 0000000000..0712750d3f --- /dev/null +++ b/testing/btest/spicy/packet-analyzer.zeek @@ -0,0 +1,45 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto spicy/raw-layer.pcap.spicy spicy/raw-layer.pcap.evt +# @TEST-EXEC: zeek -r ${TRACES}/spicy/raw-layer.pcap test.hlto %INPUT >output +# @TEST-EXEC: btest-diff output +# @TEST-EXEC: btest-diff weird.log + +module PacketAnalyzer::SPICY_RAWLAYER; + +event zeek_init() + { + PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_ETHERNET, 0x88b5, PacketAnalyzer::ANALYZER_SPICY_RAWLAYER); + + if ( ! PacketAnalyzer::try_register_packet_analyzer_by_name("spicy_RawLayer", 0x4950, "IP") ) + print "cannot register IP analyzer"; + } + +event raw::data(p: raw_pkt_hdr, data: string) + { + print fmt("MACs: src=%s dst=%s", p$l2$src, p$l2$dst); + print "raw data", data; + } + +# @TEST-START-FILE spicy/raw-layer.pcap.spicy +module RawLayer; + +import zeek; + +public type Packet = unit { + data: bytes &size=19; + protocol: uint16; + + on %done { + zeek::forward_packet(self.protocol); + zeek::weird("test_weird"); + } +}; +# @TEST-END-FILE + +# @TEST-START-FILE spicy/raw-layer.pcap.evt +packet analyzer spicy::RawLayer: + parse with RawLayer::Packet; + +on RawLayer::Packet::data -> event raw::data($packet, self.data); +# @TEST-END-FILE diff --git a/testing/btest/spicy/parse-error.zeek b/testing/btest/spicy/parse-error.zeek new file mode 100644 index 0000000000..4dbc0184e3 --- /dev/null +++ b/testing/btest/spicy/parse-error.zeek @@ -0,0 +1,36 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto test.evt test.spicy +# @TEST-EXEC: HILTI_DEBUG=zeek zeek -r ${TRACES}/ssh/single-conn.trace misc/dump-events test.hlto %INPUT +# Zeek versions differ in their quoting of the newline character in dpd.log (two slashes vs one). +# @TEST-EXEC: cat dpd.log | sed 's#\\\\#\\#g' >dpd.log.tmp && mv dpd.log.tmp dpd.log +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff dpd.log +# +# @TEST-DOC: Trigger parse error after confirmation, should be recorded in dpd.log + +# @TEST-START-FILE test.spicy +module SSH; + +import zeek; + +public type Banner = unit { + magic : /SSH-/ { zeek::confirm_protocol(); } + version : /[^-]*/; + dash : /-/; + software: /KAPUTT/; +}; +# @TEST-END-FILE + +# @TEST-START-FILE test.evt + +protocol analyzer spicy::SSH over TCP: + parse originator with SSH::Banner, + port 22/tcp + + # With Zeek < 5.0, DPD tracking doesn't work correctly for replaced + # analyzers because the ProtocolViolation() doesn't take a tag. + # + # replaces SSH + ; + +# @TEST-END-FILE diff --git a/testing/btest/spicy/port-fail.evt b/testing/btest/spicy/port-fail.evt new file mode 100644 index 0000000000..e9bae8ac13 --- /dev/null +++ b/testing/btest/spicy/port-fail.evt @@ -0,0 +1,22 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC-FAIL: spicyz %INPUT -o x.hlto >output 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff output + +protocol analyzer spicy::SSH over TCP: + port 123456/udp; + +@TEST-START-NEXT + +protocol analyzer spicy::SSH over TCP: + port -1/udp; + +@TEST-START-NEXT + +protocol analyzer spicy::SSH over TCP: + port 1/udp-2/tcp; + +@TEST-START-NEXT + +protocol analyzer spicy::SSH over TCP: + port 2/udp-1/udp; diff --git a/testing/btest/spicy/preprocessor-fail.evt b/testing/btest/spicy/preprocessor-fail.evt new file mode 100644 index 0000000000..daeb3e0848 --- /dev/null +++ b/testing/btest/spicy/preprocessor-fail.evt @@ -0,0 +1,37 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC-FAIL: spicyz -D zeek -o x.hlto %INPUT 2>output +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff output + +@if ZEEK_VERSION < 90000 + +# @TEST-START-NEXT + +@if ZEEK_VERSION >= 30000 +@else + +# @TEST-START-NEXT + +@else +@endif + +# @TEST-START-NEXT + +@endif + +# @TEST-START-NEXT + +@if ZEEK_VERSION < 90000 +@if ZEEK_VERSION < 90000 +@if ZEEK_VERSION < 90000 +@else +@endif +# @TEST-START-NEXT + +@if ZEEK_VERSION >= thirtythousand +@endif + +# @TEST-START-NEXT + +@if ZEEK_VERSION >= 3.0.0 +@endif diff --git a/testing/btest/spicy/preprocessor-spicy.spicy b/testing/btest/spicy/preprocessor-spicy.spicy new file mode 100644 index 0000000000..0c061df2b0 --- /dev/null +++ b/testing/btest/spicy/preprocessor-spicy.spicy @@ -0,0 +1,61 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: echo == spicyz >>output +# @TEST-EXEC: spicyz -c x %INPUT +# @TEST-EXEC: cat x_Foo.cc | grep print >>output +# +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Test various Spicy preprocessor constructs +# +# Same test is part of Spicy itself as well, with different results. + +module Foo; + +@if HAVE_ZEEK +print "have zeek"; +@else +print "not have zeek"; +@endif + +@if ! HAVE_ZEEK +print "not have zeek"; +@endif + +@if SPICY_VERSION +print "have spicy version"; +@endif + +@if SPICY_VERSION >= 400 +print "have spicy version >= 0.4"; +@endif + +@if SPICY_VERSION >= 40000 +print "have spicy version >= 4"; +@endif + +@if ! SPICY_VERSION >= 40000 +print "not have spicy version >= 4"; +@endif + +@if UNKNOWiN +no valid Spicy syntax here. +@endif + +@if HAVE_ZEEK + @if ZEEK_VERSION > 10000 + print "have zeek and zeek version > 1.0"; + @else + print "have zeek but zeek version < 1.0"; + @endif +@else + print "have not zeek"; +@endif + +@if HAVE_ZEEK + @if ZEEK_VERSION < 10000 + print "have zeek and zeek version < 1.0"; + @endif +@else + print "have not zeek"; +@endif diff --git a/testing/btest/spicy/preprocessor.evt b/testing/btest/spicy/preprocessor.evt new file mode 100644 index 0000000000..e58e7ed135 --- /dev/null +++ b/testing/btest/spicy/preprocessor.evt @@ -0,0 +1,40 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -D zeek -o x.hlto %INPUT 2>output +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff output + +protocol analyzer YES_1 over TCP: parse with X; + +@if ZEEK_VERSION >= 30000 + protocol analyzer YES_2 over TCP: parse with X; +@endif + +@if ZEEK_VERSION < 90000 + protocol analyzer YES_3 over TCP: parse with X; +@endif + +@if ZEEK_VERSION >= 30000 + protocol analyzer YES_4 over TCP: parse with X; +@else + protocol analyzer NO_4 over TCP: parse with X; +@endif + +@if ZEEK_VERSION < 30000 + protocol analyzer NO_5 over TCP: parse with X; +@else + protocol analyzer YES_5 over TCP: parse with X; +@endif + +@if ZEEK_VERSION >= 20000 + @if ZEEK_VERSION >= 90000 + protocol analyzer NO_6 over TCP: parse with X; + @else + protocol analyzer YES_6 over TCP: parse with X; + @endif +@else + @if ZEEK_VERSION < 90000 + protocol analyzer YES_7 over TCP: parse with X; + @else + protocol analyzer NO_7 over TCP: parse with X; + @endif +@endif diff --git a/testing/btest/spicy/profiling.zeek b/testing/btest/spicy/profiling.zeek new file mode 100644 index 0000000000..f34c2e2087 --- /dev/null +++ b/testing/btest/spicy/profiling.zeek @@ -0,0 +1,37 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -Z -o ssh.hlto ssh.spicy ./ssh.evt +# @TEST-EXEC: zeek -b -r ${TRACES}/ssh/single-conn.trace Zeek::Spicy ssh.hlto %INPUT Spicy::enable_profiling=T >output 2>prof.log.raw +# @TEST-EXEC: cat prof.log.raw | awk '{print $1, $2}' | egrep -v 'zeek/rt/debug|zeek/rt/internal_handler' >prof.log +# @TEST-EXEC: btest-diff output +# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff prof.log +# +# @TEST-DOC: Tests that we get profiling information for the Spicy analyzer. + +event ssh::banner(c: connection, is_orig: bool, version: string, software: string) + { + print "SSH banner", c$id, is_orig, version, software; + } + +# @TEST-START-FILE ssh.spicy +module SSH; + +import spicy; + +public type Banner = unit { + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; +}; +# @TEST-END-FILE + +# @TEST-START-FILE ssh.evt +protocol analyzer spicy::SSH over TCP: + # no port, we're using the signature + parse with SSH::Banner, + port 22/tcp, + replaces SSH; + +on SSH::Banner -> event ssh::banner($conn, $is_orig, self.version, self.software); +# @TEST-END-FILE diff --git a/testing/btest/spicy/protocol-analyzer-data-in.zeek b/testing/btest/spicy/protocol-analyzer-data-in.zeek new file mode 100644 index 0000000000..3c12289f77 --- /dev/null +++ b/testing/btest/spicy/protocol-analyzer-data-in.zeek @@ -0,0 +1,57 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto ssh.spicy ./ssh-cond.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT +# @TEST-EXEC: btest-diff http.log + +# @TEST-START-FILE ssh.spicy +module SSH; + +import spicy; +import zeek; + +type Context = tuple; + +public type Banner = unit { + %context = Context; + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; +}; + +on Banner::%done { + zeek::protocol_begin("HTTP"); + + zeek::protocol_data_in(True, b"GET /etc/passwd1 HTTP/1.0\r\n\r\n"); + zeek::protocol_data_in(False, b"HTTP/1.0 200 OK\r\nContent-Length: 0\r\n\r\n"); + + # We can get a handle to the existing HTTP analyzer. + local http1 = zeek::protocol_handle_get_or_create("HTTP"); + zeek::protocol_data_in(True, b"GET /etc/passwd1.1 HTTP/1.0\r\n\r\n", http1); + + # We can get another handle to the existing HTTP analyzer. + local http2 = zeek::protocol_handle_get_or_create("HTTP"); + zeek::protocol_data_in(False, b"HTTP/1.0 200 OK\r\nContent-Length: 0\r\n\r\n", http2); + zeek::protocol_handle_close(http1); + + zeek::protocol_end(); # Noop. + + # Creating a DPD child analyzer creates no handle. + zeek::protocol_begin(); # DPD + zeek::protocol_data_in(True, b"GET /etc/passwd2 HTTP/1.0\r\n\r\n"); + zeek::protocol_data_in(False, b"HTTP/1.0 200 OK\r\nContent-Length: 0\r\n\r\n"); + zeek::protocol_end(); +} +# @TEST-END-FILE + +# @TEST-START-FILE ssh-cond.evt + +import zeek; + +protocol analyzer spicy::SSH over TCP: + parse originator with SSH::Banner, + port 22/tcp, + replaces SSH; + +# @TEST-END-FILE diff --git a/testing/btest/spicy/protocol-analyzer-explicit-forwarding.zeek b/testing/btest/spicy/protocol-analyzer-explicit-forwarding.zeek new file mode 100644 index 0000000000..d12541efd5 --- /dev/null +++ b/testing/btest/spicy/protocol-analyzer-explicit-forwarding.zeek @@ -0,0 +1,48 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -d -o foo.hlto foo.spicy foo.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace foo.hlto %INPUT Spicy::enable_print=T >output +# @TEST-EXEC: btest-diff output + +# @TEST-START-FILE foo.spicy +module foo; +import zeek; + +public type X = unit { + xs: bytes &eod { + local y = zeek::protocol_handle_get_or_create("spicy_Y"); + local z = zeek::protocol_handle_get_or_create("spicy_Z"); + + zeek::protocol_data_in(zeek::is_orig(), b"only Y", y); + zeek::protocol_data_in(zeek::is_orig(), b"both Y and Z"); + + zeek::protocol_handle_close(z); + zeek::protocol_data_in(zeek::is_orig(), b"only Y after removal of Z"); + + zeek::protocol_handle_close(y); + zeek::protocol_data_in(zeek::is_orig(), b"goes nowhere"); + } +}; + +public type Y = unit { + ys: bytes &eod &chunked { print "ys=%s" % $$; } +}; + +public type Z = unit { + zs: bytes &eod &chunked { print "zs=%s" % $$; } +}; +# @TEST-END-FILE + +# @TEST-START-FILE foo.evt +# Analyzer instantiated from Zeek based on the traffic. +protocol analyzer spicy::X over TCP: + parse originator with foo::X, + port 22/tcp, + replaces SSH; + +# Analyzers which will only be instantiated explicitly by us. +protocol analyzer spicy::Y over TCP: + parse originator with foo::Y; +protocol analyzer spicy::Z over TCP: + parse originator with foo::Z; +# @TEST-END-FILE diff --git a/testing/btest/spicy/protocol-analyzer-tcp-over-udp.spicy b/testing/btest/spicy/protocol-analyzer-tcp-over-udp.spicy new file mode 100644 index 0000000000..02ebcd63dd --- /dev/null +++ b/testing/btest/spicy/protocol-analyzer-tcp-over-udp.spicy @@ -0,0 +1,28 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto %INPUT ./foo.evt +# @TEST-EXEC: zeek -Cr ${TRACES}/ssh/ssh-over-udp.pcap test.hlto +# @TEST-EXEC: btest-diff ssh.log +# +# @TEST-DOC: Pass data from inside a UDP analyzer to a Zeek analyzers that works on top of TCP. Regression tests for #92 and also #91. +# + +module Foo; + +import spicy; +import zeek; + +public type Bar = unit { + on %init { zeek::protocol_begin("SSH"); } + data: bytes &eod { zeek::protocol_data_in(zeek::is_orig(), $$); } +}; + +# @TEST-START-FILE foo.evt + +import zeek; + +protocol analyzer spicy::Foo over UDP: + parse with Foo::Bar, + port 1234/udp; + +# @TEST-END-FILE diff --git a/testing/btest/spicy/replaces-mismatch.zeek b/testing/btest/spicy/replaces-mismatch.zeek new file mode 100644 index 0000000000..593767e2a5 --- /dev/null +++ b/testing/btest/spicy/replaces-mismatch.zeek @@ -0,0 +1,25 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o ssh.hlto ssh.spicy ./ssh.evt +# @TEST-EXEC-FAIL: zeek -r ${TRACES}/ssh/single-conn.trace ssh.hlto >output 2>&1 +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Attempt to replace a packet analyzer with a protocol analyzer + +# @TEST-START-FILE ssh.spicy +module SSH; + +import zeek; + +public type Banner = unit { +}; +# @TEST-END-FILE + +# @TEST-START-FILE ssh.evt + +protocol analyzer spicy::SSH over TCP: + parse with SSH::Banner, + port 22/tcp, + replaces Ethernet; # fail + +# @TEST-END-FILE diff --git a/testing/btest/spicy/replaces.zeek b/testing/btest/spicy/replaces.zeek new file mode 100644 index 0000000000..e5105486e2 --- /dev/null +++ b/testing/btest/spicy/replaces.zeek @@ -0,0 +1,45 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: mkdir -p modules +# @TEST-EXEC: spicyz -o modules/ssh.hlto ssh.spicy ./ssh.evt +# @TEST-EXEC: ZEEK_SPICY_MODULE_PATH=$(pwd)/modules zeek -r ${TRACES}/ssh/single-conn.trace %INPUT | sort >output +# @TEST-EXEC: btest-diff output +# +# We use the module search path for loading here as a regression test for #137. +# Note that this that problem only showed up when the Spicy plugin was built +# into Zeek. + +event ssh::banner(c: connection, is_orig: bool, version: string, software: string) + { + print "SSH banner", c$id, is_orig, version, software; + } + +event analyzer_confirmation(c: connection, atype: AllAnalyzers::Tag, aid: count) + { + print atype, aid; + } + +# @TEST-START-FILE ssh.spicy +module SSH; + +import zeek; + +public type Banner = unit { + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; + + on %done { zeek::confirm_protocol(); } +}; +# @TEST-END-FILE + +# @TEST-START-FILE ssh.evt + +protocol analyzer spicy::SSH over TCP: + parse with SSH::Banner, + port 22/tcp, + replaces SSH; + +on SSH::Banner -> event ssh::banner($conn, $is_orig, self.version, self.software); +# @TEST-END-FILE diff --git a/testing/btest/spicy/resource-usage.zeek b/testing/btest/spicy/resource-usage.zeek new file mode 100644 index 0000000000..dcd4991e25 --- /dev/null +++ b/testing/btest/spicy/resource-usage.zeek @@ -0,0 +1,26 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto test.evt test.spicy +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto Zeek/Spicy/misc/resource-usage | sed 's/=[^ ]*/=XXX/g' >output +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Exercise the misc/resource-usage.zeek script. + +# @TEST-START-FILE test.spicy +module SSH; + +public type Banner = unit { + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /KAPUTT/; +}; +# @TEST-END-FILE + +# @TEST-START-FILE test.evt + +protocol analyzer spicy::SSH over TCP: + parse originator with SSH::Banner, + port 22/tcp; + +# @TEST-END-FILE diff --git a/testing/btest/spicy/reuse-parser-across-hltos.zeek b/testing/btest/spicy/reuse-parser-across-hltos.zeek new file mode 100644 index 0000000000..2cac28bf69 --- /dev/null +++ b/testing/btest/spicy/reuse-parser-across-hltos.zeek @@ -0,0 +1,70 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -d -o foo.hlto foo.spicy foo.evt +# @TEST-EXEC: spicyz -d -o bar.hlto bar.spicy bar.evt foo.spicy +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace foo.hlto %INPUT | sort >>output +# @TEST-EXEC: echo >>output +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace bar.hlto %INPUT | sort >>output +# @TEST-EXEC: echo >>output +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace foo.hlto bar.hlto %INPUT | sort >>output +# @TEST-EXEC: echo >>output +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace bar.hlto foo.hlto %INPUT | sort >>output +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Check that HLTOs remain isolated from each other when reusing another's units. +# +# The events triggered should reflect what's being loaded, and not depend on any loading ordering either. + +event foo::test(x: string) + { + print "foo", x; + } + +event bar::test(x: string) + { + print "bar", x; + } + +# @TEST-START-FILE foo.spicy +module Foo; + +public type Banner = unit { + %port = 22/tcp; + + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; +}; +# @TEST-END-FILE +# +# @TEST-START-FILE foo.evt +import Foo; + +protocol analyzer spicy::Foo over TCP: + parse with Foo::Banner; + +on Foo::Banner -> event foo::test(self.version); +# @TEST-END-FILE + +# @TEST-START-FILE bar.spicy +module Bar; + +import Foo; + +public type Banner = unit { + %port = 22/tcp; + x: Foo::Banner; +}; + +# @TEST-END-FILE +# +# @TEST-START-FILE bar.evt +import Foo; +import Bar; + +protocol analyzer spicy::Bar over TCP: + parse with Bar::Banner; + +on Foo::Banner -> event bar::test(self.version); +# @TEST-END-FILE diff --git a/testing/btest/spicy/spicy-dump.spicy b/testing/btest/spicy/spicy-dump.spicy index 5ce7eca37a..149bf87aa6 100644 --- a/testing/btest/spicy/spicy-dump.spicy +++ b/testing/btest/spicy/spicy-dump.spicy @@ -1,8 +1,9 @@ -# @TEST-DOC: Smoke test for a bundled Spicy. +# @TEST-REQUIRES: have-spicy && test -x ${BUILD}/auxil/spicy/spicy/bin/spicy-dump # -# @TEST-REQUIRES: $SCRIPTS/have-spicy # @TEST-EXEC: printf 12345 | ${BUILD}/auxil/spicy/spicy/bin/spicy-dump -d %INPUT >output # @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Smoke test for a bundled Spicy. module test; diff --git a/testing/btest/spicy/spicyz.test b/testing/btest/spicy/spicyz.test index 7abbf30745..f54ec89009 100644 --- a/testing/btest/spicy/spicyz.test +++ b/testing/btest/spicy/spicyz.test @@ -1,10 +1,11 @@ -# @TEST-DOC: Smoke test for a custom ahead-of-time compiled Spicy analyzer hooked into Zeek. +# @TEST-REQUIRES: have-spicy # -# @TEST-REQUIRES: $SCRIPTS/have-spicy -# @TEST-EXEC: ${BUILD}/src/builtin-plugins/spicy-plugin/bin/spicyz test.spicy test.evt -o test.hlto +# @TEST-EXEC: spicyz test.spicy test.evt -o test.hlto # @TEST-EXEC: zeek -NN test.hlto | grep -q ANALYZER_SPICY_TEST # @TEST-EXEC: zeek -r ${TRACES}/http/post.trace test.zeek test.hlto "Spicy::enable_print = T;" >>output 2>&1 # @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Smoke test for a custom ahead-of-time compiled Spicy analyzer hooked into Zeek. # @TEST-START-FILE test.spicy module test; diff --git a/testing/btest/spicy/ssh-banner.zeek b/testing/btest/spicy/ssh-banner.zeek new file mode 100644 index 0000000000..b4af1d682d --- /dev/null +++ b/testing/btest/spicy/ssh-banner.zeek @@ -0,0 +1,75 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o ssh.hlto ssh.spicy ./ssh.evt +# @TEST-EXEC: echo === confirmation >>output +# @TEST-EXEC: zeek -b -r ${TRACES}/ssh/single-conn.trace -s ./ssh.sig Zeek::Spicy base/frameworks/notice/weird ssh.hlto %INPUT ./extern.zeek | sort >>output +# @TEST-EXEC: btest-diff weird.log +# @TEST-EXEC: echo === violation >>output +# Note: The following removes the payload data from the violation log, as that's a recent addition that breaks older version. Can remove later. +# @TEST-EXEC: zeek -r ${TRACES}/http/post.trace -s ./ssh.sig Zeek::Spicy ssh.hlto ./extern.zeek %INPUT | sed 's/ \[POST.*//g' | sort >>output +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-remove-abspath btest-diff output +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff analyzer.log +# @TEST-EXEC: test '!' -f reporter.log + + +event ssh::banner(c: connection, is_orig: bool, version: string, software: string) + { + print "SSH banner", c$id, is_orig, version, software; + } + +event analyzer_confirmation(c: connection, atype: AllAnalyzers::Tag, aid: count) + { + if ( atype == Analyzer::ANALYZER_SPICY_SSH ) + print "confirm", atype; + } + +event analyzer_violation(c: connection, atype: AllAnalyzers::Tag, aid: count, reason: string) + { + if ( atype == Analyzer::ANALYZER_SPICY_SSH ) + print "violation", atype, reason; + } + +# @TEST-START-FILE extern.zeek + +module Foo; + +event ssh::banner(c: connection, is_orig: bool, version: string, software: string) + { + print "SSH banner in Foo", c$id, is_orig, version, software; + } +# @TEST-END-FILE + +# @TEST-START-FILE ssh.spicy +module SSH; + +import spicy; +import zeek; + +public type Banner = unit { + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/ { zeek::weird("my_weird", $$.decode()); } + + on %done { spicy::accept_input(); assert zeek::uid() == "CHhAvVGS1DHFjwGM9"; } + on %error { spicy::decline_input("kaputt"); } +}; +# @TEST-END-FILE + +# @TEST-START-FILE ssh.sig + +signature ssh_server { + ip-proto == tcp + payload /./ + enable "spicy_SSH" + tcp-state responder +} +# @TEST-END-FILE + +# @TEST-START-FILE ssh.evt +protocol analyzer spicy::SSH over TCP: + # no port, we're using the signature + parse with SSH::Banner; + +on SSH::Banner -> event ssh::banner($conn, $is_orig, self.version, self.software); +# @TEST-END-FILE diff --git a/testing/btest/spicy/terminate-session.zeek b/testing/btest/spicy/terminate-session.zeek new file mode 100644 index 0000000000..09c97d19cc --- /dev/null +++ b/testing/btest/spicy/terminate-session.zeek @@ -0,0 +1,41 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto test.spicy test.evt +# @TEST-EXEC: zeek -b -r ${TRACES}/dns/long-connection.pcap Zeek::Spicy test.hlto base/protocols/conn %INPUT +# @TEST-EXEC: cat conn.log | zeek-cut uid -C > conn.log2 && mv conn.log2 conn.log +# @TEST-EXEC: btest-diff conn.log +# +# @TEST-DOC: Validate that `terminate_session` indeed flushes Zeek-side connection state +# +# We expect to see two conn.log entries instead of one. + +redef likely_server_ports += { 53/udp }; # avoid flipping direction after termination +redef udp_inactivity_timeout = 24hrs; # avoid long gaps to trigger removal + +# @TEST-START-FILE test.spicy +module Test; + +import zeek; + +public type Foo = unit { + on %done { + self.context().counter = self.context().counter + 1; + + # close the connection if it is too long + if ( self.context().counter >= 10 ) + zeek::terminate_session(); + } + x : /./; + + %context = Counter; +}; + +type Counter = tuple; + +# @TEST-END-FILE + +# @TEST-START-FILE test.evt +protocol analyzer spicy::Test over UDP: + port 53/udp, + parse originator with Test::Foo; +# @TEST-END-FILE diff --git a/testing/btest/spicy/toggle-protocol-analyzer.zeek b/testing/btest/spicy/toggle-protocol-analyzer.zeek new file mode 100644 index 0000000000..ef0077a473 --- /dev/null +++ b/testing/btest/spicy/toggle-protocol-analyzer.zeek @@ -0,0 +1,46 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o ssh.hlto ssh.spicy ssh.evt +# @TEST-EXEC: echo === >>output +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace ssh.hlto %INPUT ENABLE=T >>output; +# @TEST-EXEC: echo === >>output +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace ssh.hlto %INPUT ENABLE=F >>output; +# @TEST-EXEC: echo === >>output +# @TEST-EXEC: btest-diff output +# +# @TEST-DOC: Check operation of Spicy::{enable,disable}_protocol_analyzer() + +const ENABLE = T &redef; + +event zeek_init() { + if ( ENABLE ) + Spicy::enable_protocol_analyzer(Analyzer::ANALYZER_SPICY_SSH); + else + Spicy::disable_protocol_analyzer(Analyzer::ANALYZER_SPICY_SSH); +} + +event ssh::banner(c: connection, is_orig: bool, version: string, software: string) + { + print "Spicy: SSH banner", c$id, is_orig, version, software; + } + +# @TEST-START-FILE ssh.spicy +module SSH; + +import zeek; + +public type Banner = unit { + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; +}; +# @TEST-END-FILE + +# @TEST-START-FILE ssh.evt +protocol analyzer spicy::SSH over TCP: + port 22/tcp, + parse originator with SSH::Banner; + +on SSH::Banner -> event ssh::banner($conn, $is_orig, self.version, self.software); +# @TEST-END-FILE diff --git a/testing/btest/spicy/tuple-arg.zeek b/testing/btest/spicy/tuple-arg.zeek new file mode 100644 index 0000000000..4b8a4be8bc --- /dev/null +++ b/testing/btest/spicy/tuple-arg.zeek @@ -0,0 +1,48 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -d -o test.hlto ssh.spicy ./ssh-tuple.evt +# @TEST-EXEC: HILTI_DEBUG=zeek zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT | sort >output +# @TEST-EXEC: grep event .stderr | sort >>output +# @TEST-EXEC: btest-diff output + +type Foo: record { + i: int; + s: string &optional; +}; + +event ssh::banner(f: Foo) + { + print f; + } + +# @TEST-START-FILE ssh.spicy + +module SSH; + +import zeek; + +public type Banner = unit { + magic : /SSH-/; + version : /[^-]*/; + dash : /-/; + software: /[^\r\n]*/; + + unset_field: uint64 if ( False ); + + on %done { zeek::confirm_protocol(); } +}; +# @TEST-END-FILE + +# @TEST-START-FILE ssh-tuple.evt + +protocol analyzer spicy::SSH over TCP: + parse originator with SSH::Banner, + port 22/tcp, + replaces SSH; + +on SSH::Banner -> event ssh::banner((1, self.software)); +on SSH::Banner -> event ssh::banner((2, self.?software)); +on SSH::Banner -> event ssh::banner((3, self.?unset_field)); +on SSH::Banner -> event ssh::banner((4, Null)); + +# @TEST-END-FILE diff --git a/testing/btest/spicy/tuple-enum.zeek b/testing/btest/spicy/tuple-enum.zeek new file mode 100644 index 0000000000..9918957b40 --- /dev/null +++ b/testing/btest/spicy/tuple-enum.zeek @@ -0,0 +1,42 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto tupleenum.spicy ./tupleenum.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT Spicy::enable_print=T | sort >output +# @TEST-EXEC: btest-diff output + +type Foo: record { + i: TupleEnum::TestEnum; + j: count; +}; + +event enum_message(f: Foo) { + print f; +} + +# @TEST-START-FILE tupleenum.evt + +protocol analyzer TupleEnum over TCP: + parse with TupleEnum::Message, + port 22/tcp, + replaces SSH; + +on TupleEnum::Message -> event enum_message( (self.a, cast(self.b)) ); + +# @TEST-END-FILE + +# @TEST-START-FILE tupleenum.spicy + +module TupleEnum; + +public type TestEnum = enum { + A = 83, B = 84, C = 85 +}; + +public type Message = unit { + a: uint8 &convert=TestEnum($$); + b: uint8; + + on %done { print self; } +}; + +# @TEST-END-FILE diff --git a/testing/btest/spicy/type-converter.zeek b/testing/btest/spicy/type-converter.zeek new file mode 100644 index 0000000000..310470fd7d --- /dev/null +++ b/testing/btest/spicy/type-converter.zeek @@ -0,0 +1,112 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto conv.spicy ./conv.evt +# @TEST-EXEC: ASAN_OPTIONS=detect_leaks=0 zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT Spicy::enable_print=T >output +# @TEST-EXEC: btest-diff output + +@TEST-START-FILE conv.spicy + +module Conv; + +public type Test = unit { + a: bytes &size=5; + b: int16; + c: uint16; + d: bytes &size=1 &convert=3.14; + e: bytes &size=1 &convert=1.2.3.4; + f: bytes &size=1 &convert=[2001:0db8::1428:57ab]; + g: bytes &size=1 &convert=True; + h: bytes &size=1 &convert="MyString"; + i: bytes &size=1 &convert=time(1295415110.5); + j: bytes &size=1 &convert=interval(4.0); + + var r: MyStruct = [$i = 11]; + var s: set = set(1,2,3); + var t: tuple = (47, "foo"); # Tuple conversion will ignore element names. + var v: vector = vector(b"A", b"B", b"C"); + var l: vector = vector(b"A", b"B", b"C"); + var m: map = map(1: "A", 2: "B", 3: "C"); + + on %done { print self; } +}; + +type MyStruct = struct { + i: int64; + s: string &optional; +}; + +@TEST-END-FILE + + +@TEST-START-FILE conv.evt + +protocol analyzer Conv over TCP: + parse originator with Conv::Test, + port 22/tcp; + +on Conv::Test -> event conv::test($conn, + $is_orig, + self.a, + self.b, + self.c, + self.d, + self.e, + self.f, + self.g, + self.h, + self.i, + self.j, + self.r, + self.s, + self.t, + self.v, + self.l, + self.m + ); + +@TEST-END-FILE + +type MyRecord: record { + i: int; + s: string &optional; +}; + +event conv::test(x: connection, + is_orig: bool, + a: string, + b: int, + c: count, + d: double, + e: addr, + f: addr, + g: bool, + h: string, + i: time, + j: interval, + r: MyRecord, + s: set[count], + t: MyRecord, + v: vector of string, + l: vector of string, + m: table[int] of string + ) + { + print x$id; + print is_orig; + print a; + print b; + print c; + print d; + print e; + print f; + print g; + print h; + print i; + print fmt("%f", j), type_name(j); # print as float as interval format differs between versions + print r; + print s; + print t; + print v; + print l; + print m; + } diff --git a/testing/btest/spicy/udp.zeek b/testing/btest/spicy/udp.zeek new file mode 100644 index 0000000000..d9eef7fd86 --- /dev/null +++ b/testing/btest/spicy/udp.zeek @@ -0,0 +1,33 @@ +# @TEST-REQUIRES: have-spicy +# +# @TEST-EXEC: spicyz -o test.hlto udp-test.spicy ./udp-test.evt +# @TEST-EXEC: zeek -Cr ${TRACES}/udp-packet.pcap test.hlto %INPUT >output +# @TEST-EXEC: btest-diff output + +event zeek_init() + { + # Check we can access the tag. + print Analyzer::ANALYZER_SPICY_UDP_TEST; + } + +event udp_test::message(c: connection, is_orig: bool, data: string) + { + print "UDP packet", c$id, is_orig, data; + } + +# @TEST-START-FILE udp-test.spicy +module UDPTest; + +public type Message = unit { + data: bytes &eod; +}; +# @TEST-END-FILE + +# @TEST-START-FILE udp-test.evt +protocol analyzer spicy::UDP_TEST over UDP: + parse with UDPTest::Message, + port 11337/udp-11340/udp, + ports {31337/udp-31340/udp}; + +on UDPTest::Message -> event udp_test::message($conn, $is_orig, self.data); +# @TEST-END-FILE diff --git a/testing/scripts/diff-canonifier-spicy b/testing/scripts/diff-canonifier-spicy new file mode 100755 index 0000000000..3fb4e2c2ba --- /dev/null +++ b/testing/scripts/diff-canonifier-spicy @@ -0,0 +1,6 @@ +#! /usr/bin/env bash +# +# Remove unstable pieces in Zeek logs produced by thge Spicy tests. + +$(dirname $0)/diff-remove-timestamps | + $(dirname $0)/diff-remove-abspath diff --git a/testing/scripts/spicy-version b/testing/scripts/spicy-version new file mode 100755 index 0000000000..d9a685c04e --- /dev/null +++ b/testing/scripts/spicy-version @@ -0,0 +1,13 @@ +#! /bin/sh +# +# Two usages: +# - Without argument, prints out the numerical Spicy version. +# - With a numerical Spicy version in $1, exit with true iff we have at least that version. + +version=$(spicy-config --version-number) + +if [ $# = 0 ]; then + echo "${version}" +else + test "${version}" -ge "$1" +fi diff --git a/testing/scripts/spicy/README b/testing/scripts/spicy/README new file mode 100644 index 0000000000..e531d0bcf6 --- /dev/null +++ b/testing/scripts/spicy/README @@ -0,0 +1,2 @@ +These are scripts kept for backwards compatibility with existing Spicy +packages. diff --git a/testing/scripts/spicy/canonify-zeek-log b/testing/scripts/spicy/canonify-zeek-log new file mode 100755 index 0000000000..c4f22cea32 --- /dev/null +++ b/testing/scripts/spicy/canonify-zeek-log @@ -0,0 +1,17 @@ +#! /usr/bin/env bash +# +# Remove unstable pieces in Zeek logs. + +# Get us "modern" regexps with sed. +if [ $(uname) == "Linux" ]; then + sed="sed -r" +else + sed="sed -E" +fi + +# File IDs changed between Zeek 3.1 and 3.2. + +${sed} 's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g' | + ${sed} 's/F[A-Za-z0-9]{14,17}/XXXXXXXXXXXXXXXXX/g' | + $(dirname $0)/diff-remove-timestamps | + $(dirname $0)/diff-remove-abspath diff --git a/testing/scripts/spicy/canonify-zeek-log-sorted b/testing/scripts/spicy/canonify-zeek-log-sorted new file mode 100755 index 0000000000..a4ff5ddbc9 --- /dev/null +++ b/testing/scripts/spicy/canonify-zeek-log-sorted @@ -0,0 +1,18 @@ +#! /usr/bin/env bash +# +# Remove unstable pieces in Zeek logs. + +# Get us "modern" regexps with sed. +if [ $(uname) == "Linux" ]; then + sed="sed -r" +else + sed="sed -E" +fi + +# File IDs changed between Zeek 3.1 and 3.2. + +${sed} 's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g' | + ${sed} 's/F[A-Za-z0-9]{15,17}/XXXXXXXXXXXXXXXXX/g' | + $(dirname $0)/diff-sort | + $(dirname $0)/diff-remove-timestamps | + $(dirname $0)/diff-remove-abspath diff --git a/testing/scripts/spicy/diff-remove-abspath b/testing/scripts/spicy/diff-remove-abspath new file mode 100755 index 0000000000..e08d64d5e2 --- /dev/null +++ b/testing/scripts/spicy/diff-remove-abspath @@ -0,0 +1,12 @@ +#! /usr/bin/env bash +# +# Replace absolute paths with the basename. + +if [ $(uname) == "Linux" ]; then + sed="sed -r" +else + sed="sed -E" +fi + +$sed 's#/+#/#g' | + $sed 's#/([^ :/]{1,}/){1,}([^ :/]{1,})#<...>/\2#g' diff --git a/testing/scripts/spicy/diff-remove-timestamps b/testing/scripts/spicy/diff-remove-timestamps new file mode 100755 index 0000000000..30ea6d1817 --- /dev/null +++ b/testing/scripts/spicy/diff-remove-timestamps @@ -0,0 +1,12 @@ +#! /usr/bin/env bash +# +# Replace anything which looks like timestamps with XXXs (including the #start/end markers in logs). + +# Get us "modern" regexps with sed. +if [ $(uname) == "Linux" ]; then + sed="sed -r" +else + sed="sed -E" +fi + +$sed -e 's/(^|[^0-9])([0-9]{9,10}\.[0-9]{1,8})/\1XXXXXXXXXX.XXXXXX/g' -e 's/^ *#(open|close).(19|20)..-..-..-..-..-../#\1 XXXX-XX-XX-XX-XX-XX/g' diff --git a/testing/scripts/spicy/diff-sort b/testing/scripts/spicy/diff-sort new file mode 100755 index 0000000000..66e2893c80 --- /dev/null +++ b/testing/scripts/spicy/diff-sort @@ -0,0 +1,19 @@ +#! /usr/bin/env bash +# +# A diff canonifier that sorts all lines but keeps all comments +# at the top. It also adds a note at the beginning as a reminder +# that the output has been sorted. + +if [ "$TMP" == "" ]; then + TMP=/tmp +fi + +tmp=$TMP/$(basename $0).$$.tmp + +cat >$tmp + +echo "### NOTE: This file has been sorted with $(basename $0)." +cat $tmp | grep ^# +cat $tmp | grep -v ^# | sort -s + +rm -f $tmp diff --git a/testing/scripts/spicy/run-zeek b/testing/scripts/spicy/run-zeek new file mode 100755 index 0000000000..db7af1df93 --- /dev/null +++ b/testing/scripts/spicy/run-zeek @@ -0,0 +1,7 @@ +#! /bin/sh +# +# Wrapper around Zeek to set environment. + +base=$(cd $(dirname $0)/.. && pwd) + +LD_PRELOAD=${ZEEK_LD_PRELOAD} DYLD_INSERT_LIBRARIES=${ZEEK_LD_PRELOAD} ASAN_OPTIONS=detect_leaks=0:detect_odr_violation=0 zeek $@ diff --git a/testing/scripts/spicy/spicy-version b/testing/scripts/spicy/spicy-version new file mode 100755 index 0000000000..bd43ce2462 --- /dev/null +++ b/testing/scripts/spicy/spicy-version @@ -0,0 +1,14 @@ +#! /bin/sh +# +# Two usages: +# - Without argument, prints out the numerical Spicy version. +# - With a numerical Spicy version in $1, exit with true iff we have at least that version. + +base=${TEST_BASE-$(cd $(dirname $0)/.. && pwd)} +nversion=$(spicy-config --version-number) + +if [ $# = 0 ]; then + echo "${nversion}" +else + test "${nversion}" -ge "$1" +fi diff --git a/testing/scripts/spicy/zeek-version b/testing/scripts/spicy/zeek-version new file mode 100755 index 0000000000..4d1a283c5b --- /dev/null +++ b/testing/scripts/spicy/zeek-version @@ -0,0 +1,20 @@ +#! /bin/sh +# +# Two usages: +# - Without argument, prints out the numerical Zeek version. +# - With a numerical Zeek version in $1, exit with true iff we have at least that version. + +base=$(cd $(dirname $0)/.. && pwd) + +version=$(zeek-config --version) +major=$(echo ${version} | cut -d . -f 1) +minor=$(echo ${version} | cut -d . -f 2) +patch=$(echo ${version} | cut -d . -f 3) + +nversion=$((${major} * 10000 + ${minor} * 100 + ${patch})) + +if [ $# = 0 ]; then + echo "${nversion}" +else + test "${nversion}" -ge "$1" +fi diff --git a/zeek-config.h.in b/zeek-config.h.in index e79286e3be..11afd6cae1 100644 --- a/zeek-config.h.in +++ b/zeek-config.h.in @@ -298,3 +298,9 @@ #else #define ZEEK_DISABLE_TSAN #endif + +/* compiled with Spicy support */ +#cmakedefine HAVE_SPICY + +// Version of Spicy that we are compiling against. +#define SPICY_VERSION_NUMBER ${SPICY_VERSION_NUMBER} diff --git a/zeek-config.in b/zeek-config.in index 91182796bc..0940f6fe09 100755 --- a/zeek-config.in +++ b/zeek-config.in @@ -32,6 +32,7 @@ add_path() { echo "$1:$2" } +# When changing any of these, also update "src/spicy/spicyz/config.h.in". include_dir=$(add_path "$include_dir" "@ZEEK_CONFIG_PCAP_INCLUDE_DIR@") include_dir=$(add_path "$include_dir" "@ZEEK_CONFIG_ZLIB_INCLUDE_DIR@") include_dir=$(add_path "$include_dir" "@ZEEK_CONFIG_OPENSSL_INCLUDE_DIR@")