diff --git a/CHANGES b/CHANGES index e6048e807d..4569a313ce 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,14 @@ +6.2.0-dev.187 | 2023-11-23 17:19:51 +0000 + + * Spicy: allow providing file id in zeek::file_begin (Johanna Amann, Corelight) + + Allow spicy parsers to generate their own file IDs and provide them to + Zeek. This duplicates functionality that is currently possible (and + used) by some binpac-based analyzers. One example for an analyzer + creating its own file IDs is the SSL analyzer. + + * Bump cmake submodule for INCLUDE_DIRS fix (Arne Welzel, Corelight) + 6.2.0-dev.180 | 2023-11-22 10:34:18 +0100 * Integrate review feedback (Dominik Charousset, Corelight) diff --git a/VERSION b/VERSION index f38b8f9bda..4bf686ac2d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -6.2.0-dev.180 +6.2.0-dev.187 diff --git a/doc b/doc index 7c2248ec62..35b5d2daf1 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 7c2248ec62ad21097c7f80874a9e46385f74c665 +Subproject commit 35b5d2daf1578d00eae08012b83d3b59f71d00e4 diff --git a/scripts/spicy/zeek.spicy b/scripts/spicy/zeek.spicy index 0afde17e8b..6cc445b628 100644 --- a/scripts/spicy/zeek.spicy +++ b/scripts/spicy/zeek.spicy @@ -107,8 +107,9 @@ public function protocol_handle_close(handle: ProtocolHandle): void &cxxname="ze ## Signals the beginning of a file to Zeek's file analysis, associating it with the current connection. ## Optionally, a mime type can be provided. It will be passed on to Zeek's file analysis framework. +## Optionally, a file ID can be provided. It will be passed on to Zeek's file analysis framework. ## Returns the Zeek-side file ID of the new file. -public function file_begin(mime_type: optional = Null) : string &cxxname="zeek::spicy::rt::file_begin"; +public function file_begin(mime_type: optional = Null, fuid: optional = Null) : string &cxxname="zeek::spicy::rt::file_begin"; ## Returns the current file's FUID. public function fuid() : string &cxxname="zeek::spicy::rt::fuid"; diff --git a/src/spicy/cookie.h b/src/spicy/cookie.h index bc59f09a7b..6823dfc488 100644 --- a/src/spicy/cookie.h +++ b/src/spicy/cookie.h @@ -50,8 +50,9 @@ public: /** * Begins analysis for a new file, pushing a new state object onto the * stack. + * @param fid Optional precomputed file ID to use for the new file. */ - FileState* push(); + FileState* push(std::optional fid = {}); /** Returns true if the stack is currently empty. */ bool isEmpty() const { return _stack.empty(); } diff --git a/src/spicy/runtime-support.cc b/src/spicy/runtime-support.cc index c59465aa4b..7f5e65f184 100644 --- a/src/spicy/runtime-support.cc +++ b/src/spicy/runtime-support.cc @@ -681,10 +681,17 @@ void rt::protocol_handle_close(const ProtocolHandle& handle) { c->analyzer->RemoveChildAnalyzer(handle.id()); } -rt::cookie::FileState* rt::cookie::FileStateStack::push() { +rt::cookie::FileState* rt::cookie::FileStateStack::push(std::optional fid_provided) { auto _ = hilti::rt::profiler::start("zeek/rt/file-stack-push"); - auto fid = file_mgr->HashHandle(hilti::rt::fmt("%s.%d", _analyzer_id, ++_id_counter)); - _stack.emplace_back(fid); + if ( fid_provided && find(*fid_provided) ) + throw InvalidValue(hilti::rt::fmt("Duplicate file id %s provided", *fid_provided)); + + std::string fid; + if ( fid_provided ) + fid = *fid_provided; + else + fid = file_mgr->HashHandle(hilti::rt::fmt("%s.%d", _analyzer_id, ++_id_counter)); + _stack.emplace_back(std::move(fid)); return &_stack.back(); } @@ -774,10 +781,10 @@ std::string rt::fuid() { throw ValueUnavailable("fuid() not available in current context"); } -std::string rt::file_begin(const std::optional& mime_type) { +std::string rt::file_begin(const std::optional& mime_type, const std::optional& fuid) { auto _ = hilti::rt::profiler::start("zeek/rt/file_begin"); auto cookie = static_cast(hilti::rt::context::cookie()); - auto* fstate = _file_state_stack(cookie)->push(); + auto* fstate = _file_state_stack(cookie)->push(fuid); fstate->mime_type = mime_type; // Feed an empty chunk into the analysis to force creating the file state inside Zeek. diff --git a/src/spicy/runtime-support.h b/src/spicy/runtime-support.h index 0ba1962d0f..e61aea53e5 100644 --- a/src/spicy/runtime-support.h +++ b/src/spicy/runtime-support.h @@ -382,9 +382,10 @@ void protocol_handle_close(const ProtocolHandle& handle); * with the current connection. * * @param mime_type optional mime type passed to Zeek + * @param fid optional file ID passed to Zeek * @returns Zeek-side file ID of the new file */ -std::string file_begin(const std::optional& mime_type); +std::string file_begin(const std::optional& mime_type, const std::optional& fid); /** * Returns the current file's FUID. diff --git a/testing/btest/Baseline/spicy.file-analysis-data-in/files-2.log b/testing/btest/Baseline/spicy.file-analysis-data-in/files-2.log new file mode 100644 index 0000000000..96b5da33b3 --- /dev/null +++ b/testing/btest/Baseline/spicy.file-analysis-data-in/files-2.log @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +FaAaAaAaAaAaAaAaAa foo-1.txt +FaAaAaAaAaAaAaAaAa foo-2.txt diff --git a/testing/btest/Baseline/spicy.file-analysis-data-in/output b/testing/btest/Baseline/spicy.file-analysis-data-in/output-1 similarity index 100% rename from testing/btest/Baseline/spicy.file-analysis-data-in/output rename to testing/btest/Baseline/spicy.file-analysis-data-in/output-1 diff --git a/testing/btest/Baseline/spicy.file-analysis-data-in/output-2 b/testing/btest/Baseline/spicy.file-analysis-data-in/output-2 new file mode 100644 index 0000000000..a1b10d3fd2 --- /dev/null +++ b/testing/btest/Baseline/spicy.file-analysis-data-in/output-2 @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +FaAaAaAaAaAaAaAaAa +FaAaAaAaAaAaAaAaAa diff --git a/testing/btest/spicy/file-analysis-data-in.zeek b/testing/btest/spicy/file-analysis-data-in.zeek index d7289455d5..c2c2e9622f 100644 --- a/testing/btest/spicy/file-analysis-data-in.zeek +++ b/testing/btest/spicy/file-analysis-data-in.zeek @@ -1,7 +1,8 @@ # @TEST-REQUIRES: have-spicy # -# @TEST-EXEC: spicyz -d -o test.hlto ssh.spicy ./ssh-cond.evt -# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT Spicy::enable_print=T | sort >output +# @TEST-EXEC: cat ssh.spicy ssh-1.spicy > ssh-test.spicy +# @TEST-EXEC: spicyz -d -o test.hlto ssh-test.spicy ./ssh-cond.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT Spicy::enable_print=T | sort >output-1 # # @TEST-EXEC: cat x509.log | grep -v ^# | cut -f 4-5 >x509.log.tmp && mv x509.log.tmp x509.log # @TEST-EXEC: btest-diff x509.log @@ -9,7 +10,15 @@ # @TEST-EXEC: cat files.log | zeek-cut sha1 filename >files.log.tmp && mv files.log.tmp files.log # @TEST-EXEC: btest-diff files.log # -# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff output +# @TEST-EXEC: cat ssh.spicy ssh-2.spicy > ssh-test.spicy +# @TEST-EXEC: spicyz -d -o test.hlto ssh-test.spicy ./ssh-cond.evt +# @TEST-EXEC: zeek -r ${TRACES}/ssh/single-conn.trace test.hlto %INPUT Spicy::enable_print=T | sort >output-2 +# +# @TEST-EXEC: cat files.log | zeek-cut fuid filename >files.log.tmp && mv files.log.tmp files-2.log +# @TEST-EXEC: btest-diff files-2.log +# +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff output-1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-canonifier-spicy btest-diff output-2 # @TEST-START-FILE ssh.spicy module SSH; @@ -33,7 +42,7 @@ public type Banner = unit { dash : /-/; software: /[^\r\n]*/; - var file_id: string = zeek::file_begin("application/x-x509-ca-cert"); + var file_id: string; var file_name: string = "foo-%d.txt" % ++file_counter; }; @@ -41,6 +50,20 @@ on Banner::%done { zeek::file_end(self.file_id); } # @TEST-END-FILE +# First test case - just let Zeek generate the File ID +# @TEST-START-FILE ssh-1.spicy + +on Banner::%init { self.file_id = zeek::file_begin("application/x-x509-ca-cert"); } + +# @TEST-END-FILE ssh-1.spicy + +# Second test case - provide a file ID +# @TEST-START-FILE ssh-2.spicy + +on Banner::%init { self.file_id = zeek::file_begin("application/x-x509-ca-cert", "FaAaAaAaAaAaAaAaAa"); } + +# @TEST-END-FILE ssh-2.spicy + # @TEST-START-FILE ssh-cond.evt import zeek;