diff --git a/.cirrus.yml b/.cirrus.yml index f021fc8f3d..d1be92dccf 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -669,3 +669,26 @@ zeekctl_debian11_task: on_failure: upload_zeekctl_testing_artifacts: path: "auxil/zeekctl/testing/.tmp/**" + +# Test building Zeek with builtin plugins available in +# testing/builtin-plugins/Files/ +include_plugins_debian11_task: + cpu: *CPUS + memory: *MEMORY + container: + # Debian 11 EOL: June 2026 + dockerfile: ci/debian-11/Dockerfile + << : *RESOURCES_TEMPLATE + sync_submodules_script: git submodule update --recursive --init + always: + ccache_cache: + folder: /tmp/ccache + fingerprint_script: echo builtin-plugins-ccache-$ZEEK_CCACHE_EPOCH-$CIRRUS_TASK_NAME-$CIRRUS_OS + reupload_on_changes: true + build_script: ZEEK_CI_CONFIGURE_FLAGS="${ZEEK_CI_CONFIGURE_FLAGS} --include-plugins='/zeek/testing/builtin-plugins/Files/protocol-plugin;/zeek/testing/builtin-plugins/Files/py-lib-plugin'" ./ci/build.sh + test_script: + - cd testing/builtin-plugins && ../../auxil/btest/btest -d -b -j ${ZEEK_CI_BTEST_JOBS} + on_failure: + upload_include_plugins_testing_artifacts: + path: "testing/builtin-plugins/.tmp/**" + << : *BUILDS_ONLY_IF_TEMPLATE diff --git a/testing/builtin-plugins/.gitignore b/testing/builtin-plugins/.gitignore new file mode 100644 index 0000000000..b4c1b7a858 --- /dev/null +++ b/testing/builtin-plugins/.gitignore @@ -0,0 +1,4 @@ +.tmp +.btest.failed.dat +diag.log +coverage.log diff --git a/testing/builtin-plugins/Baseline/tests.plugins-exist/out b/testing/builtin-plugins/Baseline/tests.plugins-exist/out new file mode 100644 index 0000000000..b2917349e8 --- /dev/null +++ b/testing/builtin-plugins/Baseline/tests.plugins-exist/out @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Zeek::PyLib - Plugin embedding Python, whoosh. (built-in) +Demo::Foo - A Foo test analyzer (built-in) diff --git a/testing/builtin-plugins/Baseline/tests.python/out b/testing/builtin-plugins/Baseline/tests.python/out new file mode 100644 index 0000000000..97ae064346 --- /dev/null +++ b/testing/builtin-plugins/Baseline/tests.python/out @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Python 3 + diff --git a/testing/builtin-plugins/Files/protocol-plugin/.btest-ignore b/testing/builtin-plugins/Files/protocol-plugin/.btest-ignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/builtin-plugins/Files/protocol-plugin/CMakeLists.txt b/testing/builtin-plugins/Files/protocol-plugin/CMakeLists.txt new file mode 100644 index 0000000000..53a50f3961 --- /dev/null +++ b/testing/builtin-plugins/Files/protocol-plugin/CMakeLists.txt @@ -0,0 +1,19 @@ + +project(Zeek-Plugin-Demo-Foo) + +cmake_minimum_required(VERSION 3.5) + +if ( NOT ZEEK_DIST ) + message(FATAL_ERROR "ZEEK_DIST not set") +endif () + +set(CMAKE_MODULE_PATH ${ZEEK_DIST}/cmake) + +include(ZeekPlugin) + +zeek_plugin_begin(Demo Foo) +zeek_plugin_cc(src/Plugin.cc) +zeek_plugin_cc(src/Foo.cc) +zeek_plugin_bif(src/events.bif) +zeek_plugin_pac(src/foo.pac src/foo-protocol.pac src/foo-analyzer.pac) +zeek_plugin_end() diff --git a/testing/builtin-plugins/Files/protocol-plugin/VERSION b/testing/builtin-plugins/Files/protocol-plugin/VERSION new file mode 100644 index 0000000000..42cf9e69db --- /dev/null +++ b/testing/builtin-plugins/Files/protocol-plugin/VERSION @@ -0,0 +1,5 @@ +# Some people put comments here + +# which isn't really recommended... + +0.1.0 diff --git a/testing/builtin-plugins/Files/protocol-plugin/scripts/Demo/Foo/base/main.zeek b/testing/builtin-plugins/Files/protocol-plugin/scripts/Demo/Foo/base/main.zeek new file mode 100644 index 0000000000..76c63723b7 --- /dev/null +++ b/testing/builtin-plugins/Files/protocol-plugin/scripts/Demo/Foo/base/main.zeek @@ -0,0 +1,7 @@ + +const ports = { 4242/tcp }; + +event zeek_init() &priority=5 + { + Analyzer::register_for_ports(Analyzer::ANALYZER_FOO, ports); + } diff --git a/testing/builtin-plugins/Files/protocol-plugin/scripts/__load__.zeek b/testing/builtin-plugins/Files/protocol-plugin/scripts/__load__.zeek new file mode 100644 index 0000000000..330718c604 --- /dev/null +++ b/testing/builtin-plugins/Files/protocol-plugin/scripts/__load__.zeek @@ -0,0 +1 @@ +@load Demo/Foo/base/main diff --git a/testing/builtin-plugins/Files/protocol-plugin/src/Foo.cc b/testing/builtin-plugins/Files/protocol-plugin/src/Foo.cc new file mode 100644 index 0000000000..4430e7664c --- /dev/null +++ b/testing/builtin-plugins/Files/protocol-plugin/src/Foo.cc @@ -0,0 +1,56 @@ + +#include "Foo.h" + +#include "zeek/analyzer/protocol/tcp/TCP_Reassembler.h" + +#include "events.bif.h" +#include "foo_pac.h" + +using namespace btest::plugin::Demo_Foo; + +Foo::Foo(zeek::Connection* conn) : zeek::analyzer::tcp::TCP_ApplicationAnalyzer("Foo", conn) + { + interp = new binpac::Foo::Foo_Conn(this); + } + +Foo::~Foo() + { + delete interp; + } + +void Foo::Done() + { + zeek::analyzer::tcp::TCP_ApplicationAnalyzer::Done(); + + interp->FlowEOF(true); + interp->FlowEOF(false); + } + +void Foo::EndpointEOF(bool is_orig) + { + zeek::analyzer::tcp::TCP_ApplicationAnalyzer::EndpointEOF(is_orig); + interp->FlowEOF(is_orig); + } + +void Foo::DeliverStream(int len, const u_char* data, bool orig) + { + zeek::analyzer::tcp::TCP_ApplicationAnalyzer::DeliverStream(len, data, orig); + + if ( TCP() && TCP()->IsPartial() ) + return; + + try + { + interp->NewData(orig, data, data + len); + } + catch ( const binpac::Exception& e ) + { + AnalyzerViolation(zeek::util::fmt("Binpac exception: %s", e.c_msg())); + } + } + +void Foo::Undelivered(uint64_t seq, int len, bool orig) + { + zeek::analyzer::tcp::TCP_ApplicationAnalyzer::Undelivered(seq, len, orig); + interp->NewGap(orig, len); + } diff --git a/testing/builtin-plugins/Files/protocol-plugin/src/Foo.h b/testing/builtin-plugins/Files/protocol-plugin/src/Foo.h new file mode 100644 index 0000000000..9d7a0e40e7 --- /dev/null +++ b/testing/builtin-plugins/Files/protocol-plugin/src/Foo.h @@ -0,0 +1,35 @@ + +#pragma once + +#include "analyzer/protocol/pia/PIA.h" +#include "analyzer/protocol/tcp/TCP.h" + +namespace binpac + { +namespace Foo + { +class Foo_Conn; + } + } + +namespace btest::plugin::Demo_Foo + { + +class Foo : public zeek::analyzer::tcp::TCP_ApplicationAnalyzer + { +public: + Foo(zeek::Connection* conn); + ~Foo(); + + virtual void Done(); + virtual void DeliverStream(int len, const u_char* data, bool orig); + virtual void Undelivered(uint64_t seq, int len, bool orig); + virtual void EndpointEOF(bool is_orig); + + static zeek::analyzer::Analyzer* Instantiate(zeek::Connection* conn) { return new Foo(conn); } + +protected: + binpac::Foo::Foo_Conn* interp; + }; + + } diff --git a/testing/builtin-plugins/Files/protocol-plugin/src/Plugin.cc b/testing/builtin-plugins/Files/protocol-plugin/src/Plugin.cc new file mode 100644 index 0000000000..ce6c02065f --- /dev/null +++ b/testing/builtin-plugins/Files/protocol-plugin/src/Plugin.cc @@ -0,0 +1,36 @@ + +#include "Plugin.h" + +#include "Foo.h" +#include "analyzer/Component.h" +#include "analyzer/Manager.h" + +namespace btest::plugin::Demo_Foo + { +Plugin plugin; + } + +using namespace btest::plugin::Demo_Foo; + +zeek::plugin::Configuration Plugin::Configure() + { + AddComponent( + new zeek::analyzer::Component("Foo", btest::plugin::Demo_Foo::Foo::Instantiate, 1)); + + zeek::plugin::Configuration config; + config.name = "Demo::Foo"; + config.description = "A Foo test analyzer"; + config.version.major = 1; + config.version.minor = 0; + config.version.patch = 0; + return config; + } + +void Plugin::InitPostScript() + { + auto tag = ::zeek::analyzer_mgr->GetAnalyzerTag("Foo"); + if ( ! tag ) + ::zeek::reporter->FatalError("cannot get analyzer Tag"); + + zeek::analyzer_mgr->RegisterAnalyzerForPort(tag, TransportProto::TRANSPORT_TCP, 4243); + } diff --git a/testing/builtin-plugins/Files/protocol-plugin/src/Plugin.h b/testing/builtin-plugins/Files/protocol-plugin/src/Plugin.h new file mode 100644 index 0000000000..1f24c7e9e4 --- /dev/null +++ b/testing/builtin-plugins/Files/protocol-plugin/src/Plugin.h @@ -0,0 +1,20 @@ + +#pragma once + +#include + +namespace btest::plugin::Demo_Foo + { + +class Plugin : public zeek::plugin::Plugin + { +protected: + // Overridden from zeek::plugin::Plugin. + zeek::plugin::Configuration Configure() override; + + void InitPostScript() override; + }; + +extern Plugin plugin; + + } diff --git a/testing/builtin-plugins/Files/protocol-plugin/src/events.bif b/testing/builtin-plugins/Files/protocol-plugin/src/events.bif new file mode 100644 index 0000000000..4603fe4cf6 --- /dev/null +++ b/testing/builtin-plugins/Files/protocol-plugin/src/events.bif @@ -0,0 +1,2 @@ + +event foo_message%(c: connection, data: string%); diff --git a/testing/builtin-plugins/Files/protocol-plugin/src/foo-analyzer.pac b/testing/builtin-plugins/Files/protocol-plugin/src/foo-analyzer.pac new file mode 100644 index 0000000000..0f847dda4d --- /dev/null +++ b/testing/builtin-plugins/Files/protocol-plugin/src/foo-analyzer.pac @@ -0,0 +1,15 @@ + +refine connection Foo_Conn += { + + function Foo_data(msg: Foo_Message): bool + %{ + auto data = zeek::make_intrusive(${msg.data}.length(), (const char*) ${msg.data}.data()); + zeek::BifEvent::enqueue_foo_message(bro_analyzer(), bro_analyzer()->Conn(), std::move(data)); + return true; + %} + +}; + +refine typeattr Foo_Message += &let { + proc: bool = $context.connection.Foo_data(this); +}; diff --git a/testing/builtin-plugins/Files/protocol-plugin/src/foo-protocol.pac b/testing/builtin-plugins/Files/protocol-plugin/src/foo-protocol.pac new file mode 100644 index 0000000000..892513c4f0 --- /dev/null +++ b/testing/builtin-plugins/Files/protocol-plugin/src/foo-protocol.pac @@ -0,0 +1,4 @@ + +type Foo_Message(is_orig: bool) = record { + data: bytestring &restofdata; +}; diff --git a/testing/builtin-plugins/Files/protocol-plugin/src/foo.pac b/testing/builtin-plugins/Files/protocol-plugin/src/foo.pac new file mode 100644 index 0000000000..193dac597d --- /dev/null +++ b/testing/builtin-plugins/Files/protocol-plugin/src/foo.pac @@ -0,0 +1,26 @@ +%include binpac.pac +%include zeek.pac + +%extern{ +#include "Foo.h" + +#include "events.bif.h" +%} + +analyzer Foo withcontext { + connection: Foo_Conn; + flow: Foo_Flow; +}; + +connection Foo_Conn(bro_analyzer: ZeekAnalyzer) { + upflow = Foo_Flow(true); + downflow = Foo_Flow(false); +}; + +%include foo-protocol.pac + +flow Foo_Flow(is_orig: bool) { + datagram = Foo_Message(is_orig) withcontext(connection, this); +}; + +%include foo-analyzer.pac diff --git a/testing/builtin-plugins/Files/py-lib-plugin/CMakeLists.txt b/testing/builtin-plugins/Files/py-lib-plugin/CMakeLists.txt new file mode 100644 index 0000000000..ce8eb32c96 --- /dev/null +++ b/testing/builtin-plugins/Files/py-lib-plugin/CMakeLists.txt @@ -0,0 +1,34 @@ +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) + +find_package(PythonLibs REQUIRED) + +file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1) + +string(REGEX REPLACE "[.-]" " " version_numbers ${VERSION}) +separate_arguments(version_numbers) +list(GET version_numbers 0 VERSION_MAJOR) +list(GET version_numbers 1 VERSION_MINOR) +list(GET version_numbers 2 VERSION_PATCH) + +# Our plugin source and scripts are in a subdirectory +set(BRO_PLUGIN_BASE "${CMAKE_CURRENT_SOURCE_DIR}/plugin") + +include(ZeekPlugin) + +include_directories(BEFORE ${PYTHON_INCLUDE_DIR}) + +zeek_plugin_begin(Zeek PyLib) + +file(GLOB cc_files RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "plugin/src/*.cc") +foreach(file ${cc_files}) + zeek_plugin_cc(${file}) +endforeach () + +file(GLOB bif_files RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "plugin/src/*.bif") +foreach(file ${bif_files}) + zeek_plugin_bif(${file}) +endforeach () + +zeek_plugin_dist_files(README VERSION) +zeek_plugin_link_library(${PYTHON_LIBRARY}) +zeek_plugin_end() diff --git a/testing/builtin-plugins/Files/py-lib-plugin/README b/testing/builtin-plugins/Files/py-lib-plugin/README new file mode 100644 index 0000000000..f4fc3b6a2c --- /dev/null +++ b/testing/builtin-plugins/Files/py-lib-plugin/README @@ -0,0 +1,8 @@ +PyLib +===== + +Embedding Python with a plugin. + + zeek -e 'Python::run_simple_string("import this")' + +Mostly to test a link dependency of a builtin plugin. diff --git a/testing/builtin-plugins/Files/py-lib-plugin/VERSION b/testing/builtin-plugins/Files/py-lib-plugin/VERSION new file mode 100644 index 0000000000..8acdd82b76 --- /dev/null +++ b/testing/builtin-plugins/Files/py-lib-plugin/VERSION @@ -0,0 +1 @@ +0.0.1 diff --git a/testing/builtin-plugins/Files/py-lib-plugin/plugin/scripts/__load__.zeek b/testing/builtin-plugins/Files/py-lib-plugin/plugin/scripts/__load__.zeek new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/builtin-plugins/Files/py-lib-plugin/plugin/scripts/__preload__.zeek b/testing/builtin-plugins/Files/py-lib-plugin/plugin/scripts/__preload__.zeek new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/builtin-plugins/Files/py-lib-plugin/plugin/src/Plugin.cc b/testing/builtin-plugins/Files/py-lib-plugin/plugin/src/Plugin.cc new file mode 100644 index 0000000000..ae05de6243 --- /dev/null +++ b/testing/builtin-plugins/Files/py-lib-plugin/plugin/src/Plugin.cc @@ -0,0 +1,34 @@ +#include "Plugin.h" + +extern "C" + { +#include + } + +namespace zeek::plugin::Zeek_PyLib + { +Plugin plugin; + } + +using namespace zeek::plugin::Zeek_PyLib; + +zeek::plugin::Configuration Plugin::Configure() + { + zeek::plugin::Configuration config; + config.name = "Zeek::PyLib"; + config.description = "Plugin embedding Python, whoosh."; + config.version.major = 0; + config.version.minor = 0; + config.version.patch = 1; + return config; + } + +void Plugin::InitPostScript() + { + Py_Initialize(); + } + +void Plugin::Done() + { + Py_FinalizeEx(); + } diff --git a/testing/builtin-plugins/Files/py-lib-plugin/plugin/src/Plugin.h b/testing/builtin-plugins/Files/py-lib-plugin/plugin/src/Plugin.h new file mode 100644 index 0000000000..2cf6d1d04d --- /dev/null +++ b/testing/builtin-plugins/Files/py-lib-plugin/plugin/src/Plugin.h @@ -0,0 +1,21 @@ +#pragma once + +#include + +namespace zeek::plugin + { +namespace Zeek_PyLib + { + +class Plugin : public zeek::plugin::Plugin + { +protected: + zeek::plugin::Configuration Configure() override; + void InitPostScript() override; + void Done() override; + }; + +extern Plugin plugin; + + } + } diff --git a/testing/builtin-plugins/Files/py-lib-plugin/plugin/src/pylib.bif b/testing/builtin-plugins/Files/py-lib-plugin/plugin/src/pylib.bif new file mode 100644 index 0000000000..7d579f9c44 --- /dev/null +++ b/testing/builtin-plugins/Files/py-lib-plugin/plugin/src/pylib.bif @@ -0,0 +1,18 @@ +module Python; + +%%{ +extern "C" { +#include +} +%%} + +function version%(%): string +%{ + return zeek::make_intrusive(Py_GetVersion()); +%} + +function run_simple_string%(s: string%): bool +%{ + PyRun_SimpleString(s->CheckString()); + return zeek::val_mgr->True(); +%} diff --git a/testing/builtin-plugins/Files/py-lib-plugin/scripts/__load__.zeek b/testing/builtin-plugins/Files/py-lib-plugin/scripts/__load__.zeek new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/builtin-plugins/README b/testing/builtin-plugins/README new file mode 100644 index 0000000000..b93bd718f8 --- /dev/null +++ b/testing/builtin-plugins/README @@ -0,0 +1 @@ +Smoke testing for builtin plugins from external sources. diff --git a/testing/builtin-plugins/btest.cfg b/testing/builtin-plugins/btest.cfg new file mode 100644 index 0000000000..b68d46ddc1 --- /dev/null +++ b/testing/builtin-plugins/btest.cfg @@ -0,0 +1,41 @@ +[DEFAULT] +# This should be a path relative to the main repository directory, and +# can be overridden with btest's -s command-line argument. +build_dir = build + +[btest] +TestDirs = tests +TmpDir = %(testbase)s/.tmp +BaselineDir = %(testbase)s/Baseline +IgnoreDirs = .svn CVS .tmp +IgnoreFiles = *.tmp *.swp .clang-format #* *.trace .DS_Store +MinVersion = 0.63 + +[environment] +ZEEKPATH=`bash -c %(testbase)s/../../%(build_dir)s/zeek-path-dev` +ZEEK_SEED_FILE=%(testbase)s/../btest/random.seed +ZEEK_PLUGIN_PATH= +TZ=UTC +LC_ALL=C +BTEST_PATH=%(testbase)s/../../auxil/btest +PATH=%(testbase)s/../../%(build_dir)s/src%(pathsep)s%(testbase)s/../scripts%(pathsep)s%(testbase)s/../../auxil/btest%(pathsep)s%(testbase)s/../../%(build_dir)s/auxil/zeek-aux/zeek-cut%(pathsep)s%(testbase)s/../../auxil/btest/sphinx%(pathsep)s%(default_path)s%(pathsep)s/sbin +TRACES=%(testbase)s/../btest/Traces +FILES=%(testbase)s/Files +SCRIPTS=%(testbase)s/../scripts +DOC_ROOT=%(testbase)s/../../doc +DIST=%(testbase)s/../.. +BUILD=%(testbase)s/../../%(build_dir)s +TEST_DIFF_CANONIFIER=%(testbase)s/../scripts/diff-canonifier +TMPDIR=%(testbase)s/.tmp +ZEEK_PROFILER_FILE=%(testbase)s/.tmp/script-coverage/XXXXXX +BTEST_RST_FILTER=$SCRIPTS/rst-filter +ZEEK_DNS_FAKE=1 +ZEEK_DEFAULT_LISTEN_ADDRESS=127.0.0.1 +ZEEK_DEFAULT_LISTEN_RETRY=1 +ZEEK_DEFAULT_CONNECT_RETRY=1 +ZEEK_DISABLE_ZEEKYGEN=1 +ZEEK_ALLOW_INIT_ERRORS=1 +ZEEK_SUPERVISOR_NO_SIGKILL=1 +UBSAN_OPTIONS=print_stacktrace=1 +SPICY_PATH=`bash -c %(testbase)s/../../%(build_dir)s/spicy-path` +HILTI_CXX_INCLUDE_DIRS=`bash -c %(testbase)s/../../%(build_dir)s/hilti-cxx-include-dirs` diff --git a/testing/builtin-plugins/tests/plugins-exist.zeek b/testing/builtin-plugins/tests/plugins-exist.zeek new file mode 100644 index 0000000000..229d809d69 --- /dev/null +++ b/testing/builtin-plugins/tests/plugins-exist.zeek @@ -0,0 +1,5 @@ +# @TEST-DOC: Assumes the plugins within Files/ have been builtin + +# @TEST-EXEC: zeek -N Zeek::PyLib >>out +# @TEST-EXEC: zeek -N Demo::Foo >>out +# @TEST-EXEC: btest-diff out diff --git a/testing/builtin-plugins/tests/python.zeek b/testing/builtin-plugins/tests/python.zeek new file mode 100644 index 0000000000..ed5450e782 --- /dev/null +++ b/testing/builtin-plugins/tests/python.zeek @@ -0,0 +1,9 @@ +# @TEST-DOC: Run a bit of Python for the kicks, assume there'll be no Python 4 in the near future. +# @TEST-EXEC: zeek %INPUT >out +# @TEST-EXEC: btest-diff out + +event zeek_init() + { + Python::run_simple_string("import sys; print(\"Python\", sys.version_info.major)"); + Python::run_simple_string("print(sys)"); + } diff --git a/testing/builtin-plugins/tests/zeek-ldd.zeek b/testing/builtin-plugins/tests/zeek-ldd.zeek new file mode 100644 index 0000000000..7d11ad0610 --- /dev/null +++ b/testing/builtin-plugins/tests/zeek-ldd.zeek @@ -0,0 +1,4 @@ +# @TEST-DOC: Run ldd on the zeek executable and check that it's linked against libpython + +# @TEST-EXEC: ldd $(which zeek) > ldd.out +# @TEST-EXEC: grep libpython ldd.out