From 54634ae241d3eef4fe0ecefa293181a80705db9a Mon Sep 17 00:00:00 2001 From: Klemens Nanni Date: Tue, 23 Sep 2025 01:42:10 +0300 Subject: [PATCH 01/14] OpaqueVal, OCSP, X509: drop outdated LibreSSL guards to fix OpenBSD build Whatever is used with recent OpenSSL is also available with latest LibreSSL on OpenBSD 7.8-beta as of today. Some of these hunks have been in the net/bro port for years, others I recently added whilst gradually updating from 6.0.5 to 8.0.1. --- src/OpaqueVal.cc | 6 +++--- src/file_analysis/analyzer/x509/OCSP.cc | 16 ++++++++-------- src/file_analysis/analyzer/x509/X509.h | 4 ++-- src/file_analysis/analyzer/x509/functions.bif | 6 +++--- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/OpaqueVal.cc b/src/OpaqueVal.cc index adc309fc4a..f318f4bdb7 100644 --- a/src/OpaqueVal.cc +++ b/src/OpaqueVal.cc @@ -27,11 +27,11 @@ #include "zeek/probabilistic/BloomFilter.h" #include "zeek/probabilistic/CardinalityCounter.h" -#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) inline void* EVP_MD_CTX_md_data(const EVP_MD_CTX* ctx) { return ctx->md_data; } #endif -#if ( OPENSSL_VERSION_NUMBER < 0x30000000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x30000000L ) #include #endif @@ -225,7 +225,7 @@ constexpr size_t SHA1VAL_STATE_SIZE = sizeof(SHA_CTX); constexpr size_t SHA256VAL_STATE_SIZE = sizeof(SHA256_CTX); -#if ( OPENSSL_VERSION_NUMBER < 0x30000000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x30000000L ) // -- MD5 diff --git a/src/file_analysis/analyzer/x509/OCSP.cc b/src/file_analysis/analyzer/x509/OCSP.cc index adee5b9624..d46eb72b2e 100644 --- a/src/file_analysis/analyzer/x509/OCSP.cc +++ b/src/file_analysis/analyzer/x509/OCSP.cc @@ -26,7 +26,7 @@ namespace zeek::file_analysis::detail { static constexpr size_t OCSP_STRING_BUF_SIZE = 2048; static bool OCSP_RESPID_bio(OCSP_BASICRESP* basic_resp, BIO* bio) { -#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) ASN1_OCTET_STRING* key = nullptr; X509_NAME* name = nullptr; @@ -353,7 +353,7 @@ void OCSP::ParseRequest(OCSP_REQUEST* req) { uint64_t version = 0; -#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) if ( req->tbsRequest->version ) version = (uint64_t)ASN1_INTEGER_get(req->tbsRequest->version); #else @@ -425,7 +425,7 @@ void OCSP::ParseResponse(OCSP_RESPONSE* resp) { if ( ! basic_resp ) goto clean_up; -#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) resp_data = basic_resp->tbsResponseData; if ( ! resp_data ) goto clean_up; @@ -434,7 +434,7 @@ void OCSP::ParseResponse(OCSP_RESPONSE* resp) { vl.emplace_back(GetFile()->ToVal()); vl.emplace_back(std::move(status_val)); -#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) vl.emplace_back(val_mgr->Count((uint64_t)ASN1_INTEGER_get(resp_data->version))); #else vl.emplace_back(parse_basic_resp_data_version(basic_resp)); @@ -452,7 +452,7 @@ void OCSP::ParseResponse(OCSP_RESPONSE* resp) { } // producedAt -#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) produced_at = resp_data->producedAt; #else produced_at = OCSP_resp_get0_produced_at(basic_resp); @@ -477,7 +477,7 @@ void OCSP::ParseResponse(OCSP_RESPONSE* resp) { // cert id const OCSP_CERTID* cert_id = nullptr; -#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) cert_id = single_resp->certId; #else cert_id = OCSP_SINGLERESP_get0_id(single_resp); @@ -550,7 +550,7 @@ void OCSP::ParseResponse(OCSP_RESPONSE* resp) { } } -#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) i2a_ASN1_OBJECT(bio, basic_resp->signatureAlgorithm->algorithm); len = BIO_read(bio, buf, sizeof(buf)); vl.emplace_back(make_intrusive(len, buf)); @@ -567,7 +567,7 @@ void OCSP::ParseResponse(OCSP_RESPONSE* resp) { certs_vector = new VectorVal(id::find_type("x509_opaque_vector")); vl.emplace_back(AdoptRef{}, certs_vector); -#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) certs = basic_resp->certs; #else certs = OCSP_resp_get0_certs(basic_resp); diff --git a/src/file_analysis/analyzer/x509/X509.h b/src/file_analysis/analyzer/x509/X509.h index 4295f68790..7fffb6729a 100644 --- a/src/file_analysis/analyzer/x509/X509.h +++ b/src/file_analysis/analyzer/x509/X509.h @@ -9,13 +9,13 @@ #include "zeek/OpaqueVal.h" #include "zeek/file_analysis/analyzer/x509/X509Common.h" -#if ( OPENSSL_VERSION_NUMBER < 0x10002000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x10002000L ) #define X509_get_signature_nid(x) OBJ_obj2nid((x)->sig_alg->algorithm) #endif -#if ( OPENSSL_VERSION_NUMBER < 0x1010000fL ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x1010000fL ) #define X509_OBJECT_new() (X509_OBJECT*)malloc(sizeof(X509_OBJECT)) #define X509_OBJECT_free(a) free(a) diff --git a/src/file_analysis/analyzer/x509/functions.bif b/src/file_analysis/analyzer/x509/functions.bif index 3be7dcd41d..b5b097f09c 100644 --- a/src/file_analysis/analyzer/x509/functions.bif +++ b/src/file_analysis/analyzer/x509/functions.bif @@ -65,7 +65,7 @@ X509* x509_get_ocsp_signer(const STACK_OF(X509)* certs, const ASN1_OCTET_STRING* key = nullptr; const X509_NAME* name = nullptr; -#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) OCSP_RESPID* resp_id = basic_resp->tbsResponseData->responderId; if ( resp_id->type == V_OCSP_RESPID_NAME ) @@ -359,7 +359,7 @@ function x509_ocsp_verify%(certs: x509_opaque_vector, ocsp_reply: string, root_c // Because we actually want to be able to give nice error messages that show why we were // not able to verify the OCSP response - do our own verification logic first. -#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) signer = x509_get_ocsp_signer(basic->certs, basic); #else signer = x509_get_ocsp_signer(OCSP_resp_get0_certs(basic), basic); @@ -730,7 +730,7 @@ function sct_verify%(cert: opaque of x509, logid: string, log_key: string, signa uint32_t cert_length; if ( precert ) { -#if ( OPENSSL_VERSION_NUMBER < 0x10002000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x10002000L ) x->cert_info->enc.modified = 1; cert_length = i2d_X509_CINF(x->cert_info, &cert_out); #else From 550c7eb0a758f0ff0e4b7c2af1621819e8dc7c12 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Wed, 24 Sep 2025 17:34:35 -0700 Subject: [PATCH 02/14] Fixes for -O gen-standalone-C++ for tracking BiFs, lambdas, attribute types, and independent globals --- src/script_opt/CPP/Driver.cc | 13 +++++++++++++ src/script_opt/ProfileFunc.cc | 36 ++++++++++++++++++++++++++++------- src/script_opt/ProfileFunc.h | 11 +++++++++++ 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/src/script_opt/CPP/Driver.cc b/src/script_opt/CPP/Driver.cc index b7cfd4a625..06037eeb42 100644 --- a/src/script_opt/CPP/Driver.cc +++ b/src/script_opt/CPP/Driver.cc @@ -83,9 +83,22 @@ void CPPCompile::Compile(bool report_uncompilable) { accessed_globals.insert(g); for ( auto& ag : pf->AllGlobals() ) all_accessed_globals.insert(ag); + for ( auto& l : pf->Lambdas() ) + // We might not have profiled this previously if none + // of the functions refer to the global. This can + // happen for example for a global "const" table that's + // made available for external lookup use. + pfs->ProfileLambda(l); } } + for ( auto& g : pfs->BiFGlobals() ) + all_accessed_globals.insert(g); + + for ( auto& t : pfs->MainTypes() ) + if ( obj_matches_opt_files(t) == AnalyzeDecision::SHOULD ) + rep_types.insert(TypeRep(t)); + for ( auto& l : pfs->Lambdas() ) if ( obj_matches_opt_files(l) == AnalyzeDecision::SHOULD ) accessed_lambdas.insert(l); diff --git a/src/script_opt/ProfileFunc.cc b/src/script_opt/ProfileFunc.cc index 95bcd405e6..0bf0d7ff96 100644 --- a/src/script_opt/ProfileFunc.cc +++ b/src/script_opt/ProfileFunc.cc @@ -622,6 +622,20 @@ ProfileFuncs::ProfileFuncs(std::vector& funcs, is_compilable_pred pred ComputeSideEffects(); } +void ProfileFuncs::ProfileLambda(const LambdaExpr* l) { + if ( lambdas.contains(l) ) + return; + + lambdas.insert(l); + pending_exprs.push_back(l); + + do + DrainPendingExprs(); + while ( ! pending_exprs.empty() ); + + AnalyzeLambdaProfile(l); +} + bool ProfileFuncs::IsTableWithDefaultAggr(const Type* t) { auto analy = tbl_has_aggr_default.find(t); if ( analy != tbl_has_aggr_default.end() ) @@ -848,14 +862,22 @@ void ProfileFuncs::ComputeBodyHashes(std::vector& funcs) { ComputeProfileHash(f.ProfilePtr()); } - for ( auto& l : lambdas ) { - auto pf = ExprProf(l); - func_profs[l->PrimaryFunc().get()] = pf; - lambda_primaries[l->Name()] = l->PrimaryFunc().get(); + for ( auto& l : lambdas ) + AnalyzeLambdaProfile(l); +} - if ( compute_func_hashes || ! pf->HasHashVal() ) - ComputeProfileHash(pf); - } +void ProfileFuncs::AnalyzeLambdaProfile(const LambdaExpr* l) { + if ( processed_lambdas.contains(l) ) + return; + + processed_lambdas.insert(l); + + auto pf = ExprProf(l); + func_profs[l->PrimaryFunc().get()] = pf; + lambda_primaries[l->Name()] = l->PrimaryFunc().get(); + + if ( compute_func_hashes || ! pf->HasHashVal() ) + ComputeProfileHash(pf); } void ProfileFuncs::ComputeProfileHash(std::shared_ptr pf) { diff --git a/src/script_opt/ProfileFunc.h b/src/script_opt/ProfileFunc.h index d4ad16300e..7b6186eb06 100644 --- a/src/script_opt/ProfileFunc.h +++ b/src/script_opt/ProfileFunc.h @@ -366,6 +366,10 @@ public: ProfileFuncs(std::vector& funcs, is_compilable_pred pred, bool compute_func_hashes, bool full_record_hashes); + // Used to profile additional lambdas that (potentially) weren't part + // of the overall function profiling. + void ProfileLambda(const LambdaExpr* l); + // The following accessors provide a global profile across all of // the (non-skipped) functions in "funcs". See the comments for // the associated member variables for documentation. @@ -449,6 +453,9 @@ protected: // Compute hashes to associate with each function void ComputeBodyHashes(std::vector& funcs); + // For a given lambda, completes analysis of its profile. + void AnalyzeLambdaProfile(const LambdaExpr* l); + // Compute the hash associated with a single function profile. void ComputeProfileHash(std::shared_ptr pf); @@ -540,6 +547,10 @@ protected: // And for lambda's. std::unordered_set lambdas; + // Lambdas that we have already processed. An optimization to avoid + // unnecessary work. + std::unordered_set processed_lambdas; + // Names of generated events. std::unordered_set events; From 01666df3d78e1c1abe416749714d6171bee788e2 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Thu, 25 Sep 2025 18:48:34 +0200 Subject: [PATCH 03/14] cluster/zeromq: Fix Cluster::subscribe() block if not initialized If Cluster::init() hasn't been invoked yet, Cluster::subscribe() with the ZeroMQ backend would block because the main_inproc socket didn't yet have a connection from the child thread. Prevent this by connecting the main and child socket pair at construction time. This will queue the subscriptions and start processing them once the child thread has started. --- src/cluster/backend/zeromq/ZeroMQ.cc | 20 ++++--- .../..manager.out | 2 + .../cluster/zeromq/subscribe-before-init.zeek | 55 +++++++++++++++++++ 3 files changed, 68 insertions(+), 9 deletions(-) create mode 100644 testing/btest/Baseline/cluster.zeromq.subscribe-before-init/..manager.out create mode 100644 testing/btest/cluster/zeromq/subscribe-before-init.zeek diff --git a/src/cluster/backend/zeromq/ZeroMQ.cc b/src/cluster/backend/zeromq/ZeroMQ.cc index 4085179b8b..4ce1525987 100644 --- a/src/cluster/backend/zeromq/ZeroMQ.cc +++ b/src/cluster/backend/zeromq/ZeroMQ.cc @@ -85,6 +85,7 @@ ZeroMQBackend::ZeroMQBackend(std::unique_ptr es, std::unique_pt : ThreadedBackend("ZeroMQ", std::move(es), std::move(ls), std::move(ehs), new zeek::detail::OnLoopProcess(this, "ZeroMQ", onloop_queue_hwm)), main_inproc(zmq::socket_t(ctx, zmq::socket_type::pair)), + child_inproc(zmq::socket_t(ctx, zmq::socket_type::pair)), // Counters for block and drop metrics. total_xpub_drops( zeek::telemetry_mgr->CounterInstance("zeek", "cluster_zeromq_xpub_drops", {}, @@ -94,7 +95,13 @@ ZeroMQBackend::ZeroMQBackend(std::unique_ptr es, std::unique_pt "Number of received events dropped due to OnLoop queue full.")), total_msg_errors( zeek::telemetry_mgr->CounterInstance("zeek", "cluster_zeromq_msg_errors", {}, - "Number of events with the wrong number of message parts.")) {} + "Number of events with the wrong number of message parts.")) { + // Establish the socket connection between main thread and child thread + // already in the constructor. This allows Subscribe() and Unsubscribe() + // calls to be delayed until DoInit() was called. + main_inproc.bind("inproc://inproc-bridge"); + child_inproc.connect("inproc://inproc-bridge"); +} ZeroMQBackend::~ZeroMQBackend() { try { @@ -169,10 +176,11 @@ void ZeroMQBackend::DoTerminate() { ctx.shutdown(); // Close the sockets that are used from the main thread, - // the remaining sockets were closed by self_thread during - // shutdown already. + // the remaining sockets except for the child_inproc one + // were closed by self_thread during shutdown already. log_push.close(); main_inproc.close(); + child_inproc.close(); ZEROMQ_DEBUG("Closing ctx"); ctx.close(); @@ -197,7 +205,6 @@ bool ZeroMQBackend::DoInit() { xpub = zmq::socket_t(ctx, zmq::socket_type::xpub); log_push = zmq::socket_t(ctx, zmq::socket_type::push); log_pull = zmq::socket_t(ctx, zmq::socket_type::pull); - child_inproc = zmq::socket_t(ctx, zmq::socket_type::pair); xpub.set(zmq::sockopt::linger, linger_ms); @@ -286,10 +293,6 @@ bool ZeroMQBackend::DoInit() { // // https://funcptr.net/2012/09/10/zeromq---edge-triggered-notification/ - // Setup connectivity between main and child thread. - main_inproc.bind("inproc://inproc-bridge"); - child_inproc.connect("inproc://inproc-bridge"); - // Thread is joined in backend->DoTerminate(), backend outlives it. self_thread = std::thread([](auto* backend) { backend->Run(); }, this); @@ -634,7 +637,6 @@ void ZeroMQBackend::Run() { // Called when Run() terminates. auto deferred_close = util::Deferred([this]() { - child_inproc.close(); xpub.close(); xsub.close(); log_pull.close(); diff --git a/testing/btest/Baseline/cluster.zeromq.subscribe-before-init/..manager.out b/testing/btest/Baseline/cluster.zeromq.subscribe-before-init/..manager.out new file mode 100644 index 0000000000..e4f236ab1a --- /dev/null +++ b/testing/btest/Baseline/cluster.zeromq.subscribe-before-init/..manager.out @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +test(), 42 diff --git a/testing/btest/cluster/zeromq/subscribe-before-init.zeek b/testing/btest/cluster/zeromq/subscribe-before-init.zeek new file mode 100644 index 0000000000..4040fd2399 --- /dev/null +++ b/testing/btest/cluster/zeromq/subscribe-before-init.zeek @@ -0,0 +1,55 @@ +# @TEST-DOC: Regression test Cluster::subscribe() blocking if called in a high-priority zeek_init() handler +# +# @TEST-REQUIRES: have-zeromq +# +# @TEST-GROUP: cluster-zeromq +# +# @TEST-PORT: XPUB_PORT +# @TEST-PORT: XSUB_PORT +# @TEST-PORT: LOG_PULL_PORT +# +# @TEST-EXEC: cp $FILES/zeromq/cluster-layout-simple.zeek cluster-layout.zeek +# @TEST-EXEC: cp $FILES/zeromq/test-bootstrap.zeek zeromq-test-bootstrap.zeek +# +# @TEST-EXEC: zeek --parse-only ./manager.zeek ./worker.zeek +# +# @TEST-EXEC: btest-bg-run manager "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=manager zeek -b ../manager.zeek >out" +# @TEST-EXEC: btest-bg-run worker "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=worker-1 zeek -b ../worker.zeek >out" +# +# @TEST-EXEC: btest-bg-wait 30 +# @TEST-EXEC: btest-diff ./manager/out + +# @TEST-START-FILE common.zeek +@load ./zeromq-test-bootstrap + +global test: event(c: count) &is_used; + +# @TEST-END-FILE + +# @TEST-START-FILE manager.zeek +@load ./common.zeek + +event zeek_init() &priority=1000000 + { + Cluster::subscribe("test.early"); + } + +event test(c: count) + { + print "test()", c; + } + +event Cluster::node_down(name: string, id: string) + { + terminate(); + } +# @TEST-END-FILE + +# @TEST-START-FILE worker.zeek +@load ./common.zeek +event Cluster::node_up(name: string, id: string) + { + Cluster::publish("test.early", test, 42); + terminate(); + } +# @TEST-END-FILE From d6795320f76e0301736264b995dd5af59b3ed24a Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 25 Sep 2025 17:52:48 -0700 Subject: [PATCH 04/14] Wrap handling of g_frame_stack in checks for DEBUG --- src/Func.cc | 6 ++++++ src/zeek-setup.cc | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/src/Func.cc b/src/Func.cc index e6047df71e..ac64d64d28 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -339,7 +339,9 @@ ValPtr ScriptFunc::Invoke(zeek::Args* args, Frame* parent) const { f->SetTriggerAssoc(parent->GetTriggerAssoc()); } +#if DEBUG g_frame_stack.push_back(f.get()); // used for backtracing +#endif const CallExpr* call_expr = parent ? parent->GetCall() : nullptr; call_stack.emplace_back(CallInfo{call_expr, this, *args}); @@ -391,7 +393,9 @@ ValPtr ScriptFunc::Invoke(zeek::Args* args, Frame* parent) const { catch ( InterpreterException& e ) { // Already reported, but now determine whether to unwind further. if ( Flavor() == FUNC_FLAVOR_FUNCTION ) { +#if DEBUG g_frame_stack.pop_back(); +#endif call_stack.pop_back(); // Result not set b/c exception was thrown throw; @@ -448,7 +452,9 @@ ValPtr ScriptFunc::Invoke(zeek::Args* args, Frame* parent) const { g_trace_state.LogTrace("Function return: %s\n", d.Description()); } +#if DEBUG g_frame_stack.pop_back(); +#endif return result; } diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index 070aad5308..119e8b1e70 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -1105,7 +1105,9 @@ SetupResult setup(int argc, char** argv, Options* zopts) { auto [body, scope] = get_global_stmts(); StmtFlowType flow; Frame f(scope->Length(), nullptr, nullptr); +#ifdef DEBUG g_frame_stack.push_back(&f); +#endif try { body->Exec(&f, flow); @@ -1113,7 +1115,9 @@ SetupResult setup(int argc, char** argv, Options* zopts) { reporter->FatalError("failed to execute script statements at top-level scope"); } +#ifdef DEBUG g_frame_stack.pop_back(); +#endif } clear_script_analysis(); From bb73499cf28cca55b53157741557fa3a30a229fa Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 25 Sep 2025 17:53:57 -0700 Subject: [PATCH 05/14] Disable script tracing on non-debug builds --- src/Debug.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Debug.h b/src/Debug.h index 3d256ca4a8..7748beb1e5 100644 --- a/src/Debug.h +++ b/src/Debug.h @@ -60,7 +60,13 @@ public: // Returns previous filename. FILE* SetTraceFile(const char* trace_filename); - bool DoTrace() const { return dbgtrace; } + bool DoTrace() const { +#if DEBUG + return dbgtrace; +#else + return false; +#endif + } void TraceOn(); void TraceOff(); From 330cb83526a2e21a69bf1bd29e6c577787c50ebe Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 25 Sep 2025 17:55:32 -0700 Subject: [PATCH 06/14] Return an error if script tracing/debugger arguments are passed to non-debug builds --- src/Options.cc | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/Options.cc b/src/Options.cc index ef771cc984..40b291f3d6 100644 --- a/src/Options.cc +++ b/src/Options.cc @@ -445,7 +445,15 @@ Options parse_cmdline(int argc, char** argv) { case 'a': rval.parse_only = true; break; case 'b': rval.bare_mode = true; break; case 'c': rval.unprocessed_output_file = optarg; break; - case 'd': rval.debug_scripts = true; break; + case 'd': +#if DEBUG + rval.debug_scripts = true; + break; +#else + fprintf(stderr, "ERROR: Debugger is disabled in non-debug builds\n"); + exit(1); + break; +#endif case 'e': rval.script_code_to_exec = optarg; break; case 'f': rval.pcap_filter = optarg; break; case 'h': rval.print_usage = true; break; @@ -489,7 +497,15 @@ Options parse_cmdline(int argc, char** argv) { rval.pcap_file = optarg; break; case 's': rval.signature_files.emplace_back(optarg); break; - case 't': rval.debug_script_tracing_file = optarg; break; + case 't': +#ifdef DEBUG + rval.debug_script_tracing_file = optarg; + break; +#else + fprintf(stderr, "ERROR: Script tracing is disabled in non-debug builds\n"); + exit(1); + break; +#endif case 'u': ++rval.analysis_options.usage_issues; break; case 'v': rval.print_version = true; break; case 'V': rval.print_build_info = true; break; From 651ac3cf793f560e4ba01fd53991756a52823d71 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 25 Sep 2025 18:35:21 -0700 Subject: [PATCH 07/14] Disable Stmt hooks for the built-in debugger if not a debug build --- src/Stmt.cc | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/Stmt.cc b/src/Stmt.cc index 9efda21df1..08a88deee0 100644 --- a/src/Stmt.cc +++ b/src/Stmt.cc @@ -147,6 +147,7 @@ const AssertStmt* Stmt::AsAssertStmt() const { } bool Stmt::SetLocationInfo(const Location* start, const Location* end) { +#if DEBUG if ( ! Obj::SetLocationInfo(start, end) ) return false; @@ -177,6 +178,9 @@ bool Stmt::SetLocationInfo(const Location* start, const Location* end) { } return true; +#else + return Obj::SetLocationInfo(start, end); +#endif } bool Stmt::IsPure() const { return false; } @@ -436,13 +440,17 @@ ValPtr IfStmt::DoExec(Frame* f, Val* v, StmtFlowType& flow) { f->SetNextStmt(do_stmt); +#if DEBUG if ( ! pre_execute_stmt(do_stmt, f) ) { // ### Abort or something } +#endif auto result = do_stmt->Exec(f, flow); +#if DEBUG if ( ! post_execute_stmt(do_stmt, f, result.get(), &flow) ) { // ### Abort or something } +#endif return result; } @@ -1409,13 +1417,17 @@ ValPtr StmtList::Exec(Frame* f, StmtFlowType& flow) { f->SetNextStmt(stmt); +#if DEBUG if ( ! pre_execute_stmt(stmt, f) ) { // ### Abort or something } +#endif auto result = stmt->Exec(f, flow); +#if DEBUG if ( ! post_execute_stmt(stmt, f, result.get(), &flow) ) { // ### Abort or something } +#endif if ( flow != FLOW_NEXT || result || f->HasDelayed() ) return result; From 65478f8dd0d31ee393b51a00300f3ef2b15d25fa Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 26 Sep 2025 04:22:46 +0000 Subject: [PATCH 08/14] fixup! Return an error if script tracing/debugger arguments are passed to non-debug builds --- testing/btest/spicy/file-analyzer-nested.zeek | 1 + 1 file changed, 1 insertion(+) diff --git a/testing/btest/spicy/file-analyzer-nested.zeek b/testing/btest/spicy/file-analyzer-nested.zeek index 98dccbe7f0..35931a312c 100644 --- a/testing/btest/spicy/file-analyzer-nested.zeek +++ b/testing/btest/spicy/file-analyzer-nested.zeek @@ -1,4 +1,5 @@ # @TEST-REQUIRES: have-spicy +# @TEST-REQUIRES: test "$($BUILD/zeek-config --build_type)" = "debug" # # @TEST-EXEC: spicyz -d -o text.hlto text.spicy ./text.evt # @TEST-EXEC: zeek -r ${TRACES}/http/post.trace text.hlto %INPUT Spicy::enable_print=T | sort -k 3 >output From 31d7df915e20b855d5b78f04d9e3ac096d2fa7e3 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Mon, 29 Sep 2025 14:41:11 +0200 Subject: [PATCH 09/14] Supervisor: Make last_signal atomic to squelch data race When the stem process terminates and SIGCHLD is sent to the supervisor, the signal might be handled by the main thread or any other threads that aren't blocking SIGCHLD explicitly. Convert last_signal to a std::atomic such that non-main threads can safely set last_signal without triggering data race as reported by TSAN. This doesn't make it less racy to work last_signal, but it appears we only use it for debug printing anyhow and another option might have been to just remove last_signal altogether. Follow-up for #4849 --- src/supervisor/Supervisor.cc | 2 +- src/supervisor/Supervisor.h | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/supervisor/Supervisor.cc b/src/supervisor/Supervisor.cc index 48894d1f0e..974367c87c 100644 --- a/src/supervisor/Supervisor.cc +++ b/src/supervisor/Supervisor.cc @@ -415,7 +415,7 @@ static ForkResult fork_with_stdio_redirect(const char* where) { void Supervisor::HandleChildSignal() { if ( last_signal >= 0 ) { - DBG_LOG(DBG_SUPERVISOR, "Supervisor received signal %d", last_signal); + DBG_LOG(DBG_SUPERVISOR, "Supervisor received signal %d", last_signal.load()); last_signal = -1; } diff --git a/src/supervisor/Supervisor.h b/src/supervisor/Supervisor.h index c433c0536a..84c5595138 100644 --- a/src/supervisor/Supervisor.h +++ b/src/supervisor/Supervisor.h @@ -3,6 +3,7 @@ #pragma once #include +#include #include #include #include @@ -323,7 +324,7 @@ private: Config config; pid_t stem_pid; - int last_signal = -1; + std::atomic last_signal = -1; std::unique_ptr stem_pipe; detail::LineBufferedPipe stem_stdout; detail::LineBufferedPipe stem_stderr; From 3c16ee9667debc2aaba6458d2e6c699f8fc7f3f8 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 29 Sep 2025 08:21:01 -0700 Subject: [PATCH 10/14] Update docs submodule [nomail] --- CHANGES | 4 ++++ VERSION | 2 +- doc | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 071e2ed187..c30086ef64 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,7 @@ +8.1.0-dev.610 | 2025-09-29 08:21:01 -0700 + + * Update docs submodule [nomail] (Tim Wojtulewicz, Corelight) + 8.1.0-dev.609 | 2025-09-29 13:08:15 +0200 * cluster/zeromq: Fix Cluster::subscribe() block if not initialized (Arne Welzel, Corelight) diff --git a/VERSION b/VERSION index 4b178aee4a..041d214e50 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -8.1.0-dev.609 +8.1.0-dev.610 diff --git a/doc b/doc index 2731def915..8f38ae2fd5 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 2731def9159247e6da8a3191783c89683363689c +Subproject commit 8f38ae2fd563314393eb1ca58c827d26e9966520 From 9e97e1f4bc0cff2d579286581206c6b90bfa6dd4 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 25 Sep 2025 17:52:48 -0700 Subject: [PATCH 11/14] Wrap handling of g_frame_stack in checks for DEBUG --- src/Func.cc | 6 ++++++ src/zeek-setup.cc | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/src/Func.cc b/src/Func.cc index e6047df71e..ac64d64d28 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -339,7 +339,9 @@ ValPtr ScriptFunc::Invoke(zeek::Args* args, Frame* parent) const { f->SetTriggerAssoc(parent->GetTriggerAssoc()); } +#if DEBUG g_frame_stack.push_back(f.get()); // used for backtracing +#endif const CallExpr* call_expr = parent ? parent->GetCall() : nullptr; call_stack.emplace_back(CallInfo{call_expr, this, *args}); @@ -391,7 +393,9 @@ ValPtr ScriptFunc::Invoke(zeek::Args* args, Frame* parent) const { catch ( InterpreterException& e ) { // Already reported, but now determine whether to unwind further. if ( Flavor() == FUNC_FLAVOR_FUNCTION ) { +#if DEBUG g_frame_stack.pop_back(); +#endif call_stack.pop_back(); // Result not set b/c exception was thrown throw; @@ -448,7 +452,9 @@ ValPtr ScriptFunc::Invoke(zeek::Args* args, Frame* parent) const { g_trace_state.LogTrace("Function return: %s\n", d.Description()); } +#if DEBUG g_frame_stack.pop_back(); +#endif return result; } diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index 070aad5308..119e8b1e70 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -1105,7 +1105,9 @@ SetupResult setup(int argc, char** argv, Options* zopts) { auto [body, scope] = get_global_stmts(); StmtFlowType flow; Frame f(scope->Length(), nullptr, nullptr); +#ifdef DEBUG g_frame_stack.push_back(&f); +#endif try { body->Exec(&f, flow); @@ -1113,7 +1115,9 @@ SetupResult setup(int argc, char** argv, Options* zopts) { reporter->FatalError("failed to execute script statements at top-level scope"); } +#ifdef DEBUG g_frame_stack.pop_back(); +#endif } clear_script_analysis(); From f7bcc63ea8132a1d5d9613fb9c1757e26fd48c96 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 25 Sep 2025 17:53:57 -0700 Subject: [PATCH 12/14] Disable script tracing on non-debug builds --- src/Debug.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Debug.h b/src/Debug.h index 3d256ca4a8..7748beb1e5 100644 --- a/src/Debug.h +++ b/src/Debug.h @@ -60,7 +60,13 @@ public: // Returns previous filename. FILE* SetTraceFile(const char* trace_filename); - bool DoTrace() const { return dbgtrace; } + bool DoTrace() const { +#if DEBUG + return dbgtrace; +#else + return false; +#endif + } void TraceOn(); void TraceOff(); From fcd94157e1804117f9322b2147adbcda9d5c9a96 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 25 Sep 2025 17:55:32 -0700 Subject: [PATCH 13/14] Return an error if script tracing/debugger arguments are passed to non-debug builds --- src/Options.cc | 20 +++++++++++++++++-- testing/btest/spicy/file-analyzer-nested.zeek | 1 + 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/Options.cc b/src/Options.cc index ef771cc984..40b291f3d6 100644 --- a/src/Options.cc +++ b/src/Options.cc @@ -445,7 +445,15 @@ Options parse_cmdline(int argc, char** argv) { case 'a': rval.parse_only = true; break; case 'b': rval.bare_mode = true; break; case 'c': rval.unprocessed_output_file = optarg; break; - case 'd': rval.debug_scripts = true; break; + case 'd': +#if DEBUG + rval.debug_scripts = true; + break; +#else + fprintf(stderr, "ERROR: Debugger is disabled in non-debug builds\n"); + exit(1); + break; +#endif case 'e': rval.script_code_to_exec = optarg; break; case 'f': rval.pcap_filter = optarg; break; case 'h': rval.print_usage = true; break; @@ -489,7 +497,15 @@ Options parse_cmdline(int argc, char** argv) { rval.pcap_file = optarg; break; case 's': rval.signature_files.emplace_back(optarg); break; - case 't': rval.debug_script_tracing_file = optarg; break; + case 't': +#ifdef DEBUG + rval.debug_script_tracing_file = optarg; + break; +#else + fprintf(stderr, "ERROR: Script tracing is disabled in non-debug builds\n"); + exit(1); + break; +#endif case 'u': ++rval.analysis_options.usage_issues; break; case 'v': rval.print_version = true; break; case 'V': rval.print_build_info = true; break; diff --git a/testing/btest/spicy/file-analyzer-nested.zeek b/testing/btest/spicy/file-analyzer-nested.zeek index 98dccbe7f0..35931a312c 100644 --- a/testing/btest/spicy/file-analyzer-nested.zeek +++ b/testing/btest/spicy/file-analyzer-nested.zeek @@ -1,4 +1,5 @@ # @TEST-REQUIRES: have-spicy +# @TEST-REQUIRES: test "$($BUILD/zeek-config --build_type)" = "debug" # # @TEST-EXEC: spicyz -d -o text.hlto text.spicy ./text.evt # @TEST-EXEC: zeek -r ${TRACES}/http/post.trace text.hlto %INPUT Spicy::enable_print=T | sort -k 3 >output From e7aad5e3e056fd3a962bb12fca814734d99e0719 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 25 Sep 2025 18:35:21 -0700 Subject: [PATCH 14/14] Disable Stmt hooks for the built-in debugger if not a debug build --- src/Stmt.cc | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/Stmt.cc b/src/Stmt.cc index 9efda21df1..08a88deee0 100644 --- a/src/Stmt.cc +++ b/src/Stmt.cc @@ -147,6 +147,7 @@ const AssertStmt* Stmt::AsAssertStmt() const { } bool Stmt::SetLocationInfo(const Location* start, const Location* end) { +#if DEBUG if ( ! Obj::SetLocationInfo(start, end) ) return false; @@ -177,6 +178,9 @@ bool Stmt::SetLocationInfo(const Location* start, const Location* end) { } return true; +#else + return Obj::SetLocationInfo(start, end); +#endif } bool Stmt::IsPure() const { return false; } @@ -436,13 +440,17 @@ ValPtr IfStmt::DoExec(Frame* f, Val* v, StmtFlowType& flow) { f->SetNextStmt(do_stmt); +#if DEBUG if ( ! pre_execute_stmt(do_stmt, f) ) { // ### Abort or something } +#endif auto result = do_stmt->Exec(f, flow); +#if DEBUG if ( ! post_execute_stmt(do_stmt, f, result.get(), &flow) ) { // ### Abort or something } +#endif return result; } @@ -1409,13 +1417,17 @@ ValPtr StmtList::Exec(Frame* f, StmtFlowType& flow) { f->SetNextStmt(stmt); +#if DEBUG if ( ! pre_execute_stmt(stmt, f) ) { // ### Abort or something } +#endif auto result = stmt->Exec(f, flow); +#if DEBUG if ( ! post_execute_stmt(stmt, f, result.get(), &flow) ) { // ### Abort or something } +#endif if ( flow != FLOW_NEXT || result || f->HasDelayed() ) return result;