diff --git a/CHANGES b/CHANGES index ad02895126..4efd02a76a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,48 @@ +8.1.0-dev.612 | 2025-09-29 18:04:24 +0200 + + * Supervisor: Make last_signal atomic to squelch data race (Arne Welzel, Corelight) + + When the stem process terminates and SIGCHLD is sent to the supervisor, + the signal might be handled by the main thread or any other threads that + aren't blocking SIGCHLD explicitly. Convert last_signal to a std::atomic + such that non-main threads can safely set last_signal without triggering + data race as reported by TSAN. This doesn't make it less racy to work + last_signal, but it appears we only use it for debug printing anyhow and + another option might have been to just remove last_signal altogether. + + Follow-up for #4849 + +8.1.0-dev.610 | 2025-09-29 08:21:01 -0700 + + * Update docs submodule [nomail] (Tim Wojtulewicz, Corelight) + +8.1.0-dev.609 | 2025-09-29 13:08:15 +0200 + + * cluster/zeromq: Fix Cluster::subscribe() block if not initialized (Arne Welzel, Corelight) + + If Cluster::init() hasn't been invoked yet, Cluster::subscribe() with the + ZeroMQ backend would block because the main_inproc socket didn't + yet have a connection from the child thread. Prevent this by connecting + the main and child socket pair at construction time. + + This will queue the subscriptions and start processing them once the + child thread has started. + +8.1.0-dev.607 | 2025-09-26 14:19:40 -0700 + + * Fixes for -O gen-standalone-C++ for tracking BiFs, lambdas, attribute types, and independent globals (Vern Paxson, Corelight) + +8.1.0-dev.605 | 2025-09-26 11:19:17 -0700 + + * OpaqueVal, OCSP, X509: drop outdated LibreSSL guards to fix OpenBSD (Klemens Nanni) + build + + Whatever is used with recent OpenSSL is also available with latest LibreSSL + on OpenBSD 7.8-beta as of today. + + Some of these hunks have been in the net/bro port for years, others I + recently added whilst gradually updating from 6.0.5 to 8.0.1. + 8.1.0-dev.603 | 2025-09-26 02:56:58 +0000 * Ignore src/3rdparty for pre-commit (Tim Wojtulewicz, Corelight) diff --git a/NEWS b/NEWS index eedb83e78c..bfb057d2f6 100644 --- a/NEWS +++ b/NEWS @@ -6,8 +6,8 @@ release. For an exhaustive list of changes, see the ``CHANGES`` file Zeek 8.1.0 ========== -We would like to thank @chrisjlly and Klemens Nanni (@klemens-ya) for their contributions -to this release. +We would like to thank @chrisjlly, Klemens Nanni (@klemensn), and Klemens Nanni +(@klemens-ya) for their contributions to this release. Breaking Changes ---------------- diff --git a/VERSION b/VERSION index 96f5f4cbeb..2e45d2803d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -8.1.0-dev.603 +8.1.0-dev.612 diff --git a/doc b/doc index 2731def915..8f38ae2fd5 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 2731def9159247e6da8a3191783c89683363689c +Subproject commit 8f38ae2fd563314393eb1ca58c827d26e9966520 diff --git a/src/OpaqueVal.cc b/src/OpaqueVal.cc index adc309fc4a..f318f4bdb7 100644 --- a/src/OpaqueVal.cc +++ b/src/OpaqueVal.cc @@ -27,11 +27,11 @@ #include "zeek/probabilistic/BloomFilter.h" #include "zeek/probabilistic/CardinalityCounter.h" -#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) inline void* EVP_MD_CTX_md_data(const EVP_MD_CTX* ctx) { return ctx->md_data; } #endif -#if ( OPENSSL_VERSION_NUMBER < 0x30000000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x30000000L ) #include #endif @@ -225,7 +225,7 @@ constexpr size_t SHA1VAL_STATE_SIZE = sizeof(SHA_CTX); constexpr size_t SHA256VAL_STATE_SIZE = sizeof(SHA256_CTX); -#if ( OPENSSL_VERSION_NUMBER < 0x30000000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x30000000L ) // -- MD5 diff --git a/src/cluster/backend/zeromq/ZeroMQ.cc b/src/cluster/backend/zeromq/ZeroMQ.cc index 4085179b8b..4ce1525987 100644 --- a/src/cluster/backend/zeromq/ZeroMQ.cc +++ b/src/cluster/backend/zeromq/ZeroMQ.cc @@ -85,6 +85,7 @@ ZeroMQBackend::ZeroMQBackend(std::unique_ptr es, std::unique_pt : ThreadedBackend("ZeroMQ", std::move(es), std::move(ls), std::move(ehs), new zeek::detail::OnLoopProcess(this, "ZeroMQ", onloop_queue_hwm)), main_inproc(zmq::socket_t(ctx, zmq::socket_type::pair)), + child_inproc(zmq::socket_t(ctx, zmq::socket_type::pair)), // Counters for block and drop metrics. total_xpub_drops( zeek::telemetry_mgr->CounterInstance("zeek", "cluster_zeromq_xpub_drops", {}, @@ -94,7 +95,13 @@ ZeroMQBackend::ZeroMQBackend(std::unique_ptr es, std::unique_pt "Number of received events dropped due to OnLoop queue full.")), total_msg_errors( zeek::telemetry_mgr->CounterInstance("zeek", "cluster_zeromq_msg_errors", {}, - "Number of events with the wrong number of message parts.")) {} + "Number of events with the wrong number of message parts.")) { + // Establish the socket connection between main thread and child thread + // already in the constructor. This allows Subscribe() and Unsubscribe() + // calls to be delayed until DoInit() was called. + main_inproc.bind("inproc://inproc-bridge"); + child_inproc.connect("inproc://inproc-bridge"); +} ZeroMQBackend::~ZeroMQBackend() { try { @@ -169,10 +176,11 @@ void ZeroMQBackend::DoTerminate() { ctx.shutdown(); // Close the sockets that are used from the main thread, - // the remaining sockets were closed by self_thread during - // shutdown already. + // the remaining sockets except for the child_inproc one + // were closed by self_thread during shutdown already. log_push.close(); main_inproc.close(); + child_inproc.close(); ZEROMQ_DEBUG("Closing ctx"); ctx.close(); @@ -197,7 +205,6 @@ bool ZeroMQBackend::DoInit() { xpub = zmq::socket_t(ctx, zmq::socket_type::xpub); log_push = zmq::socket_t(ctx, zmq::socket_type::push); log_pull = zmq::socket_t(ctx, zmq::socket_type::pull); - child_inproc = zmq::socket_t(ctx, zmq::socket_type::pair); xpub.set(zmq::sockopt::linger, linger_ms); @@ -286,10 +293,6 @@ bool ZeroMQBackend::DoInit() { // // https://funcptr.net/2012/09/10/zeromq---edge-triggered-notification/ - // Setup connectivity between main and child thread. - main_inproc.bind("inproc://inproc-bridge"); - child_inproc.connect("inproc://inproc-bridge"); - // Thread is joined in backend->DoTerminate(), backend outlives it. self_thread = std::thread([](auto* backend) { backend->Run(); }, this); @@ -634,7 +637,6 @@ void ZeroMQBackend::Run() { // Called when Run() terminates. auto deferred_close = util::Deferred([this]() { - child_inproc.close(); xpub.close(); xsub.close(); log_pull.close(); diff --git a/src/file_analysis/analyzer/x509/OCSP.cc b/src/file_analysis/analyzer/x509/OCSP.cc index adee5b9624..d46eb72b2e 100644 --- a/src/file_analysis/analyzer/x509/OCSP.cc +++ b/src/file_analysis/analyzer/x509/OCSP.cc @@ -26,7 +26,7 @@ namespace zeek::file_analysis::detail { static constexpr size_t OCSP_STRING_BUF_SIZE = 2048; static bool OCSP_RESPID_bio(OCSP_BASICRESP* basic_resp, BIO* bio) { -#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) ASN1_OCTET_STRING* key = nullptr; X509_NAME* name = nullptr; @@ -353,7 +353,7 @@ void OCSP::ParseRequest(OCSP_REQUEST* req) { uint64_t version = 0; -#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) if ( req->tbsRequest->version ) version = (uint64_t)ASN1_INTEGER_get(req->tbsRequest->version); #else @@ -425,7 +425,7 @@ void OCSP::ParseResponse(OCSP_RESPONSE* resp) { if ( ! basic_resp ) goto clean_up; -#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) resp_data = basic_resp->tbsResponseData; if ( ! resp_data ) goto clean_up; @@ -434,7 +434,7 @@ void OCSP::ParseResponse(OCSP_RESPONSE* resp) { vl.emplace_back(GetFile()->ToVal()); vl.emplace_back(std::move(status_val)); -#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) vl.emplace_back(val_mgr->Count((uint64_t)ASN1_INTEGER_get(resp_data->version))); #else vl.emplace_back(parse_basic_resp_data_version(basic_resp)); @@ -452,7 +452,7 @@ void OCSP::ParseResponse(OCSP_RESPONSE* resp) { } // producedAt -#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) produced_at = resp_data->producedAt; #else produced_at = OCSP_resp_get0_produced_at(basic_resp); @@ -477,7 +477,7 @@ void OCSP::ParseResponse(OCSP_RESPONSE* resp) { // cert id const OCSP_CERTID* cert_id = nullptr; -#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) cert_id = single_resp->certId; #else cert_id = OCSP_SINGLERESP_get0_id(single_resp); @@ -550,7 +550,7 @@ void OCSP::ParseResponse(OCSP_RESPONSE* resp) { } } -#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) i2a_ASN1_OBJECT(bio, basic_resp->signatureAlgorithm->algorithm); len = BIO_read(bio, buf, sizeof(buf)); vl.emplace_back(make_intrusive(len, buf)); @@ -567,7 +567,7 @@ void OCSP::ParseResponse(OCSP_RESPONSE* resp) { certs_vector = new VectorVal(id::find_type("x509_opaque_vector")); vl.emplace_back(AdoptRef{}, certs_vector); -#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) certs = basic_resp->certs; #else certs = OCSP_resp_get0_certs(basic_resp); diff --git a/src/file_analysis/analyzer/x509/X509.h b/src/file_analysis/analyzer/x509/X509.h index 4295f68790..7fffb6729a 100644 --- a/src/file_analysis/analyzer/x509/X509.h +++ b/src/file_analysis/analyzer/x509/X509.h @@ -9,13 +9,13 @@ #include "zeek/OpaqueVal.h" #include "zeek/file_analysis/analyzer/x509/X509Common.h" -#if ( OPENSSL_VERSION_NUMBER < 0x10002000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x10002000L ) #define X509_get_signature_nid(x) OBJ_obj2nid((x)->sig_alg->algorithm) #endif -#if ( OPENSSL_VERSION_NUMBER < 0x1010000fL ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x1010000fL ) #define X509_OBJECT_new() (X509_OBJECT*)malloc(sizeof(X509_OBJECT)) #define X509_OBJECT_free(a) free(a) diff --git a/src/file_analysis/analyzer/x509/functions.bif b/src/file_analysis/analyzer/x509/functions.bif index 3be7dcd41d..b5b097f09c 100644 --- a/src/file_analysis/analyzer/x509/functions.bif +++ b/src/file_analysis/analyzer/x509/functions.bif @@ -65,7 +65,7 @@ X509* x509_get_ocsp_signer(const STACK_OF(X509)* certs, const ASN1_OCTET_STRING* key = nullptr; const X509_NAME* name = nullptr; -#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) OCSP_RESPID* resp_id = basic_resp->tbsResponseData->responderId; if ( resp_id->type == V_OCSP_RESPID_NAME ) @@ -359,7 +359,7 @@ function x509_ocsp_verify%(certs: x509_opaque_vector, ocsp_reply: string, root_c // Because we actually want to be able to give nice error messages that show why we were // not able to verify the OCSP response - do our own verification logic first. -#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x10100000L ) signer = x509_get_ocsp_signer(basic->certs, basic); #else signer = x509_get_ocsp_signer(OCSP_resp_get0_certs(basic), basic); @@ -730,7 +730,7 @@ function sct_verify%(cert: opaque of x509, logid: string, log_key: string, signa uint32_t cert_length; if ( precert ) { -#if ( OPENSSL_VERSION_NUMBER < 0x10002000L ) || defined(LIBRESSL_VERSION_NUMBER) +#if ( OPENSSL_VERSION_NUMBER < 0x10002000L ) x->cert_info->enc.modified = 1; cert_length = i2d_X509_CINF(x->cert_info, &cert_out); #else diff --git a/src/script_opt/CPP/Driver.cc b/src/script_opt/CPP/Driver.cc index b7cfd4a625..06037eeb42 100644 --- a/src/script_opt/CPP/Driver.cc +++ b/src/script_opt/CPP/Driver.cc @@ -83,9 +83,22 @@ void CPPCompile::Compile(bool report_uncompilable) { accessed_globals.insert(g); for ( auto& ag : pf->AllGlobals() ) all_accessed_globals.insert(ag); + for ( auto& l : pf->Lambdas() ) + // We might not have profiled this previously if none + // of the functions refer to the global. This can + // happen for example for a global "const" table that's + // made available for external lookup use. + pfs->ProfileLambda(l); } } + for ( auto& g : pfs->BiFGlobals() ) + all_accessed_globals.insert(g); + + for ( auto& t : pfs->MainTypes() ) + if ( obj_matches_opt_files(t) == AnalyzeDecision::SHOULD ) + rep_types.insert(TypeRep(t)); + for ( auto& l : pfs->Lambdas() ) if ( obj_matches_opt_files(l) == AnalyzeDecision::SHOULD ) accessed_lambdas.insert(l); diff --git a/src/script_opt/ProfileFunc.cc b/src/script_opt/ProfileFunc.cc index 95bcd405e6..0bf0d7ff96 100644 --- a/src/script_opt/ProfileFunc.cc +++ b/src/script_opt/ProfileFunc.cc @@ -622,6 +622,20 @@ ProfileFuncs::ProfileFuncs(std::vector& funcs, is_compilable_pred pred ComputeSideEffects(); } +void ProfileFuncs::ProfileLambda(const LambdaExpr* l) { + if ( lambdas.contains(l) ) + return; + + lambdas.insert(l); + pending_exprs.push_back(l); + + do + DrainPendingExprs(); + while ( ! pending_exprs.empty() ); + + AnalyzeLambdaProfile(l); +} + bool ProfileFuncs::IsTableWithDefaultAggr(const Type* t) { auto analy = tbl_has_aggr_default.find(t); if ( analy != tbl_has_aggr_default.end() ) @@ -848,14 +862,22 @@ void ProfileFuncs::ComputeBodyHashes(std::vector& funcs) { ComputeProfileHash(f.ProfilePtr()); } - for ( auto& l : lambdas ) { - auto pf = ExprProf(l); - func_profs[l->PrimaryFunc().get()] = pf; - lambda_primaries[l->Name()] = l->PrimaryFunc().get(); + for ( auto& l : lambdas ) + AnalyzeLambdaProfile(l); +} - if ( compute_func_hashes || ! pf->HasHashVal() ) - ComputeProfileHash(pf); - } +void ProfileFuncs::AnalyzeLambdaProfile(const LambdaExpr* l) { + if ( processed_lambdas.contains(l) ) + return; + + processed_lambdas.insert(l); + + auto pf = ExprProf(l); + func_profs[l->PrimaryFunc().get()] = pf; + lambda_primaries[l->Name()] = l->PrimaryFunc().get(); + + if ( compute_func_hashes || ! pf->HasHashVal() ) + ComputeProfileHash(pf); } void ProfileFuncs::ComputeProfileHash(std::shared_ptr pf) { diff --git a/src/script_opt/ProfileFunc.h b/src/script_opt/ProfileFunc.h index d4ad16300e..7b6186eb06 100644 --- a/src/script_opt/ProfileFunc.h +++ b/src/script_opt/ProfileFunc.h @@ -366,6 +366,10 @@ public: ProfileFuncs(std::vector& funcs, is_compilable_pred pred, bool compute_func_hashes, bool full_record_hashes); + // Used to profile additional lambdas that (potentially) weren't part + // of the overall function profiling. + void ProfileLambda(const LambdaExpr* l); + // The following accessors provide a global profile across all of // the (non-skipped) functions in "funcs". See the comments for // the associated member variables for documentation. @@ -449,6 +453,9 @@ protected: // Compute hashes to associate with each function void ComputeBodyHashes(std::vector& funcs); + // For a given lambda, completes analysis of its profile. + void AnalyzeLambdaProfile(const LambdaExpr* l); + // Compute the hash associated with a single function profile. void ComputeProfileHash(std::shared_ptr pf); @@ -540,6 +547,10 @@ protected: // And for lambda's. std::unordered_set lambdas; + // Lambdas that we have already processed. An optimization to avoid + // unnecessary work. + std::unordered_set processed_lambdas; + // Names of generated events. std::unordered_set events; diff --git a/src/supervisor/Supervisor.cc b/src/supervisor/Supervisor.cc index 48894d1f0e..974367c87c 100644 --- a/src/supervisor/Supervisor.cc +++ b/src/supervisor/Supervisor.cc @@ -415,7 +415,7 @@ static ForkResult fork_with_stdio_redirect(const char* where) { void Supervisor::HandleChildSignal() { if ( last_signal >= 0 ) { - DBG_LOG(DBG_SUPERVISOR, "Supervisor received signal %d", last_signal); + DBG_LOG(DBG_SUPERVISOR, "Supervisor received signal %d", last_signal.load()); last_signal = -1; } diff --git a/src/supervisor/Supervisor.h b/src/supervisor/Supervisor.h index c433c0536a..84c5595138 100644 --- a/src/supervisor/Supervisor.h +++ b/src/supervisor/Supervisor.h @@ -3,6 +3,7 @@ #pragma once #include +#include #include #include #include @@ -323,7 +324,7 @@ private: Config config; pid_t stem_pid; - int last_signal = -1; + std::atomic last_signal = -1; std::unique_ptr stem_pipe; detail::LineBufferedPipe stem_stdout; detail::LineBufferedPipe stem_stderr; diff --git a/testing/btest/Baseline/cluster.zeromq.subscribe-before-init/..manager.out b/testing/btest/Baseline/cluster.zeromq.subscribe-before-init/..manager.out new file mode 100644 index 0000000000..e4f236ab1a --- /dev/null +++ b/testing/btest/Baseline/cluster.zeromq.subscribe-before-init/..manager.out @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +test(), 42 diff --git a/testing/btest/cluster/zeromq/subscribe-before-init.zeek b/testing/btest/cluster/zeromq/subscribe-before-init.zeek new file mode 100644 index 0000000000..4040fd2399 --- /dev/null +++ b/testing/btest/cluster/zeromq/subscribe-before-init.zeek @@ -0,0 +1,55 @@ +# @TEST-DOC: Regression test Cluster::subscribe() blocking if called in a high-priority zeek_init() handler +# +# @TEST-REQUIRES: have-zeromq +# +# @TEST-GROUP: cluster-zeromq +# +# @TEST-PORT: XPUB_PORT +# @TEST-PORT: XSUB_PORT +# @TEST-PORT: LOG_PULL_PORT +# +# @TEST-EXEC: cp $FILES/zeromq/cluster-layout-simple.zeek cluster-layout.zeek +# @TEST-EXEC: cp $FILES/zeromq/test-bootstrap.zeek zeromq-test-bootstrap.zeek +# +# @TEST-EXEC: zeek --parse-only ./manager.zeek ./worker.zeek +# +# @TEST-EXEC: btest-bg-run manager "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=manager zeek -b ../manager.zeek >out" +# @TEST-EXEC: btest-bg-run worker "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=worker-1 zeek -b ../worker.zeek >out" +# +# @TEST-EXEC: btest-bg-wait 30 +# @TEST-EXEC: btest-diff ./manager/out + +# @TEST-START-FILE common.zeek +@load ./zeromq-test-bootstrap + +global test: event(c: count) &is_used; + +# @TEST-END-FILE + +# @TEST-START-FILE manager.zeek +@load ./common.zeek + +event zeek_init() &priority=1000000 + { + Cluster::subscribe("test.early"); + } + +event test(c: count) + { + print "test()", c; + } + +event Cluster::node_down(name: string, id: string) + { + terminate(); + } +# @TEST-END-FILE + +# @TEST-START-FILE worker.zeek +@load ./common.zeek +event Cluster::node_up(name: string, id: string) + { + Cluster::publish("test.early", test, 42); + terminate(); + } +# @TEST-END-FILE