From 550c7eb0a758f0ff0e4b7c2af1621819e8dc7c12 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Wed, 24 Sep 2025 17:34:35 -0700 Subject: [PATCH 1/5] Fixes for -O gen-standalone-C++ for tracking BiFs, lambdas, attribute types, and independent globals --- src/script_opt/CPP/Driver.cc | 13 +++++++++++++ src/script_opt/ProfileFunc.cc | 36 ++++++++++++++++++++++++++++------- src/script_opt/ProfileFunc.h | 11 +++++++++++ 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/src/script_opt/CPP/Driver.cc b/src/script_opt/CPP/Driver.cc index b7cfd4a625..06037eeb42 100644 --- a/src/script_opt/CPP/Driver.cc +++ b/src/script_opt/CPP/Driver.cc @@ -83,9 +83,22 @@ void CPPCompile::Compile(bool report_uncompilable) { accessed_globals.insert(g); for ( auto& ag : pf->AllGlobals() ) all_accessed_globals.insert(ag); + for ( auto& l : pf->Lambdas() ) + // We might not have profiled this previously if none + // of the functions refer to the global. This can + // happen for example for a global "const" table that's + // made available for external lookup use. + pfs->ProfileLambda(l); } } + for ( auto& g : pfs->BiFGlobals() ) + all_accessed_globals.insert(g); + + for ( auto& t : pfs->MainTypes() ) + if ( obj_matches_opt_files(t) == AnalyzeDecision::SHOULD ) + rep_types.insert(TypeRep(t)); + for ( auto& l : pfs->Lambdas() ) if ( obj_matches_opt_files(l) == AnalyzeDecision::SHOULD ) accessed_lambdas.insert(l); diff --git a/src/script_opt/ProfileFunc.cc b/src/script_opt/ProfileFunc.cc index 95bcd405e6..0bf0d7ff96 100644 --- a/src/script_opt/ProfileFunc.cc +++ b/src/script_opt/ProfileFunc.cc @@ -622,6 +622,20 @@ ProfileFuncs::ProfileFuncs(std::vector& funcs, is_compilable_pred pred ComputeSideEffects(); } +void ProfileFuncs::ProfileLambda(const LambdaExpr* l) { + if ( lambdas.contains(l) ) + return; + + lambdas.insert(l); + pending_exprs.push_back(l); + + do + DrainPendingExprs(); + while ( ! pending_exprs.empty() ); + + AnalyzeLambdaProfile(l); +} + bool ProfileFuncs::IsTableWithDefaultAggr(const Type* t) { auto analy = tbl_has_aggr_default.find(t); if ( analy != tbl_has_aggr_default.end() ) @@ -848,14 +862,22 @@ void ProfileFuncs::ComputeBodyHashes(std::vector& funcs) { ComputeProfileHash(f.ProfilePtr()); } - for ( auto& l : lambdas ) { - auto pf = ExprProf(l); - func_profs[l->PrimaryFunc().get()] = pf; - lambda_primaries[l->Name()] = l->PrimaryFunc().get(); + for ( auto& l : lambdas ) + AnalyzeLambdaProfile(l); +} - if ( compute_func_hashes || ! pf->HasHashVal() ) - ComputeProfileHash(pf); - } +void ProfileFuncs::AnalyzeLambdaProfile(const LambdaExpr* l) { + if ( processed_lambdas.contains(l) ) + return; + + processed_lambdas.insert(l); + + auto pf = ExprProf(l); + func_profs[l->PrimaryFunc().get()] = pf; + lambda_primaries[l->Name()] = l->PrimaryFunc().get(); + + if ( compute_func_hashes || ! pf->HasHashVal() ) + ComputeProfileHash(pf); } void ProfileFuncs::ComputeProfileHash(std::shared_ptr pf) { diff --git a/src/script_opt/ProfileFunc.h b/src/script_opt/ProfileFunc.h index d4ad16300e..7b6186eb06 100644 --- a/src/script_opt/ProfileFunc.h +++ b/src/script_opt/ProfileFunc.h @@ -366,6 +366,10 @@ public: ProfileFuncs(std::vector& funcs, is_compilable_pred pred, bool compute_func_hashes, bool full_record_hashes); + // Used to profile additional lambdas that (potentially) weren't part + // of the overall function profiling. + void ProfileLambda(const LambdaExpr* l); + // The following accessors provide a global profile across all of // the (non-skipped) functions in "funcs". See the comments for // the associated member variables for documentation. @@ -449,6 +453,9 @@ protected: // Compute hashes to associate with each function void ComputeBodyHashes(std::vector& funcs); + // For a given lambda, completes analysis of its profile. + void AnalyzeLambdaProfile(const LambdaExpr* l); + // Compute the hash associated with a single function profile. void ComputeProfileHash(std::shared_ptr pf); @@ -540,6 +547,10 @@ protected: // And for lambda's. std::unordered_set lambdas; + // Lambdas that we have already processed. An optimization to avoid + // unnecessary work. + std::unordered_set processed_lambdas; + // Names of generated events. std::unordered_set events; From 01666df3d78e1c1abe416749714d6171bee788e2 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Thu, 25 Sep 2025 18:48:34 +0200 Subject: [PATCH 2/5] cluster/zeromq: Fix Cluster::subscribe() block if not initialized If Cluster::init() hasn't been invoked yet, Cluster::subscribe() with the ZeroMQ backend would block because the main_inproc socket didn't yet have a connection from the child thread. Prevent this by connecting the main and child socket pair at construction time. This will queue the subscriptions and start processing them once the child thread has started. --- src/cluster/backend/zeromq/ZeroMQ.cc | 20 ++++--- .../..manager.out | 2 + .../cluster/zeromq/subscribe-before-init.zeek | 55 +++++++++++++++++++ 3 files changed, 68 insertions(+), 9 deletions(-) create mode 100644 testing/btest/Baseline/cluster.zeromq.subscribe-before-init/..manager.out create mode 100644 testing/btest/cluster/zeromq/subscribe-before-init.zeek diff --git a/src/cluster/backend/zeromq/ZeroMQ.cc b/src/cluster/backend/zeromq/ZeroMQ.cc index 4085179b8b..4ce1525987 100644 --- a/src/cluster/backend/zeromq/ZeroMQ.cc +++ b/src/cluster/backend/zeromq/ZeroMQ.cc @@ -85,6 +85,7 @@ ZeroMQBackend::ZeroMQBackend(std::unique_ptr es, std::unique_pt : ThreadedBackend("ZeroMQ", std::move(es), std::move(ls), std::move(ehs), new zeek::detail::OnLoopProcess(this, "ZeroMQ", onloop_queue_hwm)), main_inproc(zmq::socket_t(ctx, zmq::socket_type::pair)), + child_inproc(zmq::socket_t(ctx, zmq::socket_type::pair)), // Counters for block and drop metrics. total_xpub_drops( zeek::telemetry_mgr->CounterInstance("zeek", "cluster_zeromq_xpub_drops", {}, @@ -94,7 +95,13 @@ ZeroMQBackend::ZeroMQBackend(std::unique_ptr es, std::unique_pt "Number of received events dropped due to OnLoop queue full.")), total_msg_errors( zeek::telemetry_mgr->CounterInstance("zeek", "cluster_zeromq_msg_errors", {}, - "Number of events with the wrong number of message parts.")) {} + "Number of events with the wrong number of message parts.")) { + // Establish the socket connection between main thread and child thread + // already in the constructor. This allows Subscribe() and Unsubscribe() + // calls to be delayed until DoInit() was called. + main_inproc.bind("inproc://inproc-bridge"); + child_inproc.connect("inproc://inproc-bridge"); +} ZeroMQBackend::~ZeroMQBackend() { try { @@ -169,10 +176,11 @@ void ZeroMQBackend::DoTerminate() { ctx.shutdown(); // Close the sockets that are used from the main thread, - // the remaining sockets were closed by self_thread during - // shutdown already. + // the remaining sockets except for the child_inproc one + // were closed by self_thread during shutdown already. log_push.close(); main_inproc.close(); + child_inproc.close(); ZEROMQ_DEBUG("Closing ctx"); ctx.close(); @@ -197,7 +205,6 @@ bool ZeroMQBackend::DoInit() { xpub = zmq::socket_t(ctx, zmq::socket_type::xpub); log_push = zmq::socket_t(ctx, zmq::socket_type::push); log_pull = zmq::socket_t(ctx, zmq::socket_type::pull); - child_inproc = zmq::socket_t(ctx, zmq::socket_type::pair); xpub.set(zmq::sockopt::linger, linger_ms); @@ -286,10 +293,6 @@ bool ZeroMQBackend::DoInit() { // // https://funcptr.net/2012/09/10/zeromq---edge-triggered-notification/ - // Setup connectivity between main and child thread. - main_inproc.bind("inproc://inproc-bridge"); - child_inproc.connect("inproc://inproc-bridge"); - // Thread is joined in backend->DoTerminate(), backend outlives it. self_thread = std::thread([](auto* backend) { backend->Run(); }, this); @@ -634,7 +637,6 @@ void ZeroMQBackend::Run() { // Called when Run() terminates. auto deferred_close = util::Deferred([this]() { - child_inproc.close(); xpub.close(); xsub.close(); log_pull.close(); diff --git a/testing/btest/Baseline/cluster.zeromq.subscribe-before-init/..manager.out b/testing/btest/Baseline/cluster.zeromq.subscribe-before-init/..manager.out new file mode 100644 index 0000000000..e4f236ab1a --- /dev/null +++ b/testing/btest/Baseline/cluster.zeromq.subscribe-before-init/..manager.out @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +test(), 42 diff --git a/testing/btest/cluster/zeromq/subscribe-before-init.zeek b/testing/btest/cluster/zeromq/subscribe-before-init.zeek new file mode 100644 index 0000000000..4040fd2399 --- /dev/null +++ b/testing/btest/cluster/zeromq/subscribe-before-init.zeek @@ -0,0 +1,55 @@ +# @TEST-DOC: Regression test Cluster::subscribe() blocking if called in a high-priority zeek_init() handler +# +# @TEST-REQUIRES: have-zeromq +# +# @TEST-GROUP: cluster-zeromq +# +# @TEST-PORT: XPUB_PORT +# @TEST-PORT: XSUB_PORT +# @TEST-PORT: LOG_PULL_PORT +# +# @TEST-EXEC: cp $FILES/zeromq/cluster-layout-simple.zeek cluster-layout.zeek +# @TEST-EXEC: cp $FILES/zeromq/test-bootstrap.zeek zeromq-test-bootstrap.zeek +# +# @TEST-EXEC: zeek --parse-only ./manager.zeek ./worker.zeek +# +# @TEST-EXEC: btest-bg-run manager "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=manager zeek -b ../manager.zeek >out" +# @TEST-EXEC: btest-bg-run worker "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=worker-1 zeek -b ../worker.zeek >out" +# +# @TEST-EXEC: btest-bg-wait 30 +# @TEST-EXEC: btest-diff ./manager/out + +# @TEST-START-FILE common.zeek +@load ./zeromq-test-bootstrap + +global test: event(c: count) &is_used; + +# @TEST-END-FILE + +# @TEST-START-FILE manager.zeek +@load ./common.zeek + +event zeek_init() &priority=1000000 + { + Cluster::subscribe("test.early"); + } + +event test(c: count) + { + print "test()", c; + } + +event Cluster::node_down(name: string, id: string) + { + terminate(); + } +# @TEST-END-FILE + +# @TEST-START-FILE worker.zeek +@load ./common.zeek +event Cluster::node_up(name: string, id: string) + { + Cluster::publish("test.early", test, 42); + terminate(); + } +# @TEST-END-FILE From 3c16ee9667debc2aaba6458d2e6c699f8fc7f3f8 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 29 Sep 2025 08:21:01 -0700 Subject: [PATCH 3/5] Update docs submodule [nomail] --- CHANGES | 4 ++++ VERSION | 2 +- doc | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 071e2ed187..c30086ef64 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,7 @@ +8.1.0-dev.610 | 2025-09-29 08:21:01 -0700 + + * Update docs submodule [nomail] (Tim Wojtulewicz, Corelight) + 8.1.0-dev.609 | 2025-09-29 13:08:15 +0200 * cluster/zeromq: Fix Cluster::subscribe() block if not initialized (Arne Welzel, Corelight) diff --git a/VERSION b/VERSION index 4b178aee4a..041d214e50 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -8.1.0-dev.609 +8.1.0-dev.610 diff --git a/doc b/doc index 2731def915..8f38ae2fd5 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 2731def9159247e6da8a3191783c89683363689c +Subproject commit 8f38ae2fd563314393eb1ca58c827d26e9966520 From 8efddf87d985fe77253795e9d8a23d1014d1145a Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 5 Sep 2025 15:39:17 -0700 Subject: [PATCH 4/5] SMB: Init lanman time and set TZ correctly before calling mktime This makes a change to the record for smb1-negotiate-response as well. Times and dates are two 2-byte sections, not three. Switch the parsing to just use two uint16s, and pass those into the lanman time conversion function. --- src/analyzer/protocol/smb/smb-time.pac | 33 ++++++++++++------ .../protocol/smb/smb1-com-negotiate.pac | 4 +-- .../out | 8 +++++ testing/btest/Traces/README | 3 ++ .../Traces/smb/cifs_negotiate_lanman.pcap | Bin 0 -> 693 bytes .../protocols/smb/smb1-negotiate-lanman.zeek | 19 ++++++++++ 6 files changed, 55 insertions(+), 12 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.protocols.smb.smb1-negotiate-lanman/out create mode 100644 testing/btest/Traces/smb/cifs_negotiate_lanman.pcap create mode 100644 testing/btest/scripts/base/protocols/smb/smb1-negotiate-lanman.zeek diff --git a/src/analyzer/protocol/smb/smb-time.pac b/src/analyzer/protocol/smb/smb-time.pac index d30a151ce1..ec2775a8d7 100644 --- a/src/analyzer/protocol/smb/smb-time.pac +++ b/src/analyzer/protocol/smb/smb-time.pac @@ -1,6 +1,6 @@ %header{ double filetime2zeektime(uint64_t ts); -double time_from_lanman(SMB_time* t, SMB_date* d, uint16_t tz); +double time_from_lanman(uint32_t smb_time, uint32_t smb_date, uint16_t tz); zeek::RecordValPtr SMB_BuildMACTimes(uint64_t modify, uint64_t access, uint64_t create, uint64_t change); @@ -14,17 +14,30 @@ double filetime2zeektime(uint64_t ts) return (static_cast(ts) / 10000000.0) - 11644473600.0; } -double time_from_lanman(SMB_time* t, SMB_date* d, uint16_t tz) +double time_from_lanman(uint32_t smb_time, uint32_t smb_date, uint16_t tz) { - tm lTime; - lTime.tm_sec = ${t.two_seconds} * 2; - lTime.tm_min = ${t.minutes}; - lTime.tm_hour = ${t.hours}; - lTime.tm_mday = ${d.day}; - lTime.tm_mon = ${d.month}; - lTime.tm_year = 1980 + ${d.year}; + tm lTime{0}; + + // Lanman uses this format for time/date: + // https://learn.microsoft.com/en-us/cpp/c-runtime-library/32-bit-windows-time-date-formats + // Seconds is in 2-second increments in the data. + lTime.tm_sec = (smb_time & 0x1f) * 2; + lTime.tm_min = (smb_time >> 5) & 0x3f; + lTime.tm_hour = (smb_time >> 11) & 0x1f; + + lTime.tm_mday = smb_date & 0x1f; + // tm_mon is zero-indexed, so adjust for that. + lTime.tm_mon = ((smb_date >> 5) & 0x0f) - 1; + // The year in the data is the number of years from 1980, while tm_year is the + // number of years since 1900. + lTime.tm_year = ((smb_date >> 9) & 0x7f) + 80; lTime.tm_isdst = -1; - return mktime(&lTime) + tz; + + // The timezone passed in the data is the number of minutes from UTC, while + // tm_gmtoff is the number of seconds east of UTC. Adjust for that. + lTime.tm_gmtoff = static_cast(tz) * 60; + + return timegm(&lTime); } zeek::RecordValPtr SMB_BuildMACTimes(uint64_t modify, uint64_t access, diff --git a/src/analyzer/protocol/smb/smb1-com-negotiate.pac b/src/analyzer/protocol/smb/smb1-com-negotiate.pac index 57529a507f..9f2828b601 100644 --- a/src/analyzer/protocol/smb/smb1-com-negotiate.pac +++ b/src/analyzer/protocol/smb/smb1-com-negotiate.pac @@ -181,8 +181,8 @@ type SMB1_negotiate_lanman_response(header: SMB_Header) = record { max_number_vcs : uint16; raw_mode : uint16; # expanded in &let session_key : uint32; - server_time : SMB_time; - server_date : SMB_date; + server_time : uint16; + server_date : uint16; server_tz : uint16; encryption_key_length : uint16; reserved : uint16; # must be zero diff --git a/testing/btest/Baseline/scripts.base.protocols.smb.smb1-negotiate-lanman/out b/testing/btest/Baseline/scripts.base.protocols.smb.smb1-negotiate-lanman/out new file mode 100644 index 0000000000..b407cefc71 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.smb.smb1-negotiate-lanman/out @@ -0,0 +1,8 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +smb1_negotiate_request +[command=114, status=0, flags=24, flags2=51283, tid=0, pid=4660, uid=0, mid=1] +[PC NETWORK PROGRAM 1.0, LANMAN1.0, LM1.2X002, LANMAN2.1] +smb1_negotiate_response +[command=114, status=0, flags=152, flags2=51283, tid=0, pid=4660, uid=0, mid=1] +[core=, lanman=[word_count=13, dialect_index=3, security_mode=[user_level=T, challenge_response=T, signatures_enabled=, signatures_required=], max_buffer_size=4356, max_mpx_count=32, max_number_vcs=1, raw_mode=[read_raw=F, write_raw=F], session_key=0, server_time=1758862898.0, encryption_key=\x11"3DUfw\x88, primary_domain=\xe4\xbd\x97\xe4\xad\x92\xe5\x89\x87\xe5\x95\x8fP], ntlm=] +Parsed Response Server Time: 2025-09-26-05:01:38T diff --git a/testing/btest/Traces/README b/testing/btest/Traces/README index a01d06902b..f2389909f5 100644 --- a/testing/btest/Traces/README +++ b/testing/btest/Traces/README @@ -56,3 +56,6 @@ Trace Index/Sources: - smb_v2_only_non_zero_reserved1.pcap Provided by @predator89090 on #4730 https://github.com/zeek/zeek/issues/4730 +- smb/cifs_negotiate_lanman.pcap + Generated with scapy/chatgpt by @Mohan-Dhawan + https://github.com/zeek/zeek/issues/4545 \ No newline at end of file diff --git a/testing/btest/Traces/smb/cifs_negotiate_lanman.pcap b/testing/btest/Traces/smb/cifs_negotiate_lanman.pcap new file mode 100644 index 0000000000000000000000000000000000000000..fffaa573caa926f98517081212e08535e78bcf34 GIT binary patch literal 693 zcmca|c+)~A1{MYw`2U}Qff2|tmb;b#5;KEh203VsMvh<3|8ph=Yh0zgx?K}<0yWnh>FbRP&~n9>39(YtSGK7yDc@#_QU zLy$IxnE%1PPDLP=MDPg^g#ac(;Lx*VUvk{ z#|(-&q;Q9r0`%)INRse}`*j9(zw$COFf*_SDu5FWC?JeAtfIXbIe;u7cnE`n!9B=7 cGyv?qgjWdf!Icn5Rwj19s5}Gq9y0?207G out +# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff out + +event smb1_negotiate_request(c: connection, hdr: SMB1::Header, dialects: string_vec) + { + print "smb1_negotiate_request"; + print hdr; + print dialects; + } + +event smb1_negotiate_response(c: connection, hdr: SMB1::Header, response: SMB1::NegotiateResponse) + { + print "smb1_negotiate_response"; + print hdr; + print response; + print fmt("Parsed Response Server Time: %DT", response$lanman$server_time); + } From c1faef38f84212ab4751b524c8e1e66986c9cb1c Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 26 Sep 2025 14:43:22 -0700 Subject: [PATCH 5/5] Remove SMB_time and SMB_date record types --- src/analyzer/protocol/smb/smb-time.pac | 12 ------------ .../protocol/smb/smb1-com-query-information.pac | 2 +- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/src/analyzer/protocol/smb/smb-time.pac b/src/analyzer/protocol/smb/smb-time.pac index ec2775a8d7..a4fcd01dc1 100644 --- a/src/analyzer/protocol/smb/smb-time.pac +++ b/src/analyzer/protocol/smb/smb-time.pac @@ -58,15 +58,3 @@ zeek::RecordValPtr SMB_BuildMACTimes(uint64_t modify, uint64_t access, type SMB_timestamp32 = uint32; type SMB_timestamp = uint64; - -type SMB_time = record { - two_seconds : uint16; - minutes : uint16; - hours : uint16; -} &byteorder = littleendian; - -type SMB_date = record { - day : uint16; - month : uint16; - year : uint16; -} &byteorder = littleendian; diff --git a/src/analyzer/protocol/smb/smb1-com-query-information.pac b/src/analyzer/protocol/smb/smb1-com-query-information.pac index b3016bfb65..17329b8af3 100644 --- a/src/analyzer/protocol/smb/smb1-com-query-information.pac +++ b/src/analyzer/protocol/smb/smb1-com-query-information.pac @@ -31,7 +31,7 @@ type SMB1_query_information_request(header: SMB_Header) = record { type SMB1_query_information_response(header: SMB_Header) = record { word_count : uint8; file_attribs : uint16; - last_write_time : SMB_time; + last_write_time : uint32; file_size : uint32; reserved : uint16[5]; byte_count : uint16;