From 35e157a0abef10593e61f2e54d76341714ebeebc Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Fri, 29 Nov 2024 16:12:05 -0800 Subject: [PATCH 01/65] skip optimization of functions with AST nodes unknown to script optimization --- src/script_opt/CPP/Util.cc | 6 ++ src/script_opt/Inline.cc | 3 +- src/script_opt/ProfileFunc.cc | 8 +-- src/script_opt/ProfileFunc.h | 10 +-- src/script_opt/ScriptOpt.cc | 119 ++++++++++++++++++++++++++++++++++ src/script_opt/ScriptOpt.h | 5 ++ src/script_opt/ZAM/Support.cc | 6 ++ 7 files changed, 148 insertions(+), 9 deletions(-) diff --git a/src/script_opt/CPP/Util.cc b/src/script_opt/CPP/Util.cc index 15fc981ede..84e1846151 100644 --- a/src/script_opt/CPP/Util.cc +++ b/src/script_opt/CPP/Util.cc @@ -39,6 +39,12 @@ string scope_prefix(const string& scope) { return "zeek::detail::CPP_" + scope; string scope_prefix(int scope) { return scope_prefix(to_string(scope)); } bool is_CPP_compilable(const ProfileFunc* pf, const char** reason) { + if ( has_AST_node_unknown_to_script_opt(pf, false) ) { + if ( reason ) + *reason = "unknown AST node type"; + return false; + } + if ( analysis_options.allow_cond ) return true; diff --git a/src/script_opt/Inline.cc b/src/script_opt/Inline.cc index ad727dd908..16c88cbd81 100644 --- a/src/script_opt/Inline.cc +++ b/src/script_opt/Inline.cc @@ -10,6 +10,7 @@ #include "zeek/script_opt/ProfileFunc.h" #include "zeek/script_opt/ScriptOpt.h" #include "zeek/script_opt/StmtOptInfo.h" +#include "zeek/script_opt/ZAM/Support.h" namespace zeek::detail { @@ -160,7 +161,7 @@ void Inliner::Analyze() { if ( non_recursive_funcs.count(func) == 0 ) continue; - if ( body->Tag() == STMT_CPP ) + if ( ! is_ZAM_compilable(f.Profile()) ) continue; inline_ables[func] = f.Profile(); diff --git a/src/script_opt/ProfileFunc.cc b/src/script_opt/ProfileFunc.cc index 25d6a5e01d..c489921f46 100644 --- a/src/script_opt/ProfileFunc.cc +++ b/src/script_opt/ProfileFunc.cc @@ -101,7 +101,7 @@ ProfileFunc::ProfileFunc(const Expr* e, bool _abs_rec_fields) { } TraversalCode ProfileFunc::PreStmt(const Stmt* s) { - stmts.push_back(s); + stmts.push_back({NewRef{}, const_cast(s)}); switch ( s->Tag() ) { case STMT_INIT: @@ -185,7 +185,7 @@ TraversalCode ProfileFunc::PreStmt(const Stmt* s) { } TraversalCode ProfileFunc::PreExpr(const Expr* e) { - exprs.push_back(e); + exprs.push_back({NewRef{}, const_cast(e)}); TrackType(e->GetType()); @@ -867,11 +867,11 @@ void ProfileFuncs::ComputeProfileHash(std::shared_ptr pf) { h = merge_p_hashes(h, p_hash(ov[i]->Name())); h = merge_p_hashes(h, p_hash("stmts")); - for ( auto i : pf->Stmts() ) + for ( auto& i : pf->Stmts() ) h = merge_p_hashes(h, p_hash(i->Tag())); h = merge_p_hashes(h, p_hash("exprs")); - for ( auto i : pf->Exprs() ) + for ( auto& i : pf->Exprs() ) h = merge_p_hashes(h, p_hash(i->Tag())); h = merge_p_hashes(h, p_hash("ids")); diff --git a/src/script_opt/ProfileFunc.h b/src/script_opt/ProfileFunc.h index 9a2c53bc02..c8bcd71d9a 100644 --- a/src/script_opt/ProfileFunc.h +++ b/src/script_opt/ProfileFunc.h @@ -66,6 +66,8 @@ inline p_hash_type merge_p_hashes(p_hash_type h1, p_hash_type h2) { using AttrSet = std::unordered_set; using AttrVec = std::vector; +class ProfileFuncs; + // Class for profiling the components of a single function (or expression). class ProfileFunc : public TraversalCallback { public: @@ -101,8 +103,8 @@ public: const auto& TableRefs() const { return tbl_refs; } const auto& AggrMods() const { return aggr_mods; } const IDSet& Inits() const { return inits; } - const std::vector& Stmts() const { return stmts; } - const std::vector& Exprs() const { return exprs; } + const std::vector& Stmts() const { return stmts; } + const std::vector& Exprs() const { return exprs; } const std::vector& Lambdas() const { return lambdas; } const std::vector& Constants() const { return constants; } const IDSet& UnorderedIdentifiers() const { return ids; } @@ -213,11 +215,11 @@ protected: // Statements seen in the function. Does not include indirect // statements, such as those in lambda bodies. - std::vector stmts; + std::vector stmts; // Expressions seen in the function. Does not include indirect // expressions (such as those appearing in attributes of types). - std::vector exprs; + std::vector exprs; // Lambdas seen in the function. We don't profile lambda bodies, // but rather make them available for separate profiling if diff --git a/src/script_opt/ScriptOpt.cc b/src/script_opt/ScriptOpt.cc index 47effda307..de53cd41ca 100644 --- a/src/script_opt/ScriptOpt.cc +++ b/src/script_opt/ScriptOpt.cc @@ -657,4 +657,123 @@ void profile_script_execution() { void finish_script_execution() { profile_script_execution(); } +// For now, we have equivalent concerns between ZAM and compile-to-C++. +bool has_AST_node_unknown_to_script_opt(const ProfileFunc* prof, bool /* is_ZAM */) { + // Note that the following sets are not comprehensive across the + // standard tags, because some tags are only generated *by* script + // optimization + // clang-format off + static const std::set known_stmts = { + STMT_PRINT, + STMT_EVENT, + STMT_EXPR, + STMT_IF, + STMT_WHEN, + STMT_SWITCH, + STMT_FOR, + STMT_NEXT, + STMT_BREAK, + STMT_RETURN, + STMT_LIST, + // STMT_EVENT_BODY_LIST, + STMT_INIT, + STMT_FALLTHROUGH, + STMT_WHILE, + // STMT_CATCH_RETURN, + // STMT_CHECK_ANY_LEN, + // STMT_CPP, + // STMT_ZAM, + STMT_NULL, + STMT_ASSERT, + // STMT_EXTERN, + // STMT_STD_FUNCTION, + }; + // clang-format on + + for ( auto& s : prof->Stmts() ) + if ( known_stmts.count(s->Tag()) == 0 ) + return true; + + // clang-format off + static const std::set known_exprs = { + // EXPR_ANY, + EXPR_NAME, + EXPR_CONST, + EXPR_CLONE, + EXPR_INCR, + EXPR_DECR, + EXPR_NOT, + EXPR_COMPLEMENT, + EXPR_POSITIVE, + EXPR_NEGATE, + EXPR_ADD, EXPR_SUB, + EXPR_AGGR_ADD, + EXPR_AGGR_DEL, + EXPR_ADD_TO, + EXPR_REMOVE_FROM, + EXPR_TIMES, + EXPR_DIVIDE, + EXPR_MASK, + EXPR_MOD, + EXPR_AND, + EXPR_OR, + EXPR_XOR, + EXPR_LSHIFT, + EXPR_RSHIFT, + EXPR_AND_AND, + EXPR_OR_OR, + EXPR_LT, + EXPR_LE, + EXPR_EQ, + EXPR_NE, + EXPR_GE, + EXPR_GT, + EXPR_COND, + EXPR_REF, + EXPR_ASSIGN, + EXPR_INDEX, + EXPR_FIELD, + EXPR_HAS_FIELD, + EXPR_RECORD_CONSTRUCTOR, + EXPR_TABLE_CONSTRUCTOR, + EXPR_SET_CONSTRUCTOR, + EXPR_VECTOR_CONSTRUCTOR, + EXPR_FIELD_ASSIGN, + EXPR_IN, + EXPR_LIST, + EXPR_CALL, + EXPR_LAMBDA, + EXPR_EVENT, + EXPR_SCHEDULE, + EXPR_ARITH_COERCE, + EXPR_RECORD_COERCE, + EXPR_TABLE_COERCE, + EXPR_VECTOR_COERCE, + EXPR_TO_ANY_COERCE, + EXPR_FROM_ANY_COERCE, + EXPR_SIZE, + EXPR_CAST, + EXPR_IS, + // EXPR_INDEX_SLICE_ASSIGN, + EXPR_INLINE, + // EXPR_APPEND_TO, + // EXPR_INDEX_ASSIGN, + // EXPR_FIELD_LHS_ASSIGN, + // EXPR_REC_ASSIGN_FIELDS, + // EXPR_REC_ADD_FIELDS, + // EXPR_REC_CONSTRUCT_WITH_REC, + // EXPR_FROM_ANY_VEC_COERCE, + // EXPR_ANY_INDEX, + // EXPR_SCRIPT_OPT_BUILTIN, + // EXPR_NOP, + }; + // clang-format on + + for ( auto& e : prof->Exprs() ) + if ( known_exprs.count(e->Tag()) == 0 ) + return true; + + return false; +} + } // namespace zeek::detail diff --git a/src/script_opt/ScriptOpt.h b/src/script_opt/ScriptOpt.h index 441c8d6ec2..4ae68870cc 100644 --- a/src/script_opt/ScriptOpt.h +++ b/src/script_opt/ScriptOpt.h @@ -263,6 +263,11 @@ extern void clear_script_analysis(); // Called when Zeek is terminating. extern void finish_script_execution(); +// Returns true if the given profile indicates the presence of an AST +// node not known to script optimization. The second argument specifies +// whether we're doing ZAM optimization; if not, compile-to-C++ is assumed. +extern bool has_AST_node_unknown_to_script_opt(const ProfileFunc* prof, bool /* is_ZAM */); + // Returns true if the given call has a specialized ZAM equivalent when // used in a conditional. extern bool IsZAM_BuiltInCond(const CallExpr* c); diff --git a/src/script_opt/ZAM/Support.cc b/src/script_opt/ZAM/Support.cc index cf8c31b301..849d0a26cc 100644 --- a/src/script_opt/ZAM/Support.cc +++ b/src/script_opt/ZAM/Support.cc @@ -117,6 +117,12 @@ bool file_mgr_set_reassembly_buffer(StringVal* file_id, uint64_t max) { bool ZAM_error = false; bool is_ZAM_compilable(const ProfileFunc* pf, const char** reason) { + if ( has_AST_node_unknown_to_script_opt(pf, true) ) { + if ( reason ) + *reason = "unknown AST node type"; + return false; + } + auto b = pf->ProfiledBody(); auto is_hook = pf->ProfiledFunc()->Flavor() == FUNC_FLAVOR_HOOK; if ( b && ! script_is_valid(b, is_hook) ) { From 4958c56c84b539b0d74e8249973846d3390af97f Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Fri, 6 Dec 2024 11:55:39 +0100 Subject: [PATCH 02/65] Add missing "COPYING" in file comments This was just done via sed. There's a number of files that don't have a license entry at all. --- src/analyzer/protocol/bittorrent/Plugin.cc | 2 +- src/analyzer/protocol/conn-size/Plugin.cc | 2 +- src/analyzer/protocol/dce-rpc/Plugin.cc | 2 +- src/analyzer/protocol/dhcp/Plugin.cc | 2 +- src/analyzer/protocol/dnp3/Plugin.cc | 2 +- src/analyzer/protocol/dns/Plugin.cc | 2 +- src/analyzer/protocol/file/Plugin.cc | 2 +- src/analyzer/protocol/finger/legacy/Plugin.cc | 2 +- src/analyzer/protocol/ftp/Plugin.cc | 2 +- src/analyzer/protocol/gnutella/Plugin.cc | 2 +- src/analyzer/protocol/gssapi/Plugin.cc | 2 +- src/analyzer/protocol/http/Plugin.cc | 2 +- src/analyzer/protocol/ident/Plugin.cc | 2 +- src/analyzer/protocol/imap/Plugin.cc | 2 +- src/analyzer/protocol/irc/Plugin.cc | 2 +- src/analyzer/protocol/krb/Plugin.cc | 2 +- src/analyzer/protocol/login/Plugin.cc | 2 +- src/analyzer/protocol/mime/Plugin.cc | 2 +- src/analyzer/protocol/modbus/Plugin.cc | 2 +- src/analyzer/protocol/mqtt/MQTT.cc | 2 +- src/analyzer/protocol/mqtt/Plugin.cc | 2 +- src/analyzer/protocol/ncp/Plugin.cc | 2 +- src/analyzer/protocol/netbios/Plugin.cc | 2 +- src/analyzer/protocol/ntlm/Plugin.cc | 2 +- src/analyzer/protocol/ntp/Plugin.cc | 2 +- src/analyzer/protocol/pia/Plugin.cc | 2 +- src/analyzer/protocol/pop3/Plugin.cc | 2 +- src/analyzer/protocol/radius/Plugin.cc | 2 +- src/analyzer/protocol/rpc/Plugin.cc | 2 +- src/analyzer/protocol/sip/Plugin.cc | 2 +- src/analyzer/protocol/smb/Plugin.cc | 2 +- src/analyzer/protocol/smtp/Plugin.cc | 2 +- src/analyzer/protocol/snmp/Plugin.cc | 2 +- src/analyzer/protocol/socks/Plugin.cc | 2 +- src/analyzer/protocol/ssh/Plugin.cc | 2 +- src/analyzer/protocol/ssl/Plugin.cc | 2 +- src/analyzer/protocol/syslog/legacy/Plugin.cc | 2 +- src/analyzer/protocol/tcp/Plugin.cc | 2 +- src/analyzer/protocol/websocket/WebSocket.cc | 2 +- src/analyzer/protocol/xmpp/Plugin.cc | 2 +- src/analyzer/protocol/zip/Plugin.cc | 2 +- src/file_analysis/analyzer/data_event/Plugin.cc | 2 +- src/file_analysis/analyzer/entropy/Plugin.cc | 2 +- src/file_analysis/analyzer/extract/Plugin.cc | 2 +- src/file_analysis/analyzer/hash/Plugin.cc | 2 +- src/file_analysis/analyzer/pe/Plugin.cc | 2 +- src/file_analysis/analyzer/x509/Plugin.cc | 2 +- src/input/readers/ascii/Plugin.cc | 2 +- src/input/readers/benchmark/Plugin.cc | 2 +- src/input/readers/binary/Plugin.cc | 2 +- src/input/readers/config/Plugin.cc | 2 +- src/input/readers/raw/Plugin.cc | 2 +- src/input/readers/raw/Plugin.h | 2 +- src/input/readers/sqlite/Plugin.cc | 2 +- src/iosource/pcap/Dumper.h | 2 +- src/iosource/pcap/Plugin.cc | 2 +- src/iosource/pcap/Source.cc | 2 +- src/logging/writers/ascii/Plugin.cc | 2 +- src/logging/writers/none/Plugin.cc | 2 +- src/logging/writers/sqlite/Plugin.cc | 2 +- src/packet_analysis/protocol/gtpv1/Plugin.cc | 2 +- src/packet_analysis/protocol/teredo/Plugin.cc | 2 +- 62 files changed, 62 insertions(+), 62 deletions(-) diff --git a/src/analyzer/protocol/bittorrent/Plugin.cc b/src/analyzer/protocol/bittorrent/Plugin.cc index 2625a4268a..b36bb9c64d 100644 --- a/src/analyzer/protocol/bittorrent/Plugin.cc +++ b/src/analyzer/protocol/bittorrent/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/conn-size/Plugin.cc b/src/analyzer/protocol/conn-size/Plugin.cc index 38b96332f3..512fa71330 100644 --- a/src/analyzer/protocol/conn-size/Plugin.cc +++ b/src/analyzer/protocol/conn-size/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/dce-rpc/Plugin.cc b/src/analyzer/protocol/dce-rpc/Plugin.cc index 5a70f5dd50..da2250adac 100644 --- a/src/analyzer/protocol/dce-rpc/Plugin.cc +++ b/src/analyzer/protocol/dce-rpc/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/dhcp/Plugin.cc b/src/analyzer/protocol/dhcp/Plugin.cc index d4c5a42d01..424f9d3a19 100644 --- a/src/analyzer/protocol/dhcp/Plugin.cc +++ b/src/analyzer/protocol/dhcp/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/dnp3/Plugin.cc b/src/analyzer/protocol/dnp3/Plugin.cc index eb0def9eb4..d9a2ad6713 100644 --- a/src/analyzer/protocol/dnp3/Plugin.cc +++ b/src/analyzer/protocol/dnp3/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/dns/Plugin.cc b/src/analyzer/protocol/dns/Plugin.cc index e5d57661ff..e57c076acd 100644 --- a/src/analyzer/protocol/dns/Plugin.cc +++ b/src/analyzer/protocol/dns/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/file/Plugin.cc b/src/analyzer/protocol/file/Plugin.cc index d107cf8a10..eb851aa570 100644 --- a/src/analyzer/protocol/file/Plugin.cc +++ b/src/analyzer/protocol/file/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/finger/legacy/Plugin.cc b/src/analyzer/protocol/finger/legacy/Plugin.cc index 8f10c0e525..69e271999a 100644 --- a/src/analyzer/protocol/finger/legacy/Plugin.cc +++ b/src/analyzer/protocol/finger/legacy/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/ftp/Plugin.cc b/src/analyzer/protocol/ftp/Plugin.cc index 8c255e8029..3f8c01760e 100644 --- a/src/analyzer/protocol/ftp/Plugin.cc +++ b/src/analyzer/protocol/ftp/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/gnutella/Plugin.cc b/src/analyzer/protocol/gnutella/Plugin.cc index f6ce79611c..c6526730b5 100644 --- a/src/analyzer/protocol/gnutella/Plugin.cc +++ b/src/analyzer/protocol/gnutella/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/gssapi/Plugin.cc b/src/analyzer/protocol/gssapi/Plugin.cc index a009e903bb..37136278c9 100644 --- a/src/analyzer/protocol/gssapi/Plugin.cc +++ b/src/analyzer/protocol/gssapi/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/http/Plugin.cc b/src/analyzer/protocol/http/Plugin.cc index a0753df0b8..fdd7b953d0 100644 --- a/src/analyzer/protocol/http/Plugin.cc +++ b/src/analyzer/protocol/http/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/ident/Plugin.cc b/src/analyzer/protocol/ident/Plugin.cc index 9e432412e2..60cf0d04d7 100644 --- a/src/analyzer/protocol/ident/Plugin.cc +++ b/src/analyzer/protocol/ident/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/imap/Plugin.cc b/src/analyzer/protocol/imap/Plugin.cc index 32b1bde50f..b55e3a72e5 100644 --- a/src/analyzer/protocol/imap/Plugin.cc +++ b/src/analyzer/protocol/imap/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/irc/Plugin.cc b/src/analyzer/protocol/irc/Plugin.cc index d1f323832a..d0d6a7950a 100644 --- a/src/analyzer/protocol/irc/Plugin.cc +++ b/src/analyzer/protocol/irc/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/krb/Plugin.cc b/src/analyzer/protocol/krb/Plugin.cc index 2cd808d6ab..7ec24f8b92 100644 --- a/src/analyzer/protocol/krb/Plugin.cc +++ b/src/analyzer/protocol/krb/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/login/Plugin.cc b/src/analyzer/protocol/login/Plugin.cc index c154ab9ba1..c61ffaffba 100644 --- a/src/analyzer/protocol/login/Plugin.cc +++ b/src/analyzer/protocol/login/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/mime/Plugin.cc b/src/analyzer/protocol/mime/Plugin.cc index 3864f40b98..24e7557a2d 100644 --- a/src/analyzer/protocol/mime/Plugin.cc +++ b/src/analyzer/protocol/mime/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/modbus/Plugin.cc b/src/analyzer/protocol/modbus/Plugin.cc index cdf9b3e18d..38ef4801ac 100644 --- a/src/analyzer/protocol/modbus/Plugin.cc +++ b/src/analyzer/protocol/modbus/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/mqtt/MQTT.cc b/src/analyzer/protocol/mqtt/MQTT.cc index 795c3e0014..dc6ca77e3f 100644 --- a/src/analyzer/protocol/mqtt/MQTT.cc +++ b/src/analyzer/protocol/mqtt/MQTT.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/analyzer/protocol/mqtt/MQTT.h" diff --git a/src/analyzer/protocol/mqtt/Plugin.cc b/src/analyzer/protocol/mqtt/Plugin.cc index b30abfaea8..be9771e4e3 100644 --- a/src/analyzer/protocol/mqtt/Plugin.cc +++ b/src/analyzer/protocol/mqtt/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/ncp/Plugin.cc b/src/analyzer/protocol/ncp/Plugin.cc index 29f40948ad..29e269d506 100644 --- a/src/analyzer/protocol/ncp/Plugin.cc +++ b/src/analyzer/protocol/ncp/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/netbios/Plugin.cc b/src/analyzer/protocol/netbios/Plugin.cc index e4cacc57fe..261ad40350 100644 --- a/src/analyzer/protocol/netbios/Plugin.cc +++ b/src/analyzer/protocol/netbios/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/ntlm/Plugin.cc b/src/analyzer/protocol/ntlm/Plugin.cc index 996bf8f013..ef3c1201ab 100644 --- a/src/analyzer/protocol/ntlm/Plugin.cc +++ b/src/analyzer/protocol/ntlm/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/ntp/Plugin.cc b/src/analyzer/protocol/ntp/Plugin.cc index d625658392..ebb07efd33 100644 --- a/src/analyzer/protocol/ntp/Plugin.cc +++ b/src/analyzer/protocol/ntp/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/pia/Plugin.cc b/src/analyzer/protocol/pia/Plugin.cc index 36ac9e90aa..1ccde06253 100644 --- a/src/analyzer/protocol/pia/Plugin.cc +++ b/src/analyzer/protocol/pia/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/pop3/Plugin.cc b/src/analyzer/protocol/pop3/Plugin.cc index 5b1d0f7f3d..9e8f41a7fe 100644 --- a/src/analyzer/protocol/pop3/Plugin.cc +++ b/src/analyzer/protocol/pop3/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/radius/Plugin.cc b/src/analyzer/protocol/radius/Plugin.cc index 2e1565dd70..ca61e3e09d 100644 --- a/src/analyzer/protocol/radius/Plugin.cc +++ b/src/analyzer/protocol/radius/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/rpc/Plugin.cc b/src/analyzer/protocol/rpc/Plugin.cc index 43ca97c49f..8b939d4e06 100644 --- a/src/analyzer/protocol/rpc/Plugin.cc +++ b/src/analyzer/protocol/rpc/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/sip/Plugin.cc b/src/analyzer/protocol/sip/Plugin.cc index f8a1ff3b67..ec6fcd10ac 100644 --- a/src/analyzer/protocol/sip/Plugin.cc +++ b/src/analyzer/protocol/sip/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/smb/Plugin.cc b/src/analyzer/protocol/smb/Plugin.cc index 6730418569..3caaf28cec 100644 --- a/src/analyzer/protocol/smb/Plugin.cc +++ b/src/analyzer/protocol/smb/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/smtp/Plugin.cc b/src/analyzer/protocol/smtp/Plugin.cc index b27747f2e3..c93c421d6e 100644 --- a/src/analyzer/protocol/smtp/Plugin.cc +++ b/src/analyzer/protocol/smtp/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/snmp/Plugin.cc b/src/analyzer/protocol/snmp/Plugin.cc index 2cfde7a67d..a7e20b1499 100644 --- a/src/analyzer/protocol/snmp/Plugin.cc +++ b/src/analyzer/protocol/snmp/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/socks/Plugin.cc b/src/analyzer/protocol/socks/Plugin.cc index 9a4f051127..ce06bf419f 100644 --- a/src/analyzer/protocol/socks/Plugin.cc +++ b/src/analyzer/protocol/socks/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/ssh/Plugin.cc b/src/analyzer/protocol/ssh/Plugin.cc index 3fa882b55f..ba2d9a8a95 100644 --- a/src/analyzer/protocol/ssh/Plugin.cc +++ b/src/analyzer/protocol/ssh/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/ssl/Plugin.cc b/src/analyzer/protocol/ssl/Plugin.cc index a807089e3c..c42d189fd6 100644 --- a/src/analyzer/protocol/ssl/Plugin.cc +++ b/src/analyzer/protocol/ssl/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/syslog/legacy/Plugin.cc b/src/analyzer/protocol/syslog/legacy/Plugin.cc index 4db0ce8262..16a879e721 100644 --- a/src/analyzer/protocol/syslog/legacy/Plugin.cc +++ b/src/analyzer/protocol/syslog/legacy/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/tcp/Plugin.cc b/src/analyzer/protocol/tcp/Plugin.cc index 74b467cd11..d21f5ccda6 100644 --- a/src/analyzer/protocol/tcp/Plugin.cc +++ b/src/analyzer/protocol/tcp/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/websocket/WebSocket.cc b/src/analyzer/protocol/websocket/WebSocket.cc index 8cd47aeefc..870a38d023 100644 --- a/src/analyzer/protocol/websocket/WebSocket.cc +++ b/src/analyzer/protocol/websocket/WebSocket.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/analyzer/protocol/websocket/WebSocket.h" diff --git a/src/analyzer/protocol/xmpp/Plugin.cc b/src/analyzer/protocol/xmpp/Plugin.cc index fd9f90a08c..f1ea2aacf4 100644 --- a/src/analyzer/protocol/xmpp/Plugin.cc +++ b/src/analyzer/protocol/xmpp/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/analyzer/protocol/zip/Plugin.cc b/src/analyzer/protocol/zip/Plugin.cc index 7347ca764b..1fffee9fe1 100644 --- a/src/analyzer/protocol/zip/Plugin.cc +++ b/src/analyzer/protocol/zip/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/file_analysis/analyzer/data_event/Plugin.cc b/src/file_analysis/analyzer/data_event/Plugin.cc index 6150af66cf..997f451584 100644 --- a/src/file_analysis/analyzer/data_event/Plugin.cc +++ b/src/file_analysis/analyzer/data_event/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/file_analysis/analyzer/entropy/Plugin.cc b/src/file_analysis/analyzer/entropy/Plugin.cc index 8238131d4c..cabb47f220 100644 --- a/src/file_analysis/analyzer/entropy/Plugin.cc +++ b/src/file_analysis/analyzer/entropy/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/file_analysis/analyzer/extract/Plugin.cc b/src/file_analysis/analyzer/extract/Plugin.cc index ea8a7201a4..f08101dfa1 100644 --- a/src/file_analysis/analyzer/extract/Plugin.cc +++ b/src/file_analysis/analyzer/extract/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/file_analysis/analyzer/hash/Plugin.cc b/src/file_analysis/analyzer/hash/Plugin.cc index 741289dc7b..833da66e1f 100644 --- a/src/file_analysis/analyzer/hash/Plugin.cc +++ b/src/file_analysis/analyzer/hash/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/file_analysis/analyzer/pe/Plugin.cc b/src/file_analysis/analyzer/pe/Plugin.cc index cee234abb4..adb5203b14 100644 --- a/src/file_analysis/analyzer/pe/Plugin.cc +++ b/src/file_analysis/analyzer/pe/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/file_analysis/analyzer/x509/Plugin.cc b/src/file_analysis/analyzer/x509/Plugin.cc index 84b8801b88..6e01516140 100644 --- a/src/file_analysis/analyzer/x509/Plugin.cc +++ b/src/file_analysis/analyzer/x509/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/input/readers/ascii/Plugin.cc b/src/input/readers/ascii/Plugin.cc index 2cbd2478cd..bec5775772 100644 --- a/src/input/readers/ascii/Plugin.cc +++ b/src/input/readers/ascii/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/input/readers/benchmark/Plugin.cc b/src/input/readers/benchmark/Plugin.cc index 13b55a447f..bc8fd989d1 100644 --- a/src/input/readers/benchmark/Plugin.cc +++ b/src/input/readers/benchmark/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/input/readers/binary/Plugin.cc b/src/input/readers/binary/Plugin.cc index f6987bb91b..1481d6be0f 100644 --- a/src/input/readers/binary/Plugin.cc +++ b/src/input/readers/binary/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/input/readers/config/Plugin.cc b/src/input/readers/config/Plugin.cc index bbf45eeabc..ac79045dc2 100644 --- a/src/input/readers/config/Plugin.cc +++ b/src/input/readers/config/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/input/readers/raw/Plugin.cc b/src/input/readers/raw/Plugin.cc index 1f1af5c931..757e989836 100644 --- a/src/input/readers/raw/Plugin.cc +++ b/src/input/readers/raw/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/input/readers/raw/Plugin.h" diff --git a/src/input/readers/raw/Plugin.h b/src/input/readers/raw/Plugin.h index 88bd662108..d15ecdbe33 100644 --- a/src/input/readers/raw/Plugin.h +++ b/src/input/readers/raw/Plugin.h @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #pragma once diff --git a/src/input/readers/sqlite/Plugin.cc b/src/input/readers/sqlite/Plugin.cc index 0448f8105e..cdba61d2b8 100644 --- a/src/input/readers/sqlite/Plugin.cc +++ b/src/input/readers/sqlite/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/iosource/pcap/Dumper.h b/src/iosource/pcap/Dumper.h index e5b0923408..81abe23936 100644 --- a/src/iosource/pcap/Dumper.h +++ b/src/iosource/pcap/Dumper.h @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #pragma once diff --git a/src/iosource/pcap/Plugin.cc b/src/iosource/pcap/Plugin.cc index 846dbc6738..f74aa39927 100644 --- a/src/iosource/pcap/Plugin.cc +++ b/src/iosource/pcap/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/iosource/pcap/Source.cc b/src/iosource/pcap/Source.cc index be01e148cc..9bb3d7833e 100644 --- a/src/iosource/pcap/Source.cc +++ b/src/iosource/pcap/Source.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/iosource/pcap/Source.h" diff --git a/src/logging/writers/ascii/Plugin.cc b/src/logging/writers/ascii/Plugin.cc index 944cdf0605..d7b640b32c 100644 --- a/src/logging/writers/ascii/Plugin.cc +++ b/src/logging/writers/ascii/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/logging/writers/none/Plugin.cc b/src/logging/writers/none/Plugin.cc index b6a093adfa..d07087006f 100644 --- a/src/logging/writers/none/Plugin.cc +++ b/src/logging/writers/none/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/logging/writers/sqlite/Plugin.cc b/src/logging/writers/sqlite/Plugin.cc index 461c78e923..97f59036e8 100644 --- a/src/logging/writers/sqlite/Plugin.cc +++ b/src/logging/writers/sqlite/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/packet_analysis/protocol/gtpv1/Plugin.cc b/src/packet_analysis/protocol/gtpv1/Plugin.cc index f19478ac0f..461832ca55 100644 --- a/src/packet_analysis/protocol/gtpv1/Plugin.cc +++ b/src/packet_analysis/protocol/gtpv1/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" diff --git a/src/packet_analysis/protocol/teredo/Plugin.cc b/src/packet_analysis/protocol/teredo/Plugin.cc index 2a35a5b847..e0a36a116a 100644 --- a/src/packet_analysis/protocol/teredo/Plugin.cc +++ b/src/packet_analysis/protocol/teredo/Plugin.cc @@ -1,4 +1,4 @@ -// See the file in the main distribution directory for copyright. +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/plugin/Plugin.h" From d93249eeab12f18563b895da3113715234f157c3 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Fri, 6 Dec 2024 12:43:39 +0100 Subject: [PATCH 03/65] pre-commit: Add license-header check inspired by Spicy --- .pre-commit-config.yaml | 11 +++++++++++ ci/license-header.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100755 ci/license-header.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1997a9be57..b0fc21d692 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,6 +2,17 @@ # See https://pre-commit.com/hooks.html for more hooks # repos: +- repo: local + hooks: + - id: license + name: Check for license headers + entry: ./ci/license-header.py + language: python + types_or: + - "c" + - "c++" + exclude: '^(testing/btest/plugins/.*|testing/builtin-plugins/.*)$' + - repo: https://github.com/pre-commit/mirrors-clang-format rev: 'v18.1.8' hooks: diff --git a/ci/license-header.py b/ci/license-header.py new file mode 100755 index 0000000000..2af95aba0c --- /dev/null +++ b/ci/license-header.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 + +import sys +import re + +exit_code = 0 + +pat1 = re.compile(r"See the file \"COPYING\" in the main distribution directory for copyright.") + +# This is the copyright line used within Spicy plugin and popular in +# Spicy analyzers. +pat2 = re.compile(r"Copyright \(c\) 2... by the Zeek Project. See COPYING for details.") + + +def match_line(line): + for pat in [pat1, pat2]: + m = pat.search(line) + if m is not None: + return True + + return False + + +for f in sys.argv[1:]: + has_license_header = False + with open(f) as fp: + for line in fp: + line = line.strip() + if has_license_header := match_line(line): + break + + if not has_license_header: + print(f"{f}:does not seem to contain a license header", file=sys.stderr) + exit_code = 1 + +sys.exit(exit_code) From 9619cd0f17d710e2ddaf8be1428a4a298ba6e7fb Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Fri, 6 Dec 2024 12:45:44 +0100 Subject: [PATCH 04/65] Add missing copyright line to headers and cc files --- src/Anon.cc | 2 ++ src/Anon.h | 2 ++ src/Base64.cc | 2 ++ src/Base64.h | 2 ++ src/DNS_Mapping.cc | 2 ++ src/DNS_Mapping.h | 2 ++ src/DbgBreakpoint.cc | 2 ++ src/DbgBreakpoint.h | 2 ++ src/DbgDisplay.h | 2 ++ src/DbgHelp.cc | 2 ++ src/DbgWatch.cc | 2 ++ src/DbgWatch.h | 2 ++ src/Debug.cc | 2 ++ src/Debug.h | 2 ++ src/DebugCmds.cc | 2 ++ src/DebugCmds.h | 2 ++ src/DebugLogger.cc | 2 ++ src/DebugLogger.h | 2 ++ src/EventHandler.cc | 2 ++ src/EventHandler.h | 2 ++ src/EventLauncher.cc | 2 ++ src/EventLauncher.h | 2 ++ src/EventRegistry.cc | 2 ++ src/EventRegistry.h | 2 ++ src/EventTrace.h | 2 ++ src/IntSet.cc | 2 ++ src/IntSet.h | 2 ++ src/List.cc | 2 ++ src/List.h | 2 ++ src/PacketFilter.cc | 2 ++ src/PacketFilter.h | 2 ++ src/PolicyFile.cc | 2 ++ src/PolicyFile.h | 2 ++ src/PrefixTable.cc | 2 ++ src/PrefixTable.h | 2 ++ src/RandTest.cc | 2 ++ src/RandTest.h | 2 ++ src/Rule.cc | 2 ++ src/Rule.h | 2 ++ src/RuleAction.cc | 2 ++ src/RuleAction.h | 2 ++ src/RuleCondition.cc | 2 ++ src/RuleCondition.h | 2 ++ src/RuleMatcher.cc | 1 + src/RuleMatcher.h | 2 ++ src/ScannedFile.cc | 2 ++ src/ScriptCoverageManager.cc | 2 ++ src/ScriptCoverageManager.h | 2 ++ src/ScriptValidation.cc | 2 ++ src/SerializationFormat.cc | 2 ++ src/SerializationFormat.h | 2 ++ src/Stats.cc | 2 ++ src/Stats.h | 2 ++ src/Trigger.cc | 2 ++ src/Trigger.h | 2 ++ src/WeirdState.cc | 2 ++ src/ZeekArgs.cc | 2 ++ src/analyzer/protocol/bittorrent/BitTorrent.cc | 2 ++ src/analyzer/protocol/bittorrent/BitTorrent.h | 2 ++ src/analyzer/protocol/bittorrent/BitTorrentTracker.cc | 2 ++ src/analyzer/protocol/bittorrent/BitTorrentTracker.h | 2 ++ src/analyzer/protocol/dhcp/DHCP.cc | 2 ++ src/analyzer/protocol/dhcp/DHCP.h | 2 ++ src/analyzer/protocol/dnp3/DNP3.cc | 3 ++- src/analyzer/protocol/dnp3/DNP3.h | 2 ++ src/analyzer/protocol/file/File.cc | 2 ++ src/analyzer/protocol/file/File.h | 2 ++ src/analyzer/protocol/irc/IRC.cc | 2 ++ src/analyzer/protocol/irc/IRC.h | 2 ++ src/analyzer/protocol/mime/MIME.cc | 2 ++ src/analyzer/protocol/mime/MIME.h | 2 ++ src/analyzer/protocol/modbus/Modbus.cc | 2 ++ src/analyzer/protocol/modbus/Modbus.h | 2 ++ src/analyzer/protocol/mqtt/MQTT.h | 2 ++ src/analyzer/protocol/ntp/NTP.cc | 2 ++ src/analyzer/protocol/ntp/NTP.h | 2 ++ src/analyzer/protocol/pia/PIA.cc | 2 ++ src/analyzer/protocol/pia/PIA.h | 2 ++ src/analyzer/protocol/pop3/POP3.cc | 2 ++ src/analyzer/protocol/pop3/POP3.h | 2 ++ src/analyzer/protocol/radius/RADIUS.cc | 2 ++ src/analyzer/protocol/radius/RADIUS.h | 2 ++ src/analyzer/protocol/rdp/Plugin.cc | 2 ++ src/analyzer/protocol/rdp/RDP.cc | 2 ++ src/analyzer/protocol/rdp/RDP.h | 2 ++ src/analyzer/protocol/rdp/RDPEUDP.cc | 2 ++ src/analyzer/protocol/rdp/RDPEUDP.h | 2 ++ src/analyzer/protocol/rfb/Plugin.cc | 2 ++ src/analyzer/protocol/rfb/RFB.cc | 2 ++ src/analyzer/protocol/rfb/RFB.h | 2 ++ src/analyzer/protocol/sip/SIP.cc | 2 ++ src/analyzer/protocol/sip/SIP.h | 2 ++ src/analyzer/protocol/smb/SMB.cc | 2 ++ src/analyzer/protocol/smb/SMB.h | 2 ++ src/analyzer/protocol/smtp/BDAT.cc | 2 ++ src/analyzer/protocol/socks/SOCKS.cc | 2 ++ src/analyzer/protocol/socks/SOCKS.h | 2 ++ src/analyzer/protocol/ssl/DTLS.cc | 2 ++ src/analyzer/protocol/ssl/DTLS.h | 2 ++ src/analyzer/protocol/ssl/SSL.cc | 2 ++ src/analyzer/protocol/ssl/SSL.h | 2 ++ src/analyzer/protocol/syslog/legacy/Syslog.cc | 2 ++ src/analyzer/protocol/syslog/legacy/Syslog.h | 2 ++ src/analyzer/protocol/tcp/ContentLine.cc | 2 ++ src/analyzer/protocol/tcp/ContentLine.h | 2 ++ src/analyzer/protocol/tcp/TCP_Flags.h | 2 ++ src/analyzer/protocol/tcp/TCP_Reassembler.cc | 2 ++ src/analyzer/protocol/tcp/TCP_Reassembler.h | 2 ++ src/binpac_zeek.h | 2 ++ src/broker/Data.cc | 2 ++ src/broker/Data.h | 2 ++ src/broker/Manager.cc | 2 ++ src/broker/Manager.h | 2 ++ src/broker/Plugin.cc | 2 ++ src/broker/Store.cc | 2 ++ src/broker/Store.h | 2 ++ src/cluster/BifSupport.cc | 2 ++ src/cluster/Manager.cc | 2 ++ src/cluster/serializer/binary-serialization-format/Plugin.cc | 2 ++ .../serializer/binary-serialization-format/Serializer.cc | 2 ++ src/cluster/serializer/broker/Plugin.cc | 2 ++ src/cluster/serializer/broker/Serializer.cc | 2 ++ src/file_analysis/analyzer/pe/PE.cc | 2 ++ src/file_analysis/analyzer/pe/PE.h | 2 ++ src/fuzzers/FuzzBuffer.cc | 2 ++ src/fuzzers/FuzzBuffer.h | 2 ++ src/fuzzers/dns-fuzzer.cc | 2 ++ src/fuzzers/fuzzer-setup.h | 2 ++ src/fuzzers/generic-analyzer-fuzzer.cc | 2 ++ src/fuzzers/packet-fuzzer.cc | 2 ++ src/fuzzers/standalone-driver.cc | 2 ++ src/fuzzers/websocket-fuzzer.cc | 2 ++ src/iosource/Packet.cc | 2 ++ src/iosource/Packet.h | 2 ++ src/logging/WriterFrontend.cc | 2 ++ src/logging/writers/none/None.cc | 2 ++ src/module_util.h | 2 ++ src/packet_analysis/protocol/gtpv1/GTPv1.h | 2 ++ src/packet_analysis/protocol/ip/SessionAdapter.cc | 2 ++ src/packet_analysis/protocol/ip/SessionAdapter.h | 2 ++ src/packet_analysis/protocol/teredo/Teredo.cc | 2 ++ src/packet_analysis/protocol/teredo/Teredo.h | 2 ++ src/plugin/ComponentManager.h | 2 ++ src/script_opt/Expr.h | 2 ++ src/session/Key.cc | 2 ++ src/telemetry/Counter.cc | 2 ++ src/telemetry/Gauge.cc | 2 ++ src/telemetry/Histogram.cc | 2 ++ src/telemetry/Opaques.cc | 2 ++ src/telemetry/ProcessStats.cc | 2 ++ src/telemetry/ProcessStats.h | 2 ++ src/telemetry/Utils.cc | 2 ++ src/telemetry/Utils.h | 2 ++ src/threading/BasicThread.cc | 2 ++ src/threading/BasicThread.h | 1 + src/threading/Manager.cc | 2 ++ src/threading/Manager.h | 2 ++ src/threading/MsgThread.cc | 2 ++ src/threading/MsgThread.h | 2 ++ src/threading/Queue.h | 2 ++ src/threading/SerialTypes.h | 1 + src/threading/formatters/detail/json.h | 2 ++ src/zeek-bif.h | 2 ++ 163 files changed, 323 insertions(+), 1 deletion(-) diff --git a/src/Anon.cc b/src/Anon.cc index 82fbd51f15..022778c9c0 100644 --- a/src/Anon.cc +++ b/src/Anon.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/Anon.h" #include diff --git a/src/Anon.h b/src/Anon.h index 454a47ca32..419a2afcf7 100644 --- a/src/Anon.h +++ b/src/Anon.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // The prefix-preserving IP address anonymization code is largely // based on (and sometimes directly copied from) Eddie Kohler's // ipsumdump-1.20 code, per: diff --git a/src/Base64.cc b/src/Base64.cc index 56d1433317..fbc6ad1eab 100644 --- a/src/Base64.cc +++ b/src/Base64.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/Base64.h" #include "zeek/zeek-config.h" diff --git a/src/Base64.h b/src/Base64.h index a6e258d05d..dd74d35c14 100644 --- a/src/Base64.h +++ b/src/Base64.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include "zeek/zeek-config.h" diff --git a/src/DNS_Mapping.cc b/src/DNS_Mapping.cc index daed31eb23..3fa14b4e93 100644 --- a/src/DNS_Mapping.cc +++ b/src/DNS_Mapping.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/DNS_Mapping.h" #include diff --git a/src/DNS_Mapping.h b/src/DNS_Mapping.h index 7aadfb6355..c2108107e3 100644 --- a/src/DNS_Mapping.h +++ b/src/DNS_Mapping.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include diff --git a/src/DbgBreakpoint.cc b/src/DbgBreakpoint.cc index fe833493f3..5d75cc1cc2 100644 --- a/src/DbgBreakpoint.cc +++ b/src/DbgBreakpoint.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // Implementation of breakpoints. #include "zeek/DbgBreakpoint.h" diff --git a/src/DbgBreakpoint.h b/src/DbgBreakpoint.h index bca79f5498..bf0aae016d 100644 --- a/src/DbgBreakpoint.h +++ b/src/DbgBreakpoint.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // Structures and methods for implementing breakpoints in the Zeek debugger. #pragma once diff --git a/src/DbgDisplay.h b/src/DbgDisplay.h index 8e79a5d736..1e36295aea 100644 --- a/src/DbgDisplay.h +++ b/src/DbgDisplay.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // Structures and methods for implementing watches in the Zeek debugger. #pragma once diff --git a/src/DbgHelp.cc b/src/DbgHelp.cc index 7f346b9a14..8ffec2054f 100644 --- a/src/DbgHelp.cc +++ b/src/DbgHelp.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // Zeek Debugger Help #include "zeek/zeek-config.h" diff --git a/src/DbgWatch.cc b/src/DbgWatch.cc index fd8700bd6e..daa865f2b5 100644 --- a/src/DbgWatch.cc +++ b/src/DbgWatch.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // Implementation of watches #include "zeek/DbgWatch.h" diff --git a/src/DbgWatch.h b/src/DbgWatch.h index 93c12f6e38..e5ba83e3d2 100644 --- a/src/DbgWatch.h +++ b/src/DbgWatch.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // Structures and methods for implementing watches in the Zeek debugger. #pragma once diff --git a/src/Debug.cc b/src/Debug.cc index 6609a3d256..1ed29ce6d6 100644 --- a/src/Debug.cc +++ b/src/Debug.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // Debugging support for Zeek policy files. #include "zeek/Debug.h" diff --git a/src/Debug.h b/src/Debug.h index 34cd81542b..4f04bc0898 100644 --- a/src/Debug.h +++ b/src/Debug.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // Debugging support for Zeek policy files. #pragma once diff --git a/src/DebugCmds.cc b/src/DebugCmds.cc index 0cb510b18c..dd5c41b3f7 100644 --- a/src/DebugCmds.cc +++ b/src/DebugCmds.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // Support routines to help deal with Zeek debugging commands and // implementation of most commands. diff --git a/src/DebugCmds.h b/src/DebugCmds.h index 5913b21144..9be4095eb5 100644 --- a/src/DebugCmds.h +++ b/src/DebugCmds.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // Support routines to help deal with Zeek debugging commands and // implementation of most commands. diff --git a/src/DebugLogger.cc b/src/DebugLogger.cc index 3feb37f8d0..6b586d5ab6 100644 --- a/src/DebugLogger.cc +++ b/src/DebugLogger.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #ifdef DEBUG #include "zeek/DebugLogger.h" diff --git a/src/DebugLogger.h b/src/DebugLogger.h index 4cfa7c9e32..63269443c9 100644 --- a/src/DebugLogger.h +++ b/src/DebugLogger.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // A logger for (selective) debugging output. Only compiled in if DEBUG is // defined. diff --git a/src/EventHandler.cc b/src/EventHandler.cc index a5701aa5fa..c375b3a3e1 100644 --- a/src/EventHandler.cc +++ b/src/EventHandler.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/EventHandler.h" #include "zeek/Desc.h" diff --git a/src/EventHandler.h b/src/EventHandler.h index 9291bda809..ce241f93c7 100644 --- a/src/EventHandler.h +++ b/src/EventHandler.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // Capsulates local and remote event handlers. #pragma once diff --git a/src/EventLauncher.cc b/src/EventLauncher.cc index e23517122b..44bdcec2a0 100644 --- a/src/EventLauncher.cc +++ b/src/EventLauncher.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/EventLauncher.h" #include "event.bif.func_def" diff --git a/src/EventLauncher.h b/src/EventLauncher.h index 4ce7cd6ad9..9595c63264 100644 --- a/src/EventLauncher.h +++ b/src/EventLauncher.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include "zeek/Conn.h" diff --git a/src/EventRegistry.cc b/src/EventRegistry.cc index b166546276..483ce88654 100644 --- a/src/EventRegistry.cc +++ b/src/EventRegistry.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/EventRegistry.h" #include diff --git a/src/EventRegistry.h b/src/EventRegistry.h index cbe9c622bf..727ba9eb0c 100644 --- a/src/EventRegistry.h +++ b/src/EventRegistry.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // Each event raised/handled by Zeek is registered in the EventRegistry. #pragma once diff --git a/src/EventTrace.h b/src/EventTrace.h index 8d391a01df..a9f7e28353 100644 --- a/src/EventTrace.h +++ b/src/EventTrace.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // Classes for tracing/dumping Zeek events. #pragma once diff --git a/src/IntSet.cc b/src/IntSet.cc index e2cac41798..6e661121a9 100644 --- a/src/IntSet.cc +++ b/src/IntSet.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/IntSet.h" #include "zeek/zeek-config.h" diff --git a/src/IntSet.h b/src/IntSet.h index 73c37a180d..a92ea2a0d2 100644 --- a/src/IntSet.h +++ b/src/IntSet.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // A simple but fast data structure for sets of integers. // Only supported operations are insert, remove and membership test. // diff --git a/src/List.cc b/src/List.cc index 690d144c69..6f29942f2a 100644 --- a/src/List.cc +++ b/src/List.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/List.h" #include "zeek/3rdparty/doctest.h" diff --git a/src/List.h b/src/List.h index 4db1ded1f3..b8351c5dc3 100644 --- a/src/List.h +++ b/src/List.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once // BaseList.h -- diff --git a/src/PacketFilter.cc b/src/PacketFilter.cc index 4fc771eec6..5fca22e5b7 100644 --- a/src/PacketFilter.cc +++ b/src/PacketFilter.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/PacketFilter.h" #include "zeek/IP.h" diff --git a/src/PacketFilter.h b/src/PacketFilter.h index 746cbe5946..76bef7b0cb 100644 --- a/src/PacketFilter.h +++ b/src/PacketFilter.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // Provides some very limited but fast packet filter mechanisms #pragma once diff --git a/src/PolicyFile.cc b/src/PolicyFile.cc index de82da11f0..b5e8662c65 100644 --- a/src/PolicyFile.cc +++ b/src/PolicyFile.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/PolicyFile.h" #include "zeek/zeek-config.h" diff --git a/src/PolicyFile.h b/src/PolicyFile.h index 91b3db5c13..be6f75c083 100644 --- a/src/PolicyFile.h +++ b/src/PolicyFile.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once // Functions for displaying the contents of policy files. diff --git a/src/PrefixTable.cc b/src/PrefixTable.cc index 9de77efa20..bc975c8fa4 100644 --- a/src/PrefixTable.cc +++ b/src/PrefixTable.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/PrefixTable.h" #include "zeek/Reporter.h" diff --git a/src/PrefixTable.h b/src/PrefixTable.h index 5a4532f136..343ff8539f 100644 --- a/src/PrefixTable.h +++ b/src/PrefixTable.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once extern "C" { diff --git a/src/RandTest.cc b/src/RandTest.cc index 23f409f7a7..3e94dd5556 100644 --- a/src/RandTest.cc +++ b/src/RandTest.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + /* Apply various randomness tests to a stream of bytes diff --git a/src/RandTest.h b/src/RandTest.h index 0a8b0f7485..6862e78504 100644 --- a/src/RandTest.h +++ b/src/RandTest.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include "zeek/zeek-config.h" diff --git a/src/Rule.cc b/src/Rule.cc index bfc00bdc6e..523ed5a61f 100644 --- a/src/Rule.cc +++ b/src/Rule.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/Rule.h" #include "zeek/zeek-config.h" diff --git a/src/Rule.h b/src/Rule.h index 5481d3d57b..7d91893260 100644 --- a/src/Rule.h +++ b/src/Rule.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include diff --git a/src/RuleAction.cc b/src/RuleAction.cc index db58b5c838..0e99cdc4c4 100644 --- a/src/RuleAction.cc +++ b/src/RuleAction.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/RuleAction.h" #include "zeek/zeek-config.h" diff --git a/src/RuleAction.h b/src/RuleAction.h index ed6ed064a7..5dbe688155 100644 --- a/src/RuleAction.h +++ b/src/RuleAction.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include // for u_char diff --git a/src/RuleCondition.cc b/src/RuleCondition.cc index 180b30b472..a888d53e53 100644 --- a/src/RuleCondition.cc +++ b/src/RuleCondition.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/RuleCondition.h" #include "zeek/zeek-config.h" diff --git a/src/RuleCondition.h b/src/RuleCondition.h index cd4331c5e3..f49e6661d2 100644 --- a/src/RuleCondition.h +++ b/src/RuleCondition.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include // for u_char diff --git a/src/RuleMatcher.cc b/src/RuleMatcher.cc index ef3860748f..4ef2211de4 100644 --- a/src/RuleMatcher.cc +++ b/src/RuleMatcher.cc @@ -1,3 +1,4 @@ +// See the file "COPYING" in the main distribution directory for copyright. #include "zeek/RuleMatcher.h" diff --git a/src/RuleMatcher.h b/src/RuleMatcher.h index 9f7e40e665..6f801645ba 100644 --- a/src/RuleMatcher.h +++ b/src/RuleMatcher.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include // for u_char diff --git a/src/ScannedFile.cc b/src/ScannedFile.cc index 44a6ddac30..f6a5aa83cc 100644 --- a/src/ScannedFile.cc +++ b/src/ScannedFile.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/ScannedFile.h" #include diff --git a/src/ScriptCoverageManager.cc b/src/ScriptCoverageManager.cc index 0b9e9c1f79..8603fb2755 100644 --- a/src/ScriptCoverageManager.cc +++ b/src/ScriptCoverageManager.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/ScriptCoverageManager.h" #include diff --git a/src/ScriptCoverageManager.h b/src/ScriptCoverageManager.h index b61a21f6aa..fe7886a969 100644 --- a/src/ScriptCoverageManager.h +++ b/src/ScriptCoverageManager.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include diff --git a/src/ScriptValidation.cc b/src/ScriptValidation.cc index 1510e1f53a..cc2bf69092 100644 --- a/src/ScriptValidation.cc +++ b/src/ScriptValidation.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/ScriptValidation.h" #include "zeek/Func.h" diff --git a/src/SerializationFormat.cc b/src/SerializationFormat.cc index 69679ab304..8f0bb88198 100644 --- a/src/SerializationFormat.cc +++ b/src/SerializationFormat.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/SerializationFormat.h" #include diff --git a/src/SerializationFormat.h b/src/SerializationFormat.h index c80f65e5d0..6f18aae740 100644 --- a/src/SerializationFormat.h +++ b/src/SerializationFormat.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // Implements different data formats for serialization. #pragma once diff --git a/src/Stats.cc b/src/Stats.cc index 7b776b0138..c4e14d9561 100644 --- a/src/Stats.cc +++ b/src/Stats.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/Stats.h" #include diff --git a/src/Stats.h b/src/Stats.h index 563e067684..129f047420 100644 --- a/src/Stats.h +++ b/src/Stats.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // Classes that collect and report statistics. #pragma once diff --git a/src/Trigger.cc b/src/Trigger.cc index 09718a99f4..eacf631bf9 100644 --- a/src/Trigger.cc +++ b/src/Trigger.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/Trigger.h" #include diff --git a/src/Trigger.h b/src/Trigger.h index 0cba28fd20..ad9929f238 100644 --- a/src/Trigger.h +++ b/src/Trigger.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include diff --git a/src/WeirdState.cc b/src/WeirdState.cc index de8f331eff..014676b3af 100644 --- a/src/WeirdState.cc +++ b/src/WeirdState.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/WeirdState.h" #include "zeek/RunState.h" diff --git a/src/ZeekArgs.cc b/src/ZeekArgs.cc index 6cc6785519..2efb8079b4 100644 --- a/src/ZeekArgs.cc +++ b/src/ZeekArgs.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/ZeekArgs.h" #include "zeek/Desc.h" diff --git a/src/analyzer/protocol/bittorrent/BitTorrent.cc b/src/analyzer/protocol/bittorrent/BitTorrent.cc index 0ae0e2888a..de9256b8f6 100644 --- a/src/analyzer/protocol/bittorrent/BitTorrent.cc +++ b/src/analyzer/protocol/bittorrent/BitTorrent.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // This code contributed by Nadi Sarrar. #include "zeek/analyzer/protocol/bittorrent/BitTorrent.h" diff --git a/src/analyzer/protocol/bittorrent/BitTorrent.h b/src/analyzer/protocol/bittorrent/BitTorrent.h index 6616ad3195..82186b3da1 100644 --- a/src/analyzer/protocol/bittorrent/BitTorrent.h +++ b/src/analyzer/protocol/bittorrent/BitTorrent.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // This code contributed by Nadi Sarrar. #pragma once diff --git a/src/analyzer/protocol/bittorrent/BitTorrentTracker.cc b/src/analyzer/protocol/bittorrent/BitTorrentTracker.cc index dd35f0e17d..52fd6d23bf 100644 --- a/src/analyzer/protocol/bittorrent/BitTorrentTracker.cc +++ b/src/analyzer/protocol/bittorrent/BitTorrentTracker.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // This code contributed by Nadi Sarrar. #include "zeek/analyzer/protocol/bittorrent/BitTorrentTracker.h" diff --git a/src/analyzer/protocol/bittorrent/BitTorrentTracker.h b/src/analyzer/protocol/bittorrent/BitTorrentTracker.h index 44768cda26..6f8812506a 100644 --- a/src/analyzer/protocol/bittorrent/BitTorrentTracker.h +++ b/src/analyzer/protocol/bittorrent/BitTorrentTracker.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // This code contributed by Nadi Sarrar. #pragma once diff --git a/src/analyzer/protocol/dhcp/DHCP.cc b/src/analyzer/protocol/dhcp/DHCP.cc index ff9d669ae0..0b49486dea 100644 --- a/src/analyzer/protocol/dhcp/DHCP.cc +++ b/src/analyzer/protocol/dhcp/DHCP.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/analyzer/protocol/dhcp/DHCP.h" #include "zeek/analyzer/protocol/dhcp/events.bif.h" diff --git a/src/analyzer/protocol/dhcp/DHCP.h b/src/analyzer/protocol/dhcp/DHCP.h index c70306fd5c..10a3e3c60b 100644 --- a/src/analyzer/protocol/dhcp/DHCP.h +++ b/src/analyzer/protocol/dhcp/DHCP.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include "analyzer/protocol/dhcp/dhcp_pac.h" diff --git a/src/analyzer/protocol/dnp3/DNP3.cc b/src/analyzer/protocol/dnp3/DNP3.cc index 7024c316cd..5da3719b26 100644 --- a/src/analyzer/protocol/dnp3/DNP3.cc +++ b/src/analyzer/protocol/dnp3/DNP3.cc @@ -1,4 +1,5 @@ -// +// See the file "COPYING" in the main distribution directory for copyright. + // DNP3 was initially used over serial links; it defined its own application // layer, transport layer, and data link layer. This hierarchy cannot be // mapped to the TCP/IP stack directly. As a result, all three DNP3 layers diff --git a/src/analyzer/protocol/dnp3/DNP3.h b/src/analyzer/protocol/dnp3/DNP3.h index e904bcc681..75007302b5 100644 --- a/src/analyzer/protocol/dnp3/DNP3.h +++ b/src/analyzer/protocol/dnp3/DNP3.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include "zeek/analyzer/protocol/tcp/TCP.h" diff --git a/src/analyzer/protocol/file/File.cc b/src/analyzer/protocol/file/File.cc index a569a00cdf..f3af1e2afd 100644 --- a/src/analyzer/protocol/file/File.cc +++ b/src/analyzer/protocol/file/File.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/analyzer/protocol/file/File.h" #include diff --git a/src/analyzer/protocol/file/File.h b/src/analyzer/protocol/file/File.h index d9073a6710..590ad01062 100644 --- a/src/analyzer/protocol/file/File.h +++ b/src/analyzer/protocol/file/File.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // Analyzer for connections that transfer binary data. #pragma once diff --git a/src/analyzer/protocol/irc/IRC.cc b/src/analyzer/protocol/irc/IRC.cc index b82e1a8a36..a0b36b6c4b 100644 --- a/src/analyzer/protocol/irc/IRC.cc +++ b/src/analyzer/protocol/irc/IRC.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // An IRC analyzer contributed by Roland Gruber. #include "zeek/analyzer/protocol/irc/IRC.h" diff --git a/src/analyzer/protocol/irc/IRC.h b/src/analyzer/protocol/irc/IRC.h index 07dc39819f..7dc40178ee 100644 --- a/src/analyzer/protocol/irc/IRC.h +++ b/src/analyzer/protocol/irc/IRC.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // An IRC analyzer contributed by Roland Gruber. #pragma once diff --git a/src/analyzer/protocol/mime/MIME.cc b/src/analyzer/protocol/mime/MIME.cc index e0773ee159..7a1180c5d8 100644 --- a/src/analyzer/protocol/mime/MIME.cc +++ b/src/analyzer/protocol/mime/MIME.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/analyzer/protocol/mime/MIME.h" #include "zeek/zeek-config.h" diff --git a/src/analyzer/protocol/mime/MIME.h b/src/analyzer/protocol/mime/MIME.h index 9b8ea9a7b2..c9079b3900 100644 --- a/src/analyzer/protocol/mime/MIME.h +++ b/src/analyzer/protocol/mime/MIME.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include diff --git a/src/analyzer/protocol/modbus/Modbus.cc b/src/analyzer/protocol/modbus/Modbus.cc index 65ab10d2dc..0dad53ad62 100644 --- a/src/analyzer/protocol/modbus/Modbus.cc +++ b/src/analyzer/protocol/modbus/Modbus.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/analyzer/protocol/modbus/Modbus.h" #include "zeek/analyzer/protocol/modbus/events.bif.h" diff --git a/src/analyzer/protocol/modbus/Modbus.h b/src/analyzer/protocol/modbus/Modbus.h index 86b516d526..c365089984 100644 --- a/src/analyzer/protocol/modbus/Modbus.h +++ b/src/analyzer/protocol/modbus/Modbus.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include "zeek/analyzer/protocol/tcp/TCP.h" diff --git a/src/analyzer/protocol/mqtt/MQTT.h b/src/analyzer/protocol/mqtt/MQTT.h index a4731dd59b..2818402b66 100644 --- a/src/analyzer/protocol/mqtt/MQTT.h +++ b/src/analyzer/protocol/mqtt/MQTT.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // Generated by binpac_quickstart #pragma once diff --git a/src/analyzer/protocol/ntp/NTP.cc b/src/analyzer/protocol/ntp/NTP.cc index f2225a7840..39e764ba36 100644 --- a/src/analyzer/protocol/ntp/NTP.cc +++ b/src/analyzer/protocol/ntp/NTP.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/analyzer/protocol/ntp/NTP.h" #include "zeek/Reporter.h" diff --git a/src/analyzer/protocol/ntp/NTP.h b/src/analyzer/protocol/ntp/NTP.h index 369a085b4b..2b1412e5dc 100644 --- a/src/analyzer/protocol/ntp/NTP.h +++ b/src/analyzer/protocol/ntp/NTP.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include "zeek/analyzer/protocol/ntp/events.bif.h" diff --git a/src/analyzer/protocol/pia/PIA.cc b/src/analyzer/protocol/pia/PIA.cc index 07cf4a49e1..884b4a67fe 100644 --- a/src/analyzer/protocol/pia/PIA.cc +++ b/src/analyzer/protocol/pia/PIA.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/analyzer/protocol/pia/PIA.h" #include "zeek/DebugLogger.h" diff --git a/src/analyzer/protocol/pia/PIA.h b/src/analyzer/protocol/pia/PIA.h index 09ab8dd0ca..b471214771 100644 --- a/src/analyzer/protocol/pia/PIA.h +++ b/src/analyzer/protocol/pia/PIA.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // An analyzer for application-layer protocol-detection. #pragma once diff --git a/src/analyzer/protocol/pop3/POP3.cc b/src/analyzer/protocol/pop3/POP3.cc index fc11b4ca41..fd2388b6ff 100644 --- a/src/analyzer/protocol/pop3/POP3.cc +++ b/src/analyzer/protocol/pop3/POP3.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // This code contributed to Zeek/Bro by Florian Schimandl, Hugh Dollman and // Robin Sommer. diff --git a/src/analyzer/protocol/pop3/POP3.h b/src/analyzer/protocol/pop3/POP3.h index 697bc73183..3191df8038 100644 --- a/src/analyzer/protocol/pop3/POP3.h +++ b/src/analyzer/protocol/pop3/POP3.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // This code contributed to Zeek/Bro by Florian Schimandl and Hugh Dollman. // // An analyser for the POP3 protocol. diff --git a/src/analyzer/protocol/radius/RADIUS.cc b/src/analyzer/protocol/radius/RADIUS.cc index 83b9d4352b..4820cda73e 100644 --- a/src/analyzer/protocol/radius/RADIUS.cc +++ b/src/analyzer/protocol/radius/RADIUS.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/analyzer/protocol/radius/RADIUS.h" #include "zeek/Reporter.h" diff --git a/src/analyzer/protocol/radius/RADIUS.h b/src/analyzer/protocol/radius/RADIUS.h index 90bfa514a9..baca03ee4f 100644 --- a/src/analyzer/protocol/radius/RADIUS.h +++ b/src/analyzer/protocol/radius/RADIUS.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include "zeek/analyzer/protocol/radius/events.bif.h" diff --git a/src/analyzer/protocol/rdp/Plugin.cc b/src/analyzer/protocol/rdp/Plugin.cc index cec1c73d8e..311ad7ceff 100644 --- a/src/analyzer/protocol/rdp/Plugin.cc +++ b/src/analyzer/protocol/rdp/Plugin.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/plugin/Plugin.h" #include "zeek/analyzer/Component.h" diff --git a/src/analyzer/protocol/rdp/RDP.cc b/src/analyzer/protocol/rdp/RDP.cc index f11bf88377..f1bce538b4 100644 --- a/src/analyzer/protocol/rdp/RDP.cc +++ b/src/analyzer/protocol/rdp/RDP.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/analyzer/protocol/rdp/RDP.h" #include "zeek/Reporter.h" diff --git a/src/analyzer/protocol/rdp/RDP.h b/src/analyzer/protocol/rdp/RDP.h index c09b4d521d..832127b3c7 100644 --- a/src/analyzer/protocol/rdp/RDP.h +++ b/src/analyzer/protocol/rdp/RDP.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include "zeek/analyzer/protocol/rdp/events.bif.h" diff --git a/src/analyzer/protocol/rdp/RDPEUDP.cc b/src/analyzer/protocol/rdp/RDPEUDP.cc index 41d62a8348..4d93a1b982 100644 --- a/src/analyzer/protocol/rdp/RDPEUDP.cc +++ b/src/analyzer/protocol/rdp/RDPEUDP.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/analyzer/protocol/rdp/RDPEUDP.h" #include "zeek/Reporter.h" diff --git a/src/analyzer/protocol/rdp/RDPEUDP.h b/src/analyzer/protocol/rdp/RDPEUDP.h index a56ec36bea..5b3222e92f 100644 --- a/src/analyzer/protocol/rdp/RDPEUDP.h +++ b/src/analyzer/protocol/rdp/RDPEUDP.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include "zeek/analyzer/protocol/rdp/events.bif.h" diff --git a/src/analyzer/protocol/rfb/Plugin.cc b/src/analyzer/protocol/rfb/Plugin.cc index 457f5c113a..16853c85f6 100644 --- a/src/analyzer/protocol/rfb/Plugin.cc +++ b/src/analyzer/protocol/rfb/Plugin.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/plugin/Plugin.h" #include "zeek/analyzer/Component.h" diff --git a/src/analyzer/protocol/rfb/RFB.cc b/src/analyzer/protocol/rfb/RFB.cc index c96a34e024..ee5e36722d 100644 --- a/src/analyzer/protocol/rfb/RFB.cc +++ b/src/analyzer/protocol/rfb/RFB.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/analyzer/protocol/rfb/RFB.h" #include "zeek/Reporter.h" diff --git a/src/analyzer/protocol/rfb/RFB.h b/src/analyzer/protocol/rfb/RFB.h index 4007bcc782..9d799a9271 100644 --- a/src/analyzer/protocol/rfb/RFB.h +++ b/src/analyzer/protocol/rfb/RFB.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include "zeek/analyzer/protocol/rfb/events.bif.h" diff --git a/src/analyzer/protocol/sip/SIP.cc b/src/analyzer/protocol/sip/SIP.cc index cbbcf5115a..7409d0d635 100644 --- a/src/analyzer/protocol/sip/SIP.cc +++ b/src/analyzer/protocol/sip/SIP.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/analyzer/protocol/sip/SIP.h" #include "zeek/analyzer/protocol/sip/events.bif.h" diff --git a/src/analyzer/protocol/sip/SIP.h b/src/analyzer/protocol/sip/SIP.h index 781a6401e7..5f0f652620 100644 --- a/src/analyzer/protocol/sip/SIP.h +++ b/src/analyzer/protocol/sip/SIP.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include "zeek/analyzer/protocol/sip/events.bif.h" diff --git a/src/analyzer/protocol/smb/SMB.cc b/src/analyzer/protocol/smb/SMB.cc index 38c0f3cb6f..6f5650b512 100644 --- a/src/analyzer/protocol/smb/SMB.cc +++ b/src/analyzer/protocol/smb/SMB.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/analyzer/protocol/smb/SMB.h" namespace zeek::analyzer::smb { diff --git a/src/analyzer/protocol/smb/SMB.h b/src/analyzer/protocol/smb/SMB.h index c0fd30d0fb..427d3fcc6a 100644 --- a/src/analyzer/protocol/smb/SMB.h +++ b/src/analyzer/protocol/smb/SMB.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include "zeek/analyzer/protocol/tcp/TCP.h" diff --git a/src/analyzer/protocol/smtp/BDAT.cc b/src/analyzer/protocol/smtp/BDAT.cc index 9bbd759c7a..f83d011592 100644 --- a/src/analyzer/protocol/smtp/BDAT.cc +++ b/src/analyzer/protocol/smtp/BDAT.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/analyzer/protocol/smtp/BDAT.h" #include "zeek/Conn.h" diff --git a/src/analyzer/protocol/socks/SOCKS.cc b/src/analyzer/protocol/socks/SOCKS.cc index fe9a87ff7a..097bde61be 100644 --- a/src/analyzer/protocol/socks/SOCKS.cc +++ b/src/analyzer/protocol/socks/SOCKS.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/analyzer/protocol/socks/SOCKS.h" #include "zeek/analyzer/protocol/socks/events.bif.h" diff --git a/src/analyzer/protocol/socks/SOCKS.h b/src/analyzer/protocol/socks/SOCKS.h index e05eb4faf6..108b4e3aa6 100644 --- a/src/analyzer/protocol/socks/SOCKS.h +++ b/src/analyzer/protocol/socks/SOCKS.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once // SOCKS v4 analyzer. diff --git a/src/analyzer/protocol/ssl/DTLS.cc b/src/analyzer/protocol/ssl/DTLS.cc index 529d167004..582b1272d3 100644 --- a/src/analyzer/protocol/ssl/DTLS.cc +++ b/src/analyzer/protocol/ssl/DTLS.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/analyzer/protocol/ssl/DTLS.h" #include "zeek/Reporter.h" diff --git a/src/analyzer/protocol/ssl/DTLS.h b/src/analyzer/protocol/ssl/DTLS.h index f6d2cbc256..8b7f95bdcb 100644 --- a/src/analyzer/protocol/ssl/DTLS.h +++ b/src/analyzer/protocol/ssl/DTLS.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include "zeek/analyzer/protocol/ssl/events.bif.h" diff --git a/src/analyzer/protocol/ssl/SSL.cc b/src/analyzer/protocol/ssl/SSL.cc index b6a3043a21..f94abc0db7 100644 --- a/src/analyzer/protocol/ssl/SSL.cc +++ b/src/analyzer/protocol/ssl/SSL.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/analyzer/protocol/ssl/SSL.h" #include diff --git a/src/analyzer/protocol/ssl/SSL.h b/src/analyzer/protocol/ssl/SSL.h index 5fb3cb8ac6..e721e0e600 100644 --- a/src/analyzer/protocol/ssl/SSL.h +++ b/src/analyzer/protocol/ssl/SSL.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include "zeek/analyzer/protocol/pia/PIA.h" diff --git a/src/analyzer/protocol/syslog/legacy/Syslog.cc b/src/analyzer/protocol/syslog/legacy/Syslog.cc index 986aef5fec..de9d1f08cf 100644 --- a/src/analyzer/protocol/syslog/legacy/Syslog.cc +++ b/src/analyzer/protocol/syslog/legacy/Syslog.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/analyzer/protocol/syslog/legacy/Syslog.h" #include "zeek/analyzer/protocol/syslog/legacy/events.bif.h" diff --git a/src/analyzer/protocol/syslog/legacy/Syslog.h b/src/analyzer/protocol/syslog/legacy/Syslog.h index 0c01c022f1..d64f8bc40b 100644 --- a/src/analyzer/protocol/syslog/legacy/Syslog.h +++ b/src/analyzer/protocol/syslog/legacy/Syslog.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include "zeek/analyzer/protocol/tcp/TCP.h" diff --git a/src/analyzer/protocol/tcp/ContentLine.cc b/src/analyzer/protocol/tcp/ContentLine.cc index 5544338434..018ab5ecf0 100644 --- a/src/analyzer/protocol/tcp/ContentLine.cc +++ b/src/analyzer/protocol/tcp/ContentLine.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/analyzer/protocol/tcp/ContentLine.h" #include "zeek/Reporter.h" diff --git a/src/analyzer/protocol/tcp/ContentLine.h b/src/analyzer/protocol/tcp/ContentLine.h index de069ebb62..6c6a557ad4 100644 --- a/src/analyzer/protocol/tcp/ContentLine.h +++ b/src/analyzer/protocol/tcp/ContentLine.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // Support-analyzer to split a reassembled stream into lines. #pragma once diff --git a/src/analyzer/protocol/tcp/TCP_Flags.h b/src/analyzer/protocol/tcp/TCP_Flags.h index ba4c7098a9..989e3feb23 100644 --- a/src/analyzer/protocol/tcp/TCP_Flags.h +++ b/src/analyzer/protocol/tcp/TCP_Flags.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once // This needs to remain the first include in this file, or some defines aren't diff --git a/src/analyzer/protocol/tcp/TCP_Reassembler.cc b/src/analyzer/protocol/tcp/TCP_Reassembler.cc index 9a63962db2..bc9df6b076 100644 --- a/src/analyzer/protocol/tcp/TCP_Reassembler.cc +++ b/src/analyzer/protocol/tcp/TCP_Reassembler.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/analyzer/protocol/tcp/TCP_Reassembler.h" #include diff --git a/src/analyzer/protocol/tcp/TCP_Reassembler.h b/src/analyzer/protocol/tcp/TCP_Reassembler.h index fc889f342b..3a8f626862 100644 --- a/src/analyzer/protocol/tcp/TCP_Reassembler.h +++ b/src/analyzer/protocol/tcp/TCP_Reassembler.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include "zeek/File.h" diff --git a/src/binpac_zeek.h b/src/binpac_zeek.h index 04849c57f8..7f3309c127 100644 --- a/src/binpac_zeek.h +++ b/src/binpac_zeek.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include diff --git a/src/broker/Data.cc b/src/broker/Data.cc index ee8777e986..e9a6492b88 100644 --- a/src/broker/Data.cc +++ b/src/broker/Data.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/broker/Data.h" #include diff --git a/src/broker/Data.h b/src/broker/Data.h index e92c75ab85..6b20fd4aeb 100644 --- a/src/broker/Data.h +++ b/src/broker/Data.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index 6e5dd26cd7..7584c05fd4 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/broker/Manager.h" #include diff --git a/src/broker/Manager.h b/src/broker/Manager.h index 9701d79d39..51974685d9 100644 --- a/src/broker/Manager.h +++ b/src/broker/Manager.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include diff --git a/src/broker/Plugin.cc b/src/broker/Plugin.cc index 3f252f2690..7fd982efa9 100644 --- a/src/broker/Plugin.cc +++ b/src/broker/Plugin.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/broker/Plugin.h" #include diff --git a/src/broker/Store.cc b/src/broker/Store.cc index 7504b3f08a..849f2697d1 100644 --- a/src/broker/Store.cc +++ b/src/broker/Store.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/broker/Store.h" #include "zeek/Desc.h" diff --git a/src/broker/Store.h b/src/broker/Store.h index db5333d3c9..8babc1d344 100644 --- a/src/broker/Store.h +++ b/src/broker/Store.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include diff --git a/src/cluster/BifSupport.cc b/src/cluster/BifSupport.cc index 1ed4b49760..40e62ed29f 100644 --- a/src/cluster/BifSupport.cc +++ b/src/cluster/BifSupport.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/cluster/BifSupport.h" #include "zeek/Desc.h" diff --git a/src/cluster/Manager.cc b/src/cluster/Manager.cc index eff6196164..e4e1ac8f71 100644 --- a/src/cluster/Manager.cc +++ b/src/cluster/Manager.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/cluster/Manager.h" #include "zeek/cluster/Serializer.h" diff --git a/src/cluster/serializer/binary-serialization-format/Plugin.cc b/src/cluster/serializer/binary-serialization-format/Plugin.cc index dc328b131a..1513321a91 100644 --- a/src/cluster/serializer/binary-serialization-format/Plugin.cc +++ b/src/cluster/serializer/binary-serialization-format/Plugin.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/cluster/serializer/binary-serialization-format/Plugin.h" #include "zeek/cluster/Component.h" diff --git a/src/cluster/serializer/binary-serialization-format/Serializer.cc b/src/cluster/serializer/binary-serialization-format/Serializer.cc index fdddeacd54..1bb2e49710 100644 --- a/src/cluster/serializer/binary-serialization-format/Serializer.cc +++ b/src/cluster/serializer/binary-serialization-format/Serializer.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/cluster/serializer/binary-serialization-format/Serializer.h" #include diff --git a/src/cluster/serializer/broker/Plugin.cc b/src/cluster/serializer/broker/Plugin.cc index aa9ab83d79..86ae0e2578 100644 --- a/src/cluster/serializer/broker/Plugin.cc +++ b/src/cluster/serializer/broker/Plugin.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "Plugin.h" #include diff --git a/src/cluster/serializer/broker/Serializer.cc b/src/cluster/serializer/broker/Serializer.cc index 068dd20767..6b1b036f22 100644 --- a/src/cluster/serializer/broker/Serializer.cc +++ b/src/cluster/serializer/broker/Serializer.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/cluster/serializer/broker/Serializer.h" #include diff --git a/src/file_analysis/analyzer/pe/PE.cc b/src/file_analysis/analyzer/pe/PE.cc index c591145c70..70d36ecf81 100644 --- a/src/file_analysis/analyzer/pe/PE.cc +++ b/src/file_analysis/analyzer/pe/PE.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/file_analysis/analyzer/pe/PE.h" #include "zeek/file_analysis/Manager.h" diff --git a/src/file_analysis/analyzer/pe/PE.h b/src/file_analysis/analyzer/pe/PE.h index 83c8cee17c..33d3617415 100644 --- a/src/file_analysis/analyzer/pe/PE.h +++ b/src/file_analysis/analyzer/pe/PE.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include diff --git a/src/fuzzers/FuzzBuffer.cc b/src/fuzzers/FuzzBuffer.cc index 25cece20c1..89608b9ed9 100644 --- a/src/fuzzers/FuzzBuffer.cc +++ b/src/fuzzers/FuzzBuffer.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #if ! defined(_GNU_SOURCE) #define _GNU_SOURCE #endif diff --git a/src/fuzzers/FuzzBuffer.h b/src/fuzzers/FuzzBuffer.h index 9188fb6221..dcd1932ef7 100644 --- a/src/fuzzers/FuzzBuffer.h +++ b/src/fuzzers/FuzzBuffer.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include diff --git a/src/fuzzers/dns-fuzzer.cc b/src/fuzzers/dns-fuzzer.cc index 4472aa09e9..cd0b9f657c 100644 --- a/src/fuzzers/dns-fuzzer.cc +++ b/src/fuzzers/dns-fuzzer.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include #include "zeek/Conn.h" diff --git a/src/fuzzers/fuzzer-setup.h b/src/fuzzers/fuzzer-setup.h index 706290a4bf..3c2d20f18f 100644 --- a/src/fuzzers/fuzzer-setup.h +++ b/src/fuzzers/fuzzer-setup.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include diff --git a/src/fuzzers/generic-analyzer-fuzzer.cc b/src/fuzzers/generic-analyzer-fuzzer.cc index d00273bda6..bdaf6c0fe8 100644 --- a/src/fuzzers/generic-analyzer-fuzzer.cc +++ b/src/fuzzers/generic-analyzer-fuzzer.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // Generic protocol analyzer fuzzer. // // Expects ZEEK_FUZZ_ANALYZER and ZEEK_FUZZ_ANALYZER_TRANSPORT to be set. diff --git a/src/fuzzers/packet-fuzzer.cc b/src/fuzzers/packet-fuzzer.cc index 1b6b55be98..25b69d6517 100644 --- a/src/fuzzers/packet-fuzzer.cc +++ b/src/fuzzers/packet-fuzzer.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #ifdef _MSC_VER #include #endif diff --git a/src/fuzzers/standalone-driver.cc b/src/fuzzers/standalone-driver.cc index f60e1519c6..3199d3ff06 100644 --- a/src/fuzzers/standalone-driver.cc +++ b/src/fuzzers/standalone-driver.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include #include #include diff --git a/src/fuzzers/websocket-fuzzer.cc b/src/fuzzers/websocket-fuzzer.cc index 1d1b47d39e..a83dfe4341 100644 --- a/src/fuzzers/websocket-fuzzer.cc +++ b/src/fuzzers/websocket-fuzzer.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include #include "zeek/Conn.h" diff --git a/src/iosource/Packet.cc b/src/iosource/Packet.cc index d5095edfca..5c5e7c631c 100644 --- a/src/iosource/Packet.cc +++ b/src/iosource/Packet.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/iosource/Packet.h" extern "C" { diff --git a/src/iosource/Packet.h b/src/iosource/Packet.h index 0a6314618b..a3796d19df 100644 --- a/src/iosource/Packet.h +++ b/src/iosource/Packet.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include "zeek/zeek-config.h" diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc index 97aee69d3a..8a51c76724 100644 --- a/src/logging/WriterFrontend.cc +++ b/src/logging/WriterFrontend.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/logging/WriterFrontend.h" #include "zeek/RunState.h" diff --git a/src/logging/writers/none/None.cc b/src/logging/writers/none/None.cc index 1227dd3eb9..d17870e331 100644 --- a/src/logging/writers/none/None.cc +++ b/src/logging/writers/none/None.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/logging/writers/none/None.h" #include diff --git a/src/module_util.h b/src/module_util.h index 81a4ce4796..c97ac3c82f 100644 --- a/src/module_util.h +++ b/src/module_util.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // // These functions are used by both Zeek and bifcl. // diff --git a/src/packet_analysis/protocol/gtpv1/GTPv1.h b/src/packet_analysis/protocol/gtpv1/GTPv1.h index c97d575d88..f48707dea1 100644 --- a/src/packet_analysis/protocol/gtpv1/GTPv1.h +++ b/src/packet_analysis/protocol/gtpv1/GTPv1.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include "zeek/packet_analysis/Analyzer.h" diff --git a/src/packet_analysis/protocol/ip/SessionAdapter.cc b/src/packet_analysis/protocol/ip/SessionAdapter.cc index 742f5638fd..7963c8e4b5 100644 --- a/src/packet_analysis/protocol/ip/SessionAdapter.cc +++ b/src/packet_analysis/protocol/ip/SessionAdapter.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/packet_analysis/protocol/ip/SessionAdapter.h" #include "zeek/File.h" diff --git a/src/packet_analysis/protocol/ip/SessionAdapter.h b/src/packet_analysis/protocol/ip/SessionAdapter.h index 0e4d74d91c..4fca79da85 100644 --- a/src/packet_analysis/protocol/ip/SessionAdapter.h +++ b/src/packet_analysis/protocol/ip/SessionAdapter.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include "zeek/analyzer/Analyzer.h" diff --git a/src/packet_analysis/protocol/teredo/Teredo.cc b/src/packet_analysis/protocol/teredo/Teredo.cc index ea4b551ab2..b12ff52235 100644 --- a/src/packet_analysis/protocol/teredo/Teredo.cc +++ b/src/packet_analysis/protocol/teredo/Teredo.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/packet_analysis/protocol/teredo/Teredo.h" #include "zeek/Conn.h" diff --git a/src/packet_analysis/protocol/teredo/Teredo.h b/src/packet_analysis/protocol/teredo/Teredo.h index d482157b92..d4577d02ae 100644 --- a/src/packet_analysis/protocol/teredo/Teredo.h +++ b/src/packet_analysis/protocol/teredo/Teredo.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include diff --git a/src/plugin/ComponentManager.h b/src/plugin/ComponentManager.h index deb2317748..9304a60cdc 100644 --- a/src/plugin/ComponentManager.h +++ b/src/plugin/ComponentManager.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include diff --git a/src/script_opt/Expr.h b/src/script_opt/Expr.h index f1b8427319..491b9c59fe 100644 --- a/src/script_opt/Expr.h +++ b/src/script_opt/Expr.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/Expr.h" namespace zeek::detail { diff --git a/src/session/Key.cc b/src/session/Key.cc index 6ab31b701a..669ddb4c61 100644 --- a/src/session/Key.cc +++ b/src/session/Key.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/session/Key.h" #include diff --git a/src/telemetry/Counter.cc b/src/telemetry/Counter.cc index 889a17f1a7..154a552a64 100644 --- a/src/telemetry/Counter.cc +++ b/src/telemetry/Counter.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/telemetry/Counter.h" using namespace zeek::telemetry; diff --git a/src/telemetry/Gauge.cc b/src/telemetry/Gauge.cc index 06ddbc95cc..21d6e50ca5 100644 --- a/src/telemetry/Gauge.cc +++ b/src/telemetry/Gauge.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/telemetry/Gauge.h" using namespace zeek::telemetry; diff --git a/src/telemetry/Histogram.cc b/src/telemetry/Histogram.cc index 6ebae60c08..d3c2d22a60 100644 --- a/src/telemetry/Histogram.cc +++ b/src/telemetry/Histogram.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/telemetry/Histogram.h" using namespace zeek::telemetry; diff --git a/src/telemetry/Opaques.cc b/src/telemetry/Opaques.cc index d5f1c50de5..fdd9edb6e0 100644 --- a/src/telemetry/Opaques.cc +++ b/src/telemetry/Opaques.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "Opaques.h" using namespace zeek; diff --git a/src/telemetry/ProcessStats.cc b/src/telemetry/ProcessStats.cc index 476efd4487..58c534c64b 100644 --- a/src/telemetry/ProcessStats.cc +++ b/src/telemetry/ProcessStats.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/telemetry/ProcessStats.h" #include "zeek/util.h" diff --git a/src/telemetry/ProcessStats.h b/src/telemetry/ProcessStats.h index d79bb2cb5f..7d4f78c33f 100644 --- a/src/telemetry/ProcessStats.h +++ b/src/telemetry/ProcessStats.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include "zeek/zeek-config.h" diff --git a/src/telemetry/Utils.cc b/src/telemetry/Utils.cc index b17ee169ae..d2c7741e18 100644 --- a/src/telemetry/Utils.cc +++ b/src/telemetry/Utils.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "Utils.h" #include "zeek/ID.h" diff --git a/src/telemetry/Utils.h b/src/telemetry/Utils.h index 77489fc260..0ce613151e 100644 --- a/src/telemetry/Utils.h +++ b/src/telemetry/Utils.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc index 222cb6406d..f5c6a588ad 100644 --- a/src/threading/BasicThread.cc +++ b/src/threading/BasicThread.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/threading/BasicThread.h" #include "zeek/zeek-config.h" diff --git a/src/threading/BasicThread.h b/src/threading/BasicThread.h index 103c11de3a..9d94a2df89 100644 --- a/src/threading/BasicThread.h +++ b/src/threading/BasicThread.h @@ -1,3 +1,4 @@ +// See the file "COPYING" in the main distribution directory for copyright. #pragma once diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index 12f0fa432e..e34cb92a6a 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/threading/Manager.h" #include diff --git a/src/threading/Manager.h b/src/threading/Manager.h index 897068bee5..6eaefe5771 100644 --- a/src/threading/Manager.h +++ b/src/threading/Manager.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc index 5bfb8ebeb4..b5621de96e 100644 --- a/src/threading/MsgThread.cc +++ b/src/threading/MsgThread.cc @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #include "zeek/threading/MsgThread.h" #include diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h index 0bb3dbaec4..0e486cf592 100644 --- a/src/threading/MsgThread.h +++ b/src/threading/MsgThread.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include diff --git a/src/threading/Queue.h b/src/threading/Queue.h index 38e78396e2..b3f949f8eb 100644 --- a/src/threading/Queue.h +++ b/src/threading/Queue.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once #include diff --git a/src/threading/SerialTypes.h b/src/threading/SerialTypes.h index 1a0ea21f1b..2e7fbb0e37 100644 --- a/src/threading/SerialTypes.h +++ b/src/threading/SerialTypes.h @@ -1,3 +1,4 @@ +// See the file "COPYING" in the main distribution directory for copyright. #pragma once diff --git a/src/threading/formatters/detail/json.h b/src/threading/formatters/detail/json.h index 7a947aaaa6..1bafc2bf6c 100644 --- a/src/threading/formatters/detail/json.h +++ b/src/threading/formatters/detail/json.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + // Not installed - used by Val.cc and formatters/JSON.cc only. #pragma once diff --git a/src/zeek-bif.h b/src/zeek-bif.h index 3e7a53da9e..8e40acafe8 100644 --- a/src/zeek-bif.h +++ b/src/zeek-bif.h @@ -1,3 +1,5 @@ +// See the file "COPYING" in the main distribution directory for copyright. + #pragma once // Headers to include by generated BiF code. From bbd7f56dcca3e0f4b6012fda7da39726094ae65c Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 6 Dec 2024 13:45:46 -0700 Subject: [PATCH 05/65] Add signatures for Python bytecode for 3.8-3.14 --- .../frameworks/files/magic/executable.sig | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/scripts/base/frameworks/files/magic/executable.sig b/scripts/base/frameworks/files/magic/executable.sig index b1dae5db8b..9a01d09d41 100644 --- a/scripts/base/frameworks/files/magic/executable.sig +++ b/scripts/base/frameworks/files/magic/executable.sig @@ -42,6 +42,11 @@ signature file-elc { file-magic /\x3bELC[\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff]/ } +# Python magic numbers can be updated/added by looking at the list at +# https://github.com/python/cpython/blob/main/Include/internal/pycore_magic_number.h +# The numbers in the list are converted to little-endian and then to hex for the +# file-magic entries below. + # Python 1 bytecode signature file-pyc-1 { file-magic /^(\xfc\xc4|\x99\x4e)\x0d\x0a/ @@ -104,3 +109,47 @@ signature file-pyc-3-7 { file-magic /^[\x3e-\x42]\x0d\x0d\x0a/ file-mime "application/x-python-bytecode", 80 } + +# Python 3.8 bytecode +signature file-pyc-3-8 { + file-magic /^[\x48\x49\x52-\x55]\x0d\x0d\x0a/ + file-mime "application/x-python-bytecode", 80 +} + +# Python 3.9 bytecode +signature file-pyc-3-9 { + file-magic /^[\x5c-\x61]\x0d\x0d\x0a/ + file-mime "application/x-python-bytecode", 80 +} + +# Python 3.10 bytecode +signature file-pyc-3-10 { + file-magic /^[\x66-\x6f]\x0d\x0d\x0a/ + file-mime "application/x-python-bytecode", 80 +} + +# Python 3.11 bytecode +signature file-pyc-3-11 { + file-magic /^[\x7a-\xa7]\x0d\x0d\x0a/ + file-mime "application/x-python-bytecode", 80 +} + +# Python 3.12 bytecode +signature file-pyc-3-12 { + file-magic /^[\xac-\xcb]\x0d\x0d\x0a/ + file-mime "application/x-python-bytecode", 80 +} + +# Python 3.13 bytecode +signature file-pyc-3-13 { + file-magic /^[\xde-\xf3]\x0d\x0d\x0a/ + file-mime "application/x-python-bytecode", 80 +} + +# Python 3.14 bytecode +# This is in pre-release at this time, and may need to be updated as new +# versions come out. +signature file-pyc-3-14 { + file-magic /^[\x10-\x19]\x0e\x0d\x0a/ + file-mime "application/x-python-bytecode", 80 +} From e81856a4af323b634af9936ead01f083ba84eceb Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Fri, 8 Nov 2024 13:03:57 -0800 Subject: [PATCH 06/65] No need to namespace Cluster:: functions in their own namespace --- scripts/base/frameworks/cluster/main.zeek | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/base/frameworks/cluster/main.zeek b/scripts/base/frameworks/cluster/main.zeek index caf2e6a11d..d610969cff 100644 --- a/scripts/base/frameworks/cluster/main.zeek +++ b/scripts/base/frameworks/cluster/main.zeek @@ -336,7 +336,7 @@ function nodes_with_type(node_type: NodeType): vector of NamedNode { return strcmp(n1$name, n2$name); }); } -function Cluster::get_node_count(node_type: NodeType): count +function get_node_count(node_type: NodeType): count { local cnt = 0; @@ -349,7 +349,7 @@ function Cluster::get_node_count(node_type: NodeType): count return cnt; } -function Cluster::get_active_node_count(node_type: NodeType): count +function get_active_node_count(node_type: NodeType): count { return node_type in active_node_ids ? |active_node_ids[node_type]| : 0; } From b9df1674b7c317fc62e3a2e994a01559cf52ed65 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Wed, 9 Oct 2024 22:35:23 -0700 Subject: [PATCH 07/65] Bump Broker to pull in disconnect feature and infinite-loop fix --- auxil/broker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auxil/broker b/auxil/broker index 2a6e6201f7..28cdb7524f 160000 --- a/auxil/broker +++ b/auxil/broker @@ -1 +1 @@ -Subproject commit 2a6e6201f7b43e213f2bac3863ca571b659e8a16 +Subproject commit 28cdb7524f73ffa37315f4058f4f48948fe1683a From 4c4eb4b8e28c7e79bebeaec549b2466095d8d97a Mon Sep 17 00:00:00 2001 From: Dominik Charousset Date: Wed, 25 Sep 2024 16:26:35 +0200 Subject: [PATCH 08/65] Add Zeek-level configurability of Broker slow-peer disconnects --- scripts/base/frameworks/broker/main.zeek | 18 ++++++++++++++ src/broker/Manager.cc | 30 ++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/scripts/base/frameworks/broker/main.zeek b/scripts/base/frameworks/broker/main.zeek index d41f64ab2e..2990f3f297 100644 --- a/scripts/base/frameworks/broker/main.zeek +++ b/scripts/base/frameworks/broker/main.zeek @@ -86,6 +86,24 @@ export { ## ZEEK_BROKER_MAX_THREADS environment variable overrides this setting. const max_threads = 1 &redef; + ## Max number of items we buffer at most per peer. What action to take when + ## the buffer reaches its maximum size is determined by + ## `peer_overflow_policy`. + const peer_buffer_size = 2048 &redef; + + ## Configures how Broker responds to peers that cannot keep up with the + ## incoming message rate. Available strategies: + ## - disconnect: drop the connection to the unresponsive peer + ## - drop_newest: replace the newest message in the buffer + ## - drop_oldest: removed the olsted message from the buffer, then append + const peer_overflow_policy = "disconnect" &redef; + + ## Same as `peer_buffer_size` but for WebSocket clients. + const web_socket_buffer_size = 512 &redef; + + ## Same as `peer_overflow_policy` but for WebSocket clients. + const web_socket_overflow_policy = "disconnect" &redef; + ## The CAF scheduling policy to use. Available options are "sharing" and ## "stealing". The "sharing" policy uses a single, global work queue along ## with mutex and condition variable used for accessing it, which may be diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index 6e5dd26cd7..e5e44e567f 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -255,6 +255,36 @@ void Manager::DoInitPostScript() { options.disable_forwarding = ! get_option("Broker::forward_messages")->AsBool(); options.use_real_time = use_real_time; + options.peer_buffer_size = get_option("Broker::peer_buffer_size")->AsCount(); + auto peer_overflow_policy = get_option("Broker::peer_overflow_policy")->AsString()->CheckString(); + if ( util::streq(peer_overflow_policy, "disconnect") ) { + options.peer_overflow_policy = broker::overflow_policy::disconnect; + } + else if ( util::streq(peer_overflow_policy, "drop_oldest") ) { + options.peer_overflow_policy = broker::overflow_policy::drop_oldest; + } + else if ( util::streq(peer_overflow_policy, "drop_newest") ) { + options.peer_overflow_policy = broker::overflow_policy::drop_newest; + } + else { + reporter->FatalError("Invalid Broker::peer_overflow_policy: %s", peer_overflow_policy); + } + + options.web_socket_buffer_size = get_option("Broker::web_socket_buffer_size")->AsCount(); + auto web_socket_overflow_policy = get_option("Broker::web_socket_overflow_policy")->AsString()->CheckString(); + if ( util::streq(web_socket_overflow_policy, "disconnect") ) { + options.web_socket_overflow_policy = broker::overflow_policy::disconnect; + } + else if ( util::streq(web_socket_overflow_policy, "drop_oldest") ) { + options.web_socket_overflow_policy = broker::overflow_policy::drop_oldest; + } + else if ( util::streq(web_socket_overflow_policy, "drop_newest") ) { + options.web_socket_overflow_policy = broker::overflow_policy::drop_newest; + } + else { + reporter->FatalError("Invalid Broker::web_socket_overflow_policy: %s", web_socket_overflow_policy); + } + broker::configuration config{std::move(options)}; config.openssl_cafile(get_option("Broker::ssl_cafile")->AsString()->CheckString()); From 0010e65f6d7b0e3216a771956d31cb9e0747084b Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Fri, 11 Oct 2024 17:12:03 -0700 Subject: [PATCH 09/65] Support re-peering with Broker peers that fall behind This adds re-peering at the Broker level for peers that Broker decided to unpeer. We keep this at the Broker level since this behavior is specific to it (as opposed to other cluster backends). Includes baseline updates for btests that pick up on the new script's @load. --- scripts/base/frameworks/broker/__load__.zeek | 1 + .../base/frameworks/broker/backpressure.zeek | 35 +++++++++++++++++++ .../canonified_loaded_scripts.log | 1 + .../canonified_loaded_scripts.log | 1 + testing/btest/Baseline/plugins.hooks/output | 6 ++++ 5 files changed, 44 insertions(+) create mode 100644 scripts/base/frameworks/broker/backpressure.zeek diff --git a/scripts/base/frameworks/broker/__load__.zeek b/scripts/base/frameworks/broker/__load__.zeek index 77dd69d554..a30468a776 100644 --- a/scripts/base/frameworks/broker/__load__.zeek +++ b/scripts/base/frameworks/broker/__load__.zeek @@ -1,3 +1,4 @@ @load ./main @load ./store @load ./log +@load ./backpressure diff --git a/scripts/base/frameworks/broker/backpressure.zeek b/scripts/base/frameworks/broker/backpressure.zeek new file mode 100644 index 0000000000..652935eed9 --- /dev/null +++ b/scripts/base/frameworks/broker/backpressure.zeek @@ -0,0 +1,35 @@ +##! This handles Broker peers that fall so far behind in handling messages that +##! this node sends it that the local Broker endpoint decides to unpeer them. +##! Zeek captures this as follows: +##! +##! - In broker.log, with a regular "peer-removed" entry indicating CAF's reason. +##! - Via eventing through :zeek:see:`Broker::peer_removed` as done in this script. +##! +##! The cluster framework additionally captures the unpeering as follows: +##! +##! - In cluster.log, with a higher-level message indicating the node names involved. +##! - Via telemetry, using a labeled counter. + +event Broker::peer_removed(endpoint: Broker::EndpointInfo, msg: string) + { + if ( "caf::sec::backpressure_overflow" !in msg ) { + return; + } + + if ( ! endpoint?$network ) { + Reporter::error(fmt("Missing network info to re-peer with %s", endpoint$id)); + return; + } + + # Re-establish the peering so Broker's reconnect behavior kicks in once + # the other endpoint catches up. Broker will periodically re-try + # connecting as necessary. If the other endpoint originally connected to + # us, our attempt will fail (since we attempt to connect to the peer's + # ephemeral port), but in that case the peer will reconnect with us once + # it recovers. + # + # We could do this more cleanly by leveraging information from the + # cluster framework (since it knows who connects to whom), but that + # would further entangle Broker into it. + Broker::peer(endpoint$network$address, endpoint$network$bound_port); +} diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 5e88f9d327..443d35b00e 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -119,6 +119,7 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/data.bif.zeek build/scripts/base/bif/store.bif.zeek scripts/base/frameworks/broker/log.zeek + scripts/base/frameworks/broker/backpressure.zeek scripts/base/frameworks/supervisor/__load__.zeek scripts/base/frameworks/supervisor/control.zeek scripts/base/frameworks/supervisor/main.zeek diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index a3f06f9db9..8a23826c17 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -119,6 +119,7 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/data.bif.zeek build/scripts/base/bif/store.bif.zeek scripts/base/frameworks/broker/log.zeek + scripts/base/frameworks/broker/backpressure.zeek scripts/base/frameworks/supervisor/__load__.zeek scripts/base/frameworks/supervisor/control.zeek scripts/base/frameworks/supervisor/main.zeek diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 48ce2af63b..42b310bcc5 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -461,6 +461,7 @@ 0.000000 MetaHookPost LoadFile(0, ./addrs, <...>/addrs.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./analyzer.bif.zeek, <...>/analyzer.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./api, <...>/api.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, ./backpressure, <...>/backpressure.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./bloom-filter.bif.zeek, <...>/bloom-filter.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./cardinality-counter.bif.zeek, <...>/cardinality-counter.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./cluster.bif.zeek, <...>/cluster.bif.zeek) -> -1 @@ -765,6 +766,7 @@ 0.000000 MetaHookPost LoadFileExtended(0, ./addrs, <...>/addrs.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./analyzer.bif.zeek, <...>/analyzer.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./api, <...>/api.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./backpressure, <...>/backpressure.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./bloom-filter.bif.zeek, <...>/bloom-filter.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./cardinality-counter.bif.zeek, <...>/cardinality-counter.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./cluster.bif.zeek, <...>/cluster.bif.zeek) -> (-1, ) @@ -1401,6 +1403,7 @@ 0.000000 MetaHookPre LoadFile(0, ./addrs, <...>/addrs.zeek) 0.000000 MetaHookPre LoadFile(0, ./analyzer.bif.zeek, <...>/analyzer.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./api, <...>/api.zeek) +0.000000 MetaHookPre LoadFile(0, ./backpressure, <...>/backpressure.zeek) 0.000000 MetaHookPre LoadFile(0, ./bloom-filter.bif.zeek, <...>/bloom-filter.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./cardinality-counter.bif.zeek, <...>/cardinality-counter.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./cluster.bif.zeek, <...>/cluster.bif.zeek) @@ -1705,6 +1708,7 @@ 0.000000 MetaHookPre LoadFileExtended(0, ./addrs, <...>/addrs.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./analyzer.bif.zeek, <...>/analyzer.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./api, <...>/api.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./backpressure, <...>/backpressure.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./bloom-filter.bif.zeek, <...>/bloom-filter.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./cardinality-counter.bif.zeek, <...>/cardinality-counter.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./cluster.bif.zeek, <...>/cluster.bif.zeek) @@ -2342,6 +2346,7 @@ 0.000000 | HookLoadFile ./api <...>/api.zeek 0.000000 | HookLoadFile ./archive <...>/archive.sig 0.000000 | HookLoadFile ./audio <...>/audio.sig +0.000000 | HookLoadFile ./backpressure <...>/backpressure.zeek 0.000000 | HookLoadFile ./bloom-filter.bif.zeek <...>/bloom-filter.bif.zeek 0.000000 | HookLoadFile ./cardinality-counter.bif.zeek <...>/cardinality-counter.bif.zeek 0.000000 | HookLoadFile ./cluster.bif.zeek <...>/cluster.bif.zeek @@ -2646,6 +2651,7 @@ 0.000000 | HookLoadFileExtended ./api <...>/api.zeek 0.000000 | HookLoadFileExtended ./archive <...>/archive.sig 0.000000 | HookLoadFileExtended ./audio <...>/audio.sig +0.000000 | HookLoadFileExtended ./backpressure <...>/backpressure.zeek 0.000000 | HookLoadFileExtended ./bloom-filter.bif.zeek <...>/bloom-filter.bif.zeek 0.000000 | HookLoadFileExtended ./cardinality-counter.bif.zeek <...>/cardinality-counter.bif.zeek 0.000000 | HookLoadFileExtended ./cluster.bif.zeek <...>/cluster.bif.zeek From 46a11ec37d9fc5d9fce488b27dda99a0733510c1 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Mon, 18 Nov 2024 16:07:26 -0800 Subject: [PATCH 10/65] Add Cluster::nodeid_to_node() helper function This translates backend-specific node identifiers (like Broker IDs) to cluster nodes and their names, if available. --- scripts/base/frameworks/cluster/main.zeek | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/scripts/base/frameworks/cluster/main.zeek b/scripts/base/frameworks/cluster/main.zeek index d610969cff..0427d6adcd 100644 --- a/scripts/base/frameworks/cluster/main.zeek +++ b/scripts/base/frameworks/cluster/main.zeek @@ -281,6 +281,15 @@ export { ## a given cluster node. global nodeid_topic: function(id: string): string; + ## Retrieve the cluster-level naming of a node based on its node ID, + ## a backend-specific identifier. + ## + ## id: the node ID of a peer. + ## + ## Returns: the :zeek:see:`Cluster::NamedNode` for the requested node, if + ## known, otherwise a "null" instance with an empty name field. + global nodeid_to_node: function(id: string): NamedNode; + ## Initialize the cluster backend. ## ## Cluster backends usually invoke this from a :zeek:see:`zeek_init` handler. @@ -394,6 +403,17 @@ function nodeid_topic(id: string): string return nodeid_topic_prefix + id + "/"; } +function nodeid_to_node(id: string): NamedNode + { + for ( name, n in nodes ) + { + if ( n?$id && n$id == id ) + return NamedNode($name=name, $node=n); + } + + return NamedNode($name="", $node=[$node_type=NONE, $ip=0.0.0.0]); + } + event Cluster::hello(name: string, id: string) &priority=10 { if ( name !in nodes ) From d260a5b7a913ec37c507f36ae4dbd994fd707f30 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Tue, 19 Nov 2024 12:11:39 -0800 Subject: [PATCH 11/65] Remove unneeded @loads from base/misc/version.zeek This module is loaded by the telemetry framework, which we're now loading via the cluster framework, i.e. also in bare mode. The resulting additional thread (for creating reporter.log) trips up a number of btest baselines. version.zeek doesn't use any of the string helper functions. --- scripts/base/misc/version.zeek | 3 --- testing/btest/Baseline/scripts.base.misc.version/.stderr | 6 +++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/scripts/base/misc/version.zeek b/scripts/base/misc/version.zeek index 14e3d4c2a7..36d8ab1fc3 100644 --- a/scripts/base/misc/version.zeek +++ b/scripts/base/misc/version.zeek @@ -2,9 +2,6 @@ ##! The most convenient way to access this are the Version::number ##! and Version::info constants. -@load base/frameworks/reporter -@load base/utils/strings - module Version; export { diff --git a/testing/btest/Baseline/scripts.base.misc.version/.stderr b/testing/btest/Baseline/scripts.base.misc.version/.stderr index 4cd9e25cb2..c7e80a0f4c 100644 --- a/testing/btest/Baseline/scripts.base.misc.version/.stderr +++ b/testing/btest/Baseline/scripts.base.misc.version/.stderr @@ -1,4 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in <...>/version.zeek, line 63: Version string 1 cannot be parsed -error in <...>/version.zeek, line 63: Version string 1.12-beta-drunk-too-much cannot be parsed -error in <...>/version.zeek, line 63: Version string JustARandomString cannot be parsed +error in <...>/version.zeek, line 60: Version string 1 cannot be parsed +error in <...>/version.zeek, line 60: Version string 1.12-beta-drunk-too-much cannot be parsed +error in <...>/version.zeek, line 60: Version string JustARandomString cannot be parsed From ead6134501cd5d7eba217f4812f4c1d99e8789bf Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Mon, 18 Nov 2024 16:09:26 -0800 Subject: [PATCH 12/65] Add backpressure disconnect notification to cluster.log and via telemetry This adds a Broker-specific script to the cluster framework, loaded only when Zeek is running in cluster mode. It adds logging in cluster.log as well as telemetry via a metrics counter for Broker-observed backpressure disconnects. The new zeek_broker_backpressure_disconnects counter, labeled by the neighboring peer that the reporting node has determined to be unresponsive, counts the number of unpeerings for this reason. Here the node "worker" has observed node "proxy" falling behind once: # HELP zeek_broker_backpressure_disconnects_total Number of Broker peering drops due to a neighbor falling too far behind in message I/O # TYPE zeek_broker_backpressure_disconnects_total counter zeek_broker_backpressure_disconnects_total{endpoint="worker",peer="proxy"} 1 Includes small btest baseline update to reflect @load of a new script. --- scripts/base/frameworks/cluster/__load__.zeek | 3 ++ .../cluster/broker-backpressure.zeek | 29 +++++++++++++++++++ .../coverage.init-default/missing_loads | 1 + 3 files changed, 33 insertions(+) create mode 100644 scripts/base/frameworks/cluster/broker-backpressure.zeek diff --git a/scripts/base/frameworks/cluster/__load__.zeek b/scripts/base/frameworks/cluster/__load__.zeek index a854302636..0d6372e3d4 100644 --- a/scripts/base/frameworks/cluster/__load__.zeek +++ b/scripts/base/frameworks/cluster/__load__.zeek @@ -14,6 +14,9 @@ redef Broker::log_topic = Cluster::rr_log_topic; # Add a cluster prefix. @prefixes += cluster +# This should soon condition on loading only when Broker is in use. +@load ./broker-backpressure + @if ( Supervisor::is_supervised() ) # When running a supervised cluster, populate Cluster::nodes from the node table # the Supervisor provides to new Zeek nodes. The management framework configures diff --git a/scripts/base/frameworks/cluster/broker-backpressure.zeek b/scripts/base/frameworks/cluster/broker-backpressure.zeek new file mode 100644 index 0000000000..e3fe4c9cdd --- /dev/null +++ b/scripts/base/frameworks/cluster/broker-backpressure.zeek @@ -0,0 +1,29 @@ +# Notifications for Broker-reported backpressure overflow. +# See base/frameworks/broker/backpressure.zeek for context. + +@load base/frameworks/telemetry + +module Cluster; + +global broker_backpressure_disconnects_cf = Telemetry::register_counter_family([ + $prefix="zeek", + $name="broker-backpressure-disconnects", + $unit="", + $label_names=vector("peer"), + $help_text="Number of Broker peerings dropped due to a neighbor falling behind in message I/O", +]); + +event Broker::peer_removed(endpoint: Broker::EndpointInfo, msg: string) + { + if ( ! endpoint?$network || "caf::sec::backpressure_overflow" !in msg ) + return; + + local nn = nodeid_to_node(endpoint$id); + + Cluster::log(fmt("removed due to backpressure overflow: %s%s:%s (%s)", + nn$name != "" ? "" : "non-cluster peer ", + endpoint$network$address, endpoint$network$bound_port, + nn$name != "" ? nn$name : endpoint$id)); + Telemetry::counter_family_inc(broker_backpressure_disconnects_cf, + vector(nn$name != "" ? nn$name : "unknown")); + } diff --git a/testing/btest/Baseline/coverage.init-default/missing_loads b/testing/btest/Baseline/coverage.init-default/missing_loads index e16624e1fb..9997ec4fd8 100644 --- a/testing/btest/Baseline/coverage.init-default/missing_loads +++ b/testing/btest/Baseline/coverage.init-default/missing_loads @@ -1,4 +1,5 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +-./frameworks/cluster/broker-backpressure.zeek -./frameworks/cluster/broker-stores.zeek -./frameworks/cluster/nodes/logger.zeek -./frameworks/cluster/nodes/manager.zeek From db79c88fb85959ad8dfa0711c02c6358de420608 Mon Sep 17 00:00:00 2001 From: zeek-bot Date: Sat, 7 Dec 2024 00:12:05 +0000 Subject: [PATCH 13/65] Update doc submodule [nomail] [skip ci] --- doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc b/doc index 9c8b992a55..50463c65a6 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 9c8b992a55908628f7b6ccc119d7cefb2c2cc0a1 +Subproject commit 50463c65a6f49c40b974f701964283f305394a0b From 7ed3f79c87f2cda1fde37b3fc6b6e272360e3b3d Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Fri, 6 Dec 2024 16:16:35 -0800 Subject: [PATCH 14/65] modified merge_types() to skip work if given identical types, which also preserves type names (useful for -O gen-C++) --- src/Type.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Type.cc b/src/Type.cc index 4b6d78a066..e32a4565bb 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -2376,6 +2376,9 @@ TypeListPtr merge_list_types(const Type* t1, const Type* t2) { } TypePtr merge_types(const TypePtr& arg_t1, const TypePtr& arg_t2) { + if ( arg_t1 == arg_t2 ) + return arg_t1; + auto t1 = arg_t1.get(); auto t2 = arg_t2.get(); // t1 = flatten_type(t1); From 62f891dcba19b07635cab6dbfcf732e54917688b Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Fri, 6 Dec 2024 16:17:44 -0800 Subject: [PATCH 15/65] modified AST profiling to mark (and fully skip) non-optimizable functions --- src/script_opt/ProfileFunc.cc | 2 ++ src/script_opt/ScriptOpt.cc | 3 +++ 2 files changed, 5 insertions(+) diff --git a/src/script_opt/ProfileFunc.cc b/src/script_opt/ProfileFunc.cc index 25d6a5e01d..07ee05675c 100644 --- a/src/script_opt/ProfileFunc.cc +++ b/src/script_opt/ProfileFunc.cc @@ -578,6 +578,8 @@ ProfileFuncs::ProfileFuncs(std::vector& funcs, is_compilable_pred pred if ( ! pred || (*pred)(pf.get(), nullptr) ) MergeInProfile(pf.get()); + else if ( pred ) + f.SetSkip(true); // Track the profile even if we're not compiling the function, since // the AST optimizer will still need it to reason about function-call diff --git a/src/script_opt/ScriptOpt.cc b/src/script_opt/ScriptOpt.cc index 47effda307..ce697ef890 100644 --- a/src/script_opt/ScriptOpt.cc +++ b/src/script_opt/ScriptOpt.cc @@ -397,6 +397,9 @@ static void use_CPP() { auto pfs = std::make_unique(funcs, is_CPP_compilable, true, false); for ( auto& f : funcs ) { + if ( f.ShouldSkip() ) + continue; + auto hash = f.Profile()->HashVal(); auto s = compiled_scripts.find(hash); From 79c5790bbf66352367bcf7732b691ca0ec7bcfb2 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Fri, 6 Dec 2024 16:18:43 -0800 Subject: [PATCH 16/65] when reporting available/unavailble C++ script bodies, flag those that are skipped --- src/script_opt/ScriptOpt.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/script_opt/ScriptOpt.cc b/src/script_opt/ScriptOpt.cc index ce697ef890..36f041fe62 100644 --- a/src/script_opt/ScriptOpt.cc +++ b/src/script_opt/ScriptOpt.cc @@ -366,6 +366,12 @@ static void report_CPP() { for ( auto& f : funcs ) { const auto& name = f.Func()->GetName(); + + if ( f.ShouldSkip() ) { + printf("script function %s: SKIP\n", name.c_str()); + continue; + } + auto hash = f.Profile()->HashVal(); bool have = compiled_scripts.count(hash) > 0; From 612d99e751481eaa4068cbc2e12ccb85537f8c37 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Fri, 6 Dec 2024 16:23:32 -0800 Subject: [PATCH 17/65] streamline generated -O C++ code by relying on per-function profiles rather than aggregate profile --- src/script_opt/CPP/Driver.cc | 37 +++++++++++++++++++++++++++--------- src/script_opt/CPP/Exprs.cc | 5 ++--- src/script_opt/CPP/Vars.h | 13 +++++++++++++ 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/src/script_opt/CPP/Driver.cc b/src/script_opt/CPP/Driver.cc index b6dc561853..c96a8012cf 100644 --- a/src/script_opt/CPP/Driver.cc +++ b/src/script_opt/CPP/Driver.cc @@ -29,6 +29,7 @@ CPPCompile::CPPCompile(vector& _funcs, std::shared_ptr _ CPPCompile::~CPPCompile() { fclose(write_file); } void CPPCompile::Compile(bool report_uncompilable) { + unordered_set rep_types; unordered_set filenames_reported_as_skipped; bool had_to_skip = false; @@ -63,6 +64,24 @@ void CPPCompile::Compile(bool report_uncompilable) { continue; } + auto pf = func.Profile(); + total_hash = merge_p_hashes(total_hash, pf->HashVal()); + + for ( auto t : pf->UnorderedTypes() ) + rep_types.insert(pfs->TypeRep(t)); + + auto& pf_all_gl = pf->AllGlobals(); + all_accessed_globals.insert(pf_all_gl.begin(), pf_all_gl.end()); + + auto& pf_gl = pf->Globals(); + accessed_globals.insert(pf_gl.begin(), pf_gl.end()); + + auto& pf_events = pf->Events(); + accessed_events.insert(pf_events.begin(), pf_events.end()); + + auto& pf_lambdas = pf->Lambdas(); + accessed_lambdas.insert(pf_lambdas.begin(), pf_lambdas.end()); + if ( is_lambda(f) || is_when_lambda(f) ) { // We deal with these separately. func.SetSkip(true); @@ -85,35 +104,35 @@ void CPPCompile::Compile(bool report_uncompilable) { } } - if ( standalone && had_to_skip ) - reporter->FatalError("aborting standalone compilation to C++ due to having to skip some functions"); - // Generate a hash unique for this compilation. for ( const auto& func : funcs ) if ( ! func.ShouldSkip() ) total_hash = merge_p_hashes(total_hash, func.Profile()->HashVal()); + if ( standalone && had_to_skip ) + reporter->FatalError("aborting standalone compilation to C++ due to having to skip some functions"); + auto t = util::current_time(); total_hash = merge_p_hashes(total_hash, hash{}(t)); GenProlog(); // Track all of the types we'll be using. - for ( const auto& t : pfs->RepTypes() ) { + for ( const auto& t : rep_types ) { TypePtr tp{NewRef{}, (Type*)(t)}; types.AddKey(tp, pfs->HashType(t)); } NL(); - for ( auto& g : pfs->AllGlobals() ) + for ( auto& g : all_accessed_globals ) CreateGlobal(g); - for ( const auto& e : pfs->Events() ) + for ( const auto& e : accessed_events ) if ( AddGlobal(e, "gl") ) Emit("EventHandlerPtr %s_ev;", globals[string(e)]); - for ( const auto& t : pfs->RepTypes() ) { + for ( const auto& t : rep_types ) { ASSERT(types.HasKey(t)); TypePtr tp{NewRef{}, (Type*)(t)}; RegisterType(tp); @@ -131,7 +150,7 @@ void CPPCompile::Compile(bool report_uncompilable) { // be identical. In that case, we don't want to generate the lambda // twice, but we do want to map the second one to the same body name. unordered_map lambda_ASTs; - for ( const auto& l : pfs->Lambdas() ) { + for ( const auto& l : accessed_lambdas ) { const auto& n = l->Name(); const auto body = l->Ingredients()->Body().get(); if ( lambda_ASTs.count(n) > 0 ) @@ -151,7 +170,7 @@ void CPPCompile::Compile(bool report_uncompilable) { CompileFunc(func); lambda_ASTs.clear(); - for ( const auto& l : pfs->Lambdas() ) { + for ( const auto& l : accessed_lambdas ) { const auto& n = l->Name(); if ( lambda_ASTs.count(n) > 0 ) continue; diff --git a/src/script_opt/CPP/Exprs.cc b/src/script_opt/CPP/Exprs.cc index 482b04f61f..ddef91b5a8 100644 --- a/src/script_opt/CPP/Exprs.cc +++ b/src/script_opt/CPP/Exprs.cc @@ -308,9 +308,8 @@ string CPPCompile::GenCallExpr(const CallExpr* c, GenType gt, bool top_level) { if ( pfs->BiFGlobals().count(f_id) == 0 ) gen += +"->AsFunc()"; - else if ( pfs->Globals().count(f_id) > 0 ) - // The BiF version has an extra "_", per - // AddBiF(..., true). + else if ( accessed_globals.count(f_id) > 0 ) + // The BiF version has an extra "_", per AddBiF(..., true). gen = globals[string(id_name) + "_"]; } diff --git a/src/script_opt/CPP/Vars.h b/src/script_opt/CPP/Vars.h index cfbeb2db3c..8a0d9dac81 100644 --- a/src/script_opt/CPP/Vars.h +++ b/src/script_opt/CPP/Vars.h @@ -53,6 +53,19 @@ std::string Canonicalize(const std::string& name) const; // be a EXPR_NAME). std::string GlobalName(const ExprPtr& e) { return globals[e->AsNameExpr()->Id()->Name()]; } +// Globals that are used (appear in the profiles) of the bodies we're +// compiling. Includes globals just used as functions to call. +std::unordered_set all_accessed_globals; + +// Same, but just the globals used in contexts beyond function calls. +std::unordered_set accessed_globals; + +// Lambdas that are accessed. +std::unordered_set accessed_lambdas; + +// Events that are accessed. +std::unordered_set accessed_events; + // Maps global names (not identifiers) to the names we use for them. std::unordered_map globals; From 2e69a8870ae7686aadef8600810f7131c97d1285 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Fri, 6 Dec 2024 16:25:22 -0800 Subject: [PATCH 18/65] introduced simplified initialization for non-standalone -O gen-C++ code tied -O gen-standalone-C++ to use of --optimize-files --- src/script_opt/CPP/Driver.h | 5 +++ src/script_opt/CPP/Inits.cc | 23 ++++++++++--- src/script_opt/CPP/InitsInfo.cc | 21 ++++++++++-- src/script_opt/CPP/InitsInfo.h | 26 ++++++++++++-- src/script_opt/CPP/README.md | 12 +++---- src/script_opt/CPP/RuntimeInits.cc | 25 ++++++++++++-- src/script_opt/CPP/RuntimeInits.h | 37 ++++++++++++++++---- src/script_opt/CPP/Types.cc | 54 ++++++++++++++++-------------- src/script_opt/CPP/Vars.cc | 18 +++++++--- src/script_opt/ScriptOpt.cc | 20 +++++++++-- src/script_opt/ScriptOpt.h | 4 +++ 11 files changed, 189 insertions(+), 56 deletions(-) diff --git a/src/script_opt/CPP/Driver.h b/src/script_opt/CPP/Driver.h index e2a097ed2e..f4f8e123d8 100644 --- a/src/script_opt/CPP/Driver.h +++ b/src/script_opt/CPP/Driver.h @@ -60,6 +60,11 @@ void GenFinishInit(); // Generate the function that registers compiled script bodies. void GenRegisterBodies(); +public: +// Whether we're generating "standalone" code. +bool TargetingStandalone() const { return standalone; } + +private: // True if the given function (plus body and profile) is one that should be // compiled. If non-nil, sets reason to the the reason why, if there's a // fundamental problem. If however the function should be skipped for other diff --git a/src/script_opt/CPP/Inits.cc b/src/script_opt/CPP/Inits.cc index 0906d1c55b..54b40b161c 100644 --- a/src/script_opt/CPP/Inits.cc +++ b/src/script_opt/CPP/Inits.cc @@ -104,11 +104,18 @@ void CPPCompile::InitializeFieldMappings() { StartBlock(); + string type_arg, attrs_arg; + if ( ! standalone ) + type_arg = attrs_arg = "DO_NOT_CONSTRUCT_VALUE_MARKER"; + for ( const auto& mapping : field_decls ) { auto rt_arg = Fmt(mapping.first); auto td = mapping.second; - auto type_arg = Fmt(TypeOffset(td->type)); - auto attrs_arg = Fmt(AttributesOffset(td->attrs)); + + if ( standalone ) { + type_arg = Fmt(TypeOffset(td->type)); + attrs_arg = Fmt(AttributesOffset(td->attrs)); + } Emit("CPP_FieldMapping(%s, \"%s\", %s, %s),", rt_arg, td->id, type_arg, attrs_arg); } @@ -121,8 +128,10 @@ void CPPCompile::InitializeEnumMappings() { StartBlock(); + auto create_if_missing = standalone ? "true" : "false"; + for ( const auto& mapping : enum_names ) - Emit("CPP_EnumMapping(%s, \"%s\"),", Fmt(mapping.first), mapping.second); + Emit("CPP_EnumMapping(%s, \"%s\", %s),", Fmt(mapping.first), mapping.second, create_if_missing); EndBlock(true); } @@ -178,9 +187,15 @@ void CPPCompile::InitializeGlobals() { Emit("Frame* f__CPP = nullptr;"); NL(); + auto& ofiles = analysis_options.only_files; + for ( const auto& ginit : IDOptInfo::GetGlobalInitExprs() ) { auto g = ginit.Id(); - if ( pfs->Globals().count(g) == 0 ) + + if ( ! ofiles.empty() && ! obj_matches_opt_files(g) ) + continue; + + if ( accessed_globals.count(g) == 0 ) continue; auto ic = ginit.IC(); diff --git a/src/script_opt/CPP/InitsInfo.cc b/src/script_opt/CPP/InitsInfo.cc index 6006b7bda0..ba8e3cd674 100644 --- a/src/script_opt/CPP/InitsInfo.cc +++ b/src/script_opt/CPP/InitsInfo.cc @@ -355,10 +355,18 @@ AttrsInfo::AttrsInfo(CPPCompile* _c, const AttributesPtr& _attrs) : CompoundItem } } -GlobalInitInfo::GlobalInitInfo(CPPCompile* c, const ID* g, string _CPP_name) +GlobalLookupInitInfo::GlobalLookupInitInfo(CPPCompile* c, const ID* g, string _CPP_name) : CPP_InitInfo(g), CPP_name(std::move(_CPP_name)) { Zeek_name = g->Name(); +} +void GlobalLookupInitInfo::InitializerVals(std::vector& ivs) const { + ivs.push_back(CPP_name); + ivs.push_back(string("\"") + Zeek_name + "\""); +} + +GlobalInitInfo::GlobalInitInfo(CPPCompile* c, const ID* g, string _CPP_name) + : GlobalLookupInitInfo(c, g, std::move(_CPP_name)) { auto& gt = g->GetType(); auto gi = c->RegisterType(gt); init_cohort = max(init_cohort, gi->InitCohort() + 1); @@ -375,7 +383,7 @@ GlobalInitInfo::GlobalInitInfo(CPPCompile* c, const ID* g, string _CPP_name) exported = g->IsExport(); val = ValElem(c, nullptr); // empty because we initialize dynamically - if ( gt->Tag() == TYPE_FUNC && ! g->GetVal() ) + if ( gt->Tag() == TYPE_FUNC && (! g->GetVal() || g->GetVal()->AsFunc()->GetKind() == Func::BUILTIN_FUNC) ) // Remember this peculiarity so we can recreate it for // error-behavior-compatibility. func_with_no_val = true; @@ -549,7 +557,7 @@ RecordTypeInfo::RecordTypeInfo(CPPCompile* _c, TypePtr _t) : AbstractTypeInfo(_c field_types.push_back(r_i->type); - if ( r_i->attrs ) { + if ( c->TargetingStandalone() && r_i->attrs ) { gi = c->RegisterAttributes(r_i->attrs); final_init_cohort = max(final_init_cohort, gi->InitCohort() + 1); field_attrs.push_back(gi->Offset()); @@ -576,6 +584,13 @@ void RecordTypeInfo::AddInitializerVals(std::vector& ivs) const { } } +NamedTypeInfo::NamedTypeInfo(CPPCompile* _c, TypePtr _t) : AbstractTypeInfo(_c, std::move(_t)) {} + +void NamedTypeInfo::AddInitializerVals(std::vector& ivs) const { + ivs.emplace_back(Fmt(NAMED_TYPE_MARKER)); + ivs.emplace_back(Fmt(c->TrackString(t->GetName()))); +} + void IndicesManager::Generate(CPPCompile* c) { c->Emit("int CPP__Indices__init[] ="); c->StartBlock(); diff --git a/src/script_opt/CPP/InitsInfo.h b/src/script_opt/CPP/InitsInfo.h index a9ae9ed37b..3432871d67 100644 --- a/src/script_opt/CPP/InitsInfo.h +++ b/src/script_opt/CPP/InitsInfo.h @@ -478,8 +478,22 @@ public: AttrsInfo(CPPCompile* c, const AttributesPtr& attrs); }; -// Information for initialization a Zeek global. -class GlobalInitInfo : public CPP_InitInfo { +// A lightweight initializer for a Zeek global that will look it up at +// initialization time but not create it if missing. +class GlobalLookupInitInfo : public CPP_InitInfo { +public: + GlobalLookupInitInfo(CPPCompile* c, const ID* g, std::string CPP_name); + + std::string InitializerType() const override { return "CPP_GlobalLookupInit"; } + void InitializerVals(std::vector& ivs) const override; + +protected: + std::string Zeek_name; + std::string CPP_name; +}; + +// Information for initializing a Zeek global. +class GlobalInitInfo : public GlobalLookupInitInfo { public: GlobalInitInfo(CPPCompile* c, const ID* g, std::string CPP_name); @@ -639,6 +653,14 @@ private: std::vector field_attrs; }; +// Class for initializing a named Zeek type that should be present at startup. +class NamedTypeInfo : public AbstractTypeInfo { +public: + NamedTypeInfo(CPPCompile* c, TypePtr _t); + + void AddInitializerVals(std::vector& ivs) const override; +}; + // Much of the table-driven initialization is based on vectors of indices, // which we represent as vectors of int's, where each int is used to index a // global C++ vector. This class manages such vectors. In particular, it diff --git a/src/script_opt/CPP/README.md b/src/script_opt/CPP/README.md index 4b3ed6b10f..ba95fc6980 100644 --- a/src/script_opt/CPP/README.md +++ b/src/script_opt/CPP/README.md @@ -90,7 +90,7 @@ and also any compiled-to-C++ bodies present in the `zeek` binary that The above workflows require the subsequent `zeek` execution to include the `target.zeek` script. You can avoid this by replacing the first step with: -1. `./src/zeek -O gen-standalone-C++ target.zeek >target-stand-in.zeek` +1. `./src/zeek -O gen-standalone-C++ --optimize-files=target.zeek target.zeek >target-stand-in.zeek` (and then building as in the 2nd step above). This option prints to _stdout_ a @@ -100,13 +100,9 @@ without needing to include `target.zeek` in the invocation (nor the `-O use-C++` option). After loading the stand-in script, you can still access types and functions declared in `target.zeek`. -Note: the implementation differences between `gen-C++` and `gen-standalone-C++` -wound up being modest enough that it might make sense to just always provide -the latter functionality, which it turns out does not introduce any -additional constraints compared to the current `gen-C++` functionality. -On the other hand, it's possible (not yet established) that code created -using `gen-C++` can be made to compile significantly faster than -standalone code. +Note: `gen-standalone-C++` _must_ be used with `--optimize-files`, as the +compiler needs the latter to determine which global declarations the +standalone code needs to initialize. There are additional workflows relating to running the test suite: see `src/script_opt/CPP/maint/README`. diff --git a/src/script_opt/CPP/RuntimeInits.cc b/src/script_opt/CPP/RuntimeInits.cc index d8b73626c5..2918a8b30a 100644 --- a/src/script_opt/CPP/RuntimeInits.cc +++ b/src/script_opt/CPP/RuntimeInits.cc @@ -235,7 +235,7 @@ void CPP_TypeInits::PreInit(InitsManager* im, int offset, ValElemVec& init_vals) if ( tag == TYPE_LIST ) inits_vec[offset] = make_intrusive(); - else if ( tag == TYPE_RECORD ) { + else if ( tag == TYPE_RECORD && init_vals[1] != NAMED_TYPE_MARKER ) { auto name = im->Strings(init_vals[1]); if ( name[0] ) inits_vec[offset] = get_record_type__CPP(name); @@ -243,7 +243,7 @@ void CPP_TypeInits::PreInit(InitsManager* im, int offset, ValElemVec& init_vals) inits_vec[offset] = get_record_type__CPP(nullptr); } - else if ( tag == TYPE_TABLE ) + else if ( tag == TYPE_TABLE && init_vals[1] != NAMED_TYPE_MARKER ) inits_vec[offset] = make_intrusive(); // else no pre-initialization needed @@ -251,6 +251,13 @@ void CPP_TypeInits::PreInit(InitsManager* im, int offset, ValElemVec& init_vals) void CPP_TypeInits::Generate(InitsManager* im, vector& ivec, int offset, ValElemVec& init_vals) const { auto tag = static_cast(init_vals[0]); + + if ( init_vals.size() > 1 && init_vals[1] == NAMED_TYPE_MARKER ) { + auto name = im->Strings(init_vals[2]); + ivec[offset] = find_global__CPP(name)->GetType(); + return; + } + TypePtr t; switch ( tag ) { case TYPE_ADDR: @@ -406,6 +413,11 @@ int CPP_FieldMapping::ComputeOffset(InitsManager* im) const { auto fm_offset = r->FieldOffset(field_name.c_str()); if ( fm_offset < 0 ) { // field does not exist, create it + if ( field_type == DO_NOT_CONSTRUCT_VALUE_MARKER ) { + reporter->CPPRuntimeError("record field \"%s\" missing in %s", field_name.c_str(), obj_desc(r).c_str()); + exit(1); + } + fm_offset = r->NumFields(); auto id = util::copy_string(field_name.c_str(), field_name.size()); @@ -429,6 +441,11 @@ int CPP_EnumMapping::ComputeOffset(InitsManager* im) const { auto em_offset = e->Lookup(e_name); if ( em_offset < 0 ) { // enum constant does not exist, create it + if ( ! construct_if_missing ) { + reporter->CPPRuntimeError("enum element \"%s\" missing in %s", e_name.c_str(), obj_desc(e).c_str()); + exit(1); + } + em_offset = e->Names().size(); if ( e->Lookup(em_offset) ) reporter->InternalError("enum inconsistency while initializing compiled scripts"); @@ -438,6 +455,10 @@ int CPP_EnumMapping::ComputeOffset(InitsManager* im) const { return em_offset; } +void CPP_GlobalLookupInit::Generate(InitsManager* im, std::vector& /* inits_vec */, int /* offset */) const { + global = find_global__CPP(name); +} + void CPP_GlobalInit::Generate(InitsManager* im, std::vector& /* inits_vec */, int /* offset */) const { auto& t = im->Types(type); global = lookup_global__CPP(name, t, exported); diff --git a/src/script_opt/CPP/RuntimeInits.h b/src/script_opt/CPP/RuntimeInits.h index 268d2ca250..ddd8cf578e 100644 --- a/src/script_opt/CPP/RuntimeInits.h +++ b/src/script_opt/CPP/RuntimeInits.h @@ -41,6 +41,14 @@ extern std::vector>> generate_indices_set(int* init #define END_OF_VEC_VEC -100 #define END_OF_VEC_VEC_VEC -200 +// A marker value for "named" types (those that are simply looked up by +// name at initialization time). +#define NAMED_TYPE_MARKER -300 + +// A marker value indicating values that should not be constructed if not +// already present. +#define DO_NOT_CONSTRUCT_VALUE_MARKER -400 + // An abstract helper class used to access elements of an initialization vector. // We need the abstraction because InitsManager below needs to be able to refer // to any of a range of templated classes. @@ -369,8 +377,19 @@ public: } }; -// Class for initializing a Zeek global. These don't go into an initialization +// Classes for initializing Zeek globals. These don't go into an initialization // vector, so we use void* as the underlying type. +class CPP_GlobalLookupInit : public CPP_Init { +public: + CPP_GlobalLookupInit(IDPtr& _global, const char* _name) : CPP_Init(), global(_global), name(_name) {} + + void Generate(InitsManager* im, std::vector& /* inits_vec */, int /* offset */) const override; + +protected: + IDPtr& global; + const char* name; +}; + class CPP_GlobalInit : public CPP_Init { public: CPP_GlobalInit(IDPtr& _global, const char* _name, int _type, int _attrs, int _val, bool _exported, @@ -463,8 +482,12 @@ public: private: int rec; // index to retrieve the record's type std::string field_name; // which field this offset pertains to - int field_type; // the field's type, in case we have to construct it - int field_attrs; // the same for the field's attributes + + // The field's type, in case we have to construct it. If + // DO_NOT_CONSTRUCT_VALUE_MARKER then it's instead an error + // if missing. + int field_type; + int field_attrs; // the same for the field's attributes }; // Constructs at run-time a mapping between abstract enum values used when @@ -473,13 +496,15 @@ private: // the enum). class CPP_EnumMapping { public: - CPP_EnumMapping(int _e_type, std::string _e_name) : e_type(_e_type), e_name(std::move(_e_name)) {} + CPP_EnumMapping(int _e_type, std::string _e_name, bool _construct_if_missing) + : e_type(_e_type), e_name(std::move(_e_name)), construct_if_missing(_construct_if_missing) {} int ComputeOffset(InitsManager* im) const; private: - int e_type; // index to EnumType - std::string e_name; // which enum constant for that type + int e_type; // index to EnumType + std::string e_name; // which enum constant for that type + bool construct_if_missing; // if true, construct constant if not present }; // Looks up a BiF of the given name, making it available to compiled diff --git a/src/script_opt/CPP/Types.cc b/src/script_opt/CPP/Types.cc index f97987a9e9..59aab07a04 100644 --- a/src/script_opt/CPP/Types.cc +++ b/src/script_opt/CPP/Types.cc @@ -191,41 +191,45 @@ shared_ptr CPPCompile::RegisterType(const TypePtr& tp) { shared_ptr gi; - switch ( t->Tag() ) { - case TYPE_ADDR: - case TYPE_ANY: - case TYPE_BOOL: - case TYPE_COUNT: - case TYPE_DOUBLE: - case TYPE_ERROR: - case TYPE_INT: - case TYPE_INTERVAL: - case TYPE_PATTERN: - case TYPE_PORT: - case TYPE_STRING: - case TYPE_TIME: - case TYPE_VOID: - case TYPE_SUBNET: - case TYPE_FILE: gi = make_shared(this, tp); break; + if ( standalone || t->GetName().empty() ) { + switch ( t->Tag() ) { + case TYPE_ADDR: + case TYPE_ANY: + case TYPE_BOOL: + case TYPE_COUNT: + case TYPE_DOUBLE: + case TYPE_ERROR: + case TYPE_INT: + case TYPE_INTERVAL: + case TYPE_PATTERN: + case TYPE_PORT: + case TYPE_STRING: + case TYPE_TIME: + case TYPE_VOID: + case TYPE_SUBNET: + case TYPE_FILE: gi = make_shared(this, tp); break; - case TYPE_ENUM: gi = make_shared(this, tp); break; + case TYPE_ENUM: gi = make_shared(this, tp); break; - case TYPE_OPAQUE: gi = make_shared(this, tp); break; + case TYPE_OPAQUE: gi = make_shared(this, tp); break; - case TYPE_TYPE: gi = make_shared(this, tp); break; + case TYPE_TYPE: gi = make_shared(this, tp); break; - case TYPE_VECTOR: gi = make_shared(this, tp); break; + case TYPE_VECTOR: gi = make_shared(this, tp); break; - case TYPE_LIST: gi = make_shared(this, tp); break; + case TYPE_LIST: gi = make_shared(this, tp); break; - case TYPE_TABLE: gi = make_shared(this, tp); break; + case TYPE_TABLE: gi = make_shared(this, tp); break; - case TYPE_RECORD: gi = make_shared(this, tp); break; + case TYPE_RECORD: gi = make_shared(this, tp); break; - case TYPE_FUNC: gi = make_shared(this, tp); break; + case TYPE_FUNC: gi = make_shared(this, tp); break; - default: reporter->InternalError("bad type in CPPCompile::RegisterType"); + default: reporter->InternalError("bad type in CPPCompile::RegisterType"); + } } + else + gi = make_shared(this, tp); type_info->AddInstance(gi); processed_types[t] = gi; diff --git a/src/script_opt/CPP/Vars.cc b/src/script_opt/CPP/Vars.cc index 38ec046aa9..34a037c056 100644 --- a/src/script_opt/CPP/Vars.cc +++ b/src/script_opt/CPP/Vars.cc @@ -11,7 +11,7 @@ void CPPCompile::CreateGlobal(const ID* g) { auto gn = string(g->Name()); bool is_bif = pfs->BiFGlobals().count(g) > 0; - if ( pfs->Globals().count(g) == 0 ) { + if ( accessed_globals.count(g) == 0 ) { // Only used in the context of calls. If it's compilable, // then we'll call it directly. if ( compilable_funcs.count(gn) > 0 ) { @@ -28,11 +28,16 @@ void CPPCompile::CreateGlobal(const ID* g) { if ( AddGlobal(gn, "gl") ) { // We'll be creating this global. Emit("IDPtr %s;", globals[gn]); - if ( pfs->Events().count(gn) > 0 ) + if ( accessed_events.count(gn) > 0 ) // This is an event that's also used as a variable. Emit("EventHandlerPtr %s_ev;", globals[gn]); - auto gi = make_shared(this, g, globals[gn]); + shared_ptr gi; + if ( standalone ) + gi = make_shared(this, g, globals[gn]); + else + gi = make_shared(this, g, globals[gn]); + global_id_info->AddInstance(gi); global_gis[g] = gi; } @@ -64,7 +69,12 @@ std::shared_ptr CPPCompile::RegisterGlobal(const ID* g) { return gg->second; } - auto gi = make_shared(this, g, globals[gn]); + shared_ptr gi; + if ( standalone ) + gi = make_shared(this, g, globals[gn]); + else + gi = make_shared(this, g, globals[gn]); + global_id_info->AddInstance(gi); global_gis[g] = gi; diff --git a/src/script_opt/ScriptOpt.cc b/src/script_opt/ScriptOpt.cc index 36f041fe62..08b32b71e1 100644 --- a/src/script_opt/ScriptOpt.cc +++ b/src/script_opt/ScriptOpt.cc @@ -109,13 +109,25 @@ bool should_analyze(const ScriptFuncPtr& f, const StmtPtr& body) { if ( ofiles.empty() && ofuncs.empty() ) return true; + if ( obj_matches_opt_files(body.get()) ) + return true; + const auto& fun = f->GetName(); for ( auto& o : ofuncs ) if ( std::regex_match(fun, o) ) return true; - auto fin = util::detail::normalize_path(body->GetLocationInfo()->filename); + return false; +} + +bool obj_matches_opt_files(const Obj* obj) { + auto& ofiles = analysis_options.only_files; + + if ( ofiles.empty() ) + return false; + + auto fin = util::detail::normalize_path(obj->GetLocationInfo()->filename); for ( auto& o : ofiles ) if ( std::regex_match(fin, o) ) @@ -285,8 +297,12 @@ static void init_options() { check_env_opt("ZEEK_USE_CPP", analysis_options.use_CPP); check_env_opt("ZEEK_ALLOW_COND", analysis_options.allow_cond); - if ( analysis_options.gen_standalone_CPP ) + if ( analysis_options.gen_standalone_CPP ) { + if ( analysis_options.only_files.empty() ) + reporter->FatalError("-O gen-standalone-C++ requires use of --optimize-files"); + analysis_options.gen_CPP = true; + } if ( analysis_options.gen_CPP ) generating_CPP = true; diff --git a/src/script_opt/ScriptOpt.h b/src/script_opt/ScriptOpt.h index 441c8d6ec2..c595a71cc6 100644 --- a/src/script_opt/ScriptOpt.h +++ b/src/script_opt/ScriptOpt.h @@ -247,6 +247,10 @@ extern void add_file_analysis_pattern(AnalyOpt& opts, const char* pat); // it should be skipped. extern bool should_analyze(const ScriptFuncPtr& f, const StmtPtr& body); +// True if the given object's location matches one specified by +// --optimize-files=... +extern bool obj_matches_opt_files(const Obj* obj); + // Analyze all of the parsed scripts collectively for usage issues (unless // suppressed by the flag) and optimization. extern void analyze_scripts(bool no_unused_warnings); From de98f1c36c1a67c6c74336111a272e82f5ef8021 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Fri, 6 Dec 2024 16:26:48 -0800 Subject: [PATCH 19/65] fix for -O gen-C++ maintenance helper to skip BTest intermediary files --- src/script_opt/CPP/maint/find-test-files.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/script_opt/CPP/maint/find-test-files.sh b/src/script_opt/CPP/maint/find-test-files.sh index 642b438d31..bf848db4ec 100755 --- a/src/script_opt/CPP/maint/find-test-files.sh +++ b/src/script_opt/CPP/maint/find-test-files.sh @@ -1,6 +1,7 @@ #! /bin/sh find ../testing/btest -type f | + grep -v '\.tmp/' | xargs grep -E -l '@TEST' | xargs grep -E -l '^[ ]*(event|print)' | xargs grep -E -c 'REQUIRES.*CPP.*((!=.*1)|(==.*0))' | From 2047ae980a1e3fa0249b421a5338e669ebf4c7be Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Fri, 6 Dec 2024 16:28:06 -0800 Subject: [PATCH 20/65] mark ZAM regression BTests as not suitable for compile-to-C++ --- testing/btest/opt/regress-any-leak.zeek | 1 + testing/btest/opt/regress-any.zeek | 1 + testing/btest/opt/regress-vector-mismatch.zeek | 1 + 3 files changed, 3 insertions(+) diff --git a/testing/btest/opt/regress-any-leak.zeek b/testing/btest/opt/regress-any-leak.zeek index 118356aa07..d049672caf 100644 --- a/testing/btest/opt/regress-any-leak.zeek +++ b/testing/btest/opt/regress-any-leak.zeek @@ -1,4 +1,5 @@ # @TEST-DOC: Regression test for leak when mixing "any" types (affected both ZAM and non-ZAM) +# @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1" # @TEST-EXEC: zeek -b -O ZAM %INPUT >output # @TEST-EXEC: btest-diff output diff --git a/testing/btest/opt/regress-any.zeek b/testing/btest/opt/regress-any.zeek index 98c4116392..0514cbac1d 100644 --- a/testing/btest/opt/regress-any.zeek +++ b/testing/btest/opt/regress-any.zeek @@ -1,4 +1,5 @@ # @TEST-DOC: Regression test for reassigning an "any" field +# @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1" # @TEST-EXEC: zeek -b -O ZAM %INPUT >output # @TEST-EXEC: btest-diff output diff --git a/testing/btest/opt/regress-vector-mismatch.zeek b/testing/btest/opt/regress-vector-mismatch.zeek index e07411750d..48a15f96ac 100644 --- a/testing/btest/opt/regress-vector-mismatch.zeek +++ b/testing/btest/opt/regress-vector-mismatch.zeek @@ -1,4 +1,5 @@ # @TEST-DOC: Regression test for coercing vectors-of-any +# @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1" # @TEST-EXEC: zeek -b -O ZAM %INPUT >output # @TEST-EXEC: btest-diff output From 96305aa4aad5b5452e1c11ca4c0b77a17c70636f Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Fri, 6 Dec 2024 16:29:28 -0800 Subject: [PATCH 21/65] BTest baseline updates for compile-to-C++ --- .../bifs.disable_analyzer-early/out | 2 +- .../bifs.disable_analyzer-hook/out | 8 +++---- .../Baseline.cpp/bifs.disable_analyzer/out | 2 +- .../btest/Baseline.cpp/bifs.type_aliases/out | 22 ------------------- .../cluster.generic.errors/.stderr | 15 +++++++++++++ .../cluster.generic.errors/.stdout | 10 +++++++++ .../cluster.generic.make_event/.stderr | 7 ++++++ .../.stdout | 4 ++-- .../.stdout | 4 ++-- .../Baseline.cpp/core.expr-exception/output | 18 +++++++-------- .../language.expire-expr-error/output | 4 ++-- .../language.expire-func-undef/output | 21 ------------------ .../.stdout | 20 ++++++++--------- 13 files changed, 63 insertions(+), 74 deletions(-) delete mode 100644 testing/btest/Baseline.cpp/bifs.type_aliases/out create mode 100644 testing/btest/Baseline.cpp/cluster.generic.errors/.stderr create mode 100644 testing/btest/Baseline.cpp/cluster.generic.errors/.stdout create mode 100644 testing/btest/Baseline.cpp/cluster.generic.make_event/.stderr delete mode 100644 testing/btest/Baseline.cpp/language.expire-func-undef/output diff --git a/testing/btest/Baseline.cpp/bifs.disable_analyzer-early/out b/testing/btest/Baseline.cpp/bifs.disable_analyzer-early/out index b72c958aef..5741db6901 100644 --- a/testing/btest/Baseline.cpp/bifs.disable_analyzer-early/out +++ b/testing/btest/Baseline.cpp/bifs.disable_analyzer-early/out @@ -3,5 +3,5 @@ proto confirm, AllAnalyzers::ANALYZER_ANALYZER_HTTP T http_request, GET, /style/enhanced.css total http messages, { -[[orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp]] = 1 +[[orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp, proto=6]] = 1 } diff --git a/testing/btest/Baseline.cpp/bifs.disable_analyzer-hook/out b/testing/btest/Baseline.cpp/bifs.disable_analyzer-hook/out index c72bc161e8..750269ab42 100644 --- a/testing/btest/Baseline.cpp/bifs.disable_analyzer-hook/out +++ b/testing/btest/Baseline.cpp/bifs.disable_analyzer-hook/out @@ -1,16 +1,16 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. proto confirm, AllAnalyzers::ANALYZER_ANALYZER_HTTP http_request, GET, /style/enhanced.css -preventing disable_analyzer, [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp], AllAnalyzers::ANALYZER_ANALYZER_HTTP, 3, 1 +preventing disable_analyzer, [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp, proto=6], AllAnalyzers::ANALYZER_ANALYZER_HTTP, 3, 1 F http_reply, 200 http_request, GET, /script/urchin.js -preventing disable_analyzer, [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp], AllAnalyzers::ANALYZER_ANALYZER_HTTP, 3, 3 +preventing disable_analyzer, [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp, proto=6], AllAnalyzers::ANALYZER_ANALYZER_HTTP, 3, 3 F http_reply, 200 http_request, GET, /images/template/screen/bullet_utility.png -allowing disable_analyzer, [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp], AllAnalyzers::ANALYZER_ANALYZER_HTTP, 3, 5 +allowing disable_analyzer, [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp, proto=6], AllAnalyzers::ANALYZER_ANALYZER_HTTP, 3, 5 T total http messages, { -[[orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp]] = 5 +[[orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp, proto=6]] = 5 } diff --git a/testing/btest/Baseline.cpp/bifs.disable_analyzer/out b/testing/btest/Baseline.cpp/bifs.disable_analyzer/out index 2a4cdae144..6c14045611 100644 --- a/testing/btest/Baseline.cpp/bifs.disable_analyzer/out +++ b/testing/btest/Baseline.cpp/bifs.disable_analyzer/out @@ -3,5 +3,5 @@ proto confirm, AllAnalyzers::ANALYZER_ANALYZER_HTTP http_request, GET, /style/enhanced.css T total http messages, { -[[orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp]] = 1 +[[orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp, proto=6]] = 1 } diff --git a/testing/btest/Baseline.cpp/bifs.type_aliases/out b/testing/btest/Baseline.cpp/bifs.type_aliases/out deleted file mode 100644 index dab1c31ead..0000000000 --- a/testing/btest/Baseline.cpp/bifs.type_aliases/out +++ /dev/null @@ -1,22 +0,0 @@ -### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -type aliases for 'RED enum val': ColorAlias Color -type aliases for 'Color enum type': ColorAlias Color -type aliases for 'MyRec val': MyRec MyRecAlias -type aliases for 'MyRecAlias val': MyRec MyRecAlias -type aliases for 'MyRec type': MyRec MyRecAlias -type aliases for 'MyRecalias type': MyRec MyRecAlias -type aliases for 'MyString val': it's just a 'string' -type aliases for 'MyString type': MyString AnotherString -type aliases for 'MyOtherString type': MyOtherString -type aliases for 'AnotherString type': MyString AnotherString -type aliases for 'string literal value': it's just a 'string' -type aliases for 'count literal value': it's just a 'count' -type aliases for 'MyTable value': it's just a 'table[count] of string' -type aliases for 'MyTable2 value': it's just a 'table[count] of string' -type aliases for 'MyTable3 value': it's just a 'table[count] of string' -type aliases for 'MyTable4 value': it's just a 'table[count] of string' -type aliases for 'MyTable type': MyTable2 MyTable3 MyTable MyTable4 -type aliases for 'MyTable2 type': MyTable2 MyTable3 MyTable MyTable4 -type aliases for 'MyTable3 type': MyTable2 MyTable3 MyTable MyTable4 -type aliases for 'MyTable4 type': MyTable2 MyTable3 MyTable MyTable4 -type aliases for 'table value': it's just a 'table[count] of string' diff --git a/testing/btest/Baseline.cpp/cluster.generic.errors/.stderr b/testing/btest/Baseline.cpp/cluster.generic.errors/.stderr new file mode 100644 index 0000000000..8e82f08d99 --- /dev/null +++ b/testing/btest/Baseline.cpp/cluster.generic.errors/.stderr @@ -0,0 +1,15 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/errors.zeek (C++), line 16: no event arguments given (<___>testing_btest__tmp_cluster_generic_errors_errors_zeek__zeek_init__36__zf: function() : void()) +error in <...>/errors.zeek (C++), line 16: not enough arguments (<___>testing_btest__tmp_cluster_generic_errors_errors_zeek__zeek_init__36__zf: function() : void()) +error in <...>/errors.zeek (C++), line 16: bad number of arguments for ping1: got 0, expect 2 +error in <...>/errors.zeek (C++), line 16: bad number of arguments for ping1: got 0, expect 2 +error in <...>/errors.zeek (C++), line 16: bad number of arguments for ping1: got 1, expect 2 +error in <...>/errors.zeek (C++), line 16: bad number of arguments for ping1: got 1, expect 2 +error in <...>/errors.zeek (C++), line 16: bad number of arguments for ping1: got 3, expect 2 +error in <...>/errors.zeek (C++), line 16: bad number of arguments for ping1: got 3, expect 2 +error in <...>/errors.zeek (C++), line 37: event parameter #2 type mismatch, got count, expecting string +error in <...>/errors.zeek (C++), line 37: event parameter #2 type mismatch, got count, expecting string +error in <...>/errors.zeek (C++), line 37: unexpected function type for hook1: hook +error in <...>/errors.zeek (C++), line 37: unexpected function type for hook1: hook +error in <...>/errors.zeek (C++), line 37: expected function or record as first argument, got count (<___>testing_btest__tmp_cluster_generic_errors_errors_zeek__zeek_init__37__zf: function() : void()) +error in <...>/errors.zeek (C++), line 37: got non-event type 'count' (<___>testing_btest__tmp_cluster_generic_errors_errors_zeek__zeek_init__37__zf: function() : void()) diff --git a/testing/btest/Baseline.cpp/cluster.generic.errors/.stdout b/testing/btest/Baseline.cpp/cluster.generic.errors/.stdout new file mode 100644 index 0000000000..ee431b2959 --- /dev/null +++ b/testing/btest/Baseline.cpp/cluster.generic.errors/.stdout @@ -0,0 +1,10 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +wrong number of args +r1, [ev=, args=[]] +r2, [ev=, args=[]] +r3, [ev=, args=[]] +r4, [ev=, args=[]] +wrong types +r1, [ev=, args=[]] +r2, [ev=, args=[]] +r3, [ev=, args=[]] diff --git a/testing/btest/Baseline.cpp/cluster.generic.make_event/.stderr b/testing/btest/Baseline.cpp/cluster.generic.make_event/.stderr new file mode 100644 index 0000000000..9e77528b32 --- /dev/null +++ b/testing/btest/Baseline.cpp/cluster.generic.make_event/.stderr @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/make_event.zeek (C++), line 30: not enough arguments (<___>testing_btest__tmp_cluster_generic_make_event_make_event_zeek__zeek_init__39__zf: function() : void()) +error in <...>/make_event.zeek (C++), line 35: got non-event type 'string' (<___>testing_btest__tmp_cluster_generic_make_event_make_event_zeek__zeek_init__40__zf: function() : void()) +error in <...>/make_event.zeek (C++), line 40: unexpected function type for test_fun: function +error in <...>/make_event.zeek (C++), line 45: unexpected function type for test_hook: hook +error in <...>/make_event.zeek (C++), line 50: bad number of arguments for test_event2: got 0, expect 1 +error in <...>/make_event.zeek (C++), line 55: bad number of arguments for test_event2: got 2, expect 1 diff --git a/testing/btest/Baseline.cpp/core.analyzer-confirmation-violation-info-ftp/.stdout b/testing/btest/Baseline.cpp/core.analyzer-confirmation-violation-info-ftp/.stdout index 1e4f9aa98a..a901d6fb47 100644 --- a/testing/btest/Baseline.cpp/core.analyzer-confirmation-violation-info-ftp/.stdout +++ b/testing/btest/Baseline.cpp/core.analyzer-confirmation-violation-info-ftp/.stdout @@ -1,3 +1,3 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -analyzer_confirmation_info, AllAnalyzers::ANALYZER_ANALYZER_FTP, [orig_h=2001:470:1f05:17a6:d69a:20ff:fefd:6b88, orig_p=24316/tcp, resp_h=2001:6a8:a40::21, resp_p=21/tcp], 3 -analyzer_violation_info, AllAnalyzers::ANALYZER_ANALYZER_FTP, non-numeric reply code, [orig_h=2001:470:1f05:17a6:d69a:20ff:fefd:6b88, orig_p=24316/tcp, resp_h=2001:6a8:a40::21, resp_p=21/tcp], 3, SSH-2.0-mod_sftp/0.9.7 +analyzer_confirmation_info, AllAnalyzers::ANALYZER_ANALYZER_FTP, [orig_h=2001:470:1f05:17a6:d69a:20ff:fefd:6b88, orig_p=24316/tcp, resp_h=2001:6a8:a40::21, resp_p=21/tcp, proto=6], 3 +analyzer_violation_info, AllAnalyzers::ANALYZER_ANALYZER_FTP, non-numeric reply code, [orig_h=2001:470:1f05:17a6:d69a:20ff:fefd:6b88, orig_p=24316/tcp, resp_h=2001:6a8:a40::21, resp_p=21/tcp, proto=6], 3, SSH-2.0-mod_sftp/0.9.7 diff --git a/testing/btest/Baseline.cpp/core.analyzer-confirmation-violation-info/.stdout b/testing/btest/Baseline.cpp/core.analyzer-confirmation-violation-info/.stdout index af8800750d..16b47e8b21 100644 --- a/testing/btest/Baseline.cpp/core.analyzer-confirmation-violation-info/.stdout +++ b/testing/btest/Baseline.cpp/core.analyzer-confirmation-violation-info/.stdout @@ -1,3 +1,3 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -analyzer_confirmation_info, AllAnalyzers::ANALYZER_ANALYZER_SSL, [orig_h=1.1.1.1, orig_p=20394/tcp, resp_h=2.2.2.2, resp_p=443/tcp], 3 -analyzer_violation_info, AllAnalyzers::ANALYZER_ANALYZER_SSL, Invalid version late in TLS connection. Packet reported version: 0, [orig_h=1.1.1.1, orig_p=20394/tcp, resp_h=2.2.2.2, resp_p=443/tcp], 3 +analyzer_confirmation_info, AllAnalyzers::ANALYZER_ANALYZER_SSL, [orig_h=1.1.1.1, orig_p=20394/tcp, resp_h=2.2.2.2, resp_p=443/tcp, proto=6], 3 +analyzer_violation_info, AllAnalyzers::ANALYZER_ANALYZER_SSL, Invalid version late in TLS connection. Packet reported version: 0, [orig_h=1.1.1.1, orig_p=20394/tcp, resp_h=2.2.2.2, resp_p=443/tcp, proto=6], 3 diff --git a/testing/btest/Baseline.cpp/core.expr-exception/output b/testing/btest/Baseline.cpp/core.expr-exception/output index 9ea22c91da..7c9dccba13 100644 --- a/testing/btest/Baseline.cpp/core.expr-exception/output +++ b/testing/btest/Baseline.cpp/core.expr-exception/output @@ -1,19 +1,19 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. ftp field missing -[orig_h=141.142.220.118, orig_p=48649/tcp, resp_h=208.80.152.118, resp_p=80/tcp] +[orig_h=141.142.220.118, orig_p=48649/tcp, resp_h=208.80.152.118, resp_p=80/tcp, proto=6] ftp field missing -[orig_h=141.142.220.118, orig_p=49997/tcp, resp_h=208.80.152.3, resp_p=80/tcp] +[orig_h=141.142.220.118, orig_p=49997/tcp, resp_h=208.80.152.3, resp_p=80/tcp, proto=6] ftp field missing -[orig_h=141.142.220.118, orig_p=49996/tcp, resp_h=208.80.152.3, resp_p=80/tcp] +[orig_h=141.142.220.118, orig_p=49996/tcp, resp_h=208.80.152.3, resp_p=80/tcp, proto=6] ftp field missing -[orig_h=141.142.220.118, orig_p=49998/tcp, resp_h=208.80.152.3, resp_p=80/tcp] +[orig_h=141.142.220.118, orig_p=49998/tcp, resp_h=208.80.152.3, resp_p=80/tcp, proto=6] ftp field missing -[orig_h=141.142.220.118, orig_p=50000/tcp, resp_h=208.80.152.3, resp_p=80/tcp] +[orig_h=141.142.220.118, orig_p=50000/tcp, resp_h=208.80.152.3, resp_p=80/tcp, proto=6] ftp field missing -[orig_h=141.142.220.118, orig_p=49999/tcp, resp_h=208.80.152.3, resp_p=80/tcp] +[orig_h=141.142.220.118, orig_p=49999/tcp, resp_h=208.80.152.3, resp_p=80/tcp, proto=6] ftp field missing -[orig_h=141.142.220.118, orig_p=50001/tcp, resp_h=208.80.152.3, resp_p=80/tcp] +[orig_h=141.142.220.118, orig_p=50001/tcp, resp_h=208.80.152.3, resp_p=80/tcp, proto=6] ftp field missing -[orig_h=141.142.220.118, orig_p=35642/tcp, resp_h=208.80.152.2, resp_p=80/tcp] +[orig_h=141.142.220.118, orig_p=35642/tcp, resp_h=208.80.152.2, resp_p=80/tcp, proto=6] ftp field missing -[orig_h=141.142.220.235, orig_p=6705/tcp, resp_h=173.192.163.128, resp_p=80/tcp] +[orig_h=141.142.220.235, orig_p=6705/tcp, resp_h=173.192.163.128, resp_p=80/tcp, proto=6] diff --git a/testing/btest/Baseline.cpp/language.expire-expr-error/output b/testing/btest/Baseline.cpp/language.expire-expr-error/output index 0220c1a064..1f379ec328 100644 --- a/testing/btest/Baseline.cpp/language.expire-expr-error/output +++ b/testing/btest/Baseline.cpp/language.expire-expr-error/output @@ -1,3 +1,3 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -runtime error in compiled code: no such index -received termination signal +event(), [] +event(s:string), [abc] diff --git a/testing/btest/Baseline.cpp/language.expire-func-undef/output b/testing/btest/Baseline.cpp/language.expire-func-undef/output deleted file mode 100644 index 4e0530b87c..0000000000 --- a/testing/btest/Baseline.cpp/language.expire-func-undef/output +++ /dev/null @@ -1,21 +0,0 @@ -### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -1299470395.000000 expression error in : value used but not set (segfault::scan_summary) -1299470405.000000 expression error in : value used but not set (segfault::scan_summary) -1299473995.000000 expression error in : value used but not set (segfault::scan_summary) -1299474005.000000 expression error in : value used but not set (segfault::scan_summary) -1299477595.000000 expression error in : value used but not set (segfault::scan_summary) -1299477605.000000 expression error in : value used but not set (segfault::scan_summary) -1299481195.000000 expression error in : value used but not set (segfault::scan_summary) -1299481205.000000 expression error in : value used but not set (segfault::scan_summary) -1299484795.000000 expression error in : value used but not set (segfault::scan_summary) -1299484805.000000 expression error in : value used but not set (segfault::scan_summary) -1299488395.000000 expression error in : value used but not set (segfault::scan_summary) -1299488405.000000 expression error in : value used but not set (segfault::scan_summary) -1299491995.000000 expression error in : value used but not set (segfault::scan_summary) -1299492005.000000 expression error in : value used but not set (segfault::scan_summary) -1299495595.000000 expression error in : value used but not set (segfault::scan_summary) -1299495605.000000 expression error in : value used but not set (segfault::scan_summary) -1299499195.000000 expression error in : value used but not set (segfault::scan_summary) -1299499205.000000 expression error in : value used but not set (segfault::scan_summary) -1299502795.000000 expression error in : value used but not set (segfault::scan_summary) -orig: 10.0.0.2: peers: {\x0a\x0910.0.0.3\x0a} diff --git a/testing/btest/Baseline.cpp/scripts.base.protocols.ssl.prevent-disable-analyzer/.stdout b/testing/btest/Baseline.cpp/scripts.base.protocols.ssl.prevent-disable-analyzer/.stdout index 229c675729..f66615e9e9 100644 --- a/testing/btest/Baseline.cpp/scripts.base.protocols.ssl.prevent-disable-analyzer/.stdout +++ b/testing/btest/Baseline.cpp/scripts.base.protocols.ssl.prevent-disable-analyzer/.stdout @@ -1,11 +1,11 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -analyzer_confirmation, [orig_h=10.0.0.80, orig_p=56637/tcp, resp_h=68.233.76.12, resp_p=443/tcp], AllAnalyzers::ANALYZER_ANALYZER_SSL, 3 -encrypted_data, [orig_h=10.0.0.80, orig_p=56637/tcp, resp_h=68.233.76.12, resp_p=443/tcp], T, 22, 32, 1 -established, [orig_h=10.0.0.80, orig_p=56637/tcp, resp_h=68.233.76.12, resp_p=443/tcp] -disabling_analyzer, [orig_h=10.0.0.80, orig_p=56637/tcp, resp_h=68.233.76.12, resp_p=443/tcp], AllAnalyzers::ANALYZER_ANALYZER_SSL, 3 -preventing disabling_analyzer, [orig_h=10.0.0.80, orig_p=56637/tcp, resp_h=68.233.76.12, resp_p=443/tcp], AllAnalyzers::ANALYZER_ANALYZER_SSL, 3 -encrypted_data, [orig_h=10.0.0.80, orig_p=56637/tcp, resp_h=68.233.76.12, resp_p=443/tcp], F, 22, 32, 2 -encrypted_data, [orig_h=10.0.0.80, orig_p=56637/tcp, resp_h=68.233.76.12, resp_p=443/tcp], T, 23, 31, 3 -encrypted_data, [orig_h=10.0.0.80, orig_p=56637/tcp, resp_h=68.233.76.12, resp_p=443/tcp], T, 23, 17, 4 -disabling_analyzer, [orig_h=10.0.0.80, orig_p=56637/tcp, resp_h=68.233.76.12, resp_p=443/tcp], AllAnalyzers::ANALYZER_ANALYZER_SSL, 3 -allowing disabling_analyzer, [orig_h=10.0.0.80, orig_p=56637/tcp, resp_h=68.233.76.12, resp_p=443/tcp], AllAnalyzers::ANALYZER_ANALYZER_SSL, 3 +analyzer_confirmation, [orig_h=10.0.0.80, orig_p=56637/tcp, resp_h=68.233.76.12, resp_p=443/tcp, proto=6], AllAnalyzers::ANALYZER_ANALYZER_SSL, 3 +encrypted_data, [orig_h=10.0.0.80, orig_p=56637/tcp, resp_h=68.233.76.12, resp_p=443/tcp, proto=6], T, 22, 32, 1 +established, [orig_h=10.0.0.80, orig_p=56637/tcp, resp_h=68.233.76.12, resp_p=443/tcp, proto=6] +disabling_analyzer, [orig_h=10.0.0.80, orig_p=56637/tcp, resp_h=68.233.76.12, resp_p=443/tcp, proto=6], AllAnalyzers::ANALYZER_ANALYZER_SSL, 3 +preventing disabling_analyzer, [orig_h=10.0.0.80, orig_p=56637/tcp, resp_h=68.233.76.12, resp_p=443/tcp, proto=6], AllAnalyzers::ANALYZER_ANALYZER_SSL, 3 +encrypted_data, [orig_h=10.0.0.80, orig_p=56637/tcp, resp_h=68.233.76.12, resp_p=443/tcp, proto=6], F, 22, 32, 2 +encrypted_data, [orig_h=10.0.0.80, orig_p=56637/tcp, resp_h=68.233.76.12, resp_p=443/tcp, proto=6], T, 23, 31, 3 +encrypted_data, [orig_h=10.0.0.80, orig_p=56637/tcp, resp_h=68.233.76.12, resp_p=443/tcp, proto=6], T, 23, 17, 4 +disabling_analyzer, [orig_h=10.0.0.80, orig_p=56637/tcp, resp_h=68.233.76.12, resp_p=443/tcp, proto=6], AllAnalyzers::ANALYZER_ANALYZER_SSL, 3 +allowing disabling_analyzer, [orig_h=10.0.0.80, orig_p=56637/tcp, resp_h=68.233.76.12, resp_p=443/tcp, proto=6], AllAnalyzers::ANALYZER_ANALYZER_SSL, 3 From e6d0c8aa0450046e4fecfa4d88ce2f2a5e5d0b0b Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Mon, 2 Dec 2024 13:35:25 -0800 Subject: [PATCH 22/65] Add sleep() BiF. Yes, really. :-) We've hit the need for this on occasion in very specific settings and always worked around it via ugly nested loops or similars. This has ample warning that folks normally won't want to use this. Not sure that ZAM btest should baseline the number of BiFs. --- src/script_opt/FuncInfo.cc | 1 + src/zeek.bif | 21 +++++++++++++++++++ .../Baseline.zam/opt.ZAM-bif-tracking/output | 2 +- testing/btest/Baseline/bifs.sleep/out | 1 + testing/btest/bifs/sleep.zeek | 21 +++++++++++++++++++ testing/btest/opt/ZAM-bif-tracking.zeek | 1 + 6 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 testing/btest/Baseline/bifs.sleep/out create mode 100644 testing/btest/bifs/sleep.zeek diff --git a/src/script_opt/FuncInfo.cc b/src/script_opt/FuncInfo.cc index 2516d1bd44..545f278f96 100644 --- a/src/script_opt/FuncInfo.cc +++ b/src/script_opt/FuncInfo.cc @@ -431,6 +431,7 @@ static std::unordered_map func_attrs = { {"skip_further_processing", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"skip_http_entity_data", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"skip_smtp_data", ATTR_NO_SCRIPT_SIDE_EFFECTS}, + {"sleep", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"split_string", ATTR_FOLDABLE}, {"split_string1", ATTR_FOLDABLE}, {"split_string_all", ATTR_FOLDABLE}, diff --git a/src/zeek.bif b/src/zeek.bif index ee14c0ddce..fbcf322c44 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -600,6 +600,27 @@ function piped_exec%(program: string, to_write: string%): bool return zeek::val_mgr->True(); %} +## Sleeps for the given amount of time. +## +## i: The time interval to sleep for. +## +## Returns: The :zeek:type:`interval` Zeek actually slept for. +## +## .. note:: +## +## This is a blocking sleep! Zeek will not run most of its processing +## during that time. You almost certainly DO NOT WANT THIS outside +## of specific testing/troubleshooting scenarios. To sleep asynchronously, +## :zeek:see:`schedule` an event, or consider :zeek:id:`Exec::run`. +function sleep%(i: interval%): interval + %{ + const auto start = std::chrono::high_resolution_clock::now(); + std::this_thread::sleep_for(std::chrono::duration(i)); + const auto end = std::chrono::high_resolution_clock::now(); + const auto slept = std::chrono::duration(end - start).count(); + return zeek::make_intrusive(slept); + %} + %%{ #include "zeek/OpaqueVal.h" %%} diff --git a/testing/btest/Baseline.zam/opt.ZAM-bif-tracking/output b/testing/btest/Baseline.zam/opt.ZAM-bif-tracking/output index 6f645f9525..0138aa7bfe 100644 --- a/testing/btest/Baseline.zam/opt.ZAM-bif-tracking/output +++ b/testing/btest/Baseline.zam/opt.ZAM-bif-tracking/output @@ -1,2 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -539 seen BiFs, 0 unseen BiFs (), 0 new BiFs () +540 seen BiFs, 0 unseen BiFs (), 0 new BiFs () diff --git a/testing/btest/Baseline/bifs.sleep/out b/testing/btest/Baseline/bifs.sleep/out new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/bifs.sleep/out @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/bifs/sleep.zeek b/testing/btest/bifs/sleep.zeek new file mode 100644 index 0000000000..7fbd3e6b46 --- /dev/null +++ b/testing/btest/bifs/sleep.zeek @@ -0,0 +1,21 @@ +# Verifies sleep()'s reported latencies. +# +# @TEST-EXEC: zeek -b %INPUT 2>out +# @TEST-EXEC: btest-diff out + +function test_sleep(i: interval) + { + local start = current_time(); + local sleep_delay = sleep(i); + local script_delay = current_time() - start; + + assert script_delay >= i, fmt("sleep() took %s, less than %s", script_delay, i); + assert sleep_delay >= i, fmt("slept for %s, less than %s", script_delay, i); + assert sleep_delay <= script_delay, fmt("sleep() claims %s, longer than %s", sleep_delay, script_delay); + } + +event zeek_init() + { + test_sleep(100msec); + test_sleep(1sec); + } diff --git a/testing/btest/opt/ZAM-bif-tracking.zeek b/testing/btest/opt/ZAM-bif-tracking.zeek index 627c21f444..e059f1f839 100644 --- a/testing/btest/opt/ZAM-bif-tracking.zeek +++ b/testing/btest/opt/ZAM-bif-tracking.zeek @@ -464,6 +464,7 @@ global known_BiFs = set( "skip_further_processing", "skip_http_entity_data", "skip_smtp_data", + "sleep", "sort", "split_string", "split_string1", From feb2aa890dd9b14eede678fa9b8fc53ed87d9eae Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Tue, 3 Dec 2024 17:38:25 -0800 Subject: [PATCH 23/65] Expand documentation of Broker events. --- src/broker/comm.bif | 76 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 69 insertions(+), 7 deletions(-) diff --git a/src/broker/comm.bif b/src/broker/comm.bif index b6d6292539..5cc2e89f8c 100644 --- a/src/broker/comm.bif +++ b/src/broker/comm.bif @@ -7,16 +7,56 @@ module Broker; -## Generated when something changes in the Broker sub-system. -event Broker::status%(endpoint: EndpointInfo, msg: string%); - -## Generated when a new peering has been established. +## Generated when a new peering has been established. Both sides of the peering +## receive this event, created independently in each endpoint. For the endpoint +## establishing the peering, the added endpoint's network information will match +## the address and port provided to :zeek:see:`Broker::peer`; for the listening +## endpoint it's the peer's TCP client's address and (likely ephemeral) TCP +## port. +## +## endpoint: the added endpoint's Broker ID and connection information. +## +## msg: a message providing additional context. +## +## .. zeek:see:: Broker::peer_removed Broker::peer_lost +## Broker::endpoint_discovered Broker::endpoint_unreachable +## Broker::status Broker::error event Broker::peer_added%(endpoint: EndpointInfo, msg: string%); -## Generated when an existing peer has been removed. +## Generated when the local endpoint has removed its peering with another +## endpoint. This event can fire for multiple reasons, such as a local call to +## :zeek:see:`Broker::unpeer`, or because Broker autonomously decides to +## unpeer. One reason it might do this is message I/O backpressure overflow, +## meaning that the remote peer cannot keep up with the stream of messages the +## local endpoint sends it. Regardless of the cause, the remote endpoint will +## locally trigger a corresponding :zeek:see:`Broker::peer_lost` event once the +## peering ends. These events are independent of the original directionality of +## TCP connection establishment and only reflect which endpoint terminates the +## peering. +## +## endpoint: the removed endpoint's Broker ID and connection information. +## +## msg: a message providing additional context. If backpressure overflow +## caused this unpeering, the message contains the string +## *caf::sec::backpressure_overflow*. +## +## .. zeek:see:: Broker::peer_added Broker::peer_lost +## Broker::endpoint_discovered Broker::endpoint_unreachable +## Broker::status Broker::error event Broker::peer_removed%(endpoint: EndpointInfo, msg: string%); -## Generated when an existing peering has been lost. +## Generated when the local endpoint has lost its peering with another +## endpoint. This event fires when the other endpoint stops or removes the +## peering for some other reason. This event is independent of the original +## directionality of connection establishment. +## +## endpoint: the lost endpoint's Broker ID and connection information. +## +## msg: a message providing additional context. +## +## .. zeek:see:: Broker::peer_added Broker::peer_removed +## Broker::endpoint_discovered Broker::endpoint_unreachable +## Broker::status Broker::error event Broker::peer_lost%(endpoint: EndpointInfo, msg: string%); ## Generated when a new Broker endpoint appeared. @@ -25,7 +65,29 @@ event Broker::endpoint_discovered%(endpoint: EndpointInfo, msg: string%); ## Generated when the last path to a Broker endpoint has been lost. event Broker::endpoint_unreachable%(endpoint: EndpointInfo, msg: string%); -## Generated when an error occurs in the Broker sub-system. +## Generated when an unspecified change occurs in Broker. This event only fires +## when the status change isn't covered by more specific Broker events. The +## provided message string may be empty. +## +## endpoint: the Broker ID and connection information, if available, +## of the endpoint the update relates to. +## +## msg: a message providing additional context. +## +## .. zeek:see:: Broker::peer_added Broker::peer_removed Broker::peer_lost +## Broker::endpoint_discovered Broker::endpoint_unreachable Broker::error +event Broker::status%(endpoint: EndpointInfo, msg: string%); + +## Generated when an error occurs in the Broker sub-system. This event +## reports local errors in Broker, as indicated by the provided +## :zeek:type:`Broker::ErrorCode`. +## +## code: the type of error that triggered this event. +## +## msg: a message providing additional context. +## +## .. zeek:see:: Broker::peer_added Broker::peer_removed Broker::peer_lost +## Broker::endpoint_discovered Broker::endpoint_unreachable Broker::status event Broker::error%(code: ErrorCode, msg: string%); ## Enumerates the possible error types. From 83d16f9ef4b7525421c38b2aeff4480611aa3231 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Mon, 2 Dec 2024 16:21:10 -0800 Subject: [PATCH 24/65] Bump cluster testsuite to pull in Broker backpressure tests --- testing/external/commit-hash.zeek-testing-cluster | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/external/commit-hash.zeek-testing-cluster b/testing/external/commit-hash.zeek-testing-cluster index da9c310b2e..b129fa1026 100644 --- a/testing/external/commit-hash.zeek-testing-cluster +++ b/testing/external/commit-hash.zeek-testing-cluster @@ -1 +1 @@ -d2987b0bc07cb70bd2f8f707b372fb852147b71f +aa361fc9f5fba202a9df68717a1d403be5f1e6b9 From 99989b8055320629e274889afe9e5237801547a4 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Mon, 9 Dec 2024 09:26:42 +0100 Subject: [PATCH 25/65] Disable CTU-SME test under TSAN --- testing/external/commit-hash.zeek-testing | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/external/commit-hash.zeek-testing b/testing/external/commit-hash.zeek-testing index 597e9dbd5b..9805198913 100644 --- a/testing/external/commit-hash.zeek-testing +++ b/testing/external/commit-hash.zeek-testing @@ -1 +1 @@ -dd4b80f8d2c2d033b5b61c95e0f4c89fcfcb29b0 +2a63b457f24133a845c2020a321b7cbc05262291 From 28f76d5da93a1fe4d1329630f21dd05d9716be6f Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Mon, 9 Dec 2024 11:02:24 +0100 Subject: [PATCH 26/65] ScriptOpt: Fail compilation if known exprs/stmts is outdated --- src/script_opt/ScriptOpt.cc | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/script_opt/ScriptOpt.cc b/src/script_opt/ScriptOpt.cc index de53cd41ca..c91c3df8b3 100644 --- a/src/script_opt/ScriptOpt.cc +++ b/src/script_opt/ScriptOpt.cc @@ -664,6 +664,7 @@ bool has_AST_node_unknown_to_script_opt(const ProfileFunc* prof, bool /* is_ZAM // optimization // clang-format off static const std::set known_stmts = { + // STMT_ALARM STMT_PRINT, STMT_EVENT, STMT_EXPR, @@ -687,9 +688,15 @@ bool has_AST_node_unknown_to_script_opt(const ProfileFunc* prof, bool /* is_ZAM STMT_ASSERT, // STMT_EXTERN, // STMT_STD_FUNCTION, +#define SCRIPT_OPT_NUM_STMTS 24 }; // clang-format on + // Fail compilation if NUM_STMT in StmtEnums.h changes. + // Update known_stmts list above appropriately after adding + // support and increase SCRIPT_OPT_NUM_STMTS. + static_assert(NUM_STMTS == SCRIPT_OPT_NUM_STMTS); + for ( auto& s : prof->Stmts() ) if ( known_stmts.count(s->Tag()) == 0 ) return true; @@ -766,9 +773,15 @@ bool has_AST_node_unknown_to_script_opt(const ProfileFunc* prof, bool /* is_ZAM // EXPR_ANY_INDEX, // EXPR_SCRIPT_OPT_BUILTIN, // EXPR_NOP, +#define SCRIPT_OPT_NUM_EXPRS 70 }; // clang-format on + // Fail compilation if NUM_EXPRS in Expr.h changes. + // Update known_exprs list above appropriately after + // adding support and increase SCRIPT_OPT_NUM_STMTS. + static_assert(NUM_EXPRS == SCRIPT_OPT_NUM_EXPRS); + for ( auto& e : prof->Exprs() ) if ( known_exprs.count(e->Tag()) == 0 ) return true; From 97c0df29d427393aac716c51ea3cea2803f4f027 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Mon, 9 Dec 2024 15:50:49 +0100 Subject: [PATCH 27/65] Bump clang-format pre-commit hooks This brings in slightly better formatting around uniform initialization, and comments after blocks not surrounded by `{ .. }`. --- .pre-commit-config.yaml | 2 +- src/DNS_Mgr.cc | 6 +++--- src/Debug.cc | 2 +- src/file_analysis/analyzer/x509/X509.cc | 6 +++--- src/script_opt/CPP/RuntimeVec.cc | 3 +-- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1997a9be57..7bff7ff2b4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ # repos: - repo: https://github.com/pre-commit/mirrors-clang-format - rev: 'v18.1.8' + rev: 'v19.1.4' hooks: - id: clang-format types_or: diff --git a/src/DNS_Mgr.cc b/src/DNS_Mgr.cc index 2bb2ba435e..42948accef 100644 --- a/src/DNS_Mgr.cc +++ b/src/DNS_Mgr.cc @@ -318,7 +318,7 @@ static void addrinfo_cb(void* arg, int status, int timeouts, struct ares_addrinf // Push a null on the end so the addr list has a final point during later parsing. addrs.push_back(NULL); - struct hostent he {}; + struct hostent he{}; he.h_name = util::copy_string(result->name); he.h_addrtype = AF_INET; he.h_length = sizeof(in_addr); @@ -333,7 +333,7 @@ static void addrinfo_cb(void* arg, int status, int timeouts, struct ares_addrinf // Push a null on the end so the addr list has a final point during later parsing. addrs6.push_back(NULL); - struct hostent he {}; + struct hostent he{}; he.h_name = util::copy_string(result->name); he.h_addrtype = AF_INET6; he.h_length = sizeof(in6_addr); @@ -370,7 +370,7 @@ static void query_cb(void* arg, ares_status_t status, size_t timeouts, const are } } else { - struct hostent he {}; + struct hostent he{}; uint32_t ttl = 0; size_t rr_cnt = ares_dns_record_rr_cnt(dnsrec, ARES_SECTION_ANSWER); diff --git a/src/Debug.cc b/src/Debug.cc index 6609a3d256..66e11b5e50 100644 --- a/src/Debug.cc +++ b/src/Debug.cc @@ -417,7 +417,7 @@ int dbg_init_debugger(const char* cmdfile) { // ### Implement this debug_msg("Command files not supported. Using interactive mode.\n"); - // ### if ( interactive ) (i.e., not reading cmds from a file) + // ### if ( interactive ) (i.e., not reading cmds from a file) #ifdef HAVE_READLINE init_readline(); #endif diff --git a/src/file_analysis/analyzer/x509/X509.cc b/src/file_analysis/analyzer/x509/X509.cc index aefdf2738c..2dab26e25a 100644 --- a/src/file_analysis/analyzer/x509/X509.cc +++ b/src/file_analysis/analyzer/x509/X509.cc @@ -308,9 +308,9 @@ void X509::ParseExtensionsSpecific(X509_EXTENSION* ex, bool global, ASN1_OBJECT* else if ( OBJ_obj2nid(ext_asn) == NID_subject_alt_name ) ParseSAN(ex); - // In OpenSSL 1.0.2+, we can get the extension by using NID_ct_precert_scts. - // In OpenSSL <= 1.0.1, this is not yet defined yet, so we have to manually - // look it up by performing a string comparison on the oid. + // In OpenSSL 1.0.2+, we can get the extension by using NID_ct_precert_scts. + // In OpenSSL <= 1.0.1, this is not yet defined yet, so we have to manually + // look it up by performing a string comparison on the oid. #ifdef NID_ct_precert_scts else if ( OBJ_obj2nid(ext_asn) == NID_ct_precert_scts ) #else diff --git a/src/script_opt/CPP/RuntimeVec.cc b/src/script_opt/CPP/RuntimeVec.cc index 62eed2dde2..fc5ccc6f1f 100644 --- a/src/script_opt/CPP/RuntimeVec.cc +++ b/src/script_opt/CPP/RuntimeVec.cc @@ -82,8 +82,7 @@ static VectorTypePtr base_vector_type__CPP(const VectorTypePtr& vt, bool is_bool // Instantiates a double_kernel for a given operation. #define VEC_OP1_WITH_DOUBLE(name, op) \ VEC_OP1( \ - name, op, case TYPE_INTERNAL_DOUBLE \ - : { \ + name, op, case TYPE_INTERNAL_DOUBLE : { \ VEC_OP1_KERNEL(AsDouble, DoubleVal, op) \ break; \ }) From 24894febb6c1d16aa100559a188dc89a7ef43ac9 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Mon, 9 Dec 2024 15:51:28 +0100 Subject: [PATCH 28/65] Bump typos pre-commit hooks This now picks up additional typical misspellings, but also triggers on more identifiers we use. I opted for fixing the obvious misspellings and updated the allowlist for anything else. --- .pre-commit-config.yaml | 4 ++-- .typos.toml | 9 +++++++++ NEWS | 2 +- ci/test.sh | 2 +- .../policy/protocols/ssl/certificate-request-info.zeek | 2 +- src/analyzer/protocol/ldap/ldap.spicy | 2 +- src/input/Manager.cc | 2 +- src/script_opt/Stmt.cc | 2 +- src/spicy/spicyz/driver.h | 2 +- 9 files changed, 18 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7bff7ff2b4..0c9db04242 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,10 +28,10 @@ repos: - id: cmake-format - repo: https://github.com/crate-ci/typos - rev: v1.16.21 + rev: v1.28.2 hooks: - id: typos - exclude: '^(.typos.toml|src/SmithWaterman.cc|testing/.*|auxil/.*|scripts/base/frameworks/files/magic/.*|CHANGES)$' + exclude: '^(.typos.toml|src/SmithWaterman.cc|testing/.*|auxil/.*|scripts/base/frameworks/files/magic/.*|CHANGES|scripts/base/protocols/ssl/mozilla-ca-list.zeek)$' - repo: https://github.com/bbannier/spicy-format rev: v0.20.0 diff --git a/.typos.toml b/.typos.toml index f8a512a8c3..4d74a52e7f 100644 --- a/.typos.toml +++ b/.typos.toml @@ -48,6 +48,15 @@ extend-ignore-identifiers-re = [ "ND_ROUTER_.*", "ND_NEIGHBOR_.*", ".*_ND_option.*", + "bck", # Used with same length as `fwd` + "pn", # Use for `PoolNode` variables + "ffrom_[ip|port|mac]", # Used in netcontrol. + "complte_flag", # Existing use in exported record in base. + "VidP(n|N)", # In SMB. + "iin", # In DNP3. + "(ScValidatePnPService|ScSendPnPMessage)", # In DCE-RPC. + "snet", # Used as shorthand for subnet in base scripts. + "(e|i)it", # Used as name for some iterators. ] [default.extend-identifiers] diff --git a/NEWS b/NEWS index 550657d6ef..142f63c9ab 100644 --- a/NEWS +++ b/NEWS @@ -3859,7 +3859,7 @@ Removed Functionality - Functionality for writing/reading binary event streams was removed. This functionality relied on the old communication code - anc was basically untested. The ``-R`` command-line option (replay) + and was basically untested. The ``-R`` command-line option (replay) as well as the ``capture_events`` function were removed. - Removed p0f (passive OS fingerprinting) support. The version of diff --git a/ci/test.sh b/ci/test.sh index ff3935579a..adea2eb292 100755 --- a/ci/test.sh +++ b/ci/test.sh @@ -41,7 +41,7 @@ function banner { function run_unit_tests { if [[ ${ZEEK_CI_SKIP_UNIT_TESTS} -eq 1 ]]; then - printf "Skipping unit tests as requested by task configureation\n\n" + printf "Skipping unit tests as requested by task configuration\n\n" return 0 fi diff --git a/scripts/policy/protocols/ssl/certificate-request-info.zeek b/scripts/policy/protocols/ssl/certificate-request-info.zeek index 2479df5efa..c318a94615 100644 --- a/scripts/policy/protocols/ssl/certificate-request-info.zeek +++ b/scripts/policy/protocols/ssl/certificate-request-info.zeek @@ -6,7 +6,7 @@ module SSL; redef record SSL::Info += { - ## List of cient certificate CAs accepted by the server + ## List of client certificate CAs accepted by the server requested_client_certificate_authorities: vector of string &optional &log; }; diff --git a/src/analyzer/protocol/ldap/ldap.spicy b/src/analyzer/protocol/ldap/ldap.spicy index 0816e6afe9..14c3607c56 100644 --- a/src/analyzer/protocol/ldap/ldap.spicy +++ b/src/analyzer/protocol/ldap/ldap.spicy @@ -935,7 +935,7 @@ type SearchFilter = unit { }; # So when you're done with recursively parsing the filters, we can now leverage the tree structure to - # recursively get the stringRepresentations for those leafs, which are SearchFilters + # recursively get the stringRepresentations for those leaves, which are SearchFilters on %done { self.stringRepresentation = string_representation(self); diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 8a056c7621..989125b90f 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -1766,7 +1766,7 @@ RecordVal* Manager::ValueToRecordVal(const Stream* stream, const Value* const* v // them and has been warned by reporter. // Hence -> assign null to the field, done. - // Better check that it really is optional. Uou never know. + // Better check that it really is optional. You never know. assert(request_type->FieldDecl(i)->GetAttr(zeek::detail::ATTR_OPTIONAL)); } else if ( ! vals[*position]->present && ! request_type->FieldDecl(i)->GetAttr(zeek::detail::ATTR_OPTIONAL) ) { diff --git a/src/script_opt/Stmt.cc b/src/script_opt/Stmt.cc index 01c9a21e77..3e45161d7c 100644 --- a/src/script_opt/Stmt.cc +++ b/src/script_opt/Stmt.cc @@ -899,7 +899,7 @@ static bool simplify_chain(const std::vector& stmts, unsigned int start // An add-chain of any size is a win. For an assign-chain to be a win, // it needs to have at least two elements, because a single "x$a = y$b" - // can be expressed using one ZAM instructino (but "x$a += y$b" cannot). + // can be expressed using one ZAM instruction (but "x$a += y$b" cannot). if ( add_chains.empty() ) { bool have_useful_assign_chain = false; for ( auto& ac : assign_chains ) diff --git a/src/spicy/spicyz/driver.h b/src/spicy/spicyz/driver.h index f6276ab0bf..a0228a4e45 100644 --- a/src/spicy/spicyz/driver.h +++ b/src/spicy/spicyz/driver.h @@ -146,7 +146,7 @@ public: /** * Parses options command-line style after Zeek-side scripts have been - * fully procssed. Most of the option processing happens here (vs. in + * fully processed. Most of the option processing happens here (vs. in * `parseOptionsPreScript()`) except for things that must be in place * already before script processing. * From 29a49a59bd26162c7881b6731045daa7dcc0f1c7 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Mon, 9 Dec 2024 15:51:05 +0100 Subject: [PATCH 29/65] Bump pre-commit hooks --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0c9db04242..1b03457c90 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: args: ["-w", "-i", "4", "-ci"] - repo: https://github.com/google/yapf - rev: v0.40.2 + rev: v0.43.0 hooks: - id: yapf @@ -34,7 +34,7 @@ repos: exclude: '^(.typos.toml|src/SmithWaterman.cc|testing/.*|auxil/.*|scripts/base/frameworks/files/magic/.*|CHANGES|scripts/base/protocols/ssl/mozilla-ca-list.zeek)$' - repo: https://github.com/bbannier/spicy-format - rev: v0.20.0 + rev: v0.22.0 hooks: - id: spicy-format # TODO: Reformat existing large analyzers just before 8.0. From 08c5a9c66d5918d5fcc874680d400d1522729f50 Mon Sep 17 00:00:00 2001 From: Jan Grashoefer Date: Fri, 23 Aug 2024 19:14:55 +0200 Subject: [PATCH 30/65] Introduce get_current_packet_ts to fix packet lag Using network_time to calculate packet lag will produce wrong results when there is no packet available but network time does not (yet) fall back to wall clock. --- scripts/policy/misc/stats.zeek | 5 ++++- src/zeek.bif | 25 +++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/scripts/policy/misc/stats.zeek b/scripts/policy/misc/stats.zeek index c324a728be..8449ccdb0d 100644 --- a/scripts/policy/misc/stats.zeek +++ b/scripts/policy/misc/stats.zeek @@ -166,6 +166,8 @@ event zeek_init() &priority=5 Log::create_stream(Stats::LOG, [$columns=Info, $ev=log_stats, $path="stats", $policy=log_policy]); } +const null_ts = double_to_time(0); + event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: ProcStats, last_es: EventStats, last_rs: ReassemblerStats, last_ts: TimerStats, last_fs: FileAnalysisStats, last_ds: DNSStats) { local nettime = network_time(); @@ -214,7 +216,8 @@ event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: Pr if ( reading_live_traffic() ) { - info$pkt_lag = current_time() - nettime; + local pkt_ts = get_current_packet_ts(); + info$pkt_lag = pkt_ts > null_ts ? current_time() - pkt_ts : 0 sec; info$pkts_dropped = ns$pkts_dropped - last_ns$pkts_dropped; info$pkts_link = ns$pkts_link - last_ns$pkts_link; diff --git a/src/zeek.bif b/src/zeek.bif index ee14c0ddce..d614bf27a0 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -3833,6 +3833,31 @@ function get_current_packet_header%(%) : raw_pkt_hdr return std::move(hdr); %} +## Returns the currently processed PCAP packet's timestamp. +## +## Returns: The currently processed packet's timestamp. +## +## .. zeek:see:: get_current_packet get_current_packet_header +## +## .. note:: +## +## For ``get_current_packet_ts()`` the same limitations as for +## :zeek:see:`get_current_packet` apply. In particular, the return value +## should be considered undefined when called within event handlers raised +## via :zeek:see:`event`, :zeek:see:`schedule` or by recipient of Broker +## messages. +function get_current_packet_ts%(%) : time + %{ + double ts = 0; + const Packet* p = nullptr; + zeek::iosource::PktSrc* pkt_src = zeek::run_state::detail::current_packet_source(); + + if ( pkt_src && pkt_src->GetCurrentPacket(&p) ) + ts = p->time; + + return zeek::make_intrusive(ts); + %} + ## Writes a given packet to a file. ## ## pkt: The PCAP packet. From 6977c07a2530fc658e159c49809c49059480176e Mon Sep 17 00:00:00 2001 From: Jan Grashoefer Date: Mon, 26 Aug 2024 12:43:14 +0200 Subject: [PATCH 31/65] Add btest for get_current_packet_ts() --- .../Baseline/core.network_time.packet_ts/output | 5 +++++ testing/btest/core/network_time/packet_ts.zeek | 17 +++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 testing/btest/Baseline/core.network_time.packet_ts/output create mode 100644 testing/btest/core/network_time/packet_ts.zeek diff --git a/testing/btest/Baseline/core.network_time.packet_ts/output b/testing/btest/Baseline/core.network_time.packet_ts/output new file mode 100644 index 0000000000..cda26603e4 --- /dev/null +++ b/testing/btest/Baseline/core.network_time.packet_ts/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +network_time_init network time: 1362692526.869344 +network_time_init packet ts: 1362692526.869344 +conn_state_remove network time: 1362692527.080972 +conn_state_remove packet ts: 0.0 diff --git a/testing/btest/core/network_time/packet_ts.zeek b/testing/btest/core/network_time/packet_ts.zeek new file mode 100644 index 0000000000..602e668d64 --- /dev/null +++ b/testing/btest/core/network_time/packet_ts.zeek @@ -0,0 +1,17 @@ +# @TEST-DOC: Test get_current_packet_ts() in comparison with network_time(). +# @TEST-EXEC: zeek -b -r $TRACES/http/get.trace %INPUT > output +# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff output + +event network_time_init() + { + print fmt("network_time_init network time: %s", network_time()); + print fmt("network_time_init packet ts: %s", get_current_packet_ts()); + } + +# Note: Gracefully closed connections will be actually removed after +# tcp_close_delay (default 5 secs). +event connection_state_remove(c: connection) + { + print fmt("conn_state_remove network time: %s", network_time()); + print fmt("conn_state_remove packet ts: %s", get_current_packet_ts()); + } From f6c44e3f7a75328ec07764a010813ba30a201028 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Mon, 9 Dec 2024 18:29:40 +0100 Subject: [PATCH 32/65] DNS/dns_binds_rr: Fix complte to complete typo, switch to count From my reading in the docs the complete_flag should only ever be a single byte, so add a weird for when it is longer, but use count as the new type. --- scripts/base/init-bare.zeek | 3 +- src/analyzer/protocol/dns/DNS.cc | 11 ++++++- .../scripts.base.protocols.dns.binds/output | 32 +++++++++---------- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index ca57925603..ab337d3591 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -4247,8 +4247,9 @@ type dns_binds_rr: record { algorithm: count; ##< Algorithm for Public Key. key_id: count; ##< key tag. removal_flag: count; ##< rm flag. - complte_flag: string; ##< complete flag. + complte_flag: string &deprecated="Remove in v8.1: Use complete_flag instead."; ##< complete flag. is_query: count; ##< The RR is a query/Response. + complete_flag: count; ##< complete flag. }; ## A Private RR type LOC record. diff --git a/src/analyzer/protocol/dns/DNS.cc b/src/analyzer/protocol/dns/DNS.cc index f317a33599..12b30f22b5 100644 --- a/src/analyzer/protocol/dns/DNS.cc +++ b/src/analyzer/protocol/dns/DNS.cc @@ -1262,6 +1262,14 @@ bool DNS_Interpreter::ParseRR_BINDS(detail::DNS_MsgInfo* msg, const u_char*& dat String* completeflag = ExtractStream(data, len, rdlength - 4); + // We exposed the complete flag as a string to script land previously, + // but there should only ever be a single byte, so raise a weird if + // it is longer than that. + // + // https://bind9.readthedocs.io/en/latest/chapter5.html#monitoring-with-private-type-records + if ( completeflag->Len() > 1 ) + analyzer->Weird("DNS_BINDS_complete_flag_length", util::fmt("%d", completeflag->Len())); + if ( dns_BINDS ) { detail::BINDS_DATA binds; binds.algorithm = algo; @@ -1855,8 +1863,9 @@ RecordValPtr DNS_MsgInfo::BuildBINDS_Val(BINDS_DATA* binds) { r->Assign(2, binds->algorithm); r->Assign(3, binds->key_id); r->Assign(4, binds->removal_flag); - r->Assign(5, binds->complete_flag); + r->Assign(5, binds->complete_flag); // Remove in v8.1: Move field 7 here. Drop String* usage. r->Assign(6, is_query); + r->Assign(7, binds->complete_flag->Len() > 0 ? binds->complete_flag->Bytes()[0] : 0); return r; } diff --git a/testing/btest/Baseline/scripts.base.protocols.dns.binds/output b/testing/btest/Baseline/scripts.base.protocols.dns.binds/output index 9e60009463..f35defd7a7 100644 --- a/testing/btest/Baseline/scripts.base.protocols.dns.binds/output +++ b/testing/btest/Baseline/scripts.base.protocols.dns.binds/output @@ -1,17 +1,17 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -BINDS, [query=example.net, answer_type=1, algorithm=7, key_id=32018, removal_flag=0, complte_flag=\x01, is_query=0] -BINDS, [query=example.net, answer_type=1, algorithm=5, key_id=2196, removal_flag=0, complte_flag=\x01, is_query=0] -BINDS, [query=example.net, answer_type=1, algorithm=15, key_id=12994, removal_flag=0, complte_flag=\x01, is_query=0] -BINDS, [query=example.net, answer_type=1, algorithm=16, key_id=23868, removal_flag=0, complte_flag=\x01, is_query=0] -BINDS, [query=example.net, answer_type=1, algorithm=7, key_id=37611, removal_flag=0, complte_flag=\x01, is_query=0] -BINDS, [query=example.net, answer_type=1, algorithm=8, key_id=9551, removal_flag=0, complte_flag=\x01, is_query=0] -BINDS, [query=example.net, answer_type=1, algorithm=5, key_id=48254, removal_flag=0, complte_flag=\x01, is_query=0] -BINDS, [query=example.net, answer_type=1, algorithm=8, key_id=33130, removal_flag=0, complte_flag=\x01, is_query=0] -BINDS, [query=example.net, answer_type=1, algorithm=14, key_id=15141, removal_flag=0, complte_flag=\x01, is_query=0] -BINDS, [query=example.net, answer_type=1, algorithm=10, key_id=41675, removal_flag=0, complte_flag=\x01, is_query=0] -BINDS, [query=example.net, answer_type=1, algorithm=10, key_id=63711, removal_flag=0, complte_flag=\x01, is_query=0] -BINDS, [query=example.net, answer_type=1, algorithm=13, key_id=65395, removal_flag=0, complte_flag=\x01, is_query=0] -BINDS, [query=example.net, answer_type=1, algorithm=13, key_id=31400, removal_flag=0, complte_flag=\x01, is_query=0] -BINDS, [query=example.net, answer_type=1, algorithm=14, key_id=60289, removal_flag=0, complte_flag=\x01, is_query=0] -BINDS, [query=example.net, answer_type=1, algorithm=15, key_id=31000, removal_flag=0, complte_flag=\x01, is_query=0] -BINDS, [query=example.net, answer_type=1, algorithm=16, key_id=40187, removal_flag=0, complte_flag=\x01, is_query=0] +BINDS, [query=example.net, answer_type=1, algorithm=7, key_id=32018, removal_flag=0, complte_flag=\x01, is_query=0, complete_flag=1] +BINDS, [query=example.net, answer_type=1, algorithm=5, key_id=2196, removal_flag=0, complte_flag=\x01, is_query=0, complete_flag=1] +BINDS, [query=example.net, answer_type=1, algorithm=15, key_id=12994, removal_flag=0, complte_flag=\x01, is_query=0, complete_flag=1] +BINDS, [query=example.net, answer_type=1, algorithm=16, key_id=23868, removal_flag=0, complte_flag=\x01, is_query=0, complete_flag=1] +BINDS, [query=example.net, answer_type=1, algorithm=7, key_id=37611, removal_flag=0, complte_flag=\x01, is_query=0, complete_flag=1] +BINDS, [query=example.net, answer_type=1, algorithm=8, key_id=9551, removal_flag=0, complte_flag=\x01, is_query=0, complete_flag=1] +BINDS, [query=example.net, answer_type=1, algorithm=5, key_id=48254, removal_flag=0, complte_flag=\x01, is_query=0, complete_flag=1] +BINDS, [query=example.net, answer_type=1, algorithm=8, key_id=33130, removal_flag=0, complte_flag=\x01, is_query=0, complete_flag=1] +BINDS, [query=example.net, answer_type=1, algorithm=14, key_id=15141, removal_flag=0, complte_flag=\x01, is_query=0, complete_flag=1] +BINDS, [query=example.net, answer_type=1, algorithm=10, key_id=41675, removal_flag=0, complte_flag=\x01, is_query=0, complete_flag=1] +BINDS, [query=example.net, answer_type=1, algorithm=10, key_id=63711, removal_flag=0, complte_flag=\x01, is_query=0, complete_flag=1] +BINDS, [query=example.net, answer_type=1, algorithm=13, key_id=65395, removal_flag=0, complte_flag=\x01, is_query=0, complete_flag=1] +BINDS, [query=example.net, answer_type=1, algorithm=13, key_id=31400, removal_flag=0, complte_flag=\x01, is_query=0, complete_flag=1] +BINDS, [query=example.net, answer_type=1, algorithm=14, key_id=60289, removal_flag=0, complte_flag=\x01, is_query=0, complete_flag=1] +BINDS, [query=example.net, answer_type=1, algorithm=15, key_id=31000, removal_flag=0, complte_flag=\x01, is_query=0, complete_flag=1] +BINDS, [query=example.net, answer_type=1, algorithm=16, key_id=40187, removal_flag=0, complte_flag=\x01, is_query=0, complete_flag=1] From ccefd66d3704c7f1d803fcc966f7b854b1c31625 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 6 Dec 2024 13:50:38 -0700 Subject: [PATCH 33/65] Move python signatures to a separate file --- .../base/frameworks/files/magic/__load__.zeek | 1 + .../frameworks/files/magic/executable.sig | 112 ------------------ .../base/frameworks/files/magic/python.sig | 111 +++++++++++++++++ testing/btest/Baseline/plugins.hooks/output | 6 + 4 files changed, 118 insertions(+), 112 deletions(-) create mode 100644 scripts/base/frameworks/files/magic/python.sig diff --git a/scripts/base/frameworks/files/magic/__load__.zeek b/scripts/base/frameworks/files/magic/__load__.zeek index cb39d21d9d..7eba65d529 100644 --- a/scripts/base/frameworks/files/magic/__load__.zeek +++ b/scripts/base/frameworks/files/magic/__load__.zeek @@ -7,6 +7,7 @@ @load-sigs ./java @load-sigs ./office @load-sigs ./programming +@load-sigs ./python @load-sigs ./video @load-sigs ./libmagic diff --git a/scripts/base/frameworks/files/magic/executable.sig b/scripts/base/frameworks/files/magic/executable.sig index 9a01d09d41..1f1f41f8c2 100644 --- a/scripts/base/frameworks/files/magic/executable.sig +++ b/scripts/base/frameworks/files/magic/executable.sig @@ -41,115 +41,3 @@ signature file-elc { file-mime "application/x-elc", 10 file-magic /\x3bELC[\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff]/ } - -# Python magic numbers can be updated/added by looking at the list at -# https://github.com/python/cpython/blob/main/Include/internal/pycore_magic_number.h -# The numbers in the list are converted to little-endian and then to hex for the -# file-magic entries below. - -# Python 1 bytecode -signature file-pyc-1 { - file-magic /^(\xfc\xc4|\x99\x4e)\x0d\x0a/ - file-mime "application/x-python-bytecode", 80 -} - -# Python 2 bytecode -signature file-pyc-2 { - file-magic /^(\x87\xc6|[\x2a\x2d]\xed|[\x3b\x45\x59\x63\x6d\x77\x81\x8b\x8c\x95\x9f\xa9\xb3\xc7\xd1\xdb\xe5\xef\xf9]\xf2|\x03\xf3)\x0d\x0a/ - file-mime "application/x-python-bytecode", 80 -} - -# Python 3.0 bytecode -signature file-pyc-3-0 { - file-magic /^([\xb8\xc2\xcc\xd6\xe0\xea\xf4\xf5\xff]\x0b|[\x09\x13\x1d\x1f\x27\x3b]\x0c)\x0d\x0a/ - file-mime "application/x-python-bytecode", 80 -} - - -# Python 3.1 bytecode -signature file-pyc-3-1 { - file-magic /^[\x45\x4f]\x0c\x0d\x0a/ - file-mime "application/x-python-bytecode", 80 -} - - -# Python 3.2 bytecode -signature file-pyc-3-2 { - file-magic /^[\x58\x62\x6c]\x0c\x0d\x0a/ - file-mime "application/x-python-bytecode", 80 -} - -# Python 3.3 bytecode -signature file-pyc-3-3 { - file-magic /^[\x76\x80\x94\x9e]\x0c\x0d\x0a/ - file-mime "application/x-python-bytecode", 80 -} - - -# Python 3.4 bytecode -signature file-pyc-3-4 { - file-magic /^[\xb2\xcc\xc6\xd0\xda\xe4\xee]\x0c\x0d\x0a/ - file-mime "application/x-python-bytecode", 80 -} - -# Python 3.5 bytecode -signature file-pyc-3-5 { - file-magic /^(\xf8\x0c|[\x02\x0c\x16\x17]\x0d)\x0d\x0a/ - file-mime "application/x-python-bytecode", 80 -} - -# Python 3.6 bytecode -signature file-pyc-3-6 { - file-magic /^[\x20\x21\x2a-\x2d\x2f-\x33]\x0d\x0d\x0a/ - file-mime "application/x-python-bytecode", 80 -} - -# Python 3.7 bytecode -signature file-pyc-3-7 { - file-magic /^[\x3e-\x42]\x0d\x0d\x0a/ - file-mime "application/x-python-bytecode", 80 -} - -# Python 3.8 bytecode -signature file-pyc-3-8 { - file-magic /^[\x48\x49\x52-\x55]\x0d\x0d\x0a/ - file-mime "application/x-python-bytecode", 80 -} - -# Python 3.9 bytecode -signature file-pyc-3-9 { - file-magic /^[\x5c-\x61]\x0d\x0d\x0a/ - file-mime "application/x-python-bytecode", 80 -} - -# Python 3.10 bytecode -signature file-pyc-3-10 { - file-magic /^[\x66-\x6f]\x0d\x0d\x0a/ - file-mime "application/x-python-bytecode", 80 -} - -# Python 3.11 bytecode -signature file-pyc-3-11 { - file-magic /^[\x7a-\xa7]\x0d\x0d\x0a/ - file-mime "application/x-python-bytecode", 80 -} - -# Python 3.12 bytecode -signature file-pyc-3-12 { - file-magic /^[\xac-\xcb]\x0d\x0d\x0a/ - file-mime "application/x-python-bytecode", 80 -} - -# Python 3.13 bytecode -signature file-pyc-3-13 { - file-magic /^[\xde-\xf3]\x0d\x0d\x0a/ - file-mime "application/x-python-bytecode", 80 -} - -# Python 3.14 bytecode -# This is in pre-release at this time, and may need to be updated as new -# versions come out. -signature file-pyc-3-14 { - file-magic /^[\x10-\x19]\x0e\x0d\x0a/ - file-mime "application/x-python-bytecode", 80 -} diff --git a/scripts/base/frameworks/files/magic/python.sig b/scripts/base/frameworks/files/magic/python.sig new file mode 100644 index 0000000000..17098bc746 --- /dev/null +++ b/scripts/base/frameworks/files/magic/python.sig @@ -0,0 +1,111 @@ +# Python magic numbers can be updated/added by looking at the list at +# https://github.com/python/cpython/blob/main/Include/internal/pycore_magic_number.h +# The numbers in the list are converted to little-endian and then to hex for the +# file-magic entries below. + +# Python 1 bytecode +signature file-pyc-1 { + file-magic /^(\xfc\xc4|\x99\x4e)\x0d\x0a/ + file-mime "application/x-python-bytecode", 80 +} + +# Python 2 bytecode +signature file-pyc-2 { + file-magic /^(\x87\xc6|[\x2a\x2d]\xed|[\x3b\x45\x59\x63\x6d\x77\x81\x8b\x8c\x95\x9f\xa9\xb3\xc7\xd1\xdb\xe5\xef\xf9]\xf2|\x03\xf3)\x0d\x0a/ + file-mime "application/x-python-bytecode", 80 +} + +# Python 3.0 bytecode +signature file-pyc-3-0 { + file-magic /^([\xb8\xc2\xcc\xd6\xe0\xea\xf4\xf5\xff]\x0b|[\x09\x13\x1d\x1f\x27\x3b]\x0c)\x0d\x0a/ + file-mime "application/x-python-bytecode", 80 +} + + +# Python 3.1 bytecode +signature file-pyc-3-1 { + file-magic /^[\x45\x4f]\x0c\x0d\x0a/ + file-mime "application/x-python-bytecode", 80 +} + + +# Python 3.2 bytecode +signature file-pyc-3-2 { + file-magic /^[\x58\x62\x6c]\x0c\x0d\x0a/ + file-mime "application/x-python-bytecode", 80 +} + +# Python 3.3 bytecode +signature file-pyc-3-3 { + file-magic /^[\x76\x80\x94\x9e]\x0c\x0d\x0a/ + file-mime "application/x-python-bytecode", 80 +} + + +# Python 3.4 bytecode +signature file-pyc-3-4 { + file-magic /^[\xb2\xcc\xc6\xd0\xda\xe4\xee]\x0c\x0d\x0a/ + file-mime "application/x-python-bytecode", 80 +} + +# Python 3.5 bytecode +signature file-pyc-3-5 { + file-magic /^(\xf8\x0c|[\x02\x0c\x16\x17]\x0d)\x0d\x0a/ + file-mime "application/x-python-bytecode", 80 +} + +# Python 3.6 bytecode +signature file-pyc-3-6 { + file-magic /^[\x20\x21\x2a-\x2d\x2f-\x33]\x0d\x0d\x0a/ + file-mime "application/x-python-bytecode", 80 +} + +# Python 3.7 bytecode +signature file-pyc-3-7 { + file-magic /^[\x3e-\x42]\x0d\x0d\x0a/ + file-mime "application/x-python-bytecode", 80 +} + +# Python 3.8 bytecode +signature file-pyc-3-8 { + file-magic /^[\x48\x49\x52-\x55]\x0d\x0d\x0a/ + file-mime "application/x-python-bytecode", 80 +} + +# Python 3.9 bytecode +signature file-pyc-3-9 { + file-magic /^[\x5c-\x61]\x0d\x0d\x0a/ + file-mime "application/x-python-bytecode", 80 +} + +# Python 3.10 bytecode +signature file-pyc-3-10 { + file-magic /^[\x66-\x6f]\x0d\x0d\x0a/ + file-mime "application/x-python-bytecode", 80 +} + +# Python 3.11 bytecode +signature file-pyc-3-11 { + file-magic /^[\x7a-\xa7]\x0d\x0d\x0a/ + file-mime "application/x-python-bytecode", 80 +} + +# Python 3.12 bytecode +signature file-pyc-3-12 { + file-magic /^[\xac-\xcb]\x0d\x0d\x0a/ + file-mime "application/x-python-bytecode", 80 +} + +# Python 3.13 bytecode +signature file-pyc-3-13 { + file-magic /^[\xde-\xf3]\x0d\x0d\x0a/ + file-mime "application/x-python-bytecode", 80 +} + +# Python 3.14 bytecode +# This is in pre-release at this time, and may need to be updated as new +# versions come out. +signature file-pyc-3-14 { + file-magic /^[\x10-\x19]\x0e\x0d\x0a/ + file-mime "application/x-python-bytecode", 80 +} diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 48ce2af63b..c799317bd1 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -632,6 +632,7 @@ 0.000000 MetaHookPost LoadFile(1, ./libmagic, <...>/libmagic.sig) -> -1 0.000000 MetaHookPost LoadFile(1, ./office, <...>/office.sig) -> -1 0.000000 MetaHookPost LoadFile(1, ./programming, <...>/programming.sig) -> -1 +0.000000 MetaHookPost LoadFile(1, ./python, <...>/python.sig) -> -1 0.000000 MetaHookPost LoadFile(1, ./video, <...>/video.sig) -> -1 0.000000 MetaHookPost LoadFile(1, s2, ./s2.sig) -> -1 0.000000 MetaHookPost LoadFileExtended(0, ./CPP-load.bif.zeek, <...>/CPP-load.bif.zeek) -> (-1, ) @@ -936,6 +937,7 @@ 0.000000 MetaHookPost LoadFileExtended(1, ./libmagic, <...>/libmagic.sig) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(1, ./office, <...>/office.sig) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(1, ./programming, <...>/programming.sig) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(1, ./python, <...>/python.sig) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(1, ./video, <...>/video.sig) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(1, s2, ./s2.sig) -> (-1, ) 0.000000 MetaHookPost QueueEvent(zeek_init()) -> false @@ -1572,6 +1574,7 @@ 0.000000 MetaHookPre LoadFile(1, ./libmagic, <...>/libmagic.sig) 0.000000 MetaHookPre LoadFile(1, ./office, <...>/office.sig) 0.000000 MetaHookPre LoadFile(1, ./programming, <...>/programming.sig) +0.000000 MetaHookPre LoadFile(1, ./python, <...>/python.sig) 0.000000 MetaHookPre LoadFile(1, ./video, <...>/video.sig) 0.000000 MetaHookPre LoadFile(1, s2, ./s2.sig) 0.000000 MetaHookPre LoadFileExtended(0, ./CPP-load.bif.zeek, <...>/CPP-load.bif.zeek) @@ -1876,6 +1879,7 @@ 0.000000 MetaHookPre LoadFileExtended(1, ./libmagic, <...>/libmagic.sig) 0.000000 MetaHookPre LoadFileExtended(1, ./office, <...>/office.sig) 0.000000 MetaHookPre LoadFileExtended(1, ./programming, <...>/programming.sig) +0.000000 MetaHookPre LoadFileExtended(1, ./python, <...>/python.sig) 0.000000 MetaHookPre LoadFileExtended(1, ./video, <...>/video.sig) 0.000000 MetaHookPre LoadFileExtended(1, s2, ./s2.sig) 0.000000 MetaHookPre QueueEvent(zeek_init()) @@ -2384,6 +2388,7 @@ 0.000000 | HookLoadFile ./pools <...>/pools.zeek 0.000000 | HookLoadFile ./postprocessors <...>/postprocessors 0.000000 | HookLoadFile ./programming <...>/programming.sig +0.000000 | HookLoadFile ./python <...>/python.sig 0.000000 | HookLoadFile ./removal-hooks <...>/removal-hooks.zeek 0.000000 | HookLoadFile ./reporter.bif.zeek <...>/reporter.bif.zeek 0.000000 | HookLoadFile ./scp <...>/scp.zeek @@ -2688,6 +2693,7 @@ 0.000000 | HookLoadFileExtended ./pools <...>/pools.zeek 0.000000 | HookLoadFileExtended ./postprocessors <...>/postprocessors 0.000000 | HookLoadFileExtended ./programming <...>/programming.sig +0.000000 | HookLoadFileExtended ./python <...>/python.sig 0.000000 | HookLoadFileExtended ./removal-hooks <...>/removal-hooks.zeek 0.000000 | HookLoadFileExtended ./reporter.bif.zeek <...>/reporter.bif.zeek 0.000000 | HookLoadFileExtended ./scp <...>/scp.zeek From c2b17f9d308be603fbb78e6d48f27e995d726e1c Mon Sep 17 00:00:00 2001 From: Jan Grashoefer Date: Mon, 9 Dec 2024 19:19:29 +0100 Subject: [PATCH 34/65] Introduce get_packet_lag() --- scripts/base/utils/time.zeek | 20 ++++++++++++++++++++ scripts/policy/misc/stats.zeek | 6 ++---- src/zeek.bif | 14 +++++++------- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/scripts/base/utils/time.zeek b/scripts/base/utils/time.zeek index c173c82878..c6f8cfe76b 100644 --- a/scripts/base/utils/time.zeek +++ b/scripts/base/utils/time.zeek @@ -1,3 +1,4 @@ +##! Time-related functions. ## Given an interval, returns a string representing the minutes and seconds ## in the interval (for example, "3m34s"). @@ -6,3 +7,22 @@ function duration_to_mins_secs(dur: interval): string local dur_count = double_to_count(interval_to_double(dur)); return fmt("%dm%ds", dur_count/60, dur_count%60); } + +## Time value representing the 0 timestamp. +const null_ts = double_to_time(0); + +## Calculate the packet lag, i.e. the difference between wall clock and the +## timestamp of the currently processed packet. If Zeek is not processing a +## packet, the function returns a 0 interval value. +function get_packet_lag(): interval + { + # We use get_current_packet_ts() instead of network_time() here, because + # network time does not immediately fall back to wall clock if there is + # no packet. Instead, network time remains set to the last seen packet's + # timestamp for ``packet_source_inactivity_timeout``. + local pkt_ts = get_current_packet_ts(); + if (pkt_ts == null_ts) + return 0 sec; + + return current_time() - pkt_ts; + } diff --git a/scripts/policy/misc/stats.zeek b/scripts/policy/misc/stats.zeek index 8449ccdb0d..cae9a3b16a 100644 --- a/scripts/policy/misc/stats.zeek +++ b/scripts/policy/misc/stats.zeek @@ -2,6 +2,7 @@ @load base/frameworks/notice @load base/frameworks/telemetry +@load base/utils/time module Stats; @@ -166,8 +167,6 @@ event zeek_init() &priority=5 Log::create_stream(Stats::LOG, [$columns=Info, $ev=log_stats, $path="stats", $policy=log_policy]); } -const null_ts = double_to_time(0); - event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: ProcStats, last_es: EventStats, last_rs: ReassemblerStats, last_ts: TimerStats, last_fs: FileAnalysisStats, last_ds: DNSStats) { local nettime = network_time(); @@ -216,8 +215,7 @@ event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: Pr if ( reading_live_traffic() ) { - local pkt_ts = get_current_packet_ts(); - info$pkt_lag = pkt_ts > null_ts ? current_time() - pkt_ts : 0 sec; + info$pkt_lag = get_packet_lag(); info$pkts_dropped = ns$pkts_dropped - last_ns$pkts_dropped; info$pkts_link = ns$pkts_link - last_ns$pkts_link; diff --git a/src/zeek.bif b/src/zeek.bif index d614bf27a0..84cc1e397a 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -3833,19 +3833,19 @@ function get_current_packet_header%(%) : raw_pkt_hdr return std::move(hdr); %} -## Returns the currently processed PCAP packet's timestamp. +## Returns the currently processed PCAP packet's timestamp or a 0 timestamp if +## there is no packet being processed at the moment. ## ## Returns: The currently processed packet's timestamp. ## -## .. zeek:see:: get_current_packet get_current_packet_header +## .. zeek:see:: get_current_packet get_current_packet_header network_time ## ## .. note:: ## -## For ``get_current_packet_ts()`` the same limitations as for -## :zeek:see:`get_current_packet` apply. In particular, the return value -## should be considered undefined when called within event handlers raised -## via :zeek:see:`event`, :zeek:see:`schedule` or by recipient of Broker -## messages. +## When there is no packet being processed, ``get_current_packet_ts()`` +## will return a 0 timestamp, while ``network_time()`` will return the +## timestamp of the last processed packet until it falls back to tracking +## wall clock after ``packet_source_inactivity_timeout``. function get_current_packet_ts%(%) : time %{ double ts = 0; From 09541d5ef2018fef4d1d798105ec74413e1c5e31 Mon Sep 17 00:00:00 2001 From: zeek-bot Date: Tue, 10 Dec 2024 00:24:53 +0000 Subject: [PATCH 35/65] Update doc submodule [nomail] [skip ci] --- doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc b/doc index 50463c65a6..27f321afb4 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 50463c65a6f49c40b974f701964283f305394a0b +Subproject commit 27f321afb47feeffaa4534093d6abfc8e2b220af From a32ea436641bdf885f4c9ee6d6d396443f3f241e Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Mon, 9 Dec 2024 23:55:15 -0800 Subject: [PATCH 36/65] NEWS tweaks [skip ci] - Switch list items back to "-" from "*" -- we hadn't used "*" since 2.5.4 but started in 7.1, probably by accident? :-) - Fix a typo. --- NEWS | 56 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/NEWS b/NEWS index 95b41a5528..3fde1b5600 100644 --- a/NEWS +++ b/NEWS @@ -9,14 +9,14 @@ Zeek 7.1.0 Breaking Changes ---------------- -* The ``OpaqueVal::DoSerialize`` and ``OpaqueVal::DoUnserialize`` methods were +- The ``OpaqueVal::DoSerialize`` and ``OpaqueVal::DoUnserialize`` methods were marked as deprecated in v7.0 and have now been removed as per the Zeek deprecation policy. Plugins that were overriding these methods and were not updated will fail to compile. Those plugins should be updated to override the new ``OpaqueVal::DoSerializeData`` and ``OpaqueVal::DoUnserializeData`` methods. -* Certain internal methods on the broker and logging classes have been changed to +- Certain internal methods on the broker and logging classes have been changed to accept std::vector parameters instead of threading::Value** to leverage automatic memory management, reduce the number of allocations and use move semantics to express ownership. @@ -25,12 +25,12 @@ Breaking Changes are not affected by this change, so we keep backwards compatibility with existing log writers. -* ``Func::Name()`` was deprecated, use ``Func::GetName()`` instead. +- ``Func::Name()`` was deprecated, use ``Func::GetName()`` instead. New Functionality ----------------- -* IP-based connections that were previously not logged due to using an unknown +- IP-based connections that were previously not logged due to using an unknown IP protocol (e.g. not TCP, UDP, or ICMP) now appear in conn.log. All conn.log entries have a new ``ip_proto`` column that indicates the numeric IP protocol identifier used by the connection. A new policy script at @@ -103,38 +103,38 @@ New Functionality These new policies fix a problem in which misbehaving nodes could trigger cascading "lockups" of nodes, each ceasing to transmit any messages. -* The LDAP analyzer now supports handling of non-sealed GSS-API WRAP tokens. +- The LDAP analyzer now supports handling of non-sealed GSS-API WRAP tokens. -* StartTLS support was added to the LDAP analyzer. The SSL analyzer is enabled +- StartTLS support was added to the LDAP analyzer. The SSL analyzer is enabled for connections where client and server negotiate to TLS through the extended request/response mechanism. -* The ``unknown_protocols()`` event now includes the name of all packet +- The ``unknown_protocols()`` event now includes the name of all packet analyzer used for processing the packet when the event is raised. The ``unknown_protocol.log`` file was extended to include this information. -* The MySQL analyzer now generates a ``mysql_user_change()`` event when +- The MySQL analyzer now generates a ``mysql_user_change()`` event when the user changes mid-session via the ``COM_USER_CHANGE`` command. -* The DNS analyzer was extended to support TKEY RRs (RFC 2390). A corresponding +- The DNS analyzer was extended to support TKEY RRs (RFC 2390). A corresponding ``dns_TKEY`` event was added. -* The ``signature_match()`` and custom signature events now receive the end of +- The ``signature_match()`` and custom signature events now receive the end of match offset within the ``data`` parameter as an optional parameter named ``end_of_match``. event signature_match(state: signature_state, msg: string, data: string, end_of_match: count); -* A we plugin hook ``InitPreExecution()`` has been added to allow introspection +- A new plugin hook ``InitPreExecution()`` has been added to allow introspection of Zeek's AST after ZAM optimizations ran. This hook executes right before the ``zeek_init()`` event is enqueued. -* The SQLite logger now supports setting the value of the SQLite synchronous mode, +- The SQLite logger now supports setting the value of the SQLite synchronous mode, as well as of the journal mode. For example, WAL mode can be enabled by setting: redef LogSQLite::journal_mode=LogSQLite::SQLITE_JOURNAL_MODE_WAL; -* A pseudo protocol analyzer StreamEvent has been added. Attaching this analyzer +- A pseudo protocol analyzer StreamEvent has been added. Attaching this analyzer to TCP connections allows processing the connection's stream data in the scripting layer. One example use-case is interactive terminal sessions over HTTP connections upgraded to TCP. @@ -151,43 +151,43 @@ New Functionality Changed Functionality --------------------- -* Heuristics for parsing SASL encrypted and signed LDAP traffic have been +- Heuristics for parsing SASL encrypted and signed LDAP traffic have been made more strict and predictable. Please provide input if this results in less visibility in your environment. -* The MySQL analyzer has been improved to better support plugin authentication +- The MySQL analyzer has been improved to better support plugin authentication mechanisms, like caching_sha2_password, as well as recognizing MySQL query attributes. -* The ``mysql.log`` for user change commands will contain *just* the username +- The ``mysql.log`` for user change commands will contain *just* the username instead of the remaining parts of the command, including auth plugin data. -* The POP3 parser has been hardened to avoid unbounded state growth in the +- The POP3 parser has been hardened to avoid unbounded state growth in the face of one-sided traffic capture or when enabled for non-POP3 traffic. Concretely, the Redis protocol's AUTH mechanism enables the POP3 analyzer for such connections through DPD. -* Batching and flushing for local log writers can now be controlled via the +- Batching and flushing for local log writers can now be controlled via the options ``Log::flush_interval`` and ``Log::write_buffer_size``. Previously the ``Threading::heartbeat_interval`` was used for flushing and the buffer size fixed at 1000. -* Logging of the FTP PASS command in ``ftp.log`` now honors ``FTP::default_capture_password`` +- Logging of the FTP PASS command in ``ftp.log`` now honors ``FTP::default_capture_password`` and the password is blanked with "". Previously, the argument for the PASS command would be logged in clear. -* The ASCII input reader now suppresses warnings for consecutive invalid lines, +- The ASCII input reader now suppresses warnings for consecutive invalid lines, producing a summary of total suppressions once a valid line is encountered. -* The `Telemetry::sync()` hook is now invoked on demand. Either when the metrics +- The `Telemetry::sync()` hook is now invoked on demand. Either when the metrics of a node are scraped via the Prometheus HTTP endpoint, or one of the collect methods is invoked from Zeek script. -* The community-id-logging.zeek policy script was used to set ``c$conn$community_id`` +- The community-id-logging.zeek policy script was used to set ``c$conn$community_id`` during ``new_connection()`` rather than ``connection_state_remove()``, allowing other scripts to reuse its value early. -* Calling ``Broker::publish()`` now uses the event time of the currently +- Calling ``Broker::publish()`` now uses the event time of the currently executing event as network time metadata attached to the remote event. Previously, ``network_time()`` was used. This matters if ``Broker::publish()`` is called within scheduled events or called within remote events. @@ -198,7 +198,7 @@ Removed Functionality Deprecated Functionality ------------------------ -* The ``Broker::auto_publish()`` function has been deprecated and should +- The ``Broker::auto_publish()`` function has been deprecated and should be replaced with explicit ``Broker::publish()`` invocations that are potentially guarded with appropriate ``@if`` or ``@ifdef`` directives. @@ -4587,14 +4587,14 @@ Bro 2.5.4 Bro 2.5.4 primarily fixes security issues: -* Multiple fixes and improvements to BinPAC generated code related to +- Multiple fixes and improvements to BinPAC generated code related to array parsing, with potential impact to all Bro's BinPAC-generated analyzers in the form of buffer over-reads or other invalid memory accesses depending on whether a particular analyzer incorrectly assumed that the evaluated-array-length expression is actually the number of elements that were parsed out from the input. -* The NCP analyzer (not enabled by default and also updated to actually +- The NCP analyzer (not enabled by default and also updated to actually work with newer Bro APIs in the release) performed a memory allocation based directly on a field in the input packet and using signed integer storage. This could result in a signed integer overflow and memory @@ -4604,9 +4604,9 @@ Bro 2.5.4 primarily fixes security issues: There's also the following bug fixes: -* A memory leak in the SMBv1 analyzer. +- A memory leak in the SMBv1 analyzer. -* The MySQL analyzer was generally not working as intended, for example, +- The MySQL analyzer was generally not working as intended, for example, it now is able to parse responses that contain multiple results/rows. Bro 2.5.3 From 9228f0ff9fdfca85984f9bef53389787bc60efc0 Mon Sep 17 00:00:00 2001 From: Jan Grashoefer Date: Tue, 10 Dec 2024 14:49:02 +0100 Subject: [PATCH 37/65] Add protocol_id count to unknown protocol record The count representation is not logged and added for access to the value in log policy hooks without converting the logged hex representation. --- scripts/policy/misc/unknown-protocols.zeek | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/policy/misc/unknown-protocols.zeek b/scripts/policy/misc/unknown-protocols.zeek index c532f7b0f5..4d5323f91c 100644 --- a/scripts/policy/misc/unknown-protocols.zeek +++ b/scripts/policy/misc/unknown-protocols.zeek @@ -19,9 +19,14 @@ export { ## The string name of the analyzer attempting to forward the protocol. analyzer: string &log; - ## The identifier of the protocol being forwarded. + ## The identifier of the protocol being forwarded in hex notation. protocol_id: string &log; + ## The identifier of the protocol being forwarded as count. + ## Note: The count value is not logged by default. It is provided for + ## easy access in log policy hooks. + protocol_id_num: count; + ## A certain number of bytes at the start of the unknown protocol's ## header. first_bytes: string &log; @@ -40,6 +45,7 @@ event unknown_protocol(analyzer_name: string, protocol: count, first_bytes: stri info$ts = network_time(); info$analyzer = analyzer_name; info$protocol_id = fmt("0x%x", protocol); + info$protocol_id_num = protocol; info$first_bytes = bytestring_to_hexstr(first_bytes); info$analyzer_history = analyzer_history; From 1805afe5d989565e61640038e9ceb9aaf93015e0 Mon Sep 17 00:00:00 2001 From: Jan Grashoefer Date: Tue, 10 Dec 2024 14:54:19 +0100 Subject: [PATCH 38/65] Add btest for unknown_protocols.log --- .../core.unknown-protocol-log/unknown_protocols.log | 11 +++++++++++ testing/btest/core/unknown-protocol-log.zeek | 6 ++++++ 2 files changed, 17 insertions(+) create mode 100644 testing/btest/Baseline/core.unknown-protocol-log/unknown_protocols.log create mode 100644 testing/btest/core/unknown-protocol-log.zeek diff --git a/testing/btest/Baseline/core.unknown-protocol-log/unknown_protocols.log b/testing/btest/Baseline/core.unknown-protocol-log/unknown_protocols.log new file mode 100644 index 0000000000..fd78878bea --- /dev/null +++ b/testing/btest/Baseline/core.unknown-protocol-log/unknown_protocols.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path unknown_protocols +#open XXXX-XX-XX-XX-XX-XX +#fields ts analyzer protocol_id protocol_id_num first_bytes analyzer_history +#types time string string count string vector[string] +XXXXXXXXXX.XXXXXX ETHERNET 0x88cc 35020 02070400222d81db1004 ETHERNET +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/core/unknown-protocol-log.zeek b/testing/btest/core/unknown-protocol-log.zeek new file mode 100644 index 0000000000..c059e1314c --- /dev/null +++ b/testing/btest/core/unknown-protocol-log.zeek @@ -0,0 +1,6 @@ +# @TEST-EXEC: zeek -b -r $TRACES/lldp.pcap %INPUT +# @TEST-EXEC: btest-diff unknown_protocols.log + +@load misc/unknown-protocols + +redef record UnknownProtocol::Info$protocol_id_num += { &log }; From 7449b050b3341e56b0c2365b7528bebc52060d7e Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Tue, 10 Dec 2024 19:51:45 +0100 Subject: [PATCH 39/65] Update BiF-tracking, add get_current_packet_ts() Also, run the ZAM-bif-tracking test in non-ZAM environments so failures are caught immediately. There's nothing overly ZAM specific about running this test. I'm not sure I like the fact that any new contributor adding a BiF will need to dig into this... it might be a bit intimidating. --- src/script_opt/FuncInfo.cc | 1 + .../{Baseline.zam => Baseline}/opt.ZAM-bif-tracking/output | 2 +- testing/btest/opt/ZAM-bif-tracking.zeek | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) rename testing/btest/{Baseline.zam => Baseline}/opt.ZAM-bif-tracking/output (70%) diff --git a/src/script_opt/FuncInfo.cc b/src/script_opt/FuncInfo.cc index 545f278f96..c21586f406 100644 --- a/src/script_opt/FuncInfo.cc +++ b/src/script_opt/FuncInfo.cc @@ -268,6 +268,7 @@ static std::unordered_map func_attrs = { {"get_current_conn_packets_threshold", ATTR_NO_ZEEK_SIDE_EFFECTS}, {"get_current_packet", ATTR_NO_ZEEK_SIDE_EFFECTS}, {"get_current_packet_header", ATTR_NO_ZEEK_SIDE_EFFECTS}, + {"get_current_packet_ts", ATTR_NO_ZEEK_SIDE_EFFECTS}, {"get_dns_stats", ATTR_NO_ZEEK_SIDE_EFFECTS}, {"get_event_handler_stats", ATTR_NO_ZEEK_SIDE_EFFECTS}, {"get_event_stats", ATTR_NO_ZEEK_SIDE_EFFECTS}, diff --git a/testing/btest/Baseline.zam/opt.ZAM-bif-tracking/output b/testing/btest/Baseline/opt.ZAM-bif-tracking/output similarity index 70% rename from testing/btest/Baseline.zam/opt.ZAM-bif-tracking/output rename to testing/btest/Baseline/opt.ZAM-bif-tracking/output index 0138aa7bfe..d17a83be72 100644 --- a/testing/btest/Baseline.zam/opt.ZAM-bif-tracking/output +++ b/testing/btest/Baseline/opt.ZAM-bif-tracking/output @@ -1,2 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -540 seen BiFs, 0 unseen BiFs (), 0 new BiFs () +541 seen BiFs, 0 unseen BiFs (), 0 new BiFs () diff --git a/testing/btest/opt/ZAM-bif-tracking.zeek b/testing/btest/opt/ZAM-bif-tracking.zeek index e059f1f839..e886fb1ce0 100644 --- a/testing/btest/opt/ZAM-bif-tracking.zeek +++ b/testing/btest/opt/ZAM-bif-tracking.zeek @@ -1,5 +1,6 @@ # @TEST-DOC: ZAM maintenance script for tracking changes in BiFs. -# @TEST-REQUIRES: test "${ZEEK_ZAM}" = "1" +# +# @TEST-REQUIRES: have-spicy # # @TEST-EXEC: zeek -b %INPUT >output # @TEST-EXEC: btest-diff output @@ -299,6 +300,7 @@ global known_BiFs = set( "get_current_conn_packets_threshold", "get_current_packet", "get_current_packet_header", + "get_current_packet_ts", "get_dns_stats", "get_event_handler_stats", "get_event_stats", From 210b54799ec261c8343c5679db943e472afcb889 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Fri, 15 Nov 2024 13:54:20 +0100 Subject: [PATCH 40/65] cluster: Move publish_hrw() and publish_rr() to cluster.bif From this point on, Cluster::publish_hrw() and Cluster::publish_rr() go through cluster/Backend.cc code. --- src/broker/messaging.bif | 80 ------------------- src/cluster/BifSupport.cc | 9 +++ src/cluster/BifSupport.h | 2 + src/cluster/cluster.bif | 78 ++++++++++++++++++ .../.stderr | 4 + .../.stdout | 13 +++ .../zeromq.err | 7 ++ .../zeromq.out | 13 +++ .../generic/cluster-publish-errors.zeek | 74 +++++++++++++++++ 9 files changed, 200 insertions(+), 80 deletions(-) create mode 100644 testing/btest/Baseline/cluster.generic.cluster-publish-errors/.stderr create mode 100644 testing/btest/Baseline/cluster.generic.cluster-publish-errors/.stdout create mode 100644 testing/btest/Baseline/cluster.generic.cluster-publish-errors/zeromq.err create mode 100644 testing/btest/Baseline/cluster.generic.cluster-publish-errors/zeromq.out create mode 100644 testing/btest/cluster/generic/cluster-publish-errors.zeek diff --git a/src/broker/messaging.bif b/src/broker/messaging.bif index aba84fe344..96008861de 100644 --- a/src/broker/messaging.bif +++ b/src/broker/messaging.bif @@ -173,83 +173,3 @@ function Broker::__unsubscribe%(topic_prefix: string%): bool auto rval = zeek::broker_mgr->Unsubscribe(topic_prefix->CheckString()); return zeek::val_mgr->Bool(rval); %} - -module Cluster; - -type Cluster::Pool: record; - -## Publishes an event to a node within a pool according to Round-Robin -## distribution strategy. -## -## pool: the pool of nodes that are eligible to receive the event. -## -## key: an arbitrary string to identify the purpose for which you're -## distributing the event. e.g. consider using namespacing of your -## script like "Intel::cluster_rr_key". -## -## args: Either the event arguments as already made by -## :zeek:see:`Broker::make_event` or the argument list to pass along -## to it. -## -## Returns: true if the message is sent. -function Cluster::publish_rr%(pool: Pool, key: string, ...%): bool - %{ - static zeek::Func* topic_func = nullptr; - - if ( ! topic_func ) - topic_func = zeek::detail::global_scope()->Find("Cluster::rr_topic")->GetVal()->AsFunc(); - - if ( ! is_cluster_pool(pool) ) - { - zeek::emit_builtin_error("expected type Cluster::Pool for pool"); - return zeek::val_mgr->False(); - } - - zeek::Args vl{{zeek::NewRef{}, pool}, {zeek::NewRef{}, key}}; - auto topic = topic_func->Invoke(&vl); - - if ( ! topic->AsString()->Len() ) - return zeek::val_mgr->False(); - - auto rval = publish_event_args(ArgsSpan{*@ARGS@}.subspan(2), - topic->AsString(), frame); - return zeek::val_mgr->Bool(rval); - %} - - -## Publishes an event to a node within a pool according to Rendezvous -## (Highest Random Weight) hashing strategy. -## -## pool: the pool of nodes that are eligible to receive the event. -## -## key: data used for input to the hashing function that will uniformly -## distribute keys among available nodes. -## -## args: Either the event arguments as already made by -## :zeek:see:`Broker::make_event` or the argument list to pass along -## to it. -## -## Returns: true if the message is sent. -function Cluster::publish_hrw%(pool: Pool, key: any, ...%): bool - %{ - static zeek::Func* topic_func = nullptr; - - if ( ! topic_func ) - topic_func = zeek::detail::global_scope()->Find("Cluster::hrw_topic")->GetVal()->AsFunc(); - - if ( ! is_cluster_pool(pool) ) - { - zeek::emit_builtin_error("expected type Cluster::Pool for pool"); - return zeek::val_mgr->False(); - } - - zeek::Args vl{{zeek::NewRef{}, pool}, {zeek::NewRef{}, key}}; - auto topic = topic_func->Invoke(&vl); - - if ( ! topic->AsString()->Len() ) - return zeek::val_mgr->False(); - - auto rval = publish_event_args(ArgsSpan{*@ARGS@}.subspan(2), - topic->AsString(), frame); - return zeek::val_mgr->Bool(rval); - %} diff --git a/src/cluster/BifSupport.cc b/src/cluster/BifSupport.cc index 40e62ed29f..871a2dfc2d 100644 --- a/src/cluster/BifSupport.cc +++ b/src/cluster/BifSupport.cc @@ -136,4 +136,13 @@ zeek::ValPtr publish_event(const zeek::ValPtr& topic, zeek::ArgsSpan args) { zeek::obj_desc_short(args[0]->GetType().get()).c_str())); return zeek::val_mgr->False(); } + +bool is_cluster_pool(const zeek::Val* pool) { + static zeek::RecordTypePtr pool_type = nullptr; + + if ( ! pool_type ) + pool_type = zeek::id::find_type("Cluster::Pool"); + + return pool->GetType() == pool_type; +} } // namespace zeek::cluster::detail::bif diff --git a/src/cluster/BifSupport.h b/src/cluster/BifSupport.h index 2434482796..b02fc97bd0 100644 --- a/src/cluster/BifSupport.h +++ b/src/cluster/BifSupport.h @@ -44,6 +44,8 @@ zeek::RecordValPtr make_event(zeek::ArgsSpan args); */ zeek::ValPtr publish_event(const zeek::ValPtr& topic, zeek::ArgsSpan args); +bool is_cluster_pool(const zeek::Val* pool); + } // namespace cluster::detail::bif } // namespace zeek diff --git a/src/cluster/cluster.bif b/src/cluster/cluster.bif index cdbe5edf9d..2ec1e81c92 100644 --- a/src/cluster/cluster.bif +++ b/src/cluster/cluster.bif @@ -69,3 +69,81 @@ function Cluster::Backend::__init%(%): bool auto rval = zeek::cluster::backend->Init(); return zeek::val_mgr->Bool(rval); %} + +type Cluster::Pool: record; + +## Publishes an event to a node within a pool according to Round-Robin +## distribution strategy. +## +## pool: the pool of nodes that are eligible to receive the event. +## +## key: an arbitrary string to identify the purpose for which you're +## distributing the event. e.g. consider using namespacing of your +## script like "Intel::cluster_rr_key". +## +## args: Either the event arguments as already made by +## :zeek:see:`Cluster::make_event` or the argument list to pass along +## to it. +## +## Returns: true if the message is sent. +function Cluster::publish_rr%(pool: Pool, key: string, ...%): bool + %{ + static zeek::Func* topic_func = nullptr; + + if ( ! topic_func ) + topic_func = zeek::detail::global_scope()->Find("Cluster::rr_topic")->GetVal()->AsFunc(); + + if ( ! is_cluster_pool(pool) ) + { + zeek::emit_builtin_error("expected type Cluster::Pool for pool"); + return zeek::val_mgr->False(); + } + + zeek::Args vl{{zeek::NewRef{}, pool}, {zeek::NewRef{}, key}}; + auto topic = topic_func->Invoke(&vl); + + if ( ! topic->AsString()->Len() ) + return zeek::val_mgr->False(); + + auto args = zeek::ArgsSpan{*@ARGS@}.subspan(2); + return publish_event(topic, args); + %} + + +## Publishes an event to a node within a pool according to Rendezvous +## (Highest Random Weight) hashing strategy. +## +## pool: the pool of nodes that are eligible to receive the event. +## +## key: data used for input to the hashing function that will uniformly +## distribute keys among available nodes. +## +## args: Either the event arguments as already made by +## :zeek:see:`Broker::make_event` or the argument list to pass along +## to it. +## +## Returns: true if the message is sent. +function Cluster::publish_hrw%(pool: Pool, key: any, ...%): bool + %{ + static zeek::Func* topic_func = nullptr; + + if ( ! topic_func ) + topic_func = zeek::detail::global_scope()->Find("Cluster::hrw_topic")->GetVal()->AsFunc(); + + if ( ! is_cluster_pool(pool) ) + { + zeek::emit_builtin_error("expected type Cluster::Pool for pool"); + return zeek::val_mgr->False(); + } + + zeek::Args vl{{zeek::NewRef{}, pool}, {zeek::NewRef{}, key}}; + auto topic = topic_func->Invoke(&vl); + + if ( ! topic->AsString()->Len() ) + return zeek::val_mgr->False(); + + auto args = zeek::ArgsSpan{*@ARGS@}.subspan(2); + + ScriptLocationScope scope{frame}; + return publish_event(topic, args); + %} diff --git a/testing/btest/Baseline/cluster.generic.cluster-publish-errors/.stderr b/testing/btest/Baseline/cluster.generic.cluster-publish-errors/.stderr new file mode 100644 index 0000000000..72b4cada12 --- /dev/null +++ b/testing/btest/Baseline/cluster.generic.cluster-publish-errors/.stderr @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/cluster-publish-errors.zeek, line 58: Publish of unknown record type 'Cluster::MyEvent' (Cluster::publish(topic, Cluster::MyEvent())) +error in <...>/cluster-publish-errors.zeek, line 65: Publish of unknown record type 'Cluster::MyEvent' (Cluster::publish_hrw(Cluster::proxy_pool, key, Cluster::MyEvent())) +error in <...>/cluster-publish-errors.zeek, line 72: Publish of unknown record type 'Cluster::MyEvent' (Cluster::publish_rr(Cluster::proxy_pool, key, Cluster::MyEvent())) diff --git a/testing/btest/Baseline/cluster.generic.cluster-publish-errors/.stdout b/testing/btest/Baseline/cluster.generic.cluster-publish-errors/.stdout new file mode 100644 index 0000000000..53ade358c6 --- /dev/null +++ b/testing/btest/Baseline/cluster.generic.cluster-publish-errors/.stdout @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Broker::make_event with Cluster::publish() +r=, T +Broker::make_event with Cluster::publish_hrw() +r=, T +Broker::make_event with Cluster::publish_rr() +r=, T +Cluster::publish() with wrong event +r=, F +Cluster::publish_hrw() with wrong event +r=, F +Cluster::publish_rr() with wrong event +r=, F diff --git a/testing/btest/Baseline/cluster.generic.cluster-publish-errors/zeromq.err b/testing/btest/Baseline/cluster.generic.cluster-publish-errors/zeromq.err new file mode 100644 index 0000000000..551c87d12e --- /dev/null +++ b/testing/btest/Baseline/cluster.generic.cluster-publish-errors/zeromq.err @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/cluster-publish-errors.zeek, line 30: Publish of Broker::Event record instance with type 'Broker::Event' to a non-Broker backend (Cluster::publish(topic, Cluster::be)) +error in <...>/cluster-publish-errors.zeek, line 39: Publish of Broker::Event record instance with type 'Broker::Event' to a non-Broker backend (Cluster::publish_hrw(Cluster::proxy_pool, key, Cluster::be)) +error in <...>/cluster-publish-errors.zeek, line 47: Publish of Broker::Event record instance with type 'Broker::Event' to a non-Broker backend (Cluster::publish_rr(Cluster::proxy_pool, key, Cluster::be)) +error in <...>/cluster-publish-errors.zeek, line 58: Publish of unknown record type 'Cluster::MyEvent' (Cluster::publish(topic, Cluster::MyEvent())) +error in <...>/cluster-publish-errors.zeek, line 65: Publish of unknown record type 'Cluster::MyEvent' (Cluster::publish_hrw(Cluster::proxy_pool, key, Cluster::MyEvent())) +error in <...>/cluster-publish-errors.zeek, line 72: Publish of unknown record type 'Cluster::MyEvent' (Cluster::publish_rr(Cluster::proxy_pool, key, Cluster::MyEvent())) diff --git a/testing/btest/Baseline/cluster.generic.cluster-publish-errors/zeromq.out b/testing/btest/Baseline/cluster.generic.cluster-publish-errors/zeromq.out new file mode 100644 index 0000000000..5c368f2c83 --- /dev/null +++ b/testing/btest/Baseline/cluster.generic.cluster-publish-errors/zeromq.out @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Broker::make_event with Cluster::publish() +r=, F +Broker::make_event with Cluster::publish_hrw() +r=, F +Broker::make_event with Cluster::publish_rr() +r=, F +Cluster::publish() with wrong event +r=, F +Cluster::publish_hrw() with wrong event +r=, F +Cluster::publish_rr() with wrong event +r=, F diff --git a/testing/btest/cluster/generic/cluster-publish-errors.zeek b/testing/btest/cluster/generic/cluster-publish-errors.zeek new file mode 100644 index 0000000000..5e106012a0 --- /dev/null +++ b/testing/btest/cluster/generic/cluster-publish-errors.zeek @@ -0,0 +1,74 @@ +# @TEST-DOC: Test errors of cluster bifs +# +# @TEST-EXEC: zeek --parse-only -b %INPUT +# @TEST-EXEC: zeek -b %INPUT +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stdout + +module Cluster; + +event ping1(c: count, how: string) &is_used + { + } + +hook hook1(c: count, how: string) &is_used + { + } + + +event zeek_init() + { + # Fake the pool! + init_pool_node(Cluster::proxy_pool, "proxy-1"); + mark_pool_node_alive(Cluster::proxy_pool, "proxy-1"); + } + +event zeek_init() &priority=-1 + { + print "Broker::make_event with Cluster::publish()"; + local be = Broker::make_event(ping1, 1, "make_event()"); + local r = Cluster::publish("topic", be); + print "r=", r; + } + +event zeek_init() &priority=-2 + { + print "Broker::make_event with Cluster::publish_hrw()"; + + local be = Broker::make_event(ping1, 1, "make_event()"); + local r = Cluster::publish_hrw(Cluster::proxy_pool, "key", be); + print "r=", r; + } + +event zeek_init() &priority=-3 + { + print "Broker::make_event with Cluster::publish_rr()"; + local be = Broker::make_event(ping1, 1, "make_event()"); + local r = Cluster::publish_rr(Cluster::proxy_pool, "key", be); + print "r=", r; + } + +type MyEvent: record { + x: count &default=1; +}; + +event zeek_init() &priority=-4 + { + print "Cluster::publish() with wrong event"; + local r = Cluster::publish("topic", MyEvent()); + print "r=", r; + } + +event zeek_init() &priority=-4 + { + print "Cluster::publish_hrw() with wrong event"; + local r = Cluster::publish_hrw(Cluster::proxy_pool, "key", MyEvent()); + print "r=", r; + } + +event zeek_init() &priority=-4 + { + print "Cluster::publish_rr() with wrong event"; + local r = Cluster::publish_rr(Cluster::proxy_pool, "key", MyEvent()); + print "r=", r; + } From a2249f7ecb94d22bfa4fe5dd936fca0e835911ba Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Fri, 15 Nov 2024 11:35:48 +0100 Subject: [PATCH 41/65] cluster: Add Cluster::node_id(), allow redef of node_topic(), nodeid_topic() This provides a way for non-broker cluster backends to override a node's identifier and its own topics that it listens on by default. --- scripts/base/frameworks/cluster/main.zeek | 13 ++++++++++--- .../base/frameworks/cluster/setup-connections.zeek | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/scripts/base/frameworks/cluster/main.zeek b/scripts/base/frameworks/cluster/main.zeek index 0427d6adcd..d56b4f5655 100644 --- a/scripts/base/frameworks/cluster/main.zeek +++ b/scripts/base/frameworks/cluster/main.zeek @@ -242,6 +242,13 @@ export { ## of the cluster that is started up. const node = getenv("CLUSTER_NODE") &redef; + ## Function returning this node's identifier. + ## + ## By default this is :zeek:see:`Broker::node_id`, but can be + ## redefined by other cluster backends. This identifier should be + ## a short lived identifier that resets when a node is restarted. + global node_id: function(): string = Broker::node_id &redef; + ## Interval for retrying failed connections between cluster nodes. ## If set, the ZEEK_DEFAULT_CONNECT_RETRY (given in number of seconds) ## environment variable overrides this option. @@ -270,7 +277,7 @@ export { ## ## Returns: a topic string that may used to send a message exclusively to ## a given cluster node. - global node_topic: function(name: string): string; + global node_topic: function(name: string): string &redef; ## Retrieve the topic associated with a specific node in the cluster. ## @@ -279,7 +286,7 @@ export { ## ## Returns: a topic string that may used to send a message exclusively to ## a given cluster node. - global nodeid_topic: function(id: string): string; + global nodeid_topic: function(id: string): string &redef; ## Retrieve the cluster-level naming of a node based on its node ID, ## a backend-specific identifier. @@ -446,7 +453,7 @@ event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) &priority= if ( ! Cluster::is_enabled() ) return; - local e = Broker::make_event(Cluster::hello, node, Broker::node_id()); + local e = Broker::make_event(Cluster::hello, node, Cluster::node_id()); Broker::publish(nodeid_topic(endpoint$id), e); } diff --git a/scripts/base/frameworks/cluster/setup-connections.zeek b/scripts/base/frameworks/cluster/setup-connections.zeek index 453658067f..93d4504a6f 100644 --- a/scripts/base/frameworks/cluster/setup-connections.zeek +++ b/scripts/base/frameworks/cluster/setup-connections.zeek @@ -94,7 +94,7 @@ event zeek_init() &priority=-10 return; } - Cluster::subscribe(nodeid_topic(Broker::node_id())); + Cluster::subscribe(nodeid_topic(Cluster::node_id())); Cluster::subscribe(node_topic(node)); From 889c7d888aad0357d17be392586fa0ce95ffdb95 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Thu, 14 Nov 2024 22:14:35 +0100 Subject: [PATCH 42/65] cluster/backend/zeromq: Add cppzmq submodule Not all supported platforms provide a recent enough cppzmq version, add a fallback as submodule. cppzmq is a header-only library, so there's no build step involved. --- .gitmodules | 3 +++ src/cluster/backend/zeromq/auxil/cppzmq | 1 + 2 files changed, 4 insertions(+) create mode 160000 src/cluster/backend/zeromq/auxil/cppzmq diff --git a/.gitmodules b/.gitmodules index 38e0606337..43d56e17f4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -76,3 +76,6 @@ [submodule "auxil/prometheus-cpp"] path = auxil/prometheus-cpp url = https://github.com/zeek/prometheus-cpp +[submodule "src/cluster/backend/zeromq/auxil/cppzmq"] + path = src/cluster/backend/zeromq/auxil/cppzmq + url = https://github.com/zeromq/cppzmq diff --git a/src/cluster/backend/zeromq/auxil/cppzmq b/src/cluster/backend/zeromq/auxil/cppzmq new file mode 160000 index 0000000000..c94c20743e --- /dev/null +++ b/src/cluster/backend/zeromq/auxil/cppzmq @@ -0,0 +1 @@ +Subproject commit c94c20743ed7d4aa37835a5c46567ab0790d4acc From 35c79ab2e3c363e690e0305383f97e7f0b9ded0e Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Thu, 14 Nov 2024 22:15:29 +0100 Subject: [PATCH 43/65] cluster/backend/zeromq: Add ZeroMQ based cluster backend This is a cluster backend implementation using a central XPUB/XSUB proxy that by default runs on the manager node. Logging is implemented leveraging PUSH/PULL sockets between logger and other nodes, rather than going through XPUB/XSUB. The test-all-policy-cluster baseline changed: Previously, Broker::peer() would be called from setup-connections.zeek, causing the IO loop to be alive. With the ZeroMQ backend, the IO loop is only alive when Cluster::init() is called, but that doesn't happen anymore. --- .../cluster/backend/zeromq/__load__.zeek | 1 + .../cluster/backend/zeromq/connect.zeek | 14 + .../cluster/backend/zeromq/main.zeek | 424 +++++++++++++ scripts/test-all-policy.zeek | 3 + scripts/zeekygen/__load__.zeek | 2 + src/cluster/CMakeLists.txt | 1 + src/cluster/backend/CMakeLists.txt | 4 + src/cluster/backend/zeromq/CMakeLists.txt | 21 + src/cluster/backend/zeromq/Plugin.cc | 22 + src/cluster/backend/zeromq/Plugin.h | 14 + src/cluster/backend/zeromq/ZeroMQ-Proxy.cc | 72 +++ src/cluster/backend/zeromq/ZeroMQ-Proxy.h | 56 ++ src/cluster/backend/zeromq/ZeroMQ.cc | 569 ++++++++++++++++++ src/cluster/backend/zeromq/ZeroMQ.h | 99 +++ .../backend/zeromq/cluster_backend_zeromq.bif | 16 + .../backend/zeromq/cmake/FindZeroMQ.cmake | 52 ++ .../cluster.log.normalized | 21 + .../cluster.zeromq.logging/manager.out | 16 + .../cluster.zeromq.logging/node_up.sorted | 21 + .../cluster.log.normalized | 13 + .../manager.out | 11 + .../node_up.sorted | 13 + .../supervisor.cluster.log | 21 + .../cluster.zeromq.two-nodes/..manager.out | 3 + .../cluster.zeromq.two-nodes/..worker.out | 2 + .../canonified_loaded_scripts.log | 1 + .../canonified_loaded_scripts.log | 1 + .../coverage.test-all-policy-cluster/.stderr | 4 - testing/btest/Baseline/plugins.hooks/output | 6 + .../zeromq/cluster-layout-no-logger.zeek | 8 + .../Files/zeromq/cluster-layout-simple.zeek | 9 + .../zeromq/cluster-layout-two-loggers.zeek | 10 + .../btest/Files/zeromq/test-bootstrap.zeek | 11 + testing/btest/cluster/zeromq/logging.zeek | 139 +++++ .../cluster/zeromq/manager-is-logger.zeek | 129 ++++ testing/btest/cluster/zeromq/supervisor.zeek | 86 +++ testing/btest/cluster/zeromq/two-nodes.zeek | 53 ++ testing/scripts/have-zeromq | 4 + 38 files changed, 1948 insertions(+), 4 deletions(-) create mode 100644 scripts/policy/frameworks/cluster/backend/zeromq/__load__.zeek create mode 100644 scripts/policy/frameworks/cluster/backend/zeromq/connect.zeek create mode 100644 scripts/policy/frameworks/cluster/backend/zeromq/main.zeek create mode 100644 src/cluster/backend/CMakeLists.txt create mode 100644 src/cluster/backend/zeromq/CMakeLists.txt create mode 100644 src/cluster/backend/zeromq/Plugin.cc create mode 100644 src/cluster/backend/zeromq/Plugin.h create mode 100644 src/cluster/backend/zeromq/ZeroMQ-Proxy.cc create mode 100644 src/cluster/backend/zeromq/ZeroMQ-Proxy.h create mode 100644 src/cluster/backend/zeromq/ZeroMQ.cc create mode 100644 src/cluster/backend/zeromq/ZeroMQ.h create mode 100644 src/cluster/backend/zeromq/cluster_backend_zeromq.bif create mode 100644 src/cluster/backend/zeromq/cmake/FindZeroMQ.cmake create mode 100644 testing/btest/Baseline/cluster.zeromq.logging/cluster.log.normalized create mode 100644 testing/btest/Baseline/cluster.zeromq.logging/manager.out create mode 100644 testing/btest/Baseline/cluster.zeromq.logging/node_up.sorted create mode 100644 testing/btest/Baseline/cluster.zeromq.manager-is-logger/cluster.log.normalized create mode 100644 testing/btest/Baseline/cluster.zeromq.manager-is-logger/manager.out create mode 100644 testing/btest/Baseline/cluster.zeromq.manager-is-logger/node_up.sorted create mode 100644 testing/btest/Baseline/cluster.zeromq.supervisor/supervisor.cluster.log create mode 100644 testing/btest/Baseline/cluster.zeromq.two-nodes/..manager.out create mode 100644 testing/btest/Baseline/cluster.zeromq.two-nodes/..worker.out create mode 100644 testing/btest/Files/zeromq/cluster-layout-no-logger.zeek create mode 100644 testing/btest/Files/zeromq/cluster-layout-simple.zeek create mode 100644 testing/btest/Files/zeromq/cluster-layout-two-loggers.zeek create mode 100644 testing/btest/Files/zeromq/test-bootstrap.zeek create mode 100644 testing/btest/cluster/zeromq/logging.zeek create mode 100644 testing/btest/cluster/zeromq/manager-is-logger.zeek create mode 100644 testing/btest/cluster/zeromq/supervisor.zeek create mode 100644 testing/btest/cluster/zeromq/two-nodes.zeek create mode 100755 testing/scripts/have-zeromq diff --git a/scripts/policy/frameworks/cluster/backend/zeromq/__load__.zeek b/scripts/policy/frameworks/cluster/backend/zeromq/__load__.zeek new file mode 100644 index 0000000000..5f39cb27df --- /dev/null +++ b/scripts/policy/frameworks/cluster/backend/zeromq/__load__.zeek @@ -0,0 +1 @@ +@load ./main.zeek diff --git a/scripts/policy/frameworks/cluster/backend/zeromq/connect.zeek b/scripts/policy/frameworks/cluster/backend/zeromq/connect.zeek new file mode 100644 index 0000000000..94aee459ae --- /dev/null +++ b/scripts/policy/frameworks/cluster/backend/zeromq/connect.zeek @@ -0,0 +1,14 @@ +##! Establish ZeroMQ connectivity with the broker. + +@load ./main + +module Cluster::Backend::ZeroMQ; + + +event zeek_init() &priority=10 + { + if ( run_proxy_thread ) + Cluster::Backend::ZeroMQ::spawn_zmq_proxy_thread(); + + Cluster::init(); + } diff --git a/scripts/policy/frameworks/cluster/backend/zeromq/main.zeek b/scripts/policy/frameworks/cluster/backend/zeromq/main.zeek new file mode 100644 index 0000000000..52e8bff74b --- /dev/null +++ b/scripts/policy/frameworks/cluster/backend/zeromq/main.zeek @@ -0,0 +1,424 @@ +##! ZeroMQ cluster backend support. +##! +##! For publish-subscribe functionality, one node in the Zeek cluster spawns a +##! thread running a central broker listening on a XPUB and XSUB socket. +##! These sockets are connected via `zmq_proxy() `_. +##! All other nodes connect to this central broker with their own XSUB and +##! XPUB sockets, establishing a global many-to-many publish-subscribe system +##! where each node sees subscriptions and messages from all other nodes in a +##! Zeek cluster. ZeroMQ's `publish-subscribe pattern `_ +##! documentation may be a good starting point. Elsewhere in ZeroMQ's documentation, +##! the central broker is also called `forwarder `_. +##! +##! For remote logging functionality, the ZeroMQ `pipeline pattern `_ +##! is used. All logger nodes listen on a PULL socket. Other nodes connect +##! via PUSH sockets to all of the loggers. Concretely, remote logging +##! functionality is not publish-subscribe, but instead leverages ZeroMQ's +##! built-in load-balancing functionality provided by PUSH and PULL +##! sockets. +##! +##! The ZeroMQ cluster backend technically allows to run a non-Zeek central +##! broker (it only needs to offer XPUB and XSUB sockets). Further, it is +##! possible to run non-Zeek logger nodes. All a logger node needs to do is +##! open a ZeroMQ PULL socket and interpret the format used by Zeek nodes +##! to send their log writes. +module Cluster::Backend::ZeroMQ; + +export { + ## The central broker's XPUB endpoint to connect to. + ## + ## A node connects with its XSUB socket to the XPUB socket + ## of the central broker. + const connect_xpub_endpoint = "tcp://127.0.0.1:5556" &redef; + + + ## The central broker's XSUB endpoint to connect to. + ## + ## A node connects with its XPUB socket to the XSUB socket + ## of the central broker. + const connect_xsub_endpoint = "tcp://127.0.0.1:5555" &redef; + + ## Vector of ZeroMQ endpoints to connect to for logging. + ## + ## A node's PUSH socket used for logging connects to each + ## of the ZeroMQ endpoints listed in this vector. + const connect_log_endpoints: vector of string &redef; + + ## Toggle for running a central ZeroMQ XPUB-XSUB broker on this node. + ## + ## If set to ``T``, :zeek:see:`Cluster::Backend::ZeroMQ::spawn_zmq_proxy_thread` + ## is called during :zeek:see:`zeek_init`. The node will listen + ## on :zeek:see:`Cluster::Backend::ZeroMQ::listen_xsub_endpoint` and + ## :zeek:see:`Cluster::Backend::ZeroMQ::listen_xpub_endpoint` and + ## forward subscriptions and messages between nodes. + ## + ## By default, this is set to ``T`` on the manager and ``F`` elsewhere. + const run_proxy_thread: bool = F &redef; + + ## XSUB listen endpoint for the central broker. + ## + ## This setting is used for the XSUB socket of the central broker started + ## when :zeek:see:`Cluster::Backend::ZeroMQ::run_proxy_thread` is ``T``. + const listen_xsub_endpoint = "tcp://127.0.0.1:5556" &redef; + + ## XPUB listen endpoint for the central broker. + ## + ## This setting is used for the XPUB socket of the central broker started + ## when :zeek:see:`Cluster::Backend::ZeroMQ::run_proxy_thread` is ``T``. + const listen_xpub_endpoint = "tcp://127.0.0.1:5555" &redef; + + ## PULL socket address to listen on for log messages. + ## + ## If empty, don't listen for log messages, otherwise + ## a ZeroMQ address to bind to. E.g., ``tcp://127.0.0.1:5555``. + const listen_log_endpoint = "" &redef; + + ## Configure the ZeroMQ's sockets linger value. + ## + ## The default used by libzmq is 30 seconds (30 000) which is very long + ## when loggers vanish before workers during a shutdown, so we reduce + ## this to 500 milliseconds by default. + ## + ## A value of ``-1`` configures blocking forever, while ``0`` would + ## immediately discard any pending messages. + ## + ## See ZeroMQ's `ZMQ_LINGER documentation `_ + ## for more details. + const linger_ms: int = 500 &redef; + + ## Configure ZeroMQ's immedidate setting on PUSH sockets + ## + ## Setting this to ``T`` will queue log writes only to completed + ## connections. By default, log writes are queued to all potential + ## endpoints listed in :zeek:see:`Cluster::Backend::ZeroMQ::connect_log_endpoints`. + ## + ## See ZeroMQ's `ZMQ_IMMEDIATE documentation `_ + ## for more details. + const log_immediate: bool = F &redef; + + ## Send high water mark value for the log PUSH sockets. + ## + ## If reached, Zeek nodes will block or drop messages. + ## + ## See ZeroMQ's `ZMQ_SNDHWM documentation `_ + ## for more details. + ## + ## TODO: Make action configurable (block vs drop) + const log_sndhwm: int = 1000 &redef; + + ## Receive high water mark value for the log PULL sockets. + ## + ## If reached, Zeek workers will block or drop messages. + ## + ## See ZeroMQ's `ZMQ_RCVHWM documentation `_ + ## for more details. + ## + ## TODO: Make action configurable (block vs drop) + const log_rcvhwm: int = 1000 &redef; + + ## Kernel transmit buffer size for log sockets. + ## + ## Using -1 will use the kernel's default. + ## + ## See ZeroMQ's `ZMQ_SNDBUF documentation `_. + const log_sndbuf: int = -1 &redef; + + ## Kernel receive buffer size for log sockets. + ## + ## Using -1 will use the kernel's default. + ## + ## See ZeroMQ's `ZMQ_RCVBUF documentation `_ + ## for more details. + const log_rcvbuf: int = -1 &redef; + + ## Do not silently drop messages if high-water-mark is reached. + ## + ## Whether to configure ``ZMQ_XPUB_NODROP`` on the XPUB socket + ## to detect when sending a message fails due to reaching + ## the high-water-mark. + ## + ## See ZeroMQ's `ZMQ_XPUB_NODROP documentation `_ + ## for more details. + const xpub_nodrop: bool = T &redef; + + ## Do not silently drop messages if high-water-mark is reached. + ## + ## Whether to configure ``ZMQ_XPUB_NODROP`` on the XPUB socket + ## to detect when sending a message fails due to reaching + ## the high-water-mark. + ## + ## This setting applies to the XPUB/XSUB broker started when + ## :zeek:see:`Cluster::Backend::ZeroMQ::run_proxy_thread` is ``T``. + ## + ## See ZeroMQ's `ZMQ_XPUB_NODROP documentation `_ + ## for more details. + const listen_xpub_nodrop: bool = T &redef; + + ## Messages to receive before yielding. + ## + ## Yield from the receive loop when this many messages have been + ## received from one of the used sockets. + const poll_max_messages = 100 &redef; + + ## Bitmask to enable low-level stderr based debug printing. + ## + ## poll debugging: 1 (produce verbose zmq::poll() output) + ## + ## Or values from the above list together and set debug_flags + ## to the result. E.g. use 7 to select 4, 2 and 1. Only use this + ## in development if something seems off. The thread used internally + ## will produce output on stderr. + const debug_flags: count = 0 &redef; + + ## The node topic prefix to use. + global node_topic_prefix = "zeek.cluster.node" &redef; + + ## The node_id topic prefix to use. + global nodeid_topic_prefix = "zeek.cluster.nodeid" &redef; + + ## Low-level event when a subscription is added. + ## + ## Every node observes all subscriptions from other nodes + ## in a cluster through its XPUB socket. Whenever a new + ## subscription topic is added, this event is raised with + ## the topic. + ## + ## topic: The topic. + global subscription: event(topic: string); + + ## Low-level event when a subscription vanishes. + ## + ## Every node observes all subscriptions from other nodes + ## in a cluster through its XPUB socket. Whenever a subscription + ## is removed from the local XPUB socket, this event is raised + ## with the topic set to the removed subscription. + ## + ## topic: The topic. + global unsubscription: event(topic: string); + + ## Low-level event send to a node in response to their subscription. + ## + ## name: The sending node's name in :zeek:see:`Cluster::nodes`. + ## + ## id: The sending node's identifier, as generated by :zeek:see:`Cluster::node_id`. + global hello: event(name: string, id: string); + + ## Expiration for hello state. + ## + ## How long to wait before expiring information about + ## subscriptions and hello messages from other + ## nodes. These expirations trigger reporter warnings. + const hello_expiration: interval = 10sec &redef; +} + +redef Cluster::backend = Cluster::CLUSTER_BACKEND_ZEROMQ; + +# By default, let the manager node run the proxy thread. +redef run_proxy_thread = Cluster::local_node_type() == Cluster::MANAGER; + +function zeromq_node_topic(name: string): string { + return node_topic_prefix + "." + name; +} + +function zeromq_nodeid_topic(id: string): string { + return nodeid_topic_prefix + "." + id; +} + +# Unique identifier for this node with some debug information. +const my_node_id = fmt("zeromq_%s_%s_%s_%s", Cluster::node, gethostname(), getpid(), unique_id("N")); + +function zeromq_node_id(): string { + return my_node_id; +} + +redef Cluster::node_topic = zeromq_node_topic; +redef Cluster::nodeid_topic = zeromq_nodeid_topic; +redef Cluster::node_id = zeromq_node_id; + +redef Cluster::logger_topic = "zeek.cluster.logger"; +redef Cluster::manager_topic = "zeek.cluster.manager"; +redef Cluster::proxy_topic = "zeek.cluster.proxy"; +redef Cluster::worker_topic = "zeek.cluster.worker"; + +redef Cluster::proxy_pool_spec = Cluster::PoolSpec( + $topic = "zeek.cluster.pool.proxy", + $node_type = Cluster::PROXY); + +redef Cluster::logger_pool_spec = Cluster::PoolSpec( + $topic = "zeek.cluster.pool.logger", + $node_type = Cluster::LOGGER); + +redef Cluster::worker_pool_spec = Cluster::PoolSpec( + $topic = "zeek.cluster.pool.worker", + $node_type = Cluster::WORKER); + + +# Configure listen_log_endpoint based on port in cluster-layout, if any. +@if ( Cluster::local_node_type() == Cluster::LOGGER || (Cluster::manager_is_logger && Cluster::local_node_type() == Cluster::MANAGER) ) +const my_node = Cluster::nodes[Cluster::node]; +@if ( my_node?$p ) +redef listen_log_endpoint = fmt("tcp://%s:%s", my_node$ip, port_to_count(my_node$p)); +@endif +@endif + +# Populate connect_log_endpoints based on Cluster::nodes on non-logger nodes. +# If you're experimenting with zero-logger clusters, ignore this code and set +# connect_log_endpoints yourself via redef. +event zeek_init() &priority=100 + { + if ( Cluster::local_node_type() == Cluster::LOGGER ) + return; + + if ( Cluster::manager_is_logger && Cluster::local_node_type() == Cluster::MANAGER ) + return; + + for ( _, node in Cluster::nodes ) + { + local endp: string; + if ( node$node_type == Cluster::LOGGER && node?$p ) + { + endp = fmt("tcp://%s:%s", node$ip, port_to_count(node$p)); + connect_log_endpoints += endp; + } + + if ( Cluster::manager_is_logger && node$node_type == Cluster::MANAGER && node?$p ) + { + endp = fmt("tcp://%s:%s", node$ip, port_to_count(node$p)); + connect_log_endpoints += endp; + } + } + + # If there's no endpoints configured, but more than a single + # node in cluster layout, log an error as that's probably not + # an intended configuration. + if ( |connect_log_endpoints| == 0 && |Cluster::nodes| > 1 ) + Reporter::error("No ZeroMQ connect_log_endpoints configured"); + } + +function nodeid_subscription_expired(nodeids: set[string], nodeid: string): interval + { + Reporter::warning(fmt("Expired subscription from nodeid %s", nodeid)); + return 0.0sec; + } + +function nodeid_hello_expired(nodeids: set[string], nodeid: string): interval + { + Reporter::warning(fmt("Expired hello from nodeid %s", nodeid)); + return 0.0sec; + } + +# State about subscriptions and hellos seen from other nodes. +global nodeid_subscriptions: set[string] &create_expire=hello_expiration &expire_func=nodeid_subscription_expired; +global nodeid_hellos: set[string] &create_expire=hello_expiration &expire_func=nodeid_hello_expired; + +# The ZeroMQ plugin notifies script land when a new subscription arrived +# on that node's XPUB socket. If the topic of such a subscription starts with +# the nodeid_topic_prefix for another node A, node B seeing the subscription +# sends ZeroMQ::hello() to the topic, announcing its own presence to node A. +# Conversely, when node A sees the subscription for node B's nodeid topic, +# it also sens ZeroMQ::hello(). In other words, every node says hello to all +# other nodes based on subscriptions they observe on their local XPUB sockets. +# +# Once node B has seen both, the nodeid topic subscription and ZeroMQ::hello() +# event from node A, it raises a Cluster::node_up() event for node A. +# +# See also the Cluster::Backend::ZeroMQ::hello() handler below. +# +# 1) node A subscribes to Cluster::nodeid_topic(Cluster::node_id()) +# 2) node B observes subscription for node A's nodeid_topic and replies with ZeroMQ::hello() +# 3) node A receives node B's nodeid_topic subscription, replies with ZeroMQ::hello() +# 4) node B receives node A's ZeroMQ::hello() and raises Cluster::node_up() +# as it has already seen node A's nodeid_topic subscription. +event Cluster::Backend::ZeroMQ::subscription(topic: string) + { + local prefix = nodeid_topic_prefix + "."; + + if ( ! starts_with(topic, prefix) ) + return; + + local nodeid = topic[|prefix|:]; + + # Do not say hello to ourselves - we won't see it anyhow. + if ( nodeid == Cluster::node_id() ) + return; + + Cluster::publish(topic, Cluster::Backend::ZeroMQ::hello, Cluster::node, Cluster::node_id()); + + # If we saw a ZeroMQ::hello from the other node already, send + # it a Cluster::hello. + if ( nodeid in nodeid_hellos ) + { + Cluster::publish(Cluster::nodeid_topic(nodeid), Cluster::hello, Cluster::node, Cluster::node_id()); + delete nodeid_hellos[nodeid]; + } + else + { + add nodeid_subscriptions[nodeid]; + } + } + +# Receiving ZeroMQ::hello() from another node: If we received a subscription +# for the node's nodeid_topic, reply with a Cluster::hello. If the node never +# properly went away, log a warning and raise a Cluster::node_down() now. +event Cluster::Backend::ZeroMQ::hello(name: string, id: string) + { + if ( name in Cluster::nodes ) + { + local n = Cluster::nodes[name]; + if ( n?$id ) + { + if ( n$id == id ) + { + # Duplicate ZeroMQ::hello(), very strange, ignore it. + Reporter::warning(fmt("node '%s' sends ZeroMQ::hello twice (id:%s)", + name, id)); + return; + } + + Reporter::warning(fmt("node '%s' never said goodbye (old id:%s new id:%s", + name, n$id, id)); + + # We raise node_down() here for the old instance, + # but it's obviously fake and somewhat lying. + event Cluster::node_down(name, n$id); + } + } + + # It is possible to publish Cluster::hello() directly if the nodeid_topic + # subscription for the other node was already seen. Otherwise, remember + # that Cluster::hello() has been seen and send Cluster::hello() in + # subscription processing further up. + if ( id in nodeid_subscriptions ) + { + Cluster::publish(Cluster::nodeid_topic(id), Cluster::hello, Cluster::node, Cluster::node_id()); + delete nodeid_subscriptions[id]; + } + else + { + add nodeid_hellos[id]; + } + } + +# If the unsubscription is for a nodeid prefix, extract the +# nodeid that's gone, find the name of the node from the +# cluster layout and raise Cluster::node_down(). +event Cluster::Backend::ZeroMQ::unsubscription(topic: string) + { + local prefix = nodeid_topic_prefix + "."; + if ( ! starts_with(topic, prefix) ) + return; + + local gone_node_id = topic[|prefix|:]; + local name = ""; + for ( node_name, n in Cluster::nodes ) { + if ( n?$id && n$id == gone_node_id ) { + name = node_name; + break; + } + } + + if ( name != "" ) + event Cluster::node_down(name, gone_node_id); + else + Reporter::warning(fmt("unsubscription of unknown node with id '%s'", gone_node_id)); + } diff --git a/scripts/test-all-policy.zeek b/scripts/test-all-policy.zeek index b515dd234b..58930dc194 100644 --- a/scripts/test-all-policy.zeek +++ b/scripts/test-all-policy.zeek @@ -11,6 +11,9 @@ # @load frameworks/control/controllee.zeek # @load frameworks/control/controller.zeek +@load frameworks/cluster/backend/zeromq/__load__.zeek +# @load frameworks/cluster/backend/zeromq/connect.zeek +@load frameworks/cluster/backend/zeromq/main.zeek @load frameworks/cluster/experimental.zeek # Loaded via the above through test-all-policy-cluster.test # when running as a manager, creates cluster.log entries diff --git a/scripts/zeekygen/__load__.zeek b/scripts/zeekygen/__load__.zeek index 5cda9d6263..d22dba2a97 100644 --- a/scripts/zeekygen/__load__.zeek +++ b/scripts/zeekygen/__load__.zeek @@ -2,6 +2,7 @@ # Scripts which are commented out in test-all-policy.zeek. @load protocols/ssl/decryption.zeek +@load frameworks/cluster/backend/zeromq/connect.zeek @load frameworks/cluster/nodes-experimental/manager.zeek @load frameworks/control/controllee.zeek @load frameworks/control/controller.zeek @@ -28,6 +29,7 @@ event zeek_init() &priority=1000 # fail when run under zeekygen. For the purpose of zeekygen, we could # probably disable all modules, too. disable_module_events("Control"); + disable_module_events("Cluster::Backend::ZeroMQ"); disable_module_events("Management::Agent::Runtime"); disable_module_events("Management::Controller::Runtime"); disable_module_events("Management::Node"); diff --git a/src/cluster/CMakeLists.txt b/src/cluster/CMakeLists.txt index 14a063943d..bb421c6c01 100644 --- a/src/cluster/CMakeLists.txt +++ b/src/cluster/CMakeLists.txt @@ -11,4 +11,5 @@ zeek_add_subdir_library( BIFS cluster.bif) +add_subdirectory(backend) add_subdirectory(serializer) diff --git a/src/cluster/backend/CMakeLists.txt b/src/cluster/backend/CMakeLists.txt new file mode 100644 index 0000000000..0e5d704186 --- /dev/null +++ b/src/cluster/backend/CMakeLists.txt @@ -0,0 +1,4 @@ +option(ENABLE_CLUSTER_BACKEND_ZEROMQ "Enable the ZeroMQ cluster backend" ON) +if (ENABLE_CLUSTER_BACKEND_ZEROMQ) + add_subdirectory(zeromq) +endif () diff --git a/src/cluster/backend/zeromq/CMakeLists.txt b/src/cluster/backend/zeromq/CMakeLists.txt new file mode 100644 index 0000000000..a15923445a --- /dev/null +++ b/src/cluster/backend/zeromq/CMakeLists.txt @@ -0,0 +1,21 @@ +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") + +find_package(ZeroMQ REQUIRED) + +message(STATUS "zeromq: ${ZeroMQ_LIBRARIES} ${ZeroMQ_INCLUDE_DIRS}") + +zeek_add_plugin( + Zeek + Cluster_Backend_ZeroMQ + INCLUDE_DIRS + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} + ${ZeroMQ_INCLUDE_DIRS} + DEPENDENCIES + ${ZeroMQ_LIBRARIES} + SOURCES + Plugin.cc + ZeroMQ-Proxy.cc + ZeroMQ.cc + BIFS + cluster_backend_zeromq.bif) diff --git a/src/cluster/backend/zeromq/Plugin.cc b/src/cluster/backend/zeromq/Plugin.cc new file mode 100644 index 0000000000..ca823f0e54 --- /dev/null +++ b/src/cluster/backend/zeromq/Plugin.cc @@ -0,0 +1,22 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/cluster/backend/zeromq/Plugin.h" + +#include "zeek/cluster/Component.h" +#include "zeek/cluster/backend/zeromq/ZeroMQ.h" + + +namespace zeek::plugin::Zeek_Cluster_Backend_ZeroMQ { + +Plugin plugin; + +zeek::plugin::Configuration Plugin::Configure() { + AddComponent(new cluster::BackendComponent("ZeroMQ", zeek::cluster::zeromq::ZeroMQBackend::Instantiate)); + + zeek::plugin::Configuration config; + config.name = "Zeek::Cluster_Backend_ZeroMQ"; + config.description = "Cluster backend using ZeroMQ"; + return config; +} + +} // namespace zeek::plugin::Zeek_Cluster_Backend_ZeroMQ diff --git a/src/cluster/backend/zeromq/Plugin.h b/src/cluster/backend/zeromq/Plugin.h new file mode 100644 index 0000000000..882a7dc9ec --- /dev/null +++ b/src/cluster/backend/zeromq/Plugin.h @@ -0,0 +1,14 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include "zeek/plugin/Plugin.h" + +namespace zeek::plugin::Zeek_Cluster_Backend_ZeroMQ { + +class Plugin : public zeek::plugin::Plugin { +public: + zeek::plugin::Configuration Configure() override; +}; + +} // namespace zeek::plugin::Zeek_Cluster_Backend_ZeroMQ diff --git a/src/cluster/backend/zeromq/ZeroMQ-Proxy.cc b/src/cluster/backend/zeromq/ZeroMQ-Proxy.cc new file mode 100644 index 0000000000..3dae7639cd --- /dev/null +++ b/src/cluster/backend/zeromq/ZeroMQ-Proxy.cc @@ -0,0 +1,72 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/cluster/backend/zeromq/ZeroMQ-Proxy.h" + +#include + +#include "zeek/Reporter.h" +#include "zeek/util.h" + + +using namespace zeek::cluster::zeromq; + +namespace { + +/** + * Function that runs zmq_proxy() that provides a central XPUB/XSUB + * broker for other Zeek nodes to connect and exchange subscription + * information. + */ +void thread_fun(ProxyThread::Args* args) { + zeek::util::detail::set_thread_name("zmq-proxy-thread"); + + try { + zmq::proxy(args->xsub, args->xpub, zmq::socket_ref{} /*capture*/); + } catch ( zmq::error_t& err ) { + args->xsub.close(); + args->xpub.close(); + + if ( err.num() != ETERM ) { + std::fprintf(stderr, "[zeromq] unexpected zmq_proxy() error: %s (%d)", err.what(), err.num()); + throw; + } + } +} + +} // namespace + +bool ProxyThread::Start() { + zmq::socket_t xpub(ctx, zmq::socket_type::xpub); + zmq::socket_t xsub(ctx, zmq::socket_type::xsub); + + xpub.set(zmq::sockopt::xpub_nodrop, xpub_nodrop); + + try { + xpub.bind(xpub_endpoint); + } catch ( zmq::error_t& err ) { + zeek::reporter->Error("Failed to bind xpub socket %s: %s (%d)", xpub_endpoint.c_str(), err.what(), err.num()); + return false; + } + + try { + xsub.bind(xsub_endpoint); + } catch ( zmq::error_t& err ) { + zeek::reporter->Error("Failed to bind xsub socket %s: %s (%d)", xpub_endpoint.c_str(), err.what(), err.num()); + return false; + } + + args = {.xpub = std::move(xpub), .xsub = std::move(xsub)}; + + thread = std::thread(thread_fun, &args); + + return true; +} + +void ProxyThread::Shutdown() { + ctx.shutdown(); + + if ( thread.joinable() ) + thread.join(); + + ctx.close(); +} diff --git a/src/cluster/backend/zeromq/ZeroMQ-Proxy.h b/src/cluster/backend/zeromq/ZeroMQ-Proxy.h new file mode 100644 index 0000000000..de33d3da1c --- /dev/null +++ b/src/cluster/backend/zeromq/ZeroMQ-Proxy.h @@ -0,0 +1,56 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include +#include +#include + + +// Central XPUB/XSUB proxy. +// +// Spawns a thread that runs zmq_proxy() for a XPUB/XSUB pair. +namespace zeek::cluster::zeromq { + +class ProxyThread { +public: + /** + * Constructor. + * + * @param xpub_endpoint the XPUB socket address to listen on. + * @param xsub_endpoint the XSUB socket address to listen on. + * @param xpub_nodrop the xpub_nodrop option to use on the XPUB socket. + */ + ProxyThread(std::string xpub_endpoint, std::string xsub_endpoint, int xpub_nodrop) + : xpub_endpoint(std::move(xpub_endpoint)), xsub_endpoint(std::move(xsub_endpoint)), xpub_nodrop(xpub_nodrop) {} + + + ~ProxyThread() { Shutdown(); } + + /** + * Data kept in object and passed to thread. + */ + struct Args { + zmq::socket_t xpub; + zmq::socket_t xsub; + }; + + /** + * Bind the sockets and spawn the thread. + */ + bool Start(); + + /** + * Shutdown the ZeroMQ context and join the thread. + */ + void Shutdown(); + +private: + zmq::context_t ctx; + std::thread thread; + Args args; + std::string xpub_endpoint; + std::string xsub_endpoint; + int xpub_nodrop = 1; +}; +} // namespace zeek::cluster::zeromq diff --git a/src/cluster/backend/zeromq/ZeroMQ.cc b/src/cluster/backend/zeromq/ZeroMQ.cc new file mode 100644 index 0000000000..02fd795bbf --- /dev/null +++ b/src/cluster/backend/zeromq/ZeroMQ.cc @@ -0,0 +1,569 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "ZeroMQ.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "zeek/DebugLogger.h" +#include "zeek/Event.h" +#include "zeek/EventRegistry.h" +#include "zeek/IntrusivePtr.h" +#include "zeek/Reporter.h" +#include "zeek/Val.h" +#include "zeek/cluster/Backend.h" +#include "zeek/cluster/Serializer.h" +#include "zeek/cluster/backend/zeromq/Plugin.h" +#include "zeek/cluster/backend/zeromq/ZeroMQ-Proxy.h" + +namespace zeek { + +namespace plugin::Zeek_Cluster_Backend_ZeroMQ { + +extern zeek::plugin::Zeek_Cluster_Backend_ZeroMQ::Plugin plugin; + +} + +namespace cluster::zeromq { + +enum class DebugFlag : zeek_uint_t { + NONE = 0, + POLL = 1, +}; + +constexpr DebugFlag operator&(zeek_uint_t x, DebugFlag y) { + return static_cast(x & static_cast(y)); +} + +#define ZEROMQ_DEBUG(...) PLUGIN_DBG_LOG(zeek::plugin::Zeek_Cluster_Backend_ZeroMQ::plugin, __VA_ARGS__) + +#define ZEROMQ_THREAD_PRINTF(...) \ + do { \ + std::fprintf(stderr, "[zeromq] " __VA_ARGS__); \ + } while ( 0 ) + +#define ZEROMQ_DEBUG_THREAD_PRINTF(flag, ...) \ + do { \ + if ( (debug_flags & flag) == flag ) { \ + ZEROMQ_THREAD_PRINTF(__VA_ARGS__); \ + } \ + } while ( 0 ) + +namespace { +void self_thread_fun(void* arg) { + auto* self = static_cast(arg); + self->Run(); +} + +} // namespace + + +// Constructor. +ZeroMQBackend::ZeroMQBackend(std::unique_ptr es, std::unique_ptr ls) + : ThreadedBackend(std::move(es), std::move(ls)) { + xsub = zmq::socket_t(ctx, zmq::socket_type::xsub); + xpub = zmq::socket_t(ctx, zmq::socket_type::xpub); + log_push = zmq::socket_t(ctx, zmq::socket_type::push); + log_pull = zmq::socket_t(ctx, zmq::socket_type::pull); + + main_inproc = zmq::socket_t(ctx, zmq::socket_type::pair); + child_inproc = zmq::socket_t(ctx, zmq::socket_type::pair); +} + +void ZeroMQBackend::DoInitPostScript() { + ThreadedBackend::DoInitPostScript(); + + my_node_id = zeek::id::find_val("Cluster::Backend::ZeroMQ::my_node_id")->ToStdString(); + listen_xpub_endpoint = + zeek::id::find_val("Cluster::Backend::ZeroMQ::listen_xpub_endpoint")->ToStdString(); + listen_xsub_endpoint = + zeek::id::find_val("Cluster::Backend::ZeroMQ::listen_xsub_endpoint")->ToStdString(); + listen_xpub_nodrop = + zeek::id::find_val("Cluster::Backend::ZeroMQ::listen_xpub_nodrop")->AsBool() ? 1 : 0; + connect_xpub_endpoint = + zeek::id::find_val("Cluster::Backend::ZeroMQ::connect_xpub_endpoint")->ToStdString(); + connect_xsub_endpoint = + zeek::id::find_val("Cluster::Backend::ZeroMQ::connect_xsub_endpoint")->ToStdString(); + listen_log_endpoint = + zeek::id::find_val("Cluster::Backend::ZeroMQ::listen_log_endpoint")->ToStdString(); + poll_max_messages = zeek::id::find_val("Cluster::Backend::ZeroMQ::poll_max_messages")->Get(); + debug_flags = zeek::id::find_val("Cluster::Backend::ZeroMQ::debug_flags")->Get(); + + event_unsubscription = zeek::event_registry->Register("Cluster::Backend::ZeroMQ::unsubscription"); + event_subscription = zeek::event_registry->Register("Cluster::Backend::ZeroMQ::subscription"); + + main_inproc.bind("inproc://publish-bridge"); + child_inproc.connect("inproc://publish-bridge"); +} + + +void ZeroMQBackend::DoTerminate() { + ZEROMQ_DEBUG("Shutting down ctx"); + ctx.shutdown(); + ZEROMQ_DEBUG("Joining self_thread"); + if ( self_thread.joinable() ) + self_thread.join(); + + log_push.close(); + log_pull.close(); + xsub.close(); + xpub.close(); + main_inproc.close(); + child_inproc.close(); + + ZEROMQ_DEBUG("Closing ctx"); + ctx.close(); + + // If running the proxy thread, terminate it, too. + if ( proxy_thread ) { + ZEROMQ_DEBUG("Shutting down proxy thread"); + proxy_thread->Shutdown(); + } + + ZEROMQ_DEBUG("Terminated"); +} + +bool ZeroMQBackend::DoInit() { + auto linger_ms = static_cast(zeek::id::find_val("Cluster::Backend::ZeroMQ::linger_ms")->AsInt()); + int xpub_nodrop = zeek::id::find_val("Cluster::Backend::ZeroMQ::xpub_nodrop")->AsBool() ? 1 : 0; + + xpub.set(zmq::sockopt::linger, linger_ms); + xpub.set(zmq::sockopt::xpub_nodrop, xpub_nodrop); + + try { + xsub.connect(connect_xsub_endpoint); + } catch ( zmq::error_t& err ) { + zeek::reporter->Error("ZeroMQ: Failed to connect to XSUB %s: %s", connect_xsub_endpoint.c_str(), err.what()); + return false; + } + + try { + xpub.connect(connect_xpub_endpoint); + } catch ( zmq::error_t& err ) { + zeek::reporter->Error("ZeroMQ: Failed to connect to XPUB %s: %s", connect_xpub_endpoint.c_str(), err.what()); + return false; + } + + + auto log_immediate = + static_cast(zeek::id::find_val("Cluster::Backend::ZeroMQ::log_immediate")->AsBool()); + + auto log_sndhwm = + static_cast(zeek::id::find_val("Cluster::Backend::ZeroMQ::log_sndhwm")->AsInt()); + + auto log_sndbuf = + static_cast(zeek::id::find_val("Cluster::Backend::ZeroMQ::log_sndbuf")->AsInt()); + + auto log_rcvhwm = + static_cast(zeek::id::find_val("Cluster::Backend::ZeroMQ::log_rcvhwm")->AsInt()); + + auto log_rcvbuf = + static_cast(zeek::id::find_val("Cluster::Backend::ZeroMQ::log_rcvbuf")->AsInt()); + + ZEROMQ_DEBUG("Setting log_sndhwm=%d log_sndbuf=%d log_rcvhwm=%d log_rcvbuf=%d linger_ms=%d", log_sndhwm, log_sndbuf, + log_rcvhwm, log_rcvbuf, linger_ms); + + log_push.set(zmq::sockopt::sndhwm, log_sndhwm); + log_push.set(zmq::sockopt::sndbuf, log_sndbuf); + log_push.set(zmq::sockopt::linger, linger_ms); + log_push.set(zmq::sockopt::immediate, log_immediate); + + log_pull.set(zmq::sockopt::rcvhwm, log_rcvhwm); + log_pull.set(zmq::sockopt::rcvbuf, log_rcvbuf); + + + if ( ! listen_log_endpoint.empty() ) { + ZEROMQ_DEBUG("Listening on log pull socket: %s", listen_log_endpoint.c_str()); + try { + log_pull.bind(listen_log_endpoint); + } catch ( zmq::error_t& err ) { + zeek::reporter->Error("ZeroMQ: Failed to bind to PULL socket %s: %s", listen_log_endpoint.c_str(), + err.what()); + return false; + } + } + + const auto& log_endpoints = zeek::id::find_val("Cluster::Backend::ZeroMQ::connect_log_endpoints"); + for ( unsigned int i = 0; i < log_endpoints->Size(); i++ ) + connect_log_endpoints.push_back(log_endpoints->StringValAt(i)->ToStdString()); + + for ( const auto& endp : connect_log_endpoints ) { + ZEROMQ_DEBUG("Connecting log_push socket with %s", endp.c_str()); + try { + log_push.connect(endp); + } catch ( zmq::error_t& err ) { + zeek::reporter->Error("ZeroMQ: Failed to connect to PUSH socket %s: %s", endp.c_str(), err.what()); + return false; + } + } + + // At this point we've connected xpub/xsub and any logging endpoints. + // However, we cannot tell if we're connected to anything as ZeroMQ does + // not trivially expose this information. + // + // There is the zmq_socket_monitor() API that we could use to get some + // more low-level events in the future for logging and possibly script + // layer eventing: http://api.zeromq.org/4-2:zmq-socket-monitor + + + // As of now, message processing happens in a separate thread that is + // started below. If we wanted to integrate ZeroMQ as a selectable IO + // source rather than going through ThreadedBackend and its flare, the + // following post might be useful: + // + // https://funcptr.net/2012/09/10/zeromq---edge-triggered-notification/ + self_thread = std::thread(self_thread_fun, this); + + // After connecting, call ThreadedBackend::DoInit() to register + // the IO source with the loop. + return ThreadedBackend::DoInit(); +} + +bool ZeroMQBackend::SpawnZmqProxyThread() { + proxy_thread = std::make_unique(listen_xpub_endpoint, listen_xsub_endpoint, listen_xpub_nodrop); + return proxy_thread->Start(); +} + +bool ZeroMQBackend::DoPublishEvent(const std::string& topic, const std::string& format, + const cluster::detail::byte_buffer& buf) { + // Publishing an event happens as a multipart message with 4 parts: + // + // * The topic to publish to - this is required by XPUB/XSUB + // * The node's identifier - see Cluster::node_id(). + // * The format used to serialize the event. + // * The serialized event itself. + std::array parts = { + zmq::const_buffer(topic.data(), topic.size()), + zmq::const_buffer(my_node_id.data(), my_node_id.size()), + zmq::const_buffer(format.data(), format.size()), + zmq::const_buffer(buf.data(), buf.size()), + }; + + ZEROMQ_DEBUG("Publishing %zu bytes to %s", buf.size(), topic.c_str()); + + for ( size_t i = 0; i < parts.size(); i++ ) { + zmq::send_flags flags = zmq::send_flags::none; + if ( i < parts.size() - 1 ) + flags = flags | zmq::send_flags::sndmore; + + // This should never fail, it will instead block + // when HWM is reached. I guess we need to see if + // and how this can happen :-/ + main_inproc.send(parts[i], flags); + } + + return true; +} + +bool ZeroMQBackend::DoSubscribe(const std::string& topic_prefix) { + ZEROMQ_DEBUG("Subscribing to %s", topic_prefix.c_str()); + try { + // Prepend 0x01 byte to indicate subscription to XSUB socket + // This is the XSUB API instead of setsockopt(ZMQ_SUBSCRIBE). + std::string msg = "\x01" + topic_prefix; + xsub.send(zmq::const_buffer(msg.data(), msg.size())); + } catch ( zmq::error_t& err ) { + zeek::reporter->Error("Failed to subscribe to topic %s: %s", topic_prefix.c_str(), err.what()); + return false; + } + + return true; +} + +bool ZeroMQBackend::DoUnsubscribe(const std::string& topic_prefix) { + ZEROMQ_DEBUG("Unsubscribing %s", topic_prefix.c_str()); + try { + // Prepend 0x00 byte to indicate subscription to XSUB socket. + // This is the XSUB API instead of setsockopt(ZMQ_SUBSCRIBE). + std::string msg = "\x00" + topic_prefix; + xsub.send(zmq::const_buffer(msg.data(), msg.size())); + } catch ( zmq::error_t& err ) { + zeek::reporter->Error("Failed to unsubscribe from topic %s: %s", topic_prefix.c_str(), err.what()); + return false; + } + + return true; +} + +bool ZeroMQBackend::DoPublishLogWrites(const logging::detail::LogWriteHeader& header, const std::string& format, + cluster::detail::byte_buffer& buf) { + ZEROMQ_DEBUG("Publishing %zu bytes of log writes (path %s)", buf.size(), header.path.c_str()); + static std::string message_type = "log-write"; + + // Publishing a log write is done using 4 parts + // + // * A constant "log-write" string + // * The node's identifier - see Cluster::node_id(). + // * The format used to serialize the log write. + // * The serialized log write itself. + std::array parts = { + zmq::const_buffer{message_type.data(), message_type.size()}, + zmq::const_buffer(my_node_id.data(), my_node_id.size()), + zmq::const_buffer{format.data(), format.size()}, + zmq::const_buffer{buf.data(), buf.size()}, + }; + + zmq::send_result_t result; + for ( size_t i = 0; i < parts.size(); i++ ) { + zmq::send_flags flags = zmq::send_flags::dontwait; + if ( i < parts.size() - 1 ) + flags = flags | zmq::send_flags::sndmore; + + result = log_push.send(parts[i], flags); + if ( ! result ) { + // XXX: Not exactly clear what we should do if we reach HWM. + // we could block and hope a logger comes along that empties + // our internal queue, or discard messages and log very loudly + // and have metrics about it. However, this may happen regularly + // at shutdown. + // + // Maybe that should be configurable? + + // If no logging endpoints were configured, that almost seems on + // purpose (and there's a warning elsewhere about this), so skip + // logging an error when sending fails. + if ( connect_log_endpoints.empty() ) + return true; + + reporter->Error("Failed to send log write. HWM reached?"); + return false; + } + } + + return true; +} + +void ZeroMQBackend::Run() { + using MultipartMessage = std::vector; + + auto HandleLogMessages = [this](const std::vector& msgs) { + QueueMessages qmsgs; + qmsgs.reserve(msgs.size()); + + for ( const auto& msg : msgs ) { + // sender, format, type, payload + if ( msg.size() != 4 ) { + ZEROMQ_THREAD_PRINTF("log: error: expected 4 parts, have %zu!\n", msg.size()); + continue; + } + + detail::byte_buffer payload{msg[3].data(), msg[3].data() + msg[3].size()}; + qmsgs.emplace_back(LogMessage{.format = std::string(msg[2].data(), msg[2].size()), + .payload = std::move(payload)}); + } + + QueueForProcessing(std::move(qmsgs)); + }; + + auto HandleInprocMessages = [this](std::vector& msgs) { + // Forward messages from the inprocess bridge to xpub. + for ( auto& msg : msgs ) { + assert(msg.size() == 4); + + for ( auto& part : msg ) { + zmq::send_flags flags = zmq::send_flags::dontwait; + if ( part.more() ) + flags = flags | zmq::send_flags::sndmore; + + zmq::send_result_t result; + do { + try { + result = xpub.send(part, flags); + } catch ( zmq::error_t& err ) { + // XXX: Not sure if the return false is so great here. + // + // Also, if we fail to publish, should we block rather + // than discard? + ZEROMQ_THREAD_PRINTF("xpub: Failed to publish: %s (%d)", err.what(), err.num()); + break; + } + // EAGAIN returns empty result, means try again! + } while ( ! result ); + } + } + }; + + auto HandleXPubMessages = [this](const std::vector& msgs) { + QueueMessages qmsgs; + qmsgs.reserve(msgs.size()); + + for ( const auto& msg : msgs ) { + if ( msg.size() != 1 ) { + ZEROMQ_THREAD_PRINTF("xpub: error: expected 1 part, have %zu!\n", msg.size()); + continue; + } + + // Check if the messages starts with \x00 or \x01 to understand if it's + // a subscription or unsubscription message. + auto first = *reinterpret_cast(msg[0].data()); + if ( first == 0 || first == 1 ) { + QueueMessage qm; + auto* start = msg[0].data() + 1; + auto* end = msg[0].data() + msg[0].size(); + detail::byte_buffer topic(start, end); + if ( first == 1 ) { + qm = BackendMessage{1, std::move(topic)}; + } + else if ( first == 0 ) { + qm = BackendMessage{0, std::move(topic)}; + } + else { + ZEROMQ_THREAD_PRINTF("xpub: error: unexpected first char: have '0x%02x'", first); + continue; + } + + qmsgs.emplace_back(std::move(qm)); + } + } + + QueueForProcessing(std::move(qmsgs)); + }; + + auto HandleXSubMessages = [this](const std::vector& msgs) { + QueueMessages qmsgs; + qmsgs.reserve(msgs.size()); + + for ( const auto& msg : msgs ) { + if ( msg.size() != 4 ) { + ZEROMQ_THREAD_PRINTF("xsub: error: expected 4 parts, have %zu!\n", msg.size()); + continue; + } + + // Filter out messages that are coming from this node. + std::string sender(msg[1].data(), msg[1].size()); + if ( sender == my_node_id ) + continue; + + detail::byte_buffer payload{msg[3].data(), msg[3].data() + msg[3].size()}; + qmsgs.emplace_back(EventMessage{.topic = std::string(msg[0].data(), msg[0].size()), + .format = std::string(msg[2].data(), msg[2].size()), + .payload = std::move(payload)}); + } + + QueueForProcessing(std::move(qmsgs)); + }; + + struct SocketInfo { + zmq::socket_ref socket; + std::string name; + std::function&)> handler; + }; + + std::vector sockets = { + {.socket = child_inproc, .name = "inproc", .handler = HandleInprocMessages}, + {.socket = xpub, .name = "xpub", .handler = HandleXPubMessages}, + {.socket = xsub, .name = "xsub", .handler = HandleXSubMessages}, + {.socket = log_pull, .name = "log_pull", .handler = HandleLogMessages}, + }; + + std::vector poll_items(sockets.size()); + + while ( true ) { + for ( size_t i = 0; i < sockets.size(); i++ ) + poll_items[i] = {.socket = sockets[i].socket.handle(), .fd = 0, .events = ZMQ_POLLIN | ZMQ_POLLERR}; + + // Awkward. + std::vector> rcv_messages(sockets.size()); + try { + int r = zmq::poll(poll_items, std::chrono::seconds(-1)); + ZEROMQ_DEBUG_THREAD_PRINTF(DebugFlag::POLL, "poll: r=%d", r); + + for ( size_t i = 0; i < poll_items.size(); i++ ) { + const auto& item = poll_items[i]; + ZEROMQ_DEBUG_THREAD_PRINTF(DebugFlag::POLL, "poll: items[%lu]=%s %s %s\n", i, sockets[i].name.c_str(), + item.revents & ZMQ_POLLIN ? "pollin " : "", + item.revents & ZMQ_POLLERR ? "err" : ""); + + if ( item.revents & ZMQ_POLLERR ) { + // What should we be doing? Re-open sockets? Terminate? + ZEROMQ_THREAD_PRINTF("poll: error: POLLERR on socket %zu %s %p revents=%x\n", i, + sockets[i].name.c_str(), item.socket, item.revents); + } + + // Nothing to do? + if ( (item.revents & ZMQ_POLLIN) == 0 ) + continue; + + bool consumed_one = false; + + // Read messages from the socket. + do { + zmq::message_t msg; + rcv_messages[i].emplace_back(); // make room for a multipart message + auto& into = rcv_messages[i].back(); + + // Only receive up to poll_max_messages from an individual + // socket. Move on to the next when exceeded. The last pushed + // message (empty) is popped at the end of the loop. + if ( poll_max_messages > 0 && rcv_messages[i].size() > poll_max_messages ) { + ZEROMQ_DEBUG_THREAD_PRINTF(DebugFlag::POLL, "poll: %s rcv_messages[%zu] full!\n", + sockets[i].name.c_str(), i); + break; + } + + consumed_one = false; + bool more = false; + + // Read a multi-part message. + do { + auto recv_result = sockets[i].socket.recv(msg, zmq::recv_flags::dontwait); + if ( recv_result ) { + consumed_one = true; + more = msg.more(); + into.emplace_back(std::move(msg)); + } + else { + // EAGAIN and more flag set? Try again! + if ( more ) + continue; + } + } while ( more ); + } while ( consumed_one ); + + assert(rcv_messages[i].back().size() == 0); + rcv_messages[i].pop_back(); + } + } catch ( zmq::error_t& err ) { + if ( err.num() == ETERM ) + return; + + throw; + } + + // At this point, we've received anything that was readable from the sockets. + // Now interpret and enqueue it into messages. + for ( size_t i = 0; i < sockets.size(); i++ ) { + if ( rcv_messages[i].empty() ) + continue; + + sockets[i].handler(rcv_messages[i]); + } + } +} + +bool ZeroMQBackend::DoProcessBackendMessage(int tag, detail::byte_buffer_span payload) { + if ( tag == 0 || tag == 1 ) { + std::string topic{reinterpret_cast(payload.data()), payload.size()}; + zeek::EventHandlerPtr eh = tag == 1 ? event_subscription : event_unsubscription; + + ZEROMQ_DEBUG("BackendMessage: %s for %s", eh->Name(), topic.c_str()); + zeek::event_mgr.Enqueue(eh, zeek::make_intrusive(topic)); + return true; + } + else { + zeek::reporter->Error("Ignoring bad BackendMessage tag=%d", tag); + return false; + } +} + + +} // namespace cluster::zeromq +} // namespace zeek diff --git a/src/cluster/backend/zeromq/ZeroMQ.h b/src/cluster/backend/zeromq/ZeroMQ.h new file mode 100644 index 0000000000..8a715b8c28 --- /dev/null +++ b/src/cluster/backend/zeromq/ZeroMQ.h @@ -0,0 +1,99 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include +#include +#include + +#include "zeek/cluster/Backend.h" +#include "zeek/cluster/Serializer.h" +#include "zeek/cluster/backend/zeromq/ZeroMQ-Proxy.h" + +namespace zeek::cluster::zeromq { + +class ZeroMQBackend : public cluster::ThreadedBackend { +public: + /** + * Constructor. + */ + ZeroMQBackend(std::unique_ptr es, std::unique_ptr ls); + + /** + * Spawns a thread running zmq_proxy() for the configured XPUB/XSUB listen + * sockets. Only one node in a cluster should do this. + */ + bool SpawnZmqProxyThread(); + + /** + * Run method for background thread. + */ + void Run(); + + /** + * Component factory. + */ + static std::unique_ptr Instantiate(std::unique_ptr event_serializer, + std::unique_ptr log_serializer) { + return std::make_unique(std::move(event_serializer), std::move(log_serializer)); + } + +private: + void DoInitPostScript() override; + + bool DoInit() override; + + void DoTerminate() override; + + bool DoPublishEvent(const std::string& topic, const std::string& format, + const cluster::detail::byte_buffer& buf) override; + + bool DoSubscribe(const std::string& topic_prefix) override; + + bool DoUnsubscribe(const std::string& topic_prefix) override; + + bool DoPublishLogWrites(const logging::detail::LogWriteHeader& header, const std::string& format, + cluster::detail::byte_buffer& buf) override; + + const char* Tag() override { return "ZeroMQ"; } + + bool DoProcessBackendMessage(int tag, detail::byte_buffer_span payload) override; + + // Script level variables. + std::string my_node_id; + std::string connect_xsub_endpoint; + std::string connect_xpub_endpoint; + std::string listen_xsub_endpoint; + std::string listen_xpub_endpoint; + std::string listen_log_endpoint; + int listen_xpub_nodrop = 1; + + zeek_uint_t poll_max_messages = 0; + zeek_uint_t debug_flags = 0; + + EventHandlerPtr event_subscription; + EventHandlerPtr event_unsubscription; + + zmq::context_t ctx; + zmq::socket_t xsub; + zmq::socket_t xpub; + + // inproc sockets used for sending + // publish messages to xpub in a + // thread safe manner. + zmq::socket_t main_inproc; + zmq::socket_t child_inproc; + + // Sockets used for logging. The log_push socket connects + // with one or more logger-like nodes. Logger nodes listen + // on the log_pull socket. + std::vector connect_log_endpoints; + zmq::socket_t log_push; + zmq::socket_t log_pull; + + std::thread self_thread; + + std::unique_ptr proxy_thread; +}; + +} // namespace zeek::cluster::zeromq diff --git a/src/cluster/backend/zeromq/cluster_backend_zeromq.bif b/src/cluster/backend/zeromq/cluster_backend_zeromq.bif new file mode 100644 index 0000000000..4721a19c6b --- /dev/null +++ b/src/cluster/backend/zeromq/cluster_backend_zeromq.bif @@ -0,0 +1,16 @@ +%%{ +#include "ZeroMQ.h" +%%} + +function Cluster::Backend::ZeroMQ::spawn_zmq_proxy_thread%(%): bool + %{ + // Spawn the ZeroMQ broker thread. + auto *zeromq_backend = dynamic_cast(zeek::cluster::backend); + if ( ! zeromq_backend ) + { + zeek::emit_builtin_error("Cluster::backend not set to ZeroMQ?"); + return zeek::val_mgr->Bool(false); + } + + return zeek::val_mgr->Bool(zeromq_backend->SpawnZmqProxyThread()); + %} diff --git a/src/cluster/backend/zeromq/cmake/FindZeroMQ.cmake b/src/cluster/backend/zeromq/cmake/FindZeroMQ.cmake new file mode 100644 index 0000000000..964f6a457d --- /dev/null +++ b/src/cluster/backend/zeromq/cmake/FindZeroMQ.cmake @@ -0,0 +1,52 @@ +include(FindPackageHandleStandardArgs) + +find_library(ZeroMQ_LIBRARY NAMES zmq HINTS ${ZeroMQ_ROOT_DIR}/lib) + +find_path(ZeroMQ_INCLUDE_DIR NAMES zmq.h HINTS ${ZeroMQ_ROOT_DIR}/include) + +find_path(ZeroMQ_CPP_INCLUDE_DIR NAMES zmq.hpp HINTS ${ZeroMQ_ROOT_DIR}/include) + +function (set_cppzmq_version) + # Extract the version from + file(STRINGS "${ZeroMQ_CPP_INCLUDE_DIR}/zmq.hpp" CPPZMQ_MAJOR_VERSION_H + REGEX "^#define CPPZMQ_VERSION_MAJOR [0-9]+$") + file(STRINGS "${ZeroMQ_CPP_INCLUDE_DIR}/zmq.hpp" CPPZMQ_MINOR_VERSION_H + REGEX "^#define CPPZMQ_VERSION_MINOR [0-9]+$") + file(STRINGS "${ZeroMQ_CPP_INCLUDE_DIR}/zmq.hpp" CPPZMQ_PATCH_VERSION_H + REGEX "^#define CPPZMQ_VERSION_PATCH [0-9]+$") + string(REGEX REPLACE "^.*MAJOR ([0-9]+)$" "\\1" CPPZMQ_MAJOR_VERSION + "${CPPZMQ_MAJOR_VERSION_H}") + string(REGEX REPLACE "^.*MINOR ([0-9]+)$" "\\1" CPPZMQ_MINOR_VERSION + "${CPPZMQ_MINOR_VERSION_H}") + string(REGEX REPLACE "^.*PATCH ([0-9]+)$" "\\1" CPPZMQ_PATCH_VERSION + "${CPPZMQ_PATCH_VERSION_H}") + + set(ZeroMQ_CPP_VERSION "${CPPZMQ_MAJOR_VERSION}.${CPPZMQ_MINOR_VERSION}.${CPPZMQ_PATCH_VERSION}" + PARENT_SCOPE) +endfunction () + +if (ZeroMQ_CPP_INCLUDE_DIR) + set_cppzmq_version() +endif () + +if (NOT ZeroMQ_CPP_VERSION) + # Probably no zmq.hpp file, use the version from auxil + set(ZeroMQ_CPP_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/auxil/cppzmq" + CACHE FILEPATH "Include path for cppzmq" FORCE) + set_cppzmq_version() +elseif (ZeroMQ_CPP_VERSION VERSION_LESS "4.9.0") + message(STATUS "Found old cppzmq version ${ZeroMQ_CPP_VERSION}, using bundled version") + set(ZeroMQ_CPP_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/auxil/cppzmq" + CACHE FILEPATH "Include path for cppzmq" FORCE) + set_cppzmq_version() +endif () + +message(STATUS "Using cppzmq ${ZeroMQ_CPP_VERSION} from ${ZeroMQ_CPP_INCLUDE_DIR}") + +find_package_handle_standard_args( + ZeroMQ FOUND_VAR ZeroMQ_FOUND REQUIRED_VARS ZeroMQ_LIBRARY ZeroMQ_INCLUDE_DIR + ZeroMQ_CPP_INCLUDE_DIR ZeroMQ_CPP_VERSION) + +set(ZeroMQ_LIBRARIES ${ZeroMQ_LIBRARY}) +set(ZeroMQ_INCLUDE_DIRS ${ZeroMQ_INCLUDE_DIR} ${ZeroMQ_CPP_INCLUDE_DIR}) +set(ZeroMQ_FOUND ${ZeroMQ_FOUND}) diff --git a/testing/btest/Baseline/cluster.zeromq.logging/cluster.log.normalized b/testing/btest/Baseline/cluster.zeromq.logging/cluster.log.normalized new file mode 100644 index 0000000000..fdebaf2784 --- /dev/null +++ b/testing/btest/Baseline/cluster.zeromq.logging/cluster.log.normalized @@ -0,0 +1,21 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +logger got hello from manager (zeromq_manager___NrFj3eGxkRR5) +logger got hello from proxy (zeromq_proxy___NrFj3eGxkRR5) +logger got hello from worker-1 (zeromq_worker-1___NrFj3eGxkRR5) +logger got hello from worker-2 (zeromq_worker-2___NrFj3eGxkRR5) +manager got hello from logger (zeromq_logger___NrFj3eGxkRR5) +manager got hello from proxy (zeromq_proxy___NrFj3eGxkRR5) +manager got hello from worker-1 (zeromq_worker-1___NrFj3eGxkRR5) +manager got hello from worker-2 (zeromq_worker-2___NrFj3eGxkRR5) +proxy got hello from logger (zeromq_logger___NrFj3eGxkRR5) +proxy got hello from manager (zeromq_manager___NrFj3eGxkRR5) +proxy got hello from worker-1 (zeromq_worker-1___NrFj3eGxkRR5) +proxy got hello from worker-2 (zeromq_worker-2___NrFj3eGxkRR5) +worker-1 got hello from logger (zeromq_logger___NrFj3eGxkRR5) +worker-1 got hello from manager (zeromq_manager___NrFj3eGxkRR5) +worker-1 got hello from proxy (zeromq_proxy___NrFj3eGxkRR5) +worker-1 got hello from worker-2 (zeromq_worker-2___NrFj3eGxkRR5) +worker-2 got hello from logger (zeromq_logger___NrFj3eGxkRR5) +worker-2 got hello from manager (zeromq_manager___NrFj3eGxkRR5) +worker-2 got hello from proxy (zeromq_proxy___NrFj3eGxkRR5) +worker-2 got hello from worker-1 (zeromq_worker-1___NrFj3eGxkRR5) diff --git a/testing/btest/Baseline/cluster.zeromq.logging/manager.out b/testing/btest/Baseline/cluster.zeromq.logging/manager.out new file mode 100644 index 0000000000..9c5b6173f4 --- /dev/null +++ b/testing/btest/Baseline/cluster.zeromq.logging/manager.out @@ -0,0 +1,16 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +A zeek_init, manager +B node_up, logger +B node_up, proxy +B node_up, worker-1 +B node_up, worker-2 +B nodes_up, 2 +B nodes_up, 3 +B nodes_up, 4 +B nodes_up, 5 +C send_finish +D node_down, logger +D node_down, proxy +D node_down, worker-1 +D node_down, worker-2 +D send_finish to logger diff --git a/testing/btest/Baseline/cluster.zeromq.logging/node_up.sorted b/testing/btest/Baseline/cluster.zeromq.logging/node_up.sorted new file mode 100644 index 0000000000..1351c3ed4a --- /dev/null +++ b/testing/btest/Baseline/cluster.zeromq.logging/node_up.sorted @@ -0,0 +1,21 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +logger manager +logger proxy +logger worker-1 +logger worker-2 +manager logger +manager proxy +manager worker-1 +manager worker-2 +proxy logger +proxy manager +proxy worker-1 +proxy worker-2 +worker-1 logger +worker-1 manager +worker-1 proxy +worker-1 worker-2 +worker-2 logger +worker-2 manager +worker-2 proxy +worker-2 worker-1 diff --git a/testing/btest/Baseline/cluster.zeromq.manager-is-logger/cluster.log.normalized b/testing/btest/Baseline/cluster.zeromq.manager-is-logger/cluster.log.normalized new file mode 100644 index 0000000000..4cb1e8ebc0 --- /dev/null +++ b/testing/btest/Baseline/cluster.zeromq.manager-is-logger/cluster.log.normalized @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +manager got hello from proxy (zeromq_proxy___NrFj3eGxkRR5) +manager got hello from worker-1 (zeromq_worker-1___NrFj3eGxkRR5) +manager got hello from worker-2 (zeromq_worker-2___NrFj3eGxkRR5) +proxy got hello from manager (zeromq_manager___NrFj3eGxkRR5) +proxy got hello from worker-1 (zeromq_worker-1___NrFj3eGxkRR5) +proxy got hello from worker-2 (zeromq_worker-2___NrFj3eGxkRR5) +worker-1 got hello from manager (zeromq_manager___NrFj3eGxkRR5) +worker-1 got hello from proxy (zeromq_proxy___NrFj3eGxkRR5) +worker-1 got hello from worker-2 (zeromq_worker-2___NrFj3eGxkRR5) +worker-2 got hello from manager (zeromq_manager___NrFj3eGxkRR5) +worker-2 got hello from proxy (zeromq_proxy___NrFj3eGxkRR5) +worker-2 got hello from worker-1 (zeromq_worker-1___NrFj3eGxkRR5) diff --git a/testing/btest/Baseline/cluster.zeromq.manager-is-logger/manager.out b/testing/btest/Baseline/cluster.zeromq.manager-is-logger/manager.out new file mode 100644 index 0000000000..df8b56a0eb --- /dev/null +++ b/testing/btest/Baseline/cluster.zeromq.manager-is-logger/manager.out @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +A node_up, proxy +A node_up, worker-1 +A node_up, worker-2 +B nodes_up, 2 +B nodes_up, 3 +B nodes_up, 4 +D node_down, proxy +D node_down, worker-1 +D node_down, worker-2 +zeek_init, manager diff --git a/testing/btest/Baseline/cluster.zeromq.manager-is-logger/node_up.sorted b/testing/btest/Baseline/cluster.zeromq.manager-is-logger/node_up.sorted new file mode 100644 index 0000000000..57fce89d58 --- /dev/null +++ b/testing/btest/Baseline/cluster.zeromq.manager-is-logger/node_up.sorted @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +manager proxy +manager worker-1 +manager worker-2 +proxy manager +proxy worker-1 +proxy worker-2 +worker-1 manager +worker-1 proxy +worker-1 worker-2 +worker-2 manager +worker-2 proxy +worker-2 worker-1 diff --git a/testing/btest/Baseline/cluster.zeromq.supervisor/supervisor.cluster.log b/testing/btest/Baseline/cluster.zeromq.supervisor/supervisor.cluster.log new file mode 100644 index 0000000000..fdebaf2784 --- /dev/null +++ b/testing/btest/Baseline/cluster.zeromq.supervisor/supervisor.cluster.log @@ -0,0 +1,21 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +logger got hello from manager (zeromq_manager___NrFj3eGxkRR5) +logger got hello from proxy (zeromq_proxy___NrFj3eGxkRR5) +logger got hello from worker-1 (zeromq_worker-1___NrFj3eGxkRR5) +logger got hello from worker-2 (zeromq_worker-2___NrFj3eGxkRR5) +manager got hello from logger (zeromq_logger___NrFj3eGxkRR5) +manager got hello from proxy (zeromq_proxy___NrFj3eGxkRR5) +manager got hello from worker-1 (zeromq_worker-1___NrFj3eGxkRR5) +manager got hello from worker-2 (zeromq_worker-2___NrFj3eGxkRR5) +proxy got hello from logger (zeromq_logger___NrFj3eGxkRR5) +proxy got hello from manager (zeromq_manager___NrFj3eGxkRR5) +proxy got hello from worker-1 (zeromq_worker-1___NrFj3eGxkRR5) +proxy got hello from worker-2 (zeromq_worker-2___NrFj3eGxkRR5) +worker-1 got hello from logger (zeromq_logger___NrFj3eGxkRR5) +worker-1 got hello from manager (zeromq_manager___NrFj3eGxkRR5) +worker-1 got hello from proxy (zeromq_proxy___NrFj3eGxkRR5) +worker-1 got hello from worker-2 (zeromq_worker-2___NrFj3eGxkRR5) +worker-2 got hello from logger (zeromq_logger___NrFj3eGxkRR5) +worker-2 got hello from manager (zeromq_manager___NrFj3eGxkRR5) +worker-2 got hello from proxy (zeromq_proxy___NrFj3eGxkRR5) +worker-2 got hello from worker-1 (zeromq_worker-1___NrFj3eGxkRR5) diff --git a/testing/btest/Baseline/cluster.zeromq.two-nodes/..manager.out b/testing/btest/Baseline/cluster.zeromq.two-nodes/..manager.out new file mode 100644 index 0000000000..7e67339b4d --- /dev/null +++ b/testing/btest/Baseline/cluster.zeromq.two-nodes/..manager.out @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +node_up, worker-1 +node_down, worker-1 diff --git a/testing/btest/Baseline/cluster.zeromq.two-nodes/..worker.out b/testing/btest/Baseline/cluster.zeromq.two-nodes/..worker.out new file mode 100644 index 0000000000..386f8ae30f --- /dev/null +++ b/testing/btest/Baseline/cluster.zeromq.two-nodes/..worker.out @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +node_up, manager diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 443d35b00e..ad6adb5cb9 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -259,6 +259,7 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/plugins/Zeek_WebSocket.functions.bif.zeek build/scripts/base/bif/plugins/Zeek_WebSocket.types.bif.zeek build/scripts/base/bif/plugins/Zeek_XMPP.events.bif.zeek + build/scripts/base/bif/plugins/Zeek_Cluster_Backend_ZeroMQ.cluster_backend_zeromq.bif.zeek build/scripts/base/bif/plugins/Zeek_ARP.events.bif.zeek build/scripts/base/bif/plugins/Zeek_UDP.events.bif.zeek build/scripts/base/bif/plugins/Zeek_ICMP.events.bif.zeek diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 8a23826c17..353e066690 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -259,6 +259,7 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/plugins/Zeek_WebSocket.functions.bif.zeek build/scripts/base/bif/plugins/Zeek_WebSocket.types.bif.zeek build/scripts/base/bif/plugins/Zeek_XMPP.events.bif.zeek + build/scripts/base/bif/plugins/Zeek_Cluster_Backend_ZeroMQ.cluster_backend_zeromq.bif.zeek build/scripts/base/bif/plugins/Zeek_ARP.events.bif.zeek build/scripts/base/bif/plugins/Zeek_UDP.events.bif.zeek build/scripts/base/bif/plugins/Zeek_ICMP.events.bif.zeek diff --git a/testing/btest/Baseline/coverage.test-all-policy-cluster/.stderr b/testing/btest/Baseline/coverage.test-all-policy-cluster/.stderr index bff9a64e41..49d861c74c 100644 --- a/testing/btest/Baseline/coverage.test-all-policy-cluster/.stderr +++ b/testing/btest/Baseline/coverage.test-all-policy-cluster/.stderr @@ -1,5 +1 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -received termination signal -received termination signal -received termination signal -received termination signal diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 9eadabd9ad..2722d1e90b 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -337,6 +337,7 @@ 0.000000 MetaHookPost LoadFile(0, ./Zeek_BenchmarkReader.benchmark.bif.zeek, <...>/Zeek_BenchmarkReader.benchmark.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_BinaryReader.binary.bif.zeek, <...>/Zeek_BinaryReader.binary.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_BitTorrent.events.bif.zeek, <...>/Zeek_BitTorrent.events.bif.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, ./Zeek_Cluster_Backend_ZeroMQ.cluster_backend_zeromq.bif.zeek, <...>/Zeek_Cluster_Backend_ZeroMQ.cluster_backend_zeromq.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_ConfigReader.config.bif.zeek, <...>/Zeek_ConfigReader.config.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_ConnSize.events.bif.zeek, <...>/Zeek_ConnSize.events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_ConnSize.functions.bif.zeek, <...>/Zeek_ConnSize.functions.bif.zeek) -> -1 @@ -643,6 +644,7 @@ 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_BenchmarkReader.benchmark.bif.zeek, <...>/Zeek_BenchmarkReader.benchmark.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_BinaryReader.binary.bif.zeek, <...>/Zeek_BinaryReader.binary.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_BitTorrent.events.bif.zeek, <...>/Zeek_BitTorrent.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Cluster_Backend_ZeroMQ.cluster_backend_zeromq.bif.zeek, <...>/Zeek_Cluster_Backend_ZeroMQ.cluster_backend_zeromq.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_ConfigReader.config.bif.zeek, <...>/Zeek_ConfigReader.config.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_ConnSize.events.bif.zeek, <...>/Zeek_ConnSize.events.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_ConnSize.functions.bif.zeek, <...>/Zeek_ConnSize.functions.bif.zeek) -> (-1, ) @@ -1281,6 +1283,7 @@ 0.000000 MetaHookPre LoadFile(0, ./Zeek_BenchmarkReader.benchmark.bif.zeek, <...>/Zeek_BenchmarkReader.benchmark.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_BinaryReader.binary.bif.zeek, <...>/Zeek_BinaryReader.binary.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_BitTorrent.events.bif.zeek, <...>/Zeek_BitTorrent.events.bif.zeek) +0.000000 MetaHookPre LoadFile(0, ./Zeek_Cluster_Backend_ZeroMQ.cluster_backend_zeromq.bif.zeek, <...>/Zeek_Cluster_Backend_ZeroMQ.cluster_backend_zeromq.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_ConfigReader.config.bif.zeek, <...>/Zeek_ConfigReader.config.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_ConnSize.events.bif.zeek, <...>/Zeek_ConnSize.events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_ConnSize.functions.bif.zeek, <...>/Zeek_ConnSize.functions.bif.zeek) @@ -1587,6 +1590,7 @@ 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_BenchmarkReader.benchmark.bif.zeek, <...>/Zeek_BenchmarkReader.benchmark.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_BinaryReader.binary.bif.zeek, <...>/Zeek_BinaryReader.binary.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_BitTorrent.events.bif.zeek, <...>/Zeek_BitTorrent.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Cluster_Backend_ZeroMQ.cluster_backend_zeromq.bif.zeek, <...>/Zeek_Cluster_Backend_ZeroMQ.cluster_backend_zeromq.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_ConfigReader.config.bif.zeek, <...>/Zeek_ConfigReader.config.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_ConnSize.events.bif.zeek, <...>/Zeek_ConnSize.events.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_ConnSize.functions.bif.zeek, <...>/Zeek_ConnSize.functions.bif.zeek) @@ -2224,6 +2228,7 @@ 0.000000 | HookLoadFile ./Zeek_BenchmarkReader.benchmark.bif.zeek <...>/Zeek_BenchmarkReader.benchmark.bif.zeek 0.000000 | HookLoadFile ./Zeek_BinaryReader.binary.bif.zeek <...>/Zeek_BinaryReader.binary.bif.zeek 0.000000 | HookLoadFile ./Zeek_BitTorrent.events.bif.zeek <...>/Zeek_BitTorrent.events.bif.zeek +0.000000 | HookLoadFile ./Zeek_Cluster_Backend_ZeroMQ.cluster_backend_zeromq.bif.zeek <...>/Zeek_Cluster_Backend_ZeroMQ.cluster_backend_zeromq.bif.zeek 0.000000 | HookLoadFile ./Zeek_ConfigReader.config.bif.zeek <...>/Zeek_ConfigReader.config.bif.zeek 0.000000 | HookLoadFile ./Zeek_ConnSize.events.bif.zeek <...>/Zeek_ConnSize.events.bif.zeek 0.000000 | HookLoadFile ./Zeek_ConnSize.functions.bif.zeek <...>/Zeek_ConnSize.functions.bif.zeek @@ -2530,6 +2535,7 @@ 0.000000 | HookLoadFileExtended ./Zeek_BenchmarkReader.benchmark.bif.zeek <...>/Zeek_BenchmarkReader.benchmark.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_BinaryReader.binary.bif.zeek <...>/Zeek_BinaryReader.binary.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_BitTorrent.events.bif.zeek <...>/Zeek_BitTorrent.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_Cluster_Backend_ZeroMQ.cluster_backend_zeromq.bif.zeek <...>/Zeek_Cluster_Backend_ZeroMQ.cluster_backend_zeromq.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_ConfigReader.config.bif.zeek <...>/Zeek_ConfigReader.config.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_ConnSize.events.bif.zeek <...>/Zeek_ConnSize.events.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_ConnSize.functions.bif.zeek <...>/Zeek_ConnSize.functions.bif.zeek diff --git a/testing/btest/Files/zeromq/cluster-layout-no-logger.zeek b/testing/btest/Files/zeromq/cluster-layout-no-logger.zeek new file mode 100644 index 0000000000..23baf76a2a --- /dev/null +++ b/testing/btest/Files/zeromq/cluster-layout-no-logger.zeek @@ -0,0 +1,8 @@ +redef Cluster::manager_is_logger = T; + +redef Cluster::nodes = { + ["manager"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=to_port(getenv("LOG_PULL_PORT"))], + ["proxy"] = [$node_type=Cluster::PROXY, $ip=127.0.0.1], + ["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1], + ["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1], +}; diff --git a/testing/btest/Files/zeromq/cluster-layout-simple.zeek b/testing/btest/Files/zeromq/cluster-layout-simple.zeek new file mode 100644 index 0000000000..be99599819 --- /dev/null +++ b/testing/btest/Files/zeromq/cluster-layout-simple.zeek @@ -0,0 +1,9 @@ +redef Cluster::manager_is_logger = F; + +redef Cluster::nodes = { + ["manager"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1], + ["logger"] = [$node_type=Cluster::LOGGER, $ip=127.0.0.1, $p=to_port(getenv("LOG_PULL_PORT"))], + ["proxy"] = [$node_type=Cluster::PROXY, $ip=127.0.0.1], + ["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1], + ["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1], +}; diff --git a/testing/btest/Files/zeromq/cluster-layout-two-loggers.zeek b/testing/btest/Files/zeromq/cluster-layout-two-loggers.zeek new file mode 100644 index 0000000000..19e6942774 --- /dev/null +++ b/testing/btest/Files/zeromq/cluster-layout-two-loggers.zeek @@ -0,0 +1,10 @@ +redef Cluster::manager_is_logger = F; + +redef Cluster::nodes = { + ["manager"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1], + ["logger-1"] = [$node_type=Cluster::LOGGER, $ip=127.0.0.1, $p=to_port(getenv("LOG_PULL_PORT_1"))], + ["logger-2"] = [$node_type=Cluster::LOGGER, $ip=127.0.0.1, $p=to_port(getenv("LOG_PULL_PORT_2"))], + ["proxy"] = [$node_type=Cluster::PROXY, $ip=127.0.0.1], + ["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1], + ["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1], +}; diff --git a/testing/btest/Files/zeromq/test-bootstrap.zeek b/testing/btest/Files/zeromq/test-bootstrap.zeek new file mode 100644 index 0000000000..116a60ea0e --- /dev/null +++ b/testing/btest/Files/zeromq/test-bootstrap.zeek @@ -0,0 +1,11 @@ +# Helper scripts for test expecting XPUB/XSUB ports allocated by +# btest and configuring the ZeroMQ globals. +@load base/utils/numbers + +@load frameworks/cluster/backend/zeromq +@load frameworks/cluster/backend/zeromq/connect + +redef Cluster::Backend::ZeroMQ::listen_xpub_endpoint = fmt("tcp://127.0.0.1:%s", extract_count(getenv("XPUB_PORT"))); +redef Cluster::Backend::ZeroMQ::listen_xsub_endpoint = fmt("tcp://127.0.0.1:%s", extract_count(getenv("XSUB_PORT"))); +redef Cluster::Backend::ZeroMQ::connect_xpub_endpoint = fmt("tcp://127.0.0.1:%s", extract_count(getenv("XSUB_PORT"))); +redef Cluster::Backend::ZeroMQ::connect_xsub_endpoint = fmt("tcp://127.0.0.1:%s", extract_count(getenv("XPUB_PORT"))); diff --git a/testing/btest/cluster/zeromq/logging.zeek b/testing/btest/cluster/zeromq/logging.zeek new file mode 100644 index 0000000000..377cf5da90 --- /dev/null +++ b/testing/btest/cluster/zeromq/logging.zeek @@ -0,0 +1,139 @@ +# @TEST-DOC: Startup a ZeroMQ cluster by hand, testing basic logging and node_up and node_down events. +# +# @TEST-REQUIRES: have-zeromq +# +# @TEST-GROUP: cluster-zeromq +# +# @TEST-PORT: XPUB_PORT +# @TEST-PORT: XSUB_PORT +# @TEST-PORT: LOG_PULL_PORT +# +# @TEST-EXEC: chmod +x ./check-cluster-log.sh +# +# @TEST-EXEC: cp $FILES/zeromq/cluster-layout-simple.zeek cluster-layout.zeek +# @TEST-EXEC: cp $FILES/zeromq/test-bootstrap.zeek zeromq-test-bootstrap.zeek +# +# @TEST-EXEC: btest-bg-run manager "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=manager zeek -b ../manager.zeek >out" +# @TEST-EXEC: btest-bg-run logger "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=logger zeek -b ../other.zeek >out" +# @TEST-EXEC: btest-bg-run proxy "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=proxy zeek -b ../other.zeek >out" +# @TEST-EXEC: btest-bg-run worker-1 "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=worker-1 zeek -b ../other.zeek >out" +# @TEST-EXEC: btest-bg-run worker-2 "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=worker-2 zeek -b ../other.zeek >out" +# +# @TEST-EXEC: btest-bg-wait 10 +# @TEST-EXEC: btest-diff cluster.log.normalized +# @TEST-EXEC: zeek-cut -F ' ' < ./logger/node_up.log | sort > node_up.sorted +# @TEST-EXEC: btest-diff node_up.sorted +# @TEST-EXEC: sort manager/out > manager.out +# @TEST-EXEC: btest-diff manager.out + +# @TEST-START-FILE common.zeek +@load ./zeromq-test-bootstrap + +redef Log::default_rotation_interval = 0sec; +redef Log::flush_interval = 0.01sec; + +type Info: record { + self: string &log &default=Cluster::node; + node: string &log; +}; + +redef enum Log::ID += { TEST_LOG }; + +global finish: event(name: string) &is_used; + +event zeek_init() { + print "A zeek_init", Cluster::node; + Log::create_stream(TEST_LOG, [$columns=Info, $path="node_up"]); +} + +event Cluster::node_up(name: string, id: string) &priority=-5 { + print "B node_up", name; + Log::write(TEST_LOG, [$node=name]); + # Log::flush(TEST_LOG); + # Log::flush(Cluster::LOG); +} +# @TEST-END-FILE + +# @TEST-START-FILE manager.zeek +@load ./common.zeek + +global nodes_up: set[string] = {"manager"}; +global nodes_down: set[string] = {"manager"}; + +event send_finish() { + print "C send_finish"; + for ( n in nodes_up ) + if ( n != "logger" ) + Cluster::publish(Cluster::node_topic(n), finish, Cluster::node); +} + +event check_cluster_log() { + if ( file_size("DONE") >= 0 ) { + event send_finish(); + return; + } + + system("../check-cluster-log.sh"); + schedule 0.1sec { check_cluster_log() }; +} + +event zeek_init() { + schedule 0.1sec { check_cluster_log() }; +} + +event Cluster::node_up(name: string, id: string) &priority=-1 { + add nodes_up[name]; + print "B nodes_up", |nodes_up|; +} + +event Cluster::node_down(name: string, id: string) { + print "D node_down", name; + add nodes_down[name]; + + if ( |nodes_down| == |Cluster::nodes| - 1 ) { + print "D send_finish to logger"; + Cluster::publish(Cluster::node_topic("logger"), finish, Cluster::node); + } + if ( |nodes_down| == |Cluster::nodes| ) + terminate(); +} +# @TEST-END-FILE + +# @TEST-START-FILE other.zeek +@load ./common.zeek + +event finish(name: string) { + print fmt("finish from %s", name); + terminate(); +} +# @TEST-END-FILE + +# @TEST-START-FILE check-cluster-log.sh +#!/bin/sh +# +# This script checks logger/cluster.log until the expected number +# of log entries have been observed and puts a normalized version +# into the testing directory for baselining. +CLUSTER_LOG=../logger/cluster.log + +if [ ! -f $CLUSTER_LOG ]; then + echo "$CLUSTER_LOG not found!" >&2 + exit 1; +fi + +if [ -f DONE ]; then + exit 0 +fi + +# Remove hostname and pid from node id in message. +zeek-cut node message < $CLUSTER_LOG | sed -r 's/_[^_]+_[0-9]+_/___/g' | sort > cluster.log.tmp + +# 4 times 5 +if [ $(wc -l < cluster.log.tmp) = 20 ]; then + echo "DONE!" >&2 + mv cluster.log.tmp ../cluster.log.normalized + echo "DONE" > DONE +fi + +exit 0 +# @TEST-END-FILE diff --git a/testing/btest/cluster/zeromq/manager-is-logger.zeek b/testing/btest/cluster/zeromq/manager-is-logger.zeek new file mode 100644 index 0000000000..c01a1e8f73 --- /dev/null +++ b/testing/btest/cluster/zeromq/manager-is-logger.zeek @@ -0,0 +1,129 @@ +# @TEST-DOC: Startup a ZeroMQ cluster without a logger, testing logging through the manager. +# +# @TEST-REQUIRES: have-zeromq +# +# @TEST-GROUP: cluster-zeromq +# +# @TEST-PORT: XPUB_PORT +# @TEST-PORT: XSUB_PORT +# @TEST-PORT: LOG_PULL_PORT +# +# @TEST-EXEC: chmod +x ./check-cluster-log.sh +# +# @TEST-EXEC: cp $FILES/zeromq/cluster-layout-no-logger.zeek cluster-layout.zeek +# @TEST-EXEC: cp $FILES/zeromq/test-bootstrap.zeek zeromq-test-bootstrap.zeek +# +# @TEST-EXEC: btest-bg-run manager "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=manager zeek -b ../manager.zeek >out" +# @TEST-EXEC: btest-bg-run proxy "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=proxy zeek -b ../other.zeek >out" +# @TEST-EXEC: btest-bg-run worker-1 "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=worker-1 zeek -b ../other.zeek >out" +# @TEST-EXEC: btest-bg-run worker-2 "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=worker-2 zeek -b ../other.zeek >out" +# +# @TEST-EXEC: btest-bg-wait 10 +# @TEST-EXEC: btest-diff cluster.log.normalized +# @TEST-EXEC: zeek-cut -F ' ' < ./manager/node_up.log | sort > node_up.sorted +# @TEST-EXEC: btest-diff node_up.sorted +# @TEST-EXEC: sort manager/out > manager.out +# @TEST-EXEC: btest-diff manager.out + +# @TEST-START-FILE common.zeek +@load ./zeromq-test-bootstrap + +redef Log::default_rotation_interval = 0sec; +redef Log::flush_interval = 0.01sec; + +type Info: record { + self: string &log &default=Cluster::node; + node: string &log; +}; + +redef enum Log::ID += { TEST_LOG }; + +global finish: event(name: string) &is_used; + +event zeek_init() { + print "zeek_init", Cluster::node; + Log::create_stream(TEST_LOG, [$columns=Info, $path="node_up"]); +} + +event Cluster::node_up(name: string, id: string) { + print "A node_up", name; + Log::write(TEST_LOG, [$node=name]); +} +# @TEST-END-FILE + +# @TEST-START-FILE manager.zeek +@load ./common.zeek + +global nodes_up: set[string] = {"manager"}; +global nodes_down: set[string] = {"manager"}; + +event send_finish() { + for ( n in nodes_up ) + Cluster::publish(Cluster::node_topic(n), finish, Cluster::node); +} + +event check_cluster_log() { + if ( file_size("DONE") >= 0 ) { + event send_finish(); + return; + } + + system("../check-cluster-log.sh"); + schedule 0.1sec { check_cluster_log() }; +} + +event zeek_init() { + schedule 0.1sec { check_cluster_log() }; +} + +event Cluster::node_up(name: string, id: string) { + add nodes_up[name]; + print "B nodes_up", |nodes_up|; +} + +event Cluster::node_down(name: string, id: string) { + print "D node_down", name; + add nodes_down[name]; + if ( |nodes_down| == |Cluster::nodes| ) + terminate(); +} +# @TEST-END-FILE + +# @TEST-START-FILE other.zeek +@load ./common.zeek + +event finish(name: string) { + print fmt("finish from %s", name); + terminate(); +} +# @TEST-END-FILE +# +# @TEST-START-FILE check-cluster-log.sh +#!/bin/sh +# +# This script checks cluster.log until the expected number +# of log entries have been observed and puts a normalized version +# into the testing directory for baselining. +CLUSTER_LOG=cluster.log + +if [ ! -f $CLUSTER_LOG ]; then + echo "$CLUSTER_LOG not found!" >&2 + exit 1; +fi + +if [ -f DONE ]; then + exit 0 +fi + +# Remove hostname and pid from node id in message. +zeek-cut node message < $CLUSTER_LOG | sed -r 's/_[^_]+_[0-9]+_/___/g' | sort > cluster.log.tmp + +# 4 times 3 +if [ $(wc -l < cluster.log.tmp) = 12 ]; then + echo "DONE!" >&2 + mv cluster.log.tmp ../cluster.log.normalized + echo "DONE" > DONE +fi + +exit 0 +# @TEST-END-FILE diff --git a/testing/btest/cluster/zeromq/supervisor.zeek b/testing/btest/cluster/zeromq/supervisor.zeek new file mode 100644 index 0000000000..62574e9867 --- /dev/null +++ b/testing/btest/cluster/zeromq/supervisor.zeek @@ -0,0 +1,86 @@ +# @TEST-DOC: Configure a ZeroMQ cluster with Zeek's supervisor. +# +# @TEST-REQUIRES: have-zeromq +# +# @TEST-GROUP: cluster-zeromq +# +# @TEST-PORT: XPUB_PORT +# @TEST-PORT: XSUB_PORT +# @TEST-PORT: LOG_PULL_PORT + +# @TEST-EXEC: chmod +x ./check-cluster-log.sh +# +# @TEST-EXEC: cp $FILES/zeromq/test-bootstrap.zeek zeromq-test-bootstrap.zeek +# +# @TEST-EXEC: btest-bg-run supervisor "ZEEKPATH=$ZEEKPATH:.. && zeek -j ../supervisor.zeek >out" +# @TEST-EXEC: btest-bg-wait 10 +# @TEST-EXEC: btest-diff supervisor/cluster.log + +redef Log::default_rotation_interval = 0secs; +redef Log::flush_interval = 0.01sec; + +@if ( ! Supervisor::is_supervisor() ) +@load ./zeromq-test-bootstrap +@else + +# The supervisor peeks into logger/cluster.log to initate a shutdown when +# all nodes have said hello to each other. See the check-cluster.log.sh +# script below. +event check_cluster_log() { + system_env("../check-cluster-log.sh", table(["SUPERVISOR_PID"] = cat(getpid()))); + + schedule 0.1sec { check_cluster_log() }; +} + +event zeek_init() + { + if ( ! Supervisor::is_supervisor() ) + return; + + Broker::listen("127.0.0.1", 9999/tcp); + + local cluster: table[string] of Supervisor::ClusterEndpoint; + cluster["manager"] = [$role=Supervisor::MANAGER, $host=127.0.0.1, $p=0/unknown]; + cluster["logger"] = [$role=Supervisor::LOGGER, $host=127.0.0.1, $p=to_port(getenv("LOG_PULL_PORT"))]; + cluster["proxy"] = [$role=Supervisor::PROXY, $host=127.0.0.1, $p=0/unknown]; + cluster["worker-1"] = [$role=Supervisor::WORKER, $host=127.0.0.1, $p=0/unknown]; + cluster["worker-2"] = [$role=Supervisor::WORKER, $host=127.0.0.1, $p=0/unknown]; + + for ( n, ep in cluster ) + { + local sn = Supervisor::NodeConfig($name=n, $bare_mode=T, $cluster=cluster, $directory=n); + local res = Supervisor::create(sn); + + if ( res != "" ) + print fmt("supervisor failed to create node '%s': %s", n, res); + } + + # Start polling the cluster.log + event check_cluster_log(); + } +@endif + +# @TEST-START-FILE check-cluster-log.sh +#!/bin/sh +# +# This script checks logger/cluster.log until the expected number +# of log entries have been observed and puts a normalized version +# into the current directory. This runs from the supervisor. +if [ ! -f logger/cluster.log ]; then + exit 1; +fi + +if [ -f DONE ]; then + exit 0 +fi + +# Remove hostname and pid from node id in message. +zeek-cut node message < logger/cluster.log | sed -r 's/_[^_]+_[0-9]+_/___/g' | sort > cluster.log + +if [ $(wc -l < cluster.log) = 20 ]; then + echo "DONE!" >&2 + # Trigger shutdown through supervisor. + kill ${ZEEK_ARG_SUPERVISOR_PID}; + echo "DONE" > DONE +fi +# @TEST-END-FILE diff --git a/testing/btest/cluster/zeromq/two-nodes.zeek b/testing/btest/cluster/zeromq/two-nodes.zeek new file mode 100644 index 0000000000..2fd01d7257 --- /dev/null +++ b/testing/btest/cluster/zeromq/two-nodes.zeek @@ -0,0 +1,53 @@ +# @TEST-DOC: Startup a manager running the ZeroMQ proxy thread, a worker connects and the manager sends a finish event to terminate the worker. +# +# @TEST-REQUIRES: have-zeromq +# +# @TEST-GROUP: cluster-zeromq +# +# @TEST-PORT: XPUB_PORT +# @TEST-PORT: XSUB_PORT +# @TEST-PORT: LOG_PULL_PORT +# +# @TEST-EXEC: cp $FILES/zeromq/cluster-layout-simple.zeek cluster-layout.zeek +# @TEST-EXEC: cp $FILES/zeromq/test-bootstrap.zeek zeromq-test-bootstrap.zeek +# +# @TEST-EXEC: btest-bg-run manager "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=manager zeek -b ../manager.zeek >out" +# @TEST-EXEC: btest-bg-run worker "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=worker-1 zeek -b ../worker.zeek >out" +# +# @TEST-EXEC: btest-bg-wait 30 +# @TEST-EXEC: btest-diff ./manager/out +# @TEST-EXEC: btest-diff ./worker/out + + +# @TEST-START-FILE common.zeek +@load ./zeromq-test-bootstrap + +global finish: event(name: string); +# @TEST-END-FILE + +# @TEST-START-FILE manager.zeek +@load ./common.zeek +# If a node comes up that isn't us, send it a finish event. +event Cluster::node_up(name: string, id: string) { + print "node_up", name; + Cluster::publish(Cluster::nodeid_topic(id), finish, Cluster::node); +} + +# If the worker vanishes, finish the test. +event Cluster::node_down(name: string, id: string) { + print "node_down", name; + terminate(); +} +# @TEST-END-FILE + +# @TEST-START-FILE worker.zeek +@load ./common.zeek + +event Cluster::node_up(name: string, id: string) { + print "node_up", name; +} + +event finish(name: string) &is_used { + terminate(); +} +# @TEST-END-FILE diff --git a/testing/scripts/have-zeromq b/testing/scripts/have-zeromq new file mode 100755 index 0000000000..4f52c32113 --- /dev/null +++ b/testing/scripts/have-zeromq @@ -0,0 +1,4 @@ +#!/bin/sh + +zeek -N Zeek::Cluster_Backend_ZeroMQ >/dev/null +exit $? From 34275afc1fc34658a19d8e366da6ae3a5bd48cd9 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Fri, 15 Nov 2024 12:29:58 +0100 Subject: [PATCH 44/65] ci: Add cppzmq and libzmq to most platforms --- ci/alpine/Dockerfile | 3 ++- ci/centos-stream-9/Dockerfile | 3 ++- ci/debian-11/Dockerfile | 3 ++- ci/debian-12/Dockerfile | 3 ++- ci/fedora-40/Dockerfile | 3 ++- ci/fedora-41/Dockerfile | 3 ++- ci/freebsd/prepare.sh | 2 +- ci/macos/prepare.sh | 2 +- ci/opensuse-leap-15.5/Dockerfile | 3 ++- ci/opensuse-leap-15.6/Dockerfile | 3 ++- ci/opensuse-tumbleweed/Dockerfile | 3 ++- ci/ubuntu-20.04/Dockerfile | 3 ++- ci/ubuntu-22.04/Dockerfile | 3 ++- ci/ubuntu-24.04/Dockerfile | 3 ++- ci/ubuntu-24.10/Dockerfile | 3 ++- 15 files changed, 28 insertions(+), 15 deletions(-) diff --git a/ci/alpine/Dockerfile b/ci/alpine/Dockerfile index ce82ab708f..7bb6e67901 100644 --- a/ci/alpine/Dockerfile +++ b/ci/alpine/Dockerfile @@ -2,7 +2,7 @@ FROM alpine:latest # A version field to invalidate Cirrus's build cache when needed, as suggested in # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822 -ENV DOCKERFILE_VERSION 20230823 +ENV DOCKERFILE_VERSION 20241024 RUN apk add --no-cache \ bash \ @@ -10,6 +10,7 @@ RUN apk add --no-cache \ bsd-compat-headers \ ccache \ cmake \ + cppzmq \ curl \ diffutils \ dnsmasq \ diff --git a/ci/centos-stream-9/Dockerfile b/ci/centos-stream-9/Dockerfile index 49414d69c6..ad8b6e7a70 100644 --- a/ci/centos-stream-9/Dockerfile +++ b/ci/centos-stream-9/Dockerfile @@ -2,7 +2,7 @@ FROM quay.io/centos/centos:stream9 # A version field to invalidate Cirrus's build cache when needed, as suggested in # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822 -ENV DOCKERFILE_VERSION 20230801 +ENV DOCKERFILE_VERSION 20241024 # dnf config-manager isn't available at first, and # we need it to install the CRB repo below. @@ -22,6 +22,7 @@ RUN dnf -y --nobest install \ bison \ ccache \ cmake \ + cppzmq-devel \ diffutils \ flex \ gcc \ diff --git a/ci/debian-11/Dockerfile b/ci/debian-11/Dockerfile index ea206a18e2..a4a5442b65 100644 --- a/ci/debian-11/Dockerfile +++ b/ci/debian-11/Dockerfile @@ -4,7 +4,7 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles" # A version field to invalidate Cirrus's build cache when needed, as suggested in # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822 -ENV DOCKERFILE_VERSION 20230801 +ENV DOCKERFILE_VERSION 20241024 RUN apt-get update && apt-get -y install \ bison \ @@ -22,6 +22,7 @@ RUN apt-get update && apt-get -y install \ libpcap-dev \ libssl-dev \ libuv1-dev \ + libzmq3-dev \ make \ python3 \ python3-dev \ diff --git a/ci/debian-12/Dockerfile b/ci/debian-12/Dockerfile index c3ae4339b7..cc94969f48 100644 --- a/ci/debian-12/Dockerfile +++ b/ci/debian-12/Dockerfile @@ -4,13 +4,14 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles" # A version field to invalidate Cirrus's build cache when needed, as suggested in # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822 -ENV DOCKERFILE_VERSION 20230801 +ENV DOCKERFILE_VERSION 20241024 RUN apt-get update && apt-get -y install \ bison \ bsdmainutils \ ccache \ cmake \ + cppzmq-dev \ curl \ dnsmasq \ flex \ diff --git a/ci/fedora-40/Dockerfile b/ci/fedora-40/Dockerfile index 45f209dbec..f2615b7007 100644 --- a/ci/fedora-40/Dockerfile +++ b/ci/fedora-40/Dockerfile @@ -2,12 +2,13 @@ FROM fedora:40 # A version field to invalidate Cirrus's build cache when needed, as suggested in # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822 -ENV DOCKERFILE_VERSION 20240617 +ENV DOCKERFILE_VERSION 20241024 RUN dnf -y install \ bison \ ccache \ cmake \ + cppzmq-devel \ diffutils \ dnsmasq \ flex \ diff --git a/ci/fedora-41/Dockerfile b/ci/fedora-41/Dockerfile index 7551a2739f..e98cd500bd 100644 --- a/ci/fedora-41/Dockerfile +++ b/ci/fedora-41/Dockerfile @@ -2,12 +2,13 @@ FROM fedora:41 # A version field to invalidate Cirrus's build cache when needed, as suggested in # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822 -ENV DOCKERFILE_VERSION 20241112 +ENV DOCKERFILE_VERSION 20241115 RUN dnf -y install \ bison \ ccache \ cmake \ + cppzmq-devel \ diffutils \ findutils \ flex \ diff --git a/ci/freebsd/prepare.sh b/ci/freebsd/prepare.sh index 5051084ac3..5fc425ddf4 100755 --- a/ci/freebsd/prepare.sh +++ b/ci/freebsd/prepare.sh @@ -6,7 +6,7 @@ set -e set -x env ASSUME_ALWAYS_YES=YES pkg bootstrap -pkg install -y bash git cmake swig bison python3 base64 flex ccache jq dnsmasq +pkg install -y bash cppzmq git cmake swig bison python3 base64 flex ccache jq dnsmasq pkg upgrade -y curl pyver=$(python3 -c 'import sys; print(f"py{sys.version_info[0]}{sys.version_info[1]}")') pkg install -y $pyver-sqlite3 diff --git a/ci/macos/prepare.sh b/ci/macos/prepare.sh index 7cc3e69cac..19272e3de0 100755 --- a/ci/macos/prepare.sh +++ b/ci/macos/prepare.sh @@ -7,7 +7,7 @@ set -x brew update brew upgrade cmake -brew install openssl@3 swig bison flex ccache libmaxminddb dnsmasq +brew install cppzmq openssl@3 swig bison flex ccache libmaxminddb dnsmasq if [ $(sw_vers -productVersion | cut -d '.' -f 1) -lt 14 ]; then python3 -m pip install --upgrade pip diff --git a/ci/opensuse-leap-15.5/Dockerfile b/ci/opensuse-leap-15.5/Dockerfile index b59f445a2a..58e247a0ea 100644 --- a/ci/opensuse-leap-15.5/Dockerfile +++ b/ci/opensuse-leap-15.5/Dockerfile @@ -2,7 +2,7 @@ FROM opensuse/leap:15.5 # A version field to invalidate Cirrus's build cache when needed, as suggested in # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822 -ENV DOCKERFILE_VERSION 20230905 +ENV DOCKERFILE_VERSION 20241024 RUN zypper addrepo https://download.opensuse.org/repositories/openSUSE:Leap:15.5:Update/standard/openSUSE:Leap:15.5:Update.repo \ && zypper refresh \ @@ -10,6 +10,7 @@ RUN zypper addrepo https://download.opensuse.org/repositories/openSUSE:Leap:15.5 bison \ ccache \ cmake \ + cppzmq-devel \ curl \ flex \ gcc12 \ diff --git a/ci/opensuse-leap-15.6/Dockerfile b/ci/opensuse-leap-15.6/Dockerfile index a40405e855..c49670ab51 100644 --- a/ci/opensuse-leap-15.6/Dockerfile +++ b/ci/opensuse-leap-15.6/Dockerfile @@ -2,7 +2,7 @@ FROM opensuse/leap:15.6 # A version field to invalidate Cirrus's build cache when needed, as suggested in # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822 -ENV DOCKERFILE_VERSION 20230905 +ENV DOCKERFILE_VERSION 20241024 RUN zypper addrepo https://download.opensuse.org/repositories/openSUSE:Leap:15.6:Update/standard/openSUSE:Leap:15.6:Update.repo \ && zypper refresh \ @@ -10,6 +10,7 @@ RUN zypper addrepo https://download.opensuse.org/repositories/openSUSE:Leap:15.6 bison \ ccache \ cmake \ + cppzmq-devel \ curl \ dnsmasq \ flex \ diff --git a/ci/opensuse-tumbleweed/Dockerfile b/ci/opensuse-tumbleweed/Dockerfile index c35a80205a..27191b665d 100644 --- a/ci/opensuse-tumbleweed/Dockerfile +++ b/ci/opensuse-tumbleweed/Dockerfile @@ -2,7 +2,7 @@ FROM opensuse/tumbleweed # A version field to invalidate Cirrus's build cache when needed, as suggested in # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822 -ENV DOCKERFILE_VERSION 20230801 +ENV DOCKERFILE_VERSION 20241024 # Remove the repo-openh264 repository, it caused intermittent issues # and we should not be needing any packages from it. @@ -14,6 +14,7 @@ RUN zypper refresh \ bison \ ccache \ cmake \ + cppzmq-devel \ curl \ diffutils \ dnsmasq \ diff --git a/ci/ubuntu-20.04/Dockerfile b/ci/ubuntu-20.04/Dockerfile index 6af6030de3..31e60690c7 100644 --- a/ci/ubuntu-20.04/Dockerfile +++ b/ci/ubuntu-20.04/Dockerfile @@ -4,7 +4,7 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles" # A version field to invalidate Cirrus's build cache when needed, as suggested in # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822 -ENV DOCKERFILE_VERSION 20240528 +ENV DOCKERFILE_VERSION 20241024 RUN apt-get update && apt-get -y install \ bc \ @@ -23,6 +23,7 @@ RUN apt-get update && apt-get -y install \ libmaxminddb-dev \ libpcap-dev \ libssl-dev \ + libzmq3-dev \ make \ python3 \ python3-dev \ diff --git a/ci/ubuntu-22.04/Dockerfile b/ci/ubuntu-22.04/Dockerfile index 12c5290aab..a34cf52619 100644 --- a/ci/ubuntu-22.04/Dockerfile +++ b/ci/ubuntu-22.04/Dockerfile @@ -4,7 +4,7 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles" # A version field to invalidate Cirrus's build cache when needed, as suggested in # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822 -ENV DOCKERFILE_VERSION 20230801 +ENV DOCKERFILE_VERSION 20241024 RUN apt-get update && apt-get -y install \ bc \ @@ -23,6 +23,7 @@ RUN apt-get update && apt-get -y install \ libmaxminddb-dev \ libpcap-dev \ libssl-dev \ + libzmq3-dev \ make \ python3 \ python3-dev \ diff --git a/ci/ubuntu-24.04/Dockerfile b/ci/ubuntu-24.04/Dockerfile index 89d999d19c..7b13638c30 100644 --- a/ci/ubuntu-24.04/Dockerfile +++ b/ci/ubuntu-24.04/Dockerfile @@ -4,7 +4,7 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles" # A version field to invalidate Cirrus's build cache when needed, as suggested in # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822 -ENV DOCKERFILE_VERSION 20240807 +ENV DOCKERFILE_VERSION 20241024 RUN apt-get update && apt-get -y install \ bc \ @@ -14,6 +14,7 @@ RUN apt-get update && apt-get -y install \ clang-18 \ clang++-18 \ cmake \ + cppzmq-dev \ curl \ dnsmasq \ flex \ diff --git a/ci/ubuntu-24.10/Dockerfile b/ci/ubuntu-24.10/Dockerfile index e31de81597..a937c7b3e7 100644 --- a/ci/ubuntu-24.10/Dockerfile +++ b/ci/ubuntu-24.10/Dockerfile @@ -4,7 +4,7 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles" # A version field to invalidate Cirrus's build cache when needed, as suggested in # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822 -ENV DOCKERFILE_VERSION 20240807 +ENV DOCKERFILE_VERSION 20241115 RUN apt-get update && apt-get -y install \ bc \ @@ -14,6 +14,7 @@ RUN apt-get update && apt-get -y install \ clang-18 \ clang++-18 \ cmake \ + cppzmq-dev \ curl \ dnsmasq \ flex \ From 2f37dcf505e7aefabf2d298a018d4114af4a88a5 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Fri, 15 Nov 2024 12:31:06 +0100 Subject: [PATCH 45/65] tsan_suppressions: Add called_from_lib: libzmq --- ci/tsan_suppressions.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ci/tsan_suppressions.txt b/ci/tsan_suppressions.txt index 7490adda94..c1a69a299f 100644 --- a/ci/tsan_suppressions.txt +++ b/ci/tsan_suppressions.txt @@ -46,3 +46,16 @@ deadlock:zeek::threading::Queue::LocksForAl # This only happens at shutdown. It was supposedly fixed in civetweb, but has cropped # up again. See https://github.com/civetweb/civetweb/issues/861 for details. race:mg_stop + +# Uninstrumented library. +# +# We'd need to build zmq with TSAN enabled, without it reports data races +# as it doesn't see the synchronization done [1], but also there's reports +# that ZeroMQ uses non-standard synchronization that may be difficult for +# TSAN to see. +# +# [1] https://groups.google.com/g/thread-sanitizer/c/7UZqM02yMYg/m/KlHOv2ckr9sJ +# [2] https://github.com/zeromq/libzmq/issues/3919 +# +called_from_lib:libzmq.so.5 +called_from_lib:libzmq.so From d946be878e544a5c58a55e4ae96be66c6c53657c Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Fri, 15 Nov 2024 12:33:00 +0100 Subject: [PATCH 46/65] docker: Add cppzmq/libzmq dependencies --- docker/builder.Dockerfile | 1 + docker/final.Dockerfile | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/builder.Dockerfile b/docker/builder.Dockerfile index 1223ddee93..72ca2f3eb3 100644 --- a/docker/builder.Dockerfile +++ b/docker/builder.Dockerfile @@ -21,6 +21,7 @@ RUN apt-get -q update \ bison \ ccache \ cmake \ + cppzmq-dev \ flex \ g++ \ gcc \ diff --git a/docker/final.Dockerfile b/docker/final.Dockerfile index 46a6b88ec2..e8bc922988 100644 --- a/docker/final.Dockerfile +++ b/docker/final.Dockerfile @@ -21,13 +21,14 @@ RUN apt-get -q update \ jq \ libmaxminddb0 \ libnode108 \ - libpython3.11 \ libpcap0.8 \ + libpython3.11 \ libssl3 \ libuv1 \ libz1 \ - python3-minimal \ + libzmq5 \ python3-git \ + python3-minimal \ python3-semantic-version \ python3-websocket \ && apt-get clean \ From df69ec12797c97c003a264fddf1c91e014bb7ef0 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Fri, 15 Nov 2024 12:33:37 +0100 Subject: [PATCH 47/65] generate-docs: Run on Ubuntu 24.04, add cppzmq --- .github/workflows/generate-docs.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml index 555961dd01..467df0ad29 100644 --- a/.github/workflows/generate-docs.yml +++ b/.github/workflows/generate-docs.yml @@ -17,7 +17,7 @@ jobs: permissions: contents: write # for Git to git push if: github.repository == 'zeek/zeek' - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: # We only perform a push if the action was triggered via a schedule @@ -51,6 +51,7 @@ jobs: bsdmainutils \ ccache \ cmake \ + cppzmq-dev \ flex \ g++ \ gcc \ @@ -71,7 +72,7 @@ jobs: # `python2` so this is a simple workaround until we drop Python 2 # support and explicitly use `python3` for all invocations. sudo ln -sf /usr/bin/python3 /usr/local/bin/python - sudo pip3 install -r doc/requirements.txt + sudo pip3 install --break-system-packages -r doc/requirements.txt - name: ccache uses: hendrikmuhs/ccache-action@v1.2 From d816bfb2493b2e96cfde5ce56cc439f23c243447 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Fri, 22 Nov 2024 12:13:08 +0100 Subject: [PATCH 48/65] btest/generic: Add publish_hrw(), publish_rr() and logging tests They currently use zeromq, but technically they should be valid for any other backend, too, even broker. --- .../.stderr | 4 + .../cluster.generic.logging-rr/rr2.log.sorted | 201 ++++++++++++++++++ .../manager.sorted | 42 ++++ .../worker-1.sorted | 16 ++ .../worker-2.sorted | 12 ++ .../cluster.generic.publish-rr/manager.sorted | 42 ++++ .../worker-1.sorted | 14 ++ .../worker-2.sorted | 14 ++ testing/btest/cluster/generic/logging-rr.zeek | 177 +++++++++++++++ .../btest/cluster/generic/publish-hrw.zeek | 101 +++++++++ testing/btest/cluster/generic/publish-rr.zeek | 101 +++++++++ 11 files changed, 724 insertions(+) create mode 100644 testing/btest/Baseline.zam/cluster.generic.cluster-publish-errors/.stderr create mode 100644 testing/btest/Baseline/cluster.generic.logging-rr/rr2.log.sorted create mode 100644 testing/btest/Baseline/cluster.generic.publish-hrw/manager.sorted create mode 100644 testing/btest/Baseline/cluster.generic.publish-hrw/worker-1.sorted create mode 100644 testing/btest/Baseline/cluster.generic.publish-hrw/worker-2.sorted create mode 100644 testing/btest/Baseline/cluster.generic.publish-rr/manager.sorted create mode 100644 testing/btest/Baseline/cluster.generic.publish-rr/worker-1.sorted create mode 100644 testing/btest/Baseline/cluster.generic.publish-rr/worker-2.sorted create mode 100644 testing/btest/cluster/generic/logging-rr.zeek create mode 100644 testing/btest/cluster/generic/publish-hrw.zeek create mode 100644 testing/btest/cluster/generic/publish-rr.zeek diff --git a/testing/btest/Baseline.zam/cluster.generic.cluster-publish-errors/.stderr b/testing/btest/Baseline.zam/cluster.generic.cluster-publish-errors/.stderr new file mode 100644 index 0000000000..77e6477e5b --- /dev/null +++ b/testing/btest/Baseline.zam/cluster.generic.cluster-publish-errors/.stderr @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/cluster-publish-errors.zeek, line 58: Publish of unknown record type 'Cluster::MyEvent' (Cluster::publish(topic, ::#0)) +error in <...>/cluster-publish-errors.zeek, line 65: Publish of unknown record type 'Cluster::MyEvent' (Cluster::publish_hrw(Cluster::proxy_pool, key, ::#0)) +error in <...>/cluster-publish-errors.zeek, line 72: Publish of unknown record type 'Cluster::MyEvent' (Cluster::publish_rr(Cluster::proxy_pool, key, ::#0)) diff --git a/testing/btest/Baseline/cluster.generic.logging-rr/rr2.log.sorted b/testing/btest/Baseline/cluster.generic.logging-rr/rr2.log.sorted new file mode 100644 index 0000000000..9b70ff71d4 --- /dev/null +++ b/testing/btest/Baseline/cluster.generic.logging-rr/rr2.log.sorted @@ -0,0 +1,201 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +0 worker-1 +0 worker-2 +1 worker-1 +1 worker-2 +2 worker-1 +2 worker-2 +3 worker-1 +3 worker-2 +4 worker-1 +4 worker-2 +5 worker-1 +5 worker-2 +6 worker-1 +6 worker-2 +7 worker-1 +7 worker-2 +8 worker-1 +8 worker-2 +9 worker-1 +9 worker-2 +10 worker-1 +10 worker-2 +11 worker-1 +11 worker-2 +12 worker-1 +12 worker-2 +13 worker-1 +13 worker-2 +14 worker-1 +14 worker-2 +15 worker-1 +15 worker-2 +16 worker-1 +16 worker-2 +17 worker-1 +17 worker-2 +18 worker-1 +18 worker-2 +19 worker-1 +19 worker-2 +20 worker-1 +20 worker-2 +21 worker-1 +21 worker-2 +22 worker-1 +22 worker-2 +23 worker-1 +23 worker-2 +24 worker-1 +24 worker-2 +25 worker-1 +25 worker-2 +26 worker-1 +26 worker-2 +27 worker-1 +27 worker-2 +28 worker-1 +28 worker-2 +29 worker-1 +29 worker-2 +30 worker-1 +30 worker-2 +31 worker-1 +31 worker-2 +32 worker-1 +32 worker-2 +33 worker-1 +33 worker-2 +34 worker-1 +34 worker-2 +35 worker-1 +35 worker-2 +36 worker-1 +36 worker-2 +37 worker-1 +37 worker-2 +38 worker-1 +38 worker-2 +39 worker-1 +39 worker-2 +40 worker-1 +40 worker-2 +41 worker-1 +41 worker-2 +42 worker-1 +42 worker-2 +43 worker-1 +43 worker-2 +44 worker-1 +44 worker-2 +45 worker-1 +45 worker-2 +46 worker-1 +46 worker-2 +47 worker-1 +47 worker-2 +48 worker-1 +48 worker-2 +49 worker-1 +49 worker-2 +50 worker-1 +50 worker-2 +51 worker-1 +51 worker-2 +52 worker-1 +52 worker-2 +53 worker-1 +53 worker-2 +54 worker-1 +54 worker-2 +55 worker-1 +55 worker-2 +56 worker-1 +56 worker-2 +57 worker-1 +57 worker-2 +58 worker-1 +58 worker-2 +59 worker-1 +59 worker-2 +60 worker-1 +60 worker-2 +61 worker-1 +61 worker-2 +62 worker-1 +62 worker-2 +63 worker-1 +63 worker-2 +64 worker-1 +64 worker-2 +65 worker-1 +65 worker-2 +66 worker-1 +66 worker-2 +67 worker-1 +67 worker-2 +68 worker-1 +68 worker-2 +69 worker-1 +69 worker-2 +70 worker-1 +70 worker-2 +71 worker-1 +71 worker-2 +72 worker-1 +72 worker-2 +73 worker-1 +73 worker-2 +74 worker-1 +74 worker-2 +75 worker-1 +75 worker-2 +76 worker-1 +76 worker-2 +77 worker-1 +77 worker-2 +78 worker-1 +78 worker-2 +79 worker-1 +79 worker-2 +80 worker-1 +80 worker-2 +81 worker-1 +81 worker-2 +82 worker-1 +82 worker-2 +83 worker-1 +83 worker-2 +84 worker-1 +84 worker-2 +85 worker-1 +85 worker-2 +86 worker-1 +86 worker-2 +87 worker-1 +87 worker-2 +88 worker-1 +88 worker-2 +89 worker-1 +89 worker-2 +90 worker-1 +90 worker-2 +91 worker-1 +91 worker-2 +92 worker-1 +92 worker-2 +93 worker-1 +93 worker-2 +94 worker-1 +94 worker-2 +95 worker-1 +95 worker-2 +96 worker-1 +96 worker-2 +97 worker-1 +97 worker-2 +98 worker-1 +98 worker-2 +99 worker-1 +99 worker-2 diff --git a/testing/btest/Baseline/cluster.generic.publish-hrw/manager.sorted b/testing/btest/Baseline/cluster.generic.publish-hrw/manager.sorted new file mode 100644 index 0000000000..e620f0a4b6 --- /dev/null +++ b/testing/btest/Baseline/cluster.generic.publish-hrw/manager.sorted @@ -0,0 +1,42 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +got pong, 0, args, worker-2, args +got pong, 0, args, worker-2, make_event +got pong, 0, make_event, worker-2, args +got pong, 0, make_event, worker-2, make_event +got pong, 1, args, worker-1, args +got pong, 1, args, worker-1, make_event +got pong, 1, make_event, worker-1, args +got pong, 1, make_event, worker-1, make_event +got pong, 2, args, worker-2, args +got pong, 2, args, worker-2, make_event +got pong, 2, make_event, worker-2, args +got pong, 2, make_event, worker-2, make_event +got pong, 3, args, worker-1, args +got pong, 3, args, worker-1, make_event +got pong, 3, make_event, worker-1, args +got pong, 3, make_event, worker-1, make_event +got pong, 4, args, worker-1, args +got pong, 4, args, worker-1, make_event +got pong, 4, make_event, worker-1, args +got pong, 4, make_event, worker-1, make_event +got pong, 5, args, worker-2, args +got pong, 5, args, worker-2, make_event +got pong, 5, make_event, worker-2, args +got pong, 5, make_event, worker-2, make_event +got pong, 6, args, worker-1, args +got pong, 6, args, worker-1, make_event +got pong, 6, make_event, worker-1, args +got pong, 6, make_event, worker-1, make_event +got pong, 7, args, worker-2, args +got pong, 7, args, worker-2, make_event +got pong, 7, make_event, worker-2, args +got pong, 7, make_event, worker-2, make_event +got pong, 8, args, worker-1, args +got pong, 8, args, worker-1, make_event +got pong, 8, make_event, worker-1, args +got pong, 8, make_event, worker-1, make_event +got pong, 9, args, worker-1, args +got pong, 9, args, worker-1, make_event +got pong, 9, make_event, worker-1, args +got pong, 9, make_event, worker-1, make_event +have 40, finish! diff --git a/testing/btest/Baseline/cluster.generic.publish-hrw/worker-1.sorted b/testing/btest/Baseline/cluster.generic.publish-hrw/worker-1.sorted new file mode 100644 index 0000000000..1f6b30f335 --- /dev/null +++ b/testing/btest/Baseline/cluster.generic.publish-hrw/worker-1.sorted @@ -0,0 +1,16 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +a node_up, manager +a node_up, worker-2 +got ping, 1, args +got ping, 1, make_event +got ping, 3, args +got ping, 3, make_event +got ping, 4, args +got ping, 4, make_event +got ping, 6, args +got ping, 6, make_event +got ping, 8, args +got ping, 8, make_event +got ping, 9, args +got ping, 9, make_event +z got finish! diff --git a/testing/btest/Baseline/cluster.generic.publish-hrw/worker-2.sorted b/testing/btest/Baseline/cluster.generic.publish-hrw/worker-2.sorted new file mode 100644 index 0000000000..6ad7198283 --- /dev/null +++ b/testing/btest/Baseline/cluster.generic.publish-hrw/worker-2.sorted @@ -0,0 +1,12 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +a node_up, manager +a node_up, worker-1 +got ping, 0, args +got ping, 0, make_event +got ping, 2, args +got ping, 2, make_event +got ping, 5, args +got ping, 5, make_event +got ping, 7, args +got ping, 7, make_event +z got finish! diff --git a/testing/btest/Baseline/cluster.generic.publish-rr/manager.sorted b/testing/btest/Baseline/cluster.generic.publish-rr/manager.sorted new file mode 100644 index 0000000000..a8e4c4926a --- /dev/null +++ b/testing/btest/Baseline/cluster.generic.publish-rr/manager.sorted @@ -0,0 +1,42 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +got pong, 0, args, worker-1, args +got pong, 0, args, worker-1, make_event +got pong, 0, make_event, worker-1, args +got pong, 0, make_event, worker-1, make_event +got pong, 1, args, worker-2, args +got pong, 1, args, worker-2, make_event +got pong, 1, make_event, worker-2, args +got pong, 1, make_event, worker-2, make_event +got pong, 2, args, worker-1, args +got pong, 2, args, worker-1, make_event +got pong, 2, make_event, worker-1, args +got pong, 2, make_event, worker-1, make_event +got pong, 3, args, worker-2, args +got pong, 3, args, worker-2, make_event +got pong, 3, make_event, worker-2, args +got pong, 3, make_event, worker-2, make_event +got pong, 4, args, worker-1, args +got pong, 4, args, worker-1, make_event +got pong, 4, make_event, worker-1, args +got pong, 4, make_event, worker-1, make_event +got pong, 5, args, worker-2, args +got pong, 5, args, worker-2, make_event +got pong, 5, make_event, worker-2, args +got pong, 5, make_event, worker-2, make_event +got pong, 6, args, worker-1, args +got pong, 6, args, worker-1, make_event +got pong, 6, make_event, worker-1, args +got pong, 6, make_event, worker-1, make_event +got pong, 7, args, worker-2, args +got pong, 7, args, worker-2, make_event +got pong, 7, make_event, worker-2, args +got pong, 7, make_event, worker-2, make_event +got pong, 8, args, worker-1, args +got pong, 8, args, worker-1, make_event +got pong, 8, make_event, worker-1, args +got pong, 8, make_event, worker-1, make_event +got pong, 9, args, worker-2, args +got pong, 9, args, worker-2, make_event +got pong, 9, make_event, worker-2, args +got pong, 9, make_event, worker-2, make_event +have 40, finish! diff --git a/testing/btest/Baseline/cluster.generic.publish-rr/worker-1.sorted b/testing/btest/Baseline/cluster.generic.publish-rr/worker-1.sorted new file mode 100644 index 0000000000..c5995b3c22 --- /dev/null +++ b/testing/btest/Baseline/cluster.generic.publish-rr/worker-1.sorted @@ -0,0 +1,14 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +a node_up, manager +a node_up, worker-2 +got ping, 0, args +got ping, 0, make_event +got ping, 2, args +got ping, 2, make_event +got ping, 4, args +got ping, 4, make_event +got ping, 6, args +got ping, 6, make_event +got ping, 8, args +got ping, 8, make_event +z got finish! diff --git a/testing/btest/Baseline/cluster.generic.publish-rr/worker-2.sorted b/testing/btest/Baseline/cluster.generic.publish-rr/worker-2.sorted new file mode 100644 index 0000000000..8ddface251 --- /dev/null +++ b/testing/btest/Baseline/cluster.generic.publish-rr/worker-2.sorted @@ -0,0 +1,14 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +a node_up, manager +a node_up, worker-1 +got ping, 1, args +got ping, 1, make_event +got ping, 3, args +got ping, 3, make_event +got ping, 5, args +got ping, 5, make_event +got ping, 7, args +got ping, 7, make_event +got ping, 9, args +got ping, 9, make_event +z got finish! diff --git a/testing/btest/cluster/generic/logging-rr.zeek b/testing/btest/cluster/generic/logging-rr.zeek new file mode 100644 index 0000000000..7ff18c18c4 --- /dev/null +++ b/testing/btest/cluster/generic/logging-rr.zeek @@ -0,0 +1,177 @@ +# @TEST-DOC: Testing round-robin of Log::write() across two loggers. +# +# @TEST-REQUIRES: have-zeromq +# +# @TEST-PORT: XPUB_PORT +# @TEST-PORT: XSUB_PORT +# @TEST-PORT: LOG_PULL_PORT_1 +# @TEST-PORT: LOG_PULL_PORT_2 +# +# @TEST-EXEC: chmod +x ./check-log.sh +# +# @TEST-EXEC: cp $FILES/zeromq/cluster-layout-two-loggers.zeek cluster-layout.zeek +# @TEST-EXEC: cp $FILES/zeromq/test-bootstrap.zeek zeromq-test-bootstrap.zeek +# +# @TEST-EXEC: zeek -b --parse-only common.zeek manager.zeek worker.zeek +# +# @TEST-EXEC: btest-bg-run manager "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=manager zeek -b ../manager.zeek >out" +# @TEST-EXEC: btest-bg-run logger-1 "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=logger-1 zeek -b ../common.zeek >out" +# @TEST-EXEC: btest-bg-run logger-2 "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=logger-2 zeek -b ../common.zeek >out" +# @TEST-EXEC: btest-bg-run worker-1 "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=worker-1 zeek -b ../worker.zeek >out" +# @TEST-EXEC: btest-bg-run worker-2 "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=worker-2 zeek -b ../worker.zeek >out" +# +# @TEST-EXEC: btest-bg-wait 10 +# +# @TEST-EXEC: test $(grep -c worker-1 logger-1/rr2.log) -gt 10 +# @TEST-EXEC: test $(grep -c worker-2 logger-1/rr2.log) -gt 10 +# @TEST-EXEC: test $(grep -c worker-1 logger-2/rr2.log) -gt 10 +# @TEST-EXEC: test $(grep -c worker-2 logger-2/rr2.log) -gt 10 + +# @TEST-EXEC: zeek-cut < logger-1/rr2.log > rr2.log +# @TEST-EXEC: zeek-cut < logger-2/rr2.log >> rr2.log +# @TEST-EXEC: sort -n rr2.log > rr2.log.sorted +# @TEST-EXEC: btest-diff rr2.log.sorted + +# @TEST-START-FILE common.zeek +@load ./zeromq-test-bootstrap.zeek + +redef Log::default_rotation_interval = 0sec; +redef Log::flush_interval = 0.03sec; +redef Log::write_buffer_size = 7; + +module LogRR; + +export { + redef enum Log::ID += { LOG1, LOG2 }; + type Info: record { + c: count &log; + from: string &log &default=Cluster::node; + }; + + global go: event(); + global finish: event(); +} + +event zeek_init() + { + Log::create_stream(LOG1, [$columns=Info, $path="rr1"]); + Log::create_stream(LOG2, [$columns=Info, $path="rr2"]); + } + +event finish() + { + terminate(); + } +# @TEST-END-FILE + +# @TEST-START-FILE manager.zeek +@load ./common.zeek + +event check_ready() + { + if ( ! piped_exec("../check-log.sh", "") ) + { + Reporter::error("check-log.sh failed"); + terminate(); + } + + if ( file_size("DONE") >= 0 ) + { + Cluster::publish(Cluster::worker_topic, LogRR::go); + return; + } + + schedule 0.1sec { check_ready() }; + } + +event zeek_init() + { + event check_ready(); + } + + +global nodes_down: set[string]; + +event Cluster::node_down(name: string, id: string) + { + print current_time(), "node_down", name; + add nodes_down[name]; + + if ( |nodes_down| == 2 ) # workers down + Cluster::publish(Cluster::logger_topic, LogRR::finish); + + if ( |nodes_down| == 4 ) # both loggers down + terminate(); + } +# @TEST-END-FILE + + +# @TEST-START-FILE worker.zeek +@load ./common.zeek + +global do_write2 = F; + +event write_log1(c: count) + { + if ( do_write2 ) + { + Log::write(LogRR::LOG1, [$c=10000000]); + return; + } + + Log::write(LogRR::LOG1, [$c=c]); + Log::flush(LogRR::LOG1); + schedule 0.05sec { write_log1(++c) }; + } + +event write_log2(c: count) + { + if ( c == 100 ) + { + terminate(); + return; + } + + Log::write(LogRR::LOG2, [$c=c]); + schedule 0.012sec { write_log2(++c) }; + } + +event LogRR::go() + { + do_write2 = T; + event write_log2(0); + } + +event zeek_init() + { + event write_log1(0); + } + +# @TEST-END-FILE + +@TEST-START-FILE check-log.sh +#!/usr/bin/env bash +# +# This script regularly checks for the loggers rr1.log file until +# both workers appear. Once this happens, creates a READY file +# which will result in workers getting the "go" and sending writes +# to rr2.log +set -eux + +LOGGERS="logger-1 logger-2" +WORKERS="worker-1 worker-2" + +for logger in $LOGGERS; do + for worker in $WORKERS; do + date +%s + echo check $logger $worker + if ! grep -q "${worker}" ../${logger}/rr1.log; then + exit 0 + fi + done +done + +echo "DONE" +echo "DONE" > DONE +exit 0 +@TEST-END-FILE diff --git a/testing/btest/cluster/generic/publish-hrw.zeek b/testing/btest/cluster/generic/publish-hrw.zeek new file mode 100644 index 0000000000..7e12e3f71f --- /dev/null +++ b/testing/btest/cluster/generic/publish-hrw.zeek @@ -0,0 +1,101 @@ +# @TEST-DOC: Send ping/pong using publish_hrw(), publish() and make_event() +# +# @TEST-REQUIRES: have-zeromq +# +# @TEST-PORT: XPUB_PORT +# @TEST-PORT: XSUB_PORT +# @TEST-PORT: LOG_PULL_PORT +# +# @TEST-EXEC: cp $FILES/zeromq/cluster-layout-no-logger.zeek cluster-layout.zeek +# @TEST-EXEC: cp $FILES/zeromq/test-bootstrap.zeek zeromq-test-bootstrap.zeek +# +# @TEST-EXEC: zeek -b --parse-only common.zeek manager.zeek worker.zeek +# +# @TEST-EXEC: btest-bg-run manager "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=manager zeek -b ../manager.zeek >out" +# @TEST-EXEC: btest-bg-run worker-1 "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=worker-1 zeek -b ../worker.zeek >out" +# @TEST-EXEC: btest-bg-run worker-2 "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=worker-2 zeek -b ../worker.zeek >out" +# +# @TEST-EXEC: btest-bg-wait 10 +# @TEST-EXEC: sort < ./manager/out > ./manager.sorted +# @TEST-EXEC: sort < ./worker-1/out > ./worker-1.sorted +# @TEST-EXEC: sort < ./worker-2/out > ./worker-2.sorted +# @TEST-EXEC: btest-diff manager.sorted +# @TEST-EXEC: btest-diff worker-1.sorted +# @TEST-EXEC: btest-diff worker-2.sorted + +# @TEST-START-FILE common.zeek +@load ./zeromq-test-bootstrap.zeek + +global finish: event(); +global ping: event(c: count, how: string); +global pong: event(c: count, how: string, from: string, from_how: string); +# @TEST-END-FILE + +# @TEST-START-FILE manager.zeek +@load ./common.zeek + +global nodes_up: set[string]; +global nodes_down: set[string]; +global pongs: set[count, string, string, string]; + +global i = 0; + +event send_hrw() + { + if (i >= 10 ) + return; + + Cluster::publish_hrw(Cluster::worker_pool, cat(i), ping, i, "args"); + local e = Cluster::make_event(ping, i, "make_event"); + Cluster::publish_hrw(Cluster::worker_pool, cat(i), e); + ++i; + + schedule 0.01sec { send_hrw() }; + } + +event pong(c: count, how: string, from: string, from_how: string) + { + print "got pong", c, how, from, from_how; + add pongs[c, how, from, from_how]; + + if ( |pongs| == 40 ) + { + print "have 40, finish!"; + Cluster::publish(Cluster::worker_topic, finish); + } + } + +event Cluster::node_up(name: string, id: string) { + add nodes_up[name]; + if ( |nodes_up| == 2 ) { + event send_hrw(); + } +} + +event Cluster::node_down(name: string, id: string) { + add nodes_down[name]; + if ( |nodes_down| == 2 ) + terminate(); +} +# @TEST-END-FILE + + +# @TEST-START-FILE worker.zeek +@load ./common.zeek + +event ping(c: count, how: string) { + print "got ping", c, how; + Cluster::publish(Cluster::manager_topic, pong, c, how, Cluster::node, "args"); + local e = Cluster::make_event(pong, c, how, Cluster::node, "make_event"); + Cluster::publish(Cluster::manager_topic, e); +} + +event Cluster::node_up(name: string, id: string) { + print "a node_up", name; +} + +event finish() &is_used { + print "z got finish!"; + terminate(); +} +# @TEST-END-FILE diff --git a/testing/btest/cluster/generic/publish-rr.zeek b/testing/btest/cluster/generic/publish-rr.zeek new file mode 100644 index 0000000000..c74fef4052 --- /dev/null +++ b/testing/btest/cluster/generic/publish-rr.zeek @@ -0,0 +1,101 @@ +# @TEST-DOC: Send ping/pong using publish_rr(), publish() and make_event() +# +# @TEST-REQUIRES: have-zeromq +# +# @TEST-PORT: XPUB_PORT +# @TEST-PORT: XSUB_PORT +# @TEST-PORT: LOG_PULL_PORT +# +# @TEST-EXEC: cp $FILES/zeromq/cluster-layout-no-logger.zeek cluster-layout.zeek +# @TEST-EXEC: cp $FILES/zeromq/test-bootstrap.zeek zeromq-test-bootstrap.zeek +# +# @TEST-EXEC: zeek -b --parse-only common.zeek manager.zeek worker.zeek +# +# @TEST-EXEC: btest-bg-run manager "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=manager zeek -b ../manager.zeek >out" +# @TEST-EXEC: btest-bg-run worker-1 "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=worker-1 zeek -b ../worker.zeek >out" +# @TEST-EXEC: btest-bg-run worker-2 "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=worker-2 zeek -b ../worker.zeek >out" +# +# @TEST-EXEC: btest-bg-wait 10 +# @TEST-EXEC: sort < ./manager/out > ./manager.sorted +# @TEST-EXEC: sort < ./worker-1/out > ./worker-1.sorted +# @TEST-EXEC: sort < ./worker-2/out > ./worker-2.sorted +# @TEST-EXEC: btest-diff manager.sorted +# @TEST-EXEC: btest-diff worker-1.sorted +# @TEST-EXEC: btest-diff worker-2.sorted + +# @TEST-START-FILE common.zeek +@load ./zeromq-test-bootstrap.zeek + +global finish: event(); +global ping: event(c: count, how: string); +global pong: event(c: count, how: string, from: string, from_how: string); +# @TEST-END-FILE + +# @TEST-START-FILE manager.zeek +@load ./common.zeek + +global nodes_up: set[string]; +global nodes_down: set[string]; +global pongs: set[count, string, string, string]; + +global i = 0; + +event send_rr() + { + if (i >= 10 ) + return; + + Cluster::publish_rr(Cluster::worker_pool, "ping-key-args", ping, i, "args"); + local e = Cluster::make_event(ping, i, "make_event"); + Cluster::publish_rr(Cluster::worker_pool, "ping-key-event", e); + ++i; + + schedule 0.01sec { send_rr() }; + } + +event pong(c: count, how: string, from: string, from_how: string) + { + print "got pong", c, how, from, from_how; + add pongs[c, how, from, from_how]; + + if ( |pongs| == 40 ) + { + print "have 40, finish!"; + Cluster::publish(Cluster::worker_topic, finish); + } + } + +event Cluster::node_up(name: string, id: string) { + add nodes_up[name]; + if ( |nodes_up| == 2 ) { + event send_rr(); + } +} + +event Cluster::node_down(name: string, id: string) { + add nodes_down[name]; + if ( |nodes_down| == 2 ) + terminate(); +} +# @TEST-END-FILE + + +# @TEST-START-FILE worker.zeek +@load ./common.zeek + +event ping(c: count, how: string) { + print "got ping", c, how; + Cluster::publish(Cluster::manager_topic, pong, c, how, Cluster::node, "args"); + local e = Cluster::make_event(pong, c, how, Cluster::node, "make_event"); + Cluster::publish(Cluster::manager_topic, e); +} + +event Cluster::node_up(name: string, id: string) { + print "a node_up", name; +} + +event finish() &is_used { + print "z got finish!"; + terminate(); +} +# @TEST-END-FILE From 07e23fb95ef003d1459ad60a974b19e9549208ce Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Wed, 4 Dec 2024 17:15:16 +0100 Subject: [PATCH 49/65] zeromq: Conditionally enable by default Instead of having ZeroMQ as a new dependency, enable the ZeroMQ backend only if ZeroMQ is available on the system as suggested by Tim. --- src/cluster/backend/CMakeLists.txt | 15 ++++++++++++++- src/cluster/backend/zeromq/CMakeLists.txt | 2 -- src/cluster/backend/zeromq/cmake/FindZeroMQ.cmake | 10 ++++------ 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/cluster/backend/CMakeLists.txt b/src/cluster/backend/CMakeLists.txt index 0e5d704186..109bdc411f 100644 --- a/src/cluster/backend/CMakeLists.txt +++ b/src/cluster/backend/CMakeLists.txt @@ -1,4 +1,17 @@ -option(ENABLE_CLUSTER_BACKEND_ZEROMQ "Enable the ZeroMQ cluster backend" ON) +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/zeromq/cmake") + +find_package(ZeroMQ) + +# Default to building ZeroMQ only if ZeroMQ was found. +# +# If a user enabled the cluster backend explicitly (-D ENABLE_CLUSTER_BACKEND_ZEROMQ:bool=ON), +# but ZeroMQ wasn' found, hard bail. +option(ENABLE_CLUSTER_BACKEND_ZEROMQ "Enable the ZeroMQ cluster backend" ${ZeroMQ_FOUND}) + if (ENABLE_CLUSTER_BACKEND_ZEROMQ) + if (NOT ZeroMQ_FOUND) + message(FATAL_ERROR "ENABLE_CLUSTER_BACKEND_ZEROMQ set, but ZeroMQ library not available") + endif () + add_subdirectory(zeromq) endif () diff --git a/src/cluster/backend/zeromq/CMakeLists.txt b/src/cluster/backend/zeromq/CMakeLists.txt index a15923445a..9895f4faf2 100644 --- a/src/cluster/backend/zeromq/CMakeLists.txt +++ b/src/cluster/backend/zeromq/CMakeLists.txt @@ -2,8 +2,6 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") find_package(ZeroMQ REQUIRED) -message(STATUS "zeromq: ${ZeroMQ_LIBRARIES} ${ZeroMQ_INCLUDE_DIRS}") - zeek_add_plugin( Zeek Cluster_Backend_ZeroMQ diff --git a/src/cluster/backend/zeromq/cmake/FindZeroMQ.cmake b/src/cluster/backend/zeromq/cmake/FindZeroMQ.cmake index 964f6a457d..92d1e94890 100644 --- a/src/cluster/backend/zeromq/cmake/FindZeroMQ.cmake +++ b/src/cluster/backend/zeromq/cmake/FindZeroMQ.cmake @@ -1,5 +1,7 @@ include(FindPackageHandleStandardArgs) +set(AUXIL_CPPZMQ_DIR ${CMAKE_CURRENT_LIST_DIR}/../auxil/cppzmq) + find_library(ZeroMQ_LIBRARY NAMES zmq HINTS ${ZeroMQ_ROOT_DIR}/lib) find_path(ZeroMQ_INCLUDE_DIR NAMES zmq.h HINTS ${ZeroMQ_ROOT_DIR}/include) @@ -31,18 +33,14 @@ endif () if (NOT ZeroMQ_CPP_VERSION) # Probably no zmq.hpp file, use the version from auxil - set(ZeroMQ_CPP_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/auxil/cppzmq" - CACHE FILEPATH "Include path for cppzmq" FORCE) + set(ZeroMQ_CPP_INCLUDE_DIR ${AUXIL_CPPZMQ_DIR} CACHE FILEPATH "Include path for cppzmq" FORCE) set_cppzmq_version() elseif (ZeroMQ_CPP_VERSION VERSION_LESS "4.9.0") message(STATUS "Found old cppzmq version ${ZeroMQ_CPP_VERSION}, using bundled version") - set(ZeroMQ_CPP_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/auxil/cppzmq" - CACHE FILEPATH "Include path for cppzmq" FORCE) + set(ZeroMQ_CPP_INCLUDE_DIR ${AUXIL_CPPZMQ_DIR} CACHE FILEPATH "Include path for cppzmq" FORCE) set_cppzmq_version() endif () -message(STATUS "Using cppzmq ${ZeroMQ_CPP_VERSION} from ${ZeroMQ_CPP_INCLUDE_DIR}") - find_package_handle_standard_args( ZeroMQ FOUND_VAR ZeroMQ_FOUND REQUIRED_VARS ZeroMQ_LIBRARY ZeroMQ_INCLUDE_DIR ZeroMQ_CPP_INCLUDE_DIR ZeroMQ_CPP_VERSION) From 9e9d0ba7aa677fdc30bed51f382e30701fa91214 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Wed, 4 Dec 2024 17:24:45 +0100 Subject: [PATCH 50/65] CMakeLists: Cluster backends output --- CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 42dc4f0a63..2aa1321a5d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1490,6 +1490,10 @@ message( "\n - debugging: ${USE_PERFTOOLS_DEBUG}" "\njemalloc: ${ENABLE_JEMALLOC}" "\n" + "\nCluster backends:" + "\n - Broker: ON" + "\n - ZeroMQ: ${ENABLE_CLUSTER_BACKEND_ZEROMQ}" + "\n" "\nFuzz Targets: ${ZEEK_ENABLE_FUZZERS}" "\nFuzz Engine: ${ZEEK_FUZZING_ENGINE}" "${_analyzer_warning}" From b93a4f5d2fcc1ef5b878f804de765a16202a35c6 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Tue, 10 Dec 2024 16:54:32 +0100 Subject: [PATCH 51/65] configure: Add --disable-cluster-backend-zeromq --- configure | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/configure b/configure index c38acf5980..707d6474ac 100755 --- a/configure +++ b/configure @@ -75,6 +75,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]... --disable-broker-tests don't try to build Broker unit tests --disable-btest don't install BTest --disable-btest-pcaps don't install Zeek's BTest input pcaps + --disable-cluster-backend-zeromq don't build Zeek's ZeroMQ cluster backend --disable-cpp-tests don't build Zeek's C++ unit tests --disable-javascript don't build Zeek's JavaScript support --disable-port-prealloc disable pre-allocating the PortVal array in ValManager @@ -333,6 +334,9 @@ while [ $# -ne 0 ]; do --disable-btest-pcaps) append_cache_entry INSTALL_BTEST_PCAPS BOOL false ;; + --disable-cluster-backend-zeromq) + append_cache_entry ENABLE_CLUSTER_BACKEND_ZEROMQ BOOL false + ;; --disable-cpp-tests) append_cache_entry ENABLE_ZEEK_UNIT_TESTS BOOL false ;; From 759281dabd0e50f677cbb70e07788e3c5731ad93 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Tue, 10 Dec 2024 19:51:45 +0100 Subject: [PATCH 52/65] Update ZAM BiF-tracking --- src/script_opt/FuncInfo.cc | 1 + testing/btest/Baseline/opt.ZAM-bif-tracking/output | 2 +- testing/btest/opt/ZAM-bif-tracking.zeek | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/script_opt/FuncInfo.cc b/src/script_opt/FuncInfo.cc index c21586f406..b244a7753c 100644 --- a/src/script_opt/FuncInfo.cc +++ b/src/script_opt/FuncInfo.cc @@ -72,6 +72,7 @@ static std::unordered_map func_attrs = { {"Analyzer::__register_for_port", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"Analyzer::__schedule_analyzer", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"Analyzer::__tag", ATTR_FOLDABLE}, + {"Cluster::Backend::ZeroMQ::spawn_zmq_proxy_thread", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"Cluster::Backend::__init", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"Cluster::__subscribe", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"Cluster::__unsubscribe", ATTR_NO_SCRIPT_SIDE_EFFECTS}, diff --git a/testing/btest/Baseline/opt.ZAM-bif-tracking/output b/testing/btest/Baseline/opt.ZAM-bif-tracking/output index d17a83be72..fc17f81c25 100644 --- a/testing/btest/Baseline/opt.ZAM-bif-tracking/output +++ b/testing/btest/Baseline/opt.ZAM-bif-tracking/output @@ -1,2 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -541 seen BiFs, 0 unseen BiFs (), 0 new BiFs () +542 seen BiFs, 0 unseen BiFs (), 0 new BiFs () diff --git a/testing/btest/opt/ZAM-bif-tracking.zeek b/testing/btest/opt/ZAM-bif-tracking.zeek index e886fb1ce0..1e46d837f1 100644 --- a/testing/btest/opt/ZAM-bif-tracking.zeek +++ b/testing/btest/opt/ZAM-bif-tracking.zeek @@ -1,6 +1,7 @@ # @TEST-DOC: ZAM maintenance script for tracking changes in BiFs. # # @TEST-REQUIRES: have-spicy +# @TEST-REQUIRES: have-zeromq # # @TEST-EXEC: zeek -b %INPUT >output # @TEST-EXEC: btest-diff output @@ -198,6 +199,7 @@ global known_BiFs = set( "Telemetry::__histogram_observe", "Telemetry::__histogram_sum", "WebSocket::__configure_analyzer", + "Cluster::Backend::ZeroMQ::spawn_zmq_proxy_thread", "__init_primary_bifs", "__init_secondary_bifs", "active_file", From f61ba4df29f35f9017168438ab608354e814259f Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Tue, 10 Dec 2024 14:45:41 -0700 Subject: [PATCH 53/65] Update zeek-testing and zeek-testing-cluster commit hashes --- testing/external/commit-hash.zeek-testing | 2 +- testing/external/commit-hash.zeek-testing-cluster | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/external/commit-hash.zeek-testing b/testing/external/commit-hash.zeek-testing index 0c933fa673..9805198913 100644 --- a/testing/external/commit-hash.zeek-testing +++ b/testing/external/commit-hash.zeek-testing @@ -1 +1 @@ -4359bd2c0e776dce08f7eca30d3d34cfe3e1d98b +2a63b457f24133a845c2020a321b7cbc05262291 diff --git a/testing/external/commit-hash.zeek-testing-cluster b/testing/external/commit-hash.zeek-testing-cluster index b129fa1026..c12bacc038 100644 --- a/testing/external/commit-hash.zeek-testing-cluster +++ b/testing/external/commit-hash.zeek-testing-cluster @@ -1 +1 @@ -aa361fc9f5fba202a9df68717a1d403be5f1e6b9 +43966c3a8c1a1a9d2cc3c77aebdbded602bf2cb3 From 567d2f356ba3f393536c6eb1b043bd6c193bacd9 Mon Sep 17 00:00:00 2001 From: zeek-bot Date: Wed, 11 Dec 2024 00:19:44 +0000 Subject: [PATCH 54/65] Update doc submodule [nomail] [skip ci] --- doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc b/doc index 27f321afb4..240a3d2d8c 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 27f321afb47feeffaa4534093d6abfc8e2b220af +Subproject commit 240a3d2d8cbbfcfd83d7e6a41c648d26ee4790d0 From 0ad32101773c5845ae7fc6014abd251c418c6eac Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Wed, 11 Dec 2024 15:34:07 +0100 Subject: [PATCH 55/65] Broker::publish: Warn on using Broker::publish() when inactive This is mostly for transitioning base scripts to Cluster::publish() and avoid silent surprises why certain things don't work when using ZeroMQ. --- src/broker/Manager.h | 2 +- src/broker/messaging.bif | 6 ++ .../..manager..stderr | 4 ++ .../..manager.out | 3 + .../..worker.out | 2 + .../btest/cluster/broker/publish-warning.zeek | 61 +++++++++++++++++++ 6 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 testing/btest/Baseline/cluster.broker.publish-warning/..manager..stderr create mode 100644 testing/btest/Baseline/cluster.broker.publish-warning/..manager.out create mode 100644 testing/btest/Baseline/cluster.broker.publish-warning/..worker.out create mode 100644 testing/btest/cluster/broker/publish-warning.zeek diff --git a/src/broker/Manager.h b/src/broker/Manager.h index 51974685d9..5b02a0ba71 100644 --- a/src/broker/Manager.h +++ b/src/broker/Manager.h @@ -109,7 +109,7 @@ public: /** * Returns true if any Broker communication is currently active. */ - [[deprecated("Remove with v8.1 - unused")]] bool Active(); + bool Active(); /** * Advances time. Broker data store expiration is driven by this diff --git a/src/broker/messaging.bif b/src/broker/messaging.bif index 96008861de..4347616a97 100644 --- a/src/broker/messaging.bif +++ b/src/broker/messaging.bif @@ -57,8 +57,14 @@ static bool publish_event_args(ArgsSpan args, const zeek::String* topic, zeek::detail::Frame* frame) { zeek::Broker::Manager::ScriptScopeGuard ssg; + zeek::ScriptLocationScope scope{frame}; + auto rval = false; + if ( zeek::broker_mgr != zeek::cluster::backend && ! zeek::broker_mgr->Active() ) + zeek::reporter->Warning("Non-broker cluster backend configured and Broker manager inactive. " + "Did you mean to use Cluster::publish() instead of Broker::publish()?"); + if ( args[0]->GetType()->Tag() == zeek::TYPE_RECORD ) rval = zeek::broker_mgr->PublishEvent(topic->CheckString(), args[0]->AsRecordVal()); diff --git a/testing/btest/Baseline/cluster.broker.publish-warning/..manager..stderr b/testing/btest/Baseline/cluster.broker.publish-warning/..manager..stderr new file mode 100644 index 0000000000..9e7871896a --- /dev/null +++ b/testing/btest/Baseline/cluster.broker.publish-warning/..manager..stderr @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +warning in ../manager.zeek, line 8: Non-broker cluster backend configured and Broker manager inactive. Did you mean to use Cluster::publish() instead of Broker::publish()? +warning in ../manager.zeek, line 16: Non-broker cluster backend configured and Broker manager inactive. Did you mean to use Cluster::publish() instead of Broker::publish()? +received termination signal diff --git a/testing/btest/Baseline/cluster.broker.publish-warning/..manager.out b/testing/btest/Baseline/cluster.broker.publish-warning/..manager.out new file mode 100644 index 0000000000..7e67339b4d --- /dev/null +++ b/testing/btest/Baseline/cluster.broker.publish-warning/..manager.out @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +node_up, worker-1 +node_down, worker-1 diff --git a/testing/btest/Baseline/cluster.broker.publish-warning/..worker.out b/testing/btest/Baseline/cluster.broker.publish-warning/..worker.out new file mode 100644 index 0000000000..386f8ae30f --- /dev/null +++ b/testing/btest/Baseline/cluster.broker.publish-warning/..worker.out @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +node_up, manager diff --git a/testing/btest/cluster/broker/publish-warning.zeek b/testing/btest/cluster/broker/publish-warning.zeek new file mode 100644 index 0000000000..38dff8e980 --- /dev/null +++ b/testing/btest/cluster/broker/publish-warning.zeek @@ -0,0 +1,61 @@ +# @TEST-DOC: When using ZeroMQ, Broker::publish() produces a warning. +# +# @TEST-REQUIRES: have-zeromq +# +# @TEST-GROUP: cluster-zeromq +# +# @TEST-PORT: XPUB_PORT +# @TEST-PORT: XSUB_PORT +# @TEST-PORT: LOG_PULL_PORT +# +# @TEST-EXEC: cp $FILES/zeromq/cluster-layout-simple.zeek cluster-layout.zeek +# @TEST-EXEC: cp $FILES/zeromq/test-bootstrap.zeek zeromq-test-bootstrap.zeek +# +# @TEST-EXEC: btest-bg-run manager "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=manager zeek -b ../manager.zeek >out" +# @TEST-EXEC: btest-bg-run worker "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=worker-1 zeek -b ../worker.zeek >out" +# +# @TEST-EXEC: btest-bg-wait 30 +# @TEST-EXEC: btest-diff ./manager/out +# @TEST-EXEC: btest-diff ./manager/.stderr +# @TEST-EXEC: btest-diff ./worker/out + + +# @TEST-START-FILE common.zeek +@load ./zeromq-test-bootstrap + +global finish: event(name: string); +# @TEST-END-FILE + +# @TEST-START-FILE manager.zeek +@load ./common.zeek +# If a node comes up that isn't us, send it a finish event. +event Cluster::node_up(name: string, id: string) { + print "node_up", name; + Cluster::publish(Cluster::nodeid_topic(id), finish, Cluster::node); + + # Also via broker, but this produces a warning which we test for. + Broker::publish(Cluster::nodeid_topic(id), finish, Cluster::node); +} + +# If the worker vanishes, finish the test. +event Cluster::node_down(name: string, id: string) { + print "node_down", name; + + # Do another Broker::publish(), just for the kicks. + Broker::publish(Cluster::nodeid_topic(id), finish, Cluster::node); + + terminate(); +} +# @TEST-END-FILE + +# @TEST-START-FILE worker.zeek +@load ./common.zeek + +event Cluster::node_up(name: string, id: string) { + print "node_up", name; +} + +event finish(name: string) &is_used { + terminate(); +} +# @TEST-END-FILE From c6c6d88b4c6cc75446443cabe2abcc4586471113 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Wed, 11 Dec 2024 17:11:00 +0100 Subject: [PATCH 56/65] btest/coverage: Avoid warnings in test-all-policy-cluster --- .../btest/Baseline/coverage.test-all-policy-cluster/.stderr | 4 ++++ testing/btest/coverage/test-all-policy-cluster.test | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/testing/btest/Baseline/coverage.test-all-policy-cluster/.stderr b/testing/btest/Baseline/coverage.test-all-policy-cluster/.stderr index 49d861c74c..bff9a64e41 100644 --- a/testing/btest/Baseline/coverage.test-all-policy-cluster/.stderr +++ b/testing/btest/Baseline/coverage.test-all-policy-cluster/.stderr @@ -1 +1,5 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +received termination signal +received termination signal +received termination signal +received termination signal diff --git a/testing/btest/coverage/test-all-policy-cluster.test b/testing/btest/coverage/test-all-policy-cluster.test index dc5e14e385..6aab29a1e7 100644 --- a/testing/btest/coverage/test-all-policy-cluster.test +++ b/testing/btest/coverage/test-all-policy-cluster.test @@ -16,6 +16,10 @@ @load test-all-policy +# Flip this to broker to avoid warnings() due Broker::publish() +# calls in some scripts. +redef Cluster::backend = Cluster::CLUSTER_BACKEND_BROKER; + @TEST-START-FILE cluster-layout.zeek redef Cluster::nodes = { ["manager"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=to_port(getenv("BROKER_PORT1"))], From 85189ca91841441e68f0ec6572f5322046c7ce36 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Wed, 11 Dec 2024 17:58:31 +0100 Subject: [PATCH 57/65] btest: Avoid loading ZeroMQ if not compiled in ...at the same time, add some `TEST-REQUIRES: have-zeromq` which unfortunately means that developers will usually want libzmq installed on their system. --- scripts/test-all-policy.zeek | 2 ++ scripts/zeekygen/__load__.zeek | 2 ++ testing/btest/coverage/bare-load-baseline.test | 1 + testing/btest/coverage/bare-mode-errors.test | 1 + testing/btest/coverage/default-load-baseline.test | 3 ++- testing/btest/plugins/hooks.zeek | 3 ++- 6 files changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/test-all-policy.zeek b/scripts/test-all-policy.zeek index 58930dc194..324d6745d6 100644 --- a/scripts/test-all-policy.zeek +++ b/scripts/test-all-policy.zeek @@ -11,9 +11,11 @@ # @load frameworks/control/controllee.zeek # @load frameworks/control/controller.zeek +@ifdef ( Cluster::CLUSTER_BACKEND_ZEROMQ ) @load frameworks/cluster/backend/zeromq/__load__.zeek # @load frameworks/cluster/backend/zeromq/connect.zeek @load frameworks/cluster/backend/zeromq/main.zeek +@endif @load frameworks/cluster/experimental.zeek # Loaded via the above through test-all-policy-cluster.test # when running as a manager, creates cluster.log entries diff --git a/scripts/zeekygen/__load__.zeek b/scripts/zeekygen/__load__.zeek index d22dba2a97..b7ac6992a6 100644 --- a/scripts/zeekygen/__load__.zeek +++ b/scripts/zeekygen/__load__.zeek @@ -2,7 +2,9 @@ # Scripts which are commented out in test-all-policy.zeek. @load protocols/ssl/decryption.zeek +@ifdef ( Cluster::CLUSTER_BACKEND_ZEROMQ ) @load frameworks/cluster/backend/zeromq/connect.zeek +@endif @load frameworks/cluster/nodes-experimental/manager.zeek @load frameworks/control/controllee.zeek @load frameworks/control/controller.zeek diff --git a/testing/btest/coverage/bare-load-baseline.test b/testing/btest/coverage/bare-load-baseline.test index 8b73fb125c..95f06a12eb 100644 --- a/testing/btest/coverage/bare-load-baseline.test +++ b/testing/btest/coverage/bare-load-baseline.test @@ -9,6 +9,7 @@ # below does. Don't ask. :-) # @TEST-REQUIRES: $SCRIPTS/have-spicy # This test logs loaded scripts, so disable it if Spicy and it associated plugin is unavailable. +# @TEST-REQUIRES: have-zeromq # Require ZeroMQ so that the plugin's bif file is loaded. # @TEST-REQUIRES: ! have-spicy-ssl # Enabling Spicy SSL changes the loaded scripts, skip in this case # @TEST-EXEC: zeek -b misc/loaded-scripts # @TEST-EXEC: test -e loaded_scripts.log diff --git a/testing/btest/coverage/bare-mode-errors.test b/testing/btest/coverage/bare-mode-errors.test index ea5b66955e..748a3f95e5 100644 --- a/testing/btest/coverage/bare-mode-errors.test +++ b/testing/btest/coverage/bare-mode-errors.test @@ -6,6 +6,7 @@ # # Require Spicy, otherwise its scripts cannot be loaded. # @TEST-REQUIRES: have-spicy +# @TEST-REQUIRES: have-zeromq # # @TEST-EXEC: test -d $DIST/scripts # @TEST-EXEC: for script in `find $DIST/scripts/ -name \*\.zeek`; do zeek -b --parse-only $script >>errors 2>&1; done diff --git a/testing/btest/coverage/default-load-baseline.test b/testing/btest/coverage/default-load-baseline.test index 2d098279fe..bbf79b6a52 100644 --- a/testing/btest/coverage/default-load-baseline.test +++ b/testing/btest/coverage/default-load-baseline.test @@ -7,7 +7,8 @@ # prefix to make the test work everywhere. That's what the sed magic # below does. Don't ask. :-) -# @TEST-REQUIRES: ${SCRIPTS}/have-spicy +# @TEST-REQUIRES: have-spicy +# @TEST-REQUIRES: have-zeromq # @TEST-REQUIRES: ! have-spicy-ssl # Enabling Spicy SSL changes the loaded scripts, skip in this case # @TEST-EXEC: zeek misc/loaded-scripts # @TEST-EXEC: test -e loaded_scripts.log diff --git a/testing/btest/plugins/hooks.zeek b/testing/btest/plugins/hooks.zeek index b65d321230..25e636a411 100644 --- a/testing/btest/plugins/hooks.zeek +++ b/testing/btest/plugins/hooks.zeek @@ -1,5 +1,6 @@ # @TEST-REQUIRES: test "${ZEEK_ZAM}" != "1" -# @TEST-REQUIRES: ${SCRIPTS}/have-spicy # This test logs loaded scripts, so disable it if Spicy and the associated plugin are unavailable. +# @TEST-REQUIRES: have-spicy # This test logs loaded scripts, so disable it if Spicy and the associated plugin are unavailable. +# @TEST-REQUIRES: have-zeromq # This test logs loaded scripts, so disable it if ZeroMQ isn't available. # @TEST-REQUIRES: ! have-spicy-ssl # Enabling Spicy SSL changes baselines and thus changes raised events. Skip in this case. # @TEST-EXEC: ${DIST}/auxil/zeek-aux/plugin-support/init-plugin -u . Demo Hooks # @TEST-EXEC: cp -r %DIR/hooks-plugin/* . From beb7b5bbbf20d3750a13a18686085f5c8a5ca58d Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Wed, 4 Dec 2024 12:19:17 -0700 Subject: [PATCH 58/65] CI: Install python 3.9 on ubuntu 20 --- ci/ubuntu-20.04/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/ubuntu-20.04/Dockerfile b/ci/ubuntu-20.04/Dockerfile index 31e60690c7..a643242370 100644 --- a/ci/ubuntu-20.04/Dockerfile +++ b/ci/ubuntu-20.04/Dockerfile @@ -25,8 +25,8 @@ RUN apt-get update && apt-get -y install \ libssl-dev \ libzmq3-dev \ make \ - python3 \ - python3-dev \ + python3.9 \ + python3.9-dev \ python3-pip\ ruby \ sqlite3 \ From 566fa7c6d0565c86ba8e07a4b91ae04b9b6eb1f6 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 5 Dec 2024 14:33:50 -0700 Subject: [PATCH 59/65] Require Python 3.9 in CMakeLists.txt --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2aa1321a5d..f2d1cd17a1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -772,7 +772,7 @@ if (NOT SED_EXE) endif () endif () -set(ZEEK_PYTHON_MIN 3.5.0) +set(ZEEK_PYTHON_MIN 3.9.0) set(Python_FIND_UNVERSIONED_NAMES FIRST) find_package(Python ${ZEEK_PYTHON_MIN} REQUIRED COMPONENTS Interpreter) find_package(FLEX REQUIRED) From 641306b912fee240e97a6d5e23767089e46b9304 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Wed, 4 Dec 2024 12:23:59 -0700 Subject: [PATCH 60/65] Upgrade btest submodule to get python upgrade --- auxil/btest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auxil/btest b/auxil/btest index 989c7513c3..9590947dc1 160000 --- a/auxil/btest +++ b/auxil/btest @@ -1 +1 @@ -Subproject commit 989c7513c3b6056a429a5d48dacdc9a2c1b216a7 +Subproject commit 9590947dc1d4e8096af21e344311c6b1d188d197 From 38fb76d52d04ead1fd60f0a1eda8499d303624ba Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Wed, 4 Dec 2024 12:24:46 -0700 Subject: [PATCH 61/65] Upgrade zeek-client submodule to get python upgrade --- auxil/zeek-client | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auxil/zeek-client b/auxil/zeek-client index 5bcc140851..1f249e911a 160000 --- a/auxil/zeek-client +++ b/auxil/zeek-client @@ -1 +1 @@ -Subproject commit 5bcc14085178ed4ddfa9ad972b441c36e8bc0787 +Subproject commit 1f249e911a1a4f7b90ec99e9aed8c3f7b7fcfb79 From c98ed621d64a197d154af895e2d17d277c05cc5a Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 9 Dec 2024 14:31:06 -0700 Subject: [PATCH 62/65] Upgrade zeekctl submodule to get python upgrade --- auxil/zeekctl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auxil/zeekctl b/auxil/zeekctl index 7e1a844808..67ae69914d 160000 --- a/auxil/zeekctl +++ b/auxil/zeekctl @@ -1 +1 @@ -Subproject commit 7e1a8448083ef0013f15e67ce001836e680589a2 +Subproject commit 67ae69914d78d987bffd7a6f22f0eead3772fe72 From e65e92edcf3787d3d23452faee6add9be80c85bd Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Tue, 10 Dec 2024 10:57:57 -0700 Subject: [PATCH 63/65] Upgrade broker submodule to get python upgrade --- auxil/broker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auxil/broker b/auxil/broker index 28cdb7524f..222571c9bc 160000 --- a/auxil/broker +++ b/auxil/broker @@ -1 +1 @@ -Subproject commit 28cdb7524f73ffa37315f4058f4f48948fe1683a +Subproject commit 222571c9bcbb84dcd68df5a02c91dec9988646d2 From b02f812e26f2d3580dea50750390a7f341ccb315 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Tue, 10 Dec 2024 12:22:17 -0700 Subject: [PATCH 64/65] Upgrade package-manager submodule to get python upgrade --- auxil/package-manager | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auxil/package-manager b/auxil/package-manager index bdc15fab95..ab6aff8929 160000 --- a/auxil/package-manager +++ b/auxil/package-manager @@ -1 +1 @@ -Subproject commit bdc15fab95b1ca2bd370fa25d91f7879b5da35fc +Subproject commit ab6aff89296d11363427beab34f88258c0abd467 From 49f82b325baefd330ddea9f9b66c4cdaa3bc9bec Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Tue, 10 Dec 2024 12:20:09 -0700 Subject: [PATCH 65/65] Swap pre-commit yapf for ruff/ruff-format, fix findings --- .pre-commit-config.yaml | 8 +++-- .style.yapf | 2 -- ci/collect-repo-info.py | 35 +++++++++++++------- ci/license-header.py | 6 ++-- ruff.toml | 8 +++++ src/make_dbg_constants.py | 49 ++++++++++++++-------------- testing/coverage/coverage_cleanup.py | 31 ++++++++---------- testing/scripts/coverage-calc | 14 ++++---- testing/scripts/httpd.py | 38 +++++++++++++-------- 9 files changed, 110 insertions(+), 81 deletions(-) delete mode 100644 .style.yapf create mode 100644 ruff.toml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 10b68d5bce..f6cfed7519 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,10 +28,12 @@ repos: - id: shfmt args: ["-w", "-i", "4", "-ci"] -- repo: https://github.com/google/yapf - rev: v0.43.0 +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.8.1 hooks: - - id: yapf + - id: ruff + args: [--fix] + - id: ruff-format - repo: https://github.com/cheshirekow/cmake-format-precommit rev: v0.6.13 diff --git a/.style.yapf b/.style.yapf deleted file mode 100644 index b05085101b..0000000000 --- a/.style.yapf +++ /dev/null @@ -1,2 +0,0 @@ -[style] -column_limit=100 diff --git a/ci/collect-repo-info.py b/ci/collect-repo-info.py index b8f0421a61..fdbbfb099f 100755 --- a/ci/collect-repo-info.py +++ b/ci/collect-repo-info.py @@ -12,8 +12,8 @@ import argparse import copy import json import logging -import pathlib import os +import pathlib import subprocess import sys @@ -38,14 +38,22 @@ def git_available(): def git_is_repo(d: pathlib.Path): try: - git("-C", str(d), "rev-parse", "--is-inside-work-tree", stderr=subprocess.DEVNULL) + git( + "-C", + str(d), + "rev-parse", + "--is-inside-work-tree", + stderr=subprocess.DEVNULL, + ) return True except subprocess.CalledProcessError: return False def git_is_dirty(d: pathlib.Path): - return (len(git("-C", str(d), "status", "--untracked=no", "--short").splitlines()) > 0) + return ( + len(git("-C", str(d), "status", "--untracked=no", "--short").splitlines()) > 0 + ) def git_generic_info(d: pathlib.Path): @@ -111,7 +119,9 @@ def collect_git_info(zeek_dir: pathlib.Path): info["name"] = "zeek" info["version"] = (zeek_dir / "VERSION").read_text().strip() info["submodules"] = collect_submodule_info(zeek_dir) - info["branch"] = git("-C", str(zeek_dir), "rev-parse", "--abbrev-ref", "HEAD").strip() + info["branch"] = git( + "-C", str(zeek_dir), "rev-parse", "--abbrev-ref", "HEAD" + ).strip() info["source"] = "git" return info @@ -156,14 +166,13 @@ def main(): for p in [p.strip() for p in v.split(";") if p.strip()]: yield pathlib.Path(p) - parser.add_argument("included_plugin_dirs", - default="", - nargs="?", - type=included_plugin_dir_conv) + parser.add_argument( + "included_plugin_dirs", default="", nargs="?", type=included_plugin_dir_conv + ) parser.add_argument("--dir", default=".") - parser.add_argument("--only-git", - action="store_true", - help="Do not try repo-info.json fallback") + parser.add_argument( + "--only-git", action="store_true", help="Do not try repo-info.json fallback" + ) args = parser.parse_args() logging.basicConfig(format="%(levelname)s: %(message)s") @@ -210,7 +219,9 @@ def main(): zkg_provides_info = copy.deepcopy(included_plugins_info) # Hardcode the former spicy-plugin so that zkg knows Spicy is available. - zkg_provides_info.append({"name": "spicy-plugin", "version": info["version"].split("-")[0]}) + zkg_provides_info.append( + {"name": "spicy-plugin", "version": info["version"].split("-")[0]} + ) info["zkg"] = {"provides": zkg_provides_info} json_str = json.dumps(info, indent=2, sort_keys=True) diff --git a/ci/license-header.py b/ci/license-header.py index 2af95aba0c..985c29a62f 100755 --- a/ci/license-header.py +++ b/ci/license-header.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 -import sys import re +import sys exit_code = 0 -pat1 = re.compile(r"See the file \"COPYING\" in the main distribution directory for copyright.") +pat1 = re.compile( + r"See the file \"COPYING\" in the main distribution directory for copyright." +) # This is the copyright line used within Spicy plugin and popular in # Spicy analyzers. diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000000..e3ce9bb17e --- /dev/null +++ b/ruff.toml @@ -0,0 +1,8 @@ +target-version = "py39" + +# Skip anything in the auxil directory. This includes pysubnetree which +# should be handled separately. +exclude = ["auxil"] + +[lint] +select = ["C4", "F", "I", "ISC", "UP"] diff --git a/src/make_dbg_constants.py b/src/make_dbg_constants.py index 3b5d7f2da6..7b2d0b3ba8 100644 --- a/src/make_dbg_constants.py +++ b/src/make_dbg_constants.py @@ -12,10 +12,10 @@ # # The input format is: # -# cmd: [DebugCmd] -# names: [space delimited names of cmd] -# resume: ['true' or 'false': should execution resume after this command?] -# help: [some help text] +# cmd: [DebugCmd] +# names: [space delimited names of cmd] +# resume: ['true' or 'false': should execution resume after this command?] +# help: [some help text] # # Blank lines are skipped. # Comments should start with // and should be on a line by themselves. @@ -24,7 +24,7 @@ import sys inputfile = sys.argv[1] -init_tmpl = ''' +init_tmpl = """ \t{ \t\tDebugCmdInfo* info; \t\t%(name_init)s @@ -32,36 +32,35 @@ init_tmpl = ''' \t\t %(repeatable)s); \t\tg_DebugCmdInfos.push_back(info); \t} -''' +""" -enum_str = ''' +enum_str = f""" // -// This file was automatically generated from %s +// This file was automatically generated from {inputfile} // DO NOT EDIT. // -enum DebugCmd { -''' % inputfile +enum DebugCmd {{ +""" -init_str = ''' +init_str = f""" // -// This file was automatically generated from %s +// This file was automatically generated from {inputfile} // DO NOT EDIT. // #include "zeek/util.h" -namespace zeek::detail {\n -void init_global_dbg_constants () { -''' % inputfile +namespace zeek::detail {{\n +void init_global_dbg_constants () {{ +""" def outputrecord(): global init_str, enum_str if dbginfo["names"]: - dbginfo["name_init"] = "const char * const names[] = {\n"\ - "\t\t\t%s\n"\ - "\t\t};\n" \ - % ",\n\t\t\t".join(dbginfo["names"]) + dbginfo["name_init"] = ( + "const char * const names[] = {{\n\t\t\t{}\n\t\t}};\n" + ).format(",\n\t\t\t".join(dbginfo["names"])) else: dbginfo["name_init"] = "const char * const names[] = { };\n" @@ -70,7 +69,7 @@ def outputrecord(): # substitute into template init_str += init_tmpl % dbginfo - enum_str += "\t%s,\n" % dbginfo["cmd"] + enum_str += "\t{},\n".format(dbginfo["cmd"]) def initdbginfo(): @@ -81,13 +80,13 @@ def initdbginfo(): "names": [], "resume": "false", "help": "", - "repeatable": "false" + "repeatable": "false", } dbginfo = initdbginfo() -inputf = open(inputfile, "r") +inputf = open(inputfile) for line in inputf: line = line.strip() if not line or line.startswith("//"): # skip empty lines and comments @@ -95,7 +94,7 @@ for line in inputf: fields = line.split(":", 1) if len(fields) != 2: - raise RuntimeError("Error in debug constant file on line: %s" % line) + raise RuntimeError(f"Error in debug constant file on line: {line}") f1, f2 = fields f2 = f2.strip() @@ -108,13 +107,13 @@ for line in inputf: dbginfo[f1] = f2 elif f1 == "names": # put quotes around the strings - dbginfo[f1] = ['"%s"' % n for n in f2.split()] + dbginfo[f1] = [f'"{n}"' for n in f2.split()] elif f1 == "help": dbginfo[f1] = f2.replace('"', '\\"') # escape quotation marks elif f1 in ("resume", "repeatable"): dbginfo[f1] = f2 else: - raise RuntimeError("Unknown command: %s" % line) + raise RuntimeError(f"Unknown command: {line}") # output the last record outputrecord() diff --git a/testing/coverage/coverage_cleanup.py b/testing/coverage/coverage_cleanup.py index 6137ec9f01..13422ae151 100755 --- a/testing/coverage/coverage_cleanup.py +++ b/testing/coverage/coverage_cleanup.py @@ -6,28 +6,26 @@ if len(sys.argv) != 2: print("Expected one argument containing the file to clean") sys.exit(-1) -with open(sys.argv[1], 'r') as f: - +with open(sys.argv[1]) as f: files = {} - cur_file = '' + cur_file = "" lines = f.readlines() for line in lines: - - if line == 'end_of_record': - cur_file = '' + if line == "end_of_record": + cur_file = "" continue - parts = line.split(':', 1) - if parts[0] == 'SF': + parts = line.split(":", 1) + if parts[0] == "SF": cur_file = parts[1].strip() - while cur_file.find('src/zeek/') != -1: - cur_file = cur_file.replace('src/zeek/', 'src/', 1) + while cur_file.find("src/zeek/") != -1: + cur_file = cur_file.replace("src/zeek/", "src/", 1) if cur_file not in files: files[cur_file] = {} - elif parts[0] == 'DA': - da_parts = parts[1].split(',') + elif parts[0] == "DA": + da_parts = parts[1].split(",") line = int(da_parts[0]) count = int(da_parts[1]) @@ -35,13 +33,12 @@ with open(sys.argv[1], 'r') as f: files[cur_file][line] = count for name in files: - - print('TN:') - print('SF:{}'.format(name)) + print("TN:") + print(f"SF:{name}") das = list(files[name].keys()) das.sort() for da in das: - print('DA:{},{}'.format(da, files[name][da])) - print('end_of_record') + print(f"DA:{da},{files[name][da]}") + print("end_of_record") diff --git a/testing/scripts/coverage-calc b/testing/scripts/coverage-calc index 0d5f2d8269..37acf15bd5 100755 --- a/testing/scripts/coverage-calc +++ b/testing/scripts/coverage-calc @@ -10,9 +10,9 @@ # that are not part of the distribution and which should not count towards # the coverage calculation. +import glob import os import sys -import glob stats = {} inputglob = sys.argv[1] @@ -20,7 +20,7 @@ outputfile = sys.argv[2] scriptdir = os.path.abspath(sys.argv[3]) for filename in glob.glob(inputglob): - with open(filename, 'r') as f: + with open(filename) as f: for line in f.read().splitlines(): parts = line.split("\t") exec_count = int(parts[0]) @@ -34,7 +34,7 @@ for filename in glob.glob(inputglob): srclines = srclines.split()[1] # For sorting purposes (so that line numbers get sorted correctly), # construct a specially-formatted key string. - sortkey = filepath + ", line " + ("%6s" % srclines.split("-")[0]) + sortkey = filepath + ", line " + ("{:<6s}".format(srclines.split("-")[0])) location = filepath + ", line " + srclines desc = parts[2] # Keying by location + desc may result in duplicate data @@ -46,9 +46,9 @@ for filename in glob.glob(inputglob): else: stats[key] = [exec_count, location, desc, sortkey] -with open(outputfile, 'w') as f: +with open(outputfile, "w") as f: for k in sorted(stats, key=lambda i: stats[i][3]): - f.write("%s\t%s\t%s\n" % (stats[k][0], stats[k][1], stats[k][2])) + f.write(f"{stats[k][0]}\t{stats[k][1]}\t{stats[k][2]}\n") num_covered = 0 for k in stats: @@ -56,5 +56,5 @@ for k in stats: num_covered += 1 if len(stats) > 0: - print("%s/%s (%.1f%%) Zeek script statements covered." % - (num_covered, len(stats), float(num_covered) / len(stats) * 100)) + pct = float(num_covered) / len(stats) * 100 + print(f"{num_covered}/{len(stats)} ({pct:.1f}%) Zeek script statements covered.") diff --git a/testing/scripts/httpd.py b/testing/scripts/httpd.py index b10c193107..53eff7c40a 100755 --- a/testing/scripts/httpd.py +++ b/testing/scripts/httpd.py @@ -4,7 +4,6 @@ import http.server as BaseHTTPServer class MyRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): - def do_GET(self): self.send_response(200) self.send_header("Content-type", "text/plain") @@ -34,19 +33,32 @@ class MyRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): if __name__ == "__main__": from optparse import OptionParser + p = OptionParser() - p.add_option("-a", - "--addr", - type="string", - default="localhost", - help=("listen on given address (numeric IP or host name), " - "an empty string (the default) means INADDR_ANY")) - p.add_option("-p", "--port", type="int", default=32123, help="listen on given TCP port number") - p.add_option("-m", - "--max", - type="int", - default=-1, - help="max number of requests to respond to, -1 means no max") + p.add_option( + "-a", + "--addr", + type="string", + default="localhost", + help=( + "listen on given address (numeric IP or host name), " + "an empty string (the default) means INADDR_ANY" + ), + ) + p.add_option( + "-p", + "--port", + type="int", + default=32123, + help="listen on given TCP port number", + ) + p.add_option( + "-m", + "--max", + type="int", + default=-1, + help="max number of requests to respond to, -1 means no max", + ) options, args = p.parse_args() httpd = BaseHTTPServer.HTTPServer((options.addr, options.port), MyRequestHandler)