Merge remote-tracking branch 'origin/topic/awelzel/log-broker-io-telemetry-rules-includes'

* origin/topic/awelzel/log-broker-io-telemetry-rules-includes:
  zeek-setup: Remove some unused headers
  clang-format: Sort doctest header at the bottom
  RuleMatcher: Move plugin/Manager.h include from .h to .cc
  iosource/Manager: Remove superflous includes
  telemetry/Manager: Remove broker header include, add fnmatch.h
  logging/Manager: Fix using filename from input.h in debug log

.clang-format

@@ -71,6 +71,7 @@ IncludeBlocks: Regroup
 # 4: any header that starts with "zeek/"
 # 5: everything else, which should catch any of the auto-generated code from the
 #    build directory as well
+# 6: third party doctest header
 #
 # Sections 0-1 and 2-3 get grouped together in their respective blocks
 IncludeCategories:
@@ -86,6 +87,8 @@ IncludeCategories:
   - Regex: '^<[[:print:]]+>'
     Priority: 2
     SortPriority: 3
+  - Regex: '^"zeek/3rdparty/doctest.h'
+    Priority: 6
   - Regex: '^"zeek/'
     Priority: 4
   - Regex: '.*'

CHANGES

@@ -1,3 +1,31 @@
+7.1.0-dev.563 | 2024-11-15 17:39:49 +0100
+
+  * zeek-setup: Remove some unused headers (Arne Welzel, Corelight)
+
+    The zeek_binpac.h one isn't used directly, but keeping it over
+    <binpac.h> include. Also do some std prefixing. binpac.h has
+    'using namespace std', so these slip through :-(
+
+  * clang-format: Sort doctest header at the bottom (Arne Welzel, Corelight)
+
+  * RuleMatcher: Move plugin/Manager.h include from .h to .cc (Arne Welzel, Corelight)
+
+  * iosource/Manager: Remove superflous includes (Arne Welzel, Corelight)
+
+  * telemetry/Manager: Remove broker header include, add fnmatch.h (Arne Welzel, Corelight)
+
+  * logging/Manager: Fix using filename from input.h in debug log (Arne Welzel, Corelight)
+
+    ...and remove network_time, it's always included.
+
+  * Update external commit hashes (Arne Welzel, Corelight)
+
+  * communityid: Do not include ports for non TCP, UDP, ICMP (Arne Welzel, Corelight)
+
+    Checked against the result of pycommunityid. The SCTP case
+    isn't quite right, because Zeek's core will not have extracted
+    any ports for SCTP.
+
 7.1.0-dev.552 | 2024-11-14 14:15:24 +0100
 
   * sumstats: Remove copy() for Broker::publish() calls (Arne Welzel, Corelight)

VERSION

@@ -1 +1 @@
-7.1.0-dev.552
+7.1.0-dev.563

src/DNS_Mapping.cc

@@ -2,10 +2,11 @@
 
 #include <ares_nameser.h>
 
-#include "zeek/3rdparty/doctest.h"
 #include "zeek/DNS_Mgr.h"
 #include "zeek/Reporter.h"
 
+#include "zeek/3rdparty/doctest.h"
+
 namespace zeek::detail {
 
 DNS_Mapping::DNS_Mapping(std::string host, struct hostent* h, uint32_t ttl, int type) {

src/DNS_Mgr.cc

@@ -32,7 +32,6 @@ using ztd::out_ptr::out_ptr;
 #include <ares_dns.h>
 #include <ares_nameser.h>
 
-#include "zeek/3rdparty/doctest.h"
 #include "zeek/DNS_Mapping.h"
 #include "zeek/Event.h"
 #include "zeek/Expr.h"
@@ -47,6 +46,8 @@ using ztd::out_ptr::out_ptr;
 #include "zeek/iosource/Manager.h"
 #include "zeek/telemetry/Manager.h"
 
+#include "zeek/3rdparty/doctest.h"
+
 // Number of seconds we'll wait for a reply.
 constexpr int DNS_TIMEOUT = 5;
 

src/Dict.cc

@@ -2,9 +2,10 @@
 
 #include "zeek/Dict.h"
 
-#include "zeek/3rdparty/doctest.h"
 #include "zeek/Hash.h"
 
+#include "zeek/3rdparty/doctest.h"
+
 namespace zeek {
 
 // namespace detail

src/Hash.cc

@@ -8,7 +8,6 @@
 #include <highwayhash/instruction_sets.h>
 #include <highwayhash/sip_hash.h>
 
-#include "zeek/3rdparty/doctest.h"
 #include "zeek/DebugLogger.h"
 #include "zeek/Desc.h"
 #include "zeek/Reporter.h"
@@ -18,6 +17,8 @@
 
 #include "const.bif.netvar_h"
 
+#include "zeek/3rdparty/doctest.h"
+
 namespace zeek::detail {
 
 alignas(32) uint64_t KeyedHash::shared_highwayhash_key[4];

src/RE.cc

@@ -7,13 +7,14 @@
 #include <cstdlib>
 #include <utility>
 
-#include "zeek/3rdparty/doctest.h"
 #include "zeek/CCL.h"
 #include "zeek/DFA.h"
 #include "zeek/EquivClass.h"
 #include "zeek/Reporter.h"
 #include "zeek/ZeekString.h"
 
+#include "zeek/3rdparty/doctest.h"
+
 zeek::detail::CCL* zeek::detail::curr_ccl = nullptr;
 zeek::detail::Specific_RE_Matcher* zeek::detail::rem = nullptr;
 zeek::detail::NFA_Machine* zeek::detail::nfa = nullptr;

src/Reporter.cc

@@ -9,7 +9,6 @@
 #include <syslog.h>
 #include <unistd.h>
 
-#include "zeek/3rdparty/doctest.h"
 #include "zeek/Conn.h"
 #include "zeek/Desc.h"
 #include "zeek/Event.h"
@@ -24,6 +23,8 @@
 #include "zeek/plugin/Manager.h"
 #include "zeek/plugin/Plugin.h"
 
+#include "zeek/3rdparty/doctest.h"
+
 #ifdef SYSLOG_INT
 extern "C" {
 int openlog(const char* ident, int logopt, int facility);

src/RuleMatcher.cc

@@ -1,8 +1,6 @@
 
 #include "zeek/RuleMatcher.h"
 
-#include "zeek/zeek-config.h"
-
 #include <algorithm>
 #include <functional>
 
@@ -24,6 +22,7 @@
 #include "zeek/ZeekString.h"
 #include "zeek/analyzer/Analyzer.h"
 #include "zeek/module_util.h"
+#include "zeek/plugin/Manager.h"
 
 using namespace std;
 

src/RuleMatcher.h

@@ -13,7 +13,6 @@
 #include "zeek/Rule.h"
 #include "zeek/ScannedFile.h"
 #include "zeek/ZeekString.h"
-#include "zeek/plugin/Manager.h"
 
 // #define MATCHER_PRINT_STATS
 

src/ZeekString.cc

@@ -9,12 +9,13 @@
 #include <iostream>
 #include <sstream> // Needed for unit testing
 
-#include "zeek/3rdparty/doctest.h"
 #include "zeek/ID.h"
 #include "zeek/Reporter.h"
 #include "zeek/Val.h"
 #include "zeek/util.h"
 
+#include "zeek/3rdparty/doctest.h"
+
 #ifdef DEBUG
 #define DEBUG_STR(msg) DBG_LOG(zeek::DBG_STRING, msg)
 #else

src/analyzer/Analyzer.cc

@@ -5,12 +5,13 @@
 #include <binpac.h>
 #include <algorithm>
 
-#include "zeek/3rdparty/doctest.h"
 #include "zeek/Event.h"
 #include "zeek/ZeekString.h"
 #include "zeek/analyzer/Manager.h"
 #include "zeek/analyzer/protocol/pia/PIA.h"
 
+#include "zeek/3rdparty/doctest.h"
+
 namespace zeek::analyzer {
 
 class AnalyzerTimer final : public zeek::detail::Timer {

src/analyzer/protocol/smtp/BDAT.cc

@@ -1,11 +1,12 @@
 #include "zeek/analyzer/protocol/smtp/BDAT.h"
 
-#include "zeek/3rdparty/doctest.h"
 #include "zeek/Conn.h"
 #include "zeek/DebugLogger.h"
 #include "zeek/analyzer/protocol/mime/MIME.h"
 #include "zeek/util.h"
 
+#include "zeek/3rdparty/doctest.h"
+
 namespace zeek::analyzer::smtp::detail {
 
 

src/broker/Data.cc

@@ -2,7 +2,6 @@
 
 #include <broker/error.hh>
 
-#include "zeek/3rdparty/doctest.h"
 #include "zeek/Desc.h"
 #include "zeek/File.h"
 #include "zeek/Func.h"
@@ -13,6 +12,8 @@
 #include "zeek/broker/data.bif.h"
 #include "zeek/module_util.h"
 
+#include "zeek/3rdparty/doctest.h"
+
 using namespace std;
 
 zeek::OpaqueTypePtr zeek::Broker::detail::opaque_of_data_type;

src/file_analysis/FileReassembler.cc

@@ -2,9 +2,10 @@
 
 #include "zeek/file_analysis/FileReassembler.h"
 
-#include "zeek/3rdparty/doctest.h"
 #include "zeek/file_analysis/File.h"
 
+#include "zeek/3rdparty/doctest.h"
+
 namespace zeek::file_analysis {
 
 class File;

src/iosource/Manager.cc

@@ -14,13 +14,11 @@
 
 #include "zeek/NetVar.h"
 #include "zeek/RunState.h"
-#include "zeek/broker/Manager.h"
 #include "zeek/iosource/Component.h"
 #include "zeek/iosource/IOSource.h"
 #include "zeek/iosource/PktDumper.h"
 #include "zeek/iosource/PktSrc.h"
 #include "zeek/plugin/Manager.h"
-#include "zeek/util.h"
 
 #define DEFAULT_PREFIX "pcap"
 

src/logging/Manager.cc

@@ -19,7 +19,6 @@
 #include "zeek/Timer.h"
 #include "zeek/Type.h"
 #include "zeek/broker/Manager.h"
-#include "zeek/input.h"
 #include "zeek/logging/WriterBackend.h"
 #include "zeek/logging/WriterFrontend.h"
 #include "zeek/logging/logging.bif.h"
@@ -2027,13 +2026,11 @@ bool Manager::FinishedRotation(WriterFrontend* writer, const char* new_name, con
     --rotations_pending;
 
     if ( ! success ) {
-        DBG_LOG(DBG_LOGGING, "Non-successful rotating writer '%s', file '%s' at %.6f,", writer->Name(), filename,
+        DBG_LOG(DBG_LOGGING, "Non-successful rotating writer '%s', new_name '%s'", writer->Name(), new_name);
-                run_state::network_time);
         return true;
     }
 
-    DBG_LOG(DBG_LOGGING, "Finished rotating %s at %.6f, new name %s", writer->Name(), run_state::network_time,
+    DBG_LOG(DBG_LOGGING, "Finished rotating %s, new name %s", writer->Name(), new_name);
-            new_name);
 
     WriterInfo* winfo = FindWriter(writer);
     if ( ! winfo )

src/logging/writers/ascii/Ascii.cc

@@ -15,7 +15,6 @@
 #include <string>
 #include <vector>
 
-#include "zeek/3rdparty/doctest.h"
 #include "zeek/Func.h"
 #include "zeek/RunState.h"
 #include "zeek/logging/Manager.h"
@@ -23,6 +22,8 @@
 #include "zeek/threading/SerialTypes.h"
 #include "zeek/util.h"
 
+#include "zeek/3rdparty/doctest.h"
+
 using namespace std;
 using zeek::threading::Field;
 using zeek::threading::Value;

src/net_util.cc

@@ -10,11 +10,12 @@
 #include <sys/types.h>
 #include <memory>
 
-#include "zeek/3rdparty/doctest.h"
 #include "zeek/IP.h"
 #include "zeek/IPAddr.h"
 #include "zeek/Reporter.h"
 
+#include "zeek/3rdparty/doctest.h"
+
 const char* transport_proto_string(TransportProto proto) {
     switch ( proto ) {
         case TRANSPORT_TCP: return "tcp";

src/telemetry/Manager.cc

@@ -6,6 +6,7 @@
 
 // CivetServer is from the civetweb submodule in prometheus-cpp
 #include <CivetServer.h>
+#include <fnmatch.h>
 #include <prometheus/collectable.h>
 #include <prometheus/exposer.h>
 #include <prometheus/registry.h>
@@ -14,18 +15,18 @@
 #include <algorithm>
 #include <thread>
 
-#include "zeek/3rdparty/doctest.h"
 #include "zeek/Func.h"
 #include "zeek/ID.h"
 #include "zeek/IPAddr.h"
 #include "zeek/RunState.h"
 #include "zeek/ZeekString.h"
-#include "zeek/broker/Manager.h"
 #include "zeek/iosource/Manager.h"
 #include "zeek/telemetry/ProcessStats.h"
 #include "zeek/telemetry/Timer.h"
 #include "zeek/threading/formatters/detail/json.h"
 
+#include "zeek/3rdparty/doctest.h"
+
 namespace zeek::telemetry {
 
 /**

src/util.cc

@@ -46,7 +46,6 @@
 #include <vector>
 
 #include "zeek/3rdparty/ConvertUTF.h"
-#include "zeek/3rdparty/doctest.h"
 #include "zeek/Desc.h"
 #include "zeek/Hash.h"
 #include "zeek/NetVar.h"
@@ -60,6 +59,8 @@
 #include "zeek/iosource/Manager.h"
 #include "zeek/iosource/PktSrc.h"
 
+#include "zeek/3rdparty/doctest.h"
+
 using namespace std;
 
 extern const char* proc_status_file;

src/zeek-setup.cc

@@ -14,17 +14,15 @@
 #include <cstdlib>
 #include <cstring>
 #include <list>
+#include <memory>
 #include <optional>
+#include <set>
+#include <string>
 
 #ifdef USE_SQLITE
 #include "zeek/3rdparty/sqlite3.h"
 #endif
 
-#define DOCTEST_CONFIG_IMPLEMENT
-
-#include "zeek/3rdparty/doctest.h"
-#include "zeek/Anon.h"
-#include "zeek/DFA.h"
 #include "zeek/DNS_Mgr.h"
 #include "zeek/Debug.h"
 #include "zeek/Desc.h"
@@ -45,8 +43,6 @@
 #include "zeek/Scope.h"
 #include "zeek/ScriptCoverageManager.h"
 #include "zeek/Stats.h"
-#include "zeek/Stmt.h"
-#include "zeek/Tag.h"
 #include "zeek/Timer.h"
 #include "zeek/Traverse.h"
 #include "zeek/Trigger.h"
@@ -57,7 +53,6 @@
 #include "zeek/file_analysis/Manager.h"
 #include "zeek/input.h"
 #include "zeek/input/Manager.h"
-#include "zeek/input/readers/raw/Raw.h"
 #include "zeek/iosource/Manager.h"
 #include "zeek/logging/Manager.h"
 #include "zeek/module_util.h"
@@ -77,6 +72,9 @@ extern "C" {
 #include "zeek/3rdparty/setsignal.h"
 };
 
+#define DOCTEST_CONFIG_IMPLEMENT
+#include "zeek/3rdparty/doctest.h"
+
 zeek::detail::ScriptCoverageManager zeek::detail::script_coverage_mgr;
 
 #ifndef HAVE_STRSEP
@@ -191,8 +189,8 @@ extern "C" char version[];
 extern "C" const char zeek_build_info[];
 
 const char* zeek::detail::command_line_policy = nullptr;
-vector<string> zeek::detail::params;
+std::vector<std::string> zeek::detail::params;
-set<string> requested_plugins;
+std::set<std::string> requested_plugins;
 const char* proc_status_file = nullptr;
 
 zeek::OpaqueTypePtr md5_type;
@@ -283,8 +281,8 @@ static bool show_plugins(int level) {
         printf("\nInactive dynamic plugins:\n");
 
         for ( plugin::Manager::inactive_plugin_list::const_iterator i = inactives.begin(); i != inactives.end(); i++ ) {
-            string name = (*i).first;
+            std::string name = (*i).first;
-            string path = (*i).second;
+            std::string path = (*i).second;
             printf("  %s (%s)\n", name.c_str(), path.c_str());
         }
     }
@@ -748,7 +746,7 @@ SetupResult setup(int argc, char** argv, Options* zopts) {
         auto ipbb = make_intrusive<BuiltinFunc>(init_bifs, ipbid->Name(), false);
 
         if ( options.event_trace_file )
-            etm = make_unique<EventTraceMgr>(*options.event_trace_file);
+            etm = std::make_unique<EventTraceMgr>(*options.event_trace_file);
 
         // Parsing involves reading input files, including any input
         // interactively provided by the user at the console. Temporarily
@@ -784,7 +782,7 @@ SetupResult setup(int argc, char** argv, Options* zopts) {
         // Assign the script_args for command line processing in Zeek scripts.
         if ( ! options.script_args.empty() ) {
             auto script_args_val = id::find_val<VectorVal>("zeek_script_args");
-            for ( const string& script_arg : options.script_args ) {
+            for ( const auto& script_arg : options.script_args ) {
                 script_args_val->Assign(script_args_val->Size(), make_intrusive<StringVal>(script_arg));
             }
         }