Merge remote-tracking branch 'origin/master' into topic/johanna/table-on-change

.cirrus.yml

@@ -1,6 +1,6 @@
 cpus: &CPUS 8
 btest_jobs: &BTEST_JOBS 8
-memory: &MEMORY 8GB
+memory: &MEMORY 6GB
 
 config: &CONFIG --build-type=release --enable-cpp-tests
 memcheck_config: &MEMCHECK_CONFIG --build-type=debug --enable-cpp-tests --sanitizers=address
@@ -95,10 +95,11 @@ ubuntu16_task:
 macos_task:
   osx_instance:
     image: catalina-base
+    # cpu/memory setting is implicitly 2 core / 4 thread and 8GB, and
+    # trying to set it explicitly results in an error.
   prepare_script: ./ci/macos/prepare.sh
   << : *CI_TEMPLATE
   env:
-    # Currently only available as 2 core / 4 thread (and 8GB) instances.
     ZEEK_CI_CPUS: 4
     ZEEK_CI_BTEST_JOBS: 4
     # No permission to write to default location of /zeek
@@ -109,7 +110,9 @@ freebsd_task:
   freebsd_instance:
     # FreeBSD 12 EOL: June 30, 2024
     image_family: freebsd-12-1
-    << : *RESOURCES_TEMPLATE
+    cpu: 8
+    # Not allowed to request less than 8GB for an 8 CPU FreeBSD VM.
+    memory: 8GB
   prepare_script: ./ci/freebsd/prepare.sh
   << : *CI_TEMPLATE
 
@@ -117,7 +120,9 @@ memcheck_task:
   container:
     # Just uses a recent/common distro to run memory error/leak checks.
     dockerfile: ci/ubuntu-18.04/Dockerfile
-    << : *RESOURCES_TEMPLATE
+    cpu: 8
+    # AddressSanitizer uses a lot more memory than a typical config.
+    memory: 16GB
   << : *CI_TEMPLATE
   env:
     ZEEK_CI_CONFIGURE_FLAGS: *MEMCHECK_CONFIG

.travis.yml

@@ -29,7 +29,8 @@ env:
   - distro: fedora_30
   - distro: ubuntu_16.04
   - distro: ubuntu_18.04
-  - distro: ubuntu_18.04_leaktest
+# Travis environment doesn't have enough memory to support ASan well.
+#  - distro: ubuntu_18.04_leaktest
 
 install: sh ci/travis-job install $distro
 

CHANGES

@@ -1,4 +1,252 @@
 
+3.1.0-dev.546 | 2020-02-05 10:51:35 -0800
+
+  * Change threading formatter errors to warnings. (Johanna Amann, Corelight)
+
+    This change standardizes threading formatter error handling and moves
+    the remaining error calls to be warnings instead.
+
+    This is in line with already existing code - in most cases warnings were
+    raised, only a few cases raised errors. These cases do not differ
+    significantly from other cases in which warnings are raised.
+
+    This also fixes GH-692, in which misformatted lines prevent future file
+    parsing.
+
+    This commit also moves the FailWarn method that is used by both the
+    config and the ascii reader up to the ReaderBackend. Furthermore it
+    makes the Warning method of ReaderBackend respect the warning
+    suppression that is introduced by the FailWarn method.
+
+  * Update NEWS about plugin breakage from the include-file changes (Tim Wojtulewicz, Corelight)
+
+3.1.0-dev.542 | 2020-02-04 14:05:19 -0700
+
+  * UID, ..: un-inline methods to reduce header dependencies
+
+    Only 1% build time speedup, but still, it declutters the headers a bit.
+
+    Before this patch:
+
+     2565.17user 141.83system 2:25.46elapsed 1860%CPU (0avgtext+0avgdata 1489076maxresident)k
+     72576inputs+9130920outputs (1667major+49400430minor)pagefaults 0swaps
+
+    After this patch:
+
+     2537.19user 142.94system 2:26.90elapsed 1824%CPU (0avgtext+0avgdata 1434268maxresident)k
+     16240inputs+8887152outputs (1931major+48728888minor)pagefaults 0swaps (Max Kellermann)
+
+  * include cleanup
+
+    The Zeek code base has very inconsistent #includes.  Many sources
+    included a few headers, and those headers included other headers, and
+    in the end, nearly everything is included everywhere, so missing
+    #includes were never noticed.  Another side effect was a lot of header
+    bloat which slows down the build.
+
+    First step to fix it: in each source file, its own header should be
+    included first to verify that each header's includes are correct, and
+    none is missing.
+
+    After adding the missing #includes, I replaced lots of #includes
+    inside headers with class forward declarations.  In most headers,
+    object pointers are never referenced, so declaring the function
+    prototypes with forward-declared classes is just fine.
+
+    This patch speeds up the build by 19%, because each compilation unit
+    gets smaller.  Here are the "time" numbers for a fresh build (with a
+    warm page cache but without ccache):
+
+    Before this patch:
+
+     3144.94user 161.63system 3:02.87elapsed 1808%CPU (0avgtext+0avgdata 2168608maxresident)k
+     760inputs+12008400outputs (1511major+57747204minor)pagefaults 0swaps
+
+    After this patch:
+
+     2565.17user 141.83system 2:25.46elapsed 1860%CPU (0avgtext+0avgdata 1489076maxresident)k
+     72576inputs+9130920outputs (1667major+49400430minor)pagefaults 0swaps (Max Kellermann)
+
+  * Updating submodule(s).
+
+     [nomail] (Jon Siwek, Corelight)
+
+
+3.1.0-dev.538 | 2020-02-04 11:57:35 +0000
+
+  * Updating submodules.
+
+3.1.0-dev.536 | 2020-02-03 15:44:10 -0700
+
+  * Change various functions to by-value std::string_view args (Jon Siwek, Corelight)
+
+3.1.0-dev.534 | 2020-02-03 15:42:24 -0700
+
+  * Change packet source fields of Options to std::optional (Jon Siwek, Corelight)
+
+3.1.0-dev.531 | 2020-02-03 13:23:22 -0800
+
+  * Update sqlite to 3.31.1 (Johanna Amann, Corelight)
+
+	* Update certificate list to state of NSS 3.49.2. (Johanna Amann, Corelight)
+
+3.1.0-dev.530 | 2020-02-03 13:21:16 -0800
+
+  * Disable Travis leak test (Jon Siwek, Corelight)
+
+    Travis environment doesn't offer enough memory for running ASan very
+    well, but Cirrus has it covered anyway.
+
+3.1.0-dev.529 | 2020-02-03 13:04:30 -0800
+
+  * CMakeLists.txt: fix cross-compilation with binpac (Fabrice Fontaine)
+
+3.1.0-dev.527 | 2020-02-03 10:11:00 -0800
+
+  * Change requested memory in Cirrus CI config (Jon Siwek, Corelight)
+
+    The ASan build does require quite a bit more, but also the default
+    release builds should do ok with slightly less.
+
+3.1.0-dev.526 | 2020-02-03 10:04:00 -0800
+
+  * Increase timeout value for live interfaces without file descriptors (Tim Wojtulewicz, Corelight)
+
+  * Use ranged-for loops in a few places in iosource Manager (Tim Wojtulewicz, Corelight)
+
+  * Change order of includes in iosource Manager, fixes build on FreeBSD 11 (Tim Wojtulewicz, Corelight)
+
+3.1.0-dev.521 | 2020-01-31 14:18:17 -0800
+
+  * util: optimize expand_escape() by avoiding sscanf() (Max Kellermann)
+
+    sscanf() is notoriously slow, and the default scripts have lots of hex
+    escapes.  This patch can reduce Zeek's startup time by 3-9%.
+
+3.1.0-dev.519 | 2020-01-31 13:19:09 -0800
+
+  * util: optimize tokenize_string() and normalize_path() (Max Kellermann)
+
+    This patch can speed up Zeek startup by 10-25%.
+
+    Adds a new tokenize_string() overload which returns string_views and
+    changes existing tokenize_string() and normalize_path() to use string_view
+    arguments.
+
+3.1.0-dev.510 | 2020-01-31 11:20:28 -0800
+
+  * Remove extra fmt() in a reporter->Error() call (Jon Siwek, Corelight)
+
+  * parse.y: Properly set location info for functions (Arne Welzel)
+
+    When defining a function, remember the location where the function header
+    was and restore it before calling `end_func()`. Inside `end_func()`, a
+    `BroFunc` object is created using the current global location information.
+
+    This came up while experimenting with zeek script profiling and wondering
+    why the locations set for `BroFunc` were "somewhere" in the middle of
+    functions instead of spanning them.
+
+  * Desc: move realloc() call out of the loop (Max Kellermann)
+
+  * SerializationFormat: move realloc() call out of the loop (Max Kellermann)
+
+  * PacketDumper: remove unused types (Max Kellermann)
+
+3.1.0-dev.503 | 2020-01-31 11:00:32 -0800
+
+  * Show percentage of packets dropped in the final process output (Tim Wojtulewicz, Corelight)
+
+  * Remove concept of multiple timer managers (Tim Wojtulewicz, Corelight)
+
+    - All timers are now handled by a single global timer manager, which
+      simplifies how they handled by the IOSource manager.
+
+    - This change flows down a number of changes to other parts of the code.
+      The timer manager tag field is removed, which means that matching
+      connections to a timer manager is also removed. This removes the ability to
+      tag a connection as internal or external, since that's how the connections
+      where differentiated. This in turn removes the `current_conns_extern` field
+      from the `ConnStats` record type in the script layer.
+
+  * Have terminate_processing() raise SIGTERM instead of calling the signal handler directly (Tim Wojtulewicz, Corelight)
+
+  * PktSrc iosource changes to match the new IOSource API (Tim Wojtulewicz, Corelight)
+
+  * Broker manager changes to match the new IOSource API and loop architecture (Tim Wojtulewicz, Corelight)
+
+  * DNS_Mgr changes to match the new IOSource API and loop architecture (Tim Wojtulewicz, Corelight)
+
+  * Threading changes for the new loop architecture (Tim Wojtulewicz, Corelight)
+
+    - threading::Manager is no longer an IOSource.
+
+    - threading::MsgThread is now an IOSource. This allows threads themselves
+      to signal when they have data to process instead of continually checking
+      each of the threads on every loop pass.
+
+    - Make the thread heartbeat timer an actual timer and let it fire as
+      necessary instead of checking to see if it should fire
+
+  * Add Trigger manager for managing triggers created by things like 'when' statements
+
+    - Adds new trigger namespace (Tim Wojtulewicz, Corelight)
+
+    - Adds trigger::Manager class as a new IOSource for keeping track of
+      triggers and integrating them into the loop. Previously the loop relied on
+      the event manager Drain() method to process all triggers on every loop, but
+      now that the loop actively waits for events to occur, triggers would not
+      fire when they needed to. Adding them as part of the loop ensures they're
+      checked.
+
+  * Make TimerMgr an IOSource (Tim Wojtulewicz, Corelight)
+
+    - This allows the loop to check what the next timeout is and use that as
+      the basis for the timeout of the poll
+
+    - This commit also removes the TimerMgr::Tag type, since it causes a name
+      clash with other code in IOSource
+
+  * Modify IOSource Manager to implement new loop architecture (Tim Wojtulewicz, Corelight)
+
+    - Removes entire FindSoonest method that includes all of the calls to select() for checking for ready sources
+
+    - Removes FD_Set checking against IOSources
+
+    - Adds system for registering and unregistering file descriptors from
+      IOSources. This allows individual sources to mark themselves as ready to be
+      checked by the loop as they become available.
+
+    - Adds entirely new loop architecture based on checking the IOSources for
+      when their next timeout is, and then waiting for either that timeout or
+      when the next source is ready. This also implements the polling based on
+      what the OS supports, instead of just calling select() on all platforms.
+      Currently it supports kqueue, epoll, and plain poll.
+
+    - Adds system for pinging the loop to force it to wake up
+
+  * Minor amount of code cleanup in Pcap IO source (Tim Wojtulewicz, Corelight)
+
+  * IOSource API changes for new loop architecture (Tim Wojtulewicz, Corelight)
+
+    - Removed GetFD and NextTimestamp methods
+
+    - Removed concept of idle sources
+
+    - Renamed Init to InitSource to avoid a name clash with the thread code
+
+    - Added GetNextTimeout method
+
+    This commit also removes the FD_Set file since it's no longer used
+
+  * Type and variable usage cleanup in Net.h (Tim Wojtulewicz, Corelight)
+
+  * Remove #include of some iosource files from Net.h (Tim Wojtulewicz, Corelight)
+
+  * Only allow a single trace file (-r) or interface (-i) option on the command-line (Tim Wojtulewicz, Corelight)
+
+  * Remove CQ_TimerMgr in favor of PQ_TimerMgr (Tim Wojtulewicz, Corelight)
+
 3.1.0-dev.482 | 2020-01-30 19:16:03 -0800
 
   * Update NEWS for naming changes. (Robin Sommer, Corelight)

CMakeLists.txt

@@ -178,8 +178,7 @@ FindRequiredPackage(OpenSSL)
 FindRequiredPackage(BIND)
 FindRequiredPackage(ZLIB)
 
-if (NOT BINPAC_EXE_PATH AND
+if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/aux/binpac/CMakeLists.txt)
-    EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/aux/binpac/CMakeLists.txt)
 
     set(ENABLE_STATIC_ONLY_SAVED ${ENABLE_STATIC_ONLY})
 
@@ -362,6 +361,7 @@ include(OpenSSLTests)
 include(CheckNameserCompat)
 include(GetArchitecture)
 include(RequireCXX17)
+include(FindKqueue)
 
 if ( (OPENSSL_VERSION VERSION_EQUAL "1.1.0") OR (OPENSSL_VERSION VERSION_GREATER "1.1.0") )
   set(ZEEK_HAVE_OPENSSL_1_1 true CACHE INTERNAL "" FORCE)

COPYING.3rdparty

@@ -21,6 +21,54 @@ a legal notice, here is a blessing:
 
 ==============================================================================
 
+%%% libkqueue - kqueue(2) compatibility library
+
+==============================================================================
+
+== all source ==
+
+Copyright (c) 2009 Mark Heily <mark@heily.com>
+
+Permission to use, copy, modify, and distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+== event.h ==
+
+Copyright (c) 1999,2000,2001 Jonathan Lemon <jlemon@FreeBSD.org>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGE.
+
+==============================================================================
+
 %%% RapidJSON - A fast JSON parser/generator for C++ with both SAX/DOM style API
 
 ==============================================================================

NEWS

@@ -102,9 +102,36 @@ Changed Functionality
   the ``portnum`` component of the returned ``URI`` value is left
   uninitialized.
 
+- Replace old ``select``-based IO loop with a new architecture that doesn't
+  spin checking for active IO sources. The new architecture now waits for the
+  the sources to actively notify it when activity occurs and only processes
+  data once it's ready. This helps heavily reduce the CPU usage on idle
+  network connections. This includes a couple of breaking changes:
+
+  - Only a single packet source is allowed to be specified from the
+    command-line now. If you pass combinations of multiple ``-r`` and/or
+    ``-i`` flags, Zeek will return an error at startup.
+  - The IOSource API changed fairly wildly. The ``GetFds()`` and
+    ``NextTimestamp`` methods no longer exist. If you had previously
+    implemented a custom IOSource, you will need to look at the new API
+    and make changes to your code to accomodate it. This does not include
+    packet sources, which should remain functional with little to no
+    changes, since the entirety of the changes should be in ``PktSrc``.
+
+- Remove a large number of headers from being included by various files across
+  the entire code base, which leads to a sizeable build time improvement. This
+  set of changes has the potential to cause plugins to not build anymore. The
+  fixes for this potential breakage should just be a matter of including the
+  necessary headers in the plugin code.
+
 Removed Functionality
 ---------------------
 
+- Removed the ``current_conns_extern`` field from the ConnStats record
+  type. Zeek only maintains a single timer manager now, and without the
+  manager tags that came with multiple tiemr managers, we don't track
+  whether a connection is external anymore.
+
 Deprecated Functionality
 ------------------------
 

VERSION

@@ -1 +1 @@
-3.1.0-dev.482
+3.1.0-dev.546

aux/bifcl

@@ -1 +1 @@
-Subproject commit e6bc87175dcdf297caae9506d326f8d0cc36b64b
+Subproject commit 3bb5f6bca3954a5ad31c7c3eaad37d1a2d7bb1d3

aux/binpac

@@ -1 +1 @@
-Subproject commit 3b6c9ebc844ca98759e183f7b07c511fa87e8163
+Subproject commit 711b365b9f8fa2e74477035060a767778ca885e7

aux/broker

@@ -1 +1 @@
-Subproject commit 89a43997d6715fca4e2242a3b88b3aa1c075a340
+Subproject commit ae9b6ffd85b1097db2a7a6acbfd708165e3a2f10

aux/zeek-aux

@@ -1 +1 @@
-Subproject commit 5b98e96695c334b6c2929fb96666d58eb22f9b66
+Subproject commit 0181cc05d079e4291a6d3982765e6b0771650857

aux/zeekctl

@@ -1 +1 @@
-Subproject commit f8941cd0ba171d9ea2b2e01724597e5e790ffb16
+Subproject commit 3ef627f10ab700c4d3f5e373dd0c8b3866ac0378

cmake

@@ -1 +1 @@
-Subproject commit d3e1884a876781dedac716d102e8a06e1cc54380
+Subproject commit 3c58b089e7533359b045b5203033e17190c22c41

configure

@@ -74,6 +74,8 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
                            (Zeek uses an embedded version by default)
     --with-caf=PATH        path to C++ Actor Framework install root
                            (a Broker dependency that is embedded by default)
+    --with-libkqueue=PATH  path to libkqueue install root
+                           (Zeek uses an embedded version by default)
 
   Optional Packages in Non-Standard Locations:
     --with-geoip=PATH      path to the libmaxminddb install root
@@ -277,7 +279,6 @@ while [ $# -ne 0 ]; do
             ;;
         --with-binpac=*)
             append_cache_entry BINPAC_EXE_PATH     PATH $optarg
-            append_cache_entry BinPAC_ROOT_DIR     PATH "$(dirname $optarg)/.."
             ;;
         --with-bifcl=*)
             append_cache_entry BIFCL_EXE_PATH      PATH   $optarg
@@ -320,6 +321,9 @@ while [ $# -ne 0 ]; do
         --with-caf=*)
             append_cache_entry CAF_ROOT_DIR      PATH   $optarg
             ;;
+        --with-libkqueue=*)
+            append_cache_entry LIBKQUEUE_ROOT_DIR      PATH   $optarg
+            ;;
         --with-rocksdb=*)
             append_cache_entry ROCKSDB_ROOT_DIR     PATH   $optarg
             ;;

doc

@@ -1 +1 @@
-Subproject commit 566174d004c14d061fcf7c03e4829f20d46aaad8
+Subproject commit 87b63d810d345172084d15c9c7feb132688456fe

scripts/base/init-bare.zeek

@@ -133,9 +133,6 @@ type PacketSource: record {
 	netmask: count;
 };
 
-## A list of packet sources being read by Zeek.
-type PacketSourceList: vector of PacketSource;
-
 ## A connection's transport-layer protocol. Note that Zeek uses the term
 ## "connection" broadly, using flow semantics for ICMP and UDP.
 type transport_proto: enum {
@@ -551,7 +548,6 @@ type NetStats: record {
 type ConnStats: record {
 	total_conns: count;           ##<
 	current_conns: count;         ##<
-	current_conns_extern: count;  ##<
 	sess_current_conns: count;    ##<
 
 	num_packets: count;

scripts/base/protocols/ssl/ct-list.zeek

@@ -1,6 +1,6 @@
 #
 # Do not edit this file. This file is automatically generated by gen-ct-list.pl
-# File generated at Fri Aug  2 12:42:14 2019
+# File generated at Mon Feb  3 12:39:48 2020
 # File generated from https://www.gstatic.com/ct/log_list/all_logs_list.json
 #
 

scripts/policy/protocols/conn/weirds.zeek

@@ -29,8 +29,6 @@ event rexmit_inconsistency(c: connection, t1: string, t2: string, tcp_flags: str
 event content_gap(c: connection, is_orig: bool, seq: count, length: count)
 	{
 	NOTICE([$note=Content_Gap, $conn=c,
-	        $msg=fmt("%s content gap (%s %d/%d)%s",
+	        $msg=fmt("%s content gap (%s %d/%d)",
-	                 id_string(c$id), is_orig ? ">" : "<", seq, length,
+	                 id_string(c$id), is_orig ? ">" : "<", seq, length)]);
-	                 is_external_connection(c) ? " [external]" : "")]);
 	}
-

src/3rdparty

@@ -1 +1 @@
-Subproject commit fae32236391d9117bf996e75d56ebd01ef076bc2
+Subproject commit 0c732695b446255047ee91456cc9aebea831fe2b

src/Anon.cc

@@ -1,3 +1,5 @@
+#include "Anon.h"
+
 #include <stdlib.h>
 #include <unistd.h>
 #include <assert.h>
@@ -5,9 +7,9 @@
 
 #include "util.h"
 #include "net_util.h"
-#include "Anon.h"
 #include "Val.h"
 #include "NetVar.h"
+#include "Reporter.h"
 
 
 AnonymizeIPAddr* ip_anonymizer[NUM_ADDR_ANONYMIZATION_METHODS] = {0};
@@ -66,6 +68,13 @@ ipaddr32_t AnonymizeIPAddr::Anonymize(ipaddr32_t addr)
 		}
 	}
 
+// Keep the specified prefix unchanged.
+int AnonymizeIPAddr::PreservePrefix(ipaddr32_t /* input */, int /* num_bits */)
+	{
+	reporter->InternalError("prefix preserving is not supported for the anonymizer");
+	return 0;
+	}
+
 int AnonymizeIPAddr::PreserveNet(ipaddr32_t input)
 	{
 	switch ( addr_to_class(ntohl(input)) ) {

src/Anon.h

@@ -11,11 +11,9 @@
 #pragma once
 
 #include <vector>
-#include <set>
 #include <map>
 
-#include "Reporter.h"
+using std::map;
-#include "net_util.h"
 
 // TODO: Anon.h may not be the right place to put these functions ...
 
@@ -46,12 +44,7 @@ public:
 
 	ipaddr32_t Anonymize(ipaddr32_t addr);
 
-	// Keep the specified prefix unchanged.
+	virtual int PreservePrefix(ipaddr32_t input, int num_bits);
-	virtual int PreservePrefix(ipaddr32_t /* input */, int /* num_bits */)
-		{
-		reporter->InternalError("prefix preserving is not supported for the anonymizer");
-		return 0;
-		}
 
 	virtual ipaddr32_t anonymize(ipaddr32_t addr) = 0;
 	

src/Attr.cc

@@ -4,6 +4,8 @@
 
 #include "Attr.h"
 #include "Expr.h"
+#include "Desc.h"
+#include "Val.h"
 #include "threading/SerialTypes.h"
 
 const char* attr_name(attr_tag t)

src/Attr.h

@@ -3,6 +3,7 @@
 #pragma once
 
 #include "Obj.h"
+#include "BroList.h"
 
 class Expr;
 

src/Base64.cc

@@ -1,5 +1,9 @@
 #include "zeek-config.h"
 #include "Base64.h"
+#include "BroString.h"
+#include "Reporter.h"
+#include "Conn.h"
+
 #include <math.h>
 
 int Base64Converter::default_base64_table[256];
@@ -215,6 +219,14 @@ int Base64Converter::Done(int* pblen, char** pbuf)
 	return 0;
 	}
 
+void Base64Converter::IllegalEncoding(const char* msg)
+		{
+		// strncpy(error_msg, msg, sizeof(error_msg));
+		if ( conn )
+			conn->Weird("base64_illegal_encoding", msg);
+		else
+			reporter->Error("%s", msg);
+		}
 
 BroString* decode_base64(const BroString* s, const BroString* a, Connection* conn)
 	{
@@ -266,4 +278,3 @@ BroString* encode_base64(const BroString* s, const BroString* a, Connection* con
 
 	return new BroString(1, (u_char*)outbuf, outlen);
 	}
-

src/Base64.h

@@ -1,13 +1,11 @@
 #pragma once
 
-#include <assert.h>
+#include <string>
-#include <stdio.h>
-#include <string.h>
 
-#include "util.h"
+using std::string;
-#include "BroString.h"
+
-#include "Reporter.h"
+class BroString;
-#include "Conn.h"
+class Connection;
 
 // Maybe we should have a base class for generic decoders?
 class Base64Converter {
@@ -40,14 +38,7 @@ public:
 	int Errored() const	{ return errored; }
 
 	const char* ErrorMsg() const	{ return error_msg; }
-	void IllegalEncoding(const char* msg)
+	void IllegalEncoding(const char* msg);
-		{
-		// strncpy(error_msg, msg, sizeof(error_msg));
-		if ( conn )
-			conn->Weird("base64_illegal_encoding", msg);
-		else
-			reporter->Error("%s", msg);
-		}
 
 protected:
 	char error_msg[256];

src/BroString.cc

@@ -1,15 +1,16 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
 #include "zeek-config.h"
+#include "BroString.h"
 
 #include <algorithm>
+#include <iostream>
 #include <ctype.h>
 
-#include <algorithm>
+#include "Val.h"
-
-#include "BroString.h"
 #include "Var.h"
 #include "Reporter.h"
+#include "util.h"
 
 #ifdef DEBUG
 #define DEBUG_STR(msg) DBG_LOG(DBG_STRING, msg)
@@ -274,6 +275,11 @@ void BroString::ToUpper()
 			b[i] = toupper(b[i]);
 	}
 
+unsigned int BroString::MemoryAllocation() const
+	{
+	return padded_sizeof(*this) + pad_size(n + final_NUL);
+	}
+
 BroString* BroString::GetSubstring(int start, int len) const
 	{
 	// This code used to live in zeek.bif's sub_bytes() routine.

src/BroString.h

@@ -4,11 +4,9 @@
 
 #include <vector>
 #include <string>
-#include <iostream>
+#include <iosfwd>
-#include <stdlib.h>
-#include <sys/types.h>
 
-#include "util.h"
+#include <sys/types.h>
 
 typedef u_char* byte_vec;
 
@@ -114,8 +112,7 @@ public:
 	// XXX and to_upper; the latter doesn't use BroString::ToUpper().
 	void ToUpper();
 
-	unsigned int MemoryAllocation() const
+	unsigned int MemoryAllocation() const;
-		{ return padded_sizeof(*this) + pad_size(n + final_NUL); }
 
 	// Returns new string containing the substring of this string,
 	// starting at @start >= 0 for going up to @length elements,

src/Brofiler.cc

@@ -1,3 +1,5 @@
+#include "Brofiler.h"
+
 #include <cstdio>
 #include <cstring>
 #include <sstream>
@@ -5,7 +7,10 @@
 #include <utility>
 #include <algorithm>
 #include <sys/stat.h>
-#include "Brofiler.h"
+
+#include "Stmt.h"
+#include "Desc.h"
+#include "Reporter.h"
 #include "util.h"
 
 Brofiler::Brofiler()

src/Brofiler.h

@@ -3,8 +3,14 @@
 #include <map>
 #include <utility>
 #include <list>
-#include <Stmt.h>
+#include <string>
 
+using std::list;
+using std::map;
+using std::pair;
+using std::string;
+
+class Stmt;
 
 /**
  * A simple class for managing stats of Bro script coverage across Bro runs.

src/CCL.cc

@@ -43,3 +43,8 @@ void CCL::Sort()
 	{
 	std::sort(syms->begin(), syms->end());
 	}
+
+unsigned int CCL::MemoryAllocation() const
+	{
+	return padded_sizeof(*this) + padded_sizeof(*syms) + pad_size(syms->size() * sizeof(int_list::value_type));
+	}

src/CCL.h

@@ -2,8 +2,9 @@
 
 #pragma once
 
+#include "util.h" // for ptr_compat_int
+
 #include <vector>
-#include "List.h"
 
 typedef std::vector<ptr_compat_int> int_list;
 
@@ -24,8 +25,7 @@ public:
 	void ReplaceSyms(int_list* new_syms)
 				{ delete syms; syms = new_syms; }
 
-	unsigned int MemoryAllocation() const
+	unsigned int MemoryAllocation() const;
-		{ return padded_sizeof(*this) + padded_sizeof(*syms) + pad_size(syms->size() * sizeof(int_list::value_type)); }
 
 protected:
 	int_list* syms;

src/CompHash.h

@@ -2,10 +2,10 @@
 
 #pragma once
 
-#include "Hash.h"
 #include "Type.h"
 
 class ListVal;
+class HashKey;
 
 class CompositeHash {
 public:

src/Conn.cc

@@ -2,20 +2,24 @@
 
 #include "zeek-config.h"
 
+#include "Conn.h"
+
 #include <ctype.h>
 
+#include "Desc.h"
 #include "Net.h"
 #include "NetVar.h"
-#include "Conn.h"
 #include "Event.h"
 #include "Sessions.h"
 #include "Reporter.h"
 #include "Timer.h"
+#include "iosource/IOSource.h"
 #include "analyzer/protocol/pia/PIA.h"
 #include "binpac.h"
 #include "TunnelEncapsulation.h"
 #include "analyzer/Analyzer.h"
 #include "analyzer/Manager.h"
+#include "iosource/IOSource.h"
 
 void ConnectionTimer::Init(Connection* arg_conn, timer_func arg_timer,
 				bool arg_do_expire)
@@ -52,7 +56,6 @@ void ConnectionTimer::Dispatch(double t, int is_expire)
 
 uint64_t Connection::total_connections = 0;
 uint64_t Connection::current_connections = 0;
-uint64_t Connection::external_connections = 0;
 
 Connection::Connection(NetSessions* s, const ConnIDKey& k, double t, const ConnID* id,
                        uint32_t flow, const Packet* pkt,
@@ -114,23 +117,10 @@ Connection::Connection(NetSessions* s, const ConnIDKey& k, double t, const ConnI
 	++current_connections;
 	++total_connections;
 
-	TimerMgr::Tag* tag = current_iosrc->GetCurrentTag();
-	conn_timer_mgr = tag ? new TimerMgr::Tag(*tag) : 0;
-
 	if ( arg_encap )
 		encapsulation = new EncapsulationStack(*arg_encap);
 	else
 		encapsulation = 0;
-
-	if ( conn_timer_mgr )
-		{
-		++external_connections;
-		// We schedule a timer which removes this connection from memory
-		// indefinitively into the future. Ii will expire when the timer
-		// mgr is drained but not before.
-		ADD_TIMER(&Connection::RemoveConnectionTimer, 1e20, 1,
-				TIMER_REMOVE_CONNECTION);
-		}
 	}
 
 Connection::~Connection()
@@ -147,12 +137,9 @@ Connection::~Connection()
 		}
 
 	delete root_analyzer;
-	delete conn_timer_mgr;
 	delete encapsulation;
 
 	--current_connections;
-	if ( conn_timer_mgr )
-		--external_connections;
 	}
 
 void Connection::CheckEncapsulation(const EncapsulationStack* arg_encap)
@@ -511,14 +498,14 @@ void Connection::ConnectionEvent(EventHandlerPtr f, analyzer::Analyzer* a, val_l
 
 	// "this" is passed as a cookie for the event
 	mgr.QueueEvent(f, std::move(vl), SOURCE_LOCAL,
-			a ? a->GetID() : 0, GetTimerMgr(), this);
+			a ? a->GetID() : 0, timer_mgr, this);
 	}
 
 void Connection::ConnectionEventFast(EventHandlerPtr f, analyzer::Analyzer* a, val_list vl)
 	{
 	// "this" is passed as a cookie for the event
 	mgr.QueueEventFast(f, std::move(vl), SOURCE_LOCAL,
-			a ? a->GetID() : 0, GetTimerMgr(), this);
+			a ? a->GetID() : 0, timer_mgr, this);
 	}
 
 void Connection::ConnectionEvent(EventHandlerPtr f, analyzer::Analyzer* a, val_list* vl)
@@ -546,7 +533,7 @@ void Connection::AddTimer(timer_func timer, double t, bool do_expire,
 		return;
 
 	Timer* conn_timer = new ConnectionTimer(this, timer, t, do_expire, type);
-	GetTimerMgr()->Add(conn_timer);
+	timer_mgr->Add(conn_timer);
 	timers.push_back(conn_timer);
 	}
 
@@ -565,25 +552,12 @@ void Connection::CancelTimers()
 	std::copy(timers.begin(), timers.end(), std::back_inserter(tmp));
 
 	for ( const auto& timer : tmp )
-		GetTimerMgr()->Cancel(timer);
+		timer_mgr->Cancel(timer);
 
 	timers_canceled = 1;
 	timers.clear();
 	}
 
-TimerMgr* Connection::GetTimerMgr() const
-	{
-	if ( ! conn_timer_mgr )
-		// Global manager.
-		return timer_mgr;
-
-	// We need to check whether the local timer manager still exists;
-	// it may have already been timed out, in which case we fall back
-	// to the global manager (though this should be rare).
-	TimerMgr* local_mgr = sessions->LookupTimerMgr(conn_timer_mgr, false);
-	return local_mgr ? local_mgr : timer_mgr;
-	}
-
 void Connection::FlipRoles()
 	{
 	IPAddr tmp_addr = resp_addr;

src/Conn.h

@@ -7,13 +7,12 @@
 #include <string>
 
 #include "Dict.h"
-#include "Val.h"
 #include "Timer.h"
-#include "RuleMatcher.h"
+#include "Rule.h"
 #include "IPAddr.h"
-#include "TunnelEncapsulation.h"
 #include "UID.h"
 #include "WeirdState.h"
+#include "iosource/Packet.h"
 
 #include "analyzer/Tag.h"
 #include "analyzer/Analyzer.h"
@@ -25,6 +24,9 @@ class LoginConn;
 class RuleHdrTest;
 class Specific_RE_Matcher;
 class RuleEndpointState;
+class EncapsulationStack;
+class Val;
+class RecordVal;
 
 namespace analyzer { class TransportLayerAnalyzer; }
 
@@ -227,11 +229,6 @@ public:
 	void Describe(ODesc* d) const override;
 	void IDString(ODesc* d) const;
 
-	TimerMgr* GetTimerMgr() const;
-
-	// Returns true if connection has been received externally.
-	bool IsExternal() const	{ return conn_timer_mgr != 0; }
-
 	// Statistics.
 
 	// Just a lower bound.
@@ -242,8 +239,6 @@ public:
 		{ return total_connections; }
 	static uint64_t CurrentConnections()
 		{ return current_connections; }
-	static uint64_t CurrentExternalConnections()
-		{ return external_connections; }
 
 	// Returns true if the history was already seen, false otherwise.
 	int CheckHistory(uint32_t mask, char code)
@@ -319,8 +314,6 @@ protected:
 	ConnIDKey key;
 	bool key_valid;
 
-	// Timer manager to use for this conn (or nil).
-	TimerMgr::Tag* conn_timer_mgr;
 	timer_list timers;
 
 	IPAddr orig_addr;
@@ -352,7 +345,6 @@ protected:
 	// Count number of connections.
 	static uint64_t total_connections;
 	static uint64_t current_connections;
-	static uint64_t external_connections;
 
 	string history;
 	uint32_t hist_seen;

src/DFA.cc

@@ -2,8 +2,9 @@
 
 #include "zeek-config.h"
 
-#include "EquivClass.h"
 #include "DFA.h"
+#include "EquivClass.h"
+#include "Desc.h"
 #include "digest.h"
 
 unsigned int DFA_State::transition_counter = 0;

src/DFA.h

@@ -3,9 +3,15 @@
 
 #pragma once
 
-#include <assert.h>
+#include "RE.h" // for typedef AcceptingSet
+#include "Obj.h"
+
+#include <map>
 #include <string>
 
+#include <assert.h>
+#include <sys/types.h> // for u_char
+
 class DFA_State;
 
 // Transitions to the uncomputed state indicate that we haven't yet

src/DNS_Mgr.cc

@@ -2,6 +2,8 @@
 
 #include "zeek-config.h"
 
+#include "DNS_Mgr.h"
+
 #include <sys/types.h>
 #include <sys/socket.h>
 #ifdef TIME_WITH_SYS_TIME
@@ -29,9 +31,9 @@
 
 #include <algorithm>
 
-#include "DNS_Mgr.h"
 #include "Event.h"
 #include "Net.h"
+#include "Val.h"
 #include "Var.h"
 #include "Reporter.h"
 #include "iosource/Manager.h"
@@ -392,7 +394,6 @@ DNS_Mgr::DNS_Mgr(DNS_MgrMode arg_mode)
 	successful = 0;
 	failed = 0;
 	nb_dns = nullptr;
-	next_timestamp = -1.0;
 	}
 
 DNS_Mgr::~DNS_Mgr()
@@ -404,7 +405,7 @@ DNS_Mgr::~DNS_Mgr()
 	delete [] dir;
 	}
 
-void DNS_Mgr::Init()
+void DNS_Mgr::InitSource()
 	{
 	if ( did_init )
 		return;
@@ -440,8 +441,15 @@ void DNS_Mgr::Init()
 		nb_dns = nb_dns_init2(err, (struct sockaddr*)&ss);
 		}
 
-	if ( ! nb_dns )
+	if ( nb_dns )
+		{
+		if ( ! iosource_mgr->RegisterFd(nb_dns_fd(nb_dns), this) )
+			reporter->FatalError("Failed to register nb_dns file descriptor with iosource_mgr");
+		}
+	else
+		{
 		reporter->Warning("problem initializing NB-DNS: %s", err);
+		}
 
 	did_init = true;
 	}
@@ -460,11 +468,6 @@ void DNS_Mgr::InitPostScript()
 	// Registering will call Init()
 	iosource_mgr->Register(this, true);
 
-	// We never set idle to false, having the main loop only calling us from
-	// time to time. If we're issuing more DNS requests than we can handle
-	// in this way, we are having problems anyway ...
-	SetIdle(true);
-
 	const char* cache_dir = dir ? dir : ".";
 	cache_name = new char[strlen(cache_dir) + 64];
 	sprintf(cache_name, "%s/%s", cache_dir, ".zeek-dns-cache");
@@ -503,7 +506,7 @@ TableVal* DNS_Mgr::LookupHost(const char* name)
 	if ( mode == DNS_FAKE )
 		return fake_name_lookup_result(name);
 
-	Init();
+	InitSource();
 
 	if ( ! nb_dns )
 		return empty_addr_set();
@@ -558,7 +561,7 @@ TableVal* DNS_Mgr::LookupHost(const char* name)
 
 Val* DNS_Mgr::LookupAddr(const IPAddr& addr)
 	{
-	Init();
+	InitSource();
 
 	if ( mode != DNS_PRIME )
 		{
@@ -1078,7 +1081,7 @@ static void resolve_lookup_cb(DNS_Mgr::LookupCallback* callback,
 
 void DNS_Mgr::AsyncLookupAddr(const IPAddr& host, LookupCallback* callback)
 	{
-	Init();
+	InitSource();
 
 	if ( mode == DNS_FAKE )
 		{
@@ -1116,7 +1119,7 @@ void DNS_Mgr::AsyncLookupAddr(const IPAddr& host, LookupCallback* callback)
 
 void DNS_Mgr::AsyncLookupName(const string& name, LookupCallback* callback)
 	{
-	Init();
+	InitSource();
 
 	if ( mode == DNS_FAKE )
 		{
@@ -1154,7 +1157,7 @@ void DNS_Mgr::AsyncLookupName(const string& name, LookupCallback* callback)
 
 void DNS_Mgr::AsyncLookupNameText(const string& name, LookupCallback* callback)
 	{
-	Init();
+	InitSource();
 
 	if ( mode == DNS_FAKE )
 		{
@@ -1242,30 +1245,6 @@ void DNS_Mgr::IssueAsyncRequests()
 		}
 	}
 
-void DNS_Mgr::GetFds(iosource::FD_Set* read, iosource::FD_Set* write,
-                     iosource::FD_Set* except)
-	{
-	if ( ! nb_dns )
-		return;
-
-	read->Insert(nb_dns_fd(nb_dns));
-	}
-
-double DNS_Mgr::NextTimestamp(double* network_time)
-	{
-	if ( asyncs_timeouts.empty() )
-		// No pending requests.
-		return -1.0;
-
-	if ( next_timestamp < 0 )
-		// Store the timestamp to help prevent starvation by some other
-		// IOSource always trying to use the same timestamp
-		// (assuming network_time does actually increase).
-		next_timestamp = timer_mgr->Time();
-
-	return next_timestamp;
-	}
-
 void DNS_Mgr::CheckAsyncAddrRequest(const IPAddr& addr, bool timeout)
 	{
 	// Note that this code is a mirror of that for CheckAsyncHostRequest.
@@ -1369,7 +1348,7 @@ void DNS_Mgr::CheckAsyncHostRequest(const char* host, bool timeout)
 
 void DNS_Mgr::Flush()
 	{
-	DoProcess();
+	Process();
 
 	HostMap::iterator it;
 	for ( it = host_mappings.begin(); it != host_mappings.end(); ++it )
@@ -1389,13 +1368,15 @@ void DNS_Mgr::Flush()
 	text_mappings.clear();
 	}
 
-void DNS_Mgr::Process()
+double DNS_Mgr::GetNextTimeout()
 	{
-	DoProcess();
+	if ( asyncs_timeouts.empty() )
-	next_timestamp = -1.0;
+		return -1;
+
+	return network_time + DNS_TIMEOUT;
 	}
 
-void DNS_Mgr::DoProcess()
+void DNS_Mgr::Process()
 	{
 	if ( ! nb_dns )
 		return;
@@ -1513,3 +1494,8 @@ void DNS_Mgr::GetStats(Stats* stats)
 	stats->cached_texts = text_mappings.size();
 	}
 
+void DNS_Mgr::Terminate()
+	{
+	if ( nb_dns )
+		iosource_mgr->UnregisterFd(nb_dns_fd(nb_dns), this);
+	}

src/DNS_Mgr.h

@@ -7,9 +7,7 @@
 #include <queue>
 #include <utility>
 
-#include "util.h"
 #include "List.h"
-#include "Dict.h"
 #include "EventHandler.h"
 #include "iosource/IOSource.h"
 #include "IPAddr.h"
@@ -91,6 +89,8 @@ public:
 
 	void GetStats(Stats* stats);
 
+	void Terminate();
+
 protected:
 	friend class LookupCallback;
 	friend class DNS_Mgr_Request;
@@ -127,16 +127,11 @@ protected:
 	void CheckAsyncHostRequest(const char* host, bool timeout);
 	void CheckAsyncTextRequest(const char* host, bool timeout);
 
-	// Process outstanding requests.
-	void DoProcess();
-
 	// IOSource interface.
-	void GetFds(iosource::FD_Set* read, iosource::FD_Set* write,
-	                    iosource::FD_Set* except) override;
-	double NextTimestamp(double* network_time) override;
 	void Process() override;
-	void Init() override;
+	void InitSource() override;
 	const char* Tag() override	{ return "DNS_Mgr"; }
+	double GetNextTimeout() override;
 
 	DNS_MgrMode mode;
 
@@ -241,7 +236,6 @@ protected:
 	unsigned long num_requests;
 	unsigned long successful;
 	unsigned long failed;
-	double next_timestamp;
 };
 
 extern DNS_Mgr* dns_mgr;

src/DbgBreakpoint.cc

@@ -2,17 +2,22 @@
 
 #include "zeek-config.h"
 
+#include "DbgBreakpoint.h"
+
 #include <assert.h>
 
+#include "Desc.h"
 #include "ID.h"
 #include "Queue.h"
 #include "Debug.h"
 #include "Scope.h"
+#include "Frame.h"
 #include "Func.h"
+#include "Val.h"
 #include "Stmt.h"
-#include "DbgBreakpoint.h"
 #include "Timer.h"
-
+#include "Reporter.h"
+#include "module_util.h"
 
 // BreakpointTimer used for time-based breakpoints
 class BreakpointTimer : public Timer {

src/DbgBreakpoint.h

@@ -2,7 +2,12 @@
 
 #pragma once
 
-#include "Debug.h"
+#include <string>
+
+using std::string;
+
+struct ParseLocationRec;
+class Stmt;
 
 enum BreakCode { bcNoHit, bcHit, bcHitAndDelete };
 class DbgBreakpoint {

src/DbgDisplay.h

@@ -2,7 +2,7 @@
 
 #pragma once
 
-#include "Debug.h"
+class Expr;
 
 // Automatic displays: display these at each stoppage.
 class DbgDisplay {

src/DbgWatch.h

@@ -2,7 +2,8 @@
 
 #pragma once
 
-#include "Debug.h"
+class BroObj;
+class Expr;
 
 class DbgWatch {
 public:

src/Debug.cc

@@ -2,6 +2,8 @@
 
 #include "zeek-config.h"
 
+#include "Debug.h"
+
 #include <stdio.h>
 #include <stdarg.h>
 #include <signal.h>
@@ -11,13 +13,20 @@
 using namespace std;
 
 #include "util.h"
-#include "Debug.h"
 #include "DebugCmds.h"
 #include "DbgBreakpoint.h"
+#include "ID.h"
+#include "Expr.h"
 #include "Stmt.h"
+#include "Frame.h"
 #include "Func.h"
 #include "Scope.h"
 #include "PolicyFile.h"
+#include "Desc.h"
+#include "Reporter.h"
+#include "Val.h"
+#include "module_util.h"
+#include "input.h"
 
 #ifdef HAVE_READLINE
 #include <readline/readline.h>

src/Debug.h

@@ -2,10 +2,15 @@
 
 #pragma once
 
+#include "Obj.h"
+#include "Queue.h"
+#include "StmtEnums.h"
+
 #include <vector>
 #include <map>
 #include <string>
 
+class Val;
 class Stmt;
 
 // This needs to be defined before we do the includes that come after it.
@@ -17,17 +22,10 @@ struct ParseLocationRec {
 	int line;
 };
 
-#include "Expr.h"
-#include "Var.h"
-#include "Frame.h"
-#include "Queue.h"
-#include "Dict.h"
-#include "StmtEnums.h"
-#include "DbgBreakpoint.h"
-
 class StmtLocMapping;
 typedef PQueue<StmtLocMapping> Filemap; // mapping for a single file
 
+class Frame;
 class DbgBreakpoint;
 class DbgWatch;
 class DbgDisplay;

src/DebugCmds.cc

@@ -2,6 +2,7 @@
 // implementation of most commands.
 
 #include "zeek-config.h"
+#include "DebugCmds.h"
 
 #include <sys/types.h>
 
@@ -9,14 +10,18 @@
 #include <string.h>
 #include <assert.h>
 
-#include "Debug.h"
-#include "DebugCmds.h"
 #include "DebugCmdInfoConstants.cc"
+#include "Debug.h"
+#include "Desc.h"
 #include "DbgBreakpoint.h"
+#include "ID.h"
+#include "Frame.h"
 #include "Func.h"
 #include "Stmt.h"
 #include "Scope.h"
+#include "Reporter.h"
 #include "PolicyFile.h"
+#include "Val.h"
 #include "util.h"
 
 //

src/Desc.cc

@@ -1,15 +1,15 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
 #include "zeek-config.h"
+#include "Desc.h"
 
 #include <stdlib.h>
+#include <string.h>
 #include <errno.h>
 #include <math.h>
 
-#include "Desc.h"
 #include "File.h"
 #include "Reporter.h"
-
 #include "ConvertUTF.h"
 
 #define DEFAULT_SIZE 128
@@ -376,11 +376,10 @@ void ODesc::AddBytesRaw(const void* bytes, unsigned int n)
 void ODesc::Grow(unsigned int n)
 	{
 	while ( offset + n + SLOP >= size )
-		{
 		size *= 2;
+
 	base = safe_realloc(base, size);
 	}
-	}
 
 void ODesc::Clear()
 	{

src/Desc.h

@@ -2,12 +2,14 @@
 
 #pragma once
 
-#include <stdio.h>
+#include "BroString.h" // for byte_vec
+#include "util.h" // for bro_int_t
+
 #include <set>
 #include <utility>
 #include <string>
 
-#include "BroString.h"
+#include <sys/types.h> // for u_char
 
 typedef enum {
 	DESC_READABLE,

src/Discard.cc

@@ -1,13 +1,18 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
-#include <algorithm>
-
 #include "zeek-config.h"
 
-#include "Net.h"
-#include "Var.h"
 #include "Discard.h"
 
+#include <algorithm>
+
+#include "Net.h"
+#include "Func.h"
+#include "Var.h"
+#include "Val.h"
+#include "IP.h"
+#include "Reporter.h" // for InterpreterException
+
 Discarder::Discarder()
 	{
 	check_ip = internal_func("discarder_check_ip");

src/Discard.h

@@ -2,14 +2,14 @@
 
 #pragma once
 
-#include "IP.h"
+#include <sys/types.h> // for u_char
-#include "Func.h"
 
 struct ip;
 struct tcphdr;
 struct udphdr;
 struct icmp;
 
+class IP_Hdr;
 class Val;
 class RecordType;
 class Func;

src/EquivClass.cc

@@ -3,6 +3,7 @@
 #include "zeek-config.h"
 
 #include "EquivClass.h"
+#include "CCL.h"
 
 EquivClass::EquivClass(int arg_size)
 	{

src/EquivClass.h

@@ -4,7 +4,7 @@
 
 #include <stdio.h>
 
-#include "CCL.h"
+class CCL;
 
 class EquivClass {
 public:

src/Event.cc

@@ -3,9 +3,11 @@
 #include "zeek-config.h"
 
 #include "Event.h"
+#include "Desc.h"
 #include "Func.h"
 #include "NetVar.h"
 #include "Trigger.h"
+#include "Val.h"
 #include "plugin/Manager.h"
 
 EventMgr mgr;
@@ -101,6 +103,19 @@ EventMgr::~EventMgr()
 	Unref(src_val);
 	}
 
+void EventMgr::QueueEvent(const EventHandlerPtr &h, val_list vl,
+			  SourceID src, analyzer::ID aid,
+			  TimerMgr* mgr, BroObj* obj)
+	{
+	if ( h )
+		QueueEvent(new Event(h, std::move(vl), src, aid, mgr, obj));
+	else
+		{
+		for ( const auto& v : vl )
+			Unref(v);
+		}
+	}
+
 void EventMgr::QueueEvent(Event* event)
 	{
 	bool done = PLUGIN_HOOK_WITH_RESULT(HOOK_QUEUE_EVENT, HookQueueEvent(event), false);
@@ -119,6 +134,13 @@ void EventMgr::QueueEvent(Event* event)
 	++num_events_queued;
 	}
 
+void EventMgr::Dispatch(Event* event, bool no_remote)
+	{
+	current_src = event->Source();
+	event->Dispatch(no_remote);
+	Unref(event);
+	}
+
 void EventMgr::Drain()
 	{
 	if ( event_queue_flush_point )
@@ -162,10 +184,9 @@ void EventMgr::Drain()
 	// do after draining events.
 	draining = false;
 
-	// We evaluate Triggers here. While this is somewhat unrelated to event
+	// Make sure all of the triggers get processed every time the events
-	// processing, we ensure that it's done at a regular basis by checking
+	// drain.
-	// them here.
+	trigger_mgr->Process();
-	Trigger::EvaluatePending();
 	}
 
 void EventMgr::Describe(ODesc* d) const

src/Event.h

@@ -2,9 +2,7 @@
 
 #pragma once
 
-#include "EventRegistry.h"
+#include "BroList.h"
-
-#include "analyzer/Tag.h"
 #include "analyzer/Analyzer.h"
 
 class EventMgr;
@@ -79,16 +77,7 @@ public:
 	// existence check.
 	void QueueEvent(const EventHandlerPtr &h, val_list vl,
 			SourceID src = SOURCE_LOCAL, analyzer::ID aid = 0,
-			TimerMgr* mgr = 0, BroObj* obj = 0)
+			TimerMgr* mgr = 0, BroObj* obj = 0);
-		{
-		if ( h )
-			QueueEvent(new Event(h, std::move(vl), src, aid, mgr, obj));
-		else
-			{
-			for ( const auto& v : vl )
-				Unref(v);
-			}
-		}
 
 	// Same as QueueEvent, except taking the event's argument list via a
 	// pointer instead of by value.  This function takes ownership of the
@@ -102,12 +91,7 @@ public:
 		delete vl;
 		}
 
-	void Dispatch(Event* event, bool no_remote = false)
+	void Dispatch(Event* event, bool no_remote = false);
-		{
-		current_src = event->Source();
-		event->Dispatch(no_remote);
-		Unref(event);
-		}
 
 	void Drain();
 	bool IsDraining() const	{ return draining; }

src/EventHandler.cc

@@ -1,5 +1,6 @@
-#include "Event.h"
 #include "EventHandler.h"
+#include "Event.h"
+#include "Desc.h"
 #include "Func.h"
 #include "Scope.h"
 #include "NetVar.h"

src/EventHandler.h

@@ -2,11 +2,10 @@
 
 #pragma once
 
-#include <assert.h>
+#include "BroList.h"
+
 #include <unordered_set>
 #include <string>
-#include "List.h"
-#include "BroList.h"
 
 class Func;
 class FuncType;

src/EventRegistry.cc

@@ -1,4 +1,5 @@
 #include "EventRegistry.h"
+#include "EventHandler.h"
 #include "RE.h"
 #include "Reporter.h"
 

src/EventRegistry.h

@@ -4,11 +4,14 @@
 
 #include <map>
 #include <string>
+#include <vector>
 
-#include "Func.h"
+using std::string;
-#include "List.h"
+using std::vector;
-#include "Dict.h"
+
-#include "EventHandler.h"
+class EventHandler;
+class EventHandlerPtr;
+class RE_Matcher;
 
 // The registry keeps track of all events that we provide or handle.
 class EventRegistry {

src/Expr.cc

@@ -4,6 +4,7 @@
 
 #include "Expr.h"
 #include "Event.h"
+#include "Desc.h"
 #include "Frame.h"
 #include "Func.h"
 #include "RE.h"
@@ -15,6 +16,8 @@
 #include "Trigger.h"
 #include "IPAddr.h"
 #include "digest.h"
+#include "module_util.h"
+#include "DebugLogger.h"
 
 #include "broker/Data.h"
 
@@ -132,12 +135,32 @@ Val* Expr::InitVal(const BroType* t, Val* aggr) const
 	return check_and_promote(Eval(0), t, 1);
 	}
 
+int Expr::IsError() const
+	{
+	return type && type->Tag() == TYPE_ERROR;
+	}
+
+void Expr::SetError()
+	{
+	SetType(error_type());
+	}
+
 void Expr::SetError(const char* msg)
 	{
 	Error(msg);
 	SetError();
 	}
 
+int Expr::IsZero() const
+	{
+	return IsConst() && ExprVal()->IsZero();
+	}
+
+int Expr::IsOne() const
+	{
+	return IsConst() && ExprVal()->IsOne();
+	}
+
 void Expr::Describe(ODesc* d) const
 	{
 	if ( IsParen() && ! d->IsBinary() )
@@ -2076,6 +2099,11 @@ AssignExpr::AssignExpr(Expr* arg_op1, Expr* arg_op2, int arg_is_init,
 	SetLocationInfo(arg_op1->GetLocationInfo(), arg_op2->GetLocationInfo());
 	}
 
+AssignExpr::~AssignExpr()
+	{
+	Unref(val);
+	}
+
 bool AssignExpr::TypeCheck(attr_list* attrs)
 	{
 	TypeTag bt1 = op1->Type()->Tag();
@@ -4258,7 +4286,7 @@ Val* CallExpr::Eval(Frame* f) const
 	// Check for that.
 	if ( f )
 		{
-		Trigger* trigger = f->GetTrigger();
+		trigger::Trigger* trigger = f->GetTrigger();
 
 		if ( trigger )
 			{

src/Expr.h

@@ -5,16 +5,17 @@
 // BRO expressions.
 
 #include "BroList.h"
-#include "ID.h"
 #include "Timer.h"
-#include "Val.h"
+#include "Type.h"
-#include "Debug.h"
 #include "EventHandler.h"
 #include "TraverseTypes.h"
 
 #include <memory>
+#include <string>
 #include <utility>
 
+using std::string;
+
 typedef enum {
 	EXPR_ANY = -1,
 	EXPR_NAME, EXPR_CONST,
@@ -117,10 +118,10 @@ public:
 	int IsConst() const	{ return tag == EXPR_CONST; }
 
 	// True if the expression is in error (to alleviate error propagation).
-	int IsError() const	{ return type && type->Tag() == TYPE_ERROR; }
+	int IsError() const;
 
 	// Mark expression as in error.
-	void SetError()		{ SetType(error_type()); }
+	void SetError();
 	void SetError(const char* msg);
 
 	// Returns the expression's constant value, or complains
@@ -128,16 +129,10 @@ public:
 	inline Val* ExprVal() const;
 
 	// True if the expression is a constant zero, false otherwise.
-	int IsZero() const
+	int IsZero() const;
-		{
-		return IsConst() && ExprVal()->IsZero();
-		}
 
 	// True if the expression is a constant one, false otherwise.
-	int IsOne() const
+	int IsOne() const;
-		{
-		return IsConst() && ExprVal()->IsOne();
-		}
 
 	// True if the expression supports the "add" or "delete" operations,
 	// false otherwise.
@@ -603,7 +598,7 @@ public:
 	// If val is given, evaluating this expression will always yield the val
 	// yet still perform the assignment.  Used for triggers.
 	AssignExpr(Expr* op1, Expr* op2, int is_init, Val* val = 0, attr_list* attrs = 0);
-	~AssignExpr() override { Unref(val); }
+	~AssignExpr() override;
 
 	Val* Eval(Frame* f) const override;
 	void EvalIntoAggregate(const BroType* t, Val* aggr, Frame* f) const override;
@@ -873,8 +868,6 @@ protected:
 	int num_fields;
 };
 
-class EventHandler;
-
 class ScheduleTimer : public Timer {
 public:
 	ScheduleTimer(EventHandlerPtr event, val_list* args, double t,

src/File.cc

@@ -1,6 +1,7 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
 #include "zeek-config.h"
+#include "File.h"
 
 #include <sys/types.h>
 #ifdef TIME_WITH_SYS_TIME
@@ -20,13 +21,14 @@
 
 #include <algorithm>
 
-#include "File.h"
+#include "Attr.h"
 #include "Type.h"
 #include "Expr.h"
 #include "NetVar.h"
 #include "Net.h"
 #include "Event.h"
 #include "Reporter.h"
+#include "Desc.h"
 
 std::list<std::pair<std::string, BroFile*>> BroFile::open_files;
 

src/File.h

@@ -2,19 +2,21 @@
 
 #pragma once
 
-#include <fcntl.h>
-#include "util.h"
 #include "Obj.h"
-#include "Attr.h"
 
 #include <list>
+#include <string>
 #include <utility>
 
+#include <fcntl.h>
+
 # ifdef NEED_KRB5_H
 #  include <krb5.h>
 # endif // NEED_KRB5_H
 
+class Attributes;
 class BroType;
+class RecordVal;
 
 class BroFile : public BroObj {
 public:

src/Frag.cc

@@ -2,11 +2,12 @@
 
 #include "zeek-config.h"
 
-#include "util.h"
-#include "Hash.h"
 #include "Frag.h"
+#include "Hash.h"
+#include "IP.h"
 #include "NetVar.h"
 #include "Sessions.h"
+#include "Reporter.h"
 
 #define MIN_ACCEPTABLE_FRAG_SIZE 64
 #define MAX_ACCEPTABLE_FRAG_SIZE 64000

src/Frag.h

@@ -2,16 +2,18 @@
 
 #pragma once
 
-#include <tuple>
+#include "util.h" // for bro_uint_t
-
+#include "IPAddr.h"
-#include "util.h"
-#include "IP.h"
-#include "Net.h"
 #include "Reassem.h"
 #include "Timer.h"
 
+#include <tuple>
+
+#include <sys/types.h> // for u_char
+
 class HashKey;
 class NetSessions;
+class IP_Hdr;
 
 class FragReassembler;
 class FragTimer;

src/Frame.cc

@@ -1,10 +1,15 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
+#include "Frame.h"
+
 #include <broker/error.hh>
 #include "broker/Data.h"
 
-#include "Frame.h"
+#include "Func.h"
+#include "Desc.h"
+#include "IntrusivePtr.h"
 #include "Trigger.h"
+#include "Val.h"
 
 vector<Frame*> g_frame_stack;
 
@@ -515,7 +520,7 @@ void Frame::CaptureClosure(Frame* c, id_list arg_outer_ids)
 	// if (c) closure = c->SelectiveClone(outer_ids);
 	}
 
-void Frame::SetTrigger(Trigger* arg_trigger)
+void Frame::SetTrigger(trigger::Trigger* arg_trigger)
 	{
 	ClearTrigger();
 
@@ -531,6 +536,14 @@ void Frame::ClearTrigger()
 	trigger = nullptr;
 	}
 
+void Frame::UnrefElement(int n)
+	{
+	if ( weak_refs && weak_refs[n] )
+		return;
+
+	Unref(frame[n]);
+	}
+
 bool Frame::IsOuterID(const ID* in) const
 	{
 	return std::any_of(outer_ids.begin(), outer_ids.end(),

src/Frame.h

@@ -2,17 +2,20 @@
 
 #pragma once
 
+#include "BroList.h" // for typedef val_list
+#include "Obj.h"
+
 #include <unordered_map>
-#include <memory>
 #include <string>
+#include <utility>
+#include <vector>
 
 #include <broker/data.hh>
 #include <broker/expected.hh>
 
-#include "Val.h"
+namespace trigger { class Trigger; }
-
-class Trigger;
 class CallExpr;
+class BroFunc;
 
 class Frame :  public BroObj {
 public:
@@ -207,9 +210,9 @@ public:
 
 	// If the frame is run in the context of a trigger condition evaluation,
 	// the trigger needs to be registered.
-	void SetTrigger(Trigger* arg_trigger);
+	void SetTrigger(trigger::Trigger* arg_trigger);
 	void ClearTrigger();
-	Trigger* GetTrigger() const		{ return trigger; }
+	trigger::Trigger* GetTrigger() const		{ return trigger; }
 
 	void SetCall(const CallExpr* arg_call)	{ call = arg_call; }
 	void ClearCall()			{ call = 0; }
@@ -232,13 +235,7 @@ private:
 	/**
 	 * Unrefs the value at offset 'n' frame unless it's a weak reference.
 	 */
-	void UnrefElement(int n)
+	void UnrefElement(int n);
-		{
-		if ( weak_refs && weak_refs[n] )
-			return;
-
-		Unref(frame[n]);
-		}
 
 	/** Have we captured this id? */
 	bool IsOuterID(const ID* in) const;
@@ -293,7 +290,7 @@ private:
 	bool break_before_next_stmt;
 	bool break_on_return;
 
-	Trigger* trigger;
+	trigger::Trigger* trigger;
 	const CallExpr* call;
 	bool delayed;
 

src/Func.cc

@@ -1,6 +1,7 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
 #include "zeek-config.h"
+#include "Func.h"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -32,12 +33,14 @@
 #include <broker/error.hh>
 
 #include "Base64.h"
+#include "Debug.h"
+#include "Desc.h"
+#include "Expr.h"
 #include "Stmt.h"
 #include "Scope.h"
 #include "Net.h"
 #include "NetVar.h"
 #include "File.h"
-#include "Func.h"
 #include "Frame.h"
 #include "Var.h"
 #include "analyzer/protocol/login/Login.h"
@@ -47,6 +50,9 @@
 #include "Traverse.h"
 #include "Reporter.h"
 #include "plugin/Manager.h"
+#include "module_util.h"
+#include "iosource/PktSrc.h"
+#include "iosource/PktDumper.h"
 
 extern	RETSIGTYPE sig_handler(int signo);
 

src/Func.h

@@ -2,16 +2,21 @@
 
 #pragma once
 
+#include "BroList.h"
+#include "Obj.h"
+#include "Type.h" /* for function_flavor */
+#include "TraverseTypes.h"
+
 #include <utility>
 #include <memory>
+#include <string>
+#include <vector>
 
 #include <broker/data.hh>
 #include <broker/expected.hh>
 
-#include "BroList.h"
+using std::string;
-#include "Obj.h"
+using std::vector;
-#include "Debug.h"
-#include "Frame.h"
 
 class Val;
 class ListExpr;
@@ -20,6 +25,7 @@ class Stmt;
 class Frame;
 class ID;
 class CallExpr;
+class Scope;
 
 class Func : public BroObj {
 public:

src/Hash.cc

@@ -19,6 +19,7 @@
 
 #include "Hash.h"
 #include "Reporter.h"
+#include "BroString.h"
 
 #include "siphash24.h"
 

src/Hash.h

@@ -2,12 +2,14 @@
 
 #pragma once
 
+#include "util.h" // for bro_int_t
+
 #include <stdlib.h>
 
-#include "BroString.h"
-
 #define UHASH_KEY_SIZE 36
 
+class BroString;
+
 typedef uint64_t hash_t;
 
 typedef enum {

src/ID.cc

@@ -3,15 +3,22 @@
 #include "zeek-config.h"
 
 #include "ID.h"
+#include "Attr.h"
+#include "Desc.h"
 #include "Expr.h"
 #include "Dict.h"
 #include "EventRegistry.h"
 #include "Func.h"
 #include "Scope.h"
+#include "Type.h"
 #include "File.h"
 #include "Scope.h"
 #include "Traverse.h"
+#include "Val.h"
 #include "zeekygen/Manager.h"
+#include "zeekygen/IdentifierInfo.h"
+#include "zeekygen/ScriptInfo.h"
+#include "module_util.h"
 
 ID::ID(const char* arg_name, IDScope arg_scope, bool arg_is_export)
 	{
@@ -51,6 +58,11 @@ string ID::ModuleName() const
 	return extract_module_name(name);
 	}
 
+void ID::SetType(BroType* t)
+	{
+	Unref(type); type = t;
+	}
+
 void ID::ClearVal()
 	{
 	if ( ! weak_ref )
@@ -143,6 +155,11 @@ void ID::SetVal(Expr* ev, init_class c)
 	EvalFunc(a->AttrExpr(), ev);
 	}
 
+bool ID::IsRedefinable() const
+	{
+	return FindAttr(ATTR_REDEF) != 0;
+	}
+
 void ID::SetAttrs(Attributes* a)
 	{
 	Unref(attrs);
@@ -189,6 +206,16 @@ void ID::UpdateValAttrs()
 		}
 	}
 
+Attr* ID::FindAttr(attr_tag t) const
+	{
+	return attrs ? attrs->FindAttr(t) : 0;
+	}
+
+bool ID::IsDeprecated() const
+	{
+	return FindAttr(ATTR_DEPRECATED) != 0;
+	}
+
 void ID::MakeDeprecated(Expr* deprecation)
 	{
 	if ( IsDeprecated() )

src/ID.h

@@ -2,14 +2,20 @@
 
 #pragma once
 
-#include "Type.h"
+#include "Obj.h"
 #include "Attr.h"
 #include "Notifier.h"
 #include "TraverseTypes.h"
+
+#include <map>
 #include <string>
+#include <vector>
 
 class Val;
+class Expr;
 class Func;
+class BroType;
+class Attributes;
 
 typedef enum { INIT_NONE, INIT_FULL, INIT_EXTRA, INIT_REMOVE, } init_class;
 typedef enum { SCOPE_FUNCTION, SCOPE_MODULE, SCOPE_GLOBAL } IDScope;
@@ -29,7 +35,7 @@ public:
 
 	std::string ModuleName() const;
 
-	void SetType(BroType* t)	{ Unref(type); type = t; }
+	void SetType(BroType* t);
 	BroType* Type()			{ return type; }
 	const BroType* Type() const	{ return type; }
 
@@ -67,7 +73,7 @@ public:
 	void SetOffset(int arg_offset)	{ offset = arg_offset; }
 	int Offset() const		{ return offset; }
 
-	bool IsRedefinable() const	{ return FindAttr(ATTR_REDEF) != 0; }
+	bool IsRedefinable() const;
 
 	void SetAttrs(Attributes* attr);
 	void AddAttrs(Attributes* attr);
@@ -75,11 +81,9 @@ public:
 	void UpdateValAttrs();
 	Attributes* Attrs() const	{ return attrs; }
 
-	Attr* FindAttr(attr_tag t) const
+	Attr* FindAttr(attr_tag t) const;
-		{ return attrs ? attrs->FindAttr(t) : 0; }
 
-	bool IsDeprecated() const
+	bool IsDeprecated() const;
-		{ return FindAttr(ATTR_DEPRECATED) != 0; }
 
 	void MakeDeprecated(Expr* deprecation);
 

src/IP.cc

@@ -1,13 +1,16 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
+#include "IP.h"
+
 #include <sys/types.h>
 #include <netinet/in.h>
 #include <netinet/icmp6.h>
 
-#include "IP.h"
+#include "IPAddr.h"
 #include "Type.h"
 #include "Val.h"
 #include "Var.h"
+#include "Reporter.h"
 
 static RecordType* ip4_hdr_type = 0;
 static RecordType* ip6_hdr_type = 0;
@@ -304,6 +307,26 @@ RecordVal* IPv6_Hdr::BuildRecordVal(VectorVal* chain) const
 	return rv;
 	}
 
+IPAddr IP_Hdr::IPHeaderSrcAddr() const
+	{
+	return ip4 ? IPAddr(ip4->ip_src) : IPAddr(ip6->ip6_src);
+	}
+
+IPAddr IP_Hdr::IPHeaderDstAddr() const
+	{
+	return ip4 ? IPAddr(ip4->ip_dst) : IPAddr(ip6->ip6_dst);
+	}
+
+IPAddr IP_Hdr::SrcAddr() const
+	{
+	return ip4 ? IPAddr(ip4->ip_src) : ip6_hdrs->SrcAddr();
+	}
+
+IPAddr IP_Hdr::DstAddr() const
+	{
+	return ip4 ? IPAddr(ip4->ip_dst) : ip6_hdrs->DstAddr();
+	}
+
 RecordVal* IP_Hdr::BuildIPHdrVal() const
 	{
 	RecordVal* rval = 0;
@@ -446,6 +469,15 @@ static inline bool isIPv6ExtHeader(uint8_t type)
 	}
 	}
 
+IPv6_Hdr_Chain::~IPv6_Hdr_Chain()
+	{
+	for ( size_t i = 0; i < chain.size(); ++i ) delete chain[i];
+#ifdef ENABLE_MOBILE_IPV6
+	delete homeAddr;
+#endif
+	delete finalDst;
+	}
+
 void IPv6_Hdr_Chain::Init(const struct ip6_hdr* ip6, int total_len,
                           bool set_next, uint16_t next)
 	{
@@ -510,6 +542,46 @@ void IPv6_Hdr_Chain::Init(const struct ip6_hdr* ip6, int total_len,
 				  isIPv6ExtHeader(next_type) );
 	}
 
+bool IPv6_Hdr_Chain::IsFragment() const
+	{
+	if ( chain.empty() )
+		{
+		reporter->InternalWarning("empty IPv6 header chain");
+		return false;
+		}
+
+	return chain[chain.size()-1]->Type() == IPPROTO_FRAGMENT;
+	}
+
+IPAddr IPv6_Hdr_Chain::SrcAddr() const
+	{
+#ifdef ENABLE_MOBILE_IPV6
+	if ( homeAddr )
+		return IPAddr(*homeAddr);
+#endif
+	if ( chain.empty() )
+		{
+		reporter->InternalWarning("empty IPv6 header chain");
+		return IPAddr();
+		}
+
+	return IPAddr(((const struct ip6_hdr*)(chain[0]->Data()))->ip6_src);
+	}
+
+IPAddr IPv6_Hdr_Chain::DstAddr() const
+	{
+	if ( finalDst )
+		return IPAddr(*finalDst);
+
+	if ( chain.empty() )
+		{
+		reporter->InternalWarning("empty IPv6 header chain");
+		return IPAddr();
+		}
+
+	return IPAddr(((const struct ip6_hdr*)(chain[0]->Data()))->ip6_dst);
+	}
+
 void IPv6_Hdr_Chain::ProcessRoutingHeader(const struct ip6_rthdr* r, uint16_t len)
 	{
 	if ( finalDst )

src/IP.h

@@ -3,15 +3,23 @@
 #pragma once
 
 #include "zeek-config.h"
-#include "net_util.h"
+
-#include "IPAddr.h"
-#include "Reporter.h"
-#include "Val.h"
-#include "Type.h"
 #include <vector>
+
+#include <sys/types.h> // for u_char
 #include <netinet/in.h>
 #include <netinet/ip.h>
 
+#ifdef HAVE_NETINET_IP6_H
+#include <netinet/ip6.h>
+#endif
+
+using std::vector;
+
+class IPAddr;
+class RecordVal;
+class VectorVal;
+
 #ifdef ENABLE_MOBILE_IPV6
 
 #ifndef IPPROTO_MOBILITY
@@ -147,14 +155,7 @@ public:
 		finalDst(0)
 		{ Init(ip6, len, false); }
 
-	~IPv6_Hdr_Chain()
+	~IPv6_Hdr_Chain();
-		{
-		for ( size_t i = 0; i < chain.size(); ++i ) delete chain[i];
-#ifdef ENABLE_MOBILE_IPV6
-		delete homeAddr;
-#endif
-		delete finalDst;
-		}
 
 	/**
 	 * @return a copy of the header chain, but with pointers to individual
@@ -180,16 +181,7 @@ public:
 	/**
 	 * Returns whether the header chain indicates a fragmented packet.
 	 */
-	bool IsFragment() const
+	bool IsFragment() const;
-		{
-		if ( chain.empty() )
-			{
-			reporter->InternalWarning("empty IPv6 header chain");
-			return false;
-			}
-
-		return chain[chain.size()-1]->Type() == IPPROTO_FRAGMENT;
-		}
 
 	/**
 	 * Returns pointer to fragment header structure if the chain contains one.
@@ -224,39 +216,14 @@ public:
 	 * option as defined by Mobile IPv6 (RFC 6275), then return it, else
 	 * return the source address in the main IPv6 header.
 	 */
-	IPAddr SrcAddr() const
+	IPAddr SrcAddr() const;
-		{
-#ifdef ENABLE_MOBILE_IPV6
-		if ( homeAddr )
-			return IPAddr(*homeAddr);
-#endif
-		if ( chain.empty() )
-			{
-			reporter->InternalWarning("empty IPv6 header chain");
-			return IPAddr();
-			}
-
-		return IPAddr(((const struct ip6_hdr*)(chain[0]->Data()))->ip6_src);
-		}
 
 	/**
 	 * If the chain contains a Routing header with non-zero segments left,
 	 * then return the last address of the first such header, else return
 	 * the destination address of the main IPv6 header.
 	 */
-	IPAddr DstAddr() const
+	IPAddr DstAddr() const;
-		{
-		if ( finalDst )
-			return IPAddr(*finalDst);
-
-		if ( chain.empty() )
-			{
-			reporter->InternalWarning("empty IPv6 header chain");
-			return IPAddr();
-			}
-
-		return IPAddr(((const struct ip6_hdr*)(chain[0]->Data()))->ip6_dst);
-		}
 
 	/**
 	 * Returns a vector of ip6_ext_hdr RecordVals that includes script-layer
@@ -400,22 +367,19 @@ public:
 	/**
 	 * Returns the source address held in the IP header.
 	 */
-	IPAddr IPHeaderSrcAddr() const
+	IPAddr IPHeaderSrcAddr() const;
-		{ return ip4 ? IPAddr(ip4->ip_src) : IPAddr(ip6->ip6_src); }
 
 	/**
 	 * Returns the destination address held in the IP header.
 	 */
-	IPAddr IPHeaderDstAddr() const
+	IPAddr IPHeaderDstAddr() const;
-		{ return ip4 ? IPAddr(ip4->ip_dst) : IPAddr(ip6->ip6_dst); }
 
 	/**
 	 * For IPv4 or IPv6 headers that don't contain a Home Address option
 	 * (Mobile IPv6, RFC 6275), return source address held in the IP header.
 	 * For IPv6 headers that contain a Home Address option, return that address.
 	 */
-	IPAddr SrcAddr() const
+	IPAddr SrcAddr() const;
-		{ return ip4 ? IPAddr(ip4->ip_src) : ip6_hdrs->SrcAddr(); }
 
 	/**
 	 * For IPv4 or IPv6 headers that don't contain a Routing header with
@@ -423,8 +387,7 @@ public:
 	 * For IPv6 headers with a Routing header that has non-zero segments left,
 	 * return the last address in the first such Routing header.
 	 */
-	IPAddr DstAddr() const
+	IPAddr DstAddr() const;
-		{ return ip4 ? IPAddr(ip4->ip_dst) : ip6_hdrs->DstAddr(); }
 
 	/**
 	 * Returns a pointer to the payload of the IP packet, usually an

src/IPAddr.cc

@@ -5,7 +5,9 @@
 #include <vector>
 #include "IPAddr.h"
 #include "Reporter.h"
+#include "BroString.h"
 #include "Conn.h"
+#include "Hash.h"
 #include "bro_inet_ntop.h"
 
 #include "analyzer/Manager.h"
@@ -45,6 +47,16 @@ ConnIDKey BuildConnIDKey(const ConnID& id)
 	return key;
 	}
 
+IPAddr::IPAddr(const BroString& s)
+	{
+	Init(s.CheckString());
+	}
+
+HashKey* IPAddr::GetHashKey() const
+	{
+	return new HashKey((void*)in6.s6_addr, sizeof(in6.s6_addr));
+	}
+
 static inline uint32_t bit_mask32(int bottom_bits)
 	{
 	if ( bottom_bits >= 32 )
@@ -290,6 +302,19 @@ string IPPrefix::AsString() const
 	return prefix.AsString() +"/" + l;
 	}
 
+HashKey* IPPrefix::GetHashKey() const
+	{
+	struct {
+		in6_addr ip;
+		uint32_t len;
+	} key;
+
+	key.ip = prefix.in6;
+	key.len = Length();
+
+	return new HashKey(&key, sizeof(key));
+	}
+
 bool IPPrefix::ConvertString(const char* text, IPPrefix* result)
 	{
 	string s(text);

src/IPAddr.h

@@ -2,18 +2,17 @@
 
 #pragma once
 
+#include "threading/SerialTypes.h"
+
 #include <netinet/in.h>
 #include <arpa/inet.h>
 #include <string.h>
 #include <string>
 
-#include "BroString.h"
+using std::string;
-#include "Hash.h"
-#include "util.h"
-#include "Type.h"
-#include "threading/SerialTypes.h"
-
 struct ConnID;
+class BroString;
+class HashKey;
 namespace analyzer { class ExpectedConn; }
 
 typedef in_addr in4_addr;
@@ -113,10 +112,7 @@ public:
 	 * @param s String containing an IP address as either a dotted IPv4
 	 * address or a hex IPv6 address.
 	 */
-	explicit IPAddr(const BroString& s)
+	explicit IPAddr(const BroString& s);
-		{
-		Init(s.CheckString());
-		}
 
 	/**
 	 * Constructs an address instance from a raw byte representation.
@@ -255,10 +251,7 @@ public:
 	 * Returns a key that can be used to lookup the IP Address in a hash
 	 * table. Passes ownership to caller.
 	 */
-	HashKey* GetHashKey() const
+	HashKey* GetHashKey() const;
-		{
-		return new HashKey((void*)in6.s6_addr, sizeof(in6.s6_addr));
-		}
 
 	/**
 	 * Masks out lower bits of the address.
@@ -640,18 +633,7 @@ public:
 	 * Returns a key that can be used to lookup the IP Prefix in a hash
 	 * table. Passes ownership to caller.
 	 */
-	HashKey* GetHashKey() const
+	HashKey* GetHashKey() const;
-		{
-		struct {
-			in6_addr ip;
-			uint32_t len;
-		} key;
-
-		key.ip = prefix.in6;
-		key.len = Length();
-
-		return new HashKey(&key, sizeof(key));
-		}
 
 	/** Converts the prefix into the type used internally by the
 	  * inter-thread communication.

src/NFA.cc

@@ -2,10 +2,12 @@
 
 #include "zeek-config.h"
 
-#include <algorithm>
-
 #include "NFA.h"
+#include "Desc.h"
 #include "EquivClass.h"
+#include "IntSet.h"
+
+#include <algorithm>
 
 static int nfa_state_id = 0;
 

src/NFA.h

@@ -2,9 +2,11 @@
 
 #pragma once
 
-#include "RE.h"
+#include "Obj.h"
-#include "IntSet.h"
+#include "List.h"
 
+class CCL;
+class Func;
 class NFA_State;
 class EquivClass;
 

src/Net.cc

@@ -1,6 +1,7 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
 #include "zeek-config.h"
+#include "Net.h"
 
 #include <sys/types.h>
 #ifdef TIME_WITH_SYS_TIME
@@ -19,13 +20,16 @@
 #include <stdlib.h>
 #include <unistd.h>
 
+extern "C" {
+#include "setsignal.h"
+};
+
 #include "NetVar.h"
 #include "Sessions.h"
 #include "Event.h"
 #include "Timer.h"
 #include "Var.h"
 #include "Reporter.h"
-#include "Net.h"
 #include "Anon.h"
 #include "PacketDumper.h"
 #include "iosource/Manager.h"
@@ -34,19 +38,15 @@
 #include "plugin/Manager.h"
 #include "broker/Manager.h"
 
-extern "C" {
-#include "setsignal.h"
-};
-
 extern "C" {
 extern int select(int, fd_set *, fd_set *, fd_set *, struct timeval *);
 }
 
-iosource::PktDumper* pkt_dumper = 0;
+iosource::PktDumper* pkt_dumper = nullptr;
 
-int reading_live = 0;
+bool reading_live = false;
-int reading_traces = 0;
+bool reading_traces = false;
-int have_pending_timers = 0;
+bool have_pending_timers = false;
 double pseudo_realtime = 0.0;
 double network_time = 0.0;	// time according to last packet timestamp
 				// (or current time)
@@ -57,11 +57,11 @@ double last_watchdog_proc_time = 0.0;	// value of above during last watchdog
 bool terminating = false;	// whether we're done reading and finishing up
 bool is_parsing = false;
 
-const Packet *current_pkt = 0;
+const Packet *current_pkt = nullptr;
 int current_dispatched = 0;
 double current_timestamp = 0.0;
-iosource::PktSrc* current_pktsrc = 0;
+iosource::PktSrc* current_pktsrc = nullptr;
-iosource::IOSource* current_iosrc = 0;
+iosource::IOSource* current_iosrc = nullptr;
 
 std::list<ScannedFile> files_scanned;
 std::vector<string> sig_files;
@@ -145,49 +145,42 @@ void net_update_time(double new_network_time)
 	PLUGIN_HOOK_VOID(HOOK_UPDATE_NETWORK_TIME, HookUpdateNetworkTime(new_network_time));
 	}
 
-void net_init(const std::vector<std::string>& interfaces,
+void net_init(const std::optional<std::string>& interface,
-              const std::vector<std::string>& pcap_input_files,
+              const std::optional<std::string>& pcap_input_file,
               const std::optional<std::string>& pcap_output_file,
               bool do_watchdog)
 	{
-	if ( ! pcap_input_files.empty() )
+	if ( pcap_input_file )
 		{
 		reading_live = pseudo_realtime > 0.0;
-		reading_traces = 1;
+		reading_traces = true;
 
-		for ( const auto& pif : pcap_input_files )
+		iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(*pcap_input_file, false);
-			{
-			iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(pif, false);
 		assert(ps);
 
 		if ( ! ps->IsOpen() )
 			reporter->FatalError("problem with trace file %s (%s)",
-				                     pif.data(), ps->ErrorMsg());
+				pcap_input_file->c_str(), ps->ErrorMsg());
 		}
-		}
+	else if ( interface )
-
-	else if ( ! interfaces.empty() )
 		{
-		reading_live = 1;
+		reading_live = true;
-		reading_traces = 0;
+		reading_traces = false;
 
-		for ( const auto& iface : interfaces )
+		iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(*interface, true);
-			{
-			iosource::PktSrc* ps = iosource_mgr->OpenPktSrc(iface, true);
 		assert(ps);
 
 		if ( ! ps->IsOpen() )
 			reporter->FatalError("problem with interface %s (%s)",
-				                     iface.data(), ps->ErrorMsg());
+				interface->c_str(), ps->ErrorMsg());
-			}
 		}
 
 	else
-		// have_pending_timers = 1, possibly.  We don't set
+		// have_pending_timers = true, possibly.  We don't set
 		// that here, though, because at this point we don't know
 		// whether the user's zeek_init() event will indeed set
 		// a timer.
-		reading_traces = reading_live = 0;
+		reading_traces = reading_live = false;
 
 	if ( pcap_output_file )
 		{
@@ -199,11 +192,10 @@ void net_init(const std::vector<std::string>& interfaces,
 			reporter->FatalError("problem opening dump file %s (%s)",
 					     writefile, pkt_dumper->ErrorMsg());
 
-		ID* id = global_scope()->Lookup("trace_output_file");
+		if ( ID* id = global_scope()->Lookup("trace_output_file") )
-		if ( ! id )
-			reporter->Error("trace_output_file not defined in bro.init");
-		else
 			id->SetVal(new StringVal(writefile));
+		else
+			reporter->Error("trace_output_file not defined in bro.init");
 		}
 
 	init_ip_addr_anonymizers();
@@ -221,12 +213,9 @@ void net_init(const std::vector<std::string>& interfaces,
 void expire_timers(iosource::PktSrc* src_ps)
 	{
 	SegmentProfiler(segment_logger, "expiring-timers");
-	TimerMgr* tmgr =
-		src_ps ? sessions->LookupTimerMgr(src_ps->GetCurrentTag())
-			: timer_mgr;
 
 	current_dispatched +=
-		tmgr->Advance(network_time,
+		timer_mgr->Advance(network_time,
 			max_timer_expires - current_dispatched);
 	}
 
@@ -235,10 +224,8 @@ void net_packet_dispatch(double t, const Packet* pkt, iosource::PktSrc* src_ps)
 	if ( ! bro_start_network_time )
 		bro_start_network_time = t;
 
-	TimerMgr* tmgr = sessions->LookupTimerMgr(src_ps->GetCurrentTag());
-
 	// network_time never goes back.
-	net_update_time(tmgr->Time() < t ? t : tmgr->Time());
+	net_update_time(timer_mgr->Time() < t ? t : timer_mgr->Time());
 
 	current_pktsrc = src_ps;
 	current_iosrc = src_ps;
@@ -246,7 +233,7 @@ void net_packet_dispatch(double t, const Packet* pkt, iosource::PktSrc* src_ps)
 
 	expire_timers(src_ps);
 
-	SegmentProfiler* sp = 0;
+	SegmentProfiler* sp = nullptr;
 
 	if ( load_sample )
 		{
@@ -273,58 +260,53 @@ void net_packet_dispatch(double t, const Packet* pkt, iosource::PktSrc* src_ps)
 		{
 		delete sp;
 		delete sample_logger;
-		sample_logger = 0;
+		sample_logger = nullptr;
 		}
 
 	processing_start_time = 0.0;	// = "we're not processing now"
 	current_dispatched = 0;
-	current_iosrc = 0;
+	current_iosrc = nullptr;
-	current_pktsrc = 0;
+	current_pktsrc = nullptr;
 	}
 
 void net_run()
 	{
 	set_processing_status("RUNNING", "net_run");
 
+	std::vector<iosource::IOSource*> ready;
+	ready.reserve(iosource_mgr->TotalSize());
+
 	while ( iosource_mgr->Size() ||
 		(BifConst::exit_only_after_terminate && ! terminating) )
 		{
-		double ts;
+		iosource_mgr->FindReadySources(&ready);
-		iosource::IOSource* src = iosource_mgr->FindSoonest(&ts);
 
 #ifdef DEBUG
 		static int loop_counter = 0;
 
 		// If no source is ready, we log only every 100th cycle,
 		// starting with the first.
-		if ( src || loop_counter++ % 100 == 0 )
+		if ( ! ready.empty() || loop_counter++ % 100 == 0 )
 			{
-			DBG_LOG(DBG_MAINLOOP, "realtime=%.6f iosrc=%s ts=%.6f",
+			DBG_LOG(DBG_MAINLOOP, "realtime=%.6f ready_count=%ld",
-					current_time(), src ? src->Tag() : "<all dry>", src ? ts : -1);
+				current_time(), ready.size());
 
-			if ( src )
+			if ( ! ready.empty() )
 				loop_counter = 0;
 			}
 #endif
-		current_iosrc = src;
+		current_iosrc = nullptr;
 		auto communication_enabled = broker_mgr->Active();
 
-		if ( src )
+		if ( ! ready.empty() )
-			src->Process();	// which will call net_packet_dispatch()
-
-		else if ( reading_live && ! pseudo_realtime)
-			{ // live but  no source is currently active
-			double ct = current_time();
-			if ( ! net_is_processing_suspended() )
 			{
-				// Take advantage of the lull to get up to
+			for ( auto src : ready )
-				// date on timers and events.
+				{
-				net_update_time(ct);
+				DBG_LOG(DBG_MAINLOOP, "processing source %s", src->Tag());
-				expire_timers();
+				current_iosrc = src;
-				usleep(1); // Just yield.
+				src->Process();
 				}
 			}
-
 		else if ( (have_pending_timers || communication_enabled ||
 		           BifConst::exit_only_after_terminate) &&
 			  ! pseudo_realtime )
@@ -335,32 +317,13 @@ void net_run()
 			// doesn't risk blocking on other inputs.
 			net_update_time(current_time());
 			expire_timers();
-
-			// Avoid busy-waiting - pause for 100 ms.
-			// We pick a sleep value of 100 msec that buys
-			// us a lot of idle time, but doesn't delay near-term
-			// timers too much.  (Delaying them somewhat is okay,
-			// since Bro timers are not high-precision anyway.)
-			if ( ! communication_enabled )
-				usleep(100000);
-			else
-				usleep(1000);
-
-			// Flawfinder says about usleep:
-			//
-			// This C routine is considered obsolete (as opposed
-			// to the shell command by the same name).   The
-			// interaction of this function with SIGALRM and
-			// other timer functions such as sleep(), alarm(),
-			// setitimer(), and nanosleep() is unspecified.
-			// Use nanosleep(2) or setitimer(2) instead.
 			}
 
 		mgr.Drain();
 
 		processing_start_time = 0.0;	// = "we're not processing now"
 		current_dispatched = 0;
-		current_iosrc = 0;
+		current_iosrc = nullptr;
 
 		if ( signal_val == SIGTERM || signal_val == SIGINT )
 			// We received a signal while processing the
@@ -377,18 +340,13 @@ void net_run()
 			{
 			auto have_active_packet_source = false;
 
-			for ( auto& ps : iosource_mgr->GetPktSrcs() )
+			iosource::PktSrc* ps = iosource_mgr->GetPktSrc();
-				{
+			if ( ps && ps->IsOpen() )
-				if ( ps->IsOpen() )
-					{
 				have_active_packet_source = true;
-					break;
-					}
-				}
 
 			if (  ! have_active_packet_source )
 				// Can turn off pseudo realtime now
-				pseudo_realtime = 0;
+				pseudo_realtime = 0.0;
 			}
 		}
 
@@ -401,20 +359,14 @@ void net_run()
 
 void net_get_final_stats()
 	{
-	const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs());
+	iosource::PktSrc* ps = iosource_mgr->GetPktSrc();
-
+	if ( ps && ps->IsLive() )
-	for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin();
-	      i != pkt_srcs.end(); i++ )
-		{
-		iosource::PktSrc* ps = *i;
-
-		if ( ps->IsLive() )
 		{
 		iosource::PktSrc::Stats s;
 		ps->Statistics(&s);
-			reporter->Info("%" PRIu64 " packets received on interface %s, %" PRIu64 " dropped",
+		double dropped_pct = s.dropped > 0.0 ? ((double)s.dropped / ((double)s.received + (double)s.dropped)) * 100.0 : 0.0;
-					s.received, ps->Path().c_str(), s.dropped);
+		reporter->Info("%" PRIu64 " packets received on interface %s, %" PRIu64 " (%.2f%%) dropped",
-			}
+			s.received, ps->Path().c_str(), s.dropped, dropped_pct);
 		}
 	}
 
@@ -468,12 +420,8 @@ void net_continue_processing()
 	if ( _processing_suspended == 1 )
 		{
 		reporter->Info("processing continued");
-
+		if ( iosource::PktSrc* ps = iosource_mgr->GetPktSrc() )
-		const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs());
+			ps->ContinueAfterSuspend();
-
-		for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin();
-		      i != pkt_srcs.end(); i++ )
-			(*i)->ContinueAfterSuspend();
 		}
 
 	--_processing_suspended;

src/Net.h

@@ -2,20 +2,25 @@
 
 #pragma once
 
+#include <list>
 #include <vector>
 #include <string>
 #include <optional>
 
-#include "net_util.h"
+#include <sys/stat.h> // for ino_t
-#include "util.h"
-#include "List.h"
-#include "Func.h"
-#include "iosource/IOSource.h"
-#include "iosource/PktSrc.h"
-#include "iosource/PktDumper.h"
 
-extern void net_init(const std::vector<std::string>& interfaces,
+using std::string;
-                     const std::vector<std::string>& pcap_input_files,
+
+namespace iosource {
+	class IOSource;
+	class PktSrc;
+	class PktDumper;
+	}
+
+class Packet;
+
+extern void net_init(const std::optional<std::string>& interfaces,
+                     const std::optional<std::string>& pcap_input_file,
                      const std::optional<std::string>& pcap_output_file,
                      bool do_watchdog);
 extern void net_run();
@@ -38,19 +43,19 @@ inline bool net_is_processing_suspended()
 	{ return _processing_suspended > 0; }
 
 // Whether we're reading live traffic.
-extern int reading_live;
+extern bool reading_live;
 
 // Same but for reading from traces instead.  We have two separate
 // variables because it's possible that neither is true, and we're
 // instead just running timers (per the variable after this one).
-extern int reading_traces;
+extern bool reading_traces;
 
 // True if we have timers scheduled for the future on which we need
 // to wait.  "Need to wait" here means that we're running live (though
 // perhaps not reading_live, but just running in real-time) as opposed
 // to reading a trace (in which case we don't want to wait in real-time
 // on future timers).
-extern int have_pending_timers;
+extern bool have_pending_timers;
 
 // If > 0, we are reading from traces but trying to mimic real-time behavior.
 // (In this case, both reading_traces and reading_live are true.)  The value

src/NetVar.cc

@@ -2,8 +2,10 @@
 
 #include "zeek-config.h"
 
-#include "Var.h"
 #include "NetVar.h"
+#include "Var.h"
+#include "EventHandler.h"
+#include "Val.h"
 
 RecordType* conn_id;
 RecordType* endpoint;

src/Notifier.cc

@@ -1,7 +1,9 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
-#include "DebugLogger.h"
 #include "Notifier.h"
+#include "DebugLogger.h"
+
+#include <set>
 
 notifier::Registry notifier::registry;
 

src/Notifier.h

@@ -7,12 +7,7 @@
 
 #pragma once
 
-#include <set>
 #include <unordered_map>
-#include <string>
-
-#include "util.h"
-#include "DebugLogger.h"
 
 namespace notifier  {
 

src/Obj.cc

@@ -1,10 +1,11 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
 #include "zeek-config.h"
+#include "Obj.h"
 
 #include <stdlib.h>
 
-#include "Obj.h"
+#include "Desc.h"
 #include "Func.h"
 #include "File.h"
 #include "plugin/Manager.h"

src/Obj.h

@@ -4,8 +4,7 @@
 
 #include <limits.h>
 
-#include "input.h"
+class ODesc;
-#include "Desc.h"
 
 class Location {
 public:

src/OpaqueVal.cc

@@ -5,9 +5,12 @@
 #include "OpaqueVal.h"
 #include "NetVar.h"
 #include "Reporter.h"
+#include "Desc.h"
+#include "Var.h"
 #include "probabilistic/BloomFilter.h"
 #include "probabilistic/CardinalityCounter.h"
 
+#include <broker/data.hh>
 #include <broker/error.hh>
 
 // Helper to retrieve a broker value out of a broker::vector at a specified

src/OpaqueVal.h

@@ -2,14 +2,16 @@
 
 #pragma once
 
-#include <broker/data.hh>
-#include <broker/expected.hh>
-
 #include "RandTest.h"
 #include "Val.h"
 #include "digest.h"
 #include "paraglob/paraglob.h"
 
+#include <broker/expected.hh>
+
+#include <sys/types.h> // for u_char
+
+namespace broker { class data; }
 class OpaqueVal;
 
 /**

src/Options.cc

@@ -1,9 +1,13 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
-#include <unistd.h>
-
 #include "zeek-config.h"
 
+#include "Options.h"
+
+#include <algorithm>
+
+#include <unistd.h>
+
 #ifdef HAVE_GETOPT_H
 #include <getopt.h>
 #endif
@@ -11,13 +15,9 @@
 #include "bsd-getopt-long.h"
 #include "logging/writers/ascii/Ascii.h"
 
-#include "Options.h"
-
 void zeek::Options::filter_supervisor_options()
 	{
 	pcap_filter = {};
-	interfaces = {};
-	pcap_files = {};
 	signature_files = {};
 	pcap_output_file = {};
 	}
@@ -49,8 +49,8 @@ void zeek::Options::filter_supervised_node_options()
 	// use-case-specific way.  e.g. interfaces is already handled for the
 	// "cluster" use-case, but don't have supervised-pcap-reading
 	// functionality yet.
-	/* interfaces = og.interfaces; */
+	/* interface = og.interface; */
-	/* pcap_files = og.pcap_files; */
+	/* pcap_file = og.pcap_file; */
 
 	pcap_output_file = og.pcap_output_file;
 	random_seed_input_file = og.random_seed_input_file;
@@ -82,9 +82,9 @@ void zeek::usage(const char* prog, int code)
 	fprintf(stderr, "    -e|--exec <zeek code>          | augment loaded scripts by given code\n");
 	fprintf(stderr, "    -f|--filter <filter>           | tcpdump filter\n");
 	fprintf(stderr, "    -h|--help                      | command line help\n");
-	fprintf(stderr, "    -i|--iface <interface>         | read from given interface\n");
+	fprintf(stderr, "    -i|--iface <interface>         | read from given interface (only one allowed)\n");
 	fprintf(stderr, "    -p|--prefix <prefix>           | add given prefix to Zeek script file resolution\n");
-	fprintf(stderr, "    -r|--readfile <readfile>       | read from given tcpdump file\n");
+	fprintf(stderr, "    -r|--readfile <readfile>       | read from given tcpdump file (only one allowed, pass '-' as the filename to read from stdin)\n");
 	fprintf(stderr, "    -s|--rulefile <rulefile>       | read rules from given file\n");
 	fprintf(stderr, "    -t|--tracefile <tracefile>     | activate execution tracing\n");
 	fprintf(stderr, "    -v|--version                   | print version and exit\n");
@@ -270,12 +270,19 @@ zeek::Options zeek::parse_cmdline(int argc, char** argv)
 			rval.print_usage = true;
 			break;
 		case 'i':
-			if ( ! rval.pcap_files.empty() )
+			if ( rval.interface )
 				{
-				fprintf(stderr, "Using -i is not allowed when reading pcap files");
+				fprintf(stderr, "ERROR: Only a single interface option (-i) is allowed.\n");
 				exit(1);
 				}
-			rval.interfaces.emplace_back(optarg);
+
+			if ( rval.pcap_file )
+				{
+				fprintf(stderr, "ERROR: Using -i is not allow when reading a pcap file.\n");
+				exit(1);
+				}
+
+			rval.interface = optarg;
 			break;
 		case 'j':
 			rval.supervisor_mode = true;
@@ -290,12 +297,19 @@ zeek::Options zeek::parse_cmdline(int argc, char** argv)
 			rval.script_prefixes.emplace_back(optarg);
 			break;
 		case 'r':
-			if ( ! rval.interfaces.empty() )
+			if ( rval.pcap_file )
 				{
-				fprintf(stderr, "Using -r is not allowed when reading a live interface");
+				fprintf(stderr, "ERROR: Only a single readfile option (-r) is allowed.\n");
 				exit(1);
 				}
-			rval.pcap_files.emplace_back(optarg);
+
+			if ( rval.interface )
+				{
+				fprintf(stderr, "Using -r is not allowed when reading a live interface.\n");
+				exit(1);
+				}
+
+			rval.pcap_file = optarg;
 			break;
 		case 's':
 			rval.signature_files.emplace_back(optarg);

src/Options.h

@@ -58,8 +58,8 @@ struct Options {
 	std::vector<std::string> doctest_args;
 
 	std::optional<std::string> pcap_filter;
-	std::vector<std::string> interfaces;
+	std::optional<std::string> interface;
-	std::vector<std::string> pcap_files;
+	std::optional<std::string> pcap_file;
 	std::vector<std::string> signature_files;
 
 	std::optional<std::string> pcap_output_file;

src/PacketDumper.cc

@@ -1,14 +1,10 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
-
 #include "zeek-config.h"
-
-#include <assert.h>
-#include <stdlib.h>
-
-#include "Event.h"
-#include "Net.h"
 #include "PacketDumper.h"
+#include "Reporter.h"
+#include "util.h"
+#include "iosource/PktDumper.h"
 
 PacketDumper::PacketDumper(pcap_dumper_t* arg_pkt_dump)
 	{

src/PacketDumper.h

@@ -2,11 +2,10 @@
 
 #pragma once
 
-#include <queue>
-#include <set>
-
 #include <pcap.h>
 
+#include <sys/types.h> // for u_char
+
 class PacketDumper {
 public:
 	explicit PacketDumper(pcap_dumper_t* pkt_dump);
@@ -20,18 +19,3 @@ protected:
 
 	void SortTimeStamp(struct timeval* timestamp);
 };
-
-struct IP_ID {
-	uint32_t ip, id;
-};
-
-struct ltipid {
-	bool operator()(IP_ID id1, IP_ID id2) const
-		{
-		return id1.ip != id2.ip ? (id1.ip < id2.ip) :
-					  (id1.id < id2.id);
-		}
-};
-
-typedef std::set<IP_ID, ltipid> IP_IDSet;
-uint16_t NextIP_ID(const uint32_t src_addr, const uint16_t id);

src/PacketFilter.cc

@@ -1,4 +1,5 @@
 #include "PacketFilter.h"
+#include "IP.h"
 
 void PacketFilter::DeleteFilter(void* data)
 	{

src/PacketFilter.h

@@ -2,9 +2,12 @@
 
 #pragma once
 
-#include "IP.h"
+#include "IPAddr.h"
 #include "PrefixTable.h"
 
+class IP_Hdr;
+class Val;
+
 class PacketFilter {
 public:
 	explicit PacketFilter(bool arg_default);

src/PrefixTable.cc

@@ -1,5 +1,6 @@
 #include "PrefixTable.h"
 #include "Reporter.h"
+#include "Val.h"
 
 prefix_t* PrefixTable::MakePrefix(const IPAddr& addr, int width)
 	{

src/PrefixTable.h

@@ -1,13 +1,19 @@
 #pragma once
 
-#include "Val.h"
-#include "net_util.h"
 #include "IPAddr.h"
 
 extern "C" {
 	#include "patricia.h"
 }
 
+#include <list>
+
+using std::list;
+using std::tuple;
+
+class Val;
+class SubNetVal;
+
 class PrefixTable {
 private:
 	struct iterator {

src/PriorityQueue.cc

@@ -27,7 +27,7 @@ PriorityQueue::~PriorityQueue()
 PQ_Element* PriorityQueue::Remove()
 	{
 	if ( heap_size == 0 )
-		return 0;
+		return nullptr;
 
 	PQ_Element* top = heap[0];
 
@@ -43,7 +43,7 @@ PQ_Element* PriorityQueue::Remove(PQ_Element* e)
 	{
 	if ( e->Offset() < 0 || e->Offset() >= heap_size ||
 	     heap[e->Offset()] != e )
-		return 0;	// not in heap
+		return nullptr;	// not in heap
 
 	e->MinimizeTime();
 	BubbleUp(e->Offset());
@@ -56,7 +56,7 @@ PQ_Element* PriorityQueue::Remove(PQ_Element* e)
 	return e2;
 	}
 
-int PriorityQueue::Add(PQ_Element* e)
+bool PriorityQueue::Add(PQ_Element* e)
 	{
 	SetElement(heap_size, e);
 
@@ -70,10 +70,10 @@ int PriorityQueue::Add(PQ_Element* e)
 	if ( heap_size >= max_heap_size )
 		return Resize(max_heap_size * 2);
 	else
-		return 1;
+		return true;
 	}
 
-int PriorityQueue::Resize(int new_size)
+bool PriorityQueue::Resize(int new_size)
 	{
 	PQ_Element** tmp = new PQ_Element*[new_size];
 	for ( int i = 0; i < max_heap_size; ++i )
@@ -84,7 +84,7 @@ int PriorityQueue::Resize(int new_size)
 
 	max_heap_size = new_size;
 
-	return heap != 0;
+	return heap != nullptr;
 	}
 
 void PriorityQueue::BubbleUp(int bin)

src/PriorityQueue.h

@@ -3,7 +3,7 @@
 #pragma once
 
 #include <math.h>
-#include "util.h"
+#include <stdint.h>
 
 class PriorityQueue;
 
@@ -43,20 +43,20 @@ public:
 	// is empty.
 	PQ_Element* Remove();
 
-	// Removes element e.  Returns e, or nil if e wasn't in the queue.
+	// Removes element e.  Returns e, or nullptr if e wasn't in the queue.
 	// Note that e will be modified via MinimizeTime().
 	PQ_Element* Remove(PQ_Element* e);
 
-	// Add a new element to the queue.  Returns 0 on failure (not enough
+	// Add a new element to the queue.  Returns false on failure (not enough
-	// memory to add the element), 1 on success.
+	// memory to add the element), true on success.
-	int Add(PQ_Element* e);
+	bool Add(PQ_Element* e);
 
 	int Size() const	{ return heap_size; }
 	int PeakSize() const	{ return peak_heap_size; }
 	uint64_t CumulativeNum() const { return cumulative_num; }
 
 protected:
-	int Resize(int new_size);
+	bool Resize(int new_size);
 
 	void BubbleUp(int bin);
 	void BubbleDown(int bin);

src/Queue.h

@@ -2,6 +2,8 @@
 
 #pragma once
 
+#include <iterator>
+
 // Queue.h --
 //	Interface for class Queue, current implementation is as an
 //	array of ent's.  This implementation was chosen to optimize

src/RE.cc

@@ -1,15 +1,16 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
 #include "zeek-config.h"
+#include "RE.h"
 
 #include <stdlib.h>
 #include <utility>
 
-#include "RE.h"
 #include "DFA.h"
 #include "CCL.h"
 #include "EquivClass.h"
 #include "Reporter.h"
+#include "BroString.h"
 
 CCL* curr_ccl = 0;
 

src/RE.h

@@ -2,9 +2,7 @@
 
 #pragma once
 
-#include "Obj.h"
+#include "List.h"
-#include "Dict.h"
-#include "BroString.h"
 #include "CCL.h"
 #include "EquivClass.h"
 
@@ -12,6 +10,7 @@
 #include <map>
 #include <string>
 
+#include <sys/types.h> // for u_char
 #include <ctype.h>
 typedef int (*cce_func)(int);
 
@@ -21,6 +20,7 @@ class DFA_Machine;
 class Specific_RE_Matcher;
 class RE_Matcher;
 class DFA_State;
+class BroString;
 
 extern int case_insensitive;
 extern CCL* curr_ccl;