Merge remote-tracking branch 'origin/topic/awelzel/builtin-af-packet'

* origin/topic/awelzel/builtin-af-packet:
  Include in Jan's AF_PACKET plugin as builtin plugin

.gitmodules

@@ -67,3 +67,6 @@
 [submodule "auxil/filesystem"]
 	path = auxil/filesystem
 	url = https://github.com/gulrak/filesystem.git
+[submodule "auxil/zeek-af_packet-plugin"]
+	path = auxil/zeek-af_packet-plugin
+	url = https://github.com/zeek/zeek-af_packet-plugin.git

.lgtm.yml

@@ -10,6 +10,7 @@ path_classifiers:
     - "auxil/rapidjson/"
     - "auxil/spicy"
     - "auxil/spicy-plugin"
+    - "auxil/zeek-af_packet-plugin/"
     - "src/3rdparty/"
 
 # Filter out alerts that aren't concerning.

CHANGES

@@ -1,3 +1,9 @@
+5.2.0-dev.150 | 2022-10-26 16:48:18 -0700
+
+  * Include in Jan's AF_PACKET plugin as builtin plugin (Arne Welzel, Corelight)
+
+  * Update doc submodule. (Robin Sommer, Corelight)
+
 5.2.0-dev.147 | 2022-10-26 11:37:24 +0200
 
   * GH-2272: Use `json_escape_utf8` in broker debug output (AmazingPP)

CMakeLists.txt

@@ -589,6 +589,16 @@ endif()
 # Tell the plugin code that we're building as part of the main tree.
 set(ZEEK_PLUGIN_INTERNAL_BUILD true CACHE INTERNAL "" FORCE)
 
+if ( ${CMAKE_SYSTEM_NAME} MATCHES Linux )
+    if ( NOT DISABLE_AF_PACKET )
+        if ( NOT AF_PACKET_PLUGIN_PATH )
+            set(AF_PACKET_PLUGIN_PATH ${CMAKE_SOURCE_DIR}/auxil/zeek-af_packet-plugin)
+        endif ()
+
+        string(APPEND ZEEK_INCLUDE_PLUGINS ";${AF_PACKET_PLUGIN_PATH}")
+    endif ()
+endif ()
+
 set(DEFAULT_ZEEKPATH .:${ZEEK_SCRIPT_INSTALL_PATH}:${ZEEK_SCRIPT_INSTALL_PATH}/policy:${ZEEK_SCRIPT_INSTALL_PATH}/site:${ZEEK_SCRIPT_INSTALL_PATH}/builtin-plugins)
 
 if ( NOT BINARY_PACKAGING_MODE )

NEWS

@@ -91,6 +91,12 @@ New Functionality
   implemented in a given module can be toggled with ``disable_module_events()``
   and ``enable_module_events()``.
 
+- On Linux, the AF_PACKET packet source plugin (https://github.com/zeek/zeek-af_packet-plugin)
+  is included as builtin plugin by default. To select this packet source, prefix
+  the interface name with ``af_packet``.
+
+	zeek -i af_packet::eth0
+
 Changed Functionality
 ---------------------
 

VERSION

@@ -1 +1 @@
-5.2.0-dev.147
+5.2.0-dev.150

auxil/zeek-af_packet-plugin

@@ -0,0 +1 @@
+Subproject commit de9713da03263b6a69573e0e0f36ac59110a9877

configure

@@ -61,6 +61,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --enable-perftools-debug use Google's perftools for debugging
     --enable-static-binpac build binpac statically (ignored if --with-binpac is specified)
     --enable-static-broker build Broker statically (ignored if --with-broker is specified)
+    --disable-af-packet    don't include native AF_PACKET support (Linux only)
     --disable-archiver     don't build or install zeek-archiver tool
     --disable-auxtools     don't build or install auxiliary tools
     --disable-broker-tests don't try to build Broker unit tests
@@ -288,6 +289,9 @@ while [ $# -ne 0 ]; do
         --enable-static-broker)
             append_cache_entry BUILD_STATIC_BROKER BOOL true
             ;;
+        --disable-af-packet)
+            append_cache_entry DISABLE_AF_PACKET BOOL true
+            ;;
         --disable-archiver)
             append_cache_entry INSTALL_ZEEK_ARCHIVER BOOL false
             ;;

testing/btest/Baseline/af_packet.plugin/.stdout

@@ -0,0 +1,5 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Zeek::AF_Packet - Packet acquisition via AF_Packet (built-in)
+buffer_size, 134217728
+enable_fanout, T
+fanout_mode, AF_Packet::FANOUT_HASH

testing/btest/af_packet/plugin.zeek

@@ -0,0 +1,10 @@
+# @TEST-DOC: On Linux, test AF_PACKET support exists when enabled and the AF_Packet module is available in script land.
+# @TEST-REQUIRES: ${SCRIPTS}/have-af-packet
+# @TEST-EXEC: zeek -N Zeek::AF_Packet
+# @TEST-EXEC: zeek -b %INPUT
+# @TEST-EXEC: btest-diff .stdout
+
+# Print some defaults for smoke checking.
+print "buffer_size", AF_Packet::buffer_size;
+print "enable_fanout", AF_Packet::enable_fanout;
+print "fanout_mode", AF_Packet::fanout_mode;

testing/btest/coverage/bare-load-baseline.test

@@ -1,7 +1,8 @@
 # This test is meant to cover whether the set of scripts that get loaded by
 # default in bare mode matches a baseline of known defaults.  The baseline
 # should only need updating if something new is @load'd from init-bare.zeek
-# (or from an @load'd descendent of it).
+# (or from an @load'd descendent of it), or when a new builtin plugin is
+# added to Zeek.
 #
 # As the output has absolute paths in it, we need to remove the common
 # prefix to make the test work everywhere. That's what the sed magic
@@ -12,5 +13,6 @@
 # @TEST-EXEC: test -e loaded_scripts.log
 # @TEST-EXEC: cat loaded_scripts.log | egrep -v '#' | awk 'NR>0{print $1}' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
 # @TEST-EXEC: (test -L $BUILD && basename $(readlink $BUILD) || basename $BUILD) >buildprefix
-# @TEST-EXEC: cat loaded_scripts.log | sed "s#`cat buildprefix`#build#g" | sed "s#`cat prefix`##g" >canonified_loaded_scripts.log
+# @TEST-EXEC: cat loaded_scripts.log | sed "s#`cat buildprefix`#build#g" | sed "s#`cat prefix`##g" >prefix_canonified_loaded_scripts.log
+# @TEST-EXEC: grep -v 'Zeek_AF_Packet' prefix_canonified_loaded_scripts.log > canonified_loaded_scripts.log
 # @TEST-EXEC: btest-diff canonified_loaded_scripts.log

testing/btest/coverage/default-load-baseline.test

@@ -12,5 +12,6 @@
 # @TEST-EXEC: test -e loaded_scripts.log
 # @TEST-EXEC: cat loaded_scripts.log | egrep -v '#' | sed 's/ //g' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
 # @TEST-EXEC: (test -L $BUILD && basename $(readlink $BUILD) || basename $BUILD) >buildprefix
-# @TEST-EXEC: cat loaded_scripts.log | sed "s#`cat buildprefix`#build#g" | sed "s#`cat prefix`##g" >canonified_loaded_scripts.log
+# @TEST-EXEC: cat loaded_scripts.log | sed "s#`cat buildprefix`#build#g" | sed "s#`cat prefix`##g" >prefix_canonified_loaded_scripts.log
+# @TEST-EXEC: grep -v 'Zeek_AF_Packet' prefix_canonified_loaded_scripts.log > canonified_loaded_scripts.log
 # @TEST-EXEC: btest-diff canonified_loaded_scripts.log

testing/btest/plugins/hooks-plugin/src/Plugin.cc

@@ -32,6 +32,21 @@ static std::set<std::string> sanitized_functions = {
 	"Telemetry::gauge_family_set",
 };
 
+// When a filename given to LOAD_FILE* hooks (and to the meta pre/post hooks)
+// contains any of these keywords, no log message is generated.
+static std::set<std::string> load_file_filter = {
+	"Zeek_AF_Packet",
+};
+
+static bool skip_load_file_logging_for(const std::string& s)
+	{
+	for ( const auto& needle : load_file_filter )
+		if ( s.find(needle) != std::string::npos )
+			return true;
+
+	return false;
+	}
+
 zeek::plugin::Configuration Plugin::Configure()
 	{
 	EnableHook(zeek::plugin::HOOK_LOAD_FILE);
@@ -93,6 +108,9 @@ static void describe_hook_args(const zeek::plugin::HookArgumentList& args, zeek:
 
 int Plugin::HookLoadFile(const LoadType type, const std::string& file, const std::string& resolved)
 	{
+	if ( skip_load_file_logging_for(resolved) )
+		return -1;
+
 	fprintf(stderr, "%.6f %-15s %s %s\n", zeek::run_state::network_time, "| HookLoadFile",
 	        file.c_str(), resolved.c_str());
 	return -1;
@@ -102,6 +120,9 @@ std::pair<int, std::optional<std::string>> Plugin::HookLoadFileExtended(const Lo
                                                                         const std::string& file,
                                                                         const std::string& resolved)
 	{
+	if ( skip_load_file_logging_for(resolved) )
+		return std::make_pair(-1, std::nullopt);
+
 	fprintf(stderr, "%.6f %-15s %s %s\n", zeek::run_state::network_time, "| HookLoadFileExtended",
 	        file.c_str(), resolved.c_str());
 	return std::make_pair(-1, std::nullopt);
@@ -177,6 +198,12 @@ void Plugin::MetaHookPre(zeek::plugin::HookType hook, const zeek::plugin::HookAr
 	zeek::ODesc d;
 	d.SetShort();
 	describe_hook_args(args, &d);
+
+	// Special case file loading filtering.
+	if ( hook == zeek::plugin::HOOK_LOAD_FILE || hook == zeek::plugin::HOOK_LOAD_FILE_EXT )
+		if ( skip_load_file_logging_for(std::string(d.Description())) )
+			return;
+
 	fprintf(stderr, "%.6f %-15s %s(%s)\n", zeek::run_state::network_time, "  MetaHookPre",
 	        hook_name(hook), d.Description());
 	}
@@ -188,6 +215,11 @@ void Plugin::MetaHookPost(zeek::plugin::HookType hook, const zeek::plugin::HookA
 	d1.SetShort();
 	describe_hook_args(args, &d1);
 
+	// Special case file loading filtering.
+	if ( hook == zeek::plugin::HOOK_LOAD_FILE || hook == zeek::plugin::HOOK_LOAD_FILE_EXT )
+		if ( skip_load_file_logging_for(std::string(d1.Description())) )
+			return;
+
 	zeek::ODesc d2;
 	d2.SetShort();
 	result.Describe(&d2);

testing/scripts/have-af-packet

@@ -0,0 +1,10 @@
+#!/bin/sh
+if [ "$(uname -s)" != "Linux" ]; then
+    exit 1
+fi
+
+if grep -q "DISABLE_AF_PACKET:BOOL=true" "${BUILD}"/CMakeCache.txt; then
+    exit 1
+fi
+
+exit 0