Merge remote-tracking branch 'origin/topic/christian/zkg-inclusion'

* origin/topic/christian/zkg-inclusion:
  Install zkg as part of the Zeek distribution.

.gitmodules

@@ -43,3 +43,6 @@
 [submodule "auxil/zeek-archiver"]
 	path = auxil/zeek-archiver
 	url = https://github.com/zeek/zeek-archiver
+[submodule "auxil/package-manager"]
+	path = auxil/package-manager
+	url = https://github.com/zeek/package-manager

CHANGES

@@ -1,4 +1,27 @@
 
+3.3.0-dev.650 | 2020-12-12 20:24:19 -0800
+
+  * Install zkg as part of the Zeek distribution. (Christian Kreibich, Corelight)
+
+    - Add auxil/package-manager submodule as an optional build
+      source. When the submodule is present, zkg gets installed into the
+      Zeek installation's bin directory, its config file into etc/zkg, and
+      its state into var/lib/zkg. Like zeekctl, zkg finds its own module
+      independently of any PYTHONPATH. Installation via pip remains
+      supported. You can skip zkg explicitly via --disable-zkg. See the
+      NEWS update for details.
+
+    - Establish a "zeek/python" subdirectory under libdir as the common place
+      for Python modules in the Zeek distribution. This now separates out
+      the Broker Python bindings, ZeekControl, and zkg's Python module.
+
+    - Add configure flags to allow customizing this Python folder, in
+      three ways: --python-dir, --python-prefix, and --python-home. These
+      differ in the logic they automatically add to the path, and build on
+      the logic already used in Broker.
+
+    - Include a (commented-out) @load for zkg's packages folder in local.zeek.
+
 3.3.0-dev.648 | 2020-12-12 13:39:28 -0800
 
   * Update Coverity Scan GitHub Action (Jon Siwek, Corelight)

CMakeLists.txt

@@ -53,8 +53,11 @@ endif ()
 get_filename_component(ZEEK_SCRIPT_INSTALL_PATH ${ZEEK_SCRIPT_INSTALL_PATH}
     ABSOLUTE)
 
-set(BRO_PLUGIN_INSTALL_PATH ${CMAKE_INSTALL_FULL_LIBDIR}/zeek/plugins CACHE STRING "Installation path for plugins" FORCE)
+# A folder for library-like Zeek-specific things: Python modules, Zeek
-set(PY_MOD_INSTALL_DIR ${CMAKE_INSTALL_FULL_LIBDIR}/zeekctl CACHE STRING "Installation path for Python modules" FORCE)
+# plugins, etc.
+set(ZEEK_LIBDIR_PATH ${CMAKE_INSTALL_FULL_LIBDIR}/zeek)
+set(BRO_PLUGIN_INSTALL_PATH ${ZEEK_LIBDIR_PATH}/plugins
+    CACHE STRING "Installation path for plugins" FORCE)
 
 configure_file(zeek-path-dev.in ${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev)
 execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink
@@ -245,6 +248,25 @@ FindRequiredPackage(OpenSSL)
 FindRequiredPackage(BIND)
 FindRequiredPackage(ZLIB)
 
+# Installation directory for the distribution's Python modules. An
+# override via configure's --python-dir wins, specifying a directory
+# explicitly. Next is --python-prefix, which includes a versioned
+# Python folder as the --prefix option in distutils does. Next
+# consider a distutils --home style override via --python-home, and
+# finally default to "zeek/python" in our libdir.
+if (ZEEK_PYTHON_DIR)
+    set(py_mod_install_dir ${ZEEK_PYTHON_DIR})
+elseif (ZEEK_PYTHON_PREFIX)
+    set(pyver ${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR})
+    set(py_mod_install_dir ${ZEEK_PYTHON_PREFIX}/lib/python${pyver}/site-packages)
+elseif (ZEEK_PYTHON_HOME)
+    set(py_mod_install_dir ${ZEEK_PYTHON_HOME}/lib/python)
+else ()
+    set(py_mod_install_dir ${ZEEK_LIBDIR_PATH}/python)
+endif ()
+set(PY_MOD_INSTALL_DIR ${py_mod_install_dir}
+    CACHE STRING "Installation path for Python modules" FORCE)
+
 if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/auxil/binpac/CMakeLists.txt)
 
     set(ENABLE_STATIC_ONLY_SAVED ${ENABLE_STATIC_ONLY})
@@ -519,6 +541,24 @@ include(InstallSymlink)
 InstallShellScript("bin" "zeek-wrapper.in" "zeek-wrapper")
 InstallSymlink("${CMAKE_INSTALL_PREFIX}/bin/zeek-wrapper" "${CMAKE_INSTALL_PREFIX}/bin/bro-config")
 
+########################################################################
+## zkg configuration
+
+if ( INSTALL_ZKG )
+    # An etc/zkg directory for zkg's config file simplifies zkg's
+    # config file code.
+    set(ZEEK_ZKG_CONFIG_DIR "${ZEEK_ETC_INSTALL_DIR}/zkg")
+    set(ZEEK_ZKG_STATE_DIR "${ZEEK_ROOT_DIR}/var/lib/zkg")
+
+    configure_file(
+        ${CMAKE_CURRENT_SOURCE_DIR}/zkg-config.in
+        ${CMAKE_CURRENT_BINARY_DIR}/zkg-config @ONLY)
+
+    install(DIRECTORY DESTINATION var/lib/zkg)
+    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zkg-config
+        DESTINATION ${ZEEK_ZKG_CONFIG_DIR} RENAME config)
+endif ()
+
 ########################################################################
 ## Recurse on sub-directories
 
@@ -528,31 +568,15 @@ add_subdirectory(man)
 
 include(CheckOptionalBuildSources)
 
+CheckOptionalBuildSources(auxil/package-manager ZKG INSTALL_ZKG)
 CheckOptionalBuildSources(auxil/zeekctl   ZeekControl INSTALL_ZEEKCTL)
 CheckOptionalBuildSources(auxil/zeek-aux  Zeek-Aux  INSTALL_AUX_TOOLS)
 CheckOptionalBuildSources(auxil/zeek-archiver ZeekArchiver INSTALL_ZEEK_ARCHIVER)
 
-########################################################################
-## Transitions and cleanups
-
-if ( NOT BINARY_PACKAGING_MODE )
-    # Remove pre-existing libdir of the old hardwired name if it is not
-    # the name we're now installing under.
-    set(_old_libdir ${CMAKE_INSTALL_PREFIX}/lib)
-
-    install(CODE "
-        if ( EXISTS \"${_old_libdir}\" AND IS_DIRECTORY \"${_old_libdir}\"
-             AND NOT \"${_old_libdir}\" STREQUAL \"${CMAKE_INSTALL_FULL_LIBDIR}\" )
-            message(STATUS \"WARNING: removing old library directory ${_old_libdir}\")
-            execute_process(COMMAND \"${CMAKE_COMMAND}\" -E remove_directory \"${_old_libdir}\")
-        endif ()
-    ")
-endif ()
-
 ########################################################################
 ## Packaging Setup
 
-if (INSTALL_ZEEKCTL)
+if ( INSTALL_ZEEKCTL OR INSTALL_ZKG )
     # CPack RPM Generator may not automatically detect this
     set(CPACK_RPM_PACKAGE_REQUIRES "python >= ${ZEEK_PYTHON_MIN}")
 endif ()
@@ -578,6 +602,8 @@ message(
     "\nBuild type:        ${CMAKE_BUILD_TYPE}"
     "\nBuild dir:         ${CMAKE_BINARY_DIR}"
     "\nInstall prefix:    ${CMAKE_INSTALL_PREFIX}"
+    "\nPlugin dir:        ${BRO_PLUGIN_INSTALL_PATH}"
+    "\nPython module dir: ${PY_MOD_INSTALL_DIR}"
     "\nZeek Script Path:  ${ZEEK_SCRIPT_INSTALL_PATH}"
     "\nDebug mode:        ${ENABLE_DEBUG}"
     "\nUnit tests:        ${ENABLE_ZEEK_UNIT_TESTS}"
@@ -590,6 +616,7 @@ message(
     "\n"
     "\nZeekControl:       ${INSTALL_ZEEKCTL}"
     "\nAux. Tools:        ${INSTALL_AUX_TOOLS}"
+    "\nzkg:               ${INSTALL_ZKG}"
     "\n"
     "\nlibmaxminddb:      ${USE_GEOIP}"
     "\nKerberos:          ${USE_KRB5}"

NEWS

@@ -120,6 +120,49 @@ New Functionality
   epochs will have to be manually ended by callis ``SumStats::next_epoch``. This
   can be convenient because epochs can be synced to other events.
 
+- The Zeek distribution now includes Zeek's package manager, zkg. Its
+  code, configuration, and state reside in Zeek's installation tree,
+  as follows:
+
+  - The toplevel script, ``zkg``, installs alongside ``zeek`` in the
+    distribution's ``$prefix/bin`` folder.
+
+  - The config file installs into ``$prefix/etc/zkg/config``. The
+    distribution's zkg command uses it by default, but you can switch
+    to a different one via the ``ZKG_CONFIG_FILE`` environment
+    variable or the ``--configfile`` command-line flag.
+
+  - zkg's package state resides in ``$prefix/var/lib/zkg``. This
+    implies that parallel Zeek installations now automatically
+    separate their package installations.
+
+  These folders have the same ownership and access permissions as the
+  rest of the installation, , meaning that in order to manage zkg
+  packages you need to run zkg as a user with corresponding access.
+  Apart from these location overrides, the bundled zkg installation
+  behaves as usual.
+
+  local.zeek now contains a (commented out) ``@load`` statement you
+  can use to source zkg's package state automatically.
+
+  zkg's own Python module resides in ``zeek/python/zeekpkg`, in the
+  installation tree's library folder. See below for additional changes
+  around the library folder.
+
+  zkg has external Python module dependencies. The Zeek configuration
+  does not verify whether these dependencies are met. A new warning
+  message at zkg launch flags missing packages and how to install them
+  (e.g. via pip).
+
+  Configuring with ``--disable-zkg`` disables the zkg inclusion. You
+  can continue to install and use zkg independently. You're also free
+  to use the config file in ``$prefix/etc/zkg/config`` with other zkg
+  installations. 
+
+  The zkg source tree resides in ``auxil/package-manager`` as an
+  additional Git submodule.
+
+
 Changed Functionality
 ---------------------
 
@@ -166,11 +209,25 @@ Changed Functionality
 - The Zeek installation tree is now more consistent in using a ``lib64/``
   (rather than ``lib/``) subdirectory for platforms where that's the common
   convention.  If the old hardcoded ``lib/`` path exists while installing Zeek
-  4.0 and the new subdirectory differs, then the old ``lib/`` will be removed.
+  4.0 and the new subdirectory differs, the old ``lib/`` remains untouched.
-  This potentially wipes out binary plugins that have already been installed
+  This clutters the installation but is safe: the new installation does not
-  there, but Zeek plugins generally have to be re-built/re-installed upon any
+  require the old location, and any files you might require still in the old
-  Zeek upgrade anyway, so no part of the usual upgrade process is expected to
+  tree (e.g. ZeekControl plugins) remain available.
-  be complicated by this cleanup operation.
+
+  Due to Zeek 4's reorganization of the installation tree we recommend
+  a clean-slate install when possible.
+
+- Python modules installed with the Zeek distribution now reside in a
+  common ``zeek/python`` directory below the library path (such as
+  ``lib64/zeek/python``) and no longer assume ZeekControl. The
+  ``zeek/python/zeekctl`` folder now contains only ZeekControl's own
+  functionality, ``zeek/python/zeekpkg`` contains zkg's Python module, and
+  Broker's Python bindings live in ``zeek/python/broker``. ``zeek-config
+  --python_dir`` now reports this new ``zeek/python`` folder. Several
+  new configure options allow you to customize the Python folder location,
+  depending on your needs.
+
+  As with the new libdir, no cleanup of the existing Python tree occurs.
 
 - Continued renaming/namespacing of many classes into either ``zeek`` or
   ``zeek::detail`` namespaces as already explained in Zeek 3.2's release notes.

VERSION

@@ -1 +1 @@
-3.3.0-dev.648
+3.3.0-dev.650

auxil/broker

@@ -1 +1 @@
-Subproject commit 8a3291800d2d40e1b7f421e1e4b9a3ffef778d59
+Subproject commit 7878a6d816fdae370e86f13ae08c6762b6d11347

auxil/package-manager

@@ -0,0 +1 @@
+Subproject commit 44d300d9507753b9d08830a8239c49e0ddaa0fbe

auxil/zeekctl

@@ -1 +1 @@
-Subproject commit c98d51bc6b7a2d40cdc095108b985088ccb7e32c
+Subproject commit 2736291513f384292c36220e9e57b439fcde3b4f

configure

@@ -43,6 +43,12 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --libdir=PATH          installation directory for library files [PREFIX/lib]
     --conf-files-dir=PATH  config files installation directory [PREFIX/etc]
     --mandir=PATH          installation path for man pages [PREFIX/share/man]
+    --python-dir=PATH      explicit installation directory for Python modules
+    --python-prefix=PATH   versioned installation directory for Python modules,
+                           like setup.py install --prefix
+                           [PATH/lib/python<Python Version>/site-packages]
+    --python-home=PATH     installation path for Python modules, like setup.py's
+                           install --home [PATH/lib/python]
 
   Optional Features:
     --enable-debug         compile in debugging mode (like --build-type=Debug)
@@ -61,6 +67,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --disable-archiver     don't build or install zeek-archiver tool
     --disable-python       don't try to build python bindings for Broker
     --disable-broker-tests don't try to build Broker unit tests
+    --disable-zkg          don't install zkg
 
   Required Packages in Non-Standard Locations:
     --with-openssl=PATH    path to OpenSSL install root
@@ -157,6 +164,7 @@ append_cache_entry BUILD_SHARED_LIBS    BOOL   true
 append_cache_entry INSTALL_AUX_TOOLS    BOOL   true
 append_cache_entry INSTALL_ZEEK_ARCHIVER BOOL  true
 append_cache_entry INSTALL_ZEEKCTL      BOOL   true
+append_cache_entry INSTALL_ZKG          BOOL   true
 append_cache_entry CPACK_SOURCE_IGNORE_FILES STRING
 append_cache_entry ENABLE_MOBILE_IPV6   BOOL   false
 append_cache_entry ZEEK_SANITIZERS      STRING ""
@@ -206,6 +214,15 @@ while [ $# -ne 0 ]; do
         --libdir=*)
             append_cache_entry CMAKE_INSTALL_LIBDIR PATH   $optarg
             ;;
+        --python-dir=*)
+            append_cache_entry ZEEK_PYTHON_DIR PATH   $optarg
+            ;;
+        --python-prefix=*)
+            append_cache_entry ZEEK_PYTHON_PREFIX PATH   $optarg
+            ;;
+        --python-home=*)
+            append_cache_entry ZEEK_PYTHON_HOME PATH   $optarg
+            ;;
         --scriptdir=*)
             append_cache_entry ZEEK_SCRIPT_INSTALL_PATH STRING $optarg
             user_set_scriptdir="true"
@@ -277,6 +294,9 @@ while [ $# -ne 0 ]; do
             append_cache_entry BROKER_DISABLE_TESTS        BOOL true
             append_cache_entry BROKER_DISABLE_DOC_EXAMPLES BOOL true
             ;;
+        --disable-zkg)
+            append_cache_entry INSTALL_ZKG BOOL false
+            ;;
         --with-openssl=*)
             append_cache_entry OPENSSL_ROOT_DIR PATH $optarg
             ;;

doc

@@ -1 +1 @@
-Subproject commit 1e5a6cc3cb4e6751cbc81d173d04f7f05eb6df90
+Subproject commit bc112906578b01df6392c804142173ed41cf8aea

scripts/site/local.zeek

@@ -104,3 +104,6 @@ redef digest_salt = "Please change this value.";
 # Uncomment the following line to enable logging of link-layer addresses. Enabling
 # this adds the link-layer address for each connection endpoint to the conn.log file.
 # @load policy/protocols/conn/mac-logging
+
+# Uncomment this to source zkg's package state
+# @load packages

zkg-config.in

@@ -0,0 +1,8 @@
+# Zeek-generated zkg config file.
+[sources]
+zeek = https://github.com/zeek/packages
+
+[paths]
+state_dir = @ZEEK_ZKG_STATE_DIR@
+script_dir = @ZEEK_SCRIPT_INSTALL_PATH@/site
+plugin_dir = @BRO_PLUGIN_INSTALL_PATH@