From d1d218b5ccf1a0a81fbca79dfe20a4c14972c46c Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Mon, 7 Dec 2020 13:46:59 -0800 Subject: [PATCH] Install zkg as part of the Zeek distribution. - Add auxil/package-manager submodule as an optional build source. When the submodule is present, zkg gets installed into the Zeek installation's bin directory, its config file into etc/zkg, and its state into var/lib/zkg. Like zeekctl, zkg finds its own module independently of any PYTHONPATH. Installation via pip remains supported. You can skip zkg explicitly via --disable-zkg. See the NEWS update for details. - Establish a "zeek/python" subdirectory under libdir as the common place for Python modules in the Zeek distribution. This now separates out the Broker Python bindings, ZeekControl, and zkg's Python module. - Add configure flags to allow customizing this Python folder, in three ways: --python-dir, --python-prefix, and --python-home. These differ in the logic they automatically add to the path, and build on the logic already used in Broker. - Include a (comented-out) @load for zkg's packages folder in local.zeek. - Bump zeekctl to move to this new location. - Bump doc to include installation instructions - Update NEWS accordingly. --- .gitmodules | 3 ++ CMakeLists.txt | 67 +++++++++++++++++++++++++++++------------ NEWS | 67 ++++++++++++++++++++++++++++++++++++++--- auxil/package-manager | 1 + auxil/zeekctl | 2 +- configure | 22 +++++++++++++- doc | 2 +- scripts/site/local.zeek | 3 ++ zkg-config.in | 8 +++++ 9 files changed, 147 insertions(+), 28 deletions(-) create mode 160000 auxil/package-manager create mode 100644 zkg-config.in diff --git a/.gitmodules b/.gitmodules index d7309796e4..326d21d7e1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -43,3 +43,6 @@ [submodule "auxil/zeek-archiver"] path = auxil/zeek-archiver url = https://github.com/zeek/zeek-archiver +[submodule "auxil/package-manager"] + path = auxil/package-manager + url = https://github.com/zeek/package-manager diff --git a/CMakeLists.txt b/CMakeLists.txt index ba2a712bff..6129b550ab 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -53,8 +53,11 @@ endif () get_filename_component(ZEEK_SCRIPT_INSTALL_PATH ${ZEEK_SCRIPT_INSTALL_PATH} ABSOLUTE) -set(BRO_PLUGIN_INSTALL_PATH ${CMAKE_INSTALL_FULL_LIBDIR}/zeek/plugins CACHE STRING "Installation path for plugins" FORCE) -set(PY_MOD_INSTALL_DIR ${CMAKE_INSTALL_FULL_LIBDIR}/zeekctl CACHE STRING "Installation path for Python modules" FORCE) +# A folder for library-like Zeek-specific things: Python modules, Zeek +# plugins, etc. +set(ZEEK_LIBDIR_PATH ${CMAKE_INSTALL_FULL_LIBDIR}/zeek) +set(BRO_PLUGIN_INSTALL_PATH ${ZEEK_LIBDIR_PATH}/plugins + CACHE STRING "Installation path for plugins" FORCE) configure_file(zeek-path-dev.in ${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev) execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink @@ -245,6 +248,25 @@ FindRequiredPackage(OpenSSL) FindRequiredPackage(BIND) FindRequiredPackage(ZLIB) +# Installation directory for the distribution's Python modules. An +# override via configure's --python-dir wins, specifying a directory +# explicitly. Next is --python-prefix, which includes a versioned +# Python folder as the --prefix option in distutils does. Next +# consider a distutils --home style override via --python-home, and +# finally default to "zeek/python" in our libdir. +if (ZEEK_PYTHON_DIR) + set(py_mod_install_dir ${ZEEK_PYTHON_DIR}) +elseif (ZEEK_PYTHON_PREFIX) + set(pyver ${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}) + set(py_mod_install_dir ${ZEEK_PYTHON_PREFIX}/lib/python${pyver}/site-packages) +elseif (ZEEK_PYTHON_HOME) + set(py_mod_install_dir ${ZEEK_PYTHON_HOME}/lib/python) +else () + set(py_mod_install_dir ${ZEEK_LIBDIR_PATH}/python) +endif () +set(PY_MOD_INSTALL_DIR ${py_mod_install_dir} + CACHE STRING "Installation path for Python modules" FORCE) + if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/auxil/binpac/CMakeLists.txt) set(ENABLE_STATIC_ONLY_SAVED ${ENABLE_STATIC_ONLY}) @@ -519,6 +541,24 @@ include(InstallSymlink) InstallShellScript("bin" "zeek-wrapper.in" "zeek-wrapper") InstallSymlink("${CMAKE_INSTALL_PREFIX}/bin/zeek-wrapper" "${CMAKE_INSTALL_PREFIX}/bin/bro-config") +######################################################################## +## zkg configuration + +if ( INSTALL_ZKG ) + # An etc/zkg directory for zkg's config file simplifies zkg's + # config file code. + set(ZEEK_ZKG_CONFIG_DIR "${ZEEK_ETC_INSTALL_DIR}/zkg") + set(ZEEK_ZKG_STATE_DIR "${ZEEK_ROOT_DIR}/var/lib/zkg") + + configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/zkg-config.in + ${CMAKE_CURRENT_BINARY_DIR}/zkg-config @ONLY) + + install(DIRECTORY DESTINATION var/lib/zkg) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zkg-config + DESTINATION ${ZEEK_ZKG_CONFIG_DIR} RENAME config) +endif () + ######################################################################## ## Recurse on sub-directories @@ -528,31 +568,15 @@ add_subdirectory(man) include(CheckOptionalBuildSources) +CheckOptionalBuildSources(auxil/package-manager ZKG INSTALL_ZKG) CheckOptionalBuildSources(auxil/zeekctl ZeekControl INSTALL_ZEEKCTL) CheckOptionalBuildSources(auxil/zeek-aux Zeek-Aux INSTALL_AUX_TOOLS) CheckOptionalBuildSources(auxil/zeek-archiver ZeekArchiver INSTALL_ZEEK_ARCHIVER) -######################################################################## -## Transitions and cleanups - -if ( NOT BINARY_PACKAGING_MODE ) - # Remove pre-existing libdir of the old hardwired name if it is not - # the name we're now installing under. - set(_old_libdir ${CMAKE_INSTALL_PREFIX}/lib) - - install(CODE " - if ( EXISTS \"${_old_libdir}\" AND IS_DIRECTORY \"${_old_libdir}\" - AND NOT \"${_old_libdir}\" STREQUAL \"${CMAKE_INSTALL_FULL_LIBDIR}\" ) - message(STATUS \"WARNING: removing old library directory ${_old_libdir}\") - execute_process(COMMAND \"${CMAKE_COMMAND}\" -E remove_directory \"${_old_libdir}\") - endif () - ") -endif () - ######################################################################## ## Packaging Setup -if (INSTALL_ZEEKCTL) +if ( INSTALL_ZEEKCTL OR INSTALL_ZKG ) # CPack RPM Generator may not automatically detect this set(CPACK_RPM_PACKAGE_REQUIRES "python >= ${ZEEK_PYTHON_MIN}") endif () @@ -578,6 +602,8 @@ message( "\nBuild type: ${CMAKE_BUILD_TYPE}" "\nBuild dir: ${CMAKE_BINARY_DIR}" "\nInstall prefix: ${CMAKE_INSTALL_PREFIX}" + "\nPlugin dir: ${BRO_PLUGIN_INSTALL_PATH}" + "\nPython module dir: ${PY_MOD_INSTALL_DIR}" "\nZeek Script Path: ${ZEEK_SCRIPT_INSTALL_PATH}" "\nDebug mode: ${ENABLE_DEBUG}" "\nUnit tests: ${ENABLE_ZEEK_UNIT_TESTS}" @@ -590,6 +616,7 @@ message( "\n" "\nZeekControl: ${INSTALL_ZEEKCTL}" "\nAux. Tools: ${INSTALL_AUX_TOOLS}" + "\nzkg: ${INSTALL_ZKG}" "\n" "\nlibmaxminddb: ${USE_GEOIP}" "\nKerberos: ${USE_KRB5}" diff --git a/NEWS b/NEWS index ee2fa48500..8bde1613fc 100644 --- a/NEWS +++ b/NEWS @@ -120,6 +120,49 @@ New Functionality epochs will have to be manually ended by callis ``SumStats::next_epoch``. This can be convenient because epochs can be synced to other events. +- The Zeek distribution now includes Zeek's package manager, zkg. Its + code, configuration, and state reside in Zeek's installation tree, + as follows: + + - The toplevel script, ``zkg``, installs alongside ``zeek`` in the + distribution's ``$prefix/bin`` folder. + + - The config file installs into ``$prefix/etc/zkg/config``. The + distribution's zkg command uses it by default, but you can switch + to a different one via the ``ZKG_CONFIG_FILE`` environment + variable or the ``--configfile`` command-line flag. + + - zkg's package state resides in ``$prefix/var/lib/zkg``. This + implies that parallel Zeek installations now automatically + separate their package installations. + + These folders have the same ownership and access permissions as the + rest of the installation, , meaning that in order to manage zkg + packages you need to run zkg as a user with corresponding access. + Apart from these location overrides, the bundled zkg installation + behaves as usual. + + local.zeek now contains a (commented out) ``@load`` statement you + can use to source zkg's package state automatically. + + zkg's own Python module resides in ``zeek/python/zeekpkg`, in the + installation tree's library folder. See below for additional changes + around the library folder. + + zkg has external Python module dependencies. The Zeek configuration + does not verify whether these dependencies are met. A new warning + message at zkg launch flags missing packages and how to install them + (e.g. via pip). + + Configuring with ``--disable-zkg`` disables the zkg inclusion. You + can continue to install and use zkg independently. You're also free + to use the config file in ``$prefix/etc/zkg/config`` with other zkg + installations. + + The zkg source tree resides in ``auxil/package-manager`` as an + additional Git submodule. + + Changed Functionality --------------------- @@ -166,11 +209,25 @@ Changed Functionality - The Zeek installation tree is now more consistent in using a ``lib64/`` (rather than ``lib/``) subdirectory for platforms where that's the common convention. If the old hardcoded ``lib/`` path exists while installing Zeek - 4.0 and the new subdirectory differs, then the old ``lib/`` will be removed. - This potentially wipes out binary plugins that have already been installed - there, but Zeek plugins generally have to be re-built/re-installed upon any - Zeek upgrade anyway, so no part of the usual upgrade process is expected to - be complicated by this cleanup operation. + 4.0 and the new subdirectory differs, the old ``lib/`` remains untouched. + This clutters the installation but is safe: the new installation does not + require the old location, and any files you might require still in the old + tree (e.g. ZeekControl plugins) remain available. + + Due to Zeek 4's reorganization of the installation tree we recommend + a clean-slate install when possible. + +- Python modules installed with the Zeek distribution now reside in a + common ``zeek/python`` directory below the library path (such as + ``lib64/zeek/python``) and no longer assume ZeekControl. The + ``zeek/python/zeekctl`` folder now contains only ZeekControl's own + functionality, ``zeek/python/zeekpkg`` contains zkg's Python module, and + Broker's Python bindings live in ``zeek/python/broker``. ``zeek-config + --python_dir`` now reports this new ``zeek/python`` folder. Several + new configure options allow you to customize the Python folder location, + depending on your needs. + + As with the new libdir, no cleanup of the existing Python tree occurs. - Continued renaming/namespacing of many classes into either ``zeek`` or ``zeek::detail`` namespaces as already explained in Zeek 3.2's release notes. diff --git a/auxil/package-manager b/auxil/package-manager new file mode 160000 index 0000000000..eaf313ae97 --- /dev/null +++ b/auxil/package-manager @@ -0,0 +1 @@ +Subproject commit eaf313ae976c6505bf4ba0efccc9ba34b65f686c diff --git a/auxil/zeekctl b/auxil/zeekctl index c98d51bc6b..fe25dba367 160000 --- a/auxil/zeekctl +++ b/auxil/zeekctl @@ -1 +1 @@ -Subproject commit c98d51bc6b7a2d40cdc095108b985088ccb7e32c +Subproject commit fe25dba36701e09a9fa3d4598307ec894e9fd535 diff --git a/configure b/configure index 162b5c4acf..e148956e69 100755 --- a/configure +++ b/configure @@ -43,6 +43,12 @@ Usage: $0 [OPTION]... [VAR=VALUE]... --libdir=PATH installation directory for library files [PREFIX/lib] --conf-files-dir=PATH config files installation directory [PREFIX/etc] --mandir=PATH installation path for man pages [PREFIX/share/man] + --python-dir=PATH explicit installation directory for Python modules + --python-prefix=PATH versioned installation directory for Python modules, + like setup.py install --prefix + [PATH/lib/python/site-packages] + --python-home=PATH installation path for Python modules, like setup.py's + install --home [PATH/lib/python] Optional Features: --enable-debug compile in debugging mode (like --build-type=Debug) @@ -61,6 +67,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]... --disable-archiver don't build or install zeek-archiver tool --disable-python don't try to build python bindings for Broker --disable-broker-tests don't try to build Broker unit tests + --disable-zkg don't install zkg Required Packages in Non-Standard Locations: --with-openssl=PATH path to OpenSSL install root @@ -157,6 +164,7 @@ append_cache_entry BUILD_SHARED_LIBS BOOL true append_cache_entry INSTALL_AUX_TOOLS BOOL true append_cache_entry INSTALL_ZEEK_ARCHIVER BOOL true append_cache_entry INSTALL_ZEEKCTL BOOL true +append_cache_entry INSTALL_ZKG BOOL true append_cache_entry CPACK_SOURCE_IGNORE_FILES STRING append_cache_entry ENABLE_MOBILE_IPV6 BOOL false append_cache_entry ZEEK_SANITIZERS STRING "" @@ -201,11 +209,20 @@ while [ $# -ne 0 ]; do --prefix=*) prefix=$optarg append_cache_entry CMAKE_INSTALL_PREFIX PATH $optarg - append_cache_entry ZEEK_ROOT_DIR PATH $optarg + append_cache_entry ZEEK_ROOT_DIR PATH $optarg ;; --libdir=*) append_cache_entry CMAKE_INSTALL_LIBDIR PATH $optarg ;; + --python-dir=*) + append_cache_entry ZEEK_PYTHON_DIR PATH $optarg + ;; + --python-prefix=*) + append_cache_entry ZEEK_PYTHON_PREFIX PATH $optarg + ;; + --python-home=*) + append_cache_entry ZEEK_PYTHON_HOME PATH $optarg + ;; --scriptdir=*) append_cache_entry ZEEK_SCRIPT_INSTALL_PATH STRING $optarg user_set_scriptdir="true" @@ -277,6 +294,9 @@ while [ $# -ne 0 ]; do append_cache_entry BROKER_DISABLE_TESTS BOOL true append_cache_entry BROKER_DISABLE_DOC_EXAMPLES BOOL true ;; + --disable-zkg) + append_cache_entry INSTALL_ZKG BOOL false + ;; --with-openssl=*) append_cache_entry OPENSSL_ROOT_DIR PATH $optarg ;; diff --git a/doc b/doc index 1e5a6cc3cb..2d1cba368f 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 1e5a6cc3cb4e6751cbc81d173d04f7f05eb6df90 +Subproject commit 2d1cba368f4e990bff705c13e38f608fa364e38c diff --git a/scripts/site/local.zeek b/scripts/site/local.zeek index 25b1ba3689..2acd297c47 100644 --- a/scripts/site/local.zeek +++ b/scripts/site/local.zeek @@ -104,3 +104,6 @@ redef digest_salt = "Please change this value."; # Uncomment the following line to enable logging of link-layer addresses. Enabling # this adds the link-layer address for each connection endpoint to the conn.log file. # @load policy/protocols/conn/mac-logging + +# Uncomment this to source zkg's package state +# @load packages diff --git a/zkg-config.in b/zkg-config.in new file mode 100644 index 0000000000..7c3fbbd3cd --- /dev/null +++ b/zkg-config.in @@ -0,0 +1,8 @@ +# Zeek-generated zkg config file. +[sources] +zeek = https://github.com/zeek/packages + +[paths] +state_dir = @ZEEK_ZKG_STATE_DIR@ +script_dir = @ZEEK_SCRIPT_INSTALL_PATH@/site +plugin_dir = @BRO_PLUGIN_INSTALL_PATH@