Merge remote-tracking branch 'origin/topic/jsiwek/python2-eol'

* origin/topic/jsiwek/python2-eol:
  Update CentOS CI Dockerfiles to fix git/diff dependencies
  Update CI scripts to remove `python` vs `python3` workarounds
  Remove Python 2 compatibility logic in httpd test script
  Update Python invocations to explicit `python3`
  Update CMake logic to enforce Python >= 3.5
  Update documentation for Python >= 3.5 requirement
  Update all submodules that have Python 2 EOL changes

CHANGES

@@ -1,4 +1,14 @@
 
+3.3.0-dev.576 | 2020-11-26 18:16:07 +0000
+
+  * Remove Python2 compatibility logic. We now require at least Python 3.5.
+    This includes script changes, improves the cmake logic to find python3,
+    makes scripts explicitly call python3 and documentation updates.
+
+    (Jon Siwek, Corelight)
+
+  * Remove Fedora 31 (EOL) from CI (Jon Siwek, Corelight)
+
 3.3.0-dev.564 | 2020-11-24 15:23:50 -0800
 
   * Improve support for custom libdir locations (Christian Kreibich, Corelight)

CMakeLists.txt

@@ -236,6 +236,7 @@ if (NOT SED_EXE)
     endif ()
 endif ()
 
+list(APPEND Python_ADDITIONAL_VERSIONS 3)
 FindRequiredPackage(PythonInterp)
 FindRequiredPackage(FLEX)
 FindRequiredPackage(BISON)
@@ -282,6 +283,12 @@ if (MISSING_PREREQS)
     message(FATAL_ERROR "Configuration aborted due to missing prerequisites")
 endif ()
 
+set(ZEEK_PYTHON_MIN 3.5.0)
+
+if ( PYTHON_VERSION_STRING VERSION_LESS ${ZEEK_PYTHON_MIN} )
+    message(FATAL_ERROR "Python ${ZEEK_PYTHON_MIN} or greater is required.")
+endif ()
+
 if ( CAF_ROOT_DIR )
   find_package(CAF COMPONENTS core io openssl REQUIRED)
 endif ()
@@ -530,7 +537,7 @@ endif ()
 
 if (INSTALL_ZEEKCTL)
     # CPack RPM Generator may not automatically detect this
-    set(CPACK_RPM_PACKAGE_REQUIRES "python >= 2.6.0")
+    set(CPACK_RPM_PACKAGE_REQUIRES "python >= ${ZEEK_PYTHON_MIN}")
 endif ()
 
 # If this CMake project is a sub-project of another, we will not

NEWS

@@ -155,6 +155,8 @@ Removed Functionality
   ``connection_state_remove`` handler can now be resolved with a less-confusing
   approach: see the ``Conn::register_removal_hook`` function.
 
+- Python 2 is no longer supported.  Python 3.5 is the new minimum requirement.
+
 Deprecated Functionality
 ------------------------
 

VERSION

@@ -1 +1 @@
-3.3.0-dev.564
+3.3.0-dev.576

auxil/bifcl

@@ -1 +1 @@
-Subproject commit 535228510006085b8300630200de318a9d7583ab
+Subproject commit e265d659fd86d7439de0f12a31f9c12a786836da

auxil/binpac

@@ -1 +1 @@
-Subproject commit 893861a62e51e6b86e3ca5f9df497cf7834019fb
+Subproject commit f1392c6af9337c9454ab43e539739b4c8abc9bae

auxil/broker

@@ -1 +1 @@
-Subproject commit 2ce76b8bda0db04d807fe85f7b959191eac84fe2
+Subproject commit 65e7ffa63c3abae1ce485154fbc2ff7c7cafbf04

auxil/btest

@@ -1 +1 @@
-Subproject commit 03b2fe6fa958a8519cb22bbf230b8a8e081a50c9
+Subproject commit ecebce5f75237f417fbc9ea254daa0d7fa4397f3

auxil/netcontrol-connectors

@@ -1 +1 @@
-Subproject commit 92d1bee12b0d92d36d784367c3c33646a7db990d
+Subproject commit 94e1c36512adb47b43c157b87c500176ffb668e2

auxil/zeek-aux

@@ -1 +1 @@
-Subproject commit fccfd7d95d7c6e5b93325365bca85734502a1b21
+Subproject commit 1ba32fede011342d0d6abeec1985a07585f90083

auxil/zeekctl

@@ -1 +1 @@
-Subproject commit d8391deb06e77515cfd73687eda74870f20e342a
+Subproject commit 6d1e1f9dbebf1d2463a0c6fb480440e7d68ba472

ci/centos-7/Dockerfile

@@ -5,7 +5,7 @@ FROM centos:7
 RUN yum -y install \
     https://repo.ius.io/ius-release-el7.rpm \
     https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm \
-  && yum -y install git2u \
+  && yum -y install git224 \
   && yum clean all && rm -rf /var/cache/yum
 
 RUN yum -y install \
@@ -38,13 +38,7 @@ RUN yum -y install \
     which \
   && yum clean all && rm -rf /var/cache/yum
 
-# Many distros adhere to PEP 394's recommendation for `python` = `python2` so
+RUN pip3 install junit2html
-# this is a simple workaround until we drop Python 2 support and explicitly
-# use `python3` for all invocations (e.g. in shebangs).
-RUN ln -sf /usr/bin/python3 /usr/local/bin/python
-RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip
-
-RUN pip install junit2html
 
 RUN echo 'unset BASH_ENV PROMPT_COMMAND ENV' > /usr/bin/zeek-ci-env && \
     echo 'source /opt/rh/devtoolset-7/enable' >> /usr/bin/zeek-ci-env

ci/centos-8/Dockerfile

@@ -23,13 +23,8 @@ RUN dnf -y update && dnf -y install \
     zlib-devel \
     libsqlite3x-devel \
     findutils \
+    diffutils \
     which \
   && dnf clean all && rm -rf /var/cache/dnf
 
-# Many distros adhere to PEP 394's recommendation for `python` = `python2` so
+RUN pip3 install junit2html
-# this is a simple workaround until we drop Python 2 support and explicitly
-# use `python3` for all invocations (e.g. in shebangs).
-RUN ln -sf /usr/bin/python3 /usr/local/bin/python
-RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip
-
-RUN pip install junit2html

ci/debian-10/Dockerfile

@@ -25,10 +25,4 @@ RUN apt-get update && apt-get -y install \
     xz-utils \
   && rm -rf /var/lib/apt/lists/*
 
-# Many distros adhere to PEP 394's recommendation for `python` = `python2` so
+RUN pip3 install junit2html
-# this is a simple workaround until we drop Python 2 support and explicitly
-# use `python3` for all invocations (e.g. in shebangs).
-RUN ln -sf /usr/bin/python3 /usr/local/bin/python
-RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip
-
-RUN pip install junit2html

ci/debian-9-32bit/Dockerfile

@@ -31,12 +31,6 @@ RUN apt-get update && apt-get -y install \
 RUN update-alternatives --install /usr/bin/cc cc /usr/bin/clang-7 100
 RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-7 100
 
-# Many distros adhere to PEP 394's recommendation for `python` = `python2` so
+RUN pip3 install junit2html
-# this is a simple workaround until we drop Python 2 support and explicitly
-# use `python3` for all invocations (e.g. in shebangs).
-RUN ln -sf /usr/bin/python3 /usr/local/bin/python
-RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip
-
-RUN pip install junit2html
 
 ENV CXXFLAGS=-stdlib=libc++

ci/debian-9/Dockerfile

@@ -28,13 +28,7 @@ RUN apt-get update && apt-get -y install \
     libc++abi-7-dev \
   && rm -rf /var/lib/apt/lists/*
 
-# Many distros adhere to PEP 394's recommendation for `python` = `python2` so
+RUN pip3 install junit2html
-# this is a simple workaround until we drop Python 2 support and explicitly
-# use `python3` for all invocations (e.g. in shebangs).
-RUN ln -sf /usr/bin/python3 /usr/local/bin/python
-RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip
-
-RUN pip install junit2html
 
 ENV CC=/usr/bin/clang-7
 ENV CXX=/usr/bin/clang++-7

ci/fedora-32/Dockerfile

@@ -22,10 +22,4 @@ RUN yum -y install \
     zlib-devel \
   && yum clean all && rm -rf /var/cache/yum
 
-# Many distros adhere to PEP 394's recommendation for `python` = `python2` so
+RUN pip3 install junit2html
-# this is a simple workaround until we drop Python 2 support and explicitly
-# use `python3` for all invocations (e.g. in shebangs).
-RUN ln -sf /usr/bin/python3 /usr/local/bin/python
-RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip
-
-RUN pip install junit2html

ci/fedora-33/Dockerfile

@@ -22,10 +22,4 @@ RUN yum -y install \
     zlib-devel \
   && yum clean all && rm -rf /var/cache/yum
 
-# Many distros adhere to PEP 394's recommendation for `python` = `python2` so
+RUN pip3 install junit2html
-# this is a simple workaround until we drop Python 2 support and explicitly
-# use `python3` for all invocations (e.g. in shebangs).
-RUN ln -sf /usr/bin/python3 /usr/local/bin/python
-RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip
-
-RUN pip install junit2html

ci/freebsd/prepare.sh

@@ -9,5 +9,4 @@ env ASSUME_ALWAYS_YES=YES pkg bootstrap
 pkg install -y bash git cmake swig bison python3 base64
 pyver=`python3 -c 'import sys; print(f"py{sys.version_info[0]}{sys.version_info[1]}")'`
 pkg install -y $pyver-sqlite3 $pyver-pip
-( cd && mkdir -p ./bin && ln -s /usr/local/bin/python3 ./bin/python )
 pip install junit2html

ci/ubuntu-16.04/Dockerfile

@@ -28,13 +28,7 @@ RUN apt-get update && apt-get -y install \
     xz-utils \
   && rm -rf /var/lib/apt/lists/*
 
-# Many distros adhere to PEP 394's recommendation for `python` = `python2` so
+RUN pip3 install junit2html
-# this is a simple workaround until we drop Python 2 support and explicitly
-# use `python3` for all invocations (e.g. in shebangs).
-RUN ln -sf /usr/bin/python3 /usr/local/bin/python
-RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip
-
-RUN pip install junit2html
 
 ENV CC=/usr/bin/clang-8
 ENV CXX=/usr/bin/clang++-8

ci/ubuntu-18.04/Dockerfile

@@ -29,11 +29,5 @@ RUN apt-get update && apt-get -y install \
     lcov \
   && rm -rf /var/lib/apt/lists/*
 
-# Many distros adhere to PEP 394's recommendation for `python` = `python2` so
+RUN pip3 install junit2html
-# this is a simple workaround until we drop Python 2 support and explicitly
-# use `python3` for all invocations (e.g. in shebangs).
-RUN ln -sf /usr/bin/python3 /usr/local/bin/python
-RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip
-
-RUN pip install junit2html
 RUN gem install coveralls-lcov

ci/ubuntu-20.04/Dockerfile

@@ -29,11 +29,5 @@ RUN apt-get update && apt-get -y install \
     lcov \
   && rm -rf /var/lib/apt/lists/*
 
-# Many distros adhere to PEP 394's recommendation for `python` = `python2` so
+RUN pip3 install junit2html
-# this is a simple workaround until we drop Python 2 support and explicitly
-# use `python3` for all invocations (e.g. in shebangs).
-RUN ln -sf /usr/bin/python3 /usr/local/bin/python
-RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip
-
-RUN pip install junit2html
 RUN gem install coveralls-lcov

cmake

@@ -1 +1 @@
-Subproject commit 3ee51ab5515f0f3089602d766aad737eb8b2c093
+Subproject commit d02e87b1b0af10c0df65f13ffc70a990411b9724

src/strings.bif

@@ -1405,7 +1405,7 @@ function swap_case%(str: string%) : string
 	%}
 
 ## Converts a string to Title Case. This changes the first character of each sequence of non-space characters
-## in the string to be capitalized. See https://docs.python.org/2/library/stdtypes.html#str.title for more info.
+## in the string to be capitalized. See https://docs.python.org/3/library/stdtypes.html#str.title for more info.
 ##
 ## str: The string to convert.
 ##

testing/btest/coverage/find-bro-logs.test

@@ -4,7 +4,7 @@
 #
 # If this test fails, then the "Log Files" documentation page should be updated.
 
-# @TEST-REQUIRES: which python
+# @TEST-REQUIRES: which python3
 # @TEST-EXEC: bash %INPUT
 # @TEST-EXEC: btest-diff out
 
@@ -15,7 +15,7 @@ if [ ! -d "${BROSCRIPTS}" ]; then
     exit 1
 fi
 
-python find_logs.py "${BROSCRIPTS}" | sort > out
+python3 find_logs.py "${BROSCRIPTS}" | sort > out
 
 @TEST-START-FILE find_logs.py
 import os, sys

testing/btest/scripts/base/utils/active-http.test

@@ -1,7 +1,7 @@
-# @TEST-REQUIRES: which python
+# @TEST-REQUIRES: which python3
 # @TEST-REQUIRES: which curl
 #
-# @TEST-EXEC: btest-bg-run httpd python $SCRIPTS/httpd.py --max 2 --addr=127.0.0.1
+# @TEST-EXEC: btest-bg-run httpd python3 $SCRIPTS/httpd.py --max 2 --addr=127.0.0.1
 # @TEST-EXEC: sleep 3
 # @TEST-EXEC: btest-bg-run zeek zeek -b %INPUT
 # @TEST-EXEC: btest-bg-wait 15

testing/scripts/coverage-calc

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 
 # This script aggregates many files containing Zeek script coverage information
 # into a single file and reports the overall coverage information. Usage:

testing/scripts/httpd.py

@@ -1,11 +1,6 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 
-try:
+import http.server as BaseHTTPServer
-    # Python 2
-    import BaseHTTPServer
-except ImportError:
-    # Python 3
-    import http.server as BaseHTTPServer
 
 class MyRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):