diff --git a/CHANGES b/CHANGES index 9b2d9f5131..2f085b8a01 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,14 @@ +3.3.0-dev.576 | 2020-11-26 18:16:07 +0000 + + * Remove Python2 compatibility logic. We now require at least Python 3.5. + This includes script changes, improves the cmake logic to find python3, + makes scripts explicitly call python3 and documentation updates. + + (Jon Siwek, Corelight) + + * Remove Fedora 31 (EOL) from CI (Jon Siwek, Corelight) + 3.3.0-dev.564 | 2020-11-24 15:23:50 -0800 * Improve support for custom libdir locations (Christian Kreibich, Corelight) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8b92e39201..e8ce79519d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -236,6 +236,7 @@ if (NOT SED_EXE) endif () endif () +list(APPEND Python_ADDITIONAL_VERSIONS 3) FindRequiredPackage(PythonInterp) FindRequiredPackage(FLEX) FindRequiredPackage(BISON) @@ -282,6 +283,12 @@ if (MISSING_PREREQS) message(FATAL_ERROR "Configuration aborted due to missing prerequisites") endif () +set(ZEEK_PYTHON_MIN 3.5.0) + +if ( PYTHON_VERSION_STRING VERSION_LESS ${ZEEK_PYTHON_MIN} ) + message(FATAL_ERROR "Python ${ZEEK_PYTHON_MIN} or greater is required.") +endif () + if ( CAF_ROOT_DIR ) find_package(CAF COMPONENTS core io openssl REQUIRED) endif () @@ -530,7 +537,7 @@ endif () if (INSTALL_ZEEKCTL) # CPack RPM Generator may not automatically detect this - set(CPACK_RPM_PACKAGE_REQUIRES "python >= 2.6.0") + set(CPACK_RPM_PACKAGE_REQUIRES "python >= ${ZEEK_PYTHON_MIN}") endif () # If this CMake project is a sub-project of another, we will not diff --git a/NEWS b/NEWS index 50458e3dd0..a020f85851 100644 --- a/NEWS +++ b/NEWS @@ -155,6 +155,8 @@ Removed Functionality ``connection_state_remove`` handler can now be resolved with a less-confusing approach: see the ``Conn::register_removal_hook`` function. +- Python 2 is no longer supported. Python 3.5 is the new minimum requirement. + Deprecated Functionality ------------------------ diff --git a/VERSION b/VERSION index ea18578fe0..ea64e30a72 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.3.0-dev.564 +3.3.0-dev.576 diff --git a/auxil/bifcl b/auxil/bifcl index 5352285100..e265d659fd 160000 --- a/auxil/bifcl +++ b/auxil/bifcl @@ -1 +1 @@ -Subproject commit 535228510006085b8300630200de318a9d7583ab +Subproject commit e265d659fd86d7439de0f12a31f9c12a786836da diff --git a/auxil/binpac b/auxil/binpac index 893861a62e..f1392c6af9 160000 --- a/auxil/binpac +++ b/auxil/binpac @@ -1 +1 @@ -Subproject commit 893861a62e51e6b86e3ca5f9df497cf7834019fb +Subproject commit f1392c6af9337c9454ab43e539739b4c8abc9bae diff --git a/auxil/broker b/auxil/broker index 2ce76b8bda..65e7ffa63c 160000 --- a/auxil/broker +++ b/auxil/broker @@ -1 +1 @@ -Subproject commit 2ce76b8bda0db04d807fe85f7b959191eac84fe2 +Subproject commit 65e7ffa63c3abae1ce485154fbc2ff7c7cafbf04 diff --git a/auxil/btest b/auxil/btest index 03b2fe6fa9..ecebce5f75 160000 --- a/auxil/btest +++ b/auxil/btest @@ -1 +1 @@ -Subproject commit 03b2fe6fa958a8519cb22bbf230b8a8e081a50c9 +Subproject commit ecebce5f75237f417fbc9ea254daa0d7fa4397f3 diff --git a/auxil/netcontrol-connectors b/auxil/netcontrol-connectors index 92d1bee12b..94e1c36512 160000 --- a/auxil/netcontrol-connectors +++ b/auxil/netcontrol-connectors @@ -1 +1 @@ -Subproject commit 92d1bee12b0d92d36d784367c3c33646a7db990d +Subproject commit 94e1c36512adb47b43c157b87c500176ffb668e2 diff --git a/auxil/zeek-aux b/auxil/zeek-aux index fccfd7d95d..1ba32fede0 160000 --- a/auxil/zeek-aux +++ b/auxil/zeek-aux @@ -1 +1 @@ -Subproject commit fccfd7d95d7c6e5b93325365bca85734502a1b21 +Subproject commit 1ba32fede011342d0d6abeec1985a07585f90083 diff --git a/auxil/zeekctl b/auxil/zeekctl index d8391deb06..6d1e1f9dbe 160000 --- a/auxil/zeekctl +++ b/auxil/zeekctl @@ -1 +1 @@ -Subproject commit d8391deb06e77515cfd73687eda74870f20e342a +Subproject commit 6d1e1f9dbebf1d2463a0c6fb480440e7d68ba472 diff --git a/ci/centos-7/Dockerfile b/ci/centos-7/Dockerfile index 759d08d19a..5ab9df2d68 100644 --- a/ci/centos-7/Dockerfile +++ b/ci/centos-7/Dockerfile @@ -5,7 +5,7 @@ FROM centos:7 RUN yum -y install \ https://repo.ius.io/ius-release-el7.rpm \ https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm \ - && yum -y install git2u \ + && yum -y install git224 \ && yum clean all && rm -rf /var/cache/yum RUN yum -y install \ @@ -38,13 +38,7 @@ RUN yum -y install \ which \ && yum clean all && rm -rf /var/cache/yum -# Many distros adhere to PEP 394's recommendation for `python` = `python2` so -# this is a simple workaround until we drop Python 2 support and explicitly -# use `python3` for all invocations (e.g. in shebangs). -RUN ln -sf /usr/bin/python3 /usr/local/bin/python -RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip - -RUN pip install junit2html +RUN pip3 install junit2html RUN echo 'unset BASH_ENV PROMPT_COMMAND ENV' > /usr/bin/zeek-ci-env && \ echo 'source /opt/rh/devtoolset-7/enable' >> /usr/bin/zeek-ci-env diff --git a/ci/centos-8/Dockerfile b/ci/centos-8/Dockerfile index 9a9df30a1e..9f4084c8b5 100644 --- a/ci/centos-8/Dockerfile +++ b/ci/centos-8/Dockerfile @@ -23,13 +23,8 @@ RUN dnf -y update && dnf -y install \ zlib-devel \ libsqlite3x-devel \ findutils \ + diffutils \ which \ && dnf clean all && rm -rf /var/cache/dnf -# Many distros adhere to PEP 394's recommendation for `python` = `python2` so -# this is a simple workaround until we drop Python 2 support and explicitly -# use `python3` for all invocations (e.g. in shebangs). -RUN ln -sf /usr/bin/python3 /usr/local/bin/python -RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip - -RUN pip install junit2html +RUN pip3 install junit2html diff --git a/ci/debian-10/Dockerfile b/ci/debian-10/Dockerfile index ea5a9ab0b3..a0f05b6f6a 100644 --- a/ci/debian-10/Dockerfile +++ b/ci/debian-10/Dockerfile @@ -25,10 +25,4 @@ RUN apt-get update && apt-get -y install \ xz-utils \ && rm -rf /var/lib/apt/lists/* -# Many distros adhere to PEP 394's recommendation for `python` = `python2` so -# this is a simple workaround until we drop Python 2 support and explicitly -# use `python3` for all invocations (e.g. in shebangs). -RUN ln -sf /usr/bin/python3 /usr/local/bin/python -RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip - -RUN pip install junit2html +RUN pip3 install junit2html diff --git a/ci/debian-9-32bit/Dockerfile b/ci/debian-9-32bit/Dockerfile index 9ff5c2161d..3a6990216d 100644 --- a/ci/debian-9-32bit/Dockerfile +++ b/ci/debian-9-32bit/Dockerfile @@ -31,12 +31,6 @@ RUN apt-get update && apt-get -y install \ RUN update-alternatives --install /usr/bin/cc cc /usr/bin/clang-7 100 RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-7 100 -# Many distros adhere to PEP 394's recommendation for `python` = `python2` so -# this is a simple workaround until we drop Python 2 support and explicitly -# use `python3` for all invocations (e.g. in shebangs). -RUN ln -sf /usr/bin/python3 /usr/local/bin/python -RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip - -RUN pip install junit2html +RUN pip3 install junit2html ENV CXXFLAGS=-stdlib=libc++ diff --git a/ci/debian-9/Dockerfile b/ci/debian-9/Dockerfile index e2dabd95d0..2fcde5fc01 100644 --- a/ci/debian-9/Dockerfile +++ b/ci/debian-9/Dockerfile @@ -28,13 +28,7 @@ RUN apt-get update && apt-get -y install \ libc++abi-7-dev \ && rm -rf /var/lib/apt/lists/* -# Many distros adhere to PEP 394's recommendation for `python` = `python2` so -# this is a simple workaround until we drop Python 2 support and explicitly -# use `python3` for all invocations (e.g. in shebangs). -RUN ln -sf /usr/bin/python3 /usr/local/bin/python -RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip - -RUN pip install junit2html +RUN pip3 install junit2html ENV CC=/usr/bin/clang-7 ENV CXX=/usr/bin/clang++-7 diff --git a/ci/fedora-32/Dockerfile b/ci/fedora-32/Dockerfile index 80b833fdb3..87e072753c 100644 --- a/ci/fedora-32/Dockerfile +++ b/ci/fedora-32/Dockerfile @@ -22,10 +22,4 @@ RUN yum -y install \ zlib-devel \ && yum clean all && rm -rf /var/cache/yum -# Many distros adhere to PEP 394's recommendation for `python` = `python2` so -# this is a simple workaround until we drop Python 2 support and explicitly -# use `python3` for all invocations (e.g. in shebangs). -RUN ln -sf /usr/bin/python3 /usr/local/bin/python -RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip - -RUN pip install junit2html +RUN pip3 install junit2html diff --git a/ci/fedora-33/Dockerfile b/ci/fedora-33/Dockerfile index 3124d85bf5..15d5f9244e 100644 --- a/ci/fedora-33/Dockerfile +++ b/ci/fedora-33/Dockerfile @@ -22,10 +22,4 @@ RUN yum -y install \ zlib-devel \ && yum clean all && rm -rf /var/cache/yum -# Many distros adhere to PEP 394's recommendation for `python` = `python2` so -# this is a simple workaround until we drop Python 2 support and explicitly -# use `python3` for all invocations (e.g. in shebangs). -RUN ln -sf /usr/bin/python3 /usr/local/bin/python -RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip - -RUN pip install junit2html +RUN pip3 install junit2html diff --git a/ci/freebsd/prepare.sh b/ci/freebsd/prepare.sh index 097c25e243..0ec60513cf 100755 --- a/ci/freebsd/prepare.sh +++ b/ci/freebsd/prepare.sh @@ -9,5 +9,4 @@ env ASSUME_ALWAYS_YES=YES pkg bootstrap pkg install -y bash git cmake swig bison python3 base64 pyver=`python3 -c 'import sys; print(f"py{sys.version_info[0]}{sys.version_info[1]}")'` pkg install -y $pyver-sqlite3 $pyver-pip -( cd && mkdir -p ./bin && ln -s /usr/local/bin/python3 ./bin/python ) pip install junit2html diff --git a/ci/ubuntu-16.04/Dockerfile b/ci/ubuntu-16.04/Dockerfile index e094e584c5..5b65acd55f 100644 --- a/ci/ubuntu-16.04/Dockerfile +++ b/ci/ubuntu-16.04/Dockerfile @@ -28,13 +28,7 @@ RUN apt-get update && apt-get -y install \ xz-utils \ && rm -rf /var/lib/apt/lists/* -# Many distros adhere to PEP 394's recommendation for `python` = `python2` so -# this is a simple workaround until we drop Python 2 support and explicitly -# use `python3` for all invocations (e.g. in shebangs). -RUN ln -sf /usr/bin/python3 /usr/local/bin/python -RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip - -RUN pip install junit2html +RUN pip3 install junit2html ENV CC=/usr/bin/clang-8 ENV CXX=/usr/bin/clang++-8 diff --git a/ci/ubuntu-18.04/Dockerfile b/ci/ubuntu-18.04/Dockerfile index e298595f39..ed5ecb3b8e 100644 --- a/ci/ubuntu-18.04/Dockerfile +++ b/ci/ubuntu-18.04/Dockerfile @@ -29,11 +29,5 @@ RUN apt-get update && apt-get -y install \ lcov \ && rm -rf /var/lib/apt/lists/* -# Many distros adhere to PEP 394's recommendation for `python` = `python2` so -# this is a simple workaround until we drop Python 2 support and explicitly -# use `python3` for all invocations (e.g. in shebangs). -RUN ln -sf /usr/bin/python3 /usr/local/bin/python -RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip - -RUN pip install junit2html +RUN pip3 install junit2html RUN gem install coveralls-lcov diff --git a/ci/ubuntu-20.04/Dockerfile b/ci/ubuntu-20.04/Dockerfile index 3c48668794..74cf7b571e 100644 --- a/ci/ubuntu-20.04/Dockerfile +++ b/ci/ubuntu-20.04/Dockerfile @@ -29,11 +29,5 @@ RUN apt-get update && apt-get -y install \ lcov \ && rm -rf /var/lib/apt/lists/* -# Many distros adhere to PEP 394's recommendation for `python` = `python2` so -# this is a simple workaround until we drop Python 2 support and explicitly -# use `python3` for all invocations (e.g. in shebangs). -RUN ln -sf /usr/bin/python3 /usr/local/bin/python -RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip - -RUN pip install junit2html +RUN pip3 install junit2html RUN gem install coveralls-lcov diff --git a/cmake b/cmake index 3ee51ab551..d02e87b1b0 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 3ee51ab5515f0f3089602d766aad737eb8b2c093 +Subproject commit d02e87b1b0af10c0df65f13ffc70a990411b9724 diff --git a/src/strings.bif b/src/strings.bif index ecabe195c8..7e4bbfdfcd 100644 --- a/src/strings.bif +++ b/src/strings.bif @@ -1405,7 +1405,7 @@ function swap_case%(str: string%) : string %} ## Converts a string to Title Case. This changes the first character of each sequence of non-space characters -## in the string to be capitalized. See https://docs.python.org/2/library/stdtypes.html#str.title for more info. +## in the string to be capitalized. See https://docs.python.org/3/library/stdtypes.html#str.title for more info. ## ## str: The string to convert. ## diff --git a/testing/btest/coverage/find-bro-logs.test b/testing/btest/coverage/find-bro-logs.test index 82b5df2445..01e822deef 100644 --- a/testing/btest/coverage/find-bro-logs.test +++ b/testing/btest/coverage/find-bro-logs.test @@ -4,7 +4,7 @@ # # If this test fails, then the "Log Files" documentation page should be updated. -# @TEST-REQUIRES: which python +# @TEST-REQUIRES: which python3 # @TEST-EXEC: bash %INPUT # @TEST-EXEC: btest-diff out @@ -15,7 +15,7 @@ if [ ! -d "${BROSCRIPTS}" ]; then exit 1 fi -python find_logs.py "${BROSCRIPTS}" | sort > out +python3 find_logs.py "${BROSCRIPTS}" | sort > out @TEST-START-FILE find_logs.py import os, sys diff --git a/testing/btest/scripts/base/utils/active-http.test b/testing/btest/scripts/base/utils/active-http.test index ff80dc5bf2..36f5ec9eab 100644 --- a/testing/btest/scripts/base/utils/active-http.test +++ b/testing/btest/scripts/base/utils/active-http.test @@ -1,7 +1,7 @@ -# @TEST-REQUIRES: which python +# @TEST-REQUIRES: which python3 # @TEST-REQUIRES: which curl # -# @TEST-EXEC: btest-bg-run httpd python $SCRIPTS/httpd.py --max 2 --addr=127.0.0.1 +# @TEST-EXEC: btest-bg-run httpd python3 $SCRIPTS/httpd.py --max 2 --addr=127.0.0.1 # @TEST-EXEC: sleep 3 # @TEST-EXEC: btest-bg-run zeek zeek -b %INPUT # @TEST-EXEC: btest-bg-wait 15 diff --git a/testing/scripts/coverage-calc b/testing/scripts/coverage-calc index 3645f57144..016cfff5c4 100755 --- a/testing/scripts/coverage-calc +++ b/testing/scripts/coverage-calc @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # This script aggregates many files containing Zeek script coverage information # into a single file and reports the overall coverage information. Usage: diff --git a/testing/scripts/httpd.py b/testing/scripts/httpd.py index 3576f09d1a..7ecfb636e9 100755 --- a/testing/scripts/httpd.py +++ b/testing/scripts/httpd.py @@ -1,11 +1,6 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 -try: - # Python 2 - import BaseHTTPServer -except ImportError: - # Python 3 - import http.server as BaseHTTPServer +import http.server as BaseHTTPServer class MyRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):