From 459dc91463bf56c298bc3bea3ab9bf99a33ee9fd Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 24 Nov 2020 20:17:13 -0800 Subject: [PATCH 1/7] Update all submodules that have Python 2 EOL changes --- auxil/broker | 2 +- auxil/btest | 2 +- auxil/netcontrol-connectors | 2 +- auxil/zeekctl | 2 +- cmake | 2 +- doc | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/auxil/broker b/auxil/broker index 2ce76b8bda..4b1acd1218 160000 --- a/auxil/broker +++ b/auxil/broker @@ -1 +1 @@ -Subproject commit 2ce76b8bda0db04d807fe85f7b959191eac84fe2 +Subproject commit 4b1acd1218f6000c6ceb605f7ea3935bc489a895 diff --git a/auxil/btest b/auxil/btest index 03b2fe6fa9..288d5ef7f4 160000 --- a/auxil/btest +++ b/auxil/btest @@ -1 +1 @@ -Subproject commit 03b2fe6fa958a8519cb22bbf230b8a8e081a50c9 +Subproject commit 288d5ef7f45e889e42e5a8e036873d9a11e0d5f8 diff --git a/auxil/netcontrol-connectors b/auxil/netcontrol-connectors index 92d1bee12b..a81f860118 160000 --- a/auxil/netcontrol-connectors +++ b/auxil/netcontrol-connectors @@ -1 +1 @@ -Subproject commit 92d1bee12b0d92d36d784367c3c33646a7db990d +Subproject commit a81f860118c1c981be0fab77d1f345ebab85460d diff --git a/auxil/zeekctl b/auxil/zeekctl index d8391deb06..6c25426bf7 160000 --- a/auxil/zeekctl +++ b/auxil/zeekctl @@ -1 +1 @@ -Subproject commit d8391deb06e77515cfd73687eda74870f20e342a +Subproject commit 6c25426bf773fe3291b8e9d840a48b24273fa549 diff --git a/cmake b/cmake index 3ee51ab551..fa6d5d2573 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 3ee51ab5515f0f3089602d766aad737eb8b2c093 +Subproject commit fa6d5d25739f60cf0e55f7a5dc65a23c07462ece diff --git a/doc b/doc index 88b48fe09d..d76c85f56f 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 88b48fe09d355239afd74a36d4f948629ee0e95d +Subproject commit d76c85f56ff7d64603eebbc49ed06ef2b816ab06 From 33a55a04c364e1ad5e3fa97dc4d5af9e5792150b Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 24 Nov 2020 20:20:02 -0800 Subject: [PATCH 2/7] Update documentation for Python >= 3.5 requirement --- NEWS | 2 ++ src/strings.bif | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 50458e3dd0..a020f85851 100644 --- a/NEWS +++ b/NEWS @@ -155,6 +155,8 @@ Removed Functionality ``connection_state_remove`` handler can now be resolved with a less-confusing approach: see the ``Conn::register_removal_hook`` function. +- Python 2 is no longer supported. Python 3.5 is the new minimum requirement. + Deprecated Functionality ------------------------ diff --git a/src/strings.bif b/src/strings.bif index ecabe195c8..7e4bbfdfcd 100644 --- a/src/strings.bif +++ b/src/strings.bif @@ -1405,7 +1405,7 @@ function swap_case%(str: string%) : string %} ## Converts a string to Title Case. This changes the first character of each sequence of non-space characters -## in the string to be capitalized. See https://docs.python.org/2/library/stdtypes.html#str.title for more info. +## in the string to be capitalized. See https://docs.python.org/3/library/stdtypes.html#str.title for more info. ## ## str: The string to convert. ## From eeec219a66ed89235093fb63753d6baf6c21de78 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 24 Nov 2020 20:23:05 -0800 Subject: [PATCH 3/7] Update CMake logic to enforce Python >= 3.5 --- CMakeLists.txt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8b92e39201..e8ce79519d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -236,6 +236,7 @@ if (NOT SED_EXE) endif () endif () +list(APPEND Python_ADDITIONAL_VERSIONS 3) FindRequiredPackage(PythonInterp) FindRequiredPackage(FLEX) FindRequiredPackage(BISON) @@ -282,6 +283,12 @@ if (MISSING_PREREQS) message(FATAL_ERROR "Configuration aborted due to missing prerequisites") endif () +set(ZEEK_PYTHON_MIN 3.5.0) + +if ( PYTHON_VERSION_STRING VERSION_LESS ${ZEEK_PYTHON_MIN} ) + message(FATAL_ERROR "Python ${ZEEK_PYTHON_MIN} or greater is required.") +endif () + if ( CAF_ROOT_DIR ) find_package(CAF COMPONENTS core io openssl REQUIRED) endif () @@ -530,7 +537,7 @@ endif () if (INSTALL_ZEEKCTL) # CPack RPM Generator may not automatically detect this - set(CPACK_RPM_PACKAGE_REQUIRES "python >= 2.6.0") + set(CPACK_RPM_PACKAGE_REQUIRES "python >= ${ZEEK_PYTHON_MIN}") endif () # If this CMake project is a sub-project of another, we will not From 9717c623c22cc6ad3ab34db4fe6a6f6b6ed2a2e8 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 24 Nov 2020 20:27:44 -0800 Subject: [PATCH 4/7] Update Python invocations to explicit `python3` --- testing/btest/coverage/find-bro-logs.test | 4 ++-- testing/btest/scripts/base/utils/active-http.test | 4 ++-- testing/scripts/coverage-calc | 2 +- testing/scripts/httpd.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/testing/btest/coverage/find-bro-logs.test b/testing/btest/coverage/find-bro-logs.test index 82b5df2445..01e822deef 100644 --- a/testing/btest/coverage/find-bro-logs.test +++ b/testing/btest/coverage/find-bro-logs.test @@ -4,7 +4,7 @@ # # If this test fails, then the "Log Files" documentation page should be updated. -# @TEST-REQUIRES: which python +# @TEST-REQUIRES: which python3 # @TEST-EXEC: bash %INPUT # @TEST-EXEC: btest-diff out @@ -15,7 +15,7 @@ if [ ! -d "${BROSCRIPTS}" ]; then exit 1 fi -python find_logs.py "${BROSCRIPTS}" | sort > out +python3 find_logs.py "${BROSCRIPTS}" | sort > out @TEST-START-FILE find_logs.py import os, sys diff --git a/testing/btest/scripts/base/utils/active-http.test b/testing/btest/scripts/base/utils/active-http.test index ff80dc5bf2..36f5ec9eab 100644 --- a/testing/btest/scripts/base/utils/active-http.test +++ b/testing/btest/scripts/base/utils/active-http.test @@ -1,7 +1,7 @@ -# @TEST-REQUIRES: which python +# @TEST-REQUIRES: which python3 # @TEST-REQUIRES: which curl # -# @TEST-EXEC: btest-bg-run httpd python $SCRIPTS/httpd.py --max 2 --addr=127.0.0.1 +# @TEST-EXEC: btest-bg-run httpd python3 $SCRIPTS/httpd.py --max 2 --addr=127.0.0.1 # @TEST-EXEC: sleep 3 # @TEST-EXEC: btest-bg-run zeek zeek -b %INPUT # @TEST-EXEC: btest-bg-wait 15 diff --git a/testing/scripts/coverage-calc b/testing/scripts/coverage-calc index 3645f57144..016cfff5c4 100755 --- a/testing/scripts/coverage-calc +++ b/testing/scripts/coverage-calc @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # This script aggregates many files containing Zeek script coverage information # into a single file and reports the overall coverage information. Usage: diff --git a/testing/scripts/httpd.py b/testing/scripts/httpd.py index 3576f09d1a..c5cc4507b7 100755 --- a/testing/scripts/httpd.py +++ b/testing/scripts/httpd.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 try: # Python 2 From 7d68f2ce9dd50c634b8dfb2b4f34d053918c805e Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 24 Nov 2020 20:28:29 -0800 Subject: [PATCH 5/7] Remove Python 2 compatibility logic in httpd test script --- testing/scripts/httpd.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/testing/scripts/httpd.py b/testing/scripts/httpd.py index c5cc4507b7..7ecfb636e9 100755 --- a/testing/scripts/httpd.py +++ b/testing/scripts/httpd.py @@ -1,11 +1,6 @@ #! /usr/bin/env python3 -try: - # Python 2 - import BaseHTTPServer -except ImportError: - # Python 3 - import http.server as BaseHTTPServer +import http.server as BaseHTTPServer class MyRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): From 2b3d216c972ecf4d96163cdb55eaccff7139161e Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 24 Nov 2020 20:35:31 -0800 Subject: [PATCH 6/7] Update CI scripts to remove `python` vs `python3` workarounds --- ci/centos-7/Dockerfile | 8 +------- ci/centos-8/Dockerfile | 8 +------- ci/debian-10/Dockerfile | 8 +------- ci/debian-9-32bit/Dockerfile | 8 +------- ci/debian-9/Dockerfile | 8 +------- ci/fedora-31/Dockerfile | 8 +------- ci/fedora-32/Dockerfile | 8 +------- ci/fedora-33/Dockerfile | 8 +------- ci/freebsd/prepare.sh | 1 - ci/ubuntu-16.04/Dockerfile | 8 +------- ci/ubuntu-18.04/Dockerfile | 8 +------- ci/ubuntu-20.04/Dockerfile | 8 +------- 12 files changed, 11 insertions(+), 78 deletions(-) diff --git a/ci/centos-7/Dockerfile b/ci/centos-7/Dockerfile index 759d08d19a..c5734f3653 100644 --- a/ci/centos-7/Dockerfile +++ b/ci/centos-7/Dockerfile @@ -38,13 +38,7 @@ RUN yum -y install \ which \ && yum clean all && rm -rf /var/cache/yum -# Many distros adhere to PEP 394's recommendation for `python` = `python2` so -# this is a simple workaround until we drop Python 2 support and explicitly -# use `python3` for all invocations (e.g. in shebangs). -RUN ln -sf /usr/bin/python3 /usr/local/bin/python -RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip - -RUN pip install junit2html +RUN pip3 install junit2html RUN echo 'unset BASH_ENV PROMPT_COMMAND ENV' > /usr/bin/zeek-ci-env && \ echo 'source /opt/rh/devtoolset-7/enable' >> /usr/bin/zeek-ci-env diff --git a/ci/centos-8/Dockerfile b/ci/centos-8/Dockerfile index 9a9df30a1e..ecdde92664 100644 --- a/ci/centos-8/Dockerfile +++ b/ci/centos-8/Dockerfile @@ -26,10 +26,4 @@ RUN dnf -y update && dnf -y install \ which \ && dnf clean all && rm -rf /var/cache/dnf -# Many distros adhere to PEP 394's recommendation for `python` = `python2` so -# this is a simple workaround until we drop Python 2 support and explicitly -# use `python3` for all invocations (e.g. in shebangs). -RUN ln -sf /usr/bin/python3 /usr/local/bin/python -RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip - -RUN pip install junit2html +RUN pip3 install junit2html diff --git a/ci/debian-10/Dockerfile b/ci/debian-10/Dockerfile index ea5a9ab0b3..a0f05b6f6a 100644 --- a/ci/debian-10/Dockerfile +++ b/ci/debian-10/Dockerfile @@ -25,10 +25,4 @@ RUN apt-get update && apt-get -y install \ xz-utils \ && rm -rf /var/lib/apt/lists/* -# Many distros adhere to PEP 394's recommendation for `python` = `python2` so -# this is a simple workaround until we drop Python 2 support and explicitly -# use `python3` for all invocations (e.g. in shebangs). -RUN ln -sf /usr/bin/python3 /usr/local/bin/python -RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip - -RUN pip install junit2html +RUN pip3 install junit2html diff --git a/ci/debian-9-32bit/Dockerfile b/ci/debian-9-32bit/Dockerfile index 9ff5c2161d..3a6990216d 100644 --- a/ci/debian-9-32bit/Dockerfile +++ b/ci/debian-9-32bit/Dockerfile @@ -31,12 +31,6 @@ RUN apt-get update && apt-get -y install \ RUN update-alternatives --install /usr/bin/cc cc /usr/bin/clang-7 100 RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-7 100 -# Many distros adhere to PEP 394's recommendation for `python` = `python2` so -# this is a simple workaround until we drop Python 2 support and explicitly -# use `python3` for all invocations (e.g. in shebangs). -RUN ln -sf /usr/bin/python3 /usr/local/bin/python -RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip - -RUN pip install junit2html +RUN pip3 install junit2html ENV CXXFLAGS=-stdlib=libc++ diff --git a/ci/debian-9/Dockerfile b/ci/debian-9/Dockerfile index e2dabd95d0..2fcde5fc01 100644 --- a/ci/debian-9/Dockerfile +++ b/ci/debian-9/Dockerfile @@ -28,13 +28,7 @@ RUN apt-get update && apt-get -y install \ libc++abi-7-dev \ && rm -rf /var/lib/apt/lists/* -# Many distros adhere to PEP 394's recommendation for `python` = `python2` so -# this is a simple workaround until we drop Python 2 support and explicitly -# use `python3` for all invocations (e.g. in shebangs). -RUN ln -sf /usr/bin/python3 /usr/local/bin/python -RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip - -RUN pip install junit2html +RUN pip3 install junit2html ENV CC=/usr/bin/clang-7 ENV CXX=/usr/bin/clang++-7 diff --git a/ci/fedora-31/Dockerfile b/ci/fedora-31/Dockerfile index bc71a8a795..e9b24a9671 100644 --- a/ci/fedora-31/Dockerfile +++ b/ci/fedora-31/Dockerfile @@ -22,10 +22,4 @@ RUN yum -y install \ zlib-devel \ && yum clean all && rm -rf /var/cache/yum -# Many distros adhere to PEP 394's recommendation for `python` = `python2` so -# this is a simple workaround until we drop Python 2 support and explicitly -# use `python3` for all invocations (e.g. in shebangs). -RUN ln -sf /usr/bin/python3 /usr/local/bin/python -RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip - -RUN pip install junit2html +RUN pip3 install junit2html diff --git a/ci/fedora-32/Dockerfile b/ci/fedora-32/Dockerfile index 80b833fdb3..87e072753c 100644 --- a/ci/fedora-32/Dockerfile +++ b/ci/fedora-32/Dockerfile @@ -22,10 +22,4 @@ RUN yum -y install \ zlib-devel \ && yum clean all && rm -rf /var/cache/yum -# Many distros adhere to PEP 394's recommendation for `python` = `python2` so -# this is a simple workaround until we drop Python 2 support and explicitly -# use `python3` for all invocations (e.g. in shebangs). -RUN ln -sf /usr/bin/python3 /usr/local/bin/python -RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip - -RUN pip install junit2html +RUN pip3 install junit2html diff --git a/ci/fedora-33/Dockerfile b/ci/fedora-33/Dockerfile index 3124d85bf5..15d5f9244e 100644 --- a/ci/fedora-33/Dockerfile +++ b/ci/fedora-33/Dockerfile @@ -22,10 +22,4 @@ RUN yum -y install \ zlib-devel \ && yum clean all && rm -rf /var/cache/yum -# Many distros adhere to PEP 394's recommendation for `python` = `python2` so -# this is a simple workaround until we drop Python 2 support and explicitly -# use `python3` for all invocations (e.g. in shebangs). -RUN ln -sf /usr/bin/python3 /usr/local/bin/python -RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip - -RUN pip install junit2html +RUN pip3 install junit2html diff --git a/ci/freebsd/prepare.sh b/ci/freebsd/prepare.sh index 097c25e243..0ec60513cf 100755 --- a/ci/freebsd/prepare.sh +++ b/ci/freebsd/prepare.sh @@ -9,5 +9,4 @@ env ASSUME_ALWAYS_YES=YES pkg bootstrap pkg install -y bash git cmake swig bison python3 base64 pyver=`python3 -c 'import sys; print(f"py{sys.version_info[0]}{sys.version_info[1]}")'` pkg install -y $pyver-sqlite3 $pyver-pip -( cd && mkdir -p ./bin && ln -s /usr/local/bin/python3 ./bin/python ) pip install junit2html diff --git a/ci/ubuntu-16.04/Dockerfile b/ci/ubuntu-16.04/Dockerfile index e094e584c5..5b65acd55f 100644 --- a/ci/ubuntu-16.04/Dockerfile +++ b/ci/ubuntu-16.04/Dockerfile @@ -28,13 +28,7 @@ RUN apt-get update && apt-get -y install \ xz-utils \ && rm -rf /var/lib/apt/lists/* -# Many distros adhere to PEP 394's recommendation for `python` = `python2` so -# this is a simple workaround until we drop Python 2 support and explicitly -# use `python3` for all invocations (e.g. in shebangs). -RUN ln -sf /usr/bin/python3 /usr/local/bin/python -RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip - -RUN pip install junit2html +RUN pip3 install junit2html ENV CC=/usr/bin/clang-8 ENV CXX=/usr/bin/clang++-8 diff --git a/ci/ubuntu-18.04/Dockerfile b/ci/ubuntu-18.04/Dockerfile index e298595f39..ed5ecb3b8e 100644 --- a/ci/ubuntu-18.04/Dockerfile +++ b/ci/ubuntu-18.04/Dockerfile @@ -29,11 +29,5 @@ RUN apt-get update && apt-get -y install \ lcov \ && rm -rf /var/lib/apt/lists/* -# Many distros adhere to PEP 394's recommendation for `python` = `python2` so -# this is a simple workaround until we drop Python 2 support and explicitly -# use `python3` for all invocations (e.g. in shebangs). -RUN ln -sf /usr/bin/python3 /usr/local/bin/python -RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip - -RUN pip install junit2html +RUN pip3 install junit2html RUN gem install coveralls-lcov diff --git a/ci/ubuntu-20.04/Dockerfile b/ci/ubuntu-20.04/Dockerfile index 3c48668794..74cf7b571e 100644 --- a/ci/ubuntu-20.04/Dockerfile +++ b/ci/ubuntu-20.04/Dockerfile @@ -29,11 +29,5 @@ RUN apt-get update && apt-get -y install \ lcov \ && rm -rf /var/lib/apt/lists/* -# Many distros adhere to PEP 394's recommendation for `python` = `python2` so -# this is a simple workaround until we drop Python 2 support and explicitly -# use `python3` for all invocations (e.g. in shebangs). -RUN ln -sf /usr/bin/python3 /usr/local/bin/python -RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip - -RUN pip install junit2html +RUN pip3 install junit2html RUN gem install coveralls-lcov From 5ff44989c4f23e91cad8b2a007837d30104d483b Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 25 Nov 2020 15:19:56 -0800 Subject: [PATCH 7/7] Update CentOS CI Dockerfiles to fix git/diff dependencies --- ci/centos-7/Dockerfile | 2 +- ci/centos-8/Dockerfile | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/centos-7/Dockerfile b/ci/centos-7/Dockerfile index c5734f3653..5ab9df2d68 100644 --- a/ci/centos-7/Dockerfile +++ b/ci/centos-7/Dockerfile @@ -5,7 +5,7 @@ FROM centos:7 RUN yum -y install \ https://repo.ius.io/ius-release-el7.rpm \ https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm \ - && yum -y install git2u \ + && yum -y install git224 \ && yum clean all && rm -rf /var/cache/yum RUN yum -y install \ diff --git a/ci/centos-8/Dockerfile b/ci/centos-8/Dockerfile index ecdde92664..9f4084c8b5 100644 --- a/ci/centos-8/Dockerfile +++ b/ci/centos-8/Dockerfile @@ -23,6 +23,7 @@ RUN dnf -y update && dnf -y install \ zlib-devel \ libsqlite3x-devel \ findutils \ + diffutils \ which \ && dnf clean all && rm -rf /var/cache/dnf