diff --git a/CHANGES b/CHANGES index cb4c84cbb9..329006f1e0 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,13 @@ +5.1.0-dev.245 | 2022-07-13 11:57:18 -0700 + + * Stop signal-masking upon running unit tests (Christian Kreibich, Corelight) + + * Pause signal-masking during script parsing (Christian Kreibich, Corelight) + + * Add btests to verify Zeek's handling of SIGTERM and reading stdin (Christian Kreibich, Corelight) + + * Add procps/procps-ng to several CI Docker images (Christian Kreibich, Corelight) + 5.1.0-dev.240 | 2022-07-13 11:23:38 -0700 * Use clang-format for all files in `testing/btest/plugins`. (Benjamin Bannier, Corelight) diff --git a/VERSION b/VERSION index 96cac64914..3d2fa0a68b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -5.1.0-dev.240 +5.1.0-dev.245 diff --git a/ci/alpine/Dockerfile b/ci/alpine/Dockerfile index 72cacecd66..0528e3f7ec 100644 --- a/ci/alpine/Dockerfile +++ b/ci/alpine/Dockerfile @@ -20,6 +20,7 @@ RUN apk add --no-cache \ make \ openssh-client \ openssl-dev \ + procps \ py3-pip \ python3 \ python3-dev \ diff --git a/ci/centos-stream-8/Dockerfile b/ci/centos-stream-8/Dockerfile index e2f31e8846..007281d7b9 100644 --- a/ci/centos-stream-8/Dockerfile +++ b/ci/centos-stream-8/Dockerfile @@ -20,6 +20,7 @@ RUN dnf -y install \ make \ openssl \ openssl-devel \ + procps-ng \ python3 \ python3-devel \ python3-pip\ diff --git a/ci/centos-stream-9/Dockerfile b/ci/centos-stream-9/Dockerfile index 4ba33dd568..5d43497c5a 100644 --- a/ci/centos-stream-9/Dockerfile +++ b/ci/centos-stream-9/Dockerfile @@ -31,6 +31,7 @@ RUN dnf -y --nobest install \ make \ openssl \ openssl-devel \ + procps-ng \ python3 \ python3-devel \ python3-pip\ diff --git a/ci/debian-10/Dockerfile b/ci/debian-10/Dockerfile index 077e2bc7d8..5b31f362c0 100644 --- a/ci/debian-10/Dockerfile +++ b/ci/debian-10/Dockerfile @@ -23,6 +23,7 @@ RUN apt-get update && apt-get -y install \ libpcap-dev \ libssl-dev \ make \ + procps \ python3 \ python3-dev \ python3-pip\ diff --git a/ci/fedora-35/Dockerfile b/ci/fedora-35/Dockerfile index a6d138563e..3801a945fe 100644 --- a/ci/fedora-35/Dockerfile +++ b/ci/fedora-35/Dockerfile @@ -18,6 +18,7 @@ RUN dnf -y install \ make \ openssl \ openssl-devel \ + procps-ng \ python3 \ python3-devel \ python3-pip\ diff --git a/ci/fedora-36/Dockerfile b/ci/fedora-36/Dockerfile index c3a6629842..62743fcd0f 100644 --- a/ci/fedora-36/Dockerfile +++ b/ci/fedora-36/Dockerfile @@ -18,6 +18,7 @@ RUN dnf -y install \ make \ openssl \ openssl-devel \ + procps-ng \ python3 \ python3-devel \ python3-pip\ diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index f35dd3c0cd..c7f90c1ccb 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -606,8 +606,9 @@ SetupResult setup(int argc, char** argv, Options* zopts) // Mask signals relevant for our signal handlers here. We unmask them // again further down, when all components that launch threads have done - // so. The launched threads inherit the active signal mask and thus - // prevent our signal handlers from running in unintended threads. + // so, and intermittently during parsing. The launched threads inherit + // the active signal mask and thus prevent our signal handlers from + // running in unintended threads. set_signal_mask(true); if ( options.supervisor_mode ) @@ -717,6 +718,7 @@ SetupResult setup(int argc, char** argv, Options* zopts) // Delay the unit test until here so that plugins have been loaded. if ( options.run_unit_tests ) { + set_signal_mask(false); // Allow ctrl-c to abort the tests early doctest::Context context; auto dargs = to_cargs(options.doctest_args); context.applyCommandLine(dargs.size(), dargs.data()); @@ -778,9 +780,15 @@ SetupResult setup(int argc, char** argv, Options* zopts) if ( options.event_trace_file ) etm = make_unique(*options.event_trace_file); + // Parsing involves reading input files, including any input + // interactively provided by the user at the console. Temporarily + // undo the signal mask to allow ctrl-c. Ideally we'd do this only + // when we actually end up reading interactively from stdin. + set_signal_mask(false); run_state::is_parsing = true; yyparse(); run_state::is_parsing = false; + set_signal_mask(true); RecordVal::DoneParsing(); TableVal::DoneParsing(); diff --git a/testing/btest/Baseline/core.load-stdin/output.explicit b/testing/btest/Baseline/core.load-stdin/output.explicit new file mode 100644 index 0000000000..4b91a7afe5 --- /dev/null +++ b/testing/btest/Baseline/core.load-stdin/output.explicit @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +stdin diff --git a/testing/btest/Baseline/core.load-stdin/output.implicit b/testing/btest/Baseline/core.load-stdin/output.implicit new file mode 100644 index 0000000000..4b91a7afe5 --- /dev/null +++ b/testing/btest/Baseline/core.load-stdin/output.implicit @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +stdin diff --git a/testing/btest/Baseline/core.load-stdin/output.mixed b/testing/btest/Baseline/core.load-stdin/output.mixed new file mode 100644 index 0000000000..62c96ffcb7 --- /dev/null +++ b/testing/btest/Baseline/core.load-stdin/output.mixed @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +test +stdin diff --git a/testing/btest/Baseline/core.load-stdin/output.nostdin b/testing/btest/Baseline/core.load-stdin/output.nostdin new file mode 100644 index 0000000000..a42080c46d --- /dev/null +++ b/testing/btest/Baseline/core.load-stdin/output.nostdin @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +test diff --git a/testing/btest/core/load-stdin.zeek b/testing/btest/core/load-stdin.zeek new file mode 100644 index 0000000000..e3aa49dc02 --- /dev/null +++ b/testing/btest/core/load-stdin.zeek @@ -0,0 +1,11 @@ +# This verifies Zeek's ability to load scripts from stdin. +# @TEST-EXEC: echo 'print "stdin";' | zeek -b >output.implicit +# @TEST-EXEC: echo 'print "stdin";' | zeek -b - >output.explicit +# @TEST-EXEC: echo 'print "stdin";' | zeek -b %INPUT >output.nostdin +# @TEST-EXEC: echo 'print "stdin";' | zeek -b %INPUT - >output.mixed +# @TEST-EXEC: btest-diff output.implicit +# @TEST-EXEC: btest-diff output.explicit +# @TEST-EXEC: btest-diff output.nostdin +# @TEST-EXEC: btest-diff output.mixed + +print "test"; diff --git a/testing/btest/core/sigterm-regular.sh b/testing/btest/core/sigterm-regular.sh new file mode 100644 index 0000000000..1cfed760a8 --- /dev/null +++ b/testing/btest/core/sigterm-regular.sh @@ -0,0 +1,49 @@ +# This test verifies that Zeek terminates upon SIGTERM during regular script +# processing. +# +# See the sigterm-stdin.sh test for additional explanation of what's happening. +# +# Use a separate output file since btest-bg-wait replaces .stdout/.stderr: +# @TEST-EXEC: bash %INPUT >output 2>&1 + +# Helper to return the PID of the Zeek process launched in the background. +zeek_pid() { + # The btest-bg-run .pid file contains the parent of the Zeek process + local ppid=$(cat zeek/.pid) + ps -xo pid,ppid,comm | awk "\$2 == \"$ppid\" && \$3 == \"zeek\" { print \$1 }" +} + +cleanup() { + btest-bg-wait -k 5 +} + +trap cleanup EXIT + +btest-bg-run zeek "zeek exit_only_after_terminate=T" + +# Wait until we see Zeek running. +for i in $(seq 10); do + pid=$(zeek_pid) + [ -n "$pid" ] && break + sleep 1 +done + +if [ -z "$pid" ]; then + echo "Couldn't determine Zeek PID" + exit 1 +fi + +for i in $(seq 10); do + kill $pid + [ -z "$(zeek_pid)" ] && break + sleep 1 +done + +pid=$(zeek_pid) + +if [ -n "$pid" ]; then + echo "Zeek PID $pid did not shut down" + exit 1 +fi + +exit 0 diff --git a/testing/btest/core/sigterm-stdin.sh b/testing/btest/core/sigterm-stdin.sh new file mode 100644 index 0000000000..fad9520db1 --- /dev/null +++ b/testing/btest/core/sigterm-stdin.sh @@ -0,0 +1,63 @@ +# This test verifies that Zeek, while reading stdin to parse scripts, terminates +# upon SIGTERM. +# +# Running Zeek in a way that portably delivers SIGINT (as ctrl-c would do) is +# tricky. With job control done locally in this script, even when run by an +# interactive bash, SIGINT is blocked. When running via btest-bg-run, the +# backgrounded processes have their SIGINT and SIGQUIT blocked, per POSIX: +# https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html +# +# Use a separate output file since btest-bg-wait replaces .stdout/.stderr: +# @TEST-EXEC: bash %INPUT >output 2>&1 + +# Helper to return the PID of the Zeek process launched in the background. +zeek_pid() { + # The btest-bg-run .pid file contains the parent of the Zeek process + local ppid=$(cat zeek/.pid) + ps -xo pid,ppid,comm | awk "\$2 == \"$ppid\" && \$3 == \"zeek\" { print \$1 }" +} + +cleanup() { + btest-bg-wait -k 5 +} + +trap cleanup EXIT + +# Launch Zeek so it stalls, reading from stdin. +mkfifo input +btest-bg-run zeek "cat ../input | zeek" + +# Wait until we see Zeek running. +for i in $(seq 10); do + pid=$(zeek_pid) + [ -n "$pid" ] && break + sleep 1 +done + +if [ -z "$pid" ]; then + echo "Couldn't determine Zeek PID" + exit 1 +fi + +# Now try several times to terminate the process via SIGTERM. We try repeatedly +# because we might hit Zeek in a brief window in time where the signal is +# blocked -- it gets unblocked during the parsing stage, since this enables +# ctrl-c to work during interactive input. +# +# Terminating Zeek does not terminate the "cat", since the latter would only +# notice upon a data write that the pipe is gone. We leave it to btest-bg-wait +# to clean up at exit. +for i in $(seq 10); do + kill $pid + [ -z "$(zeek_pid)" ] && break + sleep 1 +done + +pid=$(zeek_pid) + +if [ -n "$pid" ]; then + echo "Zeek PID $pid did not shut down" + exit 1 +fi + +exit 0