Merge branch 'topic/christian/gh-2239-stdin-ctrl-c'

* topic/christian/gh-2239-stdin-ctrl-c:
  Stop signal-masking upon running unit tests
  Pause signal-masking during script parsing
  Add btests to verify Zeek's handling of SIGTERM and reading stdin
  Add procps/procps-ng to several CI Docker images

CHANGES

@@ -1,3 +1,13 @@
+5.1.0-dev.245 | 2022-07-13 11:57:18 -0700
+
+  * Stop signal-masking upon running unit tests (Christian Kreibich, Corelight)
+
+  * Pause signal-masking during script parsing (Christian Kreibich, Corelight)
+
+  * Add btests to verify Zeek's handling of SIGTERM and reading stdin (Christian Kreibich, Corelight)
+
+  * Add procps/procps-ng to several CI Docker images (Christian Kreibich, Corelight)
+
 5.1.0-dev.240 | 2022-07-13 11:23:38 -0700
 
   * Use clang-format for all files in `testing/btest/plugins`. (Benjamin Bannier, Corelight)

VERSION

@@ -1 +1 @@
-5.1.0-dev.240
+5.1.0-dev.245

ci/alpine/Dockerfile

@@ -20,6 +20,7 @@ RUN apk add --no-cache \
   make \
   openssh-client \
   openssl-dev \
+  procps \
   py3-pip \
   python3 \
   python3-dev \

ci/centos-stream-8/Dockerfile

@@ -20,6 +20,7 @@ RUN dnf -y install \
     make \
     openssl \
     openssl-devel \
+    procps-ng \
     python3 \
     python3-devel \
     python3-pip\

ci/centos-stream-9/Dockerfile

@@ -31,6 +31,7 @@ RUN dnf -y --nobest install \
     make \
     openssl \
     openssl-devel \
+    procps-ng \
     python3 \
     python3-devel \
     python3-pip\

ci/debian-10/Dockerfile

@@ -23,6 +23,7 @@ RUN apt-get update && apt-get -y install \
     libpcap-dev \
     libssl-dev \
     make \
+    procps \
     python3 \
     python3-dev \
     python3-pip\

ci/fedora-35/Dockerfile

@@ -18,6 +18,7 @@ RUN dnf -y install \
     make \
     openssl \
     openssl-devel \
+    procps-ng \
     python3 \
     python3-devel \
     python3-pip\

ci/fedora-36/Dockerfile

@@ -18,6 +18,7 @@ RUN dnf -y install \
     make \
     openssl \
     openssl-devel \
+    procps-ng \
     python3 \
     python3-devel \
     python3-pip\

src/zeek-setup.cc

@@ -606,8 +606,9 @@ SetupResult setup(int argc, char** argv, Options* zopts)
 
 	// Mask signals relevant for our signal handlers here. We unmask them
 	// again further down, when all components that launch threads have done
-	// so. The launched threads inherit the active signal mask and thus
+	// so, and intermittently during parsing. The launched threads inherit
-	// prevent our signal handlers from running in unintended threads.
+	// the active signal mask and thus prevent our signal handlers from
+	// running in unintended threads.
 	set_signal_mask(true);
 
 	if ( options.supervisor_mode )
@@ -717,6 +718,7 @@ SetupResult setup(int argc, char** argv, Options* zopts)
 	// Delay the unit test until here so that plugins have been loaded.
 	if ( options.run_unit_tests )
 		{
+		set_signal_mask(false); // Allow ctrl-c to abort the tests early
 		doctest::Context context;
 		auto dargs = to_cargs(options.doctest_args);
 		context.applyCommandLine(dargs.size(), dargs.data());
@@ -778,9 +780,15 @@ SetupResult setup(int argc, char** argv, Options* zopts)
 		if ( options.event_trace_file )
 			etm = make_unique<EventTraceMgr>(*options.event_trace_file);
 
+		// Parsing involves reading input files, including any input
+		// interactively provided by the user at the console. Temporarily
+		// undo the signal mask to allow ctrl-c. Ideally we'd do this only
+		// when we actually end up reading interactively from stdin.
+		set_signal_mask(false);
 		run_state::is_parsing = true;
 		yyparse();
 		run_state::is_parsing = false;
+		set_signal_mask(true);
 
 		RecordVal::DoneParsing();
 		TableVal::DoneParsing();

testing/btest/Baseline/core.load-stdin/output.explicit

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+stdin

testing/btest/Baseline/core.load-stdin/output.implicit

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+stdin

testing/btest/Baseline/core.load-stdin/output.mixed

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+test
+stdin

testing/btest/Baseline/core.load-stdin/output.nostdin

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+test

testing/btest/core/load-stdin.zeek

@@ -0,0 +1,11 @@
+# This verifies Zeek's ability to load scripts from stdin.
+# @TEST-EXEC: echo 'print "stdin";' | zeek -b >output.implicit
+# @TEST-EXEC: echo 'print "stdin";' | zeek -b - >output.explicit
+# @TEST-EXEC: echo 'print "stdin";' | zeek -b %INPUT >output.nostdin
+# @TEST-EXEC: echo 'print "stdin";' | zeek -b %INPUT - >output.mixed
+# @TEST-EXEC: btest-diff output.implicit
+# @TEST-EXEC: btest-diff output.explicit
+# @TEST-EXEC: btest-diff output.nostdin
+# @TEST-EXEC: btest-diff output.mixed
+
+print "test";

testing/btest/core/sigterm-regular.sh

@@ -0,0 +1,49 @@
+# This test verifies that Zeek terminates upon SIGTERM during regular script
+# processing.
+#
+# See the sigterm-stdin.sh test for additional explanation of what's happening.
+#
+# Use a separate output file since btest-bg-wait replaces .stdout/.stderr:
+# @TEST-EXEC: bash %INPUT >output 2>&1
+
+# Helper to return the PID of the Zeek process launched in the background.
+zeek_pid() {
+    # The btest-bg-run .pid file contains the parent of the Zeek process
+    local ppid=$(cat zeek/.pid)
+    ps -xo pid,ppid,comm | awk "\$2 == \"$ppid\" && \$3 == \"zeek\" { print \$1 }"
+}
+
+cleanup() {
+    btest-bg-wait -k 5
+}
+
+trap cleanup EXIT
+
+btest-bg-run zeek "zeek exit_only_after_terminate=T"
+
+# Wait until we see Zeek running.
+for i in $(seq 10); do
+    pid=$(zeek_pid)
+    [ -n "$pid" ] && break
+    sleep 1
+done
+
+if [ -z "$pid" ]; then
+    echo "Couldn't determine Zeek PID"
+    exit 1
+fi
+
+for i in $(seq 10); do
+    kill $pid
+    [ -z "$(zeek_pid)" ] && break
+    sleep 1
+done
+
+pid=$(zeek_pid)
+
+if [ -n "$pid" ]; then
+    echo "Zeek PID $pid did not shut down"
+    exit 1
+fi
+
+exit 0

testing/btest/core/sigterm-stdin.sh

@@ -0,0 +1,63 @@
+# This test verifies that Zeek, while reading stdin to parse scripts, terminates
+# upon SIGTERM.
+#
+# Running Zeek in a way that portably delivers SIGINT (as ctrl-c would do) is
+# tricky. With job control done locally in this script, even when run by an
+# interactive bash, SIGINT is blocked. When running via btest-bg-run, the
+# backgrounded processes have their SIGINT and SIGQUIT blocked, per POSIX:
+# https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html
+#
+# Use a separate output file since btest-bg-wait replaces .stdout/.stderr:
+# @TEST-EXEC: bash %INPUT >output 2>&1
+
+# Helper to return the PID of the Zeek process launched in the background.
+zeek_pid() {
+    # The btest-bg-run .pid file contains the parent of the Zeek process
+    local ppid=$(cat zeek/.pid)
+    ps -xo pid,ppid,comm | awk "\$2 == \"$ppid\" && \$3 == \"zeek\" { print \$1 }"
+}
+
+cleanup() {
+    btest-bg-wait -k 5
+}
+
+trap cleanup EXIT
+
+# Launch Zeek so it stalls, reading from stdin.
+mkfifo input
+btest-bg-run zeek "cat ../input | zeek"
+
+# Wait until we see Zeek running.
+for i in $(seq 10); do
+    pid=$(zeek_pid)
+    [ -n "$pid" ] && break
+    sleep 1
+done
+
+if [ -z "$pid" ]; then
+    echo "Couldn't determine Zeek PID"
+    exit 1
+fi
+
+# Now try several times to terminate the process via SIGTERM. We try repeatedly
+# because we might hit Zeek in a brief window in time where the signal is
+# blocked -- it gets unblocked during the parsing stage, since this enables
+# ctrl-c to work during interactive input.
+#
+# Terminating Zeek does not terminate the "cat", since the latter would only
+# notice upon a data write that the pipe is gone. We leave it to btest-bg-wait
+# to clean up at exit.
+for i in $(seq 10); do
+    kill $pid
+    [ -z "$(zeek_pid)" ] && break
+    sleep 1
+done
+
+pid=$(zeek_pid)
+
+if [ -n "$pid" ]; then
+    echo "Zeek PID $pid did not shut down"
+    exit 1
+fi
+
+exit 0