From 442cb40db8ff618f78fc2210d88a5084b2ff8ba7 Mon Sep 17 00:00:00 2001
From: Johanna Amann <johanna@corelight.com>
Date: Fri, 4 Dec 2020 15:51:07 +0000
Subject: [PATCH] Fix binary baseline & line-end problem

By default all baslines are run through diff-remove-timestamp. On a BSD
sed implementation, this means that a newline is added to the end of the
file, if no newline was there originally. This behavior differs from GNU
sed, which does not add a newline.

In this commit we unify this behavior by always adding a newline, even
when using GNU sed. This commit also disables the canonifier for a bunch
of binary baselines, so we do not have to change them.
---
 .../Baseline/bifs.enable_raw_output/output      |   2 +-
 testing/btest/Baseline/bifs.piped_exec/test.txt | Bin 124 -> 125 bytes
 .../Baseline/language.raw_output_attr/output    |   2 +-
 .../btest/scripts/base/files/extract/limit.zeek |   6 +++---
 .../base/frameworks/file-analysis/http/get.zeek |   4 ++--
 .../file-analysis/http/multipart.zeek           |   8 ++++----
 .../frameworks/file-analysis/http/pipeline.zeek |  10 +++++-----
 .../frameworks/file-analysis/http/post.zeek     |   4 ++--
 .../base/frameworks/file-analysis/irc.zeek      |   2 +-
 .../base/frameworks/input/raw/executestdin.zeek |   2 +-
 .../base/protocols/http/content-range-gap.zeek  |   2 +-
 testing/scripts/diff-remove-timestamps          |   5 +++--
 12 files changed, 24 insertions(+), 23 deletions(-)

diff --git a/testing/btest/Baseline/bifs.enable_raw_output/output b/testing/btest/Baseline/bifs.enable_raw_output/output
index 9a4829574f..a517d4680d 100644
--- a/testing/btest/Baseline/bifs.enable_raw_output/output
+++ b/testing/btest/Baseline/bifs.enable_raw_output/output
@@ -1,2 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-helloXworldhi
\ No newline at end of file
+helloXworldhi
diff --git a/testing/btest/Baseline/bifs.piped_exec/test.txt b/testing/btest/Baseline/bifs.piped_exec/test.txt
index d245b378cc9eb01813af7787242251df4ce1ccfd..d2668b97c12161ddc55fc49dc5f09f53bccbed73 100644
GIT binary patch
delta 6
Ncmb=aolwKb1po;j0ucZJ

delta 4
Lcmb=enNR}&1qcEL

diff --git a/testing/btest/Baseline/language.raw_output_attr/output b/testing/btest/Baseline/language.raw_output_attr/output
index 9a4829574f..a517d4680d 100644
--- a/testing/btest/Baseline/language.raw_output_attr/output
+++ b/testing/btest/Baseline/language.raw_output_attr/output
@@ -1,2 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-helloXworldhi
\ No newline at end of file
+helloXworldhi
diff --git a/testing/btest/scripts/base/files/extract/limit.zeek b/testing/btest/scripts/base/files/extract/limit.zeek
index e676d0ebe0..e384dd0c9b 100644
--- a/testing/btest/scripts/base/files/extract/limit.zeek
+++ b/testing/btest/scripts/base/files/extract/limit.zeek
@@ -1,12 +1,12 @@
 # @TEST-EXEC: zeek -b -r $TRACES/ftp/retr.trace %INPUT max_extract=3000 efname=1
-# @TEST-EXEC: btest-diff extract_files/1
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff extract_files/1
 # @TEST-EXEC: btest-diff 1.out
 # @TEST-EXEC: zeek -b -r $TRACES/ftp/retr.trace %INPUT max_extract=3000 efname=2 double_it=T
-# @TEST-EXEC: btest-diff extract_files/2
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff extract_files/2
 # @TEST-EXEC: btest-diff 2.out
 # @TEST-EXEC: btest-diff files.log
 # @TEST-EXEC: zeek -b -r $TRACES/ftp/retr.trace %INPUT max_extract=7000 efname=3 unlimit_it=T
-# @TEST-EXEC: btest-diff extract_files/3
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff extract_files/3
 # @TEST-EXEC: btest-diff 3.out
 
 @load base/files/extract
diff --git a/testing/btest/scripts/base/frameworks/file-analysis/http/get.zeek b/testing/btest/scripts/base/frameworks/file-analysis/http/get.zeek
index 8112be149c..2204454e55 100644
--- a/testing/btest/scripts/base/frameworks/file-analysis/http/get.zeek
+++ b/testing/btest/scripts/base/frameworks/file-analysis/http/get.zeek
@@ -2,8 +2,8 @@
 # @TEST-EXEC: zeek -b -r $TRACES/http/get-gzip.trace $SCRIPTS/file-analysis-test.zeek %INPUT c=2 >get-gzip.out
 # @TEST-EXEC: btest-diff get.out
 # @TEST-EXEC: btest-diff get-gzip.out
-# @TEST-EXEC: btest-diff 1-file
-# @TEST-EXEC: btest-diff 2-file
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff 1-file
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff 2-file
 
 @load base/protocols/http
 
diff --git a/testing/btest/scripts/base/frameworks/file-analysis/http/multipart.zeek b/testing/btest/scripts/base/frameworks/file-analysis/http/multipart.zeek
index 206eecb285..0730ad7992 100644
--- a/testing/btest/scripts/base/frameworks/file-analysis/http/multipart.zeek
+++ b/testing/btest/scripts/base/frameworks/file-analysis/http/multipart.zeek
@@ -1,9 +1,9 @@
 # @TEST-EXEC: zeek -b -r $TRACES/http/multipart.trace $SCRIPTS/file-analysis-test.zeek %INPUT >out
 # @TEST-EXEC: btest-diff out
-# @TEST-EXEC: btest-diff 1-file
-# @TEST-EXEC: btest-diff 2-file
-# @TEST-EXEC: btest-diff 3-file
-# @TEST-EXEC: btest-diff 4-file
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff 1-file
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff 2-file
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff 3-file
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff 4-file
 
 @load base/protocols/http
 
diff --git a/testing/btest/scripts/base/frameworks/file-analysis/http/pipeline.zeek b/testing/btest/scripts/base/frameworks/file-analysis/http/pipeline.zeek
index c5d4db7cff..2cd61c0aae 100644
--- a/testing/btest/scripts/base/frameworks/file-analysis/http/pipeline.zeek
+++ b/testing/btest/scripts/base/frameworks/file-analysis/http/pipeline.zeek
@@ -1,10 +1,10 @@
 # @TEST-EXEC: zeek -b -r $TRACES/http/pipelined-requests.trace $SCRIPTS/file-analysis-test.zeek %INPUT >out
 # @TEST-EXEC: btest-diff out
-# @TEST-EXEC: btest-diff 1-file
-# @TEST-EXEC: btest-diff 2-file
-# @TEST-EXEC: btest-diff 3-file
-# @TEST-EXEC: btest-diff 4-file
-# @TEST-EXEC: btest-diff 5-file
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff 1-file
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff 2-file
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff 3-file
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff 4-file
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff 5-file
 
 @load base/protocols/http
 
diff --git a/testing/btest/scripts/base/frameworks/file-analysis/http/post.zeek b/testing/btest/scripts/base/frameworks/file-analysis/http/post.zeek
index 54e3e31313..6e9084804a 100644
--- a/testing/btest/scripts/base/frameworks/file-analysis/http/post.zeek
+++ b/testing/btest/scripts/base/frameworks/file-analysis/http/post.zeek
@@ -1,7 +1,7 @@
 # @TEST-EXEC: zeek -b -r $TRACES/http/post.trace $SCRIPTS/file-analysis-test.zeek %INPUT >out
 # @TEST-EXEC: btest-diff out
-# @TEST-EXEC: btest-diff 1-file
-# @TEST-EXEC: btest-diff 2-file
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff 1-file
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff 2-file
 
 @load base/protocols/http
 
diff --git a/testing/btest/scripts/base/frameworks/file-analysis/irc.zeek b/testing/btest/scripts/base/frameworks/file-analysis/irc.zeek
index 07c1e52845..71341dbfd9 100644
--- a/testing/btest/scripts/base/frameworks/file-analysis/irc.zeek
+++ b/testing/btest/scripts/base/frameworks/file-analysis/irc.zeek
@@ -1,6 +1,6 @@
 # @TEST-EXEC: zeek -b -r $TRACES/irc-dcc-send.trace $SCRIPTS/file-analysis-test.zeek %INPUT >out
 # @TEST-EXEC: btest-diff out
-# @TEST-EXEC: btest-diff thefile
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff thefile
 
 @load base/protocols/irc
 
diff --git a/testing/btest/scripts/base/frameworks/input/raw/executestdin.zeek b/testing/btest/scripts/base/frameworks/input/raw/executestdin.zeek
index 0beb8bca20..496e5ae3fd 100644
--- a/testing/btest/scripts/base/frameworks/input/raw/executestdin.zeek
+++ b/testing/btest/scripts/base/frameworks/input/raw/executestdin.zeek
@@ -1,6 +1,6 @@
 # @TEST-EXEC: btest-bg-run zeek zeek -b %INPUT
 # @TEST-EXEC: btest-bg-wait 15
-# @TEST-EXEC: btest-diff test.txt
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff test.txt
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out
 
 redef exit_only_after_terminate = T;
diff --git a/testing/btest/scripts/base/protocols/http/content-range-gap.zeek b/testing/btest/scripts/base/protocols/http/content-range-gap.zeek
index 81d7fe042d..1bb0916b40 100644
--- a/testing/btest/scripts/base/protocols/http/content-range-gap.zeek
+++ b/testing/btest/scripts/base/protocols/http/content-range-gap.zeek
@@ -1,5 +1,5 @@
 # @TEST-EXEC: zeek -b -r $TRACES/http/content-range-gap.trace %INPUT
-# @TEST-EXEC: btest-diff extract_files/thefile
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff extract_files/thefile
 
 @load base/protocols/http
 @load base/files/extract
diff --git a/testing/scripts/diff-remove-timestamps b/testing/scripts/diff-remove-timestamps
index f7d839c4d6..5c5956a0ed 100755
--- a/testing/scripts/diff-remove-timestamps
+++ b/testing/scripts/diff-remove-timestamps
@@ -9,5 +9,6 @@ else
    sed="sed -E"
 fi
 
-$sed 's/(^|[^0-9])([0-9]{9,10}\.[0-9]{1,8})/\1XXXXXXXXXX.XXXXXX/g' | \
-$sed 's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g'
+# $a\ adds a newline to the end of the file if it does not exist.
+# This unifies the behavior of BSD and GNU sed
+$sed -e 's/(^|[^0-9])([0-9]{9,10}\.[0-9]{1,8})/\1XXXXXXXXXX.XXXXXX/g' -e  's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g' -e '$a\'