Fix binary baseline & line-end problem

By default all baslines are run through diff-remove-timestamp. On a BSD
sed implementation, this means that a newline is added to the end of the
file, if no newline was there originally. This behavior differs from GNU
sed, which does not add a newline.

In this commit we unify this behavior by always adding a newline, even
when using GNU sed. This commit also disables the canonifier for a bunch
of binary baselines, so we do not have to change them.

testing/btest/scripts/base/files/extract/limit.zeek

@@ -1,12 +1,12 @@
 # @TEST-EXEC: zeek -b -r $TRACES/ftp/retr.trace %INPUT max_extract=3000 efname=1
-# @TEST-EXEC: btest-diff extract_files/1
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff extract_files/1
 # @TEST-EXEC: btest-diff 1.out
 # @TEST-EXEC: zeek -b -r $TRACES/ftp/retr.trace %INPUT max_extract=3000 efname=2 double_it=T
-# @TEST-EXEC: btest-diff extract_files/2
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff extract_files/2
 # @TEST-EXEC: btest-diff 2.out
 # @TEST-EXEC: btest-diff files.log
 # @TEST-EXEC: zeek -b -r $TRACES/ftp/retr.trace %INPUT max_extract=7000 efname=3 unlimit_it=T
-# @TEST-EXEC: btest-diff extract_files/3
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff extract_files/3
 # @TEST-EXEC: btest-diff 3.out
 
 @load base/files/extract

testing/btest/scripts/base/frameworks/file-analysis/http/get.zeek

@@ -2,8 +2,8 @@
 # @TEST-EXEC: zeek -b -r $TRACES/http/get-gzip.trace $SCRIPTS/file-analysis-test.zeek %INPUT c=2 >get-gzip.out
 # @TEST-EXEC: btest-diff get.out
 # @TEST-EXEC: btest-diff get-gzip.out
-# @TEST-EXEC: btest-diff 1-file
-# @TEST-EXEC: btest-diff 2-file
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff 1-file
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff 2-file
 
 @load base/protocols/http
 

testing/btest/scripts/base/frameworks/file-analysis/http/multipart.zeek

@@ -1,9 +1,9 @@
 # @TEST-EXEC: zeek -b -r $TRACES/http/multipart.trace $SCRIPTS/file-analysis-test.zeek %INPUT >out
 # @TEST-EXEC: btest-diff out
-# @TEST-EXEC: btest-diff 1-file
-# @TEST-EXEC: btest-diff 2-file
-# @TEST-EXEC: btest-diff 3-file
-# @TEST-EXEC: btest-diff 4-file
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff 1-file
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff 2-file
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff 3-file
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff 4-file
 
 @load base/protocols/http
 

testing/btest/scripts/base/frameworks/file-analysis/http/pipeline.zeek

@@ -1,10 +1,10 @@
 # @TEST-EXEC: zeek -b -r $TRACES/http/pipelined-requests.trace $SCRIPTS/file-analysis-test.zeek %INPUT >out
 # @TEST-EXEC: btest-diff out
-# @TEST-EXEC: btest-diff 1-file
-# @TEST-EXEC: btest-diff 2-file
-# @TEST-EXEC: btest-diff 3-file
-# @TEST-EXEC: btest-diff 4-file
-# @TEST-EXEC: btest-diff 5-file
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff 1-file
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff 2-file
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff 3-file
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff 4-file
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff 5-file
 
 @load base/protocols/http
 

testing/btest/scripts/base/frameworks/file-analysis/http/post.zeek

@@ -1,7 +1,7 @@
 # @TEST-EXEC: zeek -b -r $TRACES/http/post.trace $SCRIPTS/file-analysis-test.zeek %INPUT >out
 # @TEST-EXEC: btest-diff out
-# @TEST-EXEC: btest-diff 1-file
-# @TEST-EXEC: btest-diff 2-file
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff 1-file
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff 2-file
 
 @load base/protocols/http
 

testing/btest/scripts/base/frameworks/file-analysis/irc.zeek

@@ -1,6 +1,6 @@
 # @TEST-EXEC: zeek -b -r $TRACES/irc-dcc-send.trace $SCRIPTS/file-analysis-test.zeek %INPUT >out
 # @TEST-EXEC: btest-diff out
-# @TEST-EXEC: btest-diff thefile
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff thefile
 
 @load base/protocols/irc
 

testing/btest/scripts/base/frameworks/input/raw/executestdin.zeek

@@ -1,6 +1,6 @@
 # @TEST-EXEC: btest-bg-run zeek zeek -b %INPUT
 # @TEST-EXEC: btest-bg-wait 15
-# @TEST-EXEC: btest-diff test.txt
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff test.txt
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out
 
 redef exit_only_after_terminate = T;

testing/btest/scripts/base/protocols/http/content-range-gap.zeek

@@ -1,5 +1,5 @@
 # @TEST-EXEC: zeek -b -r $TRACES/http/content-range-gap.trace %INPUT
-# @TEST-EXEC: btest-diff extract_files/thefile
+# @TEST-EXEC: env -u TEST_DIFF_CANONIFIER btest-diff extract_files/thefile
 
 @load base/protocols/http
 @load base/files/extract

testing/scripts/diff-remove-timestamps

@@ -9,5 +9,6 @@ else
    sed="sed -E"
 fi
 
-$sed 's/(^|[^0-9])([0-9]{9,10}\.[0-9]{1,8})/\1XXXXXXXXXX.XXXXXX/g' | \
-$sed 's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g'
+# $a\ adds a newline to the end of the file if it does not exist.
+# This unifies the behavior of BSD and GNU sed
+$sed -e 's/(^|[^0-9])([0-9]{9,10}\.[0-9]{1,8})/\1XXXXXXXXXX.XXXXXX/g' -e  's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g' -e '$a\'