From c738701ffd295d84de5b584538e3fdee915f7e31 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Tue, 5 Jul 2011 18:36:31 -0700
Subject: [PATCH] Cleaning up some testing stuff.

- The Makefiles now run btest with "-f diag.log" so that diangnostics
  output will always be written into that file.

- Makefiles now hardcode path to btest to avoid picking up the wrong version
  if in PATH.

- The canonifier scripts now live in testing/scripts, and they are
  used from both btest/ and external/.

- There's a new diff-remove-uids scripts that removed connection UIDs
  for diffing. The external/* tests now use that by default.

- Timestamp removal now has its own script: diff-remove-timestamps.
  diff-canonifier calls that.

- All Makefile have a "brief" target that runs btest with -b.

- The higher-level directories have Makefile to call the subdirs.
---
 testing/Makefile                              |  8 +++++++
 testing/README                                |  2 ++
 testing/btest/.gitignore                      |  1 +
 testing/btest/Makefile                        | 15 +++++++------
 testing/btest/btest.cfg                       |  2 +-
 testing/external/.gitignore                   |  1 +
 testing/external/Makefile                     |  9 +++++++-
 testing/external/scripts/create-new-repo      |  5 +++--
 testing/external/scripts/diff-canonifier      |  5 -----
 testing/external/scripts/files/Makefile       |  3 ---
 testing/external/scripts/skel/.gitignore      |  1 +
 testing/external/scripts/skel/Makefile        | 11 ++++++++++
 .../scripts/{files => skel}/test.skeleton     |  0
 testing/external/subdir-btest.cfg             |  2 +-
 testing/scripts/diff-canonifier               |  5 +++++
 testing/scripts/diff-canonifier-external      |  5 +++++
 .../Scripts => scripts}/diff-remove-abspath   |  0
 .../diff-remove-timestamps}                   |  0
 testing/scripts/diff-remove-uids              | 21 +++++++++++++++++++
 .../doc/example-diff-canonifier.py            |  0
 20 files changed, 75 insertions(+), 21 deletions(-)
 create mode 100644 testing/Makefile
 delete mode 100755 testing/external/scripts/diff-canonifier
 delete mode 100644 testing/external/scripts/files/Makefile
 create mode 100644 testing/external/scripts/skel/.gitignore
 create mode 100644 testing/external/scripts/skel/Makefile
 rename testing/external/scripts/{files => skel}/test.skeleton (100%)
 create mode 100755 testing/scripts/diff-canonifier
 create mode 100755 testing/scripts/diff-canonifier-external
 rename testing/{btest/Scripts => scripts}/diff-remove-abspath (100%)
 rename testing/{btest/Scripts/diff-canonifier => scripts/diff-remove-timestamps} (100%)
 create mode 100755 testing/scripts/diff-remove-uids
 rename testing/{btest/Scripts => scripts}/doc/example-diff-canonifier.py (100%)

diff --git a/testing/Makefile b/testing/Makefile
new file mode 100644
index 0000000000..7f03a55f49
--- /dev/null
+++ b/testing/Makefile
@@ -0,0 +1,8 @@
+
+DIRS=btest external
+
+all:
+	@for repo in $(DIRS); do (cd $$repo && make ); done
+
+brief:
+	@for repo in $(DIRS); do (cd $$repo && make brief ); done
diff --git a/testing/README b/testing/README
index 75337f263a..ba407fcc67 100644
--- a/testing/README
+++ b/testing/README
@@ -11,3 +11,5 @@ operation:
         size, these are not included directly. See the README for more
         information. 
 
+    scripts/
+        Helpers scripts used by some tests.
diff --git a/testing/btest/.gitignore b/testing/btest/.gitignore
index d36977dc47..0c143f664e 100644
--- a/testing/btest/.gitignore
+++ b/testing/btest/.gitignore
@@ -1 +1,2 @@
 .tmp
+diag.log
diff --git a/testing/btest/Makefile b/testing/btest/Makefile
index ed042722b8..7489d761fb 100644
--- a/testing/btest/Makefile
+++ b/testing/btest/Makefile
@@ -1,14 +1,13 @@
 
+DIAG=diag.log
+BTEST=../../aux/btest/btest
+
 all:
     # Showing all tests.
-	@btest
+	@rm -f $(DIAG)
+	@$(BTEST) -f $(DIAG)
 
 brief:
     # Brief output showing only failed tests.
-	@btest -b
-
-brief-debug:
-    # Verbose output for failed tests, also recorded in test.log.
-	@rm -f test.log
-	@btest -b -d -f test.log
-	@echo Output in test.log
+	@rm -f $(DIAG)
+	@$(BTEST) -b -f $(DIAG)
diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg
index c46dd2d58a..f98d1e7155 100644
--- a/testing/btest/btest.cfg
+++ b/testing/btest/btest.cfg
@@ -13,6 +13,6 @@ LC_ALL=C
 PATH=%(testbase)s/../../build/src:%(testbase)s/../../aux/btest:%(default_path)s
 TEST_DIFF_CANONIFIER=%(testbase)s/Scripts/diff-canonifier
 TRACES=%(testbase)s/Traces
-SCRIPTS=%(testbase)s/Scripts
+SCRIPTS=%(testbase)s/../scripts
 DIST=%(testbase)s/../..
 BUILD=%(testbase)s/../../build
diff --git a/testing/external/.gitignore b/testing/external/.gitignore
index 444ff42e80..c55a0d04e3 100644
--- a/testing/external/.gitignore
+++ b/testing/external/.gitignore
@@ -1 +1,2 @@
 *.git
+diag.log
diff --git a/testing/external/Makefile b/testing/external/Makefile
index 5dce270206..cb215246c6 100644
--- a/testing/external/Makefile
+++ b/testing/external/Makefile
@@ -1,8 +1,15 @@
 
 PUBLIC_REPO=git://git.bro-ids.org/bro-testing
 
+DIAG=diag.log
+
 all:
-	@for repo in *.git; do (echo "==== $$repo" && cd $$repo && btest ); echo; done
+	@rm -f $(DIAG)
+	@for repo in *.git; do (cd $$repo && make ); done
+
+brief:
+	@rm -f $(DIAG)
+	@for repo in *.git; do (cd $$repo && make brief ); done
 
 init:
 	git clone $(PUBLIC_REPO) bro-testing.git
diff --git a/testing/external/scripts/create-new-repo b/testing/external/scripts/create-new-repo
index 2ed772b426..c1256d3e5d 100755
--- a/testing/external/scripts/create-new-repo
+++ b/testing/external/scripts/create-new-repo
@@ -34,8 +34,9 @@ done
 
 ln -s ../subdir-btest.cfg ./btest.cfg
 
-cp $cwd/`dirname $0`/files/test.skeleton tests
-cp $cwd/`dirname $0`/files/Makefile .
+cp $cwd/`dirname $0`/skel/test.skeleton tests
+cp $cwd/`dirname $0`/skel/Makefile .
+cp $cwd/`dirname $0`/skel/.gitignore .
 
 git add *
 
diff --git a/testing/external/scripts/diff-canonifier b/testing/external/scripts/diff-canonifier
deleted file mode 100755
index 063f1e4900..0000000000
--- a/testing/external/scripts/diff-canonifier
+++ /dev/null
@@ -1,5 +0,0 @@
-#! /usr/bin/env bash
-#
-# Replace anything which looks like timestamps with XXXs.
-
-sed 's/[0-9]\{10\}\.[0-9]\{2,8\}/XXXXXXXXXX.XXXXXX/g'
diff --git a/testing/external/scripts/files/Makefile b/testing/external/scripts/files/Makefile
deleted file mode 100644
index 7bbb5317f4..0000000000
--- a/testing/external/scripts/files/Makefile
+++ /dev/null
@@ -1,3 +0,0 @@
-
-all:
-	@btest
diff --git a/testing/external/scripts/skel/.gitignore b/testing/external/scripts/skel/.gitignore
new file mode 100644
index 0000000000..d49f35423e
--- /dev/null
+++ b/testing/external/scripts/skel/.gitignore
@@ -0,0 +1 @@
+diag.log
diff --git a/testing/external/scripts/skel/Makefile b/testing/external/scripts/skel/Makefile
new file mode 100644
index 0000000000..c535827937
--- /dev/null
+++ b/testing/external/scripts/skel/Makefile
@@ -0,0 +1,11 @@
+
+DIAG=diag.log
+BTEST=../../../aux/btest/btest
+
+all:
+	@rm -f $(DIAG)
+	@$(BTEST) -f $(DIAG)
+
+brief:
+	@rm -f $(DIAG)
+	@$(BTEST) -b -f $(DIAG)
diff --git a/testing/external/scripts/files/test.skeleton b/testing/external/scripts/skel/test.skeleton
similarity index 100%
rename from testing/external/scripts/files/test.skeleton
rename to testing/external/scripts/skel/test.skeleton
diff --git a/testing/external/subdir-btest.cfg b/testing/external/subdir-btest.cfg
index 9c7f9b7f88..18938ec985 100644
--- a/testing/external/subdir-btest.cfg
+++ b/testing/external/subdir-btest.cfg
@@ -11,7 +11,7 @@ BRO_SEED_FILE=%(testbase)s/../random.seed
 TZ=UTC
 LC_ALL=C
 PATH=%(testbase)s/../../../build/src:%(testbase)s/../../../aux/btest:%(default_path)s
-TEST_DIFF_CANONIFIER=%(testbase)s/../scripts/diff-canonifier
+TEST_DIFF_CANONIFIER=%(testbase)s/../../btest/Scripts/diff-canonifier-external
 TRACES=%(testbase)s/Traces
 SCRIPTS=%(testbase)s/../scripts
 DIST=%(testbase)s/../../..
diff --git a/testing/scripts/diff-canonifier b/testing/scripts/diff-canonifier
new file mode 100755
index 0000000000..3cb213a3f7
--- /dev/null
+++ b/testing/scripts/diff-canonifier
@@ -0,0 +1,5 @@
+#! /usr/bin/env bash
+#
+# Default canonifier used with the tests in testing/btest/*.
+
+`dirname $0`/diff-remove-timestamps
diff --git a/testing/scripts/diff-canonifier-external b/testing/scripts/diff-canonifier-external
new file mode 100755
index 0000000000..af0777facb
--- /dev/null
+++ b/testing/scripts/diff-canonifier-external
@@ -0,0 +1,5 @@
+#! /usr/bin/env bash
+#
+# Default canonifier used with the trace-based tests in testing/external/*.
+
+`dirname $0`/diff-remove-timestamps | `dirname $0`/diff-remove-uids
diff --git a/testing/btest/Scripts/diff-remove-abspath b/testing/scripts/diff-remove-abspath
similarity index 100%
rename from testing/btest/Scripts/diff-remove-abspath
rename to testing/scripts/diff-remove-abspath
diff --git a/testing/btest/Scripts/diff-canonifier b/testing/scripts/diff-remove-timestamps
similarity index 100%
rename from testing/btest/Scripts/diff-canonifier
rename to testing/scripts/diff-remove-timestamps
diff --git a/testing/scripts/diff-remove-uids b/testing/scripts/diff-remove-uids
new file mode 100755
index 0000000000..5cdd887b49
--- /dev/null
+++ b/testing/scripts/diff-remove-uids
@@ -0,0 +1,21 @@
+#! /usr/bin/awk -f
+#
+# A diff canonifier that removes all connection UIDs.
+
+BEGIN { IFS="\t"; OFS="\t"; }
+
+column > 0 {
+    $column = "XXXXXXXXXXX";
+    }
+
+/^#/ {
+    for ( i = 0; i < NF; ++i ) {
+        if ( $i == "uid" )
+            column = i - 1;
+        }
+    }
+
+{ print }
+
+
+
diff --git a/testing/btest/Scripts/doc/example-diff-canonifier.py b/testing/scripts/doc/example-diff-canonifier.py
similarity index 100%
rename from testing/btest/Scripts/doc/example-diff-canonifier.py
rename to testing/scripts/doc/example-diff-canonifier.py