Merge remote-tracking branch 'origin/topic/jsiwek/brofiler'

* origin/topic/jsiwek/brofiler:
  Fix superfluous/duplicate data getting in to testing coverage log.
  Add "# @no-test" tag to blacklist statements from test coverage analysis.
  Test coverage integration for external tests and complete suite.
  Integrate Bro script coverage profiling with the btest suite.
  Add simple profiling class to accumulate Stmt usage stats across runs.

Renaming environment variable BROFILER_FILE to BRO_PROFILER_FILE for
consistency. Yeah, I know, such a nice name! :)

testing/.gitignore

@@ -0,0 +1 @@
+coverage.log

testing/Makefile

@@ -3,6 +3,11 @@ DIRS=btest external
 
 all:
 	@for repo in $(DIRS); do (cd $$repo && make ); done
+	@cp btest/coverage.log `mktemp brocov.tmp.XXX`
+	@for f in external/*/coverage.log; do cp $$f `mktemp brocov.tmp.XXX`; done
+	@echo "Complete test suite code coverage:"
+	@./scripts/coverage-calc "brocov.tmp.*" coverage.log `pwd`/../scripts
+	@rm -f brocov.tmp.*
 
 brief:
 	@for repo in $(DIRS); do (cd $$repo && make brief ); done

testing/btest/.gitignore

@@ -1,2 +1,3 @@
 .tmp
 diag.log
+coverage.log

testing/btest/Baseline/coverage.coverage-blacklist/output

@@ -0,0 +1,5 @@
+1	/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 13	print cover me;
+1	/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 17	print always executed;
+0	/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 26	print also impossible, but included in code coverage analysis;
+1	/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 29	print success;
+1	/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 5	print first;

testing/btest/Makefile

@@ -5,7 +5,9 @@ BTEST=../../aux/btest/btest
 all:
     # Showing all tests.
 	@rm -f $(DIAG)
+	@rm -f .tmp/script-coverage*
 	@$(BTEST) -f $(DIAG)
+	@../scripts/coverage-calc ".tmp/script-coverage*" coverage.log `pwd`/../../scripts
 
 brief:
     # Brief output showing only failed tests.

testing/btest/btest.cfg

@@ -10,9 +10,12 @@ BROPATH=`bash -c %(testbase)s/../../build/bro-path-dev`
 BRO_SEED_FILE=%(testbase)s/random.seed
 TZ=UTC
 LC_ALL=C
-PATH=%(testbase)s/../../build/src:%(testbase)s/../../aux/btest:%(default_path)s
+BTEST_PATH=%(testbase)s/../../aux/btest
+PATH=%(testbase)s/../../build/src:%(testbase)s/../scripts:%(testbase)s/../../aux/btest:%(default_path)s
 TRACES=%(testbase)s/Traces
 SCRIPTS=%(testbase)s/../scripts
 DIST=%(testbase)s/../..
 BUILD=%(testbase)s/../../build
 TEST_DIFF_CANONIFIER=$SCRIPTS/diff-canonifier
+TMPDIR=%(testbase)s/.tmp
+BROFILER_FILE=%(testbase)s/.tmp/script-coverage

testing/btest/coverage/coverage-blacklist.bro

@@ -0,0 +1,29 @@
+# @TEST-EXEC: BROFILER_FILE=coverage bro -b %INPUT
+# @TEST-EXEC: grep %INPUT coverage | sort -k2 >output
+# @TEST-EXEC: btest-diff output
+
+print "first";
+
+if ( F )
+	{ # @no-test
+	print "hello";
+	print "world";
+	}
+
+print "cover me";
+
+if ( T )
+	{
+	print "always executed";
+	}
+
+print "don't cover me"; # @no-test
+
+if ( 0 + 0 == 1 ) print "impossible"; # @no-test
+
+if ( 1 == 0 )
+	{
+	print "also impossible, but included in code coverage analysis";
+	}
+
+print "success";

testing/external/subdir-btest.cfg

@@ -17,3 +17,4 @@ TRACES=%(testbase)s/Traces
 SCRIPTS=%(testbase)s/../scripts
 DIST=%(testbase)s/../../..
 BUILD=%(testbase)s/../../../build
+BROFILER_FILE=%(testbase)s/.tmp/script-coverage

testing/scripts/btest-bg-run

@@ -0,0 +1,7 @@
+#! /usr/bin/env bash
+
+# This is a wrapper script to btest's real btest-bg-run.  It's used
+# when collecting Bro script coverage statistics so that two independent
+# Bro processing don't try to write those usage statistics to the same file.
+
+BROFILER_FILE=`mktemp -t script-coverage` $BTEST_PATH/btest-bg-run $@

testing/scripts/coverage-calc

@@ -0,0 +1,51 @@
+#! /usr/bin/env python
+
+# This script aggregates many files containing Bro script coverage information
+# into a single file and reports the overall coverage information. Usage:
+#
+#   coverage-calc <quoted glob of filenames> <output file> <script dir>
+#
+# The last argument is used to point to a root directory containing all
+# the Bro distribution's scripts.  It's used to cull out test scripts
+# that are not part of the distribution and which should not count towrads
+# the coverage calculation.
+
+import os
+import sys
+import glob
+
+stats = {}
+inputglob = sys.argv[1]
+outputfile = sys.argv[2]
+scriptdir = os.path.abspath(sys.argv[3])
+
+for filename in glob.glob(inputglob):
+    with open(filename, 'r') as f:
+        for line in f.read().splitlines():
+            parts = line.split("\t")
+            exec_count = int(parts[0])
+            location = os.path.normpath(parts[1])
+            # ignore scripts that don't appear to be part of Bro distribution
+            if not location.startswith(scriptdir):
+                continue
+            desc = parts[2]
+            # keying by location + desc may result in duplicate data
+            # as some descs change as a result of differing configurations
+            # producing record (re)definitions
+            key = location
+            if key in stats:
+                stats[key][0] += exec_count
+            else:
+                stats[key] = [exec_count, location, desc]
+
+with open(outputfile, 'w') as f:
+    for k in sorted(stats, key=lambda i: stats[i][1]):
+        f.write("%s\t%s\t%s\n" % (stats[k][0], stats[k][1], stats[k][2]))
+
+num_covered = 0
+for k in stats:
+    if stats[k][0] > 0:
+        num_covered += 1
+
+if len(stats) > 0:
+    print "%s/%s (%.1f%%) Bro script statements covered." % (num_covered, len(stats), float(num_covered)/len(stats)*100)