Integrate Bro script coverage profiling with the btest suite.

src/Brofiler.cc

@@ -13,16 +13,12 @@ Brofiler::~Brofiler()
 	{
 	}
 
-void Brofiler::ReadStats()
+bool Brofiler::ReadStats()
 	{
 	char* bf = getenv("BROFILER_FILE");
-	if ( ! bf ) return;
+	if ( ! bf ) return false;
 	FILE* f = fopen(bf, "r");
-	if ( ! f )
-		{
-		fprintf(stderr, "Failed to open Brofiler file '%s' for reading\n", bf);
-		return;
-		}
+	if ( ! f ) return false;
 
 	char line[16384];
 	string delimiter;
@@ -40,18 +36,19 @@ void Brofiler::ReadStats()
 		}
 
 	fclose(f);
+	return true;
 	}
 
-void Brofiler::WriteStats()
+bool Brofiler::WriteStats()
 	{
 	char* bf = getenv("BROFILER_FILE");
-	if ( ! bf ) return;
+	if ( ! bf ) return false;
 
 	FILE* f = fopen(bf, "w");
 	if ( ! f )
 		{
-		fprintf(stderr, "Failed to open Brofiler file '%s' for writing\n", bf);
-		return;
+		reporter->Error("Failed to open Brofiler file '%s' for writing\n", bf);
+		return false;
 		}
 
 	for ( list<const Stmt*>::const_iterator it = stmts.begin();
@@ -78,5 +75,6 @@ void Brofiler::WriteStats()
 		}
 
 	fclose(f);
+	return true;
 	}
 

src/Brofiler.h

@@ -18,15 +18,19 @@ public:
 	/**
 	 * Imports Bro script Stmt usage information from file pointed to by
 	 * environment variable BROFILER_FILE.
+	 *
+	 * @return: true if usage info was read, otherwise false.
 	 */
-	void ReadStats();
+	bool ReadStats();
 
 	/**
 	 * Combines usage stats from current run with any read from ReadStats(),
 	 * then writes information to file pointed to by environment variable
 	 * BROFILER_FILE.
+	 *
+	 * @return: true when usage info is written, otherwise false.
 	 */
-	void WriteStats();
+	bool WriteStats();
 
 	void SetDelim(char d) { delim = d; }
 

testing/btest/.gitignore

@@ -1,2 +1,3 @@
 .tmp
 diag.log
+coverage.log

testing/btest/Makefile

@@ -5,7 +5,9 @@ BTEST=../../aux/btest/btest
 all:
     # Showing all tests.
 	@rm -f $(DIAG)
+	@rm -f .tmp/script-coverage*
 	@$(BTEST) -f $(DIAG)
+	@../scripts/coverage-calc ".tmp/script-coverage*" coverage.log `pwd`
 
 brief:
     # Brief output showing only failed tests.

testing/btest/btest.cfg

@@ -10,9 +10,12 @@ BROPATH=`bash -c %(testbase)s/../../build/bro-path-dev`
 BRO_SEED_FILE=%(testbase)s/random.seed
 TZ=UTC
 LC_ALL=C
-PATH=%(testbase)s/../../build/src:%(testbase)s/../../aux/btest:%(default_path)s
+BTEST_PATH=%(testbase)s/../../aux/btest
+PATH=%(testbase)s/../../build/src:%(testbase)s/../scripts:%(testbase)s/../../aux/btest:%(default_path)s
 TRACES=%(testbase)s/Traces
 SCRIPTS=%(testbase)s/../scripts
 DIST=%(testbase)s/../..
 BUILD=%(testbase)s/../../build
 TEST_DIFF_CANONIFIER=$SCRIPTS/diff-canonifier
+TMPDIR=%(testbase)s/.tmp
+BROFILER_FILE=%(testbase)s/.tmp/script-coverage

testing/scripts/btest-bg-run

@@ -0,0 +1,7 @@
+#! /usr/bin/env bash
+
+# This is a wrapper script to btest's real btest-bg-run.  It's used
+# when collecting Bro script coverage statistics so that two independent
+# Bro processing don't try to write those usage statistics to the same file.
+
+BROFILER_FILE=`mktemp -t script-coverage` $BTEST_PATH/btest-bg-run $@

testing/scripts/coverage-calc

@@ -0,0 +1,46 @@
+#! /usr/bin/env python
+
+# This script aggregates many files containing Bro script coverage information
+# into a single file and reports the overall coverage information. Usage:
+#
+#   coverage-calc <quoted glob of filenames> <output file> <ignored script dir>
+#
+# The last argument is used to ignore Bro scripts that are part of the test
+# suite itself as those should not count towards the coverage calculation.
+
+import os
+import sys
+import glob
+
+stats = {}
+inputglob = sys.argv[1]
+outputfile = sys.argv[2]
+ignoredir = os.path.abspath(sys.argv[3])
+
+for filename in glob.glob(inputglob):
+    with open(filename, 'r') as f:
+        for line in f.read().splitlines():
+            parts = line.split("\t")
+            exec_count = int(parts[0])
+            location = os.path.normpath(parts[1])
+            # ignore scripts that don't appear to be part of Bro distribution
+            if location.startswith(ignoredir) or not location.startswith("/"):
+                continue
+            desc = parts[2]
+            key = location + desc
+            if key in stats:
+                stats[key][0] += exec_count
+            else:
+                stats[key] = [exec_count, location, desc]
+
+with open(outputfile, 'w') as f:
+    for k in sorted(stats, key=lambda i: stats[i][1]):
+        f.write("%s\t%s\t%s\n" % (stats[k][0], stats[k][1], stats[k][2]))
+
+num_covered = 0
+for k in stats:
+    if stats[k][0] > 0:
+        num_covered += 1
+
+if len(stats) > 0:
+    print "%s/%s (%.1f%%) Bro script statements covered." % (num_covered, len(stats), float(num_covered)/len(stats)*100)