From 9aefeec4ce8f4a3952dc9874a99b570310fe8b39 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 11 Jan 2012 16:30:25 -0600 Subject: [PATCH] Integrate Bro script coverage profiling with the btest suite. --- src/Brofiler.cc | 20 +++++++-------- src/Brofiler.h | 8 ++++-- testing/btest/.gitignore | 1 + testing/btest/Makefile | 2 ++ testing/btest/btest.cfg | 5 +++- testing/scripts/btest-bg-run | 7 ++++++ testing/scripts/coverage-calc | 46 +++++++++++++++++++++++++++++++++++ 7 files changed, 75 insertions(+), 14 deletions(-) create mode 100755 testing/scripts/btest-bg-run create mode 100755 testing/scripts/coverage-calc diff --git a/src/Brofiler.cc b/src/Brofiler.cc index fc23f10a7d..8db5861d20 100644 --- a/src/Brofiler.cc +++ b/src/Brofiler.cc @@ -13,16 +13,12 @@ Brofiler::~Brofiler() { } -void Brofiler::ReadStats() +bool Brofiler::ReadStats() { char* bf = getenv("BROFILER_FILE"); - if ( ! bf ) return; + if ( ! bf ) return false; FILE* f = fopen(bf, "r"); - if ( ! f ) - { - fprintf(stderr, "Failed to open Brofiler file '%s' for reading\n", bf); - return; - } + if ( ! f ) return false; char line[16384]; string delimiter; @@ -40,18 +36,19 @@ void Brofiler::ReadStats() } fclose(f); + return true; } -void Brofiler::WriteStats() +bool Brofiler::WriteStats() { char* bf = getenv("BROFILER_FILE"); - if ( ! bf ) return; + if ( ! bf ) return false; FILE* f = fopen(bf, "w"); if ( ! f ) { - fprintf(stderr, "Failed to open Brofiler file '%s' for writing\n", bf); - return; + reporter->Error("Failed to open Brofiler file '%s' for writing\n", bf); + return false; } for ( list::const_iterator it = stmts.begin(); @@ -78,5 +75,6 @@ void Brofiler::WriteStats() } fclose(f); + return true; } diff --git a/src/Brofiler.h b/src/Brofiler.h index 6ded906698..0a284c62c8 100644 --- a/src/Brofiler.h +++ b/src/Brofiler.h @@ -18,15 +18,19 @@ public: /** * Imports Bro script Stmt usage information from file pointed to by * environment variable BROFILER_FILE. + * + * @return: true if usage info was read, otherwise false. */ - void ReadStats(); + bool ReadStats(); /** * Combines usage stats from current run with any read from ReadStats(), * then writes information to file pointed to by environment variable * BROFILER_FILE. + * + * @return: true when usage info is written, otherwise false. */ - void WriteStats(); + bool WriteStats(); void SetDelim(char d) { delim = d; } diff --git a/testing/btest/.gitignore b/testing/btest/.gitignore index 0c143f664e..5282177d90 100644 --- a/testing/btest/.gitignore +++ b/testing/btest/.gitignore @@ -1,2 +1,3 @@ .tmp diag.log +coverage.log diff --git a/testing/btest/Makefile b/testing/btest/Makefile index 7489d761fb..2ebd66edd2 100644 --- a/testing/btest/Makefile +++ b/testing/btest/Makefile @@ -5,7 +5,9 @@ BTEST=../../aux/btest/btest all: # Showing all tests. @rm -f $(DIAG) + @rm -f .tmp/script-coverage* @$(BTEST) -f $(DIAG) + @../scripts/coverage-calc ".tmp/script-coverage*" coverage.log `pwd` brief: # Brief output showing only failed tests. diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 7d8283587c..b37b3063be 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -10,9 +10,12 @@ BROPATH=`bash -c %(testbase)s/../../build/bro-path-dev` BRO_SEED_FILE=%(testbase)s/random.seed TZ=UTC LC_ALL=C -PATH=%(testbase)s/../../build/src:%(testbase)s/../../aux/btest:%(default_path)s +BTEST_PATH=%(testbase)s/../../aux/btest +PATH=%(testbase)s/../../build/src:%(testbase)s/../scripts:%(testbase)s/../../aux/btest:%(default_path)s TRACES=%(testbase)s/Traces SCRIPTS=%(testbase)s/../scripts DIST=%(testbase)s/../.. BUILD=%(testbase)s/../../build TEST_DIFF_CANONIFIER=$SCRIPTS/diff-canonifier +TMPDIR=%(testbase)s/.tmp +BROFILER_FILE=%(testbase)s/.tmp/script-coverage diff --git a/testing/scripts/btest-bg-run b/testing/scripts/btest-bg-run new file mode 100755 index 0000000000..462ae23fa2 --- /dev/null +++ b/testing/scripts/btest-bg-run @@ -0,0 +1,7 @@ +#! /usr/bin/env bash + +# This is a wrapper script to btest's real btest-bg-run. It's used +# when collecting Bro script coverage statistics so that two independent +# Bro processing don't try to write those usage statistics to the same file. + +BROFILER_FILE=`mktemp -t script-coverage` $BTEST_PATH/btest-bg-run $@ diff --git a/testing/scripts/coverage-calc b/testing/scripts/coverage-calc new file mode 100755 index 0000000000..a146667595 --- /dev/null +++ b/testing/scripts/coverage-calc @@ -0,0 +1,46 @@ +#! /usr/bin/env python + +# This script aggregates many files containing Bro script coverage information +# into a single file and reports the overall coverage information. Usage: +# +# coverage-calc +# +# The last argument is used to ignore Bro scripts that are part of the test +# suite itself as those should not count towards the coverage calculation. + +import os +import sys +import glob + +stats = {} +inputglob = sys.argv[1] +outputfile = sys.argv[2] +ignoredir = os.path.abspath(sys.argv[3]) + +for filename in glob.glob(inputglob): + with open(filename, 'r') as f: + for line in f.read().splitlines(): + parts = line.split("\t") + exec_count = int(parts[0]) + location = os.path.normpath(parts[1]) + # ignore scripts that don't appear to be part of Bro distribution + if location.startswith(ignoredir) or not location.startswith("/"): + continue + desc = parts[2] + key = location + desc + if key in stats: + stats[key][0] += exec_count + else: + stats[key] = [exec_count, location, desc] + +with open(outputfile, 'w') as f: + for k in sorted(stats, key=lambda i: stats[i][1]): + f.write("%s\t%s\t%s\n" % (stats[k][0], stats[k][1], stats[k][2])) + +num_covered = 0 +for k in stats: + if stats[k][0] > 0: + num_covered += 1 + +if len(stats) > 0: + print "%s/%s (%.1f%%) Bro script statements covered." % (num_covered, len(stats), float(num_covered)/len(stats)*100)