Add a script to cleanup the file listing before sending to Coveralls

GCC has a "feature" where it doesn't realize that files in the
src/zeek/... tree are the same as files in the normal 'src/' tree. This
leads the coverage script to send duplicates to Coveralls and pollute
the display with them. The new script scrapes the intermediate output
from lcov and de-duplicates anything in src/zeek/ to be just from src/.

testing/coverage/coverage_cleanup.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+import sys
+
+if len(sys.argv) != 2:
+    print("Expected one argument containing the file to clean")
+    sys.exit(-1)
+
+with open(sys.argv[1], 'r') as f:
+
+    files = {}
+    cur_file = ''
+    lines = f.readlines()
+
+    for line in lines:
+
+        if line == 'end_of_record':
+            cur_file = ''
+            continue
+
+        parts = line.split(':', 1)
+        if parts[0] == 'SF':
+            cur_file = parts[1].strip()
+            while cur_file.find('src/zeek/') != -1:
+                cur_file = cur_file.replace('src/zeek/', 'src/', 1)
+
+            if cur_file not in files:
+                files[cur_file] = {}
+        elif parts[0] == 'DA':
+            da_parts = parts[1].split(',')
+            line = int(da_parts[0])
+            count = int(da_parts[1])
+
+            if files[cur_file].get(line, 0) == 0:
+                files[cur_file][line] = count
+
+    for name in files:
+
+        print('TN:')
+        print('SF:{}'.format(name))
+
+        das = list(files[name].keys())
+        das.sort()
+
+        for da in das:
+            print('DA:{},{}'.format(da, files[name][da]))
+        print('end_of_record')

testing/coverage/lcov_html.sh

@@ -93,7 +93,7 @@ fi
 
 # Files and directories that will be removed from the counts in step 5. Directories
 # need to be surrounded by escaped wildcards.
-REMOVE_TARGETS="*.yy *.ll *.y *.l \*/bro.dir/\* *.bif \*/zeek.dir/\* \*/src/3rdparty/\* \*/auxil/\*"
+REMOVE_TARGETS="*.yy *.ll *.y *.l \*/bro.dir/\* *.bif \*/zeek.dir/\* \*/src/3rdparty/\* \*/src/zeek/3rdparty/\* \*/auxil/\* "
 
 # 1. Move to base dir, create tmp dir
 cd ../../;
@@ -131,6 +131,12 @@ if [ $HTML_REPORT -eq 1 ]; then
     echo -n "Creating HTML files... "
     verify_run "genhtml -o $COVERAGE_HTML_DIR $COVERAGE_FILE"
 else
+    # The data we send to coveralls has a lot of duplicate files in it because of the
+    # zeek symlink in the src directory. Run a script that cleans that up.
+    echo -n "Cleaning coverage data for Coveralls..."
+    COVERAGE_FILE_CLEAN="${COVERAGE_FILE}.clean"
+    verify_run "testing/coverage/coverage_cleanup.py ${COVERAGE_FILE} > ${COVERAGE_FILE_CLEAN} 2>&1"
+
     echo -n "Reporting to Coveralls..."
     coveralls_cmd="coveralls-lcov -t ${COVERALLS_REPO_TOKEN}"
 
@@ -141,7 +147,7 @@ else
 	coveralls_cmd="${coveralls_cmd} --service-name=local"
     fi
 
-    coveralls_cmd="${coveralls_cmd} ${COVERAGE_FILE}"
+    coveralls_cmd="${coveralls_cmd} ${COVERAGE_FILE_CLEAN}"
 
     verify_run "${coveralls_cmd}"
 fi