Merge remote-tracking branch 'origin/topic/jsiwek/fix-zeek-profiler-file'

* origin/topic/jsiwek/fix-zeek-profiler-file:
  Fix ZEEK_PROFILER_FILE file format/parsing

CHANGES

@@ -1,4 +1,12 @@
 
+3.1.0-dev.284 | 2019-11-21 08:29:36 -0800
+
+  * Fix ZEEK_PROFILER_FILE file format/parsing
+
+    Some Zeek script statement descriptions were exceeding the hardcoded
+    maximum length and also could contain tab characters which were
+    supposed to be reserved for use as a delimiter in the file format. (Jon Siwek, Corelight)
+
 3.1.0-dev.282 | 2019-11-18 12:06:13 +0000
 
   * GH-646: Add new "successful_connection_remove" event. (Jon Siwek, Corelight)

VERSION

@@ -1 +1 @@
-3.1.0-dev.282
+3.1.0-dev.284

src/Brofiler.cc

@@ -1,5 +1,7 @@
 #include <cstdio>
 #include <cstring>
+#include <sstream>
+#include <fstream>
 #include <utility>
 #include <algorithm>
 #include <sys/stat.h>
@@ -22,27 +24,46 @@ bool Brofiler::ReadStats()
 	if ( ! bf )
 		return false;
 
-	FILE* f = fopen(bf, "r");
+	std::ifstream ifs;
-	if ( ! f )
+	ifs.open(bf, std::ifstream::in);
+
+	if ( ! ifs )
 		return false;
 
-	char line[16384];
+	std::stringstream ss;
+	ss << ifs.rdbuf();
+	std::string file_contents = ss.str();
+	ss.clear();
+
+	std::vector<std::string> lines;
+	tokenize_string(file_contents, "\n", &lines);
 	string delimiter;
 	delimiter = delim;
 
-	while( fgets(line, sizeof(line), f) )
+	for ( const auto& line : lines )
 		{
-		line[strlen(line) - 1] = 0; //remove newline
+		if ( line.empty() )
-		string cnt(strtok(line, delimiter.c_str()));
+			continue;
-		string location(strtok(0, delimiter.c_str()));
+
-		string desc(strtok(0, delimiter.c_str()));
+		std::vector<std::string> line_components;
-		pair<string, string> location_desc(location, desc);
+		tokenize_string(line, delimiter, &line_components);
+
+		if ( line_components.size() != 3 )
+			{
+			fprintf(stderr, "invalid ZEEK_PROFILER_FILE line: %s\n", line.data());
+			continue;
+			}
+
+		std::string& cnt = line_components[0];
+		std::string& location = line_components[1];
+		std::string& desc = line_components[2];
+
+		pair<string, string> location_desc(std::move(location), std::move(desc));
 		uint64_t count;
 		atoi_n(cnt.size(), cnt.c_str(), 0, 10, count);
-		usage_map[location_desc] = count;
+		usage_map.emplace(std::move(location_desc), count);
 		}
 
-	fclose(f);
 	return true;
 	}
 
@@ -96,7 +117,8 @@ bool Brofiler::WriteStats()
 		ODesc desc_info;
 		(*it)->Describe(&desc_info);
 		string desc(desc_info.Description());
-		for_each(desc.begin(), desc.end(), canonicalize_desc());
+		canonicalize_desc cd{delim};
+		for_each(desc.begin(), desc.end(), cd);
 		pair<string, string> location_desc(location_info.Description(), desc);
 		if ( usage_map.find(location_desc) != usage_map.end() )
 			usage_map[location_desc] += (*it)->GetAccessCount();

src/Brofiler.h

@@ -70,9 +70,12 @@ private:
 	 * that don't agree with the output format of Brofiler.
 	 */
 	struct canonicalize_desc {
+		char delim;
+
 		void operator() (char& c)
 			{
 			if ( c == '\n' ) c = ' ';
+			if ( c == delim ) c = ' ';
 			}
 	};
 };

testing/btest/Baseline/coverage.zeek-profiler-file/step1.out

@@ -0,0 +1 @@
+1	./profiling-test1.zeek, line 2	print new conn;

testing/btest/Baseline/coverage.zeek-profiler-file/step2.out

@@ -0,0 +1 @@
+2	./profiling-test1.zeek, line 2	print new conn;

testing/btest/Baseline/coverage.zeek-profiler-file/step3.out

@@ -0,0 +1,2 @@
+2	./profiling-test1.zeek, line 2	print new conn;
+1	./profiling-test2.zeek, line 2	print new conn;

testing/btest/coverage/zeek-profiler-file.zeek

@@ -0,0 +1,21 @@
+# @TEST-EXEC: ZEEK_PROFILER_FILE=cov.txt zeek -b -r $TRACES/http/get.trace profiling-test1.zeek
+# @TEST-EXEC: grep profiling-test1.zeek cov.txt > step1.out
+# @TEST-EXEC: btest-diff step1.out
+
+# @TEST-EXEC: ZEEK_PROFILER_FILE=cov.txt zeek -b -r $TRACES/http/get.trace profiling-test1.zeek
+# @TEST-EXEC: grep profiling-test1.zeek cov.txt > step2.out
+# @TEST-EXEC: btest-diff step2.out
+
+# @TEST-EXEC: ZEEK_PROFILER_FILE=cov.txt zeek -r $TRACES/http/get.trace profiling-test2.zeek
+# @TEST-EXEC: grep profiling-test cov.txt > step3.out
+# @TEST-EXEC: btest-diff step3.out
+
+@TEST-START-FILE profiling-test1.zeek
+event new_connection(c: connection)
+	{ print "new conn"; }
+@TEST-END-FILE
+
+@TEST-START-FILE profiling-test2.zeek
+event new_connection(c: connection)
+	{ print "new conn"; }
+@TEST-END-FILE