Merge branch 'master' of https://github.com/ZekeMedley/zeek

* 'master' of https://github.com/ZekeMedley/zeek:
  Use the right delete and improve the leak test. Increases the size of the table being loaded in the pattern leak test and uses the right delete method.
  Fix formatting.
  Fix memory leak and add test.
  Add pattern support to input framework.

src/input/Manager.cc

@@ -224,7 +224,7 @@ ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, EnumVal* tag)
 	return backend;
 	}
 
-// Create a new input reader object to be used at whomevers leisure lateron.
+// Create a new input reader object to be used at whomevers leisure later on.
 bool Manager::CreateStream(Stream* info, RecordVal* description)
 	{
 	RecordType* rtype = description->Type()->AsRecordType();
@@ -232,7 +232,7 @@ bool Manager::CreateStream(Stream* info, RecordVal* description)
 		|| same_type(rtype, BifType::Record::Input::EventDescription, 0)
 		|| same_type(rtype, BifType::Record::Input::AnalysisDescription, 0) ) )
 		{
-		reporter->Error("Streamdescription argument not of right type for new input stream");
+		reporter->Error("Stream description argument not of right type for new input stream");
 		return false;
 		}
 
@@ -824,6 +824,7 @@ bool Manager::IsCompatibleType(BroType* t, bool atomic_only)
 	case TYPE_INTERVAL:
 	case TYPE_ENUM:
 	case TYPE_STRING:
+	case TYPE_PATTERN:
 		return true;
 
 	case TYPE_RECORD:
@@ -2074,6 +2075,12 @@ int Manager::GetValueLength(const Value* val) const
 		}
 		break;
 
+	case TYPE_PATTERN:
+		{
+		length += strlen(val->val.pattern_text_val) + 1;
+		break;
+		}
+
 	case TYPE_TABLE:
 		{
 		for ( int i = 0; i < val->val.set_val.size; i++ )
@@ -2193,6 +2200,14 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) const
 		return length;
 		}
 
+	case TYPE_PATTERN:
+		{
+		// include null-terminator
+		int length = strlen(val->val.pattern_text_val) + 1;
+		memcpy(data + startpos, val->val.pattern_text_val, length);
+		return length;
+		}
+
 	case TYPE_TABLE:
 		{
 		int length = 0;
@@ -2350,6 +2365,13 @@ Val* Manager::ValueToVal(const Stream* i, const Value* val, BroType* request_typ
 		return subnetval;
 		}
 
+	case TYPE_PATTERN:
+		{
+		RE_Matcher* re = new RE_Matcher(val->val.pattern_text_val);
+		re->Compile();
+		return new PatternVal(re);
+		}
+
 	case TYPE_TABLE:
 		{
 		// all entries have to have the same type...
@@ -2492,6 +2514,13 @@ Val* Manager::ValueToVal(const Stream* i, const Value* val, bool& have_error) co
 		return subnetval;
 		}
 
+	case TYPE_PATTERN:
+		{
+		RE_Matcher* re = new RE_Matcher(val->val.pattern_text_val);
+		re->Compile();
+		return new PatternVal(re);
+		}
+
 	case TYPE_TABLE:
 		{
 		TypeList* set_index;

src/threading/SerialTypes.cc

@@ -91,6 +91,9 @@ Value::~Value()
 	     && present )
 		delete [] val.string_val.data;
 
+	if ( type == TYPE_PATTERN && present)
+		delete [] val.pattern_text_val;
+
 	if ( type == TYPE_TABLE && present )
 		{
 		for ( int i = 0; i < val.set_val.size; i++ )
@@ -414,4 +417,3 @@ bool Value::Write(SerializationFormat* fmt) const
 
 	return false;
 	}
-

src/threading/SerialTypes.h

@@ -126,6 +126,7 @@ struct Value {
 		vec_t vector_val;
 		addr_t addr_val;
 		subnet_t subnet_val;
+		const char* pattern_text_val;
 
 		struct {
 			char* data;

src/threading/formatters/Ascii.cc

@@ -325,6 +325,29 @@ threading::Value* Ascii::ParseValue(const string& s, const string& name, TypeTag
 		break;
 		}
 
+	case TYPE_PATTERN:
+		{
+		string candidate = get_unescaped_string(s);
+		// A string is a candidate pattern iff it begins and ends with
+		// a '/'. Rather or not the rest of the string is legal will
+		// be determined later when it is given to the RE engine.
+		if ( candidate.size() >= 2 )
+			{
+			if ( candidate.front() == candidate.back() &&
+			     candidate.back() == '/' )
+				{
+				// Remove the '/'s
+				candidate.erase(0, 1);
+				candidate.erase(candidate.size() - 1);
+				val->val.pattern_text_val = copy_string(candidate.c_str());
+				break;
+				}
+			}
+
+		GetThread()->Error(GetThread()->Fmt("String '%s' contained no parseable pattern.", candidate.c_str()));
+		goto parse_error;
+		}
+
 	case TYPE_TABLE:
 	case TYPE_VECTOR:
 		// First - common initialization

testing/btest/Baseline/scripts.base.frameworks.input.bad_patterns/.stderr

@@ -0,0 +1,9 @@
+error: input.log/Input::READER_ASCII: String '/cat/sss' contained no parseable pattern.
+warning: input.log/Input::READER_ASCII: Could not convert line '2	/cat/sss' of input.log to Val. Ignoring line.
+error: input.log/Input::READER_ASCII: String '/foo|bar' contained no parseable pattern.
+warning: input.log/Input::READER_ASCII: Could not convert line '3	/foo|bar' of input.log to Val. Ignoring line.
+error: input.log/Input::READER_ASCII: String 'this is not a pattern' contained no parseable pattern.
+warning: input.log/Input::READER_ASCII: Could not convert line '4	this is not a pattern' of input.log to Val. Ignoring line.
+error: input.log/Input::READER_ASCII: String '/5' contained no parseable pattern.
+warning: input.log/Input::READER_ASCII: Could not convert line '5	/5' of input.log to Val. Ignoring line.
+received termination signal

testing/btest/Baseline/scripts.base.frameworks.input.patterns/out

@@ -0,0 +1,9 @@
+T
+F
+T
+{
+[2] = [p=/^?(cat)$?/],
+[4] = [p=/^?(^oob)$?/],
+[1] = [p=/^?(dog)$?/],
+[3] = [p=/^?(foo|bar)$?/]
+}

testing/btest/core/leaks/input-patterns.zeek

@@ -0,0 +1,52 @@
+# Needs perftools support.
+#
+# @TEST-GROUP: leaks
+#
+# @TEST-REQUIRES: zeek  --help 2>&1 | grep -q mem-leaks
+#
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run zeek zeek -m -b %INPUT
+# @TEST-EXEC: btest-bg-wait 60
+
+redef exit_only_after_terminate = T;
+
+@TEST-START-FILE input.log
+#separator \x09
+#fields	i	p
+#types	count	pattern
+1	/dog/
+2	/cat/
+3	/foo|bar/
+4	/^oob/
+@TEST-END-FILE
+
+global outfile: file;
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	p: pattern;
+};
+
+global pats: table[int] of Val = table();
+
+event zeek_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="pats", $idx=Idx, $val=Val, $destination=pats]);
+	}
+
+event Input::end_of_data(name: string, source:string)
+	{
+	print outfile, (pats[3]$p in "foobar"); # T
+	print outfile, (pats[4]$p in "foobar"); # F
+	print outfile, (pats[3]$p == "foo"); # T
+	print outfile, pats;
+	Input::remove("pats");
+	close(outfile);
+	terminate();
+	}

testing/btest/scripts/base/frameworks/input/bad_patterns.zeek

@@ -0,0 +1,38 @@
+# @TEST-EXEC: zeek -b %INPUT
+# @TEST-EXEC: btest-diff .stderr
+
+@TEST-START-FILE input.log
+#separator \x09
+#fields	i	p
+#types	count	pattern
+1	/d/og/
+2	/cat/sss
+3	/foo|bar
+4	this is not a pattern
+5	/5
+@TEST-END-FILE
+
+redef exit_only_after_terminate = T;
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	p: pattern;
+};
+
+event kill_me()
+	{
+	terminate();
+	}
+
+global pats: table[int] of Val = table();
+
+event zeek_init()
+	{
+	Input::add_table([$source="input.log", $name="pats", $idx=Idx, $val=Val, $destination=pats]);
+	schedule 10msec { kill_me() };
+	}

testing/btest/scripts/base/frameworks/input/patterns.zeek

@@ -0,0 +1,47 @@
+# @TEST-EXEC: btest-bg-run zeek zeek -b %INPUT
+# @TEST-EXEC: btest-bg-wait 10
+# @TEST-EXEC: btest-diff out
+
+redef exit_only_after_terminate = T;
+
+@TEST-START-FILE input.log
+#separator \x09
+#fields	i	p
+#types	count	pattern
+1	/dog/
+2	/cat/
+3	/foo|bar/
+4	/^oob/
+@TEST-END-FILE
+
+global outfile: file;
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	p: pattern;
+};
+
+global pats: table[int] of Val = table();
+
+event zeek_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="pats", $idx=Idx, $val=Val, $destination=pats]);
+	}
+
+event Input::end_of_data(name: string, source:string)
+	{
+	print outfile, (pats[3]$p in "foobar"); # T
+	print outfile, (pats[4]$p in "foobar"); # F
+	print outfile, (pats[3]$p == "foo"); # T
+	print outfile, pats;
+	Input::remove("pats");
+	close(outfile);
+	terminate();
+	}