Merge remote-tracking branch 'origin/master' into topic/dnthayer/language-tests

CHANGES

@@ -1,5 +1,39 @@
 
-2.1-beta-54 | 2012-08-23 11:58:50 -0700
+2.1 | 2012-08-28 16:46:42 -0700
+
+  * Make bif.identify_magic robust against FreeBSD's libmagic config.
+    (Robin Sommer)
+
+  * Remove automatic use of gperftools on non-Linux systems.
+    --enable-perftools must now explicity be supplied to ./configure
+    on non-Linux systems to link against the tcmalloc library.
+
+  * Fix uninitialized value for 'is_partial' in TCP analyzer. (Jon
+    Siwek)
+
+  * Parse 64-bit consts in Bro scripts correctly. (Bernhard Amann)
+    
+  * Output 64-bit counts correctly on 32-bit machines (Bernhard Amann)
+    
+  * Input framework fixes, including:  (Bernhard Amann)
+
+    - One of the change events got the wrong parameters.
+
+    - Escape commas in sets and vectors that were unescaped before
+      tokenization.
+
+    - Handling of zero-length-strings as last element in a set was
+      broken (sets ending with a ,).
+
+    - Hashing of lines just containing zero-length-strings was broken.
+
+    - Make set_separators different from , work for input framework.
+      
+    - Input framework was not handling counts and ints out of
+      32-bit-range correctly.
+
+    - Errors in single lines do not kill processing, but simply ignore
+      the line, log it, and continue.
 
   * Update documentation for builtin types. (Daniel Thayer)
     
@@ -19,8 +53,6 @@
   * Adding an identifier to the SMTP blocklist notices for duplicate
     suppression. (Seth Hall)
 
-     - Slight addition and revision to inline docs.
-    
 2.1-beta-45 | 2012-08-22 16:11:10 -0700
 
   * Add an option to the input framework that allows the user to chose

CMakeLists.txt

@@ -88,25 +88,31 @@ if (LIBGEOIP_FOUND)
     list(APPEND OPTLIBS ${LibGeoIP_LIBRARY})
 endif ()
 
-set(USE_PERFTOOLS false)
+set(HAVE_PERFTOOLS false)
 set(USE_PERFTOOLS_DEBUG false)
+set(USE_PERFTOOLS_TCMALLOC false)
 
 if (NOT DISABLE_PERFTOOLS)
    find_package(GooglePerftools)
 endif ()
 
 if (GOOGLEPERFTOOLS_FOUND)
-    include_directories(BEFORE ${GooglePerftools_INCLUDE_DIR})
+    set(HAVE_PERFTOOLS true)
-    set(USE_PERFTOOLS true)
+    # Non-Linux systems may not be well-supported by gperftools, so
+    # require explicit request from user to enable it in that case.
+    if (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ENABLE_PERFTOOLS)
+        set(USE_PERFTOOLS_TCMALLOC true)
 
         if (ENABLE_PERFTOOLS_DEBUG)
             # Enable heap debugging with perftools.
             set(USE_PERFTOOLS_DEBUG true)
+            include_directories(BEFORE ${GooglePerftools_INCLUDE_DIR})
             list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES_DEBUG})
         else ()
             # Link in tcmalloc for better performance.
             list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES})
         endif ()
+    endif ()
 endif ()
 
 set(USE_DATASERIES false)
@@ -224,7 +230,8 @@ message(
     "\nAux. Tools:        ${INSTALL_AUX_TOOLS}"
     "\n"
     "\nGeoIP:             ${USE_GEOIP}"
-    "\nGoogle perftools:  ${USE_PERFTOOLS}"
+    "\ngperftools found:  ${HAVE_PERFTOOLS}"
+    "\n        tcmalloc:  ${USE_PERFTOOLS_TCMALLOC}"
     "\n       debugging:  ${USE_PERFTOOLS_DEBUG}"
     "\ncURL:              ${USE_CURL}"
     "\n"

NEWS

@@ -102,9 +102,14 @@ the full set.
 
     * Bro now requires CMake >= 2.6.3.
 
-    * Bro now links in tcmalloc (part of Google perftools) if found at
+    * On Linux, Bro now links in tcmalloc (part of Google perftools)
-      configure time. Doing so can significantly improve memory and
+      if found at configure time. Doing so can significantly improve
-      CPU use.
+      memory and CPU use.
+
+      On the other platforms, the new configure option
+      --enable-perftools can be used to enable linking to tcmalloc.
+      (Note that perftools's support for non-Linux platforms may be
+      less reliable).
 
 - The configure switch --enable-brov6 is gone.
 

VERSION

@@ -1 +1 @@
-2.1-beta-54
+2.1

aux/bro-aux

@@ -1 +1 @@
-Subproject commit 4bc1a6f6a8816dfacd8288fcf182ba35520e589b
+Subproject commit 6748ec3a96d582a977cd9114ef19c76fe75c57ff

aux/broctl

@@ -1 +1 @@
-Subproject commit 5b3f9e5906c90b76c5aa1626e112d4c991cb3fd8
+Subproject commit 2fb9ff62bf08f78071753016863640022fbfe338

configure

@@ -29,6 +29,8 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
   Optional Features:
     --enable-debug         compile in debugging mode
     --enable-mobile-ipv6   analyze mobile IPv6 features defined by RFC 6275
+    --enable-perftools     force use of Google perftools on non-Linux systems
+                           (automatically on when perftools is present on Linux)
     --enable-perftools-debug use Google's perftools for debugging
     --disable-broccoli     don't build or install the Broccoli library
     --disable-broctl       don't install Broctl
@@ -98,6 +100,7 @@ append_cache_entry PY_MOD_INSTALL_DIR   PATH   $prefix/lib/broctl
 append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $prefix/share/bro
 append_cache_entry BRO_ETC_INSTALL_DIR  PATH   $prefix/etc
 append_cache_entry ENABLE_DEBUG         BOOL   false
+append_cache_entry ENABLE_PERFTOOLS     BOOL   false
 append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL false
 append_cache_entry BinPAC_SKIP_INSTALL  BOOL   true
 append_cache_entry BUILD_SHARED_LIBS    BOOL   true
@@ -146,7 +149,11 @@ while [ $# -ne 0 ]; do
         --enable-mobile-ipv6)
             append_cache_entry ENABLE_MOBILE_IPV6         BOOL   true
             ;;
+        --enable-perftools)
+            append_cache_entry ENABLE_PERFTOOLS     BOOL   true
+            ;;
         --enable-perftools-debug)
+            append_cache_entry ENABLE_PERFTOOLS     BOOL   true
             append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL   true
             ;;
         --disable-broccoli)

src/TCP.cc

@@ -46,6 +46,7 @@ TCP_Analyzer::TCP_Analyzer(Connection* conn)
 	finished = 0;
 	reassembling = 0;
 	first_packet_seen = 0;
+	is_partial = 0;
 
 	orig = new TCP_Endpoint(this, 1);
 	resp = new TCP_Endpoint(this, 0);

src/input/Manager.cc

@@ -1228,7 +1228,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader)
 			Ref(predidx);
 			Ref(val);
 			Ref(ev);
-			SendEvent(stream->event, 3, ev, predidx, val);
+			SendEvent(stream->event, 4, stream->description->Ref(), ev, predidx, val);
 			}
 
 		if ( predidx )  // if we have a stream or an event...
@@ -1748,7 +1748,7 @@ int Manager::GetValueLength(const Value* val) {
 	case TYPE_STRING:
 	case TYPE_ENUM:
 		{
-		length += val->val.string_val.length;
+		length += val->val.string_val.length + 1;
 		break;
 		}
 
@@ -1848,7 +1848,10 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val)
 	case TYPE_ENUM:
 		{
 		memcpy(data+startpos, val->val.string_val.data, val->val.string_val.length);
-		return val->val.string_val.length;
+		// Add a \0 to the end. To be able to hash zero-length
+		// strings and differentiate from !present.
+		memset(data + startpos + val->val.string_val.length, 0, 1);
+		return val->val.string_val.length + 1;
 		}
 
 	case TYPE_ADDR:
@@ -1939,13 +1942,15 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals)
 		const Value* val = vals[i];
 		if ( val->present )
 			length += GetValueLength(val);
+
+		// And in any case add 1 for the end-of-field-identifier.
+		length++;
 		}
 
-	if ( length == 0 )
+	assert ( length >= num_elements );
-		{
+
-		reporter->Error("Input reader sent line where all elements are null values. Ignoring line");
+	if ( length == num_elements )
 		return NULL;
-		}
 
 	int position = 0;
 	char *data = (char*) malloc(length);
@@ -1957,6 +1962,12 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals)
 		const Value* val = vals[i];
 		if ( val->present )
 			position += CopyValue(data, position, val);
+
+		memset(data + position, 1, 1); // Add end-of-field-marker. Does not really matter which value it is,
+		                               // it just has to be... something.
+
+		position++;
+
 		}
 
 	HashKey *key = new HashKey(data, length);

src/input/readers/Ascii.cc

@@ -11,6 +11,7 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
+#include <errno.h>
 
 using namespace input::reader;
 using threading::Value;
@@ -209,6 +210,42 @@ bool Ascii::GetLine(string& str)
 	return false;
 	}
 
+bool Ascii::CheckNumberError(const string& s, const char * end)
+	{
+	// Do this check first, before executing s.c_str() or similar.
+	// otherwise the value to which *end is pointing at the moment might
+	// be gone ...
+	bool endnotnull =  (*end != '\0');
+
+	if ( s.length() == 0 )
+		{
+		Error("Got empty string for number field");
+		return true;
+		}
+
+	if ( end == s.c_str() ) {
+		Error(Fmt("String '%s' contained no parseable number", s.c_str()));
+		return true;
+	}
+
+	if ( endnotnull )
+		Warning(Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end));
+
+	if ( errno == EINVAL )
+		{
+		Error(Fmt("String '%s' could not be converted to a number", s.c_str()));
+		return true;
+		}
+
+	else if ( errno == ERANGE )
+		{
+		Error(Fmt("Number '%s' out of supported range.", s.c_str()));
+		return true;
+		}
+
+	return false;
+	}
+
 
 Value* Ascii::EntryToVal(string s, FieldMapping field)
 	{
@@ -216,10 +253,13 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 		return new Value(field.type, false);
 
 	Value* val = new Value(field.type, true);
+	char* end = 0;
+	errno = 0;
 
 	switch ( field.type ) {
 	case TYPE_ENUM:
 	case TYPE_STRING:
+		s = get_unescaped_string(s);
 		val->val.string_val.length = s.size();
 		val->val.string_val.data = copy_string(s.c_str());
 		break;
@@ -238,27 +278,37 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 		break;
 
 	case TYPE_INT:
-		val->val.int_val = atoi(s.c_str());
+		val->val.int_val = strtoll(s.c_str(), &end, 10);
+		if ( CheckNumberError(s, end) )
+			return 0;
 		break;
 
 	case TYPE_DOUBLE:
 	case TYPE_TIME:
 	case TYPE_INTERVAL:
-		val->val.double_val = atof(s.c_str());
+		val->val.double_val = strtod(s.c_str(), &end);
+		if ( CheckNumberError(s, end) )
+			return 0;
 		break;
 
 	case TYPE_COUNT:
 	case TYPE_COUNTER:
-		val->val.uint_val = atoi(s.c_str());
+		val->val.uint_val = strtoull(s.c_str(), &end, 10);
+		if ( CheckNumberError(s, end) )
+			return 0;
 		break;
 
 	case TYPE_PORT:
-		val->val.port_val.port = atoi(s.c_str());
+		val->val.port_val.port = strtoull(s.c_str(), &end, 10);
+		if ( CheckNumberError(s, end) )
+			return 0;
+
 		val->val.port_val.proto = TRANSPORT_UNKNOWN;
 		break;
 
 	case TYPE_SUBNET:
 		{
+		s = get_unescaped_string(s);
 		size_t pos = s.find("/");
 		if ( pos == s.npos )
 			{
@@ -266,7 +316,11 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 			return 0;
 			}
 
-		int width = atoi(s.substr(pos+1).c_str());
+		uint8_t width = (uint8_t) strtol(s.substr(pos+1).c_str(), &end, 10);
+
+		if ( CheckNumberError(s, end) )
+			return 0;
+
 		string addr = s.substr(0, pos);
 
 		val->val.subnet_val.prefix = StringToAddr(addr);
@@ -275,6 +329,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 		}
 
 	case TYPE_ADDR:
+		s = get_unescaped_string(s);
 		val->val.addr_val = StringToAddr(s);
 		break;
 
@@ -288,7 +343,10 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 		// how many entries do we have...
 		unsigned int length = 1;
 		for ( unsigned int i = 0; i < s.size(); i++ )
-			if ( s[i] == ',' ) length++;
+			{
+			if ( s[i] == set_separator[0] )
+				length++;
+			}
 
 		unsigned int pos = 0;
 
@@ -342,9 +400,24 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 			pos++;
 			}
 
+		// Test if the string ends with a set_separator... or if the
+		// complete string is empty. In either of these cases we have
+		// to push an empty val on top of it.
+		if ( s.empty() || *s.rbegin() == set_separator[0] )
+			{
+			lvals[pos] = EntryToVal("", field.subType());
+			if ( lvals[pos] == 0 )
+				{
+				Error("Error while trying to add empty set element");
+				return 0;
+				}
+
+			pos++;
+			}
+
 		if ( pos != length )
 			{
-			Error("Internal error while parsing set: did not find all elements");
+			Error(Fmt("Internal error while parsing set: did not find all elements: %s", s.c_str()));
 			return 0;
 			}
 
@@ -428,6 +501,7 @@ bool Ascii::DoUpdate()
 	while ( GetLine(line ) )
 		{
 		// split on tabs
+		bool error = false;
 		istringstream splitstream(line);
 
 		map<int, string> stringfields;
@@ -438,8 +512,6 @@ bool Ascii::DoUpdate()
 			if ( ! getline(splitstream, s, separator[0]) )
 				break;
 
-			s = get_unescaped_string(s);
-
 			stringfields[pos] = s;
 			pos++;
 			}
@@ -474,8 +546,9 @@ bool Ascii::DoUpdate()
 			Value* val = EntryToVal(stringfields[(*fit).position], *fit);
 			if ( val == 0 )
 				{
-				Error("Could not convert String value to Val");
+				Error(Fmt("Could not convert line '%s' to Val. Ignoring line.", line.c_str()));
-				return false;
+				error = true;
+				break;
 				}
 
 			if ( (*fit).secondary_position != -1 )
@@ -492,6 +565,19 @@ bool Ascii::DoUpdate()
 			fpos++;
 			}
 
+		if ( error )
+			{
+			// Encountered non-fatal error, ignoring line. But
+			// first, delete all successfully read fields and the
+			// array structure.
+
+			for ( int i = 0; i < fpos; i++ )
+				delete fields[fpos];
+
+			delete [] fields;
+			continue;
+			}
+
 		//printf("fpos: %d, second.num_fields: %d\n", fpos, (*it).second.num_fields);
 		assert ( fpos == NumFields() );
 

src/input/readers/Ascii.h

@@ -48,6 +48,7 @@ private:
 	bool ReadHeader(bool useCached);
 	bool GetLine(string& str);
 	threading::Value* EntryToVal(string s, FieldMapping type);
+	bool CheckNumberError(const string& s, const char * end);
 
 	ifstream* file;
 	time_t mtime;

src/modp_numtoa.c

@@ -56,7 +56,7 @@ void modp_uitoa10(uint32_t value, char* str)
 void modp_litoa10(int64_t value, char* str)
 {
     char* wstr=str;
-    unsigned long uvalue = (value < 0) ? -value : value;
+    uint64_t uvalue = (value < 0) ? -value : value;
 
     // Conversion. Number is reversed.
     do *wstr++ = (char)(48 + (uvalue % 10)); while(uvalue /= 10);

src/scan.l

@@ -437,9 +437,7 @@ F	RET_CONST(new Val(false, TYPE_BOOL))
 	}
 
 {D}		{
- 		// TODO: check if we can use strtoull instead of atol,
+		RET_CONST(new Val(static_cast<bro_uint_t>(strtoul(yytext, (char**) NULL, 10)),
-		// and similarly for {HEX}.
-		RET_CONST(new Val(static_cast<unsigned int>(atol(yytext)),
 			  TYPE_COUNT))
 		}
 {FLOAT}		RET_CONST(new Val(atof(yytext), TYPE_DOUBLE))
@@ -483,7 +481,7 @@ F	RET_CONST(new Val(false, TYPE_BOOL))
 
 ({D}"."){3}{D}		RET_CONST(new AddrVal(yytext))
 
-"0x"{HEX}+	RET_CONST(new Val(static_cast<bro_uint_t>(strtol(yytext, 0, 16)), TYPE_COUNT))
+"0x"{HEX}+	RET_CONST(new Val(static_cast<bro_uint_t>(strtoull(yytext, 0, 16)), TYPE_COUNT))
 
 {H}("."{H})+		RET_CONST(dns_mgr->LookupHost(yytext))
 

testing/btest/Baseline/bifs.identify_data/out

@@ -1,4 +1,4 @@
 ASCII text, with no line terminators
 text/plain; charset=us-ascii
-PNG image data
+PNG image
 image/png; charset=binary

testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out

@@ -0,0 +1,4 @@
+{
+[9223372036854775800] = [c=18446744073709551612],
+[-9223372036854775800] = [c=18446744073709551612]
+}

testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/out

@@ -0,0 +1,155 @@
+============PREDICATE============
+Input::EVENT_NEW
+[i=1]
+[s=<uninitialized>, ss=TEST]
+============PREDICATE============
+Input::EVENT_NEW
+[i=2]
+[s=<uninitialized>, ss=<uninitialized>]
+============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[2] = [s=<uninitialized>, ss=<uninitialized>],
+[1] = [s=<uninitialized>, ss=TEST]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
+Type
+Input::EVENT_NEW
+Left
+[i=1]
+Right
+[s=<uninitialized>, ss=TEST]
+============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[2] = [s=<uninitialized>, ss=<uninitialized>],
+[1] = [s=<uninitialized>, ss=TEST]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
+Type
+Input::EVENT_NEW
+Left
+[i=2]
+Right
+[s=<uninitialized>, ss=<uninitialized>]
+==========SERVERS============
+{
+[2] = [s=<uninitialized>, ss=<uninitialized>],
+[1] = [s=<uninitialized>, ss=TEST]
+}
+============PREDICATE============
+Input::EVENT_CHANGED
+[i=1]
+[s=TEST, ss=<uninitialized>]
+============PREDICATE============
+Input::EVENT_CHANGED
+[i=2]
+[s=TEST, ss=TEST]
+============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[2] = [s=TEST, ss=TEST],
+[1] = [s=TEST, ss=<uninitialized>]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
+Type
+Input::EVENT_CHANGED
+Left
+[i=1]
+Right
+[s=<uninitialized>, ss=TEST]
+============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[2] = [s=TEST, ss=TEST],
+[1] = [s=TEST, ss=<uninitialized>]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
+Type
+Input::EVENT_CHANGED
+Left
+[i=2]
+Right
+[s=<uninitialized>, ss=<uninitialized>]
+==========SERVERS============
+{
+[2] = [s=TEST, ss=TEST],
+[1] = [s=TEST, ss=<uninitialized>]
+}
+done

testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline

@@ -0,0 +1,8 @@
+error: ../input.log/Input::READER_ASCII: Number '12129223372036854775800' out of supported range.
+error: ../input.log/Input::READER_ASCII: Could not convert line '12129223372036854775800	121218446744073709551612' to Val. Ignoring line.
+warning: ../input.log/Input::READER_ASCII: Number '9223372036854775801TEXTHERE' contained non-numeric trailing characters. Ignored trailing characters 'TEXTHERE'
+warning: ../input.log/Input::READER_ASCII: Number '1Justtext' contained non-numeric trailing characters. Ignored trailing characters 'Justtext'
+error: ../input.log/Input::READER_ASCII: String 'Justtext' contained no parseable number
+error: ../input.log/Input::READER_ASCII: Could not convert line 'Justtext	1' to Val. Ignoring line.
+received termination signal
+>>>

testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out

@@ -0,0 +1,4 @@
+{
+[9223372036854775800] = [c=4],
+[9223372036854775801] = [c=1]
+}

testing/btest/Baseline/scripts.base.frameworks.input.reread/out

@@ -1084,7 +1084,7 @@ BB
 }
 ============PREDICATE============
 Input::EVENT_REMOVED
-[i=-43]
+[i=-44]
 [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -1096,6 +1096,21 @@ AA,
 BB
 }, se={
 
+}, vc=[10, 20, 30], ve=[]]
+============PREDICATE============
+Input::EVENT_REMOVED
+[i=-42]
+[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
 }, vc=[10, 20, 30], ve=[]]
 ============PREDICATE============
 Input::EVENT_REMOVED
@@ -1111,21 +1126,6 @@ AA,
 BB
 }, se={
 
-}, vc=[10, 20, 30], ve=[]]
-============PREDICATE============
-Input::EVENT_REMOVED
-[i=-44]
-[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
-2,
-4,
-1,
-3
-}, ss={
-CC,
-AA,
-BB
-}, se={
-
 }, vc=[10, 20, 30], ve=[]]
 ============PREDICATE============
 Input::EVENT_REMOVED
@@ -1159,25 +1159,7 @@ BB
 }, vc=[10, 20, 30], ve=[]]
 ============PREDICATE============
 Input::EVENT_REMOVED
-[i=-42]
-[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
-2,
-4,
-1,
-3
-}, ss={
-CC,
-AA,
-BB
-}, se={
-
-}, vc=[10, 20, 30], ve=[]]
-============EVENT============
-Description
-Input::EVENT_REMOVED
-Type
 [i=-43]
-Left
 [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -1190,14 +1172,10 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
-Right
 ============EVENT============
 Description
-Input::EVENT_REMOVED
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
-Type
+[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
-[i=-46]
-Left
-[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
 1,
@@ -1209,13 +1187,32 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
-Right
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
-============EVENT============
+{ 
-Description
+print A::outfile, ============EVENT============;
-Input::EVENT_REMOVED
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
 Type
+Input::EVENT_REMOVED
+Left
 [i=-44]
-Left
+Right
 [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -1228,14 +1225,10 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
-Right
 ============EVENT============
 Description
-Input::EVENT_REMOVED
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
-Type
+[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
-[i=-47]
-Left
-[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
 1,
@@ -1247,32 +1240,32 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
-Right
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
-============EVENT============
+{ 
-Description
+print A::outfile, ============EVENT============;
-Input::EVENT_REMOVED
+print A::outfile, Description;
-Type
+print A::outfile, A::description;
-[i=-45]
+print A::outfile, Type;
-Left
+print A::outfile, A::tpe;
-[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+print A::outfile, Left;
-2,
+print A::outfile, A::left;
-4,
+print A::outfile, Right;
-1,
+print A::outfile, A::right;
-3
+}, pred=anonymous-function
-}, ss={
+{ 
-CC,
+print A::outfile, ============PREDICATE============;
-AA,
+print A::outfile, A::typ;
-BB
+print A::outfile, A::left;
-}, se={
+print A::outfile, A::right;
+return (T);
+}, config={
 
-}, vc=[10, 20, 30], ve=[]]
+}]
-Right
-============EVENT============
-Description
-Input::EVENT_REMOVED
 Type
+Input::EVENT_REMOVED
+Left
 [i=-42]
-Left
+Right
 [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -1285,7 +1278,218 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
+============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
+Type
+Input::EVENT_REMOVED
+Left
+[i=-46]
 Right
+[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
+Type
+Input::EVENT_REMOVED
+Left
+[i=-47]
+Right
+[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
+Type
+Input::EVENT_REMOVED
+Left
+[i=-45]
+Right
+[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
+Type
+Input::EVENT_REMOVED
+Left
+[i=-43]
+Right
+[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
 ==========SERVERS============
 {
 [-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={

testing/btest/Baseline/scripts.base.frameworks.input.setseparator/out

@@ -0,0 +1,10 @@
+{
+[1] = [s={
+b,
+e,
+d,
+c,
+f,
+a
+}, ss=[1, 2, 3, 4, 5, 6]]
+}

testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out

@@ -0,0 +1,23 @@
+{
+[2] = [s={
+,
+testing
+}, s=[testing, , testing]],
+[4] = [s={
+,
+testing
+}, s=[testing, ]],
+[6] = [s={
+
+}, s=[]],
+[1] = [s={
+testing,testing,testing,
+}, s=[testing,testing,testing,]],
+[5] = [s={
+
+}, s=[, , , ]],
+[3] = [s={
+,
+testing
+}, s=[, testing]]
+}

testing/btest/bifs/identify_data.bro

@@ -1,5 +1,5 @@
 #
-# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: bro %INPUT | sed 's/PNG image data/PNG image/g' >out
 # @TEST-EXEC: btest-diff out
 
 event bro_init()

testing/btest/scripts/base/frameworks/input/bignumber.bro

@@ -0,0 +1,45 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#fields	i	c
+#types	int	count
+9223372036854775800	18446744073709551612
+-9223372036854775800	18446744073709551612
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	c: count;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
+	}
+
+event Input::update_finished(name: string, source:string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}

testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro

@@ -0,0 +1,89 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: cp input1.log input.log
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp input2.log input.log
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input1.log
+#separator \x09
+#fields	i	s	ss
+#types	int	sting	string
+1	-	TEST
+2	-	-
+@TEST-END-FILE
+@TEST-START-FILE input2.log
+#separator \x09
+#fields	i	s	ss
+#types	int	sting	string
+1	TEST	-
+2	TEST	TEST
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	s: string;
+	ss: string;
+};
+
+global servers: table[int] of Val = table();
+
+global outfile: file;
+
+global try: count;
+
+event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val)
+	{
+	print outfile, "============EVENT============";
+	print outfile, "Description";
+	print outfile, description;
+	print outfile, "Type";
+	print outfile, tpe;
+	print outfile, "Left";
+	print outfile, left;
+	print outfile, "Right";
+	print outfile, right;
+	}
+
+event bro_init()
+	{
+	outfile = open("../out");
+	try = 0;
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $mode=Input::REREAD, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line,
+	$pred(typ: Input::Event, left: Idx, right: Val) = { 
+	print outfile, "============PREDICATE============";
+	print outfile, typ;
+	print outfile, left;
+	print outfile, right;
+	return T;
+	}
+	]);
+	}
+
+
+event Input::update_finished(name: string, source: string)
+	{
+	print outfile, "==========SERVERS============";
+	print outfile, servers;
+	
+	try = try + 1;
+	if ( try == 2 )
+		{
+		print outfile, "done";
+		close(outfile);
+		Input::remove("input");
+		terminate();
+		}
+	}

testing/btest/scripts/base/frameworks/input/invalidnumbers.bro

@@ -0,0 +1,48 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+# @TEST-EXEC: sed 1d .stderr > .stderrwithoutfirstline
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderrwithoutfirstline
+
+@TEST-START-FILE input.log
+#separator \x09
+#fields	i	c
+#types	int	count
+12129223372036854775800	121218446744073709551612
+9223372036854775801TEXTHERE	1Justtext
+Justtext	1
+9223372036854775800	-18446744073709551612
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	c: count;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
+	}
+
+event Input::update_finished(name: string, source:string)
+	{
+	print outfile, servers;
+	terminate();
+	}

testing/btest/scripts/base/frameworks/input/setseparator.bro

@@ -0,0 +1,46 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#fields	i	s	ss
+1	a|b|c|d|e|f	1|2|3|4|5|6
+@TEST-END-FILE
+
+redef InputAscii::set_separator = "|";
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	s: set[string];
+	ss:vector of count;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
+	}
+
+event Input::update_finished(name: string, source:string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}

testing/btest/scripts/base/frameworks/input/setspecialcases.bro

@@ -0,0 +1,50 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#fields	i	s	ss
+1	testing\x2ctesting\x2ctesting\x2c	testing\x2ctesting\x2ctesting\x2c
+2	testing,,testing	testing,,testing
+3	,testing	,testing
+4	testing,	testing,
+5	,,,	,,,
+6		
+@TEST-END-FILE
+
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	s: set[string];
+	s: vector of string;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
+	}
+
+event Input::update_finished(name: string, source:string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}