Merge remote-tracking branch 'origin' into topic/bernhard/input-threads

CHANGES

@@ -1,4 +1,27 @@
 
+2.0-150 | 2012-03-13 16:16:22 -0700
+
+  * Changing the regular expression to allow Site::local_nets in
+    signatures. (Julien Sentier)
+
+  * Removing a line of dead code. Found by . Closes #786. (Julien
+    Sentier)
+
+2.0-146 | 2012-03-13 15:39:38 -0700
+
+  * Change IPv6 literal constant syntax to require encasing square
+    brackets. (Jon Siwek)
+
+2.0-145 | 2012-03-09 15:10:35 -0800
+
+  * Remove the match expression. 'match' and 'using' are no longer
+    keywords. Addressed #753. (Jon Siwek)
+
+2.0-143 | 2012-03-09 15:07:42 -0800
+
+  * Fix a BRO_PROFILER_FILE/mkstemp portability issue. Addresses #794.
+    (Jon Siwek)
+
 2.0-139 | 2012-03-02 09:33:04 -0800
 
   * Changes to how script coverage integrates with test suites. (Jon Siwek)

NEWS

@@ -24,6 +24,13 @@ Bro 2.1
 - The connection compressor was already deprecated in 2.0 and has now
   been removed from the code base.
 
+- We removed the "match" statement, which was no longer used by any of
+  the default scripts, nor was it likely to be used by anybody anytime
+  soon. With that, "match" and "using" are no longer reserved keywords.
+
+- The syntax for IPv6 literals changed from "2607:f8b0:4009:802::1012"
+  to "[2607:f8b0:4009:802::1012]".
+
 TODO: Extend.
 
 Bro 2.0

VERSION

@@ -1 +1 @@
-2.0-139
+2.0-150

aux/broccoli

@@ -1 +1 @@
-Subproject commit 2602eb53e70d7f0afae8fac58d7636b9291974a4
+Subproject commit a08ca90727c5c4b90aa8633106ec33a5cf7378d4

aux/btest

@@ -1 +1 @@
-Subproject commit ee87db37b520b88a55323a9767234c30b801e439
+Subproject commit 9c9fde204dd5518bdfdb8b4a86d38ed06e597209

scripts/base/protocols/ftp/main.bro

@@ -275,7 +275,7 @@ event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) &prior
 			{
 			c$ftp$passive=T;
 			
-			if ( code == 229 && data$h == :: )
+			if ( code == 229 && data$h == [::] )
 				data$h = id$resp_h;
 			
 			ftp_data_expected[data$h, data$p] = c$ftp;

src/Expr.cc

@@ -3633,151 +3633,6 @@ bool FieldAssignExpr::DoUnserialize(UnserialInfo* info)
 	return true;
 	}
 
-RecordMatchExpr::RecordMatchExpr(Expr* op1 /* record to match */,
-					Expr* op2 /* cases to match against */)
-: BinaryExpr(EXPR_MATCH, op1, op2)
-	{
-	BroType* result_type = 0;
-
-	// Make sure the second argument is of a suitable type.
-	if ( ! op2->Type()->IsSet() )
-		{
-		ExprError("matching must be done against a set of match records");
-		return;
-		}
-
-	type_list* elt_types = op2->Type()->AsSetType()->Indices()->Types();
-
-	if ( ! elt_types->length() ||
-	     (*elt_types)[0]->Tag() != TYPE_RECORD )
-		{
-		ExprError("matching must be done against a set of match records");
-		return;
-		}
-
-	RecordType* case_rec_type = (*elt_types)[0]->AsRecordType();
-
-	// NOTE: The "result" and "pred" field names are hardcoded here.
-	result_field_index = case_rec_type->FieldOffset("result");
-
-	if ( result_field_index < 0 )
-		{
-		ExprError("match records must have a $result field");
-		return;
-		}
-
-	result_type = case_rec_type->FieldType("result")->Ref();
-
-	// Check that pred exists, and that the first argument matches it.
-	if ( (pred_field_index = case_rec_type->FieldOffset("pred")) < 0 ||
-	     case_rec_type->FieldType("pred")->Tag() != TYPE_FUNC )
-		{
-		ExprError("match records must have a $pred' field of function type");
-		return;
-		}
-
-	FuncType* pred_type = case_rec_type->FieldType("pred")->AsFuncType();
-	type_list* pred_arg_types = pred_type->ArgTypes()->Types();
-	if ( pred_arg_types->length() != 1 ||
-	     ! check_and_promote_expr(op1, (*pred_arg_types)[0]) )
-		ExprError("record to match does not have the same type as predicate argument");
-
-	// NOTE: The "priority" field name is hardcoded here.
-	if ( (priority_field_index = case_rec_type->FieldOffset("priority")) >= 0 &&
-	     ! IsArithmetic(case_rec_type->FieldType("priority")->Tag()) )
-		ExprError("$priority field must have a numeric type");
-
-	SetType(result_type);
-	}
-
-void RecordMatchExpr::ExprDescribe(ODesc* d) const
-	{
-	if ( d->IsReadable() )
-		{
-		d->Add("match ");
-		op1->Describe(d);
-		d->Add(" using ");
-		op2->Describe(d);
-		}
-	}
-
-Val* RecordMatchExpr::Fold(Val* v1, Val* v2) const
-	{
-	TableVal* match_set = v2->AsTableVal();
-	if ( ! match_set )
-		Internal("non-table in RecordMatchExpr");
-
-	Val* return_val = 0;
-	double highest_priority = -1e100;
-
-	ListVal* match_recs = match_set->ConvertToList(TYPE_ANY);
-	for ( int i = 0; i < match_recs->Length(); ++i )
-		{
-		val_list args(1);
-		args.append(v1->Ref());
-
-		double this_priority = 0;
-
-		// ### Get rid of the double Index if TYPE_ANY->TYPE_RECORD.
-		Val* v = match_recs->Index(i)->AsListVal()->Index(0);
-
-		const RecordVal* match_rec = v->AsRecordVal();
-		if ( ! match_rec )
-			Internal("Element of match set is not a record");
-
-		if ( priority_field_index >= 0 )
-			{
-			this_priority =
-				match_rec->Lookup(priority_field_index)->CoerceToDouble();
-			if ( this_priority <= highest_priority )
-				{
-				Unref(v1);
-				continue;
-				}
-			}
-
-		// No try/catch here; we pass exceptions upstream.
-		Val* pred_val =
-			match_rec->Lookup(pred_field_index)->AsFunc()->Call(&args);
-		bool is_zero = pred_val->IsZero();
-		Unref(pred_val);
-
-		if ( ! is_zero )
-			{
-			Val* new_return_val =
-				match_rec->Lookup(result_field_index);
-
-			Unref(return_val);
-			return_val = new_return_val->Ref();
-
-			if ( priority_field_index >= 0 )
-				highest_priority = this_priority;
-			else
-				break;
-			}
-		}
-
-	Unref(match_recs);
-
-	return return_val;
-	}
-
-IMPLEMENT_SERIAL(RecordMatchExpr, SER_RECORD_MATCH_EXPR);
-
-bool RecordMatchExpr::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_RECORD_MATCH_EXPR, BinaryExpr);
-	return SERIALIZE(pred_field_index) && SERIALIZE(result_field_index) &&
-		SERIALIZE(priority_field_index);
-	}
-
-bool RecordMatchExpr::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(BinaryExpr);
-	return UNSERIALIZE(&pred_field_index) && UNSERIALIZE(&result_field_index) &&
-		UNSERIALIZE(&priority_field_index);
-	}
-
 ArithCoerceExpr::ArithCoerceExpr(Expr* arg_op, TypeTag t)
 : UnaryExpr(EXPR_ARITH_COERCE, arg_op)
 	{

src/Expr.h

@@ -823,32 +823,6 @@ protected:
 	string field_name;
 };
 
-class RecordMatchExpr : public BinaryExpr {
-public:
-	RecordMatchExpr(Expr* op1 /* record to match */, 
-			Expr* op2 /* cases to match against */);	
-
-protected:
-	friend class Expr;
-	RecordMatchExpr()
-		{
-		pred_field_index = result_field_index =
-			priority_field_index = 0;
-		}
-
-	virtual Val* Fold(Val* v1, Val* v2) const;
-	void ExprDescribe(ODesc*) const;
-
-	DECLARE_SERIAL(RecordMatchExpr);
-
-	// The following are used to hold the field offset of
-	// $pred, $result, $priority, so the names only need to
-	// be looked up at compile-time.
-	int pred_field_index;
-	int result_field_index;
-	int priority_field_index;
-};
-
 class ArithCoerceExpr : public UnaryExpr {
 public:
 	ArithCoerceExpr(Expr* op, TypeTag t);

src/SMTP.cc

@@ -352,7 +352,6 @@ void SMTP_Analyzer::ProcessLine(int length, const char* line, bool orig)
 			const char* ext;
 			int ext_len;
 
-			line = skip_whitespace(line + ext_len, end_of_line);
 			get_word(end_of_line - line, line, ext_len, ext);
 			ProcessExtension(ext_len, ext);
 			}

src/SerialTypes.h

@@ -125,7 +125,7 @@ SERIAL_EXPR(FIELD_EXPR, 22)
 SERIAL_EXPR(HAS_FIELD_EXPR, 23)
 SERIAL_EXPR(RECORD_CONSTRUCTOR_EXPR, 24)
 SERIAL_EXPR(FIELD_ASSIGN_EXPR, 25)
-SERIAL_EXPR(RECORD_MATCH_EXPR, 26)
+// There used to be a SERIAL_EXPR(RECORD_MATCH_EXPR, 26) here
 SERIAL_EXPR(ARITH_COERCE_EXPR, 27)
 SERIAL_EXPR(RECORD_COERCE_EXPR, 28)
 SERIAL_EXPR(FLATTEN_EXPR, 29)

src/Serializer.h

@@ -125,7 +125,7 @@ protected:
 
 	// This will be increased whenever there is an incompatible change
 	// in the data format.
-	static const uint32 DATA_FORMAT_VERSION = 21;
+	static const uint32 DATA_FORMAT_VERSION = 22;
 
 	ChunkedIO* io;
 

src/parse.y

@@ -10,7 +10,7 @@
 %token TOK_CONSTANT TOK_COPY TOK_COUNT TOK_COUNTER TOK_DEFAULT TOK_DELETE
 %token TOK_DOUBLE TOK_ELSE TOK_ENUM TOK_EVENT TOK_EXPORT TOK_FILE TOK_FOR
 %token TOK_FUNCTION TOK_GLOBAL TOK_ID TOK_IF TOK_INT
-%token TOK_INTERVAL TOK_LIST TOK_LOCAL TOK_MODULE TOK_MATCH
+%token TOK_INTERVAL TOK_LIST TOK_LOCAL TOK_MODULE
 %token TOK_NEXT TOK_OF TOK_PATTERN TOK_PATTERN_TEXT
 %token TOK_PORT TOK_PRINT TOK_RECORD TOK_REDEF
 %token TOK_REMOVE_FROM TOK_RETURN TOK_SCHEDULE TOK_SET
@@ -34,7 +34,7 @@
 
 %left ',' '|'
 %right '=' TOK_ADD_TO TOK_REMOVE_FROM
-%right '?' ':' TOK_USING
+%right '?' ':'
 %left TOK_OR
 %left TOK_AND
 %nonassoc '<' '>' TOK_LE TOK_GE TOK_EQ TOK_NE
@@ -505,12 +505,6 @@ expr:
 			$$ = new VectorConstructorExpr($3);
 			}
 
-	|	TOK_MATCH expr TOK_USING expr
-			{
-			set_location(@1, @4);
-			$$ = new RecordMatchExpr($2, $4);
-			}
-
 	|	expr '(' opt_expr_list ')'
 			{
 			set_location(@1, @4);

src/rule-scan.l

@@ -18,7 +18,7 @@ WS	[ \t]+
 D	[0-9]+
 H	[0-9a-fA-F]+
 STRING	\"([^\n\"]|\\\")*\"
-ID	[0-9a-zA-Z_-]+
+ID	([0-9a-zA-Z_-]+::)*[0-9a-zA-Z_-]+
 RE	\/(\\\/)?([^/]|[^\\]\\\/)*\/
 META	\.[^ \t]+{WS}[^\n]+
 PID	([0-9a-zA-Z_-]|"::")+

src/scan.l

@@ -228,6 +228,24 @@ ESCSEQ	(\\([^\n]|[0-7]+|x[[:xdigit:]]+))
 			++yylloc.last_line;
 			}
 
+	/* IPv6 literal constant patterns */
+"["({HEX}:){7}{HEX}"]" {
+	string s(yytext+1);
+	RET_CONST(new AddrVal(s.erase(s.size()-1)))
+}
+"["0x{HEX}({HEX}|:)*"::"({HEX}|:)*"]" {
+	string s(yytext+3);
+	RET_CONST(new AddrVal(s.erase(s.size()-1)))
+}
+"["({HEX}|:)*"::"({HEX}|:)*"]" {
+	string s(yytext+1);
+	RET_CONST(new AddrVal(s.erase(s.size()-1)))
+}
+"["({HEX}|:)*"::"({HEX}|:)*({D}"."){3}{D}"]" {
+	string s(yytext+1);
+	RET_CONST(new AddrVal(s.erase(s.size()-1)))
+}
+
 [!%*/+\-,:;<=>?()\[\]{}~$|]	return yytext[0];
 
 "--"	return TOK_DECR;
@@ -273,7 +291,6 @@ int	return TOK_INT;
 interval	return TOK_INTERVAL;
 list	return TOK_LIST;
 local	return TOK_LOCAL;
-match	return TOK_MATCH;
 module	return TOK_MODULE;
 next	return TOK_NEXT;
 of	return TOK_OF;
@@ -295,7 +312,6 @@ timeout	return TOK_TIMEOUT;
 timer	return TOK_TIMER;
 type	return TOK_TYPE;
 union	return TOK_UNION;
-using	return TOK_USING;
 vector	return TOK_VECTOR;
 when	return TOK_WHEN;
 
@@ -451,11 +467,6 @@ F	RET_CONST(new Val(false, TYPE_BOOL))
 
 ({D}"."){3}{D}		RET_CONST(new AddrVal(yytext))
 
-({HEX}:){7}{HEX}	RET_CONST(new AddrVal(yytext))
-
-0x{HEX}({HEX}|:)*"::"({HEX}|:)*	RET_CONST(new AddrVal(yytext+2))
-(({D}|:)({HEX}|:)*)?"::"({HEX}|:)*	RET_CONST(new AddrVal(yytext))
-
 "0x"{HEX}+	RET_CONST(new Val(static_cast<bro_uint_t>(strtol(yytext, 0, 16)), TYPE_COUNT))
 
 {H}("."{H})+		RET_CONST(dns_mgr->LookupHost(yytext))

testing/btest/Baseline/language.ipv6-literals/output

@@ -0,0 +1,22 @@
+::1
+::ffff
+::255.255.255.255
+::10.10.255.255
+1::1
+1::a
+1::1:1
+1::1:a
+a::a
+a::1
+a::a:a
+a::a:1
+a:a::a
+aaaa::ffff
+192.168.1.100
+ffff::c0a8:164
+::192.168.1.100
+805b:2d9d:dc28::fc57:d4c8:1fff
+aaaa::bbbb
+aaaa:bbbb:cccc:dddd:eeee:ffff:1111:2222
+aaaa:bbbb:cccc:dddd:eeee:ffff:1:2222
+aaaa:bbbb:cccc:dddd:eeee:ffff:0:2222

testing/btest/Baseline/language.match-test/output

@@ -1,3 +0,0 @@
-default
-it's big
-it's really big

testing/btest/Baseline/language.match-test2/output

@@ -1 +0,0 @@
-2

testing/btest/bifs/addr_count_conversion.bro

@@ -3,7 +3,7 @@
 
 global v: index_vec;
 
-v = addr_to_counts(2001:0db8:85a3:0000:0000:8a2e:0370:7334);
+v = addr_to_counts([2001:0db8:85a3:0000:0000:8a2e:0370:7334]);
 print v;
 print counts_to_addr(v);
 v = addr_to_counts(1.2.3.4);

testing/btest/bifs/addr_to_ptr_name.bro

@@ -1,6 +1,6 @@
 # @TEST-EXEC: bro %INPUT >output
 # @TEST-EXEC: btest-diff output
 
-print addr_to_ptr_name(2607:f8b0:4009:802::1012);
+print addr_to_ptr_name([2607:f8b0:4009:802::1012]);
 print addr_to_ptr_name(74.125.225.52);
 

testing/btest/bifs/addr_version.bro

@@ -2,6 +2,6 @@
 # @TEST-EXEC: btest-diff out
 
 print is_v4_addr(1.2.3.4);
-print is_v4_addr(::1);
+print is_v4_addr([::1]);
 print is_v6_addr(1.2.3.4);
-print is_v6_addr(::1);
+print is_v6_addr([::1]);

testing/btest/bifs/to_addr.bro

@@ -17,4 +17,4 @@ test_to_addr("10.20.30.40", 10.20.30.40);
 test_to_addr("100.200.30.40", 100.200.30.40);
 test_to_addr("10.0.0.0", 10.0.0.0);
 test_to_addr("10.00.00.000", 10.0.0.0);
-test_to_addr("not an IP", ::);
+test_to_addr("not an IP", [::]);

testing/btest/bifs/to_subnet.bro

@@ -6,6 +6,6 @@ global sn: subnet;
 sn = to_subnet("10.0.0.0/8");
 print sn, sn == 10.0.0.0/8;
 sn = to_subnet("2607:f8b0::/32");
-print sn, sn == 2607:f8b0::/32;
+print sn, sn == [2607:f8b0::]/32;
 sn = to_subnet("10.0.0.0");
-print sn, sn == ::/0;
+print sn, sn == [::]/0;

testing/btest/language/ipv6-literals.bro

@@ -0,0 +1,30 @@
+# @TEST-EXEC: bro -b %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+local v: vector of addr = vector();
+
+v[|v|] = [::1];
+v[|v|] = [::ffff];
+v[|v|] = [::ffff:ffff];
+v[|v|] = [::0a0a:ffff];
+v[|v|] = [1::1];
+v[|v|] = [1::a];
+v[|v|] = [1::1:1];
+v[|v|] = [1::1:a];
+v[|v|] = [a::a];
+v[|v|] = [a::1];
+v[|v|] = [a::a:a];
+v[|v|] = [a::a:1];
+v[|v|] = [a:a::a];
+v[|v|] = [aaaa:0::ffff];
+v[|v|] = [::ffff:192.168.1.100];
+v[|v|] = [ffff::192.168.1.100];
+v[|v|] = [::192.168.1.100];
+v[|v|] = [805B:2D9D:DC28::FC57:212.200.31.255];
+v[|v|] = [0xaaaa::bbbb];
+v[|v|] = [aaaa:bbbb:cccc:dddd:eeee:ffff:1111:2222];
+v[|v|] = [aaaa:bbbb:cccc:dddd:eeee:ffff:1:2222];
+v[|v|] = [aaaa:bbbb:cccc:dddd:eeee:ffff:0:2222];
+
+for (i in v)
+    print v[i];

testing/btest/language/match-test.bro

@@ -1,20 +0,0 @@
-# @TEST-EXEC: bro %INPUT  >output 2>&1
-# @TEST-EXEC: btest-diff output
-
-global match_stuff = {
-	[$pred(a: count) = { return a > 5; },
-	 $result = "it's big",
-	 $priority = 2],
-
-	[$pred(a: count) = { return a > 15; },
-	 $result = "it's really big",
-	 $priority = 3],
-
-	[$pred(a: count) = { return T; },
-	 $result = "default",
-	 $priority = 0],
-};
-
-print match 0 using match_stuff;
-print match 10 using match_stuff;
-print match 20 using match_stuff;

testing/btest/language/match-test2.bro

@@ -1,51 +0,0 @@
-# @TEST-EXEC: bro %INPUT  >output 2>&1
-# @TEST-EXEC: btest-diff output
-
-type fakealert : record {
-     alert: string;
-};
-
-
-type match_rec : record {
-     result : count;
-     pred : function(rec : fakealert) : bool;
-     priority: count;
-};
-
-
-#global test_set : set[int] = 
-#{
-#1, 2, 3
-#};
-
-global match_set : set[match_rec] = 
-{
-  [$result = 1, $pred(a: fakealert) = { return T; }, $priority = 8 ],
-  [$result = 2, $pred(a: fakealert) = { return T; }, $priority = 9 ]
-};
-
-global al : fakealert;
-
-#global testset : set[fakealert] = 
-#{
-#       [$alert="hithere"]
-#};
-
-
-type nonalert: record {
-     alert : string;
-     pred : function(a : int) : int;
-};
-
-#global na : nonalert;
-#na$alert = "5";
-
-#al$alert = "hithere2";
-#if (al in testset)
-#   print 1;
-#else
-#   print 0;
-
-
-al$alert = "hi";
-print (match al using match_set);

testing/btest/language/sizeof.bro

@@ -20,7 +20,7 @@ type example_record: record {
 };
 
 global a:  addr = 1.2.3.4;
-global a6: addr = ::1;
+global a6: addr = [::1];
 global b:  bool = T;
 global c:  count = 10;
 global d:  double = -1.23;