Merge branch 'topic/vern/bit-ops' of https://github.com/bro/bro

* 'topic/vern/bit-ops' of https://github.com/bro/bro:
  documentation clarification for "p1 | p2"
  documentation for bitwise operators
  document the '|' operator for patterns
  test suite for bitwise operators brief NEWS blurb allow for "counter" operands (does anyone still use these?) for one (but not both) of the bitwise operands
  bitwise operations for "count" types implemented
  Starting branch for supporting bit operations on count's.

CHANGES

@@ -1,4 +1,9 @@
 
+2.5-679 | 2018-06-21 16:00:48 -0500
+
+  * Add support for bitwise operations (&, |, ^, ~) on "count" values.
+    (Vern Paxson)
+
 2.5-671 | 2018-06-21 11:55:39 -0500
 
   * Add ability for BroControl to skip cluster setup (Corelight)
@@ -55,7 +60,6 @@
 2.5-650 | 2018-06-06 16:20:18 -0500
 
   * Improve Broker performance (Corelight)
->>>>>>> b51e6f39ddc641811d4875cda4543d3a60fb5a63
 
 2.5-648 | 2018-06-05 17:32:47 -0500
 

NEWS

@@ -245,6 +245,10 @@ New Functionality
 - Bro can now decrypt Kerberos tickets, and retrieve the authentication from
   them, given a suitable keytab file.
 
+- Added support for bitwise operations on "count" values.  '&', '|' and
+  '^' are binary "and", "or" and "xor" operators, and '~' is a unary
+  ones-complement operator.
+
 Changed Functionality
 ---------------------
 

VERSION

@@ -1 +1 @@
-2.5-671
+2.5-679

doc/script-reference/types.rst

@@ -91,6 +91,10 @@ Here is a more detailed description of each type:
     type, but a unary plus or minus applied to a "count" results in an
     "int".
 
+    In addition, "count" types support bitwise operations.  You can use
+    ``&``, ``|``, and ``^`` for bitwise ``and'', ``or'', and ``xor''.  You
+    can also use ``~`` for bitwise (one's) complement.
+
 .. bro:type:: double
 
     A numeric type representing a double-precision floating-point
@@ -233,6 +237,14 @@ Here is a more detailed description of each type:
     is false since "oob" does not appear at the start of "foobar".  The
     ``!in`` operator would yield the negation of ``in``.
 
+    Finally, you can create a disjunction (either-or) of two literal patterns
+    using the ``|`` operator.  For example::
+
+	/foo/ | /bar/ in "foobar"
+
+    yields true, like in the similar example above.  (This does not presently
+    work for variables whose values are patterns, however.)
+
 .. bro:type:: port
 
     A type representing transport-level port numbers (besides TCP and
@@ -585,6 +597,9 @@ Here is a more detailed description of each type:
     The resulting vector of bool is the logical "and" (or logical "or") of
     each element of the operand vectors.
 
+    Vectors of type ``count`` can also be operands for the bitwise and/or/xor
+    operators, ``&``, ``|`` and ``^``.
+
     See the :bro:keyword:`for` statement for info on how to iterate over
     the elements in a vector.
 

src/Expr.cc

@@ -21,8 +21,10 @@ const char* expr_name(BroExprTag t)
 	static const char* expr_names[int(NUM_EXPRS)] = {
 		"name", "const",
 		"(*)",
-		"++", "--", "!", "+", "-",
-		"+", "-", "+=", "-=", "*", "/", "%", "&&", "||",
+		"++", "--", "!", "~", "+", "-",
+		"+", "-", "+=", "-=", "*", "/", "%",
+		"&", "|", "^",
+		"&&", "||",
 		"<", "<=", "==", "!=", ">=", ">", "?:", "ref",
 		"=", "~", "[]", "$", "?$", "[=]",
 		"table()", "set()", "vector()",
@@ -702,6 +704,12 @@ Val* BinaryExpr::Fold(Val* v1, Val* v2) const
 	else \
 		Internal("bad type in BinaryExpr::Fold");
 
+#define DO_UINT_FOLD(op) \
+	if ( is_unsigned ) \
+		u3 = u1 op u2; \
+	else \
+		Internal("bad type in BinaryExpr::Fold");
+
 #define DO_FOLD(op) \
 	if ( is_integral ) \
 		i3 = i1 op i2; \
@@ -775,8 +783,12 @@ Val* BinaryExpr::Fold(Val* v1, Val* v2) const
 
 		break;
 
-	case EXPR_AND:		DO_INT_FOLD(&&); break;
-	case EXPR_OR:		DO_INT_FOLD(||); break;
+	case EXPR_AND:		DO_UINT_FOLD(&); break;
+	case EXPR_OR:		DO_UINT_FOLD(|); break;
+	case EXPR_XOR:		DO_UINT_FOLD(^); break;
+
+	case EXPR_AND_AND:	DO_INT_FOLD(&&); break;
+	case EXPR_OR_OR:	DO_INT_FOLD(||); break;
 
 	case EXPR_LT:		DO_INT_VAL_FOLD(<); break;
 	case EXPR_LE:		DO_INT_VAL_FOLD(<=); break;
@@ -1077,6 +1089,39 @@ bool IncrExpr::DoUnserialize(UnserialInfo* info)
 	return true;
 	}
 
+ComplementExpr::ComplementExpr(Expr* arg_op) : UnaryExpr(EXPR_COMPLEMENT, arg_op)
+	{
+	if ( IsError() )
+		return;
+
+	BroType* t = op->Type();
+	TypeTag bt = t->Tag();
+
+	if ( bt != TYPE_COUNT )
+		ExprError("requires \"count\" operand");
+	else
+		SetType(base_type(TYPE_COUNT));
+	}
+
+Val* ComplementExpr::Fold(Val* v) const
+	{
+	return new Val(~ v->InternalUnsigned(), type->Tag());
+	}
+
+IMPLEMENT_SERIAL(ComplementExpr, SER_COMPLEMENT_EXPR);
+
+bool ComplementExpr::DoSerialize(SerialInfo* info) const
+	{
+	DO_SERIALIZE(SER_COMPLEMENT_EXPR, UnaryExpr);
+	return true;
+	}
+
+bool ComplementExpr::DoUnserialize(UnserialInfo* info)
+	{
+	DO_UNSERIALIZE(UnaryExpr);
+	return true;
+	}
+
 NotExpr::NotExpr(Expr* arg_op) : UnaryExpr(EXPR_NOT, arg_op)
 	{
 	if ( IsError() )
@@ -1670,14 +1715,14 @@ Val* BoolExpr::DoSingleEval(Frame* f, Val* v1, Expr* op2) const
 		RE_Matcher* re1 = v1->AsPattern();
 		RE_Matcher* re2 = v2->AsPattern();
 
-		RE_Matcher* res = tag == EXPR_AND ?
+		RE_Matcher* res = tag == EXPR_AND_AND ?
 			RE_Matcher_conjunction(re1, re2) :
 			RE_Matcher_disjunction(re1, re2);
 
 		return new PatternVal(res);
 		}
 
-	if ( tag == EXPR_AND )
+	if ( tag == EXPR_AND_AND )
 		{
 		if ( v1->IsZero() )
 			return v1;
@@ -1741,8 +1786,8 @@ Val* BoolExpr::Eval(Frame* f) const
 
 		VectorVal* result = 0;
 
-		// It's either and EXPR_AND or an EXPR_OR.
-		bool is_and = (tag == EXPR_AND);
+		// It's either and EXPR_AND_AND or an EXPR_OR_OR.
+		bool is_and = (tag == EXPR_AND_AND);
 
 		if ( scalar_v->IsZero() == is_and )
 			{
@@ -1783,7 +1828,7 @@ Val* BoolExpr::Eval(Frame* f) const
 		Val* op2 = vec_v2->Lookup(i);
 		if ( op1 && op2 )
 			{
-			bool local_result = (tag == EXPR_AND) ?
+			bool local_result = (tag == EXPR_AND_AND) ?
 				(! op1->IsZero() && ! op2->IsZero()) :
 				(! op1->IsZero() || ! op2->IsZero());
 
@@ -1813,6 +1858,49 @@ bool BoolExpr::DoUnserialize(UnserialInfo* info)
 	return true;
 	}
 
+BitExpr::BitExpr(BroExprTag arg_tag, Expr* arg_op1, Expr* arg_op2)
+: BinaryExpr(arg_tag, arg_op1, arg_op2)
+	{
+	if ( IsError() )
+		return;
+
+	TypeTag bt1 = op1->Type()->Tag();
+	if ( IsVector(bt1) )
+		bt1 = op1->Type()->AsVectorType()->YieldType()->Tag();
+
+	TypeTag bt2 = op2->Type()->Tag();
+	if ( IsVector(bt2) )
+		bt2 = op2->Type()->AsVectorType()->YieldType()->Tag();
+
+	if ( (bt1 == TYPE_COUNT || bt1 == TYPE_COUNTER) &&
+	     (bt2 == TYPE_COUNT || bt2 == TYPE_COUNTER) )
+		{
+		if ( bt1 == TYPE_COUNTER && bt2 == TYPE_COUNTER )
+			ExprError("cannot apply a bitwise operator to two \"counter\" operands");
+		else if ( is_vector(op1) || is_vector(op2) )
+			SetType(new VectorType(base_type(TYPE_COUNT)));
+		else
+			SetType(base_type(TYPE_COUNT));
+		}
+
+	else
+		ExprError("requires \"count\" operands");
+	}
+
+IMPLEMENT_SERIAL(BitExpr, SER_BIT_EXPR);
+
+bool BitExpr::DoSerialize(SerialInfo* info) const
+	{
+	DO_SERIALIZE(SER_BIT_EXPR, BinaryExpr);
+	return true;
+	}
+
+bool BitExpr::DoUnserialize(UnserialInfo* info)
+	{
+	DO_UNSERIALIZE(BinaryExpr);
+	return true;
+	}
+
 EqExpr::EqExpr(BroExprTag arg_tag, Expr* arg_op1, Expr* arg_op2)
 : BinaryExpr(arg_tag, arg_op1, arg_op2)
 	{

src/Expr.h

@@ -17,10 +17,13 @@ typedef enum {
 	EXPR_ANY = -1,
 	EXPR_NAME, EXPR_CONST,
 	EXPR_CLONE,
-	EXPR_INCR, EXPR_DECR, EXPR_NOT, EXPR_POSITIVE, EXPR_NEGATE,
+	EXPR_INCR, EXPR_DECR,
+	EXPR_NOT, EXPR_COMPLEMENT,
+	EXPR_POSITIVE, EXPR_NEGATE,
 	EXPR_ADD, EXPR_SUB, EXPR_ADD_TO, EXPR_REMOVE_FROM,
 	EXPR_TIMES, EXPR_DIVIDE, EXPR_MOD,
-	EXPR_AND, EXPR_OR,
+	EXPR_AND, EXPR_OR, EXPR_XOR,
+	EXPR_AND_AND, EXPR_OR_OR,
 	EXPR_LT, EXPR_LE, EXPR_EQ, EXPR_NE, EXPR_GE, EXPR_GT,
 	EXPR_COND,
 	EXPR_REF,
@@ -379,6 +382,19 @@ protected:
 	DECLARE_SERIAL(IncrExpr);
 };
 
+class ComplementExpr : public UnaryExpr {
+public:
+	explicit ComplementExpr(Expr* op);
+
+protected:
+	friend class Expr;
+	ComplementExpr()	{ }
+
+	Val* Fold(Val* v) const override;
+
+	DECLARE_SERIAL(ComplementExpr);
+};
+
 class NotExpr : public UnaryExpr {
 public:
 	explicit NotExpr(Expr* op);
@@ -532,6 +548,17 @@ protected:
 	DECLARE_SERIAL(BoolExpr);
 };
 
+class BitExpr : public BinaryExpr {
+public:
+	BitExpr(BroExprTag tag, Expr* op1, Expr* op2);
+
+protected:
+	friend class Expr;
+	BitExpr()	{ }
+
+	DECLARE_SERIAL(BitExpr);
+};
+
 class EqExpr : public BinaryExpr {
 public:
 	EqExpr(BroExprTag tag, Expr* op1, Expr* op2);

src/SerialObj.h

@@ -3,7 +3,7 @@
 // How to make objects of class Foo serializable:
 //
 //    1. Derive Foo (directly or indirectly) from SerialObj.
-//    2. Add a SER_FOO constant to SerialTypes below.
+//    2. Add a SER_FOO constant to SerialTypes in SerialTypes.h.
 //    3. Add DECLARE_SERIAL(Foo) into class definition.
 //    4. Add a (preferably protected) default ctor if it doesn't already exist.
 //    5. For non-abstract classes, add IMPLEMENT_SERIAL(Foo, SER_FOO) to *.cc

src/SerialTypes.h

@@ -164,6 +164,8 @@ SERIAL_EXPR(TABLE_COERCE_EXPR, 43)
 SERIAL_EXPR(VECTOR_COERCE_EXPR, 44)
 SERIAL_EXPR(CAST_EXPR, 45)
 SERIAL_EXPR(IS_EXPR_, 46) // Name conflict with internal SER_IS_EXPR constant.
+SERIAL_EXPR(BIT_EXPR, 47)
+SERIAL_EXPR(COMPLEMENT_EXPR, 48)
 
 #define SERIAL_STMT(name, val) SERIAL_CONST(name, val, STMT)
 SERIAL_STMT(STMT, 1)

src/parse.y

@@ -34,18 +34,21 @@
 
 %token TOK_NO_TEST
 
-%left ',' '|'
+%left ','
 %right '=' TOK_ADD_TO TOK_REMOVE_FROM
 %right '?' ':'
-%left TOK_OR
-%left TOK_AND
+%left TOK_OR_OR
+%left TOK_AND_AND
 %nonassoc TOK_HOOK
 %nonassoc '<' '>' TOK_LE TOK_GE TOK_EQ TOK_NE
 %left TOK_IN TOK_NOT_IN
+%left '|'
+%left '^'
+%left '&'
 %left '+' '-'
 %left '*' '/' '%'
 %left TOK_INCR TOK_DECR
-%right '!'
+%right '!' '~'
 %left '$' '[' ']' '(' ')' TOK_HAS_FIELD TOK_HAS_ATTR
 %nonassoc TOK_AS TOK_IS
 
@@ -338,6 +341,12 @@ expr:
 			$$ = new NotExpr($2);
 			}
 
+	|	'~' expr
+			{
+			set_location(@1, @2);
+			$$ = new ComplementExpr($2);
+			}
+
 	|	'-' expr	%prec '!'
 			{
 			set_location(@1, @2);
@@ -392,16 +401,34 @@ expr:
 			$$ = new ModExpr($1, $3);
 			}
 
-	|	expr TOK_AND expr
+	|	expr '&' expr
 			{
 			set_location(@1, @3);
-			$$ = new BoolExpr(EXPR_AND, $1, $3);
+			$$ = new BitExpr(EXPR_AND, $1, $3);
 			}
 
-	|	expr TOK_OR expr
+	|	expr '|' expr
 			{
 			set_location(@1, @3);
-			$$ = new BoolExpr(EXPR_OR, $1, $3);
+			$$ = new BitExpr(EXPR_OR, $1, $3);
+			}
+
+	|	expr '^' expr
+			{
+			set_location(@1, @3);
+			$$ = new BitExpr(EXPR_XOR, $1, $3);
+			}
+
+	|	expr TOK_AND_AND expr
+			{
+			set_location(@1, @3);
+			$$ = new BoolExpr(EXPR_AND_AND, $1, $3);
+			}
+
+	|	expr TOK_OR_OR expr
+			{
+			set_location(@1, @3);
+			$$ = new BoolExpr(EXPR_OR_OR, $1, $3);
 			}
 
 	|	expr TOK_EQ expr
@@ -704,7 +731,7 @@ expr:
 			$$ = new ConstExpr(new PatternVal($1));
 			}
 
-	|       '|' expr '|'
+	|       '|' expr '|'	%prec '('
 			{
 			set_location(@1, @3);
 			$$ = new SizeExpr($2);

src/scan.l

@@ -193,7 +193,7 @@ ESCSEQ	(\\([^\n]|[0-7]+|x[[:xdigit:]]+))
 	RET_CONST(new SubNetVal(IPPrefix(IPAddr(ip), len)))
 }
 
-[!%*/+\-,:;<=>?()\[\]{}~$|]	return yytext[0];
+[!%*/+\-,:;<=>?()\[\]{}~$|&^]	return yytext[0];
 
 "--"	return TOK_DECR;
 "++"	return TOK_INCR;
@@ -206,8 +206,8 @@ ESCSEQ	(\\([^\n]|[0-7]+|x[[:xdigit:]]+))
 ">="	return TOK_GE;
 "<="	return TOK_LE;
 
-"&&"	return TOK_AND;
-"||"	return TOK_OR;
+"&&"	return TOK_AND_AND;
+"||"	return TOK_OR_OR;
 
 add	return TOK_ADD;
 addr	return TOK_ADDR;

testing/btest/Baseline/language.count/out

@@ -14,5 +14,16 @@ modulus operator (PASS)
 division operator (PASS)
 assignment operator (PASS)
 assignment operator (PASS)
+bitwise and (PASS)
+bitwise and (PASS)
+bitwise and (PASS)
+bitwise or (PASS)
+bitwise or (PASS)
+bitwise or (PASS)
+bitwise xor (PASS)
+bitwise xor (PASS)
+bitwise xor (PASS)
+bitwise complement (PASS)
+bitwise complement (PASS)
 max count value = 18446744073709551615 (PASS)
 max count value = 18446744073709551615 (PASS)

testing/btest/language/count.bro

@@ -47,6 +47,17 @@ event bro_init()
 	test_case( "assignment operator", c2 == 8 );
 	c2 -= 2;
 	test_case( "assignment operator", c2 == 6 );
+	test_case( "bitwise and", c2 & 0x4 == 0x4 );
+	test_case( "bitwise and", c4 & 0x4 == 0x4 );
+	test_case( "bitwise and", c8 & 0x4 == 0x0 );
+	test_case( "bitwise or", c2 | 0x4 == c2 );
+	test_case( "bitwise or", c4 | 0x4 == c4 );
+	test_case( "bitwise or", c8 | 0x4 == c7 );
+	test_case( "bitwise xor", c7 ^ 0x4 == c8 );
+	test_case( "bitwise xor", c4 ^ 0x4 == 251 );
+	test_case( "bitwise xor", c8 ^ 0x4 == c7 );
+	test_case( "bitwise complement", ~c6 == 0 );
+	test_case( "bitwise complement", ~~c4 == c4 );
 
 	# Max. value tests