From d9dc8d69d7125474f6eae06718f0b91ddef9e1f2 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Thu, 26 Apr 2018 10:51:43 -0700 Subject: [PATCH 1/6] Starting branch for supporting bit operations on count's. This check-in just adds the operators to the lexer and migrates the existing operator names internally. --- src/Expr.cc | 18 ++++++++++-------- src/Expr.h | 3 ++- src/parse.y | 12 ++++++------ src/scan.l | 6 +++--- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/src/Expr.cc b/src/Expr.cc index a0456fda31..06fcf448e1 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -24,7 +24,9 @@ const char* expr_name(BroExprTag t) "name", "const", "(*)", "++", "--", "!", "+", "-", - "+", "-", "+=", "-=", "*", "/", "%", "&&", "||", + "+", "-", "+=", "-=", "*", "/", "%", + "&", "|", "^", + "&&", "||", "<", "<=", "==", "!=", ">=", ">", "?:", "ref", "=", "~", "[]", "$", "?$", "[=]", "table()", "set()", "vector()", @@ -777,8 +779,8 @@ Val* BinaryExpr::Fold(Val* v1, Val* v2) const break; - case EXPR_AND: DO_INT_FOLD(&&); break; - case EXPR_OR: DO_INT_FOLD(||); break; + case EXPR_AND_AND: DO_INT_FOLD(&&); break; + case EXPR_OR_OR: DO_INT_FOLD(||); break; case EXPR_LT: DO_INT_VAL_FOLD(<); break; case EXPR_LE: DO_INT_VAL_FOLD(<=); break; @@ -1672,14 +1674,14 @@ Val* BoolExpr::DoSingleEval(Frame* f, Val* v1, Expr* op2) const RE_Matcher* re1 = v1->AsPattern(); RE_Matcher* re2 = v2->AsPattern(); - RE_Matcher* res = tag == EXPR_AND ? + RE_Matcher* res = tag == EXPR_AND_AND ? RE_Matcher_conjunction(re1, re2) : RE_Matcher_disjunction(re1, re2); return new PatternVal(res); } - if ( tag == EXPR_AND ) + if ( tag == EXPR_AND_AND ) { if ( v1->IsZero() ) return v1; @@ -1743,8 +1745,8 @@ Val* BoolExpr::Eval(Frame* f) const VectorVal* result = 0; - // It's either and EXPR_AND or an EXPR_OR. - bool is_and = (tag == EXPR_AND); + // It's either and EXPR_AND_AND or an EXPR_OR_OR. + bool is_and = (tag == EXPR_AND_AND); if ( scalar_v->IsZero() == is_and ) { @@ -1785,7 +1787,7 @@ Val* BoolExpr::Eval(Frame* f) const Val* op2 = vec_v2->Lookup(i); if ( op1 && op2 ) { - bool local_result = (tag == EXPR_AND) ? + bool local_result = (tag == EXPR_AND_AND) ? (! op1->IsZero() && ! op2->IsZero()) : (! op1->IsZero() || ! op2->IsZero()); diff --git a/src/Expr.h b/src/Expr.h index d50506f493..89d44c1cb4 100644 --- a/src/Expr.h +++ b/src/Expr.h @@ -20,7 +20,8 @@ typedef enum { EXPR_INCR, EXPR_DECR, EXPR_NOT, EXPR_POSITIVE, EXPR_NEGATE, EXPR_ADD, EXPR_SUB, EXPR_ADD_TO, EXPR_REMOVE_FROM, EXPR_TIMES, EXPR_DIVIDE, EXPR_MOD, - EXPR_AND, EXPR_OR, + EXPR_AND, EXPR_OR, EXPR_XOR, + EXPR_AND_AND, EXPR_OR_OR, EXPR_LT, EXPR_LE, EXPR_EQ, EXPR_NE, EXPR_GE, EXPR_GT, EXPR_COND, EXPR_REF, diff --git a/src/parse.y b/src/parse.y index 89bb5adbee..90d4c71309 100644 --- a/src/parse.y +++ b/src/parse.y @@ -34,8 +34,8 @@ %left ',' '|' %right '=' TOK_ADD_TO TOK_REMOVE_FROM %right '?' ':' -%left TOK_OR -%left TOK_AND +%left TOK_OR_OR +%left TOK_AND_AND %nonassoc TOK_HOOK %nonassoc '<' '>' TOK_LE TOK_GE TOK_EQ TOK_NE %left TOK_IN TOK_NOT_IN @@ -388,16 +388,16 @@ expr: $$ = new ModExpr($1, $3); } - | expr TOK_AND expr + | expr TOK_AND_AND expr { set_location(@1, @3); - $$ = new BoolExpr(EXPR_AND, $1, $3); + $$ = new BoolExpr(EXPR_AND_AND, $1, $3); } - | expr TOK_OR expr + | expr TOK_OR_OR expr { set_location(@1, @3); - $$ = new BoolExpr(EXPR_OR, $1, $3); + $$ = new BoolExpr(EXPR_OR_OR, $1, $3); } | expr TOK_EQ expr diff --git a/src/scan.l b/src/scan.l index 05baea5735..e848b78a43 100644 --- a/src/scan.l +++ b/src/scan.l @@ -193,7 +193,7 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) RET_CONST(new SubNetVal(IPPrefix(IPAddr(ip), len))) } -[!%*/+\-,:;<=>?()\[\]{}~$|] return yytext[0]; +[!%*/+\-,:;<=>?()\[\]{}~$|&^] return yytext[0]; "--" return TOK_DECR; "++" return TOK_INCR; @@ -206,8 +206,8 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) ">=" return TOK_GE; "<=" return TOK_LE; -"&&" return TOK_AND; -"||" return TOK_OR; +"&&" return TOK_AND_AND; +"||" return TOK_OR_OR; add return TOK_ADD; addr return TOK_ADDR; From 1658931af1f5c35533457d15735c65462d1e8af1 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Thu, 26 Apr 2018 12:38:24 -0700 Subject: [PATCH 2/6] bitwise operations for "count" types implemented --- src/Expr.cc | 85 ++++++++++++++++++++++++++++++++++++++++++++++- src/Expr.h | 28 +++++++++++++++- src/SerialObj.h | 2 +- src/SerialTypes.h | 2 ++ src/parse.y | 33 ++++++++++++++++-- 5 files changed, 144 insertions(+), 6 deletions(-) diff --git a/src/Expr.cc b/src/Expr.cc index 06fcf448e1..5f79d08dbe 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -23,7 +23,7 @@ const char* expr_name(BroExprTag t) static const char* expr_names[int(NUM_EXPRS)] = { "name", "const", "(*)", - "++", "--", "!", "+", "-", + "++", "--", "!", "~", "+", "-", "+", "-", "+=", "-=", "*", "/", "%", "&", "|", "^", "&&", "||", @@ -706,6 +706,12 @@ Val* BinaryExpr::Fold(Val* v1, Val* v2) const else \ Internal("bad type in BinaryExpr::Fold"); +#define DO_UINT_FOLD(op) \ + if ( is_unsigned ) \ + u3 = u1 op u2; \ + else \ + Internal("bad type in BinaryExpr::Fold"); + #define DO_FOLD(op) \ if ( is_integral ) \ i3 = i1 op i2; \ @@ -779,6 +785,10 @@ Val* BinaryExpr::Fold(Val* v1, Val* v2) const break; + case EXPR_AND: DO_UINT_FOLD(&); break; + case EXPR_OR: DO_UINT_FOLD(|); break; + case EXPR_XOR: DO_UINT_FOLD(^); break; + case EXPR_AND_AND: DO_INT_FOLD(&&); break; case EXPR_OR_OR: DO_INT_FOLD(||); break; @@ -1081,6 +1091,39 @@ bool IncrExpr::DoUnserialize(UnserialInfo* info) return true; } +ComplementExpr::ComplementExpr(Expr* arg_op) : UnaryExpr(EXPR_COMPLEMENT, arg_op) + { + if ( IsError() ) + return; + + BroType* t = op->Type(); + TypeTag bt = t->Tag(); + + if ( bt != TYPE_COUNT ) + ExprError("requires \"count\" operand"); + else + SetType(base_type(TYPE_COUNT)); + } + +Val* ComplementExpr::Fold(Val* v) const + { + return new Val(~ v->InternalUnsigned(), type->Tag()); + } + +IMPLEMENT_SERIAL(ComplementExpr, SER_COMPLEMENT_EXPR); + +bool ComplementExpr::DoSerialize(SerialInfo* info) const + { + DO_SERIALIZE(SER_COMPLEMENT_EXPR, UnaryExpr); + return true; + } + +bool ComplementExpr::DoUnserialize(UnserialInfo* info) + { + DO_UNSERIALIZE(UnaryExpr); + return true; + } + NotExpr::NotExpr(Expr* arg_op) : UnaryExpr(EXPR_NOT, arg_op) { if ( IsError() ) @@ -1817,6 +1860,46 @@ bool BoolExpr::DoUnserialize(UnserialInfo* info) return true; } +BitExpr::BitExpr(BroExprTag arg_tag, Expr* arg_op1, Expr* arg_op2) +: BinaryExpr(arg_tag, arg_op1, arg_op2) + { + if ( IsError() ) + return; + + TypeTag bt1 = op1->Type()->Tag(); + if ( IsVector(bt1) ) + bt1 = op1->Type()->AsVectorType()->YieldType()->Tag(); + + TypeTag bt2 = op2->Type()->Tag(); + if ( IsVector(bt2) ) + bt2 = op2->Type()->AsVectorType()->YieldType()->Tag(); + + if ( bt1 == TYPE_COUNT && bt2 == bt1 ) + { + if ( is_vector(op1) || is_vector(op2) ) + SetType(new VectorType(base_type(bt1))); + else + SetType(base_type(bt1)); + } + + else + ExprError("requires \"count\" operands"); + } + +IMPLEMENT_SERIAL(BitExpr, SER_BIT_EXPR); + +bool BitExpr::DoSerialize(SerialInfo* info) const + { + DO_SERIALIZE(SER_BIT_EXPR, BinaryExpr); + return true; + } + +bool BitExpr::DoUnserialize(UnserialInfo* info) + { + DO_UNSERIALIZE(BinaryExpr); + return true; + } + EqExpr::EqExpr(BroExprTag arg_tag, Expr* arg_op1, Expr* arg_op2) : BinaryExpr(arg_tag, arg_op1, arg_op2) { diff --git a/src/Expr.h b/src/Expr.h index 89d44c1cb4..8e8b6cc96b 100644 --- a/src/Expr.h +++ b/src/Expr.h @@ -17,7 +17,9 @@ typedef enum { EXPR_ANY = -1, EXPR_NAME, EXPR_CONST, EXPR_CLONE, - EXPR_INCR, EXPR_DECR, EXPR_NOT, EXPR_POSITIVE, EXPR_NEGATE, + EXPR_INCR, EXPR_DECR, + EXPR_NOT, EXPR_COMPLEMENT, + EXPR_POSITIVE, EXPR_NEGATE, EXPR_ADD, EXPR_SUB, EXPR_ADD_TO, EXPR_REMOVE_FROM, EXPR_TIMES, EXPR_DIVIDE, EXPR_MOD, EXPR_AND, EXPR_OR, EXPR_XOR, @@ -378,6 +380,19 @@ protected: DECLARE_SERIAL(IncrExpr); }; +class ComplementExpr : public UnaryExpr { +public: + explicit ComplementExpr(Expr* op); + +protected: + friend class Expr; + ComplementExpr() { } + + Val* Fold(Val* v) const override; + + DECLARE_SERIAL(ComplementExpr); +}; + class NotExpr : public UnaryExpr { public: explicit NotExpr(Expr* op); @@ -531,6 +546,17 @@ protected: DECLARE_SERIAL(BoolExpr); }; +class BitExpr : public BinaryExpr { +public: + BitExpr(BroExprTag tag, Expr* op1, Expr* op2); + +protected: + friend class Expr; + BitExpr() { } + + DECLARE_SERIAL(BitExpr); +}; + class EqExpr : public BinaryExpr { public: EqExpr(BroExprTag tag, Expr* op1, Expr* op2); diff --git a/src/SerialObj.h b/src/SerialObj.h index 77dc28aefd..b502414f71 100644 --- a/src/SerialObj.h +++ b/src/SerialObj.h @@ -3,7 +3,7 @@ // How to make objects of class Foo serializable: // // 1. Derive Foo (directly or indirectly) from SerialObj. -// 2. Add a SER_FOO constant to SerialTypes below. +// 2. Add a SER_FOO constant to SerialTypes in SerialTypes.h. // 3. Add DECLARE_SERIAL(Foo) into class definition. // 4. Add a (preferably protected) default ctor if it doesn't already exist. // 5. For non-abstract classes, add IMPLEMENT_SERIAL(Foo, SER_FOO) to *.cc diff --git a/src/SerialTypes.h b/src/SerialTypes.h index 8a1a2abf51..c5b25d32fb 100644 --- a/src/SerialTypes.h +++ b/src/SerialTypes.h @@ -162,6 +162,8 @@ SERIAL_EXPR(SET_CONSTRUCTOR_EXPR, 41) SERIAL_EXPR(VECTOR_CONSTRUCTOR_EXPR, 42) SERIAL_EXPR(TABLE_COERCE_EXPR, 43) SERIAL_EXPR(VECTOR_COERCE_EXPR, 44) +SERIAL_EXPR(BIT_EXPR, 45) +SERIAL_EXPR(COMPLEMENT_EXPR, 46) #define SERIAL_STMT(name, val) SERIAL_CONST(name, val, STMT) SERIAL_STMT(STMT, 1) diff --git a/src/parse.y b/src/parse.y index 90d4c71309..6f7a43ae7f 100644 --- a/src/parse.y +++ b/src/parse.y @@ -31,7 +31,7 @@ %token TOK_NO_TEST -%left ',' '|' +%left ',' %right '=' TOK_ADD_TO TOK_REMOVE_FROM %right '?' ':' %left TOK_OR_OR @@ -39,10 +39,13 @@ %nonassoc TOK_HOOK %nonassoc '<' '>' TOK_LE TOK_GE TOK_EQ TOK_NE %left TOK_IN TOK_NOT_IN +%left '|' +%left '^' +%left '&' %left '+' '-' %left '*' '/' '%' %left TOK_INCR TOK_DECR -%right '!' +%right '!' '~' %left '$' '[' ']' '(' ')' TOK_HAS_FIELD TOK_HAS_ATTR %type opt_no_test opt_no_test_block opt_deprecated @@ -334,6 +337,12 @@ expr: $$ = new NotExpr($2); } + | '~' expr + { + set_location(@1, @2); + $$ = new ComplementExpr($2); + } + | '-' expr %prec '!' { set_location(@1, @2); @@ -388,6 +397,24 @@ expr: $$ = new ModExpr($1, $3); } + | expr '&' expr + { + set_location(@1, @3); + $$ = new BitExpr(EXPR_AND, $1, $3); + } + + | expr '|' expr + { + set_location(@1, @3); + $$ = new BitExpr(EXPR_OR, $1, $3); + } + + | expr '^' expr + { + set_location(@1, @3); + $$ = new BitExpr(EXPR_XOR, $1, $3); + } + | expr TOK_AND_AND expr { set_location(@1, @3); @@ -700,7 +727,7 @@ expr: $$ = new ConstExpr(new PatternVal($1)); } - | '|' expr '|' + | '|' expr '|' %prec '(' { set_location(@1, @3); $$ = new SizeExpr($2); From 58ffd61dcc2874d07089b325850c670d6f622e14 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Thu, 26 Apr 2018 13:25:04 -0700 Subject: [PATCH 3/6] test suite for bitwise operators brief NEWS blurb allow for "counter" operands (does anyone still use these?) for one (but not both) of the bitwise operands --- NEWS | 4 ++++ src/Expr.cc | 11 +++++++---- testing/btest/Baseline/language.count/out | 11 +++++++++++ testing/btest/language/count.bro | 11 +++++++++++ 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index aa43682047..eeeda2a42c 100644 --- a/NEWS +++ b/NEWS @@ -146,6 +146,10 @@ New Functionality - Added new SMB events: smb1_transaction_secondary_request, smb1_transaction2_secondary_request, smb1_transaction_response +- Added support for bitwise operations on "count" values. '&', '|' and + '^' are binary "and", "or" and "xor" operators, and '~' is a unary + ones-complement operator. + Changed Functionality --------------------- diff --git a/src/Expr.cc b/src/Expr.cc index 5f79d08dbe..f5c8e66b50 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -1874,12 +1874,15 @@ BitExpr::BitExpr(BroExprTag arg_tag, Expr* arg_op1, Expr* arg_op2) if ( IsVector(bt2) ) bt2 = op2->Type()->AsVectorType()->YieldType()->Tag(); - if ( bt1 == TYPE_COUNT && bt2 == bt1 ) + if ( (bt1 == TYPE_COUNT || bt1 == TYPE_COUNTER) && + (bt2 == TYPE_COUNT || bt2 == TYPE_COUNTER) ) { - if ( is_vector(op1) || is_vector(op2) ) - SetType(new VectorType(base_type(bt1))); + if ( bt1 == TYPE_COUNTER && bt2 == TYPE_COUNTER ) + ExprError("cannot apply a bitwise operator to two \"counter\" operands"); + else if ( is_vector(op1) || is_vector(op2) ) + SetType(new VectorType(base_type(TYPE_COUNT))); else - SetType(base_type(bt1)); + SetType(base_type(TYPE_COUNT)); } else diff --git a/testing/btest/Baseline/language.count/out b/testing/btest/Baseline/language.count/out index 4ef65b6098..f1e1eef587 100644 --- a/testing/btest/Baseline/language.count/out +++ b/testing/btest/Baseline/language.count/out @@ -14,5 +14,16 @@ modulus operator (PASS) division operator (PASS) assignment operator (PASS) assignment operator (PASS) +bitwise and (PASS) +bitwise and (PASS) +bitwise and (PASS) +bitwise or (PASS) +bitwise or (PASS) +bitwise or (PASS) +bitwise xor (PASS) +bitwise xor (PASS) +bitwise xor (PASS) +bitwise complement (PASS) +bitwise complement (PASS) max count value = 18446744073709551615 (PASS) max count value = 18446744073709551615 (PASS) diff --git a/testing/btest/language/count.bro b/testing/btest/language/count.bro index b0972e29fa..39a3786dfb 100644 --- a/testing/btest/language/count.bro +++ b/testing/btest/language/count.bro @@ -47,6 +47,17 @@ event bro_init() test_case( "assignment operator", c2 == 8 ); c2 -= 2; test_case( "assignment operator", c2 == 6 ); + test_case( "bitwise and", c2 & 0x4 == 0x4 ); + test_case( "bitwise and", c4 & 0x4 == 0x4 ); + test_case( "bitwise and", c8 & 0x4 == 0x0 ); + test_case( "bitwise or", c2 | 0x4 == c2 ); + test_case( "bitwise or", c4 | 0x4 == c4 ); + test_case( "bitwise or", c8 | 0x4 == c7 ); + test_case( "bitwise xor", c7 ^ 0x4 == c8 ); + test_case( "bitwise xor", c4 ^ 0x4 == 251 ); + test_case( "bitwise xor", c8 ^ 0x4 == c7 ); + test_case( "bitwise complement", ~c6 == 0 ); + test_case( "bitwise complement", ~~c4 == c4 ); # Max. value tests From 1ca5a5a16dd6e774b1173006c26c8e7b68c6cfb4 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Thu, 26 Apr 2018 14:57:55 -0700 Subject: [PATCH 4/6] document the '|' operator for patterns --- doc/script-reference/types.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/script-reference/types.rst b/doc/script-reference/types.rst index 651ebfb411..3d6f74aef0 100644 --- a/doc/script-reference/types.rst +++ b/doc/script-reference/types.rst @@ -233,6 +233,13 @@ Here is a more detailed description of each type: is false since "oob" does not appear at the start of "foobar". The ``!in`` operator would yield the negation of ``in``. + Finally, you can create a disjunction (either-or) of two patterns + using the ``|`` operator. For example:: + + /foo/ | /bar/ in "foobar" + + yields true, like in the similar example above. + .. bro:type:: port A type representing transport-level port numbers (besides TCP and From 2bd3a7664df2c5d3200d782ae4ec2dfccd71bc89 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Thu, 26 Apr 2018 16:22:17 -0700 Subject: [PATCH 5/6] documentation for bitwise operators --- doc/script-reference/types.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/script-reference/types.rst b/doc/script-reference/types.rst index 3d6f74aef0..fe77ece90c 100644 --- a/doc/script-reference/types.rst +++ b/doc/script-reference/types.rst @@ -91,6 +91,10 @@ Here is a more detailed description of each type: type, but a unary plus or minus applied to a "count" results in an "int". + In addition, "count" types support bitwise operations. You can use + ``&``, ``|``, and ``^`` for bitwise ``and'', ``or'', and ``xor''. You + can also use ``~`` for bitwise (one's) complement. + .. bro:type:: double A numeric type representing a double-precision floating-point @@ -592,6 +596,9 @@ Here is a more detailed description of each type: The resulting vector of bool is the logical "and" (or logical "or") of each element of the operand vectors. + Vectors of type ``count`` can also be operands for the bitwise and/or/xor + operators, ``&``, ``|`` and ``^``. + See the :bro:keyword:`for` statement for info on how to iterate over the elements in a vector. From 344382ee7beda7c610af297a1a5aa58e8af4f9a9 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Tue, 19 Jun 2018 11:50:38 -0700 Subject: [PATCH 6/6] documentation clarification for "p1 | p2" --- doc/script-reference/types.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/script-reference/types.rst b/doc/script-reference/types.rst index fe77ece90c..b446e972dd 100644 --- a/doc/script-reference/types.rst +++ b/doc/script-reference/types.rst @@ -237,12 +237,13 @@ Here is a more detailed description of each type: is false since "oob" does not appear at the start of "foobar". The ``!in`` operator would yield the negation of ``in``. - Finally, you can create a disjunction (either-or) of two patterns + Finally, you can create a disjunction (either-or) of two literal patterns using the ``|`` operator. For example:: /foo/ | /bar/ in "foobar" - yields true, like in the similar example above. + yields true, like in the similar example above. (This does not presently + work for variables whose values are patterns, however.) .. bro:type:: port