Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 06:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge remote-tracking branch 'origin/topic/vern/set-ops2'

Browse source 
* origin/topic/vern/set-ops2:
  documentation, test suite update
  implemented set relationals
  bug fix for set intersection
  set intersection implemented
  mirroring previous topic/vern/set-ops to get branch up to date, since I'm a n00b

Fixed a couple memory leaks and added a leak test
This commit is contained in:
Jon Siwek 2018-08-02 10:40:36 -05:00
commit 06c6e1188a
11 changed files with 528 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

5
CHANGES
View file

@ -1,4 +1,9 @@
2.5-802 | 2018-08-02 10:40:36 -0500
* Add set operations: union, intersection, difference, comparison
(Vern Paxson, Corelight)
2.5-796 | 2018-08-01 16:31:25 -0500
* Add 'W' connection history indicator for zero windows

9
NEWS
View file

@ -300,6 +300,15 @@ New Functionality
event tcp_multiple_retransmissions(c: connection, is_orig: bool,
threshold: count);
- Added support for set union, intersection, difference, and comparison
operations. The corresponding operators for the first three are
"s1 | s2", "s1 & s2", and "s1 - s2". Relationals are in terms
of subsets, so "s1 < s2" yields true if s1 is a proper subset of s2
and "s1 == s2" if the two sets have exactly the same elements.
"s1 <= s2" holds for subsets or equality, and similarly "s1 != s2",
"s1 > s2", and "s1 >= s2" have the expected meanings in terms
of non-equality, proper superset, and superset-or-equal.
Changed Functionality
---------------------

2
VERSION
View file

@ -1 +1 @@
2.5-796
2.5-802

9
doc/script-reference/types.rst
View file

@ -544,6 +544,15 @@ Here is a more detailed description of each type:
|s|
You can compute the union, intersection, or difference of two sets
using the ``|``, ``&``, and ``-`` operators. You can compare
sets for equality (they have exactly the same elements) using ``==``.
The ``<`` operator returns ``T`` if the lefthand operand is a proper
subset of the righthand operand. Similarly, ``<=`` returns ``T``
if the lefthand operator is a subset (not necessarily proper, i.e.,
it may be equal to the righthand operand). The operators ``!=``, ``>``
and ``>=`` provide the expected complementary operations.
See the :bro:keyword:`for` statement for info on how to iterate over
the elements in a set.

151
src/Expr.cc
View file

@ -672,6 +672,9 @@ Val* BinaryExpr::Fold(Val* v1, Val* v2) const
if ( v1->Type()->Tag() == TYPE_PATTERN )
return PatternFold(v1, v2);
if ( v1->Type()->IsSet() )
return SetFold(v1, v2);
if ( it == TYPE_INTERNAL_ADDR )
return AddrFold(v1, v2);
@ -858,6 +861,7 @@ Val* BinaryExpr::StringFold(Val* v1, Val* v2) const
return new Val(result, TYPE_BOOL);
}
Val* BinaryExpr::PatternFold(Val* v1, Val* v2) const
{
const RE_Matcher* re1 = v1->AsPattern();
@ -873,6 +877,61 @@ Val* BinaryExpr::PatternFold(Val* v1, Val* v2) const
return new PatternVal(res);
}
Val* BinaryExpr::SetFold(Val* v1, Val* v2) const
{
TableVal* tv1 = v1->AsTableVal();
TableVal* tv2 = v2->AsTableVal();
TableVal* result;
bool res;
switch ( tag ) {
case EXPR_AND:
return tv1->Intersect(tv2);
case EXPR_OR:
result = v1->Clone()->AsTableVal();
if ( ! tv2->AddTo(result, false, false) )
reporter->InternalError("set union failed to type check");
return result;
case EXPR_SUB:
result = v1->Clone()->AsTableVal();
if ( ! tv2->RemoveFrom(result) )
reporter->InternalError("set difference failed to type check");
return result;
case EXPR_EQ:
res = tv1->EqualTo(tv2);
break;
case EXPR_NE:
res = ! tv1->EqualTo(tv2);
break;
case EXPR_LT:
res = tv1->IsSubsetOf(tv2) && tv1->Size() < tv2->Size();
break;
case EXPR_LE:
res = tv1->IsSubsetOf(tv2);
break;
case EXPR_GE:
case EXPR_GT:
// These should't happen due to canonicalization.
reporter->InternalError("confusion over canonicalization in set comparison");
break;
default:
BadTag("BinaryExpr::SetFold", expr_name(tag));
return 0;
}
return new Val(res, TYPE_BOOL);
}
Val* BinaryExpr::AddrFold(Val* v1, Val* v2) const
{
IPAddr a1 = v1->AsAddr();
@ -1454,24 +1513,39 @@ SubExpr::SubExpr(Expr* arg_op1, Expr* arg_op2)
if ( IsError() )
return;
TypeTag bt1 = op1->Type()->Tag();
if ( IsVector(bt1) )
bt1 = op1->Type()->AsVectorType()->YieldType()->Tag();
const BroType* t1 = op1->Type();
const BroType* t2 = op2->Type();
TypeTag bt2 = op2->Type()->Tag();
TypeTag bt1 = t1->Tag();
if ( IsVector(bt1) )
bt1 = t1->AsVectorType()->YieldType()->Tag();
TypeTag bt2 = t2->Tag();
if ( IsVector(bt2) )
bt2 = op2->Type()->AsVectorType()->YieldType()->Tag();
bt2 = t2->AsVectorType()->YieldType()->Tag();
BroType* base_result_type = 0;
if ( bt1 == TYPE_TIME && bt2 == TYPE_INTERVAL )
base_result_type = base_type(bt1);
else if ( bt1 == TYPE_TIME && bt2 == TYPE_TIME )
SetType(base_type(TYPE_INTERVAL));
else if ( bt1 == TYPE_INTERVAL && bt2 == TYPE_INTERVAL )
base_result_type = base_type(bt1);
else if ( t1->IsSet() && t2->IsSet() )
{
if ( same_type(t1, t2) )
SetType(op1->Type()->Ref());
else
ExprError("incompatible \"set\" operands");
}
else if ( BothArithmetic(bt1, bt2) )
PromoteType(max_type(bt1, bt2), is_vector(op1) || is_vector(op2));
else
ExprError("requires arithmetic operands");
@ -1888,13 +1962,16 @@ BitExpr::BitExpr(BroExprTag arg_tag, Expr* arg_op1, Expr* arg_op2)
if ( IsError() )
return;
TypeTag bt1 = op1->Type()->Tag();
if ( IsVector(bt1) )
bt1 = op1->Type()->AsVectorType()->YieldType()->Tag();
const BroType* t1 = op1->Type();
const BroType* t2 = op2->Type();
TypeTag bt2 = op2->Type()->Tag();
TypeTag bt1 = t1->Tag();
if ( IsVector(bt1) )
bt1 = t1->AsVectorType()->YieldType()->Tag();
TypeTag bt2 = t2->Tag();
if ( IsVector(bt2) )
bt2 = op2->Type()->AsVectorType()->YieldType()->Tag();
bt2 = t2->AsVectorType()->YieldType()->Tag();
if ( (bt1 == TYPE_COUNT || bt1 == TYPE_COUNTER) &&
(bt2 == TYPE_COUNT || bt2 == TYPE_COUNTER) )
@ -1917,8 +1994,16 @@ BitExpr::BitExpr(BroExprTag arg_tag, Expr* arg_op1, Expr* arg_op2)
SetType(base_type(TYPE_PATTERN));
}
else if ( t1->IsSet() && t2->IsSet() )
{
if ( same_type(t1, t2) )
SetType(op1->Type()->Ref());
else
ExprError("incompatible \"set\" operands");
}
else
ExprError("requires \"count\" operands");
ExprError("requires \"count\" or compatible \"set\" operands");
}
IMPLEMENT_SERIAL(BitExpr, SER_BIT_EXPR);
@ -1943,13 +2028,16 @@ EqExpr::EqExpr(BroExprTag arg_tag, Expr* arg_op1, Expr* arg_op2)
Canonicize();
TypeTag bt1 = op1->Type()->Tag();
if ( IsVector(bt1) )
bt1 = op1->Type()->AsVectorType()->YieldType()->Tag();
const BroType* t1 = op1->Type();
const BroType* t2 = op2->Type();
TypeTag bt2 = op2->Type()->Tag();
TypeTag bt1 = t1->Tag();
if ( IsVector(bt1) )
bt1 = t1->AsVectorType()->YieldType()->Tag();
TypeTag bt2 = t2->Tag();
if ( IsVector(bt2) )
bt2 = op2->Type()->AsVectorType()->YieldType()->Tag();
bt2 = t2->AsVectorType()->YieldType()->Tag();
if ( is_vector(op1) || is_vector(op2) )
SetType(new VectorType(base_type(TYPE_BOOL)));
@ -1979,10 +2067,20 @@ EqExpr::EqExpr(BroExprTag arg_tag, Expr* arg_op1, Expr* arg_op2)
break;
case TYPE_ENUM:
if ( ! same_type(op1->Type(), op2->Type()) )
if ( ! same_type(t1, t2) )
ExprError("illegal enum comparison");
break;
case TYPE_TABLE:
if ( t1->IsSet() && t2->IsSet() )
{
if ( ! same_type(t1, t2) )
ExprError("incompatible sets in comparison");
break;
}
// FALL THROUGH
default:
ExprError("illegal comparison");
}
@ -2045,13 +2143,16 @@ RelExpr::RelExpr(BroExprTag arg_tag, Expr* arg_op1, Expr* arg_op2)
Canonicize();
TypeTag bt1 = op1->Type()->Tag();
if ( IsVector(bt1) )
bt1 = op1->Type()->AsVectorType()->YieldType()->Tag();
const BroType* t1 = op1->Type();
const BroType* t2 = op2->Type();
TypeTag bt2 = op2->Type()->Tag();
TypeTag bt1 = t1->Tag();
if ( IsVector(bt1) )
bt1 = t1->AsVectorType()->YieldType()->Tag();
TypeTag bt2 = t2->Tag();
if ( IsVector(bt2) )
bt2 = op2->Type()->AsVectorType()->YieldType()->Tag();
bt2 = t2->AsVectorType()->YieldType()->Tag();
if ( is_vector(op1) || is_vector(op2) )
SetType(new VectorType(base_type(TYPE_BOOL)));
@ -2061,6 +2162,12 @@ RelExpr::RelExpr(BroExprTag arg_tag, Expr* arg_op1, Expr* arg_op2)
if ( BothArithmetic(bt1, bt2) )
PromoteOps(max_type(bt1, bt2));
else if ( t1->IsSet() && t2->IsSet() )
{
if ( ! same_type(t1, t2) )
ExprError("incompatible sets in comparison");
}
else if ( bt1 != bt2 )
ExprError("operands must be of the same type");

3
src/Expr.h
View file

@ -332,6 +332,9 @@ protected:
// Same for when the constants are patterns.
virtual Val* PatternFold(Val* v1, Val* v2) const;
// Same for when the constants are sets.
virtual Val* SetFold(Val* v1, Val* v2) const;
// Same for when the constants are addresses or subnets.
virtual Val* AddrFold(Val* v1, Val* v2) const;
virtual Val* SubNetFold(Val* v1, Val* v2) const;

93
src/Val.cc
View file

@ -1706,9 +1706,11 @@ int TableVal::RemoveFrom(Val* val) const
HashKey* k;
while ( tbl->NextEntry(k, c) )
{
Val* index = RecoverIndex(k);
Unref(index);
// Not sure that this is 100% sound, since the HashKey
// comes from one table but is being used in another.
// OTOH, they are both the same type, so as long as
// we don't have hash keys that are keyed per dictionary,
// it should work ...
Unref(t->Delete(k));
delete k;
}
@ -1716,6 +1718,91 @@ int TableVal::RemoveFrom(Val* val) const
return 1;
}
TableVal* TableVal::Intersect(const TableVal* tv) const
{
TableVal* result = new TableVal(table_type);
const PDict(TableEntryVal)* t0 = AsTable();
const PDict(TableEntryVal)* t1 = tv->AsTable();
PDict(TableEntryVal)* t2 = result->AsNonConstTable();
// Figure out which is smaller; assign it to t1.
if ( t1->Length() > t0->Length() )
{ // Swap.
const PDict(TableEntryVal)* tmp = t1;
t1 = t0;
t0 = tmp;
}
IterCookie* c = t1->InitForIteration();
HashKey* k;
while ( t1->NextEntry(k, c) )
{
// Here we leverage the same assumption about consistent
// hashes as in TableVal::RemoveFrom above.
if ( t0->Lookup(k) )
t2->Insert(k, new TableEntryVal(0));
delete k;
}
return result;
}
bool TableVal::EqualTo(const TableVal* tv) const
{
const PDict(TableEntryVal)* t0 = AsTable();
const PDict(TableEntryVal)* t1 = tv->AsTable();
if ( t0->Length() != t1->Length() )
return false;
IterCookie* c = t0->InitForIteration();
HashKey* k;
while ( t0->NextEntry(k, c) )
{
// Here we leverage the same assumption about consistent
// hashes as in TableVal::RemoveFrom above.
if ( ! t1->Lookup(k) )
{
delete k;
t0->StopIteration(c);
return false;
}
delete k;
}
return true;
}
bool TableVal::IsSubsetOf(const TableVal* tv) const
{
const PDict(TableEntryVal)* t0 = AsTable();
const PDict(TableEntryVal)* t1 = tv->AsTable();
if ( t0->Length() > t1->Length() )
return false;
IterCookie* c = t0->InitForIteration();
HashKey* k;
while ( t0->NextEntry(k, c) )
{
// Here we leverage the same assumption about consistent
// hashes as in TableVal::RemoveFrom above.
if ( ! t1->Lookup(k) )
{
delete k;
t0->StopIteration(c);
return false;
}
delete k;
}
return true;
}
int TableVal::ExpandAndInit(Val* index, Val* new_val)
{
BroType* index_type = index->Type();

16
src/Val.h
View file

@ -809,6 +809,22 @@ public:
// Returns true if the addition typechecked, false if not.
int RemoveFrom(Val* v) const override;
// Returns a new table that is the intersection of this
// table and the given table. Intersection is just done
// on index, not on yield value, so this really only makes
// sense for sets.
TableVal* Intersect(const TableVal* v) const;
// Returns true if this set contains the same members as the
// given set. Note that comparisons are done using hash keys,
// so errors can arise for compound sets such as sets-of-sets.
// See https://bro-tracker.atlassian.net/browse/BIT-1949.
bool EqualTo(const TableVal* v) const;
// Returns true if this set is a subset (not necessarily proper)
// of the given set.
bool IsSubsetOf(const TableVal* v) const;
// Expands any lists in the index into multiple initializations.
// Returns true if the initializations typecheck, false if not.
int ExpandAndInit(Val* index, Val* new_val);

27
testing/btest/Baseline/language.set/out
View file

@ -42,3 +42,30 @@ remove element (PASS)
!in operator (PASS)
remove element (PASS)
!in operator (PASS)
union (PASS)
intersection (FAIL)
difference (PASS)
difference (PASS)
union/inter. (PASS)
relational (PASS)
relational (PASS)
subset (FAIL)
subset (FAIL)
subset (PASS)
superset (FAIL)
superset (FAIL)
superset (FAIL)
superset (PASS)
non-ordering (FAIL)
non-ordering (PASS)
superset (PASS)
superset (FAIL)
superset (PASS)
superset (PASS)
superset (PASS)
superset (FAIL)
equality (PASS)
equality (FAIL)
non-equality (PASS)
equality (FAIL)
magnitude (FAIL)

194
testing/btest/core/leaks/set.bro Normal file
View file

@ -0,0 +1,194 @@
# @TEST-GROUP: leaks
# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks
# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b -r $TRACES/http/get.trace %INPUT
# @TEST-EXEC: btest-bg-wait 60
function test_case(msg: string, expect: bool)
{
print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
}
# Note: only global sets can be initialized with curly braces
global sg1: set[string] = { "curly", "braces" };
global sg2: set[port, string, bool] = { [10/udp, "curly", F],
[11/udp, "braces", T] };
global sg3 = { "more", "curly", "braces" };
global did_once = F;
event new_connection(cc: connection)
{
if ( did_once )
return;
did_once = T;
local s1: set[string] = set( "test", "example" );
local s2: set[string] = set();
local s3: set[string];
local s4 = set( "type inference" );
local s5: set[port, string, bool] = set( [1/tcp, "test", T],
[2/tcp, "example", F] );
local s6: set[port, string, bool] = set();
local s7: set[port, string, bool];
local s8 = set( [8/tcp, "type inference", T] );
# Type inference tests
test_case( "type inference", type_name(s4) == "set[string]" );
test_case( "type inference", type_name(s8) == "set[port,string,bool]" );
test_case( "type inference", type_name(sg3) == "set[string]" );
# Test the size of each set
test_case( "cardinality", |s1| == 2 );
test_case( "cardinality", |s2| == 0 );
test_case( "cardinality", |s3| == 0 );
test_case( "cardinality", |s4| == 1 );
test_case( "cardinality", |s5| == 2 );
test_case( "cardinality", |s6| == 0 );
test_case( "cardinality", |s7| == 0 );
test_case( "cardinality", |s8| == 1 );
test_case( "cardinality", |sg1| == 2 );
test_case( "cardinality", |sg2| == 2 );
test_case( "cardinality", |sg3| == 3 );
# Test iterating over each set
local ct: count;
ct = 0;
for ( c in s1 )
{
if ( type_name(c) != "string" )
print "Error: wrong set element type";
++ct;
}
test_case( "iterate over set", ct == 2 );
ct = 0;
for ( c in s2 )
{
++ct;
}
test_case( "iterate over set", ct == 0 );
ct = 0;
for ( [c1,c2,c3] in s5 )
{
++ct;
}
test_case( "iterate over set", ct == 2 );
ct = 0;
for ( [c1,c2,c3] in sg2 )
{
++ct;
}
test_case( "iterate over set", ct == 2 );
# Test adding elements to each set (Note: cannot add elements to sets
# of multiple types)
add s1["added"];
add s1["added"]; # element already exists (nothing happens)
test_case( "add element", |s1| == 3 );
test_case( "in operator", "added" in s1 );
add s2["another"];
test_case( "add element", |s2| == 1 );
add s2["test"];
test_case( "add element", |s2| == 2 );
test_case( "in operator", "another" in s2 );
test_case( "in operator", "test" in s2 );
add s3["foo"];
test_case( "add element", |s3| == 1 );
test_case( "in operator", "foo" in s3 );
add s4["local"];
test_case( "add element", |s4| == 2 );
test_case( "in operator", "local" in s4 );
add sg1["global"];
test_case( "add element", |sg1| == 3 );
test_case( "in operator", "global" in sg1 );
add sg3["more global"];
test_case( "add element", |sg3| == 4 );
test_case( "in operator", "more global" in sg3 );
# Test removing elements from each set (Note: cannot remove elements
# from sets of multiple types)
delete s1["test"];
delete s1["foobar"]; # element does not exist (nothing happens)
test_case( "remove element", |s1| == 2 );
test_case( "!in operator", "test" !in s1 );
delete s2["test"];
test_case( "remove element", |s2| == 1 );
test_case( "!in operator", "test" !in s2 );
delete s3["foo"];
test_case( "remove element", |s3| == 0 );
test_case( "!in operator", "foo" !in s3 );
delete s4["type inference"];
test_case( "remove element", |s4| == 1 );
test_case( "!in operator", "type inference" !in s4 );
delete sg1["braces"];
test_case( "remove element", |sg1| == 2 );
test_case( "!in operator", "braces" !in sg1 );
delete sg3["curly"];
test_case( "remove element", |sg3| == 3 );
test_case( "!in operator", "curly" !in sg3 );
local a = set(1,5,7,9,8,14);
local b = set(1,7,9,2);
local a_plus_b = set(1,2,5,7,9,8,14);
local a_also_b = set(1,7,9);
local a_sans_b = set(5,8,14);
local b_sans_a = set(2);
local a_or_b = a | b;
local a_and_b = a & b;
test_case( "union", a_or_b == a_plus_b );
test_case( "intersection", a_and_b == a_plus_b );
test_case( "difference", a - b == a_sans_b );
test_case( "difference", b - a == b_sans_a );
test_case( "union/inter.", |b & set(1,7,9,2)| == |b | set(1,7,2,9)| );
test_case( "relational", |b & a_or_b| == |b| && |b| < |a_or_b| );
test_case( "relational", b < a_or_b && a < a_or_b && a_or_b > a_and_b );
test_case( "subset", b < a );
test_case( "subset", a < b );
test_case( "subset", b < (a | set(2)) );
test_case( "superset", b > a );
test_case( "superset", b > (a | set(2)) );
test_case( "superset", b | set(8, 14, 5) > (a | set(2)) );
test_case( "superset", b | set(8, 14, 99, 5) > (a | set(2)) );
test_case( "non-ordering", (a <= b) || (a >= b) );
test_case( "non-ordering", (a <= a_or_b) && (a_or_b >= b) );
test_case( "superset", (b | set(14, 5)) > a - set(8) );
test_case( "superset", (b | set(14)) > a - set(8) );
test_case( "superset", (b | set(14)) > a - set(8,5) );
test_case( "superset", b >= a - set(5,8,14) );
test_case( "superset", b > a - set(5,8,14) );
test_case( "superset", (b - set(2)) > a - set(5,8,14) );
test_case( "equality", a == a | set(5) );
test_case( "equality", a == a | set(5,11) );
test_case( "non-equality", a != a | set(5,11) );
test_case( "equality", a == a | set(5,11) );
test_case( "magnitude", |a_and_b| == |a_or_b|);
}

45
testing/btest/language/set.bro
View file

@ -136,5 +136,50 @@ event bro_init()
delete sg3["curly"];
test_case( "remove element", |sg3| == 3 );
test_case( "!in operator", "curly" !in sg3 );
local a = set(1,5,7,9,8,14);
local b = set(1,7,9,2);
local a_plus_b = set(1,2,5,7,9,8,14);
local a_also_b = set(1,7,9);
local a_sans_b = set(5,8,14);
local b_sans_a = set(2);
local a_or_b = a | b;
local a_and_b = a & b;
test_case( "union", a_or_b == a_plus_b );
test_case( "intersection", a_and_b == a_plus_b );
test_case( "difference", a - b == a_sans_b );
test_case( "difference", b - a == b_sans_a );
test_case( "union/inter.", |b & set(1,7,9,2)| == |b | set(1,7,2,9)| );
test_case( "relational", |b & a_or_b| == |b| && |b| < |a_or_b| );
test_case( "relational", b < a_or_b && a < a_or_b && a_or_b > a_and_b );
test_case( "subset", b < a );
test_case( "subset", a < b );
test_case( "subset", b < (a | set(2)) );
test_case( "superset", b > a );
test_case( "superset", b > (a | set(2)) );
test_case( "superset", b | set(8, 14, 5) > (a | set(2)) );
test_case( "superset", b | set(8, 14, 99, 5) > (a | set(2)) );
test_case( "non-ordering", (a <= b) || (a >= b) );
test_case( "non-ordering", (a <= a_or_b) && (a_or_b >= b) );
test_case( "superset", (b | set(14, 5)) > a - set(8) );
test_case( "superset", (b | set(14)) > a - set(8) );
test_case( "superset", (b | set(14)) > a - set(8,5) );
test_case( "superset", b >= a - set(5,8,14) );
test_case( "superset", b > a - set(5,8,14) );
test_case( "superset", (b - set(2)) > a - set(5,8,14) );
test_case( "equality", a == a | set(5) );
test_case( "equality", a == a | set(5,11) );
test_case( "non-equality", a != a | set(5,11) );
test_case( "equality", a == a | set(5,11) );
test_case( "magnitude", |a_and_b| == |a_or_b|);
}