Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 06:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Ok, this one is not really necessary for 2.1 and more of a nice-to-have

Browse source 
Before this patch, empty values were not hashed at all. Which had the unfortunate side-effect
that e.g. the lines

TEST	-
and
-	TEST

have the same hash values. On re-reads that means that the change will
be ignored.

This is probably pretty academic, but this patch changes it and adds a testcase.

Output of the reread test changes due to re-ordering of the output (probably
due to the fact that the internal hash values are changed and thus transferred
in a different order)
This commit is contained in:
Bernhard Amann 2012-08-26 20:49:21 -07:00
parent fbe464ffa3
commit 7e46936728
4 changed files with 382 additions and 127 deletions
Download patch file Download diff file Expand all files Collapse all files

17
src/input/Manager.cc
View file

@ -1911,11 +1911,16 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals)
const Value* val = vals[i]; const Value* val = vals[i];
if ( val->present ) if ( val->present )
length += GetValueLength(val); length += GetValueLength(val);
}
if ( length == 0 ) // and in any case add 1 for the end-of-field-identifier
length++;
}
assert ( length >= num_elements );
if ( length == num_elements )
{ {
reporter->Error("Input reader sent line where all elements are null values. Ignoring line");
return NULL; return NULL;
} }
@ -1929,6 +1934,12 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals)
const Value* val = vals[i]; const Value* val = vals[i];
if ( val->present ) if ( val->present )
position += CopyValue(data, position, val); position += CopyValue(data, position, val);
memset(data+position, 1, 1); // add end-of-field-marker. does not really matter which value it is,
// it just has to be... something
position++;
} }
HashKey *key = new HashKey(data, length); HashKey *key = new HashKey(data, length);

155
testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/out Normal file
View file

@ -0,0 +1,155 @@
============PREDICATE============
Input::EVENT_NEW
[i=1]
[s=<uninitialized>, ss=TEST]
============PREDICATE============
Input::EVENT_NEW
[i=2]
[s=<uninitialized>, ss=<uninitialized>]
============EVENT============
Description
[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
[2] = [s=<uninitialized>, ss=<uninitialized>],
[1] = [s=<uninitialized>, ss=TEST]
}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
{
print A::outfile, ============EVENT============;
print A::outfile, Description;
print A::outfile, A::description;
print A::outfile, Type;
print A::outfile, A::tpe;
print A::outfile, Left;
print A::outfile, A::left;
print A::outfile, Right;
print A::outfile, A::right;
}, pred=anonymous-function
{
print A::outfile, ============PREDICATE============;
print A::outfile, A::typ;
print A::outfile, A::left;
print A::outfile, A::right;
return (T);
}, config={
}]
Type
Input::EVENT_NEW
Left
[i=1]
Right
[s=<uninitialized>, ss=TEST]
============EVENT============
Description
[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
[2] = [s=<uninitialized>, ss=<uninitialized>],
[1] = [s=<uninitialized>, ss=TEST]
}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
{
print A::outfile, ============EVENT============;
print A::outfile, Description;
print A::outfile, A::description;
print A::outfile, Type;
print A::outfile, A::tpe;
print A::outfile, Left;
print A::outfile, A::left;
print A::outfile, Right;
print A::outfile, A::right;
}, pred=anonymous-function
{
print A::outfile, ============PREDICATE============;
print A::outfile, A::typ;
print A::outfile, A::left;
print A::outfile, A::right;
return (T);
}, config={
}]
Type
Input::EVENT_NEW
Left
[i=2]
Right
[s=<uninitialized>, ss=<uninitialized>]
==========SERVERS============
{
[2] = [s=<uninitialized>, ss=<uninitialized>],
[1] = [s=<uninitialized>, ss=TEST]
}
============PREDICATE============
Input::EVENT_CHANGED
[i=1]
[s=TEST, ss=<uninitialized>]
============PREDICATE============
Input::EVENT_CHANGED
[i=2]
[s=TEST, ss=TEST]
============EVENT============
Description
[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
[2] = [s=TEST, ss=TEST],
[1] = [s=TEST, ss=<uninitialized>]
}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
{
print A::outfile, ============EVENT============;
print A::outfile, Description;
print A::outfile, A::description;
print A::outfile, Type;
print A::outfile, A::tpe;
print A::outfile, Left;
print A::outfile, A::left;
print A::outfile, Right;
print A::outfile, A::right;
}, pred=anonymous-function
{
print A::outfile, ============PREDICATE============;
print A::outfile, A::typ;
print A::outfile, A::left;
print A::outfile, A::right;
return (T);
}, config={
}]
Type
Input::EVENT_CHANGED
Left
[i=1]
Right
[s=<uninitialized>, ss=TEST]
============EVENT============
Description
[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
[2] = [s=TEST, ss=TEST],
[1] = [s=TEST, ss=<uninitialized>]
}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
{
print A::outfile, ============EVENT============;
print A::outfile, Description;
print A::outfile, A::description;
print A::outfile, Type;
print A::outfile, A::tpe;
print A::outfile, Left;
print A::outfile, A::left;
print A::outfile, Right;
print A::outfile, A::right;
}, pred=anonymous-function
{
print A::outfile, ============PREDICATE============;
print A::outfile, A::typ;
print A::outfile, A::left;
print A::outfile, A::right;
return (T);
}, config={
}]
Type
Input::EVENT_CHANGED
Left
[i=2]
Right
[s=<uninitialized>, ss=<uninitialized>]
==========SERVERS============
{
[2] = [s=TEST, ss=TEST],
[1] = [s=TEST, ss=<uninitialized>]
}
done

248
testing/btest/Baseline/scripts.base.frameworks.input.reread/out
View file

@ -1084,7 +1084,7 @@ BB
} }
============PREDICATE============ ============PREDICATE============
Input::EVENT_REMOVED Input::EVENT_REMOVED
[i=-43] [i=-44]
[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
2, 2,
4, 4,
@ -1096,6 +1096,21 @@ AA,
BB BB
}, se={ }, se={
}, vc=[10, 20, 30], ve=[]]
============PREDICATE============
Input::EVENT_REMOVED
[i=-42]
[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
2,
4,
1,
3
}, ss={
CC,
AA,
BB
}, se={
}, vc=[10, 20, 30], ve=[]] }, vc=[10, 20, 30], ve=[]]
============PREDICATE============ ============PREDICATE============
Input::EVENT_REMOVED Input::EVENT_REMOVED
@ -1111,21 +1126,6 @@ AA,
BB BB
}, se={ }, se={
}, vc=[10, 20, 30], ve=[]]
============PREDICATE============
Input::EVENT_REMOVED
[i=-44]
[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
2,
4,
1,
3
}, ss={
CC,
AA,
BB
}, se={
}, vc=[10, 20, 30], ve=[]] }, vc=[10, 20, 30], ve=[]]
============PREDICATE============ ============PREDICATE============
Input::EVENT_REMOVED Input::EVENT_REMOVED
@ -1159,7 +1159,113 @@ BB
}, vc=[10, 20, 30], ve=[]] }, vc=[10, 20, 30], ve=[]]
============PREDICATE============ ============PREDICATE============
Input::EVENT_REMOVED Input::EVENT_REMOVED
[i=-43]
[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
2,
4,
1,
3
}, ss={
CC,
AA,
BB
}, se={
}, vc=[10, 20, 30], ve=[]]
============EVENT============
Description
[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
2,
4,
1,
3
}, ss={
CC,
AA,
BB
}, se={
}, vc=[10, 20, 30], ve=[]]
}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
{
print A::outfile, ============EVENT============;
print A::outfile, Description;
print A::outfile, A::description;
print A::outfile, Type;
print A::outfile, A::tpe;
print A::outfile, Left;
print A::outfile, A::left;
print A::outfile, Right;
print A::outfile, A::right;
}, pred=anonymous-function
{
print A::outfile, ============PREDICATE============;
print A::outfile, A::typ;
print A::outfile, A::left;
print A::outfile, A::right;
return (T);
}, config={
}]
Type
Input::EVENT_REMOVED
Left
[i=-44]
Right
[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
2,
4,
1,
3
}, ss={
CC,
AA,
BB
}, se={
}, vc=[10, 20, 30], ve=[]]
============EVENT============
Description
[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
2,
4,
1,
3
}, ss={
CC,
AA,
BB
}, se={
}, vc=[10, 20, 30], ve=[]]
}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
{
print A::outfile, ============EVENT============;
print A::outfile, Description;
print A::outfile, A::description;
print A::outfile, Type;
print A::outfile, A::tpe;
print A::outfile, Left;
print A::outfile, A::left;
print A::outfile, Right;
print A::outfile, A::right;
}, pred=anonymous-function
{
print A::outfile, ============PREDICATE============;
print A::outfile, A::typ;
print A::outfile, A::left;
print A::outfile, A::right;
return (T);
}, config={
}]
Type
Input::EVENT_REMOVED
Left
[i=-42] [i=-42]
Right
[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
2, 2,
4, 4,
@ -1207,59 +1313,6 @@ print A::outfile, A::right;
return (T); return (T);
}, config={ }, config={
}]
Type
Input::EVENT_REMOVED
Left
[i=-43]
Right
[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
2,
4,
1,
3
}, ss={
CC,
AA,
BB
}, se={
}, vc=[10, 20, 30], ve=[]]
============EVENT============
Description
[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
2,
4,
1,
3
}, ss={
CC,
AA,
BB
}, se={
}, vc=[10, 20, 30], ve=[]]
}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
{
print A::outfile, ============EVENT============;
print A::outfile, Description;
print A::outfile, A::description;
print A::outfile, Type;
print A::outfile, A::tpe;
print A::outfile, Left;
print A::outfile, A::left;
print A::outfile, Right;
print A::outfile, A::right;
}, pred=anonymous-function
{
print A::outfile, ============PREDICATE============;
print A::outfile, A::typ;
print A::outfile, A::left;
print A::outfile, A::right;
return (T);
}, config={
}] }]
Type Type
Input::EVENT_REMOVED Input::EVENT_REMOVED
@ -1313,59 +1366,6 @@ print A::outfile, A::right;
return (T); return (T);
}, config={ }, config={
}]
Type
Input::EVENT_REMOVED
Left
[i=-44]
Right
[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
2,
4,
1,
3
}, ss={
CC,
AA,
BB
}, se={
}, vc=[10, 20, 30], ve=[]]
============EVENT============
Description
[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
2,
4,
1,
3
}, ss={
CC,
AA,
BB
}, se={
}, vc=[10, 20, 30], ve=[]]
}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
{
print A::outfile, ============EVENT============;
print A::outfile, Description;
print A::outfile, A::description;
print A::outfile, Type;
print A::outfile, A::tpe;
print A::outfile, Left;
print A::outfile, A::left;
print A::outfile, Right;
print A::outfile, A::right;
}, pred=anonymous-function
{
print A::outfile, ============PREDICATE============;
print A::outfile, A::typ;
print A::outfile, A::left;
print A::outfile, A::right;
return (T);
}, config={
}] }]
Type Type
Input::EVENT_REMOVED Input::EVENT_REMOVED
@ -1476,9 +1476,9 @@ return (T);
Type Type
Input::EVENT_REMOVED Input::EVENT_REMOVED
Left Left
[i=-42] [i=-43]
Right Right
[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
2, 2,
4, 4,
1, 1,

89
testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro Normal file
View file

@ -0,0 +1,89 @@
# (uses listen.bro just to ensure input sources are more reliably fully-read).
# @TEST-SERIALIZE: comm
#
# @TEST-EXEC: cp input1.log input.log
# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
# @TEST-EXEC: sleep 2
# @TEST-EXEC: cp input2.log input.log
# @TEST-EXEC: btest-bg-wait -k 5
# @TEST-EXEC: btest-diff out
@TEST-START-FILE input1.log
#separator \x09
#fields i s ss
#types int sting string
1 - TEST
2 - -
@TEST-END-FILE
@TEST-START-FILE input2.log
#separator \x09
#fields i s ss
#types int sting string
1 TEST -
2 TEST TEST
@TEST-END-FILE
@load frameworks/communication/listen
module A;
type Idx: record {
i: int;
};
type Val: record {
s: string;
ss: string;
};
global servers: table[int] of Val = table();
global outfile: file;
global try: count;
event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val)
{
print outfile, "============EVENT============";
print outfile, "Description";
print outfile, description;
print outfile, "Type";
print outfile, tpe;
print outfile, "Left";
print outfile, left;
print outfile, "Right";
print outfile, right;
}
event bro_init()
{
outfile = open("../out");
try = 0;
# first read in the old stuff into the table...
Input::add_table([$source="../input.log", $mode=Input::REREAD, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line,
$pred(typ: Input::Event, left: Idx, right: Val) = {
print outfile, "============PREDICATE============";
print outfile, typ;
print outfile, left;
print outfile, right;
return T;
}
]);
}
event Input::update_finished(name: string, source: string)
{
print outfile, "==========SERVERS============";
print outfile, servers;
try = try + 1;
if ( try == 2 )
{
print outfile, "done";
close(outfile);
Input::remove("input");
terminate();
}
}