Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-12 03:28:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

input/Raw: Rework GetLine()

Browse source 
This isn't a straightforward fix, unfortunately. The existing GetLine()
implementation didn't deal well with input that's incrementally produced
where individually read chunks wouldn't end with the separator.

The prior implementation increased the buffer each time it failed to find
a separator in the current buffer, but then also ended up not searching the
full new buffer size for the terminator, doing that endlessly.

This change reworks the Raw reader to rely only on bufpos for reading
and searching purposes and skip reallocation if the buffer size if it
wasn't actually exhausted.

Closes #3957
This commit is contained in:
Arne Welzel 2024-10-02 09:27:06 +02:00 • committed by Christian Kreibich
parent 4656faed6c
commit ecfa03ea1d
10 changed files with 281 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

52
testing/btest/scripts/base/frameworks/input/raw/executestream-leftover.zeek Normal file
View file

@ -0,0 +1,52 @@
# @TEST-DOC: Launching a program that produces output slowly and strangely separated.
# @TEST-EXEC: chmod +x run.sh
# @TEST-EXEC: btest-bg-run zeek zeek -b %INPUT
# @TEST-EXEC: btest-bg-wait 10
# @TEST-EXEC: btest-diff zeek/.stdout
redef exit_only_after_terminate = T;
redef Threading::heartbeat_interval = 0.01sec;
@TEST-START-FILE run.sh
#!/usr/bin/env bash
echo -e -n "aaa\nb"
sleep 0.1
echo -e -n "bb\nfi"
sleep 0.1
echo "nal"
sleep infinity
@TEST-END-FILE
module A;
type Val: record {
s: string;
};
global lines = 0;
event one_line(description: Input::EventDescription, tpe: Input::Event, s: string)
{
print tpe, s;
++lines;
if ( lines == 3 )
{
Input::remove("input");
terminate();
}
}
event zeek_init()
{
Input::add_event([
$name="run",
$source="../run.sh |",
$reader=Input::READER_RAW,
$mode=Input::STREAM,
$fields=Val,
$ev=one_line, $want_record=F,
]);
}

53
testing/btest/scripts/base/frameworks/input/raw/executestream-no-last-separator.zeek Normal file
View file

@ -0,0 +1,53 @@
# @TEST-DOC: Launching a program that doesn't end it's final line with a \n
# @TEST-EXEC: chmod +x run.sh
# @TEST-EXEC: btest-bg-run zeek zeek -b %INPUT
# @TEST-EXEC: btest-bg-wait 10
# @TEST-EXEC: btest-diff zeek/.stdout
redef exit_only_after_terminate = T;
redef Threading::heartbeat_interval = 0.01sec;
@TEST-START-FILE run.sh
#!/usr/bin/env bash
sleep 0.1
echo "aaa"
sleep 0.1
echo "bbb"
sleep 0.1
echo -n "final"
sleep 0.1
exit 0
@TEST-END-FILE
module A;
type Val: record {
s: string;
};
global lines = 0;
event one_line(description: Input::EventDescription, tpe: Input::Event, s: string)
{
print tpe, s;
++lines;
if ( lines == 3 )
{
Input::remove("input");
terminate();
}
}
event zeek_init()
{
Input::add_event([
$name="run",
$source="../run.sh |",
$reader=Input::READER_RAW,
$mode=Input::STREAM,
$fields=Val,
$ev=one_line, $want_record=F,
]);
}

62
testing/btest/scripts/base/frameworks/input/raw/executestream-slow-long.zeek Normal file
View file

@ -0,0 +1,62 @@
# @TEST-DOC: Launching a program that produces output slowly and exercises buffering.
# @TEST-EXEC: chmod +x run.sh
# @TEST-EXEC: btest-bg-run zeek zeek -b %INPUT
# @TEST-EXEC: btest-bg-wait 10
# @TEST-EXEC: btest-diff zeek/.stdout
redef exit_only_after_terminate = T;
redef Threading::heartbeat_interval = 0.01sec;
@TEST-START-FILE run.sh
#!/usr/bin/env bash
sleep 0.1
echo -n "binary start"
sleep 0.1
dd if=/dev/zero bs=1 count=8192
sleep 0.1
echo -n "binary middle"
sleep 0.1
dd if=/dev/zero bs=1 count=8192
sleep 0.1
dd if=/dev/zero bs=1 count=8192
sleep 0.1
echo "binary done"
sleep 0.1
echo "ccc"
sleep 0.1
echo "final"
sleep infinity
@TEST-END-FILE
module A;
type Val: record {
s: string;
};
global lines = 0;
event one_line(description: Input::EventDescription, tpe: Input::Event, s: string)
{
print tpe,|s|, s[:16], s[-16:];
++lines;
if ( lines == 3 )
{
Input::remove("input");
terminate();
}
}
event zeek_init()
{
Input::add_event([
$name="run",
$source="../run.sh |",
$reader=Input::READER_RAW,
$mode=Input::STREAM,
$fields=Val,
$ev=one_line, $want_record=F,
]);
}

55
testing/btest/scripts/base/frameworks/input/raw/executestream-slow.zeek Normal file
View file

@ -0,0 +1,55 @@
# @TEST-DOC: Launching a program that produces output slowly puts the raw reader into an endless loop.
# @TEST-EXEC: chmod +x run.sh
# @TEST-EXEC: btest-bg-run zeek zeek -b %INPUT
# @TEST-EXEC: btest-bg-wait 10
# @TEST-EXEC: btest-diff zeek/.stdout
redef exit_only_after_terminate = T;
redef Threading::heartbeat_interval = 0.01sec;
@TEST-START-FILE run.sh
#!/usr/bin/env bash
sleep 0.1
echo -n "aaa-"
sleep 0.1
echo -n "bbb-"
sleep 0.1
echo "ccc"
sleep 0.1
echo "aaa-bbb-ccc"
echo "final"
sleep infinity
@TEST-END-FILE
module A;
type Val: record {
s: string;
};
global lines = 0;
event one_line(description: Input::EventDescription, tpe: Input::Event, s: string)
{
print tpe, s;
++lines;
if ( lines == 3 )
{
Input::remove("input");
terminate();
}
}
event zeek_init()
{
Input::add_event([
$name="run",
$source="../run.sh |",
$reader=Input::READER_RAW,
$mode=Input::STREAM,
$fields=Val,
$ev=one_line, $want_record=F,
]);
}