Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 06:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Include file information in input reader error messages

Browse source
This commit is contained in:
Tim Wojtulewicz 2021-12-09 12:08:33 -07:00
parent 8184073ef8
commit 098a5d3348
13 changed files with 179 additions and 57 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/input/ReaderBackend.cc
View file

@ -2,6 +2,7 @@
#include "zeek/input/ReaderBackend.h"
#include "zeek/Desc.h"
#include "zeek/input/Manager.h"
#include "zeek/input/ReaderFrontend.h"

11
src/input/ReaderBackend.h
View file

@ -7,6 +7,11 @@
#include "zeek/threading/MsgThread.h"
#include "zeek/threading/SerialTypes.h"
namespace zeek::detail
{
class Location;
}
namespace zeek::input
{
@ -55,6 +60,12 @@ enum ReaderMode
class ReaderBackend : public threading::MsgThread
{
public:
// Silence a warning from clang about hidden overloaded functions and the
// Info() function that this class provides.
using threading::MsgThread::Error;
using threading::MsgThread::Info;
using threading::MsgThread::Warning;
/**
* Constructor.
*

22
src/input/readers/ascii/Ascii.cc
View file

@ -55,9 +55,10 @@ Ascii::Ascii(ReaderFrontend* frontend) : ReaderBackend(frontend)
fail_on_invalid_lines = false;
}
Ascii::~Ascii() { }
void Ascii::DoClose() { }
void Ascii::DoClose()
{
read_location.reset();
}
bool Ascii::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fields)
{
@ -156,6 +157,9 @@ bool Ascii::OpenFile()
return ! fail_on_file_problem;
}
read_location = std::make_unique<zeek::detail::Location>();
read_location->filename = util::copy_string(fname.c_str());
StopWarningSuppression();
return true;
}
@ -253,6 +257,12 @@ bool Ascii::GetLine(string& str)
{
while ( getline(file, str) )
{
if ( read_location )
{
read_location->first_line++;
read_location->last_line++;
}
if ( ! str.size() )
continue;
@ -278,6 +288,12 @@ bool Ascii::DoUpdate()
if ( ! OpenFile() )
return ! fail_on_file_problem;
if ( read_location )
{
read_location->first_line = 0;
read_location->last_line = 0;
}
switch ( Info().mode )
{
case MODE_REREAD:

18
src/input/readers/ascii/Ascii.h
View file

@ -8,6 +8,7 @@
#include <memory>
#include <vector>
#include "zeek/Obj.h"
#include "zeek/input/ReaderBackend.h"
#include "zeek/threading/formatters/Ascii.h"
@ -20,20 +21,16 @@ struct FieldMapping
std::string name;
TypeTag type;
TypeTag subtype; // internal type for sets and vectors
int position;
int secondary_position; // for ports: pos of the second field
bool present;
int position = -1;
int secondary_position = -1; // for ports: pos of the second field
bool present = false;
FieldMapping(const std::string& arg_name, const TypeTag& arg_type, int arg_position);
FieldMapping(const std::string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype,
int arg_position);
FieldMapping(const FieldMapping& arg);
FieldMapping()
{
position = -1;
secondary_position = -1;
}
FieldMapping() = default;
FieldMapping subType();
};
@ -45,7 +42,7 @@ class Ascii : public ReaderBackend
{
public:
explicit Ascii(ReaderFrontend* frontend);
~Ascii() override;
~Ascii() override = default;
// prohibit copying and moving
Ascii(const Ascii&) = delete;
@ -62,6 +59,8 @@ protected:
bool DoUpdate() override;
bool DoHeartbeat(double network_time, double current_time) override;
zeek::detail::Location* GetLocationInfo() const override { return read_location.get(); }
private:
bool ReadHeader(bool useCached);
bool GetLine(std::string& str);
@ -92,6 +91,7 @@ private:
std::string path_prefix;
std::unique_ptr<threading::Formatter> formatter;
std::unique_ptr<zeek::detail::Location> read_location;
};
} // namespace zeek::input::reader::detail

94
src/threading/MsgThread.cc
View file

@ -5,6 +5,8 @@
#include <unistd.h>
#include "zeek/DebugLogger.h"
#include "zeek/Desc.h"
#include "zeek/Obj.h"
#include "zeek/RunState.h"
#include "zeek/iosource/Manager.h"
#include "zeek/threading/Manager.h"
@ -341,38 +343,105 @@ void MsgThread::Finished()
void MsgThread::Info(const char* msg)
{
SendOut(new detail::ReporterMessage(detail::ReporterMessage::INFO, this, msg));
ODesc desc;
if ( auto* location = GetLocationInfo() )
{
location->Describe(&desc);
desc.Add(": ");
}
desc.Add(msg);
SendOut(new detail::ReporterMessage(detail::ReporterMessage::INFO, this, desc.Description()));
}
void MsgThread::Warning(const char* msg)
{
SendOut(new detail::ReporterMessage(detail::ReporterMessage::WARNING, this, msg));
ODesc desc;
if ( auto* location = GetLocationInfo() )
{
location->Describe(&desc);
desc.Add(": ");
}
desc.Add(msg);
SendOut(
new detail::ReporterMessage(detail::ReporterMessage::WARNING, this, desc.Description()));
}
void MsgThread::Error(const char* msg)
{
SendOut(new detail::ReporterMessage(detail::ReporterMessage::ERROR, this, msg));
ODesc desc;
if ( auto* location = GetLocationInfo() )
{
location->Describe(&desc);
desc.Add(": ");
}
desc.Add(msg);
SendOut(new detail::ReporterMessage(detail::ReporterMessage::ERROR, this, desc.Description()));
}
void MsgThread::FatalError(const char* msg)
{
SendOut(new detail::ReporterMessage(detail::ReporterMessage::FATAL_ERROR, this, msg));
ODesc desc;
if ( auto* location = GetLocationInfo() )
{
location->Describe(&desc);
desc.Add(": ");
}
desc.Add(msg);
SendOut(new detail::ReporterMessage(detail::ReporterMessage::FATAL_ERROR, this,
desc.Description()));
}
void MsgThread::FatalErrorWithCore(const char* msg)
{
SendOut(new detail::ReporterMessage(detail::ReporterMessage::FATAL_ERROR_WITH_CORE, this, msg));
ODesc desc;
if ( auto* location = GetLocationInfo() )
{
location->Describe(&desc);
desc.Add(": ");
}
desc.Add(msg);
SendOut(new detail::ReporterMessage(detail::ReporterMessage::FATAL_ERROR_WITH_CORE, this,
desc.Description()));
}
void MsgThread::InternalWarning(const char* msg)
{
SendOut(new detail::ReporterMessage(detail::ReporterMessage::INTERNAL_WARNING, this, msg));
ODesc desc;
if ( auto* location = GetLocationInfo() )
{
location->Describe(&desc);
desc.Add(": ");
}
desc.Add(msg);
SendOut(new detail::ReporterMessage(detail::ReporterMessage::INTERNAL_WARNING, this,
desc.Description()));
}
void MsgThread::InternalError(const char* msg)
{
// This one aborts immediately.
fprintf(stderr, "internal error in thread: %s\n", msg);
ODesc desc;
if ( auto* location = GetLocationInfo() )
{
location->Describe(&desc);
desc.Add(": ");
}
desc.Add(msg);
fprintf(stderr, "internal error in thread: %s\n", desc.Description());
abort();
}
@ -380,7 +449,16 @@ void MsgThread::InternalError(const char* msg)
void MsgThread::Debug(DebugStream stream, const char* msg)
{
SendOut(new detail::DebugMessage(stream, this, msg));
ODesc desc;
if ( auto* location = GetLocationInfo() )
{
location->Describe(&desc);
desc.Add(": ");
}
desc.Add(msg);
SendOut(new detail::DebugMessage(stream, this, desc.Description()));
}
#endif

16
src/threading/MsgThread.h
View file

@ -8,6 +8,11 @@
#include "zeek/threading/BasicThread.h"
#include "zeek/threading/Queue.h"
namespace zeek::detail
{
class Location;
}
namespace zeek::threading
{
@ -275,6 +280,17 @@ protected:
void OnSignalStop() override;
void OnKill() override;
/**
* Method for child classes to override to provide file location
* information in log messages. This is primarily used by the input
* framework's ReaderBackend classes to give more descriptive error
* messages.
*
* @return A Location pointer containing the file location information,
* or nullptr if nothing is available.
*/
virtual zeek::detail::Location* GetLocationInfo() const { return nullptr; }
private:
/**
* Pops a message sent by the main thread from the main-to-chold

16
testing/btest/Baseline/scripts.base.frameworks.input.bad_patterns/out
View file

@ -1,10 +1,10 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
### NOTE: This file has been sorted with diff-sort.
warning: input.log/Input::READER_ASCII: Could not convert line '2 /cat/sss' of input.log to Val. Ignoring line.
warning: input.log/Input::READER_ASCII: Could not convert line '3 /foo|bar' of input.log to Val. Ignoring line.
warning: input.log/Input::READER_ASCII: Could not convert line '4 this is not a pattern' of input.log to Val. Ignoring line.
warning: input.log/Input::READER_ASCII: Could not convert line '5 /5' of input.log to Val. Ignoring line.
warning: input.log/Input::READER_ASCII: String '/5' contained no parseable pattern.
warning: input.log/Input::READER_ASCII: String '/cat/sss' contained no parseable pattern.
warning: input.log/Input::READER_ASCII: String '/foo|bar' contained no parseable pattern.
warning: input.log/Input::READER_ASCII: String 'this is not a pattern' contained no parseable pattern.
warning: input.log/Input::READER_ASCII: input.log, line 5: Could not convert line '2 /cat/sss' of input.log to Val. Ignoring line.
warning: input.log/Input::READER_ASCII: input.log, line 5: String '/cat/sss' contained no parseable pattern.
warning: input.log/Input::READER_ASCII: input.log, line 6: Could not convert line '3 /foo|bar' of input.log to Val. Ignoring line.
warning: input.log/Input::READER_ASCII: input.log, line 6: String '/foo|bar' contained no parseable pattern.
warning: input.log/Input::READER_ASCII: input.log, line 7: Could not convert line '4 this is not a pattern' of input.log to Val. Ignoring line.
warning: input.log/Input::READER_ASCII: input.log, line 7: String 'this is not a pattern' contained no parseable pattern.
warning: input.log/Input::READER_ASCII: input.log, line 8: Could not convert line '5 /5' of input.log to Val. Ignoring line.
warning: input.log/Input::READER_ASCII: input.log, line 8: String '/5' contained no parseable pattern.

14
testing/btest/Baseline/scripts.base.frameworks.input.invalid-lines/.stderrwithoutfirstline
View file

@ -1,11 +1,11 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
### NOTE: This file has been sorted with diff-sort.
>>>
error: ../input.log/Input::READER_ASCII: Init failed
error: ../input.log/Input::READER_ASCII: Not enough fields in line 'T -41 SSH::LOG 21 123 tcp 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30' of ../input.log. Found 15 fields, want positions 17 and -1
error: ../input.log/Input::READER_ASCII: terminating thread
error: ../input.log/Input::READER_ASCII: ../input.log, line 5: Init failed
error: ../input.log/Input::READER_ASCII: ../input.log, line 5: Not enough fields in line 'T -41 SSH::LOG 21 123 tcp 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30' of ../input.log. Found 15 fields, want positions 17 and -1
error: ../input.log/Input::READER_ASCII: ../input.log, line 5: terminating thread
received termination signal
warning: ../input.log/Input::READER_ASCII: Bad address: 342.2.3.4
warning: ../input.log/Input::READER_ASCII: Not enough fields in line 'T -41 SSH::LOG 21 123 tcp 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30' of ../input.log. Found 15 fields, want positions 17 and -1
warning: ../input.log/Input::READER_ASCII: Not enough fields in line 'T -41' of ../input.log. Found 1 fields, want positions 2 and -1
warning: ../input.log/Input::READER_ASCII: Tried to parse invalid/unknown protocol: whatever
warning: ../input.log/Input::READER_ASCII: ../input.log, line 5: Not enough fields in line 'T -41 SSH::LOG 21 123 tcp 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30' of ../input.log. Found 15 fields, want positions 17 and -1
warning: ../input.log/Input::READER_ASCII: ../input.log, line 7: Tried to parse invalid/unknown protocol: whatever
warning: ../input.log/Input::READER_ASCII: ../input.log, line 8: Bad address: 342.2.3.4
warning: ../input.log/Input::READER_ASCII: ../input.log, line 9: Not enough fields in line 'T -41' of ../input.log. Found 1 fields, want positions 2 and -1

16
testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline
View file

@ -1,11 +1,11 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
warning: ..<...>/Input::READER_ASCII: Number '12129223372036854775800' out of supported range.
warning: ..<...>/Input::READER_ASCII: Could not convert line '12129223372036854775800 121218446744073709551612' of ../input.log to Val. Ignoring line.
warning: ..<...>/Input::READER_ASCII: Number '9223372036854775801TEXTHERE' contained non-numeric trailing characters. Ignored trailing characters 'TEXTHERE'
warning: ..<...>/Input::READER_ASCII: Number '1Justtext' contained non-numeric trailing characters. Ignored trailing characters 'Justtext'
warning: ..<...>/Input::READER_ASCII: String 'Justtext' contained no parseable number
warning: ..<...>/Input::READER_ASCII: Could not convert line 'Justtext 1' of ../input.log to Val. Ignoring line.
warning: ..<...>/Input::READER_ASCII: Number ' -18446744073709551612' cannot be negative
warning: ..<...>/Input::READER_ASCII: Could not convert line '9223372036854775800 -18446744073709551612' of ../input.log to Val. Ignoring line.
warning: ..<...>/Input::READER_ASCII: ../input.log, line 4: Number '12129223372036854775800' out of supported range.
warning: ..<...>/Input::READER_ASCII: ../input.log, line 4: Could not convert line '12129223372036854775800 121218446744073709551612' of ../input.log to Val. Ignoring line.
warning: ..<...>/Input::READER_ASCII: ../input.log, line 5: Number '9223372036854775801TEXTHERE' contained non-numeric trailing characters. Ignored trailing characters 'TEXTHERE'
warning: ..<...>/Input::READER_ASCII: ../input.log, line 5: Number '1Justtext' contained non-numeric trailing characters. Ignored trailing characters 'Justtext'
warning: ..<...>/Input::READER_ASCII: ../input.log, line 6: String 'Justtext' contained no parseable number
warning: ..<...>/Input::READER_ASCII: ../input.log, line 6: Could not convert line 'Justtext 1' of ../input.log to Val. Ignoring line.
warning: ..<...>/Input::READER_ASCII: ../input.log, line 7: Number ' -18446744073709551612' cannot be negative
warning: ..<...>/Input::READER_ASCII: ../input.log, line 7: Could not convert line '9223372036854775800 -18446744073709551612' of ../input.log to Val. Ignoring line.
received termination signal
>>>

12
testing/btest/Baseline/scripts.base.frameworks.input.invalidset/.stderrwithoutfirstline
View file

@ -1,11 +1,11 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
warning: Skipping input with missing non-optional value
warning: ..<...>/Input::READER_ASCII: Invalid value for subnet: 127.0.0.1
warning: ..<...>/Input::READER_ASCII: Error while reading set or vector
warning: ..<...>/Input::READER_ASCII: Could not convert line 'name 127.0.0.1' of ../input.log to Val. Ignoring line.
warning: ..<...>/Input::READER_ASCII: ../input.log, line 4: Invalid value for subnet: 127.0.0.1
warning: ..<...>/Input::READER_ASCII: ../input.log, line 4: Error while reading set or vector
warning: ..<...>/Input::READER_ASCII: ../input.log, line 4: Could not convert line 'name 127.0.0.1' of ../input.log to Val. Ignoring line.
warning: Skipping input with missing non-optional value
warning: ..<...>/Input::READER_ASCII: Invalid value for subnet: 127.0.0.1
warning: ..<...>/Input::READER_ASCII: Error while reading set or vector
warning: ..<...>/Input::READER_ASCII: Could not convert line 'name 127.0.0.1' of ../input.log to Val. Ignoring line.
warning: ..<...>/Input::READER_ASCII: ../input.log, line 4: Invalid value for subnet: 127.0.0.1
warning: ..<...>/Input::READER_ASCII: ../input.log, line 4: Error while reading set or vector
warning: ..<...>/Input::READER_ASCII: ../input.log, line 4: Could not convert line 'name 127.0.0.1' of ../input.log to Val. Ignoring line.
received termination signal
>>>

8
testing/btest/Baseline/scripts.base.frameworks.input.invalidtext/.stderrwithoutfirstline
View file

@ -1,7 +1,7 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
warning: ..<...>/Input::READER_ASCII: String 'l' contained no parseable number
warning: ..<...>/Input::READER_ASCII: Could not convert line ' l' of ../input.log to Val. Ignoring line.
warning: ..<...>/Input::READER_ASCII: String 'l' contained no parseable number
warning: ..<...>/Input::READER_ASCII: Could not convert line ' l' of ../input.log to Val. Ignoring line.
warning: ..<...>/Input::READER_ASCII: ../input.log, line 4: String 'l' contained no parseable number
warning: ..<...>/Input::READER_ASCII: ../input.log, line 4: Could not convert line ' l' of ../input.log to Val. Ignoring line.
warning: ..<...>/Input::READER_ASCII: ../input.log, line 4: String 'l' contained no parseable number
warning: ..<...>/Input::READER_ASCII: ../input.log, line 4: Could not convert line ' l' of ../input.log to Val. Ignoring line.
received termination signal
>>>

2
testing/btest/Baseline/scripts.base.frameworks.input.missing-file-initially/zeek..stderr
View file

@ -4,7 +4,7 @@ error: ../does-not-exist.dat/Input::READER_ASCII: Init failed
error: ../does-not-exist.dat/Input::READER_ASCII: Init: cannot open ../does-not-exist.dat
error: ../does-not-exist.dat/Input::READER_ASCII: terminating thread
received termination signal
warning: ../does-not-exist.dat/Input::READER_ASCII: Could not get stat for ../does-not-exist.dat
warning: ../does-not-exist.dat/Input::READER_ASCII: ../does-not-exist.dat: Could not get stat for ../does-not-exist.dat
warning: ../does-not-exist.dat/Input::READER_ASCII: Init: cannot open ../does-not-exist.dat
warning: ../does-not-exist.dat/Input::READER_ASCII: Init: cannot open ../does-not-exist.dat
warning: ../does-not-exist.dat/Input::READER_ASCII: Init: cannot open ../does-not-exist.dat

6
testing/btest/Baseline/scripts.base.frameworks.input.port-embedded/zeek..stderr
View file

@ -1,5 +1,5 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
warning: ../input.log/Input::READER_ASCII: Port '50/trash' contained unknown protocol 'trash'
warning: ../input.log/Input::READER_ASCII: Number '-1' cannot be negative
warning: ../input.log/Input::READER_ASCII: Could not convert line '1.2.3.8 -1/tcp' of ../input.log to Val. Ignoring line.
warning: ../input.log/Input::READER_ASCII: ../input.log, line 5: Port '50/trash' contained unknown protocol 'trash'
warning: ../input.log/Input::READER_ASCII: ../input.log, line 6: Number '-1' cannot be negative
warning: ../input.log/Input::READER_ASCII: ../input.log, line 6: Could not convert line '1.2.3.8 -1/tcp' of ../input.log to Val. Ignoring line.
received termination signal