Include file information in input reader error messages

This commit is contained in:
Tim Wojtulewicz 2021-12-09 12:08:33 -07:00
parent 8184073ef8
commit 098a5d3348
13 changed files with 179 additions and 57 deletions

View file

@ -2,6 +2,7 @@
#include "zeek/input/ReaderBackend.h" #include "zeek/input/ReaderBackend.h"
#include "zeek/Desc.h"
#include "zeek/input/Manager.h" #include "zeek/input/Manager.h"
#include "zeek/input/ReaderFrontend.h" #include "zeek/input/ReaderFrontend.h"

View file

@ -7,6 +7,11 @@
#include "zeek/threading/MsgThread.h" #include "zeek/threading/MsgThread.h"
#include "zeek/threading/SerialTypes.h" #include "zeek/threading/SerialTypes.h"
namespace zeek::detail
{
class Location;
}
namespace zeek::input namespace zeek::input
{ {
@ -55,6 +60,12 @@ enum ReaderMode
class ReaderBackend : public threading::MsgThread class ReaderBackend : public threading::MsgThread
{ {
public: public:
// Silence a warning from clang about hidden overloaded functions and the
// Info() function that this class provides.
using threading::MsgThread::Error;
using threading::MsgThread::Info;
using threading::MsgThread::Warning;
/** /**
* Constructor. * Constructor.
* *

View file

@ -55,9 +55,10 @@ Ascii::Ascii(ReaderFrontend* frontend) : ReaderBackend(frontend)
fail_on_invalid_lines = false; fail_on_invalid_lines = false;
} }
Ascii::~Ascii() { } void Ascii::DoClose()
{
void Ascii::DoClose() { } read_location.reset();
}
bool Ascii::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fields) bool Ascii::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fields)
{ {
@ -156,6 +157,9 @@ bool Ascii::OpenFile()
return ! fail_on_file_problem; return ! fail_on_file_problem;
} }
read_location = std::make_unique<zeek::detail::Location>();
read_location->filename = util::copy_string(fname.c_str());
StopWarningSuppression(); StopWarningSuppression();
return true; return true;
} }
@ -253,6 +257,12 @@ bool Ascii::GetLine(string& str)
{ {
while ( getline(file, str) ) while ( getline(file, str) )
{ {
if ( read_location )
{
read_location->first_line++;
read_location->last_line++;
}
if ( ! str.size() ) if ( ! str.size() )
continue; continue;
@ -278,6 +288,12 @@ bool Ascii::DoUpdate()
if ( ! OpenFile() ) if ( ! OpenFile() )
return ! fail_on_file_problem; return ! fail_on_file_problem;
if ( read_location )
{
read_location->first_line = 0;
read_location->last_line = 0;
}
switch ( Info().mode ) switch ( Info().mode )
{ {
case MODE_REREAD: case MODE_REREAD:

View file

@ -8,6 +8,7 @@
#include <memory> #include <memory>
#include <vector> #include <vector>
#include "zeek/Obj.h"
#include "zeek/input/ReaderBackend.h" #include "zeek/input/ReaderBackend.h"
#include "zeek/threading/formatters/Ascii.h" #include "zeek/threading/formatters/Ascii.h"
@ -20,20 +21,16 @@ struct FieldMapping
std::string name; std::string name;
TypeTag type; TypeTag type;
TypeTag subtype; // internal type for sets and vectors TypeTag subtype; // internal type for sets and vectors
int position; int position = -1;
int secondary_position; // for ports: pos of the second field int secondary_position = -1; // for ports: pos of the second field
bool present; bool present = false;
FieldMapping(const std::string& arg_name, const TypeTag& arg_type, int arg_position); FieldMapping(const std::string& arg_name, const TypeTag& arg_type, int arg_position);
FieldMapping(const std::string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, FieldMapping(const std::string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype,
int arg_position); int arg_position);
FieldMapping(const FieldMapping& arg); FieldMapping(const FieldMapping& arg);
FieldMapping() FieldMapping() = default;
{
position = -1;
secondary_position = -1;
}
FieldMapping subType(); FieldMapping subType();
}; };
@ -45,7 +42,7 @@ class Ascii : public ReaderBackend
{ {
public: public:
explicit Ascii(ReaderFrontend* frontend); explicit Ascii(ReaderFrontend* frontend);
~Ascii() override; ~Ascii() override = default;
// prohibit copying and moving // prohibit copying and moving
Ascii(const Ascii&) = delete; Ascii(const Ascii&) = delete;
@ -62,6 +59,8 @@ protected:
bool DoUpdate() override; bool DoUpdate() override;
bool DoHeartbeat(double network_time, double current_time) override; bool DoHeartbeat(double network_time, double current_time) override;
zeek::detail::Location* GetLocationInfo() const override { return read_location.get(); }
private: private:
bool ReadHeader(bool useCached); bool ReadHeader(bool useCached);
bool GetLine(std::string& str); bool GetLine(std::string& str);
@ -92,6 +91,7 @@ private:
std::string path_prefix; std::string path_prefix;
std::unique_ptr<threading::Formatter> formatter; std::unique_ptr<threading::Formatter> formatter;
std::unique_ptr<zeek::detail::Location> read_location;
}; };
} // namespace zeek::input::reader::detail } // namespace zeek::input::reader::detail

View file

@ -5,6 +5,8 @@
#include <unistd.h> #include <unistd.h>
#include "zeek/DebugLogger.h" #include "zeek/DebugLogger.h"
#include "zeek/Desc.h"
#include "zeek/Obj.h"
#include "zeek/RunState.h" #include "zeek/RunState.h"
#include "zeek/iosource/Manager.h" #include "zeek/iosource/Manager.h"
#include "zeek/threading/Manager.h" #include "zeek/threading/Manager.h"
@ -341,38 +343,105 @@ void MsgThread::Finished()
void MsgThread::Info(const char* msg) void MsgThread::Info(const char* msg)
{ {
SendOut(new detail::ReporterMessage(detail::ReporterMessage::INFO, this, msg)); ODesc desc;
if ( auto* location = GetLocationInfo() )
{
location->Describe(&desc);
desc.Add(": ");
}
desc.Add(msg);
SendOut(new detail::ReporterMessage(detail::ReporterMessage::INFO, this, desc.Description()));
} }
void MsgThread::Warning(const char* msg) void MsgThread::Warning(const char* msg)
{ {
SendOut(new detail::ReporterMessage(detail::ReporterMessage::WARNING, this, msg)); ODesc desc;
if ( auto* location = GetLocationInfo() )
{
location->Describe(&desc);
desc.Add(": ");
}
desc.Add(msg);
SendOut(
new detail::ReporterMessage(detail::ReporterMessage::WARNING, this, desc.Description()));
} }
void MsgThread::Error(const char* msg) void MsgThread::Error(const char* msg)
{ {
SendOut(new detail::ReporterMessage(detail::ReporterMessage::ERROR, this, msg)); ODesc desc;
if ( auto* location = GetLocationInfo() )
{
location->Describe(&desc);
desc.Add(": ");
}
desc.Add(msg);
SendOut(new detail::ReporterMessage(detail::ReporterMessage::ERROR, this, desc.Description()));
} }
void MsgThread::FatalError(const char* msg) void MsgThread::FatalError(const char* msg)
{ {
SendOut(new detail::ReporterMessage(detail::ReporterMessage::FATAL_ERROR, this, msg)); ODesc desc;
if ( auto* location = GetLocationInfo() )
{
location->Describe(&desc);
desc.Add(": ");
}
desc.Add(msg);
SendOut(new detail::ReporterMessage(detail::ReporterMessage::FATAL_ERROR, this,
desc.Description()));
} }
void MsgThread::FatalErrorWithCore(const char* msg) void MsgThread::FatalErrorWithCore(const char* msg)
{ {
SendOut(new detail::ReporterMessage(detail::ReporterMessage::FATAL_ERROR_WITH_CORE, this, msg)); ODesc desc;
if ( auto* location = GetLocationInfo() )
{
location->Describe(&desc);
desc.Add(": ");
}
desc.Add(msg);
SendOut(new detail::ReporterMessage(detail::ReporterMessage::FATAL_ERROR_WITH_CORE, this,
desc.Description()));
} }
void MsgThread::InternalWarning(const char* msg) void MsgThread::InternalWarning(const char* msg)
{ {
SendOut(new detail::ReporterMessage(detail::ReporterMessage::INTERNAL_WARNING, this, msg)); ODesc desc;
if ( auto* location = GetLocationInfo() )
{
location->Describe(&desc);
desc.Add(": ");
}
desc.Add(msg);
SendOut(new detail::ReporterMessage(detail::ReporterMessage::INTERNAL_WARNING, this,
desc.Description()));
} }
void MsgThread::InternalError(const char* msg) void MsgThread::InternalError(const char* msg)
{ {
// This one aborts immediately. // This one aborts immediately.
fprintf(stderr, "internal error in thread: %s\n", msg); ODesc desc;
if ( auto* location = GetLocationInfo() )
{
location->Describe(&desc);
desc.Add(": ");
}
desc.Add(msg);
fprintf(stderr, "internal error in thread: %s\n", desc.Description());
abort(); abort();
} }
@ -380,7 +449,16 @@ void MsgThread::InternalError(const char* msg)
void MsgThread::Debug(DebugStream stream, const char* msg) void MsgThread::Debug(DebugStream stream, const char* msg)
{ {
SendOut(new detail::DebugMessage(stream, this, msg)); ODesc desc;
if ( auto* location = GetLocationInfo() )
{
location->Describe(&desc);
desc.Add(": ");
}
desc.Add(msg);
SendOut(new detail::DebugMessage(stream, this, desc.Description()));
} }
#endif #endif

View file

@ -8,6 +8,11 @@
#include "zeek/threading/BasicThread.h" #include "zeek/threading/BasicThread.h"
#include "zeek/threading/Queue.h" #include "zeek/threading/Queue.h"
namespace zeek::detail
{
class Location;
}
namespace zeek::threading namespace zeek::threading
{ {
@ -275,6 +280,17 @@ protected:
void OnSignalStop() override; void OnSignalStop() override;
void OnKill() override; void OnKill() override;
/**
* Method for child classes to override to provide file location
* information in log messages. This is primarily used by the input
* framework's ReaderBackend classes to give more descriptive error
* messages.
*
* @return A Location pointer containing the file location information,
* or nullptr if nothing is available.
*/
virtual zeek::detail::Location* GetLocationInfo() const { return nullptr; }
private: private:
/** /**
* Pops a message sent by the main thread from the main-to-chold * Pops a message sent by the main thread from the main-to-chold

View file

@ -1,10 +1,10 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
### NOTE: This file has been sorted with diff-sort. ### NOTE: This file has been sorted with diff-sort.
warning: input.log/Input::READER_ASCII: Could not convert line '2 /cat/sss' of input.log to Val. Ignoring line. warning: input.log/Input::READER_ASCII: input.log, line 5: Could not convert line '2 /cat/sss' of input.log to Val. Ignoring line.
warning: input.log/Input::READER_ASCII: Could not convert line '3 /foo|bar' of input.log to Val. Ignoring line. warning: input.log/Input::READER_ASCII: input.log, line 5: String '/cat/sss' contained no parseable pattern.
warning: input.log/Input::READER_ASCII: Could not convert line '4 this is not a pattern' of input.log to Val. Ignoring line. warning: input.log/Input::READER_ASCII: input.log, line 6: Could not convert line '3 /foo|bar' of input.log to Val. Ignoring line.
warning: input.log/Input::READER_ASCII: Could not convert line '5 /5' of input.log to Val. Ignoring line. warning: input.log/Input::READER_ASCII: input.log, line 6: String '/foo|bar' contained no parseable pattern.
warning: input.log/Input::READER_ASCII: String '/5' contained no parseable pattern. warning: input.log/Input::READER_ASCII: input.log, line 7: Could not convert line '4 this is not a pattern' of input.log to Val. Ignoring line.
warning: input.log/Input::READER_ASCII: String '/cat/sss' contained no parseable pattern. warning: input.log/Input::READER_ASCII: input.log, line 7: String 'this is not a pattern' contained no parseable pattern.
warning: input.log/Input::READER_ASCII: String '/foo|bar' contained no parseable pattern. warning: input.log/Input::READER_ASCII: input.log, line 8: Could not convert line '5 /5' of input.log to Val. Ignoring line.
warning: input.log/Input::READER_ASCII: String 'this is not a pattern' contained no parseable pattern. warning: input.log/Input::READER_ASCII: input.log, line 8: String '/5' contained no parseable pattern.

View file

@ -1,11 +1,11 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
### NOTE: This file has been sorted with diff-sort. ### NOTE: This file has been sorted with diff-sort.
>>> >>>
error: ../input.log/Input::READER_ASCII: Init failed error: ../input.log/Input::READER_ASCII: ../input.log, line 5: Init failed
error: ../input.log/Input::READER_ASCII: Not enough fields in line 'T -41 SSH::LOG 21 123 tcp 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30' of ../input.log. Found 15 fields, want positions 17 and -1 error: ../input.log/Input::READER_ASCII: ../input.log, line 5: Not enough fields in line 'T -41 SSH::LOG 21 123 tcp 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30' of ../input.log. Found 15 fields, want positions 17 and -1
error: ../input.log/Input::READER_ASCII: terminating thread error: ../input.log/Input::READER_ASCII: ../input.log, line 5: terminating thread
received termination signal received termination signal
warning: ../input.log/Input::READER_ASCII: Bad address: 342.2.3.4 warning: ../input.log/Input::READER_ASCII: ../input.log, line 5: Not enough fields in line 'T -41 SSH::LOG 21 123 tcp 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30' of ../input.log. Found 15 fields, want positions 17 and -1
warning: ../input.log/Input::READER_ASCII: Not enough fields in line 'T -41 SSH::LOG 21 123 tcp 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30' of ../input.log. Found 15 fields, want positions 17 and -1 warning: ../input.log/Input::READER_ASCII: ../input.log, line 7: Tried to parse invalid/unknown protocol: whatever
warning: ../input.log/Input::READER_ASCII: Not enough fields in line 'T -41' of ../input.log. Found 1 fields, want positions 2 and -1 warning: ../input.log/Input::READER_ASCII: ../input.log, line 8: Bad address: 342.2.3.4
warning: ../input.log/Input::READER_ASCII: Tried to parse invalid/unknown protocol: whatever warning: ../input.log/Input::READER_ASCII: ../input.log, line 9: Not enough fields in line 'T -41' of ../input.log. Found 1 fields, want positions 2 and -1

View file

@ -1,11 +1,11 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
warning: ..<...>/Input::READER_ASCII: Number '12129223372036854775800' out of supported range. warning: ..<...>/Input::READER_ASCII: ../input.log, line 4: Number '12129223372036854775800' out of supported range.
warning: ..<...>/Input::READER_ASCII: Could not convert line '12129223372036854775800 121218446744073709551612' of ../input.log to Val. Ignoring line. warning: ..<...>/Input::READER_ASCII: ../input.log, line 4: Could not convert line '12129223372036854775800 121218446744073709551612' of ../input.log to Val. Ignoring line.
warning: ..<...>/Input::READER_ASCII: Number '9223372036854775801TEXTHERE' contained non-numeric trailing characters. Ignored trailing characters 'TEXTHERE' warning: ..<...>/Input::READER_ASCII: ../input.log, line 5: Number '9223372036854775801TEXTHERE' contained non-numeric trailing characters. Ignored trailing characters 'TEXTHERE'
warning: ..<...>/Input::READER_ASCII: Number '1Justtext' contained non-numeric trailing characters. Ignored trailing characters 'Justtext' warning: ..<...>/Input::READER_ASCII: ../input.log, line 5: Number '1Justtext' contained non-numeric trailing characters. Ignored trailing characters 'Justtext'
warning: ..<...>/Input::READER_ASCII: String 'Justtext' contained no parseable number warning: ..<...>/Input::READER_ASCII: ../input.log, line 6: String 'Justtext' contained no parseable number
warning: ..<...>/Input::READER_ASCII: Could not convert line 'Justtext 1' of ../input.log to Val. Ignoring line. warning: ..<...>/Input::READER_ASCII: ../input.log, line 6: Could not convert line 'Justtext 1' of ../input.log to Val. Ignoring line.
warning: ..<...>/Input::READER_ASCII: Number ' -18446744073709551612' cannot be negative warning: ..<...>/Input::READER_ASCII: ../input.log, line 7: Number ' -18446744073709551612' cannot be negative
warning: ..<...>/Input::READER_ASCII: Could not convert line '9223372036854775800 -18446744073709551612' of ../input.log to Val. Ignoring line. warning: ..<...>/Input::READER_ASCII: ../input.log, line 7: Could not convert line '9223372036854775800 -18446744073709551612' of ../input.log to Val. Ignoring line.
received termination signal received termination signal
>>> >>>

View file

@ -1,11 +1,11 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
warning: Skipping input with missing non-optional value warning: Skipping input with missing non-optional value
warning: ..<...>/Input::READER_ASCII: Invalid value for subnet: 127.0.0.1 warning: ..<...>/Input::READER_ASCII: ../input.log, line 4: Invalid value for subnet: 127.0.0.1
warning: ..<...>/Input::READER_ASCII: Error while reading set or vector warning: ..<...>/Input::READER_ASCII: ../input.log, line 4: Error while reading set or vector
warning: ..<...>/Input::READER_ASCII: Could not convert line 'name 127.0.0.1' of ../input.log to Val. Ignoring line. warning: ..<...>/Input::READER_ASCII: ../input.log, line 4: Could not convert line 'name 127.0.0.1' of ../input.log to Val. Ignoring line.
warning: Skipping input with missing non-optional value warning: Skipping input with missing non-optional value
warning: ..<...>/Input::READER_ASCII: Invalid value for subnet: 127.0.0.1 warning: ..<...>/Input::READER_ASCII: ../input.log, line 4: Invalid value for subnet: 127.0.0.1
warning: ..<...>/Input::READER_ASCII: Error while reading set or vector warning: ..<...>/Input::READER_ASCII: ../input.log, line 4: Error while reading set or vector
warning: ..<...>/Input::READER_ASCII: Could not convert line 'name 127.0.0.1' of ../input.log to Val. Ignoring line. warning: ..<...>/Input::READER_ASCII: ../input.log, line 4: Could not convert line 'name 127.0.0.1' of ../input.log to Val. Ignoring line.
received termination signal received termination signal
>>> >>>

View file

@ -1,7 +1,7 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
warning: ..<...>/Input::READER_ASCII: String 'l' contained no parseable number warning: ..<...>/Input::READER_ASCII: ../input.log, line 4: String 'l' contained no parseable number
warning: ..<...>/Input::READER_ASCII: Could not convert line ' l' of ../input.log to Val. Ignoring line. warning: ..<...>/Input::READER_ASCII: ../input.log, line 4: Could not convert line ' l' of ../input.log to Val. Ignoring line.
warning: ..<...>/Input::READER_ASCII: String 'l' contained no parseable number warning: ..<...>/Input::READER_ASCII: ../input.log, line 4: String 'l' contained no parseable number
warning: ..<...>/Input::READER_ASCII: Could not convert line ' l' of ../input.log to Val. Ignoring line. warning: ..<...>/Input::READER_ASCII: ../input.log, line 4: Could not convert line ' l' of ../input.log to Val. Ignoring line.
received termination signal received termination signal
>>> >>>

View file

@ -4,7 +4,7 @@ error: ../does-not-exist.dat/Input::READER_ASCII: Init failed
error: ../does-not-exist.dat/Input::READER_ASCII: Init: cannot open ../does-not-exist.dat error: ../does-not-exist.dat/Input::READER_ASCII: Init: cannot open ../does-not-exist.dat
error: ../does-not-exist.dat/Input::READER_ASCII: terminating thread error: ../does-not-exist.dat/Input::READER_ASCII: terminating thread
received termination signal received termination signal
warning: ../does-not-exist.dat/Input::READER_ASCII: Could not get stat for ../does-not-exist.dat warning: ../does-not-exist.dat/Input::READER_ASCII: ../does-not-exist.dat: Could not get stat for ../does-not-exist.dat
warning: ../does-not-exist.dat/Input::READER_ASCII: Init: cannot open ../does-not-exist.dat warning: ../does-not-exist.dat/Input::READER_ASCII: Init: cannot open ../does-not-exist.dat
warning: ../does-not-exist.dat/Input::READER_ASCII: Init: cannot open ../does-not-exist.dat warning: ../does-not-exist.dat/Input::READER_ASCII: Init: cannot open ../does-not-exist.dat
warning: ../does-not-exist.dat/Input::READER_ASCII: Init: cannot open ../does-not-exist.dat warning: ../does-not-exist.dat/Input::READER_ASCII: Init: cannot open ../does-not-exist.dat

View file

@ -1,5 +1,5 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
warning: ../input.log/Input::READER_ASCII: Port '50/trash' contained unknown protocol 'trash' warning: ../input.log/Input::READER_ASCII: ../input.log, line 5: Port '50/trash' contained unknown protocol 'trash'
warning: ../input.log/Input::READER_ASCII: Number '-1' cannot be negative warning: ../input.log/Input::READER_ASCII: ../input.log, line 6: Number '-1' cannot be negative
warning: ../input.log/Input::READER_ASCII: Could not convert line '1.2.3.8 -1/tcp' of ../input.log to Val. Ignoring line. warning: ../input.log/Input::READER_ASCII: ../input.log, line 6: Could not convert line '1.2.3.8 -1/tcp' of ../input.log to Val. Ignoring line.
received termination signal received termination signal