From d6cef16f77778dce83f27e9a0dea279ac9e47fbf Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sun, 6 Mar 2011 19:28:48 -0800 Subject: [PATCH] Rotation support. This follows rather closely how rotation currently works in rotate-logs.bro. logging.bro now defines: # Default rotation interval; zero disables rotation. const default_rotation_interval = 0secs &redef; # Default naming suffix format. const default_rotation_date_format = "%y-%m-%d_%H.%M.%S" &redef; # Default postprocessor for writers outputting into files. const default_rotation_postprocessor = "" &redef; # Default function to construct the name of the rotated file. # The default implementation includes # default_rotation_date_format into the file name. global default_rotation_path_func: function(info: RotationInfo) : string &redef; Writer support for rotation is optional, usually it will only make sense for file-based writers. TODO: Currently, there's no way to customize rotation on a per file basis, there are only the global defaults as described above. Individual customization is coming next. --- policy/bro.init | 4 +- policy/logging.bro | 34 ++++- src/LogMgr.cc | 166 ++++++++++++++++++++-- src/LogMgr.h | 5 + src/LogWriter.cc | 54 ++++++- src/LogWriter.h | 38 ++--- src/LogWriterAscii.cc | 34 +++-- src/LogWriterAscii.h | 8 +- src/Scope.cc | 5 +- src/Scope.h | 2 +- src/logging.bif | 4 + src/parse.y | 18 ++- testing/btest/Baseline/logging.rotate/out | 50 +++++++ testing/btest/btest.cfg | 2 +- testing/btest/logging/rotate.bro | 31 ++++ testing/btest/logging/rotation.trace | Bin 0 -> 1144 bytes 16 files changed, 387 insertions(+), 68 deletions(-) create mode 100644 testing/btest/Baseline/logging.rotate/out create mode 100644 testing/btest/logging/rotate.bro create mode 100644 testing/btest/logging/rotation.trace diff --git a/policy/bro.init b/policy/bro.init index 3a107d865c..84f0094a71 100644 --- a/policy/bro.init +++ b/policy/bro.init @@ -273,12 +273,12 @@ type entropy_test_result: record { serial_correlation: double; }; -@load logging # sic! Not logging.bif. - # Prototypes of Bro built-in functions. @load strings.bif.bro @load bro.bif.bro +@load logging # sic! Not logging.bif. + global bro_alarm_file: file &redef; global alarm_hook: function(msg: string): bool &redef; global log_file_name: function(tag: string): string &redef; diff --git a/policy/logging.bro b/policy/logging.bro index c5613e4ada..f3d4e35c37 100644 --- a/policy/logging.bro +++ b/policy/logging.bro @@ -3,8 +3,12 @@ module Log; # Log::ID and Log::Writer are defined in bro.init due to circular dependencies. export { - # The default writer to use. - const default_writer = Log::WRITER_ASCII &redef; + # Information passed to a rotation callback function. + type RotationInfo: record { + path: string; # Original path value. + open: time; # Time when opened. + close: time; # Time when closed. + }; # If true, local logging is by default enabled for all filters. const enable_local_logging = T &redef; @@ -12,6 +16,23 @@ export { # If true, remote logging is by default enabled for all filters. const enable_remote_logging = T &redef; + # The default writer to use. + const default_writer = Log::WRITER_ASCII &redef; + + # Default rotation interval; zero disables rotation. + const default_rotation_interval = 0secs &redef; + + # Default naming suffix format. + const default_rotation_date_format = "%y-%m-%d_%H.%M.%S" &redef; + + # Default postprocessor for writers outputting into files. + const default_rotation_postprocessor = "" &redef; + + # Default function to construct the name of the rotated file. + # The default implementation includes + # default_rotation_date_format into the file name. + global default_rotation_path_func: function(info: RotationInfo) : string &redef; + # A stream defining the logging. type Stream: record { # A record type defining the log's columns. @@ -51,10 +72,10 @@ export { exclude: set[string] &optional; # If true, record all log records locally. - log_local: bool &default=Log::enable_local_logging; + log_local: bool &default=enable_local_logging; # If true, pass all log records on to remote peers if they request it. - log_remote: bool &default=Log::enable_remote_logging; + log_remote: bool &default=enable_remote_logging; # The writer to use. writer: Writer &default=Log::default_writer; @@ -74,6 +95,11 @@ export { module Log; +function default_rotation_path_func(info: RotationInfo) : string + { + return fmt("%s-%s", info$path, strftime(default_rotation_date_format, info$open)); + } + function create_stream(id: Log::ID, stream: Log::Stream) : bool { if ( ! Log::__create_stream(id, stream) ) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 15db1cb927..46293e71fe 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -3,6 +3,7 @@ #include "Event.h" #include "EventHandler.h" #include "NetVar.h" +#include "Net.h" #include "LogWriterAscii.h" @@ -22,6 +23,7 @@ LogWriterDefinition log_writers[] = { struct LogMgr::Filter { string name; + EnumVal* id; Func* pred; Func* path_func; string path; @@ -37,6 +39,12 @@ struct LogMgr::Filter { ~Filter(); }; +struct LogMgr::WriterInfo { + double open_time; + Timer* rotation_timer; + LogWriter *writer; + }; + struct LogMgr::Stream { EnumVal* id; string name; @@ -45,7 +53,9 @@ struct LogMgr::Stream { list filters; typedef pair WriterPathPair; - typedef map WriterMap; + + typedef map WriterMap; + WriterMap writers; // Writers indexed by id/path pair. ~Stream(); @@ -213,7 +223,15 @@ LogMgr::Stream::~Stream() Unref(columns); for ( WriterMap::iterator i = writers.begin(); i != writers.end(); i++ ) + { + WriterInfo* winfo = i->second; + + if ( winfo->rotation_timer ) + timer_mgr->Cancel(winfo->rotation_timer); + + delete winfo->writer; delete i->second; + } for ( list::iterator f = filters.begin(); f != filters.end(); ++f ) delete *f; @@ -248,7 +266,7 @@ void LogMgr::RemoveDisabledWriters(Stream* stream) for ( Stream::WriterMap::iterator j = stream->writers.begin(); j != stream->writers.end(); j++ ) { - if ( j->second->Disabled() ) + if ( j->second->writer->Disabled() ) { delete j->second; disabled.push_back(j->first); @@ -413,16 +431,26 @@ bool LogMgr::AddFilter(EnumVal* id, RecordVal* fval) // Create a new Filter instance. - Val* pred = fval->Lookup(rtype->FieldOffset("pred")); - Val* path_func = fval->Lookup(rtype->FieldOffset("path_func")); + Val* name = fval->LookupWithDefault(rtype->FieldOffset("name")); + Val* pred = fval->LookupWithDefault(rtype->FieldOffset("pred")); + Val* path_func = fval->LookupWithDefault(rtype->FieldOffset("path_func")); + Val* log_local = fval->LookupWithDefault(rtype->FieldOffset("log_local")); + Val* log_remote = fval->LookupWithDefault(rtype->FieldOffset("log_remote")); Filter* filter = new Filter; - filter->name = fval->Lookup(rtype->FieldOffset("name"))->AsString()->CheckString(); + filter->name = name->AsString()->CheckString(); + filter->id = id->Ref()->AsEnumVal(); filter->pred = pred ? pred->AsFunc() : 0; filter->path_func = path_func ? path_func->AsFunc() : 0; filter->writer = writer->Ref()->AsEnumVal(); - filter->local = fval->LookupWithDefault(rtype->FieldOffset("log_local"))->AsBool(); - filter->remote = fval->LookupWithDefault(rtype->FieldOffset("log_remote"))->AsBool(); + filter->local = log_local->AsBool(); + filter->remote = log_remote->AsBool(); + + Unref(name); + Unref(pred); + Unref(path_func); + Unref(log_local); + Unref(log_remote); // TODO: Check that the predciate is of the right type. @@ -580,7 +608,7 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) if ( w != stream->writers.end() ) // We have a writer already. - writer = w->second; + writer = w->second->writer; else { // No, need to create one. @@ -669,10 +697,13 @@ LogVal** LogMgr::RecordToFilterVals(Filter* filter, RecordVal* columns) case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: vals[i]->val.uint_val = val->InternalUnsigned(); break; + case TYPE_PORT: + vals[i]->val.uint_val = val->AsPortVal()->Port(); + break; + case TYPE_SUBNET: vals[i]->val.subnet_val = *val->AsSubNet(); break; @@ -719,7 +750,7 @@ LogWriter* LogMgr::CreateWriter(EnumVal* id, EnumVal* writer, string path, int n if ( w != stream->writers.end() ) // If we already have a writer for this. That's fine, we just return // it. - return w->second; + return w->second->writer; // Need to instantiate a new writer. @@ -766,7 +797,13 @@ LogWriter* LogMgr::CreateWriter(EnumVal* id, EnumVal* writer, string path, int n return 0; } - stream->writers.insert(Stream::WriterMap::value_type(Stream::WriterPathPair(writer->AsEnum(), path), writer_obj)); + WriterInfo* winfo = new WriterInfo; + winfo->writer = writer_obj; + winfo->open_time = network_time; + winfo->rotation_timer = 0; + InstallRotationTimer(winfo); + + stream->writers.insert(Stream::WriterMap::value_type(Stream::WriterPathPair(writer->AsEnum(), path), winfo)); return writer_obj; } @@ -799,7 +836,7 @@ bool LogMgr::Write(EnumVal* id, EnumVal* writer, string path, int num_fields, Lo return false; } - bool success = w->second->Write(num_fields, vals); + bool success = w->second->writer->Write(num_fields, vals); DBG_LOG(DBG_LOGGING, "Wrote pre-filtered record to path '%s' on stream '%s' [%s]", path.c_str(), stream->name.c_str(), (success ? "ok" : "error")); @@ -817,7 +854,7 @@ void LogMgr::SendAllWritersTo(RemoteSerializer::PeerID peer) for ( Stream::WriterMap::iterator i = stream->writers.begin(); i != stream->writers.end(); i++ ) { - LogWriter* writer = i->second; + LogWriter* writer = i->second->writer; EnumVal writer_val(i->first.first, BifType::Enum::Log::Writer); remote_serializer->SendLogCreateWriter(peer, (*s)->id, &writer_val, i->first.second, writer->NumFields(), writer->Fields()); } @@ -831,7 +868,7 @@ bool LogMgr::SetBuf(EnumVal* id, bool enabled) return false; for ( Stream::WriterMap::iterator i = stream->writers.begin(); i != stream->writers.end(); i++ ) - i->second->SetBuf(enabled); + i->second->writer->SetBuf(enabled); RemoveDisabledWriters(stream); @@ -845,7 +882,7 @@ bool LogMgr::Flush(EnumVal* id) return false; for ( Stream::WriterMap::iterator i = stream->writers.begin(); i != stream->writers.end(); i++ ) - i->second->Flush(); + i->second->writer->Flush(); RemoveDisabledWriters(stream); @@ -857,3 +894,102 @@ void LogMgr::Error(LogWriter* writer, const char* msg) run_time(fmt("error with writer for %s: %s", writer->Path().c_str(), msg)); } +// Timer which on dispatching rotates the filter. +class RotationTimer : public Timer { +public: + RotationTimer(double t, LogMgr::WriterInfo* arg_winfo, bool arg_rotate) + : Timer(t, TIMER_ROTATE) { winfo = arg_winfo; rotate = arg_rotate; } + ~RotationTimer(); + + void Dispatch(double t, int is_expire); + +protected: + LogMgr::WriterInfo* winfo; + bool rotate; +}; + +RotationTimer::~RotationTimer() + { + if ( winfo->rotation_timer == this ) + winfo->rotation_timer = 0; + } + +void RotationTimer::Dispatch(double t, int is_expire) + { + winfo->rotation_timer = 0; + + if ( rotate ) + log_mgr->Rotate(winfo); + + if ( ! is_expire ) + { + winfo->open_time = network_time; + log_mgr->InstallRotationTimer(winfo); + } + } + +void LogMgr::InstallRotationTimer(WriterInfo* winfo) + { + if ( terminating ) + return; + + if ( winfo->rotation_timer ) + { + timer_mgr->Cancel(winfo->rotation_timer); + winfo->rotation_timer = 0; + } + + double rotation_interval = BifConst::Log::default_rotation_interval; + + if ( rotation_interval ) + { + // When this is called for the first time, network_time can still be + // zero. If so, we set a timer which fires immediately but doesn't + // rotate when it expires. + if ( ! network_time ) + winfo->rotation_timer = new RotationTimer(1, winfo, false); + else + { + if ( ! winfo->open_time ) + winfo->open_time = network_time; + + const char* base_time = log_rotate_base_time ? + log_rotate_base_time->AsString()->CheckString() : 0; + + double delta_t = + calc_next_rotate(rotation_interval, base_time); + + winfo->rotation_timer = new RotationTimer(network_time + delta_t, winfo, true); + } + + timer_mgr->Add(winfo->rotation_timer); + + DBG_LOG(DBG_LOGGING, "Scheduled rotation timer for %s to %.6f", winfo->writer->Path().c_str(), winfo->rotation_timer->Time()); + } + } + +void LogMgr::Rotate(WriterInfo* winfo) + { + DBG_LOG(DBG_LOGGING, "Rotating %s at %.6f", winfo->writer->Path().c_str(), network_time); + + // Create the RotationInfo record. + RecordVal* info = new RecordVal(BifType::Record::Log::RotationInfo); + info->Assign(0, new StringVal(winfo->writer->Path().c_str())); + info->Assign(1, new Val(winfo->open_time, TYPE_TIME)); + info->Assign(2, new Val(network_time, TYPE_TIME)); + + // Call the function building us the new path. + + Func* rotation_path_func = internal_func("Log::default_rotation_path_func"); + string rotation_postprocessor = BifConst::Log::default_rotation_postprocessor->AsString()->CheckString(); + + val_list vl(1); + vl.append(info); + Val* result = rotation_path_func->Call(&vl); + string new_path = result->AsString()->CheckString(); + Unref(result); + + winfo->writer->Rotate(new_path, rotation_postprocessor, winfo->open_time, network_time, terminating); + } + + diff --git a/src/LogMgr.h b/src/LogMgr.h index 069ddfdff2..1188b80336 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -57,6 +57,7 @@ private: class LogWriter; class RemoteSerializer; +class RotationTimer; class LogMgr { public: @@ -75,6 +76,7 @@ public: protected: friend class LogWriter; friend class RemoteSerializer; + friend class RotationTimer; // These function are also used by the RemoteSerializer. LogWriter* CreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, LogField** fields); // takes ownership of fields. @@ -89,11 +91,14 @@ protected: private: struct Filter; struct Stream; + struct WriterInfo; bool TraverseRecord(Filter* filter, RecordType* rt, TableVal* include, TableVal* exclude, string path, list indices); LogVal** RecordToFilterVals(Filter* filter, RecordVal* columns); Stream* FindStream(EnumVal* id); void RemoveDisabledWriters(Stream* stream); + void InstallRotationTimer(WriterInfo* winfo); + void Rotate(WriterInfo* info); vector streams; // Indexed by stream enum. }; diff --git a/src/LogWriter.cc b/src/LogWriter.cc index 6c27bb55b5..2298004185 100644 --- a/src/LogWriter.cc +++ b/src/LogWriter.cc @@ -18,7 +18,7 @@ LogWriter::~LogWriter() delete [] fields; } -bool LogWriter::Init(string arg_path, int arg_num_fields, LogField** arg_fields) +bool LogWriter::Init(string arg_path, int arg_num_fields, const LogField* const * arg_fields) { path = arg_path; num_fields = arg_num_fields; @@ -75,6 +75,17 @@ bool LogWriter::SetBuf(bool enabled) return true; } +bool LogWriter::Rotate(string rotated_path, string postprocessor, double open, double close, bool terminating) + { + if ( ! DoRotate(rotated_path, postprocessor, open, close, terminating) ) + { + disabled = true; + return false; + } + + return true; + } + bool LogWriter::Flush() { if ( ! DoFlush() ) @@ -126,4 +137,45 @@ void LogWriter::DeleteVals(LogVal** vals) delete vals[i]; } +bool LogWriter::RunPostProcessor(string fname, string postprocessor, string old_name, double open, double close, bool terminating) + { + // This function operates in way backwards-compatible with the old Bro + // log rotation scheme. + + if ( ! postprocessor.size() ) + return true; + + const char* const fmt = "%y-%m-%d_%H.%M.%S"; + + struct tm tm1; + struct tm tm2; + + time_t tt1 = (time_t)open; + time_t tt2 = (time_t)close; + + localtime_r(&tt1, &tm1); + localtime_r(&tt2, &tm2); + + char buf1[128]; + char buf2[128]; + + strftime(buf1, sizeof(buf1), fmt, &tm1); + strftime(buf2, sizeof(buf2), fmt, &tm2); + + string cmd = postprocessor; + cmd += " " + fname; + cmd += " " + old_name; + cmd += " " + string(buf1); + cmd += " " + string(buf2); + cmd += " " + string(terminating ? "1" : "0"); + cmd += " &"; + + system(cmd.c_str()); + + return true; + } + + + + diff --git a/src/LogWriter.h b/src/LogWriter.h index 0dd37d2881..5a89e6be9e 100644 --- a/src/LogWriter.h +++ b/src/LogWriter.h @@ -23,7 +23,7 @@ public: // // The new instance takes ownership of "fields", and will delete them // when done. - bool Init(string path, int num_fields, LogField** fields); + bool Init(string path, int num_fields, const LogField* const * fields); // Writes one log entry. The method takes ownership of "vals" and will // return immediately after queueing the write request, potentially @@ -35,14 +35,19 @@ public: // Sets the buffering status for the writer, if the writer supports it. bool SetBuf(bool enabled); - // Flushes any currently buffered output, if the writer support it. + // Flushes any currently buffered output, if the writer supports it. bool Flush(); + // Triggers rotation, if the writer supports it. + bool Rotate(string rotated_path, string postprocessor, double open, double close, bool terminating); + // Finished writing to this logger. Will not be called if an error has // been indicated earlier. After calling this, no more writing must be // performed. void Finish(); + // Returns the path as passed to Init(). + const string Path() const { return path; } int NumFields() const { return num_fields; } const LogField* const * Fields() const { return fields; } @@ -57,10 +62,10 @@ protected: // always return true. // Called once for initialization of the Writer. - virtual bool DoInit(string path, int num_fields, LogField** fields) = 0; + virtual bool DoInit(string path, int num_fields, const LogField* const * fields) = 0; // Called once per entry to record. - virtual bool DoWrite(int num_fields, LogField** fields, LogVal** vals) = 0; + virtual bool DoWrite(int num_fields, const LogField* const * fields, LogVal** vals) = 0; // Called when the buffering status for this writer is changed. If // buffering is disabled, the writer should attempt to write out @@ -80,25 +85,24 @@ protected: virtual bool DoFlush() = 0; // Called when a log output is to be rotated. Most directly, this only - // applies to writers outputting files, thoug a writer may also trigger - // other regular actions if that fits a similar model + // applies to writers outputting files, though a writer may also trigger + // other regular actions if semantics are similar. // // The string "rotate_path" is interpreted in writer-specific way, yet - // should generally should have similar semantics that the "path" passed + // should generally should have similar semantics as the "path" passed // into DoInit(), except that now it reflects the name to where the // rotated output is to be moved. After rotation, output should continue // normally with the standard "path". As an example, for file-based // output, the rotate_path may be the original filename with an embedded - // timestamp. - // - // The writer must return false if an error occured that prevent the - // writer for continuing operation; it will then be disabled. The error - // reason should be reported via Error(). If a recoverable error occurs, - // still call Error(), but return true. + // timestamp. "postprocessor" is the name of a command to execute on the + // rotated file. If empty, no such processing should take place; if given + // but the writer doesn't support postprocessing, it can be ignored. + // "open" and "close" are the network time's at opening and closeing the + // current file, respetively. // // A writer may ignore rotation requests if it doesn't fit with its // semantics. - virtual bool DoRotate(string rotated_path) = 0; + virtual bool DoRotate(string rotated_path, string postprocessor, double open, double close, bool terminating) = 0; // Called once on termination. Not called when any of the other methods // has previously signaled an error, i.e., executing this method signals @@ -116,8 +120,8 @@ protected: // Reports an error. void Error(const char *msg); - // Returns the path as passed to Init(). - const string Path() const { return path; } + // Runs a post-processor on the given file. + bool RunPostProcessor(string fname, string postprocessor, string old_name, double open, double close, bool terminating); private: friend class LogMgr; @@ -131,7 +135,7 @@ private: string path; int num_fields; - LogField** fields; + const LogField* const * fields; bool buffering; bool disabled; diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index 902cbf7ea2..6ba3f812f7 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -6,26 +6,22 @@ LogWriterAscii::LogWriterAscii() { - fname = 0; file = 0; } LogWriterAscii::~LogWriterAscii() { - if ( fname ) - free(fname); - if ( file ) fclose(file); } -bool LogWriterAscii::DoInit(string path, int num_fields, LogField** fields) +bool LogWriterAscii::DoInit(string path, int num_fields, const LogField* const * fields) { - fname = strdup(Fmt("%s.log", path.c_str())); + fname = path + ".log"; - if ( ! (file = fopen(fname, "w")) ) + if ( ! (file = fopen(fname.c_str(), "w")) ) { - Error(Fmt("cannot open %s: %s", fname, strerror(errno))); + Error(Fmt("cannot open %s: %s", fname.c_str(), strerror(errno))); return false; } @@ -34,7 +30,7 @@ bool LogWriterAscii::DoInit(string path, int num_fields, LogField** fields) for ( int i = 0; i < num_fields; i++ ) { - LogField* field = fields[i]; + const LogField* field = fields[i]; if ( fputs(field->name.c_str(), file) == EOF ) goto write_error; @@ -48,7 +44,7 @@ bool LogWriterAscii::DoInit(string path, int num_fields, LogField** fields) return true; write_error: - Error(Fmt("error writing to %s: %s", fname, strerror(errno))); + Error(Fmt("error writing to %s: %s", fname.c_str(), strerror(errno))); return false; } @@ -62,7 +58,7 @@ void LogWriterAscii::DoFinish() { } -bool LogWriterAscii::DoWrite(int num_fields, LogField** fields, LogVal** vals) +bool LogWriterAscii::DoWrite(int num_fields, const LogField* const * fields, LogVal** vals) { ODesc desc(DESC_READABLE); @@ -72,7 +68,7 @@ bool LogWriterAscii::DoWrite(int num_fields, LogField** fields, LogVal** vals) desc.Add("\t"); LogVal* val = vals[i]; - LogField* field = fields[i]; + const LogField* field = fields[i]; if ( ! val->present ) { @@ -127,7 +123,7 @@ bool LogWriterAscii::DoWrite(int num_fields, LogField** fields, LogVal** vals) if ( fwrite(desc.Bytes(), desc.Len(), 1, file) != 1 ) { - Error(Fmt("error writing to %s: %s", fname, strerror(errno))); + Error(Fmt("error writing to %s: %s", fname.c_str(), strerror(errno))); return false; } @@ -137,9 +133,17 @@ bool LogWriterAscii::DoWrite(int num_fields, LogField** fields, LogVal** vals) return true; } -bool LogWriterAscii::DoRotate(string rotated_path) +bool LogWriterAscii::DoRotate(string rotated_path, string postprocessor, double open, double close, bool terminating) { - return true; + fclose(file); + + string nname = rotated_path + ".log"; + rename(fname.c_str(), nname.c_str()); + + if ( postprocessor.size() && ! RunPostProcessor(nname, postprocessor, fname.c_str(), open, close, terminating) ) + return false; + + return DoInit(Path(), NumFields(), Fields()); } bool LogWriterAscii::DoSetBuf(bool enabled) diff --git a/src/LogWriterAscii.h b/src/LogWriterAscii.h index e4501aa5b7..9732242fba 100644 --- a/src/LogWriterAscii.h +++ b/src/LogWriterAscii.h @@ -15,16 +15,16 @@ public: static LogWriter* Instantiate() { return new LogWriterAscii; } protected: - virtual bool DoInit(string path, int num_fields, LogField** fields); - virtual bool DoWrite(int num_fields, LogField** fields, LogVal** vals); + virtual bool DoInit(string path, int num_fields, const LogField* const * fields); + virtual bool DoWrite(int num_fields, const LogField* const * fields, LogVal** vals); virtual bool DoSetBuf(bool enabled); - virtual bool DoRotate(string rotated_path); + virtual bool DoRotate(string rotated_path, string postprocessr, double open, double close, bool terminating); virtual bool DoFlush(); virtual void DoFinish(); private: FILE* file; - char* fname; + string fname; }; #endif diff --git a/src/Scope.cc b/src/Scope.cc index 64cf61080f..eda38d9c80 100644 --- a/src/Scope.cc +++ b/src/Scope.cc @@ -113,9 +113,10 @@ TraversalCode Scope::Traverse(TraversalCallback* cb) const } -ID* lookup_ID(const char* name, const char* curr_module, bool no_global) +ID* lookup_ID(const char* name, const char* curr_module, bool no_global, bool same_module_only) { string fullname = make_full_var_name(curr_module, name); + string ID_module = extract_module_name(fullname.c_str()); bool need_export = ID_module != GLOBAL_MODULE_NAME && ID_module != curr_module; @@ -134,7 +135,7 @@ ID* lookup_ID(const char* name, const char* curr_module, bool no_global) } } - if ( ! no_global ) + if ( ! no_global && (strcmp(GLOBAL_MODULE_NAME, curr_module) == 0 || ! same_module_only) ) { string globalname = make_full_var_name(GLOBAL_MODULE_NAME, name); ID* id = global_scope()->Lookup(globalname.c_str()); diff --git a/src/Scope.h b/src/Scope.h index ca86a5ec1f..d4b7ba9ac9 100644 --- a/src/Scope.h +++ b/src/Scope.h @@ -65,7 +65,7 @@ extern bool in_debug; // If no_global is true, don't search in the default "global" namespace. extern ID* lookup_ID(const char* name, const char* module, - bool no_global = false); + bool no_global = false, bool same_module_only=false); extern ID* install_ID(const char* name, const char* module_name, bool is_global, bool is_export); diff --git a/src/logging.bif b/src/logging.bif index e31c56276e..a561c6c5a3 100644 --- a/src/logging.bif +++ b/src/logging.bif @@ -8,6 +8,10 @@ module Log; type Filter: record; type Stream: record; +type RotationInfo: record; + +const Log::default_rotation_interval: interval; +const Log::default_rotation_postprocessor: string; function Log::__create_stream%(id: Log::ID, stream: Log::Stream%) : bool %{ diff --git a/src/parse.y b/src/parse.y index a982935b2b..4f81347cde 100644 --- a/src/parse.y +++ b/src/parse.y @@ -42,7 +42,7 @@ %left '$' '[' ']' '(' ')' TOK_HAS_FIELD TOK_HAS_ATTR %type TOK_ID TOK_PATTERN_TEXT single_pattern -%type local_id global_id event_id global_or_event_id resolve_id begin_func +%type local_id global_id def_global_id event_id global_or_event_id resolve_id begin_func %type local_id_list %type init_class %type opt_init @@ -102,6 +102,7 @@ Expr* bro_this = 0; int in_init = 0; bool in_debug = false; bool resolving_global_ID = false; +bool defining_global_ID = false; ID* func_id = 0; EnumType *cur_enum_type = 0; @@ -842,10 +843,10 @@ decl: | TOK_EXPORT '{' { is_export = true; } decl_list '}' { is_export = false; } - | TOK_GLOBAL global_id opt_type init_class opt_init opt_attr ';' + | TOK_GLOBAL def_global_id opt_type init_class opt_init opt_attr ';' { add_global($2, $3, $4, $5, $6, VAR_REGULAR); } - | TOK_CONST global_id opt_type init_class opt_init opt_attr ';' + | TOK_CONST def_global_id opt_type init_class opt_init opt_attr ';' { add_global($2, $3, $4, $5, $6, VAR_CONST); } | TOK_REDEF global_id opt_type init_class opt_init opt_attr ';' @@ -855,7 +856,7 @@ decl: '{' { parser_redef_enum($3); } enum_body '}' ';' { /* no action */ } - | TOK_TYPE global_id ':' refined_type opt_attr ';' + | TOK_TYPE def_global_id ':' refined_type opt_attr ';' { add_type($2, $4, $5, 0); } @@ -883,7 +884,7 @@ conditional: ; func_hdr: - TOK_FUNCTION global_id func_params + TOK_FUNCTION def_global_id func_params { begin_func($2, current_module.c_str(), FUNC_FLAVOR_FUNCTION, 0, $3); @@ -1265,6 +1266,11 @@ global_id: { $$ = $2; } ; +def_global_id: + { defining_global_ID = 1; } global_id { defining_global_ID = 0; } + { $$ = $2; } + ; + event_id: { resolving_global_ID = 0; } global_or_event_id { $$ = $2; } @@ -1275,7 +1281,7 @@ global_or_event_id: { set_location(@1); - $$ = lookup_ID($1, current_module.c_str(), false); + $$ = lookup_ID($1, current_module.c_str(), false, defining_global_ID); if ( $$ ) { if ( ! $$->IsGlobal() ) diff --git a/testing/btest/Baseline/logging.rotate/out b/testing/btest/Baseline/logging.rotate/out new file mode 100644 index 0000000000..27bd5179cf --- /dev/null +++ b/testing/btest/Baseline/logging.rotate/out @@ -0,0 +1,50 @@ +test-11-03-06_19.00.05.log test.log 11-03-06_19.00.05 11-03-06_20.00.05 0 +test-11-03-06_20.00.05.log test.log 11-03-06_20.00.05 11-03-06_21.00.05 0 +test-11-03-06_21.00.05.log test.log 11-03-06_21.00.05 11-03-06_22.00.05 0 +test-11-03-06_22.00.05.log test.log 11-03-06_22.00.05 11-03-06_23.00.05 0 +test-11-03-06_23.00.05.log test.log 11-03-06_23.00.05 11-03-07_00.00.05 0 +test-11-03-07_00.00.05.log test.log 11-03-07_00.00.05 11-03-07_01.00.05 0 +test-11-03-07_01.00.05.log test.log 11-03-07_01.00.05 11-03-07_02.00.05 0 +test-11-03-07_02.00.05.log test.log 11-03-07_02.00.05 11-03-07_03.00.05 0 +test-11-03-07_03.00.05.log test.log 11-03-07_03.00.05 11-03-07_04.00.05 0 +test-11-03-07_04.00.05.log test.log 11-03-07_04.00.05 11-03-07_04.59.55 1 +> test-11-03-06_19.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299466805.0 10.0.0.1 20 10.0.0.2 1024 +1299470395.0 10.0.0.2 20 10.0.0.3 0 +> test-11-03-06_20.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299470405.0 10.0.0.1 20 10.0.0.2 1025 +1299473995.0 10.0.0.2 20 10.0.0.3 1 +> test-11-03-06_21.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299474005.0 10.0.0.1 20 10.0.0.2 1026 +1299477595.0 10.0.0.2 20 10.0.0.3 2 +> test-11-03-06_22.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299477605.0 10.0.0.1 20 10.0.0.2 1027 +1299481195.0 10.0.0.2 20 10.0.0.3 3 +> test-11-03-06_23.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299481205.0 10.0.0.1 20 10.0.0.2 1028 +1299484795.0 10.0.0.2 20 10.0.0.3 4 +> test-11-03-07_00.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299484805.0 10.0.0.1 20 10.0.0.2 1029 +1299488395.0 10.0.0.2 20 10.0.0.3 5 +> test-11-03-07_01.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299488405.0 10.0.0.1 20 10.0.0.2 1030 +1299491995.0 10.0.0.2 20 10.0.0.3 6 +> test-11-03-07_02.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299492005.0 10.0.0.1 20 10.0.0.2 1031 +1299495595.0 10.0.0.2 20 10.0.0.3 7 +> test-11-03-07_03.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299495605.0 10.0.0.1 20 10.0.0.2 1032 +1299499195.0 10.0.0.2 20 10.0.0.3 8 +> test-11-03-07_04.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299499205.0 10.0.0.1 20 10.0.0.2 1033 +1299502795.0 10.0.0.2 20 10.0.0.3 9 diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 1e702ce344..a49fc6af02 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -4,7 +4,7 @@ TestDirs = logging TmpDir = %(testbase)s/.tmp BaselineDir = %(testbase)s/Baseline IgnoreDirs = .svn CVS .tmp -IgnoreFiles = *.tmp *.swp #* +IgnoreFiles = *.tmp *.swp #* *.trace [environment] BROPATH=`bash -c %(testbase)s/../../build/bro-path-dev` diff --git a/testing/btest/logging/rotate.bro b/testing/btest/logging/rotate.bro new file mode 100644 index 0000000000..96d5068db7 --- /dev/null +++ b/testing/btest/logging/rotate.bro @@ -0,0 +1,31 @@ +# +# @TEST-EXEC: bro -r %DIR/rotation.trace %INPUT >out +# @TEST-EXEC: for i in test-*.log; do printf '> %s\n' $i; cat $i; done >>out +# @TEST-EXEC: btest-diff out + +module Test; + +export { + # Create a new ID for our log stream + redef enum Log::ID += { Test }; + + # Define a record with all the columns the log file can have. + # (I'm using a subset of fields from ssh-ext for demonstration.) + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + }; +} + +redef Log::default_rotation_interval = 1hr; +redef Log::default_rotation_postprocessor = "echo"; + +event bro_init() +{ + Log::create_stream(Test, [$columns=Log]); +} + +event new_connection(c: connection) + { + Log::write(Test, [$t=network_time(), $id=c$id]); + } diff --git a/testing/btest/logging/rotation.trace b/testing/btest/logging/rotation.trace new file mode 100644 index 0000000000000000000000000000000000000000..9954b22e1597624b5f9d944b219e77092d141cb7 GIT binary patch literal 1144 zcma))yGnyW5Jpe_XuQ;U083xM$|g-3MX(VBYtz_9QcNdU+1OYKf`ye!)7n#bDX@vPRvsxR>&3S#F@;Oe1a2Ipdx9EJD^I@FwIZmEn6m!