From b6c1b35bb80cb09fe0d0fefbb626358a30177ae3 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 26 Nov 2013 10:35:04 -0800 Subject: [PATCH 001/193] Updating submodule(s). [nomail] --- aux/binpac | 2 +- aux/bro-aux | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- aux/btest | 2 +- cmake | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/aux/binpac b/aux/binpac index 54b321009b..896ddedde5 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 54b321009b750268526419bdbd841f421c839313 +Subproject commit 896ddedde55c48ec2163577fc258b49c418abb3e diff --git a/aux/bro-aux b/aux/bro-aux index ebf9c0d88a..6c5db7af6e 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit ebf9c0d88ae8230845b91f15755156f93ff21aa8 +Subproject commit 6c5db7af6e93406be498b24f8a2d67ca281c1a03 diff --git a/aux/broccoli b/aux/broccoli index 17ec437752..889f9c6594 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 17ec437752837fb4214abfb0a2da49df74668d5d +Subproject commit 889f9c65944ceac20ad9230efc39d33e6e1221c3 diff --git a/aux/broctl b/aux/broctl index 6e01d6972f..0cd102805e 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 6e01d6972f02d68ee82d05f392d1a00725595b7f +Subproject commit 0cd102805e73343cab3f9fd4a76552e13940dad9 diff --git a/aux/btest b/aux/btest index 26c3136d56..ce366206e3 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit 26c3136d56493017bc33c5a2f22ae393d585c2d9 +Subproject commit ce366206e3407e534a786ad572c342e9f9fef26b diff --git a/cmake b/cmake index e7a46cb82e..0187b33a29 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit e7a46cb82ee10aa522c4d88115baf10181277d20 +Subproject commit 0187b33a29d5ec824f940feff60dc5d8c2fe314f From f0fe27002903ea7c9a17141e27b059bcc87b255d Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 26 Nov 2013 10:57:02 -0800 Subject: [PATCH 002/193] Minor interface changes to provide more accessor methods for class information. In particular, adding a few const versions of methods. --- src/Event.h | 2 ++ src/ID.h | 2 ++ src/Type.cc | 28 ++++++++++++++++++++++++---- src/Type.h | 13 +++++++++++-- 4 files changed, 39 insertions(+), 6 deletions(-) diff --git a/src/Event.h b/src/Event.h index 9d0a707cda..6f9c9d10c3 100644 --- a/src/Event.h +++ b/src/Event.h @@ -24,6 +24,8 @@ public: SourceID Source() const { return src; } analyzer::ID Analyzer() const { return aid; } TimerMgr* Mgr() const { return mgr; } + EventHandlerPtr Handler() const { return handler; } + val_list* Args() const { return args; } void Describe(ODesc* d) const; diff --git a/src/ID.h b/src/ID.h index 57e1222511..5b71830640 100644 --- a/src/ID.h +++ b/src/ID.h @@ -32,9 +32,11 @@ public: void SetType(BroType* t) { Unref(type); type = t; } BroType* Type() { return type; } + const BroType* Type() const { return type; } void MakeType() { is_type = 1; } BroType* AsType() { return is_type ? Type() : 0; } + const BroType* AsType() const { return is_type ? Type() : 0; } // If weak_ref is false, the Val is assumed to be already ref'ed // and will be deref'ed when the ID is deleted. diff --git a/src/Type.cc b/src/Type.cc index a6d8b90c6c..6b4b2eb970 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -430,6 +430,11 @@ BroType* IndexType::YieldType() return yield_type; } +const BroType* IndexType::YieldType() const + { + return yield_type; + } + void IndexType::Describe(ODesc* d) const { BroType::Describe(d); @@ -723,6 +728,11 @@ BroType* FuncType::YieldType() return yield; } +const BroType* FuncType::YieldType() const + { + return yield; + } + int FuncType::MatchesIndex(ListExpr*& index) const { return check_and_promote_args(index, args) ? @@ -1444,9 +1454,9 @@ void CommentedEnumType::AddNameInternal(const string& module_name, const char* n names[copy_string(fullname.c_str())] = val; } -bro_int_t EnumType::Lookup(const string& module_name, const char* name) +bro_int_t EnumType::Lookup(const string& module_name, const char* name) const { - NameMap::iterator pos = + NameMap::const_iterator pos = names.find(make_full_var_name(module_name.c_str(), name).c_str()); if ( pos == names.end() ) @@ -1455,9 +1465,9 @@ bro_int_t EnumType::Lookup(const string& module_name, const char* name) return pos->second; } -const char* EnumType::Lookup(bro_int_t value) +const char* EnumType::Lookup(bro_int_t value) const { - for ( NameMap::iterator iter = names.begin(); + for ( NameMap::const_iterator iter = names.begin(); iter != names.end(); ++iter ) if ( iter->second == value ) return iter->first; @@ -1465,6 +1475,16 @@ const char* EnumType::Lookup(bro_int_t value) return 0; } +EnumType::enum_name_list EnumType::Names() const + { + enum_name_list n; + for ( NameMap::const_iterator iter = names.begin(); + iter != names.end(); ++iter ) + n.push_back(std::make_pair(iter->first, iter->second)); + + return n; + } + void EnumType::DescribeReST(ODesc* d) const { d->Add(":bro:type:`"); diff --git a/src/Type.h b/src/Type.h index a6163d5152..6c162bdabb 100644 --- a/src/Type.h +++ b/src/Type.h @@ -304,6 +304,7 @@ public: TypeList* Indices() const { return indices; } const type_list* IndexTypes() const { return indices->Types(); } BroType* YieldType(); + const BroType* YieldType() const; void Describe(ODesc* d) const; void DescribeReST(ODesc* d) const; @@ -366,6 +367,7 @@ public: RecordType* Args() const { return args; } BroType* YieldType(); + const BroType* YieldType() const; void SetYieldType(BroType* arg_yield) { yield = arg_yield; } function_flavor Flavor() const { return flavor; } string FlavorString() const; @@ -522,6 +524,8 @@ protected: class EnumType : public BroType { public: + typedef std::list > enum_name_list; + EnumType(const string& arg_name); EnumType(EnumType* e); ~EnumType(); @@ -536,11 +540,15 @@ public: void AddName(const string& module_name, const char* name, bro_int_t val, bool is_export); // -1 indicates not found. - bro_int_t Lookup(const string& module_name, const char* name); - const char* Lookup(bro_int_t value); // Returns 0 if not found + bro_int_t Lookup(const string& module_name, const char* name) const; + const char* Lookup(bro_int_t value) const; // Returns 0 if not found string Name() const { return name; } + // Returns the list of defined names with their values. The names + // will be fully qualified with their module name. + enum_name_list Names() const; + void DescribeReST(ODesc* d) const; protected: @@ -592,6 +600,7 @@ public: VectorType(BroType* t); virtual ~VectorType(); BroType* YieldType() { return yield_type; } + const BroType* YieldType() const { return yield_type; } int MatchesIndex(ListExpr*& index) const; From d34f23c8d4153417d107db82167095f63dec9457 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 26 Nov 2013 11:16:58 -0800 Subject: [PATCH 003/193] A set of file analysis extensions. - Enable manager to associate analyzers with a MIME type. With that, one can now say enable all analyzers for, e.g., "image/gif". This is exposed to script-land as Files::add_analyzers_for_mime_type(f: fa_file, mtype: string) For MIME types identified via libmagic, this happens automatically (via the file_new() handler in files/main.bro). - Extend the analyzer API to better match that of protocol analyzers: - Adding unique analyzer IDs so that we can refer to instances from script-land. - Adding subtypes to Components so that a single analyzer implementation can support different types of analyzers internally. - Add an analyzer method SetTag() that allows to set the tag after construction. - Adding Init() and Done() methods for consistency with what other classes offer. - Add debug logging to the file_analysis stream. TODO: test cases missing for the new script-land functionality. --- scripts/base/frameworks/files/main.bro | 39 ++++++--- scripts/base/init-bare.bro | 7 ++ src/file_analysis/Analyzer.cc | 8 ++ src/file_analysis/Analyzer.h | 54 ++++++++++++- src/file_analysis/AnalyzerSet.cc | 10 ++- src/file_analysis/Component.cc | 8 +- src/file_analysis/Component.h | 8 +- src/file_analysis/File.cc | 42 +++++++--- src/file_analysis/File.h | 2 +- src/file_analysis/Manager.cc | 105 +++++++++++++++++++++++-- src/file_analysis/Manager.h | 74 +++++++++++++++++ src/file_analysis/file_analysis.bif | 10 +++ 12 files changed, 337 insertions(+), 30 deletions(-) diff --git a/scripts/base/frameworks/files/main.bro b/scripts/base/frameworks/files/main.bro index cf2c11be45..7fecec3034 100644 --- a/scripts/base/frameworks/files/main.bro +++ b/scripts/base/frameworks/files/main.bro @@ -56,7 +56,7 @@ export { ## local file path which was read, or some other input source. source: string &log &optional; - ## A value to represent the depth of this file in relation + ## A value to represent the depth of this file in relation ## to its source. In SMTP, it is the depth of the MIME ## attachment on the message. In HTTP, it is the depth of the ## request within the TCP connection. @@ -72,7 +72,7 @@ export { mime_type: string &log &optional; ## A filename for the file if one is available from the source - ## for the file. These will frequently come from + ## for the file. These will frequently come from ## "Content-Disposition" headers in network protocols. filename: string &log &optional; @@ -148,9 +148,18 @@ export { ## Returns: true if the analyzer will be added, or false if analysis ## for the file isn't currently active or the *args* ## were invalid for the analyzer type. - global add_analyzer: function(f: fa_file, - tag: Files::Tag, - args: AnalyzerArgs &default=AnalyzerArgs()): bool; + global add_analyzer: function(f: fa_file, + tag: Files::Tag, + args: AnalyzerArgs &default=AnalyzerArgs()): bool; + + ## Adds all analyzers associated with a give MIME type to the analysis of + ## a file. Note that analyzers added via MIME types cannot take further + ## arguments. + ## + ## f: the file. + ## + ## mtype: the MIME type; it will be compared case-insensitive. + global add_analyzers_for_mime_type: function(f: fa_file, mtype: string); ## Removes an analyzer from the analysis of a given file. ## @@ -195,7 +204,7 @@ export { ## A callback to generate a file handle on demand when ## one is needed by the core. get_file_handle: function(c: connection, is_orig: bool): string; - + ## A callback to "describe" a file. In the case of an HTTP ## transfer the most obvious description would be the URL. ## It's like an extremely compressed version of the normal log. @@ -206,7 +215,7 @@ export { ## Register callbacks for protocols that work with the Files framework. ## The callbacks must uniquely identify a file and each protocol can ## only have a single callback registered for it. - ## + ## ## tag: Tag for the protocol analyzer having a callback being registered. ## ## reg: A :bro:see:`Files::ProtoRegistration` record. @@ -258,13 +267,13 @@ function set_info(f: fa_file) f$info$source = f$source; f$info$duration = f$last_active - f$info$ts; f$info$seen_bytes = f$seen_bytes; - if ( f?$total_bytes ) + if ( f?$total_bytes ) f$info$total_bytes = f$total_bytes; f$info$missing_bytes = f$missing_bytes; f$info$overflow_bytes = f$overflow_bytes; if ( f?$is_orig ) f$info$is_orig = f$is_orig; - if ( f?$mime_type ) + if ( f?$mime_type ) f$info$mime_type = f$mime_type; } @@ -288,6 +297,15 @@ function add_analyzer(f: fa_file, tag: Files::Tag, args: AnalyzerArgs): bool return T; } +function add_analyzers_for_mime_type(f: fa_file, mtype: string) + { + local dummy_args: AnalyzerArgs; + local analyzers = __add_analyzers_for_mime_type(f$id, mtype, dummy_args); + + for ( tag in analyzers ) + add f$info$analyzers[Files::analyzer_name(tag)]; + } + function register_analyzer_add_callback(tag: Files::Tag, callback: function(f: fa_file, args: AnalyzerArgs)) { analyzer_add_callbacks[tag] = callback; @@ -311,6 +329,9 @@ function analyzer_name(tag: Files::Tag): string event file_new(f: fa_file) &priority=10 { set_info(f); + + if ( f?$mime_type ) + add_analyzers_for_mime_type(f, f$mime_type); } event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=10 diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index de26e6a41d..928697edb0 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -60,6 +60,13 @@ type addr_vec: vector of addr; ## directly and then remove this alias. type table_string_of_string: table[string] of string; +## A set of file analyzer tags. +## +## .. todo:: We need this type definition only for declaring builtin functions +## via ``bifcl``. We should extend ``bifcl`` to understand composite types +## directly and then remove this alias. +type files_tag_set: set[Files::Tag]; + ## A connection's transport-layer protocol. Note that Bro uses the term ## "connection" broadly, using flow semantics for ICMP and UDP. type transport_proto: enum { diff --git a/src/file_analysis/Analyzer.cc b/src/file_analysis/Analyzer.cc index e0b5011aa8..2d45c6579a 100644 --- a/src/file_analysis/Analyzer.cc +++ b/src/file_analysis/Analyzer.cc @@ -3,9 +3,17 @@ #include "Analyzer.h" #include "Manager.h" +file_analysis::ID file_analysis::Analyzer::id_counter = 0; + file_analysis::Analyzer::~Analyzer() { DBG_LOG(DBG_FILE_ANALYSIS, "Destroy file analyzer %s", file_mgr->GetComponentName(tag)); Unref(args); } + +void file_analysis::Analyzer::SetAnalyzerTag(const file_analysis::Tag& arg_tag) + { + assert(! tag || tag == arg_tag); + tag = arg_tag; + } diff --git a/src/file_analysis/Analyzer.h b/src/file_analysis/Analyzer.h index e20e2802cf..ab03b1f9ba 100644 --- a/src/file_analysis/Analyzer.h +++ b/src/file_analysis/Analyzer.h @@ -13,6 +13,8 @@ namespace file_analysis { class File; +typedef uint32 ID; + /** * Base class for analyzers that can be attached to file_analysis::File objects. */ @@ -25,6 +27,18 @@ public: */ virtual ~Analyzer(); + /** + * Initializes the analyzer before input processing starts. + */ + virtual void Init() + { }; + + /** + * Finishes the analyzer's operation after all input has been parsed. + */ + virtual void Done() + { }; + /** * Subclasses may override this metod to receive file data non-sequentially. * @param data points to start of a chunk of file data. @@ -72,6 +86,13 @@ public: */ file_analysis::Tag Tag() const { return tag; } + /** + * Returns the analyzer instance's internal ID. These IDs are unique + * across all analyzers instantiated and can thus be used to + * indentify a specific instance. + */ + ID GetID() const { return id; } + /** * @return the AnalyzerArgs associated with the analyzer. */ @@ -82,10 +103,19 @@ public: */ File* GetFile() const { return file; } + /** + * Sets the tag associated with the analyzer's type. Note that this + * can be called only right after construction, if the constructor + * did not receive a name or tag. The method cannot be used to change + * an existing tag. + */ + void SetAnalyzerTag(const file_analysis::Tag& tag); + protected: /** * Constructor. Only derived classes are meant to be instantiated. + * @param arg_tag the tag definining the analyzer's type. * @param arg_args an \c AnalyzerArgs (script-layer type) value specifiying * tunable options, if any, related to a particular analyzer type. * @param arg_file the file to which the the analyzer is being attached. @@ -94,13 +124,35 @@ protected: : tag(arg_tag), args(arg_args->Ref()->AsRecordVal()), file(arg_file) - {} + { + id = ++id_counter; + } + + /** + * Constructor. Only derived classes are meant to be instantiated. + * As this version of the constructor does not receive a name or tag, + * SetAnalyzerTag() must be called before the instance can be used. + * + * @param arg_args an \c AnalyzerArgs (script-layer type) value specifiying + * tunable options, if any, related to a particular analyzer type. + * @param arg_file the file to which the the analyzer is being attached. + */ + Analyzer(RecordVal* arg_args, File* arg_file) + : tag(), + args(arg_args->Ref()->AsRecordVal()), + file(arg_file) + { + id = ++id_counter; + } private: + ID id; /**< Unique instance ID. */ file_analysis::Tag tag; /**< The particular type of the analyzer instance. */ RecordVal* args; /**< \c AnalyzerArgs val gives tunable analyzer params. */ File* file; /**< The file to which the analyzer is attached. */ + + static ID id_counter; }; } // namespace file_analysis diff --git a/src/file_analysis/AnalyzerSet.cc b/src/file_analysis/AnalyzerSet.cc index 6fc3d2dfd0..aee5e0d62e 100644 --- a/src/file_analysis/AnalyzerSet.cc +++ b/src/file_analysis/AnalyzerSet.cc @@ -9,7 +9,10 @@ using namespace file_analysis; static void analyzer_del_func(void* v) { - delete (file_analysis::Analyzer*) v; + file_analysis::Analyzer* a = (file_analysis::Analyzer*)v; + + a->Done(); + delete a; } AnalyzerSet::AnalyzerSet(File* arg_file) : file(arg_file) @@ -98,6 +101,7 @@ bool AnalyzerSet::AddMod::Perform(AnalyzerSet* set) } set->Insert(a, key); + return true; } @@ -124,7 +128,9 @@ bool AnalyzerSet::Remove(file_analysis::Tag tag, HashKey* key) file_mgr->GetComponentName(tag), file->GetID().c_str()); + a->Done(); delete a; + return true; } @@ -176,6 +182,8 @@ void AnalyzerSet::Insert(file_analysis::Analyzer* a, HashKey* key) file_mgr->GetComponentName(a->Tag()), file->GetID().c_str()); analyzer_map.Insert(key, a); delete key; + + a->Init(); } void AnalyzerSet::DrainModifications() diff --git a/src/file_analysis/Component.cc b/src/file_analysis/Component.cc index 9c47f2c75e..87789eb113 100644 --- a/src/file_analysis/Component.cc +++ b/src/file_analysis/Component.cc @@ -8,13 +8,15 @@ using namespace file_analysis; -Component::Component(const char* arg_name, factory_callback arg_factory) +Component::Component(const char* arg_name, factory_callback arg_factory, Tag::subtype_t subtype) : plugin::Component(plugin::component::FILE_ANALYZER), - plugin::TaggedComponent() + plugin::TaggedComponent(subtype) { name = copy_string(arg_name); canon_name = canonify_name(arg_name); factory = arg_factory; + + file_mgr->RegisterComponent(this, "ANALYZER_"); } Component::Component(const Component& other) @@ -24,6 +26,8 @@ Component::Component(const Component& other) name = copy_string(other.name); canon_name = copy_string(other.canon_name); factory = other.factory; + + // TODO: Do we need the RegisterComponent() call here? } Component::~Component() diff --git a/src/file_analysis/Component.h b/src/file_analysis/Component.h index 4cf2dced60..6e2809349d 100644 --- a/src/file_analysis/Component.h +++ b/src/file_analysis/Component.h @@ -40,8 +40,14 @@ public: * from file_analysis::Analyzer. This is typically a static \c * Instatiate() method inside the class that just allocates and * returns a new instance. + * + * @param subtype A subtype associated with this component that + * further distinguishes it. The subtype will be integrated into the + * analyzer::Tag that the manager associates with this analyzer, and + * analyzer instances can accordingly access it via analyzer::Tag(). + * If not used, leave at zero. */ - Component(const char* name, factory_callback factory); + Component(const char* name, factory_callback factory, Tag::subtype_t subtype = 0); /** * Copy constructor. diff --git a/src/file_analysis/File.cc b/src/file_analysis/File.cc index 55b28763c8..980a8bfbce 100644 --- a/src/file_analysis/File.cc +++ b/src/file_analysis/File.cc @@ -1,6 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. #include +#include #include "File.h" #include "FileTimer.h" @@ -82,7 +83,7 @@ File::File(const string& file_id, Connection* conn, analyzer::Tag tag, { StaticInit(); - DBG_LOG(DBG_FILE_ANALYSIS, "Creating new File object %s", file_id.c_str()); + DBG_LOG(DBG_FILE_ANALYSIS, "[%s] Creating new File object", file_id.c_str()); val = new RecordVal(fa_file_type); val->Assign(id_idx, new StringVal(file_id.c_str())); @@ -100,7 +101,7 @@ File::File(const string& file_id, Connection* conn, analyzer::Tag tag, File::~File() { - DBG_LOG(DBG_FILE_ANALYSIS, "Destroying File object %s", id.c_str()); + DBG_LOG(DBG_FILE_ANALYSIS, "[%s] Destroying File object", id.c_str()); Unref(val); // Queue may not be empty in the case where only content gaps were seen. @@ -229,6 +230,7 @@ void File::IncrementByteCount(uint64 size, int field_idx) void File::SetTotalBytes(uint64 size) { + DBG_LOG(DBG_FILE_ANALYSIS, "[%s] Total bytes %" PRIu64, id.c_str(), size); val->Assign(total_bytes_idx, new Val(size, TYPE_COUNT)); } @@ -251,11 +253,17 @@ void File::ScheduleInactivityTimer() const bool File::AddAnalyzer(file_analysis::Tag tag, RecordVal* args) { + DBG_LOG(DBG_FILE_ANALYSIS, "[%s] Queuing addition of %s analyzer", + id.c_str(), file_mgr->GetComponentName(tag)); + return done ? false : analyzers.QueueAdd(tag, args); } bool File::RemoveAnalyzer(file_analysis::Tag tag, RecordVal* args) { + DBG_LOG(DBG_FILE_ANALYSIS, "[%s] Queuing remove of %s analyzer", + id.c_str(), file_mgr->GetComponentName(tag)); + return done ? false : analyzers.QueueRemove(tag, args); } @@ -284,16 +292,18 @@ bool File::DetectMIME(const u_char* data, uint64 len) if ( mime ) { + // strip off charset const char* mime_end = strchr(mime, ';'); - if ( mime_end ) - // strip off charset - val->Assign(mime_type_idx, new StringVal(mime_end - mime, mime)); - else - val->Assign(mime_type_idx, new StringVal(mime)); + StringVal* mime_val = mime_end ? + new StringVal(mime_end - mime, mime) : + new StringVal(mime); + + val->Assign(mime_type_idx, mime_val); + DBG_LOG(DBG_FILE_ANALYSIS, "[%s] Detected MIME type %s", id.c_str(), mime_val->CheckString()); } - return mime; + return true; } void File::ReplayBOF() @@ -314,7 +324,6 @@ void File::ReplayBOF() val->Assign(bof_buffer_idx, new StringVal(bs)); DetectMIME(bs->Bytes(), bs->Len()); - FileEvent(file_new); for ( size_t i = 0; i < bof_buffer.chunks.size(); ++i ) @@ -333,6 +342,11 @@ void File::DataIn(const u_char* data, uint64 len, uint64 offset) first_chunk = false; } + DBG_LOG(DBG_FILE_ANALYSIS, "[%s] %" PRIu64 " bytes in at offset" PRIu64 "; %s [%s]", + id.c_str(), len, offset, + IsComplete() ? "complete" : "incomplete", + fmt_bytes((const char*) data, min((uint64)40, len)), len > 40 ? "..." : ""); + file_analysis::Analyzer* a = 0; IterCookie* c = analyzers.InitForIteration(); @@ -367,6 +381,11 @@ void File::DataIn(const u_char* data, uint64 len) missed_bof = false; } + DBG_LOG(DBG_FILE_ANALYSIS, "[%s] %" PRIu64 " bytes in; %s [%s]", + id.c_str(), len, + IsComplete() ? "complete" : "incomplete", + fmt_bytes((const char*) data, min((uint64)40, len)), len > 40 ? "..." : ""); + file_analysis::Analyzer* a = 0; IterCookie* c = analyzers.InitForIteration(); @@ -391,6 +410,8 @@ void File::DataIn(const u_char* data, uint64 len) void File::EndOfFile() { + DBG_LOG(DBG_FILE_ANALYSIS, "[%s] End of file", id.c_str()); + if ( done ) return; @@ -417,6 +438,9 @@ void File::EndOfFile() void File::Gap(uint64 offset, uint64 len) { + DBG_LOG(DBG_FILE_ANALYSIS, "[%s] Gap of size %" PRIu64 " at offset %" PRIu64, + id.c_str(), offset, len); + analyzers.DrainModifications(); // If we were buffering the beginning of the file, a gap means we've got diff --git a/src/file_analysis/File.h b/src/file_analysis/File.h index 6354c1c7e9..b225d77856 100644 --- a/src/file_analysis/File.h +++ b/src/file_analysis/File.h @@ -229,7 +229,7 @@ protected: * field in #val. * @param data pointer to a chunk of file data. * @param len number of bytes in the data chunk. - * @return whether mime type was available. + * @return true if mime type available. */ bool DetectMIME(const u_char* data, uint64 len); diff --git a/src/file_analysis/Manager.cc b/src/file_analysis/Manager.cc index 0337dbb098..a635a5dc14 100644 --- a/src/file_analysis/Manager.cc +++ b/src/file_analysis/Manager.cc @@ -12,10 +12,12 @@ #include "UID.h" #include "plugin/Manager.h" +#include "analyzer/Manager.h" using namespace file_analysis; TableVal* Manager::disabled = 0; +TableType* Manager::tag_set_type = 0; string Manager::salt; Manager::Manager() @@ -27,15 +29,13 @@ Manager::Manager() Manager::~Manager() { Terminate(); + + for ( MIMEMap::iterator i = mime_types.begin(); i != mime_types.end(); i++ ) + delete i->second; } void Manager::InitPreScript() { - std::list analyzers = plugin_mgr->Components(); - - for ( std::list::const_iterator i = analyzers.begin(); - i != analyzers.end(); ++i ) - RegisterComponent(*i, "ANALYZER_"); } void Manager::InitPostScript() @@ -72,6 +72,7 @@ void Manager::SetHandle(const string& handle) if ( handle.empty() ) return; + DBG_LOG(DBG_FILE_ANALYSIS, "Set current handle to %s", handle.c_str()); current_file_id = HashHandle(handle); } @@ -205,6 +206,28 @@ bool Manager::AddAnalyzer(const string& file_id, file_analysis::Tag tag, return file->AddAnalyzer(tag, args); } +TableVal* Manager::AddAnalyzersForMIMEType(const string& file_id, const string& mtype, + RecordVal* args) + { + if ( ! tag_set_type ) + tag_set_type = internal_type("files_tag_set")->AsTableType(); + + TableVal* sval = new TableVal(tag_set_type); + TagSet* l = LookupMIMEType(mtype, false); + + if ( ! l ) + return sval; + + for ( TagSet::const_iterator i = l->begin(); i != l->end(); i++ ) + { + file_analysis::Tag tag = *i; + if ( AddAnalyzer(file_id, tag, args) ) + sval->Assign(tag.AsEnumVal(), 0); + } + + return sval; + } + bool Manager::RemoveAnalyzer(const string& file_id, file_analysis::Tag tag, RecordVal* args) const { @@ -327,6 +350,9 @@ void Manager::GetFileHandle(analyzer::Tag tag, Connection* c, bool is_orig) if ( ! get_file_handle ) return; + DBG_LOG(DBG_FILE_ANALYSIS, "Raise get_file_handle() for protocol analyzer %s", + analyzer_mgr->GetComponentName(tag)); + EnumVal* tagval = tag.AsEnumVal(); Ref(tagval); @@ -376,5 +402,72 @@ Analyzer* Manager::InstantiateAnalyzer(Tag tag, RecordVal* args, File* f) const return 0; } - return c->Factory()(args, f); + DBG_LOG(DBG_FILE_ANALYSIS, "Instantiate analyzer %s for file %s", + GetComponentName(tag), f->id.c_str()); + + Analyzer* a = c->Factory()(args, f); + + if ( ! a ) + reporter->InternalError("file analyzer instantiation failed"); + + a->SetAnalyzerTag(tag); + + return a; + } + +Manager::TagSet* Manager::LookupMIMEType(const string& mtype, bool add_if_not_found) + { + MIMEMap::const_iterator i = mime_types.find(to_upper(mtype)); + + if ( i != mime_types.end() ) + return i->second; + + if ( ! add_if_not_found ) + return 0; + + TagSet* l = new TagSet; + mime_types.insert(std::make_pair(to_upper(mtype), l)); + return l; + } + +bool Manager::RegisterAnalyzerForMIMEType(EnumVal* tag, StringVal* mtype) + { + Component* p = Lookup(tag); + + if ( ! p ) + return false; + + return RegisterAnalyzerForMIMEType(p->Tag(), mtype->CheckString()); + } + +bool Manager::RegisterAnalyzerForMIMEType(Tag tag, const string& mtype) + { + TagSet* l = LookupMIMEType(mtype, true); + + DBG_LOG(DBG_FILE_ANALYSIS, "Register analyzer %s for MIME type %s", + GetComponentName(tag), mtype.c_str()); + + l->insert(tag); + return true; + } + +bool Manager::UnregisterAnalyzerForMIMEType(EnumVal* tag, StringVal* mtype) + { + Component* p = Lookup(tag); + + if ( ! p ) + return false; + + return UnregisterAnalyzerForMIMEType(p->Tag(), mtype->CheckString()); + } + +bool Manager::UnregisterAnalyzerForMIMEType(Tag tag, const string& mtype) + { + TagSet* l = LookupMIMEType(mtype, true); + + DBG_LOG(DBG_FILE_ANALYSIS, "Unregister analyzer %s for MIME type %s", + GetComponentName(tag), mtype.c_str()); + + l->erase(tag); + return true; } diff --git a/src/file_analysis/Manager.h b/src/file_analysis/Manager.h index cf73c6b52d..c6f86a0480 100644 --- a/src/file_analysis/Manager.h +++ b/src/file_analysis/Manager.h @@ -198,6 +198,18 @@ public: bool AddAnalyzer(const string& file_id, file_analysis::Tag tag, RecordVal* args) const; + /** + * Queue attachment of an all analyzers associated with a given MIME + * type to the file identifier. + * + * @param file_id the file identifier/hash. + * @param mtype the MIME type; comparisions will be performanced case-insensitive. + * @param args a \c AnalyzerArgs value which describes a file analyzer. + * @return A ref'ed \c set[Tag] with all added analyzers. + */ + TableVal* AddAnalyzersForMIMEType(const string& file_id, const string& mtype, + RecordVal* args); + /** * Queue removal of an analyzer for a given file identifier. * @param file_id the file identifier/hash. @@ -224,6 +236,62 @@ public: */ Analyzer* InstantiateAnalyzer(Tag tag, RecordVal* args, File* f) const; + /** + * Registers a MIME type for an analyzer. Once registered, files of + * that MIME type will automatically get a corresponding analyzer + * assigned. + * + * @param tag The analyzer's tag as an enum of script type \c + * Files::Tag. + * + * @param mtype The MIME type. It will be matched case-insenistive. + * + * @return True if successful. + */ + bool RegisterAnalyzerForMIMEType(EnumVal* tag, StringVal* mtype); + + /** + * Registers a MIME type for an analyzer. Once registered, files of + * that MIME type will automatically get a corresponding analyzer + * assigned. + * + * @param tag The analyzer's tag as an enum of script type \c + * Files::Tag. + * + * @param mtype The MIME type. It will be matched case-insenistive. + * + * @return True if successful. + */ + bool RegisterAnalyzerForMIMEType(Tag tag, const string& mtype); + + /** + * Unregisters a MIME type for an analyzer. + * + * @param tag The analyzer's tag as an enum of script type \c + * Files::Tag. + * + * @param mtype The MIME type. It will be matched case-insenistive. + * + * @return True if successful (incl. when the type wasn't actually + * registered for the analyzer). + * + */ + bool UnregisterAnalyzerForMIMEType(EnumVal* tag, StringVal* mtype); + + /** + * Unregisters a MIME type for an analyzer. + * + * @param tag The analyzer's tag as an enum of script type \c + * Files::Tag. + * + * @param mtype The MIME type. It will be matched case-insenistive. + * + * @return True if successful (incl. when the type wasn't actually + * registered for the analyzer). + * + */ + bool UnregisterAnalyzerForMIMEType(Tag tag, const string& mtype); + protected: friend class FileTimer; @@ -297,12 +365,18 @@ protected: static bool IsDisabled(analyzer::Tag tag); private: + typedef set TagSet; + typedef map MIMEMap; + + TagSet* LookupMIMEType(const string& mtype, bool add_if_not_found); IDMap id_map; /**< Map file ID to file_analysis::File records. */ IDSet ignored; /**< Ignored files. Will be finally removed on EOF. */ string current_file_id; /**< Hash of what get_file_handle event sets. */ + MIMEMap mime_types;/**< Mapping of MIME types to analyzers. */ static TableVal* disabled; /**< Table of disabled analyzers. */ + static TableType* tag_set_type; /**< Type for set[tag]. */ static string salt; /**< A salt added to file handles before hashing. */ }; diff --git a/src/file_analysis/file_analysis.bif b/src/file_analysis/file_analysis.bif index 0e904f298f..8205b870c1 100644 --- a/src/file_analysis/file_analysis.bif +++ b/src/file_analysis/file_analysis.bif @@ -26,6 +26,16 @@ function Files::__add_analyzer%(file_id: string, tag: Files::Tag, args: any%): b return new Val(result, TYPE_BOOL); %} +## :bro:see:`Files::add_analyzers_for_mime_type`. +function Files::__add_analyzers_for_mime_type%(file_id: string, mtype: string, args: any%): files_tag_set + %{ + using BifType::Record::Files::AnalyzerArgs; + RecordVal* rv = args->AsRecordVal()->CoerceTo(AnalyzerArgs); + Val* analyzers = file_mgr->AddAnalyzersForMIMEType(file_id->CheckString(), mtype->CheckString(), rv); + Unref(rv); + return analyzers; + %} + ## :bro:see:`Files::remove_analyzer`. function Files::__remove_analyzer%(file_id: string, tag: Files::Tag, args: any%): bool %{ From d727af097b2d78a77c85d98903278a1c1dbdc4ab Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 26 Nov 2013 11:22:23 -0800 Subject: [PATCH 004/193] Minor API changes to analyzers. --- src/analyzer/Analyzer.cc | 4 ++-- src/analyzer/Analyzer.h | 9 ++++++--- src/analyzer/Component.cc | 4 ++++ src/analyzer/Manager.cc | 10 +++++----- src/analyzer/Manager.h | 8 ++++++++ src/analyzer/protocol/tcp/TCP.h | 4 ++-- 6 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/analyzer/Analyzer.cc b/src/analyzer/Analyzer.cc index 03734f1a22..2927e3ad97 100644 --- a/src/analyzer/Analyzer.cc +++ b/src/analyzer/Analyzer.cc @@ -644,12 +644,12 @@ void Analyzer::FlipRoles() resp_supporters = tmp; } -void Analyzer::ProtocolConfirmation() +void Analyzer::ProtocolConfirmation(Tag arg_tag) { if ( protocol_confirmed ) return; - EnumVal* tval = tag.AsEnumVal(); + EnumVal* tval = arg_tag ? arg_tag.AsEnumVal() : tag.AsEnumVal(); Ref(tval); val_list* vl = new val_list; diff --git a/src/analyzer/Analyzer.h b/src/analyzer/Analyzer.h index f7ca07ca51..eec0303475 100644 --- a/src/analyzer/Analyzer.h +++ b/src/analyzer/Analyzer.h @@ -97,8 +97,8 @@ public: /** * Constructor. As this version of the constructor does not receive a - * name or tag, setTag() must be called before the instance can be - * used. + * name or tag, SetAnalyzerTag() must be called before the instance + * can be used. * * @param conn The connection the analyzer is associated with. */ @@ -471,8 +471,11 @@ public: * may turn into \c protocol_confirmed event at the script-layer (but * only once per analyzer for each connection, even if the method is * called multiple times). + * + * If tag is given, it overrides the analyzer tag passed to the + * scripting layer; the default is the one of the analyzer itself. */ - virtual void ProtocolConfirmation(); + virtual void ProtocolConfirmation(Tag tag = Tag()); /** * Signals Bro's protocol detection that the analyzer has found a diff --git a/src/analyzer/Component.cc b/src/analyzer/Component.cc index 66ab2213bb..255e03435c 100644 --- a/src/analyzer/Component.cc +++ b/src/analyzer/Component.cc @@ -17,6 +17,8 @@ Component::Component(const char* arg_name, factory_callback arg_factory, Tag::su factory = arg_factory; enabled = arg_enabled; partial = arg_partial; + + analyzer_mgr->RegisterComponent(this, "ANALYZER_"); } Component::Component(const Component& other) @@ -28,6 +30,8 @@ Component::Component(const Component& other) factory = other.factory; enabled = other.enabled; partial = other.partial; + + // TODO: Do we need the RegisterComponent() call here? } Component::~Component() diff --git a/src/analyzer/Manager.cc b/src/analyzer/Manager.cc index 6268fde177..748af34dfb 100644 --- a/src/analyzer/Manager.cc +++ b/src/analyzer/Manager.cc @@ -86,11 +86,6 @@ Manager::~Manager() void Manager::InitPreScript() { - std::list analyzers = plugin_mgr->Components(); - - for ( std::list::const_iterator i = analyzers.begin(); i != analyzers.end(); i++ ) - RegisterComponent(*i, "ANALYZER_"); - // Cache these tags. analyzer_backdoor = GetComponentTag("BACKDOOR"); analyzer_connsize = GetComponentTag("CONNSIZE"); @@ -202,6 +197,11 @@ void Manager::DisableAllAnalyzers() (*i)->SetEnabled(false); } +analyzer::Tag Manager::GetAnalyzerTag(const char* name) + { + return GetComponentTag(name); + } + bool Manager::IsEnabled(Tag tag) { if ( ! tag ) diff --git a/src/analyzer/Manager.h b/src/analyzer/Manager.h index 77b40a1d68..c842794d6c 100644 --- a/src/analyzer/Manager.h +++ b/src/analyzer/Manager.h @@ -133,6 +133,14 @@ public: */ void DisableAllAnalyzers(); + /** + * Returns the tag associated with an analyer name, or the tag + * associated with an error if no such analyzer exists. + * + * @param name The canonical analyzer name to check. + */ + Tag GetAnalyzerTag(const char* name); + /** * Returns true if an analyzer is enabled. * diff --git a/src/analyzer/protocol/tcp/TCP.h b/src/analyzer/protocol/tcp/TCP.h index b2649b4ab8..e01ed3a255 100644 --- a/src/analyzer/protocol/tcp/TCP.h +++ b/src/analyzer/protocol/tcp/TCP.h @@ -294,8 +294,8 @@ public: // TCP_ENDPOINT_RESET. If gen_event is true and the connection // is now fully closed, a connection_finished event will be // generated; otherwise not. - virtual void ConnectionClosed(TCP_Endpoint* endpoint, - TCP_Endpoint* peer, int gen_event); + virtual void ConnectionClosed(analyzer::tcp::TCP_Endpoint* endpoint, + analyzer::tcp::TCP_Endpoint* peer, int gen_event); virtual void ConnectionFinished(int half_finished); virtual void ConnectionReset(); From 7412470d6616a9f8c6a9a8174af244b570ebdcee Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sat, 28 Sep 2013 16:06:40 -0700 Subject: [PATCH 005/193] Prettyfing Describe() for record types. If a record type has a name and ODesc is set to short, we now print the name instead of the full field list. --- src/Type.cc | 14 ++++++++++---- src/Var.cc | 2 +- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/Type.cc b/src/Type.cc index 6b4b2eb970..52645533c3 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -1048,10 +1048,16 @@ void RecordType::Describe(ODesc* d) const { if ( d->IsReadable() ) { - d->AddSP("record {"); - DescribeFields(d); - d->SP(); - d->Add("}"); + if ( d->IsShort() && GetTypeID() ) + d->Add(GetTypeID()); + + else + { + d->AddSP("record {"); + DescribeFields(d); + d->SP(); + d->Add("}"); + } } else diff --git a/src/Var.cc b/src/Var.cc index d384fedc74..08129bdc3e 100644 --- a/src/Var.cc +++ b/src/Var.cc @@ -314,9 +314,9 @@ void add_type(ID* id, BroType* t, attr_list* attr, int /* is_event */) delete [] data; } - tnew->SetTypeID(copy_string(id->Name())); } + tnew->SetTypeID(copy_string(id->Name())); id->SetType(tnew); id->MakeType(); From 555df1e7ea3b1baa629cb5059f5acf55be8d9114 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 26 Nov 2013 11:23:25 -0800 Subject: [PATCH 006/193] Checkpointing the dynamic plugin code. This is essentially the code from the dynamic-plugin branch except for some pieces that I have split out into separate, earlier commits. I'm going to updatre things in this branch going forward. --- CMakeLists.txt | 10 + cmake | 2 +- config.h.in | 11 + src/CMakeLists.txt | 15 +- src/DebugLogger.cc | 25 +- src/DebugLogger.h | 12 +- src/Event.cc | 6 + src/FlowSrc.cc | 2 +- src/Func.cc | 47 ++- src/Net.cc | 15 +- src/Net.h | 1 + src/RemoteSerializer.cc | 2 +- src/builtin-func.l | 6 +- src/builtin-func.y | 14 +- src/main.cc | 72 ++++- src/plugin/ComponentManager.h | 2 - src/plugin/Macros.h | 17 +- src/plugin/Manager.cc | 295 +++++++++++++++++- src/plugin/Manager.h | 122 +++++++- src/plugin/Plugin.cc | 135 +++++++- src/plugin/Plugin.h | 228 +++++++++++++- src/scan.l | 20 ++ src/util-config.h.in | 1 + src/util.cc | 87 ++++-- src/util.h | 15 +- .../Baseline/core.plugins.analyzer/output | 6 + .../Baseline/core.plugins.test-plugin/output | 11 + testing/btest/Traces/port4242.trace | Bin 0 -> 868 bytes .../plugins/analyzer-plugin/.btest-ignore | 0 .../plugins/analyzer-plugin/CMakeLists.txt | 20 ++ .../plugins/analyzer-plugin/__bro_plugin__ | 2 + .../scripts/Demo/Foo/base/main.bro | 7 + .../analyzer-plugin/scripts/__load__.bro | 1 + .../core/plugins/analyzer-plugin/src/Foo.cc | 59 ++++ .../core/plugins/analyzer-plugin/src/Foo.h | 31 ++ .../plugins/analyzer-plugin/src/Plugin.cc | 12 + .../plugins/analyzer-plugin/src/events.bif | 2 + .../analyzer-plugin/src/foo-analyzer.pac | 15 + .../analyzer-plugin/src/foo-protocol.pac | 4 + .../core/plugins/analyzer-plugin/src/foo.pac | 26 ++ .../plugins/analyzer-plugin/src/functions.bif | 0 testing/btest/core/plugins/analyzer.bro | 13 + testing/btest/core/plugins/test-plugin.sh | 45 +++ 43 files changed, 1306 insertions(+), 110 deletions(-) create mode 100644 testing/btest/Baseline/core.plugins.analyzer/output create mode 100644 testing/btest/Baseline/core.plugins.test-plugin/output create mode 100644 testing/btest/Traces/port4242.trace create mode 100644 testing/btest/core/plugins/analyzer-plugin/.btest-ignore create mode 100644 testing/btest/core/plugins/analyzer-plugin/CMakeLists.txt create mode 100644 testing/btest/core/plugins/analyzer-plugin/__bro_plugin__ create mode 100644 testing/btest/core/plugins/analyzer-plugin/scripts/Demo/Foo/base/main.bro create mode 100644 testing/btest/core/plugins/analyzer-plugin/scripts/__load__.bro create mode 100644 testing/btest/core/plugins/analyzer-plugin/src/Foo.cc create mode 100644 testing/btest/core/plugins/analyzer-plugin/src/Foo.h create mode 100644 testing/btest/core/plugins/analyzer-plugin/src/Plugin.cc create mode 100644 testing/btest/core/plugins/analyzer-plugin/src/events.bif create mode 100644 testing/btest/core/plugins/analyzer-plugin/src/foo-analyzer.pac create mode 100644 testing/btest/core/plugins/analyzer-plugin/src/foo-protocol.pac create mode 100644 testing/btest/core/plugins/analyzer-plugin/src/foo.pac create mode 100644 testing/btest/core/plugins/analyzer-plugin/src/functions.bif create mode 100644 testing/btest/core/plugins/analyzer.bro create mode 100644 testing/btest/core/plugins/test-plugin.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 1c427188ca..0738eb387d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,10 @@ project(Bro C CXX) + +# When changing the minimum version here, also adapt +# cmake/BroPluginDynamic and +# aux/bro-aux/plugin-support/skeleton/CMakeLists.txt cmake_minimum_required(VERSION 2.6.3 FATAL_ERROR) + include(cmake/CommonCMakeConfig.cmake) ######################################################################## @@ -17,6 +22,7 @@ set(BRO_SCRIPT_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/scripts) get_filename_component(BRO_SCRIPT_INSTALL_PATH ${BRO_SCRIPT_INSTALL_PATH} ABSOLUTE) +set(BRO_PLUGIN_INSTALL_PATH ${BRO_ROOT_DIR}/share/bro/plugins) set(BRO_MAGIC_INSTALL_PATH ${BRO_ROOT_DIR}/share/bro/magic) set(BRO_MAGIC_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/magic/database) @@ -183,6 +189,10 @@ include(MiscTests) include(PCAPTests) include(OpenSSLTests) include(CheckNameserCompat) +include(GetArchitecture) + +# Tell the plugin code that we're building as part of the main tree. +set(BRO_PLUGIN_INTERNAL_BUILD true CACHE INTERNAL "" FORCE) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/config.h.in ${CMAKE_CURRENT_BINARY_DIR}/config.h) diff --git a/cmake b/cmake index 0187b33a29..fc53d57770 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 0187b33a29d5ec824f940feff60dc5d8c2fe314f +Subproject commit fc53d57770c86fbf9bd2d9694f06a1d539ebe35f diff --git a/config.h.in b/config.h.in index 2d065f755e..61124d2f67 100644 --- a/config.h.in +++ b/config.h.in @@ -209,3 +209,14 @@ /* Common IPv6 extension structure */ #cmakedefine HAVE_IP6_EXT + +/* String with host architecture (e.g., "linux-x86_64") */ +#define HOST_ARCHITECTURE "@HOST_ARCHITECTURE@" + +/* String with extension of dynamic libraries (e.g., ".so") */ +#define SHARED_LIBRARY_SUFFIX "@CMAKE_SHARED_LIBRARY_SUFFIX@" + +/* True if we're building outside of the main Bro source code tree. */ +#ifndef BRO_PLUGIN_INTERNAL_BUILD +#define BRO_PLUGIN_INTERNAL_BUILD @BRO_PLUGIN_INTERNAL_BUILD@ +#endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c881cc4df1..34af4eb974 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -414,6 +414,17 @@ add_dependencies(bro bif_loader_plugins) # Install *.bif.bro. install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/base/bif DESTINATION ${BRO_SCRIPT_INSTALL_PATH}/base) -# Make clean removes the bif directory. -set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES ${CMAKE_BINARY_DIR}/scripts/base/bif) +# Install the plugin directory with a short README. +set(plugin_binary_dir "${CMAKE_BINARY_DIR}/plugins") + +add_custom_command(OUTPUT ${plugin_binary_dir}/README + COMMAND mkdir ARGS -p ${plugin_binary_dir} + COMMAND echo ARGS "This directory holds dynamic Bro plugins." >${plugin_binary_dir}/README) + +add_custom_target(plugin-dir DEPENDS ${plugin_binary_dir}/README) +add_dependencies(bro plugin-dir) + +# Make clean removes the bif and plugin directories. +set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES ${CMAKE_BINARY_DIR}/scripts/base/bif) +set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES ${plugin_binary_dir}) diff --git a/src/DebugLogger.cc b/src/DebugLogger.cc index 95049ef70b..267383503d 100644 --- a/src/DebugLogger.cc +++ b/src/DebugLogger.cc @@ -5,6 +5,7 @@ #include "DebugLogger.h" #include "Net.h" +#include "plugin/Plugin.h" DebugLogger debug_logger("debug"); @@ -73,10 +74,12 @@ void DebugLogger::EnableStreams(const char* s) { if ( strcasecmp("verbose", tok) == 0 ) verbose = true; - else + else if ( strncmp(tok, "plugin-", 7) != 0 ) reporter->FatalError("unknown debug stream %s\n", tok); } + enabled_streams.insert(tok); + tok = strtok(0, ","); } @@ -105,4 +108,24 @@ void DebugLogger::Log(DebugStream stream, const char* fmt, ...) fflush(file); } +void DebugLogger::Log(const plugin::Plugin& plugin, const char* fmt, ...) + { + string tok = string("plugin-") + plugin.Name(); + tok = strreplace(tok, "::", "-"); + + if ( enabled_streams.find(tok) == enabled_streams.end() ) + return; + + fprintf(file, "%17.06f/%17.06f [plugin %s] ", + network_time, current_time(true), plugin.Name()); + + va_list ap; + va_start(ap, fmt); + vfprintf(file, fmt, ap); + va_end(ap); + + fputc('\n', file); + fflush(file); + } + #endif diff --git a/src/DebugLogger.h b/src/DebugLogger.h index c5744642f5..a5d3c2878c 100644 --- a/src/DebugLogger.h +++ b/src/DebugLogger.h @@ -7,6 +7,8 @@ #ifdef DEBUG #include +#include +#include // To add a new debugging stream, add a constant here as well as // an entry to DebugLogger::streams in DebugLogger.cc. @@ -27,7 +29,7 @@ enum DebugStream { DBG_INPUT, // Input streams DBG_THREADING, // Threading system DBG_FILE_ANALYSIS, // File analysis - DBG_PLUGINS, + DBG_PLUGINS, // Plugin support NUM_DBGS // Has to be last }; @@ -39,6 +41,10 @@ enum DebugStream { #define DBG_PUSH(stream) debug_logger.PushIndent(stream) #define DBG_POP(stream) debug_logger.PopIndent(stream) +#define PLUGIN_DBG_LOG(plugin, args...) debug_logger.Log(plugin, args) + +namespace plugin { class Plugin; } + class DebugLogger { public: // Output goes to stderr per default. @@ -46,6 +52,7 @@ public: ~DebugLogger(); void Log(DebugStream stream, const char* fmt, ...); + void Log(const plugin::Plugin& plugin, const char* fmt, ...); void PushIndent(DebugStream stream) { ++streams[int(stream)].indent; } @@ -76,6 +83,8 @@ private: bool enabled; }; + std::set enabled_streams; + static Stream streams[NUM_DBGS]; }; @@ -86,6 +95,7 @@ extern DebugLogger debug_logger; #define DBG_LOG_VERBOSE(args...) #define DBG_PUSH(stream) #define DBG_POP(stream) +#define PLUGIN_DBG_LOG(plugin, args...) #endif #endif diff --git a/src/Event.cc b/src/Event.cc index 0fb76d10bc..7774042ac4 100644 --- a/src/Event.cc +++ b/src/Event.cc @@ -6,6 +6,7 @@ #include "Func.h" #include "NetVar.h" #include "Trigger.h" +#include "plugin/Manager.h" EventMgr mgr; @@ -77,6 +78,9 @@ EventMgr::~EventMgr() void EventMgr::QueueEvent(Event* event) { + if ( plugin_mgr->QueueEvent(event) ) + return; + if ( ! head ) head = tail = event; else @@ -115,6 +119,8 @@ void EventMgr::Drain() SegmentProfiler(segment_logger, "draining-events"); + plugin_mgr->DrainEvents(); + draining = true; while ( head ) Dispatch(); diff --git a/src/FlowSrc.cc b/src/FlowSrc.cc index 32aa4c4e3a..563153b634 100644 --- a/src/FlowSrc.cc +++ b/src/FlowSrc.cc @@ -49,7 +49,7 @@ void FlowSrc::Process() // This is normally done by calling net_packet_dispatch(), // but as we don't have a packet to dispatch ... - network_time = next_timestamp; + net_update_time(next_timestamp); expire_timers(); netflow_analyzer->downflow()->set_exporter_ip(exporter_ip); diff --git a/src/Func.cc b/src/Func.cc index 11749a8a9c..d8e0ab2c96 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -46,6 +46,7 @@ #include "Event.h" #include "Traverse.h" #include "Reporter.h" +#include "plugin/Manager.h" extern RETSIGTYPE sig_handler(int signo); @@ -226,7 +227,7 @@ TraversalCode Func::Traverse(TraversalCallback* cb) const HANDLE_TC_STMT_PRE(tc); // FIXME: Traverse arguments to builtin functions, too. - if ( kind == BRO_FUNC ) + if ( kind == BRO_FUNC && scope ) { tc = scope->Traverse(cb); HANDLE_TC_STMT_PRE(tc); @@ -281,6 +282,50 @@ Val* BroFunc::Call(val_list* args, Frame* parent) const #ifdef PROFILE_BRO_FUNCTIONS DEBUG_MSG("Function: %s\n", id->Name()); #endif + + Val* plugin_result = plugin_mgr->CallFunction(this, args); + + if ( plugin_result ) + { + // TODO: We should factor this out into its own method. + switch ( Flavor() ) { + case FUNC_FLAVOR_EVENT: + Unref(plugin_result); + plugin_result = 0; + break; + + case FUNC_FLAVOR_HOOK: + if ( plugin_result->Type()->Tag() != TYPE_BOOL ) + reporter->InternalError("plugin returned non-bool for hook"); + + break; + + case FUNC_FLAVOR_FUNCTION: + { + BroType* yt = FType()->YieldType(); + + if ( (! yt) || yt->Tag() == TYPE_VOID ) + { + Unref(plugin_result); + plugin_result = 0; + } + + else + { + if ( plugin_result->Type()->Tag() != yt->Tag() ) + reporter->InternalError("plugin returned wrong type for function call"); + } + + break; + } + } + + loop_over_list(*args, i) + Unref((*args)[i]); + + return plugin_result; + } + if ( bodies.empty() ) { // Can only happen for events and hooks. diff --git a/src/Net.cc b/src/Net.cc index ac4dacf9b8..6cc148770e 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -30,6 +30,7 @@ #include "PacketSort.h" #include "Serializer.h" #include "PacketDumper.h" +#include "plugin/Manager.h" extern "C" { #include "setsignal.h" @@ -144,13 +145,17 @@ RETSIGTYPE watchdog(int /* signo */) return RETSIGVAL; } +void net_update_time(double new_network_time) + { + network_time = new_network_time; + plugin_mgr->UpdateNetworkTime(network_time); + } + void net_init(name_list& interfaces, name_list& readfiles, name_list& netflows, name_list& flowfiles, const char* writefile, const char* filter, const char* secondary_filter, int do_watchdog) { - init_net_var(); - if ( readfiles.length() > 0 || flowfiles.length() > 0 ) { reading_live = pseudo_realtime > 0.0; @@ -323,7 +328,7 @@ void net_packet_dispatch(double t, const struct pcap_pkthdr* hdr, : timer_mgr; // network_time never goes back. - network_time = tmgr->Time() < t ? t : tmgr->Time(); + net_update_time(tmgr->Time() < t ? t : tmgr->Time()); current_pktsrc = src_ps; current_iosrc = src_ps; @@ -456,7 +461,7 @@ void net_run() { // Take advantage of the lull to get up to // date on timers and events. - network_time = ct; + net_update_time(ct); expire_timers(); usleep(1); // Just yield. } @@ -478,7 +483,7 @@ void net_run() // date on timers and events. Because we only // have timers as sources, going to sleep here // doesn't risk blocking on other inputs. - network_time = current_time(); + net_update_time(current_time()); expire_timers(); // Avoid busy-waiting - pause for 100 ms. diff --git a/src/Net.h b/src/Net.h index 5b959d1688..765e9f709d 100644 --- a/src/Net.h +++ b/src/Net.h @@ -19,6 +19,7 @@ extern void net_run(); extern void net_get_final_stats(); extern void net_finish(int drain_events); extern void net_delete(); // Reclaim all memory, etc. +extern void net_update_time(double new_network_time); extern void net_packet_arrival(double t, const struct pcap_pkthdr* hdr, const u_char* pkt, int hdr_size, PktSrc* src_ps); diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index c8cf03667b..e60ad34b46 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -1455,7 +1455,7 @@ void RemoteSerializer::Process() // FIXME: The following chunk of code is copied from // net_packet_dispatch(). We should change that function // to accept an IOSource instead of the PktSrc. - network_time = p->time; + net_update_time(p->time); SegmentProfiler(segment_logger, "expiring-timers"); TimerMgr* tmgr = sessions->LookupTimerMgr(GetCurrentTag()); diff --git a/src/builtin-func.l b/src/builtin-func.l index 3e5f7bce0a..63df96eb80 100644 --- a/src/builtin-func.l +++ b/src/builtin-func.l @@ -247,14 +247,13 @@ void init_alternative_mode() fprintf(fp_func_init, "\n"); fprintf(fp_func_init, "#include \n"); fprintf(fp_func_init, "#include \n"); + fprintf(fp_func_init, "#include \"plugin/Plugin.h\"\n"); fprintf(fp_func_init, "#include \"%s.h\"\n", input_filename); fprintf(fp_func_init, "\n"); fprintf(fp_func_init, "namespace plugin { namespace %s {\n", plugin); fprintf(fp_func_init, "\n"); - fprintf(fp_func_init, "std::list > __bif_%s_init()\n", name); + fprintf(fp_func_init, "void __bif_%s_init(plugin::Plugin* plugin)\n", name); fprintf(fp_func_init, "\t{\n"); - fprintf(fp_func_init, "\tstd::list > bifs;\n"); - fprintf(fp_func_init, "\n"); } } @@ -266,7 +265,6 @@ void finish_alternative_mode() if ( plugin ) { fprintf(fp_func_init, "\n"); - fprintf(fp_func_init, "\treturn bifs;\n"); fprintf(fp_func_init, "\t}\n"); fprintf(fp_func_init, "} }\n"); fprintf(fp_func_init, "\n"); diff --git a/src/builtin-func.y b/src/builtin-func.y index 662256b8d6..1b22436fff 100644 --- a/src/builtin-func.y +++ b/src/builtin-func.y @@ -267,12 +267,12 @@ void print_event_c_body(FILE *fp) //fprintf(fp, "%s // end namespace\n", decl.generate_c_namespace_end.c_str()); } -void record_bif_item(const char* id, int type) +void record_bif_item(const char* id, const char* type) { if ( ! plugin ) return; - fprintf(fp_func_init, "\tbifs.push_back(std::make_pair(\"%s\", %d));\n", id, type); + fprintf(fp_func_init, "\tplugin->AddBifItem(\"%s\", plugin::BifItem::%s);\n", id, type); } %} @@ -358,7 +358,7 @@ type_def: TOK_TYPE opt_ws TOK_ID opt_ws ':' opt_ws type_def_types opt_ws ';' decl.c_fullname.c_str(), decl.bro_fullname.c_str(), type_name.c_str()); - record_bif_item(decl.bro_fullname.c_str(), 5); + record_bif_item(decl.bro_fullname.c_str(), "TYPE"); } ; @@ -401,7 +401,7 @@ enum_def: enum_def_1 enum_list TOK_RPB "\t%s = internal_type(\"%s\")->AsEnumType();\n", decl.c_fullname.c_str(), decl.bro_fullname.c_str()); - record_bif_item(decl.bro_fullname.c_str(), 5); + record_bif_item(decl.bro_fullname.c_str(), "TYPE"); } ; @@ -457,7 +457,7 @@ const_def: TOK_CONST opt_ws TOK_ID opt_ws ':' opt_ws TOK_ID opt_ws ';' decl.c_fullname.c_str(), decl.bro_fullname.c_str(), accessor); - record_bif_item(decl.bro_fullname.c_str(), 3); + record_bif_item(decl.bro_fullname.c_str(), "CONSTANT"); } attr_list: @@ -545,7 +545,7 @@ head_1: TOK_ID opt_ws arg_begin "Val* %s(Frame* frame, val_list* %s)", decl.c_fullname.c_str(), arg_list_name); - record_bif_item(decl.bro_fullname.c_str(), 1); + record_bif_item(decl.bro_fullname.c_str(), "FUNCTION"); } else if ( definition_type == EVENT_DEF ) { @@ -562,7 +562,7 @@ head_1: TOK_ID opt_ws arg_begin "\t%s = internal_handler(\"%s\");\n", decl.c_fullname.c_str(), decl.bro_fullname.c_str()); - record_bif_item(decl.bro_fullname.c_str(), 2); + record_bif_item(decl.bro_fullname.c_str(), "EVENT"); // C++ prototypes of bro_event_* functions will // be generated later. diff --git a/src/main.cc b/src/main.cc index 0f60a4c70f..bfbcf77a6d 100644 --- a/src/main.cc +++ b/src/main.cc @@ -198,6 +198,7 @@ void usage() fprintf(stderr, " -N|--print-plugins | print available plugins and exit (-NN for verbose)\n"); fprintf(stderr, " -O|--optimize | optimize policy script\n"); fprintf(stderr, " -P|--prime-dns | prime DNS\n"); + fprintf(stderr, " -Q|--time | print execution time summary to stderr\n"); fprintf(stderr, " -R|--replay | replay events\n"); fprintf(stderr, " -S|--debug-rules | enable rule debugging\n"); fprintf(stderr, " -T|--re-level | set 'RE_level' for rules\n"); @@ -220,8 +221,10 @@ void usage() fprintf(stderr, " -n|--idmef-dtd | specify path to IDMEF DTD file\n"); #endif - fprintf(stderr, " $BROPATH | file search path (%s)\n", bro_path()); + fprintf(stderr, " $BROPATH | file search path (%s)\n", bro_path().c_str()); fprintf(stderr, " $BROMAGIC | libmagic mime magic database search path (%s)\n", bro_magic_path()); + fprintf(stderr, " $BRO_PLUGINS | plugin search path (%s)\n", bro_plugin_path()); + fprintf(stderr, " $BRO_PREFIXES | prefix list (%s)\n", bro_prefixes()); fprintf(stderr, " $BRO_PREFIXES | prefix list (%s)\n", bro_prefixes().c_str()); fprintf(stderr, " $BRO_DNS_FAKE | disable DNS lookups (%s)\n", bro_dns_fake()); fprintf(stderr, " $BRO_SEED_FILE | file to load seeds from (not set)\n"); @@ -435,6 +438,8 @@ int main(int argc, char** argv) { std::set_new_handler(bro_new_handler); + double time_start = current_time(true); + brofiler.ReadStats(); bro_argc = argc; @@ -464,6 +469,7 @@ int main(int argc, char** argv) int rule_debug = 0; int RE_level = 4; int print_plugins = 0; + int time_bro = 0; static struct option long_opts[] = { {"bare-mode", no_argument, 0, 'b'}, @@ -545,7 +551,7 @@ int main(int argc, char** argv) opterr = 0; char opts[256]; - safe_strncpy(opts, "B:D:e:f:I:i:K:l:n:p:R:r:s:T:t:U:w:x:X:y:Y:z:CFGLNOPSWbdghvZ", + safe_strncpy(opts, "B:D:e:f:I:i:K:l:n:p:R:r:s:T:t:U:w:x:X:y:Y:z:CFGLNOPSWbdghvZQ", sizeof(opts)); #ifdef USE_PERFTOOLS_DEBUG @@ -674,6 +680,10 @@ int main(int argc, char** argv) dns_type = DNS_PRIME; break; + case 'Q': + time_bro = 1; + break; + case 'R': events_file = optarg; break; @@ -760,6 +770,7 @@ int main(int argc, char** argv) reporter = new Reporter(); thread_mgr = new threading::Manager(); + plugin_mgr = new plugin::Manager(); #ifdef DEBUG if ( debug_streams ) @@ -810,6 +821,8 @@ int main(int argc, char** argv) if ( ! bare_mode ) add_input_file("base/init-default.bro"); + plugin_mgr->LoadPluginsFrom(bro_plugin_path()); + if ( optind == argc && read_files.length() == 0 && flow_files.length() == 0 && interfaces.length() == 0 && @@ -842,7 +855,6 @@ int main(int argc, char** argv) analyzer_mgr = new analyzer::Manager(); log_mgr = new logging::Manager(); input_mgr = new input::Manager(); - plugin_mgr = new plugin::Manager(); file_mgr = new file_analysis::Manager(); plugin_mgr->InitPreScript(); @@ -877,9 +889,13 @@ int main(int argc, char** argv) yyparse(); - plugin_mgr->InitPostScript(); - analyzer_mgr->InitPostScript(); - file_mgr->InitPostScript(); + init_general_global_var(); + init_net_var(); + + plugin_mgr->InitBifs(); + + if ( reporter->Errors() > 0 ) + exit(1); if ( print_plugins ) { @@ -887,6 +903,10 @@ int main(int argc, char** argv) exit(1); } + plugin_mgr->InitPostScript(); + analyzer_mgr->InitPostScript(); + file_mgr->InitPostScript(); + #ifdef USE_PERFTOOLS_DEBUG } #endif @@ -916,8 +936,6 @@ int main(int argc, char** argv) reporter->InitOptions(); - init_general_global_var(); - if ( user_pcap_filter ) { ID* id = global_scope()->Lookup("cmd_line_bpf_filter"); @@ -1079,7 +1097,7 @@ int main(int argc, char** argv) if ( ! reading_live && ! reading_traces ) // Set up network_time to track real-time, since // we don't have any other source for it. - network_time = current_time(); + net_update_time(current_time()); EventHandlerPtr bro_init = internal_handler("bro_init"); if ( bro_init ) //### this should be a function @@ -1165,7 +1183,43 @@ int main(int argc, char** argv) #endif + double time_net_start = current_time(true);; + + unsigned int mem_net_start_total; + unsigned int mem_net_start_malloced; + + if ( time_bro ) + { + get_memory_usage(&mem_net_start_total, &mem_net_start_malloced); + + fprintf(stderr, "# initialization %.6f\n", time_net_start - time_start); + + fprintf(stderr, "# initialization %uM/%uM\n", + mem_net_start_total / 1024 / 1024, + mem_net_start_malloced / 1024 / 1024); + } + net_run(); + + double time_net_done = current_time(true);; + + unsigned int mem_net_done_total; + unsigned int mem_net_done_malloced; + + if ( time_bro ) + { + get_memory_usage(&mem_net_done_total, &mem_net_done_malloced); + + fprintf(stderr, "# total time %.6f, processing %.6f\n", + time_net_done - time_start, time_net_done - time_net_start); + + fprintf(stderr, "# total mem %uM/%uM, processing %uM/%uM\n", + mem_net_done_total / 1024 / 1024, + mem_net_done_malloced / 1024 / 1024, + (mem_net_done_total - mem_net_start_total) / 1024 / 1024, + (mem_net_done_malloced - mem_net_start_malloced) / 1024 / 1024); + } + done_with_network(); net_delete(); diff --git a/src/plugin/ComponentManager.h b/src/plugin/ComponentManager.h index ccc076db28..71994d82d1 100644 --- a/src/plugin/ComponentManager.h +++ b/src/plugin/ComponentManager.h @@ -83,8 +83,6 @@ public: */ T GetComponentTag(Val* v) const; -protected: - /** * Add a component the internal maps used to keep track of it and create * a script-layer ID for the component's enum value. diff --git a/src/plugin/Macros.h b/src/plugin/Macros.h index 9362642e91..3248031e98 100644 --- a/src/plugin/Macros.h +++ b/src/plugin/Macros.h @@ -28,18 +28,15 @@ * must be unique across all loaded plugins. */ #define BRO_PLUGIN_BEGIN(_ns, _name) \ - namespace plugin { namespace _ns ## _ ## _name {\ + namespace plugin { namespace _ns ## _ ## _name { \ class Plugin : public plugin::Plugin { \ protected: \ - void InitPreScript() \ + void InitPreScript() \ { \ SetName(#_ns "::" #_name); \ - SetVersion(-1);\ - SetAPIVersion(BRO_PLUGIN_API_VERSION);\ - SetDynamicPlugin(false); -// TODO: The SetDynamicPlugin() call is currently hardcoded to false. Change -// once we have dynamic plugins as well. - + SetVersion(-1); \ + SetAPIVersion(BRO_PLUGIN_API_VERSION); \ + SetDynamicPlugin(! BRO_PLUGIN_INTERNAL_BUILD);\ /** * Ends the definition of a plugin. @@ -76,8 +73,8 @@ * interpreter. */ #define BRO_PLUGIN_BIF_FILE(file) \ - extern std::list > __bif_##file##_init(); \ - AddBifInitFunction(&__bif_##file##_init); + extern void __bif_##file##_init(plugin::Plugin*); \ + __AddBifInitFunction(&__bif_##file##_init); /** * Defines a component implementing a protocol analyzer. diff --git a/src/plugin/Manager.cc b/src/plugin/Manager.cc index 67f4dea2bd..ddd953ab22 100644 --- a/src/plugin/Manager.cc +++ b/src/plugin/Manager.cc @@ -1,11 +1,23 @@ // See the file "COPYING" in the main distribution directory for copyright. +#include +#include +#include +#include +#include +#include + #include "Manager.h" #include "../Reporter.h" +#include "../Func.h" +#include "../Event.h" using namespace plugin; +string Manager::current_dir; +string Manager::current_sopath; + Manager::Manager() { init = false; @@ -16,18 +28,162 @@ Manager::~Manager() assert(! init); } -bool Manager::LoadPlugin(const std::string& path) +void Manager::LoadPluginsFrom(const string& dir) { assert(! init); - reporter->InternalError("plugin::Manager::LoadPlugin not yet implemented"); - return false; + + if ( dir.empty() ) + return; + + if ( dir.find(":") != string::npos ) + { + // Split at ":". + std::stringstream s(dir); + std::string d; + + while ( std::getline(s, d, ':') ) + LoadPluginsFrom(d); + + return; + } + + if ( ! is_dir(dir) ) + { + DBG_LOG(DBG_PLUGINS, "Not a valid plugin directory: %s", dir.c_str()); + return; + } + + int rc = LoadPlugin(dir); + + if ( rc >= 0 ) + return; + + DBG_LOG(DBG_PLUGINS, "Searching directory %s recursively for plugins", dir.c_str()); + + DIR* d = opendir(dir.c_str()); + + if ( ! d ) + { + DBG_LOG(DBG_PLUGINS, "Cannot open directory %s", dir.c_str()); + return; + } + + bool found = false; + + struct dirent *dp; + + while ( (dp = readdir(d)) ) + { + struct stat st; + + if ( strcmp(dp->d_name, "..") == 0 + || strcmp(dp->d_name, ".") == 0 ) + continue; + + string path = dir + "/" + dp->d_name; + + if( stat(path.c_str(), &st) < 0 ) + { + DBG_LOG(DBG_PLUGINS, "Cannot stat %s: %s", path.c_str(), strerror(errno)); + continue; + } + + if ( st.st_mode & S_IFDIR ) + LoadPluginsFrom(path); + } } -bool Manager::LoadPluginsFrom(const std::string& dir) +int Manager::LoadPlugin(const std::string& dir) { assert(! init); - reporter->InternalError("plugin::Manager::LoadPluginsFrom not yet implemented"); - return false; + + // Check if it's a plugin dirctory. + if ( ! is_file(dir + "/__bro_plugin__") ) + return -1; + + DBG_LOG(DBG_PLUGINS, "Loading plugin from %s", dir.c_str()); + + // Add the "scripts" and "bif" directories to BROPATH. + string scripts = dir + "/scripts"; + + if ( is_dir(scripts) ) + { + DBG_LOG(DBG_PLUGINS, " Adding %s to BROPATH", scripts.c_str()); + add_to_bro_path(scripts); + } + + string bif = dir + "/bif"; + + if ( is_dir(bif) ) + { + DBG_LOG(DBG_PLUGINS, " Adding %s to BROPATH", bif.c_str()); + add_to_bro_path(bif); + } + + // Load dylib/scripts/__load__.bro automatically. + string dyinit = dir + "/dylib/scripts/__load__.bro"; + + if ( is_file(dyinit) ) + { + DBG_LOG(DBG_PLUGINS, " Adding %s for loading", dyinit.c_str()); + add_input_file(dyinit.c_str()); + } + + // Load scripts/__load__.bro automatically. + string init = scripts + "/__load__.bro"; + + if ( is_file(init) ) + { + DBG_LOG(DBG_PLUGINS, " Adding %s for loading", init.c_str()); + add_input_file(init.c_str()); + } + + // Load bif/__load__.bro automatically. + init = bif + "/__load__.bro"; + + if ( is_file(init) ) + { + DBG_LOG(DBG_PLUGINS, " Adding %s for loading", init.c_str()); + add_input_file(init.c_str()); + } + + // Load shared libraries. + + string dypattern = dir + "/dylib/*." + HOST_ARCHITECTURE + SHARED_LIBRARY_SUFFIX; + + DBG_LOG(DBG_PLUGINS, " Searching for shared libraries %s", dypattern.c_str()); + + glob_t gl; + + if ( glob(dypattern.c_str(), 0, 0, &gl) == 0 ) + { + for ( size_t i = 0; i < gl.gl_pathc; i++ ) + { + const char* path = gl.gl_pathv[i]; + + current_dir = dir; + current_sopath = path; + void* hdl = dlopen(path, RTLD_LAZY | RTLD_GLOBAL); + current_dir.clear(); + current_sopath.clear(); + + if ( ! hdl ) + { + const char* err = dlerror(); + reporter->FatalError("cannot load plugin library %s: %s", path, err ? err : ""); + } + + DBG_LOG(DBG_PLUGINS, " Loaded %s", path); + } + } + + else + { + DBG_LOG(DBG_PLUGINS, " No shared library found"); + return 1; + } + + return 1; } static bool plugin_cmp(const Plugin* a, const Plugin* b) @@ -39,22 +195,61 @@ bool Manager::RegisterPlugin(Plugin *plugin) { Manager::PluginsInternal()->push_back(plugin); + if ( current_dir.size() && current_sopath.size() ) + plugin->SetPluginLocation(current_dir.c_str(), current_sopath.c_str()); + // Sort plugins by name to make sure we have a deterministic order. PluginsInternal()->sort(plugin_cmp); return true; } +static bool interpreter_plugin_cmp(const InterpreterPlugin* a, const InterpreterPlugin* b) + { + if ( a->Priority() == b->Priority() ) + return a->Name() < b->Name(); + + // Reverse sort. + return a->Priority() > b->Priority(); + } + void Manager::InitPreScript() { assert(! init); for ( plugin_list::iterator i = Manager::PluginsInternal()->begin(); i != Manager::PluginsInternal()->end(); i++ ) - (*i)->InitPreScript(); + { + Plugin* plugin = *i; + + if ( plugin->PluginType() == Plugin::INTERPRETER ) + interpreter_plugins.push_back(dynamic_cast(plugin)); + + plugin->InitPreScript(); + + // Track the file extensions the plugin can handle. + std::stringstream ext(plugin->FileExtensions()); + + // Split at ":". + std::string e; + + while ( std::getline(ext, e, ':') ) + { + DBG_LOG(DBG_PLUGINS, "Plugin %s handles *.%s", plugin->Name(), e.c_str()); + extensions.insert(std::make_pair(e, plugin)); + } + } + + interpreter_plugins.sort(interpreter_plugin_cmp); init = true; } +void Manager::InitBifs() + { + for ( plugin_list::iterator i = Manager::PluginsInternal()->begin(); i != Manager::PluginsInternal()->end(); i++ ) + (*i)->InitBifs(); + } + void Manager::InitPostScript() { assert(init); @@ -78,6 +273,28 @@ void Manager::FinishPlugins() init = false; } +int Manager::TryLoadFile(const char* file) + { + assert(file); + const char* ext = strrchr(file, '.'); + + if ( ! ext ) + return -1; + + extension_map::iterator i = extensions.find(++ext); + if ( i == extensions.end() ) + return -1; + + Plugin* plugin = i->second; + + DBG_LOG(DBG_PLUGINS, "Loading %s with %s", file, plugin->Name()); + + if ( i->second->LoadFile(file) ) + return 1; + + return 0; + } + Manager::plugin_list Manager::Plugins() const { return *Manager::PluginsInternal(); @@ -92,3 +309,67 @@ Manager::plugin_list* Manager::PluginsInternal() return plugins; } + +Val* Manager::CallFunction(const Func* func, val_list* args) const + { + Val* result = 0; + + for ( interpreter_plugin_list::const_iterator i = interpreter_plugins.begin(); + i != interpreter_plugins.end() && ! result; i++ ) + { + result = (*i)->CallFunction(func, args); + + if ( result ) + { + DBG_LOG(DBG_PLUGINS, "Plugin %s replaced call to %s", (*i)->Name(), func->Name()); + return result; + } + } + + return 0; + } + +bool Manager::QueueEvent(Event* event) const + { + for ( interpreter_plugin_list::const_iterator i = interpreter_plugins.begin(); + i != interpreter_plugins.end(); i++ ) + { + if ( (*i)->QueueEvent(event) ) + { + DBG_LOG(DBG_PLUGINS, "Plugin %s handled queueing of event %s", (*i)->Name(), event->Handler()->Name()); + return true; + } + } + + return false; + } + + +void Manager::UpdateNetworkTime(double network_time) const + { + for ( interpreter_plugin_list::const_iterator i = interpreter_plugins.begin(); + i != interpreter_plugins.end(); i++ ) + (*i)->UpdateNetworkTime(network_time); + } + +void Manager::DrainEvents() const + { + for ( interpreter_plugin_list::const_iterator i = interpreter_plugins.begin(); + i != interpreter_plugins.end(); i++ ) + (*i)->DrainEvents(); + } + +void Manager::DisableInterpreterPlugin(const InterpreterPlugin* plugin) + { + for ( interpreter_plugin_list::iterator i = interpreter_plugins.begin(); + i != interpreter_plugins.end(); i++ ) + { + if ( *i == plugin ) + { + interpreter_plugins.erase(i); + return; + } + } + } + + diff --git a/src/plugin/Manager.h b/src/plugin/Manager.h index 2bbcaeb0f1..85b60fb763 100644 --- a/src/plugin/Manager.h +++ b/src/plugin/Manager.h @@ -3,6 +3,8 @@ #ifndef PLUGIN_MANAGER_H #define PLUGIN_MANAGER_H +#include + #include "Plugin.h" #include "Component.h" @@ -17,6 +19,7 @@ class Manager { public: typedef std::list plugin_list; + typedef std::list interpreter_plugin_list; typedef Plugin::component_list component_list; /** @@ -30,24 +33,17 @@ public: ~Manager(); /** - * Loads a plugin dynamically from a file. This must be called only - * before InitPluginsPreScript() + * Loads all plugins dynamically from a set of directories. Multiple + * directories are split by ':'. If a directory does not contain a + * plugin itself, the method searches for plugins recursively. For + * plugins found, the method loads the plugin's shared library and + * makes its scripts available to the interpreter. * - * This is not currently implemented. - * - * @param file The path to the plugin to load. - */ - bool LoadPlugin(const std::string& file); - - /** - * Loads plugins dynamically found in a directory. This must be - * called only before InitPluginsPreScript(). - * - * This is not currently implemented. + * This must be called only before InitPluginsPreScript(). * * @param dir The directory to search for plugins. */ - bool LoadPluginsFrom(const std::string& dir); + void LoadPluginsFrom(const std::string& dir); /** * First-stage initializion of the manager. This is called early on @@ -57,7 +53,13 @@ public: void InitPreScript(); /** - * Second-stage initialization of the manager. This is called late + * Second-stage initialization of the manager. This is called in + * between pre- and post-script to make BiFs available. + */ + void InitBifs(); + + /** + * Third-stage initialization of the manager. This is called late * during Bro's initialization after any scripts are processed, and * forwards to the corresponding Plugin methods. */ @@ -69,6 +71,22 @@ public: */ void FinishPlugins(); + /** + * This tries to load the given file by searching for a plugin that + * support that extension. If a correspondign plugin is found, it's + * asked to loead the file. If that fails, the method reports an + * error message. + * + * This method must be called only between InitPreScript() and + * InitPostScript(). + * + * @return 1 if the file was sucessfully loaded by a plugin; 0 if a + * plugin was found that supports the file's extension, yet it + * encountered a problem loading the file; and -1 if we don't have a + * plugin that supports this extension. + */ + int TryLoadFile(const char* file); + /** * Returns a list of all available plugins. This includes all that * are compiled in statically, as well as those loaded dynamically so @@ -83,6 +101,52 @@ public: */ template std::list Components() const; + /** + * Filters a function/event/hook call through all interpreter plugins. + * + * @param func The function to be called. + * + * @param args The function call's arguments; they may be modified. + * + * @return If a plugin handled the call, a +1 Val with the result + * value to pass back to the interpreter (for void functions and + * events, it may be any Val and must be ignored). If no plugin + * handled the call, the method returns null. + */ + Val* CallFunction(const Func* func, val_list* args) const; + + /** + * Filter the queuing of an event through all interpreter plugins. + * + * @param event The event to be queued; it may be modified. + * + * @return Returns true if a plugin handled the queuing; in that case the + * plugin will have taken ownership. + * + */ + bool QueueEvent(Event* event) const; + + /** + * Informs all interpreter plugins about an update in network time. + * + * @param networkt_time The new network time. + */ + void UpdateNetworkTime(double network_time) const; + + /** + * Informs all interpreter plugins that the event queue has been drained. + */ + void DrainEvents() const; + + /** + * Disables an interpreter plugin's hooking of the script interpreter. + * The remaining functionality of the Plugin base class remains + * available. + * + * @param plugin The plugin to disable. + */ + void DisableInterpreterPlugin(const InterpreterPlugin* plugin); + /** * Internal method that registers a freshly instantiated plugin with * the manager. @@ -91,12 +155,38 @@ public: * ownership, yet assumes the pointer will stay valid at least until * the Manager is destroyed. */ - static bool RegisterPlugin(Plugin *plugin); + static bool RegisterPlugin(Plugin* plugin); + +protected: + /** + * Loads a plugin dynamically from a given directory. It loads the + * plugin's shared library, and makes its scripts available to the + * interpreter. Different from LoadPluginsFrom() this method does not + * further descend the directory tree recursively to search for + * plugins. + * + * This must be called only before InitPluginsPreScript() + * + * @param file The path to the plugin to load. + * + * @return 0 if there's a plugin in this directory, but there was a + * problem loading it; -1 if there's no plugin at all in this + * directory; 1 if there's a plugin in this directory and we loaded + * it successfully. + */ + int LoadPlugin(const std::string& dir); private: static plugin_list* PluginsInternal(); bool init; + typedef std::map extension_map; + extension_map extensions; + + interpreter_plugin_list interpreter_plugins; + + static string current_dir; + static string current_sopath; }; template diff --git a/src/plugin/Plugin.cc b/src/plugin/Plugin.cc index eaac8a3b25..ff3cf8f414 100644 --- a/src/plugin/Plugin.cc +++ b/src/plugin/Plugin.cc @@ -10,9 +10,9 @@ using namespace plugin; -BifItem::BifItem(const std::string& arg_id, Type arg_type) +BifItem::BifItem(const char* arg_id, Type arg_type) { - id = copy_string(arg_id.c_str()); + id = copy_string(arg_id); type = arg_type; } @@ -47,6 +47,9 @@ Plugin::Plugin() version = -9999; api_version = -9999; dynamic = false; + base_dir = 0; + sopath = 0; + extensions = 0; Manager::RegisterPlugin(this); } @@ -57,6 +60,14 @@ Plugin::~Plugin() delete [] name; delete [] description; + delete [] base_dir; + delete [] sopath; + delete [] extensions; + } + +Plugin::Type Plugin::PluginType() const + { + return STANDARD; } const char* Plugin::Name() const @@ -66,6 +77,7 @@ const char* Plugin::Name() const void Plugin::SetName(const char* arg_name) { + delete [] name; name = copy_string(arg_name); } @@ -76,6 +88,7 @@ const char* Plugin::Description() const void Plugin::SetDescription(const char* arg_description) { + delete [] description; description = copy_string(arg_description); } @@ -99,6 +112,16 @@ bool Plugin::DynamicPlugin() const return dynamic; } +const char* Plugin::PluginDirectory() const + { + return base_dir; + } + +const char* Plugin::PluginPath() const + { + return sopath; + } + void Plugin::SetAPIVersion(int arg_version) { api_version = arg_version; @@ -109,22 +132,26 @@ void Plugin::SetDynamicPlugin(bool arg_dynamic) dynamic = arg_dynamic; } +void Plugin::SetPluginLocation(const char* arg_dir, const char* arg_sopath) + { + delete [] base_dir; + delete [] sopath; + base_dir = copy_string(arg_dir); + sopath = copy_string(arg_sopath); + } + void Plugin::InitPreScript() { } void Plugin::InitPostScript() { - for ( bif_init_func_list::const_iterator f = bif_inits.begin(); f != bif_inits.end(); f++ ) - { - bif_init_func_result items = (**f)(); + } - for ( bif_init_func_result::const_iterator i = items.begin(); i != items.end(); i++ ) - { - BifItem bi((*i).first, (BifItem::Type)(*i).second); - bif_items.push_back(bi); - } - } +void Plugin::InitBifs() + { + for ( bif_init_func_list::const_iterator f = bif_inits.begin(); f != bif_inits.end(); f++ ) + (**f)(this); } Plugin::bif_item_list Plugin::BifItems() const @@ -143,6 +170,22 @@ Plugin::bif_item_list Plugin::CustomBifItems() const return bif_item_list(); } +const char* Plugin::FileExtensions() const + { + return extensions ? extensions : ""; + } + +void Plugin::SetFileExtensions(const char* ext) + { + extensions = copy_string(ext); + } + +bool Plugin::LoadFile(const char* file) + { + reporter->InternalError("Plugin::LoadFile not overriden for %s", file); + return false; + } + void Plugin::Done() { for ( component_list::const_iterator i = components.begin(); i != components.end(); i++ ) @@ -170,11 +213,23 @@ void Plugin::AddComponent(Component* c) components.sort(component_cmp); } -void Plugin::AddBifInitFunction(bif_init_func c) +bool Plugin::LoadBroFile(const char* file) + { + add_input_file(file); + return true; + } + +void Plugin::__AddBifInitFunction(bif_init_func c) { bif_inits.push_back(c); } +void Plugin::AddBifItem(const char* name, BifItem::Type type) + { + BifItem bi(name, (BifItem::Type)type); + bif_items.push_back(bi); + } + void Plugin::Describe(ODesc* d) const { d->Add("Plugin: "); @@ -190,7 +245,7 @@ void Plugin::Describe(ODesc* d) const { if ( version > 0 ) { - d->Add(" (version "); + d->Add(" (dynamic, version "); d->Add(version); d->Add(")"); } @@ -201,6 +256,18 @@ void Plugin::Describe(ODesc* d) const else d->Add(" (built-in)"); + switch ( PluginType() ) { + case STANDARD: + break; + + case INTERPRETER: + d->Add( " (interpreter plugin)"); + break; + + default: + reporter->InternalError("unknown plugin type in Plugin::Describe"); + } + d->Add("\n"); if ( d->IsShort() ) @@ -252,4 +319,46 @@ void Plugin::Describe(ODesc* d) const } } +InterpreterPlugin::InterpreterPlugin(int arg_priority) + { + priority = arg_priority; + } + +InterpreterPlugin::~InterpreterPlugin() + { + } + +int InterpreterPlugin::Priority() const + { + return priority; + } + +Plugin::Type InterpreterPlugin::PluginType() const + { + return INTERPRETER; + } + +Val* InterpreterPlugin::CallFunction(const Func* func, val_list* args) + { + return 0; + } + +bool InterpreterPlugin::QueueEvent(Event* event) + { + return false; + } + +void InterpreterPlugin::UpdateNetworkTime(double network_time) + { + } + +void InterpreterPlugin::DrainEvents() + { + } + +void InterpreterPlugin::DisableInterpreterPlugin() const + { + plugin_mgr->DisableInterpreterPlugin(this); + } + diff --git a/src/plugin/Plugin.h b/src/plugin/Plugin.h index 4abd260550..306c97e59e 100644 --- a/src/plugin/Plugin.h +++ b/src/plugin/Plugin.h @@ -9,6 +9,8 @@ #include "Macros.h" class ODesc; +class Func; +class Event; namespace plugin { @@ -22,8 +24,6 @@ class BifItem { public: /** * Type of the item. - * - * The values here must match the integers that \c bifcl generated. */ enum Type { FUNCTION = 1, EVENT = 2, CONSTANT = 3, GLOBAL = 4, TYPE = 5 }; @@ -35,7 +35,7 @@ public: * * @param type The type of the item. */ - BifItem(const std::string& id, Type type); + BifItem(const char* id, Type type); /** * Copy constructor. @@ -88,6 +88,26 @@ class Plugin { public: typedef std::list component_list; typedef std::list bif_item_list; + typedef std::list > bif_init_func_result; + typedef void (*bif_init_func)(Plugin *); + + /** + * Type of a plugin. Plugin types are set implicitly by deriving from + * the corresponding base class. */ + enum Type { + /** + * A standard plugin. This is the type for all plugins + * derived directly from \a Plugin. */ + STANDARD, + + /** + * An interpreter plugin. These plugins get hooked into the + * script interpreter and can modify, or even replace, its + * execution of Bro script code. To create an interpreter + * plugin, derive from \aInterpreterPlugin. + */ + INTERPRETER + }; /** * Constructor. @@ -99,6 +119,11 @@ public: */ virtual ~Plugin(); + /** + * Returns the type of the plugin. + */ + virtual Type PluginType() const; + /** * Returns the name of the plugin. */ @@ -121,6 +146,23 @@ public: */ bool DynamicPlugin() const; + /** + * Returns a colon-separated list of file extensions the plugin handles. + */ + const char* FileExtensions() const; + + /** + * For dynamic plugins, returns the base directory from which it was + * loaded. For static plugins, returns null. + **/ + const char* PluginDirectory() const; + + /** + * For dynamic plugins, returns the full path to the shared library + * from which it was loaded. For static plugins, returns null. + **/ + const char* PluginPath() const; + /** * Returns the internal API version that this plugin relies on. Only * plugins that match Bro's current API version may be used. For @@ -173,9 +215,38 @@ public: */ void Describe(ODesc* d) const; + /** + * Registering an individual BiF that the plugin defines. The + * information is for informational purpuses only and will show up in + * the result of BifItems() as well as in the Describe() output. + * Another way to add this information is via overriding + * CustomBifItems(). + * + * @param name The name of the BiF item. + * + * @param type The item's type. + */ + void AddBifItem(const char* name, BifItem::Type type); + + /** + * Adds a file to the list of files Bro loads at startup. This will + * normally be a Bro script, but it passes through the plugin system + * as well to load files with other extensions as supported by any of + * the current plugins. In other words, calling this method is + * similar to given a file on the command line. Note that the file + * may be only queued for now, and actually loaded later. + * + * This method must not be called after InitPostScript(). + * + * @param file The file to load. It will be searched along the standard paths. + * + * @return True if successful (which however may only mean + * "successfully queued"). + */ + bool LoadBroFile(const char* file); + protected: - typedef std::list > bif_init_func_result; - typedef bif_init_func_result (*bif_init_func)(); + friend class Manager; /** * Sets the plugins name. @@ -212,6 +283,28 @@ protected: */ void SetDynamicPlugin(bool dynamic); + /** + * Reports the extensions of input files the plugin handles. If Bro + * wants to load a file with one of these extensions it will pass + * them to LoadFile() and then then ignore otherwise. + * + * ext: A list of colon-separated file extensions the plugin handles. + */ + void SetFileExtensions(const char* ext); + + /** + * Sets the base directory and shared library path from which the + * plugin was loaded. This should be called only from the manager for + * dynamic plugins. + * + * @param dir The plugin directory. The functions makes an internal + * copy of string. + * + * @param sopath The full path the shared library loaded. The + * functions makes an internal copy of string. + */ + void SetPluginLocation(const char* dir, const char* sopath); + /** * Takes ownership. */ @@ -227,17 +320,41 @@ protected: */ virtual bif_item_list CustomBifItems() const; + /** + * Virtual method that can be overriden by derived class to load + * files with extensions reported via SetFileExtension(). + * + * This method will be called between InitPreScript() and + * InitPostScript(), but with no further order or timing guaranteed. + * It will be called once for each file encountered with of the + * specificed extensions (i.e., duplicates are filtered out + * automatically). + * + * @return True if the file was loaded successfuly, false if not. Bro + * will abort in the latter case. + */ + virtual bool LoadFile(const char* file); + + /** + * Initializes the BiF items added with AddBifItem(). Internal method + * that will be called by the manager at the right time. + */ + void InitBifs(); + /** * Internal function adding an entry point for registering * auto-generated BiFs. */ - void AddBifInitFunction(bif_init_func c); + void __AddBifInitFunction(bif_init_func c); private: typedef std::list bif_init_func_list; const char* name; const char* description; + const char* base_dir; + const char* sopath; + const char* extensions; int version; int api_version; bool dynamic; @@ -247,6 +364,105 @@ private: bif_init_func_list bif_inits; }; +/** + * Class for hooking into script execution. An interpreter plugin can do + * everything a normal plugin can, yet will also be interfaced to the script + * interpreter for learning about, modidying, or potentially replacing + * standard functionality. + */ +class InterpreterPlugin : public Plugin { +public: + /** + * Constructor. + * + * @param priority Imposes an order on InterpreterPlugins in which + * they'll be chained. The higher the prioritu, the earlier a plugin + * is called when the interpreter goes through the chain. */ + InterpreterPlugin(int priority); + + /** + * Destructor. + */ + virtual ~InterpreterPlugin(); + + /** + * Returns the plugins priority. + */ + int Priority() const; + + /** + * Callback for executing a function/event/hook. Whenever the script + * interpreter is about to execution a function, it first gives all + * InterpreterPlugins a chance to handle the call (in the order of their + * priorities). A plugin can either just inspect the call, or replace it + * (i.e., prevent the interpreter from executing it). In the latter case + * it must provide a matching return value. + * + * The default implementation does never handle the call in any way. + * + * @param func The function being called. + * + * @param args The function arguments. The method can modify the list + * in place long as it ensures matching types and correct reference + * counting. + * + * @return If the plugin handled the call, a +1 Val with the result + * value to pass back to the interpreter (for void functions and + * events any \a Val is fine; it will be ignored; best to use a \c + * TYPE_ANY). If the plugin did not handle the call, it must return + * null. + */ + virtual Val* CallFunction(const Func* func, val_list* args); + + /** + * Callback for raising an event. Whenever the script interpreter is + * about to queue an event for later execution, it first gives all + * InterpreterPlugins a chance to handle the queuing otherwise (in the + * order of their priorities). A plugin can either just inspect the + * event, or take it over (i.e., prevent the interpreter from queuing it + * it). + * + * The default implementation does never handle the queuing in any way. + * + * @param event The even to be queued. The method can modify it in in + * place long as it ensures matching types and correct reference + * counting. + * + * @return True if the plugin took charge of the event; in that case it + * must have assumed ownership of the event and the intpreter will not do + * anything further with it. False otherwise. + * + */ + virtual bool QueueEvent(Event* event); + + /** + * Callback for updates in network time. This method will be called + * whenever network time is advanced. + * + * @param networkt_time The new network time. + */ + virtual void UpdateNetworkTime(double network_time); + + /** + * Callback for event queue draining. This method will be called + * whenever the event manager has drained it queue. + */ + virtual void DrainEvents(); + + /** + * Disables interpreter hooking. The functionality of the Plugin base + * class remains available. + */ + void DisableInterpreterPlugin() const; + + // Overridden from base class. + virtual Type PluginType() const; + +private: + int priority; + +}; + } #endif diff --git a/src/scan.l b/src/scan.l index 636ec5b251..4c67a2f75e 100644 --- a/src/scan.l +++ b/src/scan.l @@ -30,6 +30,8 @@ #include "analyzer/Analyzer.h" +#include "plugin/Manager.h" + extern YYLTYPE yylloc; // holds start line and column of token extern int print_loaded_scripts; extern int generate_documentation; @@ -579,6 +581,24 @@ YYLTYPE GetCurrentLocation() static int load_files(const char* orig_file) { + int rc = plugin_mgr->TryLoadFile(orig_file); + + if ( rc == 1 ) + return 0; // A plugin took care of it, just skip. + + if ( rc == 0 ) + { + if ( ! reporter->Errors() ) + // This is just in case the plugin failed to report + // the error itself, in which case we want to at + // least tell the user that something went wrong. + reporter->Error("Plugin reported error loading %s", orig_file); + + exit(1); + } + + assert(rc == -1); // No plugin in charge of this file. + // Whether we pushed on a FileInfo that will restore the // current module after the final file has been scanned. bool did_module_restore = false; diff --git a/src/util-config.h.in b/src/util-config.h.in index ff5e28537a..90919a63f4 100644 --- a/src/util-config.h.in +++ b/src/util-config.h.in @@ -3,3 +3,4 @@ #define BRO_BUILD_SOURCE_PATH "@CMAKE_BINARY_DIR@/src" #define BRO_BUILD_SCRIPTS_PATH "@CMAKE_BINARY_DIR@/scripts" #define BRO_MAGIC_INSTALL_PATH "@BRO_MAGIC_INSTALL_PATH@" +#define BRO_PLUGIN_INSTALL_PATH "@BRO_PLUGIN_INSTALL_PATH@" diff --git a/src/util.cc b/src/util.cc index dd232a83fa..3b917495bc 100644 --- a/src/util.cc +++ b/src/util.cc @@ -619,13 +619,13 @@ bool ensure_dir(const char *dirname) return true; } -bool is_dir(const char* path) +bool is_dir(const std::string& path) { struct stat st; - if ( stat(path, &st) < 0 ) + if ( stat(path.c_str(), &st) < 0 ) { if ( errno != ENOENT ) - reporter->Warning("can't stat %s: %s", path, strerror(errno)); + reporter->Warning("can't stat %s: %s", path.c_str(), strerror(errno)); return false; } @@ -633,6 +633,37 @@ bool is_dir(const char* path) return S_ISDIR(st.st_mode); } +bool is_file(const std::string& path) + { + struct stat st; + if ( stat(path.c_str(), &st) < 0 ) + { + if ( errno != ENOENT ) + reporter->Warning("can't stat %s: %s", path.c_str(), strerror(errno)); + + return false; + } + + return S_ISREG(st.st_mode); + } + +string strreplace(const string& s, const string& o, const string& n) + { + string r = s; + + while ( true ) + { + size_t i = r.find(o); + + if ( i == std::string::npos ) + break; + + r.replace(i, o.size(), n); + } + + return r; +} + int hmac_key_set = 0; uint8 shared_hmac_md5_key[16]; @@ -872,16 +903,31 @@ int int_list_cmp(const void* v1, const void* v2) return 1; } -const char* bro_path() +static string bro_path_value; + +const std::string& bro_path() { - const char* path = getenv("BROPATH"); - if ( ! path ) - path = ".:" + if ( bro_path_value.empty() ) + { + const char* path = getenv("BROPATH"); + if ( ! path ) + path = ".:" BRO_SCRIPT_INSTALL_PATH ":" BRO_SCRIPT_INSTALL_PATH "/policy" ":" BRO_SCRIPT_INSTALL_PATH "/site"; - return path; + bro_path_value = path; + } + + return bro_path_value; + } + +extern void add_to_bro_path(const string& dir) + { + // Make sure path is initialized. + bro_path(); + + bro_path_value += string(":") + dir; } const char* bro_magic_path() @@ -894,6 +940,16 @@ const char* bro_magic_path() return path; } +const char* bro_plugin_path() + { + const char* path = getenv("BRO_PLUGINS"); + + if ( ! path ) + path = BRO_PLUGIN_INSTALL_PATH; + + return path; + } + string bro_prefixes() { string rval; @@ -1089,9 +1145,9 @@ FILE* search_for_file(const char* filename, const char* ext, // @loads can be referenced relatively. if ( current_scanned_file_path != "" && filename[0] == '.' ) safe_snprintf(path, sizeof(path), "%s:%s", - current_scanned_file_path.c_str(), bro_path()); + current_scanned_file_path.c_str(), bro_path().c_str()); else - safe_strncpy(path, bro_path(), sizeof(path)); + safe_strncpy(path, bro_path().c_str(), sizeof(path)); char* dir_beginning = path; char* dir_ending = path; @@ -1505,25 +1561,16 @@ void get_memory_usage(unsigned int* total, unsigned int* malloced) if ( malloced ) *malloced = mi.uordblks; - ret_total = mi.arena; +#endif - if ( total ) - *total = ret_total; -#else struct rusage r; getrusage(RUSAGE_SELF, &r); - if ( malloced ) - *malloced = 0; - // At least on FreeBSD it's in KB. ret_total = r.ru_maxrss * 1024; if ( total ) *total = ret_total; -#endif - - // return ret_total; } #ifdef malloc diff --git a/src/util.h b/src/util.h index fcdfd6d499..a7d493485d 100644 --- a/src/util.h +++ b/src/util.h @@ -147,7 +147,13 @@ extern const char* fmt_access_time(double time); extern bool ensure_dir(const char *dirname); // Returns true if path exists and is a directory. -bool is_dir(const char* path); +bool is_dir(const std::string& path); + +// Returns true if path exists and is a file. +bool is_file(const std::string& path); + +// Replaces all occurences of *o* in *s* with *n*. +extern std::string strreplace(const std::string& s, const std::string& o, const std::string& n); extern uint8 shared_hmac_md5_key[16]; @@ -202,9 +208,12 @@ static const SourceID SOURCE_LOCAL = 0; extern void pinpoint(); extern int int_list_cmp(const void* v1, const void* v2); -extern const char* bro_path(); +extern const std::string& bro_path(); +extern void add_to_bro_path(const std::string& dir); + extern const char* bro_magic_path(); -extern std::string bro_prefixes(); +extern const char* bro_plugin_path(); +extern const char* bro_prefixes(); std::string dot_canon(std::string path, std::string file, std::string prefix = ""); const char* normalize_path(const char* path); void get_script_subpath(const std::string& full_filename, const char** subpath); diff --git a/testing/btest/Baseline/core.plugins.analyzer/output b/testing/btest/Baseline/core.plugins.analyzer/output new file mode 100644 index 0000000000..deb8a7655d --- /dev/null +++ b/testing/btest/Baseline/core.plugins.analyzer/output @@ -0,0 +1,6 @@ +Plugin: Demo::Foo - A Foo test analyzer (dynamic, version 1) + [Analyzer] Foo (ANALYZER_FOO, enabled) + [Event] foo_message + +=== +foo_message, [orig_h=::1, orig_p=37927/tcp, resp_h=::1, resp_p=4242/tcp], Hello, Foo!^J diff --git a/testing/btest/Baseline/core.plugins.test-plugin/output b/testing/btest/Baseline/core.plugins.test-plugin/output new file mode 100644 index 0000000000..e0663ab42e --- /dev/null +++ b/testing/btest/Baseline/core.plugins.test-plugin/output @@ -0,0 +1,11 @@ +Plugin: Demo::Foo - (dynamic, version 1) + [Event] plugin_event + [Function] hello_plugin_world + +=== +plugin: automatically loaded at startup +calling bif, Hello from the plugin! +=== +plugin: automatically loaded at startup +calling bif, Hello from the plugin! +plugin: manually loaded diff --git a/testing/btest/Traces/port4242.trace b/testing/btest/Traces/port4242.trace new file mode 100644 index 0000000000000000000000000000000000000000..e999b43b92fd875862e79db6ddca132548fe2413 GIT binary patch literal 868 zcmca|c+)~A1{MYw`2U}Qff2}goBcS@;yx!s9FPsd$e`_R0*I@@<^ZCQ07xS;V4R{Z zFsV{7yBj39fN9k#MxY8Nmj6dsm^ipj@AK6Ki8C@Yvx964qu4fqNmJCnJ?4x9+Ss!| zXw|Cc2T^SU*$A_(5olW&&~^|;22j5#5bd`fppEej0*qlhKLh>32=N!lCXkukqcy_&;v^S1r5b`U?|Q2`|CNXjUZcs wD7FQfGG>57PY@V-&oFJ7O0g|Sewo<-vgIkJE!!xz1?rcXK))n_Lk!Ip07J)|iU0rr literal 0 HcmV?d00001 diff --git a/testing/btest/core/plugins/analyzer-plugin/.btest-ignore b/testing/btest/core/plugins/analyzer-plugin/.btest-ignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/btest/core/plugins/analyzer-plugin/CMakeLists.txt b/testing/btest/core/plugins/analyzer-plugin/CMakeLists.txt new file mode 100644 index 0000000000..a032edbd89 --- /dev/null +++ b/testing/btest/core/plugins/analyzer-plugin/CMakeLists.txt @@ -0,0 +1,20 @@ + +project(Bro-Plugin-Demo-Foo) + +cmake_minimum_required(VERSION 2.6.3) + +if ( NOT BRO_DIST ) + message(FATAL_ERROR "BRO_DIST not set") +endif () + +set(CMAKE_MODULE_PATH ${BRO_DIST}/cmake) + +include(BroPlugin) + +bro_plugin_begin(Demo Foo) +bro_plugin_cc(src/Plugin.cc) +bro_plugin_cc(src/Foo.cc) +bro_plugin_bif(src/events.bif) +bro_plugin_bif(src/functions.bif) +bro_plugin_pac(src/foo.pac src/foo-protocol.pac src/foo-analyzer.pac) +bro_plugin_end() diff --git a/testing/btest/core/plugins/analyzer-plugin/__bro_plugin__ b/testing/btest/core/plugins/analyzer-plugin/__bro_plugin__ new file mode 100644 index 0000000000..c10d6ee17c --- /dev/null +++ b/testing/btest/core/plugins/analyzer-plugin/__bro_plugin__ @@ -0,0 +1,2 @@ +A place-holder file that marks a directory as holding a Bro plugin. +Content is ignored. diff --git a/testing/btest/core/plugins/analyzer-plugin/scripts/Demo/Foo/base/main.bro b/testing/btest/core/plugins/analyzer-plugin/scripts/Demo/Foo/base/main.bro new file mode 100644 index 0000000000..2e2d174b47 --- /dev/null +++ b/testing/btest/core/plugins/analyzer-plugin/scripts/Demo/Foo/base/main.bro @@ -0,0 +1,7 @@ + +const ports = { 4242/tcp }; + +event bro_init() &priority=5 + { + Analyzer::register_for_ports(Analyzer::ANALYZER_FOO, ports); + } diff --git a/testing/btest/core/plugins/analyzer-plugin/scripts/__load__.bro b/testing/btest/core/plugins/analyzer-plugin/scripts/__load__.bro new file mode 100644 index 0000000000..330718c604 --- /dev/null +++ b/testing/btest/core/plugins/analyzer-plugin/scripts/__load__.bro @@ -0,0 +1 @@ +@load Demo/Foo/base/main diff --git a/testing/btest/core/plugins/analyzer-plugin/src/Foo.cc b/testing/btest/core/plugins/analyzer-plugin/src/Foo.cc new file mode 100644 index 0000000000..b23c4b7817 --- /dev/null +++ b/testing/btest/core/plugins/analyzer-plugin/src/Foo.cc @@ -0,0 +1,59 @@ + +#include "Foo.h" +#include "foo_pac.h" +#include "events.bif.h" + +#include + +using namespace analyzer::Foo; + +Foo_Analyzer::Foo_Analyzer(Connection* conn) +: tcp::TCP_ApplicationAnalyzer("Foo", conn) + { + interp = new binpac::Foo::Foo_Conn(this); + } + +Foo_Analyzer::~Foo_Analyzer() + { + delete interp; + } + +void Foo_Analyzer::Done() + { + tcp::TCP_ApplicationAnalyzer::Done(); + + interp->FlowEOF(true); + interp->FlowEOF(false); + } + +void Foo_Analyzer::EndpointEOF(bool is_orig) + { + tcp::TCP_ApplicationAnalyzer::EndpointEOF(is_orig); + interp->FlowEOF(is_orig); + } + +void Foo_Analyzer::DeliverStream(int len, const u_char* data, bool orig) + { + tcp::TCP_ApplicationAnalyzer::DeliverStream(len, data, orig); + + assert(TCP()); + + if ( TCP()->IsPartial() ) + // punt on partial. + return; + + try + { + interp->NewData(orig, data, data + len); + } + catch ( const binpac::Exception& e ) + { + ProtocolViolation(fmt("Binpac exception: %s", e.c_msg())); + } + } + +void Foo_Analyzer::Undelivered(int seq, int len, bool orig) + { + tcp::TCP_ApplicationAnalyzer::Undelivered(seq, len, orig); + interp->NewGap(orig, len); + } diff --git a/testing/btest/core/plugins/analyzer-plugin/src/Foo.h b/testing/btest/core/plugins/analyzer-plugin/src/Foo.h new file mode 100644 index 0000000000..dfc50a9d76 --- /dev/null +++ b/testing/btest/core/plugins/analyzer-plugin/src/Foo.h @@ -0,0 +1,31 @@ + +#ifndef BRO_PLUGIN_DEMO_FOO_H +#define BRO_PLUGIN_DEMO_FOO_H + +#include "analyzer/protocol/tcp/TCP.h" +#include "analyzer/protocol/pia/PIA.h" + +namespace binpac { namespace Foo { class Foo_Conn; } } + +namespace analyzer { namespace Foo { + +class Foo_Analyzer : public tcp::TCP_ApplicationAnalyzer { +public: + Foo_Analyzer(Connection* conn); + ~Foo_Analyzer(); + + virtual void Done(); + virtual void DeliverStream(int len, const u_char* data, bool orig); + virtual void Undelivered(int seq, int len, bool orig); + virtual void EndpointEOF(bool is_orig); + + static analyzer::Analyzer* InstantiateAnalyzer(Connection* conn) + { return new Foo_Analyzer(conn); } + +protected: + binpac::Foo::Foo_Conn* interp; +}; + +} } // namespace analyzer::* + +#endif diff --git a/testing/btest/core/plugins/analyzer-plugin/src/Plugin.cc b/testing/btest/core/plugins/analyzer-plugin/src/Plugin.cc new file mode 100644 index 0000000000..b37a60d148 --- /dev/null +++ b/testing/btest/core/plugins/analyzer-plugin/src/Plugin.cc @@ -0,0 +1,12 @@ + +#include + +#include "Foo.h" + +BRO_PLUGIN_BEGIN(Demo, Foo) + BRO_PLUGIN_VERSION(1); + BRO_PLUGIN_DESCRIPTION("A Foo test analyzer"); + BRO_PLUGIN_ANALYZER("Foo", Foo::Foo_Analyzer); + BRO_PLUGIN_BIF_FILE(events); + BRO_PLUGIN_BIF_FILE(functions); +BRO_PLUGIN_END diff --git a/testing/btest/core/plugins/analyzer-plugin/src/events.bif b/testing/btest/core/plugins/analyzer-plugin/src/events.bif new file mode 100644 index 0000000000..4603fe4cf6 --- /dev/null +++ b/testing/btest/core/plugins/analyzer-plugin/src/events.bif @@ -0,0 +1,2 @@ + +event foo_message%(c: connection, data: string%); diff --git a/testing/btest/core/plugins/analyzer-plugin/src/foo-analyzer.pac b/testing/btest/core/plugins/analyzer-plugin/src/foo-analyzer.pac new file mode 100644 index 0000000000..a210a8430c --- /dev/null +++ b/testing/btest/core/plugins/analyzer-plugin/src/foo-analyzer.pac @@ -0,0 +1,15 @@ + +refine connection Foo_Conn += { + + function Foo_data(msg: Foo_Message): bool + %{ + StringVal* data = new StringVal(${msg.data}.length(), (const char*) ${msg.data}.data()); + BifEvent::generate_foo_message(bro_analyzer(), bro_analyzer()->Conn(), data); + return true; + %} + +}; + +refine typeattr Foo_Message += &let { + proc: bool = $context.connection.Foo_data(this); +}; diff --git a/testing/btest/core/plugins/analyzer-plugin/src/foo-protocol.pac b/testing/btest/core/plugins/analyzer-plugin/src/foo-protocol.pac new file mode 100644 index 0000000000..892513c4f0 --- /dev/null +++ b/testing/btest/core/plugins/analyzer-plugin/src/foo-protocol.pac @@ -0,0 +1,4 @@ + +type Foo_Message(is_orig: bool) = record { + data: bytestring &restofdata; +}; diff --git a/testing/btest/core/plugins/analyzer-plugin/src/foo.pac b/testing/btest/core/plugins/analyzer-plugin/src/foo.pac new file mode 100644 index 0000000000..826bcc624e --- /dev/null +++ b/testing/btest/core/plugins/analyzer-plugin/src/foo.pac @@ -0,0 +1,26 @@ +%include binpac.pac +%include bro.pac + +%extern{ +#include "Foo.h" + +#include "events.bif.h" +%} + +analyzer Foo withcontext { + connection: Foo_Conn; + flow: Foo_Flow; +}; + +connection Foo_Conn(bro_analyzer: BroAnalyzer) { + upflow = Foo_Flow(true); + downflow = Foo_Flow(false); +}; + +%include foo-protocol.pac + +flow Foo_Flow(is_orig: bool) { + datagram = Foo_Message(is_orig) withcontext(connection, this); +}; + +%include foo-analyzer.pac diff --git a/testing/btest/core/plugins/analyzer-plugin/src/functions.bif b/testing/btest/core/plugins/analyzer-plugin/src/functions.bif new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/btest/core/plugins/analyzer.bro b/testing/btest/core/plugins/analyzer.bro new file mode 100644 index 0000000000..2a72797564 --- /dev/null +++ b/testing/btest/core/plugins/analyzer.bro @@ -0,0 +1,13 @@ +# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo +# @TEST-EXEC: cp -r %DIR/analyzer-plugin/* . +# @TEST-EXEC: make BRO=${DIST} +# @TEST-EXEC: BROPLUGINS=`pwd` bro -NN | awk '/^Plugin:.*Demo/ {p=1; print; next} /^Plugin:/{p=0} p==1{print}' >>output +# @TEST-EXEC: echo === >>output +# @TEST-EXEC: BROPLUGINS=`pwd` bro -r $TRACES/port4242.trace %INPUT >>output +# @TEST-EXEC: btest-diff output + +event foo_message(c: connection, data: string) + { + print "foo_message", c$id, data; + } + diff --git a/testing/btest/core/plugins/test-plugin.sh b/testing/btest/core/plugins/test-plugin.sh new file mode 100644 index 0000000000..87b09b2e4d --- /dev/null +++ b/testing/btest/core/plugins/test-plugin.sh @@ -0,0 +1,45 @@ +# @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo +# @TEST-EXEC: bash %INPUT +# @TEST-EXEC: make BRO=${DIST} +# @TEST-EXEC: BROPLUGINS=`pwd` bro -NN | awk '/^Plugin:.*Demo/ {p=1; print; next} /^Plugin:/{p=0} p==1{print}' >>output +# @TEST-EXEC: echo === >>output +# @TEST-EXEC: BROPLUGINS=`pwd` bro -r $TRACES/empty.trace >>output +# @TEST-EXEC: echo === >>output +# @TEST-EXEC: BROPLUGINS=`pwd` bro demo/foo -r $TRACES/empty.trace >>output +# @TEST-EXEC: btest-diff output + +cat >scripts/__load__.bro <scripts/demo/foo/__load__.bro <scripts/demo/foo/manually.bro <scripts/demo/foo/base/at-startup.bro <src/functions.bif <src/events.bif < Date: Tue, 26 Nov 2013 13:55:58 -0800 Subject: [PATCH 007/193] Restructuring the plugin API to accomodate hooks. I got rid of the earlier separate InterpreterPlugin class. Instead Plugin now has a set of virtual methods HookSomething()... that plugins can override. For efficiency purposes, they however need to register first that they are interested in a hook, otherwise the virtual method will never be called. The idea is to extend the set of hooks over time as we figure out what's useful. This is a checkpoint commit that's essentially untested and probably broken. It compiles, though. --- src/Event.cc | 5 +- src/Func.cc | 2 +- src/Net.cc | 2 +- src/main.cc | 1 - src/plugin/Macros.h | 8 + src/plugin/Manager.cc | 200 +++++++++++-------- src/plugin/Manager.h | 137 ++++++++----- src/plugin/Plugin.cc | 166 ++++++++-------- src/plugin/Plugin.h | 449 ++++++++++++++++++++++-------------------- src/scan.l | 2 +- src/util.h | 2 +- 11 files changed, 529 insertions(+), 445 deletions(-) diff --git a/src/Event.cc b/src/Event.cc index 7774042ac4..6b345a9c5b 100644 --- a/src/Event.cc +++ b/src/Event.cc @@ -78,8 +78,7 @@ EventMgr::~EventMgr() void EventMgr::QueueEvent(Event* event) { - if ( plugin_mgr->QueueEvent(event) ) - return; + PLUGIN_HOOK_VOID(HOOK_QUEUE_EVENT, HookQueueEvent(event)); if ( ! head ) head = tail = event; @@ -119,7 +118,7 @@ void EventMgr::Drain() SegmentProfiler(segment_logger, "draining-events"); - plugin_mgr->DrainEvents(); + PLUGIN_HOOK_VOID(HOOK_DRAIN_EVENTS, HookDrainEvents()); draining = true; while ( head ) diff --git a/src/Func.cc b/src/Func.cc index d8e0ab2c96..f08cf095f0 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -283,7 +283,7 @@ Val* BroFunc::Call(val_list* args, Frame* parent) const DEBUG_MSG("Function: %s\n", id->Name()); #endif - Val* plugin_result = plugin_mgr->CallFunction(this, args); + Val* plugin_result = PLUGIN_HOOK_WITH_RESULT(HOOK_CALL_FUNCTION, HookCallFunction(this, args), 0); if ( plugin_result ) { diff --git a/src/Net.cc b/src/Net.cc index 6cc148770e..19663b18ed 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -148,7 +148,7 @@ RETSIGTYPE watchdog(int /* signo */) void net_update_time(double new_network_time) { network_time = new_network_time; - plugin_mgr->UpdateNetworkTime(network_time); + PLUGIN_HOOK_VOID(HOOK_UPDATE_NETWORK_TIME, HookUpdateNetworkTime(new_network_time)); } void net_init(name_list& interfaces, name_list& readfiles, diff --git a/src/main.cc b/src/main.cc index bfbcf77a6d..c3c32a4b14 100644 --- a/src/main.cc +++ b/src/main.cc @@ -224,7 +224,6 @@ void usage() fprintf(stderr, " $BROPATH | file search path (%s)\n", bro_path().c_str()); fprintf(stderr, " $BROMAGIC | libmagic mime magic database search path (%s)\n", bro_magic_path()); fprintf(stderr, " $BRO_PLUGINS | plugin search path (%s)\n", bro_plugin_path()); - fprintf(stderr, " $BRO_PREFIXES | prefix list (%s)\n", bro_prefixes()); fprintf(stderr, " $BRO_PREFIXES | prefix list (%s)\n", bro_prefixes().c_str()); fprintf(stderr, " $BRO_DNS_FAKE | disable DNS lookups (%s)\n", bro_dns_fake()); fprintf(stderr, " $BRO_SEED_FILE | file to load seeds from (not set)\n"); diff --git a/src/plugin/Macros.h b/src/plugin/Macros.h index 3248031e98..3657fb1e40 100644 --- a/src/plugin/Macros.h +++ b/src/plugin/Macros.h @@ -64,6 +64,14 @@ */ #define BRO_PLUGIN_VERSION(v) SetVersion(v) +/** + * Enables a hook for the plugin. Once enabled, Bro will call the + * corresponding virtual function. + * + * @param h The \a HookType to able. + */ +#define BRO_PLUGIN_ENABLE_HOOK(h) EnableHook(h) + /** * Adds script-level items defined in a \c *.bif file to what the plugin * provides. diff --git a/src/plugin/Manager.cc b/src/plugin/Manager.cc index ddd953ab22..be1d7827a5 100644 --- a/src/plugin/Manager.cc +++ b/src/plugin/Manager.cc @@ -21,11 +21,20 @@ string Manager::current_sopath; Manager::Manager() { init = false; + hooks = new hook_list*[NUM_HOOKS]; + + for ( int i = 0; i < NUM_HOOKS; i++ ) + hooks[i] = 0; } Manager::~Manager() { assert(! init); + + for ( int i = 0; i < NUM_HOOKS; i++ ) + delete hooks[i]; + + delete [] hooks; } void Manager::LoadPluginsFrom(const string& dir) @@ -204,15 +213,6 @@ bool Manager::RegisterPlugin(Plugin *plugin) return true; } -static bool interpreter_plugin_cmp(const InterpreterPlugin* a, const InterpreterPlugin* b) - { - if ( a->Priority() == b->Priority() ) - return a->Name() < b->Name(); - - // Reverse sort. - return a->Priority() > b->Priority(); - } - void Manager::InitPreScript() { assert(! init); @@ -221,26 +221,9 @@ void Manager::InitPreScript() { Plugin* plugin = *i; - if ( plugin->PluginType() == Plugin::INTERPRETER ) - interpreter_plugins.push_back(dynamic_cast(plugin)); - plugin->InitPreScript(); - - // Track the file extensions the plugin can handle. - std::stringstream ext(plugin->FileExtensions()); - - // Split at ":". - std::string e; - - while ( std::getline(ext, e, ':') ) - { - DBG_LOG(DBG_PLUGINS, "Plugin %s handles *.%s", plugin->Name(), e.c_str()); - extensions.insert(std::make_pair(e, plugin)); - } } - interpreter_plugins.sort(interpreter_plugin_cmp); - init = true; } @@ -273,28 +256,6 @@ void Manager::FinishPlugins() init = false; } -int Manager::TryLoadFile(const char* file) - { - assert(file); - const char* ext = strrchr(file, '.'); - - if ( ! ext ) - return -1; - - extension_map::iterator i = extensions.find(++ext); - if ( i == extensions.end() ) - return -1; - - Plugin* plugin = i->second; - - DBG_LOG(DBG_PLUGINS, "Loading %s with %s", file, plugin->Name()); - - if ( i->second->LoadFile(file) ) - return 1; - - return 0; - } - Manager::plugin_list Manager::Plugins() const { return *Manager::PluginsInternal(); @@ -310,66 +271,135 @@ Manager::plugin_list* Manager::PluginsInternal() return plugins; } -Val* Manager::CallFunction(const Func* func, val_list* args) const +static bool hook_cmp(std::pair a, std::pair b) { - Val* result = 0; + if ( a.first == b.first ) + return a.second->Name() < a.second->Name(); - for ( interpreter_plugin_list::const_iterator i = interpreter_plugins.begin(); - i != interpreter_plugins.end() && ! result; i++ ) + // Reverse sort. + return a.first > b.first; + } + +std::list > Manager::HooksEnabledForPlugin(const Plugin* plugin) const + { + std::list > enabled; + + for ( int i = 0; i < NUM_HOOKS; i++ ) { - result = (*i)->CallFunction(func, args); + hook_list* l = hooks[i]; - if ( result ) + if ( ! l ) + continue; + + for ( hook_list::iterator j = l->begin(); j != l->end(); j++ ) { - DBG_LOG(DBG_PLUGINS, "Plugin %s replaced call to %s", (*i)->Name(), func->Name()); - return result; + if ( (*j).second == plugin ) + enabled.push_back(std::make_pair((HookType)i, (*j).first)); } } + return enabled; + } + +void Manager::EnableHook(HookType hook, Plugin* plugin, int prio) + { + if ( ! hooks[hook] ) + hooks[hook] = new hook_list; + + hooks[hook]->push_back(std::make_pair(prio, plugin)); + hooks[hook]->sort(hook_cmp); + } + +void Manager::DisableHook(HookType hook, Plugin* plugin) + { + hook_list* l = hooks[hook]; + + if ( ! l ) + return; + + for ( hook_list::iterator i = l->begin(); i != l->end(); i++ ) + { + if ( (*i).second == plugin ) + { + l->erase(i); + break; + } + } + + if ( l->empty() ) + { + delete l; + hooks[hook] = 0; + } + } + +int Manager::HookLoadFile(const char* file) + { + hook_list* l = hooks[HOOK_LOAD_FILE]; + + for ( hook_list::iterator i = l->begin(); l && i != l->end(); i++ ) + { + Plugin* p = (*i).second; + + int rc = p->HookLoadFile(file); + + if ( rc >= 0 ) + return rc; + } + + return -1; + } + +Val* Manager::HookCallFunction(const Func* func, val_list* args) const + { + hook_list* l = hooks[HOOK_CALL_FUNCTION]; + + for ( hook_list::iterator i = l->begin(); l && i != l->end(); i++ ) + { + Plugin* p = (*i).second; + + Val* v = p->HookCallFunction(func, args); + + if ( v ) + return v; + } + return 0; } -bool Manager::QueueEvent(Event* event) const +bool Manager::HookQueueEvent(Event* event) const { - for ( interpreter_plugin_list::const_iterator i = interpreter_plugins.begin(); - i != interpreter_plugins.end(); i++ ) + hook_list* l = hooks[HOOK_QUEUE_EVENT]; + + for ( hook_list::iterator i = l->begin(); l && i != l->end(); i++ ) { - if ( (*i)->QueueEvent(event) ) - { - DBG_LOG(DBG_PLUGINS, "Plugin %s handled queueing of event %s", (*i)->Name(), event->Handler()->Name()); + Plugin* p = (*i).second; + + if ( p->HookQueueEvent(event) ) return true; - } } return false; } - -void Manager::UpdateNetworkTime(double network_time) const +void Manager::HookDrainEvents() const { - for ( interpreter_plugin_list::const_iterator i = interpreter_plugins.begin(); - i != interpreter_plugins.end(); i++ ) - (*i)->UpdateNetworkTime(network_time); - } + hook_list* l = hooks[HOOK_DRAIN_EVENTS]; -void Manager::DrainEvents() const - { - for ( interpreter_plugin_list::const_iterator i = interpreter_plugins.begin(); - i != interpreter_plugins.end(); i++ ) - (*i)->DrainEvents(); - } - -void Manager::DisableInterpreterPlugin(const InterpreterPlugin* plugin) - { - for ( interpreter_plugin_list::iterator i = interpreter_plugins.begin(); - i != interpreter_plugins.end(); i++ ) + for ( hook_list::iterator i = l->begin(); l && i != l->end(); i++ ) { - if ( *i == plugin ) - { - interpreter_plugins.erase(i); - return; - } + Plugin* p = (*i).second; + p->HookDrainEvents(); } } +void Manager::HookUpdateNetworkTime(double network_time) const + { + hook_list* l = hooks[HOOK_UPDATE_NETWORK_TIME]; + for ( hook_list::iterator i = l->begin(); l && i != l->end(); i++ ) + { + Plugin* p = (*i).second; + p->HookUpdateNetworkTime(network_time); + } + } diff --git a/src/plugin/Manager.h b/src/plugin/Manager.h index 85b60fb763..0c52679a64 100644 --- a/src/plugin/Manager.h +++ b/src/plugin/Manager.h @@ -12,6 +12,14 @@ namespace plugin { +// Macros that trigger a plugin hook. We put this into macros to short-cut +// the code for the most common case that no plugin defines the hook. +#define PLUGIN_HOOK_WITH_RESULT(hook, method_call, default_result) \ + (plugin_mgr->HavePluginForHook(plugin::hook) ? plugin_mgr->method_call : (default_result)) + +#define PLUGIN_HOOK_VOID(hook, method_call) \ + if ( plugin_mgr->HavePluginForHook(plugin::hook) ) plugin_mgr->method_call; + /** * A singleton object managing all plugins. */ @@ -19,7 +27,6 @@ class Manager { public: typedef std::list plugin_list; - typedef std::list interpreter_plugin_list; typedef Plugin::component_list component_list; /** @@ -30,7 +37,7 @@ public: /** * Destructor. */ - ~Manager(); + virtual ~Manager(); /** * Loads all plugins dynamically from a set of directories. Multiple @@ -71,22 +78,6 @@ public: */ void FinishPlugins(); - /** - * This tries to load the given file by searching for a plugin that - * support that extension. If a correspondign plugin is found, it's - * asked to loead the file. If that fails, the method reports an - * error message. - * - * This method must be called only between InitPreScript() and - * InitPostScript(). - * - * @return 1 if the file was sucessfully loaded by a plugin; 0 if a - * plugin was found that supports the file's extension, yet it - * encountered a problem loading the file; and -1 if we don't have a - * plugin that supports this extension. - */ - int TryLoadFile(const char* file); - /** * Returns a list of all available plugins. This includes all that * are compiled in statically, as well as those loaded dynamically so @@ -102,7 +93,65 @@ public: template std::list Components() const; /** - * Filters a function/event/hook call through all interpreter plugins. + * Returns true if there's at least one plugin interested in a given + * hook. + * + * @param The hook to check. + * + * @return True if there's a plugin for that hook. + */ + bool HavePluginForHook(HookType hook) const + { + // Inline to make avoid the function call. + return hooks[hook] != 0; + } + + /** + * Returns all the hooks, with their priorities, that are currently + * enabled for a given plugin. + * + * @param plugin The plugin to return the hooks for. + */ + std::list > HooksEnabledForPlugin(const Plugin* plugin) const; + + /** + * Enables a hook for a given plugin. + * + * hook: The hook to enable. + * + * plugin: The plugin defining the hook. + * + * prio: The priority to associate with the plugin for this hook. + */ + void EnableHook(HookType hook, Plugin* plugin, int prio); + + /** + * Disables a hook for a given plugin. + * + * hook: The hook to enable. + * + * plugin: The plugin that used to define the hook. + */ + void DisableHook(HookType hook, Plugin* plugin); + + // Hook entry functions. + + /** + * Hook that gives plugins a chance to take over loading an input + * input file. This method must be called between InitPreScript() and + * InitPostScript() for each input file Bro is about to load, either + * given on the command line or via @load script directives. The hook + * can take over the file, in which case Bro must not further process + * it otherwise. + * + * @return 1 if a plugin took over the file and loaded it + * successfully; 0 if a plugin took over the file but had trouble + * loading it; and -1 if no plugin was interested in the file at all. + */ + virtual int HookLoadFile(const char* file); + + /** + * Hook that filters calls to a script function/event/hook. * * @param func The function to be called. * @@ -113,39 +162,29 @@ public: * events, it may be any Val and must be ignored). If no plugin * handled the call, the method returns null. */ - Val* CallFunction(const Func* func, val_list* args) const; - - /** - * Filter the queuing of an event through all interpreter plugins. - * - * @param event The event to be queued; it may be modified. - * - * @return Returns true if a plugin handled the queuing; in that case the - * plugin will have taken ownership. - * - */ - bool QueueEvent(Event* event) const; + Val* HookCallFunction(const Func* func, val_list* args) const; /** - * Informs all interpreter plugins about an update in network time. + * Hook that filters the queuing of an event. * - * @param networkt_time The new network time. + * @param event The event to be queued; it may be modified. + * + * @return Returns true if a plugin handled the queuing; in that case + * the plugin will have taken ownership. */ - void UpdateNetworkTime(double network_time) const; + bool HookQueueEvent(Event* event) const; /** - * Informs all interpreter plugins that the event queue has been drained. + * Hook that informs plugins about an update in network time. + * + * @param network_time The new network time. */ - void DrainEvents() const; + void HookUpdateNetworkTime(double network_time) const; - /** - * Disables an interpreter plugin's hooking of the script interpreter. - * The remaining functionality of the Plugin base class remains - * available. - * - * @param plugin The plugin to disable. - */ - void DisableInterpreterPlugin(const InterpreterPlugin* plugin); + /** + * Hooks that informs plugins that the event queue is being drained. + */ + void HookDrainEvents() const; /** * Internal method that registers a freshly instantiated plugin with @@ -177,13 +216,17 @@ protected: int LoadPlugin(const std::string& dir); private: + // A hook list keeps pairs of plugin and priority interested in a + // given hook. + typedef std::list > hook_list; + static plugin_list* PluginsInternal(); bool init; - typedef std::map extension_map; - extension_map extensions; - interpreter_plugin_list interpreter_plugins; + // An array indexed by HookType. An entry is null if there's no hook + // of that type enabled. + hook_list** hooks; static string current_dir; static string current_sopath; diff --git a/src/plugin/Plugin.cc b/src/plugin/Plugin.cc index ff3cf8f414..20d436907b 100644 --- a/src/plugin/Plugin.cc +++ b/src/plugin/Plugin.cc @@ -10,6 +10,22 @@ using namespace plugin; +const char* hook_name(HookType h) + { + static const char* hook_names[int(NUM_HOOKS) + 1] = { + // Order must match that of HookType. + "LoadFile", + "CallFunction", + "QueueEvent", + "DrainEvents", + "UpdateNetworkTime", + // End marker. + "", + }; + + return hook_names[int(h)]; + } + BifItem::BifItem(const char* arg_id, Type arg_type) { id = copy_string(arg_id); @@ -49,7 +65,6 @@ Plugin::Plugin() dynamic = false; base_dir = 0; sopath = 0; - extensions = 0; Manager::RegisterPlugin(this); } @@ -62,12 +77,6 @@ Plugin::~Plugin() delete [] description; delete [] base_dir; delete [] sopath; - delete [] extensions; - } - -Plugin::Type Plugin::PluginType() const - { - return STANDARD; } const char* Plugin::Name() const @@ -170,22 +179,6 @@ Plugin::bif_item_list Plugin::CustomBifItems() const return bif_item_list(); } -const char* Plugin::FileExtensions() const - { - return extensions ? extensions : ""; - } - -void Plugin::SetFileExtensions(const char* ext) - { - extensions = copy_string(ext); - } - -bool Plugin::LoadFile(const char* file) - { - reporter->InternalError("Plugin::LoadFile not overriden for %s", file); - return false; - } - void Plugin::Done() { for ( component_list::const_iterator i = components.begin(); i != components.end(); i++ ) @@ -204,18 +197,9 @@ static bool component_cmp(const Component* a, const Component* b) return a->Name() < b->Name(); } -void Plugin::AddComponent(Component* c) - { - components.push_back(c); - - // Sort components by name to make sure we have a deterministic - // order. - components.sort(component_cmp); - } - bool Plugin::LoadBroFile(const char* file) { - add_input_file(file); + ::add_input_file(file); return true; } @@ -230,6 +214,53 @@ void Plugin::AddBifItem(const char* name, BifItem::Type type) bif_items.push_back(bi); } +void Plugin::AddComponent(Component* c) + { + components.push_back(c); + + // Sort components by name to make sure we have a deterministic + // order. + components.sort(component_cmp); + } + +Plugin::hook_list Plugin::EnabledHooks() const + { + return plugin_mgr->HooksEnabledForPlugin(this); + } + +void Plugin::EnableHook(HookType hook, int priority) + { + plugin_mgr->EnableHook(hook, this, priority); + } + +void Plugin::DisableHook(HookType hook) + { + plugin_mgr->DisableHook(hook, this); + } + +int Plugin::HookLoadFile(const char* file) + { + return -1; + } + +Val* Plugin::HookCallFunction(const Func* func, val_list* args) + { + return 0; + } + +bool Plugin::HookQueueEvent(Event* event) + { + return false; + } + +void Plugin::HookDrainEvents() + { + } + +void Plugin::HookUpdateNetworkTime(double network_time) + { + } + void Plugin::Describe(ODesc* d) const { d->Add("Plugin: "); @@ -256,18 +287,6 @@ void Plugin::Describe(ODesc* d) const else d->Add(" (built-in)"); - switch ( PluginType() ) { - case STANDARD: - break; - - case INTERPRETER: - d->Add( " (interpreter plugin)"); - break; - - default: - reporter->InternalError("unknown plugin type in Plugin::Describe"); - } - d->Add("\n"); if ( d->IsShort() ) @@ -317,48 +336,19 @@ void Plugin::Describe(ODesc* d) const d->Add((*i).GetID()); d->Add("\n"); } + + hook_list hooks = EnabledHooks(); + + for ( hook_list::iterator i = hooks.begin(); i != hooks.end(); i++ ) + { + HookType hook = (*i).first; + int prio = (*i).second; + + d->Add(" Implements "); + d->Add(hook_name(hook)); + d->Add(" (priority "); + d->Add(prio); + d->Add("]\n"); + } } -InterpreterPlugin::InterpreterPlugin(int arg_priority) - { - priority = arg_priority; - } - -InterpreterPlugin::~InterpreterPlugin() - { - } - -int InterpreterPlugin::Priority() const - { - return priority; - } - -Plugin::Type InterpreterPlugin::PluginType() const - { - return INTERPRETER; - } - -Val* InterpreterPlugin::CallFunction(const Func* func, val_list* args) - { - return 0; - } - -bool InterpreterPlugin::QueueEvent(Event* event) - { - return false; - } - -void InterpreterPlugin::UpdateNetworkTime(double network_time) - { - } - -void InterpreterPlugin::DrainEvents() - { - } - -void InterpreterPlugin::DisableInterpreterPlugin() const - { - plugin_mgr->DisableInterpreterPlugin(this); - } - - diff --git a/src/plugin/Plugin.h b/src/plugin/Plugin.h index 306c97e59e..915563e98c 100644 --- a/src/plugin/Plugin.h +++ b/src/plugin/Plugin.h @@ -17,6 +17,21 @@ namespace plugin { class Manager; class Component; +/** + * Hook types that a plugin may define. Each label maps to the corresponding + * virtual method in \a Plugin. + */ +enum HookType { + // Note: when changing this table, update hook_name() in Plugin.cc. + HOOK_LOAD_FILE, + HOOK_CALL_FUNCTION, + HOOK_QUEUE_EVENT, + HOOK_DRAIN_EVENTS, + HOOK_UPDATE_NETWORK_TIME, + // End marker. + NUM_HOOKS, +}; + /** * A class describing an item defined in \c *.bif file. */ @@ -70,45 +85,37 @@ private: /** * Base class for all plugins. * - * Plugins encapsulate functionality that extends one of Bro's major + * Plugins encapsulate functionality that extends one or more of Bro's major * subsystems, such as analysis of a specific protocol, or logging output in - * a particular format. A plugin is a logical container that can provide one - * or more \a components implementing functionality. For example, a RPC - * plugin could provide analyzer for set of related protocols (RPC, NFS, - * etc.), each of which would be a separate component. Likewise, a SQLite - * plugin could provide both a writer and reader component. In addition to - * components, a plugin can also provide of script-level elements defined in - * *.bif files. + * a particular format. A plugin acts a logical container that can provide a + * set of different functionality. Specifically, it may: + * + * - Provide one or more \a components implementing functionality. For + * example, a RPC plugin could provide analyzer for set of related + * protocols (RPC, NFS, etc.), each of which would be a separate component. + * Likewise, a SQLite plugin could provide both a writer and reader + * component. In addition to components, a plugin can also provide of + * script-level elements defined in *.bif files. + * + * - Provide BiF elements (functions, events, types, globals). + * + * - Provide hooks (aka callbacks) into Bro's core processing to inject + * and/or alter functionality. + * + * Note that a plugin needs to explicitly register all the functionality it + * provides. For components, it needs to call AddComponent(); for BiFs + * AddBifItem(); and for hooks EnableHook() and then also implemennt the + * corresponding virtual method). * - * Currently, all plugins are compiled statically into the final Bro binary. - * Later, we will extend the infrastructure to also support plugins loaded - * dynamically as shared libraries. */ class Plugin { public: typedef std::list component_list; typedef std::list bif_item_list; + typedef std::list > hook_list; typedef std::list > bif_init_func_result; typedef void (*bif_init_func)(Plugin *); - /** - * Type of a plugin. Plugin types are set implicitly by deriving from - * the corresponding base class. */ - enum Type { - /** - * A standard plugin. This is the type for all plugins - * derived directly from \a Plugin. */ - STANDARD, - - /** - * An interpreter plugin. These plugins get hooked into the - * script interpreter and can modify, or even replace, its - * execution of Bro script code. To create an interpreter - * plugin, derive from \aInterpreterPlugin. - */ - INTERPRETER - }; - /** * Constructor. */ @@ -119,11 +126,6 @@ public: */ virtual ~Plugin(); - /** - * Returns the type of the plugin. - */ - virtual Type PluginType() const; - /** * Returns the name of the plugin. */ @@ -146,11 +148,6 @@ public: */ bool DynamicPlugin() const; - /** - * Returns a colon-separated list of file extensions the plugin handles. - */ - const char* FileExtensions() const; - /** * For dynamic plugins, returns the base directory from which it was * loaded. For static plugins, returns null. @@ -201,8 +198,8 @@ public: /** * Finalizer method that derived classes can override for performing - * custom tasks at shutdown. Implementation must call the parent's - * version. + * custom tasks at shutdown. This can be overridden by derived + * classes; they must however call the parent's implementation. */ virtual void Done(); @@ -216,12 +213,14 @@ public: void Describe(ODesc* d) const; /** - * Registering an individual BiF that the plugin defines. The + * Registers an individual BiF that the plugin defines. The * information is for informational purpuses only and will show up in * the result of BifItems() as well as in the Describe() output. * Another way to add this information is via overriding * CustomBifItems(). * + * \todo Do we need both this an CustomBifItems()? + * * @param name The name of the BiF item. * * @param type The item's type. @@ -229,10 +228,10 @@ public: void AddBifItem(const char* name, BifItem::Type type); /** - * Adds a file to the list of files Bro loads at startup. This will - * normally be a Bro script, but it passes through the plugin system - * as well to load files with other extensions as supported by any of - * the current plugins. In other words, calling this method is + * Adds a file to the list of files that Bro loads at startup. This + * will normally be a Bro script, but it passes through the plugin + * system as well to load files with other extensions as supported by + * any of the current plugins. In other words, calling this method is * similar to given a file on the command line. Note that the file * may be only queued for now, and actually loaded later. * @@ -245,161 +244,99 @@ public: */ bool LoadBroFile(const char* file); -protected: - friend class Manager; - - /** - * Sets the plugins name. - * - * @param name The name. Makes a copy internally. - */ - void SetName(const char* name); - - /** - * Sets the plugin's textual description. - * - * @param name The description. Makes a copy internally. - */ - void SetDescription(const char* descr); - - /** - * Sets the plugin's version. - * - * @param version The version. - */ - void SetVersion(int version); - - /** - * Sets the API version the plugin requires. - * BRO_PLUGIN_VERSION_BUILTIN indicates that it's a plugin linked in - * statically. - */ - void SetAPIVersion(int version); - - /** - * Marks the plugin as statically or dynamically linked. - * - * @param dynamic True if this is a dynamically linked plugin. - */ - void SetDynamicPlugin(bool dynamic); - - /** - * Reports the extensions of input files the plugin handles. If Bro - * wants to load a file with one of these extensions it will pass - * them to LoadFile() and then then ignore otherwise. - * - * ext: A list of colon-separated file extensions the plugin handles. - */ - void SetFileExtensions(const char* ext); - - /** - * Sets the base directory and shared library path from which the - * plugin was loaded. This should be called only from the manager for - * dynamic plugins. - * - * @param dir The plugin directory. The functions makes an internal - * copy of string. - * - * @param sopath The full path the shared library loaded. The - * functions makes an internal copy of string. - */ - void SetPluginLocation(const char* dir, const char* sopath); - - /** - * Takes ownership. - */ - void AddComponent(Component* c); - - /** - * Virtual method that can be overriden by derived class to provide - * information about further script-level elements that the plugins - * provides on its own, i.e., outside of the standard mechanism - * processing *.bif files automatically. The returned information is - * for informational purpuses only and will show up in the result of - * BifItems() as well as in the Describe() output. - */ - virtual bif_item_list CustomBifItems() const; - - /** - * Virtual method that can be overriden by derived class to load - * files with extensions reported via SetFileExtension(). - * - * This method will be called between InitPreScript() and - * InitPostScript(), but with no further order or timing guaranteed. - * It will be called once for each file encountered with of the - * specificed extensions (i.e., duplicates are filtered out - * automatically). - * - * @return True if the file was loaded successfuly, false if not. Bro - * will abort in the latter case. - */ - virtual bool LoadFile(const char* file); - - /** - * Initializes the BiF items added with AddBifItem(). Internal method - * that will be called by the manager at the right time. - */ - void InitBifs(); - /** * Internal function adding an entry point for registering * auto-generated BiFs. */ void __AddBifInitFunction(bif_init_func c); -private: - typedef std::list bif_init_func_list; +protected: + friend class Manager; - const char* name; - const char* description; - const char* base_dir; - const char* sopath; - const char* extensions; - int version; - int api_version; - bool dynamic; - - component_list components; - bif_item_list bif_items; - bif_init_func_list bif_inits; -}; - -/** - * Class for hooking into script execution. An interpreter plugin can do - * everything a normal plugin can, yet will also be interfaced to the script - * interpreter for learning about, modidying, or potentially replacing - * standard functionality. - */ -class InterpreterPlugin : public Plugin { -public: /** - * Constructor. + * Registers and activates a component. * - * @param priority Imposes an order on InterpreterPlugins in which - * they'll be chained. The higher the prioritu, the earlier a plugin - * is called when the interpreter goes through the chain. */ - InterpreterPlugin(int priority); - - /** - * Destructor. + * @param c The component. The method takes ownership. */ - virtual ~InterpreterPlugin(); + void AddComponent(Component* c); /** - * Returns the plugins priority. + * Enables a hook. The corresponding virtual method will now be + * called as Bro's processing proceeds. Note that enabling hooks can + * have performance impaxct as many trigger frequently inside Bro's + * main processing path. + * + * Note that hooks may be enabled/disabled dynamically at any time, + * the output of Bro's \c -NN option will only reflect that state at + * startup time; hence usually one should call this for a plugin's + * hooks in either the plugin's ctor or in InitPreScript(). For + * consistency with other parts of the API, there's a macro + * PLUGIN_ENABLE_HOOK for use inside the ctor. + * + * @param hook The hook to enable. + * + * @param priority If multiple plugins enable the same hook, their + * priorities determine the order in which they'll be executed, from + * highest to lowest. If two plugins specify the same priority, order + * is undefined. */ - int Priority() const; + void EnableHook(HookType hook, int priority = 0); /** - * Callback for executing a function/event/hook. Whenever the script - * interpreter is about to execution a function, it first gives all - * InterpreterPlugins a chance to handle the call (in the order of their - * priorities). A plugin can either just inspect the call, or replace it - * (i.e., prevent the interpreter from executing it). In the latter case - * it must provide a matching return value. + * Disables a hook. Bro will no longer call the corresponding virtual + * method. + * + * @param hook The hook to disable. + */ + void DisableHook(HookType hook); + + /** + * Returns a list of hooks that are currently enabled for the plugin, + * along with their priorities. + */ + hook_list EnabledHooks() const; + + /** + * Virtual method that can be overriden by derived class to provide + * information about further script-level elements that the plugin + * provides on its own, i.e., outside of the standard mechanism + * processing *.bif files automatically. The returned information is + * for informational purposes only and will show up in the result of + * BifItems() as well as in the Describe() output. + * + * \todo Do we need both this an AddBifItem()? + */ + virtual bif_item_list CustomBifItems() const; + + // Hook functions. + + /** + * Hook into loading input files. This method will be called between + * InitPreScript() and InitPostScript(), but with no further order or + * timing guaranteed. It will be called once for each input file Bro + * is about to load, either given on the command line or via @load + * script directives. The hook can take over the file, in which case + * Bro not further process it otherwise. + * + * @return 1 if the plugin took over the file and loaded it + * successfully; 0 if the plugin took over the file but had trouble + * loading it (Bro will abort in this case, the plugin should have + * printed an error message); and -1 if the plugin wasn't interested + * in the file at all. + */ + virtual int HookLoadFile(const char* file); + + /** + * Hook into executing a script-level function/event/hook. Whenever + * the script interpreter is about to execution a function, it first + * gives all plugins with this hook enabled a chance to handle the + * call (in the order of their priorities). A plugin can either just + * inspect the call, or replace it (i.e., prevent the interpreter + * from executing it). In the latter case it must provide a matching + * return value. * * The default implementation does never handle the call in any way. - * + * * @param func The function being called. * * @param args The function arguments. The method can modify the list @@ -412,55 +349,133 @@ public: * TYPE_ANY). If the plugin did not handle the call, it must return * null. */ - virtual Val* CallFunction(const Func* func, val_list* args); - - /** - * Callback for raising an event. Whenever the script interpreter is - * about to queue an event for later execution, it first gives all - * InterpreterPlugins a chance to handle the queuing otherwise (in the - * order of their priorities). A plugin can either just inspect the - * event, or take it over (i.e., prevent the interpreter from queuing it - * it). - * - * The default implementation does never handle the queuing in any way. - * - * @param event The even to be queued. The method can modify it in in - * place long as it ensures matching types and correct reference - * counting. - * - * @return True if the plugin took charge of the event; in that case it - * must have assumed ownership of the event and the intpreter will not do - * anything further with it. False otherwise. - * - */ - virtual bool QueueEvent(Event* event); + virtual Val* HookCallFunction(const Func* func, val_list* args); /** - * Callback for updates in network time. This method will be called + * Hook into raising events. Whenever the script interpreter is about + * to queue an event for later execution, it first gives all plugins + * with this hook enabled a chance to handle the queuing otherwise + * (in the order of their priorities). A plugin can either just + * inspect the event, or take it over (i.e., prevent the interpreter + * from queuing it it). + * + * The default implementation does never handle the queuing in any + * way. + * + * @param event The even to be queued. The method can modify it in in + * place long as it ensures matching types and correct reference + * counting. + * + * @return True if the plugin took charge of the event; in that case + * it must have assumed ownership of the event and the intpreter will + * not do anything further with it. False otherwise. + */ + virtual bool HookQueueEvent(Event* event); + + /** + * Hook intp event queue draining. This method will be called + * whenever the event manager is draining its queue. + */ + virtual void HookDrainEvents(); + + /** + * Hook for updates to network time. This method will be called * whenever network time is advanced. * * @param networkt_time The new network time. */ - virtual void UpdateNetworkTime(double network_time); + virtual void HookUpdateNetworkTime(double network_time); + + // Methods that are used internally primarily. /** - * Callback for event queue draining. This method will be called - * whenever the event manager has drained it queue. + * Sets the plugins name. + * + * This is used primarily internally; plugin code should pass the + * name via the BRO_PLUGIN_BEGIN macro instead. + * + * @param name The name. Makes a copy internally. */ - virtual void DrainEvents(); + void SetName(const char* name); /** - * Disables interpreter hooking. The functionality of the Plugin base - * class remains available. + * Sets the plugin's textual description. + * + * This is used primarily internally; plugin code should pass the + * name via the BRO_PLUGIN_DESCRIPTION macro instead. + * + * @param name The description. Makes a copy internally. */ - void DisableInterpreterPlugin() const; + void SetDescription(const char* descr); - // Overridden from base class. - virtual Type PluginType() const; + /** + * Sets the plugin's version. + * + * This is used primarily internally; plugin code should pass the + * name via the BRO_PLUGIN_VERSION macro instead. + * + * @param version The version. + */ + void SetVersion(int version); + + /** + * Sets the API version the plugin requires. + * BRO_PLUGIN_VERSION_BUILTIN indicates that it's a plugin linked in + * statically. + * + * This is used primarily internally; plugins automatically set + * either API version of the Bro they are compiled dynamically for, + * or BRO_PLUGIN_VERSION_BUILTIN if they are linked in statically. + * + * @param version The version. + */ + void SetAPIVersion(int version); + + /** + * Marks the plugin as statically or dynamically linked. + * + * This is used primarily internally; plugins automatically set this + * based on which way they are compiled. + * + * @param dynamic True if this is a dynamically linked plugin. + */ + void SetDynamicPlugin(bool dynamic); + + /** + * Sets the base directory and shared library path from which the + * plugin was loaded. + * + * This is used primarily internally; plugins will have there + * location set automatically. + * + * @param dir The plugin directory. The functions makes an internal + * copy of string. + * + * @param sopath The full path the shared library loaded. The + * functions makes an internal copy of string. + */ + void SetPluginLocation(const char* dir, const char* sopath); private: - int priority; + /** + * Initializes the BiF items added with AddBifItem(). Internal method + * that will be called by the manager at the right time. + */ + void InitBifs(); + typedef std::list bif_init_func_list; + + const char* name; + const char* description; + const char* base_dir; + const char* sopath; + int version; + int api_version; + bool dynamic; + + component_list components; + bif_item_list bif_items; + bif_init_func_list bif_inits; }; } diff --git a/src/scan.l b/src/scan.l index 4c67a2f75e..599a660fee 100644 --- a/src/scan.l +++ b/src/scan.l @@ -581,7 +581,7 @@ YYLTYPE GetCurrentLocation() static int load_files(const char* orig_file) { - int rc = plugin_mgr->TryLoadFile(orig_file); + int rc = PLUGIN_HOOK_WITH_RESULT(HOOK_LOAD_FILE, HookLoadFile(orig_file), -1); if ( rc == 1 ) return 0; // A plugin took care of it, just skip. diff --git a/src/util.h b/src/util.h index a7d493485d..8526ded03f 100644 --- a/src/util.h +++ b/src/util.h @@ -213,7 +213,7 @@ extern void add_to_bro_path(const std::string& dir); extern const char* bro_magic_path(); extern const char* bro_plugin_path(); -extern const char* bro_prefixes(); +extern std::string bro_prefixes(); std::string dot_canon(std::string path, std::string file, std::string prefix = ""); const char* normalize_path(const char* path); void get_script_subpath(const std::string& full_filename, const char** subpath); From 93d9dde969a9bc3983b4bcb656970453730ae3eb Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 10 Dec 2013 16:26:34 -0800 Subject: [PATCH 008/193] IOSource reorg. A bunch of infrastructure work to move IOSource, IOSourceRegistry (now iosource::Manager) and PktSrc/PktDumper code into iosource/, and over to a plugin structure. Other IOSources aren't touched yet, they are still in src/*. It compiles and does something with a small trace, but that's all I've tested so far. There are quite certainly a number of problems left, as well as various TODOs and cleanup; and nothing's cast in stone yet. Will continue to work on this. --- TODO.iosources | 9 + aux/broccoli | 2 +- aux/broctl | 2 +- src/CMakeLists.txt | 7 +- src/DNS_Mgr.cc | 13 +- src/DNS_Mgr.h | 6 +- src/DebugLogger.cc | 3 +- src/DebugLogger.h | 1 + src/FlowSrc.cc | 15 +- src/FlowSrc.h | 4 +- src/Net.cc | 119 +-- src/Net.h | 22 +- src/PacketSort.cc | 2 +- src/PacketSort.h | 8 +- src/PktSrc.cc | 796 ------------------ src/PktSrc.h | 258 ------ src/RemoteSerializer.cc | 34 +- src/RemoteSerializer.h | 4 +- src/Serializer.cc | 19 +- src/Serializer.h | 4 +- src/Sessions.cc | 24 +- src/Sessions.h | 4 +- src/bro.bif | 73 +- src/iosource/CMakeLists.txt | 23 + src/iosource/Component.cc | 44 + src/iosource/Component.h | 56 ++ src/{ => iosource}/IOSource.h | 76 +- src/{IOSource.cc => iosource/Manager.cc} | 151 +++- src/iosource/Manager.h | 75 ++ src/iosource/pktsrc/CMakeLists.txt | 2 + src/iosource/pktsrc/Component.cc | 130 +++ src/iosource/pktsrc/Component.h | 132 +++ src/iosource/pktsrc/PktDumper.cc | 79 ++ src/iosource/pktsrc/PktDumper.h | 57 ++ src/iosource/pktsrc/PktSrc.cc | 411 +++++++++ src/iosource/pktsrc/PktSrc.h | 140 +++ src/iosource/pktsrc/old-2ndary-code.h | 69 ++ src/{ => iosource/pktsrc/pcap}/BPF_Program.cc | 0 src/{ => iosource/pktsrc/pcap}/BPF_Program.h | 0 src/iosource/pktsrc/pcap/CMakeLists.txt | 8 + src/iosource/pktsrc/pcap/Dumper.cc | 111 +++ src/iosource/pktsrc/pcap/Dumper.h | 40 + src/iosource/pktsrc/pcap/Plugin.cc | 12 + src/iosource/pktsrc/pcap/Source.cc | 343 ++++++++ src/iosource/pktsrc/pcap/Source.h | 60 ++ src/main.cc | 16 +- src/plugin/Component.cc | 12 + src/plugin/Component.h | 7 +- src/plugin/Macros.h | 27 + src/threading/Manager.cc | 8 +- src/threading/Manager.h | 4 +- src/util.cc | 7 +- 52 files changed, 2223 insertions(+), 1306 deletions(-) create mode 100644 TODO.iosources delete mode 100644 src/PktSrc.cc delete mode 100644 src/PktSrc.h create mode 100644 src/iosource/CMakeLists.txt create mode 100644 src/iosource/Component.cc create mode 100644 src/iosource/Component.h rename src/{ => iosource}/IOSource.h (51%) rename src/{IOSource.cc => iosource/Manager.cc} (52%) create mode 100644 src/iosource/Manager.h create mode 100644 src/iosource/pktsrc/CMakeLists.txt create mode 100644 src/iosource/pktsrc/Component.cc create mode 100644 src/iosource/pktsrc/Component.h create mode 100644 src/iosource/pktsrc/PktDumper.cc create mode 100644 src/iosource/pktsrc/PktDumper.h create mode 100644 src/iosource/pktsrc/PktSrc.cc create mode 100644 src/iosource/pktsrc/PktSrc.h create mode 100644 src/iosource/pktsrc/old-2ndary-code.h rename src/{ => iosource/pktsrc/pcap}/BPF_Program.cc (100%) rename src/{ => iosource/pktsrc/pcap}/BPF_Program.h (100%) create mode 100644 src/iosource/pktsrc/pcap/CMakeLists.txt create mode 100644 src/iosource/pktsrc/pcap/Dumper.cc create mode 100644 src/iosource/pktsrc/pcap/Dumper.h create mode 100644 src/iosource/pktsrc/pcap/Plugin.cc create mode 100644 src/iosource/pktsrc/pcap/Source.cc create mode 100644 src/iosource/pktsrc/pcap/Source.h diff --git a/TODO.iosources b/TODO.iosources new file mode 100644 index 0000000000..7380c89b92 --- /dev/null +++ b/TODO.iosources @@ -0,0 +1,9 @@ +- Move the current_{iosrc,pkt_src,etc.} into manager +- Remove all 2ndary path code +- Remove all flow src code. +- Move pktsrc/*.{h,cc} up a level? Or create a subsublibrary there? +- Create a global Packet data structure and pass that around instead + of the pcap_* stuff? +- PktDumper: Move Dump() to public and remove Record() +- Wrap BPF_Program into namespace and clean up +- Tests, in particular the packet dumping needs testing. diff --git a/aux/broccoli b/aux/broccoli index e02ccc0a27..17ec437752 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit e02ccc0a27e64b147f01e4c7deb5b897864d59d5 +Subproject commit 17ec437752837fb4214abfb0a2da49df74668d5d diff --git a/aux/broctl b/aux/broctl index 2e07720b4f..6e01d6972f 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 2e07720b4f129802e07ca99498e2aff4542c737a +Subproject commit 6e01d6972f02d68ee82d05f392d1a00725595b7f diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8e22b504e4..1aede44934 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -151,10 +151,12 @@ list(APPEND BINPAC_OUTPUTS "${BINPAC_OUTPUT_CC}") set(bro_SUBDIR_LIBS CACHE INTERNAL "subdir libraries" FORCE) set(bro_PLUGIN_LIBS CACHE INTERNAL "plugin libraries" FORCE) +add_subdirectory(iosource) add_subdirectory(analyzer) add_subdirectory(file_analysis) add_subdirectory(probabilistic) add_subdirectory(broxygen) +add_subdirectory(iosource) set(bro_SUBDIRS ${bro_SUBDIR_LIBS} @@ -249,7 +251,8 @@ set(bro_SRCS Anon.cc Attr.cc Base64.cc - BPF_Program.cc + BroDoc.cc + BroDocObj.cc Brofiler.cc BroString.cc CCL.cc @@ -281,7 +284,6 @@ set(bro_SRCS Hash.cc ID.cc IntSet.cc - IOSource.cc IP.cc IPAddr.cc List.cc @@ -295,7 +297,6 @@ set(bro_SRCS PacketFilter.cc PacketSort.cc PersistenceSerializer.cc - PktSrc.cc PolicyFile.cc PrefixTable.cc PriorityQueue.cc diff --git a/src/DNS_Mgr.cc b/src/DNS_Mgr.cc index 17409a930b..4d96e21a3e 100644 --- a/src/DNS_Mgr.cc +++ b/src/DNS_Mgr.cc @@ -34,6 +34,7 @@ #include "Net.h" #include "Var.h" #include "Reporter.h" +#include "iosource/Manager.h" extern "C" { extern int select(int, fd_set *, fd_set *, fd_set *, struct timeval *); @@ -404,17 +405,17 @@ DNS_Mgr::~DNS_Mgr() delete [] dir; } -bool DNS_Mgr::Init() +void DNS_Mgr::Init() { if ( did_init ) - return true; + return; const char* cache_dir = dir ? dir : "."; if ( mode == DNS_PRIME && ! ensure_dir(cache_dir) ) { did_init = 0; - return false; + return; } cache_name = new char[strlen(cache_dir) + 64]; @@ -433,14 +434,12 @@ bool DNS_Mgr::Init() did_init = 1; - io_sources.Register(this, true); + iosource_mgr->Register(this, true); // We never set idle to false, having the main loop only calling us from // time to time. If we're issuing more DNS requests than we can handle // in this way, we are having problems anyway ... - idle = true; - - return true; + SetIdle(true); } TableVal* DNS_Mgr::LookupHost(const char* name) diff --git a/src/DNS_Mgr.h b/src/DNS_Mgr.h index bfcc70a5c2..069c7e1a2b 100644 --- a/src/DNS_Mgr.h +++ b/src/DNS_Mgr.h @@ -12,7 +12,7 @@ #include "BroList.h" #include "Dict.h" #include "EventHandler.h" -#include "IOSource.h" +#include "iosource/IOSource.h" #include "IPAddr.h" class Val; @@ -40,12 +40,12 @@ enum DNS_MgrMode { // Number of seconds we'll wait for a reply. #define DNS_TIMEOUT 5 -class DNS_Mgr : public IOSource { +class DNS_Mgr : public iosource::IOSource { public: DNS_Mgr(DNS_MgrMode mode); virtual ~DNS_Mgr(); - bool Init(); + void Init(); void Flush(); // Looks up the address or addresses of the given host, and returns diff --git a/src/DebugLogger.cc b/src/DebugLogger.cc index 78377eafcf..05f0f9e89f 100644 --- a/src/DebugLogger.cc +++ b/src/DebugLogger.cc @@ -17,7 +17,8 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = { { "dpd", 0, false }, { "tm", 0, false }, { "logging", 0, false }, {"input", 0, false }, { "threading", 0, false }, { "file_analysis", 0, false }, - { "plugins", 0, false }, { "broxygen", 0, false } + { "plugins", 0, false }, { "broxygen", 0, false }, + { "pktio", 0, false} }; DebugLogger::DebugLogger(const char* filename) diff --git a/src/DebugLogger.h b/src/DebugLogger.h index d1f053788e..b098430a9a 100644 --- a/src/DebugLogger.h +++ b/src/DebugLogger.h @@ -29,6 +29,7 @@ enum DebugStream { DBG_FILE_ANALYSIS, // File analysis DBG_PLUGINS, DBG_BROXYGEN, + DBG_PKTIO, // Packet sources and dumpers. NUM_DBGS // Has to be last }; diff --git a/src/FlowSrc.cc b/src/FlowSrc.cc index 32aa4c4e3a..fb9676ab5b 100644 --- a/src/FlowSrc.cc +++ b/src/FlowSrc.cc @@ -15,7 +15,6 @@ FlowSrc::FlowSrc() { // TODO: v9. selectable_fd = -1; - idle = false; data = 0; pdu_len = -1; exporter_ip = 0; @@ -80,7 +79,7 @@ int FlowSocketSrc::ExtractNextPDU() reporter->Error("problem reading NetFlow data from socket"); data = 0; next_timestamp = -1.0; - closed = 1; + SetClosed(true); return 0; } @@ -115,7 +114,7 @@ FlowSocketSrc::FlowSocketSrc(const char* listen_parms) snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE, "parsing your listen-spec went nuts: laddr='%s', port='%s'\n", laddr[0] ? laddr : "", port[0] ? port : ""); - closed = 1; + SetClosed(true); return; } @@ -131,7 +130,7 @@ FlowSocketSrc::FlowSocketSrc(const char* listen_parms) snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE, "getaddrinfo(%s, %s, ...): %s", laddr, port, gai_strerror(ret)); - closed = 1; + SetClosed(true); return; } @@ -139,7 +138,7 @@ FlowSocketSrc::FlowSocketSrc(const char* listen_parms) { snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE, "socket: %s", strerror(errno)); - closed = 1; + SetClosed(true); goto cleanup; } @@ -147,7 +146,7 @@ FlowSocketSrc::FlowSocketSrc(const char* listen_parms) { snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE, "bind: %s", strerror(errno)); - closed = 1; + SetClosed(true); goto cleanup; } @@ -211,7 +210,7 @@ FlowFileSrc::FlowFileSrc(const char* readfile) selectable_fd = open(this->readfile, O_RDONLY); if ( selectable_fd < 0 ) { - closed = 1; + SetClosed(true); snprintf(errbuf, BRO_FLOW_ERRBUF_SIZE, "open: %s", strerror(errno)); } @@ -223,6 +222,6 @@ int FlowFileSrc::Error(int errlvl, const char* errmsg) "%s: %s", errmsg, strerror(errlvl)); data = 0; next_timestamp = -1.0; - closed = 1; + SetClosed(true); return 0; } diff --git a/src/FlowSrc.h b/src/FlowSrc.h index 03dda2761d..71c8b0cd11 100644 --- a/src/FlowSrc.h +++ b/src/FlowSrc.h @@ -5,7 +5,7 @@ #ifndef flowsrc_h #define flowsrc_h -#include "IOSource.h" +#include "iosource/IOSource.h" #include "NetVar.h" #include "binpac.h" @@ -28,7 +28,7 @@ namespace binpac { } } -class FlowSrc : public IOSource { +class FlowSrc : public iosource::IOSource { public: virtual ~FlowSrc(); diff --git a/src/Net.cc b/src/Net.cc index ac4dacf9b8..64baeff4aa 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -30,6 +30,9 @@ #include "PacketSort.h" #include "Serializer.h" #include "PacketDumper.h" +#include "iosource/Manager.h" +#include "iosource/pktsrc/PktSrc.h" +#include "iosource/pktsrc/PktDumper.h" extern "C" { #include "setsignal.h" @@ -39,10 +42,7 @@ extern "C" { extern int select(int, fd_set *, fd_set *, fd_set *, struct timeval *); } -PList(PktSrc) pkt_srcs; - -// FIXME: We should really merge PktDumper and PacketDumper. -PktDumper* pkt_dumper = 0; +iosource::PktDumper* pkt_dumper = 0; int reading_live = 0; int reading_traces = 0; @@ -65,8 +65,8 @@ const u_char* current_pkt = 0; int current_dispatched = 0; int current_hdr_size = 0; double current_timestamp = 0.0; -PktSrc* current_pktsrc = 0; -IOSource* current_iosrc; +iosource::PktSrc* current_pktsrc = 0; +iosource::IOSource* current_iosrc = 0; std::list files_scanned; std::vector sig_files; @@ -115,8 +115,8 @@ RETSIGTYPE watchdog(int /* signo */) // saving the packet which caused the // watchdog to trigger may be helpful, // so we'll save that one nevertheless. - pkt_dumper = new PktDumper("watchdog-pkt.pcap"); - if ( pkt_dumper->IsError() ) + pkt_dumper = iosource_mgr->OpenPktDumper("watchdog-pkt.pcap", false); + if ( ! pkt_dumper || pkt_dumper->IsError() ) { reporter->Error("watchdog: can't open watchdog-pkt.pcap for writing\n"); delete pkt_dumper; @@ -125,7 +125,12 @@ RETSIGTYPE watchdog(int /* signo */) } if ( pkt_dumper ) - pkt_dumper->Dump(current_hdr, current_pkt); + { + iosource::PktDumper::Packet p; + p.hdr = current_hdr; + p.data = current_pkt; + pkt_dumper->Record(&p); + } } net_get_final_stats(); @@ -157,18 +162,10 @@ void net_init(name_list& interfaces, name_list& readfiles, reading_traces = 1; for ( int i = 0; i < readfiles.length(); ++i ) - { - PktFileSrc* ps = new PktFileSrc(readfiles[i], filter); - - if ( ! ps->IsOpen() ) - reporter->FatalError("%s: problem with trace file %s - %s\n", - prog, readfiles[i], ps->ErrorMsg()); - else - { - pkt_srcs.append(ps); - io_sources.Register(ps); - } + iosource_mgr->OpenPktSrc(readfiles[i], filter, false); + } +#if 0 if ( secondary_filter ) { // We use a second PktFileSrc for the @@ -189,7 +186,6 @@ void net_init(name_list& interfaces, name_list& readfiles, ps->AddSecondaryTablePrograms(); } - } for ( int i = 0; i < flowfiles.length(); ++i ) { @@ -203,7 +199,7 @@ void net_init(name_list& interfaces, name_list& readfiles, io_sources.Register(fs); } } - } +#endif else if ((interfaces.length() > 0 || netflows.length() > 0)) { @@ -211,22 +207,13 @@ void net_init(name_list& interfaces, name_list& readfiles, reading_traces = 0; for ( int i = 0; i < interfaces.length(); ++i ) - { - PktSrc* ps; - ps = new PktInterfaceSrc(interfaces[i], filter); - - if ( ! ps->IsOpen() ) - reporter->FatalError("%s: problem with interface %s - %s\n", - prog, interfaces[i], ps->ErrorMsg()); - else - { - pkt_srcs.append(ps); - io_sources.Register(ps); - } + iosource_mgr->OpenPktSrc(interfaces[i], filter, true); + } +#if 0 if ( secondary_filter ) { - PktSrc* ps; + iosource::PktSrc* ps; ps = new PktInterfaceSrc(interfaces[i], filter, TYPE_FILTER_SECONDARY); @@ -258,8 +245,7 @@ void net_init(name_list& interfaces, name_list& readfiles, else io_sources.Register(fs); } - - } +#endif else // have_pending_timers = 1, possibly. We don't set @@ -270,12 +256,7 @@ void net_init(name_list& interfaces, name_list& readfiles, if ( writefile ) { - // ### This will fail horribly if there are multiple - // interfaces with different-lengthed media. - pkt_dumper = new PktDumper(writefile); - if ( pkt_dumper->IsError() ) - reporter->FatalError("%s: can't open write file \"%s\" - %s\n", - prog, writefile, pkt_dumper->ErrorMsg()); + pkt_dumper = iosource_mgr->OpenPktDumper(writefile, false); ID* id = global_scope()->Lookup("trace_output_file"); if ( ! id ) @@ -299,7 +280,7 @@ void net_init(name_list& interfaces, name_list& readfiles, } } -void expire_timers(PktSrc* src_ps) +void expire_timers(iosource::PktSrc* src_ps) { SegmentProfiler(segment_logger, "expiring-timers"); TimerMgr* tmgr = @@ -313,7 +294,7 @@ void expire_timers(PktSrc* src_ps) void net_packet_dispatch(double t, const struct pcap_pkthdr* hdr, const u_char* pkt, int hdr_size, - PktSrc* src_ps, PacketSortElement* pkt_elem) + iosource::PktSrc* src_ps, PacketSortElement* pkt_elem) { if ( ! bro_start_network_time ) bro_start_network_time = t; @@ -394,7 +375,7 @@ int process_packet_sorter(double latest_packet_time) void net_packet_arrival(double t, const struct pcap_pkthdr* hdr, const u_char* pkt, int hdr_size, - PktSrc* src_ps) + iosource::PktSrc* src_ps) { if ( packet_sorter ) { @@ -421,12 +402,12 @@ void net_run() { set_processing_status("RUNNING", "net_run"); - while ( io_sources.Size() || + while ( iosource_mgr->Size() || (packet_sorter && ! packet_sorter->Empty()) || (BifConst::exit_only_after_terminate && ! terminating) ) { double ts; - IOSource* src = io_sources.FindSoonest(&ts); + iosource::IOSource* src = iosource_mgr->FindSoonest(&ts); #ifdef DEBUG static int loop_counter = 0; @@ -535,16 +516,19 @@ void net_run() void net_get_final_stats() { - loop_over_list(pkt_srcs, i) + const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); + + for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); + i != pkt_srcs.end(); i++ ) { - PktSrc* ps = pkt_srcs[i]; + iosource::PktSrc* ps = *i; if ( ps->IsLive() ) { - struct PktSrc::Stats s; + iosource::PktSrc::Stats s; ps->Statistics(&s); reporter->Info("%d packets received on interface %s, %d dropped\n", - s.received, ps->Interface(), s.dropped); + s.received, ps->Path().c_str(), s.dropped); } } } @@ -587,29 +571,6 @@ void net_delete() delete ip_anonymizer[i]; } -// net_packet_match -// -// Description: -// - Checks if a packet matches a filter. It just wraps up a call to -// [pcap.h's] bpf_filter(). -// -// Inputs: -// - fp: a BPF-compiled filter -// - pkt: a pointer to the packet -// - len: the original packet length -// - caplen: the captured packet length. This is pkt length -// -// Output: -// - return: 1 if the packet matches the filter, 0 otherwise - -int net_packet_match(BPF_Program* fp, const u_char* pkt, - u_int len, u_int caplen) - { - // NOTE: I don't like too much un-const'ing the pkt variable. - return bpf_filter(fp->GetProgram()->bf_insns, (u_char*) pkt, len, caplen); - } - - int _processing_suspended = 0; static double suspend_start = 0; @@ -627,8 +588,12 @@ void net_continue_processing() if ( _processing_suspended == 1 ) { reporter->Info("processing continued"); - loop_over_list(pkt_srcs, i) - pkt_srcs[i]->ContinueAfterSuspend(); + + const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); + + for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); + i != pkt_srcs.end(); i++ ) + (*i)->ContinueAfterSuspend(); } --_processing_suspended; diff --git a/src/Net.h b/src/Net.h index 07c856d1dd..421bee5911 100644 --- a/src/Net.h +++ b/src/Net.h @@ -4,13 +4,18 @@ #define net_h #include "net_util.h" +<<<<<<< HEAD #include "util.h" #include "BPF_Program.h" +======= +>>>>>>> 5493253... Checkpoint. #include "List.h" -#include "PktSrc.h" #include "FlowSrc.h" #include "Func.h" #include "RemoteSerializer.h" +#include "iosource/IOSource.h" +#include "iosource/pktsrc/PktSrc.h" +#include "iosource/pktsrc/PktDumper.h" extern void net_init(name_list& interfaces, name_list& readfiles, name_list& netflows, name_list& flowfiles, @@ -22,10 +27,8 @@ extern void net_finish(int drain_events); extern void net_delete(); // Reclaim all memory, etc. extern void net_packet_arrival(double t, const struct pcap_pkthdr* hdr, const u_char* pkt, int hdr_size, - PktSrc* src_ps); -extern int net_packet_match(BPF_Program* fp, const u_char* pkt, - u_int len, u_int caplen); -extern void expire_timers(PktSrc* src_ps = 0); + iosource::PktSrc* src_ps); +extern void expire_timers(iosource::PktSrc* src_ps = 0); extern void termination_signal(); // Functions to temporarily suspend processing of live input (network packets @@ -82,13 +85,10 @@ extern const u_char* current_pkt; extern int current_dispatched; extern int current_hdr_size; extern double current_timestamp; -extern PktSrc* current_pktsrc; -extern IOSource* current_iosrc; +extern iosource::PktSrc* current_pktsrc; +extern iosource::IOSource* current_iosrc; -declare(PList,PktSrc); -extern PList(PktSrc) pkt_srcs; - -extern PktDumper* pkt_dumper; // where to save packets +extern iosource::PktDumper* pkt_dumper; // where to save packets extern char* writefile; diff --git a/src/PacketSort.cc b/src/PacketSort.cc index 429d8e2720..606d21b689 100644 --- a/src/PacketSort.cc +++ b/src/PacketSort.cc @@ -3,7 +3,7 @@ const bool DEBUG_packetsort = false; -PacketSortElement::PacketSortElement(PktSrc* arg_src, +PacketSortElement::PacketSortElement(iosource::PktSrc* arg_src, double arg_timestamp, const struct pcap_pkthdr* arg_hdr, const u_char* arg_pkt, int arg_hdr_size) { diff --git a/src/PacketSort.h b/src/PacketSort.h index 199da0732f..d61f66994e 100644 --- a/src/PacketSort.h +++ b/src/PacketSort.h @@ -16,16 +16,14 @@ enum { NUM_OF_PQ_LEVEL, }; -class PktSrc; - class PacketSortElement { public: - PacketSortElement(PktSrc* src, double timestamp, + PacketSortElement(iosource::PktSrc* src, double timestamp, const struct pcap_pkthdr* hdr, const u_char* pkt, int hdr_size); ~PacketSortElement(); - PktSrc* Src() const { return src; } + iosource::PktSrc* Src() const { return src; } double TimeStamp() const { return timestamp; } const struct pcap_pkthdr* Hdr() const { return &hdr; } const u_char* Pkt() const { return pkt; } @@ -33,7 +31,7 @@ public: const IP_Hdr* IPHdr() const { return ip_hdr; } protected: - PktSrc* src; + iosource::PktSrc* src; double timestamp; struct pcap_pkthdr hdr; u_char* pkt; diff --git a/src/PktSrc.cc b/src/PktSrc.cc deleted file mode 100644 index 9d6bce6fe9..0000000000 --- a/src/PktSrc.cc +++ /dev/null @@ -1,796 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#include -#include - -#include "config.h" - -#include "util.h" -#include "PktSrc.h" -#include "Hash.h" -#include "Net.h" -#include "Sessions.h" - - -// ### This needs auto-confing. -#ifdef HAVE_PCAP_INT_H -#include -#endif - -PktSrc::PktSrc() - { - interface = readfile = 0; - data = last_data = 0; - memset(&hdr, 0, sizeof(hdr)); - hdr_size = 0; - datalink = 0; - netmask = 0xffffff00; - pd = 0; - idle = false; - - next_sync_point = 0; - first_timestamp = current_timestamp = next_timestamp = 0.0; - first_wallclock = current_wallclock = 0; - - stats.received = stats.dropped = stats.link = 0; - } - -PktSrc::~PktSrc() - { - Close(); - - loop_over_list(program_list, i) - delete program_list[i]; - - BPF_Program* code; - IterCookie* cookie = filters.InitForIteration(); - while ( (code = filters.NextEntry(cookie)) ) - delete code; - - delete [] interface; - delete [] readfile; - } - -void PktSrc::GetFds(int* read, int* write, int* except) - { - if ( pseudo_realtime ) - { - // Select would give erroneous results. But we simulate it - // by setting idle accordingly. - idle = CheckPseudoTime() == 0; - return; - } - - if ( selectable_fd >= 0 ) - *read = selectable_fd; - } - -int PktSrc::ExtractNextPacket() - { - // Don't return any packets if processing is suspended (except for the - // very first packet which we need to set up times). - if ( net_is_processing_suspended() && first_timestamp ) - { - idle = true; - return 0; - } - - data = last_data = pcap_next(pd, &hdr); - - if ( data && (hdr.len == 0 || hdr.caplen == 0) ) - { - sessions->Weird("empty_pcap_header", &hdr, data); - return 0; - } - - if ( data ) - next_timestamp = hdr.ts.tv_sec + double(hdr.ts.tv_usec) / 1e6; - - if ( pseudo_realtime ) - current_wallclock = current_time(true); - - if ( ! first_timestamp ) - first_timestamp = next_timestamp; - - idle = (data == 0); - - if ( data ) - ++stats.received; - - // Source has gone dry. If it's a network interface, this just means - // it's timed out. If it's a file, though, then the file has been - // exhausted. - if ( ! data && ! IsLive() ) - { - closed = true; - - if ( pseudo_realtime && using_communication ) - { - if ( remote_trace_sync_interval ) - remote_serializer->SendFinalSyncPoint(); - else - remote_serializer->Terminate(); - } - } - - return data != 0; - } - -double PktSrc::NextTimestamp(double* local_network_time) - { - if ( ! data && ! ExtractNextPacket() ) - return -1.0; - - if ( pseudo_realtime ) - { - // Delay packet if necessary. - double packet_time = CheckPseudoTime(); - if ( packet_time ) - return packet_time; - - idle = true; - return -1.0; - } - - return next_timestamp; - } - -void PktSrc::ContinueAfterSuspend() - { - current_wallclock = current_time(true); - } - -double PktSrc::CurrentPacketWallClock() - { - // We stop time when we are suspended. - if ( net_is_processing_suspended() ) - current_wallclock = current_time(true); - - return current_wallclock; - } - -double PktSrc::CheckPseudoTime() - { - if ( ! data && ! ExtractNextPacket() ) - return 0; - - if ( ! current_timestamp ) - return bro_start_time; - - if ( remote_trace_sync_interval ) - { - if ( next_sync_point == 0 || next_timestamp >= next_sync_point ) - { - int n = remote_serializer->SendSyncPoint(); - next_sync_point = first_timestamp + - n * remote_trace_sync_interval; - remote_serializer->Log(RemoteSerializer::LogInfo, - fmt("stopping at packet %.6f, next sync-point at %.6f", - current_timestamp, next_sync_point)); - - return 0; - } - } - - double pseudo_time = next_timestamp - first_timestamp; - double ct = (current_time(true) - first_wallclock) * pseudo_realtime; - - return pseudo_time <= ct ? bro_start_time + pseudo_time : 0; - } - -void PktSrc::Process() - { - if ( ! data && ! ExtractNextPacket() ) - return; - - current_timestamp = next_timestamp; - - int pkt_hdr_size = hdr_size; - - // Unfortunately some packets on the link might have MPLS labels - // while others don't. That means we need to ask the link-layer if - // labels are in place. - bool have_mpls = false; - - int protocol = 0; - - switch ( datalink ) { - case DLT_NULL: - { - protocol = (data[3] << 24) + (data[2] << 16) + (data[1] << 8) + data[0]; - - // From the Wireshark Wiki: "AF_INET6, unfortunately, has - // different values in {NetBSD,OpenBSD,BSD/OS}, - // {FreeBSD,DragonFlyBSD}, and {Darwin/Mac OS X}, so an IPv6 - // packet might have a link-layer header with 24, 28, or 30 - // as the AF_ value." As we may be reading traces captured on - // platforms other than what we're running on, we accept them - // all here. - if ( protocol != AF_INET - && protocol != AF_INET6 - && protocol != 24 - && protocol != 28 - && protocol != 30 ) - { - sessions->Weird("non_ip_packet_in_null_transport", &hdr, data); - data = 0; - return; - } - - break; - } - - case DLT_EN10MB: - { - // Get protocol being carried from the ethernet frame. - protocol = (data[12] << 8) + data[13]; - - switch ( protocol ) - { - // MPLS carried over the ethernet frame. - case 0x8847: - have_mpls = true; - break; - - // VLAN carried over the ethernet frame. - case 0x8100: - data += get_link_header_size(datalink); - data += 4; // Skip the vlan header - pkt_hdr_size = 0; - - // Check for 802.1ah (Q-in-Q) containing IP. - // Only do a second layer of vlan tag - // stripping because there is no - // specification that allows for deeper - // nesting. - if ( ((data[2] << 8) + data[3]) == 0x0800 ) - data += 4; - - break; - - // PPPoE carried over the ethernet frame. - case 0x8864: - data += get_link_header_size(datalink); - protocol = (data[6] << 8) + data[7]; - data += 8; // Skip the PPPoE session and PPP header - pkt_hdr_size = 0; - - if ( protocol != 0x0021 && protocol != 0x0057 ) - { - // Neither IPv4 nor IPv6. - sessions->Weird("non_ip_packet_in_pppoe_encapsulation", &hdr, data); - data = 0; - return; - } - break; - } - - break; - } - - case DLT_PPP_SERIAL: - { - // Get PPP protocol. - protocol = (data[2] << 8) + data[3]; - - if ( protocol == 0x0281 ) - // MPLS Unicast - have_mpls = true; - - else if ( protocol != 0x0021 && protocol != 0x0057 ) - { - // Neither IPv4 nor IPv6. - sessions->Weird("non_ip_packet_in_ppp_encapsulation", &hdr, data); - data = 0; - return; - } - break; - } - } - - if ( have_mpls ) - { - // Remove the data link layer - data += get_link_header_size(datalink); - - // Denote a header size of zero before the IP header - pkt_hdr_size = 0; - - // Skip the MPLS label stack. - bool end_of_stack = false; - - while ( ! end_of_stack ) - { - end_of_stack = *(data + 2) & 0x01; - data += 4; - } - } - - if ( pseudo_realtime ) - { - current_pseudo = CheckPseudoTime(); - net_packet_arrival(current_pseudo, &hdr, data, pkt_hdr_size, this); - if ( ! first_wallclock ) - first_wallclock = current_time(true); - } - - else - net_packet_arrival(current_timestamp, &hdr, data, pkt_hdr_size, this); - - data = 0; - } - -bool PktSrc::GetCurrentPacket(const struct pcap_pkthdr** arg_hdr, - const u_char** arg_pkt) - { - if ( ! last_data ) - return false; - - *arg_hdr = &hdr; - *arg_pkt = last_data; - return true; - } - -int PktSrc::PrecompileFilter(int index, const char* filter) - { - // Compile filter. - BPF_Program* code = new BPF_Program(); - - if ( ! code->Compile(pd, filter, netmask, errbuf, sizeof(errbuf)) ) - { - delete code; - return 0; - } - - // Store it in hash. - HashKey* hash = new HashKey(HashKey(bro_int_t(index))); - BPF_Program* oldcode = filters.Lookup(hash); - if ( oldcode ) - delete oldcode; - - filters.Insert(hash, code); - delete hash; - - return 1; - } - -int PktSrc::SetFilter(int index) - { - // We don't want load-level filters for the secondary path. - if ( filter_type == TYPE_FILTER_SECONDARY && index > 0 ) - return 1; - - HashKey* hash = new HashKey(HashKey(bro_int_t(index))); - BPF_Program* code = filters.Lookup(hash); - delete hash; - - if ( ! code ) - { - safe_snprintf(errbuf, sizeof(errbuf), - "No precompiled pcap filter for index %d", - index); - return 0; - } - - if ( pcap_setfilter(pd, code->GetProgram()) < 0 ) - { - safe_snprintf(errbuf, sizeof(errbuf), - "pcap_setfilter(%d): %s", - index, pcap_geterr(pd)); - return 0; - } - -#ifndef HAVE_LINUX - // Linux doesn't clear counters when resetting filter. - stats.received = stats.dropped = stats.link = 0; -#endif - - return 1; - } - -void PktSrc::SetHdrSize() - { - int dl = pcap_datalink(pd); - hdr_size = get_link_header_size(dl); - - if ( hdr_size < 0 ) - { - safe_snprintf(errbuf, sizeof(errbuf), - "unknown data link type 0x%x", dl); - Close(); - } - - datalink = dl; - } - -void PktSrc::Close() - { - if ( pd ) - { - pcap_close(pd); - pd = 0; - closed = true; - } - } - -void PktSrc::AddSecondaryTablePrograms() - { - BPF_Program* program; - - loop_over_list(secondary_path->EventTable(), i) - { - SecondaryEvent* se = secondary_path->EventTable()[i]; - program = new BPF_Program(); - - if ( ! program->Compile(snaplen, datalink, se->Filter(), - netmask, errbuf, sizeof(errbuf)) ) - { - delete program; - Close(); - return; - } - - SecondaryProgram* sp = new SecondaryProgram(program, se); - program_list.append(sp); - } - } - -void PktSrc::Statistics(Stats* s) - { - if ( reading_traces ) - s->received = s->dropped = s->link = 0; - - else - { - struct pcap_stat pstat; - if ( pcap_stats(pd, &pstat) < 0 ) - { - reporter->Error("problem getting packet filter statistics: %s", - ErrorMsg()); - s->received = s->dropped = s->link = 0; - } - - else - { - s->dropped = pstat.ps_drop; - s->link = pstat.ps_recv; - } - } - - s->received = stats.received; - - if ( pseudo_realtime ) - s->dropped = 0; - - stats.dropped = s->dropped; - } - -PktInterfaceSrc::PktInterfaceSrc(const char* arg_interface, const char* filter, - PktSrc_Filter_Type ft) -: PktSrc() - { - char tmp_errbuf[PCAP_ERRBUF_SIZE]; - filter_type = ft; - - // Determine interface if not specified. - if ( ! arg_interface && ! (arg_interface = pcap_lookupdev(tmp_errbuf)) ) - { - safe_snprintf(errbuf, sizeof(errbuf), - "pcap_lookupdev: %s", tmp_errbuf); - return; - } - - interface = copy_string(arg_interface); - - // Determine network and netmask. - uint32 net; - if ( pcap_lookupnet(interface, &net, &netmask, tmp_errbuf) < 0 ) - { - // ### The lookup can fail if no address is assigned to - // the interface; and libpcap doesn't have any useful notion - // of error codes, just error strings - how bogus - so we - // just kludge around the error :-(. - // sprintf(errbuf, "pcap_lookupnet %s", tmp_errbuf); - // return; - net = 0; - netmask = 0xffffff00; - } - - // We use the smallest time-out possible to return almost immediately if - // no packets are available. (We can't use set_nonblocking() as it's - // broken on FreeBSD: even when select() indicates that we can read - // something, we may get nothing if the store buffer hasn't filled up - // yet.) - pd = pcap_open_live(interface, snaplen, 1, 1, tmp_errbuf); - - if ( ! pd ) - { - safe_snprintf(errbuf, sizeof(errbuf), - "pcap_open_live: %s", tmp_errbuf); - closed = true; - return; - } - - // ### This needs autoconf'ing. -#ifdef HAVE_PCAP_INT_H - reporter->Info("pcap bufsize = %d\n", ((struct pcap *) pd)->bufsize); -#endif - -#ifdef HAVE_LINUX - if ( pcap_setnonblock(pd, 1, tmp_errbuf) < 0 ) - { - safe_snprintf(errbuf, sizeof(errbuf), - "pcap_setnonblock: %s", tmp_errbuf); - pcap_close(pd); - closed = true; - return; - } -#endif - selectable_fd = pcap_fileno(pd); - - if ( PrecompileFilter(0, filter) && SetFilter(0) ) - { - SetHdrSize(); - - if ( closed ) - // Couldn't get header size. - return; - - reporter->Info("listening on %s, capture length %d bytes\n", interface, snaplen); - } - else - closed = true; - } - - -PktFileSrc::PktFileSrc(const char* arg_readfile, const char* filter, - PktSrc_Filter_Type ft) -: PktSrc() - { - readfile = copy_string(arg_readfile); - - filter_type = ft; - - pd = pcap_open_offline((char*) readfile, errbuf); - - if ( pd && PrecompileFilter(0, filter) && SetFilter(0) ) - { - SetHdrSize(); - - if ( closed ) - // Unknown link layer type. - return; - - // We don't put file sources into non-blocking mode as - // otherwise we would not be able to identify the EOF. - - selectable_fd = fileno(pcap_file(pd)); - - if ( selectable_fd < 0 ) - reporter->InternalError("OS does not support selectable pcap fd"); - } - else - closed = true; - } - - -SecondaryPath::SecondaryPath() - { - filter = 0; - - // Glue together the secondary filter, if exists. - Val* secondary_fv = internal_val("secondary_filters"); - if ( secondary_fv->AsTableVal()->Size() == 0 ) - return; - - int did_first = 0; - const TableEntryValPDict* v = secondary_fv->AsTable(); - IterCookie* c = v->InitForIteration(); - TableEntryVal* tv; - HashKey* h; - - while ( (tv = v->NextEntry(h, c)) ) - { - // Get the index values. - ListVal* index = - secondary_fv->AsTableVal()->RecoverIndex(h); - - const char* str = - index->Index(0)->Ref()->AsString()->CheckString(); - - if ( ++did_first == 1 ) - { - filter = copy_string(str); - } - else - { - if ( strlen(filter) > 0 ) - { - char* tmp_f = new char[strlen(str) + strlen(filter) + 32]; - if ( strlen(str) == 0 ) - sprintf(tmp_f, "%s", filter); - else - sprintf(tmp_f, "(%s) or (%s)", filter, str); - delete [] filter; - filter = tmp_f; - } - } - - // Build secondary_path event table item and link it. - SecondaryEvent* se = - new SecondaryEvent(index->Index(0)->Ref()->AsString()->CheckString(), - tv->Value()->AsFunc() ); - - event_list.append(se); - - delete h; - Unref(index); - } - } - -SecondaryPath::~SecondaryPath() - { - loop_over_list(event_list, i) - delete event_list[i]; - - delete [] filter; - } - - -SecondaryProgram::~SecondaryProgram() - { - delete program; - } - -PktDumper::PktDumper(const char* arg_filename, bool arg_append) - { - filename[0] = '\0'; - is_error = false; - append = arg_append; - dumper = 0; - open_time = 0.0; - - // We need a pcap_t with a reasonable link-layer type. We try to get it - // from the packet sources. If not available, we fall back to Ethernet. - // FIXME: Perhaps we should make this configurable? - int linktype = -1; - - if ( pkt_srcs.length() ) - linktype = pkt_srcs[0]->LinkType(); - - if ( linktype < 0 ) - linktype = DLT_EN10MB; - - pd = pcap_open_dead(linktype, 8192); - if ( ! pd ) - { - Error("error for pcap_open_dead"); - return; - } - - if ( arg_filename ) - Open(arg_filename); - } - -bool PktDumper::Open(const char* arg_filename) - { - if ( ! arg_filename && ! *filename ) - { - Error("no filename given"); - return false; - } - - if ( arg_filename ) - { - if ( dumper && streq(arg_filename, filename) ) - // Already open. - return true; - - safe_strncpy(filename, arg_filename, FNBUF_LEN); - } - - if ( dumper ) - Close(); - - struct stat s; - int exists = 0; - - if ( append ) - { - // See if output file already exists (and is non-empty). - exists = stat(filename, &s); ; - - if ( exists < 0 && errno != ENOENT ) - { - Error(fmt("can't stat file %s: %s", filename, strerror(errno))); - return false; - } - } - - if ( ! append || exists < 0 || s.st_size == 0 ) - { - // Open new file. - dumper = pcap_dump_open(pd, filename); - if ( ! dumper ) - { - Error(pcap_geterr(pd)); - return false; - } - } - - else - { - // Old file and we need to append, which, unfortunately, - // is not supported by libpcap. So, we have to hack a - // little bit, knowing that pcap_dumpter_t is, in fact, - // a FILE ... :-( - dumper = (pcap_dumper_t*) fopen(filename, "a"); - if ( ! dumper ) - { - Error(fmt("can't open dump %s: %s", filename, strerror(errno))); - return false; - } - } - - open_time = network_time; - is_error = false; - return true; - } - -bool PktDumper::Close() - { - if ( dumper ) - { - pcap_dump_close(dumper); - dumper = 0; - is_error = false; - } - - return true; - } - -bool PktDumper::Dump(const struct pcap_pkthdr* hdr, const u_char* pkt) - { - if ( ! dumper ) - return false; - - if ( ! open_time ) - open_time = network_time; - - pcap_dump((u_char*) dumper, hdr, pkt); - - return true; - } - -void PktDumper::Error(const char* errstr) - { - safe_strncpy(errbuf, errstr, sizeof(errbuf)); - is_error = true; - } - -int get_link_header_size(int dl) - { - switch ( dl ) { - case DLT_NULL: - return 4; - - case DLT_EN10MB: - return 14; - - case DLT_FDDI: - return 13 + 8; // fddi_header + LLC - -#ifdef DLT_LINUX_SLL - case DLT_LINUX_SLL: - return 16; -#endif - - case DLT_PPP_SERIAL: // PPP_SERIAL - return 4; - - case DLT_RAW: - return 0; - } - - return -1; - } diff --git a/src/PktSrc.h b/src/PktSrc.h deleted file mode 100644 index 70eef4dd00..0000000000 --- a/src/PktSrc.h +++ /dev/null @@ -1,258 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#ifndef pktsrc_h -#define pktsrc_h - -#include "Dict.h" -#include "Expr.h" -#include "BPF_Program.h" -#include "IOSource.h" -#include "RemoteSerializer.h" - -#define BRO_PCAP_ERRBUF_SIZE PCAP_ERRBUF_SIZE + 256 - -extern "C" { -#include -} - -declare(PDict,BPF_Program); - -// Whether a PktSrc object is used by the normal filter structure or the -// secondary-path structure. -typedef enum { - TYPE_FILTER_NORMAL, // the normal filter - TYPE_FILTER_SECONDARY, // the secondary-path filter -} PktSrc_Filter_Type; - - -// {filter,event} tuples conforming the secondary path. -class SecondaryEvent { -public: - SecondaryEvent(const char* arg_filter, Func* arg_event) - { - filter = arg_filter; - event = arg_event; - } - - const char* Filter() { return filter; } - Func* Event() { return event; } - -private: - const char* filter; - Func* event; -}; - -declare(PList,SecondaryEvent); -typedef PList(SecondaryEvent) secondary_event_list; - - - -class SecondaryPath { -public: - SecondaryPath(); - ~SecondaryPath(); - - secondary_event_list& EventTable() { return event_list; } - const char* Filter() { return filter; } - -private: - secondary_event_list event_list; - // OR'ed union of all SecondaryEvent filters - char* filter; -}; - -// Main secondary-path object. -extern SecondaryPath* secondary_path; - - -// {program, {filter,event}} tuple table. -class SecondaryProgram { -public: - SecondaryProgram(BPF_Program* arg_program, SecondaryEvent* arg_event) - { - program = arg_program; - event = arg_event; - } - - ~SecondaryProgram(); - - BPF_Program* Program() { return program; } - SecondaryEvent* Event() { return event; } - -private: - // Associated program. - BPF_Program *program; - - // Event that is run in case the program is matched. - SecondaryEvent* event; -}; - -declare(PList,SecondaryProgram); -typedef PList(SecondaryProgram) secondary_program_list; - - - -class PktSrc : public IOSource { -public: - ~PktSrc(); - - // IOSource interface - bool IsReady(); - void GetFds(int* read, int* write, int* except); - double NextTimestamp(double* local_network_time); - void Process(); - const char* Tag() { return "PktSrc"; } - - const char* ErrorMsg() const { return errbuf; } - void ClearErrorMsg() { *errbuf ='\0'; } - - // Returns the packet last processed; false if there is no - // current packet available. - bool GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt); - - int HdrSize() const { return hdr_size; } - int DataLink() const { return datalink; } - - void ConsumePacket() { data = 0; } - - int IsLive() const { return interface != 0; } - - pcap_t* PcapHandle() const { return pd; } - int LinkType() const { return pcap_datalink(pd); } - - const char* ReadFile() const { return readfile; } - const char* Interface() const { return interface; } - PktSrc_Filter_Type FilterType() const { return filter_type; } - void AddSecondaryTablePrograms(); - const secondary_program_list& ProgramTable() const - { return program_list; } - - // Signal packet source that processing was suspended and is now going - // to be continued. - void ContinueAfterSuspend(); - - // Only valid in pseudo-realtime mode. - double CurrentPacketTimestamp() { return current_pseudo; } - double CurrentPacketWallClock(); - - struct Stats { - unsigned int received; // pkts received (w/o drops) - unsigned int dropped; // pkts dropped - unsigned int link; // total packets on link - // (not always not available) - }; - - virtual void Statistics(Stats* stats); - - // Precompiles a filter and associates the given index with it. - // Returns true on success, 0 if a problem occurred. - virtual int PrecompileFilter(int index, const char* filter); - - // Activates the filter with the given index. - // Returns true on success, 0 if a problem occurred. - virtual int SetFilter(int index); - -protected: - PktSrc(); - - static const int PCAP_TIMEOUT = 20; - - void SetHdrSize(); - - virtual void Close(); - - // Returns 1 on success, 0 on time-out/gone dry. - virtual int ExtractNextPacket(); - - // Checks if the current packet has a pseudo-time <= current_time. - // If yes, returns pseudo-time, otherwise 0. - double CheckPseudoTime(); - - double current_timestamp; - double next_timestamp; - - // Only set in pseudo-realtime mode. - double first_timestamp; - double first_wallclock; - double current_wallclock; - double current_pseudo; - - struct pcap_pkthdr hdr; - const u_char* data; // contents of current packet - const u_char* last_data; // same, but unaffected by consuming - int hdr_size; - int datalink; - double next_sync_point; // For trace synchronziation in pseudo-realtime - - char* interface; // nil if not reading from an interface - char* readfile; // nil if not reading from a file - - pcap_t* pd; - int selectable_fd; - uint32 netmask; - char errbuf[BRO_PCAP_ERRBUF_SIZE]; - - Stats stats; - - PDict(BPF_Program) filters; // precompiled filters - - PktSrc_Filter_Type filter_type; // normal path or secondary path - secondary_program_list program_list; -}; - -class PktInterfaceSrc : public PktSrc { -public: - PktInterfaceSrc(const char* interface, const char* filter, - PktSrc_Filter_Type ft=TYPE_FILTER_NORMAL); -}; - -class PktFileSrc : public PktSrc { -public: - PktFileSrc(const char* readfile, const char* filter, - PktSrc_Filter_Type ft=TYPE_FILTER_NORMAL); -}; - - -extern int get_link_header_size(int dl); - -class PktDumper { -public: - PktDumper(const char* file = 0, bool append = false); - ~PktDumper() { Close(); } - - bool Open(const char* file = 0); - bool Close(); - bool Dump(const struct pcap_pkthdr* hdr, const u_char* pkt); - - pcap_dumper_t* PcapDumper() { return dumper; } - - const char* FileName() const { return filename; } - bool IsError() const { return is_error; } - const char* ErrorMsg() const { return errbuf; } - - // This heuristic will horribly fail if we're using packets - // with different link layers. (If we can't derive a reasonable value - // from the packet sources, our fall-back is Ethernet.) - int HdrSize() const - { return get_link_header_size(pcap_datalink(pd)); } - - // Network time when dump file was opened. - double OpenTime() const { return open_time; } - -private: - void InitPd(); - void Error(const char* str); - - static const int FNBUF_LEN = 1024; - char filename[FNBUF_LEN]; - - bool append; - pcap_dumper_t* dumper; - pcap_t* pd; - double open_time; - - bool is_error; - char errbuf[BRO_PCAP_ERRBUF_SIZE]; -}; - -#endif diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index c8cf03667b..b0db8fafe8 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -188,10 +188,11 @@ #include "File.h" #include "Conn.h" #include "Reporter.h" -#include "threading/SerialTypes.h" -#include "logging/Manager.h" #include "IPAddr.h" #include "bro_inet_ntop.h" +#include "threading/SerialTypes.h" +#include "logging/Manager.h" +#include "iosource/Manager.h" extern "C" { #include "setsignal.h" @@ -284,10 +285,10 @@ struct ping_args { \ if ( ! c ) \ { \ - idle = io->IsIdle();\ + SetIdle(io->IsIdle());\ return true; \ } \ - idle = false; \ + SetIdle(false); \ } static const char* msgToStr(int msg) @@ -536,7 +537,6 @@ RemoteSerializer::RemoteSerializer() current_sync_point = 0; syncing_times = false; io = 0; - closed = false; terminating = false; in_sync = 0; last_flush = 0; @@ -574,7 +574,7 @@ void RemoteSerializer::Init() Fork(); - io_sources.Register(this); + iosource_mgr->Register(this); Log(LogInfo, fmt("communication started, parent pid is %d, child pid is %d", getpid(), child_pid)); initialized = 1; @@ -1278,7 +1278,7 @@ bool RemoteSerializer::Listen(const IPAddr& ip, uint16 port, bool expect_ssl, return false; listening = true; - closed = false; + SetClosed(false); return true; } @@ -1347,7 +1347,7 @@ bool RemoteSerializer::StopListening() return false; listening = false; - closed = ! IsActive(); + SetClosed(! IsActive()); return true; } @@ -1385,7 +1385,7 @@ double RemoteSerializer::NextTimestamp(double* local_network_time) if ( received_logs > 0 ) { // If we processed logs last time, assume there's more. - idle = false; + SetIdle(false); received_logs = 0; return timer_mgr->Time(); } @@ -1400,7 +1400,7 @@ double RemoteSerializer::NextTimestamp(double* local_network_time) pt = timer_mgr->Time(); if ( packets.length() ) - idle = false; + SetIdle(false); if ( et >= 0 && (et < pt || pt < 0) ) return et; @@ -1479,7 +1479,7 @@ void RemoteSerializer::Process() } if ( packets.length() ) - idle = false; + SetIdle(false); } void RemoteSerializer::Finish() @@ -1511,7 +1511,7 @@ bool RemoteSerializer::Poll(bool may_block) } io->Flush(); - idle = false; + SetIdle(false); switch ( msgstate ) { case TYPE: @@ -1695,7 +1695,7 @@ bool RemoteSerializer::DoMessage() case MSG_TERMINATE: assert(terminating); - io_sources.Terminate(); + iosource_mgr->Terminate(); return true; case MSG_REMOTE_PRINT: @@ -1885,7 +1885,7 @@ void RemoteSerializer::RemovePeer(Peer* peer) delete peer->cache_out; delete peer; - closed = ! IsActive(); + SetClosed(! IsActive()); if ( in_sync == peer ) in_sync = 0; @@ -2850,7 +2850,7 @@ void RemoteSerializer::GotEvent(const char* name, double time, BufferedEvent* e = new BufferedEvent; // Our time, not the time when the event was generated. - e->time = pkt_srcs.length() ? + e->time = iosource_mgr->GetPktSrcs().size() ? time_t(network_time) : time_t(timer_mgr->Time()); e->src = current_peer->id; @@ -3094,7 +3094,7 @@ RecordVal* RemoteSerializer::GetPeerVal(PeerID id) void RemoteSerializer::ChildDied() { Log(LogError, "child died"); - closed = true; + SetClosed(true); child_pid = 0; // Shut down the main process as well. @@ -3188,7 +3188,7 @@ void RemoteSerializer::FatalError(const char* msg) Log(LogError, msg); reporter->Error("%s", msg); - closed = true; + SetClosed(true); if ( kill(child_pid, SIGQUIT) < 0 ) reporter->Warning("warning: cannot kill child pid %d, %s", child_pid, strerror(errno)); diff --git a/src/RemoteSerializer.h b/src/RemoteSerializer.h index 5ff7fff8d6..f8b306f002 100644 --- a/src/RemoteSerializer.h +++ b/src/RemoteSerializer.h @@ -6,7 +6,7 @@ #include "Dict.h" #include "List.h" #include "Serializer.h" -#include "IOSource.h" +#include "iosource/IOSource.h" #include "Stats.h" #include "File.h" #include "logging/WriterBackend.h" @@ -22,7 +22,7 @@ namespace threading { } // This class handles the communication done in Bro's main loop. -class RemoteSerializer : public Serializer, public IOSource { +class RemoteSerializer : public Serializer, public iosource::IOSource { public: RemoteSerializer(); virtual ~RemoteSerializer(); diff --git a/src/Serializer.cc b/src/Serializer.cc index 156ad67f2e..74740497a1 100644 --- a/src/Serializer.cc +++ b/src/Serializer.cc @@ -19,6 +19,7 @@ #include "Conn.h" #include "Timer.h" #include "RemoteSerializer.h" +#include "iosource/Manager.h" Serializer::Serializer(SerializationFormat* arg_format) { @@ -1045,7 +1046,7 @@ EventPlayer::EventPlayer(const char* file) Error(fmt("event replayer: cannot open %s", file)); if ( ReadHeader() ) - io_sources.Register(this); + iosource_mgr->Register(this); } EventPlayer::~EventPlayer() @@ -1085,7 +1086,7 @@ double EventPlayer::NextTimestamp(double* local_network_time) { UnserialInfo info(this); Unserialize(&info); - closed = io->Eof(); + SetClosed(io->Eof()); } if ( ! ne_time ) @@ -1142,7 +1143,7 @@ bool Packet::Serialize(SerialInfo* info) const static BroFile* profiling_output = 0; #ifdef DEBUG -static PktDumper* dump = 0; +static iosource::PktDumper* dump = 0; #endif Packet* Packet::Unserialize(UnserialInfo* info) @@ -1188,7 +1189,7 @@ Packet* Packet::Unserialize(UnserialInfo* info) p->hdr = hdr; p->pkt = (u_char*) pkt; p->tag = tag; - p->hdr_size = get_link_header_size(p->link_type); + p->hdr_size = iosource::PktSrc::GetLinkHeaderSize(p->link_type); delete [] tag; @@ -1213,9 +1214,15 @@ Packet* Packet::Unserialize(UnserialInfo* info) if ( debug_logger.IsEnabled(DBG_TM) ) { if ( ! dump ) - dump = new PktDumper("tm.pcap"); + dump = iosource_mgr->OpenPktDumper("tm.pcap", true); - dump->Dump(p->hdr, p->pkt); + if ( dump ) + { + iosource::PktDumper::Packet dp; + dp.hdr = p->hdr; + dp.data = p->pkt; + dump->Record(&dp); + } } #endif diff --git a/src/Serializer.h b/src/Serializer.h index af4878ccf5..3be2da5134 100644 --- a/src/Serializer.h +++ b/src/Serializer.h @@ -15,7 +15,7 @@ #include "SerialInfo.h" #include "IP.h" #include "Timer.h" -#include "IOSource.h" +#include "iosource/IOSource.h" #include "Reporter.h" class SerializationCache; @@ -350,7 +350,7 @@ public: }; // Plays a file of events back. -class EventPlayer : public FileSerializer, public IOSource { +class EventPlayer : public FileSerializer, public iosource::IOSource { public: EventPlayer(const char* file); virtual ~EventPlayer(); diff --git a/src/Sessions.cc b/src/Sessions.cc index acc306d277..c84c677db4 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -168,7 +168,7 @@ void NetSessions::Done() void NetSessions::DispatchPacket(double t, const struct pcap_pkthdr* hdr, const u_char* pkt, int hdr_size, - PktSrc* src_ps, PacketSortElement* pkt_elem) + iosource::PktSrc* src_ps, PacketSortElement* pkt_elem) { const struct ip* ip_hdr = 0; const u_char* ip_data = 0; @@ -185,10 +185,14 @@ void NetSessions::DispatchPacket(double t, const struct pcap_pkthdr* hdr, // Blanket encapsulation hdr_size += encap_hdr_size; +#if 0 if ( src_ps->FilterType() == TYPE_FILTER_NORMAL ) NextPacket(t, hdr, pkt, hdr_size, pkt_elem); else NextPacketSecondary(t, hdr, pkt, hdr_size, src_ps); +#else + NextPacket(t, hdr, pkt, hdr_size, pkt_elem); +#endif } void NetSessions::NextPacket(double t, const struct pcap_pkthdr* hdr, @@ -278,7 +282,7 @@ void NetSessions::NextPacket(double t, const struct pcap_pkthdr* hdr, void NetSessions::NextPacketSecondary(double /* t */, const struct pcap_pkthdr* hdr, const u_char* const pkt, int hdr_size, - const PktSrc* src_ps) + const iosource::PktSrc* src_ps) { SegmentProfiler(segment_logger, "processing-secondary-packet"); @@ -291,6 +295,7 @@ void NetSessions::NextPacketSecondary(double /* t */, const struct pcap_pkthdr* return; } +#if 0 const struct ip* ip = (const struct ip*) (pkt + hdr_size); if ( ip->ip_v == 4 ) { @@ -321,6 +326,7 @@ void NetSessions::NextPacketSecondary(double /* t */, const struct pcap_pkthdr* delete args; } } +#endif } int NetSessions::CheckConnectionTag(Connection* conn) @@ -1341,14 +1347,24 @@ void NetSessions::DumpPacket(const struct pcap_pkthdr* hdr, return; if ( len == 0 ) - pkt_dumper->Dump(hdr, pkt); + { + iosource::PktDumper::Packet p; + p.hdr = hdr; + p.data = pkt; + pkt_dumper->Record(&p); + } + else { struct pcap_pkthdr h = *hdr; h.caplen = len; if ( h.caplen > hdr->caplen ) reporter->InternalError("bad modified caplen"); - pkt_dumper->Dump(&h, pkt); + + iosource::PktDumper::Packet p; + p.hdr = &h; + p.data = pkt; + pkt_dumper->Record(&p); } } diff --git a/src/Sessions.h b/src/Sessions.h index 1788541f45..4f12bd1240 100644 --- a/src/Sessions.h +++ b/src/Sessions.h @@ -74,7 +74,7 @@ public: // employing the packet sorter first. void DispatchPacket(double t, const struct pcap_pkthdr* hdr, const u_char* const pkt, int hdr_size, - PktSrc* src_ps, PacketSortElement* pkt_elem); + iosource::PktSrc* src_ps, PacketSortElement* pkt_elem); void Done(); // call to drain events before destructing @@ -225,7 +225,7 @@ protected: void NextPacketSecondary(double t, const struct pcap_pkthdr* hdr, const u_char* const pkt, int hdr_size, - const PktSrc* src_ps); + const iosource::PktSrc* src_ps); // Record the given packet (if a dumper is active). If len=0 // then the whole packet is recorded, otherwise just the first diff --git a/src/bro.bif b/src/bro.bif index 24dff3c77c..2b94307143 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -21,6 +21,7 @@ #include "IPAddr.h" #include "util.h" #include "file_analysis/Manager.h" +#include "iosource/Manager.h" using namespace std; @@ -33,7 +34,7 @@ TableType* var_sizes; // and hence it's declared in NetVar.{h,cc}. extern RecordType* gap_info; -static PktDumper* addl_pkt_dumper = 0; +static iosource::PktDumper* addl_pkt_dumper = 0; bro_int_t parse_int(const char*& fmt) { @@ -1657,11 +1658,14 @@ function net_stats%(%): NetStats unsigned int drop = 0; unsigned int link = 0; - loop_over_list(pkt_srcs, i) - { - PktSrc* ps = pkt_srcs[i]; + const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); - struct PktSrc::Stats stat; + for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); + i != pkt_srcs.end(); i++ ) + { + iosource::PktSrc* ps = *i; + + struct iosource::PktSrc::Stats stat; ps->Statistics(&stat); recv += stat.received; drop += stat.dropped; @@ -3206,10 +3210,15 @@ function dump_current_packet%(file_name: string%) : bool return new Val(0, TYPE_BOOL); if ( ! addl_pkt_dumper ) - addl_pkt_dumper = new PktDumper(0, true); + addl_pkt_dumper = iosource_mgr->OpenPktDumper(file_name->CheckString(), true); - addl_pkt_dumper->Open(file_name->CheckString()); - addl_pkt_dumper->Dump(hdr, pkt); + if ( addl_pkt_dumper ) + { + iosource::PktDumper::Packet p; + p.hdr = hdr; + p.data = pkt; + addl_pkt_dumper->Record(&p); + } return new Val(! addl_pkt_dumper->IsError(), TYPE_BOOL); %} @@ -3266,10 +3275,15 @@ function dump_packet%(pkt: pcap_packet, file_name: string%) : bool hdr.len = (*pkt_vl)[3]->AsCount(); if ( ! addl_pkt_dumper ) - addl_pkt_dumper = new PktDumper(0, true); + addl_pkt_dumper = iosource_mgr->OpenPktDumper(file_name->CheckString(), true); - addl_pkt_dumper->Open(file_name->CheckString()); - addl_pkt_dumper->Dump(&hdr, (*pkt_vl)[4]->AsString()->Bytes()); + if ( addl_pkt_dumper ) + { + iosource::PktDumper::Packet p; + p.hdr = &hdr; + p.data = (*pkt_vl)[4]->AsString()->Bytes(); + addl_pkt_dumper->Record(&p); + } return new Val(addl_pkt_dumper->IsError(), TYPE_BOOL); %} @@ -4030,14 +4044,14 @@ function rotate_file_by_name%(f: string%): rotate_info bool is_addl_pkt_dumper = false; // Special case: one of current dump files. - if ( pkt_dumper && streq(pkt_dumper->FileName(), f->CheckString()) ) + if ( pkt_dumper && streq(pkt_dumper->Path().c_str(), f->CheckString()) ) { is_pkt_dumper = true; pkt_dumper->Close(); } if ( addl_pkt_dumper && - streq(addl_pkt_dumper->FileName(), f->CheckString()) ) + streq(addl_pkt_dumper->Path().c_str(), f->CheckString()) ) { is_addl_pkt_dumper = true; addl_pkt_dumper->Close(); @@ -4156,15 +4170,18 @@ function precompile_pcap_filter%(id: PcapFilterID, s: string%): bool %{ bool success = true; - loop_over_list(pkt_srcs, i) - { - pkt_srcs[i]->ClearErrorMsg(); + const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); - if ( ! pkt_srcs[i]->PrecompileFilter(id->ForceAsInt(), + for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); + i != pkt_srcs.end(); i++ ) + { + iosource::PktSrc* ps = *i; + + if ( ! ps->PrecompileFilter(id->ForceAsInt(), s->CheckString()) ) { reporter->Error("precompile_pcap_filter: %s", - pkt_srcs[i]->ErrorMsg()); + ps->ErrorMsg()); success = false; } } @@ -4194,11 +4211,14 @@ function install_pcap_filter%(id: PcapFilterID%): bool %{ bool success = true; - loop_over_list(pkt_srcs, i) - { - pkt_srcs[i]->ClearErrorMsg(); + const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); - if ( ! pkt_srcs[i]->SetFilter(id->ForceAsInt()) ) + for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); + i != pkt_srcs.end(); i++ ) + { + iosource::PktSrc* ps = *i; + + if ( ! ps->SetFilter(id->ForceAsInt()) ) success = false; } @@ -4221,9 +4241,14 @@ function install_pcap_filter%(id: PcapFilterID%): bool ## uninstall_dst_net_filter function pcap_error%(%): string %{ - loop_over_list(pkt_srcs, i) + const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); + + for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); + i != pkt_srcs.end(); i++ ) { - const char* err = pkt_srcs[i]->ErrorMsg(); + iosource::PktSrc* ps = *i; + + const char* err = ps->ErrorMsg(); if ( *err ) return new StringVal(err); } diff --git a/src/iosource/CMakeLists.txt b/src/iosource/CMakeLists.txt new file mode 100644 index 0000000000..a9246e8de9 --- /dev/null +++ b/src/iosource/CMakeLists.txt @@ -0,0 +1,23 @@ + +include(BroSubdir) + +include_directories(BEFORE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} +) + +add_subdirectory(pktsrc) + + +set(iosource_SRCS + Component.cc + Manager.cc + + pktsrc/Component.cc + pktsrc/PktDumper.cc + pktsrc/PktSrc.cc +) + +bro_add_subdir_library(iosource ${iosource_SRCS}) +add_dependencies(bro_iosource generate_outputs) + diff --git a/src/iosource/Component.cc b/src/iosource/Component.cc new file mode 100644 index 0000000000..5f916c0a11 --- /dev/null +++ b/src/iosource/Component.cc @@ -0,0 +1,44 @@ + +#include "Component.h" + +#include "Desc.h" + +using namespace iosource; + +Component::Component(const std::string& arg_name) + : plugin::Component(plugin::component::IOSOURCE) + { + name = arg_name; + } + +Component::Component(plugin::component::Type type, const std::string& arg_name) + : plugin::Component(type) + { + name = arg_name; + } + +Component::Component(const Component& other) + : plugin::Component(other) + { + name = other.name; + } + +Component::~Component() + { + } + +void Component::Describe(ODesc* d) const + { + plugin::Component::Describe(d); + d->Add(name); + } + +Component& Component::operator=(const Component& other) + { + plugin::Component::operator=(other); + + if ( &other != this ) + name = other.name; + + return *this; + } diff --git a/src/iosource/Component.h b/src/iosource/Component.h new file mode 100644 index 0000000000..b56eeb038c --- /dev/null +++ b/src/iosource/Component.h @@ -0,0 +1,56 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef IOSOURCE_PLUGIN_COMPONENT_H +#define IOSOURCE_PLUGIN_COMPONENT_H + +#include "plugin/Component.h" + +namespace iosource { + +class IOSource; + +/** + * Component description for plugins providing IOSources. + */ +class Component : public plugin::Component { +public: + typedef IOSource* (*factory_callback)(); + + /** + * XXX + */ + Component(const std::string& name); + + /** + * Copy constructor. + */ + Component(const Component& other); + + /** + * Destructor. + */ + ~Component(); + + /** + * XXX + */ + virtual const char* Name() const { return name.c_str(); } + + /** + * Generates a human-readable description of the component. This goes + * into the output of \c "bro -NN". + */ + virtual void Describe(ODesc* d) const; + + Component& operator=(const Component& other); + +protected: + Component(plugin::component::Type type, const std::string& name); + +private: + std::string name; +}; + +} + +#endif diff --git a/src/IOSource.h b/src/iosource/IOSource.h similarity index 51% rename from src/IOSource.h rename to src/iosource/IOSource.h index db50bbd2a9..3419152a9a 100644 --- a/src/IOSource.h +++ b/src/iosource/IOSource.h @@ -1,13 +1,17 @@ -// Interface for classes providing/consuming data during Bro's main loop. +// See the file "COPYING" in the main distribution directory for copyright. -#ifndef iosource_h -#define iosource_h +#ifndef IOSOURCE_IOSOURCE_H +#define IOSOURCE_IOSOURCE_H + +#include -#include #include "Timer.h" -using namespace std; +namespace iosource { +/** + * Interface class for components providing/consuming data inside Bro's main loop. + */ class IOSource { public: IOSource() { idle = closed = false; } @@ -20,6 +24,12 @@ public: // Otherwise, source may be removed. bool IsOpen() const { return ! closed; } + // XXX + virtual void Init() { } + + // XXX + virtual void Done() { } + // Returns select'able fds (leaves args untouched if we don't have // selectable fds). virtual void GetFds(int* read, int* write, int* except) = 0; @@ -46,58 +56,18 @@ public: protected: // Derived classed are to set this to true if they have gone dry // temporarily. - bool idle; + void SetIdle(bool is_idle) { idle = is_idle; } + // Derived classed are to set this to true if they have gone dry - // permanently. + // temporarily. + void SetClosed(bool is_closed) { closed = is_closed; } + +private: + bool idle; bool closed; }; -class IOSourceRegistry { -public: - IOSourceRegistry() { call_count = 0; dont_counts = 0; } - ~IOSourceRegistry(); - - // If dont_count is true, this source does not contribute to the - // number of IOSources returned by Size(). The effect is that - // if all sources but the non-counting ones have gone dry, - // processing will shut down. - void Register(IOSource* src, bool dont_count = false); - - // This may block for some time. - IOSource* FindSoonest(double* ts); - - int Size() const { return sources.size() - dont_counts; } - - // Terminate IOSource processing immediately by removing all - // sources (and therefore returning a Size() of zero). - void Terminate() { RemoveAll(); } - -protected: - // When looking for a source with something to process, - // every SELECT_FREQUENCY calls we will go ahead and - // block on a select(). - static const int SELECT_FREQUENCY = 25; - - // Microseconds to wait in an empty select if no source is ready. - static const int SELECT_TIMEOUT = 50; - - void RemoveAll(); - - unsigned int call_count; - int dont_counts; - - struct Source { - IOSource* src; - int fd_read; - int fd_write; - int fd_except; - }; - - typedef list SourceList; - SourceList sources; -}; - -extern IOSourceRegistry io_sources; +} #endif diff --git a/src/IOSource.cc b/src/iosource/Manager.cc similarity index 52% rename from src/IOSource.cc rename to src/iosource/Manager.cc index d47007caad..9c14330868 100644 --- a/src/IOSource.cc +++ b/src/iosource/Manager.cc @@ -1,3 +1,4 @@ + #include #include #include @@ -5,26 +6,37 @@ #include -#include "util.h" +#include "Manager.h" #include "IOSource.h" +#include "pktsrc/PktSrc.h" +#include "pktsrc/PktDumper.h" +#include "pktsrc/Component.h" +#include "plugin/Manager.h" -IOSourceRegistry io_sources; +#include "util.h" -IOSourceRegistry::~IOSourceRegistry() +#define DEFAULT_PREFIX "pcap" + +using namespace iosource; + +Manager::~Manager() { for ( SourceList::iterator i = sources.begin(); i != sources.end(); ++i ) + { + (*i)->src->Done(); delete *i; + } sources.clear(); } -void IOSourceRegistry::RemoveAll() +void Manager::RemoveAll() { // We're cheating a bit here ... dont_counts = sources.size(); } -IOSource* IOSourceRegistry::FindSoonest(double* ts) +IOSource* Manager::FindSoonest(double* ts) { // Remove sources which have gone dry. For simplicity, we only // remove at most one each time. @@ -101,9 +113,9 @@ IOSource* IOSourceRegistry::FindSoonest(double* ts) FD_SET(src->fd_write, &fd_write); FD_SET(src->fd_except, &fd_except); - maxx = max(src->fd_read, maxx); - maxx = max(src->fd_write, maxx); - maxx = max(src->fd_except, maxx); + maxx = std::max(src->fd_read, maxx); + maxx = std::max(src->fd_write, maxx); + maxx = std::max(src->fd_except, maxx); } // We can't block indefinitely even when all sources are dry: @@ -166,11 +178,130 @@ finished: return soonest_src; } -void IOSourceRegistry::Register(IOSource* src, bool dont_count) +void Manager::Register(IOSource* src, bool dont_count) { + src->Init(); Source* s = new Source; s->src = src; if ( dont_count ) ++dont_counts; - return sources.push_back(s); + + sources.push_back(s); + } + +void Manager::Register(PktSrc* src) + { + pkt_srcs.push_back(src); + Register(src, false); + } + +static std::pair split_prefix(std::string path) + { + // See if the path comes with a prefix telling us which type of + // PktSrc to use. If not, choose default. + std::string prefix; + + std::string::size_type i = path.find(":"); + if ( i != std::string::npos ) + { + prefix = path.substr(0, i); + path = path.substr(++i, std::string::npos); + } + + else + prefix= DEFAULT_PREFIX; + + return std::make_pair(prefix, path); + } + +PktSrc* Manager::OpenPktSrc(const std::string& path, const std::string& filter, bool is_live) + { + std::pair t = split_prefix(path); + std::string prefix = t.first; + std::string npath = t.second; + + // Find the component providing packet sources of the requested prefix. + + pktsrc::SourceComponent* component = 0; + + std::list all_components = plugin_mgr->Components(); + + for ( std::list::const_iterator i = all_components.begin(); + i != all_components.end(); i++ ) + { + pktsrc::SourceComponent* c = *i; + + if ( c->Prefix() == prefix && + (( is_live && c->DoesLive() ) || + (! is_live && c->DoesTrace())) ) + { + component = c; + break; + } + } + + + if ( ! component ) + reporter->FatalError("type of packet source '%s' not recognized", prefix.c_str()); + + // Instantiate packet source. + + PktSrc* ps = (*component->Factory())(path, filter, is_live); + + if ( ! (ps && ps->IsOpen()) ) + { + string type = (is_live ? "interface" : "trace file"); + string pserr = ps->ErrorMsg() ? (string(" - ") + ps->ErrorMsg()) : ""; + + reporter->FatalError("%s: problem with %s %s%s\n", + prog, path.c_str(), type.c_str(), pserr.c_str()); + } + + DBG_LOG(DBG_PKTIO, "Created packet source of type %s for %s\n", component->Name(), path.c_str()); + + Register(ps); + return ps; + } + + +PktDumper* Manager::OpenPktDumper(const string& path, bool append) + { + std::pair t = split_prefix(path); + std::string prefix = t.first; + std::string npath = t.second; + + // Find the component providing packet dumpers of the requested prefix. + + pktsrc::DumperComponent* component = 0; + + std::list all_components = plugin_mgr->Components(); + + for ( std::list::const_iterator i = all_components.begin(); + i != all_components.end(); i++ ) + { + if ( (*i)->Prefix() == prefix ) + { + component = (*i); + break; + } + } + + if ( ! component ) + reporter->FatalError("type of packet dumper '%s' not recognized", prefix.c_str()); + + // Instantiate packet dumper. + + PktDumper* pd = (*component->Factory())(path, append); + + if ( ! (pd && pd->IsOpen()) ) + { + string pderr = pd->ErrorMsg().size() ? (string(" - ") + pd->ErrorMsg()) : ""; + + reporter->FatalError("%s: can't open write file \"%s\"%s\n", + prog, path.c_str(), pderr.c_str()); + } + + DBG_LOG(DBG_PKTIO, "Created packer dumper of type %s for %s\n", component->Name(), path.c_str()); + + return pd; } diff --git a/src/iosource/Manager.h b/src/iosource/Manager.h new file mode 100644 index 0000000000..5a3a58d798 --- /dev/null +++ b/src/iosource/Manager.h @@ -0,0 +1,75 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef IOSOURCE_MANAGER_H +#define IOSOURCE_MANAGER_H + +#include +#include + +namespace iosource { + +class IOSource; +class PktSrc; +class PktDumper; + +class Manager { +public: + Manager() { call_count = 0; dont_counts = 0; } + ~Manager(); + + // If dont_count is true, this source does not contribute to the + // number of IOSources returned by Size(). The effect is that + // if all sources but the non-counting ones have gone dry, + // processing will shut down. + void Register(IOSource* src, bool dont_count = false); + + // This may block for some time. + IOSource* FindSoonest(double* ts); + + int Size() const { return sources.size() - dont_counts; } + + typedef std::list PktSrcList; + const PktSrcList& GetPktSrcs() const { return pkt_srcs; } + + // Terminate IOSource processing immediately by removing all + // sources (and therefore returning a Size() of zero). + void Terminate() { RemoveAll(); } + + PktSrc* OpenPktSrc(const std::string& path, const std::string& filter, bool is_live); + PktDumper* OpenPktDumper(const std::string& path, bool append); + +protected: + void Register(PktSrc* src); + + // When looking for a source with something to process, + // every SELECT_FREQUENCY calls we will go ahead and + // block on a select(). + static const int SELECT_FREQUENCY = 25; + + // Microseconds to wait in an empty select if no source is ready. + static const int SELECT_TIMEOUT = 50; + + void RemoveAll(); + + unsigned int call_count; + int dont_counts; + + struct Source { + IOSource* src; + int fd_read; + int fd_write; + int fd_except; + }; + + typedef std::list SourceList; + SourceList sources; + + PktSrcList pkt_srcs; +}; + +} + +extern iosource::Manager* iosource_mgr; + +#endif + diff --git a/src/iosource/pktsrc/CMakeLists.txt b/src/iosource/pktsrc/CMakeLists.txt new file mode 100644 index 0000000000..07303b46a3 --- /dev/null +++ b/src/iosource/pktsrc/CMakeLists.txt @@ -0,0 +1,2 @@ + +add_subdirectory(pcap) diff --git a/src/iosource/pktsrc/Component.cc b/src/iosource/pktsrc/Component.cc new file mode 100644 index 0000000000..22c49feed0 --- /dev/null +++ b/src/iosource/pktsrc/Component.cc @@ -0,0 +1,130 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "Component.h" + +#include "../Desc.h" + +using namespace iosource::pktsrc; + +SourceComponent::SourceComponent(const std::string& arg_name, const std::string& arg_prefix, InputType arg_type, factory_callback arg_factory) + : iosource::Component(plugin::component::PKTSRC, arg_name) + { + prefix = arg_prefix; + type = arg_type; + factory = arg_factory; + } + +SourceComponent::SourceComponent(const SourceComponent& other) + : iosource::Component(other) + { + prefix = other.prefix; + type = other.type; + factory = other.factory; + } + +SourceComponent::~SourceComponent() + { + } + +const std::string& SourceComponent::Prefix() const + { + return prefix; + } + +bool SourceComponent::DoesLive() const + { + return type == LIVE || type == BOTH; + } + +bool SourceComponent::DoesTrace() const + { + return type == TRACE || type == BOTH; + } + +SourceComponent::factory_callback SourceComponent::Factory() const + { + return factory; + } + + +void SourceComponent::Describe(ODesc* d) const + { + iosource::Component::Describe(d); + + d->Add(" (interface prefix: "); + d->Add(prefix); + d->Add(")"); + } + +SourceComponent& SourceComponent::operator=(const SourceComponent& other) + { + iosource::Component::operator=(other); + + if ( &other != this ) + { + prefix = other.prefix; + type = other.type; + factory = other.factory; + } + + return *this; + } + +DumperComponent::DumperComponent(const std::string& arg_name, const std::string& arg_prefix, factory_callback arg_factory) + : plugin::Component(plugin::component::PKTDUMPER) + { + name = arg_name; + factory = arg_factory; + prefix = arg_prefix; + } + +DumperComponent::DumperComponent(const DumperComponent& other) + : plugin::Component(other) + { + name = other.name; + factory = other.factory; + prefix = other.prefix; + } + +DumperComponent::~DumperComponent() + { + } + +DumperComponent::factory_callback DumperComponent::Factory() const + { + return factory; + } + +const char* DumperComponent::Name() const + { + return name.c_str(); + } + +const std::string& DumperComponent::Prefix() const + { + return prefix; + } + +void DumperComponent::Describe(ODesc* d) const + { + plugin::Component::Describe(d); + + d->Add(name); + d->Add(" (dumper prefix: "); + d->Add(prefix); + d->Add(")"); + } + +DumperComponent& DumperComponent::operator=(const DumperComponent& other) + { + plugin::Component::operator=(other); + + if ( &other != this ) + { + name = other.name; + factory = other.factory; + prefix = other.prefix; + } + + return *this; + } diff --git a/src/iosource/pktsrc/Component.h b/src/iosource/pktsrc/Component.h new file mode 100644 index 0000000000..2a62fb5503 --- /dev/null +++ b/src/iosource/pktsrc/Component.h @@ -0,0 +1,132 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef IOSOURCE_PKTSRC_PLUGIN_COMPONENT_H +#define IOSOURCE_PKTSRC_PLUGIN_COMPONENT_H + +#include "../Component.h" + +namespace iosource { + +class PktSrc; +class PktDumper; + +namespace pktsrc { + +/** + * Component description for plugins providing a PktSrc for packet input. + */ +class SourceComponent : public iosource::Component { +public: + enum InputType { LIVE, TRACE, BOTH }; + + typedef PktSrc* (*factory_callback)(const std::string& path, const std::string& filter, bool is_live); + + /** + * XXX + */ + SourceComponent(const std::string& name, const std::string& prefix, InputType type, factory_callback factory); + + /** + * Copy constructor. + */ + SourceComponent(const SourceComponent& other); + + /** + * Destructor. + */ + virtual ~SourceComponent(); + + /** + * Returns the prefix passes to the constructor. + */ + const std::string& Prefix() const; + + /** + * Returns true if packet source instantiated by the component handle + * live traffic. + */ + bool DoesLive() const; + + /** + * Returns true if packet source instantiated by the component handle + * offline traces. + */ + bool DoesTrace() const; + + /** + * Returns the source's factory function. + */ + factory_callback Factory() const; + + /** + * Generates a human-readable description of the component. This goes + * into the output of \c "bro -NN". + */ + virtual void Describe(ODesc* d) const; + + SourceComponent& operator=(const SourceComponent& other); + +private: + std::string prefix; + InputType type; + factory_callback factory; +}; + +/** + * Component description for plugins providing a PktDumper for packet output. + * + * PktDumpers aren't IOSurces but we locate them here to keep them along with + * the PktSrc. + */ +class DumperComponent : public plugin::Component { +public: + typedef PktDumper* (*factory_callback)(const std::string& path, bool append); + + /** + * XXX + */ + DumperComponent(const std::string& name, const std::string& prefix, factory_callback factory); + + /** + * Copy constructor. + */ + DumperComponent(const DumperComponent& other); + + /** + * Destructor. + */ + ~DumperComponent(); + + /** + * XXX + */ + virtual const char* Name() const; + + /** + * Returns the prefix passes to the constructor. + */ + const std::string& Prefix() const; + + /** + * Returns the source's factory function. + */ + factory_callback Factory() const; + + /** + * Generates a human-readable description of the component. This goes + * into the output of \c "bro -NN". + */ + virtual void Describe(ODesc* d) const; + + DumperComponent& operator=(const DumperComponent& other); + +private: + std::string name; + std::string prefix; + factory_callback factory; +}; + +} +} + +#endif diff --git a/src/iosource/pktsrc/PktDumper.cc b/src/iosource/pktsrc/PktDumper.cc new file mode 100644 index 0000000000..21ad79b87d --- /dev/null +++ b/src/iosource/pktsrc/PktDumper.cc @@ -0,0 +1,79 @@ + +// See the file "COPYING" in the main distribution directory for copyright. + +#include +#include + +#include "config.h" + +#include "PktDumper.h" + +using namespace iosource; + +PktDumper::PktDumper() + { + is_open = false; + errmsg = ""; + } + +PktDumper::~PktDumper() + { + } + +const std::string& PktDumper::Path() const + { + return props.path; + } + +bool PktDumper::IsOpen() const + { + return is_open; + } + +double PktDumper::OpenTime() const + { + return is_open ? props.open_time : 0; + } + +bool PktDumper::IsError() const + { + return errmsg.size(); + } + +const std::string& PktDumper::ErrorMsg() const + { + return errmsg; + } + +int PktDumper::HdrSize() const + { + return is_open ? props.hdr_size : -1; + } + +bool PktDumper::Record(const Packet* pkt) + { + return Dump(pkt); + } + +void PktDumper::Opened(const Properties& arg_props) + { + is_open = true; + props = arg_props; + DBG_LOG(DBG_PKTIO, "Opened dumper %s", props.path.c_str()); + } + +void PktDumper::Closed() + { + is_open = false; + props.path = ""; + DBG_LOG(DBG_PKTIO, "Closed dumper %s", props.path.c_str()); + } + +void PktDumper::Error(const std::string& msg) + { + errmsg = msg; + + DBG_LOG(DBG_PKTIO, "Error with dumper %s: %s", + IsOpen() ? props.path.c_str() : "", + msg.c_str()); + } diff --git a/src/iosource/pktsrc/PktDumper.h b/src/iosource/pktsrc/PktDumper.h new file mode 100644 index 0000000000..b8f3595e32 --- /dev/null +++ b/src/iosource/pktsrc/PktDumper.h @@ -0,0 +1,57 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef IOSOURCE_PKTSRC_PKTDUMPER_H +#define IOSOURCE_PKTSRC_PKTDUMPER_H + +#include "../IOSource.h" + +namespace iosource { + +class PktDumper { +public: + struct Packet { + const struct pcap_pkthdr* hdr; + const u_char* data; + }; + + PktDumper(); + virtual ~PktDumper(); + + const std::string& Path() const; + bool IsOpen() const; + double OpenTime() const; + bool IsError() const; + const std::string& ErrorMsg() const; + int HdrSize() const; + bool Record(const Packet* pkt); + + virtual void Close() = 0; + virtual void Open() = 0; + +protected: + // Methods to use by derived classed. + // + struct Properties { + std::string path; + int hdr_size; + double open_time; + }; + + void Opened(const Properties& props); + void Closed(); + void Error(const std::string& msg); + + // PktSrc interface for derived classes to implement. + + virtual bool Dump(const Packet* pkt) = 0; + +private: + bool is_open; + Properties props; + + std::string errmsg; +}; + +} + +#endif diff --git a/src/iosource/pktsrc/PktSrc.cc b/src/iosource/pktsrc/PktSrc.cc new file mode 100644 index 0000000000..703a2d634b --- /dev/null +++ b/src/iosource/pktsrc/PktSrc.cc @@ -0,0 +1,411 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include +#include + +#include "config.h" + +#include "util.h" +#include "PktSrc.h" +#include "Hash.h" +#include "Net.h" +#include "Sessions.h" + +using namespace iosource; + +PktSrc::PktSrc() + { + have_packet = false; + errbuf = ""; + + next_sync_point = 0; + first_timestamp = 0.0; + first_wallclock = current_wallclock = 0; + } + +PktSrc::~PktSrc() + { + } + +const std::string& PktSrc::Path() const + { + static std::string not_open("not open"); + return IsOpen() ? props.path : not_open; + } + +const char* PktSrc::ErrorMsg() const + { + return errbuf.c_str(); + } + +int PktSrc::LinkType() const + { + return IsOpen() ? props.link_type : -1; + } + +int PktSrc::HdrSize() const + { + return IsOpen() ? props.hdr_size : -1; + } + +int PktSrc::SnapLen() const + { + return snaplen; // That's a global. Change? + } + +bool PktSrc::IsLive() const + { + return props.is_live; + } + +double PktSrc::CurrentPacketTimestamp() + { + return current_pseudo; + } + +double PktSrc::CurrentPacketWallClock() + { + // We stop time when we are suspended. + if ( net_is_processing_suspended() ) + current_wallclock = current_time(true); + + return current_wallclock; + } + +void PktSrc::Opened(const Properties& arg_props) + { + props = arg_props; + SetClosed(false); + + DBG_LOG(DBG_PKTIO, "Opened source %s", props.path.c_str()); + } + +void PktSrc::Closed() + { + SetClosed(true); + + DBG_LOG(DBG_PKTIO, "Closed source %s", props.path.c_str()); + } + +void PktSrc::Error(const std::string& msg) + { + // We don't report this immediately, Bro will ask us for the error + // once it notices we aren't open. + errbuf = msg; + DBG_LOG(DBG_PKTIO, "Error with source %s: %s", + IsOpen() ? props.path.c_str() : "", + msg.c_str()); + } + +void PktSrc::Info(const std::string& msg) + { + reporter->Info("%s", msg.c_str()); + } + +void PktSrc::Weird(const std::string& msg, const Packet* p) + { + sessions->Weird(msg.c_str(), p->hdr, p->data, 0); + } + +void PktSrc::InternalError(const std::string& msg) + { + reporter->InternalError("%s", msg.c_str()); + } + +void PktSrc::ContinueAfterSuspend() + { + current_wallclock = current_time(true); + } + +int PktSrc::GetLinkHeaderSize(int link_type) + { + switch ( link_type ) { + case DLT_NULL: + return 4; + + case DLT_EN10MB: + return 14; + + case DLT_FDDI: + return 13 + 8; // fddi_header + LLC + +#ifdef DLT_LINUX_SLL + case DLT_LINUX_SLL: + return 16; +#endif + + case DLT_PPP_SERIAL: // PPP_SERIAL + return 4; + + case DLT_RAW: + return 0; + } + + return -1; + } + +double PktSrc::CheckPseudoTime() + { + if ( ! IsOpen() ) + return 0; + + if ( ! ExtractNextPacketInternal() ) + return 0; + + if ( remote_trace_sync_interval ) + { + if ( next_sync_point == 0 || current_packet.ts >= next_sync_point ) + { + int n = remote_serializer->SendSyncPoint(); + next_sync_point = first_timestamp + + n * remote_trace_sync_interval; + remote_serializer->Log(RemoteSerializer::LogInfo, + fmt("stopping at packet %.6f, next sync-point at %.6f", + current_packet.ts, next_sync_point)); + + return 0; + } + } + + double pseudo_time = current_packet.ts - first_timestamp; + double ct = (current_time(true) - first_wallclock) * pseudo_realtime; + + return pseudo_time <= ct ? bro_start_time + pseudo_time : 0; + } + +void PktSrc::Init() + { + Open(); + } + +void PktSrc::Done() + { + Close(); + } + +void PktSrc::GetFds(int* read, int* write, int* except) + { + if ( pseudo_realtime ) + { + // Select would give erroneous results. But we simulate it + // by setting idle accordingly. + SetIdle(CheckPseudoTime() == 0); + return; + } + + if ( IsOpen() && props.selectable_fd >= 0 ) + *read = props.selectable_fd; + } + +double PktSrc::NextTimestamp(double* local_network_time) + { + if ( ! IsOpen() ) + return -1.0; + + if ( ! ExtractNextPacketInternal() ) + return -1.0; + + if ( pseudo_realtime ) + { + // Delay packet if necessary. + double packet_time = CheckPseudoTime(); + if ( packet_time ) + return packet_time; + + SetIdle(true); + return -1.0; + } + + return current_packet.ts; + } + +void PktSrc::Process() + { + if ( ! IsOpen() ) + return; + + if ( ! ExtractNextPacketInternal() ) + return; + + int pkt_hdr_size = props.hdr_size; + + // Unfortunately some packets on the link might have MPLS labels + // while others don't. That means we need to ask the link-layer if + // labels are in place. + bool have_mpls = false; + + int protocol = 0; + const u_char* data = current_packet.data; + + switch ( props.link_type ) { + case DLT_NULL: + { + protocol = (data[3] << 24) + (data[2] << 16) + (data[1] << 8) + data[0]; + + // From the Wireshark Wiki: "AF_INET6, unfortunately, has + // different values in {NetBSD,OpenBSD,BSD/OS}, + // {FreeBSD,DragonFlyBSD}, and {Darwin/Mac OS X}, so an IPv6 + // packet might have a link-layer header with 24, 28, or 30 + // as the AF_ value." As we may be reading traces captured on + // platforms other than what we're running on, we accept them + // all here. + if ( protocol != AF_INET + && protocol != AF_INET6 + && protocol != 24 + && protocol != 28 + && protocol != 30 ) + { + Weird("non_ip_packet_in_null_transport", ¤t_packet); + data = 0; + return; + } + + break; + } + + case DLT_EN10MB: + { + // Get protocol being carried from the ethernet frame. + protocol = (data[12] << 8) + data[13]; + + switch ( protocol ) + { + // MPLS carried over the ethernet frame. + case 0x8847: + have_mpls = true; + break; + + // VLAN carried over the ethernet frame. + case 0x8100: + data += GetLinkHeaderSize(props.link_type); + data += 4; // Skip the vlan header + pkt_hdr_size = 0; + + // Check for 802.1ah (Q-in-Q) containing IP. + // Only do a second layer of vlan tag + // stripping because there is no + // specification that allows for deeper + // nesting. + if ( ((data[2] << 8) + data[3]) == 0x0800 ) + data += 4; + + break; + + // PPPoE carried over the ethernet frame. + case 0x8864: + data += GetLinkHeaderSize(props.link_type); + protocol = (data[6] << 8) + data[7]; + data += 8; // Skip the PPPoE session and PPP header + pkt_hdr_size = 0; + + if ( protocol != 0x0021 && protocol != 0x0057 ) + { + // Neither IPv4 nor IPv6. + Weird("non_ip_packet_in_pppoe_encapsulation", ¤t_packet); + data = 0; + return; + } + break; + } + + break; + } + + case DLT_PPP_SERIAL: + { + // Get PPP protocol. + protocol = (data[2] << 8) + data[3]; + + if ( protocol == 0x0281 ) + // MPLS Unicast + have_mpls = true; + + else if ( protocol != 0x0021 && protocol != 0x0057 ) + { + // Neither IPv4 nor IPv6. + Weird("non_ip_packet_in_ppp_encapsulation", ¤t_packet); + data = 0; + return; + } + break; + } + } + + if ( have_mpls ) + { + // Remove the data link layer + data += GetLinkHeaderSize(props.link_type); + + // Denote a header size of zero before the IP header + pkt_hdr_size = 0; + + // Skip the MPLS label stack. + bool end_of_stack = false; + + while ( ! end_of_stack ) + { + end_of_stack = *(data + 2) & 0x01; + data += 4; + } + } + + if ( pseudo_realtime ) + { + current_pseudo = CheckPseudoTime(); + net_packet_arrival(current_pseudo, current_packet.hdr, current_packet.data, pkt_hdr_size, this); + if ( ! first_wallclock ) + first_wallclock = current_time(true); + } + + else + net_packet_arrival(current_packet.ts, current_packet.hdr, current_packet.data, pkt_hdr_size, this); + + have_packet = 0; + DoneWithPacket(¤t_packet); + } + +const char* PktSrc::Tag() + { + return "PktSrc"; + } + +int PktSrc::ExtractNextPacketInternal() + { + if ( have_packet ) + return true; + + have_packet = false; + + // Don't return any packets if processing is suspended (except for the + // very first packet which we need to set up times). + if ( net_is_processing_suspended() && first_timestamp ) + { + SetIdle(true); + return 0; + } + + if ( pseudo_realtime ) + current_wallclock = current_time(true); + + if ( ExtractNextPacket(¤t_packet) ) + { + if ( ! first_timestamp ) + first_timestamp = current_packet.ts; + + have_packet = true; + return 1; + } + + if ( pseudo_realtime && using_communication && ! IsOpen() ) + { + // Source has gone dry, we're done. + if ( remote_trace_sync_interval ) + remote_serializer->SendFinalSyncPoint(); + else + remote_serializer->Terminate(); + } + + SetIdle(true); + return 0; + } + diff --git a/src/iosource/pktsrc/PktSrc.h b/src/iosource/pktsrc/PktSrc.h new file mode 100644 index 0000000000..3c3436bb19 --- /dev/null +++ b/src/iosource/pktsrc/PktSrc.h @@ -0,0 +1,140 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef IOSOURCE_PKTSRC_PKTSRC_H +#define IOSOURCE_PKTSRC_PKTSRC_H + +#include "../IOSource.h" + +struct pcap_pkthdr; + +namespace iosource { + +class PktSrc : public IOSource { +public: + struct Stats { + unsigned int received; // pkts received (w/o drops) + unsigned int dropped; // pkts dropped + unsigned int link; // total packets on link + // (not always not available) + // + Stats() { received = dropped = link = 0; } + }; + + PktSrc(); + virtual ~PktSrc(); + + const std::string& Path() const; + const std::string& Filter() const; + bool IsLive() const; + int LinkType() const; + const char* ErrorMsg() const; + int HdrSize() const; + int SnapLen() const; + + // Only valid in pseudo-realtime mode. + double CurrentPacketTimestamp(); + double CurrentPacketWallClock(); + + // Signal packet source that processing was suspended and is now + // going to be continued. + void ContinueAfterSuspend(); + + virtual void Statistics(Stats* stats) = 0; + + // Returns the packet last processed; false if there is no + // current packet available. + virtual bool GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt) = 0; + + // Precompiles a filter and associates the given index with it. + // Returns true on success, 0 if a problem occurred or filtering is + // not supported. + virtual int PrecompileFilter(int index, const std::string& filter) = 0; + + // Activates the filter with the given index. Returns true on + // success, 0 if a problem occurred or the filtering is not + // supported. + virtual int SetFilter(int index) = 0; + + static int GetLinkHeaderSize(int link_type); + +#if 0 + PktSrc_Filter_Type FilterType() const { return filter_type; } + + void AddSecondaryTablePrograms(); + const secondary_program_list& ProgramTable() const + { return program_list; } +#endif + +protected: + // Methods to use by derived classes. + + struct Properties { + std::string path; + std::string filter; // Maybe different than what's passed in if not (directly) supported. + int selectable_fd; + int link_type; + int hdr_size; + bool is_live; + }; + + struct Packet { + double ts; + const struct ::pcap_pkthdr* hdr; + const u_char* data; + }; + + void Opened(const Properties& props); + void Closed(); + void Info(const std::string& msg); + void Error(const std::string& msg); + void Weird(const std::string& msg, const Packet* pkt); + void InternalError(const std::string& msg); + + // PktSrc interface for derived classes to implement. + + virtual void Open() = 0; + virtual void Close() = 0; + // Returns 1 on success, 0 on time-out/gone dry. + virtual int ExtractNextPacket(Packet* pkt) = 0; + virtual void DoneWithPacket(Packet* pkt) = 0; + +private: + // Checks if the current packet has a pseudo-time <= current_time. + // If yes, returns pseudo-time, otherwise 0. + double CheckPseudoTime(); + + // XXX + int ExtractNextPacketInternal(); + + // IOSource interface implementation. + virtual void Init(); + virtual void Done(); + virtual void GetFds(int* read, int* write, int* except); + virtual double NextTimestamp(double* local_network_time); + virtual void Process(); + virtual const char* Tag(); + + Properties props; + + bool have_packet; + Packet current_packet; + + // Only set in pseudo-realtime mode. + double first_timestamp; + double first_wallclock; + double current_wallclock; + double current_pseudo; + double next_sync_point; // For trace synchronziation in pseudo-realtime + + std::string errbuf; + +#if 0 + PktSrc_Filter_Type filter_type; // normal path or secondary path + secondary_program_list program_list; +#endif +}; + +} + + +#endif diff --git a/src/iosource/pktsrc/old-2ndary-code.h b/src/iosource/pktsrc/old-2ndary-code.h new file mode 100644 index 0000000000..0b47cccdc5 --- /dev/null +++ b/src/iosource/pktsrc/old-2ndary-code.h @@ -0,0 +1,69 @@ +// Whether a PktSrc object is used by the normal filter structure or the +// secondary-path structure. +typedef enum { + TYPE_FILTER_NORMAL, // the normal filter + TYPE_FILTER_SECONDARY, // the secondary-path filter +} PktSrc_Filter_Type; + +// {filter,event} tuples conforming the secondary path. +class SecondaryEvent { +public: + SecondaryEvent(const char* arg_filter, Func* arg_event) + { + filter = arg_filter; + event = arg_event; + } + + const char* Filter() { return filter; } + Func* Event() { return event; } + +private: + const char* filter; + Func* event; +}; + +declare(PList,SecondaryEvent); +typedef PList(SecondaryEvent) secondary_event_list; + +class SecondaryPath { +public: + SecondaryPath(); + ~SecondaryPath(); + + secondary_event_list& EventTable() { return event_list; } + const char* Filter() { return filter; } + +private: + secondary_event_list event_list; + // OR'ed union of all SecondaryEvent filters + char* filter; +}; + +// Main secondary-path object. +extern SecondaryPath* secondary_path; + +// {program, {filter,event}} tuple table. +class SecondaryProgram { +public: + SecondaryProgram(BPF_Program* arg_program, SecondaryEvent* arg_event) + { + program = arg_program; + event = arg_event; + } + + ~SecondaryProgram(); + + BPF_Program* Program() { return program; } + SecondaryEvent* Event() { return event; } + +private: + // Associated program. + BPF_Program *program; + + // Event that is run in case the program is matched. + SecondaryEvent* event; +}; + +declare(PList,SecondaryProgram); +typedef PList(SecondaryProgram) secondary_program_list; + diff --git a/src/BPF_Program.cc b/src/iosource/pktsrc/pcap/BPF_Program.cc similarity index 100% rename from src/BPF_Program.cc rename to src/iosource/pktsrc/pcap/BPF_Program.cc diff --git a/src/BPF_Program.h b/src/iosource/pktsrc/pcap/BPF_Program.h similarity index 100% rename from src/BPF_Program.h rename to src/iosource/pktsrc/pcap/BPF_Program.h diff --git a/src/iosource/pktsrc/pcap/CMakeLists.txt b/src/iosource/pktsrc/pcap/CMakeLists.txt new file mode 100644 index 0000000000..b43d51b0ca --- /dev/null +++ b/src/iosource/pktsrc/pcap/CMakeLists.txt @@ -0,0 +1,8 @@ + +include(BroPlugin) + +include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) + +bro_plugin_begin(Bro Pcap) +bro_plugin_cc(Source.cc Dumper.cc BPF_Program.cc Plugin.cc) +bro_plugin_end() diff --git a/src/iosource/pktsrc/pcap/Dumper.cc b/src/iosource/pktsrc/pcap/Dumper.cc new file mode 100644 index 0000000000..ad1ac2fa73 --- /dev/null +++ b/src/iosource/pktsrc/pcap/Dumper.cc @@ -0,0 +1,111 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include +#include + +#include "Dumper.h" +#include "../PktSrc.h" + +using namespace iosource::pktsrc; + +PcapDumper::PcapDumper(const std::string& path, bool arg_append) + { + append = arg_append; + props.path = path; + dumper = 0; + pd = 0; + } + +PcapDumper::~PcapDumper() + { + } + +void PcapDumper::Open() + { + int linktype = -1; + + pd = pcap_open_dead(DLT_EN10MB, 8192); + if ( ! pd ) + { + Error("error for pcap_open_dead"); + return; + } + + if ( props.path.empty() ) + { + Error("no filename given"); + return; + } + + struct stat s; + int exists = 0; + + if ( append ) + { + // See if output file already exists (and is non-empty). + exists = stat(props.path.c_str(), &s); ; + + if ( exists < 0 && errno != ENOENT ) + { + Error(fmt("can't stat file %s: %s", props.path.c_str(), strerror(errno))); + return; + } + } + + if ( ! append || exists < 0 || s.st_size == 0 ) + { + // Open new file. + dumper = pcap_dump_open(pd, props.path.c_str()); + if ( ! dumper ) + { + Error(pcap_geterr(pd)); + return; + } + } + + else + { + // Old file and we need to append, which, unfortunately, + // is not supported by libpcap. So, we have to hack a + // little bit, knowing that pcap_dumpter_t is, in fact, + // a FILE ... :-( + dumper = (pcap_dumper_t*) fopen(props.path.c_str(), "a"); + if ( ! dumper ) + { + Error(fmt("can't open dump %s: %s", props.path.c_str(), strerror(errno))); + return; + } + } + + props.open_time = network_time; + props.hdr_size = PktSrc::GetLinkHeaderSize(pcap_datalink(pd)); + Opened(props); + } + +void PcapDumper::Close() + { + if ( ! dumper ) + return; + + pcap_dump_close(dumper); + pcap_close(pd); + dumper = 0; + pd = 0; + + Closed(); + } + +bool PcapDumper::Dump(const Packet* pkt) + { + if ( ! dumper ) + return false; + + pcap_dump((u_char*) dumper, pkt->hdr, pkt->data); + + return true; + } + +iosource::PktDumper* PcapDumper::Instantiate(const std::string& path, bool append) + { + return new PcapDumper(path, append); + } diff --git a/src/iosource/pktsrc/pcap/Dumper.h b/src/iosource/pktsrc/pcap/Dumper.h new file mode 100644 index 0000000000..c2762a2b04 --- /dev/null +++ b/src/iosource/pktsrc/pcap/Dumper.h @@ -0,0 +1,40 @@ + +#ifndef IOSOURCE_PKTSRC_PCAP_DUMPER_H +#define IOSOURCE_PKTSRC_PCAP_DUMPER_H + +extern "C" { +#include +} + +#include "../PktDumper.h" + +namespace iosource { +namespace pktsrc { + +class PcapDumper : public PktDumper { +public: + PcapDumper(const std::string& path, bool append); + virtual ~PcapDumper(); + + static PktDumper* Instantiate(const std::string& path, bool appen); + +protected: + // PktDumper interface. + virtual void Open(); + virtual void Close(); + virtual bool Dump(const Packet* pkt); + +private: + Properties props; + + bool append; + pcap_dumper_t* dumper; + pcap_t* pd; +}; + +} +} + +#endif + + diff --git a/src/iosource/pktsrc/pcap/Plugin.cc b/src/iosource/pktsrc/pcap/Plugin.cc new file mode 100644 index 0000000000..307f2da99f --- /dev/null +++ b/src/iosource/pktsrc/pcap/Plugin.cc @@ -0,0 +1,12 @@ + +#include "plugin/Plugin.h" +#include "iosource/pktsrc/Component.h" + +#include "Source.h" +#include "Dumper.h" + +BRO_PLUGIN_BEGIN(Bro, Pcap) + BRO_PLUGIN_DESCRIPTION("Packet I/O via libpcap"); + BRO_PLUGIN_PKTSRC("PcapReader", "pcap", SourceComponent::BOTH, PcapSource); + BRO_PLUGIN_PKTDUMPER("PcapTraceWriter", "pcap", PcapDumper); +BRO_PLUGIN_END diff --git a/src/iosource/pktsrc/pcap/Source.cc b/src/iosource/pktsrc/pcap/Source.cc new file mode 100644 index 0000000000..86c0273adf --- /dev/null +++ b/src/iosource/pktsrc/pcap/Source.cc @@ -0,0 +1,343 @@ + +#include "config.h" + +#include "Source.h" + +#ifdef HAVE_PCAP_INT_H +#include +#endif + +using namespace iosource::pktsrc; + +PcapSource::~PcapSource() + { + Close(); + } + +PcapSource::PcapSource(const std::string& path, const std::string& filter, bool is_live) + { + props.path = path; + props.filter = filter; + props.is_live = is_live; + last_data = 0; + } + +void PcapSource::Open() + { + if ( props.is_live ) + OpenLive(); + else + OpenOffline(); + } + +void PcapSource::Close() + { + if ( ! pd ) + return; + + BPF_Program* code; + IterCookie* cookie = filters.InitForIteration(); + while ( (code = filters.NextEntry(cookie)) ) + delete code; + + filters.Clear(); + + pcap_close(pd); + pd = 0; + last_data = 0; + + Closed(); + } + +void PcapSource::OpenLive() + { + char errbuf[PCAP_ERRBUF_SIZE]; + char tmp_errbuf[PCAP_ERRBUF_SIZE]; + +#if 0 + filter_type = ft; +#endif + + // Determine interface if not specified. + if ( props.path.empty() ) + props.path = pcap_lookupdev(tmp_errbuf); + + if ( props.path.empty() ) + { + safe_snprintf(errbuf, sizeof(errbuf), + "pcap_lookupdev: %s", tmp_errbuf); + Error(errbuf); + return; + } + + // Determine network and netmask. + uint32 net; + if ( pcap_lookupnet(props.path.c_str(), &net, &netmask, tmp_errbuf) < 0 ) + { + // ### The lookup can fail if no address is assigned to + // the interface; and libpcap doesn't have any useful notion + // of error codes, just error std::strings - how bogus - so we + // just kludge around the error :-(. + // sprintf(errbuf, "pcap_lookupnet %s", tmp_errbuf); + // return; + netmask = 0xffffff00; + } + + // We use the smallest time-out possible to return almost immediately if + // no packets are available. (We can't use set_nonblocking() as it's + // broken on FreeBSD: even when select() indicates that we can read + // something, we may get nothing if the store buffer hasn't filled up + // yet.) + pd = pcap_open_live(props.path.c_str(), SnapLen(), 1, 1, tmp_errbuf); + + if ( ! pd ) + { + safe_snprintf(errbuf, sizeof(errbuf), + "pcap_open_live: %s", tmp_errbuf); + Error(errbuf); + return; + } + + // ### This needs autoconf'ing. +#ifdef HAVE_PCAP_INT_H + Info("pcap bufsize = %d\n", ((struct pcap *) pd)->bufsize); +#endif + +#ifdef HAVE_LINUX + if ( pcap_setnonblock(pd, 1, tmp_errbuf) < 0 ) + { + safe_snprintf(errbuf, sizeof(errbuf), + "pcap_setnonblock: %s", tmp_errbuf); + Error(errbuf); + pcap_close(pd); + return; + } +#endif + + props.selectable_fd = pcap_fileno(pd); + + if ( PrecompileFilter(0, props.filter) && SetFilter(0) ) + { + SetHdrSize(); + + if ( ! pd ) + // Was closed, couldn't get header size. + return; + + Info(fmt("listening on %s, capture length %d bytes\n", props.path.c_str(), SnapLen())); + } + else + Close(); + + props.is_live = true; + Opened(props); + } + +void PcapSource::OpenOffline() + { + char errbuf[PCAP_ERRBUF_SIZE]; + char tmp_errbuf[PCAP_ERRBUF_SIZE]; + +#if 0 + filter_type = ft; +#endif + + pd = pcap_open_offline(props.path.c_str(), errbuf); + + if ( ! pd ) + { + safe_snprintf(errbuf, sizeof(errbuf), + "pcap_open_offline: %s", tmp_errbuf); + Error(errbuf); + return; + } + + if ( PrecompileFilter(0, props.filter) && SetFilter(0) ) + { + SetHdrSize(); + + if ( ! pd ) + // Was closed, unknown link layer type. + return; + + // We don't put file sources into non-blocking mode as + // otherwise we would not be able to identify the EOF. + + props.selectable_fd = fileno(pcap_file(pd)); + + if ( props.selectable_fd < 0 ) + InternalError("OS does not support selectable pcap fd"); + } + + else + Close(); + + props.is_live = false; + Opened(props); + } + +int PcapSource::ExtractNextPacket(Packet* pkt) + { + const u_char* data = pcap_next(pd, ¤t_hdr); + + if ( ! data ) + { + // Source has gone dry. If it's a network interface, this just means + // it's timed out. If it's a file, though, then the file has been + // exhausted. + if ( ! props.is_live ) + Close(); + + return 0; + } + + pkt->ts = current_hdr.ts.tv_sec + double(current_hdr.ts.tv_usec) / 1e6; + pkt->hdr = ¤t_hdr; + pkt->data = last_data = data; + + if ( current_hdr.len == 0 || current_hdr.caplen == 0 ) + { + Weird("empty_pcap_header", pkt); + return 0; + } + + last_hdr = current_hdr; + last_data = data; + ++stats.received; + return 1; + } + +void PcapSource::DoneWithPacket(Packet* pkt) + { + // Nothing to do. + } + +int PcapSource::PrecompileFilter(int index, const std::string& filter) + { + char errbuf[PCAP_ERRBUF_SIZE]; + + // Compile filter. + BPF_Program* code = new BPF_Program(); + + if ( ! code->Compile(pd, filter.c_str(), netmask, errbuf, sizeof(errbuf)) ) + { + PcapError(); + delete code; + return 0; + } + + // Store it in hash. + HashKey* hash = new HashKey(HashKey(bro_int_t(index))); + BPF_Program* oldcode = filters.Lookup(hash); + if ( oldcode ) + delete oldcode; + + filters.Insert(hash, code); + delete hash; + + return 1; + } + +int PcapSource::SetFilter(int index) + { + char errbuf[PCAP_ERRBUF_SIZE]; + +#if 0 + // We don't want load-level filters for the secondary path. + if ( filter_type == TYPE_FILTER_SECONDARY && index > 0 ) + return 1; +#endif + + HashKey* hash = new HashKey(HashKey(bro_int_t(index))); + BPF_Program* code = filters.Lookup(hash); + delete hash; + + if ( ! code ) + { + safe_snprintf(errbuf, sizeof(errbuf), + "No precompiled pcap filter for index %d", + index); + Error(errbuf); + return 0; + } + + if ( pcap_setfilter(pd, code->GetProgram()) < 0 ) + { + PcapError(); + return 0; + } + +#ifndef HAVE_LINUX + // Linux doesn't clear counters when resetting filter. + stats.received = stats.dropped = stats.link = 0; +#endif + + return 1; + } + +void PcapSource::Statistics(Stats* s) + { + char errbuf[PCAP_ERRBUF_SIZE]; + + if ( ! props.is_live ) + s->received = s->dropped = s->link = 0; + + else + { + struct pcap_stat pstat; + if ( pcap_stats(pd, &pstat) < 0 ) + { + PcapError(); + s->received = s->dropped = s->link = 0; + } + + else + { + s->dropped = pstat.ps_drop; + s->link = pstat.ps_recv; + } + } + + s->received = stats.received; + + if ( ! props.is_live ) + s->dropped = 0; + } + +bool PcapSource::GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt) + { + if ( ! last_data ) + return false; + + *hdr = &last_hdr; + *pkt = last_data; + return true; + } + +void PcapSource::PcapError() + { + assert(pd); + Error(fmt("pcap_error: %s", pcap_geterr(pd))); + Close(); + } + +void PcapSource::SetHdrSize() + { + char errbuf[PCAP_ERRBUF_SIZE]; + + props.link_type = pcap_datalink(pd); + props.hdr_size = GetLinkHeaderSize(props.link_type); + + if ( props.hdr_size < 0 ) + { + safe_snprintf(errbuf, sizeof(errbuf), + "unknown data link type 0x%x", props.link_type); + Error(errbuf); + Close(); + } + } + +iosource::PktSrc* PcapSource::Instantiate(const std::string& path, const std::string& filter, bool is_live) + { + return new PcapSource(path, filter, is_live); + } diff --git a/src/iosource/pktsrc/pcap/Source.h b/src/iosource/pktsrc/pcap/Source.h new file mode 100644 index 0000000000..9f1d7c7eb8 --- /dev/null +++ b/src/iosource/pktsrc/pcap/Source.h @@ -0,0 +1,60 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef IOSOURCE_PKTSRC_PCAP_SOURCE_H +#define IOSOURCE_PKTSRC_PCAP_SOURCE_H + +extern "C" { +#include +} + +#include "../PktSrc.h" +#include "BPF_Program.h" +#include "Dict.h" + +declare(PDict,BPF_Program); + +namespace iosource { +namespace pktsrc { + +class PcapSource : public iosource::PktSrc { +public: + // XXX + PcapSource(const std::string& path, const std::string& filter, bool is_live); + + virtual ~PcapSource(); + + static PktSrc* Instantiate(const std::string& path, const std::string& filter, bool is_live); + +protected: + // PktSrc interface. + virtual void Open(); + virtual void Close(); + virtual int ExtractNextPacket(Packet* pkt); + virtual void DoneWithPacket(Packet* pkt); + virtual int PrecompileFilter(int index, const std::string& filter); + virtual int SetFilter(int index); + virtual void Statistics(Stats* stats); + virtual bool GetCurrentPacket(const pcap_pkthdr** hdr, const u_char** pkt); + +private: + void OpenLive(); + void OpenOffline(); + void PcapError(); + void SetHdrSize(); + + Properties props; + Stats stats; + + pcap_t *pd; + uint32 netmask; + PDict(BPF_Program) filters; + + struct pcap_pkthdr current_hdr; + struct pcap_pkthdr last_hdr; + const u_char* last_data; +}; + +} +} + +#endif diff --git a/src/main.cc b/src/main.cc index 53a0cb20ee..cf4bd423b6 100644 --- a/src/main.cc +++ b/src/main.cc @@ -62,6 +62,7 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void); #include "plugin/Manager.h" #include "file_analysis/Manager.h" #include "broxygen/Manager.h" +#include "iosource/Manager.h" #include "binpac_bro.h" @@ -100,6 +101,7 @@ plugin::Manager* plugin_mgr = 0; analyzer::Manager* analyzer_mgr = 0; file_analysis::Manager* file_mgr = 0; broxygen::Manager* broxygen_mgr = 0; +iosource::Manager* iosource_mgr = 0; Stmt* stmts; EventHandlerPtr net_done = 0; RuleMatcher* rule_matcher = 0; @@ -116,7 +118,10 @@ int signal_val = 0; int optimize = 0; int do_notice_analysis = 0; int rule_bench = 0; +int generate_documentation = 0; +#if 0 SecondaryPath* secondary_path = 0; +#endif extern char version[]; char* command_line_policy = 0; vector params; @@ -379,7 +384,9 @@ void terminate_bro() delete event_serializer; delete state_serializer; delete event_registry; +#if 0 delete secondary_path; +#endif delete remote_serializer; delete analyzer_mgr; delete log_mgr; @@ -845,6 +852,7 @@ int main(int argc, char** argv) input_mgr = new input::Manager(); plugin_mgr = new plugin::Manager(); file_mgr = new file_analysis::Manager(); + iosource_mgr = new iosource::Manager(); plugin_mgr->InitPreScript(); analyzer_mgr->InitPreScript(); @@ -967,13 +975,15 @@ int main(int argc, char** argv) snaplen = internal_val("snaplen")->AsCount(); +#if 0 // Initialize the secondary path, if it's needed. secondary_path = new SecondaryPath(); +#endif if ( dns_type != DNS_PRIME ) net_init(interfaces, read_files, netflows, flow_files, writefile, "", - secondary_path->Filter(), do_watchdog); + "", do_watchdog); BroFile::SetDefaultRotation(log_rotate_interval, log_max_size); @@ -1132,9 +1142,9 @@ int main(int argc, char** argv) have_pending_timers = ! reading_traces && timer_mgr->Size() > 0; - io_sources.Register(thread_mgr, true); + iosource_mgr->Register(thread_mgr, true); - if ( io_sources.Size() > 0 || + if ( iosource_mgr->Size() > 0 || have_pending_timers || BifConst::exit_only_after_terminate ) { diff --git a/src/plugin/Component.cc b/src/plugin/Component.cc index ee18d55cdf..94de2c5446 100644 --- a/src/plugin/Component.cc +++ b/src/plugin/Component.cc @@ -43,6 +43,18 @@ void Component::Describe(ODesc* d) const d->Add("File Analyzer"); break; + case component::IOSOURCE: + d->Add("I/O Source"); + break; + + case component::PKTSRC: + d->Add("Packet Source"); + break; + + case component::PKTDUMPER: + d->Add("Packet Dumper"); + break; + default: reporter->InternalWarning("unknown component type in plugin::Component::Describe"); d->Add(""); diff --git a/src/plugin/Component.h b/src/plugin/Component.h index ad02dc7e4b..04248f54a9 100644 --- a/src/plugin/Component.h +++ b/src/plugin/Component.h @@ -3,6 +3,8 @@ #ifndef PLUGIN_COMPONENT_H #define PLUGIN_COMPONENT_H +#include + class ODesc; namespace plugin { @@ -16,7 +18,10 @@ enum Type { READER, /// An input reader (not currently used). WRITER, /// An logging writer (not currenly used). ANALYZER, /// A protocol analyzer. - FILE_ANALYZER /// A file analyzer. + FILE_ANALYZER, /// A file analyzer. + IOSOURCE, /// An I/O source, excluding packet sources. + PKTSRC, /// A packet source. + PKTDUMPER /// A packet dumper. }; } diff --git a/src/plugin/Macros.h b/src/plugin/Macros.h index 9362642e91..c2f8a2ed7a 100644 --- a/src/plugin/Macros.h +++ b/src/plugin/Macros.h @@ -130,4 +130,31 @@ #define BRO_PLUGIN_SUPPORT_ANALYZER(tag) \ AddComponent(new ::analyzer::Component(tag, 0)); +/** + * Defines a component implementing an IOSource (excluding packet sources). + * + * XXXX + * + */ +#define BRO_PLUGIN_IOSOURCE(tag, cls) \ + AddComponent(new ::iosource::pktsrc::Component(tag, iosource::cls::Instantiate)); + +/** + * Defines a component implementing a PktSrc. + * + * XXXX + * + */ +#define BRO_PLUGIN_PKTSRC(tag, prefix, type, cls) \ + AddComponent(new ::iosource::pktsrc::SourceComponent(tag, prefix, iosource::pktsrc::type, iosource::pktsrc::cls::Instantiate)); + +/** + * Defines a component implementing a PktDumper. + * + * XXXX + * + */ +#define BRO_PLUGIN_PKTDUMPER(tag, prefix, cls) \ + AddComponent(new ::iosource::pktsrc::DumperComponent(tag, prefix, iosource::pktsrc::cls::Instantiate)); + #endif diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index 4491cd42b5..fdc422bd1f 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -11,7 +11,7 @@ Manager::Manager() did_process = true; next_beat = 0; terminating = false; - idle = true; + SetIdle(true); } Manager::~Manager() @@ -47,8 +47,8 @@ void Manager::Terminate() all_threads.clear(); msg_threads.clear(); - idle = true; - closed = true; + SetIdle(true); + SetClosed(true); terminating = false; } @@ -56,7 +56,7 @@ void Manager::AddThread(BasicThread* thread) { DBG_LOG(DBG_THREADING, "Adding thread %s ...", thread->Name()); all_threads.push_back(thread); - idle = false; + SetIdle(false); } void Manager::AddMsgThread(MsgThread* thread) diff --git a/src/threading/Manager.h b/src/threading/Manager.h index e839749a91..c94cc41aaa 100644 --- a/src/threading/Manager.h +++ b/src/threading/Manager.h @@ -4,7 +4,7 @@ #include -#include "IOSource.h" +#include "iosource/IOSource.h" #include "BasicThread.h" #include "MsgThread.h" @@ -21,7 +21,7 @@ namespace threading { * their outgoing message queue on a regular basis and feeds data sent into * the rest of Bro. It also triggers the regular heartbeats. */ -class Manager : public IOSource +class Manager : public iosource::IOSource { public: /** diff --git a/src/util.cc b/src/util.cc index ad55e3f75e..358e88b0e2 100644 --- a/src/util.cc +++ b/src/util.cc @@ -43,6 +43,7 @@ #include "NetVar.h" #include "Net.h" #include "Reporter.h" +#include "iosource/Manager.h" /** * Return IP address without enclosing brackets and any leading 0x. @@ -1351,11 +1352,13 @@ double current_time(bool real) double t = double(tv.tv_sec) + double(tv.tv_usec) / 1e6; - if ( ! pseudo_realtime || real || pkt_srcs.length() == 0 ) + const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); + + if ( ! pseudo_realtime || real || pkt_srcs.empty() ) return t; // This obviously only works for a single source ... - PktSrc* src = pkt_srcs[0]; + iosource::PktSrc* src = pkt_srcs.front(); if ( net_is_processing_suspended() ) return src->CurrentPacketTimestamp(); From 61ee2b9172a10c3f0e483b90a0947fd54f3e989f Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 11 Dec 2013 20:55:17 -0800 Subject: [PATCH 009/193] Fixing rebase relicts. --- src/CMakeLists.txt | 3 --- src/Net.h | 4 ---- 2 files changed, 7 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1aede44934..516426ea02 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -151,7 +151,6 @@ list(APPEND BINPAC_OUTPUTS "${BINPAC_OUTPUT_CC}") set(bro_SUBDIR_LIBS CACHE INTERNAL "subdir libraries" FORCE) set(bro_PLUGIN_LIBS CACHE INTERNAL "plugin libraries" FORCE) -add_subdirectory(iosource) add_subdirectory(analyzer) add_subdirectory(file_analysis) add_subdirectory(probabilistic) @@ -251,8 +250,6 @@ set(bro_SRCS Anon.cc Attr.cc Base64.cc - BroDoc.cc - BroDocObj.cc Brofiler.cc BroString.cc CCL.cc diff --git a/src/Net.h b/src/Net.h index 421bee5911..b9320181f9 100644 --- a/src/Net.h +++ b/src/Net.h @@ -4,11 +4,7 @@ #define net_h #include "net_util.h" -<<<<<<< HEAD #include "util.h" -#include "BPF_Program.h" -======= ->>>>>>> 5493253... Checkpoint. #include "List.h" #include "FlowSrc.h" #include "Func.h" From e9413c936134c90d15c27639e8da124bb7aae181 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 12 Dec 2013 16:50:56 -0800 Subject: [PATCH 010/193] New default for plugin installation path. --- CMakeLists.txt | 2 +- aux/bro-aux | 2 +- cmake | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0738eb387d..c006237850 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,7 +22,7 @@ set(BRO_SCRIPT_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/scripts) get_filename_component(BRO_SCRIPT_INSTALL_PATH ${BRO_SCRIPT_INSTALL_PATH} ABSOLUTE) -set(BRO_PLUGIN_INSTALL_PATH ${BRO_ROOT_DIR}/share/bro/plugins) +set(BRO_PLUGIN_INSTALL_PATH ${BRO_ROOT_DIR}/lib/bro/plugins) set(BRO_MAGIC_INSTALL_PATH ${BRO_ROOT_DIR}/share/bro/magic) set(BRO_MAGIC_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/magic/database) diff --git a/aux/bro-aux b/aux/bro-aux index 6c5db7af6e..715937ad72 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 6c5db7af6e93406be498b24f8a2d67ca281c1a03 +Subproject commit 715937ad72d9158d8c3142f81ee7005601d75db8 diff --git a/cmake b/cmake index fc53d57770..6eb4cbfbc7 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit fc53d57770c86fbf9bd2d9694f06a1d539ebe35f +Subproject commit 6eb4cbfbc75f0bd13e247150b19932905a85e10c From 987452befff1e5bd99689eee2f97c18e43ef1cfd Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 12 Dec 2013 17:39:03 -0800 Subject: [PATCH 011/193] Cleanup of plugin component API. - Move more functionality into base class. - Remove cctors and assignment operators (weren't actually needed anymore) - Switch from const char* to std::string. --- src/BroDoc.cc | 4 +-- src/RuleAction.cc | 6 ++--- src/analyzer/Analyzer.cc | 4 +-- src/analyzer/Component.cc | 44 +++----------------------------- src/analyzer/Component.h | 34 +++++++----------------- src/analyzer/Manager.cc | 24 +++++++++-------- src/file_analysis/Analyzer.cc | 2 +- src/file_analysis/AnalyzerSet.cc | 12 ++++----- src/file_analysis/Component.cc | 41 +++-------------------------- src/file_analysis/Component.h | 34 +++++++----------------- src/file_analysis/File.cc | 4 +-- src/file_analysis/Manager.cc | 10 ++++---- src/plugin/Component.cc | 12 ++++++++- src/plugin/Component.h | 34 +++++++++++++++++++----- src/plugin/ComponentManager.h | 22 ++++++++-------- src/plugin/TaggedComponent.h | 32 ----------------------- src/util.cc | 9 +++---- src/util.h | 2 +- 18 files changed, 114 insertions(+), 216 deletions(-) diff --git a/src/BroDoc.cc b/src/BroDoc.cc index 93d8a34848..0a1006bd07 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -473,7 +473,7 @@ static void WritePluginSectionHeading(FILE* f, const plugin::Plugin* p) static void WriteAnalyzerComponent(FILE* f, const analyzer::Component* c) { EnumType* atag = analyzer_mgr->GetTagEnumType(); - string tag = fmt("ANALYZER_%s", c->CanonicalName()); + string tag = fmt("ANALYZER_%s", c->CanonicalName().c_str()); if ( atag->Lookup("Analyzer", tag.c_str()) < 0 ) reporter->InternalError("missing analyzer tag for %s", tag.c_str()); @@ -484,7 +484,7 @@ static void WriteAnalyzerComponent(FILE* f, const analyzer::Component* c) static void WriteAnalyzerComponent(FILE* f, const file_analysis::Component* c) { EnumType* atag = file_mgr->GetTagEnumType(); - string tag = fmt("ANALYZER_%s", c->CanonicalName()); + string tag = fmt("ANALYZER_%s", c->CanonicalName().c_str()); if ( atag->Lookup("Files", tag.c_str()) < 0 ) reporter->InternalError("missing analyzer tag for %s", tag.c_str()); diff --git a/src/RuleAction.cc b/src/RuleAction.cc index ec57c96bd2..8c6948ad10 100644 --- a/src/RuleAction.cc +++ b/src/RuleAction.cc @@ -60,11 +60,11 @@ RuleActionAnalyzer::RuleActionAnalyzer(const char* arg_analyzer) void RuleActionAnalyzer::PrintDebug() { if ( ! child_analyzer ) - fprintf(stderr, "|%s|\n", analyzer_mgr->GetComponentName(analyzer)); + fprintf(stderr, "|%s|\n", analyzer_mgr->GetComponentName(analyzer).c_str()); else fprintf(stderr, "|%s:%s|\n", - analyzer_mgr->GetComponentName(analyzer), - analyzer_mgr->GetComponentName(child_analyzer)); + analyzer_mgr->GetComponentName(analyzer).c_str(), + analyzer_mgr->GetComponentName(child_analyzer).c_str()); } diff --git a/src/analyzer/Analyzer.cc b/src/analyzer/Analyzer.cc index 2927e3ad97..1e4f0e20f4 100644 --- a/src/analyzer/Analyzer.cc +++ b/src/analyzer/Analyzer.cc @@ -75,7 +75,7 @@ analyzer::ID Analyzer::id_counter = 0; const char* Analyzer::GetAnalyzerName() const { assert(tag); - return analyzer_mgr->GetComponentName(tag); + return analyzer_mgr->GetComponentName(tag).c_str(); } void Analyzer::SetAnalyzerTag(const Tag& arg_tag) @@ -87,7 +87,7 @@ void Analyzer::SetAnalyzerTag(const Tag& arg_tag) bool Analyzer::IsAnalyzer(const char* name) { assert(tag); - return strcmp(analyzer_mgr->GetComponentName(tag), name) == 0; + return strcmp(analyzer_mgr->GetComponentName(tag).c_str(), name) == 0; } // Used in debugging output. diff --git a/src/analyzer/Component.cc b/src/analyzer/Component.cc index 255e03435c..91528b29d8 100644 --- a/src/analyzer/Component.cc +++ b/src/analyzer/Component.cc @@ -8,12 +8,11 @@ using namespace analyzer; -Component::Component(const char* arg_name, factory_callback arg_factory, Tag::subtype_t arg_subtype, bool arg_enabled, bool arg_partial) - : plugin::Component(plugin::component::ANALYZER), +Component::Component(const std::string& name, factory_callback arg_factory, Tag::subtype_t arg_subtype, bool arg_enabled, bool arg_partial) + : plugin::Component(plugin::component::ANALYZER, name), plugin::TaggedComponent(arg_subtype) { - name = copy_string(arg_name); - canon_name = canonify_name(arg_name); + canon_name = canonify_name(name); factory = arg_factory; enabled = arg_enabled; partial = arg_partial; @@ -21,31 +20,12 @@ Component::Component(const char* arg_name, factory_callback arg_factory, Tag::su analyzer_mgr->RegisterComponent(this, "ANALYZER_"); } -Component::Component(const Component& other) - : plugin::Component(Type()), - plugin::TaggedComponent(other) - { - name = copy_string(other.name); - canon_name = copy_string(other.canon_name); - factory = other.factory; - enabled = other.enabled; - partial = other.partial; - - // TODO: Do we need the RegisterComponent() call here? - } - Component::~Component() { - delete [] name; - delete [] canon_name; } -void Component::Describe(ODesc* d) const +void Component::DoDescribe(ODesc* d) const { - plugin::Component::Describe(d); - d->Add(name); - d->Add(" ("); - if ( factory ) { d->Add("ANALYZER_"); @@ -54,20 +34,4 @@ void Component::Describe(ODesc* d) const } d->Add(enabled ? "enabled" : "disabled"); - d->Add(")"); - } - -Component& Component::operator=(const Component& other) - { - plugin::TaggedComponent::operator=(other); - - if ( &other != this ) - { - name = copy_string(other.name); - factory = other.factory; - enabled = other.enabled; - partial = other.partial; - } - - return *this; } diff --git a/src/analyzer/Component.h b/src/analyzer/Component.h index 9bc8b357d7..d8a62bb38c 100644 --- a/src/analyzer/Component.h +++ b/src/analyzer/Component.h @@ -1,7 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. -#ifndef ANALYZER_PLUGIN_COMPONENT_H -#define ANALYZER_PLUGIN_COMPONENT_H +#ifndef ANALYZER_COMPONENT_H +#define ANALYZER_COMPONENT_H #include "Tag.h" #include "plugin/Component.h" @@ -56,33 +56,20 @@ public: * connections has generally not seen much testing yet as virtually * no existing analyzer supports it. */ - Component(const char* name, factory_callback factory, Tag::subtype_t subtype = 0, bool enabled = true, bool partial = false); - - /** - * Copy constructor. - */ - Component(const Component& other); + Component(const std::string& name, factory_callback factory, Tag::subtype_t subtype = 0, bool enabled = true, bool partial = false); /** * Destructor. */ ~Component(); - /** - * Returns the name of the analyzer. This name is unique across all - * analyzers and used to identify it. The returned name is derived - * from what's passed to the constructor but upper-cased and - * canonified to allow being part of a script-level ID. - */ - virtual const char* Name() const { return name; } - /** * Returns a canonocalized version of the analyzer's name. The * returned name is derived from what's passed to the constructor but * upper-cased and transformed to allow being part of a script-level * ID. */ - const char* CanonicalName() const { return canon_name; } + const std::string& CanonicalName() const { return canon_name; } /** * Returns the analyzer's factory function. @@ -110,17 +97,14 @@ public: */ void SetEnabled(bool arg_enabled) { enabled = arg_enabled; } +protected: /** - * Generates a human-readable description of the component's main - * parameters. This goes into the output of \c "bro -NN". - */ - virtual void Describe(ODesc* d) const; - - Component& operator=(const Component& other); + * Overriden from plugin::Component. + */ + virtual void DoDescribe(ODesc* d) const; private: - const char* name; // The analyzer's name. - const char* canon_name; // The analyzer's canonical name. + std::string canon_name; // The analyzer's canonical name. factory_callback factory; // The analyzer's factory callback. bool partial; // True if the analyzer supports partial connections. bool enabled; // True if the analyzer is enabled. diff --git a/src/analyzer/Manager.cc b/src/analyzer/Manager.cc index 748af34dfb..e2cd2a7e05 100644 --- a/src/analyzer/Manager.cc +++ b/src/analyzer/Manager.cc @@ -104,7 +104,8 @@ void Manager::DumpDebug() DBG_LOG(DBG_ANALYZER, "Available analyzers after bro_init():"); list all_analyzers = GetComponents(); for ( list::const_iterator i = all_analyzers.begin(); i != all_analyzers.end(); ++i ) - DBG_LOG(DBG_ANALYZER, " %s (%s)", (*i)->Name(), IsEnabled((*i)->Tag()) ? "enabled" : "disabled"); + DBG_LOG(DBG_ANALYZER, " %s (%s)", (*i)->Name().c_str(), + IsEnabled((*i)->Tag()) ? "enabled" : "disabled"); DBG_LOG(DBG_ANALYZER, ""); DBG_LOG(DBG_ANALYZER, "Analyzers by port:"); @@ -143,7 +144,7 @@ bool Manager::EnableAnalyzer(Tag tag) if ( ! p ) return false; - DBG_LOG(DBG_ANALYZER, "Enabling analyzer %s", p->Name()); + DBG_LOG(DBG_ANALYZER, "Enabling analyzer %s", p->Name().c_str()); p->SetEnabled(true); return true; @@ -156,7 +157,7 @@ bool Manager::EnableAnalyzer(EnumVal* val) if ( ! p ) return false; - DBG_LOG(DBG_ANALYZER, "Enabling analyzer %s", p->Name()); + DBG_LOG(DBG_ANALYZER, "Enabling analyzer %s", p->Name().c_str()); p->SetEnabled(true); return true; @@ -169,7 +170,7 @@ bool Manager::DisableAnalyzer(Tag tag) if ( ! p ) return false; - DBG_LOG(DBG_ANALYZER, "Disabling analyzer %s", p->Name()); + DBG_LOG(DBG_ANALYZER, "Disabling analyzer %s", p->Name().c_str()); p->SetEnabled(false); return true; @@ -182,7 +183,7 @@ bool Manager::DisableAnalyzer(EnumVal* val) if ( ! p ) return false; - DBG_LOG(DBG_ANALYZER, "Disabling analyzer %s", p->Name()); + DBG_LOG(DBG_ANALYZER, "Disabling analyzer %s", p->Name().c_str()); p->SetEnabled(false); return true; @@ -254,7 +255,7 @@ bool Manager::RegisterAnalyzerForPort(Tag tag, TransportProto proto, uint32 port return false; #ifdef DEBUG - const char* name = GetComponentName(tag); + const char* name = GetComponentName(tag).c_str(); DBG_LOG(DBG_ANALYZER, "Registering analyzer %s for port %" PRIu32 "/%d", name, port, proto); #endif @@ -270,7 +271,7 @@ bool Manager::UnregisterAnalyzerForPort(Tag tag, TransportProto proto, uint32 po return true; // still a "successful" unregistration #ifdef DEBUG - const char* name = GetComponentName(tag); + const char* name = GetComponentName(tag).c_str(); DBG_LOG(DBG_ANALYZER, "Unregistering analyzer %s for port %" PRIu32 "/%d", name, port, proto); #endif @@ -293,7 +294,8 @@ Analyzer* Manager::InstantiateAnalyzer(Tag tag, Connection* conn) if ( ! c->Factory() ) { - reporter->InternalWarning("analyzer %s cannot be instantiated dynamically", GetComponentName(tag)); + reporter->InternalWarning("analyzer %s cannot be instantiated dynamically", + GetComponentName(tag).c_str()); return 0; } @@ -403,7 +405,7 @@ bool Manager::BuildInitialAnalyzerTree(Connection* conn) root->AddChildAnalyzer(analyzer, false); DBG_ANALYZER_ARGS(conn, "activated %s analyzer as scheduled", - analyzer_mgr->GetComponentName(*i)); + analyzer_mgr->GetComponentName(*i).c_str()); } } @@ -429,7 +431,7 @@ bool Manager::BuildInitialAnalyzerTree(Connection* conn) root->AddChildAnalyzer(analyzer, false); DBG_ANALYZER_ARGS(conn, "activated %s analyzer due to port %d", - analyzer_mgr->GetComponentName(*j), resp_port); + analyzer_mgr->GetComponentName(*j).c_str(), resp_port); } } } @@ -555,7 +557,7 @@ void Manager::ExpireScheduledAnalyzers() conns.erase(i); DBG_LOG(DBG_ANALYZER, "Expiring expected analyzer %s for connection %s", - analyzer_mgr->GetComponentName(a->analyzer), + analyzer_mgr->GetComponentName(a->analyzer).c_str(), fmt_conn_id(a->conn.orig, 0, a->conn.resp, a->conn.resp_p)); delete a; diff --git a/src/file_analysis/Analyzer.cc b/src/file_analysis/Analyzer.cc index 2d45c6579a..c04c02f818 100644 --- a/src/file_analysis/Analyzer.cc +++ b/src/file_analysis/Analyzer.cc @@ -8,7 +8,7 @@ file_analysis::ID file_analysis::Analyzer::id_counter = 0; file_analysis::Analyzer::~Analyzer() { DBG_LOG(DBG_FILE_ANALYSIS, "Destroy file analyzer %s", - file_mgr->GetComponentName(tag)); + file_mgr->GetComponentName(tag).c_str()); Unref(args); } diff --git a/src/file_analysis/AnalyzerSet.cc b/src/file_analysis/AnalyzerSet.cc index aee5e0d62e..b3f11b6816 100644 --- a/src/file_analysis/AnalyzerSet.cc +++ b/src/file_analysis/AnalyzerSet.cc @@ -53,7 +53,7 @@ bool AnalyzerSet::Add(file_analysis::Tag tag, RecordVal* args) if ( analyzer_map.Lookup(key) ) { DBG_LOG(DBG_FILE_ANALYSIS, "Instantiate analyzer %s skipped for file id" - " %s: already exists", file_mgr->GetComponentName(tag), + " %s: already exists", file_mgr->GetComponentName(tag).c_str(), file->GetID().c_str()); delete key; return true; @@ -93,7 +93,7 @@ bool AnalyzerSet::AddMod::Perform(AnalyzerSet* set) if ( set->analyzer_map.Lookup(key) ) { DBG_LOG(DBG_FILE_ANALYSIS, "Add analyzer %s skipped for file id" - " %s: already exists", file_mgr->GetComponentName(a->Tag()), + " %s: already exists", file_mgr->GetComponentName(a->Tag()).c_str(), a->GetFile()->GetID().c_str()); Abort(); @@ -120,12 +120,12 @@ bool AnalyzerSet::Remove(file_analysis::Tag tag, HashKey* key) if ( ! a ) { DBG_LOG(DBG_FILE_ANALYSIS, "Skip remove analyzer %s for file id %s", - file_mgr->GetComponentName(tag), file->GetID().c_str()); + file_mgr->GetComponentName(tag).c_str(), file->GetID().c_str()); return false; } DBG_LOG(DBG_FILE_ANALYSIS, "Remove analyzer %s for file id %s", - file_mgr->GetComponentName(tag), + file_mgr->GetComponentName(tag).c_str(), file->GetID().c_str()); a->Done(); @@ -169,7 +169,7 @@ file_analysis::Analyzer* AnalyzerSet::InstantiateAnalyzer(Tag tag, if ( ! a ) { reporter->Error("Failed file analyzer %s instantiation for file id %s", - file_mgr->GetComponentName(tag), file->GetID().c_str()); + file_mgr->GetComponentName(tag).c_str(), file->GetID().c_str()); return 0; } @@ -179,7 +179,7 @@ file_analysis::Analyzer* AnalyzerSet::InstantiateAnalyzer(Tag tag, void AnalyzerSet::Insert(file_analysis::Analyzer* a, HashKey* key) { DBG_LOG(DBG_FILE_ANALYSIS, "Add analyzer %s for file id %s", - file_mgr->GetComponentName(a->Tag()), file->GetID().c_str()); + file_mgr->GetComponentName(a->Tag()).c_str(), file->GetID().c_str()); analyzer_map.Insert(key, a); delete key; diff --git a/src/file_analysis/Component.cc b/src/file_analysis/Component.cc index 87789eb113..dc30b95986 100644 --- a/src/file_analysis/Component.cc +++ b/src/file_analysis/Component.cc @@ -8,58 +8,25 @@ using namespace file_analysis; -Component::Component(const char* arg_name, factory_callback arg_factory, Tag::subtype_t subtype) - : plugin::Component(plugin::component::FILE_ANALYZER), +Component::Component(const std::string& name, factory_callback arg_factory, Tag::subtype_t subtype) + : plugin::Component(plugin::component::FILE_ANALYZER, name), plugin::TaggedComponent(subtype) { - name = copy_string(arg_name); - canon_name = canonify_name(arg_name); + canon_name = canonify_name(name); factory = arg_factory; file_mgr->RegisterComponent(this, "ANALYZER_"); } -Component::Component(const Component& other) - : plugin::Component(Type()), - plugin::TaggedComponent(other) - { - name = copy_string(other.name); - canon_name = copy_string(other.canon_name); - factory = other.factory; - - // TODO: Do we need the RegisterComponent() call here? - } - Component::~Component() { - delete [] name; - delete [] canon_name; } -void Component::Describe(ODesc* d) const +void Component::DoDescribe(ODesc* d) const { - plugin::Component::Describe(d); - d->Add(name); - d->Add(" ("); - if ( factory ) { d->Add("ANALYZER_"); d->Add(canon_name); } - - d->Add(")"); - } - -Component& Component::operator=(const Component& other) - { - plugin::TaggedComponent::operator=(other); - - if ( &other != this ) - { - name = copy_string(other.name); - factory = other.factory; - } - - return *this; } diff --git a/src/file_analysis/Component.h b/src/file_analysis/Component.h index 6e2809349d..8f3b628a32 100644 --- a/src/file_analysis/Component.h +++ b/src/file_analysis/Component.h @@ -1,7 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. -#ifndef FILE_ANALYZER_PLUGIN_COMPONENT_H -#define FILE_ANALYZER_PLUGIN_COMPONENT_H +#ifndef FILE_ANALYZER_COMPONENT_H +#define FILE_ANALYZER_COMPONENT_H #include "Tag.h" #include "plugin/Component.h" @@ -47,50 +47,34 @@ public: * analyzer instances can accordingly access it via analyzer::Tag(). * If not used, leave at zero. */ - Component(const char* name, factory_callback factory, Tag::subtype_t subtype = 0); - - /** - * Copy constructor. - */ - Component(const Component& other); + Component(const std::string& name, factory_callback factory, Tag::subtype_t subtype = 0); /** * Destructor. */ ~Component(); - /** - * Returns the name of the analyzer. This name is unique across all - * analyzers and used to identify it. The returned name is derived - * from what's passed to the constructor but upper-cased and - * canonified to allow being part of a script-level ID. - */ - virtual const char* Name() const { return name; } - /** * Returns a canonocalized version of the analyzer's name. The * returned name is derived from what's passed to the constructor but * upper-cased and transformed to allow being part of a script-level * ID. */ - const char* CanonicalName() const { return canon_name; } + const std::string& CanonicalName() const { return canon_name; } /** * Returns the analyzer's factory function. */ factory_callback Factory() const { return factory; } +protected: /** - * Generates a human-readable description of the component's main - * parameters. This goes into the output of \c "bro -NN". - */ - virtual void Describe(ODesc* d) const; - - Component& operator=(const Component& other); + * Overriden from plugin::Component. + */ + virtual void DoDescribe(ODesc* d) const; private: - const char* name; // The analyzer's name. - const char* canon_name; // The analyzer's canonical name. + std::string canon_name; // The analyzer's canonical name. factory_callback factory; // The analyzer's factory callback. }; diff --git a/src/file_analysis/File.cc b/src/file_analysis/File.cc index 980a8bfbce..6c15f47094 100644 --- a/src/file_analysis/File.cc +++ b/src/file_analysis/File.cc @@ -254,7 +254,7 @@ void File::ScheduleInactivityTimer() const bool File::AddAnalyzer(file_analysis::Tag tag, RecordVal* args) { DBG_LOG(DBG_FILE_ANALYSIS, "[%s] Queuing addition of %s analyzer", - id.c_str(), file_mgr->GetComponentName(tag)); + id.c_str(), file_mgr->GetComponentName(tag).c_str()); return done ? false : analyzers.QueueAdd(tag, args); } @@ -262,7 +262,7 @@ bool File::AddAnalyzer(file_analysis::Tag tag, RecordVal* args) bool File::RemoveAnalyzer(file_analysis::Tag tag, RecordVal* args) { DBG_LOG(DBG_FILE_ANALYSIS, "[%s] Queuing remove of %s analyzer", - id.c_str(), file_mgr->GetComponentName(tag)); + id.c_str(), file_mgr->GetComponentName(tag).c_str()); return done ? false : analyzers.QueueRemove(tag, args); } diff --git a/src/file_analysis/Manager.cc b/src/file_analysis/Manager.cc index a635a5dc14..de930e7589 100644 --- a/src/file_analysis/Manager.cc +++ b/src/file_analysis/Manager.cc @@ -351,7 +351,7 @@ void Manager::GetFileHandle(analyzer::Tag tag, Connection* c, bool is_orig) return; DBG_LOG(DBG_FILE_ANALYSIS, "Raise get_file_handle() for protocol analyzer %s", - analyzer_mgr->GetComponentName(tag)); + analyzer_mgr->GetComponentName(tag).c_str()); EnumVal* tagval = tag.AsEnumVal(); Ref(tagval); @@ -398,12 +398,12 @@ Analyzer* Manager::InstantiateAnalyzer(Tag tag, RecordVal* args, File* f) const if ( ! c->Factory() ) { reporter->InternalWarning("file analyzer %s cannot be instantiated " - "dynamically", c->CanonicalName()); + "dynamically", c->CanonicalName().c_str()); return 0; } DBG_LOG(DBG_FILE_ANALYSIS, "Instantiate analyzer %s for file %s", - GetComponentName(tag), f->id.c_str()); + GetComponentName(tag).c_str(), f->id.c_str()); Analyzer* a = c->Factory()(args, f); @@ -445,7 +445,7 @@ bool Manager::RegisterAnalyzerForMIMEType(Tag tag, const string& mtype) TagSet* l = LookupMIMEType(mtype, true); DBG_LOG(DBG_FILE_ANALYSIS, "Register analyzer %s for MIME type %s", - GetComponentName(tag), mtype.c_str()); + GetComponentName(tag).c_str(), mtype.c_str()); l->insert(tag); return true; @@ -466,7 +466,7 @@ bool Manager::UnregisterAnalyzerForMIMEType(Tag tag, const string& mtype) TagSet* l = LookupMIMEType(mtype, true); DBG_LOG(DBG_FILE_ANALYSIS, "Unregister analyzer %s for MIME type %s", - GetComponentName(tag), mtype.c_str()); + GetComponentName(tag).c_str(), mtype.c_str()); l->erase(tag); return true; diff --git a/src/plugin/Component.cc b/src/plugin/Component.cc index ee18d55cdf..a3d2eef194 100644 --- a/src/plugin/Component.cc +++ b/src/plugin/Component.cc @@ -7,15 +7,21 @@ using namespace plugin; -Component::Component(component::Type arg_type) +Component::Component(component::Type arg_type, const std::string& arg_name) { type = arg_type; + name = arg_name; } Component::~Component() { } +const std::string& Component::Name() const + { + return name; + } + component::Type Component::Type() const { return type; @@ -51,4 +57,8 @@ void Component::Describe(ODesc* d) const d->Add("]"); d->Add(" "); + d->Add(name); + d->Add(" ("); + DoDescribe(d); + d->Add(")"); } diff --git a/src/plugin/Component.h b/src/plugin/Component.h index ad02dc7e4b..0b6b5e9c49 100644 --- a/src/plugin/Component.h +++ b/src/plugin/Component.h @@ -3,6 +3,8 @@ #ifndef PLUGIN_COMPONENT_H #define PLUGIN_COMPONENT_H +#include + class ODesc; namespace plugin { @@ -32,8 +34,11 @@ public: * Constructor. * * @param type The type of the compoment. + * + * @param name A descriptive name for the component. This name must + * be unique across all components of the same type. */ - Component(component::Type type); + Component(component::Type type, const std::string& name); /** * Destructor. @@ -46,22 +51,37 @@ public: component::Type Type() const; /** - * Returns a descriptive name for the analyzer. This name must be - * unique across all components of the same type. + * Returns the compoment's name. */ - virtual const char* Name() const = 0; + const std::string& Name() const; /** - * Returns a textual representation of the component. The default - * version just output the type. Derived version should call the - * parent's implementation and that add further information. + * Returns a textual representation of the component. This goes into + * the output of "bro -NN". + * + * By default version, this just outputs the type and the name. + * Derived versions should override DoDescribe() to add type specific + * details. * * @param d The description object to use. */ virtual void Describe(ODesc* d) const; +protected: + /** + * Adds type specific information to the outout of Describe(). + * + * @param d The description object to use. + */ + virtual void DoDescribe(ODesc* d) const { } + private: + // Disable. + Component(const Component& other); + Component operator=(const Component& other); + component::Type type; + std::string name; }; } diff --git a/src/plugin/ComponentManager.h b/src/plugin/ComponentManager.h index 71994d82d1..b3ad944bec 100644 --- a/src/plugin/ComponentManager.h +++ b/src/plugin/ComponentManager.h @@ -37,7 +37,7 @@ public: /** * @return The script-layer module in which the component's "Tag" ID lives. */ - const char* GetModule() const; + const std::string& GetModule() const; /** * @return A list of all registered components. @@ -55,7 +55,7 @@ public: * @param tag A component's tag. * @return The canonical component name. */ - const char* GetComponentName(T tag) const; + const std::string& GetComponentName(T tag) const; /** * Get a component name from it's enum value. @@ -63,7 +63,7 @@ public: * @param val A component's enum value. * @return The canonical component name. */ - const char* GetComponentName(Val* val) const; + const std::string& GetComponentName(Val* val) const; /** * Get a component tag from its name. @@ -134,7 +134,7 @@ ComponentManager::ComponentManager(const string& arg_module) } template -const char* ComponentManager::GetModule() const +const std::string& ComponentManager::GetModule() const { return module.c_str(); } @@ -158,9 +158,9 @@ EnumType* ComponentManager::GetTagEnumType() const } template -const char* ComponentManager::GetComponentName(T tag) const +const std::string& ComponentManager::GetComponentName(T tag) const { - static const char* error = ""; + static const std::string& error = ""; if ( ! tag ) return error; @@ -176,7 +176,7 @@ const char* ComponentManager::GetComponentName(T tag) const } template -const char* ComponentManager::GetComponentName(Val* val) const +const std::string& ComponentManager::GetComponentName(Val* val) const { return GetComponentName(T(val->AsEnumVal())); } @@ -222,14 +222,14 @@ template void ComponentManager::RegisterComponent(C* component, const string& prefix) { - const char* cname = component->CanonicalName(); + std::string cname = component->CanonicalName(); if ( Lookup(cname) ) reporter->FatalError("Component '%s::%s' defined more than once", - module.c_str(), cname); + module.c_str(), cname.c_str()); DBG_LOG(DBG_PLUGINS, "Registering component %s (tag %s)", - component->Name(), component->Tag().AsString().c_str()); + component->Name().c_str(), component->Tag().AsString().c_str()); components_by_name.insert(std::make_pair(cname, component)); components_by_tag.insert(std::make_pair(component->Tag(), component)); @@ -237,7 +237,7 @@ void ComponentManager::RegisterComponent(C* component, component->Tag().AsEnumVal()->InternalInt(), component)); // Install an identfier for enum value - string id = fmt("%s%s", prefix.c_str(), cname); + string id = fmt("%s%s", prefix.c_str(), cname.c_str()); tag_enum_type->AddName(module, id.c_str(), component->Tag().AsEnumVal()->InternalInt(), true); } diff --git a/src/plugin/TaggedComponent.h b/src/plugin/TaggedComponent.h index 99eab9f230..7e3ee24bdf 100644 --- a/src/plugin/TaggedComponent.h +++ b/src/plugin/TaggedComponent.h @@ -23,28 +23,12 @@ public: */ TaggedComponent(typename T::subtype_t subtype = 0); - /** - * Copy constructor. - * - * @param other Another component from which to copy its tag value. - */ - TaggedComponent(const TaggedComponent& other); - - /** - * Assignment operator. - * - * @param other A component to assign. - * @return The assigned object. - */ - TaggedComponent& operator=(const TaggedComponent& other); - /** * @return The component's tag. */ T Tag() const; private: - T tag; /**< The automatically assigned analyzer tag. */ static typename T::type_t type_counter; /**< Used to generate globally unique tags. */ @@ -56,22 +40,6 @@ TaggedComponent::TaggedComponent(typename T::subtype_t subtype) tag = T(++type_counter, subtype); } -template -TaggedComponent::TaggedComponent(const TaggedComponent& other) - { - tag = other.tag; - } - -template -TaggedComponent& -TaggedComponent::operator =(const TaggedComponent& other) - { - if ( &other != this ) - tag = other.tag; - - return *this; - } - template T TaggedComponent::Tag() const { diff --git a/src/util.cc b/src/util.cc index 3b917495bc..e41925a414 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1701,17 +1701,16 @@ const char* bro_magic_buffer(magic_t cookie, const void* buffer, size_t length) return rval; } -const char* canonify_name(const char* name) +std::string canonify_name(const std::string& name) { - unsigned int len = strlen(name); - char* nname = new char[len + 1]; + unsigned int len = name.size(); + std::string nname; for ( unsigned int i = 0; i < len; i++ ) { char c = isalnum(name[i]) ? name[i] : '_'; - nname[i] = toupper(c); + nname += toupper(c); } - nname[len] = '\0'; return nname; } diff --git a/src/util.h b/src/util.h index 8526ded03f..e8bce8221d 100644 --- a/src/util.h +++ b/src/util.h @@ -406,6 +406,6 @@ const char* bro_magic_buffer(magic_t cookie, const void* buffer, size_t length); * @param name The string to canonicalize. * @return The canonicalized version of \a name which caller may later delete[]. */ -const char* canonify_name(const char* name); +std::string canonify_name(const std::string& name); #endif From a80dd102153be5de92778226e8f80a466981b7c2 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 16 Dec 2013 10:08:38 -0800 Subject: [PATCH 012/193] Updates of the dynamic plugin code. Includes: - Cleanup of the plugin API, in particular generally changing const char* to std::string - Renaming environment variable BRO_PLUGINS to BRO_PLUGIN_PATH, defaulting to /lib/bro/plugins - Reworking how dynamic plugins are searched and activated. See doc/devel/plugins.rst for details. - New @load-plugin directive to explicitly activate a plugin - Support for Darwin. (Linux untested right now) - The init-plugin updates come with support for "make test", "make sdist", and "make bdist" (see how-to). - Test updates. Notes: The new hook mechanism, which allows plugins to hook into Bro's core a well-defined points, is still essentially untested. --- CMakeLists.txt | 3 +- aux/bro-aux | 2 +- cmake | 2 +- config.h.in | 2 +- src/BroDoc.cc | 4 +- src/CMakeLists.txt | 3 + src/DebugLogger.cc | 2 +- src/input.h | 1 + src/main.cc | 10 +- src/plugin/Manager.cc | 156 ++++++++++++++++------ src/plugin/Manager.h | 83 +++++++----- src/plugin/Plugin.cc | 58 ++++---- src/plugin/Plugin.h | 36 ++--- src/scan.l | 16 +++ src/util.cc | 9 +- src/util.h | 3 + testing/btest/core/plugins/analyzer.bro | 4 +- testing/btest/core/plugins/test-plugin.sh | 6 +- 18 files changed, 257 insertions(+), 143 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c006237850..c71762e112 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,10 +22,11 @@ set(BRO_SCRIPT_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/scripts) get_filename_component(BRO_SCRIPT_INSTALL_PATH ${BRO_SCRIPT_INSTALL_PATH} ABSOLUTE) -set(BRO_PLUGIN_INSTALL_PATH ${BRO_ROOT_DIR}/lib/bro/plugins) +set(BRO_PLUGIN_INSTALL_PATH ${BRO_ROOT_DIR}/lib/bro/plugins CACHE STRING "Installation path for plugins" FORCE) set(BRO_MAGIC_INSTALL_PATH ${BRO_ROOT_DIR}/share/bro/magic) set(BRO_MAGIC_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/magic/database) + configure_file(bro-path-dev.in ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev) file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.sh "export BROPATH=`${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n" diff --git a/aux/bro-aux b/aux/bro-aux index 715937ad72..f89d870437 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 715937ad72d9158d8c3142f81ee7005601d75db8 +Subproject commit f89d870437dbbbb107fd29f1085aef070e3bf373 diff --git a/cmake b/cmake index 6eb4cbfbc7..763b6d2574 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 6eb4cbfbc75f0bd13e247150b19932905a85e10c +Subproject commit 763b6d2574294e62f116e167c564a6292206a655 diff --git a/config.h.in b/config.h.in index 61124d2f67..d3889a2d90 100644 --- a/config.h.in +++ b/config.h.in @@ -214,7 +214,7 @@ #define HOST_ARCHITECTURE "@HOST_ARCHITECTURE@" /* String with extension of dynamic libraries (e.g., ".so") */ -#define SHARED_LIBRARY_SUFFIX "@CMAKE_SHARED_LIBRARY_SUFFIX@" +#define DYNAMIC_PLUGIN_SUFFIX "@CMAKE_SHARED_MODULE_SUFFIX@" /* True if we're building outside of the main Bro source code tree. */ #ifndef BRO_PLUGIN_INTERNAL_BUILD diff --git a/src/BroDoc.cc b/src/BroDoc.cc index 0a1006bd07..03c29d2201 100644 --- a/src/BroDoc.cc +++ b/src/BroDoc.cc @@ -467,7 +467,7 @@ static void WritePluginSectionHeading(FILE* f, const plugin::Plugin* p) fprintf(f, "-"); fprintf(f, "\n\n"); - fprintf(f, "%s\n\n", p->Description()); + fprintf(f, "%s\n\n", p->Description().c_str()); } static void WriteAnalyzerComponent(FILE* f, const analyzer::Component* c) @@ -568,7 +568,7 @@ static void WritePluginBifItems(FILE* f, const plugin::Plugin* p, if ( o ) o->WriteReST(f); else - reporter->Warning("No docs for ID: %s\n", it->GetID()); + reporter->Warning("No docs for ID: %s\n", it->GetID().c_str()); } } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 34af4eb974..8a0b9836c7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -389,6 +389,9 @@ install(TARGETS bro DESTINATION bin) set(BRO_EXE bro CACHE STRING "Bro executable binary" FORCE) +set(BRO_EXE_PATH ${CMAKE_CURRENT_BINARY_DIR}/bro + CACHE STRING "Path to Bro executable binary" FORCE) + # Target to create all the autogenerated files. add_custom_target(generate_outputs_stage1) add_dependencies(generate_outputs_stage1 ${bro_ALL_GENERATED_OUTPUTS}) diff --git a/src/DebugLogger.cc b/src/DebugLogger.cc index 267383503d..88635a6fee 100644 --- a/src/DebugLogger.cc +++ b/src/DebugLogger.cc @@ -117,7 +117,7 @@ void DebugLogger::Log(const plugin::Plugin& plugin, const char* fmt, ...) return; fprintf(file, "%17.06f/%17.06f [plugin %s] ", - network_time, current_time(true), plugin.Name()); + network_time, current_time(true), plugin.Name().c_str()); va_list ap; va_start(ap, fmt); diff --git a/src/input.h b/src/input.h index 8fcceb256b..d07a82f2ee 100644 --- a/src/input.h +++ b/src/input.h @@ -15,6 +15,7 @@ extern int brolex(); extern char last_tok[128]; extern void add_input_file(const char* file); +extern void add_input_file_at_front(const char* file); // Adds the substrings (using the given delimiter) in a string to the // given namelist. diff --git a/src/main.cc b/src/main.cc index c3c32a4b14..abafff5a85 100644 --- a/src/main.cc +++ b/src/main.cc @@ -223,7 +223,7 @@ void usage() fprintf(stderr, " $BROPATH | file search path (%s)\n", bro_path().c_str()); fprintf(stderr, " $BROMAGIC | libmagic mime magic database search path (%s)\n", bro_magic_path()); - fprintf(stderr, " $BRO_PLUGINS | plugin search path (%s)\n", bro_plugin_path()); + fprintf(stderr, " $BRO_PLUGIN_PATH | plugin search path (%s)\n", bro_plugin_path()); fprintf(stderr, " $BRO_PREFIXES | prefix list (%s)\n", bro_prefixes().c_str()); fprintf(stderr, " $BRO_DNS_FAKE | disable DNS lookups (%s)\n", bro_dns_fake()); fprintf(stderr, " $BRO_SEED_FILE | file to load seeds from (not set)\n"); @@ -820,7 +820,7 @@ int main(int argc, char** argv) if ( ! bare_mode ) add_input_file("base/init-default.bro"); - plugin_mgr->LoadPluginsFrom(bro_plugin_path()); + plugin_mgr->SearchDynamicPlugins(bro_plugin_path()); if ( optind == argc && read_files.length() == 0 && flow_files.length() == 0 && @@ -860,6 +860,9 @@ int main(int argc, char** argv) analyzer_mgr->InitPreScript(); file_mgr->InitPreScript(); + if ( ! bare_mode ) + plugin_mgr->ActivateAllDynamicPlugins(); + if ( events_file ) event_player = new EventPlayer(events_file); @@ -896,13 +899,14 @@ int main(int argc, char** argv) if ( reporter->Errors() > 0 ) exit(1); + plugin_mgr->InitPostScript(); + if ( print_plugins ) { show_plugins(print_plugins); exit(1); } - plugin_mgr->InitPostScript(); analyzer_mgr->InitPostScript(); file_mgr->InitPostScript(); diff --git a/src/plugin/Manager.cc b/src/plugin/Manager.cc index be1d7827a5..76e154067a 100644 --- a/src/plugin/Manager.cc +++ b/src/plugin/Manager.cc @@ -1,6 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. #include +#include #include #include #include @@ -15,6 +16,7 @@ using namespace plugin; +Plugin* Manager::current_plugin = 0; string Manager::current_dir; string Manager::current_sopath; @@ -37,7 +39,7 @@ Manager::~Manager() delete [] hooks; } -void Manager::LoadPluginsFrom(const string& dir) +void Manager::SearchDynamicPlugins(const std::string& dir) { assert(! init); @@ -51,7 +53,7 @@ void Manager::LoadPluginsFrom(const string& dir) std::string d; while ( std::getline(s, d, ':') ) - LoadPluginsFrom(d); + SearchDynamicPlugins(d); return; } @@ -62,12 +64,34 @@ void Manager::LoadPluginsFrom(const string& dir) return; } - int rc = LoadPlugin(dir); + // Check if it's a plugin dirctory. - if ( rc >= 0 ) + const std::string magic = dir + "/__bro_plugin__"; + + if ( is_file(magic) ) + { + // It's a plugin, get it's name. + std::ifstream in(magic); + + if ( in.fail() ) + reporter->FatalError("cannot open plugin magic file %s", magic.c_str()); + + std::string name; + std::getline(in, name); + strstrip(name); + + if ( name.empty() ) + reporter->FatalError("empty plugin magic file %s", magic.c_str()); + + // Record it, so that we can later activate it. + + dynamic_plugins.insert(std::make_pair(name, dir)); + + DBG_LOG(DBG_PLUGINS, "Found plugin %s in %s", name.c_str(), dir.c_str()); return; + } - DBG_LOG(DBG_PLUGINS, "Searching directory %s recursively for plugins", dir.c_str()); + // No plugin here, traverse subirectories. DIR* d = opendir(dir.c_str()); @@ -98,22 +122,33 @@ void Manager::LoadPluginsFrom(const string& dir) } if ( st.st_mode & S_IFDIR ) - LoadPluginsFrom(path); + SearchDynamicPlugins(path); } } -int Manager::LoadPlugin(const std::string& dir) +bool Manager::ActivateDynamicPluginInternal(const std::string& name) { - assert(! init); + dynamic_plugin_map::iterator m = dynamic_plugins.find(name); - // Check if it's a plugin dirctory. - if ( ! is_file(dir + "/__bro_plugin__") ) - return -1; + if ( m == dynamic_plugins.end() ) + { + reporter->Error("plugin %s is not available", name.c_str()); + return false; + } - DBG_LOG(DBG_PLUGINS, "Loading plugin from %s", dir.c_str()); + std::string dir = m->second + "/"; + + if ( dir.empty() ) + { + // That's our marker that we have already activated this + // plugin. Silently ignore the new request. + return true; + } + + DBG_LOG(DBG_PLUGINS, "Activating plugin %s", name.c_str()); // Add the "scripts" and "bif" directories to BROPATH. - string scripts = dir + "/scripts"; + std::string scripts = dir + "scripts"; if ( is_dir(scripts) ) { @@ -121,44 +156,27 @@ int Manager::LoadPlugin(const std::string& dir) add_to_bro_path(scripts); } - string bif = dir + "/bif"; + // Load {bif,scripts}/__load__.bro automatically. - if ( is_dir(bif) ) - { - DBG_LOG(DBG_PLUGINS, " Adding %s to BROPATH", bif.c_str()); - add_to_bro_path(bif); - } - - // Load dylib/scripts/__load__.bro automatically. - string dyinit = dir + "/dylib/scripts/__load__.bro"; - - if ( is_file(dyinit) ) - { - DBG_LOG(DBG_PLUGINS, " Adding %s for loading", dyinit.c_str()); - add_input_file(dyinit.c_str()); - } - - // Load scripts/__load__.bro automatically. - string init = scripts + "/__load__.bro"; + string init = dir + "lib/bif/__load__.bro"; if ( is_file(init) ) { - DBG_LOG(DBG_PLUGINS, " Adding %s for loading", init.c_str()); - add_input_file(init.c_str()); + DBG_LOG(DBG_PLUGINS, " Loading %s", init.c_str()); + scripts_to_load.push_back(init); } - // Load bif/__load__.bro automatically. - init = bif + "/__load__.bro"; + init = dir + "scripts/__load__.bro"; if ( is_file(init) ) { - DBG_LOG(DBG_PLUGINS, " Adding %s for loading", init.c_str()); - add_input_file(init.c_str()); + DBG_LOG(DBG_PLUGINS, " Loading %s", init.c_str()); + scripts_to_load.push_back(init); } // Load shared libraries. - string dypattern = dir + "/dylib/*." + HOST_ARCHITECTURE + SHARED_LIBRARY_SUFFIX; + string dypattern = dir + "/lib/*." + HOST_ARCHITECTURE + DYNAMIC_PLUGIN_SUFFIX; DBG_LOG(DBG_PLUGINS, " Searching for shared libraries %s", dypattern.c_str()); @@ -170,11 +188,10 @@ int Manager::LoadPlugin(const std::string& dir) { const char* path = gl.gl_pathv[i]; + current_plugin = 0; current_dir = dir; current_sopath = path; void* hdl = dlopen(path, RTLD_LAZY | RTLD_GLOBAL); - current_dir.clear(); - current_sopath.clear(); if ( ! hdl ) { @@ -182,6 +199,24 @@ int Manager::LoadPlugin(const std::string& dir) reporter->FatalError("cannot load plugin library %s: %s", path, err ? err : ""); } + if ( ! current_plugin ) + reporter->FatalError("load plugin library %s did not instantiate a plugin", path); + + // We execute the pre-script initialization here; this in + // fact could be *during* script initialization if we got + // triggered via @load-plugin. + current_plugin->InitPreScript(); + + // Make sure the name the plugin reports is consistent with + // what we expect from its magic file. + if ( string(current_plugin->Name()) != name ) + reporter->FatalError("inconsistent plugin name: %s vs %s", + current_plugin->Name().c_str(), name.c_str()); + + current_dir.clear(); + current_sopath.clear(); + current_plugin = 0; + DBG_LOG(DBG_PLUGINS, " Loaded %s", path); } } @@ -189,10 +224,42 @@ int Manager::LoadPlugin(const std::string& dir) else { DBG_LOG(DBG_PLUGINS, " No shared library found"); - return 1; } - return 1; + // Mark this plugin as activated by clearing the path. + m->second.clear(); + + return true; + } + +bool Manager::ActivateDynamicPlugin(const std::string& name) + { + if ( ! ActivateDynamicPluginInternal(name) ) + return false; + + UpdateInputFiles(); + return true; + } + +bool Manager::ActivateAllDynamicPlugins() + { + for ( dynamic_plugin_map::const_iterator i = dynamic_plugins.begin(); + i != dynamic_plugins.end(); i++ ) + { + if ( ! ActivateDynamicPluginInternal(i->first) ) + return false; + } + + UpdateInputFiles(); + + return true; + } + +void Manager::UpdateInputFiles() + { + for ( file_list::const_reverse_iterator i = scripts_to_load.rbegin(); + i != scripts_to_load.rend(); i++ ) + add_input_file_at_front((*i).c_str()); } static bool plugin_cmp(const Plugin* a, const Plugin* b) @@ -205,11 +272,13 @@ bool Manager::RegisterPlugin(Plugin *plugin) Manager::PluginsInternal()->push_back(plugin); if ( current_dir.size() && current_sopath.size() ) + // A dynamic plugin, record its location. plugin->SetPluginLocation(current_dir.c_str(), current_sopath.c_str()); // Sort plugins by name to make sure we have a deterministic order. PluginsInternal()->sort(plugin_cmp); + current_plugin = plugin; return true; } @@ -220,7 +289,6 @@ void Manager::InitPreScript() for ( plugin_list::iterator i = Manager::PluginsInternal()->begin(); i != Manager::PluginsInternal()->end(); i++ ) { Plugin* plugin = *i; - plugin->InitPreScript(); } @@ -333,7 +401,7 @@ void Manager::DisableHook(HookType hook, Plugin* plugin) } } -int Manager::HookLoadFile(const char* file) +int Manager::HookLoadFile(const string& file) { hook_list* l = hooks[HOOK_LOAD_FILE]; diff --git a/src/plugin/Manager.h b/src/plugin/Manager.h index 0c52679a64..5c5c7430be 100644 --- a/src/plugin/Manager.h +++ b/src/plugin/Manager.h @@ -40,17 +40,41 @@ public: virtual ~Manager(); /** - * Loads all plugins dynamically from a set of directories. Multiple - * directories are split by ':'. If a directory does not contain a - * plugin itself, the method searches for plugins recursively. For - * plugins found, the method loads the plugin's shared library and - * makes its scripts available to the interpreter. + * Searches a set of directories for plugins. If a specificed + * directory does not contain a plugin itself, the method searches + * for plugins recursively. For plugins found, the method makes them + * available for later activation via ActivatePlugin(). * * This must be called only before InitPluginsPreScript(). * - * @param dir The directory to search for plugins. + * @param dir The directory to search for plugins. Multiple + * directories are split by ':'. */ - void LoadPluginsFrom(const std::string& dir); + void SearchDynamicPlugins(const std::string& dir); + + /** + * Activates a plugin that SearchPlugins() has previously discovered. + * Activing a plugin involved loading its dynamic module, making its + * bifs available, and adding its script paths to BROPATH. + * + * @param name The name of the plugin, as determined previously by + * SearchPlugin(). + * + * @return True if the plugin has been loaded successfully. + * + */ + bool ActivateDynamicPlugin(const std::string& name); + + /** + * Activates all plugins that SearchPlugins() has previously + * discovered. The effect is the same all calling \a + * ActivePlugin(name) for every plugin. + * + * @return True if all plugins have been loaded successfully. If one + * fail to load, the method stops there without loading any furthers + * and returns false. + */ + bool ActivateAllDynamicPlugins(); /** * First-stage initializion of the manager. This is called early on @@ -148,7 +172,7 @@ public: * successfully; 0 if a plugin took over the file but had trouble * loading it; and -1 if no plugin was interested in the file at all. */ - virtual int HookLoadFile(const char* file); + virtual int HookLoadFile(const string& file); /** * Hook that filters calls to a script function/event/hook. @@ -196,40 +220,37 @@ public: */ static bool RegisterPlugin(Plugin* plugin); -protected: - /** - * Loads a plugin dynamically from a given directory. It loads the - * plugin's shared library, and makes its scripts available to the - * interpreter. Different from LoadPluginsFrom() this method does not - * further descend the directory tree recursively to search for - * plugins. - * - * This must be called only before InitPluginsPreScript() - * - * @param file The path to the plugin to load. - * - * @return 0 if there's a plugin in this directory, but there was a - * problem loading it; -1 if there's no plugin at all in this - * directory; 1 if there's a plugin in this directory and we loaded - * it successfully. - */ - int LoadPlugin(const std::string& dir); - private: + bool ActivateDynamicPluginInternal(const std::string& name); + void UpdateInputFiles(); + + // All found dynamic plugins, mapping their names to base directory. + typedef std::map dynamic_plugin_map; + dynamic_plugin_map dynamic_plugins; + + // We buffer scripts to load temporarliy to get them to load in the + // right order. + typedef std::list file_list; + file_list scripts_to_load; + + bool init; + // A hook list keeps pairs of plugin and priority interested in a // given hook. typedef std::list > hook_list; - static plugin_list* PluginsInternal(); - - bool init; - // An array indexed by HookType. An entry is null if there's no hook // of that type enabled. hook_list** hooks; + static Plugin* current_plugin; static string current_dir; static string current_sopath; + + // Returns a modifiable list of all plugins, both static and dynamic. + // This is a static method so that plugins can register themselves + // even before the manager exists. + static plugin_list* PluginsInternal(); }; template diff --git a/src/plugin/Plugin.cc b/src/plugin/Plugin.cc index 20d436907b..000bb2af52 100644 --- a/src/plugin/Plugin.cc +++ b/src/plugin/Plugin.cc @@ -26,15 +26,15 @@ const char* hook_name(HookType h) return hook_names[int(h)]; } -BifItem::BifItem(const char* arg_id, Type arg_type) +BifItem::BifItem(const std::string& arg_id, Type arg_type) { - id = copy_string(arg_id); + id = arg_id; type = arg_type; } BifItem::BifItem(const BifItem& other) { - id = copy_string(other.id); + id = other.id; type = other.type; } @@ -42,7 +42,7 @@ BifItem& BifItem::operator=(const BifItem& other) { if ( this != &other ) { - id = copy_string(other.id); + id = other.id; type = other.type; } @@ -51,20 +51,19 @@ BifItem& BifItem::operator=(const BifItem& other) BifItem::~BifItem() { - delete [] id; } Plugin::Plugin() { - name = copy_string(""); - description = copy_string(""); + name = ""; + description = ""; // These will be reset by the BRO_PLUGIN_* macros. version = -9999; api_version = -9999; dynamic = false; - base_dir = 0; - sopath = 0; + base_dir = ""; + sopath = ""; Manager::RegisterPlugin(this); } @@ -72,33 +71,26 @@ Plugin::Plugin() Plugin::~Plugin() { Done(); - - delete [] name; - delete [] description; - delete [] base_dir; - delete [] sopath; } -const char* Plugin::Name() const +const std::string& Plugin::Name() const { return name; } -void Plugin::SetName(const char* arg_name) +void Plugin::SetName(const std::string& arg_name) { - delete [] name; - name = copy_string(arg_name); + name = arg_name; } -const char* Plugin::Description() const +const std::string& Plugin::Description() const { return description; } -void Plugin::SetDescription(const char* arg_description) +void Plugin::SetDescription(const std::string& arg_description) { - delete [] description; - description = copy_string(arg_description); + description = arg_description; } int Plugin::Version() const @@ -121,12 +113,12 @@ bool Plugin::DynamicPlugin() const return dynamic; } -const char* Plugin::PluginDirectory() const +const std::string& Plugin::PluginDirectory() const { return base_dir; } -const char* Plugin::PluginPath() const +const std::string& Plugin::PluginPath() const { return sopath; } @@ -141,12 +133,10 @@ void Plugin::SetDynamicPlugin(bool arg_dynamic) dynamic = arg_dynamic; } -void Plugin::SetPluginLocation(const char* arg_dir, const char* arg_sopath) +void Plugin::SetPluginLocation(const std::string& arg_dir, const std::string& arg_sopath) { - delete [] base_dir; - delete [] sopath; - base_dir = copy_string(arg_dir); - sopath = copy_string(arg_sopath); + base_dir = arg_dir; + sopath = arg_sopath; } void Plugin::InitPreScript() @@ -197,9 +187,9 @@ static bool component_cmp(const Component* a, const Component* b) return a->Name() < b->Name(); } -bool Plugin::LoadBroFile(const char* file) +bool Plugin::LoadBroFile(const std::string& file) { - ::add_input_file(file); + ::add_input_file(file.c_str()); return true; } @@ -208,7 +198,7 @@ void Plugin::__AddBifInitFunction(bif_init_func c) bif_inits.push_back(c); } -void Plugin::AddBifItem(const char* name, BifItem::Type type) +void Plugin::AddBifItem(const std::string& name, BifItem::Type type) { BifItem bi(name, (BifItem::Type)type); bif_items.push_back(bi); @@ -238,7 +228,7 @@ void Plugin::DisableHook(HookType hook) plugin_mgr->DisableHook(hook, this); } -int Plugin::HookLoadFile(const char* file) +int Plugin::HookLoadFile(const std::string& file) { return -1; } @@ -266,7 +256,7 @@ void Plugin::Describe(ODesc* d) const d->Add("Plugin: "); d->Add(name); - if ( description && *description ) + if ( description.size() ) { d->Add(" - "); d->Add(description); diff --git a/src/plugin/Plugin.h b/src/plugin/Plugin.h index 915563e98c..7da5a74c6f 100644 --- a/src/plugin/Plugin.h +++ b/src/plugin/Plugin.h @@ -50,7 +50,7 @@ public: * * @param type The type of the item. */ - BifItem(const char* id, Type type); + BifItem(const std::string& id, Type type); /** * Copy constructor. @@ -70,7 +70,7 @@ public: /** * Returns the script-level ID as passed into the constructor. */ - const char* GetID() const { return id; } + const std::string& GetID() const { return id; } /** * Returns the type as passed into the constructor. @@ -78,7 +78,7 @@ public: Type GetType() const { return type; } private: - const char* id; + std::string id; Type type; }; @@ -113,7 +113,7 @@ public: typedef std::list component_list; typedef std::list bif_item_list; typedef std::list > hook_list; - typedef std::list > bif_init_func_result; + typedef std::list > bif_init_func_result; typedef void (*bif_init_func)(Plugin *); /** @@ -129,12 +129,12 @@ public: /** * Returns the name of the plugin. */ - const char* Name() const; + const std::string& Name() const; /** * Returns a short textual description of the plugin, if provided. */ - const char* Description() const; + const std::string& Description() const; /** * Returns the version of the plugin. Version are only meaningful for @@ -152,13 +152,13 @@ public: * For dynamic plugins, returns the base directory from which it was * loaded. For static plugins, returns null. **/ - const char* PluginDirectory() const; + const std::string& PluginDirectory() const; /** * For dynamic plugins, returns the full path to the shared library * from which it was loaded. For static plugins, returns null. **/ - const char* PluginPath() const; + const std::string& PluginPath() const; /** * Returns the internal API version that this plugin relies on. Only @@ -225,7 +225,7 @@ public: * * @param type The item's type. */ - void AddBifItem(const char* name, BifItem::Type type); + void AddBifItem(const std::string& name, BifItem::Type type); /** * Adds a file to the list of files that Bro loads at startup. This @@ -242,7 +242,7 @@ public: * @return True if successful (which however may only mean * "successfully queued"). */ - bool LoadBroFile(const char* file); + bool LoadBroFile(const std::string& file); /** * Internal function adding an entry point for registering @@ -324,7 +324,7 @@ protected: * printed an error message); and -1 if the plugin wasn't interested * in the file at all. */ - virtual int HookLoadFile(const char* file); + virtual int HookLoadFile(const std::string& file); /** * Hook into executing a script-level function/event/hook. Whenever @@ -396,7 +396,7 @@ protected: * * @param name The name. Makes a copy internally. */ - void SetName(const char* name); + void SetName(const std::string& name); /** * Sets the plugin's textual description. @@ -406,7 +406,7 @@ protected: * * @param name The description. Makes a copy internally. */ - void SetDescription(const char* descr); + void SetDescription(const std::string& descr); /** * Sets the plugin's version. @@ -454,7 +454,7 @@ protected: * @param sopath The full path the shared library loaded. The * functions makes an internal copy of string. */ - void SetPluginLocation(const char* dir, const char* sopath); + void SetPluginLocation(const std::string& dir, const std::string& sopath); private: /** @@ -465,10 +465,10 @@ private: typedef std::list bif_init_func_list; - const char* name; - const char* description; - const char* base_dir; - const char* sopath; + std::string name; + std::string description; + std::string base_dir; + std::string sopath; int version; int api_version; bool dynamic; diff --git a/src/scan.l b/src/scan.l index 599a660fee..d5eedc83c3 100644 --- a/src/scan.l +++ b/src/scan.l @@ -407,6 +407,11 @@ when return TOK_WHEN; new_sig_file); } +@load-plugin{WS}{ID} { + const char* plugin = skip_whitespace(yytext + 12); + plugin_mgr->ActivateDynamicPlugin(plugin); +} + @unload{WS}{FILE} { // Skip "@unload". const char* new_file = skip_whitespace(yytext + 7); @@ -829,6 +834,17 @@ void add_input_file(const char* file) input_files.append(copy_string(file)); } +void add_input_file_at_front(const char* file) + { + if ( ! file ) + reporter->InternalError("empty filename"); + + if ( ! filename ) + (void) load_files(file); + else + input_files.insert(copy_string(file)); + } + void add_to_name_list(char* s, char delim, name_list& nl) { while ( s ) diff --git a/src/util.cc b/src/util.cc index e41925a414..917ddf9388 100644 --- a/src/util.cc +++ b/src/util.cc @@ -664,6 +664,13 @@ string strreplace(const string& s, const string& o, const string& n) return r; } +std::string strstrip(std::string s) + { + s.erase(s.begin(), std::find_if(s.begin(), s.end(), std::not1(std::ptr_fun(std::isspace)))); + s.erase(std::find_if(s.rbegin(), s.rend(), std::not1(std::ptr_fun(std::isspace))).base(), s.end()); + return s; + } + int hmac_key_set = 0; uint8 shared_hmac_md5_key[16]; @@ -942,7 +949,7 @@ const char* bro_magic_path() const char* bro_plugin_path() { - const char* path = getenv("BRO_PLUGINS"); + const char* path = getenv("BRO_PLUGIN_PATH"); if ( ! path ) path = BRO_PLUGIN_INSTALL_PATH; diff --git a/src/util.h b/src/util.h index e8bce8221d..feae01c0fb 100644 --- a/src/util.h +++ b/src/util.h @@ -155,6 +155,9 @@ bool is_file(const std::string& path); // Replaces all occurences of *o* in *s* with *n*. extern std::string strreplace(const std::string& s, const std::string& o, const std::string& n); +// Remove all leading and trainling white space from string. +extern std::string strstrip(std::string s); + extern uint8 shared_hmac_md5_key[16]; extern int hmac_key_set; diff --git a/testing/btest/core/plugins/analyzer.bro b/testing/btest/core/plugins/analyzer.bro index 2a72797564..c9831ad269 100644 --- a/testing/btest/core/plugins/analyzer.bro +++ b/testing/btest/core/plugins/analyzer.bro @@ -1,9 +1,9 @@ # @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo # @TEST-EXEC: cp -r %DIR/analyzer-plugin/* . # @TEST-EXEC: make BRO=${DIST} -# @TEST-EXEC: BROPLUGINS=`pwd` bro -NN | awk '/^Plugin:.*Demo/ {p=1; print; next} /^Plugin:/{p=0} p==1{print}' >>output +# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN | awk '/^Plugin:.*Demo/ {p=1; print; next} /^Plugin:/{p=0} p==1{print}' >>output # @TEST-EXEC: echo === >>output -# @TEST-EXEC: BROPLUGINS=`pwd` bro -r $TRACES/port4242.trace %INPUT >>output +# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r $TRACES/port4242.trace %INPUT >>output # @TEST-EXEC: btest-diff output event foo_message(c: connection, data: string) diff --git a/testing/btest/core/plugins/test-plugin.sh b/testing/btest/core/plugins/test-plugin.sh index 87b09b2e4d..3b67b9454f 100644 --- a/testing/btest/core/plugins/test-plugin.sh +++ b/testing/btest/core/plugins/test-plugin.sh @@ -1,11 +1,11 @@ # @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo # @TEST-EXEC: bash %INPUT # @TEST-EXEC: make BRO=${DIST} -# @TEST-EXEC: BROPLUGINS=`pwd` bro -NN | awk '/^Plugin:.*Demo/ {p=1; print; next} /^Plugin:/{p=0} p==1{print}' >>output +# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN | awk '/^Plugin:.*Demo/ {p=1; print; next} /^Plugin:/{p=0} p==1{print}' >>output # @TEST-EXEC: echo === >>output -# @TEST-EXEC: BROPLUGINS=`pwd` bro -r $TRACES/empty.trace >>output +# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r $TRACES/empty.trace >>output # @TEST-EXEC: echo === >>output -# @TEST-EXEC: BROPLUGINS=`pwd` bro demo/foo -r $TRACES/empty.trace >>output +# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro demo/foo -r $TRACES/empty.trace >>output # @TEST-EXEC: btest-diff output cat >scripts/__load__.bro < Date: Mon, 16 Dec 2013 10:09:16 -0800 Subject: [PATCH 013/193] Start of a plugin writing how-to. See doc/devel/plugins.rst. It includes a simple example and background on how things work. --- doc/devel/plugins.rst | 339 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 339 insertions(+) create mode 100644 doc/devel/plugins.rst diff --git a/doc/devel/plugins.rst b/doc/devel/plugins.rst new file mode 100644 index 0000000000..f00bbb4d2c --- /dev/null +++ b/doc/devel/plugins.rst @@ -0,0 +1,339 @@ + +=================== +Writing Bro Plugins +=================== + +Bro is internally moving to a plugin structure that enables extending +the system dynamically, without modifying the core code base. That way +custom code remains self-contained and can be maintained, compiled, +and installed independently. Currently, plugins can add the following +functionality to Bro: + + - Bro scripts. + + - Builtin functions/events/types for the scripting language. + + - Protocol and file analyzers. + + - Packet sources and packet dumpers. TODO: Not yet. + + - Logging framework backends. TODO: Not yet. + + - Input framework readers. TODO: Not yet. + +A plugin's functionality is available to the user just as if Bro had +the corresponding code built-in. Indeed, internally many of Bro's +pieces are structured as plugins as well, they are just statically +compiled into the binary rather than loaded dynamically at runtime, as +external plugins are. + +Quick Start +=========== + +Writing a basic plugin is quite straight-forward as long as one +follows a few conventions. In the following we walk through adding a +new built-in function (bif) to Bro; we'll add `a `rot13(s: string) : +string``, a function that rotates every character in a string by 13 +places. + +A plugin comes in the form of a directory following a certain +structure. To get started, Bro's distribution provides a helper script +``aux/bro-aux/plugin-support/init-plugin`` that creates a skeleton +plugin that can then be customized. Let's use that:: + + # mkdir rot13-plugin + # cd rot13-plugin + # init-plugin Demo Rot13 + +As you can see the script takes two arguments. The first is a +namespace the plugin will live in, and the second a descriptive name +for the plugin itself. Bro uses the combination of the two to identify +a plugin. The namespace serves to avoid naming conflicts between +plugins written by independent developers; pick, e.g., the name of +your organisation (and note that the namespace ``Bro`` is reserved for +functionality distributed by the Bro Project). In our example, the +plugin will be called ``Demo::Rot13``. + +The ``init-plugin`` script puts a number of files in place. The full +layout is described later. For now, all we need is +``src/functions.bif``. It's initially empty, but we'll add our new bif +there as follows:: + + # cat scripts/functions.bif + module CaesarCipher; + + function camel_case%(s: string%) : string + %{ + char* rot13 = copy_string(s->CheckString()); + + for ( char* p = rot13; *p; p++ ) + { + char b = islower(*p) ? 'a' : 'A'; + *p = (*p - b + 13) % 26 + b; + } + + return new StringVal(strlen(rot13), rot13); + %} + +The syntax of this file is just like any other ``*.bif`` file; we +won't go into it here. + +Now we can already compile our plugin, we just need to tell the +Makefile put in place by ``init-plugin`` where the Bro source tree is +located (Bro needs to have been built there first):: + + # make BRO=/path/to/bro/dist + [... cmake output ...] + +Now our ``rot13-plugin`` directory has everything that it needs +for Bro to recognize it as a dynamic plugin. Once we point Bro to it, +it will pull it in automatically, as we can check with the ``-N`` +option: + + # export BRO_PLUGIN_PATH=/path/to/rot13-plugin + # bro -N + [...] + Plugin: Demo::Rot13 - (dynamic, version 1) + [...] + +That looks quite good, except for the dummy description that we should +replace with something nicer so that users will know what our plugin +is about. We do this by editing the ``BRO_PLUGIN_DESCRIPTION`` line +in ``src/Plugin.cc``, like this: + + # cat src/Plugin.cc + + #include + + BRO_PLUGIN_BEGIN(Demo, Rot13) + BRO_PLUGIN_VERSION(1); + BRO_PLUGIN_DESCRIPTION("Caesar cipher rotating a string's characters by 13 places."); + BRO_PLUGIN_BIF_FILE(consts); + BRO_PLUGIN_BIF_FILE(events); + BRO_PLUGIN_BIF_FILE(functions); + BRO_PLUGIN_END + + # make + [...] + # bro -N | grep Rot13 + Plugin: Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 1) + +Better. Bro can also show us what exactly the plugin provides with the +more verbose option ``-NN``:: + + # bro -NN + [...] + Plugin: Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 1) + [Function] CaesarCipher::rot13 + [...] + +There's our function. Now let's use it:: + + # bro -e 'print CaesarCipher::rot13("Hello")' + Uryyb + +It works. We next install the plugin along with Bro itself, so that it +will find it directly without needing the ``BRO_PLUGIN_PATH`` +environment variable. If we first unset the variable, the function +will no longer be available:: + + # unset BRO_PLUGIN_PATH + # bro -e 'print CaesarCipher::rot13("Hello")' + error in , line 1: unknown identifier CaesarCipher::rot13, at or near "CaesarCipher::rot13" + +Once we install it, it works again:: + + # make install + # bro -e 'print CaesarCipher::rot13("Hello")' + Uryyb + +The installed version went into +``/lib/bro/plugins/Demo_Rot13``. + +We can distribute the plugin in either source or binary form by using +the Makefile's ``sdist`` and ``bdist`` target, respectively. Both +create corrsponding tarballs:: + + # make sdist + [...] + Source distribution in build/sdist/Demo_Rot13.tar.gz + + # make bdist + [...] + Binary distribution in build/Demo_Rot13-darwin-x86_64.tar.gz + +The source archive will contain everything in the plugin directory +except any generated files. The binary archive will contain anything +needed to install and run the plugin, i.e., just what ``make install`` +puts into place as well. As the binary distribution is +platform-dependent, its name includes the OS and architecture the +plugin was built on. + +Plugin Directory Layout +======================= + +A plugin's directory needs to follow a set of conventions so that Bro +(1) recognizes it as a plugin, and (2) knows what to load. While +``init-plugin`` takes care of most of this, the following is the full +story. We'll use ```` to represent a plugin's top-level +directory. + +``/__bro_plugin__`` + A file that marks a directory as containing a Bro plugin. The file + must exist, and its content must consist of a single line with the + qualified name of the plugin (e.g., "Demo::Rot13"). + +``/lib/--.so`` + The shared library containing the plugin's compiled code. Bro will + load this in dynamically at run-time if OS and architecture match + the current platform. + +``lib/bif/`` + Directory with auto-generated Bro scripts that declare the plugins + bif elements. The files here are produced by ``bifcl``. + +``scripts/`` + A directory with the plugin's custom Bro scripts. When the plugin + gets activated, this directory will be automatically added to + ``BROPATH``, so that any scripts/modules inside can be + ``@load``ed. + +``scripts``/__load__.bro + A Bro script that will be loaded immediately when the plugin gets + activated. See below for more information on activating plugins. + +By convention, a plugin should put its custom scripts into sub folders +of ``scripts/``, i.e., ``scripts//