diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 230c00e40e..16de055e11 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -329,7 +329,6 @@ set(bro_SRCS FTP.cc File.cc FileAnalyzer.cc - FileAnalysisManager.cc Finger.cc FlowSrc.cc Frag.cc @@ -449,6 +448,12 @@ set(bro_SRCS input/readers/Raw.cc input/readers/Benchmark.cc + file_analysis/Manager.cc + file_analysis/Info.cc + file_analysis/InfoTimer.cc + file_analysis/Action.h + file_analysis/Extract.cc + nb_dns.c digest.h ) diff --git a/src/FileAnalysisManager.h b/src/FileAnalysisManager.h deleted file mode 100644 index 40134d9f47..0000000000 --- a/src/FileAnalysisManager.h +++ /dev/null @@ -1,351 +0,0 @@ -#ifndef FILE_ANALYSIS_MANAGER_H -#define FILE_ANALYSIS_MANAGER_H - -#include -#include -#include - -#include "Conn.h" -#include "Analyzer.h" -#include "Timer.h" -#include "Val.h" -#include "Reporter.h" - -namespace file_analysis { - -class Info; - -/** - * Base class for actions that can be attached to a file_analysis::Info object. - */ -class Action { -public: - - virtual ~Action() {} - - /** - * Subclasses may override this to receive file data non-sequentially. - */ - virtual void DeliverChunk(const u_char* data, uint64 len, uint64 offset) {} - - /** - * Subclasses may override this to receive file sequentially. - */ - virtual void DeliverStream(const u_char* data, uint64 len) {} - - /** - * Subclasses may override this to specifically handle the end of a file. - */ - virtual void EndOfFile() {} - - /** - * Subclasses may override this to handle missing data in a file stream. - */ - virtual void Undelivered(uint64 offset, uint64 len) {} - -protected: - - Action(Info* arg_info); - - Info* info; -}; - -typedef Action* (*ActionInstantiator)(const RecordVal* args, Info* info); - -/** - * An action to simply extract files to disk. - */ -class Extract : Action { -public: - - static Action* Instantiate(const RecordVal* args, Info* info); - - ~Extract(); - - virtual void DeliverChunk(const u_char* data, uint64 len, uint64 offset); - -protected: - - Extract(Info* arg_info, const string& arg_filename); - - string filename; - int fd; -}; - -/** - * Wrapper class around \c FileAnalysis::Info record values from script layer. - */ -class Info { -public: - - ~Info(); - - /** - * @return value (seconds) of the "timeout_interval" field from #val record. - */ - double TimeoutInterval() const; - - /** - * @return value of the "file_id" field from #val record. - */ - string FileID() const; - - /** - * @return #last_activity_time - */ - double LastActivityTime() const { return last_activity_time; } - - /** - * Refreshes #last_activity_time with current network time. - */ - void UpdateLastActivityTime() { last_activity_time = network_time; } - - /** - * Set "total_bytes" field of #val record to \a size. - */ - void SetTotalBytes(uint64 size); - - /** - * Compares "seen_bytes" field to "total_bytes" field of #val record - * and returns true if the comparison indicates the full file was seen. - * If "total_bytes" hasn't been set yet, it returns false. - */ - bool IsComplete() const; - - /** - * Create a timer to be dispatched after the amount of time indicated by - * the "timeout_interval" field of the #val record in order to check if - * #last_activity_time is old enough to timeout analysis of the file. - */ - void ScheduleInactivityTimer() const; - - /** - * Attaches an action. Only one action per type can be attached at a time. - * @return true if the action was attached, else false. - */ - bool AddAction(EnumVal* act, RecordVal* args); - - /** - * Removes an action. - * @return true if the action was removed, else false. - */ - bool RemoveAction(EnumVal* act); - - /** - * Pass in non-sequential data and deliver to attached actions/analyzers. - */ - void DataIn(const u_char* data, uint64 len, uint64 offset); - - /** - * Pass in sequential data and deliver to attached actions/analyzers. - */ - void DataIn(const u_char* data, uint64 len); - - /** - * Inform attached actions/analyzers about end of file being seen. - */ - void EndOfFile(); - - /** - * Inform attached actions/analyzers about a gap in file stream. - */ - void Gap(uint64 offset, uint64 len); - -protected: - - friend class Manager; - - /** - * Constructor; only file_analysis::Manager should be creating these. - */ - Info(const string& file_id, Connection* conn = 0, - const string& protocol = ""); - - /** - * Updates the "conn_ids" and "conn_uids" fields in #val record with the - * \c conn_id and UID taken from \a conn. - */ - void UpdateConnectionFields(Connection* conn); - - /** - * Increment a byte count field of #val record by \a size. - */ - void IncrementByteCount(uint64 size, int field_idx); - - /** - * Wrapper to RecordVal::LookupWithDefault for the field in #val at index - * \a idx which automatically unrefs the Val and returns a converted value. - */ - uint64 LookupFieldDefaultCount(int idx) const; - - /** - * Wrapper to RecordVal::LookupWithDefault for the field in #val at index - * \a idx which automatically unrefs the Val and returns a converted value. - */ - double LookupFieldDefaultInterval(int idx) const; - - RecordVal* val; /**< \c FileAnalysis::Info from script layer. */ - double last_activity_time; /**< Time of last activity. */ - bool postpone_timeout; /**< Whether postponing timeout is requested. */ - bool need_reassembly; /**< Whether file stream reassembly is needed. */ - - typedef map ActionMap; - - ActionMap actions; - - /** - * @return the field offset in #val record corresponding to \a field_name. - */ - static int Idx(const string& field_name); - - /** - * Initializes the index offsets for fields in \c FileAnalysis::info record. - */ - static void InitFieldIndices(); - - static int file_id_idx; - static int parent_file_id_idx; - static int protocol_idx; - static int conn_uids_idx; - static int conn_ids_idx; - static int seen_bytes_idx; - static int total_bytes_idx; - static int missing_bytes_idx; - static int overflow_bytes_idx; - static int timeout_interval_idx; - static int actions_idx; - static int action_args_idx; -}; - -/** - * Timer to periodically check if file analysis for a given file is inactive. - */ -class InfoTimer : public Timer { -public: - - InfoTimer(double t, const string& id, double interval) - : Timer(t + interval, TIMER_FILE_ANALYSIS_INACTIVITY), file_id(id) { } - - ~InfoTimer() { } - - /** - * Check inactivity of file_analysis::Info corresponding to #file_id, - * reschedule if active, else call file_analysis::Manager::Timeout. - */ - void Dispatch(double t, int is_expire); - -protected: - - string file_id; -}; - -/** - * Main entry point for interacting with file analysis. - */ -class Manager { -public: - - Manager(); - - ~Manager(); - - /** - * Times out any active file analysis to prepare for shutdown. - */ - void Terminate(); - - /** - * Pass in non-sequential file data. - */ - void DataIn(const string& file_id, const u_char* data, uint64 len, - uint64 offset, Connection* conn = 0, - const string& protocol = ""); - - /** - * Pass in sequential file data. - */ - void DataIn(const string& file_id, const u_char* data, uint64 len, - Connection* conn = 0, const string& protocol = ""); - - /** - * Signal the end of file data. - */ - void EndOfFile(const string& file_id, Connection* conn = 0, - const string& protocol = ""); - - /** - * Signal a gap in the file data stream. - */ - void Gap(const string& file_id, uint64 offset, uint64 len, - Connection* conn = 0, const string& protocol = ""); - - /** - * Provide the expected number of bytes that comprise a file. - */ - void SetSize(const string& file_id, uint64 size, Connection* conn = 0, - const string& protocol = ""); - - /** - * Discard the file_analysis::Info object associated with \a file_id. - * @return false if file identifier did not map to anything, else true. - */ - bool RemoveFile(const string& file_id); - - /** - * If called during \c FileAnalysis::policy evaluation for a - * \c FileAnalysis::TRIGGER_TIMEOUT, requests deferral of analysis timeout. - */ - bool PostponeTimeout(const string& file_id) const; - - /** - * Attaches an action to the file identifier. Only one action of a given - * type can be attached per file identifier at a time. - * @return true if the action was attached, else false. - */ - bool AddAction(const string& file_id, EnumVal* act, RecordVal* args) const; - - /** - * Removes an action for a given file identifier. - * @return true if the action was removed, else false. - */ - bool RemoveAction(const string& file_id, EnumVal* act) const; - - /** - * Calls the \c FileAnalysis::policy hook. - */ - static void EvaluatePolicy(BifEnum::FileAnalysis::Trigger t, Info* info); - -protected: - - friend class InfoTimer; - - typedef map FileMap; - - /** - * @return the Info object mapped to \a file_id. One is created if mapping - * doesn't exist. If it did exist, the activity time is refreshed - * and connection-related fields of the record value may be updated. - */ - Info* IDtoInfo(const string& file_id, Connection* conn = 0, - const string& protocol = ""); - - /** - * @return the Info object mapped to \a file_id, or a null pointer if no - * mapping exists. - */ - Info* Lookup(const string& file_id) const; - - /** - * Evaluate timeout policy for a file and remove the Info object mapped to - * \a file_id if needed. - */ - void Timeout(const string& file_id, bool is_terminating = ::terminating); - - FileMap file_map; /**< Map strings to \c FileAnalysis::Info records. */ -}; - -} // namespace file_analysis - -extern file_analysis::Manager* file_mgr; - -#endif diff --git a/src/FileAnalyzer.cc b/src/FileAnalyzer.cc index 26dd141003..8e994adbf1 100644 --- a/src/FileAnalyzer.cc +++ b/src/FileAnalyzer.cc @@ -1,6 +1,6 @@ #include -#include "FileAnalysisManager.h" +#include "file_analysis/Manager.h" #include "FileAnalyzer.h" #include "Reporter.h" diff --git a/src/file_analysis.bif b/src/file_analysis.bif index f1646e5e8e..546ac5103c 100644 --- a/src/file_analysis.bif +++ b/src/file_analysis.bif @@ -3,7 +3,7 @@ module FileAnalysis; %%{ -#include "FileAnalysisManager.h" +#include "file_analysis/Manager.h" %%} type Info: record; diff --git a/src/file_analysis/Action.h b/src/file_analysis/Action.h new file mode 100644 index 0000000000..ac8713f81e --- /dev/null +++ b/src/file_analysis/Action.h @@ -0,0 +1,49 @@ +#ifndef FILE_ANALYSIS_ACTION_H +#define FILE_ANALYSIS_ACTION_H + +#include "Val.h" + +namespace file_analysis { + +class Info; + +/** + * Base class for actions that can be attached to a file_analysis::Info object. + */ +class Action { +public: + + virtual ~Action() {} + + /** + * Subclasses may override this to receive file data non-sequentially. + */ + virtual void DeliverChunk(const u_char* data, uint64 len, uint64 offset) {} + + /** + * Subclasses may override this to receive file sequentially. + */ + virtual void DeliverStream(const u_char* data, uint64 len) {} + + /** + * Subclasses may override this to specifically handle the end of a file. + */ + virtual void EndOfFile() {} + + /** + * Subclasses may override this to handle missing data in a file stream. + */ + virtual void Undelivered(uint64 offset, uint64 len) {} + +protected: + + Action(Info* arg_info) {} + + Info* info; +}; + +typedef Action* (*ActionInstantiator)(const RecordVal* args, Info* info); + +} // namespace file_analysis + +#endif diff --git a/src/file_analysis/Extract.cc b/src/file_analysis/Extract.cc new file mode 100644 index 0000000000..c580aaa0dd --- /dev/null +++ b/src/file_analysis/Extract.cc @@ -0,0 +1,46 @@ +#include + +#include "Extract.h" +#include "util.h" + +using namespace file_analysis; + +Extract::Extract(Info* arg_info, const string& arg_filename) + : Action(arg_info), filename(arg_filename) + { + fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666); + + if ( fd < 0 ) + { + fd = 0; + char buf[128]; + strerror_r(errno, buf, sizeof(buf)); + reporter->Error("cannot open %s: %s", filename.c_str(), buf); + } + } + +Extract::~Extract() + { + if ( fd ) + safe_close(fd); + } + +Action* Extract::Instantiate(const RecordVal* args, Info* info) + { + const char* field = "extract_filename"; + int off = BifType::Record::FileAnalysis::ActionArgs->FieldOffset(field); + Val* v = args->Lookup(off); + + if ( ! v ) return 0; + + return new Extract(info, v->AsString()->CheckString()); + } + +void Extract::DeliverChunk(const u_char* data, uint64 len, uint64 offset) + { + Action::DeliverChunk(data, len, offset); + + if ( ! fd ) return; + + safe_pwrite(fd, data, len, offset); + } diff --git a/src/file_analysis/Extract.h b/src/file_analysis/Extract.h new file mode 100644 index 0000000000..17c3a959ad --- /dev/null +++ b/src/file_analysis/Extract.h @@ -0,0 +1,33 @@ +#ifndef FILE_ANALYSIS_EXTRACT_H +#define FILE_ANALYSIS_EXTRACT_H + +#include + +#include "Val.h" +#include "Info.h" + +namespace file_analysis { + +/** + * An action to simply extract files to disk. + */ +class Extract : Action { +public: + + static Action* Instantiate(const RecordVal* args, Info* info); + + ~Extract(); + + virtual void DeliverChunk(const u_char* data, uint64 len, uint64 offset); + +protected: + + Extract(Info* arg_info, const string& arg_filename); + + string filename; + int fd; +}; + +} // namespace file_analysis + +#endif diff --git a/src/FileAnalysisManager.cc b/src/file_analysis/Info.cc similarity index 53% rename from src/FileAnalysisManager.cc rename to src/file_analysis/Info.cc index bd170b87bd..60729cd590 100644 --- a/src/FileAnalysisManager.cc +++ b/src/file_analysis/Info.cc @@ -1,59 +1,20 @@ -#include +#include -#include "FileAnalysisManager.h" -#include "util.h" +#include "Info.h" +#include "InfoTimer.h" +#include "Reporter.h" +#include "Val.h" + +#include "Action.h" +#include "Extract.h" using namespace file_analysis; // keep in order w/ declared enum values in file_analysis.bif static ActionInstantiator action_factory[] = { - Extract::Instantiate, + Extract::Instantiate, }; -Action::Action(Info* arg_info) : info(arg_info) - { - } - -Extract::Extract(Info* arg_info, const string& arg_filename) - : Action(arg_info), filename(arg_filename) - { - fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666); - - if ( fd < 0 ) - { - fd = 0; - char buf[128]; - strerror_r(errno, buf, sizeof(buf)); - reporter->Error("cannot open %s: %s", filename.c_str(), buf); - } - } - -Extract::~Extract() - { - if ( fd ) - safe_close(fd); - } - -Action* Extract::Instantiate(const RecordVal* args, Info* info) - { - const char* field = "extract_filename"; - int off = BifType::Record::FileAnalysis::ActionArgs->FieldOffset(field); - Val* v = args->Lookup(off); - - if ( ! v ) return 0; - - return new Extract(info, v->AsString()->CheckString()); - } - -void Extract::DeliverChunk(const u_char* data, uint64 len, uint64 offset) - { - Action::DeliverChunk(data, len, offset); - - if ( ! fd ) return; - - safe_pwrite(fd, data, len, offset); - } - static TableVal* empty_conn_id_set() { TypeList* set_index = new TypeList(conn_id); @@ -295,203 +256,3 @@ void Info::Gap(uint64 offset, uint64 len) IncrementByteCount(len, missing_bytes_idx); } - -void InfoTimer::Dispatch(double t, int is_expire) - { - Info* info = file_mgr->Lookup(file_id); - - if ( ! info ) return; - - double last_active = info->LastActivityTime(); - double inactive_time = t > last_active ? t - last_active : 0.0; - - DBG_LOG(DBG_FILE_ANALYSIS, "Checking inactivity for %s, last active at %f, " - "inactive for %f", file_id.c_str(), last_active, inactive_time); - - if ( last_active == 0.0 ) - { - // was created when network_time was zero, so re-schedule w/ valid time - info->UpdateLastActivityTime(); - info->ScheduleInactivityTimer(); - return; - } - - if ( inactive_time >= info->TimeoutInterval() ) - file_mgr->Timeout(file_id); - else if ( ! is_expire ) - info->ScheduleInactivityTimer(); - } - -Manager::Manager() - { - } - -Manager::~Manager() - { - Terminate(); - } - -void Manager::Terminate() - { - vector keys; - for ( FileMap::iterator it = file_map.begin(); it != file_map.end(); ++it ) - keys.push_back(it->first); - for ( size_t i = 0; i < keys.size(); ++i ) - Timeout(keys[i], true); - } - -static void check_file_done(Info* info) - { - if ( info->IsComplete() ) - { - Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_DONE, info); - file_mgr->RemoveFile(info->FileID()); - } - } - -void Manager::DataIn(const string& file_id, const u_char* data, uint64 len, - uint64 offset, Connection* conn, const string& protocol) - { - Info* info = IDtoInfo(file_id, conn, protocol); - info->DataIn(data, len, offset); - check_file_done(info); - } - -void Manager::DataIn(const string& file_id, const u_char* data, uint64 len, - Connection* conn, const string& protocol) - { - Info* info = IDtoInfo(file_id, conn, protocol); - info->DataIn(data, len); - check_file_done(info); - } - -void Manager::EndOfFile(const string& file_id, Connection* conn, - const string& protocol) - { - Info* info = IDtoInfo(file_id, conn, protocol); - info->EndOfFile(); - Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_EOF, info); - } - -void Manager::Gap(const string& file_id, uint64 offset, uint64 len, - Connection* conn, const string& protocol) - { - Info* info = IDtoInfo(file_id, conn, protocol); - info->Gap(offset, len); - Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_GAP, info); - } - -void Manager::SetSize(const string& file_id, uint64 size, - Connection* conn, const string& protocol) - { - Info* info = IDtoInfo(file_id, conn, protocol); - info->SetTotalBytes(size); - check_file_done(info); - } - -void Manager::EvaluatePolicy(BifEnum::FileAnalysis::Trigger t, Info* info) - { - const ID* id = global_scope()->Lookup("FileAnalysis::policy"); - assert(id); - const Func* hook = id->ID_Val()->AsFunc(); - - val_list vl(2); - vl.append(new EnumVal(t, BifType::Enum::FileAnalysis::Trigger)); - vl.append(info->val->Ref()); - - info->postpone_timeout = false; - - Val* result = hook->Call(&vl); - Unref(result); - } - -bool Manager::PostponeTimeout(const string& file_id) const - { - Info* info = Lookup(file_id); - - if ( ! info ) return false; - - info->postpone_timeout = true; - return true; - } - -bool Manager::AddAction(const string& file_id, EnumVal* act, - RecordVal* args) const - { - Info* info = Lookup(file_id); - - if ( ! info ) return false; - - return info->AddAction(act, args); - } - -bool Manager::RemoveAction(const string& file_id, EnumVal* act) const - { - Info* info = Lookup(file_id); - - if ( ! info ) return false; - - return info->RemoveAction(act); - } - -Info* Manager::IDtoInfo(const string& file_id, Connection* conn, - const string& protocol) - { - Info* rval = file_map[file_id]; - - if ( ! rval ) - { - rval = file_map[file_id] = new Info(file_id, conn, protocol); - Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_NEW, rval); - } - else - { - rval->UpdateLastActivityTime(); - rval->UpdateConnectionFields(conn); - } - - return rval; - } - -Info* Manager::Lookup(const string& file_id) const - { - FileMap::const_iterator it = file_map.find(file_id); - - if ( it == file_map.end() ) return 0; - - return it->second; - } - -void Manager::Timeout(const string& file_id, bool is_terminating) - { - Info* info = Lookup(file_id); - - if ( ! info ) return; - - Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TIMEOUT, info); - - if ( info->postpone_timeout && ! is_terminating ) - { - DBG_LOG(DBG_FILE_ANALYSIS, "Postpone file analysis timeout for %s", - info->FileID().c_str()); - info->UpdateLastActivityTime(); - info->ScheduleInactivityTimer(); - return; - } - - DBG_LOG(DBG_FILE_ANALYSIS, "File analysis timeout for %s", - info->FileID().c_str()); - - RemoveFile(file_id); - } - -bool Manager::RemoveFile(const string& file_id) - { - FileMap::iterator it = file_map.find(file_id); - - if ( it == file_map.end() ) return false; - - delete it->second; - file_map.erase(it); - return true; - } diff --git a/src/file_analysis/Info.h b/src/file_analysis/Info.h new file mode 100644 index 0000000000..2823fa2d2c --- /dev/null +++ b/src/file_analysis/Info.h @@ -0,0 +1,160 @@ +#ifndef FILE_ANALYSIS_INFO_H +#define FILE_ANALYSIS_INFO_H + +#include +#include + +#include "Conn.h" +#include "Val.h" +#include "Action.h" + +namespace file_analysis { + +/** + * Wrapper class around \c FileAnalysis::Info record values from script layer. + */ +class Info { +public: + + ~Info(); + + /** + * @return value (seconds) of the "timeout_interval" field from #val record. + */ + double TimeoutInterval() const; + + /** + * @return value of the "file_id" field from #val record. + */ + string FileID() const; + + /** + * @return #last_activity_time + */ + double LastActivityTime() const { return last_activity_time; } + + /** + * Refreshes #last_activity_time with current network time. + */ + void UpdateLastActivityTime() { last_activity_time = network_time; } + + /** + * Set "total_bytes" field of #val record to \a size. + */ + void SetTotalBytes(uint64 size); + + /** + * Compares "seen_bytes" field to "total_bytes" field of #val record + * and returns true if the comparison indicates the full file was seen. + * If "total_bytes" hasn't been set yet, it returns false. + */ + bool IsComplete() const; + + /** + * Create a timer to be dispatched after the amount of time indicated by + * the "timeout_interval" field of the #val record in order to check if + * #last_activity_time is old enough to timeout analysis of the file. + */ + void ScheduleInactivityTimer() const; + + /** + * Attaches an action. Only one action per type can be attached at a time. + * @return true if the action was attached, else false. + */ + bool AddAction(EnumVal* act, RecordVal* args); + + /** + * Removes an action. + * @return true if the action was removed, else false. + */ + bool RemoveAction(EnumVal* act); + + /** + * Pass in non-sequential data and deliver to attached actions/analyzers. + */ + void DataIn(const u_char* data, uint64 len, uint64 offset); + + /** + * Pass in sequential data and deliver to attached actions/analyzers. + */ + void DataIn(const u_char* data, uint64 len); + + /** + * Inform attached actions/analyzers about end of file being seen. + */ + void EndOfFile(); + + /** + * Inform attached actions/analyzers about a gap in file stream. + */ + void Gap(uint64 offset, uint64 len); + +protected: + + friend class Manager; + + /** + * Constructor; only file_analysis::Manager should be creating these. + */ + Info(const string& file_id, Connection* conn = 0, + const string& protocol = ""); + + /** + * Updates the "conn_ids" and "conn_uids" fields in #val record with the + * \c conn_id and UID taken from \a conn. + */ + void UpdateConnectionFields(Connection* conn); + + /** + * Increment a byte count field of #val record by \a size. + */ + void IncrementByteCount(uint64 size, int field_idx); + + /** + * Wrapper to RecordVal::LookupWithDefault for the field in #val at index + * \a idx which automatically unrefs the Val and returns a converted value. + */ + uint64 LookupFieldDefaultCount(int idx) const; + + /** + * Wrapper to RecordVal::LookupWithDefault for the field in #val at index + * \a idx which automatically unrefs the Val and returns a converted value. + */ + double LookupFieldDefaultInterval(int idx) const; + + RecordVal* val; /**< \c FileAnalysis::Info from script layer. */ + double last_activity_time; /**< Time of last activity. */ + bool postpone_timeout; /**< Whether postponing timeout is requested. */ + bool need_reassembly; /**< Whether file stream reassembly is needed. */ + + typedef map ActionMap; + + ActionMap actions; + + /** + * @return the field offset in #val record corresponding to \a field_name. + */ + static int Idx(const string& field_name); + + /** + * Initializes the index offsets for fields in \c FileAnalysis::info record. + */ + static void InitFieldIndices(); + + static int file_id_idx; + static int parent_file_id_idx; + static int protocol_idx; + static int conn_uids_idx; + static int conn_ids_idx; + static int seen_bytes_idx; + static int total_bytes_idx; + static int missing_bytes_idx; + static int overflow_bytes_idx; + static int timeout_interval_idx; + static int actions_idx; + static int action_args_idx; +}; + +} // namespace file_analysis + +#endif diff --git a/src/file_analysis/InfoTimer.cc b/src/file_analysis/InfoTimer.cc new file mode 100644 index 0000000000..f1bb524d40 --- /dev/null +++ b/src/file_analysis/InfoTimer.cc @@ -0,0 +1,30 @@ +#include "Manager.h" +#include "Info.h" + +using namespace file_analysis; + +void InfoTimer::Dispatch(double t, int is_expire) + { + Info* info = file_mgr->Lookup(file_id); + + if ( ! info ) return; + + double last_active = info->LastActivityTime(); + double inactive_time = t > last_active ? t - last_active : 0.0; + + DBG_LOG(DBG_FILE_ANALYSIS, "Checking inactivity for %s, last active at %f, " + "inactive for %f", file_id.c_str(), last_active, inactive_time); + + if ( last_active == 0.0 ) + { + // was created when network_time was zero, so re-schedule w/ valid time + info->UpdateLastActivityTime(); + info->ScheduleInactivityTimer(); + return; + } + + if ( inactive_time >= info->TimeoutInterval() ) + file_mgr->Timeout(file_id); + else if ( ! is_expire ) + info->ScheduleInactivityTimer(); + } diff --git a/src/file_analysis/InfoTimer.h b/src/file_analysis/InfoTimer.h new file mode 100644 index 0000000000..d5432e0ebc --- /dev/null +++ b/src/file_analysis/InfoTimer.h @@ -0,0 +1,33 @@ +#ifndef FILE_ANALYSIS_INFOTIMER_H +#define FILE_ANALYSIS_INFOTIMER_H + +#include "Timer.h" +#include + +namespace file_analysis { + +/** + * Timer to periodically check if file analysis for a given file is inactive. + */ +class InfoTimer : public Timer { +public: + + InfoTimer(double t, const string& id, double interval) + : Timer(t + interval, TIMER_FILE_ANALYSIS_INACTIVITY), file_id(id) {} + + ~InfoTimer() {} + + /** + * Check inactivity of file_analysis::Info corresponding to #file_id, + * reschedule if active, else call file_analysis::Manager::Timeout. + */ + void Dispatch(double t, int is_expire); + +protected: + + string file_id; +}; + +} // namespace file_analysis + +#endif diff --git a/src/file_analysis/Manager.cc b/src/file_analysis/Manager.cc new file mode 100644 index 0000000000..33011a6ec6 --- /dev/null +++ b/src/file_analysis/Manager.cc @@ -0,0 +1,181 @@ +#include +#include + +#include "Manager.h" +#include "Info.h" + +using namespace file_analysis; + +Manager::Manager() + { + } + +Manager::~Manager() + { + Terminate(); + } + +void Manager::Terminate() + { + vector keys; + for ( FileMap::iterator it = file_map.begin(); it != file_map.end(); ++it ) + keys.push_back(it->first); + for ( size_t i = 0; i < keys.size(); ++i ) + Timeout(keys[i], true); + } + +static void check_file_done(Info* info) + { + if ( info->IsComplete() ) + { + Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_DONE, info); + file_mgr->RemoveFile(info->FileID()); + } + } + +void Manager::DataIn(const string& file_id, const u_char* data, uint64 len, + uint64 offset, Connection* conn, const string& protocol) + { + Info* info = IDtoInfo(file_id, conn, protocol); + info->DataIn(data, len, offset); + check_file_done(info); + } + +void Manager::DataIn(const string& file_id, const u_char* data, uint64 len, + Connection* conn, const string& protocol) + { + Info* info = IDtoInfo(file_id, conn, protocol); + info->DataIn(data, len); + check_file_done(info); + } + +void Manager::EndOfFile(const string& file_id, Connection* conn, + const string& protocol) + { + Info* info = IDtoInfo(file_id, conn, protocol); + info->EndOfFile(); + Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_EOF, info); + } + +void Manager::Gap(const string& file_id, uint64 offset, uint64 len, + Connection* conn, const string& protocol) + { + Info* info = IDtoInfo(file_id, conn, protocol); + info->Gap(offset, len); + Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_GAP, info); + } + +void Manager::SetSize(const string& file_id, uint64 size, + Connection* conn, const string& protocol) + { + Info* info = IDtoInfo(file_id, conn, protocol); + info->SetTotalBytes(size); + check_file_done(info); + } + +void Manager::EvaluatePolicy(BifEnum::FileAnalysis::Trigger t, Info* info) + { + const ID* id = global_scope()->Lookup("FileAnalysis::policy"); + assert(id); + const Func* hook = id->ID_Val()->AsFunc(); + + val_list vl(2); + vl.append(new EnumVal(t, BifType::Enum::FileAnalysis::Trigger)); + vl.append(info->val->Ref()); + + info->postpone_timeout = false; + + Val* result = hook->Call(&vl); + Unref(result); + } + +bool Manager::PostponeTimeout(const string& file_id) const + { + Info* info = Lookup(file_id); + + if ( ! info ) return false; + + info->postpone_timeout = true; + return true; + } + +bool Manager::AddAction(const string& file_id, EnumVal* act, + RecordVal* args) const + { + Info* info = Lookup(file_id); + + if ( ! info ) return false; + + return info->AddAction(act, args); + } + +bool Manager::RemoveAction(const string& file_id, EnumVal* act) const + { + Info* info = Lookup(file_id); + + if ( ! info ) return false; + + return info->RemoveAction(act); + } + +Info* Manager::IDtoInfo(const string& file_id, Connection* conn, + const string& protocol) + { + Info* rval = file_map[file_id]; + + if ( ! rval ) + { + rval = file_map[file_id] = new Info(file_id, conn, protocol); + Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_NEW, rval); + } + else + { + rval->UpdateLastActivityTime(); + rval->UpdateConnectionFields(conn); + } + + return rval; + } + +Info* Manager::Lookup(const string& file_id) const + { + FileMap::const_iterator it = file_map.find(file_id); + + if ( it == file_map.end() ) return 0; + + return it->second; + } + +void Manager::Timeout(const string& file_id, bool is_terminating) + { + Info* info = Lookup(file_id); + + if ( ! info ) return; + + Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TIMEOUT, info); + + if ( info->postpone_timeout && ! is_terminating ) + { + DBG_LOG(DBG_FILE_ANALYSIS, "Postpone file analysis timeout for %s", + info->FileID().c_str()); + info->UpdateLastActivityTime(); + info->ScheduleInactivityTimer(); + return; + } + + DBG_LOG(DBG_FILE_ANALYSIS, "File analysis timeout for %s", + info->FileID().c_str()); + + RemoveFile(file_id); + } + +bool Manager::RemoveFile(const string& file_id) + { + FileMap::iterator it = file_map.find(file_id); + + if ( it == file_map.end() ) return false; + + delete it->second; + file_map.erase(it); + return true; + } diff --git a/src/file_analysis/Manager.h b/src/file_analysis/Manager.h new file mode 100644 index 0000000000..3601c8b43f --- /dev/null +++ b/src/file_analysis/Manager.h @@ -0,0 +1,125 @@ +#ifndef FILE_ANALYSIS_MANAGER_H +#define FILE_ANALYSIS_MANAGER_H + +#include +#include + +#include "Net.h" +#include "Conn.h" +#include "Val.h" + +#include "Info.h" +#include "InfoTimer.h" + +namespace file_analysis { + +/** + * Main entry point for interacting with file analysis. + */ +class Manager { +public: + + Manager(); + + ~Manager(); + + /** + * Times out any active file analysis to prepare for shutdown. + */ + void Terminate(); + + /** + * Pass in non-sequential file data. + */ + void DataIn(const string& file_id, const u_char* data, uint64 len, + uint64 offset, Connection* conn = 0, + const string& protocol = ""); + + /** + * Pass in sequential file data. + */ + void DataIn(const string& file_id, const u_char* data, uint64 len, + Connection* conn = 0, const string& protocol = ""); + + /** + * Signal the end of file data. + */ + void EndOfFile(const string& file_id, Connection* conn = 0, + const string& protocol = ""); + + /** + * Signal a gap in the file data stream. + */ + void Gap(const string& file_id, uint64 offset, uint64 len, + Connection* conn = 0, const string& protocol = ""); + + /** + * Provide the expected number of bytes that comprise a file. + */ + void SetSize(const string& file_id, uint64 size, Connection* conn = 0, + const string& protocol = ""); + + /** + * Discard the file_analysis::Info object associated with \a file_id. + * @return false if file identifier did not map to anything, else true. + */ + bool RemoveFile(const string& file_id); + + /** + * If called during \c FileAnalysis::policy evaluation for a + * \c FileAnalysis::TRIGGER_TIMEOUT, requests deferral of analysis timeout. + */ + bool PostponeTimeout(const string& file_id) const; + + /** + * Attaches an action to the file identifier. Only one action of a given + * type can be attached per file identifier at a time. + * @return true if the action was attached, else false. + */ + bool AddAction(const string& file_id, EnumVal* act, RecordVal* args) const; + + /** + * Removes an action for a given file identifier. + * @return true if the action was removed, else false. + */ + bool RemoveAction(const string& file_id, EnumVal* act) const; + + /** + * Calls the \c FileAnalysis::policy hook. + */ + static void EvaluatePolicy(BifEnum::FileAnalysis::Trigger t, Info* info); + +protected: + + friend class InfoTimer; + + typedef map FileMap; + + /** + * @return the Info object mapped to \a file_id. One is created if mapping + * doesn't exist. If it did exist, the activity time is refreshed + * and connection-related fields of the record value may be updated. + */ + Info* IDtoInfo(const string& file_id, Connection* conn = 0, + const string& protocol = ""); + + /** + * @return the Info object mapped to \a file_id, or a null pointer if no + * mapping exists. + */ + Info* Lookup(const string& file_id) const; + + /** + * Evaluate timeout policy for a file and remove the Info object mapped to + * \a file_id if needed. + */ + void Timeout(const string& file_id, bool is_terminating = ::terminating); + + FileMap file_map; /**< Map strings to \c FileAnalysis::Info records. */ +}; + +} // namespace file_analysis + +extern file_analysis::Manager* file_mgr; + +#endif diff --git a/src/main.cc b/src/main.cc index 1657c3b89f..c90eb73d37 100644 --- a/src/main.cc +++ b/src/main.cc @@ -58,7 +58,7 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void); #include "logging/Manager.h" #include "logging/writers/Ascii.h" -#include "FileAnalysisManager.h" +#include "file_analysis/Manager.h" #include "binpac_bro.h"