diff --git a/src/FileAnalysisManager.cc b/src/FileAnalysisManager.cc index afadfee55a..724c9dc984 100644 --- a/src/FileAnalysisManager.cc +++ b/src/FileAnalysisManager.cc @@ -5,6 +5,20 @@ using namespace file_analysis; +Action::Action(Info* arg_info) : info(arg_info) + { + } + +Extract::Extract(Info* arg_info, const string& arg_filename) + : Action(arg_info), filename(arg_filename) + { + } + +void Extract::DeliverStream(const u_char* data, uint64 len) + { + // TODO: write data to filename + } + static TableVal* empty_conn_id_set() { TypeList* set_index = new TypeList(conn_id); @@ -59,6 +73,7 @@ Info::Info(const string& file_id, Connection* conn, const string& protocol) } val = new RecordVal(BifType::Record::FileAnalysis::Info); + // TODO: hash/prettify file_id for script layer presentation val->Assign(file_id_idx, new StringVal(file_id.c_str())); UpdateConnectionFields(conn); @@ -72,6 +87,8 @@ Info::Info(const string& file_id, Connection* conn, const string& protocol) Info::~Info() { + for ( size_t i = 0; i < analyzers.size(); ++i ) + DBG_LOG(DBG_FILE_ANALYSIS, "Destroying Info object %s", FileID().c_str()); Unref(val); } @@ -91,6 +108,22 @@ void Info::UpdateConnectionFields(Connection* conn) conn_ids->AsTableVal()->Assign(get_conn_id_val(conn), 0); } +uint64 Info::FieldDefaultCount(int idx) const + { + Val* v = val->LookupWithDefault(idx); + uint64 rval = v->AsCount(); + Unref(v); + return rval; + } + +double Info::FieldDefaultInterval(int idx) const + { + Val* v = val->LookupWithDefault(idx); + double rval = v->AsInterval(); + Unref(v); + return rval; + } + int Info::Idx(const string& field) { int rval = BifType::Record::FileAnalysis::Info->FieldOffset(field.c_str()); @@ -102,7 +135,7 @@ int Info::Idx(const string& field) double Info::TimeoutInterval() const { - return val->LookupWithDefault(timeout_interval_idx)->AsInterval(); + return FieldDefaultInterval(timeout_interval_idx); } string Info::FileID() const @@ -110,15 +143,24 @@ string Info::FileID() const return val->Lookup(file_id_idx)->AsString()->CheckString(); } +void Info::IncrementSeenBytes(uint64 size) + { + uint64 old = FieldDefaultCount(seen_bytes_idx); + val->Assign(seen_bytes_idx, new Val(old + size, TYPE_COUNT)); + } + void Info::SetTotalBytes(uint64 size) { val->Assign(total_bytes_idx, new Val(size, TYPE_COUNT)); + } - if ( val->LookupWithDefault(seen_bytes_idx)->AsCount() >= size ) - { - Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_DONE, this); - file_mgr->Remove(FileID()); - } +bool Info::IsComplete() const + { + Val* total = val->Lookup(total_bytes_idx); + if ( ! total ) return false; + if ( FieldDefaultCount(seen_bytes_idx) >= total->AsCount() ) + return true; + return false; } void Info::ScheduleInactivityTimer() const @@ -170,6 +212,15 @@ void Manager::Terminate() Timeout(keys[i], true); } +static void check_file_done(Info* info) + { + if ( info->IsComplete() ) + { + Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_DONE, info); + file_mgr->Remove(info->FileID()); + } + } + void Manager::DataIn(const string& file_id, const u_char* data, uint64 len, uint64 offset, Connection* conn, const string& protocol) { @@ -177,6 +228,7 @@ void Manager::DataIn(const string& file_id, const u_char* data, uint64 len, info->UpdateLastActivityTime(); info->UpdateConnectionFields(conn); // TODO: more stuff + check_file_done(info); } void Manager::DataIn(const string& file_id, const u_char* data, uint64 len, @@ -186,9 +238,10 @@ void Manager::DataIn(const string& file_id, const u_char* data, uint64 len, info->UpdateLastActivityTime(); info->UpdateConnectionFields(conn); // TODO: more stuff + check_file_done(info); } -void Manager::EndOfData(const string& file_id, Connection* conn, +void Manager::EndOfFile(const string& file_id, Connection* conn, const string& protocol) { Info* info = IDtoInfo(file_id, conn, protocol); @@ -205,6 +258,7 @@ void Manager::SetSize(const string& file_id, uint64 size, info->UpdateLastActivityTime(); info->UpdateConnectionFields(conn); info->SetTotalBytes(size); + check_file_done(info); } void Manager::EvaluatePolicy(BifEnum::FileAnalysis::Trigger t, Info* info) diff --git a/src/FileAnalysisManager.h b/src/FileAnalysisManager.h index 59b5681eb8..ed11b9d912 100644 --- a/src/FileAnalysisManager.h +++ b/src/FileAnalysisManager.h @@ -3,6 +3,7 @@ #include #include +#include #include "Conn.h" #include "Analyzer.h" @@ -12,6 +13,42 @@ namespace file_analysis { +class Info; + +/** + * Base class for actions that can be attached to a file_analysis::Info object. + */ +class Action { +public: + + Action(Info* arg_info); + + ~Action() {} + + virtual void DeliverStream(const u_char* data, uint64 len) = 0; + +protected: + + Info* info; +}; + +/** + * An action to simply extract files to disk. + */ +class Extract : Action { +public: + + Extract(Info* arg_info, const string& arg_filename); + + ~Extract() {} + + virtual void DeliverStream(const u_char* data, uint64 len); + +protected: + + string filename; +}; + /** * Wrapper class around \c FileAnalysis::Info record values from script layer. */ @@ -41,11 +78,22 @@ public: void UpdateLastActivityTime() { last_activity_time = network_time; } /** - * Set "total_bytes" field of #val record to \a size, check if "seen_bytes" - * is greater or equal to it, and evaluate \c FileAnalysis::policy if so. + * Increments the "seen_bytes" field of #val record by \a size. + */ + void IncrementSeenBytes(uint64 size); + + /** + * Set "total_bytes" field of #val record to \a size. */ void SetTotalBytes(uint64 size); + /** + * Compares "seen_bytes" field to "total_bytes" field of #val record + * and returns true if the comparison indicates the full file was seen. + * If "total_bytes" hasn't been set yet, it returns false. + */ + bool IsComplete() const; + /** * Create a timer to be dispatched after the amount of time indicated by * the "timeout_interval" field of the #val record in order to check if @@ -69,10 +117,24 @@ protected: */ void UpdateConnectionFields(Connection* conn); + /** + * Wrapper to RecordVal::LookupWithDefault for the field in #val at index + * \a idx which automatically unrefs the Val and returns a converted value. + */ + uint64 FieldDefaultCount(int idx) const; + + /** + * Wrapper to RecordVal::LookupWithDefault for the field in #val at index + * \a idx which automatically unrefs the Val and returns a converted value. + */ + double FieldDefaultInterval(int idx) const; + RecordVal* val; /**< \c FileAnalysis::Info from script layer. */ double last_activity_time; /**< Time of last activity. */ bool postpone_timeout; /**< Whether postponing timeout is requested. */ + vector analyzers; + /** * @return the field offset in #val record corresponding to \a field_name. */ @@ -142,7 +204,7 @@ public: /** * Signal the end of file data. */ - void EndOfData(const string& file_id, Connection* conn = 0, + void EndOfFile(const string& file_id, Connection* conn = 0, const string& protocol = ""); /** diff --git a/src/FileAnalyzer.cc b/src/FileAnalyzer.cc index d4064e8144..b6cb4e74b8 100644 --- a/src/FileAnalyzer.cc +++ b/src/FileAnalyzer.cc @@ -1,5 +1,6 @@ #include +#include "FileAnalysisManager.h" #include "FileAnalyzer.h" #include "Reporter.h" @@ -16,12 +17,20 @@ File_Analyzer::File_Analyzer(Connection* conn) InitMagic(&magic, MAGIC_NONE); InitMagic(&magic_mime, MAGIC_MIME); } + + char op[256], rp[256]; + modp_ulitoa10(ntohs(conn->OrigPort()), op); + modp_ulitoa10(ntohs(conn->RespPort()), rp); + file_id = "TCPFile " + conn->OrigAddr().AsString() + ":" + op + "->" + + conn->RespAddr().AsString() + ":" + rp; } void File_Analyzer::DeliverStream(int len, const u_char* data, bool orig) { TCP_ApplicationAnalyzer::DeliverStream(len, data, orig); + file_mgr->DataIn(file_id, data, len, Conn()); + int n = min(len, BUFFER_SIZE - buffer_len); if ( n ) @@ -39,6 +48,8 @@ void File_Analyzer::Done() { TCP_ApplicationAnalyzer::Done(); + file_mgr->EndOfFile(file_id, Conn()); + if ( buffer_len && buffer_len != BUFFER_SIZE ) Identify(); } diff --git a/src/FileAnalyzer.h b/src/FileAnalyzer.h index dcf9d22e8e..dedfaf3f02 100644 --- a/src/FileAnalyzer.h +++ b/src/FileAnalyzer.h @@ -5,6 +5,7 @@ #include "TCP.h" +#include #include class File_Analyzer : public TCP_ApplicationAnalyzer { @@ -33,6 +34,8 @@ protected: static magic_t magic; static magic_t magic_mime; + + string file_id; }; #endif