Prototype file analyzer/actions, connect TCP analyzer to file analysis.

No way to add analyzers/actions and they don't do anything yet.

src/FileAnalysisManager.cc

@@ -5,6 +5,20 @@
 
 using namespace file_analysis;
 
+Action::Action(Info* arg_info) : info(arg_info)
+	{
+	}
+
+Extract::Extract(Info* arg_info, const string& arg_filename)
+    : Action(arg_info), filename(arg_filename)
+	{
+	}
+
+void Extract::DeliverStream(const u_char* data, uint64 len)
+	{
+	// TODO: write data to filename
+	}
+
 static TableVal* empty_conn_id_set()
 	{
 	TypeList* set_index = new TypeList(conn_id);
@@ -59,6 +73,7 @@ Info::Info(const string& file_id, Connection* conn, const string& protocol)
 		}
 
 	val = new RecordVal(BifType::Record::FileAnalysis::Info);
+	// TODO: hash/prettify file_id for script layer presentation
 	val->Assign(file_id_idx, new StringVal(file_id.c_str()));
 
 	UpdateConnectionFields(conn);
@@ -72,6 +87,8 @@ Info::Info(const string& file_id, Connection* conn, const string& protocol)
 
 Info::~Info()
 	{
+	for ( size_t i = 0; i < analyzers.size(); ++i )
+
 	DBG_LOG(DBG_FILE_ANALYSIS, "Destroying Info object %s", FileID().c_str());
 	Unref(val);
 	}
@@ -91,6 +108,22 @@ void Info::UpdateConnectionFields(Connection* conn)
 	conn_ids->AsTableVal()->Assign(get_conn_id_val(conn), 0);
 	}
 
+uint64 Info::FieldDefaultCount(int idx) const
+	{
+	Val* v = val->LookupWithDefault(idx);
+	uint64 rval = v->AsCount();
+	Unref(v);
+	return rval;
+	}
+
+double Info::FieldDefaultInterval(int idx) const
+	{
+	Val* v = val->LookupWithDefault(idx);
+	double rval = v->AsInterval();
+	Unref(v);
+	return rval;
+	}
+
 int Info::Idx(const string& field)
 	{
 	int rval = BifType::Record::FileAnalysis::Info->FieldOffset(field.c_str());
@@ -102,7 +135,7 @@ int Info::Idx(const string& field)
 
 double Info::TimeoutInterval() const
 	{
-	return val->LookupWithDefault(timeout_interval_idx)->AsInterval();
+	return FieldDefaultInterval(timeout_interval_idx);
 	}
 
 string Info::FileID() const
@@ -110,15 +143,24 @@ string Info::FileID() const
 	return val->Lookup(file_id_idx)->AsString()->CheckString();
 	}
 
+void Info::IncrementSeenBytes(uint64 size)
+	{
+	uint64 old = FieldDefaultCount(seen_bytes_idx);
+	val->Assign(seen_bytes_idx, new Val(old + size, TYPE_COUNT));
+	}
+
 void Info::SetTotalBytes(uint64 size)
 	{
 	val->Assign(total_bytes_idx, new Val(size, TYPE_COUNT));
-
-	if ( val->LookupWithDefault(seen_bytes_idx)->AsCount() >= size )
-		{
-		Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_DONE, this);
-		file_mgr->Remove(FileID());
 	}
+
+bool Info::IsComplete() const
+	{
+	Val* total = val->Lookup(total_bytes_idx);
+	if ( ! total ) return false;
+	if ( FieldDefaultCount(seen_bytes_idx) >= total->AsCount() )
+		return true;
+	return false;
 	}
 
 void Info::ScheduleInactivityTimer() const
@@ -170,6 +212,15 @@ void Manager::Terminate()
 		Timeout(keys[i], true);
 	}
 
+static void check_file_done(Info* info)
+	{
+	if ( info->IsComplete() )
+		{
+		Manager::EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_DONE, info);
+		file_mgr->Remove(info->FileID());
+		}
+	}
+
 void Manager::DataIn(const string& file_id, const u_char* data, uint64 len,
                      uint64 offset, Connection* conn, const string& protocol)
 	{
@@ -177,6 +228,7 @@ void Manager::DataIn(const string& file_id, const u_char* data, uint64 len,
 	info->UpdateLastActivityTime();
 	info->UpdateConnectionFields(conn);
 	// TODO: more stuff
+	check_file_done(info);
 	}
 
 void Manager::DataIn(const string& file_id, const u_char* data, uint64 len,
@@ -186,9 +238,10 @@ void Manager::DataIn(const string& file_id, const u_char* data, uint64 len,
 	info->UpdateLastActivityTime();
 	info->UpdateConnectionFields(conn);
 	// TODO: more stuff
+	check_file_done(info);
 	}
 
-void Manager::EndOfData(const string& file_id, Connection* conn,
+void Manager::EndOfFile(const string& file_id, Connection* conn,
                         const string& protocol)
 	{
 	Info* info = IDtoInfo(file_id, conn, protocol);
@@ -205,6 +258,7 @@ void Manager::SetSize(const string& file_id, uint64 size,
 	info->UpdateLastActivityTime();
 	info->UpdateConnectionFields(conn);
 	info->SetTotalBytes(size);
+	check_file_done(info);
 	}
 
 void Manager::EvaluatePolicy(BifEnum::FileAnalysis::Trigger t, Info* info)

src/FileAnalysisManager.h

@@ -3,6 +3,7 @@
 
 #include <string>
 #include <map>
+#include <vector>
 
 #include "Conn.h"
 #include "Analyzer.h"
@@ -12,6 +13,42 @@
 
 namespace file_analysis {
 
+class Info;
+
+/**
+ * Base class for actions that can be attached to a file_analysis::Info object.
+ */
+class Action {
+public:
+
+	Action(Info* arg_info);
+
+	~Action() {}
+
+	virtual void DeliverStream(const u_char* data, uint64 len) = 0;
+
+protected:
+
+	Info* info;
+};
+
+/**
+ * An action to simply extract files to disk.
+ */
+class Extract : Action {
+public:
+
+	Extract(Info* arg_info, const string& arg_filename);
+
+	~Extract() {}
+
+	virtual void DeliverStream(const u_char* data, uint64 len);
+
+protected:
+
+	string filename;
+};
+
 /**
  * Wrapper class around \c FileAnalysis::Info record values from script layer.
  */
@@ -41,11 +78,22 @@ public:
 	void UpdateLastActivityTime() { last_activity_time = network_time; }
 
 	/**
-	 * Set "total_bytes" field of #val record to \a size, check if "seen_bytes"
+	 * Increments the "seen_bytes" field of #val record by \a size.
-	 * is greater or equal to it, and evaluate \c FileAnalysis::policy if so.
+	 */
+	void IncrementSeenBytes(uint64 size);
+
+	/**
+	 * Set "total_bytes" field of #val record to \a size.
 	 */
 	void SetTotalBytes(uint64 size);
 
+	/**
+	 * Compares "seen_bytes" field to "total_bytes" field of #val record
+	 * and returns true if the comparison indicates the full file was seen.
+	 * If "total_bytes" hasn't been set yet, it returns false.
+	 */
+	bool IsComplete() const;
+
 	/**
 	 * Create a timer to be dispatched after the amount of time indicated by
 	 * the "timeout_interval" field of the #val record in order to check if
@@ -69,10 +117,24 @@ protected:
 	 */
 	void UpdateConnectionFields(Connection* conn);
 
+	/**
+	 * Wrapper to RecordVal::LookupWithDefault for the field in #val at index
+	 * \a idx which automatically unrefs the Val and returns a converted value.
+	 */
+	uint64 FieldDefaultCount(int idx) const;
+
+	/**
+	 * Wrapper to RecordVal::LookupWithDefault for the field in #val at index
+	 * \a idx which automatically unrefs the Val and returns a converted value.
+	 */
+	double FieldDefaultInterval(int idx) const;
+
 	RecordVal* val;            /**< \c FileAnalysis::Info from script layer. */
 	double last_activity_time; /**< Time of last activity. */
 	bool postpone_timeout;     /**< Whether postponing timeout is requested. */
 
+	vector<Analyzer*> analyzers;
+
 	/**
 	 * @return the field offset in #val record corresponding to \a field_name.
 	 */
@@ -142,7 +204,7 @@ public:
 	/**
 	 * Signal the end of file data.
 	 */
-	void EndOfData(const string& file_id, Connection* conn = 0,
+	void EndOfFile(const string& file_id, Connection* conn = 0,
 	               const string& protocol = "");
 
 	/**

src/FileAnalyzer.cc

@@ -1,5 +1,6 @@
 #include <algorithm>
 
+#include "FileAnalysisManager.h"
 #include "FileAnalyzer.h"
 #include "Reporter.h"
 
@@ -16,12 +17,20 @@ File_Analyzer::File_Analyzer(Connection* conn)
 		InitMagic(&magic, MAGIC_NONE);
 		InitMagic(&magic_mime, MAGIC_MIME);
 		}
+
+	char op[256], rp[256];
+	modp_ulitoa10(ntohs(conn->OrigPort()), op);
+	modp_ulitoa10(ntohs(conn->RespPort()), rp);
+	file_id = "TCPFile " + conn->OrigAddr().AsString() + ":" + op + "->" +
+	          conn->RespAddr().AsString() + ":" + rp;
 	}
 
 void File_Analyzer::DeliverStream(int len, const u_char* data, bool orig)
 	{
 	TCP_ApplicationAnalyzer::DeliverStream(len, data, orig);
 
+	file_mgr->DataIn(file_id, data, len, Conn());
+
 	int n = min(len, BUFFER_SIZE - buffer_len);
 
 	if ( n )
@@ -39,6 +48,8 @@ void File_Analyzer::Done()
 	{
 	TCP_ApplicationAnalyzer::Done();
 
+	file_mgr->EndOfFile(file_id, Conn());
+
 	if ( buffer_len && buffer_len != BUFFER_SIZE )
 		Identify();
 	}

src/FileAnalyzer.h

@@ -5,6 +5,7 @@
 
 #include "TCP.h"
 
+#include <string>
 #include <magic.h>
 
 class File_Analyzer : public TCP_ApplicationAnalyzer {
@@ -33,6 +34,8 @@ protected:
 
 	static magic_t magic;
 	static magic_t magic_mime;
+
+	string file_id;
 };
 
 #endif