FileAnalysis: optimize get_file_handle event queueing.

When a file handle is needed and the last event in the queue is also
a get_file_handle event with the same arguments, instead of queueing
a new event, just remember to cache/re-use the resulting handle from
the previous event.  This depends on get_file_handle handlers not
changing global state that is also used to derive the file handle
string.

src/file_analysis/Info.cc

@@ -185,6 +185,13 @@ double Info::GetTimeoutInterval() const
 	return LookupFieldDefaultInterval(timeout_interval_idx);
 	}
 
+string Info::GetSource() const
+	{
+	Val* v = val->Lookup(source_idx);
+	if ( ! v ) return "";
+	return v->AsStringVal()->CheckString();
+	}
+
 RecordVal* Info::GetResults(RecordVal* args) const
 	{
 	TableVal* actions_table = val->Lookup(actions_idx)->AsTableVal();
@@ -243,18 +250,6 @@ bool Info::BufferBOF(const u_char* data, uint64 len)
 
 	uint64 desired_size = LookupFieldDefaultCount(bof_buffer_size_idx);
 
-	/* Leaving out this optimization (I think) for now to keep things simpler.
-	// If first chunk satisfies desired size, do everything now without copying.
-	if ( bof_buffer.chunks.empty() && len >= desired_size )
-		{
-		bof_buffer.full = bof_buffer.replayed = true;
-		val->Assign(bof_buffer_idx, new StringVal(new BroString(data, len, 0)));
-		file_mgr->EvaluatePolicy(TRIGGER_BOF_BUFFER, this);
-		// TODO: libmagic stuff
-		return false;
-		}
-	*/
-
 	bof_buffer.chunks.push_back(new BroString(data, len, 0));
 	bof_buffer.size += len;
 

src/file_analysis/Info.h

@@ -32,6 +32,11 @@ public:
 	 */
 	double GetTimeoutInterval() const;
 
+	/**
+	 * @return value of the "source" field from #val record.
+	 */
+	string GetSource() const;
+
 	/**
 	 * @return value of the "file_id" field from #val record.
 	 */

src/file_analysis/Manager.cc

@@ -34,11 +34,17 @@ void Manager::ReceiveHandle(const string& handle)
 	if ( pending.empty() )
 		reporter->InternalError("File analysis underflow");
 
-	PendingFile* pf = pending.front();
-	if ( ! handle.empty() )
-		pf->Finish(handle);
-	delete pf;
-	pending.pop();
+	int use_count = cache.front();
+	cache.pop();
+
+	for ( int i = 0; i < use_count; ++i )
+		{
+		PendingFile* pf = pending.front();
+		if ( ! handle.empty() )
+			pf->Finish(handle);
+		delete pf;
+		pending.pop();
+		}
 	}
 
 void Manager::EventDrainDone()
@@ -330,11 +336,50 @@ bool Manager::IsDisabled(AnalyzerTag::Tag tag)
 	return rval;
 	}
 
+static bool CheckArgEquality(AnalyzerTag::Tag tag, Connection* conn,
+                             bool is_orig, val_list* other_args)
+	{
+	if ( ! other_args ) return false;
+	if ( (*other_args)[0]->AsCount() != (bro_uint_t) tag ) return false;
+	if ( (*other_args)[2]->AsBool() != is_orig ) return false;
+
+	RecordVal* id = (*other_args)[1]->AsRecordVal()->Lookup(
+	        connection_type->FieldOffset("id"))->AsRecordVal();
+
+	PortVal* orig_p = id->Lookup(
+	        conn_id->FieldOffset("orig_p"))->AsPortVal();
+
+	if ( orig_p->Port() != ntohs(conn->OrigPort()) ) return false;
+	if ( orig_p->PortType() != conn->ConnTransport() ) return false;
+
+	PortVal* resp_p = id->Lookup(
+	        conn_id->FieldOffset("resp_p"))->AsPortVal();
+
+	if ( resp_p->Port() != ntohs(conn->RespPort()) ) return false;
+
+	if ( id->Lookup(conn_id->FieldOffset("orig_h"))->AsAddr() !=
+	     conn->OrigAddr() ) return false;
+
+	if ( id->Lookup(conn_id->FieldOffset("resp_h"))->AsAddr() !=
+	     conn->RespAddr() ) return false;
+
+	return true;
+	}
+
 bool Manager::QueueHandleEvent(AnalyzerTag::Tag tag, Connection* conn,
                                bool is_orig)
 	{
 	if ( ! get_file_handle ) return false;
 
+	if ( mgr.Tail() == get_file_handle &&
+	     CheckArgEquality(tag, conn, is_orig, mgr.TailArgs()) )
+		{
+		cache.front()++;
+		return true;
+		}
+
+	cache.push(1);
+
 	val_list* vl = new val_list();
 	vl->append(new Val(tag, TYPE_COUNT));
 	vl->append(conn->BuildConnVal());

src/file_analysis/Manager.h

@@ -130,6 +130,7 @@ protected:
 	typedef set<string> StrSet;
 	typedef map<FileID, Info*> IDMap;
 	typedef queue<PendingFile*> PendingQueue;
+	typedef queue<int> HandleCache;
 
 	/**
 	 * @return the Info object mapped to \a unique or a null pointer if analysis
@@ -164,22 +165,24 @@ protected:
 	 */
 	bool IsIgnored(const string& unique);
 
-	/**
-	 * @return whether file analysis is disabled for the given analyzer.
-	 */
-	static bool IsDisabled(AnalyzerTag::Tag tag);
-
 	/**
 	 * Queues \c get_file_handle event in order to retrieve unique file handle.
 	 * @return true if there is a handler for the event, else false.
 	 */
-	static bool QueueHandleEvent(AnalyzerTag::Tag tag, Connection* conn,
-	                             bool is_orig);
+	bool QueueHandleEvent(AnalyzerTag::Tag tag, Connection* conn,
+	                      bool is_orig);
+
+	/**
+	 * @return whether file analysis is disabled for the given analyzer.
+	 */
+	static bool IsDisabled(AnalyzerTag::Tag tag);
 
 	StrMap str_map; /**< Map unique strings to \c FileAnalysis::Info records. */
 	IDMap id_map;   /**< Map file IDs to \c FileAnalysis::Info records. */
 	StrSet ignored; /**< Ignored files.  Will be finally removed on EOF. */
 	PendingQueue pending; /**< Files awaiting a unique handle. */
+	HandleCache cache; /**< The number of times a received file handle can be
+	                        used to pop the #pending queue. */
 
 	static TableVal* disabled; /**< Table of disabled analyzers. */
 };