Merge remote-tracking branch 'origin/master' into topic/vladg/kerberos

Conflicts:
	testing/btest/Baseline/core.print-bpf-filters/output2
	testing/btest/Baseline/scripts.policy.misc.dump-events/smtp-events.log

src/Attr.cc

@@ -265,6 +265,14 @@ void Attributes::CheckAttr(Attr* a)
 				// Ok.
 				break;
 
+			Expr* e = a->AttrExpr();
+			if ( check_and_promote_expr(e, type) )
+				{
+				a->SetAttrExpr(e);
+				// Ok.
+				break;
+				}
+
 			a->AttrExpr()->Error("&default value has inconsistent type", type);
 			}
 
@@ -297,8 +305,11 @@ void Attributes::CheckAttr(Attr* a)
 
 				Expr* e = a->AttrExpr();
 				if ( check_and_promote_expr(e, ytype) )
+					{
+					a->SetAttrExpr(e);
 					// Ok.
 					break;
+					}
 
 				Error("&default value has inconsistent type 2");
 				}

src/Attr.h

@@ -45,6 +45,13 @@ public:
 	attr_tag Tag() const	{ return tag; }
 	Expr* AttrExpr() const	{ return expr; }
 
+	// Up to the caller to decide if previous expr can be unref'd since it may
+	// not always be safe; e.g. expressions (at time of writing) don't always
+	// keep careful track of referencing their operands, so doing something
+	// like SetAttrExpr(coerce(AttrExpr())) must not completely unref the
+	// previous expr as the new expr depends on it.
+	void SetAttrExpr(Expr* e) { expr = e; }
+
 	int RedundantAttrOkay() const
 		{ return tag == ATTR_REDEF || tag == ATTR_OPTIONAL; }
 

src/CMakeLists.txt

@@ -48,7 +48,7 @@ set(BISON_FLAGS "--debug")
 bison_target(BIFParser builtin-func.y
              ${CMAKE_CURRENT_BINARY_DIR}/bif_parse.cc
              HEADER ${CMAKE_CURRENT_BINARY_DIR}/bif_parse.h
-             VERBOSE ${CMAKE_CURRENT_BINARY_DIR}/bif_parse.output
+             #VERBOSE ${CMAKE_CURRENT_BINARY_DIR}/bif_parse.output
              COMPILE_FLAGS "${BISON_FLAGS}")
 flex_target(BIFScanner builtin-func.l ${CMAKE_CURRENT_BINARY_DIR}/bif_lex.cc)
 add_flex_bison_dependency(BIFScanner BIFParser)
@@ -57,7 +57,7 @@ add_flex_bison_dependency(BIFScanner BIFParser)
 bison_target(RuleParser rule-parse.y
              ${CMAKE_CURRENT_BINARY_DIR}/rup.cc
              HEADER ${CMAKE_CURRENT_BINARY_DIR}/rup.h
-             VERBOSE ${CMAKE_CURRENT_BINARY_DIR}/rule_parse.output
+             #VERBOSE ${CMAKE_CURRENT_BINARY_DIR}/rule_parse.output
              COMPILE_FLAGS "${BISON_FLAGS}")
 replace_yy_prefix_target(${CMAKE_CURRENT_BINARY_DIR}/rup.cc
                          ${CMAKE_CURRENT_BINARY_DIR}/rule-parse.cc
@@ -72,7 +72,7 @@ flex_target(RuleScanner rule-scan.l ${CMAKE_CURRENT_BINARY_DIR}/rule-scan.cc
 bison_target(REParser re-parse.y
              ${CMAKE_CURRENT_BINARY_DIR}/rep.cc
              HEADER ${CMAKE_CURRENT_BINARY_DIR}/re-parse.h
-             VERBOSE ${CMAKE_CURRENT_BINARY_DIR}/re_parse.output
+             #VERBOSE ${CMAKE_CURRENT_BINARY_DIR}/re_parse.output
              COMPILE_FLAGS "${BISON_FLAGS}")
 replace_yy_prefix_target(${CMAKE_CURRENT_BINARY_DIR}/rep.cc
                          ${CMAKE_CURRENT_BINARY_DIR}/re-parse.cc
@@ -85,7 +85,7 @@ add_flex_bison_dependency(REScanner REParser)
 bison_target(Parser parse.y
              ${CMAKE_CURRENT_BINARY_DIR}/p.cc
              HEADER ${CMAKE_CURRENT_BINARY_DIR}/broparse.h
-             VERBOSE ${CMAKE_CURRENT_BINARY_DIR}/parse.output
+             #VERBOSE ${CMAKE_CURRENT_BINARY_DIR}/parse.output
              COMPILE_FLAGS "${BISON_FLAGS}")
 replace_yy_prefix_target(${CMAKE_CURRENT_BINARY_DIR}/p.cc
                          ${CMAKE_CURRENT_BINARY_DIR}/parse.cc

src/Expr.cc

@@ -1438,7 +1438,7 @@ bool AddExpr::DoUnserialize(UnserialInfo* info)
 	}
 
 AddToExpr::AddToExpr(Expr* arg_op1, Expr* arg_op2)
-: BinaryExpr(EXPR_ADD_TO, arg_op1, arg_op2)
+: BinaryExpr(EXPR_ADD_TO, arg_op1->MakeLvalue(), arg_op2)
 	{
 	if ( IsError() )
 		return;
@@ -1562,7 +1562,7 @@ bool SubExpr::DoUnserialize(UnserialInfo* info)
 	}
 
 RemoveFromExpr::RemoveFromExpr(Expr* arg_op1, Expr* arg_op2)
-: BinaryExpr(EXPR_REMOVE_FROM, arg_op1, arg_op2)
+: BinaryExpr(EXPR_REMOVE_FROM, arg_op1->MakeLvalue(), arg_op2)
 	{
 	if ( IsError() )
 		return;

src/Frag.cc

@@ -28,7 +28,7 @@ void FragTimer::Dispatch(double t, int /* is_expire */)
 FragReassembler::FragReassembler(NetSessions* arg_s,
 			const IP_Hdr* ip, const u_char* pkt,
 			HashKey* k, double t)
-	: Reassembler(0, REASSEM_IP)
+	: Reassembler(0)
 	{
 	s = arg_s;
 	key = k;

src/IP.cc

@@ -636,3 +636,51 @@ VectorVal* IPv6_Hdr_Chain::BuildVal() const
 
 	return rval;
 	}
+
+IP_Hdr* IP_Hdr::Copy() const
+	{
+	char* new_hdr = new char[HdrLen()];
+
+	if ( ip4 )
+		{
+		memcpy(new_hdr, ip4, HdrLen());
+		return new IP_Hdr((const struct ip*) new_hdr, true);
+		}
+
+	memcpy(new_hdr, ip6, HdrLen());
+	const struct ip6_hdr* new_ip6 = (const struct ip6_hdr*)new_hdr;
+	IPv6_Hdr_Chain* new_ip6_hdrs = ip6_hdrs->Copy(new_ip6);
+	return new IP_Hdr(new_ip6, true, 0, new_ip6_hdrs);
+	}
+
+IPv6_Hdr_Chain* IPv6_Hdr_Chain::Copy(const ip6_hdr* new_hdr) const
+	{
+	IPv6_Hdr_Chain* rval = new IPv6_Hdr_Chain;
+	rval->length = length;
+
+#ifdef ENABLE_MOBILE_IPV6
+	if ( homeAddr )
+		rval->homeAddr = new IPAddr(*homeAddr);
+#endif
+
+	if ( finalDst )
+		rval->finalDst = new IPAddr(*finalDst);
+
+	if ( chain.empty() )
+		{
+		reporter->InternalWarning("empty IPv6 header chain");
+		delete rval;
+		return 0;
+		}
+
+	const u_char* new_data = (const u_char*)new_hdr;
+	const u_char* old_data = chain[0]->Data();
+
+	for ( size_t i = 0; i < chain.size(); ++i )
+		{
+		int off = chain[i]->Data() - old_data;
+		rval->chain.push_back(new IPv6_Hdr(chain[i]->Type(), new_data + off));
+		}
+
+	return rval;
+	}

src/IP.h

@@ -157,6 +157,12 @@ public:
 		delete finalDst;
 		}
 
+	/**
+	 * @return a copy of the header chain, but with pointers to individual
+	 * IPv6 headers now pointing within \a new_hdr.
+	 */
+	IPv6_Hdr_Chain* Copy(const struct ip6_hdr* new_hdr) const;
+
 	/**
 	 * Returns the number of headers in the chain.
 	 */
@@ -264,6 +270,14 @@ protected:
 	// point to a fragment
 	friend class FragReassembler;
 
+	IPv6_Hdr_Chain() :
+		length(0),
+#ifdef ENABLE_MOBILE_IPV6
+		homeAddr(0),
+#endif
+		finalDst(0)
+		{}
+
 	/**
 	 * Initializes the header chain from an IPv6 header structure, and replaces
 	 * the first next protocol pointer field that points to a fragment header.
@@ -353,6 +367,13 @@ public:
 		{
 		}
 
+	/**
+	 * Copy a header.  The internal buffer which contains the header data
+	 * must not be truncated.  Also note that if that buffer points to a full
+	 * packet payload, only the IP header portion is copied.
+	 */
+	IP_Hdr* Copy() const;
+
 	/**
 	 * Destructor.
 	 */
@@ -554,6 +575,7 @@ public:
 	RecordVal* BuildPktHdrVal() const;
 
 private:
+
 	const struct ip* ip4;
 	const struct ip6_hdr* ip6;
 	bool del;

src/Reassem.cc

@@ -31,7 +31,7 @@ DataBlock::DataBlock(const u_char* data, uint64 size, uint64 arg_seq,
 
 uint64 Reassembler::total_size = 0;
 
-Reassembler::Reassembler(uint64 init_seq, ReassemblerType arg_type)
+Reassembler::Reassembler(uint64 init_seq)
 	{
 	blocks = last_block = 0;
 	trim_seq = last_reassem_seq = init_seq;

src/Reassem.h

@@ -22,11 +22,10 @@ public:
 };
 
 
-enum ReassemblerType { REASSEM_IP, REASSEM_TCP };
 
 class Reassembler : public BroObj {
 public:
-	Reassembler(uint64 init_seq, ReassemblerType arg_type);
+	Reassembler(uint64 init_seq);
 	virtual ~Reassembler();
 
 	void NewBlock(double t, uint64 seq, uint64 len, const u_char* data);

src/SerialTypes.h

@@ -87,6 +87,7 @@ SERIAL_TCP_CONTENTS(TCP_NVT, 3)
 #define SERIAL_REASSEMBLER(name, val) SERIAL_CONST(name, val, REASSEMBLER)
 SERIAL_REASSEMBLER(REASSEMBLER, 1)
 SERIAL_REASSEMBLER(TCP_REASSEMBLER, 2)
+SERIAL_REASSEMBLER(FILE_REASSEMBLER, 3)
 
 #define SERIAL_VAL(name, val) SERIAL_CONST(name, val, VAL)
 SERIAL_VAL(VAL, 1)

src/analyzer/protocol/CMakeLists.txt

@@ -22,6 +22,7 @@ add_subdirectory(krb)
 add_subdirectory(login)
 add_subdirectory(mime)
 add_subdirectory(modbus)
+add_subdirectory(mysql)
 add_subdirectory(ncp)
 add_subdirectory(netbios)
 add_subdirectory(netflow)

src/analyzer/protocol/mysql/CMakeLists.txt

@@ -0,0 +1,10 @@
+
+include(BroPlugin)
+
+include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
+
+bro_plugin_begin(Bro MySQL)
+	bro_plugin_cc(MySQL.cc Plugin.cc)
+	bro_plugin_bif(events.bif)
+	bro_plugin_pac(mysql.pac mysql-analyzer.pac mysql-protocol.pac)
+bro_plugin_end()

src/analyzer/protocol/mysql/MySQL.cc

@@ -0,0 +1,65 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "MySQL.h"
+#include "analyzer/protocol/tcp/TCP_Reassembler.h"
+#include "Reporter.h"
+#include "events.bif.h"
+
+using namespace analyzer::MySQL;
+
+MySQL_Analyzer::MySQL_Analyzer(Connection* c)
+	: tcp::TCP_ApplicationAnalyzer("MySQL", c)
+	{
+	interp = new binpac::MySQL::MySQL_Conn(this);
+	had_gap = false;
+	}
+
+MySQL_Analyzer::~MySQL_Analyzer()
+	{
+	delete interp;
+	}
+
+void MySQL_Analyzer::Done()
+	{
+	tcp::TCP_ApplicationAnalyzer::Done();
+
+	interp->FlowEOF(true);
+	interp->FlowEOF(false);
+	}
+
+void MySQL_Analyzer::EndpointEOF(bool is_orig)
+	{
+	tcp::TCP_ApplicationAnalyzer::EndpointEOF(is_orig);
+	interp->FlowEOF(is_orig);
+	}
+
+void MySQL_Analyzer::DeliverStream(int len, const u_char* data, bool orig)
+	{
+	tcp::TCP_ApplicationAnalyzer::DeliverStream(len, data, orig);
+
+	assert(TCP());
+	if ( TCP()->IsPartial() )
+		return;
+
+	if ( had_gap )
+		// If only one side had a content gap, we could still try to
+		// deliver data to the other side if the script layer can
+		// handle this.
+		return;
+
+	try
+		{
+		interp->NewData(orig, data, data + len);
+		}
+	catch ( const binpac::Exception& e )
+		{
+		reporter->Weird(e.msg().c_str());
+		}
+	}
+
+void MySQL_Analyzer::Undelivered(uint64 seq, int len, bool orig)
+	{
+	tcp::TCP_ApplicationAnalyzer::Undelivered(seq, len, orig);
+	had_gap = true;
+	interp->NewGap(orig, len);
+	}

src/analyzer/protocol/mysql/MySQL.h

@@ -0,0 +1,40 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef ANALYZER_PROTOCOL_MYSQL_MYSQL_H
+#define ANALYZER_PROTOCOL_MYSQL_MYSQL_H
+
+#include "events.bif.h"
+#include "analyzer/protocol/tcp/TCP.h"
+
+#include "mysql_pac.h"
+
+namespace analyzer { namespace MySQL {
+
+class MySQL_Analyzer
+
+: public tcp::TCP_ApplicationAnalyzer {
+
+public:
+	MySQL_Analyzer(Connection* conn);
+	virtual ~MySQL_Analyzer();
+
+	// Overriden from Analyzer.
+	virtual void Done();
+
+	virtual void DeliverStream(int len, const u_char* data, bool orig);
+	virtual void Undelivered(uint64 seq, int len, bool orig);
+
+	// Overriden from tcp::TCP_ApplicationAnalyzer.
+	virtual void EndpointEOF(bool is_orig);
+
+	static analyzer::Analyzer* Instantiate(Connection* conn)
+		{ return new MySQL_Analyzer(conn); }
+
+protected:
+	binpac::MySQL::MySQL_Conn* interp;
+	bool had_gap;
+};
+
+} } // namespace analyzer::*
+
+#endif

src/analyzer/protocol/mysql/Plugin.cc

@@ -0,0 +1,21 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "plugin/Plugin.h"
+
+#include "MySQL.h"
+
+namespace plugin {
+	namespace Bro_MySQL {
+		class Plugin : public plugin::Plugin {
+		public:
+			plugin::Configuration Configure()
+				{
+				AddComponent(new ::analyzer::Component("MySQL", ::analyzer::MySQL::MySQL_Analyzer::Instantiate));
+				plugin::Configuration config;
+				config.name = "Bro::MySQL";
+				config.description = "MySQL analyzer";
+				return config;
+				}
+		} plugin;
+	}
+}

src/analyzer/protocol/mysql/events.bif

@@ -0,0 +1,65 @@
+## Generated for a command request from a MySQL client.
+##
+## See the MySQL `documentation <http://dev.mysql.com/doc/internals/en/client-server-protocol.html>`__
+## for more information about the MySQL protocol.
+##
+## c: The connection.
+##
+## command: The numerical code of the command issued.
+##
+## arg: The argument for the command (empty string if not provided).
+##
+## .. bro:see:: mysql_error mysql_ok mysql_server_version mysql_handshake_response
+event mysql_command_request%(c: connection, command: count, arg: string%);
+
+## Generated for an unsuccessful MySQL response.
+##
+## See the MySQL `documentation <http://dev.mysql.com/doc/internals/en/client-server-protocol.html>`__
+## for more information about the MySQL protocol.
+##
+## c: The connection.
+##
+## code: The error code.
+##
+## msg: Any extra details about the error (empty string if not provided).
+##
+## .. bro:see:: mysql_command_request mysql_ok mysql_server_version mysql_handshake_response
+event mysql_error%(c: connection, code: count, msg: string%);
+
+## Generated for a successful MySQL response.
+##
+## See the MySQL `documentation <http://dev.mysql.com/doc/internals/en/client-server-protocol.html>`__
+## for more information about the MySQL protocol.
+##
+## c: The connection.
+##
+## affected_rows: The number of rows that were affected.
+##
+## .. bro:see:: mysql_command_request mysql_error mysql_server_version mysql_handshake_response
+event mysql_ok%(c: connection, affected_rows: count%);
+
+## Generated for the initial server handshake packet, which includes the MySQL server version.
+##
+## See the MySQL `documentation <http://dev.mysql.com/doc/internals/en/client-server-protocol.html>`__
+## for more information about the MySQL protocol.
+##
+## c: The connection.
+##
+## ver: The server version string.
+##
+## .. bro:see:: mysql_command_request mysql_error mysql_ok mysql_handshake_response
+event mysql_server_version%(c: connection, ver: string%);
+
+## Generated for a client handshake response packet, which includes the username the client is attempting
+## to connect as.
+##
+## See the MySQL `documentation <http://dev.mysql.com/doc/internals/en/client-server-protocol.html>`__
+## for more information about the MySQL protocol.
+##
+## c: The connection.
+##
+## username: The username supplied by the client
+##
+## .. bro:see:: mysql_command_request mysql_error mysql_ok mysql_server_version
+event mysql_handshake%(c: connection, username: string%);
+

src/analyzer/protocol/mysql/mysql-analyzer.pac

@@ -0,0 +1,98 @@
+# See the file "COPYING" in the main distribution directory for copyright.
+
+refine flow MySQL_Flow += {
+	function proc_mysql_initial_handshake_packet(msg: Initial_Handshake_Packet): bool
+		%{
+		if ( mysql_server_version )
+			{
+			if ( ${msg.version} == 10 )
+				BifEvent::generate_mysql_server_version(connection()->bro_analyzer(),
+														connection()->bro_analyzer()->Conn(),
+														bytestring_to_val(${msg.handshake10.server_version}));
+			if ( ${msg.version} == 9 )
+				BifEvent::generate_mysql_server_version(connection()->bro_analyzer(),
+														connection()->bro_analyzer()->Conn(),
+														bytestring_to_val(${msg.handshake9.server_version}));
+			}
+		return true;
+		%}
+
+	function proc_mysql_handshake_response_packet(msg: Handshake_Response_Packet): bool
+		%{
+		if ( mysql_handshake )
+			{
+			if ( ${msg.version} == 10 )
+				BifEvent::generate_mysql_handshake(connection()->bro_analyzer(),
+									    		   connection()->bro_analyzer()->Conn(),
+													bytestring_to_val(${msg.v10_response.username}));
+			if ( ${msg.version} == 9 )
+				BifEvent::generate_mysql_handshake(connection()->bro_analyzer(),
+									    		   connection()->bro_analyzer()->Conn(),
+								    	    	   bytestring_to_val(${msg.v9_response.username}));
+			}
+		return true;
+		%}
+
+	function proc_mysql_command_request_packet(msg: Command_Request_Packet): bool
+		%{
+		if ( mysql_command_request )
+			BifEvent::generate_mysql_command_request(connection()->bro_analyzer(),
+													 connection()->bro_analyzer()->Conn(),
+													 ${msg.command},
+													 bytestring_to_val(${msg.arg}));
+		return true;
+		%}
+
+	function proc_err_packet(msg: ERR_Packet): bool
+		%{
+		if ( mysql_error )
+			BifEvent::generate_mysql_error(connection()->bro_analyzer(),
+										   connection()->bro_analyzer()->Conn(),
+										   ${msg.code},
+										   bytestring_to_val(${msg.msg}));
+		return true;
+		%}
+
+	function proc_ok_packet(msg: OK_Packet): bool
+		%{
+		if ( mysql_ok )
+			BifEvent::generate_mysql_ok(connection()->bro_analyzer(),
+										connection()->bro_analyzer()->Conn(),
+										${msg.rows});
+		return true;
+		%}
+
+	function proc_resultset(msg: Resultset): bool
+		%{
+		if ( mysql_ok )
+			BifEvent::generate_mysql_ok(connection()->bro_analyzer(),
+										connection()->bro_analyzer()->Conn(),
+										${msg.rows}->size());
+		return true;
+		%}
+
+};
+
+refine typeattr Initial_Handshake_Packet += &let {
+	proc = $context.flow.proc_mysql_initial_handshake_packet(this);
+};
+
+refine typeattr Handshake_Response_Packet += &let {
+	proc = $context.flow.proc_mysql_handshake_response_packet(this);
+};
+
+refine typeattr Command_Request_Packet += &let {
+	proc = $context.flow.proc_mysql_command_request_packet(this);
+};
+
+refine typeattr ERR_Packet += &let {
+	proc = $context.flow.proc_err_packet(this);
+};
+
+refine typeattr OK_Packet += &let {
+	proc = $context.flow.proc_ok_packet(this);
+};
+
+refine typeattr Resultset += &let {
+	proc = $context.flow.proc_resultset(this);
+};

src/analyzer/protocol/mysql/mysql-protocol.pac

@@ -0,0 +1,407 @@
+# See the file "COPYING" in the main distribution directory for copyright.
+#
+# All information is from the MySQL internals documentation at:
+# <http://dev.mysql.com/doc/internals/en/client-server-protocol.html>
+#
+
+# Basic Types
+
+type uint24le = record {
+	byte3 : uint8;
+	byte2 : uint8;
+	byte1 : uint8;
+};
+
+type LengthEncodedInteger = record {
+	length : uint8;
+	integer : LengthEncodedIntegerLookahead(length);
+};
+
+type LengthEncodedIntegerLookahead(length: uint8) = record {
+	val: case length of {
+		0xfb    -> i0        : empty;
+		0xfc    -> i2        : uint16;
+		0xfd    -> i3        : uint24le;
+		0xfe    -> i4        : uint64;
+		0xff    -> err_packet: empty;
+		default -> one       : empty;
+	};
+};
+
+type LengthEncodedString = record {
+	len: LengthEncodedInteger;
+	val: bytestring &length=to_int()(len);
+};
+
+%header{
+	class to_int
+		{
+		public:
+		int operator()(uint24le* num) const
+			{
+			return (num->byte1() << 16) | (num->byte2() << 8) | num->byte3();
+			}
+
+		int operator()(LengthEncodedInteger* lei) const
+			{
+			if ( lei->length() < 0xfb )
+				return lei->length();
+			else if ( lei->length() == 0xfc )
+				return lei->integer()->i2();
+			else if ( lei->length() == 0xfd )
+				return to_int()(lei->integer()->i3());
+			else if ( lei->length() == 0xfe )
+				return lei->integer()->i4();
+			else
+				return 0;
+			}
+
+		int operator()(LengthEncodedIntegerLookahead* lei) const
+			{
+			if ( lei->length() < 0xfb )
+				return lei->length();
+			else if ( lei->length() == 0xfc )
+				return lei->i2();
+			else if ( lei->length() == 0xfd )
+				return to_int()(lei->i3());
+			else if ( lei->length() == 0xfe )
+				return lei->i4();
+			else
+				return 0;
+			}
+		};
+%}
+
+extern type to_int;
+
+# Enums
+
+enum command_consts {
+	COM_SLEEP               = 0x00,
+	COM_QUIT                = 0x01,
+	COM_INIT_DB             = 0x02,
+	COM_QUERY               = 0x03,
+	COM_FIELD_LIST          = 0x04,
+	COM_CREATE_DB           = 0x05,
+	COM_DROP_DB             = 0x06,
+	COM_REFRESH             = 0x07,
+	COM_SHUTDOWN            = 0x08,
+	COM_STATISTICS          = 0x09,
+	COM_PROCESS_INFO        = 0x0a,
+	COM_CONNECT             = 0x0b,
+	COM_PROCESS_KILL        = 0x0c,
+	COM_DEBUG               = 0x0d,
+	COM_PING                = 0x0e,
+	COM_TIME                = 0x0f,
+	COM_DELAYED_INSERT      = 0x10,
+	COM_CHANGE_USER         = 0x11,
+	COM_BINLOG_DUMP         = 0x12,
+	COM_TABLE_DUMP          = 0x13,
+	COM_CONNECT_OUT         = 0x14,
+	COM_REGISTER_SLAVE      = 0x15,
+	COM_STMT_PREPARE        = 0x16,
+	COM_STMT_EXECUTE        = 0x17,
+	COM_STMT_SEND_LONG_DATA = 0x18,
+	COM_STMT_CLOSE          = 0x19,
+	COM_STMT_RESET          = 0x1a,
+	COM_SET_OPTION          = 0x1b,
+	COM_STMT_FETCH          = 0x1c,
+	COM_DAEMON              = 0x1d,
+	COM_BINLOG_DUMP_GTID    = 0x1e
+};
+
+enum state {
+	CONNECTION_PHASE = 0,
+	COMMAND_PHASE    = 1,
+};
+
+enum Expected {
+	NO_EXPECTATION,
+	EXPECT_STATUS,
+	EXPECT_COLUMN_DEFINITION,
+	EXPECT_COLUMN_COUNT,
+	EXPECT_EOF1,
+	EXPECT_EOF2,
+	EXPECT_RESULTSET,
+	EXPECT_QUERY_RESPONSE,
+};
+
+type NUL_String = RE/[^\0]*/;
+
+# MySQL PDU
+
+type MySQL_PDU(is_orig: bool) = record {
+	hdr		: Header;
+	msg		: case is_orig of {
+		false -> server_msg: Server_Message(hdr.seq_id);
+		true  -> client_msg: Client_Message(state);
+	} &requires(state);
+} &let {
+	state	: int = $context.connection.get_state();
+} &length=hdr.len &byteorder=bigendian;
+
+type Header = record {
+	le_len: uint24le;
+	seq_id: uint8;
+} &let {
+	len   : uint32 = to_int()(le_len) + 4;
+} &length=4;
+
+type Server_Message(seq_id: uint8) = case seq_id of {
+	0       -> initial_handshake: Initial_Handshake_Packet;
+	default -> command_response : Command_Response;
+};
+
+type Client_Message(state: int) = case state of {
+	CONNECTION_PHASE -> connection_phase: Handshake_Response_Packet;
+	COMMAND_PHASE    -> command_phase   : Command_Request_Packet;
+};
+
+# Handshake Request
+
+type Initial_Handshake_Packet = record {
+	version		: uint8;
+	pkt		: case version of {
+		10      -> handshake10 : Handshake_v10;
+		9       -> handshake9  : Handshake_v9;
+		default -> error       : ERR_Packet;
+	};
+} &let {
+	set_version		: bool = $context.connection.set_version(version);
+};
+
+type Handshake_v10 = record {
+	server_version          : NUL_String;
+	connection_id           : uint32;
+	auth_plugin_data_part_1 : bytestring &length=8;
+	filler_1                : uint8;
+	capability_flag_1       : uint16;
+	character_set           : uint8;
+	status_flags            : uint16;
+	capability_flags_2      : uint16;
+	auth_plugin_data_len    : uint8;
+	auth_plugin_name        : NUL_String;
+};
+
+type Handshake_v9 = record {
+	server_version		: NUL_String;
+	connection_id		: uint32;
+	scramble			: NUL_String;
+};
+
+# Handshake Response
+
+type Handshake_Response_Packet = case $context.connection.get_version() of {
+	10	-> v10_response	: Handshake_Response_Packet_v10;
+	9	-> v9_response	: Handshake_Response_Packet_v9;
+} &let {
+	version 			: uint8 = $context.connection.get_version();
+} &byteorder=bigendian;
+
+type Handshake_Response_Packet_v10 = record {
+	cap_flags    : uint32;
+	max_pkt_size : uint32;
+	char_set     : uint8;
+	pad          : padding[23];
+	username     : NUL_String;
+	password     : bytestring &restofdata;
+};
+
+type Handshake_Response_Packet_v9 = record {
+	cap_flags		: uint16;
+	max_pkt_size 	: uint24le;
+	username     	: NUL_String;
+	auth_response	: NUL_String;
+	have_db			: case ( cap_flags & 0x8 ) of {
+		0x8	-> database : NUL_String;
+		0x0	-> none	    : empty;
+	};
+	password     	: bytestring &restofdata;
+};
+
+# Command Request
+
+type Command_Request_Packet = record {
+	command			: uint8;
+	arg    			: bytestring &restofdata;
+} &let {
+	update_expectation	: bool = $context.connection.set_next_expected(EXPECT_COLUMN_COUNT);
+};
+
+# Command Response
+
+type Command_Response = case $context.connection.get_expectation() of {
+	EXPECT_COLUMN_COUNT		-> col_count_meta	: ColumnCountMeta;
+	EXPECT_COLUMN_DEFINITION	-> col_defs		: ColumnDefinitions;
+	EXPECT_RESULTSET		-> resultset		: Resultset;
+	EXPECT_STATUS			-> status		: Command_Response_Status;
+	EXPECT_EOF1			-> eof1			: EOF1;
+	EXPECT_EOF2			-> eof2			: EOF2;
+	default				-> unknow		: empty;
+};
+
+type Command_Response_Status = record {
+	pkt_type: uint8;
+	response: case pkt_type of {
+		0x00    -> data_ok:   OK_Packet;
+		0xfe    -> data_eof:  EOF_Packet;
+		0xff    -> data_err:  ERR_Packet;
+		default -> unknown:   empty;
+	};
+};
+
+type ColumnCountMeta = record {
+	byte	: uint8;
+	pkt_type: case byte of {
+		0x00    -> ok		: OK_Packet;
+		0xff    -> err		: ERR_Packet;
+#		0xfb    -> Not implemented
+		default -> col_count: ColumnCount(byte);
+	};
+};
+
+type ColumnCount(byte: uint8) = record {
+	le_column_count		: LengthEncodedIntegerLookahead(byte);
+} &let {
+	col_num			: uint32 = to_int()(le_column_count);
+	update_col_num		: bool = $context.connection.set_col_count(col_num);
+	update_expectation	: bool = $context.connection.set_next_expected(EXPECT_COLUMN_DEFINITION);
+};
+
+type ColumnDefinitions = record {
+	defs				: ColumnDefinition41[1];
+} &let {
+	update_expectation	: bool = $context.connection.set_next_expected(EXPECT_EOF1);
+};
+
+type EOF1 = record {
+	eof					: EOF_Packet;
+} &let {
+	update_expectation	: bool = $context.connection.set_next_expected(EXPECT_RESULTSET);
+};
+
+type EOF2 = record {
+	eof					: EOF_Packet;
+} &let {
+	update_expectation	: bool = $context.connection.set_next_expected(NO_EXPECTATION);
+};
+
+type Resultset = record {
+	rows				: ResultsetRow[] &until($input.length()==0);
+} &let {
+	update_expectation	: bool = $context.connection.set_next_expected(EXPECT_EOF2);
+};
+
+type ResultsetRow = record {
+	fields: LengthEncodedString[$context.connection.get_col_count()];
+};
+
+type ColumnDefinition41 = record {
+	catalog  : LengthEncodedString;
+	schema   : LengthEncodedString;
+	table    : LengthEncodedString;
+	org_table: LengthEncodedString;
+	name     : LengthEncodedString;
+	org_name : LengthEncodedString;
+	next_len : LengthEncodedInteger;
+	char_set : uint16;
+	col_len  : uint32;
+	type     : uint8;
+	flags    : uint16;
+	decimals : uint8;
+	filler   : padding[2];
+};
+
+type ColumnDefinition320 = record {
+	table            : LengthEncodedString;
+	name             : LengthEncodedString;
+	length_of_col_len: LengthEncodedInteger;
+	col_len          : uint24le;
+	type_len         : LengthEncodedInteger;
+	type             : uint8;
+};
+
+type OK_Packet = record {
+	le_rows		: LengthEncodedInteger;
+	todo   		: bytestring &restofdata;
+} &let {
+	rows        : uint32 = to_int()(le_rows);
+	update_state: bool   = $context.connection.update_state(COMMAND_PHASE);
+};
+
+type ERR_Packet = record {
+	code : uint16;
+	state: bytestring &length=6;
+	msg  : bytestring &restofdata;
+} &let {
+	update_state: bool = $context.connection.update_state(COMMAND_PHASE);
+};
+
+type EOF_Packet = record {
+	warnings: uint16;
+	status  : uint16;
+} &let {
+	update_state: bool = $context.connection.update_state(COMMAND_PHASE);
+};
+
+# State tracking
+
+refine connection MySQL_Conn += {
+	%member{
+		uint8 version_;
+		int state_;
+		Expected expected_;
+		uint32 col_count_;
+	%}
+
+	%init{
+		version_ = 0;
+		state_ = CONNECTION_PHASE;
+		expected_ = EXPECT_STATUS;
+		col_count_ = 0;
+	%}
+
+	function get_version(): uint8
+		%{
+		return version_;
+		%}
+
+	function set_version(v: uint8): bool
+		%{
+		version_ = v;
+		return true;
+		%}
+
+	function get_state(): int
+		%{
+		return state_;
+		%}
+
+	function update_state(s: state): bool
+		%{
+		state_ = s;
+		return true;
+		%}
+
+	function get_expectation(): Expected
+		%{
+		return expected_;
+		%}
+
+	function set_next_expected(e: Expected): bool
+		%{
+		expected_ = e;
+		return true;
+		%}
+
+	function get_col_count(): uint32
+		%{
+		return col_count_;
+		%}
+
+	function set_col_count(i: uint32): bool
+		%{
+		col_count_ = i;
+		return true;
+		%}
+};

src/analyzer/protocol/mysql/mysql.pac

@@ -0,0 +1,37 @@
+# See the file "COPYING" in the main distribution directory for copyright.
+#
+# Analyzer for MySQL
+#  - mysql-protocol.pac: describes the MySQL protocol messages
+#  - mysql-analyzer.pac: describes the MySQL analyzer code
+
+%include binpac.pac
+%include bro.pac
+
+%extern{
+	#include "events.bif.h"
+%}
+
+analyzer MySQL withcontext {
+	connection: MySQL_Conn;
+	flow:       MySQL_Flow;
+};
+
+# Our connection consists of two flows, one in each direction.
+connection MySQL_Conn(bro_analyzer: BroAnalyzer) {
+	upflow   = MySQL_Flow(true);
+	downflow = MySQL_Flow(false);
+};
+
+%include mysql-protocol.pac
+
+# Now we define the flow:
+flow MySQL_Flow(is_orig: bool) {
+	# There are two options here: flowunit or datagram.
+	# flowunit = MySQL_PDU(is_orig) withcontext(connection, this);
+	flowunit = MySQL_PDU(is_orig) withcontext(connection, this);
+	# Using flowunit will cause the anlayzer to buffer incremental input.
+	# This is needed for &oneline and &length. If you don't need this, you'll
+	# get better performance with datagram.
+};
+
+%include mysql-analyzer.pac

src/analyzer/protocol/pia/PIA.cc

@@ -22,6 +22,7 @@ void PIA::ClearBuffer(Buffer* buffer)
 	for ( DataBlock* b = buffer->head; b; b = next )
 		{
 		next = b->next;
+		delete b->ip;
 		delete [] b->data;
 		delete b;
 		}
@@ -31,7 +32,7 @@ void PIA::ClearBuffer(Buffer* buffer)
 	}
 
 void PIA::AddToBuffer(Buffer* buffer, uint64 seq, int len, const u_char* data,
-			bool is_orig)
+			bool is_orig, const IP_Hdr* ip)
 	{
 	u_char* tmp = 0;
 
@@ -42,6 +43,7 @@ void PIA::AddToBuffer(Buffer* buffer, uint64 seq, int len, const u_char* data,
 		}
 
 	DataBlock* b = new DataBlock;
+	b->ip = ip ? ip->Copy() : 0;
 	b->data = tmp;
 	b->is_orig = is_orig;
 	b->len = len;
@@ -59,9 +61,10 @@ void PIA::AddToBuffer(Buffer* buffer, uint64 seq, int len, const u_char* data,
 	buffer->size += len;
 	}
 
-void PIA::AddToBuffer(Buffer* buffer, int len, const u_char* data, bool is_orig)
+void PIA::AddToBuffer(Buffer* buffer, int len, const u_char* data, bool is_orig,
+                      const IP_Hdr* ip)
 	{
-	AddToBuffer(buffer, -1, len, data, is_orig);
+	AddToBuffer(buffer, -1, len, data, is_orig, ip);
 	}
 
 void PIA::ReplayPacketBuffer(analyzer::Analyzer* analyzer)
@@ -69,7 +72,7 @@ void PIA::ReplayPacketBuffer(analyzer::Analyzer* analyzer)
 	DBG_LOG(DBG_ANALYZER, "PIA replaying %d total packet bytes", pkt_buffer.size);
 
 	for ( DataBlock* b = pkt_buffer.head; b; b = b->next )
-		analyzer->DeliverPacket(b->len, b->data, b->is_orig, -1, 0, 0);
+		analyzer->DeliverPacket(b->len, b->data, b->is_orig, -1, b->ip, 0);
 	}
 
 void PIA::PIA_Done()
@@ -96,7 +99,7 @@ void PIA::PIA_DeliverPacket(int len, const u_char* data, bool is_orig, uint64 se
 	if ( (pkt_buffer.state == BUFFERING || new_state == BUFFERING) &&
 	     len > 0 )
 		{
-		AddToBuffer(&pkt_buffer, seq, len, data, is_orig);
+		AddToBuffer(&pkt_buffer, seq, len, data, is_orig, ip);
 		if ( pkt_buffer.size > dpd_buffer_size )
 			new_state = dpd_match_only_beginning ?
 						SKIPPING : MATCHING_ONLY;

src/analyzer/protocol/pia/PIA.h

@@ -49,6 +49,7 @@ protected:
 	// Buffers one chunk of data.  Used both for packet payload (incl.
 	// sequence numbers for TCP) and chunks of a reassembled stream.
 	struct DataBlock {
+		IP_Hdr* ip;
 		const u_char* data;
 		bool is_orig;
 		int len;
@@ -66,9 +67,9 @@ protected:
 	};
 
 	void AddToBuffer(Buffer* buffer, uint64 seq, int len,
-				const u_char* data, bool is_orig);
+				const u_char* data, bool is_orig, const IP_Hdr* ip = 0);
 	void AddToBuffer(Buffer* buffer, int len,
-				const u_char* data, bool is_orig);
+				const u_char* data, bool is_orig, const IP_Hdr* ip = 0);
 	void ClearBuffer(Buffer* buffer);
 
 	DataBlock* CurrentPacket()	{ return &current_packet; }

src/analyzer/protocol/ssl/SSL.h

@@ -24,12 +24,6 @@ public:
 	static analyzer::Analyzer* Instantiate(Connection* conn)
 		{ return new SSL_Analyzer(conn); }
 
-	static bool Available()
-		{
-		return ( ssl_client_hello || ssl_server_hello ||
-			ssl_established || ssl_extension || ssl_alert );
-		}
-
 protected:
 	binpac::SSL::SSL_Conn* interp;
 	bool had_gap;

src/analyzer/protocol/ssl/ssl-analyzer.pac

@@ -112,7 +112,10 @@ refine connection SSL_Conn += {
 					cipher_suites24 : uint24[]) : bool
 		%{
 		if ( ! version_ok(version) )
+			{
 			bro_analyzer()->ProtocolViolation(fmt("unsupported client SSL version 0x%04x", version));
+			bro_analyzer()->SetSkip(true);
+			}
 		else
 			bro_analyzer()->ProtocolConfirmation();
 
@@ -152,7 +155,10 @@ refine connection SSL_Conn += {
 					comp_method : uint8) : bool
 		%{
 		if ( ! version_ok(version) )
+			{
 			bro_analyzer()->ProtocolViolation(fmt("unsupported server SSL version 0x%04x", version));
+			bro_analyzer()->SetSkip(true);
+			}
 
 		if ( ssl_server_hello )
 			{
@@ -202,6 +208,7 @@ refine connection SSL_Conn += {
 			// This should be impossible due to the binpac parser
 			// and protocol description
 			bro_analyzer()->ProtocolViolation(fmt("Impossible extension length: %lu", length));
+			bro_analyzer()->SetSkip(true);
 			return true;
 			}
 
@@ -392,7 +399,11 @@ refine connection SSL_Conn += {
 	function proc_check_v2_server_hello_version(version: uint16) : bool
 		%{
 		if ( version != SSLv20 )
+			{
 			bro_analyzer()->ProtocolViolation(fmt("Invalid version in SSL server hello. Version: %d", version));
+			bro_analyzer()->SetSkip(true);
+			return false;
+			}
 
 		return true;
 		%}
@@ -479,13 +490,13 @@ refine typeattr ServerHello += &let {
 };
 
 refine typeattr V2ServerHello += &let {
-	proc : bool = $context.connection.proc_server_hello(rec, server_version, 0,
-				conn_id_data, 0, 0, ciphers, 0);
-
 	check_v2 : bool = $context.connection.proc_check_v2_server_hello_version(server_version);
 
+	proc : bool = $context.connection.proc_server_hello(rec, server_version, 0,
+				conn_id_data, 0, 0, ciphers, 0) &requires(check_v2) &if(check_v2 == true);
+
 	cert : bool = $context.connection.proc_v2_certificate(rec, cert_data)
-		&requires(proc);
+		&requires(proc) &requires(check_v2) &if(check_v2 == true);
 };
 
 refine typeattr Certificate += &let {

src/analyzer/protocol/ssl/ssl-protocol.pac

@@ -36,7 +36,7 @@ type SSLRecord(is_orig: bool) = record {
 } &length = length+5, &byteorder=bigendian,
 	&let {
 	version : int =
-		$context.connection.determine_ssl_record_layer(head0, head1, head2, head3, head4);
+		$context.connection.determine_ssl_record_layer(head0, head1, head2, head3, head4, is_orig);
 
 	content_type : int = case version of {
 		SSLv20 -> head2+300;
@@ -748,7 +748,7 @@ refine connection SSL_Conn += {
 		%}
 
 	function determine_ssl_record_layer(head0 : uint8, head1 : uint8,
-					head2 : uint8, head3: uint8, head4: uint8) : int
+					head2 : uint8, head3: uint8, head4: uint8, is_orig: bool) : int
 		%{
 		// re-check record layer version to be sure that we still are synchronized with
 		// the data stream
@@ -759,6 +759,7 @@ refine connection SSL_Conn += {
 			     version != TLSv11 && version != TLSv12 )
 				{
 				bro_analyzer()->ProtocolViolation(fmt("Invalid version late in TLS connection. Packet reported version: %d", version));
+				bro_analyzer()->SetSkip(true);
 				return UNKNOWN_VERSION;
 				}
 			}
@@ -768,13 +769,14 @@ refine connection SSL_Conn += {
 
 		if ( head0 & 0x80 )
 			{
-			if ( head2 == 0x01 ) // SSLv2 client hello.
+			if ( head2 == 0x01 && is_orig ) // SSLv2 client hello.
 				{
 				uint16 version = (head3 << 8) | head4;
 				if ( version != SSLv20 && version != SSLv30 && version != TLSv10 &&
 				     version != TLSv11 && version != TLSv12 )
 					{
 					bro_analyzer()->ProtocolViolation(fmt("Invalid version in SSL client hello. Version: %d", version));
+					bro_analyzer()->SetSkip(true);
 					return UNKNOWN_VERSION;
 					}
 
@@ -782,7 +784,7 @@ refine connection SSL_Conn += {
 					return SSLv20;
 				}
 
-			else if ( head2 == 0x04 ) // SSLv2 server hello. This connection will continue using SSLv2.
+			else if ( head2 == 0x04 && head4 < 2 && ! is_orig ) // SSLv2 server hello. This connection will continue using SSLv2.
 				{
 				record_layer_version_ = SSLv20;
 				return SSLv20;
@@ -791,6 +793,7 @@ refine connection SSL_Conn += {
 			else // this is not SSL or TLS.
 				{
 				bro_analyzer()->ProtocolViolation(fmt("Invalid headers in SSL connection. Head1: %d, head2: %d, head3: %d", head1, head2, head3));
+				bro_analyzer()->SetSkip(true);
 				return UNKNOWN_VERSION;
 				}
 			}
@@ -800,6 +803,7 @@ refine connection SSL_Conn += {
 		     version != TLSv11 && version != TLSv12 )
 			{
 			bro_analyzer()->ProtocolViolation(fmt("Invalid version in TLS connection. Version: %d", version));
+			bro_analyzer()->SetSkip(true);
 			return UNKNOWN_VERSION;
 			}
 
@@ -810,6 +814,7 @@ refine connection SSL_Conn += {
 			}
 
 		bro_analyzer()->ProtocolViolation(fmt("Invalid type in TLS connection. Version: %d, Type: %d", version, head0));
+		bro_analyzer()->SetSkip(true);
 		return UNKNOWN_VERSION;
 		%}
 

src/analyzer/protocol/tcp/TCP_Reassembler.cc

@@ -28,7 +28,7 @@ TCP_Reassembler::TCP_Reassembler(analyzer::Analyzer* arg_dst_analyzer,
 				TCP_Analyzer* arg_tcp_analyzer,
 				TCP_Reassembler::Type arg_type,
 				TCP_Endpoint* arg_endp)
-	: Reassembler(1, REASSEM_TCP)
+	: Reassembler(1)
 	{
 	dst_analyzer = arg_dst_analyzer;
 	tcp_analyzer = arg_tcp_analyzer;

src/analyzer/protocol/tcp/TCP_Reassembler.h

@@ -11,9 +11,6 @@ namespace analyzer { namespace tcp {
 
 class TCP_Analyzer;
 
-const int STOP_ON_GAP = 1;
-const int PUNT_ON_PARTIAL = 1;
-
 class TCP_Reassembler : public Reassembler {
 public:
 	enum Type {

src/analyzer/protocol/tcp/events.bif

@@ -29,8 +29,10 @@ event new_connection_contents%(c: connection%);
 ##    new_connection new_connection_contents partial_connection
 event connection_attempt%(c: connection%);
 
-## Generated when a SYN-ACK packet is seen in response to a SYN packet during
-## a TCP handshake.  The final ACK of the handshake in response to SYN-ACK may
+## Generated when seeing a SYN-ACK packet from the responder in a TCP
+## handshake.  An associated SYN packet was not seen from the originator
+## side if its state is not set to :bro:see:`TCP_ESTABLISHED`.
+## The final ACK of the handshake in response to SYN-ACK may
 ## or may not occur later, one way to tell is to check the *history* field of
 ## :bro:type:`connection` to see if the originator sent an ACK, indicated by
 ## 'A' in the history string.

src/bro.bif

@@ -2,8 +2,8 @@
 ##! such as general programming algorithms, string processing, math functions,
 ##! introspection, type conversion, file/directory manipulation, packet
 ##! filtering, interprocess communication and controlling protocol analyzer
-##! behavior. 
-##! 
+##! behavior.
+##!
 ##! You'll find most of Bro's built-in functions that aren't protocol-specific
 ##! in this file.
 
@@ -2159,6 +2159,22 @@ function counts_to_addr%(v: index_vec%): addr
 		}
 	%}
 
+## Converts an :bro:type:`enum` to an :bro:type:`int`.
+##
+## e: The :bro:type:`enum` to convert.
+##
+## Returns: The :bro:type:`int` value that corresponds to the :bro:type:`enum`.
+function enum_to_int%(e: any%): int
+	%{
+	if ( e->Type()->Tag() != TYPE_ENUM )
+		{
+		builtin_error("enum_to_int() requires enum value");
+		return new Val(-1, TYPE_INT);
+		}
+
+	return new Val(e->AsEnum(), TYPE_INT);
+	%}
+
 ## Converts a :bro:type:`string` to an :bro:type:`int`.
 ##
 ## str: The :bro:type:`string` to convert.

src/builtin-func.l

@@ -1,4 +1,5 @@
 %{
+#include <ctype.h>
 #include <string.h>
 #include <unistd.h>
 #include "bif_arg.h"
@@ -223,7 +224,7 @@ void init_alternative_mode()
 
 	for ( char* p = guard; *p; p++ )
 		{
-		if ( strchr("/.-", *p) )
+		if ( ! isalnum(*p) )
 			*p = '_';
 		}
 

src/event.bif

@@ -905,7 +905,8 @@ event get_file_handle%(tag: Analyzer::Tag, c: connection, is_orig: bool%);
 ##
 ## f: The file.
 ##
-## .. bro:see:: file_over_new_connection file_timeout file_gap file_state_remove
+## .. bro:see:: file_over_new_connection file_timeout file_gap file_mime_type
+##    file_state_remove
 event file_new%(f: fa_file%);
 
 ## Indicates that a file has been seen being transferred over a connection
@@ -917,16 +918,39 @@ event file_new%(f: fa_file%);
 ##
 ## is_orig: true if the originator of *c* is the one sending the file.
 ##
-## .. bro:see:: file_new file_timeout file_gap file_state_remove
+## .. bro:see:: file_new file_timeout file_gap file_mime_type 
+##    file_state_remove
 event file_over_new_connection%(f: fa_file, c: connection, is_orig: bool%);
 
+## Provide the most likely matching MIME type for this file. The analysis 
+## can be augmented at this time via :bro:see:`Files::add_analyzer`.
+##
+## f: The file.
+##
+## mime_type: The mime type that was discovered.
+##
+## .. bro:see:: file_over_new_connection file_timeout file_gap file_mime_type 
+##    file_mime_types file_state_remove
+event file_mime_type%(f: fa_file, mime_type: string%);
+
+## Provide all matching MIME types for this file. The analysis can be
+## augmented at this time via :bro:see:`Files::add_analyzer`.
+##
+## f: The file.
+##
+## mime_types: The mime types that were discovered.
+##
+## .. bro:see:: file_over_new_connection file_timeout file_gap file_mime_type 
+##    file_mime_types file_state_remove
+event file_mime_types%(f: fa_file, mime_types: mime_matches%);
+
 ## Indicates that file analysis has timed out because no activity was seen
 ## for the file in a while.
 ##
 ## f: The file.
 ##
-## .. bro:see:: file_new file_over_new_connection file_gap file_state_remove
-##    default_file_timeout_interval Files::set_timeout_interval
+## .. bro:see:: file_new file_over_new_connection file_gap file_mime_type
+##    file_mime_types file_state_remove default_file_timeout_interval
 ##    Files::set_timeout_interval
 event file_timeout%(f: fa_file%);
 
@@ -938,14 +962,34 @@ event file_timeout%(f: fa_file%);
 ##
 ## len: The number of missing bytes.
 ##
-## .. bro:see:: file_new file_over_new_connection file_timeout file_state_remove
+## .. bro:see:: file_new file_over_new_connection file_timeout file_mime_type
+##    file_mime_types file_state_remove file_reassembly_overflow
 event file_gap%(f: fa_file, offset: count, len: count%);
 
+## Indicates that the file had an overflow of the reassembly buffer.
+## This is a specialization of the :bro:id:`file_gap` event.
+##
+## f: The file.
+##
+## offset: The byte offset from the start of the file at which the reassembly
+##         couldn't continue due to running out of reassembly buffer space.
+##
+## skipped: The number of bytes of the file skipped over to flush some
+##          file data and get back under the reassembly buffer size limit.
+##          This value will also be represented as a gap.
+##
+## .. bro:see:: file_new file_over_new_connection file_timeout file_mime_type
+##    file_mime_types file_state_remove file_gap Files::enable_reassembler 
+##    Files::reassembly_buffer_size Files::enable_reassembly 
+##    Files::disable_reassembly Files::set_reassembly_buffer_size
+event file_reassembly_overflow%(f: fa_file, offset: count, skipped: count%);
+
 ## This event is generated each time file analysis is ending for a given file.
 ##
 ## f: The file.
 ##
 ## .. bro:see:: file_new file_over_new_connection file_timeout file_gap
+##    file_mime_type file_mime_types
 event file_state_remove%(f: fa_file%);
 
 ## Generated when an internal DNS lookup produces the same result as last time.

src/file_analysis/Analyzer.h

@@ -111,6 +111,18 @@ public:
 	 */
 	void SetAnalyzerTag(const file_analysis::Tag& tag);
 
+	/**
+	 * @return true if the analyzer has ever seen a stream-wise delivery.
+	 */
+	bool GotStreamDelivery() const
+		{ return got_stream_delivery; }
+
+	/**
+	 * Flag the analyzer as having seen a stream-wise delivery.
+	 */
+	void SetGotStreamDelivery()
+		{ got_stream_delivery = true; }
+
 protected:
 
 	/**
@@ -123,7 +135,8 @@ protected:
 	Analyzer(file_analysis::Tag arg_tag, RecordVal* arg_args, File* arg_file)
 	    : tag(arg_tag),
 	      args(arg_args->Ref()->AsRecordVal()),
-	      file(arg_file)
+	      file(arg_file),
+	      got_stream_delivery(false)
 		{
 		id = ++id_counter;
 		}
@@ -140,7 +153,8 @@ protected:
 	Analyzer(RecordVal* arg_args, File* arg_file)
 	    : tag(),
 	      args(arg_args->Ref()->AsRecordVal()),
-	      file(arg_file)
+	      file(arg_file),
+	      got_stream_delivery(false)
 		{
 		id = ++id_counter;
 		}
@@ -151,6 +165,7 @@ private:
 	file_analysis::Tag tag;	/**< The particular type of the analyzer instance. */
 	RecordVal* args;	/**< \c AnalyzerArgs val gives tunable analyzer params. */
 	File* file;	/**< The file to which the analyzer is attached. */
+	bool got_stream_delivery;
 
 	static ID id_counter;
 };

src/file_analysis/AnalyzerSet.cc

@@ -72,7 +72,7 @@ bool AnalyzerSet::Add(file_analysis::Tag tag, RecordVal* args)
 	return true;
 	}
 
-bool AnalyzerSet::QueueAdd(file_analysis::Tag tag, RecordVal* args)
+Analyzer* AnalyzerSet::QueueAdd(file_analysis::Tag tag, RecordVal* args)
 	{
 	HashKey* key = GetKey(tag, args);
 	file_analysis::Analyzer* a = InstantiateAnalyzer(tag, args);
@@ -80,12 +80,12 @@ bool AnalyzerSet::QueueAdd(file_analysis::Tag tag, RecordVal* args)
 	if ( ! a )
 		{
 		delete key;
-		return false;
+		return 0;
 		}
 
 	mod_queue.push(new AddMod(a, key));
 
-	return true;
+	return a;
 	}
 
 bool AnalyzerSet::AddMod::Perform(AnalyzerSet* set)

src/file_analysis/AnalyzerSet.h

@@ -57,9 +57,10 @@ public:
 	 * Queue the attachment of an analyzer to #file.
 	 * @param tag the analyzer tag of the file analyzer to add.
 	 * @param args an \c AnalyzerArgs value which specifies an analyzer.
-	 * @return true if analyzer was able to be instantiated, else false.
+	 * @return if successful, a pointer to a newly instantiated analyzer else
+	 * a null pointer.  The caller does *not* take ownership of the memory.
 	 */
-	bool QueueAdd(file_analysis::Tag tag, RecordVal* args);
+	file_analysis::Analyzer* QueueAdd(file_analysis::Tag tag, RecordVal* args);
 
 	/**
 	 * Remove an analyzer from #file immediately.

src/file_analysis/CMakeLists.txt

@@ -11,6 +11,7 @@ set(file_analysis_SRCS
     Manager.cc
     File.cc
     FileTimer.cc
+    FileReassembler.cc
     Analyzer.cc
     AnalyzerSet.cc
     Component.cc

src/file_analysis/File.cc

@@ -53,8 +53,6 @@ int File::overflow_bytes_idx = -1;
 int File::timeout_interval_idx = -1;
 int File::bof_buffer_size_idx = -1;
 int File::bof_buffer_idx = -1;
-int File::mime_type_idx = -1;
-int File::mime_types_idx = -1;
 
 void File::StaticInit()
 	{
@@ -74,15 +72,14 @@ void File::StaticInit()
 	timeout_interval_idx = Idx("timeout_interval");
 	bof_buffer_size_idx = Idx("bof_buffer_size");
 	bof_buffer_idx = Idx("bof_buffer");
-	mime_type_idx = Idx("mime_type");
-	mime_types_idx = Idx("mime_types");
 	}
 
-File::File(const string& file_id, Connection* conn, analyzer::Tag tag,
-           bool is_orig)
-	: id(file_id), val(0), postpone_timeout(false), first_chunk(true),
-	  missed_bof(false), need_reassembly(false), done(false),
-	  did_file_new_event(false), analyzers(this)
+File::File(const string& file_id, const string& source_name, Connection* conn,
+           analyzer::Tag tag, bool is_orig)
+	: id(file_id), val(0), file_reassembler(0), stream_offset(0), 
+	  reassembly_max_buffer(0), did_mime_type(false), 
+	  reassembly_enabled(false), postpone_timeout(false), done(false), 
+	  analyzers(this)
 	{
 	StaticInit();
 
@@ -90,11 +87,10 @@ File::File(const string& file_id, Connection* conn, analyzer::Tag tag,
 
 	val = new RecordVal(fa_file_type);
 	val->Assign(id_idx, new StringVal(file_id.c_str()));
+	SetSource(source_name);
 
 	if ( conn )
 		{
-		// add source, connection, is_orig fields
-		SetSource(analyzer_mgr->GetComponentName(tag));
 		val->Assign(is_orig_idx, new Val(is_orig, TYPE_BOOL));
 		UpdateConnectionFields(conn, is_orig);
 		}
@@ -106,12 +102,7 @@ File::~File()
 	{
 	DBG_LOG(DBG_FILE_ANALYSIS, "[%s] Destroying File object", id.c_str());
 	Unref(val);
-
-	while ( ! fonc_queue.empty() )
-		{
-		delete_vals(fonc_queue.front().second);
-		fonc_queue.pop();
-		}
+	delete file_reassembler;
 	}
 
 void File::UpdateLastActivityTime()
@@ -124,10 +115,10 @@ double File::GetLastActivityTime() const
 	return val->Lookup(last_active_idx)->AsTime();
 	}
 
-void File::UpdateConnectionFields(Connection* conn, bool is_orig)
+bool File::UpdateConnectionFields(Connection* conn, bool is_orig)
 	{
 	if ( ! conn )
-		return;
+		return false;
 
 	Val* conns = val->Lookup(conns_idx);
 
@@ -138,27 +129,28 @@ void File::UpdateConnectionFields(Connection* conn, bool is_orig)
 		}
 
 	Val* idx = get_conn_id_val(conn);
-	if ( ! conns->AsTableVal()->Lookup(idx) )
+
+	if ( conns->AsTableVal()->Lookup(idx) )
 		{
-		Val* conn_val = conn->BuildConnVal();
-		conns->AsTableVal()->Assign(idx, conn_val);
-
-		if ( FileEventAvailable(file_over_new_connection) )
-			{
-			val_list* vl = new val_list();
-			vl->append(val->Ref());
-			vl->append(conn_val->Ref());
-			vl->append(new Val(is_orig, TYPE_BOOL));
-
-			if ( did_file_new_event )
-				FileEvent(file_over_new_connection, vl);
-			else
-				fonc_queue.push(pair<EventHandlerPtr, val_list*>(
-				        file_over_new_connection, vl));
-			}
+		Unref(idx);
+		return false;
 		}
 
+	conns->AsTableVal()->Assign(idx, conn->BuildConnVal());
 	Unref(idx);
+	return true;
+	}
+
+void File::RaiseFileOverNewConnection(Connection* conn, bool is_orig)
+	{
+	if ( conn && FileEventAvailable(file_over_new_connection) )
+		{
+		val_list* vl = new val_list();
+		vl->append(val->Ref());
+		vl->append(conn->BuildConnVal());
+		vl->append(new Val(is_orig, TYPE_BOOL));
+		FileEvent(file_over_new_connection, vl);
+		}
 	}
 
 uint64 File::LookupFieldDefaultCount(int idx) const
@@ -242,7 +234,7 @@ bool File::IsComplete() const
 	if ( ! total )
 		return false;
 
-	if ( LookupFieldDefaultCount(seen_bytes_idx) >= total->AsCount() )
+	if ( stream_offset >= total->AsCount() )
 		return true;
 
 	return false;
@@ -258,7 +250,10 @@ bool File::AddAnalyzer(file_analysis::Tag tag, RecordVal* args)
 	DBG_LOG(DBG_FILE_ANALYSIS, "[%s] Queuing addition of %s analyzer",
 		id.c_str(), file_mgr->GetComponentName(tag).c_str());
 
-	return done ? false : analyzers.QueueAdd(tag, args);
+	if ( done )
+		return false;
+
+	return analyzers.QueueAdd(tag, args) != 0;
 	}
 
 bool File::RemoveAnalyzer(file_analysis::Tag tag, RecordVal* args)
@@ -269,9 +264,70 @@ bool File::RemoveAnalyzer(file_analysis::Tag tag, RecordVal* args)
 	return done ? false : analyzers.QueueRemove(tag, args);
 	}
 
+void File::EnableReassembly()
+	{
+	reassembly_enabled = true;
+	}
+
+void File::DisableReassembly()
+	{
+	reassembly_enabled = false;
+	delete file_reassembler;
+	file_reassembler = 0;
+	}
+
+void File::SetReassemblyBuffer(uint64 max)
+	{
+	reassembly_max_buffer = max;
+	}
+
+bool File::DetectMIME()
+	{
+	did_mime_type = true;
+
+	Val* bof_buffer_val = val->Lookup(bof_buffer_idx);
+
+	if ( ! bof_buffer_val )
+		{
+		if ( bof_buffer.size == 0 )
+			return false;
+
+		BroString* bs = concatenate(bof_buffer.chunks);
+		bof_buffer_val = new StringVal(bs);
+		val->Assign(bof_buffer_idx, bof_buffer_val);
+		}
+
+	RuleMatcher::MIME_Matches matches;
+	const u_char* data = bof_buffer_val->AsString()->Bytes();
+	uint64 len = bof_buffer_val->AsString()->Len();
+	len = min(len, LookupFieldDefaultCount(bof_buffer_size_idx));
+	file_mgr->DetectMIME(data, len, &matches);
+
+	if ( matches.empty() )
+		return false;
+
+	if ( FileEventAvailable(file_mime_type) )
+		{
+		val_list* vl = new val_list();
+		vl->append(val->Ref());
+		vl->append(new StringVal(*(matches.begin()->second.begin())));
+		FileEvent(file_mime_type, vl);
+		}
+
+	if ( FileEventAvailable(file_mime_types) )
+		{
+		val_list* vl = new val_list();
+		vl->append(val->Ref());
+		vl->append(file_analysis::GenMIMEMatchesVal(matches));
+		FileEvent(file_mime_types, vl);
+		}
+
+	return true;
+	}
+
 bool File::BufferBOF(const u_char* data, uint64 len)
 	{
-	if ( bof_buffer.full || bof_buffer.replayed )
+	if ( bof_buffer.full )
 		return false;
 
 	uint64 desired_size = LookupFieldDefaultCount(bof_buffer_size_idx);
@@ -279,131 +335,154 @@ bool File::BufferBOF(const u_char* data, uint64 len)
 	bof_buffer.chunks.push_back(new BroString(data, len, 0));
 	bof_buffer.size += len;
 
-	if ( bof_buffer.size >= desired_size )
+	if ( bof_buffer.size < desired_size )
+		return true;
+
+	bof_buffer.full = true;
+
+	if ( bof_buffer.size > 0 )
 		{
-		bof_buffer.full = true;
-		ReplayBOF();
+		BroString* bs = concatenate(bof_buffer.chunks);
+		val->Assign(bof_buffer_idx, new StringVal(bs));
 		}
 
-	return true;
+	return false;
 	}
 
-bool File::DetectMIME(const u_char* data, uint64 len)
+void File::DeliverStream(const u_char* data, uint64 len)
 	{
-	RuleMatcher::MIME_Matches matches;
-	len = min(len, LookupFieldDefaultCount(bof_buffer_size_idx));
-	file_mgr->DetectMIME(data, len, &matches);
+	bool bof_was_full = bof_buffer.full;
+	// Buffer enough data for the BOF buffer
+	BufferBOF(data, len);
 
-	if ( matches.empty() )
-		return false;
+	if ( ! did_mime_type && bof_buffer.full &&
+	     LookupFieldDefaultCount(missing_bytes_idx) == 0 )
+		DetectMIME();
 
-	val->Assign(mime_type_idx,
-	            new StringVal(*(matches.begin()->second.begin())));
-	val->Assign(mime_types_idx, file_analysis::GenMIMEMatchesVal(matches));
+	DBG_LOG(DBG_FILE_ANALYSIS,
+	        "[%s] %" PRIu64 " stream bytes in at offset %" PRIu64 "; %s [%s%s]",
+	        id.c_str(), len, stream_offset,
+	        IsComplete() ? "complete" : "incomplete",
+	        fmt_bytes((const char*) data, min((uint64)40, len)),
+	        len > 40 ? "..." : "");
 
-	return true;
-	}
+	file_analysis::Analyzer* a = 0;
+	IterCookie* c = analyzers.InitForIteration();
 
-void File::ReplayBOF()
-	{
-	if ( bof_buffer.replayed )
-		return;
-
-	bof_buffer.replayed = true;
-
-	if ( bof_buffer.chunks.empty() )
+	while ( (a = analyzers.NextEntry(c)) )
 		{
-		// Since we missed the beginning, try file type detect on next data in.
-		missed_bof = true;
-		return;
+		if ( ! a->GotStreamDelivery() )
+			{
+			int num_bof_chunks_behind = bof_buffer.chunks.size();
+
+			if ( ! bof_was_full )
+				// We just added a chunk to the BOF buffer, don't count it
+				// as it will get delivered on its own.
+				num_bof_chunks_behind -= 1;
+
+			uint64 bytes_delivered = 0;
+
+			// Catch this analyzer up with the BOF buffer.
+			for ( int i = 0; i < num_bof_chunks_behind; ++i )
+				{
+				if ( ! a->DeliverStream(bof_buffer.chunks[i]->Bytes(),
+				                        bof_buffer.chunks[i]->Len()) )
+					analyzers.QueueRemove(a->Tag(), a->Args());
+
+				bytes_delivered += bof_buffer.chunks[i]->Len();
+				}
+
+			a->SetGotStreamDelivery();
+			// May need to catch analyzer up on missed gap?
+			// Analyzer should be fully caught up to stream_offset now.
+			}
+
+		if ( ! a->DeliverStream(data, len) )
+			analyzers.QueueRemove(a->Tag(), a->Args());
 		}
 
-	BroString* bs = concatenate(bof_buffer.chunks);
-	val->Assign(bof_buffer_idx, new StringVal(bs));
+	stream_offset += len;
+	IncrementByteCount(len, seen_bytes_idx);
+	}
 
-	DetectMIME(bs->Bytes(), bs->Len());
-	FileEvent(file_new);
+void File::DeliverChunk(const u_char* data, uint64 len, uint64 offset)
+	{
+	// Potentially handle reassembly and deliver to the stream analyzers.
+	if ( file_reassembler )
+		{
+		if ( reassembly_max_buffer > 0 &&
+		     reassembly_max_buffer < file_reassembler->TotalSize() )
+			{
+			uint64 current_offset = stream_offset;
+			uint64 gap_bytes = file_reassembler->Flush();
+			IncrementByteCount(gap_bytes, overflow_bytes_idx);
 
-	for ( size_t i = 0; i < bof_buffer.chunks.size(); ++i )
-		DataIn(bof_buffer.chunks[i]->Bytes(), bof_buffer.chunks[i]->Len());
+			if ( FileEventAvailable(file_reassembly_overflow) )
+				{
+				val_list* vl = new val_list();
+				vl->append(val->Ref());
+				vl->append(new Val(current_offset, TYPE_COUNT));
+				vl->append(new Val(gap_bytes, TYPE_COUNT));
+				FileEvent(file_reassembly_overflow, vl);
+				}
+			}
+
+		// Forward data to the reassembler.
+		file_reassembler->NewBlock(network_time, offset, len, data);
+		}
+	else if ( stream_offset == offset )
+		{
+		// This is the normal case where a file is transferred linearly.
+		// Nothing special should be done here.
+		DeliverStream(data, len);
+		}
+	else if ( reassembly_enabled )
+		{
+		// This is data that doesn't match the offset and the reassembler 
+		// needs to be enabled.
+		file_reassembler = new FileReassembler(this, stream_offset);
+		file_reassembler->NewBlock(network_time, offset, len, data);
+		}
+	else
+		{
+		// We can't reassemble so we throw out the data for streaming.
+		IncrementByteCount(len, overflow_bytes_idx);
+		}
+
+	DBG_LOG(DBG_FILE_ANALYSIS,
+	        "[%s] %" PRIu64 " chunk bytes in at offset %" PRIu64 "; %s [%s%s]",
+	        id.c_str(), len, offset,
+	        IsComplete() ? "complete" : "incomplete",
+	        fmt_bytes((const char*) data, min((uint64)40, len)),
+	        len > 40 ? "..." : "");
+
+	file_analysis::Analyzer* a = 0;
+	IterCookie* c = analyzers.InitForIteration();
+
+	while ( (a = analyzers.NextEntry(c)) )
+		{
+		if ( ! a->DeliverChunk(data, len, offset) )
+			{
+			analyzers.QueueRemove(a->Tag(), a->Args());
+			}
+		}
+
+	if ( IsComplete() )
+		EndOfFile();
 	}
 
 void File::DataIn(const u_char* data, uint64 len, uint64 offset)
 	{
 	analyzers.DrainModifications();
-
-	if ( first_chunk )
-		{
-		// TODO: this should all really be delayed until we attempt reassembly
-		DetectMIME(data, len);
-		FileEvent(file_new);
-		first_chunk = false;
-		}
-
-	DBG_LOG(DBG_FILE_ANALYSIS, "[%s] %" PRIu64 " bytes in at offset" PRIu64 "; %s [%s]",
-		id.c_str(), len, offset,
-		IsComplete() ? "complete" : "incomplete",
-		fmt_bytes((const char*) data, min((uint64)40, len)), len > 40 ? "..." : "");
-
-	file_analysis::Analyzer* a = 0;
-	IterCookie* c = analyzers.InitForIteration();
-
-	while ( (a = analyzers.NextEntry(c)) )
-		{
-		if ( ! a->DeliverChunk(data, len, offset) )
-			analyzers.QueueRemove(a->Tag(), a->Args());
-		}
-
+	DeliverChunk(data, len, offset);
 	analyzers.DrainModifications();
-
-	// TODO: check reassembly requirement based on buffer size in record
-	if ( need_reassembly )
-		reporter->InternalError("file_analyzer::File TODO: reassembly not yet supported");
-
-	// TODO: reassembly overflow stuff, increment overflow count, eval trigger
-
-	IncrementByteCount(len, seen_bytes_idx);
 	}
 
 void File::DataIn(const u_char* data, uint64 len)
 	{
 	analyzers.DrainModifications();
-
-	if ( BufferBOF(data, len) )
-		return;
-
-	if ( missed_bof )
-		{
-		DetectMIME(data, len);
-		FileEvent(file_new);
-		missed_bof = false;
-		}
-
-	DBG_LOG(DBG_FILE_ANALYSIS, "[%s] %" PRIu64 " bytes in; %s [%s]",
-		id.c_str(), len,
-		IsComplete() ? "complete" : "incomplete",
-		fmt_bytes((const char*) data, min((uint64)40, len)), len > 40 ? "..." : "");
-
-	file_analysis::Analyzer* a = 0;
-	IterCookie* c = analyzers.InitForIteration();
-
-	while ( (a = analyzers.NextEntry(c)) )
-		{
-		if ( ! a->DeliverStream(data, len) )
-			{
-			analyzers.QueueRemove(a->Tag(), a->Args());
-			continue;
-			}
-
-		uint64 offset = LookupFieldDefaultCount(seen_bytes_idx) +
-		                LookupFieldDefaultCount(missing_bytes_idx);
-
-		if ( ! a->DeliverChunk(data, len, offset) )
-			analyzers.QueueRemove(a->Tag(), a->Args());
-		}
-
+	DeliverChunk(data, len, stream_offset);
 	analyzers.DrainModifications();
-	IncrementByteCount(len, seen_bytes_idx);
 	}
 
 void File::EndOfFile()
@@ -413,10 +492,17 @@ void File::EndOfFile()
 	if ( done )
 		return;
 
+	if ( ! did_mime_type &&
+	     LookupFieldDefaultCount(missing_bytes_idx) == 0 )
+		DetectMIME();
+
 	analyzers.DrainModifications();
 
-	// Send along anything that's been buffered, but never flushed.
-	ReplayBOF();
+	if ( file_reassembler )
+		{
+		file_reassembler->Flush();
+		analyzers.DrainModifications();
+		}
 
 	done = true;
 
@@ -436,14 +522,17 @@ void File::EndOfFile()
 
 void File::Gap(uint64 offset, uint64 len)
 	{
-	DBG_LOG(DBG_FILE_ANALYSIS, "[%s] Gap of size %" PRIu64 " at offset %" PRIu64,
+	DBG_LOG(DBG_FILE_ANALYSIS, "[%s] Gap of size %" PRIu64 " at offset %," PRIu64,
 		id.c_str(), len, offset);
 
-	analyzers.DrainModifications();
+	if ( file_reassembler && ! file_reassembler->IsCurrentlyFlushing() )
+		{
+		file_reassembler->FlushTo(offset + len);
+		// The reassembler will call us back with all the gaps we need to know.
+		return;
+		}
 
-	// If we were buffering the beginning of the file, a gap means we've got
-	// as much contiguous stuff at the beginning as possible, so work with that.
-	ReplayBOF();
+	analyzers.DrainModifications();
 
 	file_analysis::Analyzer* a = 0;
 	IterCookie* c = analyzers.InitForIteration();
@@ -464,6 +553,8 @@ void File::Gap(uint64 offset, uint64 len)
 		}
 
 	analyzers.DrainModifications();
+
+	stream_offset += len;
 	IncrementByteCount(len, missing_bytes_idx);
 	}
 
@@ -482,30 +573,13 @@ void File::FileEvent(EventHandlerPtr h)
 	FileEvent(h, vl);
 	}
 
-static void flush_file_event_queue(queue<pair<EventHandlerPtr, val_list*> >& q)
-	{
-	while ( ! q.empty() )
-		{
-		pair<EventHandlerPtr, val_list*> p = q.front();
-		mgr.QueueEvent(p.first, p.second);
-		q.pop();
-		}
-	}
-
 void File::FileEvent(EventHandlerPtr h, val_list* vl)
 	{
-	if ( h == file_state_remove )
-		flush_file_event_queue(fonc_queue);
-
 	mgr.QueueEvent(h, vl);
 
-	if ( h == file_new )
-		{
-		did_file_new_event = true;
-		flush_file_event_queue(fonc_queue);
-		}
-
-	if ( h == file_new || h == file_timeout || h == file_extraction_limit )
+	if ( h == file_new || h == file_over_new_connection ||
+	     h == file_mime_type ||
+	     h == file_timeout || h == file_extraction_limit )
 		{
 		// immediate feedback is required for these events.
 		mgr.Drain();

src/file_analysis/File.h

@@ -3,11 +3,11 @@
 #ifndef FILE_ANALYSIS_FILE_H
 #define FILE_ANALYSIS_FILE_H
 
-#include <queue>
 #include <string>
 #include <utility>
 #include <vector>
 
+#include "FileReassembler.h"
 #include "Conn.h"
 #include "Val.h"
 #include "Tag.h"
@@ -16,6 +16,8 @@
 
 namespace file_analysis {
 
+class FileReassembler;
+
 /**
  * Wrapper class around \c fa_file record values from script layer.
  */
@@ -86,10 +88,10 @@ public:
 	void SetTotalBytes(uint64 size);
 
 	/**
-	 * Compares "seen_bytes" field to "total_bytes" field of #val record to
-	 * determine if the full file has been seen.
-	 * @return false if "total_bytes" hasn't been set yet or "seen_bytes" is
-	 *         less than it, else true.
+	 * @return true if file analysis is complete for the file, else false.
+	 * It is incomplete if the total size is unknown or if the number of bytes
+	 * streamed to analyzers (either as data delivers or gap information)
+	 * matches the known total size.
 	 */
 	bool IsComplete() const;
 
@@ -166,18 +168,20 @@ public:
 
 protected:
 	friend class Manager;
+	friend class FileReassembler;
 
 	/**
 	 * Constructor; only file_analysis::Manager should be creating these.
 	 * @param file_id an identifier string for the file in pretty hash form
 	 *        (similar to connection uids).
+	 * @param source_name the value for the source field to fill in.
 	 * @param conn a network connection over which the file is transferred.
 	 * @param tag the network protocol over which the file is transferred.
 	 * @param is_orig true if the file is being transferred from the originator
 	 *        of the connection to the responder.  False indicates the other
 	 *        direction.
 	 */
-	File(const string& file_id, Connection* conn = 0,
+	File(const string& file_id, const string& source_name, Connection* conn = 0,
 	     analyzer::Tag tag = analyzer::Tag::Error, bool is_orig = false);
 
 	/**
@@ -185,8 +189,14 @@ protected:
 	 * \c conn_id and UID taken from \a conn.
 	 * @param conn the connection over which a part of the file has been seen.
 	 * @param is_orig true if the connection originator is sending the file.
+	 * @return true if the connection was previously unknown.
 	 */
-	void UpdateConnectionFields(Connection* conn, bool is_orig);
+	bool UpdateConnectionFields(Connection* conn, bool is_orig);
+
+	/**
+	 * Raise the file_over_new_connection event with given arguments.
+	 */
+	void RaiseFileOverNewConnection(Connection* conn, bool is_orig);
 
 	/**
 	 * Increment a byte count field of #val record by \a size.
@@ -219,20 +229,40 @@ protected:
 	 */
 	bool BufferBOF(const u_char* data, uint64 len);
 
-	/**
-	 * Forward any beginning-of-file buffered data on to DataIn stream.
-	 */
-	void ReplayBOF();
-
 	/**
 	 * Does mime type detection via file magic signatures and assigns
 	 * strongest matching mime type (if available) to \c mime_type
-	 * field in #val.
-	 * @param data pointer to a chunk of file data.
-	 * @param len number of bytes in the data chunk.
+	 * field in #val.  It uses the data in the BOF buffer.
 	 * @return whether a mime type match was found.
 	 */
-	bool DetectMIME(const u_char* data, uint64 len);
+	bool DetectMIME();
+
+	/**
+	 * Enables reassembly on the file.
+	 */
+	void EnableReassembly();
+
+	/**
+	 * Disables reassembly on the file.  If there is an existing reassembler
+	 * for the file, this will cause it to be deleted and won't allow a new
+	 * one to be created until reassembly is reenabled.
+	 */
+	void DisableReassembly();
+
+	/**
+	 * Set a maximum allowed bytes of memory for file reassembly for this file.
+	 */
+	void SetReassemblyBuffer(uint64 max);
+
+	/**
+	 * Perform stream-wise delivery for analyzers that need it.
+	 */
+	void DeliverStream(const u_char* data, uint64 len);
+
+	/** 
+	 * Perform chunk-wise delivery for analyzers that need it.
+	 */
+	void DeliverChunk(const u_char* data, uint64 len, uint64 offset);
 
 	/**
 	 * Lookup a record field index/offset by name.
@@ -246,25 +276,24 @@ protected:
 	 */
 	static void StaticInit();
 
-private:
+protected:
 	string id;                 /**< A pretty hash that likely identifies file */
 	RecordVal* val;            /**< \c fa_file from script layer. */
+	FileReassembler* file_reassembler; /**< A reassembler for the file if it's needed. */
+	uint64 stream_offset;      /**< The offset of the file which has been forwarded. */
+	uint64 reassembly_max_buffer;      /**< Maximum allowed buffer for reassembly. */
+	bool did_mime_type;        /**< Whether the mime type ident has already been attempted. */
+	bool reassembly_enabled;           /**< Whether file stream reassembly is needed. */
 	bool postpone_timeout;     /**< Whether postponing timeout is requested. */
-	bool first_chunk;          /**< Track first non-linear chunk. */
-	bool missed_bof;           /**< Flags that we missed start of file. */
-	bool need_reassembly;      /**< Whether file stream reassembly is needed. */
 	bool done;                 /**< If this object is about to be deleted. */
-	bool did_file_new_event;   /**< Whether the file_new event has been done. */
-	AnalyzerSet analyzers;     /**< A set of attached file analyzer. */
-	queue<pair<EventHandlerPtr, val_list*> > fonc_queue;
+	AnalyzerSet analyzers;     /**< A set of attached file analyzers. */
 
 	struct BOF_Buffer {
-		BOF_Buffer() : full(false), replayed(false), size(0) {}
+		BOF_Buffer() : full(false), size(0) {}
 		~BOF_Buffer()
 			{ for ( size_t i = 0; i < chunks.size(); ++i ) delete chunks[i]; }
 
 		bool full;
-		bool replayed;
 		uint64 size;
 		BroString::CVec chunks;
 	} bof_buffer;              /**< Beginning of file buffer. */

src/file_analysis/FileReassembler.cc

@@ -0,0 +1,123 @@
+
+#include "FileReassembler.h"
+#include "File.h"
+
+
+namespace file_analysis {
+
+class File;
+
+FileReassembler::FileReassembler(File *f, uint64 starting_offset)
+	: Reassembler(starting_offset), the_file(f), flushing(false)
+	{
+	}
+
+FileReassembler::~FileReassembler()
+	{
+	}
+
+uint64 FileReassembler::Flush()
+	{
+	if ( flushing )
+		return 0;
+
+	if ( last_block )
+		{
+		// This is expected to call back into FileReassembler::Undelivered().
+		flushing = true;
+		uint64 rval = TrimToSeq(last_block->upper);
+		flushing = false;
+		return rval;
+		}
+
+	return 0;
+	}
+
+uint64 FileReassembler::FlushTo(uint64 sequence)
+	{
+	if ( flushing )
+		return 0;
+
+	flushing = true;
+	uint64 rval = TrimToSeq(sequence);
+	flushing = false;
+	last_reassem_seq = sequence;
+	return rval;
+	}
+
+void FileReassembler::BlockInserted(DataBlock* start_block)
+	{
+	if ( start_block->seq > last_reassem_seq ||
+	     start_block->upper <= last_reassem_seq )
+		return;
+
+	for ( DataBlock* b = start_block;
+	      b && b->seq <= last_reassem_seq; b = b->next )
+		{
+		if ( b->seq == last_reassem_seq )
+			{ // New stuff.
+			uint64 len = b->Size();
+			last_reassem_seq += len;
+			the_file->DeliverStream(b->block, len);
+			}
+		}
+
+	// Throw out forwarded data
+	TrimToSeq(last_reassem_seq);
+	}
+
+void FileReassembler::Undelivered(uint64 up_to_seq)
+	{
+	// If we have blocks that begin below up_to_seq, deliver them.
+	DataBlock* b = blocks;
+
+	while ( b )
+		{
+		if ( b->seq < last_reassem_seq )
+			{
+			// Already delivered this block.
+			b = b->next;
+			continue;
+			}
+
+		if ( b->seq >= up_to_seq )
+			// Block is beyond what we need to process at this point.
+			break;
+
+		uint64 gap_at_seq = last_reassem_seq;
+		uint64 gap_len = b->seq - last_reassem_seq;
+		the_file->Gap(gap_at_seq, gap_len);
+		last_reassem_seq += gap_len;
+		BlockInserted(b);
+		// Inserting a block may cause trimming of what's buffered,
+		// so have to assume 'b' is invalid, hence re-assign to start.
+		b = blocks;
+		}
+
+	if ( up_to_seq > last_reassem_seq )
+		{
+		the_file->Gap(last_reassem_seq, up_to_seq - last_reassem_seq);
+		last_reassem_seq = up_to_seq;
+		}
+	}
+
+void FileReassembler::Overlap(const u_char* b1, const u_char* b2, uint64 n)
+	{
+	// Not doing anything here yet.
+	}
+
+IMPLEMENT_SERIAL(FileReassembler, SER_FILE_REASSEMBLER);
+
+bool FileReassembler::DoSerialize(SerialInfo* info) const
+	{
+	reporter->InternalError("FileReassembler::DoSerialize not implemented");
+	return false; // Cannot be reached.
+	}
+
+bool FileReassembler::DoUnserialize(UnserialInfo* info)
+	{
+	reporter->InternalError("FileReassembler::DoUnserialize not implemented");
+	return false; // Cannot be reached.
+	}
+
+} // end file_analysis

src/file_analysis/FileReassembler.h

@@ -0,0 +1,65 @@
+#ifndef FILE_ANALYSIS_FILEREASSEMBLER_H
+#define FILE_ANALYSIS_FILEREASSEMBLER_H
+
+#include "Reassem.h"
+#include "File.h"
+
+class BroFile;
+class Connection;
+
+namespace file_analysis {
+
+class File;
+
+class FileReassembler : public Reassembler {
+public:
+
+	FileReassembler(File* f, uint64 starting_offset);
+	virtual ~FileReassembler();
+
+	void Done();
+
+	// Checks if we have delivered all contents that we can possibly
+	// deliver for this endpoint.
+	void CheckEOF();
+
+	/**
+	 * Discards all contents of the reassembly buffer.  This will spin through
+	 * the buffer and call File::DeliverStream() and File::Gap() wherever
+	 * appropriate.
+	 * @return the number of new bytes now detected as gaps in the file.
+	 */
+	uint64 Flush();
+
+	/**
+	 * Discards all contents of the reassembly buffer up to a given sequence
+	 * number.  This will spin through the buffer and call
+	 * File::DeliverStream() and File::Gap() wherever appropriate.
+	 * @param sequence the sequence number to flush until.
+	 * @return the number of new bytes now detected as gaps in the file.
+	 */
+	uint64 FlushTo(uint64 sequence);
+
+	/**
+	 * @return whether the reassembler is currently is the process of flushing
+	 * out the contents of its buffer.
+	 */
+	bool IsCurrentlyFlushing() const
+		{ return flushing; }
+
+protected:
+	FileReassembler()	{ }
+
+	DECLARE_SERIAL(FileReassembler);
+
+	void Undelivered(uint64 up_to_seq);
+	void BlockInserted(DataBlock* b);
+	void Overlap(const u_char* b1, const u_char* b2, uint64 n);
+
+	File* the_file;
+	bool flushing;
+};
+
+} // namespace analyzer::* 
+
+#endif

src/file_analysis/Manager.cc

@@ -154,14 +154,12 @@ string Manager::DataIn(const u_char* data, uint64 len, analyzer::Tag tag,
 void Manager::DataIn(const u_char* data, uint64 len, const string& file_id,
                      const string& source)
 	{
-	File* file = GetFile(file_id);
+	File* file = GetFile(file_id, 0, analyzer::Tag::Error, false, false,
+	                     source.c_str());
 
 	if ( ! file )
 		return;
 
-	if ( file->GetSource().empty() )
-		file->SetSource(source);
-
 	file->DataIn(data, len);
 
 	if ( file->IsComplete() )
@@ -232,6 +230,39 @@ bool Manager::SetTimeoutInterval(const string& file_id, double interval) const
 	return true;
 	}
 
+bool Manager::EnableReassembly(const string& file_id)
+	{
+	File* file = LookupFile(file_id);
+
+	if ( ! file )
+		return false;
+
+	file->EnableReassembly();
+	return true;
+	}
+
+bool Manager::DisableReassembly(const string& file_id)
+	{
+	File* file = LookupFile(file_id);
+
+	if ( ! file )
+		return false;
+
+	file->DisableReassembly();
+	return true;
+	}
+
+bool Manager::SetReassemblyBuffer(const string& file_id, uint64 max)
+	{
+	File* file = LookupFile(file_id);
+
+	if ( ! file )
+		return false;
+
+	file->SetReassemblyBuffer(max);
+	return true;
+	}
+
 bool Manager::SetExtractionLimit(const string& file_id, RecordVal* args,
                                  uint64 n) const
 	{
@@ -254,28 +285,6 @@ bool Manager::AddAnalyzer(const string& file_id, file_analysis::Tag tag,
 	return file->AddAnalyzer(tag, args);
 	}
 
-TableVal* Manager::AddAnalyzersForMIMEType(const string& file_id, const string& mtype,
-					   RecordVal* args)
-	{
-	if ( ! tag_set_type )
-		tag_set_type = internal_type("files_tag_set")->AsTableType();
-
-	TableVal* sval = new TableVal(tag_set_type);
-	TagSet* l = LookupMIMEType(mtype, false);
-
-	if ( ! l )
-		return sval;
-
-	for ( TagSet::const_iterator i = l->begin(); i != l->end(); i++ )
-		{
-		file_analysis::Tag tag = *i;
-		if ( AddAnalyzer(file_id, tag, args) )
-			sval->Assign(tag.AsEnumVal(), 0);
-		}
-
-	return sval;
-	}
-
 bool Manager::RemoveAnalyzer(const string& file_id, file_analysis::Tag tag,
                              RecordVal* args) const
 	{
@@ -288,7 +297,8 @@ bool Manager::RemoveAnalyzer(const string& file_id, file_analysis::Tag tag,
 	}
 
 File* Manager::GetFile(const string& file_id, Connection* conn,
-                       analyzer::Tag tag, bool is_orig, bool update_conn)
+                       analyzer::Tag tag, bool is_orig, bool update_conn,
+                       const char* source_name)
 	{
 	if ( file_id.empty() )
 		return 0;
@@ -300,10 +310,19 @@ File* Manager::GetFile(const string& file_id, Connection* conn,
 
 	if ( ! rval )
 		{
-		rval = new File(file_id, conn, tag, is_orig);
+		rval = new File(file_id,
+		                source_name ? source_name
+		                            : analyzer_mgr->GetComponentName(tag),
+		                conn, tag, is_orig);
 		id_map.Insert(file_id.c_str(), rval);
 		rval->ScheduleInactivityTimer();
 
+		// Generate file_new after inserting it into manager's mapping
+		// in case script-layer calls back in to core from the event.
+		rval->FileEvent(file_new);
+		// Same for file_over_new_connection.
+		rval->RaiseFileOverNewConnection(conn, is_orig);
+
 		if ( IsIgnored(file_id) )
 			return 0;
 		}
@@ -311,8 +330,8 @@ File* Manager::GetFile(const string& file_id, Connection* conn,
 		{
 		rval->UpdateLastActivityTime();
 
-		if ( update_conn )
-			rval->UpdateConnectionFields(conn, is_orig);
+		if ( update_conn && rval->UpdateConnectionFields(conn, is_orig) )
+			rval->RaiseFileOverNewConnection(conn, is_orig);
 		}
 
 	return rval;
@@ -461,63 +480,6 @@ Analyzer* Manager::InstantiateAnalyzer(Tag tag, RecordVal* args, File* f) const
 	return a;
 	}
 
-Manager::TagSet* Manager::LookupMIMEType(const string& mtype, bool add_if_not_found)
-	{
-	MIMEMap::const_iterator i = mime_types.find(to_upper(mtype));
-
-	if ( i != mime_types.end() )
-		return i->second;
-
-	if ( ! add_if_not_found )
-		return 0;
-
-	TagSet* l = new TagSet;
-	mime_types.insert(std::make_pair(to_upper(mtype), l));
-	return l;
-	}
-
-bool Manager::RegisterAnalyzerForMIMEType(EnumVal* tag, StringVal* mtype)
-	{
-	Component* p = Lookup(tag);
-
-	if ( ! p  )
-		return false;
-
-	return RegisterAnalyzerForMIMEType(p->Tag(), mtype->CheckString());
-	}
-
-bool Manager::RegisterAnalyzerForMIMEType(Tag tag, const string& mtype)
-	{
-	TagSet* l = LookupMIMEType(mtype, true);
-
-	DBG_LOG(DBG_FILE_ANALYSIS, "Register analyzer %s for MIME type %s",
-		GetComponentName(tag).c_str(), mtype.c_str());
-
-	l->insert(tag);
-	return true;
-	}
-
-bool Manager::UnregisterAnalyzerForMIMEType(EnumVal* tag, StringVal* mtype)
-	{
-	Component* p = Lookup(tag);
-
-	if ( ! p  )
-		return false;
-
-	return UnregisterAnalyzerForMIMEType(p->Tag(), mtype->CheckString());
-	}
-
-bool Manager::UnregisterAnalyzerForMIMEType(Tag tag, const string& mtype)
-	{
-	TagSet* l = LookupMIMEType(mtype, true);
-
-	DBG_LOG(DBG_FILE_ANALYSIS, "Unregister analyzer %s for MIME type %s",
-		GetComponentName(tag).c_str(), mtype.c_str());
-
-	l->erase(tag);
-	return true;
-	}
-
 RuleMatcher::MIME_Matches* Manager::DetectMIME(const u_char* data, uint64 len,
         RuleMatcher::MIME_Matches* rval) const
 	{

src/file_analysis/Manager.h

@@ -213,6 +213,21 @@ public:
 	 */
 	bool SetTimeoutInterval(const string& file_id, double interval) const;
 
+	/**
+	 * Enable the reassembler for a file.
+	 */
+	bool EnableReassembly(const string& file_id);
+	
+	/**
+	 * Disable the reassembler for a file.
+	 */
+	bool DisableReassembly(const string& file_id);
+
+	/**
+	 * Set the reassembly for a file in bytes.
+	 */
+	bool SetReassemblyBuffer(const string& file_id, uint64 max);
+
 	/**
 	 * Sets a limit on the maximum size allowed for extracting the file
 	 * to local disk;
@@ -238,18 +253,6 @@ public:
 	bool AddAnalyzer(const string& file_id, file_analysis::Tag tag,
 	                 RecordVal* args) const;
 
-	/**
-	 * Queue attachment of an all analyzers associated with a given MIME
-	 * type to the file identifier.
-	 *
-	 * @param file_id the file identifier/hash.
-	 * @param mtype the MIME type; comparisions will be performanced case-insensitive.
-	 * @param args a \c AnalyzerArgs value which describes a file analyzer.
-	 * @return A ref'ed \c set[Tag] with all added analyzers.
-	 */
-	TableVal* AddAnalyzersForMIMEType(const string& file_id, const string& mtype,
-					  RecordVal* args);
-
 	/**
 	 * Queue removal of an analyzer for a given file identifier.
 	 * @param file_id the file identifier/hash.
@@ -277,62 +280,6 @@ public:
 	Analyzer* InstantiateAnalyzer(Tag tag, RecordVal* args, File* f) const;
 
 	/**
-	 * Registers a MIME type for an analyzer. Once registered, files of
-	 * that MIME type will automatically get a corresponding analyzer
-	 * assigned.
-	 *
-	 * @param tag The analyzer's tag as an enum of script type \c
-	 * Files::Tag.
-	 *
-	 * @param mtype The MIME type. It will be matched case-insenistive.
-	 *
-	 * @return True if successful.
-	 */
-	bool RegisterAnalyzerForMIMEType(EnumVal* tag, StringVal* mtype);
-
-	/**
-	 * Registers a MIME type for an analyzer. Once registered, files of
-	 * that MIME type will automatically get a corresponding analyzer
-	 * assigned.
-	 *
-	 * @param tag The analyzer's tag as an enum of script type \c
-	 * Files::Tag.
-	 *
-	 * @param mtype The MIME type. It will be matched case-insenistive.
-	 *
-	 * @return True if successful.
-	 */
-	bool RegisterAnalyzerForMIMEType(Tag tag, const string& mtype);
-
-	/**
-	 * Unregisters a MIME type for an analyzer.
-	 *
-	 * @param tag The analyzer's tag as an enum of script type \c
-	 * Files::Tag.
-	 *
-	 * @param mtype The MIME type. It will be matched case-insenistive.
-	 *
-	 * @return True if successful (incl. when the type wasn't actually
-	 * registered for the analyzer).
-	 *
-	 */
-	bool UnregisterAnalyzerForMIMEType(EnumVal* tag, StringVal* mtype);
-
-	/**
-	 * Unregisters a MIME type for an analyzer.
-	 *
-	 * @param tag The analyzer's tag as an enum of script type \c
-	 * Files::Tag.
-	 *
-	 * @param mtype The MIME type. It will be matched case-insenistive.
-	 *
-	 * @return True if successful (incl. when the type wasn't actually
-	 * registered for the analyzer).
-	 *
-	 */
-	bool UnregisterAnalyzerForMIMEType(Tag tag, const string& mtype);
-
-    /**
 	 * Returns a set of all matching MIME magic signatures for a given
 	 * chunk of data.
 	 * @param data A chunk of bytes to match magic MIME signatures against.
@@ -372,6 +319,7 @@ protected:
 	 *        this file isn't related to a connection).
 	 * @param update_conn whether we need to update connection-related field
 	 *        in the \c fa_file record value associated with the file.
+	 * @param an optional value of the source field to fill in.
 	 * @return the File object mapped to \a file_id or a null pointer if
 	 *         analysis is being ignored for the associated file.  An File
 	 *         object may be created if a mapping doesn't exist, and if it did
@@ -380,7 +328,8 @@ protected:
 	 */
 	File* GetFile(const string& file_id, Connection* conn = 0,
 	              analyzer::Tag tag = analyzer::Tag::Error,
-	              bool is_orig = false, bool update_conn = true);
+	              bool is_orig = false, bool update_conn = true,
+	              const char* source_name = 0);
 
 	/**
 	 * Try to retrieve a file that's being analyzed, using its identifier/hash.

src/file_analysis/analyzer/extract/Extract.cc

@@ -12,9 +12,9 @@ using namespace file_analysis;
 Extract::Extract(RecordVal* args, File* file, const string& arg_filename,
                  uint64 arg_limit)
     : file_analysis::Analyzer(file_mgr->GetComponentTag("EXTRACT"), args, file),
-      filename(arg_filename), limit(arg_limit)
+      filename(arg_filename), limit(arg_limit), depth(0)
 	{
-	fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666);
+	fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_APPEND, 0666);
 
 	if ( fd < 0 )
 		{
@@ -53,7 +53,7 @@ file_analysis::Analyzer* Extract::Instantiate(RecordVal* args, File* file)
 	                   limit->AsCount());
 	}
 
-static bool check_limit_exceeded(uint64 lim, uint64 off, uint64 len, uint64* n)
+static bool check_limit_exceeded(uint64 lim, uint64 depth, uint64 len, uint64* n)
 	{
 	if ( lim == 0 )
 		{
@@ -61,29 +61,31 @@ static bool check_limit_exceeded(uint64 lim, uint64 off, uint64 len, uint64* n)
 		return false;
 		}
 
-	if ( off >= lim )
+	if ( depth >= lim )
 		{
 		*n = 0;
 		return true;
 		}
-
-	*n = lim - off;
-
-	if ( len > *n )
+	else if ( depth + len > lim )
+		{
+		*n = lim - depth;
 		return true;
+		}
 	else
+		{
 		*n = len;
+		}
 
 	return false;
 	}
 
-bool Extract::DeliverChunk(const u_char* data, uint64 len, uint64 offset)
+bool Extract::DeliverStream(const u_char* data, uint64 len)
 	{
 	if ( ! fd )
 		return false;
 
 	uint64 towrite = 0;
-	bool limit_exceeded = check_limit_exceeded(limit, offset, len, &towrite);
+	bool limit_exceeded = check_limit_exceeded(limit, depth, len, &towrite);
 
 	if ( limit_exceeded && file_extraction_limit )
 		{
@@ -92,16 +94,31 @@ bool Extract::DeliverChunk(const u_char* data, uint64 len, uint64 offset)
 		vl->append(f->GetVal()->Ref());
 		vl->append(Args()->Ref());
 		vl->append(new Val(limit, TYPE_COUNT));
-		vl->append(new Val(offset, TYPE_COUNT));
 		vl->append(new Val(len, TYPE_COUNT));
 		f->FileEvent(file_extraction_limit, vl);
 
-		// Limit may have been modified by BIF, re-check it.
-		limit_exceeded = check_limit_exceeded(limit, offset, len, &towrite);
+		// Limit may have been modified by a BIF, re-check it.
+		limit_exceeded = check_limit_exceeded(limit, depth, len, &towrite);
 		}
 
 	if ( towrite > 0 )
-		safe_pwrite(fd, data, towrite, offset);
+		{
+		safe_write(fd, reinterpret_cast<const char*>(data), towrite);
+		depth += towrite;
+		}
 
 	return ( ! limit_exceeded );
 	}
+
+bool Extract::Undelivered(uint64 offset, uint64 len)
+	{
+	if ( depth == offset )
+		{
+		char* tmp = new char[len]();
+		safe_write(fd, tmp, len);
+		delete [] tmp;
+		depth += len;
+		}
+
+	return true;
+	}

src/file_analysis/analyzer/extract/Extract.h

@@ -28,11 +28,18 @@ public:
 	 * Write a chunk of file data to the local extraction file.
 	 * @param data pointer to a chunk of file data.
 	 * @param len number of bytes in the data chunk.
-	 * @param offset number of bytes from start of file at which chunk starts.
 	 * @return false if there was no extraction file open and the data couldn't
 	 *         be written, else true.
 	 */
-	virtual bool DeliverChunk(const u_char* data, uint64 len, uint64 offset);
+	virtual bool DeliverStream(const u_char* data, uint64 len);
+
+	/**
+	 * Report undelivered bytes.
+	 * @param offset distance into the file where the gap occurred.
+	 * @param len number of bytes undelivered.
+	 * @return true
+	 */
+	virtual bool Undelivered(uint64 offset, uint64 len);
 
 	/**
 	 * Create a new instance of an Extract analyzer.
@@ -67,6 +74,7 @@ private:
 	string filename;
 	int fd;
 	uint64 limit;
+	uint64 depth;
 };
 
 } // namespace file_analysis

src/file_analysis/analyzer/extract/events.bif

@@ -11,9 +11,7 @@
 ##
 ## limit: The limit, in bytes, the extracted file is about to breach.
 ##
-## offset: The offset at which a file chunk is about to be written.
-##
 ## len: The length of the file chunk about to be written.
 ##
 ## .. bro:see:: Files::add_analyzer Files::ANALYZER_EXTRACT
-event file_extraction_limit%(f: fa_file, args: any, limit: count, offset: count, len: count%);
+event file_extraction_limit%(f: fa_file, args: any, limit: count, len: count%);

src/file_analysis/analyzer/x509/X509.cc

@@ -147,7 +147,7 @@ RecordVal* file_analysis::X509::ParseCertificate(X509Val* cert_val)
 #ifndef OPENSSL_NO_EC
 		else if ( pkey->type == EVP_PKEY_EC )
 			{
-			pX509Cert->Assign(8, new StringVal("dsa"));
+			pX509Cert->Assign(8, new StringVal("ecdsa"));
 			pX509Cert->Assign(11, KeyCurve(pkey));
 			}
 #endif

src/file_analysis/file_analysis.bif

@@ -15,6 +15,27 @@ function Files::__set_timeout_interval%(file_id: string, t: interval%): bool
 	return new Val(result, TYPE_BOOL);
 	%}
 
+## :bro:see:`Files::enable_reassembly`.
+function Files::__enable_reassembly%(file_id: string%): bool
+	%{
+	bool result = file_mgr->EnableReassembly(file_id->CheckString());
+	return new Val(result, TYPE_BOOL);
+	%}
+
+## :bro:see:`Files::disable_reassembly`.
+function Files::__disable_reassembly%(file_id: string%): bool
+	%{
+	bool result = file_mgr->DisableReassembly(file_id->CheckString());
+	return new Val(result, TYPE_BOOL);
+	%}
+
+## :bro:see:`Files::set_reassembly_buffer`.
+function Files::__set_reassembly_buffer%(file_id: string, max: count%): bool
+	%{
+	bool result = file_mgr->SetReassemblyBuffer(file_id->CheckString(), max);
+	return new Val(result, TYPE_BOOL);
+	%}
+
 ## :bro:see:`Files::add_analyzer`.
 function Files::__add_analyzer%(file_id: string, tag: Files::Tag, args: any%): bool
 	%{
@@ -26,16 +47,6 @@ function Files::__add_analyzer%(file_id: string, tag: Files::Tag, args: any%): b
 	return new Val(result, TYPE_BOOL);
 	%}
 
-## :bro:see:`Files::add_analyzers_for_mime_type`.
-function Files::__add_analyzers_for_mime_type%(file_id: string, mtype: string, args: any%): files_tag_set
-	%{
-	using BifType::Record::Files::AnalyzerArgs;
-	RecordVal* rv = args->AsRecordVal()->CoerceTo(AnalyzerArgs);
-	Val* analyzers = file_mgr->AddAnalyzersForMIMEType(file_id->CheckString(), mtype->CheckString(), rv);
-	Unref(rv);
-	return analyzers;
-	%}
-
 ## :bro:see:`Files::remove_analyzer`.
 function Files::__remove_analyzer%(file_id: string, tag: Files::Tag, args: any%): bool
 	%{
@@ -60,13 +71,6 @@ function Files::__analyzer_name%(tag: Files::Tag%) : string
 	return new StringVal(file_mgr->GetComponentName(tag));
 	%}
 
-## :bro:see:`Files::register_for_mime_type`.
-function Files::__register_for_mime_type%(id: Analyzer::Tag, mt: string%) : bool
-	%{
-	bool result = file_mgr->RegisterAnalyzerForMIMEType(id->AsEnumVal(), mt);
-	return new Val(result, TYPE_BOOL);
-	%}
-
 module GLOBAL;
 
 ## For use within a :bro:see:`get_file_handle` handler to set a unique

src/scan.l

@@ -26,6 +26,7 @@
 #include "Reporter.h"
 #include "RE.h"
 #include "Net.h"
+#include "Traverse.h"
 
 #include "analyzer/Analyzer.h"
 #include "broxygen/Manager.h"
@@ -615,11 +616,50 @@ void end_RE()
 	BEGIN(INITIAL);
 	}
 
+class LocalNameFinder : public TraversalCallback {
+public:
+	LocalNameFinder()
+		{}
+
+	virtual TraversalCode PreExpr(const Expr* expr)
+		{
+		if ( expr->Tag() != EXPR_NAME )
+			return TC_CONTINUE;
+
+		const NameExpr* name_expr = static_cast<const NameExpr*>(expr);
+
+		if ( name_expr->Id()->IsGlobal() )
+			return TC_CONTINUE;
+
+		local_names.push_back(name_expr);
+		return TC_CONTINUE;
+		}
+
+	std::vector<const NameExpr*> local_names;
+};
+
 void do_atif(Expr* expr)
 	{
 	++current_depth;
 
-	Val* val = expr->Eval(0);
+	LocalNameFinder cb;
+	expr->Traverse(&cb);
+	Val* val = 0;
+
+	if ( cb.local_names.empty() )
+		val = expr->Eval(0);
+	else
+		{
+		for ( size_t i = 0; i < cb.local_names.size(); ++i )
+			cb.local_names[i]->Error("referencing a local name in @if");
+		}
+
+	if ( ! val )
+		{
+		expr->Error("invalid expression in @if");
+		return;
+		}
+
 	if ( ! val->AsBool() )
 		{
 		if_stack.append(current_depth);