Merge remote-tracking branch 'origin/topic/johanna/imap-starttls'

BIT-1574 #merged

* origin/topic/johanna/imap-starttls:
  IMAP: add c++11 header file that gcc complains about.
  IMAP: documentation and test updates
  also generate an event when starttls is encounterd for imap.
  Add support of getting server capabilities to IMAP parser.
  Basic IMAP StartTLS analyzer.

scripts/base/init-default.bro

@@ -48,6 +48,7 @@
 @load base/protocols/dns
 @load base/protocols/ftp
 @load base/protocols/http
+@load base/protocols/imap
 @load base/protocols/irc
 @load base/protocols/krb
 @load base/protocols/modbus

scripts/base/protocols/imap/README

@@ -0,0 +1,5 @@
+Support for the Internet Message Access Protocol (IMAP).
+
+Note that currently the IMAP analyzer only supports analyzing IMAP sessions
+until they do or do not switch to TLS using StartTLS. Hence, we do not get
+mails from IMAP sessions, only X509 certificates.

scripts/base/protocols/imap/__load__.bro

@@ -0,0 +1,2 @@
+@load ./main
+

scripts/base/protocols/imap/main.bro

@@ -0,0 +1,11 @@
+
+module IMAP;
+
+const ports = { 143/tcp };
+redef likely_server_ports += { ports };
+
+event bro_init() &priority=5
+	{
+	Analyzer::register_for_ports(Analyzer::ANALYZER_IMAP, ports);
+	}
+

src/analyzer/protocol/CMakeLists.txt

@@ -16,6 +16,7 @@ add_subdirectory(gtpv1)
 add_subdirectory(http)
 add_subdirectory(icmp)
 add_subdirectory(ident)
+add_subdirectory(imap)
 add_subdirectory(interconn)
 add_subdirectory(irc)
 add_subdirectory(krb)

src/analyzer/protocol/imap/CMakeLists.txt

@@ -0,0 +1,12 @@
+
+include(BroPlugin)
+
+include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
+
+bro_plugin_begin(Bro IMAP)
+bro_plugin_cc(Plugin.cc)
+bro_plugin_cc(IMAP.cc)
+bro_plugin_bif(events.bif)
+bro_plugin_pac(imap.pac imap-analyzer.pac imap-protocol.pac)
+bro_plugin_end()
+

src/analyzer/protocol/imap/IMAP.cc

@@ -0,0 +1,85 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "IMAP.h"
+#include "analyzer/protocol/tcp/TCP_Reassembler.h"
+#include "analyzer/Manager.h"
+
+using namespace analyzer::imap;
+
+IMAP_Analyzer::IMAP_Analyzer(Connection* conn)
+	: tcp::TCP_ApplicationAnalyzer("IMAP", conn)
+	{
+	interp = new binpac::IMAP::IMAP_Conn(this);
+	had_gap = false;
+	tls_active = false;
+	}
+
+IMAP_Analyzer::~IMAP_Analyzer()
+	{
+	delete interp;
+	}
+
+void IMAP_Analyzer::Done()
+	{
+	tcp::TCP_ApplicationAnalyzer::Done();
+
+	interp->FlowEOF(true);
+	interp->FlowEOF(false);
+	}
+
+void IMAP_Analyzer::EndpointEOF(bool is_orig)
+	{
+	tcp::TCP_ApplicationAnalyzer::EndpointEOF(is_orig);
+	interp->FlowEOF(is_orig);
+	}
+
+void IMAP_Analyzer::DeliverStream(int len, const u_char* data, bool orig)
+	{
+	tcp::TCP_ApplicationAnalyzer::DeliverStream(len, data, orig);
+
+	if ( tls_active )
+		{
+		// If TLS has been initiated, forward to child and abort further
+		// processing
+		ForwardStream(len, data, orig);
+		return;
+		}
+
+	assert(TCP());
+	if ( TCP()->IsPartial() )
+		return;
+
+	if ( had_gap )
+		// If only one side had a content gap, we could still try to
+		// deliver data to the other side if the script layer can
+		// handle this.
+		return;
+
+	try
+		{
+		interp->NewData(orig, data, data + len);
+		}
+	catch ( const binpac::Exception& e )
+		{
+		ProtocolViolation(fmt("Binpac exception: %s", e.c_msg()));
+		}
+	}
+
+void IMAP_Analyzer::Undelivered(uint64 seq, int len, bool orig)
+	{
+	tcp::TCP_ApplicationAnalyzer::Undelivered(seq, len, orig);
+	had_gap = true;
+	interp->NewGap(orig, len);
+	}
+
+void IMAP_Analyzer::StartTLS()
+	{
+	// StartTLS was called. This means we saw a client starttls followed
+	// by a server proceed. From here on, everything should be a binary
+	// TLS datastream.
+	tls_active = true;
+
+	Analyzer* ssl = analyzer_mgr->InstantiateAnalyzer("SSL", Conn());
+	if ( ssl )
+		AddChildAnalyzer(ssl);
+	}

src/analyzer/protocol/imap/IMAP.h

@@ -0,0 +1,40 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef ANALYZER_PROTOCOL_IMAP_IMAP_H
+#define ANALYZER_PROTOCOL_IMAP_IMAP_H
+
+// for std::transform
+#include <algorithm>
+#include "analyzer/protocol/tcp/TCP.h"
+
+#include "imap_pac.h"
+
+namespace analyzer { namespace imap {
+
+class IMAP_Analyzer : public tcp::TCP_ApplicationAnalyzer {
+public:
+	IMAP_Analyzer(Connection* conn);
+	virtual ~IMAP_Analyzer();
+
+	virtual void Done();
+	virtual void DeliverStream(int len, const u_char* data, bool orig);
+	virtual void Undelivered(uint64 seq, int len, bool orig);
+
+	// Overriden from tcp::TCP_ApplicationAnalyzer.
+	virtual void EndpointEOF(bool is_orig);
+
+	void StartTLS();
+
+	static analyzer::Analyzer* Instantiate(Connection* conn)
+		{ return new IMAP_Analyzer(conn); }
+
+protected:
+	binpac::IMAP::IMAP_Conn* interp;
+	bool had_gap;
+
+	bool tls_active;
+};
+
+} } // namespace analyzer::*
+
+#endif /* ANALYZER_PROTOCOL_IMAP_IMAP_H */

src/analyzer/protocol/imap/Plugin.cc

@@ -0,0 +1,22 @@
+// See the file  in the main distribution directory for copyright.
+#include "plugin/Plugin.h"
+#include "IMAP.h"
+
+namespace plugin {
+namespace Bro_IMAP {
+
+class Plugin : public plugin::Plugin {
+public:
+	plugin::Configuration Configure()
+		{
+		AddComponent(new ::analyzer::Component("IMAP", ::analyzer::imap::IMAP_Analyzer::Instantiate));
+
+		plugin::Configuration config;
+		config.name = "Bro::IMAP";
+		config.description = "IMAP analyzer (StartTLS only)";
+		return config;
+		}
+} plugin;
+
+}
+}

src/analyzer/protocol/imap/events.bif

@@ -0,0 +1,13 @@
+## Generated when a server sends a capability list to the client,
+## after being queried using the CAPABILITY command.
+##
+## c: The connection.
+##
+## capabilities: The list of IMAP capabilities as sent by the server.
+event imap_capabilities%(c: connection, capabilities: string_vec%);
+
+## Generated when a IMAP connection goes encrypted after a successful
+## StartTLS exchange between the client and the server.
+##
+## c: The connection.
+event imap_starttls%(c: connection%);

src/analyzer/protocol/imap/imap-analyzer.pac

@@ -0,0 +1,76 @@
+refine connection IMAP_Conn += {
+
+	%member{
+		string client_starttls_id;
+	%}
+
+	%init{
+	%}
+
+	function proc_imap_token(is_orig: bool, tag: bytestring, command: bytestring): bool
+		%{
+		string commands = std_str(command);
+		std::transform(commands.begin(), commands.end(), commands.begin(), ::tolower);
+
+		string tags = std_str(tag);
+
+		//printf("imap %s %s\n", commands.c_str(), tags.c_str());
+
+		if ( !is_orig && tags == "*" && commands == "ok" )
+			bro_analyzer()->ProtocolConfirmation();
+
+		if ( is_orig && ( command == "capability" || commands == "starttls" ) )
+			bro_analyzer()->ProtocolConfirmation();
+
+		if ( command == "authenticate" || command == "login" || command == "examine" || command == "create" || command == "list" || command == "fetch" )
+			{
+			bro_analyzer()->ProtocolConfirmation();
+			// Handshake has passed the phase where we should see StartTLS. Simply skip from hereon...
+			bro_analyzer()->SetSkip(true);
+			return true;
+			}
+
+		if ( is_orig && commands == "starttls" )
+			{
+			if ( !client_starttls_id.empty() )
+				reporter->Weird(bro_analyzer()->Conn(), "IMAP: client sent duplicate StartTLS");
+
+			client_starttls_id = tags;
+			}
+
+		if ( !is_orig && !client_starttls_id.empty() && tags == client_starttls_id )
+			{
+			if ( commands == "ok" )
+				{
+				bro_analyzer()->StartTLS();
+				BifEvent::generate_imap_starttls(bro_analyzer(), bro_analyzer()->Conn());
+				}
+			else
+				reporter->Weird(bro_analyzer()->Conn(), "IMAP: server refused StartTLS");
+			}
+
+		return true;
+		%}
+
+	function proc_server_capability(capabilities: Capability[]): bool
+		%{
+		VectorVal* capv = new VectorVal(internal_type("string_vec")->AsVectorType());
+		for ( unsigned int i = 0; i< capabilities->size(); i++ )
+			{
+			const bytestring& capability = (*capabilities)[i]->cap();
+			capv->Assign(i, new StringVal(capability.length(), (const char*)capability.data()));
+			}
+
+		BifEvent::generate_imap_capabilities(bro_analyzer(), bro_analyzer()->Conn(), capv);
+		return true;
+		%}
+
+};
+
+refine typeattr ImapToken += &let {
+	proc: bool = $context.connection.proc_imap_token(is_orig, tag, command);
+};
+
+refine typeattr ServerCapability += &let {
+	proc: bool = $context.connection.proc_server_capability(capabilities);
+};

src/analyzer/protocol/imap/imap-protocol.pac

@@ -0,0 +1,70 @@
+# commands that we support parsing. The numbers do not really mean anything
+# in this case
+enum ImapCommand {
+	CMD_CAPABILITY,
+	CMD_UNKNOWN
+}
+
+type TAG = RE/[[:alnum:][:punct:]]+/;
+type CONTENT = RE/[^\r\n]*/;
+type SPACING = RE/[ ]+/;
+type OPTIONALSPACING = RE/[ ]*/;
+type NEWLINE = RE/[\r\n]+/;
+type OPTIONALNEWLINE = RE/[\r\n]*/;
+
+type IMAP_PDU(is_orig: bool) = ImapToken(is_orig)[] &until($input.length() == 0);
+
+type ImapToken(is_orig: bool) = record {
+	tag : TAG;
+	: SPACING;
+	command: TAG;
+	: OPTIONALSPACING;
+	client_or_server: case is_orig of {
+		true -> client: UnknownCommand(this) ;
+		false -> server: ServerContentText(this);
+	} &requires(pcommand) ;
+} &let {
+	pcommand: int = $context.connection.determine_command(is_orig, tag, command);
+};
+
+type ServerContentText(rec: ImapToken) = case rec.pcommand of {
+	CMD_CAPABILITY -> capability: ServerCapability(rec);
+	default -> unknown: UnknownCommand(rec);
+};
+
+type Capability = record {
+	cap: TAG;
+	: OPTIONALSPACING;
+	nl: OPTIONALNEWLINE;
+};
+
+type ServerCapability(rec: ImapToken) = record {
+	capabilities: Capability[] &until($context.connection.strlen($element.nl) > 0);
+};
+
+type UnknownCommand(rec: ImapToken) = record {
+	tagcontent: CONTENT;
+	: NEWLINE;
+};
+
+refine connection IMAP_Conn += {
+
+	function determine_command(is_orig: bool, tag: bytestring, command: bytestring): int
+		%{
+		string cmdstr = std_str(command);
+		std::transform(cmdstr.begin(), cmdstr.end(), cmdstr.begin(), ::tolower);
+		string tagstr = std_str(tag);
+
+		if ( !is_orig && cmdstr == "capability" && tag == "*" ) {
+			return CMD_CAPABILITY;
+		}
+
+		return CMD_UNKNOWN;
+		%}
+
+	function strlen(str: bytestring): int
+		%{
+		return str.length();
+		%}
+
+};

src/analyzer/protocol/imap/imap.pac

@@ -0,0 +1,37 @@
+# binpac file for the IMAP analyzer.
+# Note that we currently do not even try to parse the protocol
+# completely -- this is only supposed to be able to parse imap
+# till StartTLS does (or does not) kick in.
+
+%include binpac.pac
+%include bro.pac
+
+%extern{
+#include "events.bif.h"
+
+namespace analyzer { namespace imap { class IMAP_Analyzer; } }
+namespace binpac { namespace IMAP { class IMAP_Conn; } }
+typedef analyzer::imap::IMAP_Analyzer* IMAPAnalyzer;
+
+#include "IMAP.h"
+%}
+
+extern type IMAPAnalyzer;
+
+analyzer IMAP withcontext {
+	connection:	 IMAP_Conn;
+	flow:		 IMAP_Flow;
+};
+
+connection IMAP_Conn(bro_analyzer: IMAPAnalyzer) {
+	upflow = IMAP_Flow(true);
+	downflow = IMAP_Flow(false);
+};
+
+%include imap-protocol.pac
+
+flow IMAP_Flow(is_orig: bool) {
+	datagram = IMAP_PDU(is_orig) withcontext(connection, this);
+};
+
+%include imap-analyzer.pac

testing/btest/Baseline/core.print-bpf-filters/output2

@@ -1,5 +1,6 @@
 2 1080
 1 137
+1 143
 1 1434
 1 161
 1 162
@@ -47,8 +48,8 @@
 1 992
 1 993
 1 995
-54 and
+55 and
-53 or
+54 or
-54 port
+55 port
-36 tcp
+37 tcp
 18 udp

testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2016-04-22-23-21-01
+#open	2016-04-26-18-11-39
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -76,6 +76,7 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/plugins/Bro_HTTP.functions.bif.bro
     build/scripts/base/bif/plugins/Bro_ICMP.events.bif.bro
     build/scripts/base/bif/plugins/Bro_Ident.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_IMAP.events.bif.bro
     build/scripts/base/bif/plugins/Bro_InterConn.events.bif.bro
     build/scripts/base/bif/plugins/Bro_IRC.events.bif.bro
     build/scripts/base/bif/plugins/Bro_KRB.events.bif.bro
@@ -131,4 +132,4 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro
 scripts/policy/misc/loaded-scripts.bro
   scripts/base/utils/paths.bro
-#close	2016-04-22-23-21-01
+#close	2016-04-26-18-11-39

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2016-04-22-23-21-18
+#open	2016-04-26-18-11-49
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -76,6 +76,7 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/plugins/Bro_HTTP.functions.bif.bro
     build/scripts/base/bif/plugins/Bro_ICMP.events.bif.bro
     build/scripts/base/bif/plugins/Bro_Ident.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_IMAP.events.bif.bro
     build/scripts/base/bif/plugins/Bro_InterConn.events.bif.bro
     build/scripts/base/bif/plugins/Bro_IRC.events.bif.bro
     build/scripts/base/bif/plugins/Bro_KRB.events.bif.bro
@@ -252,6 +253,8 @@ scripts/base/init-default.bro
     scripts/base/protocols/http/entities.bro
     scripts/base/protocols/http/utils.bro
     scripts/base/protocols/http/files.bro
+  scripts/base/protocols/imap/__load__.bro
+    scripts/base/protocols/imap/main.bro
   scripts/base/protocols/irc/__load__.bro
     scripts/base/protocols/irc/main.bro
     scripts/base/protocols/irc/dcc-send.bro
@@ -302,4 +305,4 @@ scripts/base/init-default.bro
   scripts/base/misc/find-checksum-offloading.bro
   scripts/base/misc/find-filtered-trace.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2016-04-22-23-21-18
+#close	2016-04-26-18-11-49

testing/btest/Baseline/plugins.hooks/output

@@ -25,6 +25,7 @@
 0.000000   MetaHookPost  CallFunction(Analyzer::__register_for_port, <frame>, (Analyzer::ANALYZER_HTTP, 8080/tcp)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Analyzer::__register_for_port, <frame>, (Analyzer::ANALYZER_HTTP, 81/tcp)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Analyzer::__register_for_port, <frame>, (Analyzer::ANALYZER_HTTP, 8888/tcp)) -> <no result>
+0.000000   MetaHookPost  CallFunction(Analyzer::__register_for_port, <frame>, (Analyzer::ANALYZER_IMAP, 143/tcp)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Analyzer::__register_for_port, <frame>, (Analyzer::ANALYZER_IRC, 6666/tcp)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Analyzer::__register_for_port, <frame>, (Analyzer::ANALYZER_IRC, 6667/tcp)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Analyzer::__register_for_port, <frame>, (Analyzer::ANALYZER_IRC, 6668/tcp)) -> <no result>
@@ -83,6 +84,7 @@
 0.000000   MetaHookPost  CallFunction(Analyzer::register_for_port, <frame>, (Analyzer::ANALYZER_HTTP, 8080/tcp)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Analyzer::register_for_port, <frame>, (Analyzer::ANALYZER_HTTP, 81/tcp)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Analyzer::register_for_port, <frame>, (Analyzer::ANALYZER_HTTP, 8888/tcp)) -> <no result>
+0.000000   MetaHookPost  CallFunction(Analyzer::register_for_port, <frame>, (Analyzer::ANALYZER_IMAP, 143/tcp)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Analyzer::register_for_port, <frame>, (Analyzer::ANALYZER_IRC, 6666/tcp)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Analyzer::register_for_port, <frame>, (Analyzer::ANALYZER_IRC, 6667/tcp)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Analyzer::register_for_port, <frame>, (Analyzer::ANALYZER_IRC, 6668/tcp)) -> <no result>
@@ -122,6 +124,7 @@
 0.000000   MetaHookPost  CallFunction(Analyzer::register_for_ports, <frame>, (Analyzer::ANALYZER_FTP, {2811<...>/tcp})) -> <no result>
 0.000000   MetaHookPost  CallFunction(Analyzer::register_for_ports, <frame>, (Analyzer::ANALYZER_GTPV1, {2152<...>/udp})) -> <no result>
 0.000000   MetaHookPost  CallFunction(Analyzer::register_for_ports, <frame>, (Analyzer::ANALYZER_HTTP, {631<...>/tcp})) -> <no result>
+0.000000   MetaHookPost  CallFunction(Analyzer::register_for_ports, <frame>, (Analyzer::ANALYZER_IMAP, {143/tcp})) -> <no result>
 0.000000   MetaHookPost  CallFunction(Analyzer::register_for_ports, <frame>, (Analyzer::ANALYZER_IRC, {6669<...>/tcp})) -> <no result>
 0.000000   MetaHookPost  CallFunction(Analyzer::register_for_ports, <frame>, (Analyzer::ANALYZER_KRB, {88/udp})) -> <no result>
 0.000000   MetaHookPost  CallFunction(Analyzer::register_for_ports, <frame>, (Analyzer::ANALYZER_KRB_TCP, {88/tcp})) -> <no result>
@@ -230,7 +233,7 @@
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])) -> <no result>
-0.000000   MetaHookPost  CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1461367323.154279, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
+0.000000   MetaHookPost  CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1461694342.200388, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, <frame>, (Cluster::LOG)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, <frame>, (Communication::LOG)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, <frame>, (Conn::LOG)) -> <no result>
@@ -351,7 +354,7 @@
 0.000000   MetaHookPost  CallFunction(Log::create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])) -> <no result>
-0.000000   MetaHookPost  CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1461367323.154279, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
+0.000000   MetaHookPost  CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1461694342.200388, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
 0.000000   MetaHookPost  CallFunction(NetControl::check_plugins, <frame>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(NetControl::init, <null>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(Notice::want_pp, <frame>, ()) -> <no result>
@@ -416,6 +419,7 @@
 0.000000   MetaHookPost  LoadFile(./Bro_HTTP.events.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_HTTP.functions.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_ICMP.events.bif.bro) -> -1
+0.000000   MetaHookPost  LoadFile(./Bro_IMAP.events.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_IRC.events.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_Ident.events.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./Bro_InterConn.events.bif.bro) -> -1
@@ -587,6 +591,7 @@
 0.000000   MetaHookPost  LoadFile(base<...>/ftp) -> -1
 0.000000   MetaHookPost  LoadFile(base<...>/hash) -> -1
 0.000000   MetaHookPost  LoadFile(base<...>/http) -> -1
+0.000000   MetaHookPost  LoadFile(base<...>/imap) -> -1
 0.000000   MetaHookPost  LoadFile(base<...>/input) -> -1
 0.000000   MetaHookPost  LoadFile(base<...>/input.bif) -> -1
 0.000000   MetaHookPost  LoadFile(base<...>/intel) -> -1
@@ -665,6 +670,7 @@
 0.000000   MetaHookPre   CallFunction(Analyzer::__register_for_port, <frame>, (Analyzer::ANALYZER_HTTP, 8080/tcp))
 0.000000   MetaHookPre   CallFunction(Analyzer::__register_for_port, <frame>, (Analyzer::ANALYZER_HTTP, 81/tcp))
 0.000000   MetaHookPre   CallFunction(Analyzer::__register_for_port, <frame>, (Analyzer::ANALYZER_HTTP, 8888/tcp))
+0.000000   MetaHookPre   CallFunction(Analyzer::__register_for_port, <frame>, (Analyzer::ANALYZER_IMAP, 143/tcp))
 0.000000   MetaHookPre   CallFunction(Analyzer::__register_for_port, <frame>, (Analyzer::ANALYZER_IRC, 6666/tcp))
 0.000000   MetaHookPre   CallFunction(Analyzer::__register_for_port, <frame>, (Analyzer::ANALYZER_IRC, 6667/tcp))
 0.000000   MetaHookPre   CallFunction(Analyzer::__register_for_port, <frame>, (Analyzer::ANALYZER_IRC, 6668/tcp))
@@ -723,6 +729,7 @@
 0.000000   MetaHookPre   CallFunction(Analyzer::register_for_port, <frame>, (Analyzer::ANALYZER_HTTP, 8080/tcp))
 0.000000   MetaHookPre   CallFunction(Analyzer::register_for_port, <frame>, (Analyzer::ANALYZER_HTTP, 81/tcp))
 0.000000   MetaHookPre   CallFunction(Analyzer::register_for_port, <frame>, (Analyzer::ANALYZER_HTTP, 8888/tcp))
+0.000000   MetaHookPre   CallFunction(Analyzer::register_for_port, <frame>, (Analyzer::ANALYZER_IMAP, 143/tcp))
 0.000000   MetaHookPre   CallFunction(Analyzer::register_for_port, <frame>, (Analyzer::ANALYZER_IRC, 6666/tcp))
 0.000000   MetaHookPre   CallFunction(Analyzer::register_for_port, <frame>, (Analyzer::ANALYZER_IRC, 6667/tcp))
 0.000000   MetaHookPre   CallFunction(Analyzer::register_for_port, <frame>, (Analyzer::ANALYZER_IRC, 6668/tcp))
@@ -762,6 +769,7 @@
 0.000000   MetaHookPre   CallFunction(Analyzer::register_for_ports, <frame>, (Analyzer::ANALYZER_FTP, {2811<...>/tcp}))
 0.000000   MetaHookPre   CallFunction(Analyzer::register_for_ports, <frame>, (Analyzer::ANALYZER_GTPV1, {2152<...>/udp}))
 0.000000   MetaHookPre   CallFunction(Analyzer::register_for_ports, <frame>, (Analyzer::ANALYZER_HTTP, {631<...>/tcp}))
+0.000000   MetaHookPre   CallFunction(Analyzer::register_for_ports, <frame>, (Analyzer::ANALYZER_IMAP, {143/tcp}))
 0.000000   MetaHookPre   CallFunction(Analyzer::register_for_ports, <frame>, (Analyzer::ANALYZER_IRC, {6669<...>/tcp}))
 0.000000   MetaHookPre   CallFunction(Analyzer::register_for_ports, <frame>, (Analyzer::ANALYZER_KRB, {88/udp}))
 0.000000   MetaHookPre   CallFunction(Analyzer::register_for_ports, <frame>, (Analyzer::ANALYZER_KRB_TCP, {88/tcp}))
@@ -870,7 +878,7 @@
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird]))
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509]))
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql]))
-0.000000   MetaHookPre   CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1461367323.154279, node=bro, filter=ip or not ip, init=T, success=T]))
+0.000000   MetaHookPre   CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1461694342.200388, node=bro, filter=ip or not ip, init=T, success=T]))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, <frame>, (Cluster::LOG))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, <frame>, (Communication::LOG))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, <frame>, (Conn::LOG))
@@ -991,7 +999,7 @@
 0.000000   MetaHookPre   CallFunction(Log::create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird]))
 0.000000   MetaHookPre   CallFunction(Log::create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509]))
 0.000000   MetaHookPre   CallFunction(Log::create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql]))
-0.000000   MetaHookPre   CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1461367323.154279, node=bro, filter=ip or not ip, init=T, success=T]))
+0.000000   MetaHookPre   CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1461694342.200388, node=bro, filter=ip or not ip, init=T, success=T]))
 0.000000   MetaHookPre   CallFunction(NetControl::check_plugins, <frame>, ())
 0.000000   MetaHookPre   CallFunction(NetControl::init, <null>, ())
 0.000000   MetaHookPre   CallFunction(Notice::want_pp, <frame>, ())
@@ -1056,6 +1064,7 @@
 0.000000   MetaHookPre   LoadFile(./Bro_HTTP.events.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_HTTP.functions.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_ICMP.events.bif.bro)
+0.000000   MetaHookPre   LoadFile(./Bro_IMAP.events.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_IRC.events.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_Ident.events.bif.bro)
 0.000000   MetaHookPre   LoadFile(./Bro_InterConn.events.bif.bro)
@@ -1227,6 +1236,7 @@
 0.000000   MetaHookPre   LoadFile(base<...>/ftp)
 0.000000   MetaHookPre   LoadFile(base<...>/hash)
 0.000000   MetaHookPre   LoadFile(base<...>/http)
+0.000000   MetaHookPre   LoadFile(base<...>/imap)
 0.000000   MetaHookPre   LoadFile(base<...>/input)
 0.000000   MetaHookPre   LoadFile(base<...>/input.bif)
 0.000000   MetaHookPre   LoadFile(base<...>/intel)
@@ -1305,6 +1315,7 @@
 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_HTTP, 8080/tcp)
 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_HTTP, 81/tcp)
 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_HTTP, 8888/tcp)
+0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_IMAP, 143/tcp)
 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_IRC, 6666/tcp)
 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_IRC, 6667/tcp)
 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_IRC, 6668/tcp)
@@ -1363,6 +1374,7 @@
 0.000000 | HookCallFunction Analyzer::register_for_port(Analyzer::ANALYZER_HTTP, 8080/tcp)
 0.000000 | HookCallFunction Analyzer::register_for_port(Analyzer::ANALYZER_HTTP, 81/tcp)
 0.000000 | HookCallFunction Analyzer::register_for_port(Analyzer::ANALYZER_HTTP, 8888/tcp)
+0.000000 | HookCallFunction Analyzer::register_for_port(Analyzer::ANALYZER_IMAP, 143/tcp)
 0.000000 | HookCallFunction Analyzer::register_for_port(Analyzer::ANALYZER_IRC, 6666/tcp)
 0.000000 | HookCallFunction Analyzer::register_for_port(Analyzer::ANALYZER_IRC, 6667/tcp)
 0.000000 | HookCallFunction Analyzer::register_for_port(Analyzer::ANALYZER_IRC, 6668/tcp)
@@ -1402,6 +1414,7 @@
 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_FTP, {2811<...>/tcp})
 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_GTPV1, {2152<...>/udp})
 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_HTTP, {631<...>/tcp})
+0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_IMAP, {143/tcp})
 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_IRC, {6669<...>/tcp})
 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_KRB, {88/udp})
 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_KRB_TCP, {88/tcp})
@@ -1509,7 +1522,7 @@
 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])
 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])
 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])
-0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1461367323.154279, node=bro, filter=ip or not ip, init=T, success=T])
+0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1461694342.200388, node=bro, filter=ip or not ip, init=T, success=T])
 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG)
 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG)
 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG)
@@ -1630,7 +1643,7 @@
 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])
 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])
 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])
-0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1461367323.154279, node=bro, filter=ip or not ip, init=T, success=T])
+0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1461694342.200388, node=bro, filter=ip or not ip, init=T, success=T])
 0.000000 | HookCallFunction NetControl::check_plugins()
 0.000000 | HookCallFunction NetControl::init()
 0.000000 | HookCallFunction Notice::want_pp()

testing/btest/Baseline/scripts.base.protocols.imap.capabilities/.stdout

@@ -0,0 +1 @@
+[IMAP4rev1, CHILDREN, ENABLE, ID, IDLE, LIST-EXTENDED, LIST-STATUS, LITERAL+, MOVE, NAMESPACE, SASL-IR, SORT, SPECIAL-USE, THREAD=ORDEREDSUBJECT, UIDPLUS, UNSELECT, WITHIN, STARTTLS, AUTH=LOGIN, AUTH=PLAIN]

testing/btest/Baseline/scripts.base.protocols.imap.starttls/.stdout

@@ -0,0 +1 @@
+Tls started for connection

testing/btest/Baseline/scripts.base.protocols.imap.starttls/conn.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	conn
+#open	2015-07-22-17-31-02
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	local_resp	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	bool	count	string	count	count	count	count	set[string]
+1437584567.812552	CXWv6p3arKYeMETxOg	192.168.17.53	49640	212.227.17.186	143	tcp	ssl,imap	2.827002	540	5653	SF	-	-	0	ShAdDafFr	18	1284	14	6225	(empty)
+#close	2015-07-22-17-31-02

testing/btest/Baseline/scripts.base.protocols.imap.starttls/ssl.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	ssl
+#open	2015-07-22-17-31-02
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	curve	server_name	resumed	last_alert	next_protocol	established	cert_chain_fuids	client_cert_chain_fuids	subject	issuer	client_subject	client_issuer
+#types	time	string	addr	port	addr	port	string	string	string	string	bool	string	string	bool	vector[string]	vector[string]	string	string	string	string
+1437584568.570497	CXWv6p3arKYeMETxOg	192.168.17.53	49640	212.227.17.186	143	TLSv12	TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384	secp256r1	-	F	-	-	T	FOWmhO3rUj3SEB5RTb,FjH9n52SzEIJ9UoVK9,FisDHa396LIaZadgG9	(empty)	CN=imap.gmx.net,emailAddress=server-certs@1und1.de,L=Montabaur,ST=Rhineland-Palatinate,O=1&1 Mail & Media GmbH,C=DE	CN=TeleSec ServerPass DE-1,street=Untere Industriestr. 20,L=Netphen,postalCode=57250,ST=NRW,OU=T-Systems Trust Center,O=T-Systems International GmbH,C=DE	-	-
+#close	2015-07-22-17-31-02

testing/btest/Baseline/scripts.base.protocols.imap.starttls/x509.log

@@ -0,0 +1,12 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	x509
+#open	2015-07-22-17-31-02
+#fields	ts	id	certificate.version	certificate.serial	certificate.subject	certificate.issuer	certificate.not_valid_before	certificate.not_valid_after	certificate.key_alg	certificate.sig_alg	certificate.key_type	certificate.key_length	certificate.exponent	certificate.curve	san.dns	san.uri	san.email	san.ip	basic_constraints.ca	basic_constraints.path_len
+#types	time	string	count	string	string	string	time	time	string	string	string	count	string	string	vector[string]	vector[string]	vector[string]	vector[addr]	bool	count
+1437584568.769690	FOWmhO3rUj3SEB5RTb	3	339D9ED8E73927C9	CN=imap.gmx.net,emailAddress=server-certs@1und1.de,L=Montabaur,ST=Rhineland-Palatinate,O=1&1 Mail & Media GmbH,C=DE	CN=TeleSec ServerPass DE-1,street=Untere Industriestr. 20,L=Netphen,postalCode=57250,ST=NRW,OU=T-Systems Trust Center,O=T-Systems International GmbH,C=DE	1384251451.000000	1479427199.000000	rsaEncryption	sha1WithRSAEncryption	rsa	2048	65537	-	imap.gmx.net,imap.gmx.de	-	-	-	F	-
+1437584568.769690	FjH9n52SzEIJ9UoVK9	3	21B6777E8CBD0EA8	CN=TeleSec ServerPass DE-1,street=Untere Industriestr. 20,L=Netphen,postalCode=57250,ST=NRW,OU=T-Systems Trust Center,O=T-Systems International GmbH,C=DE	CN=Deutsche Telekom Root CA 2,OU=T-TeleSec Trust Center,O=Deutsche Telekom AG,C=DE	1362146309.000000	1562716740.000000	rsaEncryption	sha1WithRSAEncryption	rsa	2048	65537	-	-	-	-	-	T	0
+1437584568.769690	FisDHa396LIaZadgG9	3	26	CN=Deutsche Telekom Root CA 2,OU=T-TeleSec Trust Center,O=Deutsche Telekom AG,C=DE	CN=Deutsche Telekom Root CA 2,OU=T-TeleSec Trust Center,O=Deutsche Telekom AG,C=DE	931522260.000000	1562716740.000000	rsaEncryption	sha1WithRSAEncryption	rsa	2048	65537	-	-	-	-	-	T	5
+#close	2015-07-22-17-31-02

testing/btest/scripts/base/protocols/imap/capabilities.test

@@ -0,0 +1,12 @@
+# @TEST-EXEC: bro -b -C -r $TRACES/tls/imap-starttls.pcap %INPUT
+# @TEST-EXEC: btest-diff .stdout
+
+@load base/protocols/ssl
+@load base/protocols/conn
+@load base/frameworks/dpd
+@load base/protocols/imap
+
+event imap_capabilities(c: connection, capabilities: string_vec)
+	{
+	print capabilities;
+	}

testing/btest/scripts/base/protocols/imap/starttls.test

@@ -0,0 +1,15 @@
+# @TEST-EXEC: bro -b -C -r $TRACES/tls/imap-starttls.pcap %INPUT
+# @TEST-EXEC: btest-diff conn.log
+# @TEST-EXEC: btest-diff ssl.log
+# @TEST-EXEC: btest-diff x509.log
+# @TEST-EXEC: btest-diff .stdout
+
+@load base/protocols/ssl
+@load base/protocols/conn
+@load base/frameworks/dpd
+@load base/protocols/imap
+
+event imap_starttls(c: connection)
+	{
+	print "Tls started for connection";
+	}