Merge remote-tracking branch 'origin/topic/johanna/intel-uid-fuid'

BIT-1572 #merged

* origin/topic/johanna/intel-uid-fuid:
  Intel: Allow to provide uid/fuid instead of conn/f.

CHANGES

@@ -1,4 +1,23 @@
 
+2.4-485 | 2016-04-28 10:18:46 -0700
+
+  * Intel: Allow to provide uid/fuid instead of conn/file. (Johanna
+    Amann)
+
+  * Provide file IDs for hostname matches in certificates. (Johanna
+    Amann)
+
+  * Rudimentary IMAP StartTLS analyzer. It parses certificates out of
+    IMAP connections using StartTLS. It aborts processing if StartTLS
+    is not found. (Johanna Amann)
+
+  * ARP: remove unnecessary variables and add testcase. (Johanna
+    Amann)
+
+  * Fix parsing of x509 pre-y2k dates. (Johanna Amann)
+
+  * Fix small error in bif documentation. (Johanna Amann)
+
 2.4-471 | 2016-04-25 15:37:15 -0700
 
   * Add DNS tests for huge TLLs and CAA. (Johanna Amann)

NEWS

@@ -33,6 +33,11 @@ New Functionality
 
 - Bro now supports the Radiotap header for 802.11 frames.
 
+- Bro now has a rudimentary IMAP analyzer examinig the initial phase
+  of the protocol. Right now the analyzer only identify STARTTLS
+  sessions, handing them over to TLS analysis. The analyzer does not
+  yet analyze any further IMAP content.
+
 - Bro now tracks VLAN IDs. To record them inside the connection log,
   load protocols/conn/vlan-logging.bro.
 

VERSION

@@ -1 +1 @@
-2.4-471
+2.4-485

scripts/base/frameworks/intel/main.bro

@@ -77,23 +77,34 @@ export {
 		## The type of data that the indicator represents.
 		indicator_type:  Type          &log &optional;
 
-		## If the indicator type was :bro:enum:`Intel::ADDR`, then this 
+		## If the indicator type was :bro:enum:`Intel::ADDR`, then this
 		## field will be present.
 		host:            addr          &optional;
 
 		## Where the data was discovered.
 		where:           Where         &log;
-		
+
 		## The name of the node where the match was discovered.
 		node:            string        &optional &log;
 
-		## If the data was discovered within a connection, the 
+		## If the data was discovered within a connection, the
 		## connection record should go here to give context to the data.
 		conn:            connection    &optional;
 
+		## If the data was discovered within a connection, the
+		## connection uid should go here to give context to the data.
+		## If the *conn* field is provided, this will be automatically
+		## filled out.
+		uid:             string        &optional;
+
 		## If the data was discovered within a file, the file record
 		## should go here to provide context to the data.
 		f:               fa_file       &optional;
+
+		## If the data was discovered within a file, the file uid should
+		## go here to provide context to the data. If the *f* field is
+		## provided, this will be automatically filled out.
+		fuid:            string        &optional;
 	};
 
 	## Record used for the logging framework representing a positive
@@ -112,7 +123,8 @@ export {
 		## If a file was associated with this intelligence hit,
 		## this is the uid for the file.
 		fuid:           string   &log &optional;
-		## A mime type if the intelligence hit is related to a file.  
+
+		## A mime type if the intelligence hit is related to a file.
 		## If the $f field is provided this will be automatically filled
 		## out.
 		file_mime_type: string   &log &optional;
@@ -283,15 +295,14 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5
 
 	if ( s?$f )
 		{
+		s$fuid = s$f$id;
+
 		if ( s$f?$conns && |s$f$conns| == 1 )
 			{
 			for ( cid in s$f$conns )
 				s$conn = s$f$conns[cid];
 			}
 
-		if ( ! info?$fuid )
-			info$fuid = s$f$id;
-
 		if ( ! info?$file_mime_type && s$f?$info && s$f$info?$mime_type )
 			info$file_mime_type = s$f$info$mime_type;
 
@@ -299,12 +310,18 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5
 			info$file_desc = Files::describe(s$f);
 		}
 
+	if ( s?$fuid )
+		info$fuid = s$fuid;
+
 	if ( s?$conn )
 		{
-		info$uid = s$conn$uid;
+		s$uid = s$conn$uid;
 		info$id  = s$conn$id;
 		}
 
+	if ( s?$uid )
+		info$uid = s$uid;
+
 	for ( item in items )
 		add info$sources[item$meta$source];
 

scripts/policy/frameworks/intel/seen/ssl.bro

@@ -20,6 +20,7 @@ event ssl_established(c: connection)
 	if ( c$ssl$cert_chain[0]$x509?$certificate && c$ssl$cert_chain[0]$x509$certificate?$cn )
 		Intel::seen([$indicator=c$ssl$cert_chain[0]$x509$certificate$cn,
 			$indicator_type=Intel::DOMAIN,
+			$fuid=c$ssl$cert_chain_fuids[0],
 			$conn=c,
 			$where=X509::IN_CERT]);
 	}

testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log

@@ -3,23 +3,23 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-04-11-13-48-49
+#open	2016-04-25-23-53-37
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	sources
 #types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[string]
 1416942644.593119	CXWv6p3arKYeMETxOg	192.168.4.149	49422	23.92.19.75	443	F0txuw2pvrkZOn04a8	application/pkix-cert	23.92.19.75:443/tcp	www.pantz.org	Intel::DOMAIN	X509::IN_CERT	bro	source1
-#close	2016-04-11-13-48-49
+#close	2016-04-25-23-53-37
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-04-11-13-48-49
+#open	2016-04-25-23-53-38
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	sources
 #types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[string]
 1170717505.735416	CXWv6p3arKYeMETxOg	192.150.187.164	58868	194.127.84.106	443	FeCwNK3rzqPnZ7eBQ5	application/pkix-cert	194.127.84.106:443/tcp	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	source1
-1170717505.934612	CXWv6p3arKYeMETxOg	192.150.187.164	58868	194.127.84.106	443	-	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	source1
+1170717505.934612	CXWv6p3arKYeMETxOg	192.150.187.164	58868	194.127.84.106	443	FeCwNK3rzqPnZ7eBQ5	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	source1
 1170717508.883051	CjhGID4nQcgTWjvg4c	192.150.187.164	58869	194.127.84.106	443	FjkLnG4s34DVZlaBNc	application/pkix-cert	194.127.84.106:443/tcp	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	source1
-1170717509.082241	CjhGID4nQcgTWjvg4c	192.150.187.164	58869	194.127.84.106	443	-	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	source1
+1170717509.082241	CjhGID4nQcgTWjvg4c	192.150.187.164	58869	194.127.84.106	443	FjkLnG4s34DVZlaBNc	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	source1
 1170717511.909717	CCvvfg3TEfuqmmG4bh	192.150.187.164	58870	194.127.84.106	443	FQXAWgI2FB5STbrff	application/pkix-cert	194.127.84.106:443/tcp	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	source1
-1170717512.108799	CCvvfg3TEfuqmmG4bh	192.150.187.164	58870	194.127.84.106	443	-	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	source1
+1170717512.108799	CCvvfg3TEfuqmmG4bh	192.150.187.164	58870	194.127.84.106	443	FQXAWgI2FB5STbrff	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	source1
-#close	2016-04-11-13-48-49
+#close	2016-04-25-23-53-38