Merge branch 'topic/seth/rdp' of ssh://git.bro.org/bro into topic/seth/rdp

CHANGES

@@ -1,4 +1,17 @@
 
+2.3-518 | 2015-03-04 13:13:50 -0800
+
+  * Add bytes_recvd to stats.log recording the number of bytes
+    received, according to packet headers. (Mike Smiley)
+
+2.3-516 | 2015-03-04 12:30:06 -0800
+
+  * Extract most specific Common Name from SSL certificates (Johanna
+    Amann)
+
+  * Send CN and SAN fields of SSL certificates to the Intel framework.
+    (Johanna Amann)
+
 2.3-511 | 2015-03-02 18:07:17 -0800
 
   * Changes to plugin meta hooks for function calls. (Gilbert Clark)

NEWS

@@ -61,6 +61,8 @@ New Functionality
 - The SSL analysis now alert when encountering SSL connections with
   old protocol versions or unsafe cipher suites.
 
+- [TODO] Add new BroControl features.
+
 Changed Functionality
 ---------------------
 
@@ -89,6 +91,8 @@ Changed Functionality
 - conn.log gained a new field local_resp that works like local_orig,
   just for the responder address of the connection.
 
+- [TODO] Add changed BroControl features.
+
 Deprecated Functionality
 ------------------------
 

VERSION

@@ -1 +1 @@
-2.3-511
+2.3-518

aux/broctl

@@ -1 +1 @@
-Subproject commit 95b48726af25dad03e0c89029c3a4aaa739e4fed
+Subproject commit f7b9ef4d245114180c5932f362310632b8cf5174

scripts/base/init-bare.bro

@@ -440,6 +440,7 @@ type NetStats: record {
 	## packet capture system, this value may not be available and will then
 	## be always set to zero.
 	pkts_link:    count &default=0;
+	bytes_recvd:  count &default=0;	##< Bytes received by Bro.
 };
 
 ## Statistics about Bro's resource consumption.
@@ -2774,19 +2775,20 @@ export {
 module X509;
 export {
 	type Certificate: record {
-		version: count;	##< Version number.
+		version: count &log;	##< Version number.
-		serial: string;	##< Serial number.
+		serial: string &log;	##< Serial number.
-		subject: string;	##< Subject.
+		subject: string &log;	##< Subject.
-		issuer: string;	##< Issuer.
+		issuer: string &log;	##< Issuer.
-		not_valid_before: time;	##< Timestamp before when certificate is not valid.
+		cn: string &optional; ##< Last (most specific) common name.
-		not_valid_after: time;	##< Timestamp after when certificate is not valid.
+		not_valid_before: time &log;	##< Timestamp before when certificate is not valid.
-		key_alg: string;	##< Name of the key algorithm
+		not_valid_after: time &log;	##< Timestamp after when certificate is not valid.
-		sig_alg: string;	##< Name of the signature algorithm
+		key_alg: string &log;	##< Name of the key algorithm
-		key_type: string &optional;	##< Key type, if key parseable by openssl (either rsa, dsa or ec)
+		sig_alg: string &log;	##< Name of the signature algorithm
-		key_length: count &optional;	##< Key length in bits
+		key_type: string &optional &log;	##< Key type, if key parseable by openssl (either rsa, dsa or ec)
-		exponent: string &optional;	##< Exponent, if RSA-certificate
+		key_length: count &optional &log;	##< Key length in bits
-		curve: string &optional;	##< Curve, if EC-certificate
+		exponent: string &optional &log;	##< Exponent, if RSA-certificate
-	} &log;
+		curve: string &optional &log;	##< Curve, if EC-certificate
+	};
 
 	type Extension: record {
 		name: string;	##< Long name of extension. oid if name not known

scripts/policy/frameworks/intel/seen/ssl.bro

@@ -10,3 +10,16 @@ event ssl_extension_server_name(c: connection, is_orig: bool, names: string_vec)
 		             $conn=c,
 		             $where=SSL::IN_SERVER_NAME]);
 	}
+
+event ssl_established(c: connection)
+	{
+	if ( ! c$ssl?$cert_chain || |c$ssl$cert_chain| == 0 ||
+	     ! c$ssl$cert_chain[0]?$x509 )
+		return;
+
+	if ( c$ssl$cert_chain[0]$x509?$certificate && c$ssl$cert_chain[0]$x509$certificate?$cn )
+		Intel::seen([$indicator=c$ssl$cert_chain[0]$x509$certificate$cn,
+			$indicator_type=Intel::DOMAIN,
+			$conn=c,
+			$where=X509::IN_CERT]);
+	}

scripts/policy/frameworks/intel/seen/x509.bro

@@ -2,6 +2,18 @@
 @load base/files/x509
 @load ./where-locations
 
+event x509_ext_subject_alternative_name(f: fa_file, ext: X509::SubjectAlternativeName)
+	{
+	if ( ext?$dns )
+		{
+		for ( i in ext$dns )
+			Intel::seen([$indicator=ext$dns[i],
+				$indicator_type=Intel::DOMAIN,
+				$f=f,
+				$where=X509::IN_CERT]);
+		}
+	}
+
 event x509_certificate(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate)
 	{
 	if ( /emailAddress=/ in cert$subject )

scripts/policy/misc/stats.bro

@@ -39,6 +39,9 @@ export {
 		## Number of packets seen on the link since the last stats
 		## interval if reading live traffic.
 		pkts_link:     count     &log &optional;
+		## Number of bytes received since the last stats interval if
+		## reading live traffic.
+		bytes_recv:   count     &log &optional;
 	};
 
 	## Event to catch stats as they are written to the logging stream.
@@ -74,6 +77,7 @@ event check_stats(last_ts: time, last_ns: NetStats, last_res: bro_resources)
 		info$pkts_recv = ns$pkts_recvd - last_ns$pkts_recvd;
 		info$pkts_dropped = ns$pkts_dropped  - last_ns$pkts_dropped;
 		info$pkts_link = ns$pkts_link  - last_ns$pkts_link;
+		info$bytes_recv = ns$bytes_recvd  - last_ns$bytes_recvd;
 		}
 
 	Log::write(Stats::LOG, info);

src/bro.bif

@@ -1675,6 +1675,7 @@ function net_stats%(%): NetStats
 	unsigned int recv = 0;
 	unsigned int drop = 0;
 	unsigned int link = 0;
+	unsigned int bytes_recv = 0;
 
 	const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs());
 
@@ -1688,12 +1689,14 @@ function net_stats%(%): NetStats
 		recv += stat.received;
 		drop += stat.dropped;
 		link += stat.link;
+		bytes_recv += stat.bytes_received;
 		}
 
 	RecordVal* ns = new RecordVal(net_stats);
 	ns->Assign(0, new Val(recv, TYPE_COUNT));
 	ns->Assign(1, new Val(drop, TYPE_COUNT));
 	ns->Assign(2, new Val(link, TYPE_COUNT));
+	ns->Assign(3, new Val(bytes_recv, TYPE_COUNT));
 
 	return ns;
 	%}

src/file_analysis/analyzer/x509/X509.cc

@@ -104,13 +104,35 @@ RecordVal* file_analysis::X509::ParseCertificate(X509Val* cert_val)
 	len = BIO_gets(bio, buf, sizeof(buf));
 	pX509Cert->Assign(2, new StringVal(len, buf));
 	BIO_reset(bio);
+
+	X509_NAME *subject_name = X509_get_subject_name(ssl_cert);
+	// extract the most specific (last) common name from the subject
+	int namepos = -1;
+	for ( ;; )
+		{
+		int j = X509_NAME_get_index_by_NID(subject_name, NID_commonName, namepos);
+		if ( j == -1 )
+			break;
+
+		namepos = j;
+		}
+
+	if ( namepos != -1 )
+		{
+		// we found a common name
+		ASN1_STRING_print(bio, X509_NAME_ENTRY_get_data(X509_NAME_get_entry(subject_name, namepos)));
+		len = BIO_gets(bio, buf, sizeof(buf));
+		pX509Cert->Assign(4, new StringVal(len, buf));
+		BIO_reset(bio);
+		}
+
 	X509_NAME_print_ex(bio, X509_get_issuer_name(ssl_cert), 0, XN_FLAG_RFC2253);
 	len = BIO_gets(bio, buf, sizeof(buf));
 	pX509Cert->Assign(3, new StringVal(len, buf));
 	BIO_free(bio);
 
-	pX509Cert->Assign(4, new Val(GetTimeFromAsn1(X509_get_notBefore(ssl_cert)), TYPE_TIME));
+	pX509Cert->Assign(5, new Val(GetTimeFromAsn1(X509_get_notBefore(ssl_cert)), TYPE_TIME));
-	pX509Cert->Assign(5, new Val(GetTimeFromAsn1(X509_get_notAfter(ssl_cert)), TYPE_TIME));
+	pX509Cert->Assign(6, new Val(GetTimeFromAsn1(X509_get_notAfter(ssl_cert)), TYPE_TIME));
 
 	// we only read 255 bytes because byte 256 is always 0.
 	// if the string is longer than 255, that will be our null-termination,
@@ -118,7 +140,7 @@ RecordVal* file_analysis::X509::ParseCertificate(X509Val* cert_val)
 	if ( ! i2t_ASN1_OBJECT(buf, 255, ssl_cert->cert_info->key->algor->algorithm) )
 		buf[0] = 0;
 
-	pX509Cert->Assign(6, new StringVal(buf));
+	pX509Cert->Assign(7, new StringVal(buf));
 
 	// Special case for RDP server certificates. For some reason some (all?) RDP server
 	// certificates like to specify their key algorithm as md5WithRSAEncryption, which
@@ -136,23 +158,23 @@ RecordVal* file_analysis::X509::ParseCertificate(X509Val* cert_val)
 	if ( ! i2t_ASN1_OBJECT(buf, 255, ssl_cert->sig_alg->algorithm) )
 		buf[0] = 0;
 
-	pX509Cert->Assign(7, new StringVal(buf));
+	pX509Cert->Assign(8, new StringVal(buf));
 
 	// Things we can do when we have the key...
 	EVP_PKEY *pkey = X509_extract_key(ssl_cert);
 	if ( pkey != NULL )
 		{
 		if ( pkey->type == EVP_PKEY_DSA )
-			pX509Cert->Assign(8, new StringVal("dsa"));
+			pX509Cert->Assign(9, new StringVal("dsa"));
 
 		else if ( pkey->type == EVP_PKEY_RSA )
 			{
-			pX509Cert->Assign(8, new StringVal("rsa"));
+			pX509Cert->Assign(9, new StringVal("rsa"));
 
 			char *exponent = BN_bn2dec(pkey->pkey.rsa->e);
 			if ( exponent != NULL )
 				{
-				pX509Cert->Assign(10, new StringVal(exponent));
+				pX509Cert->Assign(11, new StringVal(exponent));
 				OPENSSL_free(exponent);
 				exponent = NULL;
 				}
@@ -160,8 +182,8 @@ RecordVal* file_analysis::X509::ParseCertificate(X509Val* cert_val)
 #ifndef OPENSSL_NO_EC
 		else if ( pkey->type == EVP_PKEY_EC )
 			{
-			pX509Cert->Assign(8, new StringVal("ecdsa"));
+			pX509Cert->Assign(9, new StringVal("ecdsa"));
-			pX509Cert->Assign(11, KeyCurve(pkey));
+			pX509Cert->Assign(12, KeyCurve(pkey));
 			}
 #endif
 
@@ -172,7 +194,7 @@ RecordVal* file_analysis::X509::ParseCertificate(X509Val* cert_val)
 
 		unsigned int length = KeyLength(pkey);
 		if ( length > 0 )
-			pX509Cert->Assign(9, new Val(length, TYPE_COUNT));
+			pX509Cert->Assign(10, new Val(length, TYPE_COUNT));
 
 		EVP_PKEY_free(pkey);
 		}

src/iosource/PktSrc.h

@@ -38,7 +38,12 @@ public:
 		 */
 		unsigned int link;
 
-		Stats()	{ received = dropped = link = 0; }
+		/**
+		  * Bytes received by source after filtering (w/o drops).
+		*/
+		uint64 bytes_received;
+
+		Stats()	{ received = dropped = link = bytes_received = 0; }
 	};
 
 	/**

src/iosource/pcap/Source.cc

@@ -180,6 +180,8 @@ bool PcapSource::ExtractNextPacket(Packet* pkt)
 	last_hdr = current_hdr;
 	last_data = data;
 	++stats.received;
+	stats.bytes_received += current_hdr.len;
+
 	return true;
 	}
 
@@ -219,7 +221,7 @@ bool PcapSource::SetFilter(int index)
 
 #ifndef HAVE_LINUX
 	// Linux doesn't clear counters when resetting filter.
-	stats.received = stats.dropped = stats.link = 0;
+	stats.received = stats.dropped = stats.link = stats.bytes_received = 0;
 #endif
 
 	return true;
@@ -230,7 +232,7 @@ void PcapSource::Statistics(Stats* s)
 	char errbuf[PCAP_ERRBUF_SIZE];
 
 	if ( ! (props.is_live && pd) )
-		s->received = s->dropped = s->link = 0;
+		s->received = s->dropped = s->link = s->bytes_received = 0;
 
 	else
 		{
@@ -238,7 +240,7 @@ void PcapSource::Statistics(Stats* s)
 		if ( pcap_stats(pd, &pstat) < 0 )
 			{
 			PcapError();
-			s->received = s->dropped = s->link = 0;
+			s->received = s->dropped = s->link = s->bytes_received = 0;
 			}
 
 		else
@@ -249,6 +251,7 @@ void PcapSource::Statistics(Stats* s)
 		}
 
 	s->received = stats.received;
+	s->bytes_received = stats.bytes_received;
 
 	if ( ! props.is_live )
 		s->dropped = 0;

testing/btest/Baseline/bifs.net_stats_trace/output

@@ -1 +1 @@
-[pkts_recvd=136, pkts_dropped=0, pkts_link=0]
+[pkts_recvd=136, pkts_dropped=0, pkts_link=0, bytes_recvd=25260]

testing/btest/Baseline/scripts.base.protocols.ssl.common_name/.stdout

@@ -0,0 +1,3 @@
+*.gstatic.com
+Google Internet Authority
+No CN

testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log

@@ -0,0 +1,22 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	intel
+#open	2015-03-04-01-12-47
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	sources
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[string]
+1416942644.593119	CXWv6p3arKYeMETxOg	192.168.4.149	49422	23.92.19.75	443	F0txuw2pvrkZOn04a8	-	23.92.19.75:443/tcp	www.pantz.org	Intel::DOMAIN	X509::IN_CERT	bro	source1
+#close	2015-03-04-01-12-47
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	intel
+#open	2015-03-04-01-12-47
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	sources
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[string]
+1170717505.934612	CXWv6p3arKYeMETxOg	192.150.187.164	58868	194.127.84.106	443	-	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	source1
+1170717509.082241	CjhGID4nQcgTWjvg4c	192.150.187.164	58869	194.127.84.106	443	-	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	source1
+1170717512.108799	CCvvfg3TEfuqmmG4bh	192.150.187.164	58870	194.127.84.106	443	-	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	source1
+#close	2015-03-04-01-12-47

testing/btest/scripts/base/protocols/ssl/common_name.test

@@ -0,0 +1,13 @@
+# This tests a normal SSL connection and the log it outputs.
+
+# @TEST-EXEC: bro -r $TRACES/tls/tls-conn-with-extensions.trace %INPUT
+# @TEST-EXEC: bro -C -r $TRACES/tls/cert-no-cn.pcap %INPUT
+# @TEST-EXEC: btest-diff .stdout
+
+event x509_certificate(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate)
+	{
+	if ( cert?$cn )
+		print cert$cn;
+	else
+		print "No CN";
+	}

testing/btest/scripts/policy/frameworks/intel/seen/certs.bro

@@ -0,0 +1,18 @@
+# @TEST-EXEC: bro -r $TRACES/tls/ecdsa-cert.pcap %INPUT
+# @TEST-EXEC: cat intel.log > intel-all.log
+# @TEST-EXEC: bro -r $TRACES/tls/ssl.v3.trace %INPUT
+# @TEST-EXEC: cat intel.log >> intel-all.log
+# @TEST-EXEC: btest-diff intel-all.log
+
+@TEST-START-FILE intel.dat
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.url
+www.pantz.org	Intel::DOMAIN	source1	test entry	http://some-data-distributor.com/100000
+www.dresdner-privat.de	Intel::DOMAIN	source1	test entry	http://some-data-distributor.com/100000
+@TEST-END-FILE
+
+@load base/frameworks/intel
+@load base/protocols/ssl
+@load frameworks/intel/seen
+
+redef Intel::read_files += { "intel.dat" };
+