Merge branch 'master' of ssh://git.bro.org/bro

CHANGES

@@ -1,4 +1,13 @@
 
+2.2-120 | 2014-01-28 10:25:23 -0800
+
+  * Fix and extend x509_extension() event, which now actually returns
+    the extension. (Bernhard Amann)
+
+    New event signauture:
+
+        event x509_extension(c: connection, is_orig: bool, cert:X509, extension: X509_extension_info)
+
 2.2-117 | 2014-01-23 14:18:19 -0800
 
   * Fixing initialization context in anonymous functions. (Robin

NEWS

@@ -36,6 +36,11 @@ Changed Functionality
 
 - Notice::end_suppression() has been removed.
 
+- Bro now parses X.509 extensions headers and, as a result, the
+  corresponding event got a new signature:
+
+      event x509_extension(c: connection, is_orig: bool, cert: X509, ext: X509_extension_info);
+
 Bro 2.2
 =======
 

VERSION

@@ -1 +1 @@
-2.2-117
+2.2-120

aux/broctl

@@ -1 +1 @@
-Subproject commit c4b5fb7336f2b598cf69777a7ec91b4aa16cacd1
+Subproject commit 9ff2e2ced64a3bd4af1268154e261671a1153481

scripts/base/init-bare.bro

@@ -2432,6 +2432,17 @@ type X509: record {
 	not_valid_after: time;	##< Timestamp after when certificate is not valid.
 };
 
+## An X509 extension.
+##
+## .. bro:see:: x509_extension
+type X509_extension_info: record {
+	name: string;	##< Long name of extension; oid if name not known.
+	short_name: string &optional;	##< Short name of extension if known.
+	oid: string;	##< Oid of extension.
+	critical: bool;	##< True if extension is critical.
+	value: string;	##< Extension content parsed to string for known extensions. Raw data otherwise.
+};
+
 ## HTTP session statistics.
 ##
 ## .. bro:see:: http_stats
@@ -2849,6 +2860,12 @@ global load_sample_freq = 20 &redef;
 ## .. bro:see:: gap_report
 const gap_report_freq = 1.0 sec &redef;
 
+## Whether to attempt to automatically detect SYN/FIN/RST-filtered trace
+## and not report missing segments for such connections.
+## If this is enabled, then missing data at the end of connections may not
+## be reported via :bro:see:`content_gap`.
+const detect_filtered_trace = F &redef;
+
 ## Whether we want :bro:see:`content_gap` and :bro:see:`gap_report` for partial
 ## connections. A connection is partial if it is missing a full handshake. Note
 ## that gap reports for partial connections might not be reliable.

src/NetVar.cc

@@ -48,6 +48,7 @@ int tcp_max_above_hole_without_any_acks;
 int tcp_excessive_data_without_further_acks;
 
 RecordType* x509_type;
+RecordType* x509_extension_type;
 
 RecordType* socks_address;
 
@@ -356,6 +357,7 @@ void init_net_var()
 		opt_internal_int("tcp_excessive_data_without_further_acks");
 
 	x509_type = internal_type("X509")->AsRecordType();
+	x509_extension_type = internal_type("X509_extension_info")->AsRecordType();
 
 	socks_address = internal_type("SOCKS::Address")->AsRecordType();
 

src/NetVar.h

@@ -51,6 +51,7 @@ extern int tcp_max_above_hole_without_any_acks;
 extern int tcp_excessive_data_without_further_acks;
 
 extern RecordType* x509_type;
+extern RecordType* x509_extension_type;
 
 extern RecordType* socks_address;
 

src/analyzer/protocol/ssl/events.bif

@@ -178,11 +178,13 @@ event x509_certificate%(c: connection, is_orig: bool, cert: X509, chain_idx: cou
 ##
 ## is_orig: True if event is raised for originator side of the connection.
 ##
-## data: The raw data associated with the extension.
+## cert: The parsed certificate.
+##
+## extension: The parsed extension.
 ##
 ## .. bro:see:: ssl_alert ssl_client_hello ssl_established ssl_extension
 ##    ssl_server_hello x509_certificate x509_error  x509_verify
-event x509_extension%(c: connection, is_orig: bool, data: string%);
+event x509_extension%(c: connection, is_orig: bool, cert: X509, extension: X509_extension_info%);
 
 ## Generated when errors occur during parsing an X509 certificate.
 ##

src/analyzer/protocol/ssl/ssl-analyzer.pac

@@ -9,6 +9,7 @@
 #include "util.h"
 
 #include <openssl/x509.h>
+#include <openssl/x509v3.h>
 #include <openssl/asn1.h>
 %}
 
@@ -211,7 +212,7 @@ refine connection SSL_Conn += {
 
 			BifEvent::generate_ssl_server_hello(bro_analyzer(),
 							bro_analyzer()->Conn(),
-							version, ts, new StringVal(server_random.length(), 
+							version, ts, new StringVal(server_random.length(),
 							(const char*) server_random.data()),
 							to_string_val(session_id),
 							ciphers->size()==0 ? 0 : ciphers->at(0), comp_method);
@@ -298,27 +299,61 @@ refine connection SSL_Conn += {
 					int num_ext = X509_get_ext_count(pTemp);
 					for ( int k = 0; k < num_ext; ++k )
 						{
-						unsigned char *pBuffer = 0;
-						int length = 0;
+						char name[256];
+						char oid[256];
+
+						memset(name, 0, sizeof(name));
+						memset(oid, 0, sizeof(oid));
 
 						X509_EXTENSION* ex = X509_get_ext(pTemp, k);
-						if (ex)
-							{
-							ASN1_STRING *pString = X509_EXTENSION_get_data(ex);
-							length = ASN1_STRING_to_UTF8(&pBuffer, pString);
-							//i2t_ASN1_OBJECT(&pBuffer, length, obj)
-							// printf("extension length: %d\n", length);
-							// -1 indicates an error.
-							if ( length >= 0 )
-								{
-								StringVal* value = new StringVal(length, (char*)pBuffer);
-								BifEvent::generate_x509_extension(bro_analyzer(),
-											bro_analyzer()->Conn(), ${rec.is_orig}, value);
-								}
-							OPENSSL_free(pBuffer);
-							}
+
+						if ( ! ex )
+							continue;
+
+						ASN1_OBJECT* ext_asn = X509_EXTENSION_get_object(ex);
+						const char* short_name = OBJ_nid2sn(OBJ_obj2nid(ext_asn));
+
+						OBJ_obj2txt(name, sizeof(name) - 1, ext_asn, 0);
+						OBJ_obj2txt(oid, sizeof(oid) - 1, ext_asn, 1);
+
+						int critical = 0;
+						if ( X509_EXTENSION_get_critical(ex) != 0 )
+							critical = 1;
+
+						BIO *bio = BIO_new(BIO_s_mem());
+						if( ! X509V3_EXT_print(bio, ex, 0, 0))
+							M_ASN1_OCTET_STRING_print(bio, ex->value);
+
+						BIO_flush(bio);
+						int length = BIO_pending(bio);
+
+						// Use OPENSSL_malloc here. Using new or anything else can lead
+						// to interesting, hard to debug segfaults.
+						char *buffer = (char*) OPENSSL_malloc(length);
+						BIO_read(bio, buffer, length);
+						StringVal* ext_val = new StringVal(length, buffer);
+						OPENSSL_free(buffer);
+
+						BIO_free_all(bio);
+
+						RecordVal* pX509Ext = new RecordVal(x509_extension_type);
+						pX509Ext->Assign(0, new StringVal(name));
+
+						if ( short_name && strlen(short_name) > 0 )
+							pX509Ext->Assign(1, new StringVal(short_name));
+
+						pX509Ext->Assign(2, new StringVal(oid));
+						pX509Ext->Assign(3, new Val(critical, TYPE_BOOL));
+						pX509Ext->Assign(4, ext_val);
+
+						BifEvent::generate_x509_extension(bro_analyzer(),
+									bro_analyzer()->Conn(),
+									${rec.is_orig},
+									pX509Cert->Ref(),
+									pX509Ext);
 						}
 					}
+
 				X509_free(pTemp);
 				}
 			}

src/analyzer/protocol/tcp/TCP.cc

@@ -373,14 +373,11 @@ void TCP_Analyzer::ProcessSYN(const IP_Hdr* ip, const struct tcphdr* tp,
 void TCP_Analyzer::ProcessFIN(double t, TCP_Endpoint* endpoint,
 				int& seq_len, uint32 base_seq)
 	{
-	if ( endpoint->FIN_cnt == 0 )
-		{
-		++seq_len;	// FIN consumes a byte of sequence space
-		++endpoint->FIN_cnt;	// remember that we've seen a FIN
-		}
+	++seq_len;  // FIN consumes a byte of sequence space.
+	++endpoint->FIN_cnt;  // remember that we've seen a FIN
 
-	else if ( t < endpoint->last_time + tcp_storm_interarrival_thresh &&
-		  ++endpoint->FIN_cnt == tcp_storm_thresh )
+	if ( t < endpoint->last_time + tcp_storm_interarrival_thresh &&
+	     endpoint->FIN_cnt == tcp_storm_thresh )
 		Weird("FIN_storm");
 
 	// Remember the relative seq in FIN_seq.

src/analyzer/protocol/tcp/TCP_Endpoint.cc

@@ -161,6 +161,13 @@ void TCP_Endpoint::SetState(EndpointState new_state)
 
 bro_int_t TCP_Endpoint::Size() const
 	{
+	if ( prev_state == TCP_ENDPOINT_SYN_SENT && state == TCP_ENDPOINT_RESET &&
+	     peer->state == TCP_ENDPOINT_INACTIVE && ! NoDataAcked() )
+		// This looks like a half-open connection was discovered and aborted.
+		// Sequence numbers could be misleading if used in context of data size
+		// and there was never a chance for this endpoint to send data anyway.
+		return 0;
+
 	bro_int_t size;
 
 	uint64 last_seq_64 = (uint64(last_seq_high) << 32) | last_seq;

src/analyzer/protocol/tcp/TCP_Reassembler.cc

@@ -178,7 +178,7 @@ void TCP_Reassembler::Undelivered(int up_to_seq)
 		// to this method and only if this condition is not true).
 		reporter->InternalError("Calling Undelivered for data that has already been delivered (or has already been marked as undelivered");
 
-	if ( last_reassem_seq == 1 &&
+	if ( BifConst::detect_filtered_trace && last_reassem_seq == 1 &&
 	     (endpoint->FIN_cnt > 0 || endpoint->RST_cnt > 0 ||
 	      peer->FIN_cnt > 0 || peer->RST_cnt > 0) )
 		{

src/const.bif

@@ -5,6 +5,7 @@
 const ignore_keep_alive_rexmit: bool;
 const skip_http_data: bool;
 const use_conn_size_analyzer: bool;
+const detect_filtered_trace: bool;
 const report_gaps_for_partial: bool;
 const exit_only_after_terminate: bool;
 

src/file_analysis/File.cc

@@ -103,7 +103,6 @@ File::~File()
 	DBG_LOG(DBG_FILE_ANALYSIS, "Destroying File object %s", id.c_str());
 	Unref(val);
 
-	// Queue may not be empty in the case where only content gaps were seen.
 	while ( ! fonc_queue.empty() )
 		{
 		delete_vals(fonc_queue.front().second);
@@ -460,20 +459,27 @@ void File::FileEvent(EventHandlerPtr h)
 	FileEvent(h, vl);
 	}
 
+static void flush_file_event_queue(queue<pair<EventHandlerPtr, val_list*> >& q)
+	{
+	while ( ! q.empty() )
+		{
+		pair<EventHandlerPtr, val_list*> p = q.front();
+		mgr.QueueEvent(p.first, p.second);
+		q.pop();
+		}
+	}
+
 void File::FileEvent(EventHandlerPtr h, val_list* vl)
 	{
+	if ( h == file_state_remove )
+		flush_file_event_queue(fonc_queue);
+
 	mgr.QueueEvent(h, vl);
 
 	if ( h == file_new )
 		{
 		did_file_new_event = true;
-
-		while ( ! fonc_queue.empty() )
-			{
-			pair<EventHandlerPtr, val_list*> p = fonc_queue.front();
-			mgr.QueueEvent(p.first, p.second);
-			fonc_queue.pop();
-			}
+		flush_file_event_queue(fonc_queue);
 		}
 
 	if ( h == file_new || h == file_timeout || h == file_extraction_limit )

testing/btest/Baseline/core.tcp.fin-retransmit/out

@@ -0,0 +1,2 @@
+[size=0, state=5, num_pkts=3, num_bytes_ip=156, flow_label=0]
+[size=0, state=6, num_pkts=2, num_bytes_ip=92, flow_label=0]

testing/btest/Baseline/core.tcp.miss-end-data/conn.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	conn
+#open	2014-01-24-22-19-38
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
+1331764471.664131	CXWv6p3arKYeMETxOg	192.168.122.230	60648	77.238.160.184	80	tcp	http	10.048360	538	2902	SF	-	2902	ShADafF	5	750	4	172	(empty)
+#close	2014-01-24-22-19-38

testing/btest/Baseline/core.tcp.miss-end-data/out

@@ -0,0 +1 @@
+content_gap, [orig_h=192.168.122.230, orig_p=60648/tcp, resp_h=77.238.160.184, resp_p=80/tcp], F, 1, 2902

testing/btest/Baseline/scripts.base.protocols.ssl.x509_extensions/.stdout

@@ -0,0 +1,20 @@
+[name=X509v3 Authority Key Identifier, short_name=authorityKeyIdentifier, oid=2.5.29.35, critical=F, value=keyid:3F:D5:B5:D0:D6:44:79:50:4A:17:A3:9B:8C:4A:DC:B8:B0:22:64:6B^J]
+[name=X509v3 Subject Key Identifier, short_name=subjectKeyIdentifier, oid=2.5.29.14, critical=F, value=A2:76:09:20:A8:40:FD:A1:AC:C8:E9:35:B9:11:A6:61:FF:8C:FF:A3]
+[name=X509v3 Key Usage, short_name=keyUsage, oid=2.5.29.15, critical=T, value=Digital Signature, Key Encipherment]
+[name=X509v3 Basic Constraints, short_name=basicConstraints, oid=2.5.29.19, critical=T, value=CA:FALSE]
+[name=X509v3 Extended Key Usage, short_name=extendedKeyUsage, oid=2.5.29.37, critical=F, value=TLS Web Server Authentication, TLS Web Client Authentication]
+[name=X509v3 Certificate Policies, short_name=certificatePolicies, oid=2.5.29.32, critical=F, value=Policy: 1.3.6.1.4.1.6449.1.2.1.3.4^J  CPS: https://secure.comodo.com/CPS^J]
+[name=X509v3 CRL Distribution Points, short_name=crlDistributionPoints, oid=2.5.29.31, critical=F, value=^JFull Name:^J  URI:http://crl.comodoca.com/COMODOHigh-AssuranceSecureServerCA.crl^J]
+[name=Authority Information Access, short_name=authorityInfoAccess, oid=1.3.6.1.5.5.7.1.1, critical=F, value=CA Issuers - URI:http://crt.comodoca.com/COMODOHigh-AssuranceSecureServerCA.crt^JOCSP - URI:http://ocsp.comodoca.com^J]
+[name=X509v3 Subject Alternative Name, short_name=subjectAltName, oid=2.5.29.17, critical=F, value=DNS:*.taleo.net, DNS:taleo.net]
+[name=X509v3 Authority Key Identifier, short_name=authorityKeyIdentifier, oid=2.5.29.35, critical=F, value=keyid:AD:BD:98:7A:34:B4:26:F7:FA:C4:26:54:EF:03:BD:E0:24:CB:54:1A^J]
+[name=X509v3 Subject Key Identifier, short_name=subjectKeyIdentifier, oid=2.5.29.14, critical=F, value=3F:D5:B5:D0:D6:44:79:50:4A:17:A3:9B:8C:4A:DC:B8:B0:22:64:6B]
+[name=X509v3 Key Usage, short_name=keyUsage, oid=2.5.29.15, critical=T, value=Certificate Sign, CRL Sign]
+[name=X509v3 Basic Constraints, short_name=basicConstraints, oid=2.5.29.19, critical=T, value=CA:TRUE, pathlen:0]
+[name=X509v3 Certificate Policies, short_name=certificatePolicies, oid=2.5.29.32, critical=F, value=Policy: X509v3 Any Policy^J]
+[name=X509v3 CRL Distribution Points, short_name=crlDistributionPoints, oid=2.5.29.31, critical=F, value=^JFull Name:^J  URI:http://crl.usertrust.com/AddTrustExternalCARoot.crl^J]
+[name=Authority Information Access, short_name=authorityInfoAccess, oid=1.3.6.1.5.5.7.1.1, critical=F, value=CA Issuers - URI:http://crt.usertrust.com/AddTrustExternalCARoot.p7c^JCA Issuers - URI:http://crt.usertrust.com/AddTrustUTNSGCCA.crt^JOCSP - URI:http://ocsp.usertrust.com^J]
+[name=X509v3 Subject Key Identifier, short_name=subjectKeyIdentifier, oid=2.5.29.14, critical=F, value=AD:BD:98:7A:34:B4:26:F7:FA:C4:26:54:EF:03:BD:E0:24:CB:54:1A]
+[name=X509v3 Key Usage, short_name=keyUsage, oid=2.5.29.15, critical=F, value=Certificate Sign, CRL Sign]
+[name=X509v3 Basic Constraints, short_name=basicConstraints, oid=2.5.29.19, critical=T, value=CA:TRUE]
+[name=X509v3 Authority Key Identifier, short_name=authorityKeyIdentifier, oid=2.5.29.35, critical=F, value=keyid:AD:BD:98:7A:34:B4:26:F7:FA:C4:26:54:EF:03:BD:E0:24:CB:54:1A^JDirName:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root^Jserial:01^J]

testing/btest/core/tcp/fin-retransmit.bro

@@ -0,0 +1,8 @@
+# @TEST-EXEC: bro -b -r $TRACES/tcp/fin_retransmission.pcap %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event connection_state_remove(c: connection)
+    {
+    print c$orig;
+    print c$resp;
+    }

testing/btest/core/tcp/miss-end-data.bro

@@ -0,0 +1,10 @@
+# @TEST-EXEC: bro -r $TRACES/tcp/miss_end_data.pcap %INPUT >out
+# @TEST-EXEC: btest-diff out
+# @TEST-EXEC: btest-diff conn.log
+
+redef report_gaps_for_partial = T;
+
+event content_gap(c: connection, is_orig: bool, seq: count, length: count)
+	{
+	print "content_gap", c$id, is_orig, seq, length;
+	}

testing/btest/scripts/base/protocols/ssl/x509_extensions.test

@@ -0,0 +1,7 @@
+# @TEST-EXEC: bro -r $TRACES/tls1.2.trace %INPUT
+# @TEST-EXEC: btest-diff .stdout
+
+event x509_extension(c: connection, is_orig: bool, cert:X509, extension: X509_extension_info) 
+{
+	print extension;
+}