Merge branch 'master' into topic/jsiwek/ipv6-ext-headers

aux/broccoli

@@ -1 +1 @@
-Subproject commit d6e36c95e0335f7cc081191c8612085bd12706f9
+Subproject commit 2602eb53e70d7f0afae8fac58d7636b9291974a4

scripts/base/init-bare.bro

@@ -1863,13 +1863,6 @@ global dns_skip_all_addl = T &redef;
 ## traffic and do not process it.  Set to 0 to turn off this functionality. 
 global dns_max_queries = 5;
 
-## The maxiumum size in bytes for an SSL cipher specifcation.  If we see a packet
-## that has bigger cipherspecs, we won't do a comparisons of cipherspecs.
-const ssl_max_cipherspec_size = 68 &redef;
-
-# todo::Is this still used?
-# type X509_extensions: table[count] of string;
-
 ## An X509 certificate.
 ##
 ## .. bro:see:: x509_certificate 
@@ -1882,10 +1875,6 @@ type X509: record {
 	not_valid_after: time;	##< Timestamp after when certificate is not valid.
 };
 
-# This is indexed with the CA's name and yields a DER (binary) encoded certificate.
-# todo::Is this still used?
-# const root_ca_certs: table[string] of string = {} &redef;
-
 ## HTTP session statistics.
 ##
 ## .. bro:see:: http_stats 

src/NetVar.cc

@@ -47,15 +47,6 @@ int tcp_max_initial_window;
 int tcp_max_above_hole_without_any_acks;
 int tcp_excessive_data_without_further_acks;
 
-int ssl_compare_cipherspecs;
-int ssl_analyze_certificates;
-int ssl_store_certificates;
-int ssl_verify_certificates;
-int ssl_store_key_material;
-int ssl_max_cipherspec_size;
-StringVal* ssl_store_cert_path;
-StringVal* x509_trusted_cert_path;
-TableType* cipher_suites_list;
 RecordType* x509_type;
 
 double non_analyzed_lifetime;
@@ -192,8 +183,6 @@ StringVal* ssl_ca_certificate;
 StringVal* ssl_private_key;
 StringVal* ssl_passphrase;
 
-StringVal* x509_crl_file;
-
 Val* profiling_file;
 double profiling_interval;
 int expensive_profiling_multiple;
@@ -353,17 +342,7 @@ void init_net_var()
 	tcp_excessive_data_without_further_acks =
 		opt_internal_int("tcp_excessive_data_without_further_acks");
 
-	ssl_compare_cipherspecs  = opt_internal_int("ssl_compare_cipherspecs");
-	ssl_analyze_certificates = opt_internal_int("ssl_analyze_certificates");
-	ssl_store_certificates   = opt_internal_int("ssl_store_certificates");
-	ssl_verify_certificates  = opt_internal_int("ssl_verify_certificates");
-	ssl_store_key_material = opt_internal_int("ssl_store_key_material");
-	ssl_max_cipherspec_size  = opt_internal_int("ssl_max_cipherspec_size");
-
-	x509_trusted_cert_path = opt_internal_string("X509_trusted_cert_path");
-	ssl_store_cert_path = opt_internal_string("ssl_store_cert_path");
 	x509_type = internal_type("X509")->AsRecordType();
-	x509_crl_file = opt_internal_string("X509_crl_file");
 
 	non_analyzed_lifetime = opt_internal_double("non_analyzed_lifetime");
 	tcp_inactivity_timeout = opt_internal_double("tcp_inactivity_timeout");

src/NetVar.h

@@ -50,17 +50,7 @@ extern int tcp_max_initial_window;
 extern int tcp_max_above_hole_without_any_acks;
 extern int tcp_excessive_data_without_further_acks;
 
-// see policy/ssl.bro for details
-extern int ssl_compare_cipherspecs;
-extern int ssl_analyze_certificates;
-extern int ssl_store_certificates;
-extern int ssl_verify_certificates;
-extern int ssl_store_key_material;
-extern int ssl_max_cipherspec_size;
-extern StringVal* ssl_store_cert_path;
-extern StringVal* x509_trusted_cert_path;
 extern RecordType* x509_type;
-extern StringVal* x509_crl_file;
 
 extern double non_analyzed_lifetime;
 extern double tcp_inactivity_timeout;

src/X509.cc

@@ -1,263 +0,0 @@
-#include <openssl/err.h>
-
-#include "X509.h"
-#include "config.h"
-
-// ### NOTE: while d2i_X509 does not take a const u_char** pointer,
-// here we assume d2i_X509 does not write to <data>, so it is safe to
-// convert data to a non-const pointer.  Could some X509 guru verify
-// this?
-
-X509* d2i_X509_(X509** px, const u_char** in, int len)
-	{
-#ifdef OPENSSL_D2I_X509_USES_CONST_CHAR
-	  return d2i_X509(px, in, len);
-#else
-	  return d2i_X509(px, (u_char**)in, len);
-#endif
-	}
-
-X509_STORE* X509_Cert::ctx = 0;
-X509_LOOKUP* X509_Cert::lookup = 0;
-X509_STORE_CTX X509_Cert::csc;
-bool X509_Cert::bInited = false;
-
-// TODO: Check if Key < 768 Bits => Weakness!
-// FIXME: Merge verify and verifyChain.
-
-void X509_Cert::sslCertificateEvent(Contents_SSL* e, X509* pCert)
-	{
-	EventHandlerPtr event = ssl_certificate;
-	if ( ! event )
-		return;
-
-	char tmp[256];
-	RecordVal* pX509Cert = new RecordVal(x509_type);
-
-	X509_NAME_oneline(X509_get_issuer_name(pCert), tmp, sizeof tmp);
-	pX509Cert->Assign(0, new StringVal(tmp));
-	X509_NAME_oneline(X509_get_subject_name(pCert), tmp, sizeof tmp);
-	pX509Cert->Assign(1, new StringVal(tmp));
-	pX509Cert->Assign(2, new AddrVal(e->Conn()->OrigAddr()));
-
-	val_list* vl = new val_list;
-	vl->append(e->BuildConnVal());
-	vl->append(pX509Cert);
-	vl->append(new Val(e->IsOrig(), TYPE_BOOL));
-
-	e->Conn()->ConnectionEvent(event, e, vl);
-	}
-
-void X509_Cert::sslCertificateError(Contents_SSL* e, int error_numbe)
-	{
-	Val* err_str = new StringVal(X509_verify_cert_error_string(csc.error));
-	val_list* vl = new val_list;
-
-	vl->append(e->BuildConnVal());
-	vl->append(new Val(csc.error, TYPE_INT));
-	vl->append(err_str);
-
-	e->Conn()->ConnectionEvent(ssl_X509_error, e, vl);
-	}
-
-int X509_Cert::init()
-	{
-#if 0
-	OpenSSL_add_all_algorithms();
-#endif
-
-	ctx = X509_STORE_new();
-	int flag = 0;
-	int ret = 0;
-
-	if ( x509_trusted_cert_path &&
-	     x509_trusted_cert_path->AsString()->Len() > 0 )
-		{ // add the path(s) for the local CA's certificates
-		const BroString* pString = x509_trusted_cert_path->AsString();
-
-		lookup = X509_STORE_add_lookup(ctx, X509_LOOKUP_hash_dir());
-		if ( ! lookup )
-			{
-			reporter->Error("X509_Cert::init(): initing lookup failed\n");
-			flag = 1;
-			}
-
-		int i = X509_LOOKUP_add_dir(lookup,
-				(const char*) pString->Bytes(),
-				X509_FILETYPE_PEM);
-		if ( ! i )
-			{
-			reporter->Error("X509_Cert::init(): error adding lookup directory\n");
-			ret = 0;
-			}
-		}
-	else
-		{
-		printf("X509: Using the default trusted cert path.\n");
-		X509_STORE_set_default_paths(ctx);
-		}
-
-	// Add crl functionality - will only add if defined and
-	// X509_STORE_add_lookup was successful.
-	if ( ! flag && x509_crl_file && x509_crl_file->AsString()->Len() > 0 )
-		{
-		const BroString* rString = x509_crl_file->AsString();
-
-		if ( X509_load_crl_file(lookup, (const char*) rString->Bytes(),
-					X509_FILETYPE_PEM) != 1 )
-			{
-			reporter->Error("X509_Cert::init(): error reading CRL file\n");
-			ret = 1;
-			}
-
-#if 0
-		// Note, openssl version must be > 0.9.7(a).
-		X509_STORE_set_flags(ctx,
-			X509_V_FLAG_CRL_CHECK | X509_V_FLAG_CRL_CHECK_ALL);
-#endif
-		}
-
-	bInited = true;
-	return ret;
-	}
-
-int X509_Cert::verify(Contents_SSL* e, const u_char* data, uint32 len)
-	{
-	if ( ! bInited )
-		init();
-
-	X509* pCert = d2i_X509_(NULL, &data, len);
-	if ( ! pCert )
-		{
-		// 5 = X509_V_ERR_UNABLE_TO_DECODE_ISSUER_PUBLIC_KEY
-		sslCertificateError(e, 5);
-		return -1;
-		}
-
-	sslCertificateEvent(e, pCert);
-
-	X509_STORE_CTX_init(&csc, ctx, pCert, 0);
-	X509_STORE_CTX_set_time(&csc, 0, (time_t) network_time);
-	int i = X509_verify_cert(&csc);
-	X509_STORE_CTX_cleanup(&csc);
-	int ret = 0;
-
-	int ext = X509_get_ext_count(pCert);
-
-	if ( ext > 0 )
-		{
-		TableVal* x509ex = new TableVal(x509_extension);
-		val_list* vl = new val_list;
-		char buf[256];
-
-		for ( int k = 0; k < ext; ++k )
-			{
-			X509_EXTENSION* ex = X509_get_ext(pCert, k);
-			ASN1_OBJECT* obj = X509_EXTENSION_get_object(ex);
-			i2t_ASN1_OBJECT(buf, sizeof(buf), obj);
-
-			Val* index = new Val(k+1, TYPE_COUNT);
-			Val* value = new StringVal(strlen(buf), buf);
-			x509ex->Assign(index, value);
-			Unref(index);
-			// later we can do critical extensions like:
-			// X509_EXTENSION_get_critical(ex);
-			}
-
-		vl->append(e->BuildConnVal());
-		vl->append(x509ex);
-		e->Conn()->ConnectionEvent(process_X509_extensions, e, vl);
-		}
-
-	if ( ! i )
-		{
-		sslCertificateError(e, csc.error);
-		ret = csc.error;
-		}
-	else
-		ret = 0;
-
-	delete pCert;
-	return ret;
-	}
-
-int X509_Cert::verifyChain(Contents_SSL* e, const u_char* data, uint32 len)
-	{
-	if ( ! bInited )
-		init();
-
-	// Gets an ssl3x cert chain (could be one single cert, too,
-	// but in chain format).
-
-	// Init the stack.
-	STACK_OF(X509)* untrustedCerts = sk_X509_new_null();
-	if ( ! untrustedCerts )
-		{
-		// Internal error allocating stack of untrusted certs.
-		// 11 = X509_V_ERR_OUT_OF_MEM
-		sslCertificateError(e, 11);
-		return -1;
-		}
-
-	// NOT AGAIN!!!
-	// Extract certificates and put them into an OpenSSL Stack.
-	uint tempLength = 0;
-	int certCount = 0;
-	X509* pCert = 0; // base cert, this one is to be verified
-
-	while ( tempLength < len )
-		{
-		++certCount;
-		uint32 certLength =
-			uint32((data[tempLength + 0] << 16) |
-			       data[tempLength + 1] << 8) |
-			data[tempLength + 2];
-
-		// Points to current cert.
-		const u_char* pCurrentCert = &data[tempLength+3];
-
-		X509* pTemp = d2i_X509_(0, &pCurrentCert, certLength);
-		if ( ! pTemp )
-			{ // error is somewhat of a misnomer
-			// 5 = X509_V_ERR_UNABLE_TO_DECODE_ISSUER_PUBLIC_KEY
-			sslCertificateError(e, 5);
-			//FIXME: free ptrs
-			return -1;
-			}
-
-		if ( certCount == 1 )
-			// The first certificate goes directly into the ctx.
-			pCert = pTemp;
-		else
-			// The remaining certificates (if any) are put into
-			// the list of untrusted certificates
-			sk_X509_push(untrustedCerts, pTemp);
-
-		tempLength += certLength + 3;
-		}
-
-	sslCertificateEvent(e, pCert);
-
-	X509_STORE_CTX_init(&csc, ctx, pCert, untrustedCerts);
-	X509_STORE_CTX_set_time(&csc, 0, (time_t) network_time);
-	int i = X509_verify_cert(&csc);
-	X509_STORE_CTX_cleanup(&csc);
-	//X509_STORE_CTX_free(&csc);
-	int ret = 0;
-
-	if ( ! i )
-		{
-		sslCertificateError(e, csc.error);
-		ret = csc.error;
-		}
-	else
-		ret = 0;
-
-	delete pCert;
-	// Free the stack, incuding. contents.
-
-	// FIXME: could this break Bro's memory tracking?
-	sk_X509_pop_free(untrustedCerts, X509_free);
-
-	return ret;
-	}

src/event.bif

@@ -4613,7 +4613,6 @@ event ssh_server_version%(c: connection, version: string%);
 ##
 ## .. bro:see:: ssl_alert  ssl_established ssl_extension ssl_server_hello
 ##    ssl_session_ticket_handshake x509_certificate x509_error x509_extension
-##    ssl_max_cipherspec_size
 event ssl_client_hello%(c: connection, version: count, possible_ts: time, session_id: string, ciphers: count_set%);
 
 ## Generated for an SSL/TLS servers's initial *hello* message. SSL/TLS sessions
@@ -4644,7 +4643,6 @@ event ssl_client_hello%(c: connection, version: count, possible_ts: time, sessio
 ##
 ## .. bro:see:: ssl_alert ssl_client_hello ssl_established ssl_extension
 ##    ssl_session_ticket_handshake x509_certificate x509_error x509_extension
-##    ssl_max_cipherspec_size
 event ssl_server_hello%(c: connection, version: count, possible_ts: time, session_id: string, cipher: count, comp_method: count%);
 
 ## Generated for SSL/TLS extensions seen in an initial handshake.  SSL/TLS sessions

testing/btest/core/leaks/dns.bro

@@ -1,5 +1,7 @@
 # Needs perftools support.
 #
+# @TEST-GROUP: leaks
+#
 # @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
 #
 # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -r $TRACES/wikipedia.trace %INPUT

testing/btest/core/leaks/test-all.bro

@@ -1,5 +1,7 @@
 # Needs perftools support.
 #
+# @TEST-GROUP: leaks
+#
 # @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
 #
 # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -r $TRACES/wikipedia.trace test-all-policy

testing/btest/istate/broccoli-ipv6.bro

@@ -1,3 +1,5 @@
+# @TEST-GROUP: comm
+#
 # @TEST-REQUIRES: test -e $BUILD/aux/broccoli/src/libbroccoli.so || test -e $BUILD/aux/broccoli/src/libbroccoli.dylib
 #
 # @TEST-EXEC: btest-bg-run bro bro %INPUT $DIST/aux/broccoli/test/broccoli-v6addrs.bro

testing/btest/istate/broccoli.bro

@@ -1,3 +1,5 @@
+# @TEST-GROUP: comm
+#
 # @TEST-REQUIRES: test -e $BUILD/aux/broccoli/src/libbroccoli.so || test -e $BUILD/aux/broccoli/src/libbroccoli.dylib
 #
 # @TEST-EXEC: btest-bg-run bro bro %INPUT $DIST/aux/broccoli/test/broping-record.bro

testing/btest/istate/events-ssl.bro

@@ -1,4 +1,5 @@
-# 
+# @TEST-GROUP: comm
+#
 # @TEST-EXEC: btest-bg-run sender   bro -C -r $TRACES/web.trace --pseudo-realtime ../sender.bro
 # @TEST-EXEC: btest-bg-run receiver bro ../receiver.bro
 # @TEST-EXEC: btest-bg-wait -k 20

testing/btest/istate/events.bro

@@ -1,4 +1,5 @@
-# 
+# @TEST-GROUP: comm
+#
 # @TEST-EXEC: btest-bg-run sender   bro -C -r $TRACES/web.trace --pseudo-realtime ../sender.bro
 # @TEST-EXEC: btest-bg-run receiver bro ../receiver.bro
 # @TEST-EXEC: btest-bg-wait -k 20

testing/btest/istate/pybroccoli.py

@@ -1,3 +1,5 @@
+# @TEST-GROUP: comm
+#
 # @TEST-REQUIRES: test -e $BUILD/aux/broccoli/src/libbroccoli.so || test -e $BUILD/aux/broccoli/src/libbroccoli.dylib
 # @TEST-REQUIRES: test -e $BUILD/aux/broccoli/bindings/broccoli-python/_broccoli_intern.so
 #

testing/btest/istate/sync.bro

@@ -1,3 +1,4 @@
+# @TEST-GROUP: comm
 #
 # @TEST-EXEC: btest-bg-run sender   bro %INPUT ../sender.bro
 # @TEST-EXEC: btest-bg-run receiver bro %INPUT ../receiver.bro

testing/btest/scripts/base/frameworks/cluster/start-it-up.bro

@@ -1,3 +1,5 @@
+# @TEST-GROUP: comm
+#
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run proxy-2   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-2 bro %INPUT

testing/btest/scripts/base/frameworks/communication/communication_log_baseline.bro

@@ -1,4 +1,5 @@
-# 
+# @TEST-GROUP: comm
+#
 # @TEST-EXEC: btest-bg-run receiver bro -b ../receiver.bro
 # @TEST-EXEC: btest-bg-run sender   bro -b ../sender.bro
 # @TEST-EXEC: btest-bg-wait -k 10

testing/btest/scripts/base/frameworks/control/configuration_update.bro

@@ -1,3 +1,5 @@
+# @TEST-GROUP: comm
+#
 # @TEST-EXEC: btest-bg-run controllee  BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controllee Communication::listen_port=65531/tcp 
 # @TEST-EXEC: btest-bg-run controller  BROPATH=$BROPATH:.. bro %INPUT test-redef frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65531/tcp Control::cmd=configuration_update
 # @TEST-EXEC: btest-bg-run controller2 BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65531/tcp Control::cmd=shutdown

testing/btest/scripts/base/frameworks/control/id_value.bro

@@ -1,3 +1,5 @@
+# @TEST-GROUP: comm
+#
 # @TEST-EXEC: btest-bg-run controllee  BROPATH=$BROPATH:.. bro %INPUT only-for-controllee frameworks/control/controllee Communication::listen_port=65532/tcp 
 # @TEST-EXEC: btest-bg-run controller  BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65532/tcp Control::cmd=id_value Control::arg=test_var
 # @TEST-EXEC: btest-bg-wait -k 10

testing/btest/scripts/base/frameworks/control/shutdown.bro

@@ -1,3 +1,5 @@
+# @TEST-GROUP: comm
+#
 # @TEST-EXEC: btest-bg-run controllee BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controllee Communication::listen_port=65530/tcp 
 # @TEST-EXEC: btest-bg-run controller BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65530/tcp Control::cmd=shutdown
 # @TEST-EXEC: btest-bg-wait 10

testing/btest/scripts/base/frameworks/logging/remote-types.bro

@@ -1,3 +1,4 @@
+# @TEST-GROUP: comm
 #
 # @TEST-EXEC: btest-bg-run sender bro --pseudo-realtime %INPUT ../sender.bro
 # @TEST-EXEC: btest-bg-run receiver bro --pseudo-realtime %INPUT ../receiver.bro

testing/btest/scripts/base/frameworks/logging/remote.bro

@@ -1,3 +1,4 @@
+# @TEST-GROUP: comm
 #
 # @TEST-EXEC: btest-bg-run sender bro --pseudo-realtime %INPUT ../sender.bro
 # @TEST-EXEC: sleep 1

testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro

@@ -1,3 +1,5 @@
+# @TEST-GROUP: comm
+#
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
 # @TEST-EXEC: sleep 1

testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro

@@ -1,3 +1,5 @@
+# @TEST-GROUP: comm
+#
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
 # @TEST-EXEC: sleep 1

testing/btest/scripts/base/frameworks/notice/cluster.bro

@@ -1,3 +1,5 @@
+# @TEST-GROUP: comm
+#
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
 # @TEST-EXEC: sleep 1

testing/btest/scripts/base/frameworks/notice/suppression-cluster.bro

@@ -1,3 +1,5 @@
+# @TEST-GROUP: comm
+#
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
 # @TEST-EXEC: sleep 1

testing/scripts/diff-canonifier-external

@@ -6,3 +6,4 @@
     | `dirname $0`/diff-remove-uids \
     | `dirname $0`/diff-remove-mime-types \
     | `dirname $0`/diff-remove-x509-names \
+    | `dirname $0`/diff-sort

testing/scripts/diff-sort

@@ -0,0 +1,19 @@
+#! /usr/bin/env bash
+#
+# A diff canonifier that sorts all lines but keeps all comments
+# at the top. It also adds a note at the beginning as a reminder
+# that the output has been sorted.
+
+if [ "$TMP" == "" ]; then
+    TMP=/tmp
+fi
+
+tmp=$TMP/`basename $0`.$$.tmp
+
+cat >$tmp
+
+echo "### NOTE: This file has been sorted with `basename $0`."
+cat $tmp | grep ^#
+cat $tmp | grep -v ^# | sort -s
+
+rm -f $tmp