Merge remote-tracking branch 'origin/topic/bernhard/base64'

* origin/topic/bernhard/base64:
  and re-enable caching of extracted certs
  and add bae64 bif tests.
  re-unify classes
  and modernize script.
  add base64-encode functionality and bif.

Closes #965.

CHANGES

@@ -1,4 +1,12 @@
 
+2.1-373 | 2013-03-17 12:58:39 -0700
+
+  * Add base64 encoding functionality, including new BiFs
+	encode_base64() and encode_base64_custom(). (Bernhard Amann)
+
+  * Replace call to external "openssl" in extract-certs-pem.bro with
+	that encode_base64(). (Bernhard Amann)
+
 2.1-366 | 2013-03-17 12:35:59 -0700
 
   * Correctly handle DNS lookups for software version ranges. (Seth

VERSION

@@ -1 +1 @@
-2.1-366
+2.1-373

scripts/base/protocols/ssl/main.bro

@@ -67,11 +67,6 @@ export {
 	## (especially with large file transfers).
 	const disable_analyzer_after_detection = T &redef;
 
-	## The openssl command line utility.  If it's in the path the default
-	## value will work, otherwise a full path string can be supplied for the
-	## utility.
-	const openssl_util = "openssl" &redef;
-
 	## The maximum amount of time a script can delay records from being logged.
 	const max_log_delay = 15secs &redef;
 

scripts/policy/protocols/ssl/extract-certs-pem.bro

@@ -4,14 +4,10 @@
 ##!
 ##! ..note::
 ##!
-##!     - It doesn't work well on a cluster because each worker will write its 
+##!     - It doesn't work well on a cluster because each worker will write its
 ##!       own certificate files and no duplicate checking is done across
 ##!       clusters so each node would log each certificate.
 ##!
-##!     - If there is a certificate input based vulnerability found in the
-##!       openssl command line utility, you could be in trouble because this
-##!       script uses that utility to convert from DER to PEM certificates.
-##!
 
 @load base/protocols/ssl
 @load base/utils/directions-and-hosts
@@ -20,7 +16,7 @@
 module SSL;
 
 export {
-	## Control if host certificates offered by the defined hosts 
+	## Control if host certificates offered by the defined hosts
 	## will be written to the PEM certificates file.
 	## Choices are: LOCAL_HOSTS, REMOTE_HOSTS, ALL_HOSTS, NO_HOSTS
 	const extract_certs_pem = LOCAL_HOSTS &redef;
@@ -35,15 +31,33 @@ event ssl_established(c: connection) &priority=5
 	{
 	if ( ! c$ssl?$cert )
 		return;
+
 	if ( ! addr_matches_host(c$id$resp_h, extract_certs_pem) )
 		return;
-	
+
 	if ( c$ssl$cert_hash in extracted_certs )
 		# If we already extracted this cert, don't do it again.
 		return;
-	
+
 	add extracted_certs[c$ssl$cert_hash];
-	local side = Site::is_local_addr(c$id$resp_h) ? "local" : "remote";
+	local filename = Site::is_local_addr(c$id$resp_h) ? "certs-local.pem" : "certs-remote.pem";
-	local cmd = fmt("%s x509 -inform DER -outform PEM >> certs-%s.pem", openssl_util, side);
+	local outfile = open_for_append(filename);
-	piped_exec(cmd, c$ssl$cert);
+
+	print outfile, "-----BEGIN CERTIFICATE-----";
+
+	# Encode to base64 and format to fit 50 lines. Otherwise openssl won't like it later.
+	local lines = split_all(encode_base64(c$ssl$cert), /.{50}/);
+	local i = 1;
+	for ( line in lines )
+		{
+		if ( |lines[i]| > 0 )
+			{
+			print outfile, lines[i];
+			}
+		i+=1;
+		}
+
+	print outfile, "-----END CERTIFICATE-----";
+	print outfile, "";
+	close(outfile);
 	}

src/Base64.cc

@@ -1,9 +1,47 @@
 #include "config.h"
 #include "Base64.h"
+#include <math.h>
 
 int Base64Decoder::default_base64_table[256];
 const string Base64Decoder::default_alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
 
+void Base64Decoder::Encode(int len, const unsigned char* data, int* pblen, char** pbuf)
+	{
+	int blen;
+	char *buf;
+
+	if ( ! pbuf )
+		reporter->InternalError("nil pointer to encoding result buffer");
+
+	if ( *pbuf && (*pblen % 4 != 0) )
+		reporter->InternalError("Base64 encode buffer not a multiple of 4");
+
+	if ( *pbuf )
+		{
+		buf = *pbuf;
+		blen = *pblen;
+		}
+	else
+		{
+		blen = (int)(4 * ceil((double)len / 3));
+		*pbuf = buf = new char[blen];
+		*pblen = blen;
+		}
+
+	for ( int i = 0, j = 0; (i < len) && ( j < blen ); )
+		{
+			uint32_t bit32 = ((i < len ? data[i++] : 0) << 16) +
+					 ((i < len ? data[i++] : 0 & i++) << 8) +
+					 ( i < len ? data[i++] : 0 & i++);
+
+			buf[j++] = alphabet[(bit32 >> 18) & 0x3f];
+			buf[j++] = alphabet[(bit32 >> 12) & 0x3f];
+			buf[j++] = (i == (len+2)) ? '=' : alphabet[(bit32 >> 6) & 0x3f];
+			buf[j++] = (i >= (len+1)) ? '=' : alphabet[bit32 & 0x3f];
+		}
+	}
+
+
 int* Base64Decoder::InitBase64Table(const string& alphabet)
 	{
 	assert(alphabet.size() == 64);
@@ -44,9 +82,21 @@ int* Base64Decoder::InitBase64Table(const string& alphabet)
 	return base64_table;
 	}
 
-Base64Decoder::Base64Decoder(Analyzer* arg_analyzer, const string& alphabet)
+
+
+Base64Decoder::Base64Decoder(Analyzer* arg_analyzer, const string& arg_alphabet)
 	{
-	base64_table = InitBase64Table(alphabet.size() ? alphabet : default_alphabet);
+	if ( arg_alphabet.size() > 0 )
+		{
+		assert(arg_alphabet.size() == 64);
+		alphabet = arg_alphabet;
+		}
+	else
+		{
+		alphabet = default_alphabet;
+		}
+
+	base64_table = 0;
 	base64_group_next = 0;
 	base64_padding = base64_after_padding = 0;
 	errored = 0;
@@ -64,6 +114,10 @@ int Base64Decoder::Decode(int len, const char* data, int* pblen, char** pbuf)
 	int blen;
 	char* buf;
 
+	// Initialization of table on first_time call of Decode.
+	if ( ! base64_table )
+		base64_table = InitBase64Table(alphabet);
+
 	if ( ! pbuf )
 		reporter->InternalError("nil pointer to decoding result buffer");
 
@@ -195,3 +249,21 @@ err:
 	delete [] rbuf;
 	return 0;
 	}
+
+BroString* encode_base64(const BroString* s, const BroString* a)
+	{
+	if ( a && a->Len() != 64 )
+		{
+		reporter->Error("base64 alphabet is not 64 characters: %s",
+		                a->CheckString());
+		return 0;
+		}
+
+	char* outbuf = 0;
+	int outlen = 0;
+	Base64Decoder enc(0, a ? a->CheckString() : "");
+	enc.Encode(s->Len(), (const unsigned char*) s->Bytes(), &outlen, &outbuf);
+
+	return new BroString(1, (u_char*)outbuf, outlen);
+	}
+

src/Base64.h

@@ -10,11 +10,10 @@
 #include "Analyzer.h"
 
 // Maybe we should have a base class for generic decoders?
-
 class Base64Decoder {
 public:
 	// <analyzer> is used for error reporting, and it should be zero when
-	// the decoder is called by the built-in function decode_base64().
+	// the decoder is called by the built-in function decode_base64() or encode_base64().
 	// Empty alphabet indicates the default base64 alphabet.
 	Base64Decoder(Analyzer* analyzer, const string& alphabet = "");
 	~Base64Decoder();
@@ -30,6 +29,7 @@ public:
 	// is not enough output buffer space.
 
 	int Decode(int len, const char* data, int* blen, char** buf);
+	void Encode(int len, const unsigned char* data, int* blen, char** buf);
 
 	int Done(int* pblen, char** pbuf);
 	int HasData() const { return base64_group_next != 0; }
@@ -39,7 +39,7 @@ public:
 
 	const char* ErrorMsg() const	{ return error_msg; }
 	void IllegalEncoding(const char* msg)
-		{ 
+		{
 		// strncpy(error_msg, msg, sizeof(error_msg));
 		if ( analyzer )
 			analyzer->Weird("base64_illegal_encoding", msg);
@@ -51,19 +51,22 @@ protected:
 	char error_msg[256];
 
 protected:
+	static const string default_alphabet;
+	string alphabet;
+
+	static int* InitBase64Table(const string& alphabet);
+	static int default_base64_table[256];
 	char base64_group[4];
 	int base64_group_next;
 	int base64_padding;
 	int base64_after_padding;
+	int* base64_table;
 	int errored;	// if true, we encountered an error - skip further processing
 	Analyzer* analyzer;
-	int* base64_table;
 
-	static int* InitBase64Table(const string& alphabet);
-	static int default_base64_table[256];
-	static const string default_alphabet;
 };
 
 BroString* decode_base64(const BroString* s, const BroString* a = 0);
+BroString* encode_base64(const BroString* s, const BroString* a = 0);
 
 #endif /* base64_h */

src/bro.bif

@@ -2829,13 +2829,55 @@ function bytestring_to_hexstr%(bytestring: string%): string
 	return new StringVal(hexstr);
 	%}
 
+## Encodes a Base64-encoded string.
+##
+## s: The string to encode
+##
+## Returns: The encoded version of *s*.
+##
+## .. bro:see:: encode_base64_custom, decode_base64
+function encode_base64%(s: string%): string
+	%{
+	BroString* t = encode_base64(s->AsString());
+	if ( t )
+		return new StringVal(t);
+	else
+		{
+		reporter->Error("error in encoding string %s", s->CheckString());
+		return new StringVal("");
+		}
+	%}
+
+## Encodes a Base64-encoded string with a custom alphabet.
+##
+## s: The string to encode
+##
+## a: The custom alphabet. The empty string indicates the default alphabet. The
+##    length of *a* must be 64. For example, a custom alphabet could be
+##    ``"!#$%&/(),-.:;<>@[]^ `_{|}~abcdefghijklmnopqrstuvwxyz0123456789+?"``.
+##
+## Returns: The encoded version of *s*.
+##
+## .. bro:see:: encode_base64, decode_base64_custom
+function encode_base64_custom%(s: string, a: string%): string
+	%{
+	BroString* t = encode_base64(s->AsString(), a->AsString());
+	if ( t )
+		return new StringVal(t);
+	else
+		{
+		reporter->Error("error in encoding string %s", s->CheckString());
+		return new StringVal("");
+		}
+	%}
+
 ## Decodes a Base64-encoded string.
 ##
 ## s: The Base64-encoded string.
 ##
 ## Returns: The decoded version of *s*.
 ##
-## .. bro:see:: decode_base64_custom
+## .. bro:see:: decode_base64_custom, encode_base64
 function decode_base64%(s: string%): string
 	%{
 	BroString* t = decode_base64(s->AsString());
@@ -2858,7 +2900,7 @@ function decode_base64%(s: string%): string
 ##
 ## Returns: The decoded version of *s*.
 ##
-## .. bro:see:: decode_base64
+## .. bro:see:: decode_base64, encode_base64_custom
 function decode_base64_custom%(s: string, a: string%): string
 	%{
 	BroString* t = decode_base64(s->AsString(), a->AsString());

testing/btest/Baseline/bifs.encode_base64/out

@@ -0,0 +1,6 @@
+YnJv
+YnJv
+}n-v
+cGFkZGluZw==
+cGFkZGluZzE=
+cGFkZGluZzEy

testing/btest/bifs/encode_base64.bro

@@ -0,0 +1,14 @@
+# @TEST-EXEC: bro -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+global default_alphabet: string = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+global my_alphabet: string = "!#$%&/(),-.:;<>@[]^ `_{|}~abcdefghijklmnopqrstuvwxyz0123456789+?";
+
+print encode_base64("bro");
+print encode_base64_custom("bro", default_alphabet);
+print encode_base64_custom("bro", my_alphabet);
+
+print encode_base64("padding");
+print encode_base64("padding1");
+print encode_base64("padding12");