Tell OpenSSL that MD5 is not used for security in order to allow bro to work properly on a FIPS system

src/DFA.cc

@@ -6,6 +6,7 @@
 
 #include "EquivClass.h"
 #include "DFA.h"
+#include "digest.h"
 
 unsigned int DFA_State::transition_counter = 0;
 
@@ -337,7 +338,7 @@ DFA_State* DFA_State_Cache::Lookup(const NFA_state_list& nfas,
 	// We use the short MD5 instead of the full string for the
 	// HashKey because the data is copied into the key.
 	u_char digest[16];
-	MD5(id_tag, p - id_tag, digest);
+	internal_md5(id_tag, p - id_tag, digest);
 
 	*hash = new HashKey(&digest, sizeof(digest));
 	CacheEntry* e = states.Lookup(*hash);

src/DNS_Mgr.cc

@@ -2,7 +2,6 @@
 
 #include "bro-config.h"
 
-#include <openssl/md5.h>
 #include <sys/types.h>
 #include <sys/socket.h>
 #ifdef TIME_WITH_SYS_TIME
@@ -36,6 +35,7 @@
 #include "Var.h"
 #include "Reporter.h"
 #include "iosource/Manager.h"
+#include "digest.h"
 
 extern "C" {
 extern int select(int, fd_set *, fd_set *, fd_set *, struct timeval *);
@@ -468,7 +468,7 @@ void DNS_Mgr::InitPostScript()
 static TableVal* fake_name_lookup_result(const char* name)
 	{
 	uint32 hash[4];
-	MD5(reinterpret_cast<const u_char*>(name), strlen(name),
+	internal_md5(reinterpret_cast<const u_char*>(name), strlen(name),
 	    reinterpret_cast<u_char*>(hash));
 	ListVal* hv = new ListVal(TYPE_ADDR);
 	hv->Append(new AddrVal(hash));

src/OpaqueVal.cc

@@ -83,7 +83,7 @@ MD5Val::MD5Val() : HashVal(md5_type)
 
 void MD5Val::digest(val_list& vlist, u_char result[MD5_DIGEST_LENGTH])
 	{
-	MD5_CTX h;
+	EVP_MD_CTX *h;
 	md5_init(&h);
 
 	loop_over_list(vlist, i)
@@ -92,17 +92,17 @@ void MD5Val::digest(val_list& vlist, u_char result[MD5_DIGEST_LENGTH])
 		if ( v->Type()->Tag() == TYPE_STRING )
 			{
 			const BroString* str = v->AsString();
-			md5_update(&h, str->Bytes(), str->Len());
+			md5_update(h, str->Bytes(), str->Len());
 			}
 		else
 			{
 			ODesc d(DESC_BINARY);
 			v->Describe(&d);
-			md5_update(&h, (const u_char *) d.Bytes(), d.Len());
+			md5_update(h, (const u_char *) d.Bytes(), d.Len());
 			}
 		}
 
-	md5_final(&h, result);
+	md5_final(h, result);
 	}
 
 void MD5Val::hmac(val_list& vlist,
@@ -113,7 +113,7 @@ void MD5Val::hmac(val_list& vlist,
 	for ( int i = 0; i < MD5_DIGEST_LENGTH; ++i )
 		result[i] ^= key[i];
 
-	MD5(result, MD5_DIGEST_LENGTH, result);
+	internal_md5(result, MD5_DIGEST_LENGTH, result);
 	}
 
 bool MD5Val::DoInit()
@@ -128,7 +128,7 @@ bool MD5Val::DoFeed(const void* data, size_t size)
 	if ( ! IsValid() )
 		return false;
 
-	md5_update(&ctx, data, size);
+	md5_update(ctx, data, size);
 	return true;
 	}
 
@@ -138,7 +138,7 @@ StringVal* MD5Val::DoGet()
 		return new StringVal("");
 
 	u_char digest[MD5_DIGEST_LENGTH];
-	md5_final(&ctx, digest);
+	md5_final(ctx, digest);
 	return new StringVal(md5_digest_print(digest));
 	}
 
@@ -146,26 +146,27 @@ IMPLEMENT_SERIAL(MD5Val, SER_MD5_VAL);
 
 bool MD5Val::DoSerialize(SerialInfo* info) const
 	{
+	MD5_CTX *md = (MD5_CTX *) EVP_MD_CTX_md_data(ctx);
 	DO_SERIALIZE(SER_MD5_VAL, HashVal);
 
 	if ( ! IsValid() )
 		return true;
 
-	if ( ! (SERIALIZE(ctx.A) &&
+	if ( ! (SERIALIZE(md->A) &&
-		SERIALIZE(ctx.B) &&
+		SERIALIZE(md->B) &&
-		SERIALIZE(ctx.C) &&
+		SERIALIZE(md->C) &&
-		SERIALIZE(ctx.D) &&
+		SERIALIZE(md->D) &&
-		SERIALIZE(ctx.Nl) &&
+		SERIALIZE(md->Nl) &&
-		SERIALIZE(ctx.Nh)) )
+		SERIALIZE(md->Nh)) )
 		return false;
 
 	for ( int i = 0; i < MD5_LBLOCK; ++i )
 		{
-		if ( ! SERIALIZE(ctx.data[i]) )
+		if ( ! SERIALIZE(md->data[i]) )
 			return false;
 		}
 
-	if ( ! SERIALIZE(ctx.num) )
+	if ( ! SERIALIZE(md->num) )
 		return false;
 
 	return true;
@@ -173,26 +174,27 @@ bool MD5Val::DoSerialize(SerialInfo* info) const
 
 bool MD5Val::DoUnserialize(UnserialInfo* info)
 	{
+	MD5_CTX *md = (MD5_CTX *) EVP_MD_CTX_md_data(ctx);
 	DO_UNSERIALIZE(HashVal);
 
 	if ( ! IsValid() )
 		return true;
 
-	if ( ! (UNSERIALIZE(&ctx.A) &&
+	if ( ! (UNSERIALIZE(&md->A) &&
-		UNSERIALIZE(&ctx.B) &&
+		UNSERIALIZE(&md->B) &&
-		UNSERIALIZE(&ctx.C) &&
+		UNSERIALIZE(&md->C) &&
-		UNSERIALIZE(&ctx.D) &&
+		UNSERIALIZE(&md->D) &&
-		UNSERIALIZE(&ctx.Nl) &&
+		UNSERIALIZE(&md->Nl) &&
-		UNSERIALIZE(&ctx.Nh)) )
+		UNSERIALIZE(&md->Nh)) )
 		return false;
 
 	for ( int i = 0; i < MD5_LBLOCK; ++i )
 		{
-		if ( ! UNSERIALIZE(&ctx.data[i]) )
+		if ( ! UNSERIALIZE(&md->data[i]) )
 			return false;
 		}
 
-	if ( ! UNSERIALIZE(&ctx.num) )
+	if ( ! UNSERIALIZE(&md->num) )
 		return false;
 
 	return true;

src/OpaqueVal.h

@@ -56,7 +56,7 @@ protected:
 	DECLARE_SERIAL(MD5Val);
 
 private:
-	MD5_CTX ctx;
+	EVP_MD_CTX* ctx;
 };
 
 class SHA1Val : public HashVal {

src/analyzer/protocol/mime/MIME.cc

@@ -1310,7 +1310,7 @@ TableVal* MIME_Message::BuildHeaderTable(MIME_HeaderList& hlist)
 	}
 
 MIME_Mail::MIME_Mail(analyzer::Analyzer* mail_analyzer, bool orig, int buf_size)
-    : MIME_Message(mail_analyzer), md5_hash()
+: MIME_Message(mail_analyzer), md5_hash()
 	{
 	analyzer = mail_analyzer;
 
@@ -1355,7 +1355,7 @@ void MIME_Mail::Done()
 	if ( compute_content_hash && mime_content_hash )
 		{
 		u_char* digest = new u_char[16];
-		md5_final(&md5_hash, digest);
+		md5_final(md5_hash, digest);
 
 		val_list* vl = new val_list;
 		vl->append(analyzer->BuildConnVal());
@@ -1456,7 +1456,7 @@ void MIME_Mail::SubmitData(int len, const char* buf)
 	if ( compute_content_hash )
 		{
 		content_hash_length += len;
-		md5_update(&md5_hash, (const u_char*) buf, len);
+		md5_update(md5_hash, (const u_char*) buf, len);
 		}
 
 	if ( mime_entity_data || mime_all_data )

src/analyzer/protocol/mime/MIME.h

@@ -2,7 +2,7 @@
 #define ANALYZER_PROTOCOL_MIME_MIME_H
 
 #include <assert.h>
-#include <openssl/md5.h>
+#include <openssl/evp.h>
 #include <stdio.h>
 #include <vector>
 #include <queue>
@@ -252,7 +252,7 @@ protected:
 	int data_start;
 	int compute_content_hash;
 	int content_hash_length;
-	MD5_CTX md5_hash;
+	EVP_MD_CTX* md5_hash;
 	vector<const BroString*> entity_content;
 	vector<const BroString*> all_content;
 

src/digest.h

@@ -9,6 +9,17 @@
 
 #include <openssl/md5.h>
 #include <openssl/sha.h>
+#include <openssl/evp.h>
+
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+#define EVP_MD_CTX_new EVP_MD_CTX_create
+#define EVP_MD_CTX_free EVP_MD_CTX_destroy
+
+inline void *EVP_MD_CTX_md_data(const EVP_MD_CTX* ctx)
+	{
+	return ctx->md_data;
+	}
+#endif
 
 #include "Reporter.h"
 
@@ -35,24 +46,58 @@ inline const char* sha256_digest_print(const u_char digest[SHA256_DIGEST_LENGTH]
 	return digest_print(digest, SHA256_DIGEST_LENGTH);
 	}
 
-inline void md5_init(MD5_CTX* c)
+inline void md5_init(EVP_MD_CTX** c)
 	{
-	if ( ! MD5_Init(c) )
+	*c = EVP_MD_CTX_new();
+	/* Allow this to work even if FIPS disables it */
+#ifdef EVP_MD_CTX_FLAG_NON_FIPS_ALLOW
+	EVP_MD_CTX_set_flags(*c, EVP_MD_CTX_FLAG_NON_FIPS_ALLOW);
+#endif
+	if ( ! EVP_DigestInit_ex(*c, EVP_md5(), NULL) )
 		reporter->InternalError("MD5_Init failed");
 	}
 
-inline void md5_update(MD5_CTX* c, const void* data, unsigned long len)
+inline void md5_update(EVP_MD_CTX* c, const void* data, unsigned long len)
 	{
-	if ( ! MD5_Update(c, data, len) )
+	if ( ! EVP_DigestUpdate(c, data, len) )
 		reporter->InternalError("MD5_Update failed");
 	}
 
-inline void md5_final(MD5_CTX* c, u_char md[MD5_DIGEST_LENGTH])
+inline void md5_final(EVP_MD_CTX* c, u_char md[MD5_DIGEST_LENGTH])
 	{
-	if ( ! MD5_Final(md, c) )
+	if ( ! EVP_DigestFinal(c, md, NULL) )
 		reporter->InternalError("MD5_Final failed");
 	}
 
+inline unsigned char* internal_md5(const unsigned char *d, size_t n, unsigned char *md)
+	{
+		EVP_MD_CTX *c;
+		static unsigned char m[MD5_DIGEST_LENGTH];
+
+		if (md == NULL)
+			md = m;
+		md5_init(&c);
+	#ifndef CHARSET_EBCDIC
+		md5_update(c, d, n);
+	#else
+		{
+			char temp[1024];
+			unsigned long chunk;
+
+			while (n > 0) {
+				chunk = (n > sizeof(temp)) ? sizeof(temp) : n;
+				ebcdic2ascii(temp, d, chunk);
+				md5_update(c, temp, chunk);
+				n -= chunk;
+				d += chunk;
+			}
+		}
+	#endif
+		md5_final(c, md);
+		EVP_MD_CTX_free(c);
+		return md;
+	}
+
 inline void sha1_init(SHA_CTX* c)
 	{
 	if ( ! SHA1_Init(c) )

src/file_analysis/Manager.cc

@@ -10,6 +10,7 @@
 #include "Var.h"
 #include "Event.h"
 #include "UID.h"
+#include "digest.h"
 
 #include "plugin/Manager.h"
 #include "analyzer/Manager.h"
@@ -93,7 +94,7 @@ string Manager::HashHandle(const string& handle) const
 	uint64 hash[2];
 	string msg(handle + salt);
 
-	MD5(reinterpret_cast<const u_char*>(msg.data()), msg.size(),
+	internal_md5(reinterpret_cast<const u_char*>(msg.data()), msg.size(),
 	    reinterpret_cast<u_char*>(hash));
 
 	return Bro::UID(bits_per_uid, hash, 2).Base62("F");

src/probabilistic/Hasher.cc

@@ -1,7 +1,7 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
 #include <typeinfo>
-#include <openssl/md5.h>
+#include <openssl/evp.h>
 
 #include "Hasher.h"
 #include "NetVar.h"
@@ -123,13 +123,13 @@ Hasher::digest UHF::hash(const void* x, size_t n) const
 		Hasher::digest rval;
 	} u;
 
-	MD5(reinterpret_cast<const unsigned char*>(x), n, u.d);
+	internal_md5(reinterpret_cast<const unsigned char*>(x), n, u.d);
 
 	const unsigned char* s = reinterpret_cast<const unsigned char*>(&seed);
 	for ( size_t i = 0; i < 16; ++i )
 		u.d[i] ^= s[i % sizeof(seed)];
 
-	MD5(u.d, 16, u.d);
+	internal_md5(u.d, 16, u.d);
 	return u.rval;
 	}
 

src/util.cc

@@ -41,6 +41,7 @@
 # include <malloc.h>
 #endif
 
+#include "digest.h"
 #include "input.h"
 #include "util.h"
 #include "Obj.h"
@@ -712,12 +713,12 @@ void hmac_md5(size_t size, const unsigned char* bytes, unsigned char digest[16])
 	if ( ! hmac_key_set )
 		reporter->InternalError("HMAC-MD5 invoked before the HMAC key is set");
 
-	MD5(bytes, size, digest);
+	internal_md5(bytes, size, digest);
 
 	for ( int i = 0; i < 16; ++i )
 		digest[i] ^= shared_hmac_md5_key[i];
 
-	MD5(digest, 16, digest);
+	internal_md5(digest, 16, digest);
 	}
 
 static bool read_random_seeds(const char* read_file, uint32* seed,
@@ -871,7 +872,7 @@ void init_random_seed(const char* read_file, const char* write_file)
 	if ( ! hmac_key_set )
 		{
 		assert(sizeof(buf) - 16 == 64);
-		MD5((const u_char*) buf, sizeof(buf) - 16, shared_hmac_md5_key); // The last 128 bits of buf are for siphash
+		internal_md5((const u_char*) buf, sizeof(buf) - 16, shared_hmac_md5_key); // The last 128 bits of buf are for siphash
 		hmac_key_set = true;
 		}