From a72e9a8126566aaee8408f7324fb63f3adbeaba5 Mon Sep 17 00:00:00 2001 From: Robert Clark Date: Fri, 26 Oct 2018 10:32:21 -0400 Subject: [PATCH 1/2] Tell OpenSSL that MD5 is not used for security in order to allow bro to work properly on a FIPS system --- src/DFA.cc | 5 +-- src/DNS_Mgr.cc | 4 +-- src/OpaqueVal.cc | 48 +++++++++++++------------ src/OpaqueVal.h | 2 +- src/analyzer/protocol/mime/MIME.cc | 8 ++--- src/analyzer/protocol/mime/MIME.h | 4 +-- src/digest.h | 57 ++++++++++++++++++++++++++---- src/file_analysis/Manager.cc | 3 +- src/probabilistic/Hasher.cc | 6 ++-- src/util.cc | 7 ++-- 10 files changed, 97 insertions(+), 47 deletions(-) diff --git a/src/DFA.cc b/src/DFA.cc index 5885a9bf3b..4ad6a90141 100644 --- a/src/DFA.cc +++ b/src/DFA.cc @@ -6,6 +6,7 @@ #include "EquivClass.h" #include "DFA.h" +#include "digest.h" unsigned int DFA_State::transition_counter = 0; @@ -337,7 +338,7 @@ DFA_State* DFA_State_Cache::Lookup(const NFA_state_list& nfas, // We use the short MD5 instead of the full string for the // HashKey because the data is copied into the key. u_char digest[16]; - MD5(id_tag, p - id_tag, digest); + internal_md5(id_tag, p - id_tag, digest); *hash = new HashKey(&digest, sizeof(digest)); CacheEntry* e = states.Lookup(*hash); @@ -395,7 +396,7 @@ DFA_Machine::DFA_Machine(NFA_Machine* n, EquivClass* arg_ec) { state_count = 0; - nfa = n; + nfa = n; Ref(n); ec = arg_ec; diff --git a/src/DNS_Mgr.cc b/src/DNS_Mgr.cc index 266217c02a..d6bdbb4f84 100644 --- a/src/DNS_Mgr.cc +++ b/src/DNS_Mgr.cc @@ -2,7 +2,6 @@ #include "bro-config.h" -#include #include #include #ifdef TIME_WITH_SYS_TIME @@ -36,6 +35,7 @@ #include "Var.h" #include "Reporter.h" #include "iosource/Manager.h" +#include "digest.h" extern "C" { extern int select(int, fd_set *, fd_set *, fd_set *, struct timeval *); @@ -468,7 +468,7 @@ void DNS_Mgr::InitPostScript() static TableVal* fake_name_lookup_result(const char* name) { uint32 hash[4]; - MD5(reinterpret_cast(name), strlen(name), + internal_md5(reinterpret_cast(name), strlen(name), reinterpret_cast(hash)); ListVal* hv = new ListVal(TYPE_ADDR); hv->Append(new AddrVal(hash)); diff --git a/src/OpaqueVal.cc b/src/OpaqueVal.cc index ea0c73d297..f88e3debbd 100644 --- a/src/OpaqueVal.cc +++ b/src/OpaqueVal.cc @@ -83,7 +83,7 @@ MD5Val::MD5Val() : HashVal(md5_type) void MD5Val::digest(val_list& vlist, u_char result[MD5_DIGEST_LENGTH]) { - MD5_CTX h; + EVP_MD_CTX *h; md5_init(&h); loop_over_list(vlist, i) @@ -92,17 +92,17 @@ void MD5Val::digest(val_list& vlist, u_char result[MD5_DIGEST_LENGTH]) if ( v->Type()->Tag() == TYPE_STRING ) { const BroString* str = v->AsString(); - md5_update(&h, str->Bytes(), str->Len()); + md5_update(h, str->Bytes(), str->Len()); } else { ODesc d(DESC_BINARY); v->Describe(&d); - md5_update(&h, (const u_char *) d.Bytes(), d.Len()); + md5_update(h, (const u_char *) d.Bytes(), d.Len()); } } - md5_final(&h, result); + md5_final(h, result); } void MD5Val::hmac(val_list& vlist, @@ -113,7 +113,7 @@ void MD5Val::hmac(val_list& vlist, for ( int i = 0; i < MD5_DIGEST_LENGTH; ++i ) result[i] ^= key[i]; - MD5(result, MD5_DIGEST_LENGTH, result); + internal_md5(result, MD5_DIGEST_LENGTH, result); } bool MD5Val::DoInit() @@ -128,7 +128,7 @@ bool MD5Val::DoFeed(const void* data, size_t size) if ( ! IsValid() ) return false; - md5_update(&ctx, data, size); + md5_update(ctx, data, size); return true; } @@ -138,7 +138,7 @@ StringVal* MD5Val::DoGet() return new StringVal(""); u_char digest[MD5_DIGEST_LENGTH]; - md5_final(&ctx, digest); + md5_final(ctx, digest); return new StringVal(md5_digest_print(digest)); } @@ -146,26 +146,27 @@ IMPLEMENT_SERIAL(MD5Val, SER_MD5_VAL); bool MD5Val::DoSerialize(SerialInfo* info) const { + MD5_CTX *md = (MD5_CTX *) EVP_MD_CTX_md_data(ctx); DO_SERIALIZE(SER_MD5_VAL, HashVal); if ( ! IsValid() ) return true; - if ( ! (SERIALIZE(ctx.A) && - SERIALIZE(ctx.B) && - SERIALIZE(ctx.C) && - SERIALIZE(ctx.D) && - SERIALIZE(ctx.Nl) && - SERIALIZE(ctx.Nh)) ) + if ( ! (SERIALIZE(md->A) && + SERIALIZE(md->B) && + SERIALIZE(md->C) && + SERIALIZE(md->D) && + SERIALIZE(md->Nl) && + SERIALIZE(md->Nh)) ) return false; for ( int i = 0; i < MD5_LBLOCK; ++i ) { - if ( ! SERIALIZE(ctx.data[i]) ) + if ( ! SERIALIZE(md->data[i]) ) return false; } - if ( ! SERIALIZE(ctx.num) ) + if ( ! SERIALIZE(md->num) ) return false; return true; @@ -173,26 +174,27 @@ bool MD5Val::DoSerialize(SerialInfo* info) const bool MD5Val::DoUnserialize(UnserialInfo* info) { + MD5_CTX *md = (MD5_CTX *) EVP_MD_CTX_md_data(ctx); DO_UNSERIALIZE(HashVal); if ( ! IsValid() ) return true; - if ( ! (UNSERIALIZE(&ctx.A) && - UNSERIALIZE(&ctx.B) && - UNSERIALIZE(&ctx.C) && - UNSERIALIZE(&ctx.D) && - UNSERIALIZE(&ctx.Nl) && - UNSERIALIZE(&ctx.Nh)) ) + if ( ! (UNSERIALIZE(&md->A) && + UNSERIALIZE(&md->B) && + UNSERIALIZE(&md->C) && + UNSERIALIZE(&md->D) && + UNSERIALIZE(&md->Nl) && + UNSERIALIZE(&md->Nh)) ) return false; for ( int i = 0; i < MD5_LBLOCK; ++i ) { - if ( ! UNSERIALIZE(&ctx.data[i]) ) + if ( ! UNSERIALIZE(&md->data[i]) ) return false; } - if ( ! UNSERIALIZE(&ctx.num) ) + if ( ! UNSERIALIZE(&md->num) ) return false; return true; diff --git a/src/OpaqueVal.h b/src/OpaqueVal.h index 61549f414a..62a78d867c 100644 --- a/src/OpaqueVal.h +++ b/src/OpaqueVal.h @@ -56,7 +56,7 @@ protected: DECLARE_SERIAL(MD5Val); private: - MD5_CTX ctx; + EVP_MD_CTX* ctx; }; class SHA1Val : public HashVal { diff --git a/src/analyzer/protocol/mime/MIME.cc b/src/analyzer/protocol/mime/MIME.cc index 19d3dbe5d3..99776c9d9e 100644 --- a/src/analyzer/protocol/mime/MIME.cc +++ b/src/analyzer/protocol/mime/MIME.cc @@ -886,7 +886,7 @@ int MIME_Entity::ParseFieldParameters(int len, const char* data) // token or quoted-string (and some lenience for characters // not explicitly allowed by the RFC, but encountered in the wild) offset = MIME_get_value(len, data, val, true); - + if ( ! val ) { IllegalFormat("Could not parse multipart boundary"); @@ -1310,7 +1310,7 @@ TableVal* MIME_Message::BuildHeaderTable(MIME_HeaderList& hlist) } MIME_Mail::MIME_Mail(analyzer::Analyzer* mail_analyzer, bool orig, int buf_size) - : MIME_Message(mail_analyzer), md5_hash() +: MIME_Message(mail_analyzer), md5_hash() { analyzer = mail_analyzer; @@ -1355,7 +1355,7 @@ void MIME_Mail::Done() if ( compute_content_hash && mime_content_hash ) { u_char* digest = new u_char[16]; - md5_final(&md5_hash, digest); + md5_final(md5_hash, digest); val_list* vl = new val_list; vl->append(analyzer->BuildConnVal()); @@ -1456,7 +1456,7 @@ void MIME_Mail::SubmitData(int len, const char* buf) if ( compute_content_hash ) { content_hash_length += len; - md5_update(&md5_hash, (const u_char*) buf, len); + md5_update(md5_hash, (const u_char*) buf, len); } if ( mime_entity_data || mime_all_data ) diff --git a/src/analyzer/protocol/mime/MIME.h b/src/analyzer/protocol/mime/MIME.h index a9ef89b932..9fcf2ea468 100644 --- a/src/analyzer/protocol/mime/MIME.h +++ b/src/analyzer/protocol/mime/MIME.h @@ -2,7 +2,7 @@ #define ANALYZER_PROTOCOL_MIME_MIME_H #include -#include +#include #include #include #include @@ -252,7 +252,7 @@ protected: int data_start; int compute_content_hash; int content_hash_length; - MD5_CTX md5_hash; + EVP_MD_CTX* md5_hash; vector entity_content; vector all_content; diff --git a/src/digest.h b/src/digest.h index a6057c53b2..b01981c56b 100644 --- a/src/digest.h +++ b/src/digest.h @@ -9,6 +9,17 @@ #include #include +#include + +#if OPENSSL_VERSION_NUMBER < 0x10100000L +#define EVP_MD_CTX_new EVP_MD_CTX_create +#define EVP_MD_CTX_free EVP_MD_CTX_destroy + +inline void *EVP_MD_CTX_md_data(const EVP_MD_CTX* ctx) + { + return ctx->md_data; + } +#endif #include "Reporter.h" @@ -35,24 +46,58 @@ inline const char* sha256_digest_print(const u_char digest[SHA256_DIGEST_LENGTH] return digest_print(digest, SHA256_DIGEST_LENGTH); } -inline void md5_init(MD5_CTX* c) +inline void md5_init(EVP_MD_CTX** c) { - if ( ! MD5_Init(c) ) + *c = EVP_MD_CTX_new(); + /* Allow this to work even if FIPS disables it */ +#ifdef EVP_MD_CTX_FLAG_NON_FIPS_ALLOW + EVP_MD_CTX_set_flags(*c, EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); +#endif + if ( ! EVP_DigestInit_ex(*c, EVP_md5(), NULL) ) reporter->InternalError("MD5_Init failed"); } -inline void md5_update(MD5_CTX* c, const void* data, unsigned long len) +inline void md5_update(EVP_MD_CTX* c, const void* data, unsigned long len) { - if ( ! MD5_Update(c, data, len) ) + if ( ! EVP_DigestUpdate(c, data, len) ) reporter->InternalError("MD5_Update failed"); } -inline void md5_final(MD5_CTX* c, u_char md[MD5_DIGEST_LENGTH]) +inline void md5_final(EVP_MD_CTX* c, u_char md[MD5_DIGEST_LENGTH]) { - if ( ! MD5_Final(md, c) ) + if ( ! EVP_DigestFinal(c, md, NULL) ) reporter->InternalError("MD5_Final failed"); } +inline unsigned char* internal_md5(const unsigned char *d, size_t n, unsigned char *md) + { + EVP_MD_CTX *c; + static unsigned char m[MD5_DIGEST_LENGTH]; + + if (md == NULL) + md = m; + md5_init(&c); + #ifndef CHARSET_EBCDIC + md5_update(c, d, n); + #else + { + char temp[1024]; + unsigned long chunk; + + while (n > 0) { + chunk = (n > sizeof(temp)) ? sizeof(temp) : n; + ebcdic2ascii(temp, d, chunk); + md5_update(c, temp, chunk); + n -= chunk; + d += chunk; + } + } + #endif + md5_final(c, md); + EVP_MD_CTX_free(c); + return md; + } + inline void sha1_init(SHA_CTX* c) { if ( ! SHA1_Init(c) ) diff --git a/src/file_analysis/Manager.cc b/src/file_analysis/Manager.cc index b095315de8..acc95a9981 100644 --- a/src/file_analysis/Manager.cc +++ b/src/file_analysis/Manager.cc @@ -10,6 +10,7 @@ #include "Var.h" #include "Event.h" #include "UID.h" +#include "digest.h" #include "plugin/Manager.h" #include "analyzer/Manager.h" @@ -93,7 +94,7 @@ string Manager::HashHandle(const string& handle) const uint64 hash[2]; string msg(handle + salt); - MD5(reinterpret_cast(msg.data()), msg.size(), + internal_md5(reinterpret_cast(msg.data()), msg.size(), reinterpret_cast(hash)); return Bro::UID(bits_per_uid, hash, 2).Base62("F"); diff --git a/src/probabilistic/Hasher.cc b/src/probabilistic/Hasher.cc index 150be57224..048c7d02d2 100644 --- a/src/probabilistic/Hasher.cc +++ b/src/probabilistic/Hasher.cc @@ -1,7 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. #include -#include +#include #include "Hasher.h" #include "NetVar.h" @@ -123,13 +123,13 @@ Hasher::digest UHF::hash(const void* x, size_t n) const Hasher::digest rval; } u; - MD5(reinterpret_cast(x), n, u.d); + internal_md5(reinterpret_cast(x), n, u.d); const unsigned char* s = reinterpret_cast(&seed); for ( size_t i = 0; i < 16; ++i ) u.d[i] ^= s[i % sizeof(seed)]; - MD5(u.d, 16, u.d); + internal_md5(u.d, 16, u.d); return u.rval; } diff --git a/src/util.cc b/src/util.cc index a1cd138b1e..cce49a7f6d 100644 --- a/src/util.cc +++ b/src/util.cc @@ -41,6 +41,7 @@ # include #endif +#include "digest.h" #include "input.h" #include "util.h" #include "Obj.h" @@ -712,12 +713,12 @@ void hmac_md5(size_t size, const unsigned char* bytes, unsigned char digest[16]) if ( ! hmac_key_set ) reporter->InternalError("HMAC-MD5 invoked before the HMAC key is set"); - MD5(bytes, size, digest); + internal_md5(bytes, size, digest); for ( int i = 0; i < 16; ++i ) digest[i] ^= shared_hmac_md5_key[i]; - MD5(digest, 16, digest); + internal_md5(digest, 16, digest); } static bool read_random_seeds(const char* read_file, uint32* seed, @@ -871,7 +872,7 @@ void init_random_seed(const char* read_file, const char* write_file) if ( ! hmac_key_set ) { assert(sizeof(buf) - 16 == 64); - MD5((const u_char*) buf, sizeof(buf) - 16, shared_hmac_md5_key); // The last 128 bits of buf are for siphash + internal_md5((const u_char*) buf, sizeof(buf) - 16, shared_hmac_md5_key); // The last 128 bits of buf are for siphash hmac_key_set = true; } From 86161c85c497207583519fcd100be6e7d432bfa5 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Thu, 24 Jan 2019 09:19:29 -0800 Subject: [PATCH 2/2] A few more updates to the digest functions. This builds upon the previous commit to make Zeek compile on FIPS systems. This patch makes the changes a bit more aggressive. Instead of having a number of different hash functions with different return values, we now standardize on EVP_MD_CTX and just have one set of functions, to which the hash algorithm that is desired is passed. On the positive side, this enables us to support a wider range of hash algorithm (and to easily add to them in the future). I reimplemented the internal_md5 function - we don't support ebdic systems in any case. The md5/sha1 serialization functions are now also tested (I don't think they were before). --- src/DFA.cc | 2 - src/OpaqueVal.cc | 131 ++++++++++-------- src/OpaqueVal.h | 4 +- src/analyzer/protocol/mime/MIME.cc | 6 +- src/digest.h | 115 ++++++--------- src/probabilistic/BitVector.cc | 7 +- src/probabilistic/Hasher.cc | 11 +- .../broker.store.type-conversion/master.out | 2 + .../btest/broker/store/type-conversion.bro | 14 ++ 9 files changed, 149 insertions(+), 143 deletions(-) diff --git a/src/DFA.cc b/src/DFA.cc index 4ad6a90141..00f56ef16e 100644 --- a/src/DFA.cc +++ b/src/DFA.cc @@ -2,8 +2,6 @@ #include "bro-config.h" -#include - #include "EquivClass.h" #include "DFA.h" #include "digest.h" diff --git a/src/OpaqueVal.cc b/src/OpaqueVal.cc index e1efb5686d..d27cd84964 100644 --- a/src/OpaqueVal.cc +++ b/src/OpaqueVal.cc @@ -79,12 +79,14 @@ bool HashVal::DoUnserialize(UnserialInfo* info) MD5Val::MD5Val() : HashVal(md5_type) { + if ( IsValid() ) + // prevent leaks... + EVP_MD_CTX_free(ctx); } void MD5Val::digest(val_list& vlist, u_char result[MD5_DIGEST_LENGTH]) { - EVP_MD_CTX *h; - md5_init(&h); + EVP_MD_CTX* h = hash_init(Hash_MD5); loop_over_list(vlist, i) { @@ -92,17 +94,17 @@ void MD5Val::digest(val_list& vlist, u_char result[MD5_DIGEST_LENGTH]) if ( v->Type()->Tag() == TYPE_STRING ) { const BroString* str = v->AsString(); - md5_update(h, str->Bytes(), str->Len()); + hash_update(h, str->Bytes(), str->Len()); } else { ODesc d(DESC_BINARY); v->Describe(&d); - md5_update(h, (const u_char *) d.Bytes(), d.Len()); + hash_update(h, (const u_char *) d.Bytes(), d.Len()); } } - md5_final(h, result); + hash_final(h, result); } void MD5Val::hmac(val_list& vlist, @@ -119,7 +121,7 @@ void MD5Val::hmac(val_list& vlist, bool MD5Val::DoInit() { assert(! IsValid()); - md5_init(&ctx); + ctx = hash_init(Hash_MD5); return true; } @@ -128,7 +130,7 @@ bool MD5Val::DoFeed(const void* data, size_t size) if ( ! IsValid() ) return false; - md5_update(ctx, data, size); + hash_update(ctx, data, size); return true; } @@ -138,7 +140,7 @@ StringVal* MD5Val::DoGet() return val_mgr->GetEmptyString(); u_char digest[MD5_DIGEST_LENGTH]; - md5_final(ctx, digest); + hash_final(ctx, digest); return new StringVal(md5_digest_print(digest)); } @@ -146,12 +148,13 @@ IMPLEMENT_SERIAL(MD5Val, SER_MD5_VAL); bool MD5Val::DoSerialize(SerialInfo* info) const { - MD5_CTX *md = (MD5_CTX *) EVP_MD_CTX_md_data(ctx); DO_SERIALIZE(SER_MD5_VAL, HashVal); if ( ! IsValid() ) return true; + MD5_CTX *md = (MD5_CTX *) EVP_MD_CTX_md_data(ctx); + if ( ! (SERIALIZE(md->A) && SERIALIZE(md->B) && SERIALIZE(md->C) && @@ -174,12 +177,14 @@ bool MD5Val::DoSerialize(SerialInfo* info) const bool MD5Val::DoUnserialize(UnserialInfo* info) { - MD5_CTX *md = (MD5_CTX *) EVP_MD_CTX_md_data(ctx); DO_UNSERIALIZE(HashVal); if ( ! IsValid() ) return true; + ctx = hash_init(Hash_MD5); + MD5_CTX *md = (MD5_CTX *) EVP_MD_CTX_md_data(ctx); + if ( ! (UNSERIALIZE(&md->A) && UNSERIALIZE(&md->B) && UNSERIALIZE(&md->C) && @@ -202,12 +207,14 @@ bool MD5Val::DoUnserialize(UnserialInfo* info) SHA1Val::SHA1Val() : HashVal(sha1_type) { + if ( IsValid() ) + // prevent leaks... + EVP_MD_CTX_free(ctx); } void SHA1Val::digest(val_list& vlist, u_char result[SHA_DIGEST_LENGTH]) { - SHA_CTX h; - sha1_init(&h); + EVP_MD_CTX* h = hash_init(Hash_SHA1); loop_over_list(vlist, i) { @@ -215,23 +222,23 @@ void SHA1Val::digest(val_list& vlist, u_char result[SHA_DIGEST_LENGTH]) if ( v->Type()->Tag() == TYPE_STRING ) { const BroString* str = v->AsString(); - sha1_update(&h, str->Bytes(), str->Len()); + hash_update(h, str->Bytes(), str->Len()); } else { ODesc d(DESC_BINARY); v->Describe(&d); - sha1_update(&h, (const u_char *) d.Bytes(), d.Len()); + hash_update(h, (const u_char *) d.Bytes(), d.Len()); } } - sha1_final(&h, result); + hash_final(h, result); } bool SHA1Val::DoInit() { assert(! IsValid()); - sha1_init(&ctx); + ctx = hash_init(Hash_SHA1); return true; } @@ -240,7 +247,7 @@ bool SHA1Val::DoFeed(const void* data, size_t size) if ( ! IsValid() ) return false; - sha1_update(&ctx, data, size); + hash_update(ctx, data, size); return true; } @@ -250,7 +257,7 @@ StringVal* SHA1Val::DoGet() return val_mgr->GetEmptyString(); u_char digest[SHA_DIGEST_LENGTH]; - sha1_final(&ctx, digest); + hash_final(ctx, digest); return new StringVal(sha1_digest_print(digest)); } @@ -263,22 +270,24 @@ bool SHA1Val::DoSerialize(SerialInfo* info) const if ( ! IsValid() ) return true; - if ( ! (SERIALIZE(ctx.h0) && - SERIALIZE(ctx.h1) && - SERIALIZE(ctx.h2) && - SERIALIZE(ctx.h3) && - SERIALIZE(ctx.h4) && - SERIALIZE(ctx.Nl) && - SERIALIZE(ctx.Nh)) ) + SHA_CTX *md = (SHA_CTX *) EVP_MD_CTX_md_data(ctx); + + if ( ! (SERIALIZE(md->h0) && + SERIALIZE(md->h1) && + SERIALIZE(md->h2) && + SERIALIZE(md->h3) && + SERIALIZE(md->h4) && + SERIALIZE(md->Nl) && + SERIALIZE(md->Nh)) ) return false; for ( int i = 0; i < SHA_LBLOCK; ++i ) { - if ( ! SERIALIZE(ctx.data[i]) ) + if ( ! SERIALIZE(md->data[i]) ) return false; } - if ( ! SERIALIZE(ctx.num) ) + if ( ! SERIALIZE(md->num) ) return false; return true; @@ -291,22 +300,25 @@ bool SHA1Val::DoUnserialize(UnserialInfo* info) if ( ! IsValid() ) return true; - if ( ! (UNSERIALIZE(&ctx.h0) && - UNSERIALIZE(&ctx.h1) && - UNSERIALIZE(&ctx.h2) && - UNSERIALIZE(&ctx.h3) && - UNSERIALIZE(&ctx.h4) && - UNSERIALIZE(&ctx.Nl) && - UNSERIALIZE(&ctx.Nh)) ) + ctx = hash_init(Hash_SHA1); + SHA_CTX *md = (SHA_CTX *) EVP_MD_CTX_md_data(ctx); + + if ( ! (UNSERIALIZE(&md->h0) && + UNSERIALIZE(&md->h1) && + UNSERIALIZE(&md->h2) && + UNSERIALIZE(&md->h3) && + UNSERIALIZE(&md->h4) && + UNSERIALIZE(&md->Nl) && + UNSERIALIZE(&md->Nh)) ) return false; for ( int i = 0; i < SHA_LBLOCK; ++i ) { - if ( ! UNSERIALIZE(&ctx.data[i]) ) + if ( ! UNSERIALIZE(&md->data[i]) ) return false; } - if ( ! UNSERIALIZE(&ctx.num) ) + if ( ! UNSERIALIZE(&md->num) ) return false; return true; @@ -314,12 +326,14 @@ bool SHA1Val::DoUnserialize(UnserialInfo* info) SHA256Val::SHA256Val() : HashVal(sha256_type) { + if ( IsValid() ) + // prevent leaks... + EVP_MD_CTX_free(ctx); } void SHA256Val::digest(val_list& vlist, u_char result[SHA256_DIGEST_LENGTH]) { - SHA256_CTX h; - sha256_init(&h); + EVP_MD_CTX* h = hash_init(Hash_SHA256); loop_over_list(vlist, i) { @@ -327,23 +341,23 @@ void SHA256Val::digest(val_list& vlist, u_char result[SHA256_DIGEST_LENGTH]) if ( v->Type()->Tag() == TYPE_STRING ) { const BroString* str = v->AsString(); - sha256_update(&h, str->Bytes(), str->Len()); + hash_update(h, str->Bytes(), str->Len()); } else { ODesc d(DESC_BINARY); v->Describe(&d); - sha256_update(&h, (const u_char *) d.Bytes(), d.Len()); + hash_update(h, (const u_char *) d.Bytes(), d.Len()); } } - sha256_final(&h, result); + hash_final(h, result); } bool SHA256Val::DoInit() { assert( ! IsValid() ); - sha256_init(&ctx); + ctx = hash_init(Hash_SHA256); return true; } @@ -352,7 +366,7 @@ bool SHA256Val::DoFeed(const void* data, size_t size) if ( ! IsValid() ) return false; - sha256_update(&ctx, data, size); + hash_update(ctx, data, size); return true; } @@ -362,7 +376,7 @@ StringVal* SHA256Val::DoGet() return val_mgr->GetEmptyString(); u_char digest[SHA256_DIGEST_LENGTH]; - sha256_final(&ctx, digest); + hash_final(ctx, digest); return new StringVal(sha256_digest_print(digest)); } @@ -375,24 +389,26 @@ bool SHA256Val::DoSerialize(SerialInfo* info) const if ( ! IsValid() ) return true; + SHA256_CTX *md = (SHA256_CTX *) EVP_MD_CTX_md_data(ctx); + for ( int i = 0; i < 8; ++i ) { - if ( ! SERIALIZE(ctx.h[i]) ) + if ( ! SERIALIZE(md->h[i]) ) return false; } - if ( ! (SERIALIZE(ctx.Nl) && - SERIALIZE(ctx.Nh)) ) + if ( ! (SERIALIZE(md->Nl) && + SERIALIZE(md->Nh)) ) return false; for ( int i = 0; i < SHA_LBLOCK; ++i ) { - if ( ! SERIALIZE(ctx.data[i]) ) + if ( ! SERIALIZE(md->data[i]) ) return false; } - if ( ! (SERIALIZE(ctx.num) && - SERIALIZE(ctx.md_len)) ) + if ( ! (SERIALIZE(md->num) && + SERIALIZE(md->md_len)) ) return false; return true; @@ -405,25 +421,28 @@ bool SHA256Val::DoUnserialize(UnserialInfo* info) if ( ! IsValid() ) return true; + ctx = hash_init(Hash_SHA256); + SHA256_CTX *md = (SHA256_CTX *) EVP_MD_CTX_md_data(ctx); + for ( int i = 0; i < 8; ++i ) { - if ( ! UNSERIALIZE(&ctx.h[i]) ) + if ( ! UNSERIALIZE(&md->h[i]) ) return false; } - if ( ! (UNSERIALIZE(&ctx.Nl) && - UNSERIALIZE(&ctx.Nh)) ) + if ( ! (UNSERIALIZE(&md->Nl) && + UNSERIALIZE(&md->Nh)) ) return false; for ( int i = 0; i < SHA_LBLOCK; ++i ) { - if ( ! UNSERIALIZE(&ctx.data[i]) ) + if ( ! UNSERIALIZE(&md->data[i]) ) return false; } - if ( ! (UNSERIALIZE(&ctx.num) && - UNSERIALIZE(&ctx.md_len)) ) + if ( ! (UNSERIALIZE(&md->num) && + UNSERIALIZE(&md->md_len)) ) return false; return true; diff --git a/src/OpaqueVal.h b/src/OpaqueVal.h index 62a78d867c..f061cbc376 100644 --- a/src/OpaqueVal.h +++ b/src/OpaqueVal.h @@ -75,7 +75,7 @@ protected: DECLARE_SERIAL(SHA1Val); private: - SHA_CTX ctx; + EVP_MD_CTX* ctx; }; class SHA256Val : public HashVal { @@ -94,7 +94,7 @@ protected: DECLARE_SERIAL(SHA256Val); private: - SHA256_CTX ctx; + EVP_MD_CTX* ctx; }; class EntropyVal : public OpaqueVal { diff --git a/src/analyzer/protocol/mime/MIME.cc b/src/analyzer/protocol/mime/MIME.cc index 8563c9ed22..2798fe3bb4 100644 --- a/src/analyzer/protocol/mime/MIME.cc +++ b/src/analyzer/protocol/mime/MIME.cc @@ -1335,7 +1335,7 @@ MIME_Mail::MIME_Mail(analyzer::Analyzer* mail_analyzer, bool orig, int buf_size) if ( mime_content_hash ) { compute_content_hash = 1; - md5_init(&md5_hash); + md5_hash = hash_init(Hash_MD5); } else compute_content_hash = 0; @@ -1355,7 +1355,7 @@ void MIME_Mail::Done() if ( compute_content_hash && mime_content_hash ) { u_char* digest = new u_char[16]; - md5_final(md5_hash, digest); + hash_final(md5_hash, digest); val_list* vl = new val_list; vl->append(analyzer->BuildConnVal()); @@ -1456,7 +1456,7 @@ void MIME_Mail::SubmitData(int len, const char* buf) if ( compute_content_hash ) { content_hash_length += len; - md5_update(md5_hash, (const u_char*) buf, len); + hash_update(md5_hash, (const u_char*) buf, len); } if ( mime_entity_data || mime_all_data ) diff --git a/src/digest.h b/src/digest.h index b01981c56b..b9dedfc382 100644 --- a/src/digest.h +++ b/src/digest.h @@ -23,6 +23,8 @@ inline void *EVP_MD_CTX_md_data(const EVP_MD_CTX* ctx) #include "Reporter.h" +enum HashAlgorithm { Hash_MD5, Hash_SHA1, Hash_SHA224, Hash_SHA256, Hash_SHA384, Hash_SHA512 }; + inline const char* digest_print(const u_char* digest, size_t n) { static char buf[256]; // big enough for any of md5/sha1/sha256 @@ -46,92 +48,65 @@ inline const char* sha256_digest_print(const u_char digest[SHA256_DIGEST_LENGTH] return digest_print(digest, SHA256_DIGEST_LENGTH); } -inline void md5_init(EVP_MD_CTX** c) +inline EVP_MD_CTX* hash_init(HashAlgorithm alg) { - *c = EVP_MD_CTX_new(); + EVP_MD_CTX *c = EVP_MD_CTX_new(); /* Allow this to work even if FIPS disables it */ + const EVP_MD* md; + switch (alg) + { + case Hash_MD5: #ifdef EVP_MD_CTX_FLAG_NON_FIPS_ALLOW - EVP_MD_CTX_set_flags(*c, EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); + EVP_MD_CTX_set_flags(c, EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); #endif - if ( ! EVP_DigestInit_ex(*c, EVP_md5(), NULL) ) - reporter->InternalError("MD5_Init failed"); + md = EVP_md5(); + break; + case Hash_SHA1: + md = EVP_sha1(); + break; + case Hash_SHA224: + md = EVP_sha224(); + break; + case Hash_SHA256: + md = EVP_sha256(); + break; + case Hash_SHA384: + md = EVP_sha384(); + break; + case Hash_SHA512: + md = EVP_sha512(); + break; + default: + reporter->InternalError("Unknown hash algorithm passed to hash_init"); + } + if ( ! EVP_DigestInit_ex(c, md, NULL) ) + reporter->InternalError("EVP_DigestInit failed"); + return c; } -inline void md5_update(EVP_MD_CTX* c, const void* data, unsigned long len) +inline void hash_update(EVP_MD_CTX* c, const void* data, unsigned long len) { if ( ! EVP_DigestUpdate(c, data, len) ) - reporter->InternalError("MD5_Update failed"); + reporter->InternalError("EVP_DigestUpdate failed"); } -inline void md5_final(EVP_MD_CTX* c, u_char md[MD5_DIGEST_LENGTH]) +inline void hash_final(EVP_MD_CTX* c, u_char md[MD5_DIGEST_LENGTH]) { if ( ! EVP_DigestFinal(c, md, NULL) ) - reporter->InternalError("MD5_Final failed"); + reporter->InternalError("EVP_DigestFinal failed"); + EVP_MD_CTX_free(c); } -inline unsigned char* internal_md5(const unsigned char *d, size_t n, unsigned char *md) +inline unsigned char* internal_md5(const unsigned char *data, unsigned long len, unsigned char *out) { - EVP_MD_CTX *c; - static unsigned char m[MD5_DIGEST_LENGTH]; + static unsigned char static_out[MD5_DIGEST_LENGTH]; + if ( ! out ) + out = static_out; // use static array for return, see OpenSSL man page - if (md == NULL) - md = m; - md5_init(&c); - #ifndef CHARSET_EBCDIC - md5_update(c, d, n); - #else - { - char temp[1024]; - unsigned long chunk; - - while (n > 0) { - chunk = (n > sizeof(temp)) ? sizeof(temp) : n; - ebcdic2ascii(temp, d, chunk); - md5_update(c, temp, chunk); - n -= chunk; - d += chunk; - } - } - #endif - md5_final(c, md); - EVP_MD_CTX_free(c); - return md; - } - -inline void sha1_init(SHA_CTX* c) - { - if ( ! SHA1_Init(c) ) - reporter->InternalError("SHA_Init failed"); - } - -inline void sha1_update(SHA_CTX* c, const void* data, unsigned long len) - { - if ( ! SHA1_Update(c, data, len) ) - reporter->InternalError("SHA_Update failed"); - } - -inline void sha1_final(SHA_CTX* c, u_char md[SHA_DIGEST_LENGTH]) - { - if ( ! SHA1_Final(md, c) ) - reporter->InternalError("SHA_Final failed"); - } - -inline void sha256_init(SHA256_CTX* c) - { - if ( ! SHA256_Init(c) ) - reporter->InternalError("SHA256_Init failed"); - } - -inline void sha256_update(SHA256_CTX* c, const void* data, unsigned long len) - { - if ( ! SHA256_Update(c, data, len) ) - reporter->InternalError("SHA256_Update failed"); - } - -inline void sha256_final(SHA256_CTX* c, u_char md[SHA256_DIGEST_LENGTH]) - { - if ( ! SHA256_Final(md, c) ) - reporter->InternalError("SHA256_Final failed"); + EVP_MD_CTX *c = hash_init(Hash_MD5); + hash_update(c, data, len); + hash_final(c, out); + return out; } #endif //bro_digest_h diff --git a/src/probabilistic/BitVector.cc b/src/probabilistic/BitVector.cc index 79b403960e..7fa80c206b 100644 --- a/src/probabilistic/BitVector.cc +++ b/src/probabilistic/BitVector.cc @@ -496,13 +496,12 @@ uint64 BitVector::Hash() const { u_char buf[SHA256_DIGEST_LENGTH]; uint64 digest; - SHA256_CTX ctx; - sha256_init(&ctx); + EVP_MD_CTX* ctx = hash_init(Hash_SHA256); for ( size_type i = 0; i < Blocks(); ++i ) - sha256_update(&ctx, &bits[i], sizeof(bits[i])); + hash_update(ctx, &bits[i], sizeof(bits[i])); - sha256_final(&ctx, buf); + hash_final(ctx, buf); memcpy(&digest, buf, sizeof(digest)); // Use the first bytes as digest return digest; } diff --git a/src/probabilistic/Hasher.cc b/src/probabilistic/Hasher.cc index 048c7d02d2..d21efbed41 100644 --- a/src/probabilistic/Hasher.cc +++ b/src/probabilistic/Hasher.cc @@ -15,24 +15,23 @@ Hasher::seed_t Hasher::MakeSeed(const void* data, size_t size) { u_char buf[SHA256_DIGEST_LENGTH]; seed_t tmpseed; - SHA256_CTX ctx; - sha256_init(&ctx); + EVP_MD_CTX* ctx = hash_init(Hash_SHA256); assert(sizeof(tmpseed) == 16); if ( data ) - sha256_update(&ctx, data, size); + hash_update(ctx, data, size); else if ( global_hash_seed && global_hash_seed->Len() > 0 ) - sha256_update(&ctx, global_hash_seed->Bytes(), global_hash_seed->Len()); + hash_update(ctx, global_hash_seed->Bytes(), global_hash_seed->Len()); else { unsigned int first_seed = initial_seed(); - sha256_update(&ctx, &first_seed, sizeof(first_seed)); + hash_update(ctx, &first_seed, sizeof(first_seed)); } - sha256_final(&ctx, buf); + hash_final(ctx, buf); memcpy(&tmpseed, buf, sizeof(tmpseed)); // Use the first bytes as seed. return tmpseed; } diff --git a/testing/btest/Baseline/broker.store.type-conversion/master.out b/testing/btest/Baseline/broker.store.type-conversion/master.out index 0ef9bd4144..e7c6056367 100644 --- a/testing/btest/Baseline/broker.store.type-conversion/master.out +++ b/testing/btest/Baseline/broker.store.type-conversion/master.out @@ -44,4 +44,6 @@ three [zero, one, two] [s=abc] [c=123, r1=[s=xyz]] +opaque of md5, T +opaque of sha1, T opaque of sha256, T diff --git a/testing/btest/broker/store/type-conversion.bro b/testing/btest/broker/store/type-conversion.bro index 916c3f349d..c92c1ea4c9 100644 --- a/testing/btest/broker/store/type-conversion.bro +++ b/testing/btest/broker/store/type-conversion.bro @@ -57,6 +57,20 @@ event bro_init() print (Broker::data(R1($s="abc")) as R1); print (Broker::data(R2($c=123, $r1=R1($s="xyz"))) as R2); + local md5h1 = md5_hash_init(); + md5_hash_update(md5h1, "abc"); + local md5h2 = (Broker::data(md5h1) as opaque of md5); + local md5s1 = md5_hash_finish(md5h1); + local md5s2 = md5_hash_finish(md5h2); + print "opaque of md5", md5s1 == md5s2; + + local sha1h1 = sha1_hash_init(); + sha1_hash_update(sha1h1, "abc"); + local sha1h2 = (Broker::data(sha1h1) as opaque of sha1); + local sha1s1 = sha1_hash_finish(sha1h1); + local sha1s2 = sha1_hash_finish(sha1h2); + print "opaque of sha1", sha1s1 == sha1s2; + local h1 = sha256_hash_init(); sha256_hash_update(h1, "abc"); local h2 = (Broker::data(h1) as opaque of sha256);