Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-03 23:28:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

A few more updates to the digest functions.

Browse source 
This builds upon the previous commit to make Zeek compile on FIPS
systems.

This patch makes the changes a bit more aggressive. Instead of having a
number of different hash functions with different return values, we now
standardize on EVP_MD_CTX and just have one set of functions, to which
the hash algorithm that is desired is passed.

On the positive side, this enables us to support a wider range of hash
algorithm (and to easily add to them in the future).

I reimplemented the internal_md5 function - we don't support ebdic
systems in any case.

The md5/sha1 serialization functions are now also tested (I don't think
they were before).
This commit is contained in:
Johanna Amann 2019-01-24 09:19:29 -08:00
parent ffa6756255
commit 86161c85c4
9 changed files with 149 additions and 143 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/DFA.cc
View file

@ -2,8 +2,6 @@
#include "bro-config.h"
#include <openssl/md5.h>
#include "EquivClass.h"
#include "DFA.h"
#include "digest.h"

131
src/OpaqueVal.cc
View file

@ -79,12 +79,14 @@ bool HashVal::DoUnserialize(UnserialInfo* info)
MD5Val::MD5Val() : HashVal(md5_type)
{
if ( IsValid() )
// prevent leaks...
EVP_MD_CTX_free(ctx);
}
void MD5Val::digest(val_list& vlist, u_char result[MD5_DIGEST_LENGTH])
{
EVP_MD_CTX *h;
md5_init(&h);
EVP_MD_CTX* h = hash_init(Hash_MD5);
loop_over_list(vlist, i)
{
@ -92,17 +94,17 @@ void MD5Val::digest(val_list& vlist, u_char result[MD5_DIGEST_LENGTH])
if ( v->Type()->Tag() == TYPE_STRING )
{
const BroString* str = v->AsString();
md5_update(h, str->Bytes(), str->Len());
hash_update(h, str->Bytes(), str->Len());
}
else
{
ODesc d(DESC_BINARY);
v->Describe(&d);
md5_update(h, (const u_char *) d.Bytes(), d.Len());
hash_update(h, (const u_char *) d.Bytes(), d.Len());
}
}
md5_final(h, result);
hash_final(h, result);
}
void MD5Val::hmac(val_list& vlist,
@ -119,7 +121,7 @@ void MD5Val::hmac(val_list& vlist,
bool MD5Val::DoInit()
{
assert(! IsValid());
md5_init(&ctx);
ctx = hash_init(Hash_MD5);
return true;
}
@ -128,7 +130,7 @@ bool MD5Val::DoFeed(const void* data, size_t size)
if ( ! IsValid() )
return false;
md5_update(ctx, data, size);
hash_update(ctx, data, size);
return true;
}
@ -138,7 +140,7 @@ StringVal* MD5Val::DoGet()
return val_mgr->GetEmptyString();
u_char digest[MD5_DIGEST_LENGTH];
md5_final(ctx, digest);
hash_final(ctx, digest);
return new StringVal(md5_digest_print(digest));
}
@ -146,12 +148,13 @@ IMPLEMENT_SERIAL(MD5Val, SER_MD5_VAL);
bool MD5Val::DoSerialize(SerialInfo* info) const
{
MD5_CTX *md = (MD5_CTX *) EVP_MD_CTX_md_data(ctx);
DO_SERIALIZE(SER_MD5_VAL, HashVal);
if ( ! IsValid() )
return true;
MD5_CTX *md = (MD5_CTX *) EVP_MD_CTX_md_data(ctx);
if ( ! (SERIALIZE(md->A) &&
SERIALIZE(md->B) &&
SERIALIZE(md->C) &&
@ -174,12 +177,14 @@ bool MD5Val::DoSerialize(SerialInfo* info) const
bool MD5Val::DoUnserialize(UnserialInfo* info)
{
MD5_CTX *md = (MD5_CTX *) EVP_MD_CTX_md_data(ctx);
DO_UNSERIALIZE(HashVal);
if ( ! IsValid() )
return true;
ctx = hash_init(Hash_MD5);
MD5_CTX *md = (MD5_CTX *) EVP_MD_CTX_md_data(ctx);
if ( ! (UNSERIALIZE(&md->A) &&
UNSERIALIZE(&md->B) &&
UNSERIALIZE(&md->C) &&
@ -202,12 +207,14 @@ bool MD5Val::DoUnserialize(UnserialInfo* info)
SHA1Val::SHA1Val() : HashVal(sha1_type)
{
if ( IsValid() )
// prevent leaks...
EVP_MD_CTX_free(ctx);
}
void SHA1Val::digest(val_list& vlist, u_char result[SHA_DIGEST_LENGTH])
{
SHA_CTX h;
sha1_init(&h);
EVP_MD_CTX* h = hash_init(Hash_SHA1);
loop_over_list(vlist, i)
{
@ -215,23 +222,23 @@ void SHA1Val::digest(val_list& vlist, u_char result[SHA_DIGEST_LENGTH])
if ( v->Type()->Tag() == TYPE_STRING )
{
const BroString* str = v->AsString();
sha1_update(&h, str->Bytes(), str->Len());
hash_update(h, str->Bytes(), str->Len());
}
else
{
ODesc d(DESC_BINARY);
v->Describe(&d);
sha1_update(&h, (const u_char *) d.Bytes(), d.Len());
hash_update(h, (const u_char *) d.Bytes(), d.Len());
}
}
sha1_final(&h, result);
hash_final(h, result);
}
bool SHA1Val::DoInit()
{
assert(! IsValid());
sha1_init(&ctx);
ctx = hash_init(Hash_SHA1);
return true;
}
@ -240,7 +247,7 @@ bool SHA1Val::DoFeed(const void* data, size_t size)
if ( ! IsValid() )
return false;
sha1_update(&ctx, data, size);
hash_update(ctx, data, size);
return true;
}
@ -250,7 +257,7 @@ StringVal* SHA1Val::DoGet()
return val_mgr->GetEmptyString();
u_char digest[SHA_DIGEST_LENGTH];
sha1_final(&ctx, digest);
hash_final(ctx, digest);
return new StringVal(sha1_digest_print(digest));
}
@ -263,22 +270,24 @@ bool SHA1Val::DoSerialize(SerialInfo* info) const
if ( ! IsValid() )
return true;
if ( ! (SERIALIZE(ctx.h0) &&
SERIALIZE(ctx.h1) &&
SERIALIZE(ctx.h2) &&
SERIALIZE(ctx.h3) &&
SERIALIZE(ctx.h4) &&
SERIALIZE(ctx.Nl) &&
SERIALIZE(ctx.Nh)) )
SHA_CTX *md = (SHA_CTX *) EVP_MD_CTX_md_data(ctx);
if ( ! (SERIALIZE(md->h0) &&
SERIALIZE(md->h1) &&
SERIALIZE(md->h2) &&
SERIALIZE(md->h3) &&
SERIALIZE(md->h4) &&
SERIALIZE(md->Nl) &&
SERIALIZE(md->Nh)) )
return false;
for ( int i = 0; i < SHA_LBLOCK; ++i )
{
if ( ! SERIALIZE(ctx.data[i]) )
if ( ! SERIALIZE(md->data[i]) )
return false;
}
if ( ! SERIALIZE(ctx.num) )
if ( ! SERIALIZE(md->num) )
return false;
return true;
@ -291,22 +300,25 @@ bool SHA1Val::DoUnserialize(UnserialInfo* info)
if ( ! IsValid() )
return true;
if ( ! (UNSERIALIZE(&ctx.h0) &&
UNSERIALIZE(&ctx.h1) &&
UNSERIALIZE(&ctx.h2) &&
UNSERIALIZE(&ctx.h3) &&
UNSERIALIZE(&ctx.h4) &&
UNSERIALIZE(&ctx.Nl) &&
UNSERIALIZE(&ctx.Nh)) )
ctx = hash_init(Hash_SHA1);
SHA_CTX *md = (SHA_CTX *) EVP_MD_CTX_md_data(ctx);
if ( ! (UNSERIALIZE(&md->h0) &&
UNSERIALIZE(&md->h1) &&
UNSERIALIZE(&md->h2) &&
UNSERIALIZE(&md->h3) &&
UNSERIALIZE(&md->h4) &&
UNSERIALIZE(&md->Nl) &&
UNSERIALIZE(&md->Nh)) )
return false;
for ( int i = 0; i < SHA_LBLOCK; ++i )
{
if ( ! UNSERIALIZE(&ctx.data[i]) )
if ( ! UNSERIALIZE(&md->data[i]) )
return false;
}
if ( ! UNSERIALIZE(&ctx.num) )
if ( ! UNSERIALIZE(&md->num) )
return false;
return true;
@ -314,12 +326,14 @@ bool SHA1Val::DoUnserialize(UnserialInfo* info)
SHA256Val::SHA256Val() : HashVal(sha256_type)
{
if ( IsValid() )
// prevent leaks...
EVP_MD_CTX_free(ctx);
}
void SHA256Val::digest(val_list& vlist, u_char result[SHA256_DIGEST_LENGTH])
{
SHA256_CTX h;
sha256_init(&h);
EVP_MD_CTX* h = hash_init(Hash_SHA256);
loop_over_list(vlist, i)
{
@ -327,23 +341,23 @@ void SHA256Val::digest(val_list& vlist, u_char result[SHA256_DIGEST_LENGTH])
if ( v->Type()->Tag() == TYPE_STRING )
{
const BroString* str = v->AsString();
sha256_update(&h, str->Bytes(), str->Len());
hash_update(h, str->Bytes(), str->Len());
}
else
{
ODesc d(DESC_BINARY);
v->Describe(&d);
sha256_update(&h, (const u_char *) d.Bytes(), d.Len());
hash_update(h, (const u_char *) d.Bytes(), d.Len());
}
}
sha256_final(&h, result);
hash_final(h, result);
}
bool SHA256Val::DoInit()
{
assert( ! IsValid() );
sha256_init(&ctx);
ctx = hash_init(Hash_SHA256);
return true;
}
@ -352,7 +366,7 @@ bool SHA256Val::DoFeed(const void* data, size_t size)
if ( ! IsValid() )
return false;
sha256_update(&ctx, data, size);
hash_update(ctx, data, size);
return true;
}
@ -362,7 +376,7 @@ StringVal* SHA256Val::DoGet()
return val_mgr->GetEmptyString();
u_char digest[SHA256_DIGEST_LENGTH];
sha256_final(&ctx, digest);
hash_final(ctx, digest);
return new StringVal(sha256_digest_print(digest));
}
@ -375,24 +389,26 @@ bool SHA256Val::DoSerialize(SerialInfo* info) const
if ( ! IsValid() )
return true;
SHA256_CTX *md = (SHA256_CTX *) EVP_MD_CTX_md_data(ctx);
for ( int i = 0; i < 8; ++i )
{
if ( ! SERIALIZE(ctx.h[i]) )
if ( ! SERIALIZE(md->h[i]) )
return false;
}
if ( ! (SERIALIZE(ctx.Nl) &&
SERIALIZE(ctx.Nh)) )
if ( ! (SERIALIZE(md->Nl) &&
SERIALIZE(md->Nh)) )
return false;
for ( int i = 0; i < SHA_LBLOCK; ++i )
{
if ( ! SERIALIZE(ctx.data[i]) )
if ( ! SERIALIZE(md->data[i]) )
return false;
}
if ( ! (SERIALIZE(ctx.num) &&
SERIALIZE(ctx.md_len)) )
if ( ! (SERIALIZE(md->num) &&
SERIALIZE(md->md_len)) )
return false;
return true;
@ -405,25 +421,28 @@ bool SHA256Val::DoUnserialize(UnserialInfo* info)
if ( ! IsValid() )
return true;
ctx = hash_init(Hash_SHA256);
SHA256_CTX *md = (SHA256_CTX *) EVP_MD_CTX_md_data(ctx);
for ( int i = 0; i < 8; ++i )
{
if ( ! UNSERIALIZE(&ctx.h[i]) )
if ( ! UNSERIALIZE(&md->h[i]) )
return false;
}
if ( ! (UNSERIALIZE(&ctx.Nl) &&
UNSERIALIZE(&ctx.Nh)) )
if ( ! (UNSERIALIZE(&md->Nl) &&
UNSERIALIZE(&md->Nh)) )
return false;
for ( int i = 0; i < SHA_LBLOCK; ++i )
{
if ( ! UNSERIALIZE(&ctx.data[i]) )
if ( ! UNSERIALIZE(&md->data[i]) )
return false;
}
if ( ! (UNSERIALIZE(&ctx.num) &&
UNSERIALIZE(&ctx.md_len)) )
if ( ! (UNSERIALIZE(&md->num) &&
UNSERIALIZE(&md->md_len)) )
return false;
return true;

4
src/OpaqueVal.h
View file

@ -75,7 +75,7 @@ protected:
DECLARE_SERIAL(SHA1Val);
private:
SHA_CTX ctx;
EVP_MD_CTX* ctx;
};
class SHA256Val : public HashVal {
@ -94,7 +94,7 @@ protected:
DECLARE_SERIAL(SHA256Val);
private:
SHA256_CTX ctx;
EVP_MD_CTX* ctx;
};
class EntropyVal : public OpaqueVal {

6
src/analyzer/protocol/mime/MIME.cc
View file

@ -1335,7 +1335,7 @@ MIME_Mail::MIME_Mail(analyzer::Analyzer* mail_analyzer, bool orig, int buf_size)
if ( mime_content_hash )
{
compute_content_hash = 1;
md5_init(&md5_hash);
md5_hash = hash_init(Hash_MD5);
}
else
compute_content_hash = 0;
@ -1355,7 +1355,7 @@ void MIME_Mail::Done()
if ( compute_content_hash && mime_content_hash )
{
u_char* digest = new u_char[16];
md5_final(md5_hash, digest);
hash_final(md5_hash, digest);
val_list* vl = new val_list;
vl->append(analyzer->BuildConnVal());
@ -1456,7 +1456,7 @@ void MIME_Mail::SubmitData(int len, const char* buf)
if ( compute_content_hash )
{
content_hash_length += len;
md5_update(md5_hash, (const u_char*) buf, len);
hash_update(md5_hash, (const u_char*) buf, len);
}
if ( mime_entity_data || mime_all_data )

113
src/digest.h
View file

@ -23,6 +23,8 @@ inline void *EVP_MD_CTX_md_data(const EVP_MD_CTX* ctx)
#include "Reporter.h"
enum HashAlgorithm { Hash_MD5, Hash_SHA1, Hash_SHA224, Hash_SHA256, Hash_SHA384, Hash_SHA512 };
inline const char* digest_print(const u_char* digest, size_t n)
{
static char buf[256]; // big enough for any of md5/sha1/sha256
@ -46,92 +48,65 @@ inline const char* sha256_digest_print(const u_char digest[SHA256_DIGEST_LENGTH]
return digest_print(digest, SHA256_DIGEST_LENGTH);
}
inline void md5_init(EVP_MD_CTX** c)
inline EVP_MD_CTX* hash_init(HashAlgorithm alg)
{
*c = EVP_MD_CTX_new();
EVP_MD_CTX *c = EVP_MD_CTX_new();
/* Allow this to work even if FIPS disables it */
const EVP_MD* md;
switch (alg)
{
case Hash_MD5:
#ifdef EVP_MD_CTX_FLAG_NON_FIPS_ALLOW
EVP_MD_CTX_set_flags(*c, EVP_MD_CTX_FLAG_NON_FIPS_ALLOW);
EVP_MD_CTX_set_flags(c, EVP_MD_CTX_FLAG_NON_FIPS_ALLOW);
#endif
if ( ! EVP_DigestInit_ex(*c, EVP_md5(), NULL) )
reporter->InternalError("MD5_Init failed");
md = EVP_md5();
break;
case Hash_SHA1:
md = EVP_sha1();
break;
case Hash_SHA224:
md = EVP_sha224();
break;
case Hash_SHA256:
md = EVP_sha256();
break;
case Hash_SHA384:
md = EVP_sha384();
break;
case Hash_SHA512:
md = EVP_sha512();
break;
default:
reporter->InternalError("Unknown hash algorithm passed to hash_init");
}
if ( ! EVP_DigestInit_ex(c, md, NULL) )
reporter->InternalError("EVP_DigestInit failed");
return c;
}
inline void md5_update(EVP_MD_CTX* c, const void* data, unsigned long len)
inline void hash_update(EVP_MD_CTX* c, const void* data, unsigned long len)
{
if ( ! EVP_DigestUpdate(c, data, len) )
reporter->InternalError("MD5_Update failed");
reporter->InternalError("EVP_DigestUpdate failed");
}
inline void md5_final(EVP_MD_CTX* c, u_char md[MD5_DIGEST_LENGTH])
inline void hash_final(EVP_MD_CTX* c, u_char md[MD5_DIGEST_LENGTH])
{
if ( ! EVP_DigestFinal(c, md, NULL) )
reporter->InternalError("MD5_Final failed");
}
inline unsigned char* internal_md5(const unsigned char *d, size_t n, unsigned char *md)
{
EVP_MD_CTX *c;
static unsigned char m[MD5_DIGEST_LENGTH];
if (md == NULL)
md = m;
md5_init(&c);
#ifndef CHARSET_EBCDIC
md5_update(c, d, n);
#else
{
char temp[1024];
unsigned long chunk;
while (n > 0) {
chunk = (n > sizeof(temp)) ? sizeof(temp) : n;
ebcdic2ascii(temp, d, chunk);
md5_update(c, temp, chunk);
n -= chunk;
d += chunk;
}
}
#endif
md5_final(c, md);
reporter->InternalError("EVP_DigestFinal failed");
EVP_MD_CTX_free(c);
return md;
}
inline void sha1_init(SHA_CTX* c)
inline unsigned char* internal_md5(const unsigned char *data, unsigned long len, unsigned char *out)
{
if ( ! SHA1_Init(c) )
reporter->InternalError("SHA_Init failed");
}
static unsigned char static_out[MD5_DIGEST_LENGTH];
if ( ! out )
out = static_out; // use static array for return, see OpenSSL man page
inline void sha1_update(SHA_CTX* c, const void* data, unsigned long len)
{
if ( ! SHA1_Update(c, data, len) )
reporter->InternalError("SHA_Update failed");
}
inline void sha1_final(SHA_CTX* c, u_char md[SHA_DIGEST_LENGTH])
{
if ( ! SHA1_Final(md, c) )
reporter->InternalError("SHA_Final failed");
}
inline void sha256_init(SHA256_CTX* c)
{
if ( ! SHA256_Init(c) )
reporter->InternalError("SHA256_Init failed");
}
inline void sha256_update(SHA256_CTX* c, const void* data, unsigned long len)
{
if ( ! SHA256_Update(c, data, len) )
reporter->InternalError("SHA256_Update failed");
}
inline void sha256_final(SHA256_CTX* c, u_char md[SHA256_DIGEST_LENGTH])
{
if ( ! SHA256_Final(md, c) )
reporter->InternalError("SHA256_Final failed");
EVP_MD_CTX *c = hash_init(Hash_MD5);
hash_update(c, data, len);
hash_final(c, out);
return out;
}
#endif //bro_digest_h

7
src/probabilistic/BitVector.cc
View file

@ -496,13 +496,12 @@ uint64 BitVector::Hash() const
{
u_char buf[SHA256_DIGEST_LENGTH];
uint64 digest;
SHA256_CTX ctx;
sha256_init(&ctx);
EVP_MD_CTX* ctx = hash_init(Hash_SHA256);
for ( size_type i = 0; i < Blocks(); ++i )
sha256_update(&ctx, &bits[i], sizeof(bits[i]));
hash_update(ctx, &bits[i], sizeof(bits[i]));
sha256_final(&ctx, buf);
hash_final(ctx, buf);
memcpy(&digest, buf, sizeof(digest)); // Use the first bytes as digest
return digest;
}

11
src/probabilistic/Hasher.cc
View file

@ -15,24 +15,23 @@ Hasher::seed_t Hasher::MakeSeed(const void* data, size_t size)
{
u_char buf[SHA256_DIGEST_LENGTH];
seed_t tmpseed;
SHA256_CTX ctx;
sha256_init(&ctx);
EVP_MD_CTX* ctx = hash_init(Hash_SHA256);
assert(sizeof(tmpseed) == 16);
if ( data )
sha256_update(&ctx, data, size);
hash_update(ctx, data, size);
else if ( global_hash_seed && global_hash_seed->Len() > 0 )
sha256_update(&ctx, global_hash_seed->Bytes(), global_hash_seed->Len());
hash_update(ctx, global_hash_seed->Bytes(), global_hash_seed->Len());
else
{
unsigned int first_seed = initial_seed();
sha256_update(&ctx, &first_seed, sizeof(first_seed));
hash_update(ctx, &first_seed, sizeof(first_seed));
}
sha256_final(&ctx, buf);
hash_final(ctx, buf);
memcpy(&tmpseed, buf, sizeof(tmpseed)); // Use the first bytes as seed.
return tmpseed;
}

2
testing/btest/Baseline/broker.store.type-conversion/master.out
View file

@ -44,4 +44,6 @@ three
[zero, one, two]
[s=abc]
[c=123, r1=[s=xyz]]
opaque of md5, T
opaque of sha1, T
opaque of sha256, T

14
testing/btest/broker/store/type-conversion.bro
View file

@ -57,6 +57,20 @@ event bro_init()
print (Broker::data(R1($s="abc")) as R1);
print (Broker::data(R2($c=123, $r1=R1($s="xyz"))) as R2);
local md5h1 = md5_hash_init();
md5_hash_update(md5h1, "abc");
local md5h2 = (Broker::data(md5h1) as opaque of md5);
local md5s1 = md5_hash_finish(md5h1);
local md5s2 = md5_hash_finish(md5h2);
print "opaque of md5", md5s1 == md5s2;
local sha1h1 = sha1_hash_init();
sha1_hash_update(sha1h1, "abc");
local sha1h2 = (Broker::data(sha1h1) as opaque of sha1);
local sha1s1 = sha1_hash_finish(sha1h1);
local sha1s2 = sha1_hash_finish(sha1h2);
print "opaque of sha1", sha1s1 == sha1s2;
local h1 = sha256_hash_init();
sha256_hash_update(h1, "abc");
local h2 = (Broker::data(h1) as opaque of sha256);