diff --git a/CHANGES b/CHANGES index 148376714e..b9daa5fcfc 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,23 @@ +2.1-263 | 2012-12-20 16:22:09 -0800 + + * Bro's language now has a new set of types "opaque of X". (Matthias + Vallentin) + + Opaque values can be passed around like other values but they can + only be manipulated with BiF functions, not with other operators. + Currently, the following opaque types are supported: + + - opaque of md5 + - opaque of sha1 + - opaque of sha256 + - opaquey of entropy. + + They go along with the corrsponding BiF functions md5_*, sha1_*, + sha256_*, and entropy_*, respectively. Note that these functions + have changed their signatures to work with opaques types rather + than global state as it was before. + 2.1-240 | 2012-12-20 15:21:07 -0800 * Improve error for invalid use of types as values. Addresses #923. diff --git a/NEWS b/NEWS index 696e738048..a852e71289 100644 --- a/NEWS +++ b/NEWS @@ -31,6 +31,22 @@ New Functionality "return" or "break" statement will fall through to subsequent cases. A default case label is allowed. +- Bro's language now has a new set of types "opaque of X". Opaque + values can be passed around like other values but they can only be + manipulated with BiF functions, not with other operators. Currently, + the following opaque types are supported: + + - opaque of md5 + - opaque of sha1 + - opaque of sha256 + - opaquey of entropy. + + They go along with the corrsponding BiF functions md5_*, sha1_*, + sha256_*, and entropy_*, respectively. Note that these functions + have changed their signatures to work with opaques types rather + than global state as it was before. + + Changed Functionality ~~~~~~~~~~~~~~~~~~~~~ @@ -44,6 +60,9 @@ Changed Functionality make_connection_persistent(), generate_idmef(), split_complete() + - md5_*, sha1_*, sha256_*, and entropy_* have all changed + their signatures to work with opaque types (see above). + - Removed a now unused argument from "do_split" helper function. - "this" is no longer a reserved keyword. diff --git a/VERSION b/VERSION index 943afac387..99a494f7c0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1-240 +2.1-263 diff --git a/scripts/base/protocols/http/file-hash.bro b/scripts/base/protocols/http/file-hash.bro index 7e8e5cceaf..bc7547e51a 100644 --- a/scripts/base/protocols/http/file-hash.bro +++ b/scripts/base/protocols/http/file-hash.bro @@ -13,16 +13,16 @@ export { redef record Info += { ## MD5 sum for a file transferred over HTTP calculated from the ## response body. - md5: string &log &optional; + md5: string &log &optional; ## This value can be set per-transfer to determine per request ## if a file should have an MD5 sum generated. It must be ## set to T at the time of or before the first chunk of body data. - calc_md5: bool &default=F; + calc_md5: bool &default=F; ## Indicates if an MD5 sum is being calculated for the current ## request/response pair. - calculating_md5: bool &default=F; + md5_handle: opaque of md5 &optional; }; ## Generate MD5 sums for these filetypes. @@ -41,13 +41,12 @@ event http_entity_data(c: connection, is_orig: bool, length: count, data: string if ( c$http$calc_md5 || (c$http?$mime_type && generate_md5 in c$http$mime_type) ) { - c$http$calculating_md5 = T; - md5_hash_init(c$id); + c$http$md5_handle = md5_hash_init(); } } - if ( c$http$calculating_md5 ) - md5_hash_update(c$id, data); + if ( c$http?$md5_handle ) + md5_hash_update(c$http$md5_handle, data); } ## In the event of a content gap during a file transfer, detect the state for @@ -55,11 +54,11 @@ event http_entity_data(c: connection, is_orig: bool, length: count, data: string ## incorrect anyway. event content_gap(c: connection, is_orig: bool, seq: count, length: count) &priority=5 { - if ( is_orig || ! c?$http || ! c$http$calculating_md5 ) return; + if ( is_orig || ! c?$http || ! c$http?$md5_handle ) return; set_state(c, F, is_orig); - c$http$calculating_md5 = F; - md5_hash_finish(c$id); + md5_hash_finish(c$http$md5_handle); # Ignore return value. + delete c$http$md5_handle; } ## When the file finishes downloading, finish the hash and generate a notice. @@ -67,11 +66,11 @@ event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) & { if ( is_orig || ! c?$http ) return; - if ( c$http$calculating_md5 ) + if ( c$http?$md5_handle ) { local url = build_url_http(c$http); - c$http$calculating_md5 = F; - c$http$md5 = md5_hash_finish(c$id); + c$http$md5 = md5_hash_finish(c$http$md5_handle); + delete c$http$md5_handle; NOTICE([$note=MD5, $msg=fmt("%s %s %s", c$id$orig_h, c$http$md5, url), $sub=c$http$md5, $conn=c, $URL=url]); @@ -82,11 +81,12 @@ event connection_state_remove(c: connection) &priority=-5 { if ( c?$http_state && c$http_state$current_response in c$http_state$pending && - c$http_state$pending[c$http_state$current_response]$calculating_md5 ) + c$http_state$pending[c$http_state$current_response]?$md5_handle ) { # The MD5 sum isn't going to be saved anywhere since the entire # body wouldn't have been seen anyway and we'd just be giving an # incorrect MD5 sum. - md5_hash_finish(c$id); + md5_hash_finish(c$http$md5_handle); + delete c$http$md5_handle; } } diff --git a/scripts/base/protocols/smtp/entities.bro b/scripts/base/protocols/smtp/entities.bro index b4b5fd1f03..80d6089ce7 100644 --- a/scripts/base/protocols/smtp/entities.bro +++ b/scripts/base/protocols/smtp/entities.bro @@ -16,33 +16,33 @@ export { type EntityInfo: record { ## This is the timestamp of when the MIME content transfer began. - ts: time &log; - uid: string &log; - id: conn_id &log; + ts: time &log; + uid: string &log; + id: conn_id &log; ## A count to represent the depth of this message transaction in a ## single connection where multiple messages were transferred. - trans_depth: count &log; + trans_depth: count &log; ## The filename seen in the Content-Disposition header. - filename: string &log &optional; + filename: string &log &optional; ## Track how many bytes of the MIME encoded file have been seen. - content_len: count &log &default=0; + content_len: count &log &default=0; ## The mime type of the entity discovered through magic bytes identification. - mime_type: string &log &optional; + mime_type: string &log &optional; ## The calculated MD5 sum for the MIME entity. - md5: string &log &optional; + md5: string &log &optional; ## Optionally calculate the file's MD5 sum. Must be set prior to the ## first data chunk being see in an event. - calc_md5: bool &default=F; + calc_md5: bool &default=F; ## This boolean value indicates if an MD5 sum is being calculated ## for the current file transfer. - calculating_md5: bool &default=F; + md5_handle: opaque of md5 &optional; ## Optionally write the file to disk. Must be set prior to first ## data chunk being seen in an event. - extract_file: bool &default=F; + extract_file: bool &default=F; ## Store the file handle here for the file currently being extracted. - extraction_file: file &log &optional; + extraction_file: file &log &optional; }; redef record Info += { @@ -126,18 +126,16 @@ event mime_segment_data(c: connection, length: count, data: string) &priority=-5 if ( c$smtp$current_entity$content_len == 0 ) { - if ( generate_md5 in c$smtp$current_entity$mime_type && ! never_calc_md5 ) - c$smtp$current_entity$calc_md5 = T; + local entity = c$smtp$current_entity; + if ( generate_md5 in entity$mime_type && ! never_calc_md5 ) + entity$calc_md5 = T; - if ( c$smtp$current_entity$calc_md5 ) - { - c$smtp$current_entity$calculating_md5 = T; - md5_hash_init(c$id); - } + if ( entity$calc_md5 ) + entity$md5_handle = md5_hash_init(); } - if ( c$smtp$current_entity$calculating_md5 ) - md5_hash_update(c$id, data); + if ( c$smtp$current_entity?$md5_handle ) + md5_hash_update(entity$md5_handle, data); } ## In the event of a content gap during the MIME transfer, detect the state for @@ -147,10 +145,11 @@ event content_gap(c: connection, is_orig: bool, seq: count, length: count) &prio { if ( is_orig || ! c?$smtp || ! c$smtp?$current_entity ) return; - if ( c$smtp$current_entity$calculating_md5 ) + local entity = c$smtp$current_entity; + if ( entity?$md5_handle ) { - c$smtp$current_entity$calculating_md5 = F; - md5_hash_finish(c$id); + md5_hash_finish(entity$md5_handle); + delete entity$md5_handle; } } @@ -161,12 +160,14 @@ event mime_end_entity(c: connection) &priority=-3 if ( ! c?$smtp || ! c$smtp?$current_entity ) return; - if ( c$smtp$current_entity$calculating_md5 ) + local entity = c$smtp$current_entity; + if ( entity?$md5_handle ) { - c$smtp$current_entity$md5 = md5_hash_finish(c$id); + entity$md5 = md5_hash_finish(entity$md5_handle); + delete entity$md5_handle; NOTICE([$note=MD5, $msg=fmt("Calculated a hash for a MIME entity from %s", c$id$orig_h), - $sub=c$smtp$current_entity$md5, $conn=c]); + $sub=entity$md5, $conn=c]); } } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d304604dcd..71adee9158 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -361,6 +361,7 @@ set(bro_SRCS NetVar.cc NetbiosSSN.cc Obj.cc + OpaqueVal.cc OSFinger.cc PacketFilter.cc PacketSort.cc diff --git a/src/OpaqueVal.cc b/src/OpaqueVal.cc new file mode 100644 index 0000000000..51f975edf8 --- /dev/null +++ b/src/OpaqueVal.cc @@ -0,0 +1,501 @@ +#include "OpaqueVal.h" +#include "Reporter.h" +#include "Serializer.h" + +bool HashVal::IsValid() const + { + return valid; + } + +bool HashVal::Init() + { + if ( valid ) + return false; + + valid = DoInit(); + return valid; + } + +StringVal* HashVal::Get() + { + if ( ! valid ) + return new StringVal(""); + + StringVal* result = DoGet(); + valid = false; + return result; + } + +bool HashVal::Feed(const void* data, size_t size) + { + if ( valid ) + return DoFeed(data, size); + + reporter->InternalError("invalid opaque hash value"); + return false; + } + +bool HashVal::DoInit() + { + assert(! "missing implementation of DoInit()"); + return false; + } + +bool HashVal::DoFeed(const void*, size_t) + { + assert(! "missing implementation of DoFeed()"); + return false; + } + +StringVal* HashVal::DoGet() + { + assert(! "missing implementation of DoGet()"); + return new StringVal(""); + } + +HashVal::HashVal(OpaqueType* t) : OpaqueVal(t) + { + valid = false; + } + +IMPLEMENT_SERIAL(HashVal, SER_HASH_VAL); + +bool HashVal::DoSerialize(SerialInfo* info) const + { + DO_SERIALIZE(SER_HASH_VAL, OpaqueVal); + return SERIALIZE(valid); + } + +bool HashVal::DoUnserialize(UnserialInfo* info) + { + DO_UNSERIALIZE(OpaqueVal); + return UNSERIALIZE(&valid); + } + +void MD5Val::digest(val_list& vlist, u_char result[MD5_DIGEST_LENGTH]) + { + MD5_CTX h; + md5_init(&h); + + loop_over_list(vlist, i) + { + Val* v = vlist[i]; + if ( v->Type()->Tag() == TYPE_STRING ) + { + const BroString* str = v->AsString(); + md5_update(&h, str->Bytes(), str->Len()); + } + else + { + ODesc d(DESC_BINARY); + v->Describe(&d); + md5_update(&h, (const u_char *) d.Bytes(), d.Len()); + } + } + + md5_final(&h, result); + } + +void MD5Val::hmac(val_list& vlist, + u_char key[MD5_DIGEST_LENGTH], + u_char result[MD5_DIGEST_LENGTH]) + { + digest(vlist, result); + for ( int i = 0; i < MD5_DIGEST_LENGTH; ++i ) + result[i] ^= key[i]; + + MD5(result, MD5_DIGEST_LENGTH, result); + } + +bool MD5Val::DoInit() + { + assert(! IsValid()); + md5_init(&ctx); + return true; + } + +bool MD5Val::DoFeed(const void* data, size_t size) + { + if ( ! IsValid() ) + return false; + + md5_update(&ctx, data, size); + return true; + } + +StringVal* MD5Val::DoGet() + { + if ( ! IsValid() ) + return new StringVal(""); + + u_char digest[MD5_DIGEST_LENGTH]; + md5_final(&ctx, digest); + return new StringVal(md5_digest_print(digest)); + } + +IMPLEMENT_SERIAL(MD5Val, SER_MD5_VAL); + +bool MD5Val::DoSerialize(SerialInfo* info) const + { + DO_SERIALIZE(SER_MD5_VAL, HashVal); + + if ( ! IsValid() ) + return true; + + if ( ! (SERIALIZE(ctx.A) && + SERIALIZE(ctx.B) && + SERIALIZE(ctx.C) && + SERIALIZE(ctx.D) && + SERIALIZE(ctx.Nl) && + SERIALIZE(ctx.Nh)) ) + return false; + + for ( int i = 0; i < MD5_LBLOCK; ++i ) + { + if ( ! SERIALIZE(ctx.data[i]) ) + return false; + } + + if ( ! SERIALIZE(ctx.num) ) + return false; + + return true; + } + +bool MD5Val::DoUnserialize(UnserialInfo* info) + { + DO_UNSERIALIZE(HashVal); + + if ( ! IsValid() ) + return true; + + if ( ! (UNSERIALIZE(&ctx.A) && + UNSERIALIZE(&ctx.B) && + UNSERIALIZE(&ctx.C) && + UNSERIALIZE(&ctx.D) && + UNSERIALIZE(&ctx.Nl) && + UNSERIALIZE(&ctx.Nh)) ) + return false; + + for ( int i = 0; i < MD5_LBLOCK; ++i ) + { + if ( ! UNSERIALIZE(&ctx.data[i]) ) + return false; + } + + if ( ! UNSERIALIZE(&ctx.num) ) + return false; + + return true; + } + +void SHA1Val::digest(val_list& vlist, u_char result[SHA_DIGEST_LENGTH]) + { + SHA_CTX h; + sha1_init(&h); + + loop_over_list(vlist, i) + { + Val* v = vlist[i]; + if ( v->Type()->Tag() == TYPE_STRING ) + { + const BroString* str = v->AsString(); + sha1_update(&h, str->Bytes(), str->Len()); + } + else + { + ODesc d(DESC_BINARY); + v->Describe(&d); + sha1_update(&h, (const u_char *) d.Bytes(), d.Len()); + } + } + + sha1_final(&h, result); + } + +bool SHA1Val::DoInit() + { + assert(! IsValid()); + sha1_init(&ctx); + return true; + } + +bool SHA1Val::DoFeed(const void* data, size_t size) + { + if ( ! IsValid() ) + return false; + + sha1_update(&ctx, data, size); + return true; + } + +StringVal* SHA1Val::DoGet() + { + if ( ! IsValid() ) + return new StringVal(""); + + u_char digest[SHA_DIGEST_LENGTH]; + sha1_final(&ctx, digest); + return new StringVal(sha1_digest_print(digest)); + } + +IMPLEMENT_SERIAL(SHA1Val, SER_SHA1_VAL); + +bool SHA1Val::DoSerialize(SerialInfo* info) const + { + DO_SERIALIZE(SER_SHA1_VAL, HashVal); + + if ( ! IsValid() ) + return true; + + if ( ! (SERIALIZE(ctx.h0) && + SERIALIZE(ctx.h1) && + SERIALIZE(ctx.h2) && + SERIALIZE(ctx.h3) && + SERIALIZE(ctx.h4) && + SERIALIZE(ctx.Nl) && + SERIALIZE(ctx.Nh)) ) + return false; + + for ( int i = 0; i < SHA_LBLOCK; ++i ) + { + if ( ! SERIALIZE(ctx.data[i]) ) + return false; + } + + if ( ! SERIALIZE(ctx.num) ) + return false; + + return true; + } + +bool SHA1Val::DoUnserialize(UnserialInfo* info) + { + DO_UNSERIALIZE(HashVal); + + if ( ! IsValid() ) + return true; + + if ( ! (UNSERIALIZE(&ctx.h0) && + UNSERIALIZE(&ctx.h1) && + UNSERIALIZE(&ctx.h2) && + UNSERIALIZE(&ctx.h3) && + UNSERIALIZE(&ctx.h4) && + UNSERIALIZE(&ctx.Nl) && + UNSERIALIZE(&ctx.Nh)) ) + return false; + + for ( int i = 0; i < SHA_LBLOCK; ++i ) + { + if ( ! UNSERIALIZE(&ctx.data[i]) ) + return false; + } + + if ( ! UNSERIALIZE(&ctx.num) ) + return false; + + return true; + } + +void SHA256Val::digest(val_list& vlist, u_char result[SHA256_DIGEST_LENGTH]) + { + SHA256_CTX h; + sha256_init(&h); + + loop_over_list(vlist, i) + { + Val* v = vlist[i]; + if ( v->Type()->Tag() == TYPE_STRING ) + { + const BroString* str = v->AsString(); + sha256_update(&h, str->Bytes(), str->Len()); + } + else + { + ODesc d(DESC_BINARY); + v->Describe(&d); + sha256_update(&h, (const u_char *) d.Bytes(), d.Len()); + } + } + + sha256_final(&h, result); + } + +bool SHA256Val::DoInit() + { + assert( ! IsValid() ); + sha256_init(&ctx); + return true; + } + +bool SHA256Val::DoFeed(const void* data, size_t size) + { + if ( ! IsValid() ) + return false; + + sha256_update(&ctx, data, size); + return true; + } + +StringVal* SHA256Val::DoGet() + { + if ( ! IsValid() ) + return new StringVal(""); + + u_char digest[SHA256_DIGEST_LENGTH]; + sha256_final(&ctx, digest); + return new StringVal(sha256_digest_print(digest)); + } + +IMPLEMENT_SERIAL(SHA256Val, SER_SHA256_VAL); + +bool SHA256Val::DoSerialize(SerialInfo* info) const + { + DO_SERIALIZE(SER_SHA256_VAL, HashVal); + + if ( ! IsValid() ) + return true; + + for ( int i = 0; i < 8; ++i ) + { + if ( ! SERIALIZE(ctx.h[i]) ) + return false; + } + + if ( ! (SERIALIZE(ctx.Nl) && + SERIALIZE(ctx.Nh)) ) + return false; + + for ( int i = 0; i < SHA_LBLOCK; ++i ) + { + if ( ! SERIALIZE(ctx.data[i]) ) + return false; + } + + if ( ! (SERIALIZE(ctx.num) && + SERIALIZE(ctx.md_len)) ) + return false; + + return true; + } + +bool SHA256Val::DoUnserialize(UnserialInfo* info) + { + DO_UNSERIALIZE(HashVal); + + if ( ! IsValid() ) + return true; + + for ( int i = 0; i < 8; ++i ) + { + if ( ! UNSERIALIZE(&ctx.h[i]) ) + return false; + } + + if ( ! (UNSERIALIZE(&ctx.Nl) && + UNSERIALIZE(&ctx.Nh)) ) + return false; + + for ( int i = 0; i < SHA_LBLOCK; ++i ) + { + if ( ! UNSERIALIZE(&ctx.data[i]) ) + return false; + } + + + if ( ! (UNSERIALIZE(&ctx.num) && + UNSERIALIZE(&ctx.md_len)) ) + return false; + + return true; + } + + +bool EntropyVal::Feed(const void* data, size_t size) + { + state.add(data, size); + return true; + } + +bool EntropyVal::Get(double *r_ent, double *r_chisq, double *r_mean, + double *r_montepicalc, double *r_scc) + { + state.end(r_ent, r_chisq, r_mean, r_montepicalc, r_scc); + return true; + } + +IMPLEMENT_SERIAL(EntropyVal, SER_ENTROPY_VAL); + +bool EntropyVal::DoSerialize(SerialInfo* info) const + { + DO_SERIALIZE(SER_ENTROPY_VAL, OpaqueVal); + + for ( int i = 0; i < 256; ++i ) + { + if ( ! SERIALIZE(state.ccount[i]) ) + return false; + } + + if ( ! (SERIALIZE(state.totalc) && + SERIALIZE(state.mp) && + SERIALIZE(state.sccfirst)) ) + return false; + + for ( int i = 0; i < RT_MONTEN; ++i ) + { + if ( ! SERIALIZE(state.monte[i]) ) + return false; + } + + if ( ! (SERIALIZE(state.inmont) && + SERIALIZE(state.mcount) && + SERIALIZE(state.cexp) && + SERIALIZE(state.montex) && + SERIALIZE(state.montey) && + SERIALIZE(state.montepi) && + SERIALIZE(state.sccu0) && + SERIALIZE(state.scclast) && + SERIALIZE(state.scct1) && + SERIALIZE(state.scct2) && + SERIALIZE(state.scct3)) ) + return false; + + return true; + } + +bool EntropyVal::DoUnserialize(UnserialInfo* info) + { + DO_UNSERIALIZE(OpaqueVal); + + for ( int i = 0; i < 256; ++i ) + { + if ( ! UNSERIALIZE(&state.ccount[i]) ) + return false; + } + + if ( ! (UNSERIALIZE(&state.totalc) && + UNSERIALIZE(&state.mp) && + UNSERIALIZE(&state.sccfirst)) ) + return false; + + for ( int i = 0; i < RT_MONTEN; ++i ) + { + if ( ! UNSERIALIZE(&state.monte[i]) ) + return false; + } + + if ( ! (UNSERIALIZE(&state.inmont) && + UNSERIALIZE(&state.mcount) && + UNSERIALIZE(&state.cexp) && + UNSERIALIZE(&state.montex) && + UNSERIALIZE(&state.montey) && + UNSERIALIZE(&state.montepi) && + UNSERIALIZE(&state.sccu0) && + UNSERIALIZE(&state.scclast) && + UNSERIALIZE(&state.scct1) && + UNSERIALIZE(&state.scct2) && + UNSERIALIZE(&state.scct3)) ) + return false; + + return true; + } diff --git a/src/OpaqueVal.h b/src/OpaqueVal.h new file mode 100644 index 0000000000..0428e50bdb --- /dev/null +++ b/src/OpaqueVal.h @@ -0,0 +1,110 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef OPAQUEVAL_H +#define OPAQUEVAL_H + +#include "RandTest.h" +#include "Val.h" +#include "digest.h" + +class HashVal : public OpaqueVal { +public: + virtual bool IsValid() const; + virtual bool Init(); + virtual bool Feed(const void* data, size_t size); + virtual StringVal* Get(); + +protected: + HashVal() { }; + HashVal(OpaqueType* t); + virtual bool DoInit(); + virtual bool DoFeed(const void* data, size_t size); + virtual StringVal* DoGet(); + + DECLARE_SERIAL(HashVal); + +private: + // This flag exists because Get() can only be called once. + bool valid; +}; + +class MD5Val : public HashVal { +public: + static void digest(val_list& vlist, u_char result[MD5_DIGEST_LENGTH]); + + static void hmac(val_list& vlist, + u_char key[MD5_DIGEST_LENGTH], + u_char result[MD5_DIGEST_LENGTH]); + + MD5Val() : HashVal(new OpaqueType("md5")) { } + +protected: + friend class Val; + + virtual bool DoInit() /* override */; + virtual bool DoFeed(const void* data, size_t size) /* override */; + virtual StringVal* DoGet() /* override */; + + DECLARE_SERIAL(MD5Val); + +private: + MD5_CTX ctx; +}; + +class SHA1Val : public HashVal { +public: + static void digest(val_list& vlist, u_char result[SHA_DIGEST_LENGTH]); + + SHA1Val() : HashVal(new OpaqueType("sha1")) { } + +protected: + friend class Val; + + virtual bool DoInit() /* override */; + virtual bool DoFeed(const void* data, size_t size) /* override */; + virtual StringVal* DoGet() /* override */; + + DECLARE_SERIAL(SHA1Val); + +private: + SHA_CTX ctx; +}; + +class SHA256Val : public HashVal { +public: + static void digest(val_list& vlist, u_char result[SHA256_DIGEST_LENGTH]); + + SHA256Val() : HashVal(new OpaqueType("sha256")) { } + +protected: + friend class Val; + + virtual bool DoInit() /* override */; + virtual bool DoFeed(const void* data, size_t size) /* override */; + virtual StringVal* DoGet() /* override */; + + DECLARE_SERIAL(SHA256Val); + +private: + SHA256_CTX ctx; +}; + +class EntropyVal : public OpaqueVal { +public: + EntropyVal() : OpaqueVal(new OpaqueType("entropy")) { } + + bool Feed(const void* data, size_t size); + bool Get(double *r_ent, double *r_chisq, double *r_mean, + double *r_montepicalc, double *r_scc); + +protected: + friend class Val; + EntropyVal(OpaqueType* t); + + DECLARE_SERIAL(EntropyVal); + +private: + RandTest state; +}; + +#endif diff --git a/src/RandTest.cc b/src/RandTest.cc index 638cc6c765..94e76500b5 100644 --- a/src/RandTest.cc +++ b/src/RandTest.cc @@ -12,7 +12,18 @@ Modified for Bro by Seth Hall - July 2010 */ -#include +#include +#include "RandTest.h" + +#define log2of10 3.32192809488736234787 +/* RT_LOG2 -- Calculate log to the base 2 */ +static double rt_log2(double x) +{ + return log2of10 * log10(x); +} + +// RT_INCIRC = pow(pow(256.0, (double) (RT_MONTEN / 2)) - 1, 2.0); +#define RT_INCIRC 281474943156225.0 RandTest::RandTest() { @@ -28,9 +39,9 @@ RandTest::RandTest() } } -void RandTest::add(void *buf, int bufl) +void RandTest::add(const void *buf, int bufl) { - unsigned char *bp = (unsigned char*)buf; + const unsigned char *bp = static_cast(buf); int oc; while (bufl-- > 0) @@ -78,8 +89,8 @@ void RandTest::add(void *buf, int bufl) } } -void RandTest::end(double *r_ent, double *r_chisq, - double *r_mean, double *r_montepicalc, double *r_scc) +void RandTest::end(double* r_ent, double* r_chisq, + double* r_mean, double* r_montepicalc, double* r_scc) { int i; double ent, chisq, scc, datasum; diff --git a/src/RandTest.h b/src/RandTest.h index a4f551b602..bb1eb3c6b4 100644 --- a/src/RandTest.h +++ b/src/RandTest.h @@ -1,34 +1,33 @@ -#include +#ifndef RANDTEST_H +#define RANDTEST_H -#define log2of10 3.32192809488736234787 -/* RT_LOG2 -- Calculate log to the base 2 */ -static double rt_log2(double x) -{ - return log2of10 * log10(x); -} +#include "util.h" #define RT_MONTEN 6 /* Bytes used as Monte Carlo co-ordinates. This should be no more bits than the mantissa of your "double" floating point type. */ +class EntropyVal; -// RT_INCIRC = pow(pow(256.0, (double) (RT_MONTEN / 2)) - 1, 2.0); -#define RT_INCIRC 281474943156225.0 class RandTest { public: RandTest(); - void add(void *buf, int bufl); - void end(double *r_ent, double *r_chisq, double *r_mean, - double *r_montepicalc, double *r_scc); + void add(const void* buf, int bufl); + void end(double* r_ent, double* r_chisq, double* r_mean, + double* r_montepicalc, double* r_scc); private: - long ccount[256]; /* Bins to count occurrences of values */ - long totalc; /* Total bytes counted */ + friend class EntropyVal; + + int64 ccount[256]; /* Bins to count occurrences of values */ + int64 totalc; /* Total bytes counted */ int mp; int sccfirst; unsigned int monte[RT_MONTEN]; - long inmont, mcount; + int64 inmont, mcount; double cexp, montex, montey, montepi, sccu0, scclast, scct1, scct2, scct3; - }; +}; + +#endif diff --git a/src/SerialTypes.h b/src/SerialTypes.h index c47ff19298..e103c1c40e 100644 --- a/src/SerialTypes.h +++ b/src/SerialTypes.h @@ -98,6 +98,12 @@ SERIAL_VAL(RECORD_VAL, 10) SERIAL_VAL(ENUM_VAL, 11) SERIAL_VAL(VECTOR_VAL, 12) SERIAL_VAL(MUTABLE_VAL, 13) +SERIAL_VAL(OPAQUE_VAL, 14) +SERIAL_VAL(HASH_VAL, 15) +SERIAL_VAL(MD5_VAL, 16) +SERIAL_VAL(SHA1_VAL, 17) +SERIAL_VAL(SHA256_VAL, 18) +SERIAL_VAL(ENTROPY_VAL, 19) #define SERIAL_EXPR(name, val) SERIAL_CONST(name, val, EXPR) SERIAL_EXPR(EXPR, 1) @@ -178,6 +184,7 @@ SERIAL_TYPE(SUBNET_TYPE, 8) SERIAL_TYPE(FILE_TYPE, 9) SERIAL_TYPE(ENUM_TYPE, 10) SERIAL_TYPE(VECTOR_TYPE, 11) +SERIAL_TYPE(OPAQUE_TYPE, 12) SERIAL_CONST2(ATTRIBUTES) SERIAL_CONST2(EVENT_HANDLER) diff --git a/src/Type.cc b/src/Type.cc index 062d11d7aa..43e342f505 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -30,6 +30,7 @@ const char* type_name(TypeTag t) "table", "union", "record", "types", "func", "file", + "opaque", "vector", "type", "error", @@ -96,6 +97,7 @@ BroType::BroType(TypeTag t, bool arg_base_type) case TYPE_LIST: case TYPE_FUNC: case TYPE_FILE: + case TYPE_OPAQUE: case TYPE_VECTOR: case TYPE_TYPE: internal_tag = TYPE_INTERNAL_OTHER; @@ -1262,6 +1264,41 @@ bool FileType::DoUnserialize(UnserialInfo* info) return yield != 0; } +OpaqueType::OpaqueType(const string& arg_name) : BroType(TYPE_OPAQUE) + { + name = arg_name; + } + +void OpaqueType::Describe(ODesc* d) const + { + if ( d->IsReadable() ) + d->AddSP("opaque of"); + else + d->Add(int(Tag())); + + d->Add(name.c_str()); + } + +IMPLEMENT_SERIAL(OpaqueType, SER_OPAQUE_TYPE); + +bool OpaqueType::DoSerialize(SerialInfo* info) const + { + DO_SERIALIZE(SER_OPAQUE_TYPE, BroType); + return SERIALIZE(name); + } + +bool OpaqueType::DoUnserialize(UnserialInfo* info) + { + DO_UNSERIALIZE(BroType); + + char const* n; + if ( ! UNSERIALIZE_STR(&n, 0) ) + return false; + + name = n; + return true; + } + EnumType::EnumType(const string& arg_name) : BroType(TYPE_ENUM) { @@ -1716,6 +1753,13 @@ int same_type(const BroType* t1, const BroType* t2, int is_init) case TYPE_FILE: return same_type(t1->YieldType(), t2->YieldType(), is_init); + case TYPE_OPAQUE: + { + const OpaqueType* ot1 = (const OpaqueType*) t1; + const OpaqueType* ot2 = (const OpaqueType*) t2; + return ot1->Name() == ot2->Name() ? 1 : 0; + } + case TYPE_TYPE: return same_type(t1, t2, is_init); @@ -1805,6 +1849,7 @@ int is_assignable(BroType* t) case TYPE_VECTOR: case TYPE_FILE: + case TYPE_OPAQUE: case TYPE_TABLE: case TYPE_TYPE: return 1; diff --git a/src/Type.h b/src/Type.h index cc6ddcd8fd..249d8709c5 100644 --- a/src/Type.h +++ b/src/Type.h @@ -29,6 +29,7 @@ typedef enum { TYPE_LIST, TYPE_FUNC, TYPE_FILE, + TYPE_OPAQUE, TYPE_VECTOR, TYPE_TYPE, TYPE_ERROR @@ -499,6 +500,23 @@ protected: BroType* yield; }; +class OpaqueType : public BroType { +public: + OpaqueType(const string& name); + virtual ~OpaqueType() { }; + + const string& Name() const { return name; } + + void Describe(ODesc* d) const; + +protected: + OpaqueType() { } + + DECLARE_SERIAL(OpaqueType) + + string name; +}; + class EnumType : public BroType { public: EnumType(const string& arg_name); diff --git a/src/Val.cc b/src/Val.cc index a62c00dcea..20051aff5f 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -3114,6 +3114,27 @@ void VectorVal::ValDescribe(ODesc* d) const d->Add("]"); } +OpaqueVal::OpaqueVal(OpaqueType* t) : Val(t) + { + } + +OpaqueVal::~OpaqueVal() + { + } + +IMPLEMENT_SERIAL(OpaqueVal, SER_OPAQUE_VAL); + +bool OpaqueVal::DoSerialize(SerialInfo* info) const + { + DO_SERIALIZE(SER_OPAQUE_VAL, Val); + return true; + } + +bool OpaqueVal::DoUnserialize(UnserialInfo* info) + { + DO_UNSERIALIZE(Val); + return true; + } Val* check_and_promote(Val* v, const BroType* t, int is_init) { diff --git a/src/Val.h b/src/Val.h index c3ec5b04fb..8ebfb2de0e 100644 --- a/src/Val.h +++ b/src/Val.h @@ -1013,6 +1013,20 @@ protected: VectorType* vector_type; }; +// Base class for values with types that are managed completely internally, +// with no further script-level operators provided (other than bif +// functions). See OpaqueVal.h for derived classes. +class OpaqueVal : public Val { +public: + OpaqueVal(OpaqueType* t); + virtual ~OpaqueVal(); + +protected: + friend class Val; + OpaqueVal() { } + + DECLARE_SERIAL(OpaqueVal); +}; // Checks the given value for consistency with the given type. If an // exact match, returns it. If promotable, returns the promoted version, diff --git a/src/bro.bif b/src/bro.bif index bebcde18c3..8cea9d9123 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -530,82 +530,7 @@ function piped_exec%(program: string, to_write: string%): bool %} %%{ -static void hash_md5_val(val_list& vlist, unsigned char digest[16]) - { - MD5_CTX h; - - md5_init(&h); - loop_over_list(vlist, i) - { - Val* v = vlist[i]; - if ( v->Type()->Tag() == TYPE_STRING ) - { - const BroString* str = v->AsString(); - md5_update(&h, str->Bytes(), str->Len()); - } - else - { - ODesc d(DESC_BINARY); - v->Describe(&d); - md5_update(&h, (const u_char *) d.Bytes(), d.Len()); - } - } - md5_final(&h, digest); - } - -static void hmac_md5_val(val_list& vlist, unsigned char digest[16]) - { - hash_md5_val(vlist, digest); - for ( int i = 0; i < 16; ++i ) - digest[i] = digest[i] ^ shared_hmac_md5_key[i]; - MD5(digest, 16, digest); - } - -static void hash_sha1_val(val_list& vlist, unsigned char digest[20]) - { - SHA_CTX h; - - sha1_init(&h); - loop_over_list(vlist, i) - { - Val* v = vlist[i]; - if ( v->Type()->Tag() == TYPE_STRING ) - { - const BroString* str = v->AsString(); - sha1_update(&h, str->Bytes(), str->Len()); - } - else - { - ODesc d(DESC_BINARY); - v->Describe(&d); - sha1_update(&h, (const u_char *) d.Bytes(), d.Len()); - } - } - sha1_final(&h, digest); - } - -static void hash_sha256_val(val_list& vlist, unsigned char digest[32]) - { - SHA256_CTX h; - - sha256_init(&h); - loop_over_list(vlist, i) - { - Val* v = vlist[i]; - if ( v->Type()->Tag() == TYPE_STRING ) - { - const BroString* str = v->AsString(); - sha256_update(&h, str->Bytes(), str->Len()); - } - else - { - ODesc d(DESC_BINARY); - v->Describe(&d); - sha256_update(&h, (const u_char *) d.Bytes(), d.Len()); - } - } - sha256_final(&h, digest); - } +#include "OpaqueVal.h" %%} ## Computes the MD5 hash value of the provided list of arguments. @@ -623,8 +548,8 @@ static void hash_sha256_val(val_list& vlist, unsigned char digest[32]) ## friends. function md5_hash%(...%): string %{ - unsigned char digest[16]; - hash_md5_val(@ARG@, digest); + unsigned char digest[MD5_DIGEST_LENGTH]; + MD5Val::digest(@ARG@, digest); return new StringVal(md5_digest_print(digest)); %} @@ -643,8 +568,8 @@ function md5_hash%(...%): string ## friends. function sha1_hash%(...%): string %{ - unsigned char digest[20]; - hash_sha1_val(@ARG@, digest); + unsigned char digest[SHA_DIGEST_LENGTH]; + SHA1Val::digest(@ARG@, digest); return new StringVal(sha1_digest_print(digest)); %} @@ -663,8 +588,8 @@ function sha1_hash%(...%): string ## friends. function sha256_hash%(...%): string %{ - unsigned char digest[32]; - hash_sha256_val(@ARG@, digest); + unsigned char digest[SHA256_DIGEST_LENGTH]; + SHA256Val::digest(@ARG@, digest); return new StringVal(sha256_digest_print(digest)); %} @@ -679,288 +604,183 @@ function sha256_hash%(...%): string ## sha256_hash sha256_hash_init sha256_hash_update sha256_hash_finish function md5_hmac%(...%): string %{ - unsigned char digest[16]; - hmac_md5_val(@ARG@, digest); - return new StringVal(md5_digest_print(digest)); + unsigned char hmac[MD5_DIGEST_LENGTH]; + MD5Val::hmac(@ARG@, shared_hmac_md5_key, hmac); + return new StringVal(md5_digest_print(hmac)); %} -%%{ -static map md5_states; -static map sha1_states; -static map sha256_states; - -BroString* convert_index_to_string(Val* index) - { - ODesc d; - index->Describe(&d); - BroString* s = new BroString(1, d.TakeBytes(), d.Len()); - s->SetUseFreeToDelete(1); - return s; - } -%%} - -## Initializes MD5 state to enable incremental hash computation. After -## initializing the MD5 state with this function, you can feed data to -## :bro:id:`md5_hash_update` and finally need to call :bro:id:`md5_hash_finish` -## to finish the computation and get the final hash value. +## Constructs an MD5 handle to enable incremental hash computation. You can +## feed data to the returned opaque value with :bro:id:`md5_hash_update` and +## eventually need to call :bro:id:`md5_hash_finish` to finish the computation +## and get the hash digest as result. ## ## For example, when computing incremental MD5 values of transferred files in -## multiple concurrent HTTP connections, one would call ``md5_hash_init(c$id)`` -## once before invoking ``md5_hash_update(c$id, some_more_data)`` in the +## multiple concurrent HTTP connections, one keeps an optional handle in the +## HTTP session record. Then, one would call +## ``c$http$md5_handle = md5_hash_init()`` once before invoking +## ``md5_hash_update(c$http$md5_handle, some_more_data)`` in the ## :bro:id:`http_entity_data` event handler. When all data has arrived, a call ## to :bro:id:`md5_hash_finish` returns the final hash value. -## -## index: The unique identifier to associate with this hash computation. +## +## Returns: The opaque handle associated with this hash computation. ## ## .. bro:see:: md5_hmac md5_hash md5_hash_update md5_hash_finish ## sha1_hash sha1_hash_init sha1_hash_update sha1_hash_finish ## sha256_hash sha256_hash_init sha256_hash_update sha256_hash_finish -function md5_hash_init%(index: any%): bool +function md5_hash_init%(%): opaque of md5 %{ - BroString* s = convert_index_to_string(index); - int status = 0; - - if ( md5_states.count(*s) < 1 ) - { - MD5_CTX h; - md5_init(&h); - md5_states[*s] = h; - status = 1; - } - - delete s; - return new Val(status, TYPE_BOOL); + HashVal* digest = new MD5Val(); + digest->Init(); + return digest; %} -## Initializes SHA1 state to enable incremental hash computation. After -## initializing the SHA1 state with this function, you can feed data to -## :bro:id:`sha1_hash_update` and finally need to call -## :bro:id:`sha1_hash_finish` to finish the computation and get the final hash -## value. +## Constructs an SHA1 handle to enable incremental hash computation. You can +## feed data to the returned opaque value with :bro:id:`sha1_hash_update` and +## finally need to call :bro:id:`sha1_hash_finish` to finish the computation +## and get the hash digest as result. ## ## For example, when computing incremental SHA1 values of transferred files in -## multiple concurrent HTTP connections, one would call ``sha1_hash_init(c$id)`` -## once before invoking ``sha1_hash_update(c$id, some_more_data)`` in the +## multiple concurrent HTTP connections, one keeps an optional handle in the +## HTTP session record. Then, one would call +## ``c$http$sha1_handle = sha1_hash_init()`` ## once before invoking +## ``sha1_hash_update(c$http$sha1_handle, some_more_data)`` in the ## :bro:id:`http_entity_data` event handler. When all data has arrived, a call ## to :bro:id:`sha1_hash_finish` returns the final hash value. ## -## index: The unique identifier to associate with this hash computation. +## Returns: The opaque handle associated with this hash computation. ## ## .. bro:see:: md5_hmac md5_hash md5_hash_init md5_hash_update md5_hash_finish ## sha1_hash sha1_hash_update sha1_hash_finish ## sha256_hash sha256_hash_init sha256_hash_update sha256_hash_finish -function sha1_hash_init%(index: any%): bool +function sha1_hash_init%(%): opaque of sha1 %{ - BroString* s = convert_index_to_string(index); - int status = 0; - - if ( sha1_states.count(*s) < 1 ) - { - SHA_CTX h; - sha1_init(&h); - sha1_states[*s] = h; - status = 1; - } - - delete s; - return new Val(status, TYPE_BOOL); + HashVal* digest = new SHA1Val(); + digest->Init(); + return digest; %} -## Initializes SHA256 state to enable incremental hash computation. After -## initializing the SHA256 state with this function, you can feed data to -## :bro:id:`sha256_hash_update` and finally need to call -## :bro:id:`sha256_hash_finish` to finish the computation and get the final hash -## value. +## Constructs an SHA256 handle to enable incremental hash computation. You can +## feed data to the returned opaque value with :bro:id:`sha256_hash_update` and +## finally need to call :bro:id:`sha256_hash_finish` to finish the computation +## and get the hash digest as result. ## ## For example, when computing incremental SHA256 values of transferred files in -## multiple concurrent HTTP connections, one would call -## ``sha256_hash_init(c$id)`` once before invoking -## ``sha256_hash_update(c$id, some_more_data)`` in the +## multiple concurrent HTTP connections, one keeps an optional handle in the +## HTTP session record. Then, one would call +## ``c$http$sha256_handle = sha256_hash_init()`` ## once before invoking +## ``sha256_hash_update(c$http$sha256_handle, some_more_data)`` in the ## :bro:id:`http_entity_data` event handler. When all data has arrived, a call ## to :bro:id:`sha256_hash_finish` returns the final hash value. ## -## index: The unique identifier to associate with this hash computation. +## Returns: The opaque handle associated with this hash computation. ## ## .. bro:see:: md5_hmac md5_hash md5_hash_init md5_hash_update md5_hash_finish ## sha1_hash sha1_hash_init sha1_hash_update sha1_hash_finish ## sha256_hash sha256_hash_update sha256_hash_finish -function sha256_hash_init%(index: any%): bool +function sha256_hash_init%(%): opaque of sha256 %{ - BroString* s = convert_index_to_string(index); - int status = 0; - - if ( sha256_states.count(*s) < 1 ) - { - SHA256_CTX h; - sha256_init(&h); - sha256_states[*s] = h; - status = 1; - } - - delete s; - return new Val(status, TYPE_BOOL); + HashVal* digest = new SHA256Val(); + digest->Init(); + return digest; %} -## Update the MD5 value associated with a given index. It is required to +## Updates the MD5 value associated with a given index. It is required to ## call :bro:id:`md5_hash_init` once before calling this ## function. ## -## index: The unique identifier to associate with this hash computation. +## handle: The opaque handle associated with this hash computation. ## ## data: The data to add to the hash computation. ## +## Returns: True on success. +## ## .. bro:see:: md5_hmac md5_hash md5_hash_init md5_hash_finish ## sha1_hash sha1_hash_init sha1_hash_update sha1_hash_finish ## sha256_hash sha256_hash_init sha256_hash_update sha256_hash_finish -function md5_hash_update%(index: any, data: string%): bool +function md5_hash_update%(handle: opaque of md5, data: string%): bool %{ - BroString* s = convert_index_to_string(index); - int status = 0; - - if ( md5_states.count(*s) > 0 ) - { - md5_update(&md5_states[*s], data->Bytes(), data->Len()); - status = 1; - } - - delete s; - return new Val(status, TYPE_BOOL); + bool rc = static_cast(handle)->Feed(data->Bytes(), data->Len()); + return new Val(rc, TYPE_BOOL); %} -## Update the SHA1 value associated with a given index. It is required to +## Updates the SHA1 value associated with a given index. It is required to ## call :bro:id:`sha1_hash_init` once before calling this ## function. ## -## index: The unique identifier to associate with this hash computation. +## handle: The opaque handle associated with this hash computation. ## ## data: The data to add to the hash computation. ## +## Returns: True on success. +## ## .. bro:see:: md5_hmac md5_hash md5_hash_init md5_hash_update md5_hash_finish ## sha1_hash sha1_hash_init sha1_hash_finish ## sha256_hash sha256_hash_init sha256_hash_update sha256_hash_finish -function sha1_hash_update%(index: any, data: string%): bool +function sha1_hash_update%(handle: opaque of sha1, data: string%): bool %{ - BroString* s = convert_index_to_string(index); - int status = 0; - - if ( sha1_states.count(*s) > 0 ) - { - sha1_update(&sha1_states[*s], data->Bytes(), data->Len()); - status = 1; - } - - delete s; - return new Val(status, TYPE_BOOL); + bool rc = static_cast(handle)->Feed(data->Bytes(), data->Len()); + return new Val(rc, TYPE_BOOL); %} -## Update the SHA256 value associated with a given index. It is required to +## Updates the SHA256 value associated with a given index. It is required to ## call :bro:id:`sha256_hash_init` once before calling this ## function. ## -## index: The unique identifier to associate with this hash computation. +## handle: The opaque handle associated with this hash computation. ## ## data: The data to add to the hash computation. ## +## Returns: True on success. +## ## .. bro:see:: md5_hmac md5_hash md5_hash_init md5_hash_update md5_hash_finish ## sha1_hash sha1_hash_init sha1_hash_update sha1_hash_finish ## sha256_hash sha256_hash_init sha256_hash_finish -function sha256_hash_update%(index: any, data: string%): bool +function sha256_hash_update%(handle: opaque of sha256, data: string%): bool %{ - BroString* s = convert_index_to_string(index); - int status = 0; - - if ( sha256_states.count(*s) > 0 ) - { - sha256_update(&sha256_states[*s], data->Bytes(), data->Len()); - status = 1; - } - - delete s; - return new Val(status, TYPE_BOOL); + bool rc = static_cast(handle)->Feed(data->Bytes(), data->Len()); + return new Val(rc, TYPE_BOOL); %} ## Returns the final MD5 digest of an incremental hash computation. ## -## index: The unique identifier of this hash computation. +## handle: The opaque handle associated with this hash computation. ## -## Returns: The hash value associated with the computation at *index*. +## Returns: The hash value associated with the computation of *handle*. ## ## .. bro:see:: md5_hmac md5_hash md5_hash_init md5_hash_update ## sha1_hash sha1_hash_init sha1_hash_update sha1_hash_finish ## sha256_hash sha256_hash_init sha256_hash_update sha256_hash_finish -function md5_hash_finish%(index: any%): string +function md5_hash_finish%(handle: opaque of md5%): string %{ - BroString* s = convert_index_to_string(index); - StringVal* printable_digest; - - if ( md5_states.count(*s) > 0 ) - { - unsigned char digest[16]; - md5_final(&md5_states[*s], digest); - md5_states.erase(*s); - printable_digest = new StringVal(md5_digest_print(digest)); - } - else - printable_digest = new StringVal(""); - - delete s; - return printable_digest; + return static_cast(handle)->Get(); %} ## Returns the final SHA1 digest of an incremental hash computation. ## -## index: The unique identifier of this hash computation. +## handle: The opaque handle associated with this hash computation. ## -## Returns: The hash value associated with the computation at *index*. +## Returns: The hash value associated with the computation of *handle*. ## ## .. bro:see:: md5_hmac md5_hash md5_hash_init md5_hash_update md5_hash_finish ## sha1_hash sha1_hash_init sha1_hash_update ## sha256_hash sha256_hash_init sha256_hash_update sha256_hash_finish -function sha1_hash_finish%(index: any%): string +function sha1_hash_finish%(handle: opaque of sha1%): string %{ - BroString* s = convert_index_to_string(index); - StringVal* printable_digest; - - if ( sha1_states.count(*s) > 0 ) - { - unsigned char digest[20]; - sha1_final(&sha1_states[*s], digest); - sha1_states.erase(*s); - printable_digest = new StringVal(sha1_digest_print(digest)); - } - else - printable_digest = new StringVal(""); - - delete s; - return printable_digest; + return static_cast(handle)->Get(); %} ## Returns the final SHA256 digest of an incremental hash computation. ## -## index: The unique identifier of this hash computation. +## handle: The opaque handle associated with this hash computation. ## -## Returns: The hash value associated with the computation at *index*. +## Returns: The hash value associated with the computation of *handle*. ## ## .. bro:see:: md5_hmac md5_hash md5_hash_init md5_hash_update md5_hash_finish ## sha1_hash sha1_hash_init sha1_hash_update sha1_hash_finish ## sha256_hash sha256_hash_init sha256_hash_update -function sha256_hash_finish%(index: any%): string +function sha256_hash_finish%(handle: opaque of sha256%): string %{ - BroString* s = convert_index_to_string(index); - StringVal* printable_digest; - - if ( sha256_states.count(*s) > 0 ) - { - unsigned char digest[32]; - sha256_final(&sha256_states[*s], digest); - sha256_states.erase(*s); - printable_digest = new StringVal(sha256_digest_print(digest)); - } - else - printable_digest = new StringVal(""); - - delete s; - return printable_digest; + return static_cast(handle)->Get(); %} ## Generates a random number. @@ -1058,11 +878,6 @@ function identify_data%(data: string, return_mime: bool%): string return new StringVal(descr); %} -%%{ -#include -static map entropy_states; -%%} - ## Performs an entropy test on the given data. ## See http://www.fourmilab.ch/random. ## @@ -1107,13 +922,11 @@ function find_entropy%(data: string%): entropy_test_result %{ double montepi, scc, ent, mean, chisq; montepi = scc = ent = mean = chisq = 0.0; + EntropyVal e; + e.Feed(data->Bytes(), data->Len()); + e.Get(&ent, &chisq, &mean, &montepi, &scc); + RecordVal* ent_result = new RecordVal(entropy_test_result); - RandTest *rt = new RandTest(); - - rt->add((char*) data->Bytes(), data->Len()); - rt->end(&ent, &chisq, &mean, &montepi, &scc); - delete rt; - ent_result->Assign(0, new Val(ent, TYPE_DOUBLE)); ent_result->Assign(1, new Val(chisq, TYPE_DOUBLE)); ent_result->Assign(2, new Val(mean, TYPE_DOUBLE)); @@ -1124,85 +937,52 @@ function find_entropy%(data: string%): entropy_test_result ## Initializes data structures for incremental entropy calculation. ## -## index: An arbitrary unique value per distinct computation. -## -## Returns: True on success. +## Returns: An opaque handle to be used in subsequent operations. ## ## .. bro:see:: find_entropy entropy_test_add entropy_test_finish -function entropy_test_init%(index: any%): bool +function entropy_test_init%(%): opaque of entropy %{ - BroString* s = convert_index_to_string(index); - int status = 0; - - if ( entropy_states.count(*s) < 1 ) - { - entropy_states[*s] = new RandTest(); - status = 1; - } - - delete s; - return new Val(status, TYPE_BOOL); + return new EntropyVal(); %} -## Adds data to an incremental entropy calculation. Before using this function, -## one needs to invoke :bro:id:`entropy_test_init`. +## Adds data to an incremental entropy calculation. +## +## handle: The opaque handle representing the entropy calculation state. ## ## data: The data to add to the entropy calculation. ## -## index: An arbitrary unique value that identifies a particular entropy -## computation. -## ## Returns: True on success. ## ## .. bro:see:: find_entropy entropy_test_add entropy_test_finish -function entropy_test_add%(index: any, data: string%): bool +function entropy_test_add%(handle: opaque of entropy, data: string%): bool %{ - BroString* s = convert_index_to_string(index); - int status = 0; - - if ( entropy_states.count(*s) > 0 ) - { - entropy_states[*s]->add((char*) data->Bytes(), data->Len()); - status = 1; - } - - delete s; + bool status = static_cast(handle)->Feed(data->Bytes(), + data->Len()); return new Val(status, TYPE_BOOL); %} ## Finishes an incremental entropy calculation. Before using this function, -## one needs to initialize the computation with :bro:id:`entropy_test_init` and +## one needs to obtain an opaque handle with :bro:id:`entropy_test_init` and ## add data to it via :bro:id:`entropy_test_add`. ## -## index: An arbitrary unique value that identifies a particular entropy -## computation. +## handle: The opaque handle representing the entropy calculation state. ## ## Returns: The result of the entropy test. See :bro:id:`find_entropy` for a ## description of the individual components. ## ## .. bro:see:: find_entropy entropy_test_init entropy_test_add -function entropy_test_finish%(index: any%): entropy_test_result +function entropy_test_finish%(handle: opaque of entropy%): entropy_test_result %{ - BroString* s = convert_index_to_string(index); double montepi, scc, ent, mean, chisq; montepi = scc = ent = mean = chisq = 0.0; + static_cast(handle)->Get(&ent, &chisq, &mean, &montepi, &scc); + RecordVal* ent_result = new RecordVal(entropy_test_result); - - if ( entropy_states.count(*s) > 0 ) - { - RandTest *rt = entropy_states[*s]; - rt->end(&ent, &chisq, &mean, &montepi, &scc); - entropy_states.erase(*s); - delete rt; - } - ent_result->Assign(0, new Val(ent, TYPE_DOUBLE)); ent_result->Assign(1, new Val(chisq, TYPE_DOUBLE)); ent_result->Assign(2, new Val(mean, TYPE_DOUBLE)); ent_result->Assign(3, new Val(montepi, TYPE_DOUBLE)); ent_result->Assign(4, new Val(scc, TYPE_DOUBLE)); - - delete s; return ent_result; %} diff --git a/src/builtin-func.l b/src/builtin-func.l index 1d61f31734..9baeb1a9f9 100644 --- a/src/builtin-func.l +++ b/src/builtin-func.l @@ -74,7 +74,9 @@ HEX [0-9a-fA-F]+ "set" return check_c_mode(TOK_SET); "table" return check_c_mode(TOK_TABLE); "vector" return check_c_mode(TOK_VECTOR); -"module" return check_c_mode(TOK_MODULE); +"of" return check_c_mode(TOK_OF); +"opaque" return check_c_mode(TOK_OPAQUE); +"module" return check_c_mode(TOK_MODULE); "@ARG@" return TOK_ARG; "@ARGS@" return TOK_ARGS; diff --git a/src/builtin-func.y b/src/builtin-func.y index fd40613236..474f321ccd 100644 --- a/src/builtin-func.y +++ b/src/builtin-func.y @@ -269,15 +269,15 @@ void print_event_c_body(FILE *fp) %token TOK_LPP TOK_RPP TOK_LPB TOK_RPB TOK_LPPB TOK_RPPB TOK_VAR_ARG %token TOK_BOOL -%token TOK_FUNCTION TOK_EVENT TOK_CONST TOK_ENUM -%token TOK_TYPE TOK_RECORD TOK_SET TOK_VECTOR TOK_TABLE TOK_MODULE +%token TOK_FUNCTION TOK_EVENT TOK_CONST TOK_ENUM TOK_OF +%token TOK_TYPE TOK_RECORD TOK_SET TOK_VECTOR TOK_OPAQUE TOK_TABLE TOK_MODULE %token TOK_ARGS TOK_ARG TOK_ARGC %token TOK_ID TOK_ATTR TOK_CSTR TOK_LF TOK_WS TOK_COMMENT %token TOK_ATOM TOK_INT TOK_C_TOKEN %left ',' ':' -%type TOK_C_TOKEN TOK_ID TOK_CSTR TOK_WS TOK_COMMENT TOK_ATTR TOK_INT opt_ws +%type TOK_C_TOKEN TOK_ID TOK_CSTR TOK_WS TOK_COMMENT TOK_ATTR TOK_INT opt_ws type %type TOK_ATOM TOK_BOOL %union { @@ -584,7 +584,17 @@ args_1: args_1 ',' opt_ws arg opt_ws { /* empty */ } ; -arg: TOK_ID opt_ws ':' opt_ws TOK_ID +// TODO: Migrate all other compound types to this rule. Once the BiF language +// can parse all regular Bro types, we can throw out the unnecessary +// boilerplate typedefs for addr_set, string_set, etc. +type: + TOK_OPAQUE opt_ws TOK_OF opt_ws TOK_ID + { $$ = concat("opaque of ", $5); } + | TOK_ID + { $$ = $1; } + ; + +arg: TOK_ID opt_ws ':' opt_ws type { args.push_back(new BuiltinFuncArg($1, $5)); } | TOK_VAR_ARG { @@ -594,7 +604,7 @@ arg: TOK_ID opt_ws ':' opt_ws TOK_ID } ; -return_type: ':' opt_ws TOK_ID opt_ws +return_type: ':' opt_ws type opt_ws { BuiltinFuncArg* ret = new BuiltinFuncArg("", $3); ret->PrintBro(fp_bro_init); diff --git a/src/parse.y b/src/parse.y index 079fc16c43..a16b742728 100644 --- a/src/parse.y +++ b/src/parse.y @@ -11,7 +11,7 @@ %token TOK_DOUBLE TOK_ELSE TOK_ENUM TOK_EVENT TOK_EXPORT TOK_FILE TOK_FOR %token TOK_FUNCTION TOK_GLOBAL TOK_HOOK TOK_ID TOK_IF TOK_INT %token TOK_INTERVAL TOK_LIST TOK_LOCAL TOK_MODULE -%token TOK_NEXT TOK_OF TOK_PATTERN TOK_PATTERN_TEXT +%token TOK_NEXT TOK_OF TOK_OPAQUE TOK_PATTERN TOK_PATTERN_TEXT %token TOK_PORT TOK_PRINT TOK_RECORD TOK_REDEF %token TOK_REMOVE_FROM TOK_RETURN TOK_SCHEDULE TOK_SET %token TOK_STRING TOK_SUBNET TOK_SWITCH TOK_TABLE @@ -899,6 +899,12 @@ type: $$ = new FileType(base_type(TYPE_STRING)); } + | TOK_OPAQUE TOK_OF TOK_ID + { + set_location(@1, @3); + $$ = new OpaqueType($3); + } + | resolve_id { if ( ! $1 || ! ($$ = $1->AsType()) ) diff --git a/src/scan.l b/src/scan.l index bf4cd76f23..efcd273e36 100644 --- a/src/scan.l +++ b/src/scan.l @@ -298,6 +298,7 @@ local return TOK_LOCAL; module return TOK_MODULE; next return TOK_NEXT; of return TOK_OF; +opaque return TOK_OPAQUE; pattern return TOK_PATTERN; port return TOK_PORT; print return TOK_PRINT; diff --git a/testing/btest/Baseline/istate.opaque/expected.log b/testing/btest/Baseline/istate.opaque/expected.log new file mode 100644 index 0000000000..1386a47db1 --- /dev/null +++ b/testing/btest/Baseline/istate.opaque/expected.log @@ -0,0 +1,4 @@ +acbd18db4cc2f85cedef654fccc4a4d8 +0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33 +2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae +[entropy=0.918296, chi_square=423.666667, mean=108.0, monte_carlo_pi=nan, serial_correlation=-0.5] diff --git a/testing/btest/Baseline/istate.opaque/output.log b/testing/btest/Baseline/istate.opaque/output.log new file mode 100644 index 0000000000..1386a47db1 --- /dev/null +++ b/testing/btest/Baseline/istate.opaque/output.log @@ -0,0 +1,4 @@ +acbd18db4cc2f85cedef654fccc4a4d8 +0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33 +2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae +[entropy=0.918296, chi_square=423.666667, mean=108.0, monte_carlo_pi=nan, serial_correlation=-0.5] diff --git a/testing/btest/bifs/entropy_test.bro b/testing/btest/bifs/entropy_test.bro index 8dc54e09b2..2a2dd422d1 100644 --- a/testing/btest/bifs/entropy_test.bro +++ b/testing/btest/bifs/entropy_test.bro @@ -5,20 +5,14 @@ event bro_init() { local a = "dh3Hie02uh^s#Sdf9L3frd243h$d78r2G4cM6*Q05d(7rh46f!0|4-f"; - if ( entropy_test_init(1) != T ) + local handle = entropy_test_init(); + if ( ! entropy_test_add(handle, a) ) exit(1); - - if ( entropy_test_add(1, a) != T ) - exit(1); - - print entropy_test_finish(1); + print entropy_test_finish(handle); local b = "0011000aaabbbbcccc000011111000000000aaaabbbbcccc0000000"; - if ( entropy_test_init(2) != T ) + handle = entropy_test_init(); + if ( ! entropy_test_add(handle, b) ) exit(1); - - if ( entropy_test_add(2, b) != T ) - exit(1); - - print entropy_test_finish(2); + print entropy_test_finish(handle); } diff --git a/testing/btest/bifs/md5.test b/testing/btest/bifs/md5.test index 5a9715edf1..b022302c59 100644 --- a/testing/btest/bifs/md5.test +++ b/testing/btest/bifs/md5.test @@ -4,16 +4,16 @@ print md5_hash("one"); print md5_hash("one", "two", "three"); -md5_hash_init("a"); -md5_hash_init("b"); +local a = md5_hash_init(); +local b = md5_hash_init(); -md5_hash_update("a", "one"); -md5_hash_update("b", "one"); -md5_hash_update("b", "two"); -md5_hash_update("b", "three"); +md5_hash_update(a, "one"); +md5_hash_update(b, "one"); +md5_hash_update(b, "two"); +md5_hash_update(b, "three"); -print md5_hash_finish("a"); -print md5_hash_finish("b"); +print md5_hash_finish(a); +print md5_hash_finish(b); print md5_hmac("one"); print md5_hmac("one", "two", "three"); diff --git a/testing/btest/bifs/sha1.test b/testing/btest/bifs/sha1.test index 85c8df99c5..7bbd8b002e 100644 --- a/testing/btest/bifs/sha1.test +++ b/testing/btest/bifs/sha1.test @@ -4,13 +4,13 @@ print sha1_hash("one"); print sha1_hash("one", "two", "three"); -sha1_hash_init("a"); -sha1_hash_init("b"); +local a = sha1_hash_init(); +local b = sha1_hash_init(); -sha1_hash_update("a", "one"); -sha1_hash_update("b", "one"); -sha1_hash_update("b", "two"); -sha1_hash_update("b", "three"); +sha1_hash_update(a, "one"); +sha1_hash_update(b, "one"); +sha1_hash_update(b, "two"); +sha1_hash_update(b, "three"); -print sha1_hash_finish("a"); -print sha1_hash_finish("b"); +print sha1_hash_finish(a); +print sha1_hash_finish(b); diff --git a/testing/btest/bifs/sha256.test b/testing/btest/bifs/sha256.test index 7451f2fad3..a1c17f7113 100644 --- a/testing/btest/bifs/sha256.test +++ b/testing/btest/bifs/sha256.test @@ -4,13 +4,13 @@ print sha256_hash("one"); print sha256_hash("one", "two", "three"); -sha256_hash_init("a"); -sha256_hash_init("b"); +local a = sha256_hash_init(); +local b = sha256_hash_init(); -sha256_hash_update("a", "one"); -sha256_hash_update("b", "one"); -sha256_hash_update("b", "two"); -sha256_hash_update("b", "three"); +sha256_hash_update(a, "one"); +sha256_hash_update(b, "one"); +sha256_hash_update(b, "two"); +sha256_hash_update(b, "three"); -print sha256_hash_finish("a"); -print sha256_hash_finish("b"); +print sha256_hash_finish(a); +print sha256_hash_finish(b); diff --git a/testing/btest/istate/opaque.bro b/testing/btest/istate/opaque.bro new file mode 100644 index 0000000000..84818a5e70 --- /dev/null +++ b/testing/btest/istate/opaque.bro @@ -0,0 +1,77 @@ +# +# @TEST-EXEC: bro -r $TRACES/empty.trace write.bro +# @TEST-EXEC: bro read.bro +# @TEST-EXEC: btest-diff expected.log +# @TEST-EXEC: btest-diff output.log +# @TEST-EXEC: cmp output.log expected.log + +@TEST-START-FILE read.bro + +global md5_handle: opaque of md5 &persistent &synchronized; +global sha1_handle: opaque of sha1 &persistent &synchronized; +global sha256_handle: opaque of sha256 &persistent &synchronized; +global entropy_handle: opaque of entropy &persistent &synchronized; + +event bro_done() + { + local out = open("output.log"); + + # Finish incremental operations started by a previous Bro. + if ( md5_hash_update(md5_handle, "oo") ) + print out, md5_hash_finish(md5_handle); + else + print out, "md5_hash_update() failed"; + + if ( sha1_hash_update(sha1_handle, "oo") ) + print out, sha1_hash_finish(sha1_handle); + else + print out, "sha1_hash_update() failed"; + + if ( sha256_hash_update(sha256_handle, "oo") ) + print out, sha256_hash_finish(sha256_handle); + else + print out, "sha256_hash_update() failed"; + + if ( entropy_test_add(entropy_handle, "oo") ) + print out, entropy_test_finish(entropy_handle); + else + print out, "entropy_test_add() failed"; + } + +@TEST-END-FILE + +@TEST-START-FILE write.bro + +global md5_handle: opaque of md5 &persistent &synchronized; +global sha1_handle: opaque of sha1 &persistent &synchronized; +global sha256_handle: opaque of sha256 &persistent &synchronized; +global entropy_handle: opaque of entropy &persistent &synchronized; + +event bro_init() + { + local out = open("expected.log"); + print out, md5_hash("foo"); + print out, sha1_hash("foo"); + print out, sha256_hash("foo"); + print out, find_entropy("foo"); + + # Begin incremental operations. Our goal is to feed the data string "foo" to + # the computation, but split into "f" and "oo" in two instances.. + md5_handle = md5_hash_init(); + if ( ! md5_hash_update(md5_handle, "f") ) + print out, "md5_hash_update() failed"; + + sha1_handle = sha1_hash_init(); + if ( ! sha1_hash_update(sha1_handle, "f") ) + print out, "sha1_hash_update() failed"; + + sha256_handle = sha256_hash_init(); + if ( ! sha256_hash_update(sha256_handle, "f") ) + print out, "sha256_hash_update() failed"; + + entropy_handle = entropy_test_init(); + if ( ! entropy_test_add(entropy_handle, "f") ) + print out, "entropy_test_add() failed"; + } + +@TEST-END-FILE