From aba8275346c61cd474bdc13ec160e6a9b7658e2a Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Wed, 28 Nov 2012 19:49:46 -0800 Subject: [PATCH 01/35] Add opaque type to lexer, parser, and BroType. --- src/Type.cc | 42 ++++++++++++++++++++++++++++++++++++++++++ src/Type.h | 19 +++++++++++++++++++ src/parse.y | 8 +++++++- src/scan.l | 1 + 4 files changed, 69 insertions(+), 1 deletion(-) diff --git a/src/Type.cc b/src/Type.cc index e9b0949d13..34ad207842 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -96,6 +96,7 @@ BroType::BroType(TypeTag t, bool arg_base_type) case TYPE_LIST: case TYPE_FUNC: case TYPE_FILE: + case TYPE_OPAQUE: case TYPE_VECTOR: case TYPE_TYPE: internal_tag = TYPE_INTERNAL_OTHER; @@ -1262,6 +1263,42 @@ bool FileType::DoUnserialize(UnserialInfo* info) return yield != 0; } +OpaqueType::OpaqueType(const string& name) +: BroType(TYPE_OPAQUE), name(name) + { + } + +void OpaqueType::Describe(ODesc* d) const + { + if ( d->IsReadable() ) + d->AddSP("opaque of"); + else + d->Add(int(Tag())); + + d->Add(name.c_str()); + } + +// TODO: Serialization semantics not yet defined. +// +// IMPLEMENT_SERIAL(OpaqueType, SER_OPAQUE_TYPE); +// +// bool OpaqueType::DoSerialize(SerialInfo* info) const +// { +// DO_SERIALIZE(SER_OPAQUE_TYPE, BroType); +// return SERIALIZE(name); +// } +// +// bool OpaqueType::DoUnserialize(UnserialInfo* info) +// { +// DO_UNSERIALIZE(BroType); +// +// char const* n; +// if ( ! UNSERIALIZE_STR(&n, 0) ); +// return false; +// name = n; +// return true; +// } + EnumType::EnumType(const string& arg_name) : BroType(TYPE_ENUM) { @@ -1716,6 +1753,10 @@ int same_type(const BroType* t1, const BroType* t2, int is_init) case TYPE_FILE: return same_type(t1->YieldType(), t2->YieldType(), is_init); + case TYPE_OPAQUE: + // FIXME: Should we downcast here and compare the opaque type names? + return 1; + case TYPE_TYPE: return same_type(t1, t2, is_init); @@ -1805,6 +1846,7 @@ int is_assignable(BroType* t) case TYPE_VECTOR: case TYPE_FILE: + case TYPE_OPAQUE: case TYPE_TABLE: case TYPE_TYPE: return 1; diff --git a/src/Type.h b/src/Type.h index 8e2bb099d8..21b4bc2023 100644 --- a/src/Type.h +++ b/src/Type.h @@ -29,6 +29,7 @@ typedef enum { TYPE_LIST, TYPE_FUNC, TYPE_FILE, + TYPE_OPAQUE, TYPE_VECTOR, TYPE_TYPE, TYPE_ERROR @@ -499,6 +500,24 @@ protected: BroType* yield; }; +class OpaqueType : public BroType { +public: + OpaqueType(const string& name); + virtual ~OpaqueType() { }; + + const string& Name() { return name; } + + void Describe(ODesc* d) const; + +protected: + OpaqueType() { } + + // TODO: Serialization semantics not yet defined. + //DECLARE_SERIAL(OpaqueType) + + const string name; +}; + class EnumType : public BroType { public: EnumType(const string& arg_name); diff --git a/src/parse.y b/src/parse.y index b4eee1a56c..c442ffbd72 100644 --- a/src/parse.y +++ b/src/parse.y @@ -11,7 +11,7 @@ %token TOK_DOUBLE TOK_ELSE TOK_ENUM TOK_EVENT TOK_EXPORT TOK_FILE TOK_FOR %token TOK_FUNCTION TOK_GLOBAL TOK_HOOK TOK_ID TOK_IF TOK_INT %token TOK_INTERVAL TOK_LIST TOK_LOCAL TOK_MODULE -%token TOK_NEXT TOK_OF TOK_PATTERN TOK_PATTERN_TEXT +%token TOK_NEXT TOK_OF TOK_OPAQUE TOK_PATTERN TOK_PATTERN_TEXT %token TOK_PORT TOK_PRINT TOK_RECORD TOK_REDEF %token TOK_REMOVE_FROM TOK_RETURN TOK_SCHEDULE TOK_SET %token TOK_STRING TOK_SUBNET TOK_SWITCH TOK_TABLE @@ -890,6 +890,12 @@ type: $$ = new FileType(base_type(TYPE_STRING)); } + | TOK_OPAQUE TOK_OF TOK_ID + { + set_location(@1, @3); + $$ = new OpaqueType($3); + } + | resolve_id { if ( ! $1 || ! ($$ = $1->AsType()) ) diff --git a/src/scan.l b/src/scan.l index 4e1a66144e..b231e11d74 100644 --- a/src/scan.l +++ b/src/scan.l @@ -298,6 +298,7 @@ local return TOK_LOCAL; module return TOK_MODULE; next return TOK_NEXT; of return TOK_OF; +opaque return TOK_OPAQUE; pattern return TOK_PATTERN; port return TOK_PORT; print return TOK_PRINT; From fa9aeeaaf1030da3eb12d391e283d0b68dca3dca Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Tue, 4 Dec 2012 21:44:28 -1000 Subject: [PATCH 02/35] Support basic serialization of opaque. --- src/SerialTypes.h | 1 + src/Type.cc | 40 ++++++++++++++++++++-------------------- src/Type.h | 5 ++--- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/SerialTypes.h b/src/SerialTypes.h index a18c9bcc65..52ef1651de 100644 --- a/src/SerialTypes.h +++ b/src/SerialTypes.h @@ -179,6 +179,7 @@ SERIAL_TYPE(SUBNET_TYPE, 8) SERIAL_TYPE(FILE_TYPE, 9) SERIAL_TYPE(ENUM_TYPE, 10) SERIAL_TYPE(VECTOR_TYPE, 11) +SERIAL_TYPE(OPAQUE_TYPE, 12) SERIAL_CONST2(ATTRIBUTES) SERIAL_CONST2(EVENT_HANDLER) diff --git a/src/Type.cc b/src/Type.cc index 34ad207842..8b4cfca88c 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -1278,26 +1278,26 @@ void OpaqueType::Describe(ODesc* d) const d->Add(name.c_str()); } -// TODO: Serialization semantics not yet defined. -// -// IMPLEMENT_SERIAL(OpaqueType, SER_OPAQUE_TYPE); -// -// bool OpaqueType::DoSerialize(SerialInfo* info) const -// { -// DO_SERIALIZE(SER_OPAQUE_TYPE, BroType); -// return SERIALIZE(name); -// } -// -// bool OpaqueType::DoUnserialize(UnserialInfo* info) -// { -// DO_UNSERIALIZE(BroType); -// -// char const* n; -// if ( ! UNSERIALIZE_STR(&n, 0) ); -// return false; -// name = n; -// return true; -// } +IMPLEMENT_SERIAL(OpaqueType, SER_OPAQUE_TYPE); + +bool OpaqueType::DoSerialize(SerialInfo* info) const + { + DO_SERIALIZE(SER_OPAQUE_TYPE, BroType); + + // For now we only serialize the name of the type but will add type-specific + // serialization code here when necessary. + return SERIALIZE(name); + } + +bool OpaqueType::DoUnserialize(UnserialInfo* info) + { + DO_UNSERIALIZE(BroType); + char const* n; + if ( ! UNSERIALIZE_STR(&n, 0) ); + return false; + name = n; + return true; + } EnumType::EnumType(const string& arg_name) : BroType(TYPE_ENUM) diff --git a/src/Type.h b/src/Type.h index 21b4bc2023..0f356dca31 100644 --- a/src/Type.h +++ b/src/Type.h @@ -512,10 +512,9 @@ public: protected: OpaqueType() { } - // TODO: Serialization semantics not yet defined. - //DECLARE_SERIAL(OpaqueType) + DECLARE_SERIAL(OpaqueType) - const string name; + string name; }; class EnumType : public BroType { From 2494940d95f2963b052205748703b76070feed7d Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Wed, 5 Dec 2012 18:52:52 -1000 Subject: [PATCH 03/35] Implement equivalence relation for opaque types. --- src/Type.cc | 7 +++++-- src/Type.h | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Type.cc b/src/Type.cc index 8b4cfca88c..0baff8b2ee 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -1754,8 +1754,11 @@ int same_type(const BroType* t1, const BroType* t2, int is_init) return same_type(t1->YieldType(), t2->YieldType(), is_init); case TYPE_OPAQUE: - // FIXME: Should we downcast here and compare the opaque type names? - return 1; + { + const OpaqueType* ot1 = (const OpaqueType*) t1; + const OpaqueType* ot2 = (const OpaqueType*) t2; + return ot1->Name() == ot2->Name() ? 1 : 0; + } case TYPE_TYPE: return same_type(t1, t2, is_init); diff --git a/src/Type.h b/src/Type.h index 0f356dca31..0f48684675 100644 --- a/src/Type.h +++ b/src/Type.h @@ -505,7 +505,7 @@ public: OpaqueType(const string& name); virtual ~OpaqueType() { }; - const string& Name() { return name; } + const string& Name() const { return name; } void Describe(ODesc* d) const; From 622190b4efad61d33444af948c9fe0156578edca Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Fri, 7 Dec 2012 22:17:59 -1000 Subject: [PATCH 04/35] More lexer/parser work. --- src/parse.y | 17 ++++++++++++----- src/scan.l | 5 ++++- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/parse.y b/src/parse.y index c442ffbd72..3caff2581f 100644 --- a/src/parse.y +++ b/src/parse.y @@ -11,11 +11,12 @@ %token TOK_DOUBLE TOK_ELSE TOK_ENUM TOK_EVENT TOK_EXPORT TOK_FILE TOK_FOR %token TOK_FUNCTION TOK_GLOBAL TOK_HOOK TOK_ID TOK_IF TOK_INT %token TOK_INTERVAL TOK_LIST TOK_LOCAL TOK_MODULE -%token TOK_NEXT TOK_OF TOK_OPAQUE TOK_PATTERN TOK_PATTERN_TEXT +%token TOK_NEXT TOK_OF TOK_PATTERN TOK_PATTERN_TEXT %token TOK_PORT TOK_PRINT TOK_RECORD TOK_REDEF %token TOK_REMOVE_FROM TOK_RETURN TOK_SCHEDULE TOK_SET %token TOK_STRING TOK_SUBNET TOK_SWITCH TOK_TABLE %token TOK_TIME TOK_TIMEOUT TOK_TIMER TOK_TYPE TOK_UNION TOK_VECTOR TOK_WHEN +%token TOK_OPAQUE TOK_OPAQUE_MD5 TOK_OPAQUE_HYPERLOGLOG %token TOK_ATTR_ADD_FUNC TOK_ATTR_ATTR TOK_ATTR_ENCRYPT TOK_ATTR_DEFAULT %token TOK_ATTR_OPTIONAL TOK_ATTR_REDEF TOK_ATTR_ROTATE_INTERVAL @@ -46,7 +47,7 @@ %left '$' '[' ']' '(' ')' TOK_HAS_FIELD TOK_HAS_ATTR %type opt_no_test opt_no_test_block -%type TOK_ID TOK_PATTERN_TEXT single_pattern TOK_DOC TOK_POST_DOC +%type TOK_ID TOK_PATTERN_TEXT single_pattern TOK_DOC TOK_POST_DOC opaque_id %type opt_doc_list opt_post_doc_list %type local_id global_id def_global_id event_id global_or_event_id resolve_id begin_func %type local_id_list @@ -565,7 +566,6 @@ expr: id->SetType(error_type()); $$ = new NameExpr(id); } - else if ( id->IsEnumConst() ) { EnumType* t = id->Type()->AsEnumType(); @@ -890,10 +890,10 @@ type: $$ = new FileType(base_type(TYPE_STRING)); } - | TOK_OPAQUE TOK_OF TOK_ID + | TOK_OPAQUE TOK_OF opaque_id { set_location(@1, @3); - $$ = new OpaqueType($3); + $$ = new OpaqueType(@3); } | resolve_id @@ -910,6 +910,13 @@ type: } ; +opaque_id: + TOK_OPAQUE_MD5 + { } + | TOK_OPAQUE_HYPERLOGLOG + { } + ; + type_list: type_list ',' type { $1->AppendEvenIfNotPure($3); } diff --git a/src/scan.l b/src/scan.l index b231e11d74..0f9f1b0e14 100644 --- a/src/scan.l +++ b/src/scan.l @@ -298,7 +298,6 @@ local return TOK_LOCAL; module return TOK_MODULE; next return TOK_NEXT; of return TOK_OF; -opaque return TOK_OPAQUE; pattern return TOK_PATTERN; port return TOK_PORT; print return TOK_PRINT; @@ -319,6 +318,10 @@ union return TOK_UNION; vector return TOK_VECTOR; when return TOK_WHEN; +opaque return TOK_OPAQUE; +md5 return TOK_OPAQUE_MD5; +hyper-log-log return TOK_OPAQUE_HYPERLOGLOG; + &add_func return TOK_ATTR_ADD_FUNC; &attr return TOK_ATTR_ATTR; &create_expire return TOK_ATTR_EXPIRE_CREATE; From 23ca1c90ef032b3aadd8400102cfd45ebf5e58f5 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Mon, 10 Dec 2012 18:29:06 -0800 Subject: [PATCH 05/35] Adapt BiF & Bro parser to handle opaque types. --- src/builtin-func.l | 2 ++ src/builtin-func.y | 20 +++++++++++++++----- src/parse.y | 16 ++++------------ src/scan.l | 5 +---- 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/builtin-func.l b/src/builtin-func.l index 1d61f31734..3b2301e951 100644 --- a/src/builtin-func.l +++ b/src/builtin-func.l @@ -74,6 +74,8 @@ HEX [0-9a-fA-F]+ "set" return check_c_mode(TOK_SET); "table" return check_c_mode(TOK_TABLE); "vector" return check_c_mode(TOK_VECTOR); +"of" return check_c_mode(TOK_OF); +"opaque" return check_c_mode(TOK_OPAQUE); "module" return check_c_mode(TOK_MODULE); "@ARG@" return TOK_ARG; diff --git a/src/builtin-func.y b/src/builtin-func.y index fd40613236..737f5e0f3e 100644 --- a/src/builtin-func.y +++ b/src/builtin-func.y @@ -269,15 +269,15 @@ void print_event_c_body(FILE *fp) %token TOK_LPP TOK_RPP TOK_LPB TOK_RPB TOK_LPPB TOK_RPPB TOK_VAR_ARG %token TOK_BOOL -%token TOK_FUNCTION TOK_EVENT TOK_CONST TOK_ENUM -%token TOK_TYPE TOK_RECORD TOK_SET TOK_VECTOR TOK_TABLE TOK_MODULE +%token TOK_FUNCTION TOK_EVENT TOK_CONST TOK_ENUM TOK_OF +%token TOK_TYPE TOK_RECORD TOK_SET TOK_VECTOR TOK_OPAQUE TOK_TABLE TOK_MODULE %token TOK_ARGS TOK_ARG TOK_ARGC %token TOK_ID TOK_ATTR TOK_CSTR TOK_LF TOK_WS TOK_COMMENT %token TOK_ATOM TOK_INT TOK_C_TOKEN %left ',' ':' -%type TOK_C_TOKEN TOK_ID TOK_CSTR TOK_WS TOK_COMMENT TOK_ATTR TOK_INT opt_ws +%type TOK_C_TOKEN TOK_ID TOK_CSTR TOK_WS TOK_COMMENT TOK_ATTR TOK_INT opt_ws type %type TOK_ATOM TOK_BOOL %union { @@ -584,7 +584,17 @@ args_1: args_1 ',' opt_ws arg opt_ws { /* empty */ } ; -arg: TOK_ID opt_ws ':' opt_ws TOK_ID +// TODO: Migrate all other compound types to this rule. Once the BiF language +// can parse all regular Bro types, we can throw out the unnecessary +// boilerplate typedefs for addr_set, string_set, etc. +type: + TOK_OPAQUE opt_ws TOK_OF opt_ws TOK_ID + { $$ = concat("opaque of ", $5); } + | TOK_ID + { $$ = $1; } + ; + +arg: TOK_ID opt_ws ':' opt_ws type { args.push_back(new BuiltinFuncArg($1, $5)); } | TOK_VAR_ARG { @@ -594,7 +604,7 @@ arg: TOK_ID opt_ws ':' opt_ws TOK_ID } ; -return_type: ':' opt_ws TOK_ID opt_ws +return_type: ':' opt_ws type opt_ws { BuiltinFuncArg* ret = new BuiltinFuncArg("", $3); ret->PrintBro(fp_bro_init); diff --git a/src/parse.y b/src/parse.y index 3caff2581f..a2bd9664ea 100644 --- a/src/parse.y +++ b/src/parse.y @@ -11,12 +11,11 @@ %token TOK_DOUBLE TOK_ELSE TOK_ENUM TOK_EVENT TOK_EXPORT TOK_FILE TOK_FOR %token TOK_FUNCTION TOK_GLOBAL TOK_HOOK TOK_ID TOK_IF TOK_INT %token TOK_INTERVAL TOK_LIST TOK_LOCAL TOK_MODULE -%token TOK_NEXT TOK_OF TOK_PATTERN TOK_PATTERN_TEXT +%token TOK_NEXT TOK_OF TOK_OPAQUE TOK_PATTERN TOK_PATTERN_TEXT %token TOK_PORT TOK_PRINT TOK_RECORD TOK_REDEF %token TOK_REMOVE_FROM TOK_RETURN TOK_SCHEDULE TOK_SET %token TOK_STRING TOK_SUBNET TOK_SWITCH TOK_TABLE %token TOK_TIME TOK_TIMEOUT TOK_TIMER TOK_TYPE TOK_UNION TOK_VECTOR TOK_WHEN -%token TOK_OPAQUE TOK_OPAQUE_MD5 TOK_OPAQUE_HYPERLOGLOG %token TOK_ATTR_ADD_FUNC TOK_ATTR_ATTR TOK_ATTR_ENCRYPT TOK_ATTR_DEFAULT %token TOK_ATTR_OPTIONAL TOK_ATTR_REDEF TOK_ATTR_ROTATE_INTERVAL @@ -47,7 +46,7 @@ %left '$' '[' ']' '(' ')' TOK_HAS_FIELD TOK_HAS_ATTR %type opt_no_test opt_no_test_block -%type TOK_ID TOK_PATTERN_TEXT single_pattern TOK_DOC TOK_POST_DOC opaque_id +%type TOK_ID TOK_PATTERN_TEXT single_pattern TOK_DOC TOK_POST_DOC %type opt_doc_list opt_post_doc_list %type local_id global_id def_global_id event_id global_or_event_id resolve_id begin_func %type local_id_list @@ -890,10 +889,10 @@ type: $$ = new FileType(base_type(TYPE_STRING)); } - | TOK_OPAQUE TOK_OF opaque_id + | TOK_OPAQUE TOK_OF TOK_ID { set_location(@1, @3); - $$ = new OpaqueType(@3); + $$ = new OpaqueType($3); } | resolve_id @@ -910,13 +909,6 @@ type: } ; -opaque_id: - TOK_OPAQUE_MD5 - { } - | TOK_OPAQUE_HYPERLOGLOG - { } - ; - type_list: type_list ',' type { $1->AppendEvenIfNotPure($3); } diff --git a/src/scan.l b/src/scan.l index 0f9f1b0e14..b231e11d74 100644 --- a/src/scan.l +++ b/src/scan.l @@ -298,6 +298,7 @@ local return TOK_LOCAL; module return TOK_MODULE; next return TOK_NEXT; of return TOK_OF; +opaque return TOK_OPAQUE; pattern return TOK_PATTERN; port return TOK_PORT; print return TOK_PRINT; @@ -318,10 +319,6 @@ union return TOK_UNION; vector return TOK_VECTOR; when return TOK_WHEN; -opaque return TOK_OPAQUE; -md5 return TOK_OPAQUE_MD5; -hyper-log-log return TOK_OPAQUE_HYPERLOGLOG; - &add_func return TOK_ATTR_ADD_FUNC; &attr return TOK_ATTR_ATTR; &create_expire return TOK_ATTR_EXPIRE_CREATE; From 624003f036ee61c5a3dde26b5a93d1e84b20ec2e Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Tue, 11 Dec 2012 16:24:52 -0800 Subject: [PATCH 06/35] Add support for opaque hash values. --- src/CMakeLists.txt | 1 + src/OpaqueVal.cc | 200 +++++++++++++++++++++++++++++++++++++++++++++ src/OpaqueVal.h | 84 +++++++++++++++++++ src/SerialTypes.h | 5 ++ src/Val.cc | 32 ++++++++ src/Val.h | 18 ++++ 6 files changed, 340 insertions(+) create mode 100644 src/OpaqueVal.cc create mode 100644 src/OpaqueVal.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6867b9639c..69559d6258 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -358,6 +358,7 @@ set(bro_SRCS NetVar.cc NetbiosSSN.cc Obj.cc + OpaqueVal.cc OSFinger.cc PacketFilter.cc PacketSort.cc diff --git a/src/OpaqueVal.cc b/src/OpaqueVal.cc new file mode 100644 index 0000000000..478a2f502d --- /dev/null +++ b/src/OpaqueVal.cc @@ -0,0 +1,200 @@ +#include "OpaqueVal.h" +#include "Reporter.h" +#include "Serializer.h" + +bool HashVal::IsValid() const + { + return valid; + } + +bool HashVal::Init() + { + assert(! "missing implementation of Init()"); + return false; + } + +StringVal* HashVal::Get() + { + if ( ! valid ) + return new StringVal(""); + + StringVal* result = Finish(); + valid = false; + return result; + } + +bool HashVal::Feed(const void* data, size_t size) + { + if ( valid ) + return Update(data, size); + + reporter->InternalError("invalidated opaque handle"); + return false; + } + +bool HashVal::Update(const void*, size_t) + { + assert(! "missing implementation of Update()"); + return false; + } + +StringVal* HashVal::Finish() + { + assert(! "missing implementation of Finish()"); + return new StringVal(""); + } + +HashVal::HashVal(OpaqueType* t) : OpaqueVal(t), valid(true) { } + +IMPLEMENT_SERIAL(HashVal, SER_HASH_VAL); + +bool HashVal::DoSerialize(SerialInfo* info) const + { + return SERIALIZE(valid); + } + +bool HashVal::DoUnserialize(UnserialInfo* info) + { + return UNSERIALIZE(&valid); + } + + +void MD5Val::digest(val_list& vlist, u_char result[MD5_DIGEST_LENGTH]) + { + MD5_CTX h; + + md5_init(&h); + loop_over_list(vlist, i) + { + Val* v = vlist[i]; + if ( v->Type()->Tag() == TYPE_STRING ) + { + const BroString* str = v->AsString(); + md5_update(&h, str->Bytes(), str->Len()); + } + else + { + ODesc d(DESC_BINARY); + v->Describe(&d); + md5_update(&h, (const u_char *) d.Bytes(), d.Len()); + } + } + md5_final(&h, result); + } + +void MD5Val::hmac(val_list& vlist, + u_char key[MD5_DIGEST_LENGTH], + u_char result[MD5_DIGEST_LENGTH]) + { + digest(vlist, result); + for ( int i = 0; i < MD5_DIGEST_LENGTH; ++i ) + result[i] ^= key[i]; + MD5(result, MD5_DIGEST_LENGTH, result); + } + +bool MD5Val::Init() + { + md5_init(&ctx); + return true; + } + +bool MD5Val::Update(const void* data, size_t size) + { + assert(IsValid()); + md5_update(&ctx, data, size); + return true; + } + +StringVal* MD5Val::Finish() + { + assert(IsValid()); + u_char digest[MD5_DIGEST_LENGTH]; + md5_final(&ctx, digest); + return new StringVal(md5_digest_print(digest)); + } + +IMPLEMENT_SERIAL(MD5Val, SER_MD5_VAL); + +bool MD5Val::DoSerialize(SerialInfo* info) const + { + // TODO: Implement serialization of MD5 state. + return false; + } + +bool MD5Val::DoUnserialize(UnserialInfo* info) + { + // TODO: Implement deserialization of MD5 state. + return false; + } + + +bool SHA1Val::Init() + { + sha1_init(&ctx); + return true; + } + +bool SHA1Val::Update(const void* data, size_t size) + { + assert(IsValid()); + sha1_update(&ctx, data, size); + return true; + } + +StringVal* SHA1Val::Finish() + { + assert(IsValid()); + u_char digest[SHA_DIGEST_LENGTH]; + sha1_final(&ctx, digest); + return new StringVal(sha1_digest_print(digest)); + } + +IMPLEMENT_SERIAL(SHA1Val, SER_SHA1_VAL); + +bool SHA1Val::DoSerialize(SerialInfo* info) const + { + // TODO: Implement serialization of SHA1 state. + return false; + } + +bool SHA1Val::DoUnserialize(UnserialInfo* info) + { + // TODO: Implement deserialization of SHA1 state. + return false; + } + + +bool SHA256Val::Init() + { + sha256_init(&ctx); + return true; + } + +bool SHA256Val::Update(const void* data, size_t size) + { + assert(IsValid()); + sha256_update(&ctx, data, size); + return true; + } + +StringVal* SHA256Val::Finish() + { + assert(IsValid()); + u_char digest[SHA256_DIGEST_LENGTH]; + sha256_final(&ctx, digest); + return new StringVal(sha256_digest_print(digest)); + } + +IMPLEMENT_SERIAL(SHA256Val, SER_SHA256_VAL); + +bool SHA256Val::DoSerialize(SerialInfo* info) const + { + // TODO: Implement serialization of SHA256 state. + return false; + } + +bool SHA256Val::DoUnserialize(UnserialInfo* info) + { + // TODO: Implement deserialization of SHA256 state. + return false; + } diff --git a/src/OpaqueVal.h b/src/OpaqueVal.h new file mode 100644 index 0000000000..6b86b96999 --- /dev/null +++ b/src/OpaqueVal.h @@ -0,0 +1,84 @@ +#ifndef OPAQUEVAL_H +#define OPAQUEVAL_H + +#include "Val.h" +#include "digest.h" + +class HashVal : public OpaqueVal { +public: + virtual bool IsValid() const; + virtual bool Init(); + virtual bool Feed(const void* data, size_t size); + virtual StringVal* Get(); + +protected: + HashVal() { }; + HashVal(OpaqueType* t); + virtual bool Update(const void* data, size_t size); + virtual StringVal* Finish(); + + DECLARE_SERIAL(HashVal); + +private: + // This flag exists because Get() can only be called once. + bool valid; +}; + +class MD5Val : public HashVal { +public: + static void digest(val_list& vlist, u_char result[MD5_DIGEST_LENGTH]); + + static void hmac(val_list& vlist, + u_char key[MD5_DIGEST_LENGTH], + u_char result[MD5_DIGEST_LENGTH]); + + MD5Val() : HashVal(new OpaqueType("md5")) { } + +protected: + friend class Val; + + virtual bool Init() /* override */; + virtual bool Update(const void* data, size_t size) /* override */; + virtual StringVal* Finish() /* override */; + + DECLARE_SERIAL(MD5Val); + +private: + MD5_CTX ctx; +}; + +class SHA1Val : public HashVal { +public: + SHA1Val() : HashVal(new OpaqueType("sha1")) { } + +protected: + friend class Val; + + virtual bool Init() /* override */; + virtual bool Update(const void* data, size_t size) /* override */; + virtual StringVal* Finish() /* override */; + + DECLARE_SERIAL(SHA1Val); + +private: + SHA_CTX ctx; +}; + +class SHA256Val : public HashVal { +public: + SHA256Val() : HashVal(new OpaqueType("sha256")) { } + +protected: + friend class Val; + + virtual bool Init() /* override */; + virtual bool Update(const void* data, size_t size) /* override */; + virtual StringVal* Finish() /* override */; + + DECLARE_SERIAL(SHA256Val); + +private: + SHA256_CTX ctx; +}; + +#endif diff --git a/src/SerialTypes.h b/src/SerialTypes.h index 52ef1651de..a25a73945b 100644 --- a/src/SerialTypes.h +++ b/src/SerialTypes.h @@ -98,6 +98,11 @@ SERIAL_VAL(RECORD_VAL, 10) SERIAL_VAL(ENUM_VAL, 11) SERIAL_VAL(VECTOR_VAL, 12) SERIAL_VAL(MUTABLE_VAL, 13) +SERIAL_VAL(OPAQUE_VAL, 14) +SERIAL_VAL(HASH_VAL, 15) +SERIAL_VAL(MD5_VAL, 16) +SERIAL_VAL(SHA1_VAL, 17) +SERIAL_VAL(SHA256_VAL, 18) #define SERIAL_EXPR(name, val) SERIAL_CONST(name, val, EXPR) SERIAL_EXPR(EXPR, 1) diff --git a/src/Val.cc b/src/Val.cc index 79fa8a0c69..f9a59f6743 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -3114,6 +3114,38 @@ void VectorVal::ValDescribe(ODesc* d) const d->Add("]"); } +OpaqueVal::OpaqueVal(OpaqueType* t) : opaque_type(t) { } + +OpaqueVal::~OpaqueVal() + { + Unref(opaque_type); + } + +bool OpaqueVal::IsValid() const + { + return false; + } + +IMPLEMENT_SERIAL(OpaqueVal, SER_OPAQUE_VAL); + +bool OpaqueVal::DoSerialize(SerialInfo* info) const + { + DO_SERIALIZE(SER_OPAQUE_VAL, Val); + assert(opaque_type); + // TODO: how to serialize a serializable class? + //return SERIALIZE(*opaque_type); + return false; + } + +bool OpaqueVal::DoUnserialize(UnserialInfo* info) + { + DO_UNSERIALIZE(Val); + // TODO: how to deserialize a serializable class? + //opaque_type = new OpaqueType(); + //return UNSERIALIZE(opaque_type); + return false; + } + Val* check_and_promote(Val* v, const BroType* t, int is_init) { diff --git a/src/Val.h b/src/Val.h index c3ec5b04fb..925fb5a2b0 100644 --- a/src/Val.h +++ b/src/Val.h @@ -1013,6 +1013,24 @@ protected: VectorType* vector_type; }; +// See OpaqueVal.h for derived classes. +class OpaqueVal : public Val { +public: + OpaqueVal(OpaqueType* t); + virtual ~OpaqueVal(); + + // Determines whether the opaque value is in a valid state. + virtual bool IsValid() const; + +protected: + friend class Val; + OpaqueVal() { } + + DECLARE_SERIAL(OpaqueVal); + + OpaqueType* opaque_type; +}; + // Checks the given value for consistency with the given type. If an // exact match, returns it. If promotable, returns the promoted version, From 833a559cac7b5c98d83d625bb238edc0015628f6 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Tue, 11 Dec 2012 16:25:56 -0800 Subject: [PATCH 07/35] Simplify hash function BiFs. --- src/bro.bif | 267 +++++++++++++++------------------------------------- 1 file changed, 76 insertions(+), 191 deletions(-) diff --git a/src/bro.bif b/src/bro.bif index d945e54ef4..b4ce3b8530 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -530,37 +530,7 @@ function piped_exec%(program: string, to_write: string%): bool %} %%{ -static void hash_md5_val(val_list& vlist, unsigned char digest[16]) - { - MD5_CTX h; - - md5_init(&h); - loop_over_list(vlist, i) - { - Val* v = vlist[i]; - if ( v->Type()->Tag() == TYPE_STRING ) - { - const BroString* str = v->AsString(); - md5_update(&h, str->Bytes(), str->Len()); - } - else - { - ODesc d(DESC_BINARY); - v->Describe(&d); - md5_update(&h, (const u_char *) d.Bytes(), d.Len()); - } - } - md5_final(&h, digest); - } - -static void hmac_md5_val(val_list& vlist, unsigned char digest[16]) - { - hash_md5_val(vlist, digest); - for ( int i = 0; i < 16; ++i ) - digest[i] = digest[i] ^ shared_hmac_md5_key[i]; - MD5(digest, 16, digest); - } - +// TODO: Migrate these functions into SHA*Val, in the same vein as MD5Val. static void hash_sha1_val(val_list& vlist, unsigned char digest[20]) { SHA_CTX h; @@ -608,6 +578,10 @@ static void hash_sha256_val(val_list& vlist, unsigned char digest[32]) } %%} +%%{ +#include "OpaqueVal.h" +%%} + ## Computes the MD5 hash value of the provided list of arguments. ## ## Returns: The MD5 hash value of the concatenated arguments. @@ -623,8 +597,8 @@ static void hash_sha256_val(val_list& vlist, unsigned char digest[32]) ## friends. function md5_hash%(...%): string %{ - unsigned char digest[16]; - hash_md5_val(@ARG@, digest); + unsigned char digest[MD5_DIGEST_LENGTH]; + MD5Val::digest(@ARG@, digest); return new StringVal(md5_digest_print(digest)); %} @@ -643,7 +617,7 @@ function md5_hash%(...%): string ## friends. function sha1_hash%(...%): string %{ - unsigned char digest[20]; + unsigned char digest[SHA_DIGEST_LENGTH]; hash_sha1_val(@ARG@, digest); return new StringVal(sha1_digest_print(digest)); %} @@ -663,7 +637,7 @@ function sha1_hash%(...%): string ## friends. function sha256_hash%(...%): string %{ - unsigned char digest[32]; + unsigned char digest[SHA256_DIGEST_LENGTH]; hash_sha256_val(@ARG@, digest); return new StringVal(sha256_digest_print(digest)); %} @@ -679,59 +653,38 @@ function sha256_hash%(...%): string ## sha256_hash sha256_hash_init sha256_hash_update sha256_hash_finish function md5_hmac%(...%): string %{ - unsigned char digest[16]; - hmac_md5_val(@ARG@, digest); - return new StringVal(md5_digest_print(digest)); + unsigned char hmac[MD5_DIGEST_LENGTH]; + MD5Val::hmac(@ARG@, shared_hmac_md5_key, hmac); + return new StringVal(md5_digest_print(hmac)); %} -%%{ -static map md5_states; -static map sha1_states; -static map sha256_states; - -BroString* convert_index_to_string(Val* index) - { - ODesc d; - index->Describe(&d); - BroString* s = new BroString(1, d.TakeBytes(), d.Len()); - s->SetUseFreeToDelete(1); - return s; - } -%%} - -## Initializes MD5 state to enable incremental hash computation. After -## initializing the MD5 state with this function, you can feed data to -## :bro:id:`md5_hash_update` and finally need to call :bro:id:`md5_hash_finish` -## to finish the computation and get the final hash value. +## Constructs an MD5 handle to enable incremental hash computation. You can +## feed data to the ## returned opaque value with ## :bro:id:`md5_hash_update` +## and finally need to call :bro:id:`md5_hash_finish` to finish the computation +## and get the final hash value. +## +## Returns: The opaque handle associated with this hash computation. ## ## For example, when computing incremental MD5 values of transferred files in -## multiple concurrent HTTP connections, one would call ``md5_hash_init(c$id)`` -## once before invoking ``md5_hash_update(c$id, some_more_data)`` in the +## multiple concurrent HTTP connections, one keep an optional handle in the +## HTTP session record. Then, one would call +## ``c$http$md5_handle = md5_hash_init()`` ## once before invoking +## ``md5_hash_update(c$http$md5_handle, some_more_data)`` in the ## :bro:id:`http_entity_data` event handler. When all data has arrived, a call ## to :bro:id:`md5_hash_finish` returns the final hash value. ## -## index: The unique identifier to associate with this hash computation. -## ## .. bro:see:: md5_hmac md5_hash md5_hash_update md5_hash_finish ## sha1_hash sha1_hash_init sha1_hash_update sha1_hash_finish ## sha256_hash sha256_hash_init sha256_hash_update sha256_hash_finish -function md5_hash_init%(index: any%): bool +function md5_hash_init%(%): opaque of md5 %{ - BroString* s = convert_index_to_string(index); - int status = 0; - - if ( md5_states.count(*s) < 1 ) - { - MD5_CTX h; - md5_init(&h); - md5_states[*s] = h; - status = 1; - } - - delete s; - return new Val(status, TYPE_BOOL); + HashVal* digest = new MD5Val(); + digest->Init(); + return digest; %} +# TODO: Update documentation. +# ## Initializes SHA1 state to enable incremental hash computation. After ## initializing the SHA1 state with this function, you can feed data to ## :bro:id:`sha1_hash_update` and finally need to call @@ -749,23 +702,15 @@ function md5_hash_init%(index: any%): bool ## .. bro:see:: md5_hmac md5_hash md5_hash_init md5_hash_update md5_hash_finish ## sha1_hash sha1_hash_update sha1_hash_finish ## sha256_hash sha256_hash_init sha256_hash_update sha256_hash_finish -function sha1_hash_init%(index: any%): bool +function sha1_hash_init%(%): opaque of sha1 %{ - BroString* s = convert_index_to_string(index); - int status = 0; - - if ( sha1_states.count(*s) < 1 ) - { - SHA_CTX h; - sha1_init(&h); - sha1_states[*s] = h; - status = 1; - } - - delete s; - return new Val(status, TYPE_BOOL); + HashVal* digest = new SHA1Val(); + digest->Init(); + return digest; %} +# TODO: Update documentation. +# ## Initializes SHA256 state to enable incremental hash computation. After ## initializing the SHA256 state with this function, you can feed data to ## :bro:id:`sha256_hash_update` and finally need to call @@ -784,49 +729,34 @@ function sha1_hash_init%(index: any%): bool ## .. bro:see:: md5_hmac md5_hash md5_hash_init md5_hash_update md5_hash_finish ## sha1_hash sha1_hash_init sha1_hash_update sha1_hash_finish ## sha256_hash sha256_hash_update sha256_hash_finish -function sha256_hash_init%(index: any%): bool +function sha256_hash_init%(%): opaque of sha256 %{ - BroString* s = convert_index_to_string(index); - int status = 0; - - if ( sha256_states.count(*s) < 1 ) - { - SHA256_CTX h; - sha256_init(&h); - sha256_states[*s] = h; - status = 1; - } - - delete s; - return new Val(status, TYPE_BOOL); + HashVal* digest = new SHA256Val(); + digest->Init(); + return digest; %} -## Update the MD5 value associated with a given index. It is required to +## Updates the MD5 value associated with a given index. It is required to ## call :bro:id:`md5_hash_init` once before calling this ## function. ## -## index: The unique identifier to associate with this hash computation. +## handle: The opaque handle associated with this hash computation. ## ## data: The data to add to the hash computation. ## +## Returns: True on success. +## ## .. bro:see:: md5_hmac md5_hash md5_hash_init md5_hash_finish ## sha1_hash sha1_hash_init sha1_hash_update sha1_hash_finish ## sha256_hash sha256_hash_init sha256_hash_update sha256_hash_finish -function md5_hash_update%(index: any, data: string%): bool +function md5_hash_update%(handle: opaque of md5, data: string%): bool %{ - BroString* s = convert_index_to_string(index); - int status = 0; - - if ( md5_states.count(*s) > 0 ) - { - md5_update(&md5_states[*s], data->Bytes(), data->Len()); - status = 1; - } - - delete s; - return new Val(status, TYPE_BOOL); + bool rc = static_cast(handle)->Feed(data->Bytes(), data->Len()); + return new Val(rc, TYPE_BOOL); %} +# TODO: Update documentation. +# ## Update the SHA1 value associated with a given index. It is required to ## call :bro:id:`sha1_hash_init` once before calling this ## function. @@ -838,21 +768,14 @@ function md5_hash_update%(index: any, data: string%): bool ## .. bro:see:: md5_hmac md5_hash md5_hash_init md5_hash_update md5_hash_finish ## sha1_hash sha1_hash_init sha1_hash_finish ## sha256_hash sha256_hash_init sha256_hash_update sha256_hash_finish -function sha1_hash_update%(index: any, data: string%): bool +function sha1_hash_update%(handle: opaque of sha1, data: string%): bool %{ - BroString* s = convert_index_to_string(index); - int status = 0; - - if ( sha1_states.count(*s) > 0 ) - { - sha1_update(&sha1_states[*s], data->Bytes(), data->Len()); - status = 1; - } - - delete s; - return new Val(status, TYPE_BOOL); + bool rc = static_cast(handle)->Feed(data->Bytes(), data->Len()); + return new Val(rc, TYPE_BOOL); %} +# TODO: Update documentation. +# ## Update the SHA256 value associated with a given index. It is required to ## call :bro:id:`sha256_hash_init` once before calling this ## function. @@ -864,49 +787,28 @@ function sha1_hash_update%(index: any, data: string%): bool ## .. bro:see:: md5_hmac md5_hash md5_hash_init md5_hash_update md5_hash_finish ## sha1_hash sha1_hash_init sha1_hash_update sha1_hash_finish ## sha256_hash sha256_hash_init sha256_hash_finish -function sha256_hash_update%(index: any, data: string%): bool +function sha256_hash_update%(handle: opaque of sha256, data: string%): bool %{ - BroString* s = convert_index_to_string(index); - int status = 0; - - if ( sha256_states.count(*s) > 0 ) - { - sha256_update(&sha256_states[*s], data->Bytes(), data->Len()); - status = 1; - } - - delete s; - return new Val(status, TYPE_BOOL); + bool rc = static_cast(handle)->Feed(data->Bytes(), data->Len()); + return new Val(rc, TYPE_BOOL); %} ## Returns the final MD5 digest of an incremental hash computation. ## -## index: The unique identifier of this hash computation. +## handle: The opaque handle associated with this hash computation. ## -## Returns: The hash value associated with the computation at *index*. +## Returns: The hash value associated with the computation of *handle*. ## ## .. bro:see:: md5_hmac md5_hash md5_hash_init md5_hash_update ## sha1_hash sha1_hash_init sha1_hash_update sha1_hash_finish ## sha256_hash sha256_hash_init sha256_hash_update sha256_hash_finish -function md5_hash_finish%(index: any%): string +function md5_hash_finish%(handle: opaque of md5%): string %{ - BroString* s = convert_index_to_string(index); - StringVal* printable_digest; - - if ( md5_states.count(*s) > 0 ) - { - unsigned char digest[16]; - md5_final(&md5_states[*s], digest); - md5_states.erase(*s); - printable_digest = new StringVal(md5_digest_print(digest)); - } - else - printable_digest = new StringVal(""); - - delete s; - return printable_digest; + return static_cast(handle)->Get(); %} +# TODO: Update documentation. +# ## Returns the final SHA1 digest of an incremental hash computation. ## ## index: The unique identifier of this hash computation. @@ -916,23 +818,9 @@ function md5_hash_finish%(index: any%): string ## .. bro:see:: md5_hmac md5_hash md5_hash_init md5_hash_update md5_hash_finish ## sha1_hash sha1_hash_init sha1_hash_update ## sha256_hash sha256_hash_init sha256_hash_update sha256_hash_finish -function sha1_hash_finish%(index: any%): string +function sha1_hash_finish%(handle: opaque of sha1%): string %{ - BroString* s = convert_index_to_string(index); - StringVal* printable_digest; - - if ( sha1_states.count(*s) > 0 ) - { - unsigned char digest[20]; - sha1_final(&sha1_states[*s], digest); - sha1_states.erase(*s); - printable_digest = new StringVal(sha1_digest_print(digest)); - } - else - printable_digest = new StringVal(""); - - delete s; - return printable_digest; + return static_cast(handle)->Get(); %} ## Returns the final SHA256 digest of an incremental hash computation. @@ -944,23 +832,9 @@ function sha1_hash_finish%(index: any%): string ## .. bro:see:: md5_hmac md5_hash md5_hash_init md5_hash_update md5_hash_finish ## sha1_hash sha1_hash_init sha1_hash_update sha1_hash_finish ## sha256_hash sha256_hash_init sha256_hash_update -function sha256_hash_finish%(index: any%): string +function sha256_hash_finish%(handle: opaque of sha256%): string %{ - BroString* s = convert_index_to_string(index); - StringVal* printable_digest; - - if ( sha256_states.count(*s) > 0 ) - { - unsigned char digest[32]; - sha256_final(&sha256_states[*s], digest); - sha256_states.erase(*s); - printable_digest = new StringVal(sha256_digest_print(digest)); - } - else - printable_digest = new StringVal(""); - - delete s; - return printable_digest; + return static_cast(handle)->Get(); %} ## Generates a random number. @@ -1122,6 +996,17 @@ function find_entropy%(data: string%): entropy_test_result return ent_result; %} +%%{ +BroString* convert_index_to_string(Val* index) + { + ODesc d; + index->Describe(&d); + BroString* s = new BroString(1, d.TakeBytes(), d.Len()); + s->SetUseFreeToDelete(1); + return s; + } +%%} + ## Initializes data structures for incremental entropy calculation. ## ## index: An arbitrary unique value per distinct computation. From 30bab14dbf24ad18d303a500638a75bf04868b98 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Tue, 11 Dec 2012 16:26:17 -0800 Subject: [PATCH 08/35] Update base scripts and unit tests. --- scripts/base/protocols/http/file-hash.bro | 30 ++++++------- scripts/base/protocols/smtp/entities.bro | 55 ++++++++++++----------- testing/btest/bifs/md5.test | 16 +++---- testing/btest/bifs/sha1.test | 16 +++---- testing/btest/bifs/sha256.test | 16 +++---- 5 files changed, 67 insertions(+), 66 deletions(-) diff --git a/scripts/base/protocols/http/file-hash.bro b/scripts/base/protocols/http/file-hash.bro index 7e8e5cceaf..aa0a8298a0 100644 --- a/scripts/base/protocols/http/file-hash.bro +++ b/scripts/base/protocols/http/file-hash.bro @@ -13,16 +13,16 @@ export { redef record Info += { ## MD5 sum for a file transferred over HTTP calculated from the ## response body. - md5: string &log &optional; + md5: string &log &optional; ## This value can be set per-transfer to determine per request ## if a file should have an MD5 sum generated. It must be ## set to T at the time of or before the first chunk of body data. - calc_md5: bool &default=F; + calc_md5: bool &default=F; ## Indicates if an MD5 sum is being calculated for the current ## request/response pair. - calculating_md5: bool &default=F; + md5_handle: opaque of md5 &optional; }; ## Generate MD5 sums for these filetypes. @@ -41,13 +41,12 @@ event http_entity_data(c: connection, is_orig: bool, length: count, data: string if ( c$http$calc_md5 || (c$http?$mime_type && generate_md5 in c$http$mime_type) ) { - c$http$calculating_md5 = T; - md5_hash_init(c$id); + c$http$md5_handle = md5_hash_init(); } } - if ( c$http$calculating_md5 ) - md5_hash_update(c$id, data); + if ( c$http?$md5_handle ) + md5_hash_update(c$http$md5_handle, data); } ## In the event of a content gap during a file transfer, detect the state for @@ -55,11 +54,11 @@ event http_entity_data(c: connection, is_orig: bool, length: count, data: string ## incorrect anyway. event content_gap(c: connection, is_orig: bool, seq: count, length: count) &priority=5 { - if ( is_orig || ! c?$http || ! c$http$calculating_md5 ) return; + if ( is_orig || ! c?$http || ! c$http?$md5_handle ) return; set_state(c, F, is_orig); - c$http$calculating_md5 = F; - md5_hash_finish(c$id); + md5_hash_finish(c$http$md5_handle); # Ignore return value. + delete c$http$md5_handle; } ## When the file finishes downloading, finish the hash and generate a notice. @@ -67,11 +66,11 @@ event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) & { if ( is_orig || ! c?$http ) return; - if ( c$http$calculating_md5 ) + if ( c$http?$md5_handle ) { local url = build_url_http(c$http); - c$http$calculating_md5 = F; - c$http$md5 = md5_hash_finish(c$id); + c$http$md5 = md5_hash_finish(c$http$md5_handle); + delete c$http$md5_handle; NOTICE([$note=MD5, $msg=fmt("%s %s %s", c$id$orig_h, c$http$md5, url), $sub=c$http$md5, $conn=c, $URL=url]); @@ -82,11 +81,12 @@ event connection_state_remove(c: connection) &priority=-5 { if ( c?$http_state && c$http_state$current_response in c$http_state$pending && - c$http_state$pending[c$http_state$current_response]$calculating_md5 ) + c$http_state$pending[c$http_state$current_response]?$md5_handle ) { # The MD5 sum isn't going to be saved anywhere since the entire # body wouldn't have been seen anyway and we'd just be giving an # incorrect MD5 sum. - md5_hash_finish(c$id); + md5_hash_finish(c$http$md5_handle); + delete c$http$md5_handle; } } diff --git a/scripts/base/protocols/smtp/entities.bro b/scripts/base/protocols/smtp/entities.bro index e158d045e0..a20c262d56 100644 --- a/scripts/base/protocols/smtp/entities.bro +++ b/scripts/base/protocols/smtp/entities.bro @@ -16,33 +16,33 @@ export { type EntityInfo: record { ## This is the timestamp of when the MIME content transfer began. - ts: time &log; - uid: string &log; - id: conn_id &log; + ts: time &log; + uid: string &log; + id: conn_id &log; ## A count to represent the depth of this message transaction in a ## single connection where multiple messages were transferred. - trans_depth: count &log; + trans_depth: count &log; ## The filename seen in the Content-Disposition header. - filename: string &log &optional; + filename: string &log &optional; ## Track how many bytes of the MIME encoded file have been seen. - content_len: count &log &default=0; + content_len: count &log &default=0; ## The mime type of the entity discovered through magic bytes identification. - mime_type: string &log &optional; + mime_type: string &log &optional; ## The calculated MD5 sum for the MIME entity. - md5: string &log &optional; + md5: string &log &optional; ## Optionally calculate the file's MD5 sum. Must be set prior to the ## first data chunk being see in an event. - calc_md5: bool &default=F; + calc_md5: bool &default=F; ## This boolean value indicates if an MD5 sum is being calculated ## for the current file transfer. - calculating_md5: bool &default=F; + md5_handle: opaque of md5 &optional; ## Optionally write the file to disk. Must be set prior to first ## data chunk being seen in an event. - extract_file: bool &default=F; + extract_file: bool &default=F; ## Store the file handle here for the file currently being extracted. - extraction_file: file &log &optional; + extraction_file: file &log &optional; }; redef record Info += { @@ -126,18 +126,16 @@ event mime_segment_data(c: connection, length: count, data: string) &priority=-5 if ( c$smtp$current_entity$content_len == 0 ) { - if ( generate_md5 in c$smtp$current_entity$mime_type && ! never_calc_md5 ) - c$smtp$current_entity$calc_md5 = T; + local entity = c$smtp$current_entity; + if ( generate_md5 in entity$mime_type && ! never_calc_md5 ) + entity$calc_md5 = T; - if ( c$smtp$current_entity$calc_md5 ) - { - c$smtp$current_entity$calculating_md5 = T; - md5_hash_init(c$id); - } + if ( entity$calc_md5 ) + entity$md5_handle = md5_hash_init(); } - if ( c$smtp$current_entity$calculating_md5 ) - md5_hash_update(c$id, data); + if ( c$smtp$current_entity?$md5_handle ) + md5_hash_update(entity$md5_handle, data); } ## In the event of a content gap during the MIME transfer, detect the state for @@ -147,10 +145,11 @@ event content_gap(c: connection, is_orig: bool, seq: count, length: count) &prio { if ( is_orig || ! c?$smtp || ! c$smtp?$current_entity ) return; - if ( c$smtp$current_entity$calculating_md5 ) + local entity = c$smtp$current_entity; + if ( entity?$md5_handle ) { - c$smtp$current_entity$calculating_md5 = F; - md5_hash_finish(c$id); + md5_hash_finish(entity$md5_handle); + delete entity$md5_handle; } } @@ -161,12 +160,14 @@ event mime_end_entity(c: connection) &priority=-3 if ( ! c?$smtp || ! c$smtp?$current_entity ) return; - if ( c$smtp$current_entity$calculating_md5 ) + local entity = c$smtp$current_entity; + if ( entity?$md5_handle ) { - c$smtp$current_entity$md5 = md5_hash_finish(c$id); + entity$md5 = md5_hash_finish(entity$md5_handle); + delete entity$md5_handle; NOTICE([$note=MD5, $msg=fmt("Calculated a hash for a MIME entity from %s", c$id$orig_h), - $sub=c$smtp$current_entity$md5, $conn=c]); + $sub=entity$md5, $conn=c]); } } diff --git a/testing/btest/bifs/md5.test b/testing/btest/bifs/md5.test index 5a9715edf1..b022302c59 100644 --- a/testing/btest/bifs/md5.test +++ b/testing/btest/bifs/md5.test @@ -4,16 +4,16 @@ print md5_hash("one"); print md5_hash("one", "two", "three"); -md5_hash_init("a"); -md5_hash_init("b"); +local a = md5_hash_init(); +local b = md5_hash_init(); -md5_hash_update("a", "one"); -md5_hash_update("b", "one"); -md5_hash_update("b", "two"); -md5_hash_update("b", "three"); +md5_hash_update(a, "one"); +md5_hash_update(b, "one"); +md5_hash_update(b, "two"); +md5_hash_update(b, "three"); -print md5_hash_finish("a"); -print md5_hash_finish("b"); +print md5_hash_finish(a); +print md5_hash_finish(b); print md5_hmac("one"); print md5_hmac("one", "two", "three"); diff --git a/testing/btest/bifs/sha1.test b/testing/btest/bifs/sha1.test index 85c8df99c5..7bbd8b002e 100644 --- a/testing/btest/bifs/sha1.test +++ b/testing/btest/bifs/sha1.test @@ -4,13 +4,13 @@ print sha1_hash("one"); print sha1_hash("one", "two", "three"); -sha1_hash_init("a"); -sha1_hash_init("b"); +local a = sha1_hash_init(); +local b = sha1_hash_init(); -sha1_hash_update("a", "one"); -sha1_hash_update("b", "one"); -sha1_hash_update("b", "two"); -sha1_hash_update("b", "three"); +sha1_hash_update(a, "one"); +sha1_hash_update(b, "one"); +sha1_hash_update(b, "two"); +sha1_hash_update(b, "three"); -print sha1_hash_finish("a"); -print sha1_hash_finish("b"); +print sha1_hash_finish(a); +print sha1_hash_finish(b); diff --git a/testing/btest/bifs/sha256.test b/testing/btest/bifs/sha256.test index 7451f2fad3..a1c17f7113 100644 --- a/testing/btest/bifs/sha256.test +++ b/testing/btest/bifs/sha256.test @@ -4,13 +4,13 @@ print sha256_hash("one"); print sha256_hash("one", "two", "three"); -sha256_hash_init("a"); -sha256_hash_init("b"); +local a = sha256_hash_init(); +local b = sha256_hash_init(); -sha256_hash_update("a", "one"); -sha256_hash_update("b", "one"); -sha256_hash_update("b", "two"); -sha256_hash_update("b", "three"); +sha256_hash_update(a, "one"); +sha256_hash_update(b, "one"); +sha256_hash_update(b, "two"); +sha256_hash_update(b, "three"); -print sha256_hash_finish("a"); -print sha256_hash_finish("b"); +print sha256_hash_finish(a); +print sha256_hash_finish(b); From 91f2cb2b64500f9c65ece76d977423202b7efcaa Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Tue, 11 Dec 2012 16:49:33 -0800 Subject: [PATCH 09/35] Add missing type name that caused failing tests. --- src/Type.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Type.cc b/src/Type.cc index 0baff8b2ee..30ff3ce897 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -30,6 +30,7 @@ const char* type_name(TypeTag t) "table", "union", "record", "types", "func", "file", + "opaque", "vector", "type", "error", From ddd306f00f337659a652c3da8fbfba63d1b6ebb2 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Wed, 12 Dec 2012 10:28:56 -0800 Subject: [PATCH 10/35] Migrate free SHA* functions to SHA*Val::digest(). --- aux/broccoli | 2 +- aux/broctl | 2 +- src/OpaqueVal.cc | 98 +++++++++++++++++++++++++++++++++++------------- src/OpaqueVal.h | 4 ++ src/bro.bif | 55 ++------------------------- 5 files changed, 80 insertions(+), 81 deletions(-) diff --git a/aux/broccoli b/aux/broccoli index 06682dbb15..a8846fc5b0 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 06682dbb15d26d2688bdc9ad76efec17d38dc80f +Subproject commit a8846fc5b004ffe4e3d00e826d0077ba19518192 diff --git a/aux/broctl b/aux/broctl index 91e3b8ad44..834131cd0e 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 91e3b8ad445cb4d8919b4ee1cc6f0753c3fa0a55 +Subproject commit 834131cd0ec0f63cce9de818726fe6167dedbf34 diff --git a/src/OpaqueVal.cc b/src/OpaqueVal.cc index 478a2f502d..fb70a85293 100644 --- a/src/OpaqueVal.cc +++ b/src/OpaqueVal.cc @@ -29,7 +29,7 @@ bool HashVal::Feed(const void* data, size_t size) return Update(data, size); reporter->InternalError("invalidated opaque handle"); - return false; + return false; } bool HashVal::Update(const void*, size_t) @@ -60,37 +60,37 @@ bool HashVal::DoUnserialize(UnserialInfo* info) void MD5Val::digest(val_list& vlist, u_char result[MD5_DIGEST_LENGTH]) - { - MD5_CTX h; + { + MD5_CTX h; - md5_init(&h); - loop_over_list(vlist, i) - { - Val* v = vlist[i]; - if ( v->Type()->Tag() == TYPE_STRING ) - { - const BroString* str = v->AsString(); - md5_update(&h, str->Bytes(), str->Len()); - } - else - { - ODesc d(DESC_BINARY); - v->Describe(&d); - md5_update(&h, (const u_char *) d.Bytes(), d.Len()); - } - } - md5_final(&h, result); - } + md5_init(&h); + loop_over_list(vlist, i) + { + Val* v = vlist[i]; + if ( v->Type()->Tag() == TYPE_STRING ) + { + const BroString* str = v->AsString(); + md5_update(&h, str->Bytes(), str->Len()); + } + else + { + ODesc d(DESC_BINARY); + v->Describe(&d); + md5_update(&h, (const u_char *) d.Bytes(), d.Len()); + } + } + md5_final(&h, result); + } void MD5Val::hmac(val_list& vlist, u_char key[MD5_DIGEST_LENGTH], u_char result[MD5_DIGEST_LENGTH]) - { - digest(vlist, result); - for ( int i = 0; i < MD5_DIGEST_LENGTH; ++i ) - result[i] ^= key[i]; - MD5(result, MD5_DIGEST_LENGTH, result); - } + { + digest(vlist, result); + for ( int i = 0; i < MD5_DIGEST_LENGTH; ++i ) + result[i] ^= key[i]; + MD5(result, MD5_DIGEST_LENGTH, result); + } bool MD5Val::Init() { @@ -128,6 +128,28 @@ bool MD5Val::DoUnserialize(UnserialInfo* info) } +void SHA1Val::digest(val_list& vlist, u_char result[SHA_DIGEST_LENGTH]) + { + SHA_CTX h; + sha1_init(&h); + loop_over_list(vlist, i) + { + Val* v = vlist[i]; + if ( v->Type()->Tag() == TYPE_STRING ) + { + const BroString* str = v->AsString(); + sha1_update(&h, str->Bytes(), str->Len()); + } + else + { + ODesc d(DESC_BINARY); + v->Describe(&d); + sha1_update(&h, (const u_char *) d.Bytes(), d.Len()); + } + } + sha1_final(&h, result); + } + bool SHA1Val::Init() { sha1_init(&ctx); @@ -164,6 +186,28 @@ bool SHA1Val::DoUnserialize(UnserialInfo* info) } +void SHA256Val::digest(val_list& vlist, u_char result[SHA256_DIGEST_LENGTH]) + { + SHA256_CTX h; + sha256_init(&h); + loop_over_list(vlist, i) + { + Val* v = vlist[i]; + if ( v->Type()->Tag() == TYPE_STRING ) + { + const BroString* str = v->AsString(); + sha256_update(&h, str->Bytes(), str->Len()); + } + else + { + ODesc d(DESC_BINARY); + v->Describe(&d); + sha256_update(&h, (const u_char *) d.Bytes(), d.Len()); + } + } + sha256_final(&h, result); + } + bool SHA256Val::Init() { sha256_init(&ctx); diff --git a/src/OpaqueVal.h b/src/OpaqueVal.h index 6b86b96999..3d665446aa 100644 --- a/src/OpaqueVal.h +++ b/src/OpaqueVal.h @@ -49,6 +49,8 @@ private: class SHA1Val : public HashVal { public: + static void digest(val_list& vlist, u_char result[SHA_DIGEST_LENGTH]); + SHA1Val() : HashVal(new OpaqueType("sha1")) { } protected: @@ -66,6 +68,8 @@ private: class SHA256Val : public HashVal { public: + static void digest(val_list& vlist, u_char result[SHA256_DIGEST_LENGTH]); + SHA256Val() : HashVal(new OpaqueType("sha256")) { } protected: diff --git a/src/bro.bif b/src/bro.bif index b4ce3b8530..36e86efd52 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -529,55 +529,6 @@ function piped_exec%(program: string, to_write: string%): bool return new Val(1, TYPE_BOOL); %} -%%{ -// TODO: Migrate these functions into SHA*Val, in the same vein as MD5Val. -static void hash_sha1_val(val_list& vlist, unsigned char digest[20]) - { - SHA_CTX h; - - sha1_init(&h); - loop_over_list(vlist, i) - { - Val* v = vlist[i]; - if ( v->Type()->Tag() == TYPE_STRING ) - { - const BroString* str = v->AsString(); - sha1_update(&h, str->Bytes(), str->Len()); - } - else - { - ODesc d(DESC_BINARY); - v->Describe(&d); - sha1_update(&h, (const u_char *) d.Bytes(), d.Len()); - } - } - sha1_final(&h, digest); - } - -static void hash_sha256_val(val_list& vlist, unsigned char digest[32]) - { - SHA256_CTX h; - - sha256_init(&h); - loop_over_list(vlist, i) - { - Val* v = vlist[i]; - if ( v->Type()->Tag() == TYPE_STRING ) - { - const BroString* str = v->AsString(); - sha256_update(&h, str->Bytes(), str->Len()); - } - else - { - ODesc d(DESC_BINARY); - v->Describe(&d); - sha256_update(&h, (const u_char *) d.Bytes(), d.Len()); - } - } - sha256_final(&h, digest); - } -%%} - %%{ #include "OpaqueVal.h" %%} @@ -618,7 +569,7 @@ function md5_hash%(...%): string function sha1_hash%(...%): string %{ unsigned char digest[SHA_DIGEST_LENGTH]; - hash_sha1_val(@ARG@, digest); + SHA1Val::digest(@ARG@, digest); return new StringVal(sha1_digest_print(digest)); %} @@ -638,7 +589,7 @@ function sha1_hash%(...%): string function sha256_hash%(...%): string %{ unsigned char digest[SHA256_DIGEST_LENGTH]; - hash_sha256_val(@ARG@, digest); + SHA256Val::digest(@ARG@, digest); return new StringVal(sha256_digest_print(digest)); %} @@ -659,7 +610,7 @@ function md5_hmac%(...%): string %} ## Constructs an MD5 handle to enable incremental hash computation. You can -## feed data to the ## returned opaque value with ## :bro:id:`md5_hash_update` +## feed data to the returned opaque value with ## :bro:id:`md5_hash_update` ## and finally need to call :bro:id:`md5_hash_finish` to finish the computation ## and get the final hash value. ## From 97e3ea1efe6e7500fa8b14db3bdc04e8623b9c50 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Wed, 12 Dec 2012 10:39:48 -0800 Subject: [PATCH 11/35] Update hash BiF documentation. --- src/bro.bif | 71 +++++++++++++++++++++++++---------------------------- 1 file changed, 33 insertions(+), 38 deletions(-) diff --git a/src/bro.bif b/src/bro.bif index 36e86efd52..aec9a56609 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -610,9 +610,9 @@ function md5_hmac%(...%): string %} ## Constructs an MD5 handle to enable incremental hash computation. You can -## feed data to the returned opaque value with ## :bro:id:`md5_hash_update` -## and finally need to call :bro:id:`md5_hash_finish` to finish the computation -## and get the final hash value. +## feed data to the returned opaque value with :bro:id:`md5_hash_update` and +## finally need to call :bro:id:`md5_hash_finish` to finish the computation +## and get the hash digest as result. ## ## Returns: The opaque handle associated with this hash computation. ## @@ -634,21 +634,20 @@ function md5_hash_init%(%): opaque of md5 return digest; %} -# TODO: Update documentation. -# -## Initializes SHA1 state to enable incremental hash computation. After -## initializing the SHA1 state with this function, you can feed data to -## :bro:id:`sha1_hash_update` and finally need to call -## :bro:id:`sha1_hash_finish` to finish the computation and get the final hash -## value. +## Constructs an SHA1 handle to enable incremental hash computation. You can +## feed data to the returned opaque value with :bro:id:`sha1_hash_update` and +## finally need to call :bro:id:`sha1_hash_finish` to finish the computation +## and get the hash digest as result. ## ## For example, when computing incremental SHA1 values of transferred files in -## multiple concurrent HTTP connections, one would call ``sha1_hash_init(c$id)`` -## once before invoking ``sha1_hash_update(c$id, some_more_data)`` in the +## multiple concurrent HTTP connections, one keep an optional handle in the +## HTTP session record. Then, one would call +## ``c$http$sha1_handle = sha1_hash_init()`` ## once before invoking +## ``sha1_hash_update(c$http$sha1_handle, some_more_data)`` in the ## :bro:id:`http_entity_data` event handler. When all data has arrived, a call ## to :bro:id:`sha1_hash_finish` returns the final hash value. ## -## index: The unique identifier to associate with this hash computation. +## Returns: The opaque handle associated with this hash computation. ## ## .. bro:see:: md5_hmac md5_hash md5_hash_init md5_hash_update md5_hash_finish ## sha1_hash sha1_hash_update sha1_hash_finish @@ -660,22 +659,20 @@ function sha1_hash_init%(%): opaque of sha1 return digest; %} -# TODO: Update documentation. -# -## Initializes SHA256 state to enable incremental hash computation. After -## initializing the SHA256 state with this function, you can feed data to -## :bro:id:`sha256_hash_update` and finally need to call -## :bro:id:`sha256_hash_finish` to finish the computation and get the final hash -## value. +## Constructs an SHA256 handle to enable incremental hash computation. You can +## feed data to the returned opaque value with :bro:id:`sha256_hash_update` and +## finally need to call :bro:id:`sha256_hash_finish` to finish the computation +## and get the hash digest as result. ## ## For example, when computing incremental SHA256 values of transferred files in -## multiple concurrent HTTP connections, one would call -## ``sha256_hash_init(c$id)`` once before invoking -## ``sha256_hash_update(c$id, some_more_data)`` in the +## multiple concurrent HTTP connections, one keep an optional handle in the +## HTTP session record. Then, one would call +## ``c$http$sha256_handle = sha256_hash_init()`` ## once before invoking +## ``sha256_hash_update(c$http$sha256_handle, some_more_data)`` in the ## :bro:id:`http_entity_data` event handler. When all data has arrived, a call ## to :bro:id:`sha256_hash_finish` returns the final hash value. ## -## index: The unique identifier to associate with this hash computation. +## Returns: The opaque handle associated with this hash computation. ## ## .. bro:see:: md5_hmac md5_hash md5_hash_init md5_hash_update md5_hash_finish ## sha1_hash sha1_hash_init sha1_hash_update sha1_hash_finish @@ -706,16 +703,16 @@ function md5_hash_update%(handle: opaque of md5, data: string%): bool return new Val(rc, TYPE_BOOL); %} -# TODO: Update documentation. -# -## Update the SHA1 value associated with a given index. It is required to +## Updates the SHA1 value associated with a given index. It is required to ## call :bro:id:`sha1_hash_init` once before calling this ## function. ## -## index: The unique identifier to associate with this hash computation. +## handle: The opaque handle associated with this hash computation. ## ## data: The data to add to the hash computation. ## +## Returns: True on success. +## ## .. bro:see:: md5_hmac md5_hash md5_hash_init md5_hash_update md5_hash_finish ## sha1_hash sha1_hash_init sha1_hash_finish ## sha256_hash sha256_hash_init sha256_hash_update sha256_hash_finish @@ -725,16 +722,16 @@ function sha1_hash_update%(handle: opaque of sha1, data: string%): bool return new Val(rc, TYPE_BOOL); %} -# TODO: Update documentation. -# -## Update the SHA256 value associated with a given index. It is required to +## Updates the SHA256 value associated with a given index. It is required to ## call :bro:id:`sha256_hash_init` once before calling this ## function. ## -## index: The unique identifier to associate with this hash computation. +## handle: The opaque handle associated with this hash computation. ## ## data: The data to add to the hash computation. ## +## Returns: True on success. +## ## .. bro:see:: md5_hmac md5_hash md5_hash_init md5_hash_update md5_hash_finish ## sha1_hash sha1_hash_init sha1_hash_update sha1_hash_finish ## sha256_hash sha256_hash_init sha256_hash_finish @@ -758,13 +755,11 @@ function md5_hash_finish%(handle: opaque of md5%): string return static_cast(handle)->Get(); %} -# TODO: Update documentation. -# ## Returns the final SHA1 digest of an incremental hash computation. ## -## index: The unique identifier of this hash computation. +## handle: The opaque handle associated with this hash computation. ## -## Returns: The hash value associated with the computation at *index*. +## Returns: The hash value associated with the computation of *handle*. ## ## .. bro:see:: md5_hmac md5_hash md5_hash_init md5_hash_update md5_hash_finish ## sha1_hash sha1_hash_init sha1_hash_update @@ -776,9 +771,9 @@ function sha1_hash_finish%(handle: opaque of sha1%): string ## Returns the final SHA256 digest of an incremental hash computation. ## -## index: The unique identifier of this hash computation. +## handle: The opaque handle associated with this hash computation. ## -## Returns: The hash value associated with the computation at *index*. +## Returns: The hash value associated with the computation of *handle*. ## ## .. bro:see:: md5_hmac md5_hash md5_hash_init md5_hash_update md5_hash_finish ## sha1_hash sha1_hash_init sha1_hash_update sha1_hash_finish From 238cffaac4578bd5817266513663734b1587c394 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Wed, 12 Dec 2012 14:44:28 -0800 Subject: [PATCH 12/35] Implement serialization of opaque types. This means that, conceptually, incremental hash computations now can start at one node and finish at another one. --- src/OpaqueVal.cc | 156 +++++++++++++++++++++++++++++++++++++++++++---- src/Val.cc | 12 ++-- 2 files changed, 148 insertions(+), 20 deletions(-) diff --git a/src/OpaqueVal.cc b/src/OpaqueVal.cc index fb70a85293..c3b127534d 100644 --- a/src/OpaqueVal.cc +++ b/src/OpaqueVal.cc @@ -50,11 +50,13 @@ IMPLEMENT_SERIAL(HashVal, SER_HASH_VAL); bool HashVal::DoSerialize(SerialInfo* info) const { + DO_SERIALIZE(SER_HASH_VAL, OpaqueVal); return SERIALIZE(valid); } bool HashVal::DoUnserialize(UnserialInfo* info) { + DO_UNSERIALIZE(OpaqueVal); return UNSERIALIZE(&valid); } @@ -117,14 +119,58 @@ IMPLEMENT_SERIAL(MD5Val, SER_MD5_VAL); bool MD5Val::DoSerialize(SerialInfo* info) const { - // TODO: Implement serialization of MD5 state. - return false; + DO_SERIALIZE(SER_MD5_VAL, HashVal); + + if ( ! IsValid() ) + return true; + + if ( ! SERIALIZE(ctx.A) ) + return false; + if ( ! SERIALIZE(ctx.B) ) + return false; + if ( ! SERIALIZE(ctx.C) ) + return false; + if ( ! SERIALIZE(ctx.D) ) + return false; + if ( ! SERIALIZE(ctx.Nl) ) + return false; + if ( ! SERIALIZE(ctx.Nh) ) + return false; + for ( int i = 0; i < MD5_LBLOCK; ++i ) + if ( ! SERIALIZE(ctx.data[i]) ) + return false; + if ( ! SERIALIZE(ctx.num) ) + return false; + + return true; } bool MD5Val::DoUnserialize(UnserialInfo* info) { - // TODO: Implement deserialization of MD5 state. - return false; + DO_UNSERIALIZE(HashVal); + + if (! IsValid()) + return true; + + if ( ! UNSERIALIZE(&ctx.A) ) + return false; + if ( ! UNSERIALIZE(&ctx.B) ) + return false; + if ( ! UNSERIALIZE(&ctx.C) ) + return false; + if ( ! UNSERIALIZE(&ctx.D) ) + return false; + if ( ! UNSERIALIZE(&ctx.Nl) ) + return false; + if ( ! UNSERIALIZE(&ctx.Nh) ) + return false; + for ( int i = 0; i < MD5_LBLOCK; ++i ) + if ( ! UNSERIALIZE(&ctx.data[i]) ) + return false; + if ( ! UNSERIALIZE(&ctx.num) ) + return false; + + return true; } @@ -175,14 +221,62 @@ IMPLEMENT_SERIAL(SHA1Val, SER_SHA1_VAL); bool SHA1Val::DoSerialize(SerialInfo* info) const { - // TODO: Implement serialization of SHA1 state. - return false; + DO_SERIALIZE(SER_SHA1_VAL, HashVal); + + if ( ! IsValid() ) + return true; + + if ( ! SERIALIZE(ctx.h0) ) + return false; + if ( ! SERIALIZE(ctx.h1) ) + return false; + if ( ! SERIALIZE(ctx.h2) ) + return false; + if ( ! SERIALIZE(ctx.h3) ) + return false; + if ( ! SERIALIZE(ctx.h4) ) + return false; + if ( ! SERIALIZE(ctx.Nl) ) + return false; + if ( ! SERIALIZE(ctx.Nh) ) + return false; + for ( int i = 0; i < SHA_LBLOCK; ++i ) + if ( ! SERIALIZE(ctx.data[i]) ) + return false; + if ( ! SERIALIZE(ctx.num) ) + return false; + + return true; } bool SHA1Val::DoUnserialize(UnserialInfo* info) { - // TODO: Implement deserialization of SHA1 state. - return false; + DO_UNSERIALIZE(HashVal); + + if ( ! IsValid() ) + return true; + + if ( ! UNSERIALIZE(&ctx.h0) ) + return false; + if ( ! UNSERIALIZE(&ctx.h1) ) + return false; + if ( ! UNSERIALIZE(&ctx.h2) ) + return false; + if ( ! UNSERIALIZE(&ctx.h3) ) + return false; + if ( ! UNSERIALIZE(&ctx.h4) ) + return false; + if ( ! UNSERIALIZE(&ctx.Nl) ) + return false; + if ( ! UNSERIALIZE(&ctx.Nh) ) + return false; + for ( int i = 0; i < SHA_LBLOCK; ++i ) + if ( ! UNSERIALIZE(&ctx.data[i]) ) + return false; + if ( ! UNSERIALIZE(&ctx.num) ) + return false; + + return true; } @@ -233,12 +327,50 @@ IMPLEMENT_SERIAL(SHA256Val, SER_SHA256_VAL); bool SHA256Val::DoSerialize(SerialInfo* info) const { - // TODO: Implement serialization of SHA256 state. - return false; + DO_SERIALIZE(SER_SHA256_VAL, HashVal); + + if ( ! IsValid() ) + return true; + + for ( int i = 0; i < 8; ++i ) + if ( ! SERIALIZE(ctx.h[i]) ) + return false; + if ( ! SERIALIZE(ctx.Nl) ) + return false; + if ( ! SERIALIZE(ctx.Nh) ) + return false; + for ( int i = 0; i < SHA_LBLOCK; ++i ) + if ( ! SERIALIZE(ctx.data[i]) ) + return false; + if ( ! SERIALIZE(ctx.num) ) + return false; + if ( ! SERIALIZE(ctx.md_len) ) + return false; + + return true; } bool SHA256Val::DoUnserialize(UnserialInfo* info) { - // TODO: Implement deserialization of SHA256 state. - return false; + DO_UNSERIALIZE(HashVal); + + if ( ! IsValid() ) + return true; + + for ( int i = 0; i < 8; ++i ) + if ( ! UNSERIALIZE(&ctx.h[i]) ) + return false; + if ( ! UNSERIALIZE(&ctx.Nl) ) + return false; + if ( ! UNSERIALIZE(&ctx.Nh) ) + return false; + for ( int i = 0; i < SHA_LBLOCK; ++i ) + if ( ! UNSERIALIZE(&ctx.data[i]) ) + return false; + if ( ! UNSERIALIZE(&ctx.num) ) + return false; + if ( ! UNSERIALIZE(&ctx.md_len) ) + return false; + + return true; } diff --git a/src/Val.cc b/src/Val.cc index e60a9c1fdb..67a4579295 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -3131,19 +3131,15 @@ IMPLEMENT_SERIAL(OpaqueVal, SER_OPAQUE_VAL); bool OpaqueVal::DoSerialize(SerialInfo* info) const { DO_SERIALIZE(SER_OPAQUE_VAL, Val); - assert(opaque_type); - // TODO: how to serialize a serializable class? - //return SERIALIZE(*opaque_type); - return false; + assert(opaque_type); + return opaque_type->Serialize(info); } bool OpaqueVal::DoUnserialize(UnserialInfo* info) { DO_UNSERIALIZE(Val); - // TODO: how to deserialize a serializable class? - //opaque_type = new OpaqueType(); - //return UNSERIALIZE(opaque_type); - return false; + opaque_type = static_cast(BroType::Unserialize(info)); + return opaque_type != 0; } From 652a015522aef79a362c032b966839354c134c16 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Wed, 12 Dec 2012 14:54:07 -0800 Subject: [PATCH 13/35] Be clearer about delegation. Bro uses the Do* prefix to signify the implementation of an aspect. This commit adopts the opaque values to use this pattern. --- src/OpaqueVal.cc | 68 ++++++++++++++++++++++++++++++++---------------- src/OpaqueVal.h | 23 ++++++++-------- 2 files changed, 57 insertions(+), 34 deletions(-) diff --git a/src/OpaqueVal.cc b/src/OpaqueVal.cc index c3b127534d..2a6c8dd384 100644 --- a/src/OpaqueVal.cc +++ b/src/OpaqueVal.cc @@ -9,8 +9,9 @@ bool HashVal::IsValid() const bool HashVal::Init() { - assert(! "missing implementation of Init()"); - return false; + assert(! valid); + valid = DoInit(); + return valid; } StringVal* HashVal::Get() @@ -18,7 +19,7 @@ StringVal* HashVal::Get() if ( ! valid ) return new StringVal(""); - StringVal* result = Finish(); + StringVal* result = DoGet(); valid = false; return result; } @@ -26,21 +27,27 @@ StringVal* HashVal::Get() bool HashVal::Feed(const void* data, size_t size) { if ( valid ) - return Update(data, size); + return DoFeed(data, size); reporter->InternalError("invalidated opaque handle"); return false; } -bool HashVal::Update(const void*, size_t) +bool HashVal::DoInit() { - assert(! "missing implementation of Update()"); + assert(! "missing implementation of DoInit()"); return false; } -StringVal* HashVal::Finish() +bool HashVal::DoFeed(const void*, size_t) { - assert(! "missing implementation of Finish()"); + assert(! "missing implementation of DoFeed()"); + return false; + } + +StringVal* HashVal::DoGet() + { + assert(! "missing implementation of DoGet()"); return new StringVal(""); } @@ -94,22 +101,27 @@ void MD5Val::hmac(val_list& vlist, MD5(result, MD5_DIGEST_LENGTH, result); } -bool MD5Val::Init() +bool MD5Val::DoInit() { + assert(! IsValid()); md5_init(&ctx); return true; } -bool MD5Val::Update(const void* data, size_t size) +bool MD5Val::DoFeed(const void* data, size_t size) { - assert(IsValid()); + if ( ! IsValid() ) + return new StringVal(""); + md5_update(&ctx, data, size); return true; } -StringVal* MD5Val::Finish() +StringVal* MD5Val::DoGet() { - assert(IsValid()); + if ( ! IsValid() ) + return new StringVal(""); + u_char digest[MD5_DIGEST_LENGTH]; md5_final(&ctx, digest); return new StringVal(md5_digest_print(digest)); @@ -196,22 +208,27 @@ void SHA1Val::digest(val_list& vlist, u_char result[SHA_DIGEST_LENGTH]) sha1_final(&h, result); } -bool SHA1Val::Init() +bool SHA1Val::DoInit() { + assert(! IsValid()); sha1_init(&ctx); return true; } -bool SHA1Val::Update(const void* data, size_t size) +bool SHA1Val::DoFeed(const void* data, size_t size) { - assert(IsValid()); + if ( ! IsValid() ) + return new StringVal(""); + sha1_update(&ctx, data, size); return true; } -StringVal* SHA1Val::Finish() +StringVal* SHA1Val::DoGet() { - assert(IsValid()); + if ( ! IsValid() ) + return new StringVal(""); + u_char digest[SHA_DIGEST_LENGTH]; sha1_final(&ctx, digest); return new StringVal(sha1_digest_print(digest)); @@ -302,22 +319,27 @@ void SHA256Val::digest(val_list& vlist, u_char result[SHA256_DIGEST_LENGTH]) sha256_final(&h, result); } -bool SHA256Val::Init() +bool SHA256Val::DoInit() { + assert( ! IsValid() ); sha256_init(&ctx); return true; } -bool SHA256Val::Update(const void* data, size_t size) +bool SHA256Val::DoFeed(const void* data, size_t size) { - assert(IsValid()); + if ( ! IsValid() ) + return new StringVal(""); + sha256_update(&ctx, data, size); return true; } -StringVal* SHA256Val::Finish() +StringVal* SHA256Val::DoGet() { - assert(IsValid()); + if ( ! IsValid() ) + return new StringVal(""); + u_char digest[SHA256_DIGEST_LENGTH]; sha256_final(&ctx, digest); return new StringVal(sha256_digest_print(digest)); diff --git a/src/OpaqueVal.h b/src/OpaqueVal.h index 3d665446aa..04f614e095 100644 --- a/src/OpaqueVal.h +++ b/src/OpaqueVal.h @@ -14,8 +14,9 @@ public: protected: HashVal() { }; HashVal(OpaqueType* t); - virtual bool Update(const void* data, size_t size); - virtual StringVal* Finish(); + virtual bool DoInit(); + virtual bool DoFeed(const void* data, size_t size); + virtual StringVal* DoGet(); DECLARE_SERIAL(HashVal); @@ -37,9 +38,9 @@ public: protected: friend class Val; - virtual bool Init() /* override */; - virtual bool Update(const void* data, size_t size) /* override */; - virtual StringVal* Finish() /* override */; + virtual bool DoInit() /* override */; + virtual bool DoFeed(const void* data, size_t size) /* override */; + virtual StringVal* DoGet() /* override */; DECLARE_SERIAL(MD5Val); @@ -56,9 +57,9 @@ public: protected: friend class Val; - virtual bool Init() /* override */; - virtual bool Update(const void* data, size_t size) /* override */; - virtual StringVal* Finish() /* override */; + virtual bool DoInit() /* override */; + virtual bool DoFeed(const void* data, size_t size) /* override */; + virtual StringVal* DoGet() /* override */; DECLARE_SERIAL(SHA1Val); @@ -75,9 +76,9 @@ public: protected: friend class Val; - virtual bool Init() /* override */; - virtual bool Update(const void* data, size_t size) /* override */; - virtual StringVal* Finish() /* override */; + virtual bool DoInit() /* override */; + virtual bool DoFeed(const void* data, size_t size) /* override */; + virtual StringVal* DoGet() /* override */; DECLARE_SERIAL(SHA256Val); From 8454f692036772def1bbec3813a791320d36626e Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Wed, 12 Dec 2012 15:04:12 -0800 Subject: [PATCH 14/35] Fix initialization of hash values. --- src/OpaqueVal.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/OpaqueVal.cc b/src/OpaqueVal.cc index 2a6c8dd384..22d1dc9b59 100644 --- a/src/OpaqueVal.cc +++ b/src/OpaqueVal.cc @@ -9,7 +9,9 @@ bool HashVal::IsValid() const bool HashVal::Init() { - assert(! valid); + if ( ! valid ) + return false; + valid = DoInit(); return valid; } @@ -51,7 +53,7 @@ StringVal* HashVal::DoGet() return new StringVal(""); } -HashVal::HashVal(OpaqueType* t) : OpaqueVal(t), valid(true) { } +HashVal::HashVal(OpaqueType* t) : OpaqueVal(t), valid(false) { } IMPLEMENT_SERIAL(HashVal, SER_HASH_VAL); From c3428ea49166cf6a14b3f6ee7aab15b878cd45d6 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Wed, 12 Dec 2012 15:12:25 -0800 Subject: [PATCH 15/35] Fix the fix :-/. --- src/OpaqueVal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OpaqueVal.cc b/src/OpaqueVal.cc index 22d1dc9b59..7b3dfc38f8 100644 --- a/src/OpaqueVal.cc +++ b/src/OpaqueVal.cc @@ -9,7 +9,7 @@ bool HashVal::IsValid() const bool HashVal::Init() { - if ( ! valid ) + if ( valid ) return false; valid = DoInit(); From 09aa14d03729824f24bc2f4abc9abe119af60a11 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Wed, 12 Dec 2012 15:13:28 -0800 Subject: [PATCH 16/35] Use more descriptive error message. --- src/OpaqueVal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OpaqueVal.cc b/src/OpaqueVal.cc index 7b3dfc38f8..d2f95da29b 100644 --- a/src/OpaqueVal.cc +++ b/src/OpaqueVal.cc @@ -31,7 +31,7 @@ bool HashVal::Feed(const void* data, size_t size) if ( valid ) return DoFeed(data, size); - reporter->InternalError("invalidated opaque handle"); + reporter->InternalError("invalid opaque hash value"); return false; } From 0cf98ac3259d503f479460b721833d43095d216b Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 13 Dec 2012 10:27:08 -0500 Subject: [PATCH 17/35] Improved file name extraction for SMTP when file name is included in Content-Type header. --- scripts/base/protocols/smtp/entities.bro | 4 ++++ scripts/base/utils/files.bro | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/base/protocols/smtp/entities.bro b/scripts/base/protocols/smtp/entities.bro index e158d045e0..b4b5fd1f03 100644 --- a/scripts/base/protocols/smtp/entities.bro +++ b/scripts/base/protocols/smtp/entities.bro @@ -177,6 +177,10 @@ event mime_one_header(c: connection, h: mime_header_rec) if ( h$name == "CONTENT-DISPOSITION" && /[fF][iI][lL][eE][nN][aA][mM][eE]/ in h$value ) c$smtp$current_entity$filename = extract_filename_from_content_disposition(h$value); + + if ( h$name == "CONTENT-TYPE" && + /[nN][aA][mM][eE][:blank:]*=/ in h$value ) + c$smtp$current_entity$filename = extract_filename_from_content_disposition(h$value); } event mime_end_entity(c: connection) &priority=-5 diff --git a/scripts/base/utils/files.bro b/scripts/base/utils/files.bro index e1e599b0b6..76d2ecea4f 100644 --- a/scripts/base/utils/files.bro +++ b/scripts/base/utils/files.bro @@ -19,7 +19,7 @@ function generate_extraction_filename(prefix: string, c: connection, suffix: str ## the filename. function extract_filename_from_content_disposition(data: string): string { - local filename = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE][[:blank:]]*=[[:blank:]]*/, ""); + local filename = sub(data, /^.*[nN][aA][mM][eE][[:blank:]]*=[[:blank:]]*/, ""); # Remove quotes around the filename if they are there. if ( /^\"/ in filename ) filename = split_n(filename, /\"/, F, 2)[2]; From 3c27267223bbd3f6c9910ef20acb9ce0e42cfacb Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 13 Dec 2012 11:09:41 -0500 Subject: [PATCH 18/35] Improvements for the "bad checksums" detector to make it detect bad TCP checksums. --- .../base/misc/find-checksum-offloading.bro | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/scripts/base/misc/find-checksum-offloading.bro b/scripts/base/misc/find-checksum-offloading.bro index a0a2c692b7..f371628c33 100644 --- a/scripts/base/misc/find-checksum-offloading.bro +++ b/scripts/base/misc/find-checksum-offloading.bro @@ -14,7 +14,8 @@ export { } # Keep track of how many bad checksums have been seen. -global bad_checksums = 0; +global bad_ip_checksums = 0; +global bad_tcp_checksums = 0; # Track to see if this script is done so that messages aren't created multiple times. global done = F; @@ -25,10 +26,19 @@ event ChecksumOffloading::check() return; local pkts_recvd = net_stats()$pkts_recvd; - if ( (bad_checksums*1.0 / net_stats()$pkts_recvd*1.0) > 0.05 ) + local bad_ip_checksum_pct = bad_ip_checksums*1.0 / net_stats()$pkts_recvd*1.0; + local bad_tcp_checksum_pct = bad_tcp_checksums*1.0 / net_stats()$pkts_recvd*1.0; + if ( bad_ip_checksum_pct > 0.05 || bad_tcp_checksum_pct > 0.05 ) { local packet_src = reading_traces() ? "trace file likely has" : "interface is likely receiving"; - local message = fmt("Your %s invalid IP checksums, most likely from NIC checksum offloading.", packet_src); + local bad_checksum_msg = bad_ip_checksum_pct > 0.0 ? "IP" : ""; + if ( bad_tcp_checksum_pct > 0.0 ) + { + if ( |bad_checksum_msg| > 0 ) + bad_checksum_msg += " and "; + bad_checksum_msg += "TCP"; + } + local message = fmt("Your %s invalid %s checksums, most likely from NIC checksum offloading.", packet_src, bad_checksum_msg); Reporter::warning(message); done = T; } @@ -48,7 +58,13 @@ event bro_init() event net_weird(name: string) { if ( name == "bad_IP_checksum" ) - ++bad_checksums; + ++bad_ip_checksums; + } + +event conn_weird(name: string, c: connection, addl: string) + { + if ( name == "bad_TCP_checksum" ) + ++bad_tcp_checksums; } event bro_done() From 524e15f2c5d49e8a3883e83f01078487c289cacd Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 13 Dec 2012 11:41:13 -0800 Subject: [PATCH 19/35] Fix memory leak in ascii input reader. In case an error was encountered while parsing an element, that element itself was not deleted, but remained in memory indefenitely. --- src/input/readers/Ascii.cc | 42 ++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 173ce418ca..e2ddffd9ff 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -273,14 +273,14 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { Error(Fmt("Field: %s Invalid value for boolean: %s", field.name.c_str(), s.c_str())); - return 0; + goto parse_error; } break; case TYPE_INT: val->val.int_val = strtoll(s.c_str(), &end, 10); if ( CheckNumberError(s, end) ) - return 0; + goto parse_error; break; case TYPE_DOUBLE: @@ -288,20 +288,20 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) case TYPE_INTERVAL: val->val.double_val = strtod(s.c_str(), &end); if ( CheckNumberError(s, end) ) - return 0; + goto parse_error; break; case TYPE_COUNT: case TYPE_COUNTER: val->val.uint_val = strtoull(s.c_str(), &end, 10); if ( CheckNumberError(s, end) ) - return 0; + goto parse_error; break; case TYPE_PORT: val->val.port_val.port = strtoull(s.c_str(), &end, 10); if ( CheckNumberError(s, end) ) - return 0; + goto parse_error; val->val.port_val.proto = TRANSPORT_UNKNOWN; break; @@ -313,13 +313,13 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) if ( pos == s.npos ) { Error(Fmt("Invalid value for subnet: %s", s.c_str())); - return 0; + goto parse_error; } uint8_t width = (uint8_t) strtol(s.substr(pos+1).c_str(), &end, 10); if ( CheckNumberError(s, end) ) - return 0; + goto parse_error; string addr = s.substr(0, pos); @@ -349,6 +349,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) } unsigned int pos = 0; + bool error = false; if ( s.compare(empty_field) == 0 ) length = 0; @@ -385,14 +386,16 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { Error(Fmt("Internal error while parsing set. pos %d >= length %d." " Element: %s", pos, length, element.c_str())); + error = true; break; } Value* newval = EntryToVal(element, field.subType()); if ( newval == 0 ) { - Error("Error while reading set"); - return 0; + Error("Error while reading set or vector"); + error = true; + break; } lvals[pos] = newval; @@ -403,22 +406,31 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) // Test if the string ends with a set_separator... or if the // complete string is empty. In either of these cases we have // to push an empty val on top of it. - if ( s.empty() || *s.rbegin() == set_separator[0] ) + if ( !error && ( s.empty() || *s.rbegin() == set_separator[0] ) ) { lvals[pos] = EntryToVal("", field.subType()); if ( lvals[pos] == 0 ) { Error("Error while trying to add empty set element"); - return 0; + goto parse_error; } pos++; } + if ( error ) { + // we had an error while reading a set or a vector. + // hence we have to clean up the values that have been read so far + for ( int i = 0; i < pos; i++ ) + delete lvals[i]; + + goto parse_error; + } + if ( pos != length ) { Error(Fmt("Internal error while parsing set: did not find all elements: %s", s.c_str())); - return 0; + goto parse_error; } break; @@ -427,10 +439,14 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) default: Error(Fmt("unsupported field format %d for %s", field.type, field.name.c_str())); - return 0; + goto parse_error; } return val; + +parse_error: + delete val; + return 0; } // read the entire file and send appropriate thingies back to InputMgr From 290c2a0b4df2db38ade684cf386a5c9b6b271d9e Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 13 Dec 2012 15:05:29 -0600 Subject: [PATCH 20/35] Make const variables actually constant. Addresses #922. Both local and global variables declared with "const" could be modified, but now expressions that would modify them should generate an error message at parse-time. --- scripts/base/frameworks/notice/cluster.bro | 6 +- src/Expr.cc | 15 +++- src/Expr.h | 3 +- src/Var.cc | 4 +- .../Baseline/language.const/invalid.stderr | 13 +++ .../Baseline/language.const/invalid.stdout | 0 .../Baseline/language.const/valid.stderr | 0 .../Baseline/language.const/valid.stdout | 10 +++ testing/btest/language/const.bro | 79 +++++++++++++++++++ 9 files changed, 120 insertions(+), 10 deletions(-) create mode 100644 testing/btest/Baseline/language.const/invalid.stderr create mode 100644 testing/btest/Baseline/language.const/invalid.stdout create mode 100644 testing/btest/Baseline/language.const/valid.stderr create mode 100644 testing/btest/Baseline/language.const/valid.stdout create mode 100644 testing/btest/language/const.bro diff --git a/scripts/base/frameworks/notice/cluster.bro b/scripts/base/frameworks/notice/cluster.bro index 3ee113acf3..e812c3fdca 100644 --- a/scripts/base/frameworks/notice/cluster.bro +++ b/scripts/base/frameworks/notice/cluster.bro @@ -21,12 +21,10 @@ redef Cluster::manager2worker_events += /Notice::begin_suppression/; redef Cluster::worker2manager_events += /Notice::cluster_notice/; @if ( Cluster::local_node_type() != Cluster::MANAGER ) + # The notice policy is completely handled by the manager and shouldn't be # done by workers or proxies to save time for packet processing. -event bro_init() &priority=11 - { - Notice::policy = table(); - } +redef Notice::policy = table(); event Notice::begin_suppression(n: Notice::Info) { diff --git a/src/Expr.cc b/src/Expr.cc index 7995d5d495..3a4e8add70 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -229,9 +229,10 @@ bool Expr::DoUnserialize(UnserialInfo* info) } -NameExpr::NameExpr(ID* arg_id) : Expr(EXPR_NAME) +NameExpr::NameExpr(ID* arg_id, bool const_init) : Expr(EXPR_NAME) { id = arg_id; + in_const_init = const_init; SetType(id->Type()->Ref()); EventHandler* h = event_registry->Lookup(id->Name()); @@ -287,6 +288,9 @@ Expr* NameExpr::MakeLvalue() if ( id->AsType() ) ExprError("Type name is not an lvalue"); + if ( id->IsConst() && ! in_const_init ) + ExprError("const is not a modifiable lvalue"); + return new RefExpr(this); } @@ -337,9 +341,11 @@ bool NameExpr::DoSerialize(SerialInfo* info) const // Write out just the name of the function if requested. if ( info->globals_as_names && id->IsGlobal() ) - return SERIALIZE('n') && SERIALIZE(id->Name()); + return SERIALIZE('n') && SERIALIZE(id->Name()) && + SERIALIZE(in_const_init); else - return SERIALIZE('f') && id->Serialize(info); + return SERIALIZE('f') && id->Serialize(info) && + SERIALIZE(in_const_init); } bool NameExpr::DoUnserialize(UnserialInfo* info) @@ -370,6 +376,9 @@ bool NameExpr::DoUnserialize(UnserialInfo* info) if ( ! id ) return false; + if ( ! UNSERIALIZE(&in_const_init) ) + return false; + return true; } diff --git a/src/Expr.h b/src/Expr.h index afdf02c124..ea17c735b5 100644 --- a/src/Expr.h +++ b/src/Expr.h @@ -198,7 +198,7 @@ protected: class NameExpr : public Expr { public: - NameExpr(ID* id); + NameExpr(ID* id, bool const_init = false); ~NameExpr(); ID* Id() const { return id; } @@ -220,6 +220,7 @@ protected: DECLARE_SERIAL(NameExpr); ID* id; + bool in_const_init; }; class ConstExpr : public Expr { diff --git a/src/Var.cc b/src/Var.cc index 2e9fdbe946..b4d76097d3 100644 --- a/src/Var.cc +++ b/src/Var.cc @@ -210,7 +210,6 @@ static void make_var(ID* id, BroType* t, init_class c, Expr* init, // defined. Func* f = new BroFunc(id, 0, 0, 0, 0); id->SetVal(new Val(f)); - id->SetConst(); } } @@ -233,8 +232,9 @@ Stmt* add_local(ID* id, BroType* t, init_class c, Expr* init, Ref(id); + Expr* name_expr = new NameExpr(id, dt == VAR_CONST); Stmt* stmt = - new ExprStmt(new AssignExpr(new NameExpr(id), init, 0, 0, + new ExprStmt(new AssignExpr(name_expr, init, 0, 0, id->Attrs() ? id->Attrs()->Attrs() : 0 )); stmt->SetLocationInfo(init->GetLocationInfo()); diff --git a/testing/btest/Baseline/language.const/invalid.stderr b/testing/btest/Baseline/language.const/invalid.stderr new file mode 100644 index 0000000000..b08c472708 --- /dev/null +++ b/testing/btest/Baseline/language.const/invalid.stderr @@ -0,0 +1,13 @@ +error in ./invalid.bro, line 15: const is not a modifiable lvalue (foo) +error in ./invalid.bro, line 16: const is not a modifiable lvalue (foo) +error in ./invalid.bro, line 17: const is not a modifiable lvalue (bar) +error in ./invalid.bro, line 17: const is not a modifiable lvalue (foo) +error in ./invalid.bro, line 18: const is not a modifiable lvalue (foo) +error in ./invalid.bro, line 19: const is not a modifiable lvalue (foo) +error in ./invalid.bro, line 20: const is not a modifiable lvalue (foo) +error in ./invalid.bro, line 22: const is not a modifiable lvalue (foo) +error in ./invalid.bro, line 25: const is not a modifiable lvalue (bar) +error in ./invalid.bro, line 26: const is not a modifiable lvalue (baz) +error in ./invalid.bro, line 27: const is not a modifiable lvalue (bar) +error in ./invalid.bro, line 28: const is not a modifiable lvalue (baz) +error in ./invalid.bro, line 33: const is not a modifiable lvalue (foo) diff --git a/testing/btest/Baseline/language.const/invalid.stdout b/testing/btest/Baseline/language.const/invalid.stdout new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/btest/Baseline/language.const/valid.stderr b/testing/btest/Baseline/language.const/valid.stderr new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/btest/Baseline/language.const/valid.stdout b/testing/btest/Baseline/language.const/valid.stdout new file mode 100644 index 0000000000..5e3a76f060 --- /dev/null +++ b/testing/btest/Baseline/language.const/valid.stdout @@ -0,0 +1,10 @@ +40 +enter f, 10 +exit f, 110 +enter f, 9 +exit f, 109 +enter f, 7 +exit f, 107 +foo, 10 +bar, 9 +baz, 7 diff --git a/testing/btest/language/const.bro b/testing/btest/language/const.bro new file mode 100644 index 0000000000..ee938e8d45 --- /dev/null +++ b/testing/btest/language/const.bro @@ -0,0 +1,79 @@ +# @TEST-EXEC: bro -b valid.bro 2>valid.stderr 1>valid.stdout +# @TEST-EXEC: btest-diff valid.stderr +# @TEST-EXEC: btest-diff valid.stdout + +# @TEST-EXEC-FAIL: bro -b invalid.bro 2>invalid.stderr 1>invalid.stdout +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff invalid.stderr +# @TEST-EXEC: btest-diff invalid.stdout + +@TEST-START-FILE valid.bro +# First some simple code that should be valid and error-free. + +function f(c: count) + { + print "enter f", c; + c = c + 100; + print "exit f", c; + } + +const foo = 0 &redef; +redef foo = 10; + +const bar = 9; + +event bro_init() + { + const baz = 7; + local i = foo; + i = i + bar + 2; + i = i + baz + 11; + ++i; + print i; + --i; + f(foo); + f(bar); + f(baz); + print "foo", foo; + print "bar", bar; + print "baz", baz; + } + +@TEST-END-FILE + +@TEST-START-FILE invalid.bro +# Now some const assignments that should generate errors at parse-time. + +const foo = 0 &redef; +redef foo = 10; + +const bar = 9; + +event bro_init() + { + const baz = 7; + local s = 0; + + print "nope"; + + foo = 100; + foo = bar; + foo = bar = baz; + foo = s; + ++foo; + s = foo = bar; + + if ( foo = 0 ) + print "nope"; + + bar = 1 + 1; + baz = s; + ++bar; + --baz; + + print "foo", foo; + print "bar", bar; + print "baz", baz; + print "foo=foo", foo = foo; + } + +@TEST-END-FILE From 483cc6bd9eebb4883b5784f39325253581e9cb30 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Thu, 13 Dec 2012 17:51:42 -0800 Subject: [PATCH 21/35] Fix a hard-to-spot bug. --- src/Type.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Type.cc b/src/Type.cc index 30ff3ce897..a78da7f3c6 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -1294,7 +1294,7 @@ bool OpaqueType::DoUnserialize(UnserialInfo* info) { DO_UNSERIALIZE(BroType); char const* n; - if ( ! UNSERIALIZE_STR(&n, 0) ); + if ( ! UNSERIALIZE_STR(&n, 0) ) return false; name = n; return true; From 86faab1e066d08d8888b5bd606d869fceafdae17 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Thu, 13 Dec 2012 18:39:29 -0800 Subject: [PATCH 22/35] C++ify RandTest.* Specifically: - Move implementation details into *.cc. - Const correctness: do not require superfluous cast. - Style: asterisk "binds" to type, not name. --- src/RandTest.cc | 20 +++++++++++++++----- src/RandTest.h | 18 ++++++------------ 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/src/RandTest.cc b/src/RandTest.cc index 638cc6c765..99237e114e 100644 --- a/src/RandTest.cc +++ b/src/RandTest.cc @@ -12,7 +12,17 @@ Modified for Bro by Seth Hall - July 2010 */ -#include +#include "RandTest.h" + +#define log2of10 3.32192809488736234787 +/* RT_LOG2 -- Calculate log to the base 2 */ +static double rt_log2(double x) +{ + return log2of10 * log10(x); +} + +// RT_INCIRC = pow(pow(256.0, (double) (RT_MONTEN / 2)) - 1, 2.0); +#define RT_INCIRC 281474943156225.0 RandTest::RandTest() { @@ -28,9 +38,9 @@ RandTest::RandTest() } } -void RandTest::add(void *buf, int bufl) +void RandTest::add(const void *buf, int bufl) { - unsigned char *bp = (unsigned char*)buf; + const unsigned char *bp = static_cast(buf); int oc; while (bufl-- > 0) @@ -78,8 +88,8 @@ void RandTest::add(void *buf, int bufl) } } -void RandTest::end(double *r_ent, double *r_chisq, - double *r_mean, double *r_montepicalc, double *r_scc) +void RandTest::end(double* r_ent, double* r_chisq, + double* r_mean, double* r_montepicalc, double* r_scc) { int i; double ent, chisq, scc, datasum; diff --git a/src/RandTest.h b/src/RandTest.h index a4f551b602..d787392e14 100644 --- a/src/RandTest.h +++ b/src/RandTest.h @@ -1,28 +1,22 @@ #include -#define log2of10 3.32192809488736234787 -/* RT_LOG2 -- Calculate log to the base 2 */ -static double rt_log2(double x) -{ - return log2of10 * log10(x); -} +class EntropyVal; #define RT_MONTEN 6 /* Bytes used as Monte Carlo co-ordinates. This should be no more bits than the mantissa of your "double" floating point type. */ -// RT_INCIRC = pow(pow(256.0, (double) (RT_MONTEN / 2)) - 1, 2.0); -#define RT_INCIRC 281474943156225.0 - class RandTest { public: RandTest(); - void add(void *buf, int bufl); - void end(double *r_ent, double *r_chisq, double *r_mean, - double *r_montepicalc, double *r_scc); + void add(const void* buf, int bufl); + void end(double* r_ent, double* r_chisq, double* r_mean, + double* r_montepicalc, double* r_scc); private: + friend class EntropyVal; + long ccount[256]; /* Bins to count occurrences of values */ long totalc; /* Total bytes counted */ int mp; From b9d05f56d063bee4d03d8356513f493c2118c6a3 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Thu, 13 Dec 2012 19:18:47 -0800 Subject: [PATCH 23/35] Migrate entropy testing to opaque. --- src/OpaqueVal.cc | 100 ++++++++++++++++++++++++++++ src/OpaqueVal.h | 19 ++++++ src/RandTest.cc | 1 + src/RandTest.h | 17 +++-- src/SerialTypes.h | 1 + src/bro.bif | 87 +++++------------------- testing/btest/bifs/entropy_test.bro | 18 ++--- 7 files changed, 156 insertions(+), 87 deletions(-) diff --git a/src/OpaqueVal.cc b/src/OpaqueVal.cc index d2f95da29b..a5e65e7e7e 100644 --- a/src/OpaqueVal.cc +++ b/src/OpaqueVal.cc @@ -398,3 +398,103 @@ bool SHA256Val::DoUnserialize(UnserialInfo* info) return true; } + + +bool EntropyVal::Feed(const void* data, size_t size) + { + state.add(data, size); + return true; + } + +bool EntropyVal::Get(double *r_ent, double *r_chisq, double *r_mean, + double *r_montepicalc, double *r_scc) + { + state.end(r_ent, r_chisq, r_mean, r_montepicalc, r_scc); + return true; + } + +IMPLEMENT_SERIAL(EntropyVal, SER_ENTROPY_VAL); + +bool EntropyVal::DoSerialize(SerialInfo* info) const + { + DO_SERIALIZE(SER_ENTROPY_VAL, OpaqueVal); + + for ( int i = 0; i < 256; ++i ) + if ( ! SERIALIZE(state.ccount[i]) ) + return false; + if ( ! SERIALIZE(state.totalc) ) + return false; + if ( ! SERIALIZE(state.mp) ) + return false; + if ( ! SERIALIZE(state.sccfirst) ) + return false; + for ( int i = 0; i < RT_MONTEN; ++i ) + if ( ! SERIALIZE(state.monte[i]) ) + return false; + if ( ! SERIALIZE(state.inmont) ) + return false; + if ( ! SERIALIZE(state.mcount) ) + return false; + if ( ! SERIALIZE(state.cexp) ) + return false; + if ( ! SERIALIZE(state.montex) ) + return false; + if ( ! SERIALIZE(state.montey) ) + return false; + if ( ! SERIALIZE(state.montepi) ) + return false; + if ( ! SERIALIZE(state.sccu0) ) + return false; + if ( ! SERIALIZE(state.scclast) ) + return false; + if ( ! SERIALIZE(state.scct1) ) + return false; + if ( ! SERIALIZE(state.scct2) ) + return false; + if ( ! SERIALIZE(state.scct3) ) + return false; + + return true; + } + +bool EntropyVal::DoUnserialize(UnserialInfo* info) + { + DO_UNSERIALIZE(OpaqueVal); + + for ( int i = 0; i < 256; ++i ) + if ( ! UNSERIALIZE(&state.ccount[i]) ) + return false; + if ( ! UNSERIALIZE(&state.totalc) ) + return false; + if ( ! UNSERIALIZE(&state.mp) ) + return false; + if ( ! UNSERIALIZE(&state.sccfirst) ) + return false; + for ( int i = 0; i < RT_MONTEN; ++i ) + if ( ! UNSERIALIZE(&state.monte[i]) ) + return false; + if ( ! UNSERIALIZE(&state.inmont) ) + return false; + if ( ! UNSERIALIZE(&state.mcount) ) + return false; + if ( ! UNSERIALIZE(&state.cexp) ) + return false; + if ( ! UNSERIALIZE(&state.montex) ) + return false; + if ( ! UNSERIALIZE(&state.montey) ) + return false; + if ( ! UNSERIALIZE(&state.montepi) ) + return false; + if ( ! UNSERIALIZE(&state.sccu0) ) + return false; + if ( ! UNSERIALIZE(&state.scclast) ) + return false; + if ( ! UNSERIALIZE(&state.scct1) ) + return false; + if ( ! UNSERIALIZE(&state.scct2) ) + return false; + if ( ! UNSERIALIZE(&state.scct3) ) + return false; + + return true; + } diff --git a/src/OpaqueVal.h b/src/OpaqueVal.h index 04f614e095..02e532aeec 100644 --- a/src/OpaqueVal.h +++ b/src/OpaqueVal.h @@ -1,6 +1,7 @@ #ifndef OPAQUEVAL_H #define OPAQUEVAL_H +#include "RandTest.h" #include "Val.h" #include "digest.h" @@ -86,4 +87,22 @@ private: SHA256_CTX ctx; }; +class EntropyVal : public OpaqueVal { +public: + EntropyVal() : OpaqueVal(new OpaqueType("entropy")) { } + + bool Feed(const void* data, size_t size); + bool Get(double *r_ent, double *r_chisq, double *r_mean, + double *r_montepicalc, double *r_scc); + +protected: + friend class Val; + EntropyVal(OpaqueType* t); + + DECLARE_SERIAL(EntropyVal); + +private: + RandTest state; +}; + #endif diff --git a/src/RandTest.cc b/src/RandTest.cc index 99237e114e..94e76500b5 100644 --- a/src/RandTest.cc +++ b/src/RandTest.cc @@ -12,6 +12,7 @@ Modified for Bro by Seth Hall - July 2010 */ +#include #include "RandTest.h" #define log2of10 3.32192809488736234787 diff --git a/src/RandTest.h b/src/RandTest.h index d787392e14..bb1eb3c6b4 100644 --- a/src/RandTest.h +++ b/src/RandTest.h @@ -1,11 +1,14 @@ -#include +#ifndef RANDTEST_H +#define RANDTEST_H -class EntropyVal; +#include "util.h" #define RT_MONTEN 6 /* Bytes used as Monte Carlo co-ordinates. This should be no more bits than the mantissa of your "double" floating point type. */ +class EntropyVal; + class RandTest { public: @@ -17,12 +20,14 @@ class RandTest { private: friend class EntropyVal; - long ccount[256]; /* Bins to count occurrences of values */ - long totalc; /* Total bytes counted */ + int64 ccount[256]; /* Bins to count occurrences of values */ + int64 totalc; /* Total bytes counted */ int mp; int sccfirst; unsigned int monte[RT_MONTEN]; - long inmont, mcount; + int64 inmont, mcount; double cexp, montex, montey, montepi, sccu0, scclast, scct1, scct2, scct3; - }; +}; + +#endif diff --git a/src/SerialTypes.h b/src/SerialTypes.h index 0eac79a04b..e103c1c40e 100644 --- a/src/SerialTypes.h +++ b/src/SerialTypes.h @@ -103,6 +103,7 @@ SERIAL_VAL(HASH_VAL, 15) SERIAL_VAL(MD5_VAL, 16) SERIAL_VAL(SHA1_VAL, 17) SERIAL_VAL(SHA256_VAL, 18) +SERIAL_VAL(ENTROPY_VAL, 19) #define SERIAL_EXPR(name, val) SERIAL_CONST(name, val, EXPR) SERIAL_EXPR(EXPR, 1) diff --git a/src/bro.bif b/src/bro.bif index aec9a56609..260e19c00c 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -878,11 +878,6 @@ function identify_data%(data: string, return_mime: bool%): string return new StringVal(descr); %} -%%{ -#include -static map entropy_states; -%%} - ## Performs an entropy test on the given data. ## See http://www.fourmilab.ch/random. ## @@ -927,13 +922,11 @@ function find_entropy%(data: string%): entropy_test_result %{ double montepi, scc, ent, mean, chisq; montepi = scc = ent = mean = chisq = 0.0; + EntropyVal e; + e.Feed(data->Bytes(), data->Len()); + e.Get(&ent, &chisq, &mean, &montepi, &scc); + RecordVal* ent_result = new RecordVal(entropy_test_result); - RandTest *rt = new RandTest(); - - rt->add((char*) data->Bytes(), data->Len()); - rt->end(&ent, &chisq, &mean, &montepi, &scc); - delete rt; - ent_result->Assign(0, new Val(ent, TYPE_DOUBLE)); ent_result->Assign(1, new Val(chisq, TYPE_DOUBLE)); ent_result->Assign(2, new Val(mean, TYPE_DOUBLE)); @@ -942,98 +935,54 @@ function find_entropy%(data: string%): entropy_test_result return ent_result; %} -%%{ -BroString* convert_index_to_string(Val* index) - { - ODesc d; - index->Describe(&d); - BroString* s = new BroString(1, d.TakeBytes(), d.Len()); - s->SetUseFreeToDelete(1); - return s; - } -%%} - ## Initializes data structures for incremental entropy calculation. ## -## index: An arbitrary unique value per distinct computation. -## -## Returns: True on success. +## Returns: An opaque handle to be used in subsequent operations. ## ## .. bro:see:: find_entropy entropy_test_add entropy_test_finish -function entropy_test_init%(index: any%): bool +function entropy_test_init%(%): opaque of entropy %{ - BroString* s = convert_index_to_string(index); - int status = 0; - - if ( entropy_states.count(*s) < 1 ) - { - entropy_states[*s] = new RandTest(); - status = 1; - } - - delete s; - return new Val(status, TYPE_BOOL); + return new EntropyVal(); %} -## Adds data to an incremental entropy calculation. Before using this function, -## one needs to invoke :bro:id:`entropy_test_init`. +## Adds data to an incremental entropy calculation. +## +## handle: The opaque handle representing the entropy calculation state. ## ## data: The data to add to the entropy calculation. ## -## index: An arbitrary unique value that identifies a particular entropy -## computation. -## ## Returns: True on success. ## ## .. bro:see:: find_entropy entropy_test_add entropy_test_finish -function entropy_test_add%(index: any, data: string%): bool +function entropy_test_add%(handle: opaque of entropy, data: string%): bool %{ - BroString* s = convert_index_to_string(index); - int status = 0; - - if ( entropy_states.count(*s) > 0 ) - { - entropy_states[*s]->add((char*) data->Bytes(), data->Len()); - status = 1; - } - - delete s; + bool status = static_cast(handle)->Feed(data->Bytes(), + data->Len()); return new Val(status, TYPE_BOOL); %} ## Finishes an incremental entropy calculation. Before using this function, -## one needs to initialize the computation with :bro:id:`entropy_test_init` and +## one needs to obtain an opaque handle with :bro:id:`entropy_test_init` and ## add data to it via :bro:id:`entropy_test_add`. ## -## index: An arbitrary unique value that identifies a particular entropy -## computation. +## handle: The opaque handle representing the entropy calculation state. ## ## Returns: The result of the entropy test. See :bro:id:`find_entropy` for a ## description of the individual components. ## ## .. bro:see:: find_entropy entropy_test_init entropy_test_add -function entropy_test_finish%(index: any%): entropy_test_result +function entropy_test_finish%(handle: opaque of entropy%): entropy_test_result %{ - BroString* s = convert_index_to_string(index); double montepi, scc, ent, mean, chisq; montepi = scc = ent = mean = chisq = 0.0; + static_cast(handle)->Get(&ent, &chisq, &mean, &montepi, &scc); + RecordVal* ent_result = new RecordVal(entropy_test_result); - - if ( entropy_states.count(*s) > 0 ) - { - RandTest *rt = entropy_states[*s]; - rt->end(&ent, &chisq, &mean, &montepi, &scc); - entropy_states.erase(*s); - delete rt; - } - ent_result->Assign(0, new Val(ent, TYPE_DOUBLE)); ent_result->Assign(1, new Val(chisq, TYPE_DOUBLE)); ent_result->Assign(2, new Val(mean, TYPE_DOUBLE)); ent_result->Assign(3, new Val(montepi, TYPE_DOUBLE)); ent_result->Assign(4, new Val(scc, TYPE_DOUBLE)); - - delete s; return ent_result; %} diff --git a/testing/btest/bifs/entropy_test.bro b/testing/btest/bifs/entropy_test.bro index 8dc54e09b2..2a2dd422d1 100644 --- a/testing/btest/bifs/entropy_test.bro +++ b/testing/btest/bifs/entropy_test.bro @@ -5,20 +5,14 @@ event bro_init() { local a = "dh3Hie02uh^s#Sdf9L3frd243h$d78r2G4cM6*Q05d(7rh46f!0|4-f"; - if ( entropy_test_init(1) != T ) + local handle = entropy_test_init(); + if ( ! entropy_test_add(handle, a) ) exit(1); - - if ( entropy_test_add(1, a) != T ) - exit(1); - - print entropy_test_finish(1); + print entropy_test_finish(handle); local b = "0011000aaabbbbcccc000011111000000000aaaabbbbcccc0000000"; - if ( entropy_test_init(2) != T ) + handle = entropy_test_init(); + if ( ! entropy_test_add(handle, b) ) exit(1); - - if ( entropy_test_add(2, b) != T ) - exit(1); - - print entropy_test_finish(2); + print entropy_test_finish(handle); } From 69d8d29fbd9c873fb9f859f355f9e1827f844502 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Fri, 14 Dec 2012 13:03:15 -0800 Subject: [PATCH 24/35] Add new unit test for opaque serialization. One Bro begins a computation, another finishes it. (This commit also fixes a problem with the serialization OpaqueVals.) --- src/OpaqueVal.cc | 12 ++- src/Val.cc | 17 ++-- src/Val.h | 5 -- .../btest/Baseline/istate.opaque/expected.log | 4 + .../btest/Baseline/istate.opaque/output.log | 4 + testing/btest/istate/opaque.bro | 77 +++++++++++++++++++ 6 files changed, 99 insertions(+), 20 deletions(-) create mode 100644 testing/btest/Baseline/istate.opaque/expected.log create mode 100644 testing/btest/Baseline/istate.opaque/output.log create mode 100644 testing/btest/istate/opaque.bro diff --git a/src/OpaqueVal.cc b/src/OpaqueVal.cc index a5e65e7e7e..12581f3a77 100644 --- a/src/OpaqueVal.cc +++ b/src/OpaqueVal.cc @@ -53,7 +53,11 @@ StringVal* HashVal::DoGet() return new StringVal(""); } -HashVal::HashVal(OpaqueType* t) : OpaqueVal(t), valid(false) { } +HashVal::HashVal(OpaqueType* t) + : OpaqueVal(t) + { + valid = false; + } IMPLEMENT_SERIAL(HashVal, SER_HASH_VAL); @@ -113,7 +117,7 @@ bool MD5Val::DoInit() bool MD5Val::DoFeed(const void* data, size_t size) { if ( ! IsValid() ) - return new StringVal(""); + return false; md5_update(&ctx, data, size); return true; @@ -220,7 +224,7 @@ bool SHA1Val::DoInit() bool SHA1Val::DoFeed(const void* data, size_t size) { if ( ! IsValid() ) - return new StringVal(""); + return false; sha1_update(&ctx, data, size); return true; @@ -331,7 +335,7 @@ bool SHA256Val::DoInit() bool SHA256Val::DoFeed(const void* data, size_t size) { if ( ! IsValid() ) - return new StringVal(""); + return false; sha256_update(&ctx, data, size); return true; diff --git a/src/Val.cc b/src/Val.cc index 67a4579295..be7c4557fa 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -3114,16 +3114,13 @@ void VectorVal::ValDescribe(ODesc* d) const d->Add("]"); } -OpaqueVal::OpaqueVal(OpaqueType* t) : opaque_type(t) { } +OpaqueVal::OpaqueVal(OpaqueType* t) + : Val(t) + { + } OpaqueVal::~OpaqueVal() { - Unref(opaque_type); - } - -bool OpaqueVal::IsValid() const - { - return false; } IMPLEMENT_SERIAL(OpaqueVal, SER_OPAQUE_VAL); @@ -3131,15 +3128,13 @@ IMPLEMENT_SERIAL(OpaqueVal, SER_OPAQUE_VAL); bool OpaqueVal::DoSerialize(SerialInfo* info) const { DO_SERIALIZE(SER_OPAQUE_VAL, Val); - assert(opaque_type); - return opaque_type->Serialize(info); + return true; } bool OpaqueVal::DoUnserialize(UnserialInfo* info) { DO_UNSERIALIZE(Val); - opaque_type = static_cast(BroType::Unserialize(info)); - return opaque_type != 0; + return true; } diff --git a/src/Val.h b/src/Val.h index 925fb5a2b0..d6893306f2 100644 --- a/src/Val.h +++ b/src/Val.h @@ -1019,16 +1019,11 @@ public: OpaqueVal(OpaqueType* t); virtual ~OpaqueVal(); - // Determines whether the opaque value is in a valid state. - virtual bool IsValid() const; - protected: friend class Val; OpaqueVal() { } DECLARE_SERIAL(OpaqueVal); - - OpaqueType* opaque_type; }; diff --git a/testing/btest/Baseline/istate.opaque/expected.log b/testing/btest/Baseline/istate.opaque/expected.log new file mode 100644 index 0000000000..1386a47db1 --- /dev/null +++ b/testing/btest/Baseline/istate.opaque/expected.log @@ -0,0 +1,4 @@ +acbd18db4cc2f85cedef654fccc4a4d8 +0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33 +2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae +[entropy=0.918296, chi_square=423.666667, mean=108.0, monte_carlo_pi=nan, serial_correlation=-0.5] diff --git a/testing/btest/Baseline/istate.opaque/output.log b/testing/btest/Baseline/istate.opaque/output.log new file mode 100644 index 0000000000..1386a47db1 --- /dev/null +++ b/testing/btest/Baseline/istate.opaque/output.log @@ -0,0 +1,4 @@ +acbd18db4cc2f85cedef654fccc4a4d8 +0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33 +2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae +[entropy=0.918296, chi_square=423.666667, mean=108.0, monte_carlo_pi=nan, serial_correlation=-0.5] diff --git a/testing/btest/istate/opaque.bro b/testing/btest/istate/opaque.bro new file mode 100644 index 0000000000..18b2b7021b --- /dev/null +++ b/testing/btest/istate/opaque.bro @@ -0,0 +1,77 @@ +# +# @TEST-EXEC: bro -r $TRACES/empty.trace write.bro +# @TEST-EXEC: bro read.bro +# @TEST-EXEC: btest-diff expected.log +# @TEST-EXEC: btest-diff output.log +# @TEST-EXEC: cmp output.log expected.log + +@TEST-START-FILE read.bro + +global md5_handle: opaque of md5 &persistent &synchronized; +global sha1_handle: opaque of sha1 &persistent &synchronized; +global sha256_handle: opaque of sha256 &persistent &synchronized; +global entropy_handle: opaque of entropy &persistent &synchronized; + +event bro_done() + { + local out = open("output.log"); + + # Finish incremental operations started by a previous Bro. + if ( md5_hash_update(md5_handle, "oo") ) + print out, md5_hash_finish(md5_handle); + else + print out, "md5_hash_update() failed"; + + if ( sha1_hash_update(sha1_handle, "oo") ) + print out, sha1_hash_finish(sha1_handle); + else + print out, "sha1_hash_update() failed"; + + if ( sha256_hash_update(sha256_handle, "oo") ) + print out, sha256_hash_finish(sha256_handle); + else + print out, "sha256_hash_update() failed"; + + if ( entropy_test_add(entropy_handle, "oo") ) + print out, entropy_test_finish(entropy_handle); + else + print out, "entropy_test_add() failed"; + } + +@TEST-END-FILE + +@TEST-START-FILE write.bro + +global md5_handle: opaque of md5 &persistent &synchronized; +global sha1_handle: opaque of sha1 &persistent &synchronized; +global sha256_handle: opaque of sha256 &persistent &synchronized; +global entropy_handle: opaque of entropy &persistent &synchronized; + +event bro_init() + { + local out = open("expected.log"); + print out, md5_hash("foo"); + print out, sha1_hash("foo"); + print out, sha256_hash("foo"); + print out, find_entropy("foo"); + + # Begin incremental operations. Our goal is to feed the data string "foo" to + # the computation, but split into "f" and "oo" in two instances.. + md5_handle = md5_hash_init(); + if ( ! md5_hash_update(md5_handle, "f") ) + print out, "md5_hash_update() failed"; + + sha1_handle = sha1_hash_init(); + if ( ! sha1_hash_update(sha1_handle, "f") ) + print out, "sha1_hash_update() failed"; + + sha256_handle = sha256_hash_init(); + if ( ! sha256_hash_update(sha256_handle, "f") ) + print out, "sha256_hash_update() failed"; + + entropy_handle = entropy_test_init(); + if ( ! entropy_test_add(entropy_handle, "f") ) + print out, "entropy_test_add() failed"; + } + +@TEST-END-FILE From 7a2901ec6b7649611e12b9fe8ff090390e8c681e Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 14 Dec 2012 14:35:23 -0800 Subject: [PATCH 25/35] Fixing checksums in test trace because Bro now reports them. :-) --- CHANGES | 8 +++ VERSION | 2 +- .../conn.ds.txt | 46 +++++++++--------- .../conn.ds.txt | 46 +++++++++--------- .../http.ds.txt | 28 +++++------ testing/btest/Traces/wikipedia.trace | Bin 27460 -> 27460 bytes 6 files changed, 69 insertions(+), 61 deletions(-) diff --git a/CHANGES b/CHANGES index e26e7c375d..d0a64e1f0a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,12 @@ +2.1-229 | 2012-12-14 14:46:12 -0800 + + * Fix memory leak in ASCII reader when encoutering errors in input. + (Bernhard Amann) + + * Improvements for the "bad checksums" detector to make it detect + bad TCP checksums. (Seth Hall) + 2.1-223 | 2012-12-12 14:25:15 -0800 * Trick for parallelizing input framework unit tests. Instead of diff --git a/VERSION b/VERSION index 42edbac7b2..44d3909c31 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1-223 +2.1-229 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.time-as-int/conn.ds.txt b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.time-as-int/conn.ds.txt index c4ac546ab6..5d95fdc0c0 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.time-as-int/conn.ds.txt +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.time-as-int/conn.ds.txt @@ -56,20 +56,20 @@ ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes 1300475167096535 UWkUyAuUGXf 141.142.220.202 5353 224.0.0.251 5353 udp dns 0 0 0 S0 F 0 D 1 73 0 0 1300475167097012 arKYeMETxOg fe80::217:f2ff:fed7:cf65 5353 ff02::fb 5353 udp 0 0 0 S0 F 0 D 1 199 0 0 1300475167099816 k6kgXLOoSKl 141.142.220.50 5353 224.0.0.251 5353 udp 0 0 0 S0 F 0 D 1 179 0 0 -1300475168853899 TEfuqmmG4bh 141.142.220.118 43927 141.142.2.2 53 udp dns 435 0 89 SHR F 0 Cd 0 0 1 117 -1300475168854378 FrJExwHcSal 141.142.220.118 37676 141.142.2.2 53 udp dns 420 0 99 SHR F 0 Cd 0 0 1 127 -1300475168854837 5OKnoww6xl4 141.142.220.118 40526 141.142.2.2 53 udp dns 391 0 183 SHR F 0 Cd 0 0 1 211 -1300475168857956 fRFu0wcOle6 141.142.220.118 32902 141.142.2.2 53 udp dns 317 0 89 SHR F 0 Cd 0 0 1 117 -1300475168858306 qSsw6ESzHV4 141.142.220.118 59816 141.142.2.2 53 udp dns 343 0 99 SHR F 0 Cd 0 0 1 127 -1300475168858713 iE6yhOq3SF 141.142.220.118 59714 141.142.2.2 53 udp dns 375 0 183 SHR F 0 Cd 0 0 1 211 -1300475168891644 qCaWGmzFtM5 141.142.220.118 58206 141.142.2.2 53 udp dns 339 0 89 SHR F 0 Cd 0 0 1 117 -1300475168892037 70MGiRM1Qf4 141.142.220.118 38911 141.142.2.2 53 udp dns 334 0 99 SHR F 0 Cd 0 0 1 127 -1300475168892414 h5DsfNtYzi1 141.142.220.118 59746 141.142.2.2 53 udp dns 420 0 183 SHR F 0 Cd 0 0 1 211 -1300475168893988 c4Zw9TmAE05 141.142.220.118 45000 141.142.2.2 53 udp dns 384 0 89 SHR F 0 Cd 0 0 1 117 -1300475168894422 EAr0uf4mhq 141.142.220.118 48479 141.142.2.2 53 udp dns 316 0 99 SHR F 0 Cd 0 0 1 127 -1300475168894787 GvmoxJFXdTa 141.142.220.118 48128 141.142.2.2 53 udp dns 422 0 183 SHR F 0 Cd 0 0 1 211 -1300475168901749 slFea8xwSmb 141.142.220.118 56056 141.142.2.2 53 udp dns 402 0 131 SHR F 0 Cd 0 0 1 159 -1300475168902195 UfGkYA2HI2g 141.142.220.118 55092 141.142.2.2 53 udp dns 374 0 198 SHR F 0 Cd 0 0 1 226 +1300475168853899 TEfuqmmG4bh 141.142.220.118 43927 141.142.2.2 53 udp dns 435 38 89 SF F 0 Dd 1 66 1 117 +1300475168854378 FrJExwHcSal 141.142.220.118 37676 141.142.2.2 53 udp dns 420 52 99 SF F 0 Dd 1 80 1 127 +1300475168854837 5OKnoww6xl4 141.142.220.118 40526 141.142.2.2 53 udp dns 391 38 183 SF F 0 Dd 1 66 1 211 +1300475168857956 fRFu0wcOle6 141.142.220.118 32902 141.142.2.2 53 udp dns 317 38 89 SF F 0 Dd 1 66 1 117 +1300475168858306 qSsw6ESzHV4 141.142.220.118 59816 141.142.2.2 53 udp dns 343 52 99 SF F 0 Dd 1 80 1 127 +1300475168858713 iE6yhOq3SF 141.142.220.118 59714 141.142.2.2 53 udp dns 375 38 183 SF F 0 Dd 1 66 1 211 +1300475168891644 qCaWGmzFtM5 141.142.220.118 58206 141.142.2.2 53 udp dns 339 38 89 SF F 0 Dd 1 66 1 117 +1300475168892037 70MGiRM1Qf4 141.142.220.118 38911 141.142.2.2 53 udp dns 334 52 99 SF F 0 Dd 1 80 1 127 +1300475168892414 h5DsfNtYzi1 141.142.220.118 59746 141.142.2.2 53 udp dns 420 38 183 SF F 0 Dd 1 66 1 211 +1300475168893988 c4Zw9TmAE05 141.142.220.118 45000 141.142.2.2 53 udp dns 384 38 89 SF F 0 Dd 1 66 1 117 +1300475168894422 EAr0uf4mhq 141.142.220.118 48479 141.142.2.2 53 udp dns 316 52 99 SF F 0 Dd 1 80 1 127 +1300475168894787 GvmoxJFXdTa 141.142.220.118 48128 141.142.2.2 53 udp dns 422 38 183 SF F 0 Dd 1 66 1 211 +1300475168901749 slFea8xwSmb 141.142.220.118 56056 141.142.2.2 53 udp dns 402 36 131 SF F 0 Dd 1 64 1 159 +1300475168902195 UfGkYA2HI2g 141.142.220.118 55092 141.142.2.2 53 udp dns 374 36 198 SF F 0 Dd 1 64 1 226 1300475169899438 BWaU4aSuwkc 141.142.220.44 5353 224.0.0.251 5353 udp dns 0 0 0 S0 F 0 D 1 85 0 0 1300475170862384 10XodEwRycf 141.142.220.226 137 141.142.220.255 137 udp dns 2613016 350 0 S0 F 0 D 7 546 0 0 1300475171675372 zno26fFZkrh fe80::3074:17d5:2052:c324 65373 ff02::1:3 5355 udp dns 100096 66 0 S0 F 0 D 2 162 0 0 @@ -77,13 +77,13 @@ ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes 1300475173116749 eWZCH7OONC1 fe80::3074:17d5:2052:c324 54213 ff02::1:3 5355 udp dns 99801 66 0 S0 F 0 D 2 162 0 0 1300475173117362 0Pwk3ntf8O3 141.142.220.226 55671 224.0.0.252 5355 udp dns 99848 66 0 S0 F 0 D 2 122 0 0 1300475173153679 0HKorjr8Zp7 141.142.220.238 56641 141.142.220.255 137 udp dns 0 0 0 S0 F 0 D 1 78 0 0 -1300475168859163 GSxOnSLghOa 141.142.220.118 49998 208.80.152.3 80 tcp 215893 1130 734 S1 F 1130 ShACad 4 216 4 950 -1300475168652003 nQcgTWjvg4c 141.142.220.118 35634 208.80.152.2 80 tcp 61328 0 350 OTH F 0 CdA 1 52 1 402 -1300475168895267 0Q4FH8sESw5 141.142.220.118 50001 208.80.152.3 80 tcp 227283 1178 734 S1 F 1178 ShACad 4 216 4 950 -1300475168902635 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 tcp 120040 534 412 S1 F 534 ShACad 3 164 3 576 -1300475168892936 Tw8jXtpTGu6 141.142.220.118 50000 208.80.152.3 80 tcp 229603 1148 734 S1 F 1148 ShACad 4 216 4 950 -1300475168855305 3PKsZ2Uye21 141.142.220.118 49996 208.80.152.3 80 tcp 218501 1171 733 S1 F 1171 ShACad 4 216 4 949 -1300475168892913 P654jzLoe3a 141.142.220.118 49999 208.80.152.3 80 tcp 220960 1137 733 S1 F 1137 ShACad 4 216 4 949 +1300475168859163 GSxOnSLghOa 141.142.220.118 49998 208.80.152.3 80 tcp http 215893 1130 734 S1 F 0 ShADad 6 1450 4 950 +1300475168652003 nQcgTWjvg4c 141.142.220.118 35634 208.80.152.2 80 tcp 61328 463 350 OTH F 0 DdA 2 567 1 402 +1300475168895267 0Q4FH8sESw5 141.142.220.118 50001 208.80.152.3 80 tcp http 227283 1178 734 S1 F 0 ShADad 6 1498 4 950 +1300475168902635 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 tcp http 120040 534 412 S1 F 0 ShADad 4 750 3 576 +1300475168892936 Tw8jXtpTGu6 141.142.220.118 50000 208.80.152.3 80 tcp http 229603 1148 734 S1 F 0 ShADad 6 1468 4 950 +1300475168855305 3PKsZ2Uye21 141.142.220.118 49996 208.80.152.3 80 tcp http 218501 1171 733 S1 F 0 ShADad 6 1491 4 949 +1300475168892913 P654jzLoe3a 141.142.220.118 49999 208.80.152.3 80 tcp http 220960 1137 733 S1 F 0 ShADad 6 1457 4 949 1300475169780331 2cx26uAvUPl 141.142.220.235 6705 173.192.163.128 80 tcp 0 0 0 OTH F 0 h 0 0 1 48 -1300475168724007 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 tcp 119904 525 232 S1 F 525 ShACad 3 164 3 396 -1300475168855330 VW0XPVINV8a 141.142.220.118 49997 208.80.152.3 80 tcp 219720 1125 734 S1 F 1125 ShACad 4 216 4 950 +1300475168724007 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 tcp http 119904 525 232 S1 F 0 ShADad 4 741 3 396 +1300475168855330 VW0XPVINV8a 141.142.220.118 49997 208.80.152.3 80 tcp http 219720 1125 734 S1 F 0 ShADad 6 1445 4 950 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/conn.ds.txt b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/conn.ds.txt index b74b9fd7e3..5af6f702b8 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/conn.ds.txt +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/conn.ds.txt @@ -56,20 +56,20 @@ ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes 1300475167.096535 UWkUyAuUGXf 141.142.220.202 5353 224.0.0.251 5353 udp dns 0.000000 0 0 S0 F 0 D 1 73 0 0 1300475167.097012 arKYeMETxOg fe80::217:f2ff:fed7:cf65 5353 ff02::fb 5353 udp 0.000000 0 0 S0 F 0 D 1 199 0 0 1300475167.099816 k6kgXLOoSKl 141.142.220.50 5353 224.0.0.251 5353 udp 0.000000 0 0 S0 F 0 D 1 179 0 0 -1300475168.853899 TEfuqmmG4bh 141.142.220.118 43927 141.142.2.2 53 udp dns 0.000435 0 89 SHR F 0 Cd 0 0 1 117 -1300475168.854378 FrJExwHcSal 141.142.220.118 37676 141.142.2.2 53 udp dns 0.000420 0 99 SHR F 0 Cd 0 0 1 127 -1300475168.854837 5OKnoww6xl4 141.142.220.118 40526 141.142.2.2 53 udp dns 0.000392 0 183 SHR F 0 Cd 0 0 1 211 -1300475168.857956 fRFu0wcOle6 141.142.220.118 32902 141.142.2.2 53 udp dns 0.000317 0 89 SHR F 0 Cd 0 0 1 117 -1300475168.858306 qSsw6ESzHV4 141.142.220.118 59816 141.142.2.2 53 udp dns 0.000343 0 99 SHR F 0 Cd 0 0 1 127 -1300475168.858713 iE6yhOq3SF 141.142.220.118 59714 141.142.2.2 53 udp dns 0.000375 0 183 SHR F 0 Cd 0 0 1 211 -1300475168.891644 qCaWGmzFtM5 141.142.220.118 58206 141.142.2.2 53 udp dns 0.000339 0 89 SHR F 0 Cd 0 0 1 117 -1300475168.892037 70MGiRM1Qf4 141.142.220.118 38911 141.142.2.2 53 udp dns 0.000335 0 99 SHR F 0 Cd 0 0 1 127 -1300475168.892414 h5DsfNtYzi1 141.142.220.118 59746 141.142.2.2 53 udp dns 0.000421 0 183 SHR F 0 Cd 0 0 1 211 -1300475168.893988 c4Zw9TmAE05 141.142.220.118 45000 141.142.2.2 53 udp dns 0.000384 0 89 SHR F 0 Cd 0 0 1 117 -1300475168.894422 EAr0uf4mhq 141.142.220.118 48479 141.142.2.2 53 udp dns 0.000317 0 99 SHR F 0 Cd 0 0 1 127 -1300475168.894787 GvmoxJFXdTa 141.142.220.118 48128 141.142.2.2 53 udp dns 0.000423 0 183 SHR F 0 Cd 0 0 1 211 -1300475168.901749 slFea8xwSmb 141.142.220.118 56056 141.142.2.2 53 udp dns 0.000402 0 131 SHR F 0 Cd 0 0 1 159 -1300475168.902195 UfGkYA2HI2g 141.142.220.118 55092 141.142.2.2 53 udp dns 0.000374 0 198 SHR F 0 Cd 0 0 1 226 +1300475168.853899 TEfuqmmG4bh 141.142.220.118 43927 141.142.2.2 53 udp dns 0.000435 38 89 SF F 0 Dd 1 66 1 117 +1300475168.854378 FrJExwHcSal 141.142.220.118 37676 141.142.2.2 53 udp dns 0.000420 52 99 SF F 0 Dd 1 80 1 127 +1300475168.854837 5OKnoww6xl4 141.142.220.118 40526 141.142.2.2 53 udp dns 0.000392 38 183 SF F 0 Dd 1 66 1 211 +1300475168.857956 fRFu0wcOle6 141.142.220.118 32902 141.142.2.2 53 udp dns 0.000317 38 89 SF F 0 Dd 1 66 1 117 +1300475168.858306 qSsw6ESzHV4 141.142.220.118 59816 141.142.2.2 53 udp dns 0.000343 52 99 SF F 0 Dd 1 80 1 127 +1300475168.858713 iE6yhOq3SF 141.142.220.118 59714 141.142.2.2 53 udp dns 0.000375 38 183 SF F 0 Dd 1 66 1 211 +1300475168.891644 qCaWGmzFtM5 141.142.220.118 58206 141.142.2.2 53 udp dns 0.000339 38 89 SF F 0 Dd 1 66 1 117 +1300475168.892037 70MGiRM1Qf4 141.142.220.118 38911 141.142.2.2 53 udp dns 0.000335 52 99 SF F 0 Dd 1 80 1 127 +1300475168.892414 h5DsfNtYzi1 141.142.220.118 59746 141.142.2.2 53 udp dns 0.000421 38 183 SF F 0 Dd 1 66 1 211 +1300475168.893988 c4Zw9TmAE05 141.142.220.118 45000 141.142.2.2 53 udp dns 0.000384 38 89 SF F 0 Dd 1 66 1 117 +1300475168.894422 EAr0uf4mhq 141.142.220.118 48479 141.142.2.2 53 udp dns 0.000317 52 99 SF F 0 Dd 1 80 1 127 +1300475168.894787 GvmoxJFXdTa 141.142.220.118 48128 141.142.2.2 53 udp dns 0.000423 38 183 SF F 0 Dd 1 66 1 211 +1300475168.901749 slFea8xwSmb 141.142.220.118 56056 141.142.2.2 53 udp dns 0.000402 36 131 SF F 0 Dd 1 64 1 159 +1300475168.902195 UfGkYA2HI2g 141.142.220.118 55092 141.142.2.2 53 udp dns 0.000374 36 198 SF F 0 Dd 1 64 1 226 1300475169.899438 BWaU4aSuwkc 141.142.220.44 5353 224.0.0.251 5353 udp dns 0.000000 0 0 S0 F 0 D 1 85 0 0 1300475170.862384 10XodEwRycf 141.142.220.226 137 141.142.220.255 137 udp dns 2.613017 350 0 S0 F 0 D 7 546 0 0 1300475171.675372 zno26fFZkrh fe80::3074:17d5:2052:c324 65373 ff02::1:3 5355 udp dns 0.100096 66 0 S0 F 0 D 2 162 0 0 @@ -77,13 +77,13 @@ ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes 1300475173.116749 eWZCH7OONC1 fe80::3074:17d5:2052:c324 54213 ff02::1:3 5355 udp dns 0.099801 66 0 S0 F 0 D 2 162 0 0 1300475173.117362 0Pwk3ntf8O3 141.142.220.226 55671 224.0.0.252 5355 udp dns 0.099849 66 0 S0 F 0 D 2 122 0 0 1300475173.153679 0HKorjr8Zp7 141.142.220.238 56641 141.142.220.255 137 udp dns 0.000000 0 0 S0 F 0 D 1 78 0 0 -1300475168.859163 GSxOnSLghOa 141.142.220.118 49998 208.80.152.3 80 tcp 0.215893 1130 734 S1 F 1130 ShACad 4 216 4 950 -1300475168.652003 nQcgTWjvg4c 141.142.220.118 35634 208.80.152.2 80 tcp 0.061329 0 350 OTH F 0 CdA 1 52 1 402 -1300475168.895267 0Q4FH8sESw5 141.142.220.118 50001 208.80.152.3 80 tcp 0.227284 1178 734 S1 F 1178 ShACad 4 216 4 950 -1300475168.902635 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 tcp 0.120041 534 412 S1 F 534 ShACad 3 164 3 576 -1300475168.892936 Tw8jXtpTGu6 141.142.220.118 50000 208.80.152.3 80 tcp 0.229603 1148 734 S1 F 1148 ShACad 4 216 4 950 -1300475168.855305 3PKsZ2Uye21 141.142.220.118 49996 208.80.152.3 80 tcp 0.218501 1171 733 S1 F 1171 ShACad 4 216 4 949 -1300475168.892913 P654jzLoe3a 141.142.220.118 49999 208.80.152.3 80 tcp 0.220961 1137 733 S1 F 1137 ShACad 4 216 4 949 +1300475168.859163 GSxOnSLghOa 141.142.220.118 49998 208.80.152.3 80 tcp http 0.215893 1130 734 S1 F 0 ShADad 6 1450 4 950 +1300475168.652003 nQcgTWjvg4c 141.142.220.118 35634 208.80.152.2 80 tcp 0.061329 463 350 OTH F 0 DdA 2 567 1 402 +1300475168.895267 0Q4FH8sESw5 141.142.220.118 50001 208.80.152.3 80 tcp http 0.227284 1178 734 S1 F 0 ShADad 6 1498 4 950 +1300475168.902635 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 tcp http 0.120041 534 412 S1 F 0 ShADad 4 750 3 576 +1300475168.892936 Tw8jXtpTGu6 141.142.220.118 50000 208.80.152.3 80 tcp http 0.229603 1148 734 S1 F 0 ShADad 6 1468 4 950 +1300475168.855305 3PKsZ2Uye21 141.142.220.118 49996 208.80.152.3 80 tcp http 0.218501 1171 733 S1 F 0 ShADad 6 1491 4 949 +1300475168.892913 P654jzLoe3a 141.142.220.118 49999 208.80.152.3 80 tcp http 0.220961 1137 733 S1 F 0 ShADad 6 1457 4 949 1300475169.780331 2cx26uAvUPl 141.142.220.235 6705 173.192.163.128 80 tcp 0.000000 0 0 OTH F 0 h 0 0 1 48 -1300475168.724007 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 tcp 0.119905 525 232 S1 F 525 ShACad 3 164 3 396 -1300475168.855330 VW0XPVINV8a 141.142.220.118 49997 208.80.152.3 80 tcp 0.219720 1125 734 S1 F 1125 ShACad 4 216 4 950 +1300475168.724007 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 tcp http 0.119905 525 232 S1 F 0 ShADad 4 741 3 396 +1300475168.855330 VW0XPVINV8a 141.142.220.118 49997 208.80.152.3 80 tcp http 0.219720 1125 734 S1 F 0 ShADad 6 1445 4 950 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt index ae62fbec3d..756a49a427 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt @@ -65,17 +65,17 @@ # Extent, type='http' ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file -1300475168.843894 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 0 0 0 304 Not Modified 0 -1300475168.975800 VW0XPVINV8a 141.142.220.118 49997 208.80.152.3 80 0 0 0 304 Not Modified 0 -1300475168.976327 3PKsZ2Uye21 141.142.220.118 49996 208.80.152.3 80 0 0 0 304 Not Modified 0 -1300475168.979160 GSxOnSLghOa 141.142.220.118 49998 208.80.152.3 80 0 0 0 304 Not Modified 0 -1300475169.012666 Tw8jXtpTGu6 141.142.220.118 50000 208.80.152.3 80 0 0 0 304 Not Modified 0 -1300475169.012730 P654jzLoe3a 141.142.220.118 49999 208.80.152.3 80 0 0 0 304 Not Modified 0 -1300475169.014860 0Q4FH8sESw5 141.142.220.118 50001 208.80.152.3 80 0 0 0 304 Not Modified 0 -1300475169.022665 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 0 0 0 304 Not Modified 0 -1300475169.036294 VW0XPVINV8a 141.142.220.118 49997 208.80.152.3 80 0 0 0 304 Not Modified 0 -1300475169.036798 3PKsZ2Uye21 141.142.220.118 49996 208.80.152.3 80 0 0 0 304 Not Modified 0 -1300475169.039923 GSxOnSLghOa 141.142.220.118 49998 208.80.152.3 80 0 0 0 304 Not Modified 0 -1300475169.074793 Tw8jXtpTGu6 141.142.220.118 50000 208.80.152.3 80 0 0 0 304 Not Modified 0 -1300475169.074938 P654jzLoe3a 141.142.220.118 49999 208.80.152.3 80 0 0 0 304 Not Modified 0 -1300475169.075065 0Q4FH8sESw5 141.142.220.118 50001 208.80.152.3 80 0 0 0 304 Not Modified 0 +1300475168.784020 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 1 GET bits.wikimedia.org /skins-1.5/monobook/main.css http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 +1300475168.916018 VW0XPVINV8a 141.142.220.118 49997 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/6/63/Wikipedia-logo.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 +1300475168.916183 3PKsZ2Uye21 141.142.220.118 49996 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/b/bb/Wikipedia_wordmark.svg/174px-Wikipedia_wordmark.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 +1300475168.918358 GSxOnSLghOa 141.142.220.118 49998 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/b/bd/Bookshelf-40x201_6.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 +1300475168.952307 Tw8jXtpTGu6 141.142.220.118 50000 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/8/8a/Wikinews-logo.png/35px-Wikinews-logo.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 +1300475168.952296 P654jzLoe3a 141.142.220.118 49999 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/4/4a/Wiktionary-logo-en-35px.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 +1300475168.954820 0Q4FH8sESw5 141.142.220.118 50001 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/f/fa/Wikiquote-logo.svg/35px-Wikiquote-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 +1300475168.962687 i2rO3KD1Syg 141.142.220.118 35642 208.80.152.2 80 1 GET meta.wikimedia.org /images/wikimedia-button.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 +1300475168.975934 VW0XPVINV8a 141.142.220.118 49997 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/f/fa/Wikibooks-logo.svg/35px-Wikibooks-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 +1300475168.976436 3PKsZ2Uye21 141.142.220.118 49996 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/d/df/Wikispecies-logo.svg/35px-Wikispecies-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 +1300475168.979264 GSxOnSLghOa 141.142.220.118 49998 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/4/4c/Wikisource-logo.svg/35px-Wikisource-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 +1300475169.014619 Tw8jXtpTGu6 141.142.220.118 50000 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/4/4a/Commons-logo.svg/35px-Commons-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 +1300475169.014593 P654jzLoe3a 141.142.220.118 49999 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/9/91/Wikiversity-logo.svg/35px-Wikiversity-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 +1300475169.014927 0Q4FH8sESw5 141.142.220.118 50001 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/7/75/Wikimedia_Community_Logo.svg/35px-Wikimedia_Community_Logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 diff --git a/testing/btest/Traces/wikipedia.trace b/testing/btest/Traces/wikipedia.trace index 68d85e0190b0fbefe12ba1bb066c4d06e987c9cf..9e21966243dacf96c2c88650a4cfe7eed70a3709 100644 GIT binary patch delta 309 zcmX?djq%7e#tr<;OuoUJ1)0@ZnRIt;w&E~kVX4}={L5rPN%_frJQtYUmTpeqR@D%zjRsz zY7pmSXOZJ9OD?k5g4Mc+US!!VqWKam3o_AB>?R|Nqw3voxOz8<<4g~qZ045)S{}rA zAP}z9MXH{WNq_t1qtcR$OeW!*AIRh|G0k|k*-u`MnQ5}#<~$`2W+skfn|G+5WoELp z-rTNP#l-Yx%VuUBOJ*jilbh}I?U|Xbdv2~bl4WJ`c)NL?`BD}pjUSs`ZT_$@ae8ld hcHGF$l+V2Ro5yq}rsY#N_j@m5VVb>rvvGi+Gyp4ZYU}_2 delta 309 zcmX?djq%7e#tr<;Ojmq23o@&-GF|z<*^0xEh2`g!x6UQR}! zVCo03V80OO=Ba!)8JT`wnOq?7k>%&3*=;}xv&pjsF9KEFu>h&s+%MDt(sNn_Y7pmS zXOZJTwJf$^wJxF{wVE%%vLF*3#cqPs-wlVWcat~{vYcNMZ25sexKbCXdZ6W(HXoIi z1k(>>a+sK|EZpoTF9-C?>CJgc9zc4@<{heMf%MAF?V43U`8}JNbu5AWL!0gN?Sb@^ z&GkmIKtCVaJkNY73)7Wjn_X@G0O=!}ogFu_gZSS(rUUiu+uZNH2q=DSvvGi+GyqRs BezyPs From 81ae68be16c919c4a662aed5d29cdca86e401b15 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 14 Dec 2012 16:55:41 -0600 Subject: [PATCH 26/35] Fix a case where c$resp$size is misrepresented. Addresses #730. That field is based on TCP sequence numbers and on seeing a SYN followed by a failed RST injection response, the initial sequence number tracked the value in the injection (most likely zero) instead of value in subsequent SYN response. This could make c$resp$size be set to large values when it's not really. Also removed some dead code paths. --- src/TCP.cc | 40 +++++++++++------- src/TCP.h | 7 +-- .../Baseline/core.tcp.rst-after-syn/.stdout | 3 ++ testing/btest/Traces/tcp/rst-inject-rae.trace | Bin 0 -> 943 bytes testing/btest/core/tcp/rst-after-syn.bro | 12 ++++++ 5 files changed, 44 insertions(+), 18 deletions(-) create mode 100644 testing/btest/Baseline/core.tcp.rst-after-syn/.stdout create mode 100644 testing/btest/Traces/tcp/rst-inject-rae.trace create mode 100644 testing/btest/core/tcp/rst-after-syn.bro diff --git a/src/TCP.cc b/src/TCP.cc index 555adf1b57..da977d8157 100644 --- a/src/TCP.cc +++ b/src/TCP.cc @@ -382,7 +382,7 @@ void TCP_Analyzer::ProcessFIN(double t, TCP_Endpoint* endpoint, endpoint->FIN_seq = base_seq - endpoint->StartSeq() + seq_len; } -bool TCP_Analyzer::ProcessRST(double t, TCP_Endpoint* endpoint, +void TCP_Analyzer::ProcessRST(double t, TCP_Endpoint* endpoint, const IP_Hdr* ip, uint32 base_seq, int len, int& seq_len) { @@ -406,11 +406,9 @@ bool TCP_Analyzer::ProcessRST(double t, TCP_Endpoint* endpoint, } PacketWithRST(); - - return true; } -int TCP_Analyzer::ProcessFlags(double t, +void TCP_Analyzer::ProcessFlags(double t, const IP_Hdr* ip, const struct tcphdr* tp, uint32 tcp_hdr_len, int len, int& seq_len, TCP_Endpoint* endpoint, TCP_Endpoint* peer, @@ -425,14 +423,11 @@ int TCP_Analyzer::ProcessFlags(double t, if ( flags.FIN() ) ProcessFIN(t, endpoint, seq_len, base_seq); - if ( flags.RST() && - ! ProcessRST(t, endpoint, ip, base_seq, len, seq_len) ) - return 0; + if ( flags.RST() ) + ProcessRST(t, endpoint, ip, base_seq, len, seq_len); if ( flags.ACK() ) ProcessACK(endpoint, peer, ack_seq, is_orig, flags); - - return 1; } void TCP_Analyzer::TransitionFromInactive(double t, TCP_Endpoint* endpoint, @@ -825,10 +820,27 @@ void TCP_Analyzer::UpdateClosedState(double t, TCP_Endpoint* endpoint, } } -void TCP_Analyzer::UpdateResetState(int len, TCP_Flags flags) +void TCP_Analyzer::UpdateResetState(int len, TCP_Flags flags, + TCP_Endpoint* endpoint, uint32 base_seq, + uint32 last_seq) { if ( flags.SYN() ) + { Weird("SYN_after_reset"); + + if ( endpoint->prev_state == TCP_ENDPOINT_INACTIVE ) + { + // Seq. numbers were initialized by a RST packet from this endpoint, + // but now that a SYN is seen from it, that could mean the earlier + // RST was spoofed/injected, so re-initialize. This mostly just + // helps prevent misrepresentations of payload sizes that are based + // on bad initial sequence values. + endpoint->InitStartSeq(base_seq); + endpoint->InitAckSeq(base_seq); + endpoint->InitLastSeq(last_seq); + } + } + if ( flags.FIN() ) Weird("FIN_after_reset"); @@ -871,7 +883,7 @@ void TCP_Analyzer::UpdateStateMachine(double t, break; case TCP_ENDPOINT_RESET: - UpdateResetState(len, flags); + UpdateResetState(len, flags, endpoint, base_seq, last_seq); break; } } @@ -996,10 +1008,8 @@ void TCP_Analyzer::DeliverPacket(int len, const u_char* data, bool is_orig, int seq_len = len; // length in terms of sequence space - if ( ! ProcessFlags(t, ip, tp, tcp_hdr_len, len, seq_len, - endpoint, peer, base_seq, ack_seq, - orig_addr, is_orig, flags) ) - return; + ProcessFlags(t, ip, tp, tcp_hdr_len, len, seq_len, endpoint, peer, base_seq, + ack_seq, orig_addr, is_orig, flags); uint32 last_seq = base_seq + seq_len; diff --git a/src/TCP.h b/src/TCP.h index c84202fcf6..635fda7960 100644 --- a/src/TCP.h +++ b/src/TCP.h @@ -135,13 +135,13 @@ protected: void ProcessFIN(double t, TCP_Endpoint* endpoint, int& seq_len, uint32 base_seq); - bool ProcessRST(double t, TCP_Endpoint* endpoint, const IP_Hdr* ip, + void ProcessRST(double t, TCP_Endpoint* endpoint, const IP_Hdr* ip, uint32 base_seq, int len, int& seq_len); void ProcessACK(TCP_Endpoint* endpoint, TCP_Endpoint* peer, uint32 ack_seq, int is_orig, TCP_Flags flags); - int ProcessFlags(double t, const IP_Hdr* ip, const struct tcphdr* tp, + void ProcessFlags(double t, const IP_Hdr* ip, const struct tcphdr* tp, uint32 tcp_hdr_len, int len, int& seq_len, TCP_Endpoint* endpoint, TCP_Endpoint* peer, uint32 base_seq, uint32 ack_seq, @@ -186,7 +186,8 @@ protected: int delta_last, TCP_Flags flags, int& do_close); - void UpdateResetState(int len, TCP_Flags flags); + void UpdateResetState(int len, TCP_Flags flags, TCP_Endpoint* endpoint, + uint32 base_seq, uint32 last_seq); void GeneratePacketEvent(TCP_Endpoint* endpoint, TCP_Endpoint* peer, uint32 base_seq, uint32 ack_seq, diff --git a/testing/btest/Baseline/core.tcp.rst-after-syn/.stdout b/testing/btest/Baseline/core.tcp.rst-after-syn/.stdout new file mode 100644 index 0000000000..25ed566cd0 --- /dev/null +++ b/testing/btest/Baseline/core.tcp.rst-after-syn/.stdout @@ -0,0 +1,3 @@ +[orig_h=1.2.0.2, orig_p=2527/tcp, resp_h=1.2.0.3, resp_p=6649/tcp] +orig:, [size=175, state=1, num_pkts=4, num_bytes_ip=395, flow_label=0] +resp:, [size=0, state=6, num_pkts=5, num_bytes_ip=236, flow_label=0] diff --git a/testing/btest/Traces/tcp/rst-inject-rae.trace b/testing/btest/Traces/tcp/rst-inject-rae.trace new file mode 100644 index 0000000000000000000000000000000000000000..7225cc0d35fedd0a020b4143f51bbc99f33a9ae8 GIT binary patch literal 943 zcmca|c+)~A1{MYw`2U}Qff2~j&&&6;4rFEU1F}IF3|JUM#kp$|IT&0S7#u`H9T@D` zRHGT07?^;Vne)En&)6=r8j#8jOb!e-MGOo~EUYn%%*+goj2v8G6MzIGBMTGAbmvDL z3^qVM2!jE#=^7x@_1H{{K_-K2ko?JcAEXLoL%fIs!?KqUO$-bMAeSa`;&Excj{}1t z+lgdkvl1fp1c7GNZV>wa|Lq2_OBVoL%7pGxkm*y>IT@UQ)`2h>AiLB=YAz@o6y!m! z1la%$hgzTwLJb1W0n65b{0#Izr%Mt@H^@Y$Y)*z~ARC0i0NF%esd-=%S+SZZ;T$mS zA*zYj-vP}@`0S~&?6Jqe^Z&hO`g&aIdtAzDWdyY91RpaV$2SOrg68-Nw4f0J2Fa(mzpbjDkgYgtn|!MiC>RRH;5Q6-(GiF|DCIe_}4RD`@D=U-+pDY?UkVSqx|x@ zQU+4Dri!m#&QbXMn8+8|?=fCa7=G&f>iF6t#o0T(=bpQZaObUcp*1|;Iy808b%<4L z{dlF}sPftW(R*KgeHU#cESi1#=GXmVB0L_(K!09}V#gO|4;X-9#`Xd|%s~DO0OrkH JO=##a008us*i--j literal 0 HcmV?d00001 diff --git a/testing/btest/core/tcp/rst-after-syn.bro b/testing/btest/core/tcp/rst-after-syn.bro new file mode 100644 index 0000000000..38976909d7 --- /dev/null +++ b/testing/btest/core/tcp/rst-after-syn.bro @@ -0,0 +1,12 @@ +# @TEST-EXEC: bro -b -r $TRACES/tcp/rst-inject-rae.trace %INPUT +# @TEST-EXEC: btest-diff .stdout + +# Mostly just checking that c$resp$size isn't huge due to the injected +# RST packet being used to initialize sequence number in TCP analyzer. + +event connection_state_remove(c: connection) + { + print c$id; + print "orig:", c$orig; + print "resp:", c$resp; + } From 2f0c698ed568afbfa3c20300a2d87cfa11d631d0 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 18 Dec 2012 14:31:39 -0600 Subject: [PATCH 27/35] Improve error for invalid use of types as values (addresses #923). This scripting error can now generate an error message at parse-time instead of run-time and also includes location information. --- scripts/policy/protocols/http/detect-webapps.bro | 2 +- src/Expr.cc | 6 +++++- .../Baseline/language.type-type-error/.stderr | 1 + testing/btest/language/type-type-error.bro | 14 ++++++++++++++ 4 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 testing/btest/Baseline/language.type-type-error/.stderr create mode 100644 testing/btest/language/type-type-error.bro diff --git a/scripts/policy/protocols/http/detect-webapps.bro b/scripts/policy/protocols/http/detect-webapps.bro index fb805bfd33..53d7109468 100644 --- a/scripts/policy/protocols/http/detect-webapps.bro +++ b/scripts/policy/protocols/http/detect-webapps.bro @@ -28,7 +28,7 @@ event signature_match(state: signature_state, msg: string, data: string) &priori if ( /^webapp-/ !in state$sig_id ) return; local c = state$conn; - local si = Software::Info; + local si: Software::Info; si = [$name=msg, $unparsed_version=msg, $host=c$id$resp_h, $host_p=c$id$resp_p, $software_type=WEB_APPLICATION]; si$url = build_url_http(c$http); if ( c$id$resp_h in Software::tracked && diff --git a/src/Expr.cc b/src/Expr.cc index 3a4e8add70..926c537720 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -233,7 +233,11 @@ NameExpr::NameExpr(ID* arg_id, bool const_init) : Expr(EXPR_NAME) { id = arg_id; in_const_init = const_init; - SetType(id->Type()->Ref()); + + if ( id->AsType() ) + SetType(new TypeType(id->AsType())); + else + SetType(id->Type()->Ref()); EventHandler* h = event_registry->Lookup(id->Name()); if ( h ) diff --git a/testing/btest/Baseline/language.type-type-error/.stderr b/testing/btest/Baseline/language.type-type-error/.stderr new file mode 100644 index 0000000000..95cb065ece --- /dev/null +++ b/testing/btest/Baseline/language.type-type-error/.stderr @@ -0,0 +1 @@ +error in /home/jsiwek/bro/testing/btest/.tmp/language.type-type-error/type-type-error.bro, line 13: not a record (r$a) diff --git a/testing/btest/language/type-type-error.bro b/testing/btest/language/type-type-error.bro new file mode 100644 index 0000000000..047e4b34ef --- /dev/null +++ b/testing/btest/language/type-type-error.bro @@ -0,0 +1,14 @@ +# @TEST-EXEC-FAIL: bro -b %INPUT +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr + +type r: record { + a: string; +}; + +event bro_init() + { + # This should generate a parse error indicating that the type identifier + # is incorrectly used in an expression expecting a real value and not + # a value of type TypeType. + print r$a; + } From 4a09c128822570b08bb19b306a5f7662718e36cd Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 18 Dec 2012 15:08:18 -0600 Subject: [PATCH 28/35] Fix to_port() BIF for port strings with a port number of zero. --- src/bro.bif | 3 ++- testing/btest/Baseline/bifs.to_port/out | 3 +++ testing/btest/bifs/to_port.bro | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/bro.bif b/src/bro.bif index d945e54ef4..bebcde18c3 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -2684,8 +2684,9 @@ function to_port%(s: string%): port if ( s->Len() < 10 ) { char* slash; + errno = 0; port = strtol(s->CheckString(), &slash, 10); - if ( port ) + if ( ! errno ) { ++slash; if ( streq(slash, "tcp") ) diff --git a/testing/btest/Baseline/bifs.to_port/out b/testing/btest/Baseline/bifs.to_port/out index 79796d605e..7744914c30 100644 --- a/testing/btest/Baseline/bifs.to_port/out +++ b/testing/btest/Baseline/bifs.to_port/out @@ -1,6 +1,9 @@ 123/tcp 123/udp 123/icmp +0/tcp +0/udp +0/icmp 0/unknown 256/tcp 256/udp diff --git a/testing/btest/bifs/to_port.bro b/testing/btest/bifs/to_port.bro index 0dfecac43e..b2289b8a21 100644 --- a/testing/btest/bifs/to_port.bro +++ b/testing/btest/bifs/to_port.bro @@ -7,6 +7,9 @@ event bro_init() print to_port("123/tcp"); print to_port("123/udp"); print to_port("123/icmp"); + print to_port("0/tcp"); + print to_port("0/udp"); + print to_port("0/icmp"); print to_port("not a port"); local a: transport_proto = tcp; From 98663fd534837f59806401f01499cd0d82bca22c Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 18 Dec 2012 15:31:50 -0600 Subject: [PATCH 29/35] Fix return value of hook calls that have no handlers. For this case, the return value is always true. --- src/Func.cc | 2 +- testing/btest/Baseline/language.hook/out | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Func.cc b/src/Func.cc index cf548b2d95..37ab7deee7 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -288,7 +288,7 @@ Val* BroFunc::Call(val_list* args, Frame* parent) const assert(Flavor() == FUNC_FLAVOR_EVENT || Flavor() == FUNC_FLAVOR_HOOK); loop_over_list(*args, i) Unref((*args)[i]); - return 0 ; + return Flavor() == FUNC_FLAVOR_HOOK ? new Val(true, TYPE_BOOL) : 0; } SegmentProfiler(segment_logger, location); diff --git a/testing/btest/Baseline/language.hook/out b/testing/btest/Baseline/language.hook/out index bef25193b8..d4f367f875 100644 --- a/testing/btest/Baseline/language.hook/out +++ b/testing/btest/Baseline/language.hook/out @@ -3,6 +3,7 @@ myhook return F myhook return T myhook, &priority=5, [a=37, b=goobye world] F +T myhook3, 8 T myhook4, 2 From 939a64b73a777822b77a9a5e25171e50c105adb0 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 20 Dec 2012 12:49:50 -0600 Subject: [PATCH 30/35] Finish implementation of script-layer switch statement. Addresses #754. They behave like C-style switches except case labels can be comprised of multiple literal constants delimited by commas. Only atomic types are allowed for now. Case label bodies that don't execute a "return" or "break" statement will fall through to subsequent cases. A default case label is allowed. --- src/Stmt.cc | 181 ++++++++++- src/Stmt.h | 23 +- src/Type.cc | 15 + src/Type.h | 3 + src/Val.cc | 12 +- .../Baseline/language.switch-statement/out | 1 + testing/btest/core/leaks/switch-statement.bro | 289 ++++++++++++++++++ testing/btest/language/switch-statement.bro | 284 +++++++++++++++++ 8 files changed, 786 insertions(+), 22 deletions(-) create mode 100644 testing/btest/Baseline/language.switch-statement/out create mode 100644 testing/btest/core/leaks/switch-statement.bro create mode 100644 testing/btest/language/switch-statement.bro diff --git a/src/Stmt.cc b/src/Stmt.cc index c65b44a9bd..27c3a2db69 100644 --- a/src/Stmt.cc +++ b/src/Stmt.cc @@ -592,6 +592,21 @@ Case::~Case() void Case::Describe(ODesc* d) const { + if ( ! Cases() ) + { + if ( ! d->IsBinary() ) + d->Add("default:"); + + d->AddCount(0); + + d->PushIndent(); + Body()->AccessStats(d); + Body()->Describe(d); + d->PopIndent(); + + return; + } + const expr_list& e = Cases()->Exprs(); if ( ! d->IsBinary() ) @@ -658,13 +673,64 @@ bool Case::DoUnserialize(UnserialInfo* info) return this->s != 0; } -SwitchStmt::SwitchStmt(Expr* index, case_list* arg_cases) : - ExprStmt(STMT_SWITCH, index) +static void int_del_func(void* v) { - cases = arg_cases; + delete (int*) v; + } - //### need to loop over cases and make sure their type matches - //### the index, and they're constant and not redundant +void SwitchStmt::Init() + { + TypeList* t = new TypeList(); + t->Append(e->Type()->Ref()); + comp_hash = new CompositeHash(t); + Unref(t); + + case_label_map.SetDeleteFunc(int_del_func); + } + +SwitchStmt::SwitchStmt(Expr* index, case_list* arg_cases) : + ExprStmt(STMT_SWITCH, index), cases(arg_cases), default_case_idx(-1) + { + Init(); + + if ( ! is_atomic_type(e->Type()) ) + e->Error("switch expression must be of an atomic type"); + + loop_over_list(*cases, i) + { + const Case* c = (*cases)[i]; + const ListExpr* le = c->Cases(); + + if ( le ) + { + if ( ! le->Type()->AsTypeList()->AllMatch(e->Type(), false) ) + { + le->Error("case expression type differs from switch type", e); + continue; + } + + const expr_list& exprs = le->Exprs(); + + loop_over_list(exprs, j) + { + if ( ! exprs[j]->IsConst() ) + exprs[j]->Error("case label expression isn't constant"); + else + { + if ( ! AddCaseLabelMapping(exprs[j]->ExprVal(), i) ) + exprs[j]->Error("duplicate case label"); + } + } + } + + else + { + if ( default_case_idx != -1 ) + c->Error("multiple default labels", (*cases)[default_case_idx]); + else + default_case_idx = i; + } + } } SwitchStmt::~SwitchStmt() @@ -673,12 +739,80 @@ SwitchStmt::~SwitchStmt() Unref((*cases)[i]); delete cases; + delete comp_hash; } -Val* SwitchStmt::DoExec(Frame* /* f */, Val* /* v */, stmt_flow_type& /* flow */) const +bool SwitchStmt::AddCaseLabelMapping(const Val* v, int idx) { - printf("switch statement not implemented\n"); - return 0; + HashKey* hk = comp_hash->ComputeHash(v, 1); + + if ( ! hk ) + { + reporter->PushLocation(e->GetLocationInfo()); + reporter->InternalError("switch expression type mismatch (%s/%s)", + type_name(v->Type()->Tag()), type_name(e->Type()->Tag())); + } + + int* label_idx = case_label_map.Lookup(hk); + + if ( label_idx ) + { + delete hk; + return false; + } + + case_label_map.Insert(hk, new int(idx)); + return true; + } + +int SwitchStmt::FindCaseLabelMatch(const Val* v) const + { + HashKey* hk = comp_hash->ComputeHash(v, 1); + + if ( ! hk ) + { + reporter->PushLocation(e->GetLocationInfo()); + reporter->InternalError("switch expression type mismatch (%s/%s)", + type_name(v->Type()->Tag()), type_name(e->Type()->Tag())); + } + + int* label_idx = case_label_map.Lookup(hk); + + delete hk; + + if ( ! label_idx ) + return default_case_idx; + else + return *label_idx; + } + +Val* SwitchStmt::DoExec(Frame* f, Val* v, stmt_flow_type& flow) const + { + Val* rval = 0; + + int matching_label_idx = FindCaseLabelMatch(v); + + if ( matching_label_idx == -1 ) + return 0; + + for ( int i = matching_label_idx; i < cases->length(); ++i ) + { + const Case* c = (*cases)[i]; + + flow = FLOW_NEXT; + rval = c->Body()->Exec(f, flow); + + if ( flow == FLOW_BREAK ) + { + flow = FLOW_NEXT; + break; + } + + if ( flow == FLOW_RETURN ) + break; + } + + return rval; } Stmt* SwitchStmt::DoSimplify() @@ -697,7 +831,13 @@ Stmt* SwitchStmt::DoSimplify() } if ( e->IsConst() ) - { // ### go through cases and pull out the one it matches + { + // Could possibly remove all case labels before the one + // that will match, but may be tricky to tell if any + // subsequent ones can also be removed since it depends + // on the evaluation of the body executing a break/return + // statement. Then still need a way to bypass the lookup + // DoExec for it to be beneficial. if ( ! optimize ) Warn("constant in switch"); } @@ -770,6 +910,9 @@ bool SwitchStmt::DoSerialize(SerialInfo* info) const if ( ! (*cases)[i]->Serialize(info) ) return false; + if ( ! SERIALIZE(default_case_idx) ) + return false; + return true; } @@ -777,6 +920,8 @@ bool SwitchStmt::DoUnserialize(UnserialInfo* info) { DO_UNSERIALIZE(ExprStmt); + Init(); + int len; if ( ! UNSERIALIZE(&len) ) return false; @@ -790,6 +935,24 @@ bool SwitchStmt::DoUnserialize(UnserialInfo* info) cases->append(c); } + if ( ! UNSERIALIZE(&default_case_idx) ) + return false; + + loop_over_list(*cases, i) + { + const ListExpr* le = (*cases)[i]->Cases(); + + if ( ! le ) continue; + + const expr_list& exprs = le->Exprs(); + + loop_over_list(exprs, j) + { + if ( ! AddCaseLabelMapping(exprs[j]->ExprVal(), i) ) + return false; + } + } + return true; } diff --git a/src/Stmt.h b/src/Stmt.h index 7c3b42609b..497d7c97b1 100644 --- a/src/Stmt.h +++ b/src/Stmt.h @@ -17,6 +17,8 @@ class StmtList; class ForStmt; +declare(PDict, int); + class Stmt : public BroObj { public: BroStmtTag Tag() const { return tag; } @@ -187,7 +189,8 @@ protected: class Case : public BroObj { public: - Case(ListExpr* c, Stmt* arg_s) { cases = c; s = arg_s; } + Case(ListExpr* c, Stmt* arg_s) : + cases(simplify_expr_list(c,SIMPLIFY_GENERAL)), s(arg_s) { } ~Case(); const ListExpr* Cases() const { return cases; } @@ -226,7 +229,7 @@ public: protected: friend class Stmt; - SwitchStmt() { cases = 0; } + SwitchStmt() { cases = 0; default_case_idx = -1; comp_hash = 0; } Val* DoExec(Frame* f, Val* v, stmt_flow_type& flow) const; Stmt* DoSimplify(); @@ -234,7 +237,23 @@ protected: DECLARE_SERIAL(SwitchStmt); + // Initialize composite hash and case label map. + void Init(); + + // Adds an entry in case_label_map for the given value to associate it + // with the given index in the cases list. If the entry already exists, + // returns false, else returns true. + bool AddCaseLabelMapping(const Val* v, int idx); + + // Returns index of a case label that's equal to the value, or + // default_case_idx if no case label matches (which may be -1 if there's + // no default label). + int FindCaseLabelMatch(const Val* v) const; + case_list* cases; + int default_case_idx; + CompositeHash* comp_hash; + PDict(int) case_label_map; }; class AddStmt : public ExprStmt { diff --git a/src/Type.cc b/src/Type.cc index e9b0949d13..062d11d7aa 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -2190,3 +2190,18 @@ BroType* init_type(Expr* init) return new SetType(t->AsTypeList(), 0); } + +bool is_atomic_type(const BroType* t) + { + switch ( t->InternalType() ) { + case TYPE_INTERNAL_INT: + case TYPE_INTERNAL_UNSIGNED: + case TYPE_INTERNAL_DOUBLE: + case TYPE_INTERNAL_STRING: + case TYPE_INTERNAL_ADDR: + case TYPE_INTERNAL_SUBNET: + return true; + default: + return false; + } + } diff --git a/src/Type.h b/src/Type.h index 8e2bb099d8..cc6ddcd8fd 100644 --- a/src/Type.h +++ b/src/Type.h @@ -625,6 +625,9 @@ BroType* merge_type_list(ListExpr* elements); // Given an expression, infer its type when used for an initialization. extern BroType* init_type(Expr* init); +// Returns true if argument is an atomic type. +bool is_atomic_type(const BroType* t); + // True if the given type tag corresponds to an integral type. #define IsIntegral(t) (t == TYPE_INT || t == TYPE_COUNT || t == TYPE_COUNTER) diff --git a/src/Val.cc b/src/Val.cc index 3e60fffc82..a62c00dcea 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -3209,17 +3209,7 @@ int same_val(const Val* /* v1 */, const Val* /* v2 */) bool is_atomic_val(const Val* v) { - switch ( v->Type()->InternalType() ) { - case TYPE_INTERNAL_INT: - case TYPE_INTERNAL_UNSIGNED: - case TYPE_INTERNAL_DOUBLE: - case TYPE_INTERNAL_STRING: - case TYPE_INTERNAL_ADDR: - case TYPE_INTERNAL_SUBNET: - return true; - default: - return false; - } + return is_atomic_type(v->Type()); } int same_atomic_val(const Val* v1, const Val* v2) diff --git a/testing/btest/Baseline/language.switch-statement/out b/testing/btest/Baseline/language.switch-statement/out new file mode 100644 index 0000000000..19f86f493a --- /dev/null +++ b/testing/btest/Baseline/language.switch-statement/out @@ -0,0 +1 @@ +done diff --git a/testing/btest/core/leaks/switch-statement.bro b/testing/btest/core/leaks/switch-statement.bro new file mode 100644 index 0000000000..24829006b5 --- /dev/null +++ b/testing/btest/core/leaks/switch-statement.bro @@ -0,0 +1,289 @@ +# Needs perftools support. +# +# @TEST-GROUP: leaks +# +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -b -r $TRACES/wikipedia.trace %INPUT + +type MyEnum: enum { + RED, + GREEN, + BLUE, + PINK, +}; + +function switch_bool(v: bool): string + { + switch (v) { + case T: + return "true"; + case F: + return "false"; + } + return "n/a"; + } + +function switch_int(v: int): string + { + switch (v) { + case +1: + return "one"; + case +2: + return "two"; + case -3: + return "minus three"; + } + return "n/a"; + } + +function switch_enum(v: MyEnum): string + { + switch (v) { + case RED: + return "red"; + case GREEN: + return "green"; + case BLUE: + return "blue"; + } + return "n/a"; + } + +function switch_count(v: count): string + { + switch (v) { + case 1: + return "1"; + case 2: + return "2"; + case 3: + return "3"; + } + return "n/a"; + } + +function switch_port(v: port): string + { + switch (v) { + case 22/tcp: + return "ssh"; + case 53/udp: + return "dns"; + case 0/icmp: + return "echo"; + } + return "n/a"; + } + +function switch_double(v: double): string + { + switch (v) { + case 1.1: + return "1.1"; + case 2.2: + return "2.2"; + case 3.3: + return "3.3"; + } + return "n/a"; + } + +function switch_interval(v: interval): string + { + switch (v) { + case 1sec: + return "1sec"; + case 2day: + return "2day"; + case 3min: + return "3min"; + } + return "n/a"; + } + +function switch_string(v: string): string + { + switch (v) { + case "one": + return "first"; + case "two": + return "second"; + case "three": + return "third"; + } + return "n/a"; + } + +function switch_addr(v: addr): string + { + switch (v) { + case 1.2.3.4: + return "ipv4"; + case [fe80::1]: + return "ipv6"; + case 0.0.0.0: + return "unspec"; + } + return "n/a"; + } + +function switch_subnet(v: subnet): string + { + switch (v) { + case 1.2.3.0/24: + return "1.2.3.0/24"; + case [fe80::0]/96: + return "[fe80::0]"; + case 192.168.0.0/16: + return "192.168.0.0/16"; + } + return "n/a"; + } + +function switch_empty(v: count): string + { + switch ( v ) { + } + return "n/a"; + } + +function switch_break(v: count): string + { + local rval = ""; + switch ( v ) { + case 1: + rval += "test"; + case 2: + rval += "testing"; + break; + case 3: + rval += "tested"; + } + return rval + "return"; + } + +function switch_default(v: count): string + { + local rval = ""; + switch ( v ) { + case 1: + rval += "1"; + case 2: + rval += "2"; + case 3: + rval += "3"; + default: + rval += "d"; + } + return rval + "r"; + } + +function switch_default_placement(v: count): string + { + local rval = ""; + switch ( v ) { + case 1: + rval += "1"; + default: + rval += "d"; + case 2: + rval += "2"; + break; + case 3: + rval += "3"; + } + return rval + "r"; + } + +function switch_case_list(v: count): string + { + switch ( v ) { + case 1, 2: + return "1,2"; + case 3, 4, 5: + return "3,4,5"; + case 6, 7, 8, 9: + return "6,7,8,9"; + } + return "n/a"; + } + +function test_switch(actual: string, expect: string) + { + if ( actual != expect ) + print fmt("%s != %s", actual, expect); + } + +event new_connection(c: connection) + { + test_switch( switch_bool(T) , "true" ); + test_switch( switch_bool(F) , "false" ); + test_switch( switch_int(+1) , "one" ); + test_switch( switch_int(+2) , "two" ); + test_switch( switch_int(-3) , "minus three" ); + test_switch( switch_int(40) , "n/a" ); + test_switch( switch_enum(RED) , "red" ); + test_switch( switch_enum(BLUE) , "blue" ); + test_switch( switch_enum(GREEN) , "green" ); + test_switch( switch_enum(PINK) , "n/a" ); + test_switch( switch_count(1) , "1" ); + test_switch( switch_count(2) , "2" ); + test_switch( switch_count(3) , "3" ); + test_switch( switch_count(100) , "n/a" ); + test_switch( switch_port(22/tcp) , "ssh" ); + test_switch( switch_port(53/udp) , "dns" ); + test_switch( switch_port(0/icmp) , "echo" ); + test_switch( switch_port(1000/tcp) , "n/a" ); + test_switch( switch_double(1.1) , "1.1" ); + test_switch( switch_double(2.2) , "2.2" ); + test_switch( switch_double(3.3) , "3.3" ); + test_switch( switch_interval(1sec) , "1sec" ); + test_switch( switch_interval(2day) , "2day" ); + test_switch( switch_interval(3min) , "3min" ); + test_switch( switch_string("one") , "first" ); + test_switch( switch_string("two") , "second" ); + test_switch( switch_string("three") , "third" ); + test_switch( switch_addr(1.2.3.4) , "ipv4" ); + test_switch( switch_addr([fe80::1]) , "ipv6" ); + test_switch( switch_addr(0.0.0.0) , "unspec" ); + test_switch( switch_subnet(1.2.3.4/24) , "1.2.3.0/24" ); + test_switch( switch_subnet([fe80::1]/96) , "[fe80::0]" ); + test_switch( switch_subnet(192.168.1.100/16) , "192.168.0.0/16" ); + test_switch( switch_empty(2) , "n/a" ); + test_switch( switch_break(1) , "testtestingreturn" ); + test_switch( switch_break(2) , "testingreturn" ); + test_switch( switch_break(3) , "testedreturn" ); + test_switch( switch_default(1) , "123dr" ); + test_switch( switch_default(2) , "23dr" ); + test_switch( switch_default(3) , "3dr" ); + test_switch( switch_default(4) , "dr" ); + test_switch( switch_default_placement(1) , "1d2r" ); + test_switch( switch_default_placement(2) , "2r" ); + test_switch( switch_default_placement(3) , "3r" ); + test_switch( switch_default_placement(4) , "d2r" ); + + local v = vector(0,1,2,3,4,5,6,7,9,10); + local expect: string; + + for ( i in v ) + { + switch ( v[i] ) { + case 1, 2: + expect = "1,2"; + break; + case 3, 4, 5: + expect = "3,4,5"; + break; + case 6, 7, 8, 9: + expect = "6,7,8,9"; + break; + default: + expect = "n/a"; + break; + } + test_switch( switch_case_list(v[i]) , expect ); + } + + print "done"; + } diff --git a/testing/btest/language/switch-statement.bro b/testing/btest/language/switch-statement.bro new file mode 100644 index 0000000000..b8c34f77dc --- /dev/null +++ b/testing/btest/language/switch-statement.bro @@ -0,0 +1,284 @@ +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +type MyEnum: enum { + RED, + GREEN, + BLUE, + PINK, +}; + +function switch_bool(v: bool): string + { + switch (v) { + case T: + return "true"; + case F: + return "false"; + } + return "n/a"; + } + +function switch_int(v: int): string + { + switch (v) { + case +1: + return "one"; + case +2: + return "two"; + case -3: + return "minus three"; + } + return "n/a"; + } + +function switch_enum(v: MyEnum): string + { + switch (v) { + case RED: + return "red"; + case GREEN: + return "green"; + case BLUE: + return "blue"; + } + return "n/a"; + } + +function switch_count(v: count): string + { + switch (v) { + case 1: + return "1"; + case 2: + return "2"; + case 3: + return "3"; + } + return "n/a"; + } + +function switch_port(v: port): string + { + switch (v) { + case 22/tcp: + return "ssh"; + case 53/udp: + return "dns"; + case 0/icmp: + return "echo"; + } + return "n/a"; + } + +function switch_double(v: double): string + { + switch (v) { + case 1.1: + return "1.1"; + case 2.2: + return "2.2"; + case 3.3: + return "3.3"; + } + return "n/a"; + } + +function switch_interval(v: interval): string + { + switch (v) { + case 1sec: + return "1sec"; + case 2day: + return "2day"; + case 3min: + return "3min"; + } + return "n/a"; + } + +function switch_string(v: string): string + { + switch (v) { + case "one": + return "first"; + case "two": + return "second"; + case "three": + return "third"; + } + return "n/a"; + } + +function switch_addr(v: addr): string + { + switch (v) { + case 1.2.3.4: + return "ipv4"; + case [fe80::1]: + return "ipv6"; + case 0.0.0.0: + return "unspec"; + } + return "n/a"; + } + +function switch_subnet(v: subnet): string + { + switch (v) { + case 1.2.3.0/24: + return "1.2.3.0/24"; + case [fe80::0]/96: + return "[fe80::0]"; + case 192.168.0.0/16: + return "192.168.0.0/16"; + } + return "n/a"; + } + +function switch_empty(v: count): string + { + switch ( v ) { + } + return "n/a"; + } + +function switch_break(v: count): string + { + local rval = ""; + switch ( v ) { + case 1: + rval += "test"; + case 2: + rval += "testing"; + break; + case 3: + rval += "tested"; + } + return rval + "return"; + } + +function switch_default(v: count): string + { + local rval = ""; + switch ( v ) { + case 1: + rval += "1"; + case 2: + rval += "2"; + case 3: + rval += "3"; + default: + rval += "d"; + } + return rval + "r"; + } + +function switch_default_placement(v: count): string + { + local rval = ""; + switch ( v ) { + case 1: + rval += "1"; + default: + rval += "d"; + case 2: + rval += "2"; + break; + case 3: + rval += "3"; + } + return rval + "r"; + } + +function switch_case_list(v: count): string + { + switch ( v ) { + case 1, 2: + return "1,2"; + case 3, 4, 5: + return "3,4,5"; + case 6, 7, 8, 9: + return "6,7,8,9"; + } + return "n/a"; + } + +function test_switch(actual: string, expect: string) + { + if ( actual != expect ) + print fmt("%s != %s", actual, expect); + } + +event bro_init() + { + test_switch( switch_bool(T) , "true" ); + test_switch( switch_bool(F) , "false" ); + test_switch( switch_int(+1) , "one" ); + test_switch( switch_int(+2) , "two" ); + test_switch( switch_int(-3) , "minus three" ); + test_switch( switch_int(40) , "n/a" ); + test_switch( switch_enum(RED) , "red" ); + test_switch( switch_enum(BLUE) , "blue" ); + test_switch( switch_enum(GREEN) , "green" ); + test_switch( switch_enum(PINK) , "n/a" ); + test_switch( switch_count(1) , "1" ); + test_switch( switch_count(2) , "2" ); + test_switch( switch_count(3) , "3" ); + test_switch( switch_count(100) , "n/a" ); + test_switch( switch_port(22/tcp) , "ssh" ); + test_switch( switch_port(53/udp) , "dns" ); + test_switch( switch_port(0/icmp) , "echo" ); + test_switch( switch_port(1000/tcp) , "n/a" ); + test_switch( switch_double(1.1) , "1.1" ); + test_switch( switch_double(2.2) , "2.2" ); + test_switch( switch_double(3.3) , "3.3" ); + test_switch( switch_interval(1sec) , "1sec" ); + test_switch( switch_interval(2day) , "2day" ); + test_switch( switch_interval(3min) , "3min" ); + test_switch( switch_string("one") , "first" ); + test_switch( switch_string("two") , "second" ); + test_switch( switch_string("three") , "third" ); + test_switch( switch_addr(1.2.3.4) , "ipv4" ); + test_switch( switch_addr([fe80::1]) , "ipv6" ); + test_switch( switch_addr(0.0.0.0) , "unspec" ); + test_switch( switch_subnet(1.2.3.4/24) , "1.2.3.0/24" ); + test_switch( switch_subnet([fe80::1]/96) , "[fe80::0]" ); + test_switch( switch_subnet(192.168.1.100/16) , "192.168.0.0/16" ); + test_switch( switch_empty(2) , "n/a" ); + test_switch( switch_break(1) , "testtestingreturn" ); + test_switch( switch_break(2) , "testingreturn" ); + test_switch( switch_break(3) , "testedreturn" ); + test_switch( switch_default(1) , "123dr" ); + test_switch( switch_default(2) , "23dr" ); + test_switch( switch_default(3) , "3dr" ); + test_switch( switch_default(4) , "dr" ); + test_switch( switch_default_placement(1) , "1d2r" ); + test_switch( switch_default_placement(2) , "2r" ); + test_switch( switch_default_placement(3) , "3r" ); + test_switch( switch_default_placement(4) , "d2r" ); + + local v = vector(0,1,2,3,4,5,6,7,9,10); + local expect: string; + + for ( i in v ) + { + switch ( v[i] ) { + case 1, 2: + expect = "1,2"; + break; + case 3, 4, 5: + expect = "3,4,5"; + break; + case 6, 7, 8, 9: + expect = "6,7,8,9"; + break; + default: + expect = "n/a"; + break; + } + test_switch( switch_case_list(v[i]) , expect ); + } + + print "done"; + } From 55c515d50a29fffc9748eb847b16d9b92b8b0017 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 20 Dec 2012 17:13:06 -0600 Subject: [PATCH 31/35] Add array-style index accessor for strings. Addresses #422. The index expression can take up to two indices for the start and end index of the substring to return (e.g. "mystring[1,3]"). Negative indices are allowed, with -1 representing the last character in the string. The indexing is not cyclic -- if the starting index is >= the length of the string an empty string is returned, and if the ending index is >= the length of the string then it's interpreted as the last index of the string. Assigning to substrings accessed like this isn't allowed. --- src/Expr.cc | 59 ++++++++++++++++--- src/Type.cc | 10 +++- .../Baseline/language.string-indexing/out | 13 ++++ testing/btest/core/leaks/string-indexing.bro | 26 ++++++++ testing/btest/language/string-indexing.bro | 17 ++++++ 5 files changed, 115 insertions(+), 10 deletions(-) create mode 100644 testing/btest/Baseline/language.string-indexing/out create mode 100644 testing/btest/core/leaks/string-indexing.bro create mode 100644 testing/btest/language/string-indexing.bro diff --git a/src/Expr.cc b/src/Expr.cc index 3a4e8add70..92fdf76a7d 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -2808,11 +2808,17 @@ IndexExpr::IndexExpr(Expr* arg_op1, ListExpr* arg_op2) SetError("not an index type"); else if ( ! op1->Type()->YieldType() ) + { + if ( IsString(op1->Type()->Tag()) && + match_type == MATCHES_INDEX_SCALAR ) + SetType(base_type(TYPE_STRING)); + else // It's a set - so indexing it yields void. We don't // directly generate an error message, though, since this // expression might be part of an add/delete statement, // rather than yielding a value. - SetType(base_type(TYPE_VOID)); + SetType(base_type(TYPE_VOID)); + } else if ( match_type == MATCHES_INDEX_SCALAR ) SetType(op1->Type()->YieldType()->Ref()); @@ -2888,6 +2894,9 @@ void IndexExpr::Delete(Frame* f) Expr* IndexExpr::MakeLvalue() { + if ( IsString(op1->Type()->Tag()) ) + ExprError("cannot assign to string index expression"); + return new RefExpr(this); } @@ -2961,10 +2970,34 @@ Val* IndexExpr::Fold(Val* v1, Val* v2) const Val* v = 0; - if ( v1->Type()->Tag() == TYPE_VECTOR ) + switch ( v1->Type()->Tag() ) { + case TYPE_VECTOR: v = v1->AsVectorVal()->Lookup(v2); - else + break; + case TYPE_TABLE: v = v1->AsTableVal()->Lookup(v2); + break; + case TYPE_STRING: + { + const ListVal* lv = v2->AsListVal(); + const BroString* s = v1->AsString(); + int len = s->Len(); + bro_int_t first = lv->Index(0)->AsInt(); + bro_int_t last = lv->Length() > 1 ? lv->Index(1)->AsInt() : first; + + if ( first < 0 ) + first += len; + if ( last < 0 ) + last += len; + + BroString* substring = s->GetSubstring(first, last - first + 1); + return new StringVal(substring ? substring : new BroString("")); + break; + } + default: + Error("type cannot be indexed"); + break; + } if ( v ) return v->Ref(); @@ -2991,14 +3024,22 @@ void IndexExpr::Assign(Frame* f, Val* v, Opcode op) return; } - if ( v1->Type()->Tag() == TYPE_VECTOR ) - { + switch ( v1->Type()->Tag() ) { + case TYPE_VECTOR: if ( ! v1->AsVectorVal()->Assign(v2, v, this, op) ) Internal("assignment failed"); - } - - else if ( ! v1->AsTableVal()->Assign(v2, v, op) ) - Internal("assignment failed"); + break; + case TYPE_TABLE: + if ( ! v1->AsTableVal()->Assign(v2, v, op) ) + Internal("assignment failed"); + break; + case TYPE_STRING: + Internal("assignment via string index accessor not allowed"); + break; + default: + Internal("bad index expression type in assignment"); + break; + } Unref(v1); Unref(v2); diff --git a/src/Type.cc b/src/Type.cc index e9b0949d13..74c0eb05cb 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -114,8 +114,16 @@ BroType::~BroType() delete [] type_id; } -int BroType::MatchesIndex(ListExpr*& /* index */) const +int BroType::MatchesIndex(ListExpr*& index) const { + if ( Tag() == TYPE_STRING ) + { + if ( index->Exprs().length() != 1 && index->Exprs().length() != 2 ) + return DOES_NOT_MATCH_INDEX; + if ( check_and_promote_exprs_to_type(index, ::base_type(TYPE_INT)) ) + return MATCHES_INDEX_SCALAR; + } + return DOES_NOT_MATCH_INDEX; } diff --git a/testing/btest/Baseline/language.string-indexing/out b/testing/btest/Baseline/language.string-indexing/out new file mode 100644 index 0000000000..3359187d4c --- /dev/null +++ b/testing/btest/Baseline/language.string-indexing/out @@ -0,0 +1,13 @@ +1 +12 +123456 +0123456789 +8 +789 +9 +9 +9 + + +2 +1 diff --git a/testing/btest/core/leaks/string-indexing.bro b/testing/btest/core/leaks/string-indexing.bro new file mode 100644 index 0000000000..40af2317e6 --- /dev/null +++ b/testing/btest/core/leaks/string-indexing.bro @@ -0,0 +1,26 @@ +# Needs perftools support. +# +# @TEST-GROUP: leaks +# +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -b -r $TRACES/wikipedia.trace %INPUT + + +event new_connection(c: connection) + { + local s = "0123456789"; + print s[1]; + print s[1,2]; + print s[1,6]; + print s[0,20]; + print s[-2]; + print s[-3,-1]; + print s[-1,-10]; + print s[-1,0]; + print s[-1,5]; + print s[20, 23]; + print s[-20, 23]; + print s[0,5][2]; + print s[0,5][1,3][0]; + } diff --git a/testing/btest/language/string-indexing.bro b/testing/btest/language/string-indexing.bro new file mode 100644 index 0000000000..d05c1b6c5e --- /dev/null +++ b/testing/btest/language/string-indexing.bro @@ -0,0 +1,17 @@ +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +local s = "0123456789"; +print s[1]; +print s[1,2]; +print s[1,6]; +print s[0,20]; +print s[-2]; +print s[-3,-1]; +print s[-1,-10]; +print s[-1,0]; +print s[-1,5]; +print s[20, 23]; +print s[-20, 23]; +print s[0,5][2]; +print s[0,5][1,3][0]; From 5526f09b602d030dacb0a302ff7f19630bafa70c Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 20 Dec 2012 17:54:44 -0800 Subject: [PATCH 32/35] Updating submodule(s). [nomail] --- aux/broctl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broctl b/aux/broctl index f4d6a2af15..2bf6b37177 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit f4d6a2af15404dc1349d12d2ad21a3eebcb2ff1e +Subproject commit 2bf6b37177b895329173acac2bb98f38a8783bc1 From 8000c40fee329e25d146b68e15fbb6ef82c599eb Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 7 Jan 2013 11:41:36 -0600 Subject: [PATCH 33/35] Fix memory leak in OpaqueType::DoUnserialize. --- src/Type.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Type.cc b/src/Type.cc index 412c25459d..1fb813efa1 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -1305,6 +1305,7 @@ bool OpaqueType::DoUnserialize(UnserialInfo* info) return false; name = n; + delete [] n; return true; } From e638f043010e14d4599fea53a75d6dbad66f3749 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 7 Jan 2013 09:47:09 -0800 Subject: [PATCH 34/35] Updating submodule(s). [nomail] --- aux/btest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/btest b/aux/btest index d9c747f268..3585dc9a7a 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit d9c747f268ca50275f3bc2e1581464b599d5a3ea +Subproject commit 3585dc9a7afe20d70cb77fc2472cc6bce3850b67 From 8b46bbb1c0b3aefea7fa683b53165e497a14056d Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 7 Jan 2013 13:29:05 -0600 Subject: [PATCH 35/35] Change substring index notation to use a colon (addresses #422). String slice notation is written as `s[1:2]` instead of `s[1, 2]` because the later is ambiguous with composite index types. --- src/Expr.cc | 16 +++++++++++++- src/Expr.h | 2 +- src/parse.y | 8 +++++++ testing/btest/core/leaks/string-indexing.bro | 22 ++++++++++---------- testing/btest/language/string-indexing.bro | 22 ++++++++++---------- 5 files changed, 46 insertions(+), 24 deletions(-) diff --git a/src/Expr.cc b/src/Expr.cc index eef0e7b69e..b3127d6a7f 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -2801,12 +2801,26 @@ bool AssignExpr::DoUnserialize(UnserialInfo* info) return UNSERIALIZE(&is_init); } -IndexExpr::IndexExpr(Expr* arg_op1, ListExpr* arg_op2) +IndexExpr::IndexExpr(Expr* arg_op1, ListExpr* arg_op2, bool is_string_slice) : BinaryExpr(EXPR_INDEX, arg_op1, arg_op2) { if ( IsError() ) return; + if ( is_string_slice ) + { + if ( ! IsString(op1->Type()->Tag()) ) + ExprError("slice notation indexing can apply only to strings"); + } + else if ( IsString(op1->Type()->Tag()) ) + { + if ( arg_op2->Exprs().length() != 1 ) + ExprError("invalid string index expression"); + } + + if ( IsError() ) + return; + int match_type = op1->Type()->MatchesIndex(arg_op2); if ( match_type == DOES_NOT_MATCH_INDEX ) SetError("not an index type"); diff --git a/src/Expr.h b/src/Expr.h index ea17c735b5..ac3ab0f4f2 100644 --- a/src/Expr.h +++ b/src/Expr.h @@ -646,7 +646,7 @@ protected: class IndexExpr : public BinaryExpr { public: - IndexExpr(Expr* op1, ListExpr* op2); + IndexExpr(Expr* op1, ListExpr* op2, bool is_string_slice = false); int CanAdd() const; int CanDel() const; diff --git a/src/parse.y b/src/parse.y index a16b742728..090786647e 100644 --- a/src/parse.y +++ b/src/parse.y @@ -418,6 +418,14 @@ expr: $$ = new IndexExpr($1, $3); } + | expr '[' expr ':' expr ']' + { + set_location(@1, @6); + ListExpr* le = new ListExpr($3); + le->Append($5); + $$ = new IndexExpr($1, le, true); + } + | expr '$' TOK_ID { set_location(@1, @3); diff --git a/testing/btest/core/leaks/string-indexing.bro b/testing/btest/core/leaks/string-indexing.bro index 40af2317e6..f9ea000ef9 100644 --- a/testing/btest/core/leaks/string-indexing.bro +++ b/testing/btest/core/leaks/string-indexing.bro @@ -11,16 +11,16 @@ event new_connection(c: connection) { local s = "0123456789"; print s[1]; - print s[1,2]; - print s[1,6]; - print s[0,20]; + print s[1:2]; + print s[1:6]; + print s[0:20]; print s[-2]; - print s[-3,-1]; - print s[-1,-10]; - print s[-1,0]; - print s[-1,5]; - print s[20, 23]; - print s[-20, 23]; - print s[0,5][2]; - print s[0,5][1,3][0]; + print s[-3:1]; + print s[-1:10]; + print s[-1:0]; + print s[-1:5]; + print s[20:23]; + print s[-20:23]; + print s[0:5][2]; + print s[0:5][1:3][0]; } diff --git a/testing/btest/language/string-indexing.bro b/testing/btest/language/string-indexing.bro index d05c1b6c5e..f991b3c5fa 100644 --- a/testing/btest/language/string-indexing.bro +++ b/testing/btest/language/string-indexing.bro @@ -3,15 +3,15 @@ local s = "0123456789"; print s[1]; -print s[1,2]; -print s[1,6]; -print s[0,20]; +print s[1:2]; +print s[1:6]; +print s[0:20]; print s[-2]; -print s[-3,-1]; -print s[-1,-10]; -print s[-1,0]; -print s[-1,5]; -print s[20, 23]; -print s[-20, 23]; -print s[0,5][2]; -print s[0,5][1,3][0]; +print s[-3:-1]; +print s[-1:-10]; +print s[-1:0]; +print s[-1:5]; +print s[20:23]; +print s[-20:23]; +print s[0:5][2]; +print s[0:5][1:3][0];