From b9dad026150a74d145fe5fd1450301ffcd038cda Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 14 Jun 2017 07:23:37 -0700 Subject: [PATCH 1/4] Reimplement copy(). The old implementation used the serialization framework, which is going away. This is a new standalone implementation that should also be quite a bit faster. WIP: Not fully implemented and tested yet. --- src/Val.cc | 203 ++++++++++++++++++--- src/Val.h | 38 +++- testing/btest/language/copy-all-types.zeek | 27 +++ 3 files changed, 244 insertions(+), 24 deletions(-) create mode 100644 testing/btest/language/copy-all-types.zeek diff --git a/src/Val.cc b/src/Val.cc index 340cef6bb5..fe83c8d583 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -72,31 +72,59 @@ Val::~Val() #endif } -Val* Val::Clone() const +Val* Val::Clone() { - SerializationFormat* form = new BinarySerializationFormat(); - form->StartWrite(); - - CloneSerializer ss(form); - SerialInfo sinfo(&ss); - sinfo.cache = false; - sinfo.include_locations = false; - - if ( ! this->Serialize(&sinfo) ) - return 0; - - char* data; - uint32 len = form->EndWrite(&data); - form->StartRead(data, len); - - UnserialInfo uinfo(&ss); - uinfo.cache = false; - Val* clone = Unserialize(&uinfo, type); - - free(data); - return clone; + Val::CloneState state; + return Clone(&state); } +Val* Val::Clone(CloneState* state) + { + auto i = state->clones.find(this); + + if ( i != state->clones.end() ) + return i->second->Ref(); + + auto c = DoClone(state); + assert(c); + + state->clones.insert(std::make_pair(this, c)); + return c; + } + +Val* Val::DoClone(CloneState* state) + { + switch ( type->InternalType() ) { + case TYPE_INTERNAL_INT: + case TYPE_INTERNAL_UNSIGNED: + case TYPE_INTERNAL_DOUBLE: + // Immutable. + return Ref(); + + case TYPE_INTERNAL_OTHER: + // Derived classes are responsible for this. Exception: + // Functions and files. There aren't any derived classes. + if ( type->Tag() == TYPE_FUNC ) + // Immutable. + return Ref(); + + if ( type->Tag() == TYPE_FILE ) + { + auto f = AsFile(); + ::Ref(f); + return new Val(f); + } + + // Fall-through. + + default: + reporter->InternalError("cloning illegal base type"); + } + + reporter->InternalError("cannot be reached"); + return nullptr; + } + bool Val::Serialize(SerialInfo* info) const { return SerialObj::Serialize(info); @@ -862,6 +890,12 @@ void PortVal::ValDescribe(ODesc* d) const d->Add("/unknown"); } +Val* PortVal::DoClone(CloneState* state) + { + // Immutable. + return Ref(); + } + IMPLEMENT_SERIAL(PortVal, SER_PORT_VAL); bool PortVal::DoSerialize(SerialInfo* info) const @@ -920,6 +954,12 @@ Val* AddrVal::SizeVal() const return val_mgr->GetCount(128); } +Val* AddrVal::DoClone(CloneState* state) + { + // Immutable. + return Ref(); + } + IMPLEMENT_SERIAL(AddrVal, SER_ADDR_VAL); bool AddrVal::DoSerialize(SerialInfo* info) const @@ -1044,6 +1084,12 @@ bool SubNetVal::Contains(const IPAddr& addr) const return val.subnet_val->Contains(a); } +Val* SubNetVal::DoClone(CloneState* state) + { + // Immutable. + return Ref(); + } + IMPLEMENT_SERIAL(SubNetVal, SER_SUBNET_VAL); bool SubNetVal::DoSerialize(SerialInfo* info) const @@ -1100,6 +1146,11 @@ unsigned int StringVal::MemoryAllocation() const return padded_sizeof(*this) + val.string_val->MemoryAllocation(); } +Val* StringVal::DoClone(CloneState* state) + { + return new StringVal(new BroString((u_char*) val.string_val->Bytes(), val.string_val->Len(), 1)); + } + IMPLEMENT_SERIAL(StringVal, SER_STRING_VAL); bool StringVal::DoSerialize(SerialInfo* info) const @@ -1162,6 +1213,13 @@ unsigned int PatternVal::MemoryAllocation() const return padded_sizeof(*this) + val.re_val->MemoryAllocation(); } +Val* PatternVal::DoClone(CloneState* state) + { + // TODO: Double-check + auto re = new RE_Matcher(val.re_val->PatternText(), val.re_val->AnywherePatternText()); + return new PatternVal(re); + } + IMPLEMENT_SERIAL(PatternVal, SER_PATTERN_VAL); bool PatternVal::DoSerialize(SerialInfo* info) const @@ -1260,6 +1318,16 @@ void ListVal::Describe(ODesc* d) const } } +Val* ListVal::DoClone(CloneState* state) + { + auto lv = new ListVal(tag); + + loop_over_list(vals, i) + lv->Append(vals[i]->Clone(state)); + + return lv; + } + IMPLEMENT_SERIAL(ListVal, SER_LIST_VAL); bool ListVal::DoSerialize(SerialInfo* info) const @@ -2498,6 +2566,55 @@ void TableVal::ReadOperation(Val* index, TableEntryVal* v) } } +Val* TableVal::DoClone(CloneState* state) + { + auto tv = new TableVal(table_type); + + const PDict(TableEntryVal)* tbl = AsTable(); + IterCookie* cookie = tbl->InitForIteration(); + + HashKey* key; + TableEntryVal* val; + while ( (val = tbl->NextEntry(key, cookie)) ) + { + Val* idx = RecoverIndex(key); + TableEntryVal* nval = val ? new TableEntryVal(*val) : nullptr; + tv->AsNonConstTable()->Insert(key, nval); + + if ( subnets ) + { + tv->subnets->Insert(idx, nval); + Unref(idx); + } + + delete key; + } + + if ( attrs ) + { + ::Ref(attrs); + tv->attrs = attrs; + } + + if ( expire_time ) + { + tv->expire_time = expire_time->Ref(); + + // As network_time is not necessarily initialized yet, we set + // a timer which fires immediately. + timer = new TableValTimer(this, 1); + timer_mgr->Add(timer); + } + + if ( expire_func ) + tv->expire_func = expire_func->Ref(); + + if ( def_val ) + tv->def_val = def_val->Ref(); + + return tv; + } + IMPLEMENT_SERIAL(TableVal, SER_TABLE_VAL); // This is getting rather complex due to the ability to suspend even within @@ -3052,7 +3169,7 @@ void RecordVal::Describe(ODesc* d) const void RecordVal::DescribeReST(ODesc* d) const { const val_list* vl = AsRecord(); - int n = vl->length(); + int n = vl->length(); d->Add("{"); d->PushIndent(); @@ -3077,6 +3194,21 @@ void RecordVal::DescribeReST(ODesc* d) const d->Add("}"); } +Val* RecordVal::DoClone(CloneState* state) + { + // TODO: We leave origin unset, ok? + ::Ref(record_type); + auto rv = new RecordVal(record_type); + + loop_over_list(*val.val_list_val, i) + { + Val* v = (*val.val_list_val)[i]->Clone(state); + rv->val.val_list_val->append(v); + } + + return nullptr; + } + IMPLEMENT_SERIAL(RecordVal, SER_RECORD_VAL); bool RecordVal::DoSerialize(SerialInfo* info) const @@ -3193,6 +3325,12 @@ void EnumVal::ValDescribe(ODesc* d) const d->Add(ename); } +Val* EnumVal::DoClone(CloneState* state) + { + // Immutable. + return Ref(); + } + IMPLEMENT_SERIAL(EnumVal, SER_ENUM_VAL); bool EnumVal::DoSerialize(SerialInfo* info) const @@ -3378,6 +3516,19 @@ bool VectorVal::RemoveProperties(Properties arg_props) return true; } + +Val* VectorVal::DoClone(CloneState* state) + { + auto vv = new VectorVal(vector_type); + for ( unsigned int i = 0; i < val.vector_val->size(); ++i ) + { + auto v = (*val.vector_val)[i]->Clone(state); + vv->val.vector_val->push_back(v); + } + + return vv; + } + IMPLEMENT_SERIAL(VectorVal, SER_VECTOR_VAL); bool VectorVal::DoSerialize(SerialInfo* info) const @@ -3450,6 +3601,12 @@ OpaqueVal::~OpaqueVal() { } +Val* OpaqueVal::DoClone(CloneState* state) + { + // TODO + return nullptr; + } + IMPLEMENT_SERIAL(OpaqueVal, SER_OPAQUE_VAL); bool OpaqueVal::DoSerialize(SerialInfo* info) const diff --git a/src/Val.h b/src/Val.h index 63e790848d..5104c1933e 100644 --- a/src/Val.h +++ b/src/Val.h @@ -172,7 +172,7 @@ public: ~Val() override; Val* Ref() { ::Ref(this); return this; } - virtual Val* Clone() const; + Val* Clone(); int IsZero() const; int IsOne() const; @@ -370,6 +370,9 @@ public: protected: friend class EnumType; + friend class ListVal; + friend class RecordVal; + friend class VectorVal; friend class ValManager; virtual void ValDescribe(ODesc* d) const; @@ -419,6 +422,14 @@ protected: static Val* Unserialize(UnserialInfo* info, TypeTag type, const BroType* exact_type); + // For internal use by the Val::Clone() methods. + struct CloneState { + std::unordered_map clones; + }; + + Val* Clone(CloneState* state); + virtual Val* DoClone(CloneState* state); + BroValUnion val; BroType* type; @@ -639,6 +650,7 @@ protected: PortVal(uint32 p, bool unused); void ValDescribe(ODesc* d) const override; + Val* DoClone(CloneState* state) override; DECLARE_SERIAL(PortVal); }; @@ -664,6 +676,8 @@ protected: explicit AddrVal(TypeTag t) : Val(t) { } explicit AddrVal(BroType* t) : Val(t) { } + Val* DoClone(CloneState* state) override; + DECLARE_SERIAL(AddrVal); }; @@ -692,6 +706,7 @@ protected: SubNetVal() {} void ValDescribe(ODesc* d) const override; + Val* DoClone(CloneState* state) override; DECLARE_SERIAL(SubNetVal); }; @@ -724,6 +739,7 @@ protected: StringVal() {} void ValDescribe(ODesc* d) const override; + Val* DoClone(CloneState* state) override; DECLARE_SERIAL(StringVal); }; @@ -744,6 +760,7 @@ protected: PatternVal() {} void ValDescribe(ODesc* d) const override; + Val* DoClone(CloneState* state) override; DECLARE_SERIAL(PatternVal); }; @@ -789,6 +806,8 @@ protected: friend class Val; ListVal() {} + Val* DoClone(CloneState* state) override; + DECLARE_SERIAL(ListVal); val_list vals; @@ -806,6 +825,15 @@ public: expire_access_time = last_read_update = int(network_time - bro_start_network_time); } + + TableEntryVal(const TableEntryVal& other) + { + val = other.val->Ref(); + last_access_time = other.last_access_time; + expire_access_time = other.expire_access_time; + last_read_update = other.last_read_update; + } + ~TableEntryVal() { } Val* Value() { return val; } @@ -997,6 +1025,8 @@ protected: // Propagates a read operation if necessary. void ReadOperation(Val* index, TableEntryVal *v); + Val* DoClone(CloneState* state) override; + DECLARE_SERIAL(TableVal); TableType* table_type; @@ -1069,6 +1099,8 @@ protected: bool AddProperties(Properties arg_state) override; bool RemoveProperties(Properties arg_state) override; + Val* DoClone(CloneState* state) override; + DECLARE_SERIAL(RecordVal); RecordType* record_type; @@ -1100,6 +1132,7 @@ protected: EnumVal() {} void ValDescribe(ODesc* d) const override; + Val* DoClone(CloneState* state) override; DECLARE_SERIAL(EnumVal); }; @@ -1160,6 +1193,7 @@ protected: bool AddProperties(Properties arg_state) override; bool RemoveProperties(Properties arg_state) override; void ValDescribe(ODesc* d) const override; + Val* DoClone(CloneState* state) override; DECLARE_SERIAL(VectorVal); @@ -1178,6 +1212,8 @@ protected: friend class Val; OpaqueVal() { } + Val* DoClone(CloneState* state) override; + DECLARE_SERIAL(OpaqueVal); }; diff --git a/testing/btest/language/copy-all-types.zeek b/testing/btest/language/copy-all-types.zeek new file mode 100644 index 0000000000..e39308b8e0 --- /dev/null +++ b/testing/btest/language/copy-all-types.zeek @@ -0,0 +1,27 @@ +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +function check(o1: any, o2: any, equal: bool, expect_same: bool) + { + local expect_msg = (equal ? "ok" : "FAIL0"); + local same = same_object(o1, o2); + + if ( expect_same && ! same ) + expect_msg = "FAIL1"; + + if ( ! expect_same && same ) + expect_msg = "FAIL2"; + + print fmt("orig=%s (%s) clone=%s (%s) equal=%s same_object=%s (%s)", o1, type_name(o1), o2, type_name(o2), equal, same, expect_msg); + } + +event zeek_init() + { + local i1 = -42; + local i2 = copy(i1); + check(i1, i2, i1 == i2, T); + + local s1 = "Foo"; + local s2 = copy(s1); + check(s1, s2, s1 == s2, F); + } From 2efbe76920b61a12630851fd517a410dccd68b5f Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Tue, 21 May 2019 15:06:39 -0700 Subject: [PATCH 2/4] Checkpoint - all non-opaque-types can be cloned. --- src/Val.cc | 21 ++- testing/btest/language/copy-all-types.zeek | 167 ++++++++++++++++++++- 2 files changed, 180 insertions(+), 8 deletions(-) diff --git a/src/Val.cc b/src/Val.cc index 2b577f58e3..101ff4b983 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -1216,6 +1216,7 @@ Val* PatternVal::DoClone(CloneState* state) { // TODO: Double-check auto re = new RE_Matcher(val.re_val->PatternText(), val.re_val->AnywherePatternText()); + re->Compile(); return new PatternVal(re); } @@ -2557,7 +2558,14 @@ Val* TableVal::DoClone(CloneState* state) while ( (val = tbl->NextEntry(key, cookie)) ) { Val* idx = RecoverIndex(key); - TableEntryVal* nval = val ? new TableEntryVal(*val) : nullptr; + TableEntryVal* nval = nullptr; + if ( val ) + { + if ( val->Value() ) + nval = new TableEntryVal(val->Value()->Clone()); + else + nval = new TableEntryVal(nullptr); + } tv->AsNonConstTable()->Insert(key, nval); if ( subnets ) @@ -3175,17 +3183,22 @@ void RecordVal::DescribeReST(ODesc* d) const Val* RecordVal::DoClone(CloneState* state) { - // TODO: We leave origin unset, ok? + // We set origin to 0 here. + // Origin only seems to be used for exactly one purpose - to find the connection record that is associated + // with a record. As we cannot guarantee that it will ber zeroed out at the approproate time (as it seems to be + // guaranteed for the original record) we don't touch it. ::Ref(record_type); auto rv = new RecordVal(record_type); + rv->origin = nullptr; + rv->val.val_list_val = new val_list(val.val_list_val->length()); loop_over_list(*val.val_list_val, i) { - Val* v = (*val.val_list_val)[i]->Clone(state); + Val* v = (*val.val_list_val)[i] ? (*val.val_list_val)[i]->Clone(state) : nullptr; rv->val.val_list_val->append(v); } - return nullptr; + return rv; } IMPLEMENT_SERIAL(RecordVal, SER_RECORD_VAL); diff --git a/testing/btest/language/copy-all-types.zeek b/testing/btest/language/copy-all-types.zeek index e39308b8e0..515f83dce4 100644 --- a/testing/btest/language/copy-all-types.zeek +++ b/testing/btest/language/copy-all-types.zeek @@ -1,6 +1,56 @@ -# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: zeek -b %INPUT >out # @TEST-EXEC: btest-diff out +type MyEnum: enum { ENUMME }; + +type InnerTestRecord: record { + a: string; +}; + +type TestRecord: record { + s1: string; + s2: string; + i1: InnerTestRecord; + i2: InnerTestRecord &optional; + donotset: InnerTestRecord &optional; + def: count &default=5; +}; + +function join_count_set(ss: set[count], j: string): string + { + local output=""; + local i=0; + for ( s in ss ) + { + if ( i > 0 ) + output = cat(output, j); + + output = cat(output, s); + ++i; + } + return output; + } + +function do_format(i: any): any + { + local tpe = type_name(i); + + switch ( tpe ) + { + case "set[count]": + return join_count_set(i, ","); + case "table[string] of string": + local cast: table[string] of string = i; + local vout: vector of string = vector(); + for ( el in cast ) + { + vout += cat(el, "=", cast[el]); + } + return join_string_vec(vout, ";"); + } + return i; + } + function check(o1: any, o2: any, equal: bool, expect_same: bool) { local expect_msg = (equal ? "ok" : "FAIL0"); @@ -11,17 +61,126 @@ function check(o1: any, o2: any, equal: bool, expect_same: bool) if ( ! expect_same && same ) expect_msg = "FAIL2"; - - print fmt("orig=%s (%s) clone=%s (%s) equal=%s same_object=%s (%s)", o1, type_name(o1), o2, type_name(o2), equal, same, expect_msg); + + print fmt("orig=%s (%s) clone=%s (%s) equal=%s same_object=%s (%s)", do_format(o1), type_name(o1), do_format(o2), type_name(o2), equal, same, expect_msg); } +function check_vector_equal(a: vector of count, b: vector of count): bool + { + if ( |a| != |b| ) + return F; + + for ( i in a ) + { + if ( a[i] != b[i] ) + return F; + } + + return T; + } + +function check_string_table_equal(a: table[string] of string, b: table[string] of string): bool + { + if ( |a| != |b| ) + return F; + + for ( i in a ) + { + if ( a[i] != b[i] ) + return F; + } + + return T; + } + +function compare_otr(a: TestRecord, b: TestRecord): bool + { + if ( a$s1 != b$s1 ) + return F; + if ( a$s2 != b$s2 ) + return F; + if ( a$i1$a != b$i1$a ) + return F; + if ( a$i2$a != b$i2$a ) + return F; + + if ( same_object(a$i1, b$i1) ) + return F; + if ( same_object(a$i2, b$i2) ) + return F; + + # check that we restroe that i1 & i2 point to same object + if ( ! same_object(a$i1, a$i2) ) + return F; + if ( ! same_object(b$i1, b$i2) ) + return F; + + if ( a$def != b$def ) + return F; + + return T; + } + + event zeek_init() { local i1 = -42; local i2 = copy(i1); check(i1, i2, i1 == i2, T); - + + local c1 : count = 42; + local c2 = copy(c1); + check(c1, c2, c1 == c2, T); + + local a1 = 127.0.0.1; + local a2 = copy(a1); + check(a1, a2, a1 == a2, T); + + local p1 = 42/tcp; + local p2 = copy(p1); + check(p1, p2, p1 == p2, T); + + local sn1 = 127.0.0.1/24; + local sn2 = copy(sn1); + check(sn1, sn2, sn1 == sn2, T); + local s1 = "Foo"; local s2 = copy(s1); check(s1, s2, s1 == s2, F); + + local pat1 = /.*PATTERN.*/; + local pat2 = copy(pat1); + # patterns cannot be directoy compared + if ( same_object(pat1, pat2) ) + print "FAIL P1"; + if ( ! ( pat1 == "PATTERN" ) ) + print "FAIL P2"; + if ( ! ( pat2 == "PATTERN" ) ) + print "FAIL P3"; + if ( pat2 == "PATERN" ) + print "FAIL P4"; + print fmt("orig=%s (%s) clone=%s (%s) same_object=%s", pat1, type_name(pat1), pat2, type_name(pat2), same_object(pat1, pat2)); + + local set1 = [1, 2, 3, 4, 5]; + local set2 = copy(set1); + check(set1, set2, set1 == set2, F); + + local v1 = vector(1, 2, 3, 4, 5); + local v2 = copy(v1); + check(v1, v2, check_vector_equal(v1, v2), F); + + local t1 : table[string] of string = table(); + t1["a"] = "va"; + t1["b"] = "vb"; + local t2 = copy(t1); + check(t1, t2, check_string_table_equal(t1, t2), F); + + local e1 = ENUMME; + local e2 = copy(ENUMME); + check(e1, e2, e1 == e2, T); + + local itr = InnerTestRecord($a="a"); + local otr1 = TestRecord($s1="s1", $s2="s2", $i1=itr, $i2=itr); + local otr2 = copy(otr1); + check(otr1, otr2, compare_otr(otr1, otr2), F); } From 74bb7716f6e4da946e6dfe663d0b89e86ff606e3 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Wed, 22 May 2019 11:36:52 -0700 Subject: [PATCH 3/4] Finish implementation of copy method. All types (besides EntropyVal) now support a native copy operation, which uses primitives of the underlying datatypes to perform a quick copy, without serialization. EntropyVal is the one exception - since that type is rather complex (many members) and will probably not be copied a lot, if at all, it makes sense to just use the serialization function. This will have to be slightly re-written in the near-term-future to use the new serialization function for that opaque type. This change also introduces a new x509_from_der bif, which allows to parse a der into an opaque of x509. This change removes the d2i_X509_ wrapper function; this was a remnant when d2i_X509 took non-const arguments. We directly use d2i_X509 at several places assuming const-ness, so there does not seem to ba a reason to keep the wrapper. This change also exposed a problem in the File cache - cases in which an object was brought back into the cache, and writing occurred in the file_open event were never correctly handeled as far as I can tell. --- src/File.cc | 5 +- src/OpaqueVal.cc | 77 ++++++++++++++++- src/OpaqueVal.h | 15 +++- src/Val.cc | 13 ++- src/file_analysis/analyzer/x509/X509.cc | 9 ++ src/file_analysis/analyzer/x509/X509.h | 9 ++ src/file_analysis/analyzer/x509/functions.bif | 29 ++++--- src/probabilistic/Topk.cc | 7 ++ src/probabilistic/Topk.h | 9 ++ .../one0 | 0 .../one1 | 0 .../one2 | 0 .../two0 | 0 .../two1 | 0 .../two2 | 0 .../language.copy-all-opaques/.stderr | 1 + .../Baseline/language.copy-all-opaques/out | 24 ++++++ .../Baseline/language.copy-all-types/out | 12 +++ ...ization.test => file-caching-cloning.test} | 8 +- testing/btest/language/copy-all-opaques.zeek | 82 +++++++++++++++++++ testing/btest/language/copy-all-types.zeek | 1 + .../base/frameworks/sumstats/topk.zeek | 12 +-- 22 files changed, 280 insertions(+), 33 deletions(-) rename testing/btest/Baseline/{core.file-caching-serialization => core.file-caching-cloning}/one0 (100%) rename testing/btest/Baseline/{core.file-caching-serialization => core.file-caching-cloning}/one1 (100%) rename testing/btest/Baseline/{core.file-caching-serialization => core.file-caching-cloning}/one2 (100%) rename testing/btest/Baseline/{core.file-caching-serialization => core.file-caching-cloning}/two0 (100%) rename testing/btest/Baseline/{core.file-caching-serialization => core.file-caching-cloning}/two1 (100%) rename testing/btest/Baseline/{core.file-caching-serialization => core.file-caching-cloning}/two2 (100%) create mode 100644 testing/btest/Baseline/language.copy-all-opaques/.stderr create mode 100644 testing/btest/Baseline/language.copy-all-opaques/out create mode 100644 testing/btest/Baseline/language.copy-all-types/out rename testing/btest/core/{file-caching-serialization.test => file-caching-cloning.test} (81%) create mode 100644 testing/btest/language/copy-all-opaques.zeek diff --git a/src/File.cc b/src/File.cc index 710693fe0b..57ec463337 100644 --- a/src/File.cc +++ b/src/File.cc @@ -169,7 +169,7 @@ const char* BroFile::Name() const return name; if ( f == stdin ) - return"/dev/stdin"; + return "/dev/stdin"; if ( f == stdout ) return "/dev/stdout"; @@ -316,7 +316,6 @@ FILE* BroFile::BringIntoCache() return 0; } - RaiseOpenEvent(); UpdateFileSize(); if ( fseek(f, position, SEEK_SET) < 0 ) @@ -327,6 +326,8 @@ FILE* BroFile::BringIntoCache() InsertAtBeginning(); + RaiseOpenEvent(); + return f; } diff --git a/src/OpaqueVal.cc b/src/OpaqueVal.cc index ce25ea5475..5b6c9aa483 100644 --- a/src/OpaqueVal.cc +++ b/src/OpaqueVal.cc @@ -87,6 +87,19 @@ MD5Val::~MD5Val() EVP_MD_CTX_free(ctx); } +Val* MD5Val::DoClone(CloneState* state) + { + auto out = new MD5Val(); + if ( IsValid() ) + { + if ( ! out->Init() ) + return nullptr; + EVP_MD_CTX_copy_ex(out->ctx, ctx); + } + + return out; + } + void MD5Val::digest(val_list& vlist, u_char result[MD5_DIGEST_LENGTH]) { EVP_MD_CTX* h = hash_init(Hash_MD5); @@ -218,6 +231,19 @@ SHA1Val::~SHA1Val() EVP_MD_CTX_free(ctx); } +Val* SHA1Val::DoClone(CloneState* state) + { + auto out = new SHA1Val(); + if ( IsValid() ) + { + if ( ! out->Init() ) + return nullptr; + EVP_MD_CTX_copy_ex(out->ctx, ctx); + } + + return out; + } + void SHA1Val::digest(val_list& vlist, u_char result[SHA_DIGEST_LENGTH]) { EVP_MD_CTX* h = hash_init(Hash_SHA1); @@ -340,6 +366,19 @@ SHA256Val::~SHA256Val() EVP_MD_CTX_free(ctx); } +Val* SHA256Val::DoClone(CloneState* state) + { + auto out = new SHA256Val(); + if ( IsValid() ) + { + if ( ! out->Init() ) + return nullptr; + EVP_MD_CTX_copy_ex(out->ctx, ctx); + } + + return out; + } + void SHA256Val::digest(val_list& vlist, u_char result[SHA256_DIGEST_LENGTH]) { EVP_MD_CTX* h = hash_init(Hash_SHA256); @@ -461,6 +500,26 @@ EntropyVal::EntropyVal() : OpaqueVal(entropy_type) { } +Val* EntropyVal::DoClone(CloneState* state) + { + SerializationFormat* form = new BinarySerializationFormat(); + form->StartWrite(); + CloneSerializer ss(form); + SerialInfo sinfo(&ss); + sinfo.cache = false; + sinfo.include_locations = false; + if ( ! this->Serialize(&sinfo) ) + return nullptr; + char* data; + uint32 len = form->EndWrite(&data); + form->StartRead(data, len); + UnserialInfo uinfo(&ss); + uinfo.cache = false; + Val* clone = Unserialize(&uinfo, type); + free(data); + return clone; + } + bool EntropyVal::Feed(const void* data, size_t size) { state.add(data, size); @@ -574,6 +633,18 @@ BloomFilterVal::BloomFilterVal(probabilistic::BloomFilter* bf) bloom_filter = bf; } +Val* BloomFilterVal::DoClone(CloneState* state) + { + if ( bloom_filter ) + { + auto bf = new BloomFilterVal(bloom_filter->Clone()); + bf->Typify(type); + return bf; + } + + return new BloomFilterVal(); + } + bool BloomFilterVal::Typify(BroType* arg_type) { if ( type ) @@ -728,6 +799,11 @@ CardinalityVal::~CardinalityVal() delete hash; } +Val* CardinalityVal::DoClone(CloneState* state) + { + return new CardinalityVal(new probabilistic::CardinalityCounter(*c)); + } + IMPLEMENT_SERIAL(CardinalityVal, SER_CARDINALITY_VAL); bool CardinalityVal::DoSerialize(SerialInfo* info) const @@ -793,4 +869,3 @@ void CardinalityVal::Add(const Val* val) c->AddElement(key->Hash()); delete key; } - diff --git a/src/OpaqueVal.h b/src/OpaqueVal.h index 89c7b2a8d2..5fe0823436 100644 --- a/src/OpaqueVal.h +++ b/src/OpaqueVal.h @@ -47,6 +47,8 @@ public: MD5Val(); ~MD5Val(); + Val* DoClone(CloneState* state) override; + protected: friend class Val; @@ -67,6 +69,8 @@ public: SHA1Val(); ~SHA1Val(); + Val* DoClone(CloneState* state) override; + protected: friend class Val; @@ -87,6 +91,8 @@ public: SHA256Val(); ~SHA256Val(); + Val* DoClone(CloneState* state) override; + protected: friend class Val; @@ -104,6 +110,8 @@ class EntropyVal : public OpaqueVal { public: EntropyVal(); + Val* DoClone(CloneState* state) override; + bool Feed(const void* data, size_t size); bool Get(double *r_ent, double *r_chisq, double *r_mean, double *r_montepicalc, double *r_scc); @@ -122,6 +130,8 @@ public: explicit BloomFilterVal(probabilistic::BloomFilter* bf); ~BloomFilterVal() override; + Val* DoClone(CloneState* state) override; + BroType* Type() const; bool Typify(BroType* type); @@ -149,7 +159,7 @@ private: BroType* type; CompositeHash* hash; probabilistic::BloomFilter* bloom_filter; - }; +}; class CardinalityVal: public OpaqueVal { @@ -157,11 +167,14 @@ public: explicit CardinalityVal(probabilistic::CardinalityCounter*); ~CardinalityVal() override; + Val* DoClone(CloneState* state) override; + void Add(const Val* val); BroType* Type() const; bool Typify(BroType* type); + probabilistic::CardinalityCounter* Get() { return c; }; protected: diff --git a/src/Val.cc b/src/Val.cc index 101ff4b983..ce8e135aec 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -109,9 +109,14 @@ Val* Val::DoClone(CloneState* state) if ( type->Tag() == TYPE_FILE ) { - auto f = AsFile(); - ::Ref(f); - return new Val(f); + // I think we can just ref the file here - it is unclear what else to do. + // In the case of cached files, I think this is equivalent to what happened before + // - serialization + unserialization just have you the same pointer that you already had. + // In the case of non-cached files, the behavior now is different; in the past, serialize + + // unserialize gave you a new file object because the old one was not in the list anymore. This object + // was automatically opened. This does not happen anymore - instead you get the non-cached pointer back + // which is brought back into the cache when written too. + return Ref(); } // Fall-through. @@ -3595,7 +3600,7 @@ OpaqueVal::~OpaqueVal() Val* OpaqueVal::DoClone(CloneState* state) { - // TODO + reporter->InternalError("cloning opaque type without clone implementation"); return nullptr; } diff --git a/src/file_analysis/analyzer/x509/X509.cc b/src/file_analysis/analyzer/x509/X509.cc index 524aae1f27..2db537bfc2 100644 --- a/src/file_analysis/analyzer/x509/X509.cc +++ b/src/file_analysis/analyzer/x509/X509.cc @@ -477,6 +477,15 @@ X509Val::~X509Val() X509_free(certificate); } +Val* X509Val::DoClone(CloneState* state) + { + auto copy = new X509Val(); + if ( certificate ) + copy->certificate = X509_dup(certificate); + + return copy; + } + ::X509* X509Val::GetCertificate() const { return certificate; diff --git a/src/file_analysis/analyzer/x509/X509.h b/src/file_analysis/analyzer/x509/X509.h index a3dc62e533..01ec5bc6ae 100644 --- a/src/file_analysis/analyzer/x509/X509.h +++ b/src/file_analysis/analyzer/x509/X509.h @@ -123,6 +123,15 @@ public: */ explicit X509Val(::X509* certificate); + /** + * Clone an X509Val + * + * @param state certifies the state of the clone operation (duplicate tracking) + * + * @return A cloned X509Val. + */ + Val* DoClone(CloneState* state) override; + /** * Destructor. */ diff --git a/src/file_analysis/analyzer/x509/functions.bif b/src/file_analysis/analyzer/x509/functions.bif index 40d4ec6da8..b2521141c4 100644 --- a/src/file_analysis/analyzer/x509/functions.bif +++ b/src/file_analysis/analyzer/x509/functions.bif @@ -13,20 +13,6 @@ // This is the indexed map of X509 certificate stores. static map x509_stores; -// ### NOTE: while d2i_X509 does not take a const u_char** pointer, -// here we assume d2i_X509 does not write to , so it is safe to -// convert data to a non-const pointer. Could some X509 guru verify -// this? - -X509* d2i_X509_(X509** px, const u_char** in, int len) - { -#ifdef OPENSSL_D2I_X509_USES_CONST_CHAR - return d2i_X509(px, in, len); -#else - return d2i_X509(px, (u_char**)in, len); -#endif - } - // construct an error record RecordVal* x509_result_record(uint64_t num, const char* reason, Val* chainVector = 0) { @@ -56,7 +42,7 @@ X509_STORE* x509_get_root_store(TableVal* root_certs) StringVal *sv = root_certs->Lookup(key)->AsStringVal(); assert(sv); const uint8* data = sv->Bytes(); - X509* x = d2i_X509_(NULL, &data, sv->Len()); + X509* x = d2i_X509(NULL, &data, sv->Len()); if ( ! x ) { builtin_error(fmt("Root CA error: %s", ERR_error_string(ERR_get_error(),NULL))); @@ -203,6 +189,19 @@ function x509_parse%(cert: opaque of x509%): X509::Certificate return file_analysis::X509::ParseCertificate(h); %} +## Constructs an opaque of X509 from a der-formatted string. +## +## Note: this function is mostly meant for testing purposes +## +## .. zeek:see:: x509_certificate x509_extension x509_ext_basic_constraints +## x509_ext_subject_alternative_name x509_verify +## x509_get_certificate_string x509_parse +function x509_from_der%(der: string%): opaque of x509 + %{ + const u_char* data = der->Bytes(); + return new file_analysis::X509Val(d2i_X509(nullptr, &data, der->Len())); + %} + ## Returns the string form of a certificate. ## ## cert: The X509 certificate opaque handle. diff --git a/src/probabilistic/Topk.cc b/src/probabilistic/Topk.cc index d3d3d6a132..d143c5834e 100644 --- a/src/probabilistic/Topk.cc +++ b/src/probabilistic/Topk.cc @@ -183,6 +183,13 @@ void TopkVal::Merge(const TopkVal* value, bool doPrune) } } +Val* TopkVal::DoClone(CloneState* state) + { + auto clone = new TopkVal(size); + clone->Merge(this); + return clone; + } + bool TopkVal::DoSerialize(SerialInfo* info) const { DO_SERIALIZE(SER_TOPK_VAL, OpaqueVal); diff --git a/src/probabilistic/Topk.h b/src/probabilistic/Topk.h index fac677a454..22c7aef1e1 100644 --- a/src/probabilistic/Topk.h +++ b/src/probabilistic/Topk.h @@ -122,6 +122,15 @@ public: */ void Merge(const TopkVal* value, bool doPrune=false); + /** + * Clone the Opaque Type + * + * @param state Clone state (tracking duplicate pointers) + * + * @returns cloned TopkVal + */ + Val* DoClone(CloneState* state) override; + protected: /** * Construct an empty TopkVal. Only used for deserialization diff --git a/testing/btest/Baseline/core.file-caching-serialization/one0 b/testing/btest/Baseline/core.file-caching-cloning/one0 similarity index 100% rename from testing/btest/Baseline/core.file-caching-serialization/one0 rename to testing/btest/Baseline/core.file-caching-cloning/one0 diff --git a/testing/btest/Baseline/core.file-caching-serialization/one1 b/testing/btest/Baseline/core.file-caching-cloning/one1 similarity index 100% rename from testing/btest/Baseline/core.file-caching-serialization/one1 rename to testing/btest/Baseline/core.file-caching-cloning/one1 diff --git a/testing/btest/Baseline/core.file-caching-serialization/one2 b/testing/btest/Baseline/core.file-caching-cloning/one2 similarity index 100% rename from testing/btest/Baseline/core.file-caching-serialization/one2 rename to testing/btest/Baseline/core.file-caching-cloning/one2 diff --git a/testing/btest/Baseline/core.file-caching-serialization/two0 b/testing/btest/Baseline/core.file-caching-cloning/two0 similarity index 100% rename from testing/btest/Baseline/core.file-caching-serialization/two0 rename to testing/btest/Baseline/core.file-caching-cloning/two0 diff --git a/testing/btest/Baseline/core.file-caching-serialization/two1 b/testing/btest/Baseline/core.file-caching-cloning/two1 similarity index 100% rename from testing/btest/Baseline/core.file-caching-serialization/two1 rename to testing/btest/Baseline/core.file-caching-cloning/two1 diff --git a/testing/btest/Baseline/core.file-caching-serialization/two2 b/testing/btest/Baseline/core.file-caching-cloning/two2 similarity index 100% rename from testing/btest/Baseline/core.file-caching-serialization/two2 rename to testing/btest/Baseline/core.file-caching-cloning/two2 diff --git a/testing/btest/Baseline/language.copy-all-opaques/.stderr b/testing/btest/Baseline/language.copy-all-opaques/.stderr new file mode 100644 index 0000000000..bf07a71a21 --- /dev/null +++ b/testing/btest/Baseline/language.copy-all-opaques/.stderr @@ -0,0 +1 @@ +error: incompatible Bloom filter types diff --git a/testing/btest/Baseline/language.copy-all-opaques/out b/testing/btest/Baseline/language.copy-all-opaques/out new file mode 100644 index 0000000000..ad38ca1a8d --- /dev/null +++ b/testing/btest/Baseline/language.copy-all-opaques/out @@ -0,0 +1,24 @@ +============ Topk +[b, a, c] +[b, a, c] +============ HLL +3.000069 +3.000069 +============ Bloom +0 +1 +0 +1 +============ Hashes +5b9164ad6f496d9dee12ec7634ce253f +5b9164ad6f496d9dee12ec7634ce253f +30ae97492ce1da88d0e7117ace0a60a6f9e1e0bc +30ae97492ce1da88d0e7117ace0a60a6f9e1e0bc +25b6746d5172ed6352966a013d93ac846e1110d5a25e8f183b5931f4688842a1 +25b6746d5172ed6352966a013d93ac846e1110d5a25e8f183b5931f4688842a1 +============ X509 +[version=3, serial=040000000001154B5AC394, subject=CN=GlobalSign Root CA,OU=Root CA,O=GlobalSign nv-sa,C=BE, issuer=CN=GlobalSign Root CA,OU=Root CA,O=GlobalSign nv-sa,C=BE, cn=GlobalSign Root CA, not_valid_before=904651200.0, not_valid_after=1832673600.0, key_alg=rsaEncryption, sig_alg=sha1WithRSAEncryption, key_type=rsa, key_length=2048, exponent=65537, curve=] +[version=3, serial=040000000001154B5AC394, subject=CN=GlobalSign Root CA,OU=Root CA,O=GlobalSign nv-sa,C=BE, issuer=CN=GlobalSign Root CA,OU=Root CA,O=GlobalSign nv-sa,C=BE, cn=GlobalSign Root CA, not_valid_before=904651200.0, not_valid_after=1832673600.0, key_alg=rsaEncryption, sig_alg=sha1WithRSAEncryption, key_type=rsa, key_length=2048, exponent=65537, curve=] +============ Entropy +[entropy=4.715374, chi_square=591.981818, mean=75.472727, monte_carlo_pi=4.0, serial_correlation=-0.11027] +[entropy=4.715374, chi_square=591.981818, mean=75.472727, monte_carlo_pi=4.0, serial_correlation=-0.11027] diff --git a/testing/btest/Baseline/language.copy-all-types/out b/testing/btest/Baseline/language.copy-all-types/out new file mode 100644 index 0000000000..230550b90e --- /dev/null +++ b/testing/btest/Baseline/language.copy-all-types/out @@ -0,0 +1,12 @@ +orig=-42 (int) clone=-42 (int) equal=T same_object=T (ok) +orig=42 (count) clone=42 (count) equal=T same_object=T (ok) +orig=127.0.0.1 (addr) clone=127.0.0.1 (addr) equal=T same_object=T (ok) +orig=42/tcp (port) clone=42/tcp (port) equal=T same_object=T (ok) +orig=127.0.0.0/24 (subnet) clone=127.0.0.0/24 (subnet) equal=T same_object=T (ok) +orig=Foo (string) clone=Foo (string) equal=T same_object=F (ok) +orig=/^?(.*PATTERN.*)$?/ (pattern) clone=/^?(.*PATTERN.*)$?/ (pattern) same_object=F +orig=2,4,1,5,3 (set[count]) clone=2,4,1,5,3 (set[count]) equal=T same_object=F (ok) +orig=[1, 2, 3, 4, 5] (vector of count) clone=[1, 2, 3, 4, 5] (vector of count) equal=T same_object=F (ok) +orig=b=vb;a=va (table[string] of string) clone=b=vb;a=va (table[string] of string) equal=T same_object=F (ok) +orig=ENUMME (enum) clone=ENUMME (enum) equal=T same_object=T (ok) +orig=[s1=s1, s2=s2, i1=[a=a], i2=[a=a], donotset=, def=5] (record { s1:string; s2:string; i1:record { a:string; }; i2:record { a:string; }; donotset:record { a:string; }; def:count; }) clone=[s1=s1, s2=s2, i1=[a=a], i2=[a=a], donotset=, def=5] (record { s1:string; s2:string; i1:record { a:string; }; i2:record { a:string; }; donotset:record { a:string; }; def:count; }) equal=T same_object=F (ok) diff --git a/testing/btest/core/file-caching-serialization.test b/testing/btest/core/file-caching-cloning.test similarity index 81% rename from testing/btest/core/file-caching-serialization.test rename to testing/btest/core/file-caching-cloning.test index 6588dc96e4..03d0f5021e 100644 --- a/testing/btest/core/file-caching-serialization.test +++ b/testing/btest/core/file-caching-cloning.test @@ -1,8 +1,8 @@ # This checks that the interactions between open-file caching and -# serialization works ok. In the first case, all files can fit -# in the cache, but get serialized before every write. In the -# second case, files are eventually forced out of the cache and -# undergo serialization, which requires re-opening. +# cloning works ok. In the first case, all files can fit +# in the cache, but get cloned before every write. In the +# second case, files are eventually forced out of the cache; later writing +# requires re-opening. # @TEST-EXEC: zeek -b %INPUT "test_file_prefix=one" # @TEST-EXEC: btest-diff one0 diff --git a/testing/btest/language/copy-all-opaques.zeek b/testing/btest/language/copy-all-opaques.zeek new file mode 100644 index 0000000000..7e5fa0c9ef --- /dev/null +++ b/testing/btest/language/copy-all-opaques.zeek @@ -0,0 +1,82 @@ +# Note: opaque types in separate test +# @TEST-EXEC: zeek -b %INPUT >out +# @TEST-EXEC: btest-diff out +# @TEST-EXEC: btest-diff .stderr + +event zeek_init() + { + print "============ Topk"; + local k1: opaque of topk = topk_init(4); + topk_add(k1, "a"); + topk_add(k1, "b"); + topk_add(k1, "b"); + topk_add(k1, "c"); + local k2 = copy(k1); + print topk_get_top(k1, 5); + topk_add(k1, "shoulnotshowup"); + print topk_get_top(k2, 5); + + + print "============ HLL"; + local c1 = hll_cardinality_init(0.01, 0.95); + hll_cardinality_add(c1, 2001); + hll_cardinality_add(c1, 2002); + hll_cardinality_add(c1, 2003); + + print hll_cardinality_estimate(c1); + local c2 = copy(c1); + hll_cardinality_add(c1, 2004); + print hll_cardinality_estimate(c2); + + print "============ Bloom"; + local bf_cnt = bloomfilter_basic_init(0.1, 1000); + bloomfilter_add(bf_cnt, 42); + bloomfilter_add(bf_cnt, 84); + bloomfilter_add(bf_cnt, 168); + print bloomfilter_lookup(bf_cnt, 0); + print bloomfilter_lookup(bf_cnt, 42); + local bf_copy = copy(bf_cnt); + bloomfilter_add(bf_cnt, 0); + print bloomfilter_lookup(bf_copy, 0); + print bloomfilter_lookup(bf_copy, 42); + # check that typefication transfered. + bloomfilter_add(bf_copy, 0.5); # causes stderr output + + print "============ Hashes"; + local md5a = md5_hash_init(); + md5_hash_update(md5a, "one"); + local md5b = copy(md5a); + md5_hash_update(md5a, "two"); + md5_hash_update(md5b, "two"); + print md5_hash_finish(md5a); + print md5_hash_finish(md5b); + + local sha1a = sha1_hash_init(); + sha1_hash_update(sha1a, "one"); + local sha1b = copy(sha1a); + sha1_hash_update(sha1a, "two"); + sha1_hash_update(sha1b, "two"); + print sha1_hash_finish(sha1a); + print sha1_hash_finish(sha1b); + + local sha256a = sha256_hash_init(); + sha256_hash_update(sha256a, "one"); + local sha256b = copy(sha256a); + sha256_hash_update(sha256a, "two"); + sha256_hash_update(sha256b, "two"); + print sha256_hash_finish(sha256a); + print sha256_hash_finish(sha256b); + + print "============ X509"; + local x509 = x509_from_der("\x30\x82\x03\x75\x30\x82\x02\x5D\xA0\x03\x02\x01\x02\x02\x0B\x04\x00\x00\x00\x00\x01\x15\x4B\x5A\xC3\x94\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x57\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x42\x45\x31\x19\x30\x17\x06\x03\x55\x04\x0A\x13\x10\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x6E\x76\x2D\x73\x61\x31\x10\x30\x0E\x06\x03\x55\x04\x0B\x13\x07\x52\x6F\x6F\x74\x20\x43\x41\x31\x1B\x30\x19\x06\x03\x55\x04\x03\x13\x12\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x1E\x17\x0D\x39\x38\x30\x39\x30\x31\x31\x32\x30\x30\x30\x30\x5A\x17\x0D\x32\x38\x30\x31\x32\x38\x31\x32\x30\x30\x30\x30\x5A\x30\x57\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x42\x45\x31\x19\x30\x17\x06\x03\x55\x04\x0A\x13\x10\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x6E\x76\x2D\x73\x61\x31\x10\x30\x0E\x06\x03\x55\x04\x0B\x13\x07\x52\x6F\x6F\x74\x20\x43\x41\x31\x1B\x30\x19\x06\x03\x55\x04\x03\x13\x12\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xDA\x0E\xE6\x99\x8D\xCE\xA3\xE3\x4F\x8A\x7E\xFB\xF1\x8B\x83\x25\x6B\xEA\x48\x1F\xF1\x2A\xB0\xB9\x95\x11\x04\xBD\xF0\x63\xD1\xE2\x67\x66\xCF\x1C\xDD\xCF\x1B\x48\x2B\xEE\x8D\x89\x8E\x9A\xAF\x29\x80\x65\xAB\xE9\xC7\x2D\x12\xCB\xAB\x1C\x4C\x70\x07\xA1\x3D\x0A\x30\xCD\x15\x8D\x4F\xF8\xDD\xD4\x8C\x50\x15\x1C\xEF\x50\xEE\xC4\x2E\xF7\xFC\xE9\x52\xF2\x91\x7D\xE0\x6D\xD5\x35\x30\x8E\x5E\x43\x73\xF2\x41\xE9\xD5\x6A\xE3\xB2\x89\x3A\x56\x39\x38\x6F\x06\x3C\x88\x69\x5B\x2A\x4D\xC5\xA7\x54\xB8\x6C\x89\xCC\x9B\xF9\x3C\xCA\xE5\xFD\x89\xF5\x12\x3C\x92\x78\x96\xD6\xDC\x74\x6E\x93\x44\x61\xD1\x8D\xC7\x46\xB2\x75\x0E\x86\xE8\x19\x8A\xD5\x6D\x6C\xD5\x78\x16\x95\xA2\xE9\xC8\x0A\x38\xEB\xF2\x24\x13\x4F\x73\x54\x93\x13\x85\x3A\x1B\xBC\x1E\x34\xB5\x8B\x05\x8C\xB9\x77\x8B\xB1\xDB\x1F\x20\x91\xAB\x09\x53\x6E\x90\xCE\x7B\x37\x74\xB9\x70\x47\x91\x22\x51\x63\x16\x79\xAE\xB1\xAE\x41\x26\x08\xC8\x19\x2B\xD1\x46\xAA\x48\xD6\x64\x2A\xD7\x83\x34\xFF\x2C\x2A\xC1\x6C\x19\x43\x4A\x07\x85\xE7\xD3\x7C\xF6\x21\x68\xEF\xEA\xF2\x52\x9F\x7F\x93\x90\xCF\x02\x03\x01\x00\x01\xA3\x42\x30\x40\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x60\x7B\x66\x1A\x45\x0D\x97\xCA\x89\x50\x2F\x7D\x04\xCD\x34\xA8\xFF\xFC\xFD\x4B\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\xD6\x73\xE7\x7C\x4F\x76\xD0\x8D\xBF\xEC\xBA\xA2\xBE\x34\xC5\x28\x32\xB5\x7C\xFC\x6C\x9C\x2C\x2B\xBD\x09\x9E\x53\xBF\x6B\x5E\xAA\x11\x48\xB6\xE5\x08\xA3\xB3\xCA\x3D\x61\x4D\xD3\x46\x09\xB3\x3E\xC3\xA0\xE3\x63\x55\x1B\xF2\xBA\xEF\xAD\x39\xE1\x43\xB9\x38\xA3\xE6\x2F\x8A\x26\x3B\xEF\xA0\x50\x56\xF9\xC6\x0A\xFD\x38\xCD\xC4\x0B\x70\x51\x94\x97\x98\x04\xDF\xC3\x5F\x94\xD5\x15\xC9\x14\x41\x9C\xC4\x5D\x75\x64\x15\x0D\xFF\x55\x30\xEC\x86\x8F\xFF\x0D\xEF\x2C\xB9\x63\x46\xF6\xAA\xFC\xDF\xBC\x69\xFD\x2E\x12\x48\x64\x9A\xE0\x95\xF0\xA6\xEF\x29\x8F\x01\xB1\x15\xB5\x0C\x1D\xA5\xFE\x69\x2C\x69\x24\x78\x1E\xB3\xA7\x1C\x71\x62\xEE\xCA\xC8\x97\xAC\x17\x5D\x8A\xC2\xF8\x47\x86\x6E\x2A\xC4\x56\x31\x95\xD0\x67\x89\x85\x2B\xF9\x6C\xA6\x5D\x46\x9D\x0C\xAA\x82\xE4\x99\x51\xDD\x70\xB7\xDB\x56\x3D\x61\xE4\x6A\xE1\x5C\xD6\xF6\xFE\x3D\xDE\x41\xCC\x07\xAE\x63\x52\xBF\x53\x53\xF4\x2B\xE9\xC7\xFD\xB6\xF7\x82\x5F\x85\xD2\x41\x18\xDB\x81\xB3\x04\x1C\xC5\x1F\xA4\x80\x6F\x15\x20\xC9\xDE\x0C\x88\x0A\x1D\xD6\x66\x55\xE2\xFC\x48\xC9\x29\x26\x69\xE0"); + local x5092 = copy(x509); + print x509_parse(x509); + print x509_parse(x5092); + + print "============ Entropy"; + local handle = entropy_test_init(); + entropy_test_add(handle, "dh3Hie02uh^s#Sdf9L3frd243h$d78r2G4cM6*Q05d(7rh46f!0|4-f"); + local handle2 = copy(handle); + print entropy_test_finish(handle); + print entropy_test_finish(handle2); + } diff --git a/testing/btest/language/copy-all-types.zeek b/testing/btest/language/copy-all-types.zeek index 515f83dce4..10bf182ef5 100644 --- a/testing/btest/language/copy-all-types.zeek +++ b/testing/btest/language/copy-all-types.zeek @@ -1,3 +1,4 @@ +# Note: opaque types in separate test # @TEST-EXEC: zeek -b %INPUT >out # @TEST-EXEC: btest-diff out diff --git a/testing/btest/scripts/base/frameworks/sumstats/topk.zeek b/testing/btest/scripts/base/frameworks/sumstats/topk.zeek index a30d3ce4c8..2375cddd10 100644 --- a/testing/btest/scripts/base/frameworks/sumstats/topk.zeek +++ b/testing/btest/scripts/base/frameworks/sumstats/topk.zeek @@ -3,7 +3,7 @@ event zeek_init() &priority=5 { - local r1: SumStats::Reducer = [$stream="test.metric", + local r1: SumStats::Reducer = [$stream="test.metric", $apply=set(SumStats::TOPK)]; # Merge two empty sets local topk1: opaque of topk = topk_init(4); @@ -18,9 +18,9 @@ event zeek_init() &priority=5 local r = result["test.metric"]; local s: vector of SumStats::Observation; s = topk_get_top(r$topk, 5); - + print fmt("Top entries for key %s", key$str); - for ( element in s ) + for ( element in s ) { print fmt("Num: %d, count: %d, epsilon: %d", s[element]$num, topk_count(r$topk, s[element]), topk_epsilon(r$topk, s[element])); } @@ -32,16 +32,16 @@ event zeek_init() &priority=5 local a: count; a = 0; - for ( i in loop_v ) + for ( i in loop_v ) { a = a + 1; for ( j in loop_v ) { - if ( i < j ) + if ( i < j ) SumStats::observe("test.metric", [$str="counter"], [$num=a]); } } - + SumStats::observe("test.metric", [$str="two"], [$num=1]); SumStats::observe("test.metric", [$str="two"], [$num=1]); From 63e6921895a1a669c92e19140a5ae1ad6e6b38fb Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Wed, 22 May 2019 14:57:55 -0700 Subject: [PATCH 4/4] Add leak-checks for new copy operations --- .../btest/core/leaks/copy-all-opaques.zeek | 83 ++++++++ testing/btest/core/leaks/copy-all-types.zeek | 190 ++++++++++++++++++ testing/btest/language/copy-all-opaques.zeek | 1 - 3 files changed, 273 insertions(+), 1 deletion(-) create mode 100644 testing/btest/core/leaks/copy-all-opaques.zeek create mode 100644 testing/btest/core/leaks/copy-all-types.zeek diff --git a/testing/btest/core/leaks/copy-all-opaques.zeek b/testing/btest/core/leaks/copy-all-opaques.zeek new file mode 100644 index 0000000000..c70238dd07 --- /dev/null +++ b/testing/btest/core/leaks/copy-all-opaques.zeek @@ -0,0 +1,83 @@ +# @TEST-GROUP: leaks +# @TEST-REQUIRES: zeek --help 2>&1 | grep -q mem-leaks + +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run zeek zeek -m -b %INPUT +# @TEST-EXEC: btest-bg-wait 60 + +event zeek_init() + { + print "============ Topk"; + local k1: opaque of topk = topk_init(4); + topk_add(k1, "a"); + topk_add(k1, "b"); + topk_add(k1, "b"); + topk_add(k1, "c"); + local k2 = copy(k1); + print topk_get_top(k1, 5); + topk_add(k1, "shoulnotshowup"); + print topk_get_top(k2, 5); + + + print "============ HLL"; + local c1 = hll_cardinality_init(0.01, 0.95); + hll_cardinality_add(c1, 2001); + hll_cardinality_add(c1, 2002); + hll_cardinality_add(c1, 2003); + + print hll_cardinality_estimate(c1); + local c2 = copy(c1); + hll_cardinality_add(c1, 2004); + print hll_cardinality_estimate(c2); + + print "============ Bloom"; + local bf_cnt = bloomfilter_basic_init(0.1, 1000); + bloomfilter_add(bf_cnt, 42); + bloomfilter_add(bf_cnt, 84); + bloomfilter_add(bf_cnt, 168); + print bloomfilter_lookup(bf_cnt, 0); + print bloomfilter_lookup(bf_cnt, 42); + local bf_copy = copy(bf_cnt); + bloomfilter_add(bf_cnt, 0); + print bloomfilter_lookup(bf_copy, 0); + print bloomfilter_lookup(bf_copy, 42); + # check that typefication transfered. + bloomfilter_add(bf_copy, 0.5); # causes stderr output + + print "============ Hashes"; + local md5a = md5_hash_init(); + md5_hash_update(md5a, "one"); + local md5b = copy(md5a); + md5_hash_update(md5a, "two"); + md5_hash_update(md5b, "two"); + print md5_hash_finish(md5a); + print md5_hash_finish(md5b); + + local sha1a = sha1_hash_init(); + sha1_hash_update(sha1a, "one"); + local sha1b = copy(sha1a); + sha1_hash_update(sha1a, "two"); + sha1_hash_update(sha1b, "two"); + print sha1_hash_finish(sha1a); + print sha1_hash_finish(sha1b); + + local sha256a = sha256_hash_init(); + sha256_hash_update(sha256a, "one"); + local sha256b = copy(sha256a); + sha256_hash_update(sha256a, "two"); + sha256_hash_update(sha256b, "two"); + print sha256_hash_finish(sha256a); + print sha256_hash_finish(sha256b); + + print "============ X509"; + local x509 = x509_from_der("\x30\x82\x03\x75\x30\x82\x02\x5D\xA0\x03\x02\x01\x02\x02\x0B\x04\x00\x00\x00\x00\x01\x15\x4B\x5A\xC3\x94\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x57\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x42\x45\x31\x19\x30\x17\x06\x03\x55\x04\x0A\x13\x10\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x6E\x76\x2D\x73\x61\x31\x10\x30\x0E\x06\x03\x55\x04\x0B\x13\x07\x52\x6F\x6F\x74\x20\x43\x41\x31\x1B\x30\x19\x06\x03\x55\x04\x03\x13\x12\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x1E\x17\x0D\x39\x38\x30\x39\x30\x31\x31\x32\x30\x30\x30\x30\x5A\x17\x0D\x32\x38\x30\x31\x32\x38\x31\x32\x30\x30\x30\x30\x5A\x30\x57\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x42\x45\x31\x19\x30\x17\x06\x03\x55\x04\x0A\x13\x10\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x6E\x76\x2D\x73\x61\x31\x10\x30\x0E\x06\x03\x55\x04\x0B\x13\x07\x52\x6F\x6F\x74\x20\x43\x41\x31\x1B\x30\x19\x06\x03\x55\x04\x03\x13\x12\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xDA\x0E\xE6\x99\x8D\xCE\xA3\xE3\x4F\x8A\x7E\xFB\xF1\x8B\x83\x25\x6B\xEA\x48\x1F\xF1\x2A\xB0\xB9\x95\x11\x04\xBD\xF0\x63\xD1\xE2\x67\x66\xCF\x1C\xDD\xCF\x1B\x48\x2B\xEE\x8D\x89\x8E\x9A\xAF\x29\x80\x65\xAB\xE9\xC7\x2D\x12\xCB\xAB\x1C\x4C\x70\x07\xA1\x3D\x0A\x30\xCD\x15\x8D\x4F\xF8\xDD\xD4\x8C\x50\x15\x1C\xEF\x50\xEE\xC4\x2E\xF7\xFC\xE9\x52\xF2\x91\x7D\xE0\x6D\xD5\x35\x30\x8E\x5E\x43\x73\xF2\x41\xE9\xD5\x6A\xE3\xB2\x89\x3A\x56\x39\x38\x6F\x06\x3C\x88\x69\x5B\x2A\x4D\xC5\xA7\x54\xB8\x6C\x89\xCC\x9B\xF9\x3C\xCA\xE5\xFD\x89\xF5\x12\x3C\x92\x78\x96\xD6\xDC\x74\x6E\x93\x44\x61\xD1\x8D\xC7\x46\xB2\x75\x0E\x86\xE8\x19\x8A\xD5\x6D\x6C\xD5\x78\x16\x95\xA2\xE9\xC8\x0A\x38\xEB\xF2\x24\x13\x4F\x73\x54\x93\x13\x85\x3A\x1B\xBC\x1E\x34\xB5\x8B\x05\x8C\xB9\x77\x8B\xB1\xDB\x1F\x20\x91\xAB\x09\x53\x6E\x90\xCE\x7B\x37\x74\xB9\x70\x47\x91\x22\x51\x63\x16\x79\xAE\xB1\xAE\x41\x26\x08\xC8\x19\x2B\xD1\x46\xAA\x48\xD6\x64\x2A\xD7\x83\x34\xFF\x2C\x2A\xC1\x6C\x19\x43\x4A\x07\x85\xE7\xD3\x7C\xF6\x21\x68\xEF\xEA\xF2\x52\x9F\x7F\x93\x90\xCF\x02\x03\x01\x00\x01\xA3\x42\x30\x40\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x60\x7B\x66\x1A\x45\x0D\x97\xCA\x89\x50\x2F\x7D\x04\xCD\x34\xA8\xFF\xFC\xFD\x4B\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\xD6\x73\xE7\x7C\x4F\x76\xD0\x8D\xBF\xEC\xBA\xA2\xBE\x34\xC5\x28\x32\xB5\x7C\xFC\x6C\x9C\x2C\x2B\xBD\x09\x9E\x53\xBF\x6B\x5E\xAA\x11\x48\xB6\xE5\x08\xA3\xB3\xCA\x3D\x61\x4D\xD3\x46\x09\xB3\x3E\xC3\xA0\xE3\x63\x55\x1B\xF2\xBA\xEF\xAD\x39\xE1\x43\xB9\x38\xA3\xE6\x2F\x8A\x26\x3B\xEF\xA0\x50\x56\xF9\xC6\x0A\xFD\x38\xCD\xC4\x0B\x70\x51\x94\x97\x98\x04\xDF\xC3\x5F\x94\xD5\x15\xC9\x14\x41\x9C\xC4\x5D\x75\x64\x15\x0D\xFF\x55\x30\xEC\x86\x8F\xFF\x0D\xEF\x2C\xB9\x63\x46\xF6\xAA\xFC\xDF\xBC\x69\xFD\x2E\x12\x48\x64\x9A\xE0\x95\xF0\xA6\xEF\x29\x8F\x01\xB1\x15\xB5\x0C\x1D\xA5\xFE\x69\x2C\x69\x24\x78\x1E\xB3\xA7\x1C\x71\x62\xEE\xCA\xC8\x97\xAC\x17\x5D\x8A\xC2\xF8\x47\x86\x6E\x2A\xC4\x56\x31\x95\xD0\x67\x89\x85\x2B\xF9\x6C\xA6\x5D\x46\x9D\x0C\xAA\x82\xE4\x99\x51\xDD\x70\xB7\xDB\x56\x3D\x61\xE4\x6A\xE1\x5C\xD6\xF6\xFE\x3D\xDE\x41\xCC\x07\xAE\x63\x52\xBF\x53\x53\xF4\x2B\xE9\xC7\xFD\xB6\xF7\x82\x5F\x85\xD2\x41\x18\xDB\x81\xB3\x04\x1C\xC5\x1F\xA4\x80\x6F\x15\x20\xC9\xDE\x0C\x88\x0A\x1D\xD6\x66\x55\xE2\xFC\x48\xC9\x29\x26\x69\xE0"); + local x5092 = copy(x509); + print x509_parse(x509); + print x509_parse(x5092); + + print "============ Entropy"; + local handle = entropy_test_init(); + entropy_test_add(handle, "dh3Hie02uh^s#Sdf9L3frd243h$d78r2G4cM6*Q05d(7rh46f!0|4-f"); + local handle2 = copy(handle); + print entropy_test_finish(handle); + print entropy_test_finish(handle2); + } diff --git a/testing/btest/core/leaks/copy-all-types.zeek b/testing/btest/core/leaks/copy-all-types.zeek new file mode 100644 index 0000000000..f83b4ee53e --- /dev/null +++ b/testing/btest/core/leaks/copy-all-types.zeek @@ -0,0 +1,190 @@ +# Note: opaque types in separate test +# @TEST-GROUP: leaks +# @TEST-REQUIRES: zeek --help 2>&1 | grep -q mem-leaks + +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run zeek zeek -m -b %INPUT +# @TEST-EXEC: btest-bg-wait 60 + +type MyEnum: enum { ENUMME }; + +type InnerTestRecord: record { + a: string; +}; + +type TestRecord: record { + s1: string; + s2: string; + i1: InnerTestRecord; + i2: InnerTestRecord &optional; + donotset: InnerTestRecord &optional; + def: count &default=5; +}; + +function join_count_set(ss: set[count], j: string): string + { + local output=""; + local i=0; + for ( s in ss ) + { + if ( i > 0 ) + output = cat(output, j); + + output = cat(output, s); + ++i; + } + return output; + } + +function do_format(i: any): any + { + local tpe = type_name(i); + + switch ( tpe ) + { + case "set[count]": + return join_count_set(i, ","); + case "table[string] of string": + local cast: table[string] of string = i; + local vout: vector of string = vector(); + for ( el in cast ) + { + vout += cat(el, "=", cast[el]); + } + return join_string_vec(vout, ";"); + } + return i; + } + +function check(o1: any, o2: any, equal: bool, expect_same: bool) + { + local expect_msg = (equal ? "ok" : "FAIL0"); + local same = same_object(o1, o2); + + if ( expect_same && ! same ) + expect_msg = "FAIL1"; + + if ( ! expect_same && same ) + expect_msg = "FAIL2"; + + print fmt("orig=%s (%s) clone=%s (%s) equal=%s same_object=%s (%s)", do_format(o1), type_name(o1), do_format(o2), type_name(o2), equal, same, expect_msg); + } + +function check_vector_equal(a: vector of count, b: vector of count): bool + { + if ( |a| != |b| ) + return F; + + for ( i in a ) + { + if ( a[i] != b[i] ) + return F; + } + + return T; + } + +function check_string_table_equal(a: table[string] of string, b: table[string] of string): bool + { + if ( |a| != |b| ) + return F; + + for ( i in a ) + { + if ( a[i] != b[i] ) + return F; + } + + return T; + } + +function compare_otr(a: TestRecord, b: TestRecord): bool + { + if ( a$s1 != b$s1 ) + return F; + if ( a$s2 != b$s2 ) + return F; + if ( a$i1$a != b$i1$a ) + return F; + if ( a$i2$a != b$i2$a ) + return F; + + if ( same_object(a$i1, b$i1) ) + return F; + if ( same_object(a$i2, b$i2) ) + return F; + + # check that we restroe that i1 & i2 point to same object + if ( ! same_object(a$i1, a$i2) ) + return F; + if ( ! same_object(b$i1, b$i2) ) + return F; + + if ( a$def != b$def ) + return F; + + return T; + } + + +event zeek_init() + { + local i1 = -42; + local i2 = copy(i1); + check(i1, i2, i1 == i2, T); + + local c1 : count = 42; + local c2 = copy(c1); + check(c1, c2, c1 == c2, T); + + local a1 = 127.0.0.1; + local a2 = copy(a1); + check(a1, a2, a1 == a2, T); + + local p1 = 42/tcp; + local p2 = copy(p1); + check(p1, p2, p1 == p2, T); + + local sn1 = 127.0.0.1/24; + local sn2 = copy(sn1); + check(sn1, sn2, sn1 == sn2, T); + + local s1 = "Foo"; + local s2 = copy(s1); + check(s1, s2, s1 == s2, F); + + local pat1 = /.*PATTERN.*/; + local pat2 = copy(pat1); + # patterns cannot be directoy compared + if ( same_object(pat1, pat2) ) + print "FAIL P1"; + if ( ! ( pat1 == "PATTERN" ) ) + print "FAIL P2"; + if ( ! ( pat2 == "PATTERN" ) ) + print "FAIL P3"; + if ( pat2 == "PATERN" ) + print "FAIL P4"; + print fmt("orig=%s (%s) clone=%s (%s) same_object=%s", pat1, type_name(pat1), pat2, type_name(pat2), same_object(pat1, pat2)); + + local set1 = [1, 2, 3, 4, 5]; + local set2 = copy(set1); + check(set1, set2, set1 == set2, F); + + local v1 = vector(1, 2, 3, 4, 5); + local v2 = copy(v1); + check(v1, v2, check_vector_equal(v1, v2), F); + + local t1 : table[string] of string = table(); + t1["a"] = "va"; + t1["b"] = "vb"; + local t2 = copy(t1); + check(t1, t2, check_string_table_equal(t1, t2), F); + + local e1 = ENUMME; + local e2 = copy(ENUMME); + check(e1, e2, e1 == e2, T); + + local itr = InnerTestRecord($a="a"); + local otr1 = TestRecord($s1="s1", $s2="s2", $i1=itr, $i2=itr); + local otr2 = copy(otr1); + check(otr1, otr2, compare_otr(otr1, otr2), F); + } diff --git a/testing/btest/language/copy-all-opaques.zeek b/testing/btest/language/copy-all-opaques.zeek index 7e5fa0c9ef..25ca89fd80 100644 --- a/testing/btest/language/copy-all-opaques.zeek +++ b/testing/btest/language/copy-all-opaques.zeek @@ -1,4 +1,3 @@ -# Note: opaque types in separate test # @TEST-EXEC: zeek -b %INPUT >out # @TEST-EXEC: btest-diff out # @TEST-EXEC: btest-diff .stderr