Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-10 02:28:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge remote-tracking branch 'origin/topic/johanna/clone'

Browse source 
* origin/topic/johanna/clone:
  Add leak-checks for new copy operations
  Finish implementation of copy method.
  Checkpoint - all non-opaque-types can be cloned.
  Reimplement copy().

Fixed a couple leaks during merge.
This commit is contained in:
Jon Siwek 2019-05-22 23:56:23 -07:00
commit 0f2e778216
28 changed files with 1005 additions and 58 deletions
Download patch file Download diff file Expand all files Collapse all files

5
src/File.cc
View file

@ -115,7 +115,7 @@ const char* BroFile::Name() const
return name;
if ( f == stdin )
return"/dev/stdin";
return "/dev/stdin";
if ( f == stdout )
return "/dev/stdout";
@ -243,8 +243,6 @@ FILE* BroFile::BringIntoCache()
return 0;
}
RaiseOpenEvent();
if ( fseek(f, position, SEEK_SET) < 0 )
{
bro_strerror_r(errno, buf, sizeof(buf));
@ -252,6 +250,7 @@ FILE* BroFile::BringIntoCache()
}
InsertAtBeginning();
RaiseOpenEvent();
return f;
}

77
src/OpaqueVal.cc
View file

@ -87,6 +87,19 @@ MD5Val::~MD5Val()
EVP_MD_CTX_free(ctx);
}
Val* MD5Val::DoClone(CloneState* state)
{
auto out = new MD5Val();
if ( IsValid() )
{
if ( ! out->Init() )
return nullptr;
EVP_MD_CTX_copy_ex(out->ctx, ctx);
}
return out;
}
void MD5Val::digest(val_list& vlist, u_char result[MD5_DIGEST_LENGTH])
{
EVP_MD_CTX* h = hash_init(Hash_MD5);
@ -218,6 +231,19 @@ SHA1Val::~SHA1Val()
EVP_MD_CTX_free(ctx);
}
Val* SHA1Val::DoClone(CloneState* state)
{
auto out = new SHA1Val();
if ( IsValid() )
{
if ( ! out->Init() )
return nullptr;
EVP_MD_CTX_copy_ex(out->ctx, ctx);
}
return out;
}
void SHA1Val::digest(val_list& vlist, u_char result[SHA_DIGEST_LENGTH])
{
EVP_MD_CTX* h = hash_init(Hash_SHA1);
@ -340,6 +366,19 @@ SHA256Val::~SHA256Val()
EVP_MD_CTX_free(ctx);
}
Val* SHA256Val::DoClone(CloneState* state)
{
auto out = new SHA256Val();
if ( IsValid() )
{
if ( ! out->Init() )
return nullptr;
EVP_MD_CTX_copy_ex(out->ctx, ctx);
}
return out;
}
void SHA256Val::digest(val_list& vlist, u_char result[SHA256_DIGEST_LENGTH])
{
EVP_MD_CTX* h = hash_init(Hash_SHA256);
@ -461,6 +500,26 @@ EntropyVal::EntropyVal() : OpaqueVal(entropy_type)
{
}
Val* EntropyVal::DoClone(CloneState* state)
{
SerializationFormat* form = new BinarySerializationFormat();
form->StartWrite();
CloneSerializer ss(form);
SerialInfo sinfo(&ss);
sinfo.cache = false;
sinfo.include_locations = false;
if ( ! this->Serialize(&sinfo) )
return nullptr;
char* data;
uint32 len = form->EndWrite(&data);
form->StartRead(data, len);
UnserialInfo uinfo(&ss);
uinfo.cache = false;
Val* clone = Unserialize(&uinfo, type);
free(data);
return clone;
}
bool EntropyVal::Feed(const void* data, size_t size)
{
state.add(data, size);
@ -574,6 +633,18 @@ BloomFilterVal::BloomFilterVal(probabilistic::BloomFilter* bf)
bloom_filter = bf;
}
Val* BloomFilterVal::DoClone(CloneState* state)
{
if ( bloom_filter )
{
auto bf = new BloomFilterVal(bloom_filter->Clone());
bf->Typify(type);
return bf;
}
return new BloomFilterVal();
}
bool BloomFilterVal::Typify(BroType* arg_type)
{
if ( type )
@ -728,6 +799,11 @@ CardinalityVal::~CardinalityVal()
delete hash;
}
Val* CardinalityVal::DoClone(CloneState* state)
{
return new CardinalityVal(new probabilistic::CardinalityCounter(*c));
}
IMPLEMENT_SERIAL(CardinalityVal, SER_CARDINALITY_VAL);
bool CardinalityVal::DoSerialize(SerialInfo* info) const
@ -793,4 +869,3 @@ void CardinalityVal::Add(const Val* val)
c->AddElement(key->Hash());
delete key;
}

15
src/OpaqueVal.h
View file

@ -47,6 +47,8 @@ public:
MD5Val();
~MD5Val();
Val* DoClone(CloneState* state) override;
protected:
friend class Val;
@ -67,6 +69,8 @@ public:
SHA1Val();
~SHA1Val();
Val* DoClone(CloneState* state) override;
protected:
friend class Val;
@ -87,6 +91,8 @@ public:
SHA256Val();
~SHA256Val();
Val* DoClone(CloneState* state) override;
protected:
friend class Val;
@ -104,6 +110,8 @@ class EntropyVal : public OpaqueVal {
public:
EntropyVal();
Val* DoClone(CloneState* state) override;
bool Feed(const void* data, size_t size);
bool Get(double *r_ent, double *r_chisq, double *r_mean,
double *r_montepicalc, double *r_scc);
@ -122,6 +130,8 @@ public:
explicit BloomFilterVal(probabilistic::BloomFilter* bf);
~BloomFilterVal() override;
Val* DoClone(CloneState* state) override;
BroType* Type() const;
bool Typify(BroType* type);
@ -149,7 +159,7 @@ private:
BroType* type;
CompositeHash* hash;
probabilistic::BloomFilter* bloom_filter;
};
};
class CardinalityVal: public OpaqueVal {
@ -157,11 +167,14 @@ public:
explicit CardinalityVal(probabilistic::CardinalityCounter*);
~CardinalityVal() override;
Val* DoClone(CloneState* state) override;
void Add(const Val* val);
BroType* Type() const;
bool Typify(BroType* type);
probabilistic::CardinalityCounter* Get() { return c; };
protected:

225
src/Val.cc
View file

@ -71,31 +71,67 @@ Val::~Val()
#endif
}
Val* Val::Clone() const
Val* Val::Clone()
{
SerializationFormat* form = new BinarySerializationFormat();
form->StartWrite();
CloneSerializer ss(form);
SerialInfo sinfo(&ss);
sinfo.cache = false;
sinfo.include_locations = false;
if ( ! this->Serialize(&sinfo) )
return 0;
char* data;
uint32 len = form->EndWrite(&data);
form->StartRead(data, len);
UnserialInfo uinfo(&ss);
uinfo.cache = false;
Val* clone = Unserialize(&uinfo, type);
free(data);
return clone;
Val::CloneState state;
return Clone(&state);
}
Val* Val::Clone(CloneState* state)
{
auto i = state->clones.find(this);
if ( i != state->clones.end() )
return i->second->Ref();
auto c = DoClone(state);
assert(c);
state->clones.insert(std::make_pair(this, c));
return c;
}
Val* Val::DoClone(CloneState* state)
{
switch ( type->InternalType() ) {
case TYPE_INTERNAL_INT:
case TYPE_INTERNAL_UNSIGNED:
case TYPE_INTERNAL_DOUBLE:
// Immutable.
return Ref();
case TYPE_INTERNAL_OTHER:
// Derived classes are responsible for this. Exception:
// Functions and files. There aren't any derived classes.
if ( type->Tag() == TYPE_FUNC )
// Immutable.
return Ref();
if ( type->Tag() == TYPE_FILE )
{
// I think we can just ref the file here - it is unclear what else
// to do. In the case of cached files, I think this is equivalent
// to what happened before - serialization + unserialization just
// have you the same pointer that you already had. In the case of
// non-cached files, the behavior now is different; in the past,
// serialize + unserialize gave you a new file object because the
// old one was not in the list anymore. This object was
// automatically opened. This does not happen anymore - instead you
// get the non-cached pointer back which is brought back into the
// cache when written too.
return Ref();
}
// Fall-through.
default:
reporter->InternalError("cloning illegal base type");
}
reporter->InternalError("cannot be reached");
return nullptr;
}
bool Val::Serialize(SerialInfo* info) const
{
return SerialObj::Serialize(info);
@ -861,6 +897,12 @@ void PortVal::ValDescribe(ODesc* d) const
d->Add("/unknown");
}
Val* PortVal::DoClone(CloneState* state)
{
// Immutable.
return Ref();
}
IMPLEMENT_SERIAL(PortVal, SER_PORT_VAL);
bool PortVal::DoSerialize(SerialInfo* info) const
@ -919,6 +961,12 @@ Val* AddrVal::SizeVal() const
return val_mgr->GetCount(128);
}
Val* AddrVal::DoClone(CloneState* state)
{
// Immutable.
return Ref();
}
IMPLEMENT_SERIAL(AddrVal, SER_ADDR_VAL);
bool AddrVal::DoSerialize(SerialInfo* info) const
@ -1043,6 +1091,12 @@ bool SubNetVal::Contains(const IPAddr& addr) const
return val.subnet_val->Contains(a);
}
Val* SubNetVal::DoClone(CloneState* state)
{
// Immutable.
return Ref();
}
IMPLEMENT_SERIAL(SubNetVal, SER_SUBNET_VAL);
bool SubNetVal::DoSerialize(SerialInfo* info) const
@ -1099,6 +1153,12 @@ unsigned int StringVal::MemoryAllocation() const
return padded_sizeof(*this) + val.string_val->MemoryAllocation();
}
Val* StringVal::DoClone(CloneState* state)
{
return new StringVal(new BroString((u_char*) val.string_val->Bytes(),
val.string_val->Len(), 1));
}
IMPLEMENT_SERIAL(StringVal, SER_STRING_VAL);
bool StringVal::DoSerialize(SerialInfo* info) const
@ -1161,6 +1221,14 @@ unsigned int PatternVal::MemoryAllocation() const
return padded_sizeof(*this) + val.re_val->MemoryAllocation();
}
Val* PatternVal::DoClone(CloneState* state)
{
auto re = new RE_Matcher(val.re_val->PatternText(),
val.re_val->AnywherePatternText());
re->Compile();
return new PatternVal(re);
}
IMPLEMENT_SERIAL(PatternVal, SER_PATTERN_VAL);
bool PatternVal::DoSerialize(SerialInfo* info) const
@ -1259,6 +1327,17 @@ void ListVal::Describe(ODesc* d) const
}
}
Val* ListVal::DoClone(CloneState* state)
{
auto lv = new ListVal(tag);
lv->vals.resize(vals.length());
loop_over_list(vals, i)
lv->Append(vals[i]->Clone(state));
return lv;
}
IMPLEMENT_SERIAL(ListVal, SER_LIST_VAL);
bool ListVal::DoSerialize(SerialInfo* info) const
@ -2459,6 +2538,55 @@ void TableVal::ReadOperation(Val* index, TableEntryVal* v)
}
}
Val* TableVal::DoClone(CloneState* state)
{
auto tv = new TableVal(table_type);
const PDict(TableEntryVal)* tbl = AsTable();
IterCookie* cookie = tbl->InitForIteration();
HashKey* key;
TableEntryVal* val;
while ( (val = tbl->NextEntry(key, cookie)) )
{
TableEntryVal* nval = val->Clone();
tv->AsNonConstTable()->Insert(key, nval);
if ( subnets )
{
Val* idx = RecoverIndex(key);
tv->subnets->Insert(idx, nval);
Unref(idx);
}
delete key;
}
if ( attrs )
{
::Ref(attrs);
tv->attrs = attrs;
}
if ( expire_time )
{
tv->expire_time = expire_time->Ref();
// As network_time is not necessarily initialized yet, we set
// a timer which fires immediately.
timer = new TableValTimer(this, 1);
timer_mgr->Add(timer);
}
if ( expire_func )
tv->expire_func = expire_func->Ref();
if ( def_val )
tv->def_val = def_val->Ref();
return tv;
}
IMPLEMENT_SERIAL(TableVal, SER_TABLE_VAL);
// This is getting rather complex due to the ability to suspend even within
@ -2752,13 +2880,16 @@ unsigned int TableVal::MemoryAllocation() const
vector<RecordVal*> RecordVal::parse_time_records;
RecordVal::RecordVal(RecordType* t) : MutableVal(t)
RecordVal::RecordVal(RecordType* t, bool init_fields) : MutableVal(t)
{
origin = 0;
record_type = t;
int n = record_type->NumFields();
val_list* vl = val.val_list_val = new val_list(n);
if ( ! init_fields )
return;
// Initialize to default values from RecordType (which are nil
// by default).
for ( int i = 0; i < n; ++i )
@ -2992,7 +3123,7 @@ void RecordVal::Describe(ODesc* d) const
void RecordVal::DescribeReST(ODesc* d) const
{
const val_list* vl = AsRecord();
int n = vl->length();
int n = vl->length();
d->Add("{");
d->PushIndent();
@ -3017,6 +3148,25 @@ void RecordVal::DescribeReST(ODesc* d) const
d->Add("}");
}
Val* RecordVal::DoClone(CloneState* state)
{
// We set origin to 0 here. Origin only seems to be used for exactly one
// purpose - to find the connection record that is associated with a
// record. As we cannot guarantee that it will ber zeroed out at the
// approproate time (as it seems to be guaranteed for the original record)
// we don't touch it.
auto rv = new RecordVal(record_type, false);
rv->origin = nullptr;
loop_over_list(*val.val_list_val, i)
{
Val* v = (*val.val_list_val)[i] ? (*val.val_list_val)[i]->Clone(state) : nullptr;
rv->val.val_list_val->append(v);
}
return rv;
}
IMPLEMENT_SERIAL(RecordVal, SER_RECORD_VAL);
bool RecordVal::DoSerialize(SerialInfo* info) const
@ -3133,6 +3283,12 @@ void EnumVal::ValDescribe(ODesc* d) const
d->Add(ename);
}
Val* EnumVal::DoClone(CloneState* state)
{
// Immutable.
return Ref();
}
IMPLEMENT_SERIAL(EnumVal, SER_ENUM_VAL);
bool EnumVal::DoSerialize(SerialInfo* info) const
@ -3294,6 +3450,21 @@ bool VectorVal::RemoveProperties(Properties arg_props)
return true;
}
Val* VectorVal::DoClone(CloneState* state)
{
auto vv = new VectorVal(vector_type);
vv->val.vector_val->reserve(val.vector_val->size());
for ( unsigned int i = 0; i < val.vector_val->size(); ++i )
{
auto v = (*val.vector_val)[i]->Clone(state);
vv->val.vector_val->push_back(v);
}
return vv;
}
IMPLEMENT_SERIAL(VectorVal, SER_VECTOR_VAL);
bool VectorVal::DoSerialize(SerialInfo* info) const
@ -3366,6 +3537,12 @@ OpaqueVal::~OpaqueVal()
{
}
Val* OpaqueVal::DoClone(CloneState* state)
{
reporter->InternalError("cloning opaque type without clone implementation");
return nullptr;
}
IMPLEMENT_SERIAL(OpaqueVal, SER_OPAQUE_VAL);
bool OpaqueVal::DoSerialize(SerialInfo* info) const

41
src/Val.h
View file

@ -172,7 +172,7 @@ public:
~Val() override;
Val* Ref() { ::Ref(this); return this; }
virtual Val* Clone() const;
Val* Clone();
int IsZero() const;
int IsOne() const;
@ -370,6 +370,9 @@ public:
protected:
friend class EnumType;
friend class ListVal;
friend class RecordVal;
friend class VectorVal;
friend class ValManager;
virtual void ValDescribe(ODesc* d) const;
@ -419,6 +422,14 @@ protected:
static Val* Unserialize(UnserialInfo* info, TypeTag type,
const BroType* exact_type);
// For internal use by the Val::Clone() methods.
struct CloneState {
std::unordered_map<const Val*, Val*> clones;
};
Val* Clone(CloneState* state);
virtual Val* DoClone(CloneState* state);
BroValUnion val;
BroType* type;
@ -636,6 +647,7 @@ protected:
PortVal(uint32 p, bool unused);
void ValDescribe(ODesc* d) const override;
Val* DoClone(CloneState* state) override;
DECLARE_SERIAL(PortVal);
};
@ -661,6 +673,8 @@ protected:
explicit AddrVal(TypeTag t) : Val(t) { }
explicit AddrVal(BroType* t) : Val(t) { }
Val* DoClone(CloneState* state) override;
DECLARE_SERIAL(AddrVal);
};
@ -689,6 +703,7 @@ protected:
SubNetVal() {}
void ValDescribe(ODesc* d) const override;
Val* DoClone(CloneState* state) override;
DECLARE_SERIAL(SubNetVal);
};
@ -721,6 +736,7 @@ protected:
StringVal() {}
void ValDescribe(ODesc* d) const override;
Val* DoClone(CloneState* state) override;
DECLARE_SERIAL(StringVal);
};
@ -741,6 +757,7 @@ protected:
PatternVal() {}
void ValDescribe(ODesc* d) const override;
Val* DoClone(CloneState* state) override;
DECLARE_SERIAL(PatternVal);
};
@ -786,6 +803,8 @@ protected:
friend class Val;
ListVal() {}
Val* DoClone(CloneState* state) override;
DECLARE_SERIAL(ListVal);
val_list vals;
@ -803,6 +822,16 @@ public:
expire_access_time = last_read_update =
int(network_time - bro_start_network_time);
}
TableEntryVal* Clone()
{
auto rval = new TableEntryVal(val ? val->Clone() : nullptr);
rval->last_access_time = last_access_time;
rval->expire_access_time = expire_access_time;
rval->last_read_update = last_read_update;
return rval;
}
~TableEntryVal() { }
Val* Value() { return val; }
@ -994,6 +1023,8 @@ protected:
// Propagates a read operation if necessary.
void ReadOperation(Val* index, TableEntryVal *v);
Val* DoClone(CloneState* state) override;
DECLARE_SERIAL(TableVal);
TableType* table_type;
@ -1009,7 +1040,7 @@ protected:
class RecordVal : public MutableVal {
public:
explicit RecordVal(RecordType* t);
explicit RecordVal(RecordType* t, bool init_fields = true);
~RecordVal() override;
Val* SizeVal() const override
@ -1066,6 +1097,8 @@ protected:
bool AddProperties(Properties arg_state) override;
bool RemoveProperties(Properties arg_state) override;
Val* DoClone(CloneState* state) override;
DECLARE_SERIAL(RecordVal);
RecordType* record_type;
@ -1097,6 +1130,7 @@ protected:
EnumVal() {}
void ValDescribe(ODesc* d) const override;
Val* DoClone(CloneState* state) override;
DECLARE_SERIAL(EnumVal);
};
@ -1157,6 +1191,7 @@ protected:
bool AddProperties(Properties arg_state) override;
bool RemoveProperties(Properties arg_state) override;
void ValDescribe(ODesc* d) const override;
Val* DoClone(CloneState* state) override;
DECLARE_SERIAL(VectorVal);
@ -1175,6 +1210,8 @@ protected:
friend class Val;
OpaqueVal() { }
Val* DoClone(CloneState* state) override;
DECLARE_SERIAL(OpaqueVal);
};

9
src/file_analysis/analyzer/x509/X509.cc
View file

@ -477,6 +477,15 @@ X509Val::~X509Val()
X509_free(certificate);
}
Val* X509Val::DoClone(CloneState* state)
{
auto copy = new X509Val();
if ( certificate )
copy->certificate = X509_dup(certificate);
return copy;
}
::X509* X509Val::GetCertificate() const
{
return certificate;

9
src/file_analysis/analyzer/x509/X509.h
View file

@ -123,6 +123,15 @@ public:
*/
explicit X509Val(::X509* certificate);
/**
* Clone an X509Val
*
* @param state certifies the state of the clone operation (duplicate tracking)
*
* @return A cloned X509Val.
*/
Val* DoClone(CloneState* state) override;
/**
* Destructor.
*/

29
src/file_analysis/analyzer/x509/functions.bif
View file

@ -13,20 +13,6 @@
// This is the indexed map of X509 certificate stores.
static map<Val*, X509_STORE*> x509_stores;
// ### NOTE: while d2i_X509 does not take a const u_char** pointer,
// here we assume d2i_X509 does not write to <data>, so it is safe to
// convert data to a non-const pointer. Could some X509 guru verify
// this?
X509* d2i_X509_(X509** px, const u_char** in, int len)
{
#ifdef OPENSSL_D2I_X509_USES_CONST_CHAR
return d2i_X509(px, in, len);
#else
return d2i_X509(px, (u_char**)in, len);
#endif
}
// construct an error record
RecordVal* x509_result_record(uint64_t num, const char* reason, Val* chainVector = 0)
{
@ -56,7 +42,7 @@ X509_STORE* x509_get_root_store(TableVal* root_certs)
StringVal *sv = root_certs->Lookup(key)->AsStringVal();
assert(sv);
const uint8* data = sv->Bytes();
X509* x = d2i_X509_(NULL, &data, sv->Len());
X509* x = d2i_X509(NULL, &data, sv->Len());
if ( ! x )
{
builtin_error(fmt("Root CA error: %s", ERR_error_string(ERR_get_error(),NULL)));
@ -203,6 +189,19 @@ function x509_parse%(cert: opaque of x509%): X509::Certificate
return file_analysis::X509::ParseCertificate(h);
%}
## Constructs an opaque of X509 from a der-formatted string.
##
## Note: this function is mostly meant for testing purposes
##
## .. zeek:see:: x509_certificate x509_extension x509_ext_basic_constraints
## x509_ext_subject_alternative_name x509_verify
## x509_get_certificate_string x509_parse
function x509_from_der%(der: string%): opaque of x509
%{
const u_char* data = der->Bytes();
return new file_analysis::X509Val(d2i_X509(nullptr, &data, der->Len()));
%}
## Returns the string form of a certificate.
##
## cert: The X509 certificate opaque handle.

7
src/probabilistic/Topk.cc
View file

@ -183,6 +183,13 @@ void TopkVal::Merge(const TopkVal* value, bool doPrune)
}
}
Val* TopkVal::DoClone(CloneState* state)
{
auto clone = new TopkVal(size);
clone->Merge(this);
return clone;
}
bool TopkVal::DoSerialize(SerialInfo* info) const
{
DO_SERIALIZE(SER_TOPK_VAL, OpaqueVal);

9
src/probabilistic/Topk.h
View file

@ -122,6 +122,15 @@ public:
*/
void Merge(const TopkVal* value, bool doPrune=false);
/**
* Clone the Opaque Type
*
* @param state Clone state (tracking duplicate pointers)
*
* @returns cloned TopkVal
*/
Val* DoClone(CloneState* state) override;
protected:
/**
* Construct an empty TopkVal. Only used for deserialization