Merge remote-tracking branch 'origin/master' into topic/johanna/remove-serializer

CHANGES

@@ -1,4 +1,39 @@
 
+2.6-325 | 2019-05-22 23:56:23 -0700
+
+  * Add leak-checks for new copy operations (Johanna Amann, Corelight)
+
+  * Finish implementation of new copy method. (Johanna Amann, Corelight)
+
+    All types (besides EntropyVal) now support a native copy operation,
+    which uses primitives of the underlying datatypes to perform a quick
+    copy, without serialization.
+
+    EntropyVal is the one exception - since that type is rather complex
+    (many members) and will probably not be copied a lot, if at all, it
+    makes sense to just use the serialization function.
+
+    This will have to be slightly re-written in the near-term-future to use
+    the new serialization function for that opaque type.
+
+    This change also introduces a new x509_from_der bif, which allows to
+    parse a der into an opaque of x509.
+
+    This change removes the d2i_X509_ wrapper function; this was a remnant
+    when d2i_X509 took non-const arguments. We directly use d2i_X509 at
+    several places assuming const-ness, so there does not seem to ba a
+    reason to keep the wrapper.
+
+    This change also exposed a problem in the File cache - cases in which an
+    object was brought back into the cache, and writing occurred in the
+    file_open event were never correctly handeled as far as I can tell.
+
+  * Reimplement copy(). (Robin Sommer, Corelight)
+
+    The old implementation used the serialization framework, which is
+    going away. This is a new standalone implementation that should also
+    be quite a bit faster.
+
 2.6-318 | 2019-05-21 09:17:53 -0700
 
   * Remove state_dir and state_write_delay options (Jon Siwek, Corelight)

VERSION

@@ -1 +1 @@
-2.6-318
+2.6-325

doc

@@ -1 +1 @@
-Subproject commit 8c3395b7e3b1bdbe03bbdd3e290f98414a307651
+Subproject commit e5422eafff850708f4d4ff590e54299ddc97ca42

src/File.cc

@@ -114,7 +114,7 @@ const char* BroFile::Name() const
 		return name;
 
 	if ( f == stdin )
-		return"/dev/stdin";
+		return "/dev/stdin";
 
 	if ( f == stdout )
 		return "/dev/stdout";
@@ -242,8 +242,6 @@ FILE* BroFile::BringIntoCache()
 		return 0;
 		}
 
-	RaiseOpenEvent();
-
 	if ( fseek(f, position, SEEK_SET) < 0 )
 		{
 		bro_strerror_r(errno, buf, sizeof(buf));
@@ -251,6 +249,7 @@ FILE* BroFile::BringIntoCache()
 		}
 
 	InsertAtBeginning();
+	RaiseOpenEvent();
 
 	return f;
 	}

src/OpaqueVal.cc

@@ -72,6 +72,19 @@ MD5Val::~MD5Val()
 		EVP_MD_CTX_free(ctx);
 	}
 
+Val* MD5Val::DoClone(CloneState* state)
+	{
+	auto out = new MD5Val();
+	if ( IsValid() )
+		{
+		if ( ! out->Init() )
+			return nullptr;
+		EVP_MD_CTX_copy_ex(out->ctx, ctx);
+		}
+
+	return out;
+	}
+
 void MD5Val::digest(val_list& vlist, u_char result[MD5_DIGEST_LENGTH])
 	{
 	EVP_MD_CTX* h = hash_init(Hash_MD5);
@@ -142,6 +155,19 @@ SHA1Val::~SHA1Val()
 		EVP_MD_CTX_free(ctx);
 	}
 
+Val* SHA1Val::DoClone(CloneState* state)
+	{
+	auto out = new SHA1Val();
+	if ( IsValid() )
+		{
+		if ( ! out->Init() )
+			return nullptr;
+		EVP_MD_CTX_copy_ex(out->ctx, ctx);
+		}
+
+	return out;
+	}
+
 void SHA1Val::digest(val_list& vlist, u_char result[SHA_DIGEST_LENGTH])
 	{
 	EVP_MD_CTX* h = hash_init(Hash_SHA1);
@@ -201,6 +227,19 @@ SHA256Val::~SHA256Val()
 		EVP_MD_CTX_free(ctx);
 	}
 
+Val* SHA256Val::DoClone(CloneState* state)
+	{
+	auto out = new SHA256Val();
+	if ( IsValid() )
+		{
+		if ( ! out->Init() )
+			return nullptr;
+		EVP_MD_CTX_copy_ex(out->ctx, ctx);
+		}
+
+	return out;
+	}
+
 void SHA256Val::digest(val_list& vlist, u_char result[SHA256_DIGEST_LENGTH])
 	{
 	EVP_MD_CTX* h = hash_init(Hash_SHA256);
@@ -254,6 +293,12 @@ EntropyVal::EntropyVal() : OpaqueVal(entropy_type)
 	{
 	}
 
+Val* EntropyVal::DoClone(CloneState* state)
+	{
+	// Fixme
+	return nullptr;
+	}
+
 bool EntropyVal::Feed(const void* data, size_t size)
 	{
 	state.add(data, size);
@@ -291,6 +336,18 @@ BloomFilterVal::BloomFilterVal(probabilistic::BloomFilter* bf)
 	bloom_filter = bf;
 	}
 
+Val* BloomFilterVal::DoClone(CloneState* state)
+	{
+	if ( bloom_filter )
+		{
+		auto bf = new BloomFilterVal(bloom_filter->Clone());
+		bf->Typify(type);
+		return bf;
+		}
+
+	return new BloomFilterVal();
+	}
+
 bool BloomFilterVal::Typify(BroType* arg_type)
 	{
 	if ( type )
@@ -407,6 +464,11 @@ CardinalityVal::~CardinalityVal()
 	delete hash;
 	}
 
+Val* CardinalityVal::DoClone(CloneState* state)
+	{
+	return new CardinalityVal(new probabilistic::CardinalityCounter(*c));
+	}
+
 bool CardinalityVal::Typify(BroType* arg_type)
 	{
 	if ( type )
@@ -434,4 +496,3 @@ void CardinalityVal::Add(const Val* val)
 	c->AddElement(key->Hash());
 	delete key;
 	}
-

src/OpaqueVal.h

@@ -45,6 +45,8 @@ public:
 	MD5Val();
 	~MD5Val();
 
+	Val* DoClone(CloneState* state) override;
+
 protected:
 	friend class Val;
 
@@ -63,6 +65,8 @@ public:
 	SHA1Val();
 	~SHA1Val();
 
+	Val* DoClone(CloneState* state) override;
+
 protected:
 	friend class Val;
 
@@ -81,6 +85,8 @@ public:
 	SHA256Val();
 	~SHA256Val();
 
+	Val* DoClone(CloneState* state) override;
+
 protected:
 	friend class Val;
 
@@ -96,6 +102,8 @@ class EntropyVal : public OpaqueVal {
 public:
 	EntropyVal();
 
+	Val* DoClone(CloneState* state) override;
+
 	bool Feed(const void* data, size_t size);
 	bool Get(double *r_ent, double *r_chisq, double *r_mean,
 		 double *r_montepicalc, double *r_scc);
@@ -112,6 +120,8 @@ public:
 	explicit BloomFilterVal(probabilistic::BloomFilter* bf);
 	~BloomFilterVal() override;
 
+	Val* DoClone(CloneState* state) override;
+
 	BroType* Type() const;
 	bool Typify(BroType* type);
 
@@ -137,7 +147,7 @@ private:
 	BroType* type;
 	CompositeHash* hash;
 	probabilistic::BloomFilter* bloom_filter;
-	};
+};
 
 
 class CardinalityVal: public OpaqueVal {
@@ -145,11 +155,14 @@ public:
 	explicit CardinalityVal(probabilistic::CardinalityCounter*);
 	~CardinalityVal() override;
 
+	Val* DoClone(CloneState* state) override;
+
 	void Add(const Val* val);
 
 	BroType* Type() const;
 	bool Typify(BroType* type);
 
+
 	probabilistic::CardinalityCounter* Get()	{ return c; };
 
 protected:

src/Val.cc

@@ -70,12 +70,67 @@ Val::~Val()
 #endif
 	}
 
-Val* Val::Clone() const
+Val* Val::Clone()
 	{
-	// Fixme: Johanna
+	Val::CloneState state;
-	return nullptr;
+	return Clone(&state);
 	}
 
+Val* Val::Clone(CloneState* state)
+	{
+	auto i = state->clones.find(this);
+
+	if ( i != state->clones.end() )
+		return i->second->Ref();
+
+	auto c = DoClone(state);
+	assert(c);
+
+	state->clones.insert(std::make_pair(this, c));
+	return c;
+	}
+
+Val* Val::DoClone(CloneState* state)
+	{
+	switch ( type->InternalType() ) {
+	case TYPE_INTERNAL_INT:
+	case TYPE_INTERNAL_UNSIGNED:
+	case TYPE_INTERNAL_DOUBLE:
+	 	// Immutable.
+		return Ref();
+
+	case TYPE_INTERNAL_OTHER:
+		// Derived classes are responsible for this. Exception:
+		// Functions and files. There aren't any derived classes.
+		if ( type->Tag() == TYPE_FUNC )
+			// Immutable.
+			return Ref();
+
+		if ( type->Tag() == TYPE_FILE )
+			{
+			// I think we can just ref the file here - it is unclear what else
+			// to do.  In the case of cached files, I think this is equivalent
+			// to what happened before - serialization + unserialization just
+			// have you the same pointer that you already had.  In the case of
+			// non-cached files, the behavior now is different; in the past,
+			// serialize + unserialize gave you a new file object because the
+			// old one was not in the list anymore. This object was
+			// automatically opened. This does not happen anymore - instead you
+			// get the non-cached pointer back which is brought back into the
+			// cache when written too.
+			return Ref();
+			}
+
+		// Fall-through.
+
+	default:
+		reporter->InternalError("cloning illegal base type");
+	}
+
+	reporter->InternalError("cannot be reached");
+	return nullptr;
+ 	}
+
 int Val::IsZero() const
 	{
 	switch ( type->InternalType() ) {
@@ -556,6 +611,12 @@ void PortVal::ValDescribe(ODesc* d) const
 		d->Add("/unknown");
 	}
 
+Val* PortVal::DoClone(CloneState* state)
+	{
+	// Immutable.
+	return Ref();
+	}
+
 AddrVal::AddrVal(const char* text) : Val(TYPE_ADDR)
 	{
 	val.addr_val = new IPAddr(text);
@@ -600,6 +661,12 @@ Val* AddrVal::SizeVal() const
 		return val_mgr->GetCount(128);
 	}
 
+Val* AddrVal::DoClone(CloneState* state)
+	{
+	// Immutable.
+	return Ref();
+	}
+
 SubNetVal::SubNetVal(const char* text) : Val(TYPE_SUBNET)
 	{
 	string s(text);
@@ -710,6 +777,12 @@ bool SubNetVal::Contains(const IPAddr& addr) const
 	return val.subnet_val->Contains(a);
 	}
 
+Val* SubNetVal::DoClone(CloneState* state)
+	{
+	// Immutable.
+	return Ref();
+	}
+
 StringVal::StringVal(BroString* s) : Val(TYPE_STRING)
 	{
 	val.string_val = s;
@@ -752,6 +825,12 @@ unsigned int StringVal::MemoryAllocation() const
 	return padded_sizeof(*this) + val.string_val->MemoryAllocation();
 	}
 
+Val* StringVal::DoClone(CloneState* state)
+	{
+	return new StringVal(new BroString((u_char*) val.string_val->Bytes(),
+	                                   val.string_val->Len(), 1));
+	}
+
 PatternVal::PatternVal(RE_Matcher* re) : Val(base_type(TYPE_PATTERN))
 	{
 	val.re_val = re;
@@ -800,6 +879,14 @@ unsigned int PatternVal::MemoryAllocation() const
 	return padded_sizeof(*this) + val.re_val->MemoryAllocation();
 	}
 
+Val* PatternVal::DoClone(CloneState* state)
+	{
+	auto re = new RE_Matcher(val.re_val->PatternText(),
+	                         val.re_val->AnywherePatternText());
+	re->Compile();
+	return new PatternVal(re);
+	}
+
 ListVal::ListVal(TypeTag t)
 : Val(new TypeList(t == TYPE_ANY ? 0 : base_type_no_ref(t)))
 	{
@@ -882,6 +969,17 @@ void ListVal::Describe(ODesc* d) const
 		}
 	}
 
+Val* ListVal::DoClone(CloneState* state)
+	{
+	auto lv = new ListVal(tag);
+	lv->vals.resize(vals.length());
+
+	loop_over_list(vals, i)
+		lv->Append(vals[i]->Clone(state));
+
+	return lv;
+	}
+
 unsigned int ListVal::MemoryAllocation() const
 	{
 	unsigned int size = 0;
@@ -2032,6 +2130,55 @@ void TableVal::ReadOperation(Val* index, TableEntryVal* v)
 		}
 	}
 
+Val* TableVal::DoClone(CloneState* state)
+	{
+	auto tv = new TableVal(table_type);
+
+	const PDict(TableEntryVal)* tbl = AsTable();
+	IterCookie* cookie = tbl->InitForIteration();
+
+	HashKey* key;
+	TableEntryVal* val;
+	while ( (val = tbl->NextEntry(key, cookie)) )
+		{
+		TableEntryVal* nval = val->Clone();
+		tv->AsNonConstTable()->Insert(key, nval);
+
+		if ( subnets )
+			{
+			Val* idx = RecoverIndex(key);
+			tv->subnets->Insert(idx, nval);
+			Unref(idx);
+			}
+
+		delete key;
+		}
+
+	if ( attrs )
+		{
+		::Ref(attrs);
+		tv->attrs = attrs;
+		}
+
+	if ( expire_time )
+		{
+		tv->expire_time = expire_time->Ref();
+
+		// As network_time is not necessarily initialized yet, we set
+		// a timer which fires immediately.
+		timer = new TableValTimer(this, 1);
+		timer_mgr->Add(timer);
+		}
+
+	if ( expire_func )
+		tv->expire_func = expire_func->Ref();
+
+	if ( def_val )
+		tv->def_val = def_val->Ref();
+
+	return tv;
+	}
+
 bool TableVal::AddProperties(Properties arg_props)
 	{
 	if ( ! MutableVal::AddProperties(arg_props) )
@@ -2095,13 +2242,16 @@ unsigned int TableVal::MemoryAllocation() const
 
 vector<RecordVal*> RecordVal::parse_time_records;
 
-RecordVal::RecordVal(RecordType* t) : MutableVal(t)
+RecordVal::RecordVal(RecordType* t, bool init_fields) : MutableVal(t)
 	{
 	origin = 0;
 	record_type = t;
 	int n = record_type->NumFields();
 	val_list* vl = val.val_list_val = new val_list(n);
 
+	if ( ! init_fields )
+		return;
+
 	// Initialize to default values from RecordType (which are nil
 	// by default).
 	for ( int i = 0; i < n; ++i )
@@ -2334,7 +2484,7 @@ void RecordVal::Describe(ODesc* d) const
 void RecordVal::DescribeReST(ODesc* d) const
 	{
 	const val_list* vl = AsRecord();
-	int n = vl->length();
+     	int n = vl->length();
 
 	d->Add("{");
 	d->PushIndent();
@@ -2359,6 +2509,25 @@ void RecordVal::DescribeReST(ODesc* d) const
 	d->Add("}");
 	}
 
+Val* RecordVal::DoClone(CloneState* state)
+	{
+	// We set origin to 0 here.  Origin only seems to be used for exactly one
+	// purpose - to find the connection record that is associated with a
+	// record. As we cannot guarantee that it will ber zeroed out at the
+	// approproate time (as it seems to be guaranteed for the original record)
+	// we don't touch it.
+	auto rv = new RecordVal(record_type, false);
+	rv->origin = nullptr;
+
+	loop_over_list(*val.val_list_val, i)
+		{
+		Val* v = (*val.val_list_val)[i] ? (*val.val_list_val)[i]->Clone(state) : nullptr;
+  		rv->val.val_list_val->append(v);
+		}
+
+	return rv;
+	}
+
 bool RecordVal::AddProperties(Properties arg_props)
 	{
 	if ( ! MutableVal::AddProperties(arg_props) )
@@ -2420,6 +2589,12 @@ void EnumVal::ValDescribe(ODesc* d) const
 	d->Add(ename);
 	}
 
+Val* EnumVal::DoClone(CloneState* state)
+	{
+	// Immutable.
+	return Ref();
+	}
+
 VectorVal::VectorVal(VectorType* t) : MutableVal(t)
 	{
 	vector_type = t->Ref()->AsVectorType();
@@ -2566,6 +2741,21 @@ bool VectorVal::RemoveProperties(Properties arg_props)
 	return true;
 	}
 
+
+Val* VectorVal::DoClone(CloneState* state)
+	{
+	auto vv = new VectorVal(vector_type);
+	vv->val.vector_val->reserve(val.vector_val->size());
+
+	for ( unsigned int i = 0; i < val.vector_val->size(); ++i )
+		{
+		auto v = (*val.vector_val)[i]->Clone(state);
+		vv->val.vector_val->push_back(v);
+		}
+
+	return vv;
+	}
+
 void VectorVal::ValDescribe(ODesc* d) const
 	{
 	d->Add("[");
@@ -2593,6 +2783,12 @@ OpaqueVal::~OpaqueVal()
 	{
 	}
 
+Val* OpaqueVal::DoClone(CloneState* state)
+	{
+	reporter->InternalError("cloning opaque type without clone implementation");
+	return nullptr;
+	}
+
 Val* check_and_promote(Val* v, const BroType* t, int is_init)
 	{
 	if ( ! v )

src/Val.h

@@ -172,7 +172,7 @@ public:
 	~Val() override;
 
 	Val* Ref()			{ ::Ref(this); return this; }
-	virtual Val* Clone() const;
+	Val* Clone();
 
 	int IsZero() const;
 	int IsOne() const;
@@ -362,6 +362,9 @@ public:
 protected:
 
 	friend class EnumType;
+	friend class ListVal;
+	friend class RecordVal;
+	friend class VectorVal;
 	friend class ValManager;
 
 	virtual void ValDescribe(ODesc* d) const;
@@ -407,6 +410,14 @@ protected:
 	ACCESSOR(TYPE_TABLE, PDict(TableEntryVal)*, table_val, AsNonConstTable)
 	ACCESSOR(TYPE_RECORD, val_list*, val_list_val, AsNonConstRecord)
 
+	// For internal use by the Val::Clone() methods.
+	struct CloneState {
+	    std::unordered_map<const Val*, Val*> clones;
+	};
+
+	Val* Clone(CloneState* state);
+	virtual Val* DoClone(CloneState* state);
+
 	BroValUnion val;
 	BroType* type;
 
@@ -612,6 +623,7 @@ protected:
 	PortVal(uint32 p, bool unused);
 
 	void ValDescribe(ODesc* d) const override;
+	Val* DoClone(CloneState* state) override;
 };
 
 class AddrVal : public Val {
@@ -634,6 +646,8 @@ protected:
 	AddrVal()	{}
 	explicit AddrVal(TypeTag t) : Val(t)	{ }
 	explicit AddrVal(BroType* t) : Val(t)	{ }
+
+	Val* DoClone(CloneState* state) override;
 };
 
 class SubNetVal : public Val {
@@ -661,6 +675,7 @@ protected:
 	SubNetVal()	{}
 
 	void ValDescribe(ODesc* d) const override;
+	Val* DoClone(CloneState* state) override;
 };
 
 class StringVal : public Val {
@@ -691,6 +706,7 @@ protected:
 	StringVal()	{}
 
 	void ValDescribe(ODesc* d) const override;
+	Val* DoClone(CloneState* state) override;
 };
 
 class PatternVal : public Val {
@@ -709,6 +725,7 @@ protected:
 	PatternVal()	{}
 
 	void ValDescribe(ODesc* d) const override;
+	Val* DoClone(CloneState* state) override;
 };
 
 // ListVals are mainly used to index tables that have more than one
@@ -752,6 +769,8 @@ protected:
 	friend class Val;
 	ListVal()	{}
 
+	Val* DoClone(CloneState* state) override;
+
 	val_list vals;
 	TypeTag tag;
 };
@@ -767,6 +786,16 @@ public:
 		expire_access_time = last_read_update =
 			int(network_time - bro_start_network_time);
 		}
+
+	TableEntryVal* Clone()
+		{
+		auto rval = new TableEntryVal(val ? val->Clone() : nullptr);
+		rval->last_access_time = last_access_time;
+		rval->expire_access_time = expire_access_time;
+		rval->last_read_update = last_read_update;
+		return rval;
+		}
+
 	~TableEntryVal()	{ }
 
 	Val* Value()	{ return val; }
@@ -958,6 +987,8 @@ protected:
 	// Propagates a read operation if necessary.
 	void ReadOperation(Val* index, TableEntryVal *v);
 
+	Val* DoClone(CloneState* state) override;
+
 	TableType* table_type;
 	CompositeHash* table_hash;
 	Attributes* attrs;
@@ -971,7 +1002,7 @@ protected:
 
 class RecordVal : public MutableVal {
 public:
-	explicit RecordVal(RecordType* t);
+	explicit RecordVal(RecordType* t, bool init_fields = true);
 	~RecordVal() override;
 
 	Val* SizeVal() const override
@@ -1028,6 +1059,8 @@ protected:
 	bool AddProperties(Properties arg_state) override;
 	bool RemoveProperties(Properties arg_state) override;
 
+	Val* DoClone(CloneState* state) override;
+
 	RecordType* record_type;
 	BroObj* origin;
 
@@ -1057,6 +1090,7 @@ protected:
 	EnumVal()	{}
 
 	void ValDescribe(ODesc* d) const override;
+	Val* DoClone(CloneState* state) override;
 };
 
 
@@ -1115,6 +1149,7 @@ protected:
 	bool AddProperties(Properties arg_state) override;
 	bool RemoveProperties(Properties arg_state) override;
 	void ValDescribe(ODesc* d) const override;
+	Val* DoClone(CloneState* state) override;
 
 	VectorType* vector_type;
 };
@@ -1130,6 +1165,8 @@ public:
 protected:
 	friend class Val;
 	OpaqueVal() { }
+
+	Val* DoClone(CloneState* state) override;
 };
 
 // Checks the given value for consistency with the given type.  If an

src/file_analysis/analyzer/x509/X509.cc

@@ -475,6 +475,15 @@ X509Val::~X509Val()
 		X509_free(certificate);
 	}
 
+Val* X509Val::DoClone(CloneState* state)
+	{
+	auto copy = new X509Val();
+	if ( certificate )
+		copy->certificate = X509_dup(certificate);
+
+	return copy;
+	}
+
 ::X509* X509Val::GetCertificate() const
 	{
 	return certificate;

src/file_analysis/analyzer/x509/X509.h

@@ -123,6 +123,15 @@ public:
 	 */
 	explicit X509Val(::X509* certificate);
 
+	/**
+	 * Clone an X509Val
+	 *
+	 * @param state certifies the state of the clone operation (duplicate tracking)
+	 *
+	 * @return A cloned X509Val.
+	 */
+	Val* DoClone(CloneState* state) override;
+
 	/**
 	 * Destructor.
 	 */

src/file_analysis/analyzer/x509/functions.bif

@@ -13,20 +13,6 @@
 // This is the indexed map of X509 certificate stores.
 static map<Val*, X509_STORE*> x509_stores;
 
-// ### NOTE: while d2i_X509 does not take a const u_char** pointer,
-// here we assume d2i_X509 does not write to <data>, so it is safe to
-// convert data to a non-const pointer.  Could some X509 guru verify
-// this?
-
-X509* d2i_X509_(X509** px, const u_char** in, int len)
-	{
-#ifdef OPENSSL_D2I_X509_USES_CONST_CHAR
-	  return d2i_X509(px, in, len);
-#else
-	  return d2i_X509(px, (u_char**)in, len);
-#endif
-	}
-
 // construct an error record
 RecordVal* x509_result_record(uint64_t num, const char* reason, Val* chainVector = 0)
 	{
@@ -56,7 +42,7 @@ X509_STORE* x509_get_root_store(TableVal* root_certs)
 		StringVal *sv = root_certs->Lookup(key)->AsStringVal();
 		assert(sv);
 		const uint8* data = sv->Bytes();
-		X509* x = d2i_X509_(NULL, &data, sv->Len());
+		X509* x = d2i_X509(NULL, &data, sv->Len());
 		if ( ! x )
 			{
 			builtin_error(fmt("Root CA error: %s", ERR_error_string(ERR_get_error(),NULL)));
@@ -203,6 +189,19 @@ function x509_parse%(cert: opaque of x509%): X509::Certificate
 	return file_analysis::X509::ParseCertificate(h);
 	%}
 
+## Constructs an opaque of X509 from a der-formatted string.
+##
+## Note: this function is mostly meant for testing purposes
+##
+## .. zeek:see:: x509_certificate x509_extension x509_ext_basic_constraints
+##              x509_ext_subject_alternative_name x509_verify
+##              x509_get_certificate_string x509_parse
+function x509_from_der%(der: string%): opaque of x509
+    %{
+    const u_char* data = der->Bytes();
+    return new file_analysis::X509Val(d2i_X509(nullptr, &data, der->Len()));
+    %}
+
 ## Returns the string form of a certificate.
 ##
 ## cert: The X509 certificate opaque handle.

src/probabilistic/Topk.cc

@@ -180,6 +180,13 @@ void TopkVal::Merge(const TopkVal* value, bool doPrune)
 		}
 	}
 
+Val* TopkVal::DoClone(CloneState* state)
+	{
+	auto clone = new TopkVal(size);
+	clone->Merge(this);
+	return clone;
+	}
+
 VectorVal* TopkVal::GetTopK(int k) const // returns vector
 	{
 	if ( numElements == 0 )

src/probabilistic/Topk.h

@@ -122,6 +122,15 @@ public:
 	 */
 	void Merge(const TopkVal* value, bool doPrune=false);
 
+	/**
+	 * Clone the Opaque Type
+	 *
+	 * @param state Clone state (tracking duplicate pointers)
+	 *
+	 * @returns cloned TopkVal
+	 */
+	Val* DoClone(CloneState* state) override;
+
 protected:
 	/**
 	 * Construct an empty TopkVal. Only used for deserialization

testing/btest/Baseline/language.copy-all-opaques/.stderr

@@ -0,0 +1 @@
+error: incompatible Bloom filter types

testing/btest/Baseline/language.copy-all-opaques/out

@@ -0,0 +1,24 @@
+============ Topk
+[b, a, c]
+[b, a, c]
+============ HLL
+3.000069
+3.000069
+============ Bloom
+0
+1
+0
+1
+============ Hashes
+5b9164ad6f496d9dee12ec7634ce253f
+5b9164ad6f496d9dee12ec7634ce253f
+30ae97492ce1da88d0e7117ace0a60a6f9e1e0bc
+30ae97492ce1da88d0e7117ace0a60a6f9e1e0bc
+25b6746d5172ed6352966a013d93ac846e1110d5a25e8f183b5931f4688842a1
+25b6746d5172ed6352966a013d93ac846e1110d5a25e8f183b5931f4688842a1
+============ X509
+[version=3, serial=040000000001154B5AC394, subject=CN=GlobalSign Root CA,OU=Root CA,O=GlobalSign nv-sa,C=BE, issuer=CN=GlobalSign Root CA,OU=Root CA,O=GlobalSign nv-sa,C=BE, cn=GlobalSign Root CA, not_valid_before=904651200.0, not_valid_after=1832673600.0, key_alg=rsaEncryption, sig_alg=sha1WithRSAEncryption, key_type=rsa, key_length=2048, exponent=65537, curve=<uninitialized>]
+[version=3, serial=040000000001154B5AC394, subject=CN=GlobalSign Root CA,OU=Root CA,O=GlobalSign nv-sa,C=BE, issuer=CN=GlobalSign Root CA,OU=Root CA,O=GlobalSign nv-sa,C=BE, cn=GlobalSign Root CA, not_valid_before=904651200.0, not_valid_after=1832673600.0, key_alg=rsaEncryption, sig_alg=sha1WithRSAEncryption, key_type=rsa, key_length=2048, exponent=65537, curve=<uninitialized>]
+============ Entropy
+[entropy=4.715374, chi_square=591.981818, mean=75.472727, monte_carlo_pi=4.0, serial_correlation=-0.11027]
+[entropy=4.715374, chi_square=591.981818, mean=75.472727, monte_carlo_pi=4.0, serial_correlation=-0.11027]

testing/btest/Baseline/language.copy-all-types/out

@@ -0,0 +1,12 @@
+orig=-42 (int) clone=-42 (int) equal=T same_object=T (ok)
+orig=42 (count) clone=42 (count) equal=T same_object=T (ok)
+orig=127.0.0.1 (addr) clone=127.0.0.1 (addr) equal=T same_object=T (ok)
+orig=42/tcp (port) clone=42/tcp (port) equal=T same_object=T (ok)
+orig=127.0.0.0/24 (subnet) clone=127.0.0.0/24 (subnet) equal=T same_object=T (ok)
+orig=Foo (string) clone=Foo (string) equal=T same_object=F (ok)
+orig=/^?(.*PATTERN.*)$?/ (pattern) clone=/^?(.*PATTERN.*)$?/ (pattern) same_object=F
+orig=2,4,1,5,3 (set[count]) clone=2,4,1,5,3 (set[count]) equal=T same_object=F (ok)
+orig=[1, 2, 3, 4, 5] (vector of count) clone=[1, 2, 3, 4, 5] (vector of count) equal=T same_object=F (ok)
+orig=b=vb;a=va (table[string] of string) clone=b=vb;a=va (table[string] of string) equal=T same_object=F (ok)
+orig=ENUMME (enum) clone=ENUMME (enum) equal=T same_object=T (ok)
+orig=[s1=s1, s2=s2, i1=[a=a], i2=[a=a], donotset=<uninitialized>, def=5] (record { s1:string; s2:string; i1:record { a:string; }; i2:record { a:string; }; donotset:record { a:string; }; def:count; }) clone=[s1=s1, s2=s2, i1=[a=a], i2=[a=a], donotset=<uninitialized>, def=5] (record { s1:string; s2:string; i1:record { a:string; }; i2:record { a:string; }; donotset:record { a:string; }; def:count; }) equal=T same_object=F (ok)

testing/btest/core/file-caching-cloning.test

@@ -1,8 +1,8 @@
 # This checks that the interactions between open-file caching and
-# serialization works ok.  In the first case, all files can fit
+# cloning works ok.  In the first case, all files can fit
-# in the cache, but get serialized before every write.  In the
+# in the cache, but get cloned before every write.  In the
-# second case, files are eventually forced out of the cache and
+# second case, files are eventually forced out of the cache; later writing
-# undergo serialization, which requires re-opening.
+# requires re-opening.
 
 # @TEST-EXEC: zeek -b %INPUT "test_file_prefix=one"
 # @TEST-EXEC: btest-diff one0

testing/btest/core/leaks/copy-all-opaques.zeek

@@ -0,0 +1,83 @@
+# @TEST-GROUP: leaks
+# @TEST-REQUIRES: zeek --help 2>&1 | grep -q mem-leaks
+
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run zeek zeek -m -b %INPUT
+# @TEST-EXEC: btest-bg-wait 60
+
+event zeek_init()
+	{
+	print "============ Topk";
+	local k1: opaque of topk = topk_init(4);
+	topk_add(k1, "a");
+	topk_add(k1, "b");
+	topk_add(k1, "b");
+	topk_add(k1, "c");
+	local k2 = copy(k1);
+	print topk_get_top(k1, 5);
+	topk_add(k1, "shoulnotshowup");
+	print topk_get_top(k2, 5);
+
+
+	print "============ HLL";
+	local c1 = hll_cardinality_init(0.01, 0.95);
+	hll_cardinality_add(c1, 2001);
+	hll_cardinality_add(c1, 2002);
+	hll_cardinality_add(c1, 2003);
+
+	print hll_cardinality_estimate(c1);
+	local c2 = copy(c1);
+	hll_cardinality_add(c1, 2004);
+	print hll_cardinality_estimate(c2);
+
+	print "============ Bloom";
+	local bf_cnt = bloomfilter_basic_init(0.1, 1000);
+	bloomfilter_add(bf_cnt, 42);
+	bloomfilter_add(bf_cnt, 84);
+	bloomfilter_add(bf_cnt, 168);
+	print bloomfilter_lookup(bf_cnt, 0);
+	print bloomfilter_lookup(bf_cnt, 42);
+	local bf_copy = copy(bf_cnt);
+	bloomfilter_add(bf_cnt, 0);
+	print bloomfilter_lookup(bf_copy, 0);
+	print bloomfilter_lookup(bf_copy, 42);
+	# check that typefication transfered.
+	bloomfilter_add(bf_copy, 0.5); # causes stderr output
+
+	print "============ Hashes";
+	local md5a = md5_hash_init();
+	md5_hash_update(md5a, "one");
+	local md5b = copy(md5a);
+	md5_hash_update(md5a, "two");
+	md5_hash_update(md5b, "two");
+	print md5_hash_finish(md5a);
+	print md5_hash_finish(md5b);
+
+	local sha1a = sha1_hash_init();
+	sha1_hash_update(sha1a, "one");
+	local sha1b = copy(sha1a);
+	sha1_hash_update(sha1a, "two");
+	sha1_hash_update(sha1b, "two");
+	print sha1_hash_finish(sha1a);
+	print sha1_hash_finish(sha1b);
+
+	local sha256a = sha256_hash_init();
+	sha256_hash_update(sha256a, "one");
+	local sha256b = copy(sha256a);
+	sha256_hash_update(sha256a, "two");
+	sha256_hash_update(sha256b, "two");
+	print sha256_hash_finish(sha256a);
+	print sha256_hash_finish(sha256b);
+
+	print "============ X509";
+	local x509 = x509_from_der("\x30\x82\x03\x75\x30\x82\x02\x5D\xA0\x03\x02\x01\x02\x02\x0B\x04\x00\x00\x00\x00\x01\x15\x4B\x5A\xC3\x94\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x57\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x42\x45\x31\x19\x30\x17\x06\x03\x55\x04\x0A\x13\x10\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x6E\x76\x2D\x73\x61\x31\x10\x30\x0E\x06\x03\x55\x04\x0B\x13\x07\x52\x6F\x6F\x74\x20\x43\x41\x31\x1B\x30\x19\x06\x03\x55\x04\x03\x13\x12\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x1E\x17\x0D\x39\x38\x30\x39\x30\x31\x31\x32\x30\x30\x30\x30\x5A\x17\x0D\x32\x38\x30\x31\x32\x38\x31\x32\x30\x30\x30\x30\x5A\x30\x57\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x42\x45\x31\x19\x30\x17\x06\x03\x55\x04\x0A\x13\x10\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x6E\x76\x2D\x73\x61\x31\x10\x30\x0E\x06\x03\x55\x04\x0B\x13\x07\x52\x6F\x6F\x74\x20\x43\x41\x31\x1B\x30\x19\x06\x03\x55\x04\x03\x13\x12\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xDA\x0E\xE6\x99\x8D\xCE\xA3\xE3\x4F\x8A\x7E\xFB\xF1\x8B\x83\x25\x6B\xEA\x48\x1F\xF1\x2A\xB0\xB9\x95\x11\x04\xBD\xF0\x63\xD1\xE2\x67\x66\xCF\x1C\xDD\xCF\x1B\x48\x2B\xEE\x8D\x89\x8E\x9A\xAF\x29\x80\x65\xAB\xE9\xC7\x2D\x12\xCB\xAB\x1C\x4C\x70\x07\xA1\x3D\x0A\x30\xCD\x15\x8D\x4F\xF8\xDD\xD4\x8C\x50\x15\x1C\xEF\x50\xEE\xC4\x2E\xF7\xFC\xE9\x52\xF2\x91\x7D\xE0\x6D\xD5\x35\x30\x8E\x5E\x43\x73\xF2\x41\xE9\xD5\x6A\xE3\xB2\x89\x3A\x56\x39\x38\x6F\x06\x3C\x88\x69\x5B\x2A\x4D\xC5\xA7\x54\xB8\x6C\x89\xCC\x9B\xF9\x3C\xCA\xE5\xFD\x89\xF5\x12\x3C\x92\x78\x96\xD6\xDC\x74\x6E\x93\x44\x61\xD1\x8D\xC7\x46\xB2\x75\x0E\x86\xE8\x19\x8A\xD5\x6D\x6C\xD5\x78\x16\x95\xA2\xE9\xC8\x0A\x38\xEB\xF2\x24\x13\x4F\x73\x54\x93\x13\x85\x3A\x1B\xBC\x1E\x34\xB5\x8B\x05\x8C\xB9\x77\x8B\xB1\xDB\x1F\x20\x91\xAB\x09\x53\x6E\x90\xCE\x7B\x37\x74\xB9\x70\x47\x91\x22\x51\x63\x16\x79\xAE\xB1\xAE\x41\x26\x08\xC8\x19\x2B\xD1\x46\xAA\x48\xD6\x64\x2A\xD7\x83\x34\xFF\x2C\x2A\xC1\x6C\x19\x43\x4A\x07\x85\xE7\xD3\x7C\xF6\x21\x68\xEF\xEA\xF2\x52\x9F\x7F\x93\x90\xCF\x02\x03\x01\x00\x01\xA3\x42\x30\x40\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x60\x7B\x66\x1A\x45\x0D\x97\xCA\x89\x50\x2F\x7D\x04\xCD\x34\xA8\xFF\xFC\xFD\x4B\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\xD6\x73\xE7\x7C\x4F\x76\xD0\x8D\xBF\xEC\xBA\xA2\xBE\x34\xC5\x28\x32\xB5\x7C\xFC\x6C\x9C\x2C\x2B\xBD\x09\x9E\x53\xBF\x6B\x5E\xAA\x11\x48\xB6\xE5\x08\xA3\xB3\xCA\x3D\x61\x4D\xD3\x46\x09\xB3\x3E\xC3\xA0\xE3\x63\x55\x1B\xF2\xBA\xEF\xAD\x39\xE1\x43\xB9\x38\xA3\xE6\x2F\x8A\x26\x3B\xEF\xA0\x50\x56\xF9\xC6\x0A\xFD\x38\xCD\xC4\x0B\x70\x51\x94\x97\x98\x04\xDF\xC3\x5F\x94\xD5\x15\xC9\x14\x41\x9C\xC4\x5D\x75\x64\x15\x0D\xFF\x55\x30\xEC\x86\x8F\xFF\x0D\xEF\x2C\xB9\x63\x46\xF6\xAA\xFC\xDF\xBC\x69\xFD\x2E\x12\x48\x64\x9A\xE0\x95\xF0\xA6\xEF\x29\x8F\x01\xB1\x15\xB5\x0C\x1D\xA5\xFE\x69\x2C\x69\x24\x78\x1E\xB3\xA7\x1C\x71\x62\xEE\xCA\xC8\x97\xAC\x17\x5D\x8A\xC2\xF8\x47\x86\x6E\x2A\xC4\x56\x31\x95\xD0\x67\x89\x85\x2B\xF9\x6C\xA6\x5D\x46\x9D\x0C\xAA\x82\xE4\x99\x51\xDD\x70\xB7\xDB\x56\x3D\x61\xE4\x6A\xE1\x5C\xD6\xF6\xFE\x3D\xDE\x41\xCC\x07\xAE\x63\x52\xBF\x53\x53\xF4\x2B\xE9\xC7\xFD\xB6\xF7\x82\x5F\x85\xD2\x41\x18\xDB\x81\xB3\x04\x1C\xC5\x1F\xA4\x80\x6F\x15\x20\xC9\xDE\x0C\x88\x0A\x1D\xD6\x66\x55\xE2\xFC\x48\xC9\x29\x26\x69\xE0");
+	local x5092 = copy(x509);
+	print x509_parse(x509);
+	print x509_parse(x5092);
+
+	print "============ Entropy";
+	local handle = entropy_test_init();
+	entropy_test_add(handle, "dh3Hie02uh^s#Sdf9L3frd243h$d78r2G4cM6*Q05d(7rh46f!0|4-f");
+	local handle2 = copy(handle);
+	print entropy_test_finish(handle);
+	print entropy_test_finish(handle2);
+	}

testing/btest/core/leaks/copy-all-types.zeek

@@ -0,0 +1,190 @@
+# Note: opaque types in separate test
+# @TEST-GROUP: leaks
+# @TEST-REQUIRES: zeek --help 2>&1 | grep -q mem-leaks
+
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run zeek zeek -m -b %INPUT
+# @TEST-EXEC: btest-bg-wait 60
+
+type MyEnum: enum { ENUMME };
+
+type InnerTestRecord: record {
+	a: string;
+};
+
+type TestRecord: record {
+	s1: string;
+	s2: string;
+	i1: InnerTestRecord;
+	i2: InnerTestRecord &optional;
+	donotset: InnerTestRecord &optional;
+	def: count &default=5;
+};
+
+function join_count_set(ss: set[count], j: string): string
+	{
+	local output="";
+	local i=0;
+	for ( s in ss )
+		{
+		if ( i > 0 )
+			output = cat(output, j);
+
+		output = cat(output, s);
+		++i;
+		}
+	return output;
+	}
+
+function do_format(i: any): any
+	{
+	local tpe = type_name(i);
+
+	switch ( tpe )
+		{
+		case "set[count]":
+			return join_count_set(i, ",");
+		case "table[string] of string":
+			local cast: table[string] of string = i;
+			local vout: vector of string = vector();
+			for ( el in cast )
+				{
+				vout += cat(el, "=", cast[el]);
+				}
+			return join_string_vec(vout, ";");
+		}
+	return i;
+	}
+
+function check(o1: any, o2: any, equal: bool, expect_same: bool)
+	{
+	local expect_msg = (equal ? "ok" : "FAIL0");
+	local same = same_object(o1, o2);
+
+	if ( expect_same && ! same )
+		expect_msg = "FAIL1";
+
+	if ( ! expect_same && same )
+		expect_msg = "FAIL2";
+
+	print fmt("orig=%s (%s) clone=%s (%s) equal=%s same_object=%s (%s)", do_format(o1), type_name(o1), do_format(o2), type_name(o2), equal, same, expect_msg);
+	}
+
+function check_vector_equal(a: vector of count, b: vector of count): bool
+	{
+	if ( |a| != |b| )
+		return F;
+
+	for ( i in a )
+		{
+		if ( a[i] != b[i] )
+			return F;
+		}
+
+	return T;
+	}
+
+function check_string_table_equal(a: table[string] of string, b: table[string] of string): bool
+	{
+	if ( |a| != |b| )
+		return F;
+
+	for ( i in a )
+		{
+		if ( a[i] != b[i] )
+			return F;
+		}
+
+	return T;
+	}
+
+function compare_otr(a: TestRecord, b: TestRecord): bool
+	{
+	if ( a$s1 != b$s1 )
+		return F;
+	if ( a$s2 != b$s2 )
+		return F;
+	if ( a$i1$a != b$i1$a )
+		return F;
+	if ( a$i2$a != b$i2$a )
+		return F;
+
+	if ( same_object(a$i1, b$i1) )
+		return F;
+	if ( same_object(a$i2, b$i2) )
+		return F;
+
+	# check that we restroe that i1 & i2 point to same object
+	if ( ! same_object(a$i1, a$i2) )
+		return F;
+	if ( ! same_object(b$i1, b$i2) )
+		return F;
+
+	if ( a$def != b$def )
+		return F;
+
+	return T;
+	}
+
+
+event zeek_init()
+	{
+	local i1 = -42;
+	local i2 = copy(i1);
+	check(i1, i2, i1 == i2, T);
+
+	local c1 : count = 42;
+	local c2 = copy(c1);
+	check(c1, c2, c1 == c2, T);
+
+	local a1 = 127.0.0.1;
+	local a2 = copy(a1);
+	check(a1, a2, a1 == a2, T);
+
+	local p1 = 42/tcp;
+	local p2 = copy(p1);
+	check(p1, p2, p1 == p2, T);
+
+	local sn1 = 127.0.0.1/24;
+	local sn2 = copy(sn1);
+	check(sn1, sn2, sn1 == sn2, T);
+
+	local s1 = "Foo";
+	local s2 = copy(s1);
+	check(s1, s2, s1 == s2, F);
+
+	local pat1 = /.*PATTERN.*/;
+	local pat2 = copy(pat1);
+	# patterns cannot be directoy compared
+	if ( same_object(pat1, pat2) )
+		print "FAIL P1";
+	if ( ! ( pat1 == "PATTERN" ) )
+		print "FAIL P2";
+	if ( ! ( pat2 == "PATTERN" ) )
+		print "FAIL P3";
+	if ( pat2 == "PATERN" )
+		print "FAIL P4";
+	print fmt("orig=%s (%s) clone=%s (%s) same_object=%s", pat1, type_name(pat1), pat2, type_name(pat2), same_object(pat1, pat2));
+
+	local set1 = [1, 2, 3, 4, 5];
+	local set2 = copy(set1);
+	check(set1, set2, set1 == set2, F);
+
+	local v1 = vector(1, 2, 3, 4, 5);
+	local v2 = copy(v1);
+	check(v1, v2, check_vector_equal(v1, v2), F);
+
+	local t1 : table[string] of string = table();
+	t1["a"] = "va";
+	t1["b"] = "vb";
+	local t2 = copy(t1);
+	check(t1, t2, check_string_table_equal(t1, t2), F);
+
+	local e1 = ENUMME;
+	local e2 = copy(ENUMME);
+	check(e1, e2, e1 == e2, T);
+
+	local itr = InnerTestRecord($a="a");
+	local otr1 = TestRecord($s1="s1", $s2="s2", $i1=itr, $i2=itr);
+	local otr2 = copy(otr1);
+	check(otr1, otr2, compare_otr(otr1, otr2), F);
+	}

testing/btest/language/copy-all-opaques.zeek

@@ -0,0 +1,81 @@
+# @TEST-EXEC: zeek -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+# @TEST-EXEC: btest-diff .stderr
+
+event zeek_init()
+	{
+	print "============ Topk";
+	local k1: opaque of topk = topk_init(4);
+	topk_add(k1, "a");
+	topk_add(k1, "b");
+	topk_add(k1, "b");
+	topk_add(k1, "c");
+	local k2 = copy(k1);
+	print topk_get_top(k1, 5);
+	topk_add(k1, "shoulnotshowup");
+	print topk_get_top(k2, 5);
+
+
+	print "============ HLL";
+	local c1 = hll_cardinality_init(0.01, 0.95);
+	hll_cardinality_add(c1, 2001);
+	hll_cardinality_add(c1, 2002);
+	hll_cardinality_add(c1, 2003);
+
+	print hll_cardinality_estimate(c1);
+	local c2 = copy(c1);
+	hll_cardinality_add(c1, 2004);
+	print hll_cardinality_estimate(c2);
+
+	print "============ Bloom";
+	local bf_cnt = bloomfilter_basic_init(0.1, 1000);
+	bloomfilter_add(bf_cnt, 42);
+	bloomfilter_add(bf_cnt, 84);
+	bloomfilter_add(bf_cnt, 168);
+	print bloomfilter_lookup(bf_cnt, 0);
+	print bloomfilter_lookup(bf_cnt, 42);
+	local bf_copy = copy(bf_cnt);
+	bloomfilter_add(bf_cnt, 0);
+	print bloomfilter_lookup(bf_copy, 0);
+	print bloomfilter_lookup(bf_copy, 42);
+	# check that typefication transfered.
+	bloomfilter_add(bf_copy, 0.5); # causes stderr output
+
+	print "============ Hashes";
+	local md5a = md5_hash_init();
+	md5_hash_update(md5a, "one");
+	local md5b = copy(md5a);
+	md5_hash_update(md5a, "two");
+	md5_hash_update(md5b, "two");
+	print md5_hash_finish(md5a);
+	print md5_hash_finish(md5b);
+
+	local sha1a = sha1_hash_init();
+	sha1_hash_update(sha1a, "one");
+	local sha1b = copy(sha1a);
+	sha1_hash_update(sha1a, "two");
+	sha1_hash_update(sha1b, "two");
+	print sha1_hash_finish(sha1a);
+	print sha1_hash_finish(sha1b);
+
+	local sha256a = sha256_hash_init();
+	sha256_hash_update(sha256a, "one");
+	local sha256b = copy(sha256a);
+	sha256_hash_update(sha256a, "two");
+	sha256_hash_update(sha256b, "two");
+	print sha256_hash_finish(sha256a);
+	print sha256_hash_finish(sha256b);
+
+	print "============ X509";
+	local x509 = x509_from_der("\x30\x82\x03\x75\x30\x82\x02\x5D\xA0\x03\x02\x01\x02\x02\x0B\x04\x00\x00\x00\x00\x01\x15\x4B\x5A\xC3\x94\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x57\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x42\x45\x31\x19\x30\x17\x06\x03\x55\x04\x0A\x13\x10\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x6E\x76\x2D\x73\x61\x31\x10\x30\x0E\x06\x03\x55\x04\x0B\x13\x07\x52\x6F\x6F\x74\x20\x43\x41\x31\x1B\x30\x19\x06\x03\x55\x04\x03\x13\x12\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x1E\x17\x0D\x39\x38\x30\x39\x30\x31\x31\x32\x30\x30\x30\x30\x5A\x17\x0D\x32\x38\x30\x31\x32\x38\x31\x32\x30\x30\x30\x30\x5A\x30\x57\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x42\x45\x31\x19\x30\x17\x06\x03\x55\x04\x0A\x13\x10\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x6E\x76\x2D\x73\x61\x31\x10\x30\x0E\x06\x03\x55\x04\x0B\x13\x07\x52\x6F\x6F\x74\x20\x43\x41\x31\x1B\x30\x19\x06\x03\x55\x04\x03\x13\x12\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xDA\x0E\xE6\x99\x8D\xCE\xA3\xE3\x4F\x8A\x7E\xFB\xF1\x8B\x83\x25\x6B\xEA\x48\x1F\xF1\x2A\xB0\xB9\x95\x11\x04\xBD\xF0\x63\xD1\xE2\x67\x66\xCF\x1C\xDD\xCF\x1B\x48\x2B\xEE\x8D\x89\x8E\x9A\xAF\x29\x80\x65\xAB\xE9\xC7\x2D\x12\xCB\xAB\x1C\x4C\x70\x07\xA1\x3D\x0A\x30\xCD\x15\x8D\x4F\xF8\xDD\xD4\x8C\x50\x15\x1C\xEF\x50\xEE\xC4\x2E\xF7\xFC\xE9\x52\xF2\x91\x7D\xE0\x6D\xD5\x35\x30\x8E\x5E\x43\x73\xF2\x41\xE9\xD5\x6A\xE3\xB2\x89\x3A\x56\x39\x38\x6F\x06\x3C\x88\x69\x5B\x2A\x4D\xC5\xA7\x54\xB8\x6C\x89\xCC\x9B\xF9\x3C\xCA\xE5\xFD\x89\xF5\x12\x3C\x92\x78\x96\xD6\xDC\x74\x6E\x93\x44\x61\xD1\x8D\xC7\x46\xB2\x75\x0E\x86\xE8\x19\x8A\xD5\x6D\x6C\xD5\x78\x16\x95\xA2\xE9\xC8\x0A\x38\xEB\xF2\x24\x13\x4F\x73\x54\x93\x13\x85\x3A\x1B\xBC\x1E\x34\xB5\x8B\x05\x8C\xB9\x77\x8B\xB1\xDB\x1F\x20\x91\xAB\x09\x53\x6E\x90\xCE\x7B\x37\x74\xB9\x70\x47\x91\x22\x51\x63\x16\x79\xAE\xB1\xAE\x41\x26\x08\xC8\x19\x2B\xD1\x46\xAA\x48\xD6\x64\x2A\xD7\x83\x34\xFF\x2C\x2A\xC1\x6C\x19\x43\x4A\x07\x85\xE7\xD3\x7C\xF6\x21\x68\xEF\xEA\xF2\x52\x9F\x7F\x93\x90\xCF\x02\x03\x01\x00\x01\xA3\x42\x30\x40\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x60\x7B\x66\x1A\x45\x0D\x97\xCA\x89\x50\x2F\x7D\x04\xCD\x34\xA8\xFF\xFC\xFD\x4B\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\xD6\x73\xE7\x7C\x4F\x76\xD0\x8D\xBF\xEC\xBA\xA2\xBE\x34\xC5\x28\x32\xB5\x7C\xFC\x6C\x9C\x2C\x2B\xBD\x09\x9E\x53\xBF\x6B\x5E\xAA\x11\x48\xB6\xE5\x08\xA3\xB3\xCA\x3D\x61\x4D\xD3\x46\x09\xB3\x3E\xC3\xA0\xE3\x63\x55\x1B\xF2\xBA\xEF\xAD\x39\xE1\x43\xB9\x38\xA3\xE6\x2F\x8A\x26\x3B\xEF\xA0\x50\x56\xF9\xC6\x0A\xFD\x38\xCD\xC4\x0B\x70\x51\x94\x97\x98\x04\xDF\xC3\x5F\x94\xD5\x15\xC9\x14\x41\x9C\xC4\x5D\x75\x64\x15\x0D\xFF\x55\x30\xEC\x86\x8F\xFF\x0D\xEF\x2C\xB9\x63\x46\xF6\xAA\xFC\xDF\xBC\x69\xFD\x2E\x12\x48\x64\x9A\xE0\x95\xF0\xA6\xEF\x29\x8F\x01\xB1\x15\xB5\x0C\x1D\xA5\xFE\x69\x2C\x69\x24\x78\x1E\xB3\xA7\x1C\x71\x62\xEE\xCA\xC8\x97\xAC\x17\x5D\x8A\xC2\xF8\x47\x86\x6E\x2A\xC4\x56\x31\x95\xD0\x67\x89\x85\x2B\xF9\x6C\xA6\x5D\x46\x9D\x0C\xAA\x82\xE4\x99\x51\xDD\x70\xB7\xDB\x56\x3D\x61\xE4\x6A\xE1\x5C\xD6\xF6\xFE\x3D\xDE\x41\xCC\x07\xAE\x63\x52\xBF\x53\x53\xF4\x2B\xE9\xC7\xFD\xB6\xF7\x82\x5F\x85\xD2\x41\x18\xDB\x81\xB3\x04\x1C\xC5\x1F\xA4\x80\x6F\x15\x20\xC9\xDE\x0C\x88\x0A\x1D\xD6\x66\x55\xE2\xFC\x48\xC9\x29\x26\x69\xE0");
+	local x5092 = copy(x509);
+	print x509_parse(x509);
+	print x509_parse(x5092);
+
+	print "============ Entropy";
+	local handle = entropy_test_init();
+	entropy_test_add(handle, "dh3Hie02uh^s#Sdf9L3frd243h$d78r2G4cM6*Q05d(7rh46f!0|4-f");
+	local handle2 = copy(handle);
+	print entropy_test_finish(handle);
+	print entropy_test_finish(handle2);
+	}

testing/btest/language/copy-all-types.zeek

@@ -0,0 +1,187 @@
+# Note: opaque types in separate test
+# @TEST-EXEC: zeek -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+type MyEnum: enum { ENUMME };
+
+type InnerTestRecord: record {
+	a: string;
+};
+
+type TestRecord: record {
+	s1: string;
+	s2: string;
+	i1: InnerTestRecord;
+	i2: InnerTestRecord &optional;
+	donotset: InnerTestRecord &optional;
+	def: count &default=5;
+};
+
+function join_count_set(ss: set[count], j: string): string
+	{
+	local output="";
+	local i=0;
+	for ( s in ss )
+		{
+		if ( i > 0 )
+			output = cat(output, j);
+
+		output = cat(output, s);
+		++i;
+		}
+	return output;
+	}
+
+function do_format(i: any): any
+	{
+	local tpe = type_name(i);
+
+	switch ( tpe )
+		{
+		case "set[count]":
+			return join_count_set(i, ",");
+		case "table[string] of string":
+			local cast: table[string] of string = i;
+			local vout: vector of string = vector();
+			for ( el in cast )
+				{
+				vout += cat(el, "=", cast[el]);
+				}
+			return join_string_vec(vout, ";");
+		}
+	return i;
+	}
+
+function check(o1: any, o2: any, equal: bool, expect_same: bool)
+	{
+	local expect_msg = (equal ? "ok" : "FAIL0");
+	local same = same_object(o1, o2);
+
+	if ( expect_same && ! same )
+		expect_msg = "FAIL1";
+
+	if ( ! expect_same && same )
+		expect_msg = "FAIL2";
+
+	print fmt("orig=%s (%s) clone=%s (%s) equal=%s same_object=%s (%s)", do_format(o1), type_name(o1), do_format(o2), type_name(o2), equal, same, expect_msg);
+	}
+
+function check_vector_equal(a: vector of count, b: vector of count): bool
+	{
+	if ( |a| != |b| )
+		return F;
+
+	for ( i in a )
+		{
+		if ( a[i] != b[i] )
+			return F;
+		}
+
+	return T;
+	}
+
+function check_string_table_equal(a: table[string] of string, b: table[string] of string): bool
+	{
+	if ( |a| != |b| )
+		return F;
+
+	for ( i in a )
+		{
+		if ( a[i] != b[i] )
+			return F;
+		}
+
+	return T;
+	}
+
+function compare_otr(a: TestRecord, b: TestRecord): bool
+	{
+	if ( a$s1 != b$s1 )
+		return F;
+	if ( a$s2 != b$s2 )
+		return F;
+	if ( a$i1$a != b$i1$a )
+		return F;
+	if ( a$i2$a != b$i2$a )
+		return F;
+
+	if ( same_object(a$i1, b$i1) )
+		return F;
+	if ( same_object(a$i2, b$i2) )
+		return F;
+
+	# check that we restroe that i1 & i2 point to same object
+	if ( ! same_object(a$i1, a$i2) )
+		return F;
+	if ( ! same_object(b$i1, b$i2) )
+		return F;
+
+	if ( a$def != b$def )
+		return F;
+
+	return T;
+	}
+
+
+event zeek_init()
+	{
+	local i1 = -42;
+	local i2 = copy(i1);
+	check(i1, i2, i1 == i2, T);
+
+	local c1 : count = 42;
+	local c2 = copy(c1);
+	check(c1, c2, c1 == c2, T);
+
+	local a1 = 127.0.0.1;
+	local a2 = copy(a1);
+	check(a1, a2, a1 == a2, T);
+
+	local p1 = 42/tcp;
+	local p2 = copy(p1);
+	check(p1, p2, p1 == p2, T);
+
+	local sn1 = 127.0.0.1/24;
+	local sn2 = copy(sn1);
+	check(sn1, sn2, sn1 == sn2, T);
+
+	local s1 = "Foo";
+	local s2 = copy(s1);
+	check(s1, s2, s1 == s2, F);
+
+	local pat1 = /.*PATTERN.*/;
+	local pat2 = copy(pat1);
+	# patterns cannot be directoy compared
+	if ( same_object(pat1, pat2) )
+		print "FAIL P1";
+	if ( ! ( pat1 == "PATTERN" ) )
+		print "FAIL P2";
+	if ( ! ( pat2 == "PATTERN" ) )
+		print "FAIL P3";
+	if ( pat2 == "PATERN" )
+		print "FAIL P4";
+	print fmt("orig=%s (%s) clone=%s (%s) same_object=%s", pat1, type_name(pat1), pat2, type_name(pat2), same_object(pat1, pat2));
+
+	local set1 = [1, 2, 3, 4, 5];
+	local set2 = copy(set1);
+	check(set1, set2, set1 == set2, F);
+
+	local v1 = vector(1, 2, 3, 4, 5);
+	local v2 = copy(v1);
+	check(v1, v2, check_vector_equal(v1, v2), F);
+
+	local t1 : table[string] of string = table();
+	t1["a"] = "va";
+	t1["b"] = "vb";
+	local t2 = copy(t1);
+	check(t1, t2, check_string_table_equal(t1, t2), F);
+
+	local e1 = ENUMME;
+	local e2 = copy(ENUMME);
+	check(e1, e2, e1 == e2, T);
+
+	local itr = InnerTestRecord($a="a");
+	local otr1 = TestRecord($s1="s1", $s2="s2", $i1=itr, $i2=itr);
+	local otr2 = copy(otr1);
+	check(otr1, otr2, compare_otr(otr1, otr2), F);
+	}

testing/btest/scripts/base/frameworks/sumstats/topk.zeek

@@ -3,7 +3,7 @@
 
 event zeek_init() &priority=5
 	{
-	local r1: SumStats::Reducer = [$stream="test.metric", 
+	local r1: SumStats::Reducer = [$stream="test.metric",
 	                               $apply=set(SumStats::TOPK)];
 	# Merge two empty sets
 	local topk1: opaque of topk = topk_init(4);
@@ -18,9 +18,9 @@ event zeek_init() &priority=5
 	                  	local r = result["test.metric"];
 	                  	local s: vector of SumStats::Observation;
 	                  	s = topk_get_top(r$topk, 5);
-	                  	
+
 	                  	print fmt("Top entries for key %s", key$str);
-	                  	for ( element in s ) 
+	                  	for ( element in s )
 	                  		{
 	                  		print fmt("Num: %d, count: %d, epsilon: %d", s[element]$num, topk_count(r$topk, s[element]), topk_epsilon(r$topk, s[element]));
 	                  		}
@@ -32,16 +32,16 @@ event zeek_init() &priority=5
 	local a: count;
 	a = 0;
 
-	for ( i in loop_v ) 
+	for ( i in loop_v )
 		{
 		a = a + 1;
 		for ( j in loop_v )
 			{
-			if ( i < j ) 
+			if ( i < j )
 				SumStats::observe("test.metric", [$str="counter"], [$num=a]);
 			}
 		}
-	
+
 
 	SumStats::observe("test.metric", [$str="two"], [$num=1]);
 	SumStats::observe("test.metric", [$str="two"], [$num=1]);