Merge remote-tracking branch 'origin/topic/johanna/remove-serializer'

* origin/topic/johanna/remove-serializer:
  Fix memory leak introduced by removing opaque of ocsp_resp.
  Change return value of OpaqueVal::DoSerialize.
  Add missing ShallowClone implementation for SetType
  Remove opaque of ocsp_resp.
  Remove remnants of event serializer.
  Fix cardinalitycounter deserialization.
  Smaller compile fixes for the new opaque serialization.
  Reimplement serialization infrastructure for OpaqueVals.
  Couple of compile fixes.
  Remove const from ShallowClone.
  Remove test-case for removed functionality
  Implement a Shallow Clone operation for types.
  Remove value serialization.

Various changes I made:

- Fix memory leak in type-checker for opaque vals wrapped in broker::data

- Noticed the two "copy-all" leak tests weren't actually checking for
  memory leaks because the heap checker isn't active until after zeek_init()
  is evaluated.

- Change OpaqueVal::DoClone to use the clone caching mechanism

- Improve copy elision for broker::expected return types in the various
  OpaqueVal serialize methods

  - Not all compilers end up properly treating the return of
    local/automatic variable as an rvalue that can be moved, and ends up
    copying it instead.

  - Particularly, until GCC 8, this pattern ends up copying instead of
    moving, and we still support platforms whose default compiler
    pre-dates that version.

  - Generally seems it's something that wasn't addressed until C++14.
    See http://www.open-std.org/jtc1/sc22/wg21/docs/cwg_defects.html#1579

- Change OpaqueVal::SerializeType to return broker::expected

- Change probabilistic DoSerialize methods to return broker::expected

src/OpaqueVal.h

@@ -3,12 +3,151 @@
 #ifndef OPAQUEVAL_H
 #define OPAQUEVAL_H
 
-#include <typeinfo>
-
 #include "RandTest.h"
 #include "Val.h"
 #include "digest.h"
 
+class OpaqueVal;
+
+/**
+  * Singleton that registers all available all available types of opaque
+  * values. This faciliates their serialization into Broker values.
+  */
+class OpaqueMgr {
+public:
+	using Factory = OpaqueVal* ();
+
+	/**
+	 * Return's a unique ID for the type of an opaque value.
+	 * @param v opaque value to return type for; its class must have been
+	 * registered with the manager, otherwise this method will abort
+	 * execution.
+	 *
+	 * @return type ID, which can used with *Instantiate()* to create a
+	 * new instance of the same type.
+	 */
+	const std::string& TypeID(const OpaqueVal* v) const;
+
+	/**
+	 * Instantiates a new opaque value of a specific opaque type.
+	 *
+	 * @param id unique type ID for the class to instantiate; this will
+	 * normally have been returned earlier by *TypeID()*.
+	 *
+	 * @return A freshly instantiated value of the OpaqueVal-derived
+	 * classes that *id* specifies, with reference count at +1. If *id*
+	 * is unknown, this will return null.
+	 *
+	 */
+	OpaqueVal* Instantiate(const std::string& id) const;
+
+	/** Returns the global manager singleton object. */
+	static OpaqueMgr* mgr();
+
+	/**
+	 * Internal helper class to register an OpaqueVal-derived classes
+	 * with the manager.
+	 */
+	template<class T>
+	class Register {
+	public:
+		Register(const char* id)
+			{ OpaqueMgr::mgr()->_types.emplace(id, &T::OpaqueInstantiate); }
+	};
+
+private:
+	std::unordered_map<std::string, Factory*> _types;
+};
+
+/** Macro to insert into an OpaqueVal-derived class's declaration. */
+#define DECLARE_OPAQUE_VALUE(T)                            \
+    friend class OpaqueMgr::Register<T>;                   \
+    broker::expected<broker::data> DoSerialize() const override;             \
+    bool DoUnserialize(const broker::data& data) override; \
+    const char* OpaqueName() const override { return #T; } \
+    static OpaqueVal* OpaqueInstantiate() { return new T(); }
+
+#define __OPAQUE_MERGE(a, b) a ## b
+#define __OPAQUE_ID(x) __OPAQUE_MERGE(_opaque, x)
+
+/** Macro to insert into an OpaqueVal-derived class's implementation file. */
+#define IMPLEMENT_OPAQUE_VALUE(T) static OpaqueMgr::Register<T> __OPAQUE_ID(__LINE__)(#T);
+
+/**
+ * Base class for all opaque values. Opaque values are types that are managed
+ * completely internally, with no further script-level operators provided
+ * (other than bif functions). See OpaqueVal.h for derived classes.
+ */
+class OpaqueVal : public Val {
+public:
+	explicit OpaqueVal(OpaqueType* t);
+	~OpaqueVal() override;
+
+	/**
+	 * Serializes the value into a Broker representation.
+	 *
+	 * @return the broker representation, or an error if serialization
+	 * isn't supported or failed.
+	 */
+	broker::expected<broker::data> Serialize() const;
+
+	/**
+	 * Reinstantiates a value from its serialized Broker representation.
+	 *
+	 * @param data Broker representation as returned by *Serialize()*.
+	 * @return unserialized instances with reference count at +1
+	 */
+	static OpaqueVal* Unserialize(const broker::data& data);
+
+protected:
+	friend class Val;
+	friend class OpaqueMgr;
+	OpaqueVal() { }
+
+	/**
+	 * Must be overridden to provide a serialized version of the derived
+	 * class' state.
+	 *
+	 * @return the serialized data or an error if serialization
+	 * isn't supported or failed.
+	 */
+	virtual broker::expected<broker::data> DoSerialize() const = 0;
+
+	/**
+	 * Must be overridden to recreate the the derived class' state from a
+	 * serialization.
+	 *
+	 * @return true if successful.
+	 */
+	virtual bool DoUnserialize(const broker::data& data) = 0;
+
+	/**
+	 * Internal helper for the serialization machinery. Automatically
+	 * overridden by the `DECLARE_OPAQUE_VALUE` macro.
+	 */
+	virtual const char* OpaqueName() const = 0;
+
+	/**
+	 * Provides an implementation of *Val::DoClone()* that leverages the
+	 * serialization methods to deep-copy an instance. Derived classes
+	 * may also override this with a more efficient custom clone
+	 * implementation of their own.
+	 */
+	Val* DoClone(CloneState* state) override;
+
+	/**
+	 * Helper function for derived class that need to record a type
+	 * during serialization.
+	 */
+	static broker::expected<broker::data> SerializeType(BroType* t);
+
+	/**
+	 * Helper function for derived class that need to restore a type
+	 * during unserialization. Returns the type at reference count +1.
+	 */
+	static BroType* UnserializeType(const broker::data& data);
+};
+
 namespace probabilistic {
 	class BloomFilter;
 	class CardinalityCounter;
@@ -22,15 +161,13 @@ public:
 	virtual StringVal* Get();
 
 protected:
-	HashVal() { };
+	HashVal()	{ valid = false; }
 	explicit HashVal(OpaqueType* t);
 
 	virtual bool DoInit();
 	virtual bool DoFeed(const void* data, size_t size);
 	virtual StringVal* DoGet();
 
-	DECLARE_SERIAL(HashVal);
-
 private:
 	// This flag exists because Get() can only be called once.
 	bool valid;
@@ -56,8 +193,7 @@ protected:
 	bool DoFeed(const void* data, size_t size) override;
 	StringVal* DoGet() override;
 
-	DECLARE_SERIAL(MD5Val);
-
+	DECLARE_OPAQUE_VALUE(MD5Val)
 private:
 	EVP_MD_CTX* ctx;
 };
@@ -78,8 +214,7 @@ protected:
 	bool DoFeed(const void* data, size_t size) override;
 	StringVal* DoGet() override;
 
-	DECLARE_SERIAL(SHA1Val);
-
+	DECLARE_OPAQUE_VALUE(SHA1Val)
 private:
 	EVP_MD_CTX* ctx;
 };
@@ -100,8 +235,7 @@ protected:
 	bool DoFeed(const void* data, size_t size) override;
 	StringVal* DoGet() override;
 
-	DECLARE_SERIAL(SHA256Val);
-
+	DECLARE_OPAQUE_VALUE(SHA256Val)
 private:
 	EVP_MD_CTX* ctx;
 };
@@ -110,8 +244,6 @@ class EntropyVal : public OpaqueVal {
 public:
 	EntropyVal();
 
-	Val* DoClone(CloneState* state) override;
-
 	bool Feed(const void* data, size_t size);
 	bool Get(double *r_ent, double *r_chisq, double *r_mean,
 		 double *r_montepicalc, double *r_scc);
@@ -119,8 +251,7 @@ public:
 protected:
 	friend class Val;
 
-	DECLARE_SERIAL(EntropyVal);
-
+	DECLARE_OPAQUE_VALUE(EntropyVal)
 private:
 	RandTest state;
 };
@@ -149,8 +280,7 @@ protected:
 	BloomFilterVal();
 	explicit BloomFilterVal(OpaqueType* t);
 
-	DECLARE_SERIAL(BloomFilterVal);
-
+	DECLARE_OPAQUE_VALUE(BloomFilterVal)
 private:
 	// Disable.
 	BloomFilterVal(const BloomFilterVal&);
@@ -174,18 +304,16 @@ public:
 	BroType* Type() const;
 	bool Typify(BroType* type);
 
-
 	probabilistic::CardinalityCounter* Get()	{ return c; };
 
 protected:
 	CardinalityVal();
 
+	DECLARE_OPAQUE_VALUE(CardinalityVal)
 private:
 	BroType* type;
 	CompositeHash* hash;
 	probabilistic::CardinalityCounter* c;
-
-	DECLARE_SERIAL(CardinalityVal);
 };
 
 #endif