Reimplement serialization infrastructure for OpaqueVals.

We need this to sender through Broker, and we also leverage it for
cloning opaques. The serialization methods now produce Broker data
instances directly, and no longer go through the binary formatter.

Summary of the new API for types derived from OpaqueVal:

    - Add DECLARE_OPAQUE_VALUE(<class>) to the class declaration
    - Add IMPLEMENT_OPAQUE_VALUE(<class>) to the class' implementation file
    - Implement these two methods (which are declated by the 1st macro):
        - broker::data DoSerialize() const
        - bool DoUnserialize(const broker::data& data)

This machinery should work correctly from dynamic plugins as well.

OpaqueVal provides a default implementation of DoClone() as well that
goes through serialization. Derived classes can provide a more
efficient version if they want.

The declaration of the "OpaqueVal" class has moved into the header
file "OpaqueVal.h", along with the new serialization infrastructure.
This is breaking existing code that relies on the location, but
because the API is changing anyways that seems fine.

This adds an internal BiF
"Broker::__opaque_clone_through_serialization" that does what the name
says: deep-copying an opaque by serializing, then-deserializing. That
can be used to tests the new functionality from btests.

Not quite done yet. TODO:
    - Not all tests pass yet:
        [  0%] language.named-set-ctors ... failed
        [ 16%] language.copy-all-opaques ... failed
        [ 33%] language.set-type-checking ... failed
        [ 50%] language.table-init-container-ctors ... failed
        [ 66%] coverage.sphinx-zeekygen-docs ... failed
        [ 83%] scripts.base.frameworks.sumstats.basic-cluster ... failed

      (Some of the serialization may still be buggy.)

    - Clean up the code a bit more.

src/broker/Data.cc

@@ -126,11 +126,6 @@ struct val_converter {
 
 			return rval->Ref();
 			}
-		case TYPE_OPAQUE:
-			{
-			// Fixme: Johanna
-			return nullptr;
-			}
 		default:
 			return nullptr;
 		}
@@ -430,6 +425,8 @@ struct val_converter {
 			auto rval = new PatternVal(re);
 			return rval;
 			}
+		else if ( type->Tag() == TYPE_OPAQUE )
+			return OpaqueVal::Unserialize(a);
 
 		return nullptr;
 		}
@@ -504,12 +501,6 @@ struct type_checker {
 
 			return true;
 			}
-		case TYPE_OPAQUE:
-			{
-			// TODO
-			// Fixme: johanna
-			return false;
-			}
 		default:
 			return false;
 		}
@@ -756,6 +747,11 @@ struct type_checker {
 
 			return true;
 			}
+		else if ( type->Tag() == TYPE_OPAQUE )
+			// Correct if successful. TODO: Should avoid do the
+			// full unserialization here, and just check the
+			// type.
+			return OpaqueVal::Unserialize(a);
 
 		return false;
 		}
@@ -970,10 +966,17 @@ broker::expected<broker::data> bro_broker::val_to_data(Val* v)
 		broker::vector rval = {p->PatternText(), p->AnywherePatternText()};
 		return {std::move(rval)};
 		}
-	// Fixme: johanna
-	// case TYPE_OPAQUE:
-	//	{
-	//	}
+	case TYPE_OPAQUE:
+		{
+		auto c = v->AsOpaqueVal()->Serialize();
+		if ( ! c )
+			{
+			reporter->Error("unsupported opaque type for serialization");
+			break;
+			}
+
+		return {c};
+		}
 	default:
 		reporter->Error("unsupported Broker::Data type: %s",
 		                type_name(v->Type()->Tag()));
@@ -1109,6 +1112,115 @@ Val* bro_broker::DataVal::castTo(BroType* t)
 	return data_to_val(data, t);
 	}
 
+IMPLEMENT_OPAQUE_VALUE(bro_broker::DataVal)
+
+broker::data bro_broker::DataVal::DoSerialize() const
+	{
+	return data;
+	}
+
+bool bro_broker::DataVal::DoUnserialize(const broker::data& data_)
+	{
+	data = data_;
+	return true;
+	}
+
+IMPLEMENT_OPAQUE_VALUE(bro_broker::SetIterator)
+
+broker::data bro_broker::SetIterator::DoSerialize() const
+	{
+	return broker::vector{dat, *it};
+	}
+
+bool bro_broker::SetIterator::DoUnserialize(const broker::data& data)
+	{
+	auto v = caf::get_if<broker::vector>(&data);
+	if ( ! (v && v->size() == 2) )
+		return false;
+
+	auto x = caf::get_if<broker::set>(&(*v)[0]);
+
+	// We set the iterator by finding the element it used to point to.
+	// This is not perfect, as there's no guarantee that the restored
+	// container will list the elements in the same order. But it's as
+	// good as we can do, and it should generally work out.
+	if( x->find((*v)[1]) == x->end() )
+		return false;
+
+	dat = *x;
+	it = dat.find((*v)[1]);
+	return true;
+	}
+
+IMPLEMENT_OPAQUE_VALUE(bro_broker::TableIterator)
+
+broker::data bro_broker::TableIterator::DoSerialize() const
+	{
+	return broker::vector{dat, it->first};
+	}
+
+bool bro_broker::TableIterator::DoUnserialize(const broker::data& data)
+	{
+	auto v = caf::get_if<broker::vector>(&data);
+	if ( ! (v && v->size() == 2) )
+		return false;
+
+	auto x = caf::get_if<broker::table>(&(*v)[0]);
+
+	// We set the iterator by finding the element it used to point to.
+	// This is not perfect, as there's no guarantee that the restored
+	// container will list the elements in the same order. But it's as
+	// good as we can do, and it should generally work out.
+	if( x->find((*v)[1]) == x->end() )
+		return false;
+
+	dat = *x;
+	it = dat.find((*v)[1]);
+	return true;
+	}
+
+IMPLEMENT_OPAQUE_VALUE(bro_broker::VectorIterator)
+
+broker::data bro_broker::VectorIterator::DoSerialize() const
+	{
+	return broker::vector{dat, it - dat.begin()};
+	}
+
+bool bro_broker::VectorIterator::DoUnserialize(const broker::data& data)
+	{
+	auto v = caf::get_if<broker::vector>(&data);
+	if ( ! (v && v->size() == 2) )
+		return false;
+
+	auto x = caf::get_if<broker::vector>(&(*v)[0]);
+	auto y = caf::get_if<broker::vector::difference_type>(&(*v)[1]);
+
+	dat = *x;
+	it = dat.begin() + *y;
+	return true;
+	}
+
+IMPLEMENT_OPAQUE_VALUE(bro_broker::RecordIterator)
+
+broker::data bro_broker::RecordIterator::DoSerialize() const
+	{
+	return broker::vector{dat, it - dat.begin()};
+	}
+
+bool bro_broker::RecordIterator::DoUnserialize(const broker::data& data)
+	{
+	auto v = caf::get_if<broker::vector>(&data);
+	if ( ! (v && v->size() == 2) )
+		return false;
+
+	auto x = caf::get_if<broker::vector>(&(*v)[0]);
+	auto y = caf::get_if<broker::vector::difference_type>(&(*v)[1]);
+
+	dat = *x;
+	it = dat.begin() + *y;
+	return true;
+	}
+
 broker::data bro_broker::threading_field_to_data(const threading::Field* f)
 	{
 	auto name = f->name;