Merge remote-tracking branch 'origin/topic/awelzel/4177-4178-custom-event-metadata-part-2'

* origin/topic/awelzel/4177-4178-custom-event-metadata-part-2:
  Event: Bail on add_missing_remote_network_timestamp without add_network_timestamp
  btest/plugin: Test custom metadata publish
  NEWS: Add note about generic event metadata
  cluster: Remove deprecated Event constructor
  cluster: Remove some explicit timestamp handling
  broker/Manager: Fetch and forward all metadata from events
  Event/init-bare: Add add_missing_remote_network_timestamp logic
  cluster/Backend/DoProcessEvent: Use generic metadata, not just timestamps
  cluster/Event: Support moving args and metadata from event
  cluster/serializer/broker: Support generic metadata
  cluster/Event: Generic metadata support
  Event: Use -1.0 for undefined/unset timestamps
  cluster: Use shorter obj_desc versions
  Desc: Add obj_desc() / obj_desc_short() overloads for IntrusivePtr

testing/btest/Baseline/core.event-metadata.add-missing-remote-network-timestamp-error/.stderr

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+fatal error: Setting EventMetadata::add_missing_remote_network_timestamp is only valid together with EventMetadata::add_network_timestamp

testing/btest/Baseline/core.event-metadata.current-event-time/output

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+new_connection add_network_timestamp=T current_event_time=1362692526.869344 network_timestamp=[1362692526.869344]
+new_connection add_network_timestamp=F current_event_time=-1.0 network_timestamp=[]

testing/btest/Baseline/plugins.publish-event-metadata/manager..stderr

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+received termination signal

testing/btest/Baseline/plugins.publish-event-metadata/manager..stdout

@@ -0,0 +1,33 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+0.000000   InitPostScript
+App::test_event(1) |mdv|=1
+[id=App::CUSTOM_METADATA_STRING, val=testing string metadata]
+custom metadata string, [testing string metadata]
+custom metadata count, []
+custom metadata table, []
+App::test_event(2) |mdv|=1
+[id=App::CUSTOM_METADATA_COUNT, val=42424242]
+custom metadata string, []
+custom metadata count, [42424242]
+custom metadata table, []
+App::test_event(3) |mdv|=1
+[id=App::CUSTOM_METADATA_TABLE, val={
+[key1] = val1
+}]
+custom metadata string, []
+custom metadata count, []
+custom metadata table, [{
+[key1] = val1
+}]
+App::test_event(4) |mdv|=4
+[id=App::CUSTOM_METADATA_TABLE, val={
+[key1] = val1
+}]
+[id=App::CUSTOM_METADATA_COUNT, val=41414242]
+[id=App::CUSTOM_METADATA_STRING, val=testing string metadata]
+[id=App::CUSTOM_METADATA_STRING, val=more string metadata]
+custom metadata string, [testing string metadata, more string metadata]
+custom metadata count, [41414242]
+custom metadata table, [{
+[key1] = val1
+}]

testing/btest/Baseline/plugins.publish-event-metadata/output

@@ -0,0 +1,4 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+0.000000   InitPostScript
+0.000000   HookPublishEvent backend=Broker topic=/test/topic event=App::test_event
+0.000000   HookPublishEvent /test/topic(App::test_event)

testing/btest/Baseline/plugins.publish-event-metadata/worker-1..stderr

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+received termination signal

testing/btest/Baseline/plugins.publish-event-metadata/worker-1..stdout

@@ -0,0 +1,6 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+0.000000   InitPostScript
+0.000000 HookPublishEvent backend=Broker topic=topic1 event=App::test_event
+0.000000 HookPublishEvent backend=Broker topic=topic2 event=App::test_event
+0.000000 HookPublishEvent backend=Broker topic=topic3 event=App::test_event
+0.000000 HookPublishEvent backend=Broker topic=topic4 event=App::test_event

testing/btest/broker/remote_event_ts_compat.zeek

@@ -19,6 +19,11 @@ redef exit_only_after_terminate = T;
 redef allow_network_time_forward = F;
 redef EventMetadata::add_network_timestamp = T;
 
+# This is needed so that the receiving node sets its
+# own local network timestamp on remote events that do
+# not have network timestamp metadata.
+redef EventMetadata::add_missing_remote_network_timestamp = T;
+
 event zeek_init()
 	{
 	Broker::subscribe(getenv("TOPIC"));

testing/btest/core/event-metadata/add-missing-remote-network-timestamp-error.zeek

@@ -0,0 +1,7 @@
+# @TEST-DOC: Using add_missing_remote_network_timestamp without add_network_timestamp is an error.
+#
+# @TEST-EXEC-FAIL: zeek -b %INPUT
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr
+
+redef EventMetadata::add_network_timestamp = F;
+redef EventMetadata::add_missing_remote_network_timestamp = T;

testing/btest/core/event-metadata/current-event-time.zeek

@@ -0,0 +1,14 @@
+# @TEST-DOC: Check current_event_time() produces the same as event metadata, or else -1.0
+#
+# @TEST-EXEC: zeek -r $TRACES/http/get.trace %INPUT EventMetadata::add_network_timestamp=T >> output 2>&1
+# @TEST-EXEC: zeek -r $TRACES/http/get.trace %INPUT EventMetadata::add_network_timestamp=F >> output 2>&1
+#
+# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff output
+
+
+event new_connection(c: connection)
+	{
+	print fmt("new_connection add_network_timestamp=%s current_event_time=%s network_timestamp=%s",
+	          EventMetadata::add_network_timestamp, current_event_time(),
+	          EventMetadata::current(EventMetadata::NETWORK_TIMESTAMP));
+	}

testing/btest/plugins/publish-event-metadata-plugin/src/Plugin.cc

@@ -0,0 +1,96 @@
+
+#include "Plugin.h"
+
+#include <zeek/Desc.h>
+#include <zeek/ID.h>
+#include <zeek/Reporter.h>
+#include <zeek/Val.h>
+#include <zeek/cluster/Backend.h>
+#include <cstdio>
+#include <string>
+
+namespace btest::plugin::Demo_PublishEventMetadata {
+Plugin plugin;
+}
+
+using namespace btest::plugin::Demo_PublishEventMetadata;
+
+zeek::plugin::Configuration Plugin::Configure() {
+    EnableHook(zeek::plugin::HOOK_PUBLISH_EVENT);
+
+    zeek::plugin::Configuration config;
+    config.name = "Demo::PublishEventMetadata";
+    config.description = "For testing metadata publish";
+    config.version.major = 1;
+    config.version.minor = 0;
+    config.version.patch = 0;
+    return config;
+}
+
+void Plugin::InitPostScript() {
+    std::fprintf(stdout, "%.6f %-15s\n", zeek::run_state::network_time, "  InitPostScript");
+}
+
+bool Plugin::HookPublishEvent(zeek::cluster::Backend& backend, const std::string& topic,
+                              zeek::cluster::detail::Event& event) {
+    std::fprintf(stdout, "%.6f %s backend=%s topic=%s event=%s\n", zeek::run_state::network_time, "HookPublishEvent",
+                 backend.Name().c_str(), topic.c_str(), std::string(event.HandlerName()).c_str());
+
+    const auto& table_type = zeek::id::find_type<zeek::TableType>("table_string_of_string");
+    const auto& string_md = zeek::id::find_val<zeek::EnumVal>("App::CUSTOM_METADATA_STRING");
+    auto count_md = zeek::id::find_val<zeek::EnumVal>("App::CUSTOM_METADATA_COUNT");
+    auto table_md = zeek::id::find_val<zeek::EnumVal>("App::CUSTOM_METADATA_TABLE");
+
+    if ( ! count_md || ! table_md )
+        zeek::reporter->FatalError("Could not find required enum values");
+
+    if ( topic == "topic1" ) {
+        if ( ! event.AddMetadata(string_md, zeek::make_intrusive<zeek::StringVal>("testing string metadata")) ) {
+            zeek::reporter->FatalError("Failed to add string metadata");
+        }
+    }
+    else if ( topic == "topic2" ) {
+        if ( ! event.AddMetadata(count_md, zeek::val_mgr->Count(42424242)) ) {
+            zeek::reporter->FatalError("Failed to add count metadata");
+        }
+    }
+    else if ( topic == "topic3" ) {
+        auto tv = zeek::make_intrusive<zeek::TableVal>(table_type);
+        if ( ! tv->Assign(zeek::make_intrusive<zeek::StringVal>("key1"),
+                          zeek::make_intrusive<zeek::StringVal>("val1")) )
+            zeek::reporter->FatalError("Could not update table value");
+
+        if ( ! event.AddMetadata(table_md, tv) ) {
+            zeek::reporter->FatalError("Failed to add table metadata");
+        }
+    }
+    else if ( topic == "topic4" ) {
+        auto tv = zeek::make_intrusive<zeek::TableVal>(table_type);
+        if ( ! tv->Assign(zeek::make_intrusive<zeek::StringVal>("key1"),
+                          zeek::make_intrusive<zeek::StringVal>("val1")) )
+            zeek::reporter->FatalError("Could not update table value");
+
+        if ( ! event.AddMetadata(table_md, tv) ) {
+            zeek::reporter->FatalError("Failed to add table metadata");
+        }
+
+        if ( ! event.AddMetadata(count_md, zeek::val_mgr->Count(41414242)) ) {
+            zeek::reporter->FatalError("Failed to add string metadata");
+        }
+
+        if ( ! event.AddMetadata(string_md, zeek::make_intrusive<zeek::StringVal>("testing string metadata")) ) {
+            zeek::reporter->FatalError("Failed to add string metadata");
+        }
+
+        // Event metadata is just a vector, so can have duplicate entries.
+        if ( ! event.AddMetadata(string_md, zeek::make_intrusive<zeek::StringVal>("more string metadata")) ) {
+            zeek::reporter->FatalError("Failed to add string metadata");
+        }
+    }
+    else {
+        zeek::reporter->FatalError("Unhandled topic %s", topic.c_str());
+    }
+
+
+    return true;
+}

testing/btest/plugins/publish-event-metadata-plugin/src/Plugin.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include <zeek/plugin/Plugin.h>
+#include <string>
+
+namespace btest::plugin::Demo_PublishEventMetadata {
+
+class Plugin : public zeek::plugin::Plugin {
+protected:
+    zeek::plugin::Configuration Configure() override;
+    void InitPostScript() override;
+
+    bool HookPublishEvent(zeek::cluster::Backend& backend, const std::string& topic,
+                          zeek::cluster::detail::Event& event) override;
+};
+
+extern Plugin plugin;
+
+} // namespace btest::plugin::Demo_PublishEventMetadata

testing/btest/plugins/publish-event-metadata.zeek

@@ -0,0 +1,77 @@
+# @TEST-DOC: Smoke test sending metadata from a worker to a manager. The manager uses script level functions.
+#
+# @TEST-EXEC: ${DIST}/auxil/zeek-aux/plugin-support/init-plugin -u . Demo PublishEventMetadata
+# @TEST-EXEC: cp -r %DIR/publish-event-metadata-plugin/* .
+# @TEST-EXEC: ./configure --zeek-dist=${DIST} && make
+#
+# @TEST-PORT: BROKER_MANAGER_PORT
+# @TEST-PORT: BROKER_WORKER1_PORT
+#
+# @TEST-EXEC: cp $FILES/broker/cluster-layout.zeek .
+#
+# @TEST-EXEC: btest-bg-run manager ZEEKPATH=$ZEEKPATH:.. ZEEK_PLUGIN_PATH=`pwd` CLUSTER_NODE=manager zeek -b Demo::PublishEventMetadata %INPUT
+# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. ZEEK_PLUGIN_PATH=`pwd` CLUSTER_NODE=worker-1 zeek -b Demo::PublishEventMetadata %INPUT
+# @TEST-EXEC: btest-bg-wait 10
+# @TEST-EXEC: btest-diff manager/.stdout
+# @TEST-EXEC: btest-diff manager/.stderr
+# @TEST-EXEC: btest-diff worker-1/.stdout
+# @TEST-EXEC: TEST_DIFF_CANONIFIER='grep -v PEER_UNAVAILABLE' btest-diff worker-1/.stderr
+
+redef allow_network_time_forward = F;
+
+@load frameworks/cluster/experimental
+
+module App;
+
+export {
+	global test_event: event(c: count);
+
+	redef enum EventMetadata::ID += {
+		CUSTOM_METADATA_STRING = 4711,
+		CUSTOM_METADATA_COUNT  = 4712,
+		CUSTOM_METADATA_TABLE  = 4713,
+	};
+}
+
+event App::test_event(c: count)
+	{
+	local mdv = EventMetadata::current_all();
+	print fmt("App::test_event(%s) |mdv|=%s", c, |mdv|);
+	for ( _, md in mdv )
+		print md;
+
+	print "custom metadata string", EventMetadata::current(App::CUSTOM_METADATA_STRING);
+	print "custom metadata count", EventMetadata::current(App::CUSTOM_METADATA_COUNT);
+	print "custom metadata table", EventMetadata::current(App::CUSTOM_METADATA_TABLE);
+
+	if ( c == 4 )
+		terminate();
+	}
+
+event zeek_init() &priority=20
+	{
+	assert EventMetadata::register(CUSTOM_METADATA_STRING, string);
+	assert EventMetadata::register(CUSTOM_METADATA_COUNT, count);
+	assert EventMetadata::register(CUSTOM_METADATA_TABLE, table[string] of string);
+
+	Cluster::subscribe("topic1");
+	Cluster::subscribe("topic2");
+	Cluster::subscribe("topic3");
+	Cluster::subscribe("topic4");
+	}
+
+event Cluster::Experimental::cluster_started()
+	{
+	if ( Cluster::node == "worker-1" )
+		{
+		Cluster::publish("topic1", test_event, 1);
+		Cluster::publish("topic2", test_event, 2);
+		Cluster::publish("topic3", test_event, 3);
+		Cluster::publish("topic4", test_event, 4);
+		}
+	}
+
+event Cluster::node_down(name: string, id: string)
+	{
+	terminate();
+	}