Merge branch 'paraglob' of https://github.com/ZekeMedley/zeek

* 'paraglob' of https://github.com/ZekeMedley/zeek:
  Add leak test to paraglob.
  Catch paraglob serialization errors in DoClone.
  Update paraglob serialization.
  Stop execution on paraglob error.
  Update paraglob submodule
  Change C++11 detection in paraglob.
  Make paraglob serializable and copyable.
  Initial paraglob integration.

I made a bunch of small changes:
 * paraglob now deals better with \0 characters
 * I rolled back the changes to Binary Serialization format,
 * there were some small formatting issue
 * the error output was slightly unsafe
 * build_unique is now in util.h.

and perhaps a few more small things.

testing/btest/Baseline/language.copy-all-opaques/out

@@ -23,3 +23,6 @@
 ============ Entropy
 [entropy=4.715374, chi_square=591.981818, mean=75.472727, monte_carlo_pi=4.0, serial_correlation=-0.11027]
 [entropy=4.715374, chi_square=591.981818, mean=75.472727, monte_carlo_pi=4.0, serial_correlation=-0.11027]
+============ Paraglob
+T
+T

testing/btest/Baseline/language.paraglob-serialization/recv.recv.out

@@ -0,0 +1,12 @@
+receiver added peer: endpoint=127.0.0.1 msg=handshake successful
+is_remote should be T, and is, T
+receiver got ping number: 1
+[*, *ello, hello]
+is_remote should be T, and is, T
+receiver got ping number: 2
+[*, *ello, hello]
+is_remote should be T, and is, T
+receiver got ping number: 3
+[*, *ello, hello]
+is_remote should be T, and is, T
+[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=4, num_events_outgoing=3, num_logs_incoming=0, num_logs_outgoing=1, num_ids_incoming=0, num_ids_outgoing=0]

testing/btest/Baseline/language.paraglob-serialization/send.send.out

@@ -0,0 +1,11 @@
+Starting send.
+[*, *ello, hello]
+is_remote should be F, and is, F
+sender added peer: endpoint=127.0.0.1 msg=received handshake from remote core
+is_remote should be T, and is, T
+sender got pong number: 1
+is_remote should be T, and is, T
+sender got pong number: 2
+is_remote should be T, and is, T
+sender got pong number: 3
+sender lost peer: endpoint=127.0.0.1 msg=lost remote peer

testing/btest/Baseline/language.paraglob/out

@@ -0,0 +1,9 @@
+[T, T, T, T, T]
+T
+F
+[*, *og, d?g, d[!wl]g]
+[once]
+[]
+[*.gov*, *malware*]
+[z*ro]
+[*.gov*, *malware*]

testing/btest/core/leaks/paraglob.zeek

@@ -0,0 +1,34 @@
+# Needs perftools support.
+#
+# @TEST-GROUP: leaks
+#
+# @TEST-REQUIRES: zeek  --help 2>&1 | grep -q mem-leaks
+#
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run zeek zeek -m -b -r $TRACES/http/get.trace %INPUT
+# @TEST-EXEC: btest-bg-wait 120
+
+event new_connection (c : connection)
+{
+  local v1 = vector("*", "d?g", "*og", "d?", "d[!wl]g");
+  local v2 = vector("once", "!o*", "once");
+  local v3 = vector("https://*.google.com/*", "*malware*", "*.gov*");
+
+  local p1 = paraglob_init(v1);
+  local p2: opaque of paraglob = paraglob_init(v2);
+  local p3 = paraglob_init(v3);
+  local p_eq = paraglob_init(v1);
+
+  # paraglob_init should not modify v1
+  print (v1 == vector("*", "d?g", "*og", "d?", "d[!wl]g"));
+  # p_eq and p1 should be the same paraglobs
+  print paraglob_equals(p_eq, p1);
+
+  print paraglob_get(p1, "dog");
+
+
+  print paraglob_get(p2, "once");
+  print paraglob_get(p3, "www.strange-malware-domain.gov");
+
+  local large_glob: opaque of paraglob = paraglob_init(v3);
+  print paraglob_get(large_glob, "www.strange-malware-domain.gov");
+}

testing/btest/language/copy-all-opaques.zeek

@@ -82,4 +82,12 @@ event zeek_init()
 	local handle2 = copy(handle);
 	print entropy_test_finish(handle);
 	print entropy_test_finish(handle2);
+
+	print "============ Paraglob";
+	local p = paraglob_init(vector("https://*.google.com/*", "*malware*", "*.gov*"));
+	local p2 = copy(p);
+	print paraglob_equals(p, p2);
+	# A get operation shouldn't change the paraglob
+	paraglob_get(p, "whitehouse.gov");
+	print paraglob_equals(p, p2);
 	}

testing/btest/language/paraglob-serialization.zeek

@@ -0,0 +1,102 @@
+# @TEST-PORT: BROKER_PORT
+#
+# @TEST-EXEC: btest-bg-run recv "zeek -B broker -b ../recv.zeek >recv.out"
+# @TEST-EXEC: btest-bg-run send "zeek -B broker -b ../send.zeek >send.out"
+#
+# @TEST-EXEC: btest-bg-wait 30
+# @TEST-EXEC: btest-diff recv/recv.out
+# @TEST-EXEC: btest-diff send/send.out
+
+@TEST-START-FILE send.zeek
+
+redef exit_only_after_terminate = T;
+
+global event_count = 0;
+global p: opaque of paraglob = paraglob_init(vector("hello", "*ello", "*"));
+
+global ping: event(msg: opaque of paraglob, c: count);
+
+event zeek_init()
+    {
+    print "Starting send.";
+    print paraglob_get(p, "hello");
+    Broker::subscribe("bro/event/my_topic");
+    Broker::peer("127.0.0.1", 9999/tcp);
+    print "is_remote should be F, and is", is_remote_event();
+    }
+
+function send_event()
+    {
+    ++event_count;
+    local e = Broker::make_event(ping, p, event_count);
+    Broker::publish("bro/event/my_topic", e);
+    }
+
+event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
+    {
+    print fmt("sender added peer: endpoint=%s msg=%s",
+    endpoint$network$address, msg);
+    send_event();
+    }
+
+event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
+    {
+    print fmt("sender lost peer: endpoint=%s msg=%s",
+    endpoint$network$address, msg);
+    terminate();
+    }
+
+event pong(msg: opaque of paraglob, n: count)
+    {
+    print "is_remote should be T, and is", is_remote_event();
+    print fmt("sender got pong number: %s", n);
+    send_event();
+    }
+
+@TEST-END-FILE
+
+
+@TEST-START-FILE recv.zeek
+
+redef exit_only_after_terminate = T;
+
+const events_to_recv = 3;
+
+global handler: event(msg: string, c: count);
+global auto_handler: event(msg: string, c: count);
+
+global pong: event(msg: opaque of paraglob, c: count);
+
+event zeek_init()
+        {
+        Broker::subscribe("bro/event/my_topic");
+        Broker::listen("127.0.0.1", 9999/tcp);
+        }
+
+event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
+        {
+        print fmt("receiver added peer: endpoint=%s msg=%s", endpoint$network$address, msg);
+        }
+
+event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
+        {
+        print fmt("receiver lost peer: endpoint=%s msg=%s", endpoint$network$address, msg);
+        }
+
+event ping(msg: opaque of paraglob, n: count)
+  {
+    print "is_remote should be T, and is", is_remote_event();
+    if ( n > events_to_recv )
+      {
+      print get_broker_stats();
+      terminate();
+      return;
+      }
+    print fmt("receiver got ping number: %s", n);
+    print paraglob_get(msg, "hello");
+
+    local e = Broker::make_event(pong, msg, n);
+    Broker::publish("bro/event/my_topic", e);
+  }
+
+@TEST-END-FILE

testing/btest/language/paraglob.zeek

@@ -0,0 +1,41 @@
+# @TEST-EXEC: zeek -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event zeek_init ()
+{
+	local v1 = vector("*", "d?g", "*og", "d?", "d[!wl]g");
+	local v2 = vector("once", "!o*", "once");
+	local v3 = vector("https://*.google.com/*", "*malware*", "*.gov*");
+	local v4 = vector("z*ro");
+
+	local p1 = paraglob_init(v1);
+	local p2: opaque of paraglob = paraglob_init(v2);
+	local p3 = paraglob_init(v3);
+	local p4 = paraglob_init(v4);
+	local p_eq = paraglob_init(v1);
+
+	# paraglob_init should not modify v1
+	print (v1 == vector("*", "d?g", "*og", "d?", "d[!wl]g"));
+	# p_eq and p1 should be the same paraglobs
+	print paraglob_equals(p_eq, p1);
+	print paraglob_equals(p1, p2);
+
+	print paraglob_get(p1, "dog");
+
+
+	print paraglob_get(p2, "once");
+	print paraglob_get(p2, "nothing");
+	print paraglob_get(p3, "www.strange-malware-domain.gov");
+	print paraglob_get(p4, "zero\0zero");
+
+	# This looks like a lot, but really should complete quickly.
+	# Paraglob should stop addition of duplicate patterns.
+	local i = 1000000;
+	while (i > 0) {
+		i = i - 1;
+		v3 += v3[1];
+	}
+
+	local large_glob: opaque of paraglob = paraglob_init(v3);
+	print paraglob_get(large_glob, "www.strange-malware-domain.gov");
+}