Merge remote-tracking branch 'origin/topic/awelzel/4605-conn-id-context'

* origin/topic/awelzel/4605-conn-id-context:
  NEWS: Adapt for conn_id$ctx introduction
  conn_key/fivetuple: Drop support for non conn_id records
  Conn: Move conn_id init and flip to IPBasedConnKey
  IPBasedConnKey: Add GetTransportProto() helper
  input/Manager: Ignore empty record types
  external: Bump commit hashes for external suites
  ip/vlan_fivetuple: Populate nested conn_id_context, not conn_id
  ConnKey: Extend DoPopulateConnIdVal() with ctx
  btest: Update tests and baselines after adding ctx to conn_id
  init-bare: Add conn_id_ctx to conn_id

testing/btest/scripts/base/frameworks/input/empty-nested-record.zeek

@@ -0,0 +1,50 @@
+# @TEST-DOC: Check that empty record types are ignored.
+#
+# @TEST-EXEC: zeek -b %INPUT >out
+# @TEST-EXEC: btest-diff .stderr
+# @TEST-EXEC: btest-diff out
+
+# @TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	s	r0.c0	r0.c1	r0.s
+#types	string	count	string
+string1	4242	4711	r0s-1
+string2	4343	-	r0s-2
+# @TEST-END-FILE
+
+redef exit_only_after_terminate = T;
+
+module Test;
+
+type EmptyRec: record { };
+
+type MyRec: record {
+	e0: EmptyRec;  # stuffing
+	c0: count;
+	c1: count &optional;
+	e1: EmptyRec;  # stuffing
+	s: string;
+};
+
+type Val: record {
+	s: string;
+	e0: EmptyRec;  # stuffing
+	r0: MyRec;
+	e1: EmptyRec;  # stuffing
+};
+
+event Test::line(description: Input::EventDescription, tpe: Input::Event, v: Val)
+	{
+	print tpe, v;
+	}
+
+event Input::end_of_data(name: string, source:string)
+	{
+	terminate();
+	}
+
+event zeek_init()
+	{
+	Input::add_event([$source="input.log", $name="file", $fields=Val, $ev=Test::line]);
+	}

testing/btest/scripts/base/utils/conn-ids.test

@@ -4,7 +4,7 @@
 @load base/utils/conn-ids
 
 global c: conn_id = [ $orig_h = 10.0.0.100, $orig_p = 10000/tcp,
-                      $resp_h = 10.0.0.200, $resp_p = 20000/tcp, $proto=6 ];
+                      $resp_h = 10.0.0.200, $resp_p = 20000/tcp, $proto=6, $ctx=[] ];
 
 print id_string(c);
 print reverse_id_string(c);

testing/btest/scripts/base/utils/directions-and-hosts.test

@@ -11,19 +11,19 @@ global remote_ip = 192.168.1.100;
 
 global local2local: conn_id = [
     $orig_h = 10.0.0.100, $orig_p = 10000/tcp,
-    $resp_h = 10.0.0.200, $resp_p = 20000/tcp, $proto=6 ];
+    $resp_h = 10.0.0.200, $resp_p = 20000/tcp, $proto=6, $ctx=[] ];
 
 global local2remote: conn_id = [
     $orig_h = 10.0.0.100,    $orig_p = 10000/tcp,
-    $resp_h = 192.168.1.100, $resp_p = 20000/tcp, $proto=6 ];
+    $resp_h = 192.168.1.100, $resp_p = 20000/tcp, $proto=6, $ctx=[] ];
 
 global remote2local: conn_id = [
     $orig_h = 192.168.1.100,    $orig_p = 10000/tcp,
-    $resp_h = 10.0.0.100,       $resp_p = 20000/tcp, $proto=6 ];
+    $resp_h = 10.0.0.100,       $resp_p = 20000/tcp, $proto=6, $ctx=[] ];
 
 global remote2remote: conn_id = [
     $orig_h = 192.168.1.100, $orig_p = 10000/tcp,
-    $resp_h = 192.168.1.200, $resp_p = 20000/tcp, $proto=6 ];
+    $resp_h = 192.168.1.200, $resp_p = 20000/tcp, $proto=6, $ctx=[] ];
 
 function test_host(ip: addr, h: Host, expect: bool)
 	{

testing/btest/scripts/policy/frameworks/conn_key/vlan_fivetuple.zeek

@@ -4,7 +4,7 @@
 # To create: tcprewrite --enet-vlan=add --enet-vlan-tag 20 --enet-vlan-cfi=1 --enet-vlan-pri=2 -i in.pcap -o out.pcap
 #
 # @TEST-EXEC: zeek -r $TRACES/vlan-collisions.pcap %INPUT
-# @TEST-EXEC: zeek-cut -m ts uid id.orig_h id.orig_p id.resp_h id.resp_p id.vlan id.inner_vlan orig_pkts resp_pkts service <conn.log >conn.log.cut
+# @TEST-EXEC: zeek-cut -m ts uid id.orig_h id.orig_p id.resp_h id.resp_p id.ctx.vlan id.ctx.inner_vlan orig_pkts resp_pkts service <conn.log >conn.log.cut
 # @TEST-EXEC: btest-diff conn.log.cut
 
 # Default operation: Zeek isn't VLAN-aware, a single conn.log entry results.
@@ -27,7 +27,7 @@ redef ConnKey::factory = ConnKey::CONNKEY_VLAN_FIVETUPLE;
 
 # Add an extra field before the VLAN ones, to throw off any fixed-offset code.
 
-redef record conn_id += {
+redef record conn_id_ctx += {
 	foo: int &default=1;
 };
 
@@ -35,11 +35,11 @@ redef record conn_id += {
 
 # @TEST-START-NEXT
 
-# Add the right fields, but in the wrong order. (zeek-cut obscures the difference.)
+# Add the right fields, but in a different order. (zeek-cut obscures the difference.)
 
-redef record conn_id += {
-	inner_vlan: int      &log &optional;
-	vlan: int      &log &optional;
+redef record conn_id_ctx += {
+	inner_vlan: int &log &optional;
+	vlan: int &log &optional;
 };
 
 redef ConnKey::factory = ConnKey::CONNKEY_VLAN_FIVETUPLE;
@@ -48,9 +48,9 @@ redef ConnKey::factory = ConnKey::CONNKEY_VLAN_FIVETUPLE;
 
 # Add the right fields, but with the wrong types.
 
-redef record conn_id += {
-	vlan: string      &log &optional;
-	inner_vlan: string      &log &optional;
+redef record conn_id_ctx += {
+	vlan: string &log &optional;
+	inner_vlan: string &log &optional;
 };
 
 redef ConnKey::factory = ConnKey::CONNKEY_VLAN_FIVETUPLE;

testing/btest/scripts/policy/frameworks/conn_key/vlan_fivetuple_conn_id_ctx_usage.zeek

@@ -0,0 +1,31 @@
+# @TEST-DOC: Demo for using conn_id$ctx in a table to track HTTP request methods per originator IP and their context.
+#
+# The test pcap has 3 overlapping healthy TCP connections, each with different VLAN tagging: none, one VLAN tag, two VLAN tags.
+# To create: tcprewrite --enet-vlan=add --enet-vlan-tag 20 --enet-vlan-cfi=1 --enet-vlan-pri=2 -i in.pcap -o out.pcap
+#
+# @TEST-EXEC: zeek -b -r $TRACES/vlan-collisions.pcap base/protocols/http ./count-http-request-methods.zeek %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+# Default operation: Zeek isn't VLAN-aware, a single conn.log entry results.
+
+# @TEST-START-NEXT
+
+# Switch to VLAN-aware flow tuples: multiple conn.log entries with full
+# information.
+
+@load frameworks/conn_key/vlan_fivetuple
+
+# @TEST-START-FILE count-http-request-methods.zeek
+global http_requests: table[addr, conn_id_ctx, string] of count &default=0;
+
+event http_request(c: connection, method: string, original_URI: string, unescaped_URI: string, version: string)
+	{
+	++http_requests[c$id$orig_h, c$id$ctx, method];
+	}
+
+event zeek_done()
+	{
+	for ( [h, ctx, method], c in http_requests )
+		print h, ctx, method, c;
+	}
+# @TEST-END-FILE

testing/btest/scripts/policy/frameworks/conn_key/vlan_fivetuple_lookup_connection.zeek

@@ -0,0 +1,37 @@
+# @TEST-DOC: Verify connections can be be looked up using lookup_connection() when using vlan aware conn_id's
+#
+# The test pcap has 3 overlapping healthy TCP connections, each with different VLAN tagging: none, one VLAN tag, two VLAN tags.
+# To create: tcprewrite --enet-vlan=add --enet-vlan-tag 20 --enet-vlan-cfi=1 --enet-vlan-pri=2 -i in.pcap -o out.pcap
+#
+# @TEST-EXEC: zeek -r $TRACES/vlan-collisions.pcap %INPUT
+# @TEST-EXEC: zeek-cut -m ts uid id.orig_h id.orig_p id.resp_h id.resp_p id.ctx.vlan id.ctx.inner_vlan orig_pkts resp_pkts service <conn.log >conn.log.cut
+#
+# @TEST-EXEC: btest-diff conn.log.cut
+# @TEST-EXEC: btest-diff .stderr
+
+@load frameworks/conn_key/vlan_fivetuple
+
+event new_connection(c: connection)
+	{
+	local c1 = lookup_connection(c$id);
+	local c2 = lookup_connection(copy(c$id));
+
+	local c3_id = conn_id($orig_h=c$id$orig_h, $orig_p=c$id$orig_p,
+	                      $resp_h=c$id$resp_h, $resp_p=c$id$resp_p,
+	                      $proto=c$id$proto, $ctx=copy(c$id$ctx));
+	local c3 = lookup_connection(c3_id);
+
+	# Ensure all the uids are the same!
+	assert c$uid == c1$uid && c1$uid == c2$uid && c2$uid == c3$uid;
+	}
+
+event new_connection(c: connection)
+	{
+	assert connection_exists(c$id);
+
+	local nx_id = copy(c$id);
+	nx_id$ctx = copy(c$id$ctx);
+	nx_id$ctx$vlan = 1000;
+	nx_id$ctx$inner_vlan = 2000;
+	assert ! connection_exists(nx_id);
+	}

testing/btest/scripts/policy/frameworks/software/vulnerable.zeek

@@ -13,11 +13,11 @@ redef Software::vulnerable_versions += {
 
 event zeek_init()
 	{
-	Software::found([$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=4.3.2.1, $resp_p=80/tcp, $proto=6],
+	Software::found([$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=4.3.2.1, $resp_p=80/tcp, $proto=6, $ctx=[]],
 	                [$name="Java", $host=1.2.3.4, $version=[$major=1, $minor=7, $minor2=0, $minor3=15]]);
-	Software::found([$orig_h=1.2.3.5, $orig_p=1234/tcp, $resp_h=4.3.2.1, $resp_p=80/tcp, $proto=6],
+	Software::found([$orig_h=1.2.3.5, $orig_p=1234/tcp, $resp_h=4.3.2.1, $resp_p=80/tcp, $proto=6, $ctx=[]],
 	                [$name="Java", $host=1.2.3.5, $version=[$major=1, $minor=6, $minor2=0, $minor3=43]]);
-	Software::found([$orig_h=1.2.3.6, $orig_p=1234/tcp, $resp_h=4.3.2.1, $resp_p=80/tcp, $proto=6],
+	Software::found([$orig_h=1.2.3.6, $orig_p=1234/tcp, $resp_h=4.3.2.1, $resp_p=80/tcp, $proto=6, $ctx=[]],
 	                [$name="Java", $host=1.2.3.6, $version=[$major=1, $minor=6, $minor2=0, $minor3=50]]);
 
 	}