Merge remote-tracking branch 'origin/topic/vern/script-opt-maint.Dec22'

* origin/topic/vern/script-opt-maint.Dec22:
  maintenance updates for -O C++
  maintenance updates for -O ZAM
  update test suite to avoid GH-2385 problems, including incorrect typing

CHANGES

@@ -1,3 +1,9 @@
+5.2.0-dev.422 | 2022-12-09 16:07:47 +0100
+
+  * maintenance updates for -O C++ (Vern Paxson, Corelight)
+
+  * maintenance updates for -O ZAM (Vern Paxson, Corelight)
+
 5.2.0-dev.418 | 2022-12-08 10:28:01 +0100
 
   * dfa: Remove transition_counter (Arne Welzel, Corelight)

VERSION

@@ -1 +1 @@
-5.2.0-dev.418
+5.2.0-dev.422

src/script_opt/CPP/Exprs.cc

@@ -1136,7 +1136,11 @@ string CPPCompile::GenDirectAssign(const ExprPtr& lhs, const string& rhs_native,
 			}
 		}
 	else
+		{
 		gen = name + " = " + rhs_native;
+		if ( ! top_level )
+			gen = NativeToGT("(" + gen + ")", n->GetType(), gt);
+		}
 
 	return gen;
 	}

src/script_opt/CPP/maint/README

@@ -15,7 +15,7 @@ The maintenance workflow:
     to check in updates to the list of how the compiler currently fares
     on various btests (see end of this doc):
 
-	Mon Nov  7 14:30:51 PST 2022
+	Sat Dec  3 13:20:43 PST 2022
 
 2.  Run "find-test-files.sh" to generate a list (to stdout) of all of the
     possible Zeek source files found in the test suite.
@@ -69,12 +69,18 @@ These BTests won't successfully run due to the indicated issue:
 		pass to fail when it tries to start up a new httpd
 	skipped - test can be skipped due to environmental reasons (e.g.,
 		whether we have a certain Kerberos setup)
+	start-next - test uses @TEST-START-NEXT.  As long as the diagnostic
+		output doesn't have errors for language.blank-local-4, it's ok
+	race? - appears to have a race condition, can fail even when
+		running with interpreter, and can pass with -O C++
 
-Database Of Known Issues (keep sorted)
+Database Of Known Issues
 
-../testing/btest/bifs/table_values.zeek bad-constructor
+../testing/btest/language/blank-local.zeek start-next
+../testing/btest/language/when-aggregates.zeek bad-when
 ../testing/btest/scripts/base/protocols/krb/smb2_krb.test skipped
 ../testing/btest/scripts/base/protocols/krb/smb2_krb_nokeytab.test skipped
 ../testing/btest/scripts/base/utils/active-http.test test-glitch
 ../testing/btest/scripts/policy/frameworks/dpd/packet-segment-logging.zeek cond
 ../testing/btest/scripts/policy/misc/dump-events.zeek skipped
+../testing/btest/supervisor/revive-leaf.zeek race?

src/script_opt/ZAM/Ops.in

@@ -344,7 +344,10 @@ eval	--frame[z.v1].int_val;
 op DecrU
 op1-read-write
 type V
-eval	--frame[z.v1].uint_val;
+eval	auto& u = frame[z.v1].uint_val;
+	if ( u == 0 )
+		ZAM_run_time_warning(z.loc, "count underflow");
+	--u;
 
 unary-op AppendTo
 # Note, even though it feels like appending both reads and modifies
@@ -1559,6 +1562,9 @@ type CiH
 op1-read
 eval	EvalScheduleArgs(z.c, z.v1,)
 
+macro QueueEvent(eh, args)
+	if ( *eh )
+		event_mgr.Enqueue(eh, std::move(args));
 
 op Event
 type HL
@@ -1566,19 +1572,19 @@ op1-read
 custom-method return CompileEvent(h, l);
 eval	ValVec args;
 	z.aux->FillValVec(args, frame);
-	event_mgr.Enqueue(z.event_handler, std::move(args));
+	QueueEvent(z.event_handler, args);
 
 internal-op Event0
 type X
 eval	ValVec args(0);
-	event_mgr.Enqueue(z.event_handler, std::move(args));
+	QueueEvent(z.event_handler, args);
 
 internal-op Event1
 type V
 op1-read
 eval	ValVec args(1);
 	args[0] = frame[z.v1].ToVal(z.t);
-	event_mgr.Enqueue(z.event_handler, std::move(args));
+	QueueEvent(z.event_handler, args);
 
 internal-op Event2
 type VV
@@ -1586,7 +1592,7 @@ op1-read
 eval	ValVec args(2);
 	args[0] = frame[z.v1].ToVal(z.t);
 	args[1] = frame[z.v2].ToVal(z.t2);
-	event_mgr.Enqueue(z.event_handler, std::move(args));
+	QueueEvent(z.event_handler, args);
 
 internal-op Event3
 type VVV
@@ -1596,7 +1602,7 @@ eval	ValVec args(3);
 	args[1] = frame[z.v2].ToVal(z.t2);
 	auto types = z.aux->types;
 	args[2] = frame[z.v3].ToVal(types[2]);
-	event_mgr.Enqueue(z.event_handler, std::move(args));
+	QueueEvent(z.event_handler, args);
 
 internal-op Event4
 type VVVV
@@ -1607,7 +1613,7 @@ eval	ValVec args(4);
 	auto types = z.aux->types;
 	args[2] = frame[z.v3].ToVal(types[2]);
 	args[3] = frame[z.v4].ToVal(types[3]);
-	event_mgr.Enqueue(z.event_handler, std::move(args));
+	QueueEvent(z.event_handler, args);
 
 
 op Return

testing/btest/Baseline.cpp/core.max-analyzer-violations/output

@@ -0,0 +1,15 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+reply, CHhAvVGS1DHFjwGM9, OK, example.com POP3-Server
+request, CHhAvVGS1DHFjwGM9, USER, zeek@zeek.org
+reply, CHhAvVGS1DHFjwGM9, OK, Please enter your password
+request, CHhAvVGS1DHFjwGM9, PASS, zeek
+reply, CHhAvVGS1DHFjwGM9, OK, mailbox locked and ready
+request, CHhAvVGS1DHFjwGM9, STAT, 
+1, violation, AllAnalyzers::ANALYZER_ANALYZER_POP3, CHhAvVGS1DHFjwGM9, 4, unknown server command (+)
+2, violation, AllAnalyzers::ANALYZER_ANALYZER_POP3, CHhAvVGS1DHFjwGM9, 4, unknown server command (???)
+3, violation, AllAnalyzers::ANALYZER_ANALYZER_POP3, CHhAvVGS1DHFjwGM9, 4, unknown server command (..)
+4, violation, AllAnalyzers::ANALYZER_ANALYZER_POP3, CHhAvVGS1DHFjwGM9, 4, unknown server command (???)
+5, violation, AllAnalyzers::ANALYZER_ANALYZER_POP3, CHhAvVGS1DHFjwGM9, 4, unknown server command (..x)
+reply, CHhAvVGS1DHFjwGM9, OK, 1 236
+request, CHhAvVGS1DHFjwGM9, QUIT, 
+reply, CHhAvVGS1DHFjwGM9, OK, Bye

testing/btest/Baseline.cpp/core.max-analyzer-violations/weird.log

@@ -0,0 +1,13 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	weird
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer	source
+#types	time	string	addr	port	addr	port	string	string	bool	string	string
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	58246	127.0.0.1	110	pop3_server_command_unknown	-	F	zeek	POP3
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	58246	127.0.0.1	110	line_terminated_with_single_CR	-	F	zeek	CONTENTLINE
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	58246	127.0.0.1	110	too_many_analyzer_violations	-	F	zeek	POP3
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline.cpp/language.count-underflow/.stderr

@@ -0,0 +1 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

testing/btest/Baseline.cpp/language.count-underflow/out

@@ -0,0 +1,8 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+local c1 = 0; --c1; c1 == 18446744073709551615; T
+local c2 = 0; c2 -= 1; c2 == 18446744073709551615; T
+local c3 = 0; c3 = c3 - 1; c3 == 18446744073709551615; T
+1 - 2, 18446744073709551615
+local c4 = count_max; ++c4; c4 == 0; T
+local c5 = count_max; c5 += 1; c5 == 0; T
+local c6 = count_max; c6 = c6 + 1; c6 == 0; T

testing/btest/Baseline.cpp/language.sizeof/.stderr

@@ -0,0 +1 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

testing/btest/Baseline.cpp/language.sizeof/output

@@ -0,0 +1,20 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+IPv4 Address 1.2.3.4: 32
+IPv6 Address ::1: 128
+Boolean T: 1
+Count 10: 10
+Expr: 18446744073709551612
+Signed Expr: 4
+Double -1.23: 1.230000
+Enum ENUM3: 2
+File 21.000000
+Function add_interface: 2
+Integer -10: 10
+Interval -5.0 secs: 5.000000
+Port 80/tcp: 65616
+Record [i=10, j=<uninitialized>, k=<uninitialized>]: 3
+Set: 3
+String 'Hello': 5
+Subnet 192.168.0.0/24: 256.000000
+Table 2
+Vector [Hello, , , , World]: 5

testing/btest/Baseline.cpp/language.vector-coerce-expr2/output

@@ -2,3 +2,7 @@
 [5.5, 5.2, -3.9, 20.0]
 [11, 5, , 107, , , 1046]
 [-2, -4, , -7, , -18, -999]
+[connT
+compiled-C++ , connF
+compiled-C++ , connMaybe
+compiled-C++ ]

testing/btest/Baseline.zam/core.max-analyzer-violations/output

@@ -0,0 +1,15 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+reply, CHhAvVGS1DHFjwGM9, OK, example.com POP3-Server
+request, CHhAvVGS1DHFjwGM9, USER, zeek@zeek.org
+reply, CHhAvVGS1DHFjwGM9, OK, Please enter your password
+request, CHhAvVGS1DHFjwGM9, PASS, zeek
+reply, CHhAvVGS1DHFjwGM9, OK, mailbox locked and ready
+request, CHhAvVGS1DHFjwGM9, STAT, 
+1, violation, AllAnalyzers::ANALYZER_ANALYZER_POP3, CHhAvVGS1DHFjwGM9, 4, unknown server command (+)
+2, violation, AllAnalyzers::ANALYZER_ANALYZER_POP3, CHhAvVGS1DHFjwGM9, 4, unknown server command (???)
+3, violation, AllAnalyzers::ANALYZER_ANALYZER_POP3, CHhAvVGS1DHFjwGM9, 4, unknown server command (..)
+4, violation, AllAnalyzers::ANALYZER_ANALYZER_POP3, CHhAvVGS1DHFjwGM9, 4, unknown server command (???)
+5, violation, AllAnalyzers::ANALYZER_ANALYZER_POP3, CHhAvVGS1DHFjwGM9, 4, unknown server command (..x)
+reply, CHhAvVGS1DHFjwGM9, OK, 1 236
+request, CHhAvVGS1DHFjwGM9, QUIT, 
+reply, CHhAvVGS1DHFjwGM9, OK, Bye

testing/btest/Baseline.zam/core.max-analyzer-violations/weird.log

@@ -0,0 +1,13 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	weird
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer	source
+#types	time	string	addr	port	addr	port	string	string	bool	string	string
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	58246	127.0.0.1	110	pop3_server_command_unknown	-	F	zeek	POP3
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	58246	127.0.0.1	110	line_terminated_with_single_CR	-	F	zeek	CONTENTLINE
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	127.0.0.1	58246	127.0.0.1	110	too_many_analyzer_violations	-	F	zeek	POP3
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline.zam/language.count-underflow/.stderr

@@ -0,0 +1,5 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+expression warning in <...>/count-underflow.zeek, line 22: count underflow (1 - 2)
+expression warning in <...>/count-underflow.zeek, line 14: count underflow (0 - 1)
+expression warning in <...>/count-underflow.zeek, line 18: count underflow (0 - 1)
+warning: <...>/count-underflow.zeek, line 10: count underflow

testing/btest/Baseline.zam/language.count-underflow/out

@@ -0,0 +1,8 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+local c1 = 0; --c1; c1 == 18446744073709551615; T
+local c2 = 0; c2 -= 1; c2 == 18446744073709551615; T
+local c3 = 0; c3 = c3 - 1; c3 == 18446744073709551615; T
+1 - 2, 18446744073709551615
+local c4 = count_max; ++c4; c4 == 0; T
+local c5 = count_max; c5 += 1; c5 == 0; T
+local c6 = count_max; c6 = c6 + 1; c6 == 0; T

testing/btest/Baseline.zam/language.vector-coerce-expr2/output

@@ -2,3 +2,7 @@
 [5.5, 5.2, -3.9, 20.0]
 [11, 5, , 107, , , 1046]
 [-2, -4, , -7, , -18, -999]
+[connT
+ZAM-code connT , connF
+ZAM-code connF , connMaybe
+ZAM-code connMaybe ]

testing/btest/Baseline.zam/plugins.file/output

@@ -8,14 +8,15 @@ analyzer_confirmation_info, AllAnalyzers::FILES_ANALYZER_FOO, FCceqBvpMfirSN0Ri
 foo_piece, FCceqBvpMfirSN0Ri, The National Center 
 foo_piece, FCceqBvpMfirSN0Ri, net, consult your lo
 foo_piece, FCceqBvpMfirSN0Ri, most everything else
+analyzer_violation_info, AllAnalyzers::FILES_ANALYZER_FOO, FCceqBvpMfirSN0Ri, test violation 3, most everything 
 foo_piece, FCceqBvpMfirSN0Ri, low:\x0a\x0a   /Mac       
 foo_piece, FCceqBvpMfirSN0Ri, es and directories o
-analyzer_violation_info, AllAnalyzers::FILES_ANALYZER_FOO, FCceqBvpMfirSN0Ri, test violation 5, es and directori
 foo_piece, FCceqBvpMfirSN0Ri, r example, here is a
+analyzer_violation_info, AllAnalyzers::FILES_ANALYZER_FOO, FCceqBvpMfirSN0Ri, test violation 6, r example, here 
 foo_piece, FCceqBvpMfirSN0Ri, application, StuffIt
 foo_piece, FCceqBvpMfirSN0Ri, tion BinHex by doubl
 foo_piece, FCceqBvpMfirSN0Ri, laced, or are going 
+analyzer_violation_info, AllAnalyzers::FILES_ANALYZER_FOO, FCceqBvpMfirSN0Ri, test violation 9, laced, or are go
 foo_piece, FCceqBvpMfirSN0Ri, sers several documen
-analyzer_violation_info, AllAnalyzers::FILES_ANALYZER_FOO, FCceqBvpMfirSN0Ri, test violation 10, sers several doc
 foo_piece, FCceqBvpMfirSN0Ri, er or can be printed
 foo_piece, FCceqBvpMfirSN0Ri,  \x0a\x0aBug reports shoul

testing/btest/Baseline.zam/plugins.file/weird.log

@@ -0,0 +1,11 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	weird
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer	source
+#types	time	string	addr	port	addr	port	string	string	bool	string	string
+XXXXXXXXXX.XXXXXX	-	-	-	-	-	too_many_analyzer_violations	FCceqBvpMfirSN0Ri	F	zeek	FOO
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/bifs.table_values/out

@@ -1,3 +1,9 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-[example, test]
+[example, test], vector of string
-[80/tcp, http, 443/tcp, https, 21/tcp, ftp, 23/tcp, telnet]
+[{
+[80/tcp, http] ,
+[443/tcp, https] 
+}, {
+[21/tcp, ftp] ,
+[23/tcp, telnet] 
+}], vector of set[port,string]

testing/btest/Baseline/language.trailing-comma/out

@@ -38,9 +38,15 @@ tab6, table[string,string] of vector of count, {
 [a, b] = [1, 2],
 [c, d] = [3, 4]
 }
-tab7, table[string,string] of list of count,count, {
+tab7, table[string,string] of set[count], {
-[a, b] = 1, 2,
+[a, b] = {
-[c, d] = 3, 4
+2,
+1
+},
+[c, d] = {
+4,
+3
+}
 }
 tab8, table[MyRecord] of count, {
 [[a=c, b=d]] = 43,

testing/btest/bifs/table_values.zeek

@@ -6,12 +6,12 @@ event zeek_init()
 	{
 	local t1: table[count] of string = table([5] = "test", [0] = "example");
 	local t2 = table(
-		["web"] = { [80/tcp, "http"], [443/tcp, "https"] },
+		["web"] = set( [80/tcp, "http"], [443/tcp, "https"] ),
-		["login"] = { [21/tcp, "ftp"], [23/tcp, "telnet"] });
+		["login"] = set( [21/tcp, "ftp"], [23/tcp, "telnet"] ));
 	
-	local v1: vector of set[string] = table_values(t1);
+	local v1: vector of string = table_values(t1);
 	local v2: vector of set[port, string] = table_values(t2);
 
-	print v1;
+	print v1, type_name(v1);
-	print v2;
+	print v2, type_name(v2);
 	}

testing/btest/doc/zeekygen/declaring_script_errors.zeek

@@ -1,4 +1,5 @@
 # @TEST-DOC: Test zeekygen declaring script functions error/empty cases.
+# @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1"
 #
 # @TEST-EXEC: unset ZEEK_DISABLE_ZEEKYGEN; zeek -b `basename %INPUT` >out
 # @TEST-EXEC: btest-diff out

testing/btest/language/trailing-comma.zeek

@@ -79,8 +79,8 @@ event zeek_init()
 	print "tab6", type_name(tab6), tab6;
 
 	local tab7 = table(
-		["a", "b"] = [1, 2, ],
+		["a", "b"] = set(1, 2, ),
-		["c", "d"] = [3, 4, ],
+		["c", "d"] = set(3, 4, ),
 	);
 	print "tab7", type_name(tab7), tab7;