updates for ZAM optimization

src/script_opt/ZAM/AM-Opt.cc

@@ -670,7 +670,15 @@ void ZAMCompiler::ReMapVar(const ID* id, int slot, bro_uint_t inst)
 	// powerful allocation method like graph coloring.  However, far and
 	// away the bulk of our variables are short-lived temporaries,
 	// for which greedy should work fine.
-	bool is_managed = ZVal::IsManagedType(id->GetType());
+	//
+	// Note, we also need to make sure that denizens sharing a slot
+	// are all consistently either managed, or non-managed, types.
+	// One subtlety in this regard is that identifiers that are types
+	// should always be deemed "managed", even if the type they refer
+	// to is not managed, because what matters for uses of those
+	// identifiers is interpreting them as "any" values having an
+	// internal type of TYPE_TYPE.
+	bool is_managed = ZVal::IsManagedType(id->GetType()) || id->IsType();
 
 	int apt_slot = -1;
 	for ( unsigned int i = 0; i < shared_frame_denizens.size(); ++i )

src/script_opt/ZAM/Compile.h

@@ -178,6 +178,8 @@ private:
 
 	const ZAMStmt CompileIncrExpr(const IncrExpr* e);
 	const ZAMStmt CompileAppendToExpr(const AppendToExpr* e);
+	const ZAMStmt CompileAddToExpr(const AddToExpr* e);
+	const ZAMStmt CompileRemoveFromExpr(const RemoveFromExpr* e);
 	const ZAMStmt CompileAssignExpr(const AssignExpr* e);
 	const ZAMStmt CompileAssignToIndex(const NameExpr* lhs, const IndexExpr* rhs);
 	const ZAMStmt CompileFieldLHSAssignExpr(const FieldLHSAssignExpr* e);
@@ -233,9 +235,6 @@ private:
 	const ZAMStmt AssignVecElems(const Expr* e);
 	const ZAMStmt AssignTableElem(const Expr* e);
 
-	const ZAMStmt AppendToField(const NameExpr* n1, const NameExpr* n2, const ConstExpr* c,
-	                            int offset);
-
 	const ZAMStmt ConstructTable(const NameExpr* n, const Expr* e);
 	const ZAMStmt ConstructSet(const NameExpr* n, const Expr* e);
 	const ZAMStmt ConstructRecord(const NameExpr* n, const Expr* e);

src/script_opt/ZAM/Expr.cc

@@ -20,6 +20,12 @@ const ZAMStmt ZAMCompiler::CompileExpr(const Expr* e)
 		case EXPR_APPEND_TO:
 			return CompileAppendToExpr(static_cast<const AppendToExpr*>(e));
 
+		case EXPR_ADD_TO:
+			return CompileAddToExpr(static_cast<const AddToExpr*>(e));
+
+		case EXPR_REMOVE_FROM:
+			return CompileRemoveFromExpr(static_cast<const RemoveFromExpr*>(e));
+
 		case EXPR_ASSIGN:
 			return CompileAssignExpr(static_cast<const AssignExpr*>(e));
 
@@ -76,44 +82,72 @@ const ZAMStmt ZAMCompiler::CompileIncrExpr(const IncrExpr* e)
 
 const ZAMStmt ZAMCompiler::CompileAppendToExpr(const AppendToExpr* e)
 	{
-	auto op1 = e->GetOp1();
+	auto n1 = e->GetOp1()->AsNameExpr();
 	auto op2 = e->GetOp2();
+	auto n2 = op2->Tag() == EXPR_NAME ? op2->AsNameExpr() : nullptr;
+	auto cc = op2->Tag() != EXPR_NAME ? op2->AsConstExpr() : nullptr;
 
+	if ( n1->GetType()->Yield()->Tag() == TYPE_ANY )
+		return n2 ? AppendToAnyVecVV(n1, n2) : AppendToAnyVecVC(n1, cc);
+
+	return n2 ? AppendToVV(n1, n2) : AppendToVC(n1, cc);
+	}
+
+const ZAMStmt ZAMCompiler::CompileAddToExpr(const AddToExpr* e)
+	{
+	auto op1 = e->GetOp1();
+	auto t1 = op1->GetType()->Tag();
+
+	auto op2 = e->GetOp2();
 	auto n2 = op2->Tag() == EXPR_NAME ? op2->AsNameExpr() : nullptr;
 	auto cc = op2->Tag() != EXPR_NAME ? op2->AsConstExpr() : nullptr;
 
 	if ( op1->Tag() == EXPR_FIELD )
 		{
+		assert(t1 == TYPE_PATTERN);
 		auto f = op1->AsFieldExpr()->Field();
 		auto n1 = op1->GetOp1()->AsNameExpr();
-		return AppendToField(n1, n2, cc, f);
+
+		ZInstI z;
+
+		if ( n2 )
+			{
+			z = ZInstI(OP_ADDPATTERNTOFIELD_VVi, FrameSlot(n1), FrameSlot(n2), f);
+			z.op_type = OP_VVV_I3;
+			}
+		else
+			{
+			z = ZInstI(OP_ADDPATTERNTOFIELD_VCi, FrameSlot(n1), f, cc);
+			z.op_type = OP_VVC_I2;
+			}
+
+		z.SetType(n2 ? n2->GetType() : cc->GetType());
+
+		return AddInst(z);
 		}
 
 	auto n1 = op1->AsNameExpr();
 
-	return n2 ? AppendToVV(n1, n2) : AppendToVC(n1, cc);
+	if ( t1 == TYPE_PATTERN )
+		return n2 ? ExtendPatternVV(n1, n2) : ExtendPatternVC(n1, cc);
+
+	if ( t1 == TYPE_VECTOR )
+		return n2 ? AddVecToVecVV(n1, n2) : AddVecToVecVC(n1, cc);
+
+	assert(t1 == TYPE_TABLE);
+
+	return n2 ? AddTableToTableVV(n1, n2) : AddTableToTableVC(n1, cc);
 	}
 
-const ZAMStmt ZAMCompiler::AppendToField(const NameExpr* n1, const NameExpr* n2, const ConstExpr* c,
+const ZAMStmt ZAMCompiler::CompileRemoveFromExpr(const RemoveFromExpr* e)
-                                         int offset)
 	{
-	ZInstI z;
+	auto n1 = e->GetOp1()->AsNameExpr();
+	auto op2 = e->GetOp2();
 
-	if ( n2 )
+	auto n2 = op2->Tag() == EXPR_NAME ? op2->AsNameExpr() : nullptr;
-		{
+	auto cc = op2->Tag() != EXPR_NAME ? op2->AsConstExpr() : nullptr;
-		z = ZInstI(OP_APPENDTOFIELD_VVi, FrameSlot(n1), FrameSlot(n2), offset);
-		z.op_type = OP_VVV_I3;
-		}
-	else
-		{
-		ASSERT(c);
-		z = ZInstI(OP_APPENDTOFIELD_VCi, FrameSlot(n1), offset, c);
-		z.op_type = OP_VVC_I2;
-		}
 
-	z.SetType(n2 ? n2->GetType() : c->GetType());
+	return n2 ? RemoveTableFromTableVV(n1, n2) : RemoveTableFromTableVC(n1, cc);
-
-	return AddInst(z);
 	}
 
 const ZAMStmt ZAMCompiler::CompileAssignExpr(const AssignExpr* e)

src/script_opt/ZAM/Ops.in

@@ -353,31 +353,73 @@ unary-op AppendTo
 # value itself.
 op1-read
 set-type $1
-eval	auto copy = CopyVal($1);
+eval	auto vv = frame[z.v1].vector_val;
-	auto vv = frame[z.v1].vector_val;
+	if ( vv->Size() == 0 )
-	vv->RawVec()->push_back(copy);
+		// Use the slightly more expensive Assign(), since it
-	vv->Modified();
+		// knows how to deal with empty vectors that do not yet
+		// have concrete types.
+		vv->Assign(0, $1.ToVal(z.t));
+	else
+		{
+		vv->RawVec()->push_back(CopyVal($1));
+		vv->Modified();
+		}
 
-internal-op AppendToField
+# For vectors-of-any, we always go through the Assign() interface because
+# it's needed for tracking the potentially differing types.
+unary-op AppendToAnyVec
+op1-read
+set-type $1
+eval	auto vv = frame[z.v1].vector_val;
+	vv->Assign(vv->Size(), $1.ToVal(z.t));
+
+internal-op AddPatternToField
 type VVi
 op1-read
-eval	EvalAppendToField(frame[z.v2], v3)
+eval	EvalAddPatternToField(frame[z.v2], v3)
 
-macro EvalAppendToField(val, f)
+macro EvalAddPatternToField(v, f)
-	auto v = CopyVal(val);
+	auto fpat = frame[z.v1].record_val->GetField(z.f)->AsPatternVal();
-	auto fvv = frame[z.v1].record_val->GetField(z.f)->AsVectorVal();
+	if ( fpat )
-	if ( fvv )
 		{
-		fvv->RawVec()->push_back(v);
+		v.re_val->AddTo(fpat, false);
-		fvv->Modified();
+		frame[z.v1].record_val->Modified();
 		}
 	else
 		ZAM_run_time_error(z.loc, util::fmt("field value missing: $%s", frame[z.v1].record_val->GetType()->AsRecordType()->FieldName(z.f)));
 
-internal-op AppendToField
+internal-op AddPatternToField
 type VCi
 op1-read
-eval	EvalAppendToField(z.c, v2)
+eval	EvalAddPatternToField(z.c, v2)
+
+unary-op ExtendPattern
+op1-read
+eval	$1.re_val->AddTo(frame[z.v1].re_val, false);
+
+unary-op AddVecToVec
+op1-read
+eval	$1.vector_val->AddTo(frame[z.v1].vector_val, false);
+
+unary-op AddTableToTable
+op1-read
+eval	auto t = frame[z.v1].table_val;
+	auto v = $1.table_val;
+	if ( v->Size() > 0 )
+		{
+		v->AddTo(t, false);
+		t->Modified();
+		}
+
+unary-op RemoveTableFromTable
+op1-read
+eval	auto t = frame[z.v1].table_val;
+	auto v = $1.table_val;
+	if ( v->Size() > 0 )
+		{
+		v->RemoveFrom(t);
+		t->Modified();
+		}
 
 unary-expr-op Cast
 op-type X
@@ -1714,7 +1756,7 @@ eval	step_iters[z.v2].InitLoop(frame[z.v1].string_val->AsString());
 
 internal-op Init-String-Loop
 type VC
-eval	step_iters[z.v2].InitLoop(z.c.string_val->AsString());
+eval	step_iters[z.v1].InitLoop(z.c.string_val->AsString());
 
 internal-op Next-String-Iter
 # v1 = iteration variable

testing/btest/Baseline.zam/bifs.to_addr/error

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+error in <...>/to_addr.zeek, line 20: failed converting string to IP address (not an IP)

testing/btest/Baseline.zam/bifs.to_addr/output

@@ -0,0 +1,10 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+to_addr(0.0.0.0) = 0.0.0.0 (SUCCESS)
+to_addr(1.2.3.4) = 1.2.3.4 (SUCCESS)
+to_addr(01.02.03.04) = 1.2.3.4 (SUCCESS)
+to_addr(001.002.003.004) = 1.2.3.4 (SUCCESS)
+to_addr(10.20.30.40) = 10.20.30.40 (SUCCESS)
+to_addr(100.200.30.40) = 100.200.30.40 (SUCCESS)
+to_addr(10.0.0.0) = 10.0.0.0 (SUCCESS)
+to_addr(10.00.00.000) = 10.0.0.0 (SUCCESS)
+to_addr(not an IP) = :: (SUCCESS)

testing/btest/Baseline.zam/bifs.to_double_from_string/error

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+error in <...>/to_double_from_string.zeek, line 15: bad conversion to double (NotADouble)
+error in <...>/to_double_from_string.zeek, line 16: bad conversion to double ()

testing/btest/Baseline.zam/bifs.to_double_from_string/output

@@ -0,0 +1,6 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+to_double(3.14) = 3.14 (SUCCESS)
+to_double(-3.14) = -3.14 (SUCCESS)
+to_double(0) = 0.0 (SUCCESS)
+to_double(NotADouble) = 0.0 (SUCCESS)
+to_double() = 0.0 (SUCCESS)