diff --git a/CHANGES b/CHANGES index 96d32960ec..5fc112f432 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,26 @@ +6.1.0-dev.42 | 2023-06-14 12:51:08 +0200 + + * NEWS: Small section about assert statement (Arne Welzel, Corelight) + + * Stmt: Rework assertion hooks break semantics (Arne Welzel, Corelight) + + Using break in either of the hooks allows to suppress the default reporter + error message rather than suppressing solely based on the existence of an + assertion_failure() handler. + + * Stmt: Introduce assert statement and related hooks (Arne Welzel, Corelight) + + including two hooks called assertion_failure() and assertion_result() for + customization and tracking of assertion results. + + * ZeekArgs: Helper for empty arguments (Arne Welzel, Corelight) + + * Reporter: Allow AssertStmt to throw InterpreterException (Arne Welzel, Corelight) + + * Lift backtrace() code into Func.{h,cc} (Arne Welzel, Corelight) + + This is to be re-used by the assertion facility. + 6.1.0-dev.32 | 2023-06-13 11:29:36 -0700 * Clarify subitem relationship in CMake configure summary. (Benjamin Bannier, Corelight) diff --git a/NEWS b/NEWS index b6ca019137..4a4bd94e08 100644 --- a/NEWS +++ b/NEWS @@ -9,9 +9,28 @@ Zeek 6.1.0 Breaking Changes ---------------- +* ``assert`` is now a reserved keyword for the new ``assert`` statement. + New Functionality ----------------- +- Added a new ``assert`` statement for assertion based testing and asserting + runtime state. + + assert [, ]; + + This statement comes with two hooks. First, ``assertion_failure()`` that + is invoked for every failing assert statement. Second, ``assertion_result()`` + which is invoked for every assert statement and its outcome. The latter allows + to construct a summary of failing and passing assert statements. Both hooks + receive the location and call stack for the ``assert`` statement via a + ``Backtrace`` vector. + + A failing assert will abort execution of the current event handler similar + to scripting errors. By default, a reporter error message is logged. Using + the break statement within ``assertion_failure()`` or ``assertion_result()`` + allows to suppress the default message. + Changed Functionality --------------------- diff --git a/VERSION b/VERSION index 36cfa15110..8a8455a4b8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -6.1.0-dev.32 +6.1.0-dev.42 diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index fb15bea8dd..ec809377de 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -942,6 +942,45 @@ type BacktraceElement: record { ## .. zeek:see:: backtrace print_backtrace type Backtrace: vector of BacktraceElement; +## A hook that is invoked when an assert statement fails. +## +## By default, a reporter error message is logged describing the failing +## assert similarly to how scripting errors are reported after invoking +## this hook. Using the :zeek:see:`break` statement in an assertion_failure +## hook handler allows to suppress this message. +## +## cond: The string representation of the condition. +## +## msg: Evaluated message as string given to the assert statement. +## +## bt: Backtrace of the assertion error. The top element will contain +## the location of the assert statement that failed. +## +## .. zeek:see:: assertion_result +type assertion_failure: hook(cond: string, msg: string, bt: Backtrace); + +## A hook that is invoked with the result of every assert statement. +## +## This is a potentially expensive hook meant to be used by testing +## frameworks to summarize assert results. In a production setup, +## this hook is likely detrimental to performance. +## +## Using the :zeek:see:`break` statement within an assertion_failure hook +## handler allows to suppress the reporter error message generated for +## failing assert statements. +## +## result: The result of evaluating **cond**. +## +## cond: The string representation of the condition. +## +## msg: Evaluated message as string given to the assert statement. +## +## bt: Backtrace of the assertion error. The top element will contain +## the location of the assert statement that failed. +## +## .. zeek:see:: assertion_failure +type assertion_result: hook(result: bool, cond: string, msg: string, bt: Backtrace); + # todo:: Do we still need these here? Can they move into the packet filter # framework? # diff --git a/src/Func.cc b/src/Func.cc index f56719ee00..fa08d81709 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -903,6 +903,59 @@ FunctionIngredients::FunctionIngredients(ScopePtr _scope, StmtPtr _body, } } +zeek::RecordValPtr make_backtrace_element(std::string_view name, const VectorValPtr args, + const zeek::detail::Location* loc) + { + static auto elem_type = id::find_type("BacktraceElement"); + static auto function_name_idx = elem_type->FieldOffset("function_name"); + static auto function_args_idx = elem_type->FieldOffset("function_args"); + static auto file_location_idx = elem_type->FieldOffset("file_location"); + static auto line_location_idx = elem_type->FieldOffset("line_location"); + + auto elem = make_intrusive(elem_type); + elem->Assign(function_name_idx, name.data()); + elem->Assign(function_args_idx, std::move(args)); + + if ( loc ) + { + elem->Assign(file_location_idx, loc->filename); + elem->Assign(line_location_idx, loc->first_line); + } + + return elem; + } + +zeek::VectorValPtr get_current_script_backtrace() + { + static auto backtrace_type = id::find_type("Backtrace"); + + auto rval = make_intrusive(backtrace_type); + + // The body of the following loop can wind up adding items to + // the call stack (because MakeCallArgumentVector() evaluates + // default arguments, which can in turn involve calls to script + // functions), so we work from a copy of the current call stack + // to prevent problems with iterator invalidation. + auto cs_copy = zeek::detail::call_stack; + + for ( auto it = cs_copy.rbegin(); it != cs_copy.rend(); ++it ) + { + const auto& ci = *it; + if ( ! ci.func ) + // This happens for compiled code. + continue; + + const auto& params = ci.func->GetType()->Params(); + auto args = MakeCallArgumentVector(ci.args, params); + + auto elem = make_backtrace_element(ci.func->Name(), std::move(args), + ci.call ? ci.call->GetLocationInfo() : nullptr); + rval->Append(std::move(elem)); + } + + return rval; + } + static void emit_builtin_error_common(const char* msg, Obj* arg, bool unwind) { auto emit = [=](const CallExpr* ce) diff --git a/src/Func.h b/src/Func.h index a27eb892b7..06f2d26306 100644 --- a/src/Func.h +++ b/src/Func.h @@ -364,6 +364,25 @@ private: extern std::vector call_stack; +/** + * Create a single BacktraceElement record val. + * + * @param name the name of the function. + * @param args call argument vector created by MakeCallArgumentVector(). + * @param loc optional location information of the caller. + * + * @return record value representing a BacktraceElement. + */ +zeek::RecordValPtr make_backtrace_element(std::string_view name, const VectorValPtr args, + const zeek::detail::Location* loc); + +/** + * Create a Zeek script Backtrace of the current script call_stack. + * + * @return VectorValPtr containing BacktraceElement entries. + */ +zeek::VectorValPtr get_current_script_backtrace(); + // This is set to true after the built-in functions have been initialized. extern bool did_builtin_init; extern std::vector bif_initializers; diff --git a/src/Reporter.h b/src/Reporter.h index c243d6e98d..8f8e54e251 100644 --- a/src/Reporter.h +++ b/src/Reporter.h @@ -31,6 +31,7 @@ using StringValPtr = IntrusivePtr; namespace detail { +class AssertStmt; class Location; class Expr; @@ -59,6 +60,7 @@ class InterpreterException : public ReporterException { protected: friend class Reporter; + friend class detail::AssertStmt; InterpreterException() { } }; diff --git a/src/Stmt.cc b/src/Stmt.cc index 1150740a9d..acdad6b8f8 100644 --- a/src/Stmt.cc +++ b/src/Stmt.cc @@ -54,6 +54,7 @@ const char* stmt_name(StmtTag t) "ZAM", "ZAM-resumption", "null", + "assert", }; return stmt_names[int(t)]; @@ -1864,6 +1865,140 @@ TraversalCode NullStmt::Traverse(TraversalCallback* cb) const HANDLE_TC_STMT_POST(tc); } +AssertStmt::AssertStmt(ExprPtr arg_cond, ExprPtr arg_msg) + : Stmt(STMT_ASSERT), cond(std::move(arg_cond)), msg(std::move(arg_msg)) + { + if ( ! IsBool(cond->GetType()->Tag()) ) + cond->Error("conditional must be boolean"); + + if ( msg && ! IsString(msg->GetType()->Tag()) ) + msg->Error("message must be string"); + } + +ValPtr AssertStmt::Exec(Frame* f, StmtFlowType& flow) + { + RegisterAccess(); + flow = FLOW_NEXT; + + static auto assertion_failure_hook = id::find_func("assertion_failure"); + static auto assertion_result_hook = id::find_func("assertion_result"); + + bool run_result_hook = assertion_result_hook && assertion_result_hook->HasEnabledBodies(); + bool run_failure_hook = assertion_failure_hook && assertion_failure_hook->HasEnabledBodies(); + + auto assert_result = cond->Eval(f)->AsBool(); + + if ( assert_result && ! run_result_hook ) + return Val::nil; + + // Textual representation of cond from the AST. + static zeek::ODesc desc; + desc.Clear(); + desc.SetShort(true); + desc.SetQuotes(true); + cond->Describe(&desc); + auto cond_val = zeek::make_intrusive(desc.Len(), (const char*)desc.Bytes()); + + zeek::StringValPtr msg_val = zeek::val_mgr->EmptyString(); + if ( msg ) + { + // Eval() may fail if expression assumes assert + // condition is F, but we still try to get it for + // the assertion_result hook. + try + { + msg_val = cast_intrusive(msg->Eval(f)); + } + catch ( InterpreterException& e ) + { + desc.Clear(); + desc.Add("Describe(&desc); + desc.Add(">"); + msg_val = zeek::make_intrusive(desc.Len(), (const char*)desc.Bytes()); + } + } + + VectorValPtr bt = nullptr; + if ( run_result_hook || run_failure_hook ) + { + bt = get_current_script_backtrace(); + auto assert_elem = make_backtrace_element("assert", MakeEmptyCallArgumentVector(), + GetLocationInfo()); + bt->Insert(0, assert_elem); + } + + // Breaking from either the assertion_failure() or assertion_result() + // hook can be used to suppress the default log message. + bool report_error = true; + + if ( run_result_hook ) + report_error &= assertion_result_hook + ->Invoke(zeek::val_mgr->Bool(assert_result), cond_val, msg_val, bt) + ->AsBool(); + + if ( assert_result ) + return Val::nil; + + if ( run_failure_hook ) + report_error &= assertion_failure_hook->Invoke(cond_val, msg_val, bt)->AsBool(); + + if ( report_error ) + { + std::string reporter_msg = util::fmt("assertion failure: %s", cond_val->CheckString()); + if ( msg_val->Len() > 0 ) + reporter_msg += util::fmt(" (%s)", msg_val->CheckString()); + + reporter->PushLocation(GetLocationInfo()); + reporter->Error("%s", reporter_msg.c_str()); + reporter->PopLocation(); + } + + throw InterpreterException(); + } + +void AssertStmt::StmtDescribe(ODesc* d) const + { + Stmt::StmtDescribe(d); + + // Quoting strings looks better when describing assert + // statements. So turn it on explicitly. + // + // E.g., md5_hash("") ends up as md5_hash() without quoting. + auto orig_quotes = d->WantQuotes(); + d->SetQuotes(true); + + cond->Describe(d); + + if ( msg ) + { + d->Add(","); + d->SP(); + msg->Describe(d); + } + + DescribeDone(d); + + d->SetQuotes(orig_quotes); + } + +TraversalCode AssertStmt::Traverse(TraversalCallback* cb) const + { + TraversalCode tc = cb->PreStmt(this); + HANDLE_TC_STMT_PRE(tc); + + tc = cond->Traverse(cb); + HANDLE_TC_STMT_PRE(tc); + if ( msg ) + { + tc = msg->Traverse(cb); + HANDLE_TC_STMT_PRE(tc); + } + + tc = cb->PostStmt(this); + HANDLE_TC_STMT_POST(tc); + } + WhenInfo::WhenInfo(ExprPtr arg_cond, FuncType::CaptureList* arg_cl, bool arg_is_return) : cond(std::move(arg_cond)), cl(arg_cl), is_return(arg_is_return) { diff --git a/src/Stmt.h b/src/Stmt.h index 5625b54502..bd91bdd7e4 100644 --- a/src/Stmt.h +++ b/src/Stmt.h @@ -544,6 +544,25 @@ private: bool is_directive; }; +class AssertStmt final : public Stmt + { +public: + explicit AssertStmt(ExprPtr cond, ExprPtr msg = nullptr); + + ValPtr Exec(Frame* f, StmtFlowType& flow) override; + + void StmtDescribe(ODesc* d) const override; + + TraversalCode Traverse(TraversalCallback* cb) const override; + + // Optimization-related: + StmtPtr Duplicate() override; + +private: + ExprPtr cond; + ExprPtr msg; + }; + // A helper class for tracking all of the information associated with // a "when" statement, and constructing the necessary components in support // of lambda-style captures. diff --git a/src/StmtBase.h b/src/StmtBase.h index dc7bb375c3..17188e0ce1 100644 --- a/src/StmtBase.h +++ b/src/StmtBase.h @@ -29,6 +29,7 @@ namespace detail class CompositeHash; class Frame; +class AssertStmt; class CatchReturnStmt; class ExprStmt; class ForStmt; @@ -94,6 +95,7 @@ public: const WhenStmt* AsWhenStmt() const; const SwitchStmt* AsSwitchStmt() const; const NullStmt* AsNullStmt() const; + const AssertStmt* AsAssertStmt() const; void RegisterAccess() const { diff --git a/src/StmtEnums.h b/src/StmtEnums.h index 1252edb43d..75f500f91c 100644 --- a/src/StmtEnums.h +++ b/src/StmtEnums.h @@ -32,8 +32,9 @@ enum StmtTag STMT_CPP, // compiled C++ STMT_ZAM, // a ZAM function body STMT_ZAM_RESUMPTION, // resumes ZAM execution for "when" statements - STMT_NULL -#define NUM_STMTS (int(STMT_NULL) + 1) + STMT_NULL, + STMT_ASSERT, +#define NUM_STMTS (int(STMT_ASSERT) + 1) }; enum StmtFlowType diff --git a/src/ZeekArgs.cc b/src/ZeekArgs.cc index 503cd98ff4..8db4264cf7 100644 --- a/src/ZeekArgs.cc +++ b/src/ZeekArgs.cc @@ -52,4 +52,10 @@ VectorValPtr MakeCallArgumentVector(const Args& vals, const RecordTypePtr& types return rval; } +VectorValPtr MakeEmptyCallArgumentVector() + { + static auto call_argument_vector = id::find_type("call_argument_vector"); + return make_intrusive(call_argument_vector); + } + } // namespace zeek diff --git a/src/ZeekArgs.h b/src/ZeekArgs.h index 27acea28b1..6d4cf15d74 100644 --- a/src/ZeekArgs.h +++ b/src/ZeekArgs.h @@ -39,4 +39,11 @@ Args val_list_to_args(const ValPList& vl); */ VectorValPtr MakeCallArgumentVector(const Args& vals, const RecordTypePtr& types); +/** + * Creates an empty "call_argument_vector" vector. + * + * @return empty vector of script-level type "call_argument_vector" + */ +VectorValPtr MakeEmptyCallArgumentVector(); + } // namespace zeek diff --git a/src/parse.y b/src/parse.y index 4db15571ab..d16fc5042c 100644 --- a/src/parse.y +++ b/src/parse.y @@ -7,7 +7,7 @@ %expect 211 -%token TOK_ADD TOK_ADD_TO TOK_ADDR TOK_ANY +%token TOK_ADD TOK_ADD_TO TOK_ADDR TOK_ANY TOK_ASSERT %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF %token TOK_BOOL TOK_BREAK TOK_CASE TOK_OPTION TOK_CONST %token TOK_CONSTANT TOK_COPY TOK_COUNT TOK_DEFAULT TOK_DELETE @@ -78,6 +78,7 @@ %type capture_list opt_captures when_captures %type when_head when_start when_clause %type TOK_PATTERN_END +%type opt_assert_msg %{ #include @@ -1802,6 +1803,11 @@ stmt: script_coverage_mgr.DecIgnoreDepth(); } + | TOK_ASSERT expr opt_assert_msg ';' + { + $$ = new AssertStmt(IntrusivePtr{AdoptRef{}, $2}, {AdoptRef{}, $3}); + } + | TOK_PRINT expr_list ';' opt_no_test { set_location(@1, @3); @@ -2228,6 +2234,13 @@ resolve_id: } ; +opt_assert_msg: + ',' expr + { $$ = $2; } + | + { $$ = nullptr; } + ; + opt_no_test: TOK_NO_TEST { $$ = true; } diff --git a/src/scan.l b/src/scan.l index c961b928e2..f1a8ecd201 100644 --- a/src/scan.l +++ b/src/scan.l @@ -321,6 +321,7 @@ add return TOK_ADD; addr return TOK_ADDR; any return TOK_ANY; as return TOK_AS; +assert return TOK_ASSERT; bool return TOK_BOOL; break return TOK_BREAK; case return TOK_CASE; diff --git a/src/script_opt/Stmt.cc b/src/script_opt/Stmt.cc index bb3e0c551b..02afbe7ae3 100644 --- a/src/script_opt/Stmt.cc +++ b/src/script_opt/Stmt.cc @@ -919,6 +919,11 @@ StmtPtr InitStmt::DoReduce(Reducer* c) return ThisPtr(); } +StmtPtr AssertStmt::Duplicate() + { + return SetSucc(new AssertStmt(cond->Duplicate(), msg ? msg->Duplicate() : nullptr)); + } + StmtPtr WhenStmt::Duplicate() { FuncType::CaptureList* cl_dup = nullptr; diff --git a/src/zeek.bif b/src/zeek.bif index c97e6e7d9c..1ad2324e04 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -2368,6 +2368,10 @@ function is_v6_subnet%(s: subnet%): bool return zeek::val_mgr->False(); %} +%%{ +#include "zeek/Func.h" +%%} + ## Returns a representation of the call stack as a vector of call stack ## elements, each containing call location information. ## @@ -2375,49 +2379,7 @@ function is_v6_subnet%(s: subnet%): bool ## location information. function backtrace%(%): Backtrace %{ - using zeek::detail::call_stack; - static auto backtrace_type = id::find_type("Backtrace"); - static auto elem_type = id::find_type("BacktraceElement"); - static auto function_name_idx = elem_type->FieldOffset("function_name"); - static auto function_args_idx = elem_type->FieldOffset("function_args"); - static auto file_location_idx = elem_type->FieldOffset("file_location"); - static auto line_location_idx = elem_type->FieldOffset("line_location"); - - auto rval = make_intrusive(backtrace_type); - - // The body of the following loop can wind up adding items to - // the call stack (because MakeCallArgumentVector() evaluates - // default arguments, which can in turn involve calls to script - // functions), so we work from a copy of the current call stack - // to prevent problems with iterator invalidation. - auto cs_copy = call_stack; - - for ( auto it = cs_copy.rbegin(); it != cs_copy.rend(); ++it ) - { - const auto& ci = *it; - if ( ! ci.func ) - // This happens for compiled code. - continue; - - auto elem = make_intrusive(elem_type); - - const auto& params = ci.func->GetType()->Params(); - auto args = MakeCallArgumentVector(ci.args, params); - - elem->Assign(function_name_idx, ci.func->Name()); - elem->Assign(function_args_idx, std::move(args)); - - if ( ci.call ) - { - auto loc = ci.call->GetLocationInfo(); - elem->Assign(file_location_idx, loc->filename); - elem->Assign(line_location_idx, loc->first_line); - } - - rval->Assign(rval->Size(), std::move(elem)); - } - - return rval; + return zeek::detail::get_current_script_backtrace(); %} # =========================================================================== diff --git a/testing/btest/Baseline/language.assert-2/out b/testing/btest/Baseline/language.assert-2/out new file mode 100644 index 0000000000..40ed4ca014 --- /dev/null +++ b/testing/btest/Baseline/language.assert-2/out @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/assert.zeek, line 3: assertion failure: fmt("%s", 1) == "2" ("1" != "2") +fatal error: errors occurred while initializing diff --git a/testing/btest/Baseline/language.assert-3/out b/testing/btest/Baseline/language.assert-3/out new file mode 100644 index 0000000000..5a58147fa5 --- /dev/null +++ b/testing/btest/Baseline/language.assert-3/out @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/assert.zeek, line 3: assertion failure: (coerce to_count("42") to double) == 42.5 (always failing) +fatal error: errors occurred while initializing diff --git a/testing/btest/Baseline/language.assert-4/out b/testing/btest/Baseline/language.assert-4/out new file mode 100644 index 0000000000..3146002d53 --- /dev/null +++ b/testing/btest/Baseline/language.assert-4/out @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/assert.zeek, line 4: assertion failure: 1 == x (Expected x to be 1, have 2) +fatal error: errors occurred while initializing diff --git a/testing/btest/Baseline/language.assert-5/out b/testing/btest/Baseline/language.assert-5/out new file mode 100644 index 0000000000..166362e0c3 --- /dev/null +++ b/testing/btest/Baseline/language.assert-5/out @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/assert.zeek, line 9: assertion failure: "ghi" in tbl ({ + [abc] = 123, + [def] = 456 +}) +fatal error: errors occurred while initializing diff --git a/testing/btest/Baseline/language.assert-6/out b/testing/btest/Baseline/language.assert-6/out new file mode 100644 index 0000000000..4bcf7b428e --- /dev/null +++ b/testing/btest/Baseline/language.assert-6/out @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/assert.zeek, line 10: assertion failure: r?$b (r$b is not set in [a=1234, b=]) +fatal error: errors occurred while initializing diff --git a/testing/btest/Baseline/language.assert-7/out b/testing/btest/Baseline/language.assert-7/out new file mode 100644 index 0000000000..c28e697ecb --- /dev/null +++ b/testing/btest/Baseline/language.assert-7/out @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +expression error in <...>/assert.zeek, line 10: field value missing (r$b) +error in <...>/assert.zeek, line 10: assertion failure: r?$b () +fatal error: errors occurred while initializing diff --git a/testing/btest/Baseline/language.assert-8/out b/testing/btest/Baseline/language.assert-8/out new file mode 100644 index 0000000000..7d877b65b5 --- /dev/null +++ b/testing/btest/Baseline/language.assert-8/out @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/assert.zeek, line 2: assertion failure: 1 == 2 (always false) +fatal error: failed to execute script statements at top-level scope diff --git a/testing/btest/Baseline/language.assert-error-2/.stderr b/testing/btest/Baseline/language.assert-error-2/.stderr new file mode 100644 index 0000000000..ffba84808c --- /dev/null +++ b/testing/btest/Baseline/language.assert-error-2/.stderr @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/assert-error.zeek, line 3: message must be string (1234) diff --git a/testing/btest/Baseline/language.assert-error-3/.stderr b/testing/btest/Baseline/language.assert-error-3/.stderr new file mode 100644 index 0000000000..8bbc43e570 --- /dev/null +++ b/testing/btest/Baseline/language.assert-error-3/.stderr @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/assert-error.zeek, line 3: syntax error, at or near ";" diff --git a/testing/btest/Baseline/language.assert-error-4/.stderr b/testing/btest/Baseline/language.assert-error-4/.stderr new file mode 100644 index 0000000000..90ff2d507a --- /dev/null +++ b/testing/btest/Baseline/language.assert-error-4/.stderr @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/assert-error.zeek, line 3: syntax error, at or near "," diff --git a/testing/btest/Baseline/language.assert-error/.stderr b/testing/btest/Baseline/language.assert-error/.stderr new file mode 100644 index 0000000000..1f636a5228 --- /dev/null +++ b/testing/btest/Baseline/language.assert-error/.stderr @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/assert-error.zeek, line 8: conditional must be boolean (1) diff --git a/testing/btest/Baseline/language.assert-hook-2/.stderr b/testing/btest/Baseline/language.assert-hook-2/.stderr new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/language.assert-hook-2/.stderr @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/language.assert-hook-2/out b/testing/btest/Baseline/language.assert-hook-2/out new file mode 100644 index 0000000000..f898a52732 --- /dev/null +++ b/testing/btest/Baseline/language.assert-hook-2/out @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +assertion_failure, to_count("5") == 4, 5 is not 4 +assert <...>/assert-hook.zeek:21 + f <...>/assert-hook.zeek:25 + g <...>/assert-hook.zeek:26 + h <...>/assert-hook.zeek:30 + zeek_init :0 diff --git a/testing/btest/Baseline/language.assert-hook-3/.stderr b/testing/btest/Baseline/language.assert-hook-3/.stderr new file mode 100644 index 0000000000..a1707e3408 --- /dev/null +++ b/testing/btest/Baseline/language.assert-hook-3/.stderr @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/assert-hook.zeek, line 12: assertion failure: F (terminate me!) +received termination signal diff --git a/testing/btest/Baseline/language.assert-hook-3/out b/testing/btest/Baseline/language.assert-hook-3/out new file mode 100644 index 0000000000..a09aae6630 --- /dev/null +++ b/testing/btest/Baseline/language.assert-hook-3/out @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +assertion_failure, terminate me! +zeek_done() diff --git a/testing/btest/Baseline/language.assert-hook-4/.stderr b/testing/btest/Baseline/language.assert-hook-4/.stderr new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/language.assert-hook-4/.stderr @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/language.assert-hook-4/out b/testing/btest/Baseline/language.assert-hook-4/out new file mode 100644 index 0000000000..13dabea91e --- /dev/null +++ b/testing/btest/Baseline/language.assert-hook-4/out @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +assertion_failure, calling exit! diff --git a/testing/btest/Baseline/language.assert-hook-5/.stderr b/testing/btest/Baseline/language.assert-hook-5/.stderr new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/language.assert-hook-5/.stderr @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/language.assert-hook-5/out b/testing/btest/Baseline/language.assert-hook-5/out new file mode 100644 index 0000000000..cfd12f766e --- /dev/null +++ b/testing/btest/Baseline/language.assert-hook-5/out @@ -0,0 +1,8 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +assertion_result T at <...>/assert-hook.zeek:25: md5_hash("") == "d41d8cd98f00b204e9800998ecf8427e" +assertion_result T at <...>/assert-hook.zeek:30: sha1_hash("") == "da39a3ee5e6b4b0d3255bfef95601890afd80709" +assertion_result F at <...>/assert-hook.zeek:35: sha1_hash("") == "wrong" +assertion_failure at <...>/assert-hook.zeek:35: sha1_hash("") == "wrong" +assertion_result F at <...>/assert-hook.zeek:40: md5_hash("") == "wrong" +assertion_failure at <...>/assert-hook.zeek:40: md5_hash("") == "wrong" +2 of 4 assertions failed diff --git a/testing/btest/Baseline/language.assert-hook-6/.stderr b/testing/btest/Baseline/language.assert-hook-6/.stderr new file mode 100644 index 0000000000..13dbb41dae --- /dev/null +++ b/testing/btest/Baseline/language.assert-hook-6/.stderr @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +expression error in <...>/assert-hook.zeek, line 15: field value missing (get_current_packet_header()$ip) +expression error in <...>/assert-hook.zeek, line 17: field value missing (get_current_packet_header()$ip) +error in <...>/assert-hook.zeek, line 17: assertion failure: 2 + 2 == 5 () +error in <...>/assert-hook.zeek, line 22: assertion failure: 2 + 2 == 5 ({"msg":"false and works"}) diff --git a/testing/btest/Baseline/language.assert-hook-6/out b/testing/btest/Baseline/language.assert-hook-6/out new file mode 100644 index 0000000000..37d07df4b9 --- /dev/null +++ b/testing/btest/Baseline/language.assert-hook-6/out @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +assertion_result, T, 2 + 2 == 4, , <...>/assert-hook.zeek, 15 +assertion_result, T, 2 + 2 == 4, {"msg":"true and works"}, <...>/assert-hook.zeek, 16 +assertion_result, F, 2 + 2 == 5, , <...>/assert-hook.zeek, 17 +assertion_failure, 2 + 2 == 5, , <...>/assert-hook.zeek, 17 +assertion_result, F, 2 + 2 == 5, {"msg":"false and works"}, <...>/assert-hook.zeek, 22 +assertion_failure, 2 + 2 == 5, {"msg":"false and works"}, <...>/assert-hook.zeek, 22 diff --git a/testing/btest/Baseline/language.assert-hook-7/.stderr b/testing/btest/Baseline/language.assert-hook-7/.stderr new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/language.assert-hook-7/.stderr @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/language.assert-hook-7/out b/testing/btest/Baseline/language.assert-hook-7/out new file mode 100644 index 0000000000..820b6a66a1 --- /dev/null +++ b/testing/btest/Baseline/language.assert-hook-7/out @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +assertion_result, T, 2 + 2 == 4, this is true, <...>/assert-hook.zeek, 10 +assertion_result, T, 2 + 2 == 4, {"msg":"this is also true"}, <...>/assert-hook.zeek, 11 +assertion_result, F, 2 + 2 == 5, this is false, <...>/assert-hook.zeek, 12 +assertion_result, F, 2 + 2 == 5, this is false, <...>/assert-hook.zeek, 18 diff --git a/testing/btest/Baseline/language.assert-hook/.stderr b/testing/btest/Baseline/language.assert-hook/.stderr new file mode 100644 index 0000000000..70a0f9c3a1 --- /dev/null +++ b/testing/btest/Baseline/language.assert-hook/.stderr @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/assert-hook.zeek, line 15: assertion failure: 1 != 1 diff --git a/testing/btest/Baseline/language.assert-hook/out b/testing/btest/Baseline/language.assert-hook/out new file mode 100644 index 0000000000..f92c1a466f --- /dev/null +++ b/testing/btest/Baseline/language.assert-hook/out @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +assertion_failure, 1 != 1, , <...>/assert-hook.zeek, 15 diff --git a/testing/btest/Baseline/language.assert-misc/out b/testing/btest/Baseline/language.assert-misc/out new file mode 100644 index 0000000000..1298f95384 --- /dev/null +++ b/testing/btest/Baseline/language.assert-misc/out @@ -0,0 +1,13 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +f, lambda_<505728364269398358> +{ +assert 0 < getpid(), fmt("my pid is funny: %s", getpid()); +} +g, lambda_<8496146571423528161> +{ +assert to_count("42") == 42; +} +test_function, test_function +{ +assert 0 < getpid(); +} diff --git a/testing/btest/Baseline/language.assert-top-level/.stderr b/testing/btest/Baseline/language.assert-top-level/.stderr new file mode 100644 index 0000000000..18c84b3171 --- /dev/null +++ b/testing/btest/Baseline/language.assert-top-level/.stderr @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/assert-top-level.zeek, line 5: assertion failure: getpid() == 0 (my pid greater 0? T) +fatal error: failed to execute script statements at top-level scope diff --git a/testing/btest/Baseline/language.assert/out b/testing/btest/Baseline/language.assert/out new file mode 100644 index 0000000000..256b93f1c8 --- /dev/null +++ b/testing/btest/Baseline/language.assert/out @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in <...>/assert.zeek, line 8: assertion failure: fmt("%s", 1) == "2" +fatal error: errors occurred while initializing diff --git a/testing/btest/language/assert-error.zeek b/testing/btest/language/assert-error.zeek new file mode 100644 index 0000000000..3ae8384e3d --- /dev/null +++ b/testing/btest/language/assert-error.zeek @@ -0,0 +1,27 @@ +# @TEST-DOC: Assert statement wrong usage +# +# @TEST-EXEC-FAIL: zeek -b %INPUT +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr + +event zeek_init() + { + assert 1; + } + +@TEST-START-NEXT +event zeek_init() + { + assert T, 1234; + } + +@TEST-START-NEXT +event zeek_init() + { + assert; + } + +@TEST-START-NEXT +event zeek_init() + { + assert T, "extra", "something"; + } diff --git a/testing/btest/language/assert-hook.zeek b/testing/btest/language/assert-hook.zeek new file mode 100644 index 0000000000..1edca8ee6b --- /dev/null +++ b/testing/btest/language/assert-hook.zeek @@ -0,0 +1,195 @@ +# @TEST-DOC: Assert statement testing with assertion_failure and assertion_result implementation. +# +# @TEST-EXEC: zeek -b %INPUT >out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr + +# Hook is not calling break: Reporter log is produced. +hook assertion_failure(cond: string, msg: string, bt: Backtrace) + { + print "assertion_failure", cond, msg, bt[0]$file_location, bt[0]$line_location; + } + +event zeek_init() + { + assert 1 != 1; + print "not reached"; + } + +@TEST-START-NEXT +# Test the backtrace location, also calling break to suppress reporter log. +hook assertion_failure(cond: string, msg: string, bt: Backtrace) + { + print "assertion_failure", cond, msg; + local indent = ""; + for ( _, e in bt ) + { + local file_name = e?$file_location ? e$file_location : ""; + local line_number = e?$line_location ? e$line_location : 0; + print fmt("%s%s %s:%s", indent, e$function_name, file_name, line_number); + indent = fmt("%s ", indent); + } + + break; + } + + +function f() + { + assert md5_hash("") == "d41d8cd98f00b204e9800998ecf8427e"; + assert to_count("5") == 4, fmt("5 is not 4"); + assert sha1_hash("") == "da39a3ee5e6b4b0d3255bfef95601890afd80709"; + } + +function g() { f(); } +function h() { g(); } + +event zeek_init() + { + h(); + print "not reached"; + } + +@TEST-START-NEXT +# Calling terminate() from the assertion hook. +redef exit_only_after_terminate = T; + +hook assertion_failure(cond: string, msg: string, bt: Backtrace) + { + print "assertion_failure", msg; + terminate(); + } + +event zeek_init() + { + assert F, "terminate me!"; + print "not reached"; + } + +event zeek_done() + { + print "zeek_done()"; + assert zeek_is_terminating(), "zeek_done() should have zeek terminating"; + } + +@TEST-START-NEXT +# Calling exit() from the assertion hook. +redef exit_only_after_terminate = T; + +hook assertion_failure(cond: string, msg: string, bt: Backtrace) + { + print "assertion_failure", msg; + exit(0); # in real tests use exit(1), this is to please btest. + } + +event zeek_init() + { + assert F, "calling exit!"; + print "not reached"; + } + +event zeek_done() + { + assert F, "zeek_done() not executed with exit()"; + } + +@TEST-START-NEXT +global assertion_failures = 0; +global assertions_total = 0; + +hook assertion_failure(cond: string, msg: string, bt: Backtrace) + { + print fmt("assertion_failure at %s:%s: %s%s%s", + bt[0]$file_location, bt[0]$line_location, + cond, |msg| > 0 ? " - " : "", msg); + + ++assertion_failures; + break; + } + +hook assertion_result(result: bool, cond: string, msg: string, bt: Backtrace) + { + print fmt("assertion_result %s at %s:%s: %s%s%s", + result, bt[0]$file_location, bt[0]$line_location, + cond, |msg| > 0 ? " - " : "", msg); + + ++assertions_total; + } + +event zeek_test() + { + assert md5_hash("") == "d41d8cd98f00b204e9800998ecf8427e"; + } + +event zeek_test() + { + assert sha1_hash("") == "da39a3ee5e6b4b0d3255bfef95601890afd80709"; + } + +event zeek_test() + { + assert sha1_hash("") == "wrong"; + } + +event zeek_test() + { + assert md5_hash("") == "wrong"; + } + +event zeek_init() + { + event zeek_test(); + } + +event zeek_done() + { + print fmt("%d of %d assertions failed", assertion_failures, assertions_total); + } + +@TEST-START-NEXT +# Evaluating the msg expression can cause errors, see if we deal +# with that gracefully. +hook assertion_failure(cond: string, msg: string, bt: Backtrace) + { + print "assertion_failure", cond, msg, bt[0]$file_location, bt[0]$line_location; + } + +hook assertion_result(result: bool, cond: string, msg: string, bt: Backtrace) + { + print "assertion_result", result, cond, msg, bt[0]$file_location, bt[0]$line_location; + } + +event zeek_init() + { + assert 2 + 2 == 4, cat(get_current_packet_header()$ip); + assert 2 + 2 == 4, to_json([$msg="true and works"]); + assert 2 + 2 == 5, cat(get_current_packet_header()$ip); + } + +event zeek_done() + { + assert 2 + 2 == 5, to_json([$msg="false and works"]); + assert 2 + 2 == 5, cat(get_current_packet_header()$ip); + } + +@TEST-START-NEXT +# Breaking in assertion_result() also suppresses the reporter errors. +hook assertion_result(result: bool, cond: string, msg: string, bt: Backtrace) + { + print "assertion_result", result, cond, msg, bt[0]$file_location, bt[0]$line_location; + break; + } + +event zeek_init() + { + assert 2 + 2 == 4, "this is true"; + assert 2 + 2 == 4, to_json([$msg="this is also true"]); + assert 2 + 2 == 5, "this is false"; + print "not reached"; + } + +event zeek_done() + { + assert 2 + 2 == 5, "this is false"; + print "not reached"; + } diff --git a/testing/btest/language/assert-misc.zeek b/testing/btest/language/assert-misc.zeek new file mode 100644 index 0000000000..996ae3efd4 --- /dev/null +++ b/testing/btest/language/assert-misc.zeek @@ -0,0 +1,28 @@ +# @TEST-DOC: Test Describe() of assert statement. Expressions may be canonicalized. +# +# @TEST-EXEC: zeek -b %INPUT >out 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out + +function test_function() + { + assert getpid() > 0; + } + +event zeek_init() + { + local f = function() { + assert getpid() > 0, fmt("my pid is funny: %s", getpid()); + }; + local g = function() { + assert to_count("42") == 42; + }; + + print "f", f; + f(); + + print "g", g; + g(); + + print "test_function", test_function; + test_function(); + } diff --git a/testing/btest/language/assert-top-level.zeek b/testing/btest/language/assert-top-level.zeek new file mode 100644 index 0000000000..738beca071 --- /dev/null +++ b/testing/btest/language/assert-top-level.zeek @@ -0,0 +1,5 @@ +# @TEST-EXEC-FAIL: zeek -b %INPUT >out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr + +assert getpid() > 0; +assert getpid() == 0, fmt("my pid greater 0? %s", getpid() > 0); diff --git a/testing/btest/language/assert.zeek b/testing/btest/language/assert.zeek new file mode 100644 index 0000000000..9fdffeaeae --- /dev/null +++ b/testing/btest/language/assert.zeek @@ -0,0 +1,75 @@ +# @TEST-DOC: Assert statement behavior testing without an assertion_failure() hook. +# +# @TEST-EXEC-FAIL: unset ZEEK_ALLOW_INIT_ERRORS; zeek -b %INPUT >out 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out + +event zeek_init() + { + assert fmt("%s", 1) == "2"; + print "not reached"; + } + +@TEST-START-NEXT +event zeek_init() + { + assert fmt("%s", 1) == "2", fmt("\"%s\" != \"2\"", 1); + print "not reached"; + } + +@TEST-START-NEXT +event zeek_init() + { + assert to_count("42") == 42.5, "always failing"; + print "not reached"; + } + +@TEST-START-NEXT +event zeek_init() + { + local x = 2; + assert x == 1, fmt("Expected x to be 1, have %s", x); + print "not reached"; + } + +@TEST-START-NEXT +event zeek_init() + { + local tbl: table[string] of string = [ + ["abc"] = "123", + ["def"] = "456", + ]; + assert "abc" in tbl, cat(tbl); + assert "def" in tbl, cat(tbl); + assert "ghi" in tbl, cat(tbl); + } + +@TEST-START-NEXT +type MyRecord: record { + a: count; + b: count &optional; +}; + +event zeek_init() + { + local r: MyRecord = [$a=1234]; + assert ! r?$b, fmt("Unexpected r$b is set to %s", r$b); + assert r?$b, fmt("r$b is not set in %s", r); + } + +@TEST-START-NEXT +type MyRecord: record { + a: count; + b: count &optional; +}; + +event zeek_init() + { + local r: MyRecord = [$a=1234]; + assert ! r?$b, fmt("Unexpected r$b is set to %s", r$b); + assert r?$b, fmt("r$b is not set trying anyway: %s", r$b); + } + +@TEST-START-NEXT +assert 1 == 1, "always true"; +assert 1 == 2, "always false"; +print "not reached";