diff --git a/.cirrus.yml b/.cirrus.yml index 6cde730bb3..e2cd173550 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -161,6 +161,13 @@ debian9_32bit_task: << : *RESOURCES_TEMPLATE << : *CI_TEMPLATE +opensuse_leap_15_2_task: + container: + # Opensuse Leap 15.2 EOL: Dec 2021 + dockerfile: ci/opensuse-leap-15.2/Dockerfile + << : *RESOURCES_TEMPLATE + << : *CI_TEMPLATE + ubuntu20_task: container: # Ubuntu 20.04 EOL: April 2025 @@ -182,6 +189,15 @@ ubuntu16_task: << : *RESOURCES_TEMPLATE << : *CI_TEMPLATE +alpine_task: + container: + # Alpine releases typically happen every 6 months w/ support for 2 years. + # The Dockerfile simply tracks latest Alpine release and shouldn't + # generally need updating based on particular Alpine release timelines. + dockerfile: ci/alpine/Dockerfile + << : *RESOURCES_TEMPLATE + << : *CI_TEMPLATE + # Apple doesn't publish official long-term support timelines. # We aim to support both the current and previous macOS release. macos_big_sur_task: diff --git a/CHANGES b/CHANGES index a8ef6a31d0..b3988bb2e4 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,133 @@ +4.1.0-dev.451 | 2021-03-31 11:58:08 -0700 + + * Add ssh to Alpine Dockerfile for retrieving external test repos (Jon Siwek, Corelight) + +4.1.0-dev.449 | 2021-03-31 10:47:22 -0700 + + * Change ci/init-external-repos.sh to use `base64 -d` (Jon Siwek, Corelight) + + The BusyBox version of `base64` does not have `--decode`. + +4.1.0-dev.448 | 2021-03-31 10:29:17 -0700 + + * Teach ci/init-external-repos.sh to consider user permissions (Jon Siwek, Corelight) + + Any errors while setting up external/private test repo should fail the + task for PRs submitted by a user with write/admin permission. + +4.1.0-dev.447 | 2021-03-31 09:55:05 -0700 + + * CI: Add OpenSUSE Leap 15.2 (Johanna Amann, Corelight) + + * Add CI task for Alpine Linux (Jon Siwek, Corelight) + + * Separate stdout from stderr in btest baselines (Jon Siwek, Corelight) + + Redirecting both to the same file can show platform-specific differences + (e.g. Alpine), likely due to different buffering defaults. + + * Remove newline-eof canonification attempt in diff-remove-timestamps (Jon Siwek, Corelight) + + * Change a include to (Jon Siwek, Corelight) + + Some systems (e.g. Alpine) may warn that that the former is incorrect. + +4.1.0-dev.440 | 2021-03-30 14:57:07 -0700 + + * Fix incomplete-type for struct timeval (Andrew Benson) + + This fixes building on musl (e.g. Void, Alpine, etc.) + +4.1.0-dev.438 | 2021-03-29 15:42:25 -0700 + + * "balance" tests with multiple Zeek scripts to load the same elements (Vern Paxson, Corelight) + + * put global statements into a quasi-function to support script optimization (Vern Paxson, Corelight) + +4.1.0-dev.434 | 2021-03-29 13:18:18 -0700 + + * Fix sign-compare compiler warning in coerce_to_record() (Jon Siwek, Corelight) + + * Fix maybe-uninitialized warning in ZVal::ToVal() (Jon Siwek, Corelight) + + * Change RecordVal::GetFieldAs() to use std::vector::operator[] (Jon Siwek, Corelight) + + Since the method claims it's up to the user to ensure the field exists + before calling, the extra bounds-checking done by std::vector::at() + isn't needed. + + * Add RecordVal::AssignField() and use it in supervisor code (Jon Siwek, Corelight) + + This is a convenience method to assign a known record field value by + field name. May also be useful to reduce warnings from static analysis + (e.g. Coverity) about not checking for negative return values before + assigning since that now flows through a [[noreturn]] error path. + + * GH-960: Fix include order of bundled header files (Jon Siwek, Corelight) + + Previously, a system-wide installation of any bundled auxil/ software + (like CAF) may get found/included rather than the bundled version and + possibly break the build. + +4.1.0-dev.427 | 2021-03-27 14:18:16 -0700 + + * Update bundled CAF to 0.18.2 (Jon Siwek, Corelight) + +4.1.0-dev.426 | 2021-03-26 17:17:45 -0700 + + * GH-1463: Rename supervisor_rotation_format_func to archiver_rotation_format_func (Vlad Grigorescu) + + And expose it for non-supervised setups. Also deprecates the old name. + +4.1.0-dev.423 | 2021-03-26 16:44:26 -0700 + + * GH-1454: Heartbleed: fix substraction order. (Johanna Amann) + + Fixes incorrect/overflowed `n` value for `SSL_Heartbeat_Many_Requests` + notices where number of server heartbeats is greater than number of client + heartbeats. + + The larger number was substracted from the smaller one leading to an + integer overflow. However, no information was lost due to everything + also being present in the notice message. + +4.1.0-dev.421 | 2021-03-26 16:39:22 -0700 + + * Fix crash in Analyzer::ForwardPacket due to recursive analyzer calls. (Tim Wojtulewicz, Corelight) + + The change in 44f558df7b5a85bae40945de653bcb2448e0a7f4 that made analyzer_list + a std::vector instead of a std::list doesn't take into account that in some + cases an analyzer may chain back into itself, such as with UDP-in-UDP tunnels. + In these cases, the second call to ForwardPacket may cause iterator + invalidation, leading to a crash, so this reverts back to using an std::list. + + * Include git sha in request to benchmark host (Tim Wojtulewicz, Corelight) + +4.1.0-dev.417 | 2021-03-25 11:37:55 -0700 + + * test suite update due to factoring out coerce_to_record() (Vern Paxson, Corelight) + + * removal of vestigial #include's (breaking an include loop in the process) (Vern Paxson, Corelight) + + * better method name: HasCopySemantics() (Vern Paxson, Corelight) + + * Use STL functionality to initialize coercion map (Vern Paxson, Corelight) + + * comments for factored-out index slice functions (Vern Paxson, Corelight) + + * support for subclassing ScriptFunc's, esp. for alternate lambda closures (Vern Paxson, Corelight) + + * factor out record coercion; modernize management of coercion "map" (Vern Paxson, Corelight) + + * lower-level method for adding fields to records (Vern Paxson, Corelight) + + * factor out "print" statement's execution functionality (Vern Paxson, Corelight) + + * functions for indexing slices and strings (Vern Paxson, Corelight) + + * new function for getting the location, if any, associated with the current call (Vern Paxson, Corelight) + 4.1.0-dev.404 | 2021-03-24 16:58:50 -0700 * Fix missing `user_agent` existence check in smtp/software.zeek (Michael Dopheide) diff --git a/CMakeLists.txt b/CMakeLists.txt index 17edb97e8c..6c3112c527 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -376,9 +376,6 @@ endif () # we have to care about CAF here because Broker headers can pull in CAF # headers. set(zeekdeps ${zeekdeps} ${CAF_LIBRARIES}) -include_directories(BEFORE ${broker_includes} ${CAF_INCLUDE_DIRS}) -include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/auxil/paraglob/include) -include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/auxil/rapidjson/include) include_directories(BEFORE ${PCAP_INCLUDE_DIR} ${BIND_INCLUDE_DIR} @@ -441,9 +438,6 @@ endif () # if one specifies --with-openssl (which may be common). include_directories(BEFORE ${OPENSSL_INCLUDE_DIR}) -# Make everyone find the highwayhash includes -include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/auxil/highwayhash) - # Determine if libfts is external to libc, i.e. musl find_package(FTS) if ( FTS_FOUND ) @@ -451,6 +445,14 @@ if ( FTS_FOUND ) include_directories(BEFORE ${FTS_INCLUDE_DIR}) endif () +# Any headers that are possibly bundled in the Zeek source-tree and that are supposed +# to have priority over any pre-existing/system-wide headers need to appear early in +# compiler search path. +include_directories(BEFORE ${broker_includes} ${CAF_INCLUDE_DIRS}) +include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/auxil/highwayhash) +include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/auxil/paraglob/include) +include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/auxil/rapidjson/include) + set(zeekdeps ${zeekdeps} ${BinPAC_LIBRARY} ${PCAP_LIBRARY} diff --git a/NEWS b/NEWS index 7d7a74a44b..410302b161 100644 --- a/NEWS +++ b/NEWS @@ -49,11 +49,6 @@ New Functionality Changed Functionality --------------------- -- The ``zeek::analyzer::analyzer_list`` type-alias changed from an - ``std::list`` to ``std::vector`` which, in practice, is not expected to be - used from plugins in API-incompatible way and may result in ~1-2% overall - performance benefit. - Removed Functionality --------------------- @@ -79,6 +74,8 @@ Deprecated Functionality environment variables prefixed by ``BRO_`` is now removed and calling ``getenv()`` directly with ``ZEEK_`` environment variables can be done. +- ``supervisor_rotation_format_func`` is renamed to ``archiver_rotation_format_func`` + Zeek 4.0.0 ========== diff --git a/VERSION b/VERSION index db2f2d0682..d8df332c52 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -4.1.0-dev.404 +4.1.0-dev.451 diff --git a/auxil/broker b/auxil/broker index 7810ec9936..938fd7de65 160000 --- a/auxil/broker +++ b/auxil/broker @@ -1 +1 @@ -Subproject commit 7810ec9936d1aa65c7c7d21b87e43c7d67481da0 +Subproject commit 938fd7de65220a986361ec596f2335ad471102d7 diff --git a/auxil/package-manager b/auxil/package-manager index cd89d32f1f..ccdd3f305c 160000 --- a/auxil/package-manager +++ b/auxil/package-manager @@ -1 +1 @@ -Subproject commit cd89d32f1f856544479e4c3d3d994eadf6759034 +Subproject commit ccdd3f305cb56040e212dab9ec9239a4bd0cc62b diff --git a/ci/alpine/Dockerfile b/ci/alpine/Dockerfile new file mode 100644 index 0000000000..be93e0c3d0 --- /dev/null +++ b/ci/alpine/Dockerfile @@ -0,0 +1,24 @@ +FROM alpine:latest + +RUN apk add --no-cache \ + cmake \ + make \ + g++ \ + python3 \ + python3-dev \ + flex \ + bison \ + libpcap-dev \ + openssl-dev \ + zlib-dev \ + swig \ + bash \ + bsd-compat-headers \ + linux-headers \ + fts-dev \ + git \ + curl \ + openssh-client \ + py3-pip + +RUN pip3 install junit2html diff --git a/ci/benchmark.sh b/ci/benchmark.sh index 7823915f63..71c65ab55b 100755 --- a/ci/benchmark.sh +++ b/ci/benchmark.sh @@ -35,7 +35,7 @@ TARGET="https://${ZEEK_BENCHMARK_HOST}:${ZEEK_BENCHMARK_PORT}${ZEEK_BENCHMARK_EN set +e # Make a request to the benchmark host. -RESULTS=$(curl -sS --stderr - --fail --insecure -X POST -H "Zeek-HMAC: ${HMAC_DIGEST}" -H "Zeek-HMAC-Timestamp: ${TIMESTAMP}" "${TARGET}?branch=${CIRRUS_BRANCH}&build=${BUILD_URL}&build_hash=${BUILD_HASH}") +RESULTS=$(curl -sS --stderr - --fail --insecure -X POST -H "Zeek-HMAC: ${HMAC_DIGEST}" -H "Zeek-HMAC-Timestamp: ${TIMESTAMP}" "${TARGET}?branch=${CIRRUS_BRANCH}&build=${BUILD_URL}&build_hash=${BUILD_HASH}&commit=${CIRRUS_CHANGE_IN_REPO}") STATUS=$? # If we got a bad status back from the host, we want to make sure to mask the host diff --git a/ci/init-external-repos.sh b/ci/init-external-repos.sh index 40bc07c243..277e372eb5 100755 --- a/ci/init-external-repos.sh +++ b/ci/init-external-repos.sh @@ -39,14 +39,20 @@ if [[ -n "${CIRRUS_CI}" ]] && [[ ! -d zeek-testing-private ]]; then # the key is also available in PRs for people with write access to the # repo, so we can still try for those cases). if [[ -n "${CIRRUS_PR}" ]]; then - set +e + if [[ "${CIRRUS_USER_PERMISSION}" == "write" ]]; then + set -e + elif [[ "${CIRRUS_USER_PERMISSION}" == "admin" ]]; then + set -e + else + set +e + fi else set -e fi banner "Trying to clone zeek-testing-private git repo" echo "${ZEEK_TESTING_PRIVATE_SSH_KEY}" > cirrus_key.b64 - base64 --decode cirrus_key.b64 > cirrus_key + base64 -d cirrus_key.b64 > cirrus_key rm cirrus_key.b64 chmod 600 cirrus_key git --version diff --git a/ci/opensuse-leap-15.2/Dockerfile b/ci/opensuse-leap-15.2/Dockerfile new file mode 100644 index 0000000000..7b23204cde --- /dev/null +++ b/ci/opensuse-leap-15.2/Dockerfile @@ -0,0 +1,25 @@ +FROM opensuse/leap:15.2 + +RUN zypper in -y \ + cmake \ + make \ + gcc \ + gcc-c++ \ + python3 \ + python3-devel \ + flex \ + bison \ + libpcap-devel \ + libopenssl-devel \ + zlib-devel \ + swig \ + git \ + curl \ + python3-pip \ + which \ + gzip \ + tar \ + && rm -rf /var/cache/zypp + + +RUN pip3 install junit2html diff --git a/doc b/doc index afd337f4b0..4bd4cf0233 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit afd337f4b00477401514ae3dc9c20476229318dc +Subproject commit 4bd4cf0233ba4d7585ce664f686eb4aff5ccb8dc diff --git a/scripts/base/frameworks/cluster/nodes/logger.zeek b/scripts/base/frameworks/cluster/nodes/logger.zeek index 772d512658..9d0d4121c8 100644 --- a/scripts/base/frameworks/cluster/nodes/logger.zeek +++ b/scripts/base/frameworks/cluster/nodes/logger.zeek @@ -22,11 +22,10 @@ redef Log::default_rotation_interval = 1 hrs; ## Alarm summary mail interval. redef Log::default_mail_alarms_interval = 24 hrs; -@if ( Supervisor::is_supervised() ) - -redef Log::default_rotation_dir = "log-queue"; - -function supervisor_rotation_format_func(ri: Log::RotationFmtInfo): Log::RotationPath +## This function will rotate logs in a format compatible with zeek-archiver. +## If you're using the Supervisor framework, this function will be used, +## if not, you can set :zeek:see:`Log::rotation_format_func` to this function. +function archiver_rotation_format_func(ri: Log::RotationFmtInfo): Log::RotationPath { local open_str = strftime(Log::default_rotation_date_format, ri$open); local close_str = strftime(Log::default_rotation_date_format, ri$close); @@ -35,7 +34,13 @@ function supervisor_rotation_format_func(ri: Log::RotationFmtInfo): Log::Rotatio return rval; } -redef Log::rotation_format_func = supervisor_rotation_format_func; +@if ( Supervisor::is_supervised() ) + +global supervisor_rotation_format_func = archiver_rotation_format_func &deprecated="Remove in v5.1. Use 'archiver_rotation_format_func'."; + +redef Log::default_rotation_dir = "log-queue"; + +redef Log::rotation_format_func = archiver_rotation_format_func; redef LogAscii::enable_leftover_log_rotation = T; diff --git a/scripts/policy/protocols/ssl/heartbleed.zeek b/scripts/policy/protocols/ssl/heartbleed.zeek index 483c1f4ce1..aabafbff14 100644 --- a/scripts/policy/protocols/ssl/heartbleed.zeek +++ b/scripts/policy/protocols/ssl/heartbleed.zeek @@ -154,7 +154,7 @@ event ssl_encrypted_heartbeat(c: connection, is_orig: bool, length: count) NOTICE([$note=SSL_Heartbeat_Many_Requests, $msg=fmt("Server sending more heartbeat responses than requests seen. Possible attack. Client count: %d, server count: %d", c$ssl$originator_heartbeats, c$ssl$responder_heartbeats), $conn=c, - $n=(c$ssl$originator_heartbeats-c$ssl$responder_heartbeats), + $n=(c$ssl$responder_heartbeats-c$ssl$originator_heartbeats), $identifier=fmt("%s%d", c$uid, c$ssl$responder_heartbeats/1000) # re-throw every 1000 heartbeats ]); diff --git a/src/Expr.cc b/src/Expr.cc index 1f5afbf8a7..c23fdb6b16 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -2936,24 +2936,7 @@ ValPtr IndexExpr::Fold(Val* v1, Val* v2) const if ( lv->Length() == 1 ) v = vect->ValAt(lv->Idx(0)->CoerceToUnsigned()); else - { - size_t len = vect->Size(); - auto result = make_intrusive(vect->GetType()); - - bro_int_t first = get_slice_index(lv->Idx(0)->CoerceToInt(), len); - bro_int_t last = get_slice_index(lv->Idx(1)->CoerceToInt(), len); - bro_int_t sub_length = last - first; - - if ( sub_length >= 0 ) - { - result->Resize(sub_length); - - for ( bro_int_t idx = first; idx < last; idx++ ) - result->Assign(idx - first, vect->ValAt(idx)); - } - - return result; - } + return index_slice(vect, lv); } break; @@ -2962,36 +2945,7 @@ ValPtr IndexExpr::Fold(Val* v1, Val* v2) const break; case TYPE_STRING: - { - const ListVal* lv = v2->AsListVal(); - const String* s = v1->AsString(); - int len = s->Len(); - String* substring = nullptr; - - if ( lv->Length() == 1 ) - { - bro_int_t idx = lv->Idx(0)->AsInt(); - - if ( idx < 0 ) - idx += len; - - // Out-of-range index will return null pointer. - substring = s->GetSubstring(idx, 1); - } - else - { - bro_int_t first = get_slice_index(lv->Idx(0)->AsInt(), len); - bro_int_t last = get_slice_index(lv->Idx(1)->AsInt(), len); - bro_int_t substring_len = last - first; - - if ( substring_len < 0 ) - substring = nullptr; - else - substring = s->GetSubstring(first, substring_len); - } - - return make_intrusive(substring ? substring : new String("")); - } + return index_string(v1->AsString(), v2->AsListVal()); default: RuntimeError("type cannot be indexed"); @@ -3005,6 +2959,63 @@ ValPtr IndexExpr::Fold(Val* v1, Val* v2) const return nullptr; } +StringValPtr index_string(const String* s, const ListVal* lv) + { + int len = s->Len(); + String* substring = nullptr; + + if ( lv->Length() == 1 ) + { + bro_int_t idx = lv->Idx(0)->AsInt(); + + if ( idx < 0 ) + idx += len; + + // Out-of-range index will return null pointer. + substring = s->GetSubstring(idx, 1); + } + else + { + bro_int_t first = get_slice_index(lv->Idx(0)->AsInt(), len); + bro_int_t last = get_slice_index(lv->Idx(1)->AsInt(), len); + bro_int_t substring_len = last - first; + + if ( substring_len < 0 ) + substring = nullptr; + else + substring = s->GetSubstring(first, substring_len); + } + + return make_intrusive(substring ? substring : new String("")); + } + +VectorValPtr index_slice(VectorVal* vect, const ListVal* lv) + { + auto first = lv->Idx(0)->CoerceToInt(); + auto last = lv->Idx(1)->CoerceToInt(); + return index_slice(vect, first, last); + } + +VectorValPtr index_slice(VectorVal* vect, int _first, int _last) + { + size_t len = vect->Size(); + auto result = make_intrusive(vect->GetType()); + + bro_int_t first = get_slice_index(_first, len); + bro_int_t last = get_slice_index(_last, len); + bro_int_t sub_length = last - first; + + if ( sub_length >= 0 ) + { + result->Resize(sub_length); + + for ( bro_int_t idx = first; idx < last; idx++ ) + result->Assign(idx - first, vect->ValAt(idx)); + } + + return result; + } + void IndexExpr::Assign(Frame* f, ValPtr v) { if ( IsError() ) @@ -3748,8 +3759,7 @@ ValPtr ArithCoerceExpr::Fold(Val* v) const } RecordCoerceExpr::RecordCoerceExpr(ExprPtr arg_op, RecordTypePtr r) - : UnaryExpr(EXPR_RECORD_COERCE, std::move(arg_op)), - map(nullptr), map_size(0) + : UnaryExpr(EXPR_RECORD_COERCE, std::move(arg_op)) { if ( IsError() ) return; @@ -3767,13 +3777,10 @@ RecordCoerceExpr::RecordCoerceExpr(ExprPtr arg_op, RecordTypePtr r) RecordType* t_r = type->AsRecordType(); RecordType* sub_r = op->GetType()->AsRecordType(); - map_size = t_r->NumFields(); - map = new int[map_size]; + int map_size = t_r->NumFields(); + map.resize(map_size, -1); // -1 = field is not mapped int i; - for ( i = 0; i < map_size; ++i ) - map[i] = -1; // -1 = field is not mapped - for ( i = 0; i < sub_r->NumFields(); ++i ) { int t_i = t_r->FieldOffset(sub_r->FieldName(i)); @@ -3854,11 +3861,6 @@ RecordCoerceExpr::RecordCoerceExpr(ExprPtr arg_op, RecordTypePtr r) } } -RecordCoerceExpr::~RecordCoerceExpr() - { - delete [] map; - } - ValPtr RecordCoerceExpr::InitVal(const zeek::Type* t, ValPtr aggr) const { if ( auto v = Eval(nullptr) ) @@ -3881,7 +3883,15 @@ ValPtr RecordCoerceExpr::Fold(Val* v) const if ( same_type(GetType(), Op()->GetType()) ) return IntrusivePtr{NewRef{}, v}; - auto val = make_intrusive(GetType()); + auto rt = cast_intrusive(GetType()); + return coerce_to_record(rt, v, map); + } + +RecordValPtr coerce_to_record(RecordTypePtr rt, Val* v, + const std::vector& map) + { + int map_size = map.size(); + auto val = make_intrusive(rt); RecordType* val_type = val->GetType()->AsRecordType(); RecordVal* rv = v->AsRecordVal(); @@ -3894,14 +3904,15 @@ ValPtr RecordCoerceExpr::Fold(Val* v) const if ( ! rhs ) { - const auto& def = rv->GetType()->AsRecordType()->FieldDecl( - map[i])->GetAttr(ATTR_DEFAULT); + auto rv_rt = rv->GetType()->AsRecordType(); + const auto& def = rv_rt->FieldDecl(map[i])-> + GetAttr(ATTR_DEFAULT); if ( def ) rhs = def->GetExpr()->Eval(nullptr); } - assert(rhs || GetType()->AsRecordType()->FieldDecl(i)->GetAttr(ATTR_OPTIONAL)); + assert(rhs || rt->FieldDecl(i)->GetAttr(ATTR_OPTIONAL)); if ( ! rhs ) { @@ -3923,21 +3934,19 @@ ValPtr RecordCoerceExpr::Fold(Val* v) const else if ( BothArithmetic(rhs_type->Tag(), field_type->Tag()) && ! same_type(rhs_type, field_type) ) { - if ( auto new_val = check_and_promote(rhs, field_type.get(), false, op->GetLocationInfo()) ) - rhs = std::move(new_val); - else - RuntimeError("Failed type conversion"); + auto new_val = check_and_promote(rhs, field_type.get(), false); + rhs = std::move(new_val); } val->Assign(i, std::move(rhs)); } else { - if ( const auto& def = GetType()->AsRecordType()->FieldDecl(i)->GetAttr(ATTR_DEFAULT) ) + if ( const auto& def = rt->FieldDecl(i)->GetAttr(ATTR_DEFAULT) ) { auto def_val = def->GetExpr()->Eval(nullptr); const auto& def_type = def_val->GetType(); - const auto& field_type = GetType()->AsRecordType()->GetFieldType(i); + const auto& field_type = rt->GetFieldType(i); if ( def_type->Tag() == TYPE_RECORD && field_type->Tag() == TYPE_RECORD && diff --git a/src/Expr.h b/src/Expr.h index ae275a7a84..3cdecfbb5e 100644 --- a/src/Expr.h +++ b/src/Expr.h @@ -941,6 +941,22 @@ protected: bool is_slice; }; +// The following execute the heart of IndexExpr functionality for +// vector slices and strings. + +// Extracts a slice of a vector, where the span of the slice is specified +// by a list of (exactly) two values. This is how the interpreter develops +// the components of a slice. +extern VectorValPtr index_slice(VectorVal* vect, const ListVal* lv); + +// Lower-level access to the slice, where its span is expressed +// directly as integers. +extern VectorValPtr index_slice(VectorVal* vect, int first, int last); + +// Returns a subset of a string, with the span specified by a list of +// (exactly) two values. +extern StringValPtr index_string(const String* s, const ListVal* lv); + class IndexExprWhen final : public IndexExpr { public: static inline std::vector results = {}; @@ -1166,21 +1182,24 @@ protected: class RecordCoerceExpr final : public UnaryExpr { public: RecordCoerceExpr(ExprPtr op, RecordTypePtr r); - ~RecordCoerceExpr() override; // Optimization-related: ExprPtr Duplicate() override; + const std::vector& Map() const { return map; } + protected: ValPtr InitVal(const zeek::Type* t, ValPtr aggr) const override; ValPtr Fold(Val* v) const override; // For each super-record slot, gives subrecord slot with which to // fill it. - int* map; - int map_size; // equivalent to Type()->AsRecordType()->NumFields() + std::vector map; }; +extern RecordValPtr coerce_to_record(RecordTypePtr rt, Val* v, + const std::vector& map); + class TableCoerceExpr final : public UnaryExpr { public: TableCoerceExpr(ExprPtr op, TableTypePtr r); diff --git a/src/File.h b/src/File.h index 8ed5d0a13e..78524826b2 100644 --- a/src/File.h +++ b/src/File.h @@ -12,8 +12,7 @@ #include #endif // NEED_KRB5_H -#include "zeek/Obj.h" -#include "zeek/IntrusivePtr.h" +#include "zeek/Val.h" #include "zeek/util.h" namespace zeek { @@ -23,6 +22,8 @@ namespace detail { class PrintStmt; class Attributes; +extern void do_print_stmt(const std::vector& vals); + } // namespace detail; class RecordVal; @@ -84,7 +85,7 @@ public: protected: - friend class detail::PrintStmt; + friend void detail::do_print_stmt(const std::vector& vals); File() { Init(); } void Init(); diff --git a/src/Frame.cc b/src/Frame.cc index c58d830d89..dd6e391adb 100644 --- a/src/Frame.cc +++ b/src/Frame.cc @@ -616,6 +616,11 @@ void Frame::CaptureClosure(Frame* c, IDPList arg_outer_ids) // if (c) closure = c->SelectiveClone(outer_ids); } +const detail::Location* Frame::GetCallLocation() const + { + return call ? call->GetLocationInfo() : nullptr; + } + void Frame::SetTrigger(trigger::TriggerPtr arg_trigger) { trigger = std::move(arg_trigger); diff --git a/src/Frame.h b/src/Frame.h index 36d775ef9f..4188dded27 100644 --- a/src/Frame.h +++ b/src/Frame.h @@ -262,6 +262,7 @@ public: void SetCall(const CallExpr* arg_call) { call = arg_call; } void ClearCall() { call = nullptr; } const CallExpr* GetCall() const { return call; } + const detail::Location* GetCallLocation() const; void SetDelayed() { delayed = true; } bool HasDelayed() const { return delayed; } diff --git a/src/Func.cc b/src/Func.cc index 319868395e..a01362c05e 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -587,6 +587,11 @@ bool ScriptFunc::StrengthenClosureReference(Frame* f) return true; } +bool ScriptFunc::HasCopySemantics() const + { + return type->GetCaptures().has_value(); + } + void ScriptFunc::SetClosureFrame(Frame* f) { if ( closure ) diff --git a/src/Func.h b/src/Func.h index db15fde849..b76603eba0 100644 --- a/src/Func.h +++ b/src/Func.h @@ -145,7 +145,7 @@ protected: namespace detail { -class ScriptFunc final : public Func { +class ScriptFunc : public Func { public: ScriptFunc(const IDPtr& id, StmtPtr body, const std::vector& inits, @@ -210,12 +210,17 @@ public: */ bool StrengthenClosureReference(Frame* f); + /** + * Whether the function's closure uses copy semantics. + */ + virtual bool HasCopySemantics() const; + /** * Serializes this function's closure or capture frame. * * @return a serialized version of the function's closure/capture frame. */ - broker::expected SerializeClosure() const; + virtual broker::expected SerializeClosure() const; /** * Sets the captures frame to one built from *data*. @@ -257,6 +262,7 @@ public: protected: ScriptFunc() : Func(SCRIPT_FUNC) {} + StmtPtr AddInits( StmtPtr body, const std::vector& inits); @@ -280,7 +286,7 @@ protected: * * @param f the frame holding the values of capture variables */ - void SetCaptures(Frame* f); + virtual void SetCaptures(Frame* f); private: size_t frame_size; diff --git a/src/ScannedFile.cc b/src/ScannedFile.cc index 77a6f1cb60..c7c7cafb0e 100644 --- a/src/ScannedFile.cc +++ b/src/ScannedFile.cc @@ -1,6 +1,6 @@ #include "zeek/ScannedFile.h" -#include +#include #include // for PATH_MAX #include "zeek/DebugLogger.h" diff --git a/src/Stmt.cc b/src/Stmt.cc index fed57f9496..77beb1c9e9 100644 --- a/src/Stmt.cc +++ b/src/Stmt.cc @@ -306,18 +306,23 @@ ValPtr PrintStmt::DoExec(std::vector vals, StmtFlowType& /* flow */) { RegisterAccess(); + do_print_stmt(vals); + return nullptr; + } +void do_print_stmt(const std::vector& vals) + { if ( ! print_stdout ) print_stdout = new File(stdout); File* f = print_stdout; int offset = 0; - if ( vals.size() > 0 && (vals)[0]->GetType()->Tag() == TYPE_FILE ) + if ( vals.size() > 0 && vals[0] && vals[0]->GetType()->Tag() == TYPE_FILE ) { f = (vals)[0]->AsFile(); if ( ! f->IsOpen() ) - return nullptr; + return; ++offset; } @@ -331,7 +336,7 @@ ValPtr PrintStmt::DoExec(std::vector vals, case BifEnum::Log::REDIRECT_ALL: { print_log(vals); - return nullptr; + return; } case BifEnum::Log::REDIRECT_STDOUT: if ( f->FileHandle() == stdout ) @@ -339,7 +344,7 @@ ValPtr PrintStmt::DoExec(std::vector vals, // Should catch even printing to a "manually opened" stdout file, // like "/dev/stdout" or "-". print_log(vals); - return nullptr; + return; } break; default: @@ -368,8 +373,6 @@ ValPtr PrintStmt::DoExec(std::vector vals, describe_vals(vals, &d, offset); f->Write("\n", 1); } - - return nullptr; } ExprStmt::ExprStmt(ExprPtr arg_e) : Stmt(STMT_EXPR), e(std::move(arg_e)) diff --git a/src/Stmt.h b/src/Stmt.h index bd26492afe..9bf13d9c9a 100644 --- a/src/Stmt.h +++ b/src/Stmt.h @@ -62,6 +62,8 @@ protected: StmtPtr DoSubclassReduce(ListExprPtr singletons, Reducer* c) override; }; +extern void do_print_stmt(const std::vector& vals); + class ExprStmt : public Stmt { public: explicit ExprStmt(ExprPtr e); diff --git a/src/Type.cc b/src/Type.cc index 6b39ef9198..97fb94ea03 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -1043,6 +1043,17 @@ const char* RecordType::AddFields(const type_decl_list& others, TableVal::SaveParseTimeTableState(this); + AddFieldsDirectly(others, add_log_attr); + + RecordVal::ResizeParseTimeRecords(this); + TableVal::RebuildParseTimeTables(); + + return nullptr; + } + +void RecordType::AddFieldsDirectly(const type_decl_list& others, + bool add_log_attr) + { for ( const auto& td : others ) { if ( add_log_attr ) @@ -1059,9 +1070,6 @@ const char* RecordType::AddFields(const type_decl_list& others, } num_fields = types->length(); - RecordVal::ResizeParseTimeRecords(this); - TableVal::RebuildParseTimeTables(); - return nullptr; } void RecordType::DescribeFields(ODesc* d) const diff --git a/src/Type.h b/src/Type.h index 09689ecf90..5a16463263 100644 --- a/src/Type.h +++ b/src/Type.h @@ -623,6 +623,9 @@ public: const char* AddFields(const type_decl_list& types, bool add_log_attr = false); + void AddFieldsDirectly(const type_decl_list& types, + bool add_log_attr = false); + void Describe(ODesc* d) const override; void DescribeReST(ODesc* d, bool roles_only = false) const override; void DescribeFields(ODesc* d) const; diff --git a/src/Val.h b/src/Val.h index 4d4984db0a..bfbca2b6a6 100644 --- a/src/Val.h +++ b/src/Val.h @@ -1152,6 +1152,19 @@ public: void Assign(int field, String* new_val) { Assign(field, new StringVal(new_val)); } + /** + * Assign a value of type @c T to a record field of the given name. + * A fatal error occurs if the no such field name exists. + */ + template + void AssignField(const char* field_name, T&& val) + { + int idx = GetType()->AsRecordType()->FieldOffset(field_name); + if ( idx < 0 ) + reporter->InternalError("missing record field: %s", field_name); + Assign(idx, std::forward(val)); + } + /** * Appends a value to the record's fields. The caller is responsible * for ensuring that fields are appended in the correct order and @@ -1288,33 +1301,33 @@ public: if constexpr ( std::is_same_v || std::is_same_v || std::is_same_v ) - return record_val->at(field).int_val; + return record_val->operator[](field).int_val; else if constexpr ( std::is_same_v ) - return record_val->at(field).uint_val; + return record_val->operator[](field).uint_val; else if constexpr ( std::is_same_v || std::is_same_v || std::is_same_v ) - return record_val->at(field).double_val; + return record_val->operator[](field).double_val; else if constexpr ( std::is_same_v ) return val_mgr->Port(record_val->at(field).uint_val); else if constexpr ( std::is_same_v ) - return record_val->at(field).string_val->Get(); + return record_val->operator[](field).string_val->Get(); else if constexpr ( std::is_same_v ) - return record_val->at(field).addr_val->Get(); + return record_val->operator[](field).addr_val->Get(); else if constexpr ( std::is_same_v ) - return record_val->at(field).subnet_val->Get(); + return record_val->operator[](field).subnet_val->Get(); else if constexpr ( std::is_same_v ) - return *(record_val->at(field).file_val); + return *(record_val->operator[](field).file_val); else if constexpr ( std::is_same_v ) - return *(record_val->at(field).func_val); + return *(record_val->operator[](field).func_val); else if constexpr ( std::is_same_v ) - return record_val->at(field).re_val->Get(); + return record_val->operator[](field).re_val->Get(); else if constexpr ( std::is_same_v ) - return record_val->at(field).record_val; + return record_val->operator[](field).record_val; else if constexpr ( std::is_same_v ) - return record_val->at(field).vector_val; + return record_val->operator[](field).vector_val; else if constexpr ( std::is_same_v ) - return record_val->at(field).table_val->Get(); + return record_val->operator[](field).table_val->Get(); else { // It's an error to reach here, although because of @@ -1329,12 +1342,12 @@ public: T GetFieldAs(int field) const { if constexpr ( std::is_integral_v && std::is_signed_v ) - return record_val->at(field).int_val; + return record_val->operator[](field).int_val; else if constexpr ( std::is_integral_v && std::is_unsigned_v ) - return record_val->at(field).uint_val; + return record_val->operator[](field).uint_val; else if constexpr ( std::is_floating_point_v ) - return record_val->at(field).double_val; + return record_val->operator[](field).double_val; // Note: we could add other types here using type traits, // such as is_same_v, etc. diff --git a/src/ZVal.cc b/src/ZVal.cc index a5dfea8a7e..0adaa70941 100644 --- a/src/ZVal.cc +++ b/src/ZVal.cc @@ -5,7 +5,6 @@ #include "zeek/Func.h" #include "zeek/OpaqueVal.h" #include "zeek/Reporter.h" -#include "zeek/Desc.h" using namespace zeek; @@ -264,8 +263,9 @@ ValPtr ZVal::ToVal(const TypePtr& t) const case TYPE_TIMER: case TYPE_UNION: case TYPE_VOID: + default: v = nullptr; - reporter->InternalError("bad ret type return tag"); + reporter->InternalError("bad type in ZVal::ToVal: %s", type_name(t->Tag())); } if ( v ) diff --git a/src/ZVal.h b/src/ZVal.h index 7cfb228047..1e1af6e900 100644 --- a/src/ZVal.h +++ b/src/ZVal.h @@ -6,11 +6,6 @@ #include "zeek/zeek-config.h" -#include - -#include "zeek/Dict.h" -#include "zeek/Expr.h" - namespace zeek { class StringVal; diff --git a/src/analyzer/Analyzer.h b/src/analyzer/Analyzer.h index 8de42eeff7..5999063fe6 100644 --- a/src/analyzer/Analyzer.h +++ b/src/analyzer/Analyzer.h @@ -38,7 +38,12 @@ class AnalyzerTimer; class SupportAnalyzer; class OutputHandler; -using analyzer_list = std::vector; +// This needs to remain a std::list because of the usage of iterators in the +// Analyzer::Forward methods. These methods have the chance to loop back +// into the same analyzer in the case of tunnels. If the recursive call adds +// to the children list, it can invalidate iterators in the outer call, +// causing a crash. +using analyzer_list = std::list; typedef uint32_t ID; typedef void (Analyzer::*analyzer_timer_func)(double t); diff --git a/src/broker/Data.cc b/src/broker/Data.cc index a54e30287c..6395796801 100644 --- a/src/broker/Data.cc +++ b/src/broker/Data.cc @@ -392,6 +392,17 @@ struct val_converter { if ( a.size() == 2 ) // we have a closure/capture frame { + // Note, seems if we already have a separate + // instance of the same lambda, then unless + // we use a cloned value, we'll step on that + // one's captures, too. This is because + // the capture mapping lives with the Func + // object rather than the FuncVal. However, + // we can't readily Clone() here because + // rval is const (and, grrr, Clone() is not). + // -VP + // rval = rval->Clone(); + auto frame = broker::get_if(a[1]); if ( ! frame ) return nullptr; @@ -400,9 +411,7 @@ struct val_converter { if ( ! b ) return nullptr; - auto copy_semantics = b->GetType()->GetCaptures().has_value(); - - if ( copy_semantics ) + if ( b->HasCopySemantics() ) { if ( ! b->DeserializeCaptures(*frame) ) return nullptr; @@ -1177,7 +1186,7 @@ broker::data& opaque_field_to_data(RecordVal* v, zeek::detail::Frame* f) const auto& d = v->GetField(0); if ( ! d ) - reporter->RuntimeError(f->GetCall()->GetLocationInfo(), + reporter->RuntimeError(f->GetCallLocation(), "Broker::Data's opaque field is not set"); // RuntimeError throws an exception which causes this line to never exceute. diff --git a/src/broker/Data.h b/src/broker/Data.h index a3db6c1514..feb58e3806 100644 --- a/src/broker/Data.h +++ b/src/broker/Data.h @@ -194,7 +194,7 @@ T& require_data_type(broker::data& d, zeek::TypeTag tag, zeek::detail::Frame* f) { auto ptr = caf::get_if(&d); if ( ! ptr ) - zeek::reporter->RuntimeError(f->GetCall()->GetLocationInfo(), + zeek::reporter->RuntimeError(f->GetCallLocation(), "data is of type '%s' not of type '%s'", caf::visit(type_name_getter{tag}, d), zeek::type_name(tag)); diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index 3d682ea4ab..0ce23d9508 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -74,7 +74,7 @@ int Manager::script_scope = 0; struct scoped_reporter_location { scoped_reporter_location(zeek::detail::Frame* frame) { - reporter->PushLocation(frame->GetCall()->GetLocationInfo()); + reporter->PushLocation(frame->GetCallLocation()); } ~scoped_reporter_location() diff --git a/src/parse.y b/src/parse.y index ac7a924e40..03a9f4cb68 100644 --- a/src/parse.y +++ b/src/parse.y @@ -266,7 +266,7 @@ static bool expr_is_table_type_name(const zeek::detail::Expr* expr) %% -bro: +zeek: decl_list stmt_list { if ( zeek::detail::stmts ) diff --git a/src/reporter.bif b/src/reporter.bif index 950b58f846..523195debb 100644 --- a/src/reporter.bif +++ b/src/reporter.bif @@ -22,7 +22,7 @@ module Reporter; ## .. zeek:see:: reporter_info function Reporter::info%(msg: string%): bool %{ - reporter->PushLocation(frame->GetCall()->GetLocationInfo()); + reporter->PushLocation(frame->GetCallLocation()); reporter->Info("%s", msg->CheckString()); reporter->PopLocation(); return zeek::val_mgr->True(); @@ -37,7 +37,7 @@ function Reporter::info%(msg: string%): bool ## .. zeek:see:: reporter_warning function Reporter::warning%(msg: string%): bool %{ - reporter->PushLocation(frame->GetCall()->GetLocationInfo()); + reporter->PushLocation(frame->GetCallLocation()); reporter->Warning("%s", msg->CheckString()); reporter->PopLocation(); return zeek::val_mgr->True(); @@ -53,7 +53,7 @@ function Reporter::warning%(msg: string%): bool ## .. zeek:see:: reporter_error function Reporter::error%(msg: string%): bool %{ - reporter->PushLocation(frame->GetCall()->GetLocationInfo()); + reporter->PushLocation(frame->GetCallLocation()); reporter->Error("%s", msg->CheckString()); reporter->PopLocation(); return zeek::val_mgr->True(); @@ -66,7 +66,7 @@ function Reporter::error%(msg: string%): bool ## Returns: Always true. function Reporter::fatal%(msg: string%): bool %{ - reporter->PushLocation(frame->GetCall()->GetLocationInfo()); + reporter->PushLocation(frame->GetCallLocation()); reporter->FatalError("%s", msg->CheckString()); reporter->PopLocation(); return zeek::val_mgr->True(); @@ -80,7 +80,7 @@ function Reporter::fatal%(msg: string%): bool ## Returns: Always true. function Reporter::fatal_error_with_core%(msg: string%): bool %{ - reporter->PushLocation(frame->GetCall()->GetLocationInfo()); + reporter->PushLocation(frame->GetCallLocation()); reporter->FatalErrorWithCore("%s", msg->CheckString()); reporter->PopLocation(); return zeek::val_mgr->True(); diff --git a/src/script_opt/ScriptOpt.cc b/src/script_opt/ScriptOpt.cc index 936ac7eea9..e083c5ab17 100644 --- a/src/script_opt/ScriptOpt.cc +++ b/src/script_opt/ScriptOpt.cc @@ -3,6 +3,7 @@ #include "zeek/Options.h" #include "zeek/Reporter.h" #include "zeek/Desc.h" +#include "zeek/module_util.h" #include "zeek/script_opt/ScriptOpt.h" #include "zeek/script_opt/ProfileFunc.h" #include "zeek/script_opt/Inline.h" @@ -153,6 +154,27 @@ void analyze_func(ScriptFuncPtr f) funcs.emplace_back(f, ScopePtr{NewRef{}, f->GetScope()}, f->CurrentBody()); } +const FuncInfo* analyze_global_stmts(Stmt* stmts) + { + // We ignore analysis_options.only_func - if it's in use, later + // logic will keep this function from being compiled, but it's handy + // now to enter it into "funcs" so we have a FuncInfo to return. + + auto id = install_ID("", GLOBAL_MODULE_NAME, true, false); + auto empty_args_t = make_intrusive(nullptr); + auto func_t = make_intrusive(empty_args_t, nullptr, FUNC_FLAVOR_FUNCTION); + id->SetType(func_t); + + auto sc = current_scope(); + std::vector empty_inits; + StmtPtr stmts_p{NewRef{}, stmts}; + auto sf = make_intrusive(id, stmts_p, empty_inits, sc->Length(), 0); + + funcs.emplace_back(sf, ScopePtr{NewRef{}, sc}, stmts_p); + + return &funcs.back(); + } + static void check_env_opt(const char* opt, bool& opt_flag) { if ( getenv(opt) ) diff --git a/src/script_opt/ScriptOpt.h b/src/script_opt/ScriptOpt.h index bee7f93f4c..11da7c22f3 100644 --- a/src/script_opt/ScriptOpt.h +++ b/src/script_opt/ScriptOpt.h @@ -106,6 +106,11 @@ extern std::unordered_set non_recursive_funcs; // Analyze a given function for optimization. extern void analyze_func(ScriptFuncPtr f); +// Analyze the given top-level statement(s) for optimization. Returns +// a pointer to a FuncInfo for an argument-less quasi-function that can +// be Invoked, or its body executed directly, to execute the statements. +extern const FuncInfo* analyze_global_stmts(Stmt* stmts); + // Analyze all of the parsed scripts collectively for optimization. extern void analyze_scripts(); diff --git a/src/supervisor/Supervisor.cc b/src/supervisor/Supervisor.cc index ce3fe8d220..d751387a1e 100644 --- a/src/supervisor/Supervisor.cc +++ b/src/supervisor/Supervisor.cc @@ -1348,22 +1348,22 @@ RecordValPtr Supervisor::NodeConfig::ToRecord() const { const auto& rt = BifType::Record::Supervisor::NodeConfig; auto rval = make_intrusive(rt); - rval->Assign(rt->FieldOffset("name"), name); + rval->AssignField("name", name); if ( interface ) - rval->Assign(rt->FieldOffset("interface"), *interface); + rval->AssignField("interface", *interface); if ( directory ) - rval->Assign(rt->FieldOffset("directory"), *directory); + rval->AssignField("directory", *directory); if ( stdout_file ) - rval->Assign(rt->FieldOffset("stdout_file"), *stdout_file); + rval->AssignField("stdout_file", *stdout_file); if ( stderr_file ) - rval->Assign(rt->FieldOffset("stderr_file"), *stderr_file); + rval->AssignField("stderr_file", *stderr_file); if ( cpu_affinity ) - rval->Assign(rt->FieldOffset("cpu_affinity"), *cpu_affinity); + rval->AssignField("cpu_affinity", *cpu_affinity); auto st = rt->GetFieldType("scripts"); auto scripts_val = make_intrusive(std::move(st)); @@ -1371,11 +1371,11 @@ RecordValPtr Supervisor::NodeConfig::ToRecord() const for ( const auto& s : scripts ) scripts_val->Assign(scripts_val->Size(), make_intrusive(s)); - rval->Assign(rt->FieldOffset("scripts"), std::move(scripts_val)); + rval->AssignField("scripts", std::move(scripts_val)); auto tt = rt->GetFieldType("cluster"); auto cluster_val = make_intrusive(std::move(tt)); - rval->Assign(rt->FieldOffset("cluster"), cluster_val); + rval->AssignField("cluster", cluster_val); for ( const auto& e : cluster ) { @@ -1385,12 +1385,12 @@ RecordValPtr Supervisor::NodeConfig::ToRecord() const const auto& ept = BifType::Record::Supervisor::ClusterEndpoint; auto val = make_intrusive(ept); - val->Assign(ept->FieldOffset("role"), BifType::Enum::Supervisor::ClusterRole->GetEnumVal(ep.role)); - val->Assign(ept->FieldOffset("host"), make_intrusive(ep.host)); - val->Assign(ept->FieldOffset("p"), val_mgr->Port(ep.port, TRANSPORT_TCP)); + val->AssignField("role", BifType::Enum::Supervisor::ClusterRole->GetEnumVal(ep.role)); + val->AssignField("host", make_intrusive(ep.host)); + val->AssignField("p", val_mgr->Port(ep.port, TRANSPORT_TCP)); if ( ep.interface ) - val->Assign(ept->FieldOffset("interface"), *ep.interface); + val->AssignField("interface", *ep.interface); cluster_val->Assign(std::move(key), std::move(val)); } @@ -1403,10 +1403,10 @@ RecordValPtr SupervisorNode::ToRecord() const const auto& rt = BifType::Record::Supervisor::NodeStatus; auto rval = make_intrusive(rt); - rval->Assign(rt->FieldOffset("node"), config.ToRecord()); + rval->AssignField("node", config.ToRecord()); if ( pid ) - rval->Assign(rt->FieldOffset("pid"), pid); + rval->AssignField("pid", pid); return rval; } @@ -1458,17 +1458,15 @@ bool SupervisedNode::InitCluster() const auto val = make_intrusive(cluster_node_type); auto node_type = supervisor_role_to_cluster_node_type(ep.role); - val->Assign(cluster_node_type->FieldOffset("node_type"), std::move(node_type)); - val->Assign(cluster_node_type->FieldOffset("ip"), make_intrusive(ep.host)); - val->Assign(cluster_node_type->FieldOffset("p"), val_mgr->Port(ep.port, TRANSPORT_TCP)); + val->AssignField("node_type", std::move(node_type)); + val->AssignField("ip", make_intrusive(ep.host)); + val->AssignField("p", val_mgr->Port(ep.port, TRANSPORT_TCP)); if ( ep.interface ) - val->Assign(cluster_node_type->FieldOffset("interface"), - *ep.interface); + val->AssignField("interface", *ep.interface); if ( manager_name && ep.role != BifEnum::Supervisor::MANAGER ) - val->Assign(cluster_node_type->FieldOffset("manager"), - *manager_name); + val->AssignField("manager", *manager_name); cluster_nodes->Assign(std::move(key), std::move(val)); } diff --git a/src/supervisor/supervisor.bif b/src/supervisor/supervisor.bif index c143bc1dcf..3d65ab31a8 100644 --- a/src/supervisor/supervisor.bif +++ b/src/supervisor/supervisor.bif @@ -86,7 +86,7 @@ function Supervisor::__node%(%): Supervisor::NodeConfig zeek::emit_builtin_error("not a supervised process"); const auto& rt = zeek::BifType::Record::Supervisor::NodeConfig; auto rval = zeek::make_intrusive(rt); - rval->Assign(rt->FieldOffset("name"), ""); + rval->AssignField("name", ""); return rval; } diff --git a/src/util.cc b/src/util.cc index ce4f74cc0b..9c73c33deb 100644 --- a/src/util.cc +++ b/src/util.cc @@ -5,17 +5,6 @@ #include "zeek/util.h" -#ifdef TIME_WITH_SYS_TIME -# include -# include -#else -# ifdef HAVE_SYS_TIME_H -# include -# else -# include -# endif -#endif - #ifdef HAVE_DARWIN #include #include diff --git a/src/util.h b/src/util.h index a4d5c6a524..4da4a10735 100644 --- a/src/util.h +++ b/src/util.h @@ -29,6 +29,17 @@ #include #include // std::unique_ptr +#ifdef TIME_WITH_SYS_TIME +# include +# include +#else +# ifdef HAVE_SYS_TIME_H +# include +# else +# include +# endif +#endif + #ifdef DEBUG #include diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index 51e3162c2b..a03f9b3696 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -752,6 +752,8 @@ SetupResult setup(int argc, char** argv, Options* zopts) } } + auto init_stmts = stmts ? analyze_global_stmts(stmts) : nullptr; + analyze_scripts(); if ( analysis_options.report_recursive ) @@ -838,15 +840,15 @@ SetupResult setup(int argc, char** argv, Options* zopts) // cause more severe problems. ZEEK_LSAN_ENABLE(); - if ( stmts ) + if ( init_stmts ) { StmtFlowType flow; - Frame f(current_scope()->Length(), nullptr, nullptr); + Frame f(init_stmts->Scope()->Length(), nullptr, nullptr); g_frame_stack.push_back(&f); try { - stmts->Exec(&f, flow); + init_stmts->Body()->Exec(&f, flow); } catch ( InterpreterException& ) { diff --git a/testing/btest/Baseline/bifs.bloomfilter/err b/testing/btest/Baseline/bifs.bloomfilter/err new file mode 100644 index 0000000000..2a4c6d7420 --- /dev/null +++ b/testing/btest/Baseline/bifs.bloomfilter/err @@ -0,0 +1,7 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error: incompatible Bloom filter types +error: incompatible Bloom filter types +error: incompatible Bloom filter types +error: incompatible Bloom filter types +error: false-positive rate must take value between 0 and 1 +error: false-positive rate must take value between 0 and 1 diff --git a/testing/btest/Baseline/bifs.bloomfilter/output b/testing/btest/Baseline/bifs.bloomfilter/output index 1bbe4592d2..2e6dfc8abe 100644 --- a/testing/btest/Baseline/bifs.bloomfilter/output +++ b/testing/btest/Baseline/bifs.bloomfilter/output @@ -1,10 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error: incompatible Bloom filter types -error: incompatible Bloom filter types -error: incompatible Bloom filter types -error: incompatible Bloom filter types -error: false-positive rate must take value between 0 and 1 -error: false-positive rate must take value between 0 and 1 0 1 1 diff --git a/testing/btest/Baseline/bifs.strptime/err b/testing/btest/Baseline/bifs.strptime/err new file mode 100644 index 0000000000..7770adb5ba --- /dev/null +++ b/testing/btest/Baseline/bifs.strptime/err @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +warning: strptime conversion failed: fmt:%m d:1980-10-24 diff --git a/testing/btest/Baseline/bifs.strptime/out b/testing/btest/Baseline/bifs.strptime/out index a6c49aa77c..8784f8dc3e 100644 --- a/testing/btest/Baseline/bifs.strptime/out +++ b/testing/btest/Baseline/bifs.strptime/out @@ -1,4 +1,3 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -warning: strptime conversion failed: fmt:%m d:1980-10-24 XXXXXXXXXX.XXXXXX 0.0 diff --git a/testing/btest/Baseline/core.init-error/err b/testing/btest/Baseline/core.init-error/err new file mode 100644 index 0000000000..2d1ce60bf3 --- /dev/null +++ b/testing/btest/Baseline/core.init-error/err @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +expression error in <...>/init-error.zeek, line 16: no such index (v[10]) +fatal error: errors occurred while initializing diff --git a/testing/btest/Baseline/core.init-error/out b/testing/btest/Baseline/core.init-error/out index a8f510074e..14ab5a1282 100644 --- a/testing/btest/Baseline/core.init-error/out +++ b/testing/btest/Baseline/core.init-error/out @@ -1,6 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -expression error in <...>/init-error.zeek, line 15: no such index (v[10]) -fatal error: errors occurred while initializing 1st event 2nd event 3rd event diff --git a/testing/btest/Baseline/core.pcap.dumper/output b/testing/btest/Baseline/core.pcap.dumper/output index 9ccfcee235..0bfe0ff5ac 100644 --- a/testing/btest/Baseline/core.pcap.dumper/output +++ b/testing/btest/Baseline/core.pcap.dumper/output @@ -1,5 +1,8 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -2c2 -< 00000010 ff ff 00 00 01 00 00 00 1d a2 b2 4e 73 00 07 00 |...........Ns...| ---- -> 00000010 00 24 00 00 01 00 00 00 1d a2 b2 4e 73 00 07 00 |.$.........Ns...| +@@ -1,5 +1,5 @@ + 00000000 d4 c3 b2 a1 02 00 04 00 00 00 00 00 00 00 00 00 |................| +-00000010 ff ff 00 00 01 00 00 00 1d a2 b2 4e 73 00 07 00 |...........Ns...| ++00000010 00 24 00 00 01 00 00 00 1d a2 b2 4e 73 00 07 00 |.$.........Ns...| + 00000020 4a 00 00 00 4a 00 00 00 52 54 00 12 35 02 08 00 |J...J...RT..5...| + 00000030 27 87 94 24 08 00 45 00 00 3c c1 d8 40 00 40 06 |'..$..E..<..@.@.| + 00000040 f1 12 0a 00 02 0f c0 96 bb 2b c0 82 00 50 01 28 |.........+...P.(| diff --git a/testing/btest/Baseline/core.reporter-error-in-handler/err b/testing/btest/Baseline/core.reporter-error-in-handler/err new file mode 100644 index 0000000000..1c7f00473e --- /dev/null +++ b/testing/btest/Baseline/core.reporter-error-in-handler/err @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +expression error in <...>/reporter-error-in-handler.zeek, line 29: no such index (a[1]) +expression error in <...>/reporter-error-in-handler.zeek, line 23: no such index (a[2]) diff --git a/testing/btest/Baseline/core.reporter-error-in-handler/output b/testing/btest/Baseline/core.reporter-error-in-handler/output index f233ff3d18..4ffc7f805d 100644 --- a/testing/btest/Baseline/core.reporter-error-in-handler/output +++ b/testing/btest/Baseline/core.reporter-error-in-handler/output @@ -1,4 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -expression error in <...>/reporter-error-in-handler.zeek, line 28: no such index (a[1]) -expression error in <...>/reporter-error-in-handler.zeek, line 22: no such index (a[2]) 1st error printed on script level diff --git a/testing/btest/Baseline/core.tunnels.teredo-udp-in-udp/tunnel.log b/testing/btest/Baseline/core.tunnels.teredo-udp-in-udp/tunnel.log new file mode 100644 index 0000000000..e33e982a23 --- /dev/null +++ b/testing/btest/Baseline/core.tunnels.teredo-udp-in-udp/tunnel.log @@ -0,0 +1,12 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path tunnel +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p tunnel_type action +#types time string addr port addr port enum enum +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 ::38:5f55:6265:726b 25977 2090:9090:9090:9090:9090:9000:: 25964 Tunnel::TEREDO Tunnel::DISCOVER +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 ::38:5f55:6265:726b 25977 2090:9090:9090:9090:9090:9000:: 25964 Tunnel::TEREDO Tunnel::CLOSE +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/language.common-mistakes/1.err b/testing/btest/Baseline/language.common-mistakes/1.err new file mode 100644 index 0000000000..571bb4a831 --- /dev/null +++ b/testing/btest/Baseline/language.common-mistakes/1.err @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +expression error in ./1.zeek, line 9: field value missing (mr$f) diff --git a/testing/btest/Baseline/language.common-mistakes/1.out b/testing/btest/Baseline/language.common-mistakes/1.out index 5496a099cd..9169b4f45e 100644 --- a/testing/btest/Baseline/language.common-mistakes/1.out +++ b/testing/btest/Baseline/language.common-mistakes/1.out @@ -1,5 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -expression error in ./1.zeek, line 9: field value missing (mr$f) bar start foo start other zeek_init diff --git a/testing/btest/Baseline/language.common-mistakes/2.err b/testing/btest/Baseline/language.common-mistakes/2.err new file mode 100644 index 0000000000..c8e7c86b12 --- /dev/null +++ b/testing/btest/Baseline/language.common-mistakes/2.err @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +expression error in ./2.zeek, line 7: no such index (t[nope]) diff --git a/testing/btest/Baseline/language.common-mistakes/2.out b/testing/btest/Baseline/language.common-mistakes/2.out index db87dbf7ec..8e1dd6eefb 100644 --- a/testing/btest/Baseline/language.common-mistakes/2.out +++ b/testing/btest/Baseline/language.common-mistakes/2.out @@ -1,3 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -expression error in ./2.zeek, line 7: no such index (t[nope]) in foo diff --git a/testing/btest/Baseline/language.common-mistakes/3.err b/testing/btest/Baseline/language.common-mistakes/3.err new file mode 100644 index 0000000000..41ae45fdce --- /dev/null +++ b/testing/btest/Baseline/language.common-mistakes/3.err @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +expression error in ./3.zeek, line 5: type-checking failed in vector append (v += ok) diff --git a/testing/btest/Baseline/language.common-mistakes/3.out b/testing/btest/Baseline/language.common-mistakes/3.out index c6fc21fcc3..8e1dd6eefb 100644 --- a/testing/btest/Baseline/language.common-mistakes/3.out +++ b/testing/btest/Baseline/language.common-mistakes/3.out @@ -1,3 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -expression error in ./3.zeek, line 5: type-checking failed in vector append (v += ok) in foo diff --git a/testing/btest/Baseline/language.invalid_index/err b/testing/btest/Baseline/language.invalid_index/err new file mode 100644 index 0000000000..5910767e32 --- /dev/null +++ b/testing/btest/Baseline/language.invalid_index/err @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +expression error in <...>/invalid_index.zeek, line 11: no such index (foo[1]) +expression error in <...>/invalid_index.zeek, line 17: no such index (foo2[1]) diff --git a/testing/btest/Baseline/language.invalid_index/out b/testing/btest/Baseline/language.invalid_index/out index 80fe32a510..a36cdc6472 100644 --- a/testing/btest/Baseline/language.invalid_index/out +++ b/testing/btest/Baseline/language.invalid_index/out @@ -1,6 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -expression error in <...>/invalid_index.zeek, line 10: no such index (foo[1]) -expression error in <...>/invalid_index.zeek, line 16: no such index (foo2[1]) foo[0], 42 foo2[0], 13 done diff --git a/testing/btest/Baseline/language.subnet-errors/err b/testing/btest/Baseline/language.subnet-errors/err new file mode 100644 index 0000000000..9bd82b682e --- /dev/null +++ b/testing/btest/Baseline/language.subnet-errors/err @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +expression error in <...>/subnet-errors.zeek, line 10: bad IPv4 subnet prefix length: 33 (1.2.3.4 / i) +expression error in <...>/subnet-errors.zeek, line 19: bad IPv6 subnet prefix length: 129 (:: / i) diff --git a/testing/btest/Baseline/language.subnet-errors/out b/testing/btest/Baseline/language.subnet-errors/out index 06dac194d1..b7c2e2e895 100644 --- a/testing/btest/Baseline/language.subnet-errors/out +++ b/testing/btest/Baseline/language.subnet-errors/out @@ -1,6 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -expression error in <...>/subnet-errors.zeek, line 9: bad IPv4 subnet prefix length: 33 (1.2.3.4 / i) -expression error in <...>/subnet-errors.zeek, line 18: bad IPv6 subnet prefix length: 129 (:: / i) 1.2.3.4/32 ::/128 init last diff --git a/testing/btest/Baseline/language.table-set-iterator-invalidation/err b/testing/btest/Baseline/language.table-set-iterator-invalidation/err new file mode 100644 index 0000000000..e33c980fc3 --- /dev/null +++ b/testing/btest/Baseline/language.table-set-iterator-invalidation/err @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +warning in <...>/table-set-iterator-invalidation.zeek, line 22: possible loop/iterator invalidation caused by expression: t[4] +warning in <...>/table-set-iterator-invalidation.zeek, line 31: possible loop/iterator invalidation caused by expression: t[4] +warning in <...>/table-set-iterator-invalidation.zeek, line 54: possible loop/iterator invalidation caused by expression: s[4] +warning in <...>/table-set-iterator-invalidation.zeek, line 63: possible loop/iterator invalidation caused by expression: s[4] diff --git a/testing/btest/Baseline/language.table-set-iterator-invalidation/out b/testing/btest/Baseline/language.table-set-iterator-invalidation/out index 38cadd94a2..3c6cc09e27 100644 --- a/testing/btest/Baseline/language.table-set-iterator-invalidation/out +++ b/testing/btest/Baseline/language.table-set-iterator-invalidation/out @@ -1,8 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -warning in <...>/table-set-iterator-invalidation.zeek, line 21: possible loop/iterator invalidation caused by expression: t[4] -warning in <...>/table-set-iterator-invalidation.zeek, line 30: possible loop/iterator invalidation caused by expression: t[4] -warning in <...>/table-set-iterator-invalidation.zeek, line 53: possible loop/iterator invalidation caused by expression: s[4] -warning in <...>/table-set-iterator-invalidation.zeek, line 62: possible loop/iterator invalidation caused by expression: s[4] { [2] = 2, [1] = 1, diff --git a/testing/btest/Baseline/language.type-cast-error-dynamic/err b/testing/btest/Baseline/language.type-cast-error-dynamic/err new file mode 100644 index 0000000000..bc7a985010 --- /dev/null +++ b/testing/btest/Baseline/language.type-cast-error-dynamic/err @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +expression error in <...>/type-cast-error-dynamic.zeek, line 12: invalid cast of value with type 'count' to type 'string' (a as string) +expression error in <...>/type-cast-error-dynamic.zeek, line 12: invalid cast of value with type 'record { a:addr; b:port; }' to type 'string' (a as string) +expression error in <...>/type-cast-error-dynamic.zeek, line 12: invalid cast of value with type 'record { data:opaque of Broker::Data &optional; }' to type 'string' (nil $data field) (a as string) diff --git a/testing/btest/Baseline/language.type-cast-error-dynamic/output b/testing/btest/Baseline/language.type-cast-error-dynamic/output index bf6438e3e6..ec70675fe2 100644 --- a/testing/btest/Baseline/language.type-cast-error-dynamic/output +++ b/testing/btest/Baseline/language.type-cast-error-dynamic/output @@ -1,5 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -expression error in <...>/type-cast-error-dynamic.zeek, line 11: invalid cast of value with type 'count' to type 'string' (a as string) -expression error in <...>/type-cast-error-dynamic.zeek, line 11: invalid cast of value with type 'record { a:addr; b:port; }' to type 'string' (a as string) -expression error in <...>/type-cast-error-dynamic.zeek, line 11: invalid cast of value with type 'record { data:opaque of Broker::Data &optional; }' to type 'string' (nil $data field) (a as string) data is string, F diff --git a/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure1.err b/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure1.err new file mode 100644 index 0000000000..833eb76999 --- /dev/null +++ b/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure1.err @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in ./double_convert_failure1.zeek, line 7 and double: type clash for field "cc" ((coerce [$cc=5.0] to myrecord) and double) diff --git a/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure1.out b/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure1.out index 833eb76999..49d861c74c 100644 --- a/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure1.out +++ b/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure1.out @@ -1,2 +1 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in ./double_convert_failure1.zeek, line 7 and double: type clash for field "cc" ((coerce [$cc=5.0] to myrecord) and double) diff --git a/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure2.err b/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure2.err new file mode 100644 index 0000000000..c89e0282e1 --- /dev/null +++ b/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure2.err @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in ./double_convert_failure2.zeek, line 7 and double: type clash for field "cc" ((coerce [$cc=-5.0] to myrecord) and double) diff --git a/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure2.out b/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure2.out index c89e0282e1..49d861c74c 100644 --- a/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure2.out +++ b/testing/btest/Baseline/language.type-coerce-numerics/double_convert_failure2.out @@ -1,2 +1 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in ./double_convert_failure2.zeek, line 7 and double: type clash for field "cc" ((coerce [$cc=-5.0] to myrecord) and double) diff --git a/testing/btest/Baseline/language.type-coerce-numerics/first_set.err b/testing/btest/Baseline/language.type-coerce-numerics/first_set.err new file mode 100644 index 0000000000..20e1c94aae --- /dev/null +++ b/testing/btest/Baseline/language.type-coerce-numerics/first_set.err @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in int: overflow promoting from unsigned/double to signed arithmetic value (int and 9223372036854775808) diff --git a/testing/btest/Baseline/language.type-coerce-numerics/first_set.out b/testing/btest/Baseline/language.type-coerce-numerics/first_set.out index f053ab7046..b31869c4eb 100644 --- a/testing/btest/Baseline/language.type-coerce-numerics/first_set.out +++ b/testing/btest/Baseline/language.type-coerce-numerics/first_set.out @@ -1,6 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in int and ./first_set.zeek, line 46: overflow promoting from unsigned/double to signed arithmetic value (int and 9223372036854775808) -expression error in ./first_set.zeek, line 46: Failed type conversion ((coerce [$ii=9223372036854775808] to record { ii:int &optional; cc:count &optional; dd:double &optional; })) 3 int 4 diff --git a/testing/btest/Baseline/language.type-coerce-numerics/int_convert_failure.err b/testing/btest/Baseline/language.type-coerce-numerics/int_convert_failure.err new file mode 100644 index 0000000000..9bca49e323 --- /dev/null +++ b/testing/btest/Baseline/language.type-coerce-numerics/int_convert_failure.err @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error in ./int_convert_failure.zeek, line 7 and int: type clash for field "cc" ((coerce [$cc=-5] to myrecord) and int) diff --git a/testing/btest/Baseline/language.type-coerce-numerics/int_convert_failure.out b/testing/btest/Baseline/language.type-coerce-numerics/int_convert_failure.out index 9bca49e323..49d861c74c 100644 --- a/testing/btest/Baseline/language.type-coerce-numerics/int_convert_failure.out +++ b/testing/btest/Baseline/language.type-coerce-numerics/int_convert_failure.out @@ -1,2 +1 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error in ./int_convert_failure.zeek, line 7 and int: type clash for field "cc" ((coerce [$cc=-5] to myrecord) and int) diff --git a/testing/btest/Baseline/language.type-coerce-numerics/vectors.err b/testing/btest/Baseline/language.type-coerce-numerics/vectors.err new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/language.type-coerce-numerics/vectors.err @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/language.uninitialized-local2/err b/testing/btest/Baseline/language.uninitialized-local2/err new file mode 100644 index 0000000000..bf4ff14005 --- /dev/null +++ b/testing/btest/Baseline/language.uninitialized-local2/err @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +expression error in <...>/uninitialized-local2.zeek, line 20: value used but not set (var_b) diff --git a/testing/btest/Baseline/language.uninitialized-local2/out b/testing/btest/Baseline/language.uninitialized-local2/out index 8d041a13c0..ba5c5cef5e 100644 --- a/testing/btest/Baseline/language.uninitialized-local2/out +++ b/testing/btest/Baseline/language.uninitialized-local2/out @@ -1,3 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -expression error in <...>/uninitialized-local2.zeek, line 19: value used but not set (var_b) var_a is, baz diff --git a/testing/btest/Baseline/language.uninitialized-local3/err b/testing/btest/Baseline/language.uninitialized-local3/err new file mode 100644 index 0000000000..ae81d66dbd --- /dev/null +++ b/testing/btest/Baseline/language.uninitialized-local3/err @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +warning in <...>/uninitialized-local3.zeek, line 39: possibly used without definition (x4) +expression error in <...>/uninitialized-local3.zeek, line 39: value used but not set (x4) diff --git a/testing/btest/Baseline/language.uninitialized-local3/out b/testing/btest/Baseline/language.uninitialized-local3/out index e65bf87f36..ee2f49d506 100644 --- a/testing/btest/Baseline/language.uninitialized-local3/out +++ b/testing/btest/Baseline/language.uninitialized-local3/out @@ -1,11 +1,9 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -warning in <...>/uninitialized-local3.zeek, line 38: possibly used without definition (x4) -expression error in <...>/uninitialized-local3.zeek, line 38: value used but not set (x4) -x$a (x <...>/uninitialized-local3.zeek, line 20) possibly used without being set -x$e (x <...>/uninitialized-local3.zeek, line 20) possibly used without being set -x$e (x <...>/uninitialized-local3.zeek, line 24) possibly used without being set -x2$worries$a (x2 <...>/uninitialized-local3.zeek, line 27) possibly used without being set -x2$worries$e (x2 <...>/uninitialized-local3.zeek, line 27) possibly used without being set +x$a (x <...>/uninitialized-local3.zeek, line 21) possibly used without being set +x$e (x <...>/uninitialized-local3.zeek, line 21) possibly used without being set +x$e (x <...>/uninitialized-local3.zeek, line 25) possibly used without being set +x2$worries$a (x2 <...>/uninitialized-local3.zeek, line 28) possibly used without being set +x2$worries$e (x2 <...>/uninitialized-local3.zeek, line 28) possibly used without being set [a=, b=, c=9, d=, e=] [a=, b=, c=9, d=, e=] [no_worries=[a=, b=, c=9, d=, e=], worries=[a=, b=, c=9, d=, e=]] diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.file_exists_lookup_file/.stdout b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.file_exists_lookup_file/.stdout index 4ac2117737..ccf66e3400 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.file_exists_lookup_file/.stdout +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.file_exists_lookup_file/.stdout @@ -1,6 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -error: file ID asdf not a known file -warning: non-void function returning without a value: Files::lookup_file This should fail but not crash This should return F F diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.file_exists_lookup_file/err b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.file_exists_lookup_file/err new file mode 100644 index 0000000000..f92621dcef --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.file_exists_lookup_file/err @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +error: file ID asdf not a known file +warning: non-void function returning without a value: Files::lookup_file diff --git a/testing/btest/Traces/tunnels/teredo-udp-in-udp.pcap b/testing/btest/Traces/tunnels/teredo-udp-in-udp.pcap new file mode 100644 index 0000000000..3f08887024 Binary files /dev/null and b/testing/btest/Traces/tunnels/teredo-udp-in-udp.pcap differ diff --git a/testing/btest/bifs/bloomfilter.zeek b/testing/btest/bifs/bloomfilter.zeek index c0ccbd7080..26b5cdbf5f 100644 --- a/testing/btest/bifs/bloomfilter.zeek +++ b/testing/btest/bifs/bloomfilter.zeek @@ -1,5 +1,6 @@ -# @TEST-EXEC: zeek -b %INPUT >output 2>&1 +# @TEST-EXEC: zeek -b %INPUT >output 2>err # @TEST-EXEC: btest-diff output +# @TEST-EXEC: btest-diff err function test_basic_bloom_filter() { diff --git a/testing/btest/bifs/strptime.zeek b/testing/btest/bifs/strptime.zeek index 3923ced4c0..e870b5c0f1 100644 --- a/testing/btest/bifs/strptime.zeek +++ b/testing/btest/bifs/strptime.zeek @@ -1,6 +1,7 @@ # -# @TEST-EXEC: zeek -b %INPUT >out 2>&1 +# @TEST-EXEC: zeek -b %INPUT >out 2>err # @TEST-EXEC: btest-diff out +# @TEST-EXEC: btest-diff err event zeek_init() { diff --git a/testing/btest/core/init-error.zeek b/testing/btest/core/init-error.zeek index 82226e9dfa..70c9239bad 100644 --- a/testing/btest/core/init-error.zeek +++ b/testing/btest/core/init-error.zeek @@ -1,7 +1,8 @@ # The default is for an initialization error to be a hard failure. -# @TEST-EXEC-FAIL: unset ZEEK_ALLOW_INIT_ERRORS && zeek -b %INPUT >out 2>&1 +# @TEST-EXEC-FAIL: unset ZEEK_ALLOW_INIT_ERRORS && zeek -b %INPUT >out 2>err # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff err event zeek_init() &priority=10 { diff --git a/testing/btest/core/pcap/dumper.zeek b/testing/btest/core/pcap/dumper.zeek index b53c01bc37..21c2148a3b 100644 --- a/testing/btest/core/pcap/dumper.zeek +++ b/testing/btest/core/pcap/dumper.zeek @@ -2,7 +2,9 @@ # @TEST-EXEC: zeek -b -r $TRACES/workshop_2011_browse.trace -w dump # @TEST-EXEC: hexdump -C $TRACES/workshop_2011_browse.trace >1 # @TEST-EXEC: hexdump -C dump >2 -# @TEST-EXEC: diff 1 2 >output || true +# @TEST-EXEC: diff -U3 1 2 >diff.output || true +# Remove first two lines of diff since they may contain timestamps +# @TEST-EXEC: awk 'NR>2' diff.output >output # Note that we're diff'ing the diff because there is an expected # difference in the pcaps: namely, the snaplen setting stored in the diff --git a/testing/btest/core/reporter-error-in-handler.zeek b/testing/btest/core/reporter-error-in-handler.zeek index 398c275c32..0b15e445b5 100644 --- a/testing/btest/core/reporter-error-in-handler.zeek +++ b/testing/btest/core/reporter-error-in-handler.zeek @@ -2,8 +2,9 @@ # This test procudes a recursive error: the error handler is itself broken. Rather # than looping indefinitly, the error inside the handler should reported to stderr. # -# @TEST-EXEC: zeek -b %INPUT >output 2>&1 +# @TEST-EXEC: zeek -b %INPUT >output 2>err # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff output +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff err global a: table[count] of count; diff --git a/testing/btest/core/tunnels/teredo-udp-in-udp.zeek b/testing/btest/core/tunnels/teredo-udp-in-udp.zeek new file mode 100644 index 0000000000..25caf94c5d --- /dev/null +++ b/testing/btest/core/tunnels/teredo-udp-in-udp.zeek @@ -0,0 +1,4 @@ +# @TEST-EXEC: zeek -r $TRACES/tunnels/teredo-udp-in-udp.pcap %INPUT +# @TEST-EXEC: btest-diff tunnel.log + +@load base/frameworks/tunnels diff --git a/testing/btest/language/common-mistakes.zeek b/testing/btest/language/common-mistakes.zeek index b829b5315b..b810dd22ea 100644 --- a/testing/btest/language/common-mistakes.zeek +++ b/testing/btest/language/common-mistakes.zeek @@ -2,14 +2,17 @@ # handled internally by way of throwing an exception to unwind out # of the current event handler body. -# @TEST-EXEC: zeek -b 1.zeek >1.out 2>&1 +# @TEST-EXEC: zeek -b 1.zeek >1.out 2>1.err # @TEST-EXEC: btest-diff 1.out +# @TEST-EXEC: btest-diff 1.err -# @TEST-EXEC: zeek -b 2.zeek >2.out 2>&1 +# @TEST-EXEC: zeek -b 2.zeek >2.out 2>2.err # @TEST-EXEC: btest-diff 2.out +# @TEST-EXEC: btest-diff 2.err -# @TEST-EXEC: zeek -b 3.zeek >3.out 2>&1 +# @TEST-EXEC: zeek -b 3.zeek >3.out 2>3.err # @TEST-EXEC: btest-diff 3.out +# @TEST-EXEC: btest-diff 3.err @TEST-START-FILE 1.zeek type myrec: record { diff --git a/testing/btest/language/invalid_index.zeek b/testing/btest/language/invalid_index.zeek index 80f294c68b..831de104df 100644 --- a/testing/btest/language/invalid_index.zeek +++ b/testing/btest/language/invalid_index.zeek @@ -1,5 +1,6 @@ -# @TEST-EXEC: zeek -b %INPUT >out 2>&1 +# @TEST-EXEC: zeek -b %INPUT >out 2>err # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff err global foo: vector of count = { 42 }; global foo2: table[count] of count = { [0] = 13 }; diff --git a/testing/btest/language/subnet-errors.zeek b/testing/btest/language/subnet-errors.zeek index 875817c433..4226232cb3 100644 --- a/testing/btest/language/subnet-errors.zeek +++ b/testing/btest/language/subnet-errors.zeek @@ -1,5 +1,6 @@ -# @TEST-EXEC: zeek -b %INPUT >out 2>&1 +# @TEST-EXEC: zeek -b %INPUT >out 2>err # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff err event zeek_init() { diff --git a/testing/btest/language/table-set-iterator-invalidation.zeek b/testing/btest/language/table-set-iterator-invalidation.zeek index aee45a6cea..6347c69125 100644 --- a/testing/btest/language/table-set-iterator-invalidation.zeek +++ b/testing/btest/language/table-set-iterator-invalidation.zeek @@ -1,5 +1,6 @@ -# @TEST-EXEC: zeek -b %INPUT >out 2>&1 +# @TEST-EXEC: zeek -b %INPUT >out 2>err # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff err # Note that while modifying container membership during for-loop iteration is # supposed to be undefined-behavior, it should be practically ok to have this diff --git a/testing/btest/language/type-cast-error-dynamic.zeek b/testing/btest/language/type-cast-error-dynamic.zeek index 1edf9e3d2a..c636eee0b8 100644 --- a/testing/btest/language/type-cast-error-dynamic.zeek +++ b/testing/btest/language/type-cast-error-dynamic.zeek @@ -1,5 +1,6 @@ -# @TEST-EXEC: zeek -b %INPUT >output 2>&1 +# @TEST-EXEC: zeek -b %INPUT >output 2>err # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff output +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff err type X: record { a: addr; diff --git a/testing/btest/language/type-coerce-numerics.zeek b/testing/btest/language/type-coerce-numerics.zeek index 7633c02b62..d31d877b35 100644 --- a/testing/btest/language/type-coerce-numerics.zeek +++ b/testing/btest/language/type-coerce-numerics.zeek @@ -1,17 +1,22 @@ -# @TEST-EXEC: zeek -b first_set.zeek >first_set.out 2>&1 +# @TEST-EXEC: zeek -b first_set.zeek >first_set.out 2>first_set.err # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff first_set.out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff first_set.err -# @TEST-EXEC-FAIL: zeek -b double_convert_failure1.zeek >double_convert_failure1.out 2>&1 +# @TEST-EXEC-FAIL: zeek -b double_convert_failure1.zeek >double_convert_failure1.out 2>double_convert_failure1.err # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff double_convert_failure1.out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff double_convert_failure1.err -# @TEST-EXEC-FAIL: zeek -b double_convert_failure2.zeek >double_convert_failure2.out 2>&1 +# @TEST-EXEC-FAIL: zeek -b double_convert_failure2.zeek >double_convert_failure2.out 2>double_convert_failure2.err # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff double_convert_failure2.out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff double_convert_failure2.err -# @TEST-EXEC-FAIL: zeek -b int_convert_failure.zeek >int_convert_failure.out 2>&1 +# @TEST-EXEC-FAIL: zeek -b int_convert_failure.zeek >int_convert_failure.out 2>int_convert_failure.err # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff int_convert_failure.out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff int_convert_failure.err -# @TEST-EXEC: zeek -b vectors.zeek >vectors.out 2>&1 +# @TEST-EXEC: zeek -b vectors.zeek >vectors.out 2>vectors.err # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff vectors.out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff vectors.err @TEST-START-FILE first_set.zeek type myrecord : record { diff --git a/testing/btest/language/uninitialized-local2.zeek b/testing/btest/language/uninitialized-local2.zeek index 4b8f0c8275..118ab77c83 100644 --- a/testing/btest/language/uninitialized-local2.zeek +++ b/testing/btest/language/uninitialized-local2.zeek @@ -1,5 +1,6 @@ -# @TEST-EXEC: zeek -b %INPUT >out 2>&1 +# @TEST-EXEC: zeek -b %INPUT >out 2>err # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff err event test() { diff --git a/testing/btest/language/uninitialized-local3.zeek b/testing/btest/language/uninitialized-local3.zeek index f3aae38f5b..79629d2138 100644 --- a/testing/btest/language/uninitialized-local3.zeek +++ b/testing/btest/language/uninitialized-local3.zeek @@ -1,5 +1,6 @@ -# @TEST-EXEC: ZEEK_USAGE_ISSUES=2 zeek -b %INPUT >out 2>&1 +# @TEST-EXEC: ZEEK_USAGE_ISSUES=2 zeek -b %INPUT >out 2>err # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff err type r: record { a: count; diff --git a/testing/btest/scripts/base/frameworks/file-analysis/bifs/file_exists_lookup_file.zeek b/testing/btest/scripts/base/frameworks/file-analysis/bifs/file_exists_lookup_file.zeek index 1a8c6e27a1..07e2cccbea 100644 --- a/testing/btest/scripts/base/frameworks/file-analysis/bifs/file_exists_lookup_file.zeek +++ b/testing/btest/scripts/base/frameworks/file-analysis/bifs/file_exists_lookup_file.zeek @@ -1,5 +1,6 @@ -# @TEST-EXEC: zeek -b -r $TRACES/http/get.trace %INPUT 2>&1 +# @TEST-EXEC: zeek -b -r $TRACES/http/get.trace %INPUT 2>err # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stdout +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff err @load base/protocols/http diff --git a/testing/btest/scripts/base/frameworks/netcontrol/acld-hook.zeek b/testing/btest/scripts/base/frameworks/netcontrol/acld-hook.zeek index 2698a3bfab..0b0bab2e6b 100644 --- a/testing/btest/scripts/base/frameworks/netcontrol/acld-hook.zeek +++ b/testing/btest/scripts/base/frameworks/netcontrol/acld-hook.zeek @@ -9,6 +9,7 @@ @TEST-START-FILE send.zeek @load base/frameworks/netcontrol +@load base/frameworks/broker redef exit_only_after_terminate = T; global have_peer = F; diff --git a/testing/btest/scripts/base/frameworks/openflow/broker-basic.zeek b/testing/btest/scripts/base/frameworks/openflow/broker-basic.zeek index e1d05db7a5..f6f166013b 100644 --- a/testing/btest/scripts/base/frameworks/openflow/broker-basic.zeek +++ b/testing/btest/scripts/base/frameworks/openflow/broker-basic.zeek @@ -82,6 +82,7 @@ event OpenFlow::flow_mod_failure(name: string, match: OpenFlow::ofp_match, flow_ @TEST-START-FILE recv.zeek +@load base/protocols/conn @load base/frameworks/openflow redef exit_only_after_terminate = T; diff --git a/testing/scripts/diff-remove-timestamps b/testing/scripts/diff-remove-timestamps index 5c5956a0ed..0bcc458739 100755 --- a/testing/scripts/diff-remove-timestamps +++ b/testing/scripts/diff-remove-timestamps @@ -9,6 +9,4 @@ else sed="sed -E" fi -# $a\ adds a newline to the end of the file if it does not exist. -# This unifies the behavior of BSD and GNU sed -$sed -e 's/(^|[^0-9])([0-9]{9,10}\.[0-9]{1,8})/\1XXXXXXXXXX.XXXXXX/g' -e 's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g' -e '$a\' +$sed -e 's/(^|[^0-9])([0-9]{9,10}\.[0-9]{1,8})/\1XXXXXXXXXX.XXXXXX/g' -e 's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g'