diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 0000000000..0f0d9b609d --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,40 @@ +Checks: '*, + -abseil-string-find-startswith, + -bugprone-exception-escape, + -bugprone-macro-parentheses, + -bugprone-suspicious-semicolon, + -cert-err58-cpp, + -cppcoreguidelines-avoid-c-arrays, + -cppcoreguidelines-avoid-goto, + -cppcoreguidelines-avoid-magic-numbers, + -cppcoreguidelines-macro-usage, + -cppcoreguidelines-non-private-member-variables-in-classes, + -cppcoreguidelines-pro-bounds-array-to-pointer-decay, + -cppcoreguidelines-pro-bounds-constant-array-index, + -cppcoreguidelines-pro-bounds-pointer-arithmetic, + -cppcoreguidelines-pro-type-const-cast, + -cppcoreguidelines-pro-type-reinterpret-cast, + -fuchsia-default-arguments, + -fuchsia-multiple-inheritance, + -fuchsia-overloaded-operator, + -fuchsia-statically-constructed-objects, + -fuchsia-trailing-return, + -google-build-using-namespace, + -google-explicit-constructor, + -google-readability-braces-around-statements, + -hicpp-avoid-c-arrays, + -hicpp-avoid-goto, + -hicpp-braces-around-statements, + -hicpp-explicit-conversions, + -hicpp-no-array-decay, + -llvm-header-guard, + -misc-macro-parentheses, + -misc-non-private-member-variables-in-classes, + -misc-suspicious-semicolon, + -misc-unused-parameters, + -modernize-avoid-c-arrays, + -modernize-use-nodiscard, + -readability-braces-around-statements, + -readability-container-size-empty, + -readability-implicit-bool-conversion, + -readability-magic-numbers' diff --git a/CHANGES b/CHANGES index 525c88a89b..59b35511d5 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,68 @@ +2.6-598 | 2019-07-12 18:20:12 -0700 + + * Fix canonification of timestamps with a decisecond multiple (Jon Siwek, Corelight) + +2.6-597 | 2019-07-12 15:01:56 -0700 + + * Fix a couple of resource leaks from JSON changes (Tim Wojtulewicz, Corelight) + +2.6-595 | 2019-07-12 13:34:08 -0700 + + * GH-157: Mark some attributes as not allowed for global variables (Tim Wojtulewicz, Corelight) + + This disallows &default for global values that are not tables, and &optional for all globals. + + * Fix uncaught exceptions from Val cloning failures (Jon Siwek, Corelight) + +2.6-591 | 2019-07-11 13:29:28 -0700 + + * Fix potential thread safety issue with zeekenv util function + + Observed segfault accessing the local static std::map of zeekenv() from + a logging thread, but only in non-debug builds using Apple/Clang + compiler, not in a debug build or GCC. Don't quite get this behavior + since static local variable initialization is supposed to be thread-safe + since C++11, but moving to a global static works and is "more efficient" + anyway since there's no longer any run-time overhead. (Jon Siwek, Corelight) + +2.6-589 | 2019-07-11 13:14:52 -0700 + + * GH-421: fix bugs/regressions in DNP3 analyzer (Hui Lin) + +2.6-587 | 2019-07-11 12:13:48 -0700 + + * Fix a sign-compare compiler warning (Jon Siwek, Corelight) + +2.6-586 | 2019-07-11 11:15:40 -0700 + + * Convert all JSON output to use an external library for better consistency (Tim Wojtulewicz, Corelight) + + See NEWS for more details; this makes to_json a bif and causes slight changes in its + output, as well as the output of the JSON logger. + +2.6-576 | 2019-07-10 18:38:54 -0700 + + * Remove unused option: chunked_io_buffer_soft_cap (Jon Siwek, Corelight) + +2.6-575 | 2019-07-09 18:28:03 -0700 + + * Avoid a null dereference (Coverity-1402816) (Tim Wojtulewicz, Corelight) + + * Avoid resource leaks (Coverity-1402818, Coverity-1402812) (Tim Wojtulewicz, Corelight) + + * Avoid null dereference in broker (Coverity-1402824, Coverity-1402814) (Tim Wojtulewicz, Corelight) + + * Improve stability of a unit test (Jon Siwek, Corelight) + +2.6-569 | 2019-07-03 13:03:22 -0700 + + * Improve stability of a unit test (Jon Siwek, Corelight) + +2.6-568 | 2019-07-03 11:50:56 -0700 + + * Add clang-tidy rule to CMake including a base configuration (Tim Wojtulewicz, Corelight) + 2.6-566 | 2019-07-03 11:08:24 -0700 * Improve Zeekygen output for long attribute expressions (Jon Siwek, Corelight) diff --git a/NEWS b/NEWS index acd25aff53..a4e19ab60e 100644 --- a/NEWS +++ b/NEWS @@ -347,6 +347,16 @@ Changed Functionality of each other on separate cluster nodes to all be logged rather than suppressed and de-duplicated into a single notice. + +- to_json is now a bif, no longer a script. Loading base/utils/json.zeek is no + longer necessary and has been deprecated. to_json should yield much better, always + valid json. There are some small differences in output; unnecessary spaces are removed + and port values are rendered differently, now including the port and the protocol. + +- The output of the JSON logger now uses an external library to generate json. There + are small changes to the output; most visibly double numbers are now rounded slightly + differently. The way in which port values are rendered does _not_ change for JSON logs. + Removed Functionality --------------------- @@ -479,6 +489,7 @@ Removed Functionality - ``backdoor_stat_period`` - ``backdoor_stat_backoff`` - ``backdoor_endp_stats`` + - ``chunked_io_buffer_soft_cap`` - The following constants were used as part of deprecated functionality in version 2.6 or below and are removed from this release: diff --git a/VERSION b/VERSION index 8b96eff2a8..eec6d0de55 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.6-566 +2.6-598 diff --git a/aux/paraglob b/aux/paraglob index 1a8d674d2c..4b0c213ad6 160000 --- a/aux/paraglob +++ b/aux/paraglob @@ -1 +1 @@ -Subproject commit 1a8d674d2ccbef06a6e4e6f1a9c8747a2eadf026 +Subproject commit 4b0c213ad64737fd1694216fe136b5665f932e22 diff --git a/doc b/doc index 80c0cb3d9d..bcfe6ffc88 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 80c0cb3d9db1cea6661b3d3c79d17e3b67533507 +Subproject commit bcfe6ffc88e0a89e7ade664113d458bae9e5e5fc diff --git a/scripts/base/frameworks/openflow/plugins/ryu.zeek b/scripts/base/frameworks/openflow/plugins/ryu.zeek index cc400293a0..08e8c8d022 100644 --- a/scripts/base/frameworks/openflow/plugins/ryu.zeek +++ b/scripts/base/frameworks/openflow/plugins/ryu.zeek @@ -3,7 +3,6 @@ @load base/frameworks/openflow @load base/utils/active-http @load base/utils/exec -@load base/utils/json module OpenFlow; diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index bfb49579cd..7d539f3948 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -4608,11 +4608,6 @@ const sig_max_group_size = 50 &redef; ## Description transmitted to remote communication peers for identification. const peer_description = "zeek" &redef; -## The number of IO chunks allowed to be buffered between the child -## and parent process of remote communication before Zeek starts dropping -## connections to remote peers in an attempt to catch up. -const chunked_io_buffer_soft_cap = 800000 &redef; - ## Reassemble the beginning of all TCP connections before doing ## signature matching. Enabling this provides more accurate matching at the ## expense of CPU cycles. diff --git a/scripts/base/utils/json.zeek b/scripts/base/utils/json.zeek index 5bce89d18b..a9662e74a2 100644 --- a/scripts/base/utils/json.zeek +++ b/scripts/base/utils/json.zeek @@ -1,109 +1,2 @@ -##! Functions to assist with generating JSON data from Zeek data scructures. -# We might want to implement this in core somtime, this looks... hacky at best. - -@load base/utils/strings - -## A function to convert arbitrary Zeek data into a JSON string. -## -## v: The value to convert to JSON. Typically a record. -## -## only_loggable: If the v value is a record this will only cause -## fields with the &log attribute to be included in the JSON. -## -## returns: a JSON formatted string. -function to_json(v: any, only_loggable: bool &default=F, field_escape_pattern: pattern &default=/^_/): string - { - local tn = type_name(v); - switch ( tn ) - { - case "type": - return ""; - - case "string": - return cat("\"", gsub(gsub(clean(v), /\\/, "\\\\"), /\"/, "\\\""), "\""); - - case "port": - return cat(port_to_count(to_port(cat(v)))); - - case "enum": - fallthrough; - case "interval": - fallthrough; - case "addr": - fallthrough; - case "subnet": - return cat("\"", v, "\""); - - case "int": - fallthrough; - case "count": - fallthrough; - case "time": - return cat(v); - - case "double": - return fmt("%.16g", v); - - case "bool": - local bval: bool = v; - return bval ? "true" : "false"; - - default: - break; - } - - if ( /^record/ in tn ) - { - local rec_parts: string_vec = vector(); - - local ft = record_fields(v); - for ( field, field_desc in ft ) - { - # replace the escape pattern in the field. - if( field_escape_pattern in field ) - field = cat(sub(field, field_escape_pattern, "")); - if ( field_desc?$value && (!only_loggable || field_desc$log) ) - { - local onepart = cat("\"", field, "\": ", to_json(field_desc$value, only_loggable)); - rec_parts += onepart; - } - } - return cat("{", join_string_vec(rec_parts, ", "), "}"); - } - - # None of the following are supported. - else if ( /^set/ in tn ) - { - local set_parts: string_vec = vector(); - local sa: set[bool] = v; - for ( sv in sa ) - { - set_parts += to_json(sv, only_loggable); - } - return cat("[", join_string_vec(set_parts, ", "), "]"); - } - else if ( /^table/ in tn ) - { - local tab_parts: vector of string = vector(); - local ta: table[bool] of any = v; - for ( ti, tv in ta ) - { - local ts = to_json(ti); - local if_quotes = (ts[0] == "\"") ? "" : "\""; - tab_parts += cat(if_quotes, ts, if_quotes, ": ", to_json(tv, only_loggable)); - } - return cat("{", join_string_vec(tab_parts, ", "), "}"); - } - else if ( /^vector/ in tn ) - { - local vec_parts: string_vec = vector(); - local va: vector of any = v; - for ( vi in va ) - { - vec_parts += to_json(va[vi], only_loggable); - } - return cat("[", join_string_vec(vec_parts, ", "), "]"); - } - - return "\"\""; - } +## This file is deprecated in favor of to_json in zeek.bif +@deprecated="Remove in 3.1. to_json is now always available as a built-in function." diff --git a/src/3rdparty b/src/3rdparty index 785e581f00..1e9d49362d 160000 --- a/src/3rdparty +++ b/src/3rdparty @@ -1 +1 @@ -Subproject commit 785e581f00a1efae3fca7a62fb15d8756c5aedb1 +Subproject commit 1e9d49362d2c3bb2f43abcd8eebe47be045659a5 diff --git a/src/Attr.cc b/src/Attr.cc index 36050d2262..ce93061cef 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -130,11 +130,12 @@ void Attr::AddTag(ODesc* d) const d->Add(attr_name(Tag())); } -Attributes::Attributes(attr_list* a, BroType* t, bool arg_in_record) +Attributes::Attributes(attr_list* a, BroType* t, bool arg_in_record, bool is_global) { attrs = new attr_list(a->length()); type = t->Ref(); in_record = arg_in_record; + global_var = is_global; SetLocationInfo(&start_location, &end_location); @@ -250,10 +251,14 @@ void Attributes::CheckAttr(Attr* a) { switch ( a->Tag() ) { case ATTR_DEPRECATED: - case ATTR_OPTIONAL: case ATTR_REDEF: break; + case ATTR_OPTIONAL: + if ( global_var ) + Error("&optional is not valid for global variables"); + break; + case ATTR_ADD_FUNC: case ATTR_DEL_FUNC: { @@ -283,6 +288,14 @@ void Attributes::CheckAttr(Attr* a) case ATTR_DEFAULT: { + // &default is allowed for global tables, since it's used in initialization + // of table fields. it's not allowed otherwise. + if ( global_var && ! type->IsSet() && type->Tag() != TYPE_TABLE ) + { + Error("&default is not valid for global variables"); + break; + } + BroType* atype = a->AttrExpr()->Type(); if ( type->Tag() != TYPE_TABLE || (type->IsSet() && ! in_record) ) @@ -410,9 +423,10 @@ void Attributes::CheckAttr(Attr* a) #if 0 //### not easy to test this w/o knowing the ID. - if ( ! IsGlobal() ) + if ( ! global_var ) Error("expiration not supported for local variables"); #endif + break; case ATTR_EXPIRE_FUNC: diff --git a/src/Attr.h b/src/Attr.h index efbac63747..09a8266929 100644 --- a/src/Attr.h +++ b/src/Attr.h @@ -77,7 +77,7 @@ protected: // Manages a collection of attributes. class Attributes : public BroObj { public: - Attributes(attr_list* a, BroType* t, bool in_record); + Attributes(attr_list* a, BroType* t, bool in_record, bool is_global); ~Attributes() override; void AddAttr(Attr* a); @@ -101,6 +101,7 @@ protected: BroType* type; attr_list* attrs; bool in_record; + bool global_var; }; #endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ce3dd897f3..a9d5501718 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -220,23 +220,7 @@ add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/DebugCmdConstants.h set_source_files_properties(nb_dns.c PROPERTIES COMPILE_FLAGS -fno-strict-aliasing) -set(bro_SRCS - ${CMAKE_CURRENT_BINARY_DIR}/version.c - ${BIF_SRCS} - ${BINPAC_AUXSRC} - ${BINPAC_OUTPUTS} - ${TRANSFORMED_BISON_OUTPUTS} - ${FLEX_RuleScanner_OUTPUTS} - ${FLEX_RuleScanner_INPUT} - ${BISON_RuleParser_INPUT} - ${FLEX_REScanner_OUTPUTS} - ${FLEX_REScanner_INPUT} - ${BISON_REParser_INPUT} - ${FLEX_Scanner_OUTPUTS} - ${FLEX_Scanner_INPUT} - ${BISON_Parser_INPUT} - ${CMAKE_CURRENT_BINARY_DIR}/DebugCmdConstants.h - ${PLUGIN_INIT} +set(MAIN_SRCS main.cc net_util.cc util.cc @@ -332,8 +316,6 @@ set(bro_SRCS threading/formatters/Ascii.cc threading/formatters/JSON.cc - 3rdparty/sqlite3.c - plugin/Component.cc plugin/ComponentManager.h plugin/TaggedComponent.h @@ -344,6 +326,31 @@ set(bro_SRCS digest.h ) +set(THIRD_PARTY_SRCS + 3rdparty/sqlite3.c +) + +set(bro_SRCS + ${CMAKE_CURRENT_BINARY_DIR}/version.c + ${BIF_SRCS} + ${BINPAC_AUXSRC} + ${BINPAC_OUTPUTS} + ${TRANSFORMED_BISON_OUTPUTS} + ${FLEX_RuleScanner_OUTPUTS} + ${FLEX_RuleScanner_INPUT} + ${BISON_RuleParser_INPUT} + ${FLEX_REScanner_OUTPUTS} + ${FLEX_REScanner_INPUT} + ${BISON_REParser_INPUT} + ${FLEX_Scanner_OUTPUTS} + ${FLEX_Scanner_INPUT} + ${BISON_Parser_INPUT} + ${CMAKE_CURRENT_BINARY_DIR}/DebugCmdConstants.h + ${PLUGIN_INIT} + ${THIRD_PARTY_SRCS} + ${MAIN_SRCS} +) + collect_headers(bro_HEADERS ${bro_SRCS}) if ( bro_HAVE_OBJECT_LIBRARIES ) @@ -433,3 +440,18 @@ install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/sqlite3.h DESTINATION include/zeek/3rdparty ) + +find_program(CLANG_TIDY NAMES clang-tidy) +if (CLANG_TIDY) + set(TIDY_SRCS "") + foreach(f ${MAIN_SRCS}) + list(APPEND TIDY_SRCS "src/${f}") + endforeach(f) + # TODO: this currently doesn't include many of the subdirectories/plugins + # that build static libraries for inclusion into the final zeek binary + # (analyzers, broker, etc.) or generated code (BIFs, BinPAC, etc.). + add_custom_target(clang-tidy + COMMAND ${CLANG_TIDY} -p ${CMAKE_BINARY_DIR} ${TIDY_SRCS} + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + ) +endif() diff --git a/src/Expr.cc b/src/Expr.cc index 450ac73437..7235b14d35 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -3133,7 +3133,7 @@ TableConstructorExpr::TableConstructorExpr(ListExpr* constructor_list, } } - attrs = arg_attrs ? new Attributes(arg_attrs, type, false) : 0; + attrs = arg_attrs ? new Attributes(arg_attrs, type, false, false) : 0; type_list* indices = type->AsTableType()->Indices()->Types(); const expr_list& cle = constructor_list->Exprs(); @@ -3240,7 +3240,7 @@ SetConstructorExpr::SetConstructorExpr(ListExpr* constructor_list, else if ( type->Tag() != TYPE_TABLE || ! type->AsTableType()->IsSet() ) SetError("values in set(...) constructor do not specify a set"); - attrs = arg_attrs ? new Attributes(arg_attrs, type, false) : 0; + attrs = arg_attrs ? new Attributes(arg_attrs, type, false, false) : 0; type_list* indices = type->AsTableType()->Indices()->Types(); expr_list& cle = constructor_list->Exprs(); diff --git a/src/ID.cc b/src/ID.cc index c249697b4d..6029cb9862 100644 --- a/src/ID.cc +++ b/src/ID.cc @@ -181,7 +181,7 @@ void ID::UpdateValAttrs() TypeDecl* fd = rt->FieldDecl(i); if ( ! fd->attrs ) - fd->attrs = new Attributes(new attr_list, rt->FieldType(i), true); + fd->attrs = new Attributes(new attr_list, rt->FieldType(i), true, IsGlobal()); fd->attrs->AddAttr(new Attr(ATTR_LOG)); } @@ -195,7 +195,7 @@ void ID::MakeDeprecated(Expr* deprecation) return; attr_list* attr = new attr_list{new Attr(ATTR_DEPRECATED, deprecation)}; - AddAttrs(new Attributes(attr, Type(), false)); + AddAttrs(new Attributes(attr, Type(), false, IsGlobal())); } string ID::GetDeprecationWarning() const @@ -245,7 +245,7 @@ void ID::SetOption() if ( ! IsRedefinable() ) { attr_list* attr = new attr_list{new Attr(ATTR_REDEF)}; - AddAttrs(new Attributes(attr, Type(), false)); + AddAttrs(new Attributes(attr, Type(), false, IsGlobal())); } } diff --git a/src/NetVar.cc b/src/NetVar.cc index c256cfd496..981ca005ff 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -141,7 +141,6 @@ RecordType* rotate_info; StringVal* log_rotate_base_time; StringVal* peer_description; -bro_uint_t chunked_io_buffer_soft_cap; Val* profiling_file; double profiling_interval; @@ -213,7 +212,6 @@ void init_general_global_var() peer_description = internal_val("peer_description")->AsStringVal(); - chunked_io_buffer_soft_cap = opt_internal_unsigned("chunked_io_buffer_soft_cap"); packet_filter_default = opt_internal_int("packet_filter_default"); diff --git a/src/NetVar.h b/src/NetVar.h index 5dc5fcfb79..41b2028064 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -144,7 +144,6 @@ extern RecordType* rotate_info; extern StringVal* log_rotate_base_time; extern StringVal* peer_description; -extern bro_uint_t chunked_io_buffer_soft_cap; extern Val* profiling_file; extern double profiling_interval; diff --git a/src/Type.cc b/src/Type.cc index 60461e026f..efe3d254c5 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -650,7 +650,7 @@ void FuncType::DescribeReST(ODesc* d, bool roles_only) const TypeDecl::TypeDecl(BroType* t, const char* i, attr_list* arg_attrs, bool in_record) { type = t; - attrs = arg_attrs ? new Attributes(arg_attrs, t, in_record) : 0; + attrs = arg_attrs ? new Attributes(arg_attrs, t, in_record, false) : 0; id = i; } @@ -841,7 +841,7 @@ const char* RecordType::AddFields(type_decl_list* others, attr_list* attr) if ( log ) { if ( ! td->attrs ) - td->attrs = new Attributes(new attr_list, td->type, true); + td->attrs = new Attributes(new attr_list, td->type, true, false); td->attrs->AddAttr(new Attr(ATTR_LOG)); } diff --git a/src/Val.cc b/src/Val.cc index 32d64c5cdd..d0f11a2062 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -27,6 +27,16 @@ #include "broker/Data.h" +#include "3rdparty/json.hpp" +#include "3rdparty/fifo_map.hpp" + +// Define a class for use with the json library that orders the keys in the same order that +// they were inserted. By default, the json library orders them alphabetically and we don't +// want it like that. +template +using json_fifo_map = nlohmann::fifo_map, A>; +using ZeekJson = nlohmann::basic_json; + Val::Val(Func* f) { val.func_val = f; @@ -380,6 +390,276 @@ bool Val::WouldOverflow(const BroType* from_type, const BroType* to_type, const return false; } +TableVal* Val::GetRecordFields() + { + TableVal* fields = new TableVal(internal_type("record_field_table")->AsTableType()); + + auto t = Type(); + + if ( t->Tag() != TYPE_RECORD && t->Tag() != TYPE_TYPE ) + { + reporter->Error("non-record value/type passed to record_fields"); + return fields; + } + + RecordType* rt = nullptr; + RecordVal* rv = nullptr; + + if ( t->Tag() == TYPE_RECORD ) + { + rt = t->AsRecordType(); + rv = AsRecordVal(); + } + else + { + t = t->AsTypeType()->Type(); + + if ( t->Tag() != TYPE_RECORD ) + { + reporter->Error("non-record value/type passed to record_fields"); + return fields; + } + + rt = t->AsRecordType(); + } + + for ( int i = 0; i < rt->NumFields(); ++i ) + { + BroType* ft = rt->FieldType(i); + TypeDecl* fd = rt->FieldDecl(i); + Val* fv = nullptr; + + if ( rv ) + fv = rv->Lookup(i); + + if ( fv ) + ::Ref(fv); + + bool logged = (fd->attrs && fd->FindAttr(ATTR_LOG) != 0); + + RecordVal* nr = new RecordVal(internal_type("record_field")->AsRecordType()); + + if ( ft->Tag() == TYPE_RECORD ) + nr->Assign(0, new StringVal("record " + ft->GetName())); + else + nr->Assign(0, new StringVal(type_name(ft->Tag()))); + + nr->Assign(1, val_mgr->GetBool(logged)); + nr->Assign(2, fv); + nr->Assign(3, rt->FieldDefault(i)); + + Val* field_name = new StringVal(rt->FieldName(i)); + fields->Assign(field_name, nr); + Unref(field_name); + } + + return fields; + } + +// This is a static method in this file to avoid including json.hpp in Val.h since it's huge. +static ZeekJson BuildJSON(Val* val, bool only_loggable=false, RE_Matcher* re=new RE_Matcher("^_")) + { + ZeekJson j; + BroType* type = val->Type(); + switch ( type->Tag() ) + { + case TYPE_BOOL: + j = val->AsBool(); + break; + + case TYPE_INT: + j = val->AsInt(); + break; + + case TYPE_COUNT: + j = val->AsCount(); + break; + + case TYPE_COUNTER: + j = val->AsCounter(); + break; + + case TYPE_TIME: + j = val->AsTime(); + break; + + case TYPE_DOUBLE: + j = val->AsDouble(); + break; + + case TYPE_PORT: + { + auto* pval = val->AsPortVal(); + j["port"] = pval->Port(); + j["proto"] = pval->Protocol(); + break; + } + + case TYPE_PATTERN: + case TYPE_INTERVAL: + case TYPE_ADDR: + case TYPE_SUBNET: + { + ODesc d; + d.SetStyle(RAW_STYLE); + val->Describe(&d); + + auto* bs = new BroString(1, d.TakeBytes(), d.Len()); + j = string((char*)bs->Bytes(), bs->Len()); + + delete bs; + break; + } + + case TYPE_FILE: + case TYPE_FUNC: + case TYPE_ENUM: + case TYPE_STRING: + { + ODesc d; + d.SetStyle(RAW_STYLE); + val->Describe(&d); + + auto* bs = new BroString(1, d.TakeBytes(), d.Len()); + j = json_escape_utf8(string((char*)bs->Bytes(), bs->Len())); + + delete bs; + break; + } + + case TYPE_TABLE: + { + auto* table = val->AsTable(); + auto* tval = val->AsTableVal(); + + if ( tval->Type()->IsSet() ) + j = ZeekJson::array(); + else + j = ZeekJson::object(); + + HashKey* k; + auto c = table->InitForIteration(); + while ( table->NextEntry(k, c) ) + { + auto lv = tval->RecoverIndex(k); + delete k; + + if ( tval->Type()->IsSet() ) + { + auto* value = lv->Index(0)->Ref(); + j.push_back(BuildJSON(value, only_loggable, re)); + Unref(value); + } + else + { + ZeekJson key_json; + Val* entry_value; + if ( lv->Length() == 1 ) + { + Val* entry_key = lv->Index(0)->Ref(); + entry_value = tval->Lookup(entry_key, true); + key_json = BuildJSON(entry_key, only_loggable, re); + Unref(entry_key); + } + else + { + entry_value = tval->Lookup(lv, true); + key_json = BuildJSON(lv, only_loggable, re); + } + + string key_string; + if ( key_json.is_string() ) + key_string = key_json; + else + key_string = key_json.dump(); + + j[key_string] = BuildJSON(entry_value, only_loggable, re); + } + + Unref(lv); + } + + break; + } + + case TYPE_RECORD: + { + j = ZeekJson::object(); + auto* rval = val->AsRecordVal(); + TableVal* fields = rval->GetRecordFields(); + auto* field_indexes = fields->ConvertToPureList(); + int num_indexes = field_indexes->Length(); + + for ( int i = 0; i < num_indexes; ++i ) + { + Val* key = field_indexes->Index(i); + auto* key_field = fields->Lookup(key)->AsRecordVal(); + + auto* key_val = key->AsStringVal(); + string key_string; + if ( re->MatchAnywhere(key_val->AsString()) != 0 ) + { + StringVal blank(""); + key_val = key_val->Substitute(re, &blank, 0)->AsStringVal(); + key_string = key_val->ToStdString(); + delete key_val; + } + else + key_string = key_val->ToStdString(); + + Val* value = key_field->Lookup("value", true); + + if ( value && ( ! only_loggable || key_field->Lookup("log")->AsBool() ) ) + j[key_string] = BuildJSON(value, only_loggable, re); + } + + delete fields; + delete field_indexes; + break; + } + + case TYPE_LIST: + { + j = ZeekJson::array(); + auto* lval = val->AsListVal(); + size_t size = lval->Length(); + for (size_t i = 0; i < size; i++) + j.push_back(BuildJSON(lval->Index(i), only_loggable, re)); + + break; + } + + case TYPE_VECTOR: + { + j = ZeekJson::array(); + auto* vval = val->AsVectorVal(); + size_t size = vval->SizeVal()->AsCount(); + for (size_t i = 0; i < size; i++) + j.push_back(BuildJSON(vval->Lookup(i), only_loggable, re)); + + break; + } + + case TYPE_OPAQUE: + { + j = ZeekJson::object(); + auto* oval = val->AsOpaqueVal(); + j["opaque_type"] = OpaqueMgr::mgr()->TypeID(oval); + break; + } + + default: break; + } + + return j; + } + +StringVal* Val::ToJSON(bool only_loggable, RE_Matcher* re) + { + ZeekJson j = BuildJSON(this, only_loggable, re); + return new StringVal(j.dump()); + } + IntervalVal::IntervalVal(double quantity, double units) : Val(quantity * units, TYPE_INTERVAL) { @@ -491,6 +771,18 @@ uint32 PortVal::Port() const return p & ~PORT_SPACE_MASK; } +string PortVal::Protocol() const + { + if ( IsUDP() ) + return "udp"; + else if ( IsTCP() ) + return "tcp"; + else if ( IsICMP() ) + return "icmp"; + else + return "unknown"; + } + int PortVal::IsTCP() const { return (val.uint_val & PORT_SPACE_MASK) == TCP_PORT_MASK; @@ -510,14 +802,8 @@ void PortVal::ValDescribe(ODesc* d) const { uint32 p = static_cast(val.uint_val); d->Add(p & ~PORT_SPACE_MASK); - if ( IsUDP() ) - d->Add("/udp"); - else if ( IsTCP() ) - d->Add("/tcp"); - else if ( IsICMP() ) - d->Add("/icmp"); - else - d->Add("/unknown"); + d->Add("/"); + d->Add(Protocol()); } Val* PortVal::DoClone(CloneState* state) @@ -713,6 +999,12 @@ StringVal::StringVal(const string& s) : Val(TYPE_STRING) val.string_val = new BroString(reinterpret_cast(s.data()), s.length(), 1); } +string StringVal::ToStdString() const + { + auto* bs = AsString(); + return string((char*)bs->Bytes(), bs->Len()); + } + StringVal* StringVal::ToUpper() { val.string_val->ToUpper(); @@ -734,6 +1026,92 @@ unsigned int StringVal::MemoryAllocation() const return padded_sizeof(*this) + val.string_val->MemoryAllocation(); } +Val* StringVal::Substitute(RE_Matcher* re, StringVal* repl, bool do_all) + { + const u_char* s = Bytes(); + int offset = 0; + int n = Len(); + + // cut_points is a set of pairs of indices in str that should + // be removed/replaced. A pair means "delete starting + // at offset x, up to but not including offset y". + List(ptr_compat_int) cut_points; // where RE matches pieces of str + + int size = 0; // size of result + + while ( n > 0 ) + { + // Find next match offset. + int end_of_match; + while ( n > 0 && + (end_of_match = re->MatchPrefix(&s[offset], n)) <= 0 ) + { + // This character is going to be copied to the result. + ++size; + + // Move on to next character. + ++offset; + --n; + } + + if ( n <= 0 ) + break; + + // s[offset .. offset+end_of_match-1] matches re. + cut_points.append(offset); + cut_points.append(offset + end_of_match); + + offset += end_of_match; + n -= end_of_match; + + if ( ! do_all ) + { + // We've now done the first substitution - finished. + // Include the remainder of the string in the result. + size += n; + break; + } + } + + // size now reflects amount of space copied. Factor in amount + // of space for replacement text. + int num_cut_points = cut_points.length() / 2; + size += num_cut_points * repl->Len(); + + // And a final NUL for good health. + ++size; + + byte_vec result = new u_char[size]; + byte_vec r = result; + + // Copy it all over. + int start_offset = 0; + for ( int i = 0; i < cut_points.length(); i += 2 /* loop over pairs */ ) + { + int num_to_copy = cut_points[i] - start_offset; + memcpy(r, s + start_offset, num_to_copy); + + r += num_to_copy; + start_offset = cut_points[i+1]; + + // Now add in replacement text. + memcpy(r, repl->Bytes(), repl->Len()); + r += repl->Len(); + } + + // Copy final trailing characters. + int num_to_copy = Len() - start_offset; + memcpy(r, s + start_offset, num_to_copy); + r += num_to_copy; + + // Final NUL. No need to increment r, since the length + // computed from it in the next statement does not include + // the NUL. + r[0] = '\0'; + + return new StringVal(new BroString(1, result, r - result)); + } + Val* StringVal::DoClone(CloneState* state) { // We could likely treat this type as immutable and return a reference @@ -1371,7 +1749,20 @@ Val* TableVal::Default(Val* index) if ( def_val->Type()->Tag() != TYPE_FUNC || same_type(def_val->Type(), Type()->YieldType()) ) - return def_attr->AttrExpr()->IsConst() ? def_val->Ref() : def_val->Clone(); + { + if ( def_attr->AttrExpr()->IsConst() ) + return def_val->Ref(); + + try + { + return def_val->Clone(); + } + catch ( InterpreterException& e ) + { /* Already reported. */ } + + Error("&default value for table is not clone-able"); + return 0; + } const Func* f = def_val->AsFunc(); val_list vl; @@ -2020,7 +2411,7 @@ vector RecordVal::parse_time_records; RecordVal::RecordVal(RecordType* t, bool init_fields) : Val(t) { - origin = 0; + origin = nullptr; int n = t->NumFields(); val_list* vl = val.val_list_val = new val_list(n); diff --git a/src/Val.h b/src/Val.h index 2ce61100b5..fb85ce2794 100644 --- a/src/Val.h +++ b/src/Val.h @@ -20,6 +20,7 @@ #include "Notifier.h" #include "IPAddr.h" #include "DebugLogger.h" +#include "RE.h" // We have four different port name spaces: TCP, UDP, ICMP, and UNKNOWN. // We distinguish between them based on the bits specified in the *_PORT_MASK @@ -34,7 +35,6 @@ class Val; class Func; class BroFile; -class RE_Matcher; class PrefixTable; class PortVal; @@ -347,6 +347,10 @@ public: static bool WouldOverflow(const BroType* from_type, const BroType* to_type, const Val* val); + TableVal* GetRecordFields(); + + StringVal* ToJSON(bool only_loggable=false, RE_Matcher* re=new RE_Matcher("^_")); + protected: friend class EnumType; @@ -530,6 +534,7 @@ public: // Returns the port number in host order (not including the mask). uint32 Port() const; + string Protocol() const; // Tests for protocol types. int IsTCP() const; @@ -632,10 +637,13 @@ public: // char* ExpandedString(int format = BroString::EXPANDED_STRING) // { return AsString()->ExpandedString(format); } + std::string ToStdString() const; StringVal* ToUpper(); unsigned int MemoryAllocation() const override; + Val* Substitute(RE_Matcher* re, StringVal* repl, bool do_all); + protected: friend class Val; StringVal() {} @@ -981,7 +989,6 @@ protected: Val* DoClone(CloneState* state) override; - RecordType* record_type; BroObj* origin; static vector parse_time_records; diff --git a/src/Var.cc b/src/Var.cc index 74cfee291f..3f8f411dc3 100644 --- a/src/Var.cc +++ b/src/Var.cc @@ -108,7 +108,7 @@ static void make_var(ID* id, BroType* t, init_class c, Expr* init, id->SetType(t); if ( attr ) - id->AddAttrs(new Attributes(attr, t, false)); + id->AddAttrs(new Attributes(attr, t, false, id->IsGlobal())); if ( init ) { @@ -286,7 +286,7 @@ void add_type(ID* id, BroType* t, attr_list* attr) id->MakeType(); if ( attr ) - id->SetAttrs(new Attributes(attr, tnew, false)); + id->SetAttrs(new Attributes(attr, tnew, false, false)); } static void transfer_arg_defaults(RecordType* args, RecordType* recv) @@ -304,7 +304,7 @@ static void transfer_arg_defaults(RecordType* args, RecordType* recv) if ( ! recv_i->attrs ) { attr_list* a = new attr_list{def}; - recv_i->attrs = new Attributes(a, recv_i->type, true); + recv_i->attrs = new Attributes(a, recv_i->type, true, false); } else if ( ! recv_i->attrs->FindAttr(ATTR_DEFAULT) ) diff --git a/src/analyzer/protocol/dnp3/dnp3-objects.pac b/src/analyzer/protocol/dnp3/dnp3-objects.pac index 3061500307..3aabe07e2a 100644 --- a/src/analyzer/protocol/dnp3/dnp3-objects.pac +++ b/src/analyzer/protocol/dnp3/dnp3-objects.pac @@ -215,10 +215,10 @@ type Request_Data_Object(function_code: uint8, qualifier_field: uint8, object_ty 0x2804 -> aos_dp: empty; # analog ouput g41 - 0x2901 -> ao_32: AnaOut32; - 0x2902 -> ao_16: AnaOut16; - 0x2903 -> ao_sp: AnaOutSP; - 0x2904 -> ao_dp: AnaOutDP; + 0x2901 -> ao_32: empty; + 0x2902 -> ao_16: empty; + 0x2903 -> ao_sp: empty; + 0x2904 -> ao_dp: empty; # analog output event g42 0x2a00 -> aoe_default: empty; @@ -258,7 +258,6 @@ type Request_Data_Object(function_code: uint8, qualifier_field: uint8, object_ty # class objects g60 0x3C01 -> class0data: empty; # &check(qualifier_field == 0x06); - #0x3C02 -> class1data: uint8; # &check(qualifier_field == 0x06); 0x3C02 -> class1data: empty; # &check(qualifier_field == 0x06 || qualifier_field == 0x07 || qualifier_field == 0x08); 0x3C03 -> class2data: empty; # &check(qualifier_field == 0x06 || qualifier_field == 0x07 || qualifier_field == 0x08); 0x3C04 -> class3data: empty; # &check(qualifier_field == 0x06 || qualifier_field == 0x07 || qualifier_field == 0x08); @@ -266,11 +265,9 @@ type Request_Data_Object(function_code: uint8, qualifier_field: uint8, object_ty 0x4601 -> file_control_id: File_Control_ID; 0x4602 -> file_control_auth: File_Control_Auth_Wrap(function_code); 0x4603 -> file_control_cmd: File_Control_Cmd; # &check( file_control_cmd.op_mode == 0 || file_control_cmd.op_mode == 1 || file_control_cmd.op_mode == 2 || file_control_cmd.op_mode == 3 ); - #0x4604 -> file_control_cmd_status: File_Control_Cmd_Status_Wrap(function_code, prefix.prefix_value); # example shown in P66 0x4604 -> file_control_cmd_status: File_Control_Cmd_Status(prefix.prefix_value); # example shown in P66 0x4605 -> file_trans: File_Transport(prefix.prefix_value); 0x4606 -> file_trans_status: File_Transport_Status(prefix.prefix_value); - #0x4607 -> file_desc: File_Desc_Wrap(function_code); 0x4607 -> file_desc: File_Desc; # internal indication g80 @@ -318,13 +315,20 @@ type Request_Data_Object(function_code: uint8, qualifier_field: uint8, object_ty # authentication challenge g120 0x7801 -> challenge: AuthChallenge(prefix.prefix_value); - 0x7802 -> reply: AuthRely(prefix.prefix_value); + 0x7802 -> reply: AuthReply(prefix.prefix_value); 0x7803 -> aggrRequest: AuthAggrRequest(prefix.prefix_value); - 0x7804 -> seesionKeyRequest: uint8; + 0x7804 -> seesionKeyRequest: uint16; 0x7805 -> status: AuthSessionKeyStatus(prefix.prefix_value); 0x7806 -> keyChange: AuthSessionKeyChange(prefix.prefix_value); 0x7807 -> error: AuthError(prefix.prefix_value); - + 0x7808 -> user_cert: UserCert(prefix.prefix_value); + 0x7809 -> mac: MAC(prefix.prefix_value); + 0x780A -> user_status_change: UserStatusChange(prefix.prefix_value); + 0x780B -> update_key_req: UpdateKeyReq(prefix.prefix_value); + 0x780C -> update_key_rep: UpdateKeyRep(prefix.prefix_value); + 0x780D -> update_key: UpdateKey(prefix.prefix_value); + 0x780E -> update_key_sig: UpdateKeySig(prefix.prefix_value); + 0x780F -> update_key_con: UpdateKeyCon(prefix.prefix_value); default -> unmatched: Default_Wrap(object_type_field); }; }; @@ -468,10 +472,10 @@ type Response_Data_Object(function_code: uint8, qualifier_field: uint8, object_t 0x1f02 -> f_ai_16_wflag: FrozenAnalogInput16wFlag; 0x1f03 -> f_ai_32_wtime: FrozenAnalogInput32wTime; 0x1f04 -> f_ai_16_wtime: FrozenAnalogInput16wTime; - 0x1f05 -> f_ai_32_woflag: AnalogInput32woFlag; - 0x1f06 -> f_ai_16_woflag: AnalogInput16woFlag; - 0x1f07 -> f_ai_sp_wflag: AnalogInputSPwFlag; - 0x1f08 -> f_ai_dp_wflag: AnalogInputDPwFlag; + 0x1f05 -> f_ai_32_woflag: FrozenAnalogInput32woFlag; + 0x1f06 -> f_ai_16_woflag: FrozenAnalogInput16woFlag; + 0x1f07 -> f_ai_sp_wflag: FrozenAnalogInputSPwFlag; + 0x1f08 -> f_ai_dp_wflag: FrozenAnalogInputDPwFlag; # analog input event g32 0x2001 -> ai32wotime: AnalogInput32woTime; @@ -592,12 +596,20 @@ type Response_Data_Object(function_code: uint8, qualifier_field: uint8, object_t # authentication challenge g120 0x7801 -> challenge: AuthChallenge(prefix.prefix_value); - 0x7802 -> reply: AuthRely(prefix.prefix_value); + 0x7802 -> reply: AuthReply(prefix.prefix_value); 0x7803 -> aggrRequest: AuthAggrRequest(prefix.prefix_value); - 0x7804 -> seesionKeyRequest: uint8; + 0x7804 -> seesionKeyRequest: uint16; 0x7805 -> status: AuthSessionKeyStatus(prefix.prefix_value); 0x7806 -> keyChange: AuthSessionKeyChange(prefix.prefix_value); 0x7807 -> error: AuthError(prefix.prefix_value); + 0x7808 -> user_cert: UserCert(prefix.prefix_value); + 0x7809 -> mac: MAC(prefix.prefix_value); + 0x780A -> user_status_change: UserStatusChange(prefix.prefix_value); + 0x780B -> update_key_req: UpdateKeyReq(prefix.prefix_value); + 0x780C -> update_key_rep: UpdateKeyRep(prefix.prefix_value); + 0x780D -> update_key: UpdateKey(prefix.prefix_value); + 0x780E -> update_key_sig: UpdateKeySig(prefix.prefix_value); + 0x780F -> update_key_con: UpdateKeyCon(prefix.prefix_value); #default -> unkonwndata: Debug_Byte; # &check( T ); default -> unmatched: Default_Wrap(object_type_field); @@ -1381,41 +1393,115 @@ type BCD_Large = record { # g120v1 type AuthChallenge(prefix: uint16) = record { - csqUsr: uint32; - hal: uint8; + cha_seq_num: uint32; + user_num: uint16; + mac_alg: uint8; reason: uint8; - chan_data: bytestring &length = (prefix - 10); + chan_data: bytestring &length = (prefix - 8); } &byteorder = littleendian; # g120v2 -type AuthRely(prefix: uint16) = record { - csqUsr: uint32; - chan_data: bytestring &length = (prefix - 4); +type AuthReply(prefix: uint16) = record { + cha_seq_num: uint32; + user_num : uint16; + mac: bytestring &length = (prefix - 6); } &byteorder = littleendian; # g120v3 type AuthAggrRequest(prefix: uint16) = record { - csqUsr: uint32; - chan_data: bytestring &length = (prefix - 4); + cha_seq_num: uint32; + user_num: uint16; } &byteorder = littleendian; # g120v5 type AuthSessionKeyStatus(prefix: uint16) = record { - csqUsr: uint32; + cha_seq_num: uint32; + user_num: uint16; key_alg: uint8; key_status: uint8; - chan_data: bytestring &length = (prefix - 10); + mac_alg: uint8; + cha_data_len : uint16; + chan_data: bytestring &length = cha_data_len; + mac: bytestring &length = (prefix - 11 - cha_data_len); } &byteorder = littleendian; # g120v6 type AuthSessionKeyChange(prefix: uint16) = record { - csqUsr: uint32; - key_wrap_data: bytestring &length = (prefix - 5); + key_change_num: uint32; + user_num: uint16; + key_wrap_data: bytestring &length = (prefix - 6); } &byteorder = littleendian; # g120v7 type AuthError(prefix: uint16) = record { - csqUsr: uint32; + cha_seq_num: uint32; + user_num: uint16; + id: uint16; error_code: uint8; - key_wrap_data: bytestring &length = (prefix - 6); + time_error: bytestring &length = 6; + error_text: bytestring &length = (prefix - 15); } &byteorder = littleendian; + +# g120v8 +type UserCert(prefix: uint16) = record { + method: uint8; + cert_type: uint8; + cert_text: bytestring &length = (prefix - 2); +} &byteorder = littleendian; + +# g120v9 +type MAC(prefix: uint16) = record { + mac_text: bytestring &length = prefix; +} &byteorder = littleendian; + +# g120v10 +type UserStatusChange(prefix: uint16) = record { + method: uint8; + operation: uint8; + seq_num: uint32; + user_role: uint16; + user_role_exp: uint16; + user_name_len: uint16; + user_pubkey_len: uint16; + cert_data_len: uint16; + user_name: bytestring &length = user_name_len; + user_pubkey: bytestring &length = user_pubkey_len; + cert_data: bytestring &length = cert_data_len; +} &byteorder = littleendian; + +# g120v11 +type UpdateKeyReq(prefix: uint16) = record { + method: uint8; + user_name_len: uint16; + master_cha_data_len: uint16; + user_name: bytestring &length = user_name_len; + master_cha_data: bytestring &length = master_cha_data_len; +} &byteorder = littleendian; + +# g120v12 +type UpdateKeyRep(prefix: uint16) = record { + seq_num: uint32; + user_num: uint16; + user_name_len: uint16; + outstation_cha_data_len: uint16; + outstation_cha_data: bytestring &length = outstation_cha_data_len; +} &byteorder = littleendian; + +# g120v13 +type UpdateKey(prefix: uint16) = record { + seq_num: uint32; + user_num: uint16; + update_key_len: uint16; + update_key_data: bytestring &length = update_key_len; +} &byteorder = littleendian; + +# g120v14 +type UpdateKeySig(prefix: uint16) = record { + digital_sig: bytestring &length = prefix; +} &byteorder = littleendian; + +# g120v15 +type UpdateKeyCon(prefix: uint16) = record { + mac: bytestring &length = prefix; +} &byteorder = littleendian; + diff --git a/src/analyzer/protocol/dnp3/dnp3-protocol.pac b/src/analyzer/protocol/dnp3/dnp3-protocol.pac index 154fdc8be3..5c20379c8b 100644 --- a/src/analyzer/protocol/dnp3/dnp3-protocol.pac +++ b/src/analyzer/protocol/dnp3/dnp3-protocol.pac @@ -91,10 +91,57 @@ type DNP3_Application_Response_Header = record { type Request_Objects(function_code: uint8) = record { object_header: Object_Header(function_code); data: case (object_header.object_type_field) of { + # binary output command g12 + 0x0c01 -> g12v1_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + 0x0c02 -> g12v2_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; 0x0c03 -> bocmd_PM: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ ( object_header.number_of_item / 8 ) + 1*( object_header.number_of_item > ( (object_header.number_of_item / 8)*8 ) ) ]; - 0x3202 -> time_interval_ojbects: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + + # time data interval data object g50 + 0x3201 -> g50v1_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + #0x3202 -> time_interval_ojbects: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; # &check( object_header.qualifier_field == 0x0f && object_header.number_of_item == 0x01); - default -> ojbects: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + 0x3202 -> g50v2_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + 0x3203 -> g50v3_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + + # Time and Date Common Time-of-Occurrence g51 + 0x3301 -> g51v1_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + 0x3302 -> g51v2_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + + # time delay g52 + 0x3401 -> g52v1_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + 0x3402 -> g52v2_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + + # file control g70 + 0x4601 -> g70v1_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + 0x4602 -> g70v2_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + 0x4603 -> g70v3_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + 0x4604 -> g70v4_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + 0x4605 -> g70v5_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + 0x4606 -> g70v6_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + 0x4607 -> g70v7_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + + # internal indication g80 + 0x5001 -> g80v1_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + + # authentication challenge g120 + 0x7801 -> g120v1_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + 0x7802 -> g120v2_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + 0x7803 -> g120v3_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + 0x7804 -> g120v4_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + 0x7805 -> g120v5_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + 0x7806 -> g120v6_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + 0x7807 -> g120v7_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + 0x7808 -> g120v8_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + 0x7809 -> g120v9_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + 0x780A -> g120v10_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + 0x780B -> g120v11_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + 0x780C -> g120v12_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + 0x780D -> g120v13_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + 0x780E -> g120v14_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + 0x780F -> g120v15_objs: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + + # default -> ojbects: Request_Data_Object(function_code, object_header.qualifier_field, object_header.object_type_field )[ object_header.number_of_item]; + default -> objects: empty; }; # dump_data is always empty; I intend to use it for checking some conditions; # However, in the current binpac implementation, &check is not implemented diff --git a/src/broker/Data.cc b/src/broker/Data.cc index 657ddd3551..8572745474 100644 --- a/src/broker/Data.cc +++ b/src/broker/Data.cc @@ -1200,6 +1200,9 @@ bool bro_broker::VectorIterator::DoUnserialize(const broker::data& data) auto x = caf::get_if(&(*v)[0]); auto y = caf::get_if(&(*v)[1]); + if ( ! (x && y) ) + return false; + dat = *x; it = dat.begin() + *y; return true; @@ -1222,6 +1225,9 @@ bool bro_broker::RecordIterator::DoUnserialize(const broker::data& data) auto x = caf::get_if(&(*v)[0]); auto y = caf::get_if(&(*v)[1]); + if ( ! (x && y) ) + return false; + dat = *x; it = dat.begin() + *y; return true; diff --git a/src/probabilistic/CounterVector.cc b/src/probabilistic/CounterVector.cc index b9a173356e..6b5a6ed92e 100644 --- a/src/probabilistic/CounterVector.cc +++ b/src/probabilistic/CounterVector.cc @@ -179,6 +179,9 @@ std::unique_ptr CounterVector::Unserialize(const broker::data& da auto width = caf::get_if(&(*v)[0]); auto bits = BitVector::Unserialize((*v)[1]); + if ( ! (width && bits) ) + return nullptr; + auto cv = std::unique_ptr(new CounterVector()); cv->width = *width; cv->bits = bits.release(); diff --git a/src/probabilistic/Topk.cc b/src/probabilistic/Topk.cc index 8ff158e10d..29f47ec753 100644 --- a/src/probabilistic/Topk.cc +++ b/src/probabilistic/Topk.cc @@ -492,25 +492,25 @@ bool TopkVal::DoUnserialize(const broker::data& data) while ( i < numElements ) { - Bucket* b = new Bucket(); auto elements_count = caf::get_if(&(*v)[idx++]); auto count = caf::get_if(&(*v)[idx++]); if ( ! (elements_count && count) ) return false; + Bucket* b = new Bucket(); b->count = *count; b->bucketPos = buckets.insert(buckets.end(), b); for ( uint64_t j = 0; j < *elements_count; j++ ) { - Element* e = new Element(); auto epsilon = caf::get_if(&(*v)[idx++]); Val* val = bro_broker::data_to_val((*v)[idx++], type); if ( ! (epsilon && val) ) return false; + Element* e = new Element(); e->epsilon = *epsilon; e->value = val; e->parent = b; diff --git a/src/strings.bif b/src/strings.bif index f2661f8cc9..42630e4b6b 100644 --- a/src/strings.bif +++ b/src/strings.bif @@ -351,91 +351,6 @@ Val* do_split(StringVal* str_val, RE_Matcher* re, int incl_sep, int max_num_sep) return a; } -Val* do_sub(StringVal* str_val, RE_Matcher* re, StringVal* repl, int do_all) - { - const u_char* s = str_val->Bytes(); - int offset = 0; - int n = str_val->Len(); - - // cut_points is a set of pairs of indices in str that should - // be removed/replaced. A pair means "delete starting - // at offset x, up to but not including offset y". - List(ptr_compat_int) cut_points; // where RE matches pieces of str - - int size = 0; // size of result - - while ( n > 0 ) - { - // Find next match offset. - int end_of_match; - while ( n > 0 && - (end_of_match = re->MatchPrefix(&s[offset], n)) <= 0 ) - { - // This character is going to be copied to the result. - ++size; - - // Move on to next character. - ++offset; - --n; - } - - if ( n <= 0 ) - break; - - // s[offset .. offset+end_of_match-1] matches re. - cut_points.append(offset); - cut_points.append(offset + end_of_match); - - offset += end_of_match; - n -= end_of_match; - - if ( ! do_all ) - { - // We've now done the first substitution - finished. - // Include the remainder of the string in the result. - size += n; - break; - } - } - - // size now reflects amount of space copied. Factor in amount - // of space for replacement text. - int num_cut_points = cut_points.length() / 2; - size += num_cut_points * repl->Len(); - - // And a final NUL for good health. - ++size; - - byte_vec result = new u_char[size]; - byte_vec r = result; - - // Copy it all over. - int start_offset = 0; - for ( int i = 0; i < cut_points.length(); i += 2 /* loop over pairs */ ) - { - int num_to_copy = cut_points[i] - start_offset; - memcpy(r, s + start_offset, num_to_copy); - - r += num_to_copy; - start_offset = cut_points[i+1]; - - // Now add in replacement text. - memcpy(r, repl->Bytes(), repl->Len()); - r += repl->Len(); - } - - // Copy final trailing characters. - int num_to_copy = str_val->Len() - start_offset; - memcpy(r, s + start_offset, num_to_copy); - r += num_to_copy; - - // Final NUL. No need to increment r, since the length - // computed from it in the next statement does not include - // the NUL. - r[0] = '\0'; - - return new StringVal(new BroString(1, result, r - result)); - } %%} ## Splits a string into an array of strings according to a pattern. @@ -535,7 +450,7 @@ function split_string_n%(str: string, re: pattern, ## .. zeek:see:: gsub subst_string function sub%(str: string, re: pattern, repl: string%): string %{ - return do_sub(str, re, repl, 0); + return str->Substitute(re, repl, false); %} ## Substitutes a given replacement string for all occurrences of a pattern @@ -552,7 +467,7 @@ function sub%(str: string, re: pattern, repl: string%): string ## .. zeek:see:: sub subst_string function gsub%(str: string, re: pattern, repl: string%): string %{ - return do_sub(str, re, repl, 1); + return str->Substitute(re, repl, true); %} diff --git a/src/threading/Formatter.cc b/src/threading/Formatter.cc index 395a7fefa6..5fa8ab6fa9 100644 --- a/src/threading/Formatter.cc +++ b/src/threading/Formatter.cc @@ -111,3 +111,14 @@ string Formatter::Render(double d) return buf; } +string Formatter::Render(TransportProto proto) + { + if ( proto == TRANSPORT_UDP ) + return "udp"; + else if ( proto == TRANSPORT_TCP ) + return "tcp"; + else if ( proto == TRANSPORT_ICMP ) + return "icmp"; + else + return "unknown"; + } diff --git a/src/threading/Formatter.h b/src/threading/Formatter.h index c564f3c945..53ca22254b 100644 --- a/src/threading/Formatter.h +++ b/src/threading/Formatter.h @@ -112,6 +112,17 @@ public: */ static string Render(double d); + /** + * Convert a transport protocol into a string. + * + * This is a helper function that formatter implementations may use. + * + * @param proto The transport protocol. + * + * @return An ASCII representation of the protocol. + */ + static string Render(TransportProto proto); + /** * Convert a string into a TransportProto. The string must be one of * \c tcp, \c udp, \c icmp, or \c unknown. diff --git a/src/threading/formatters/JSON.cc b/src/threading/formatters/JSON.cc index a324a08530..58b1ffd779 100644 --- a/src/threading/formatters/JSON.cc +++ b/src/threading/formatters/JSON.cc @@ -11,7 +11,7 @@ #include #include -#include "./JSON.h" +#include "JSON.h" using namespace threading::formatter; @@ -27,78 +27,83 @@ JSON::~JSON() bool JSON::Describe(ODesc* desc, int num_fields, const Field* const * fields, Value** vals) const { - if ( surrounding_braces ) - desc->AddRaw("{"); + ZeekJson j = ZeekJson::object(); for ( int i = 0; i < num_fields; i++ ) { - const u_char* bytes = desc->Bytes(); - int len = desc->Len(); + if ( vals[i]->present ) + { + ZeekJson new_entry = BuildJSON(vals[i]); + if ( new_entry.is_null() ) + return false; - if ( i > 0 && - len > 0 && - bytes[len-1] != ',' && - bytes[len-1] != '{' && - bytes[len-1] != '[' && - vals[i]->present ) - desc->AddRaw(","); - - if ( ! Describe(desc, vals[i], fields[i]->name) ) - return false; + j[fields[i]->name] = new_entry; + } } - if ( surrounding_braces ) - desc->AddRaw("}"); + desc->Add(j.dump()); return true; } bool JSON::Describe(ODesc* desc, Value* val, const string& name) const { + if ( desc->IsBinary() ) + { + GetThread()->Error("json formatter: binary format not supported"); + return false; + } + if ( ! val->present ) return true; - if ( name.size() ) - { - desc->AddRaw("\"", 1); - desc->Add(name); - desc->AddRaw("\":", 2); - } + ZeekJson j = BuildJSON(val, name); + if ( j.is_null() ) + return false; + desc->Add(j.dump()); + return true; + } + +threading::Value* JSON::ParseValue(const string& s, const string& name, TypeTag type, TypeTag subtype) const + { + GetThread()->Error("JSON formatter does not support parsing yet."); + return nullptr; + } + +ZeekJson JSON::BuildJSON(Value* val, const string& name) const + { + ZeekJson j; switch ( val->type ) { case TYPE_BOOL: - desc->AddRaw(val->val.int_val == 0 ? "false" : "true"); + j = val->val.int_val != 0; break; case TYPE_INT: - desc->Add(val->val.int_val); + j = val->val.int_val; break; case TYPE_COUNT: case TYPE_COUNTER: - desc->Add(val->val.uint_val); + j = val->val.uint_val; break; case TYPE_PORT: - desc->Add(val->val.port_val.port); + j = val->val.port_val.port; break; case TYPE_SUBNET: - desc->AddRaw("\"", 1); - desc->Add(Render(val->val.subnet_val)); - desc->AddRaw("\"", 1); + j = Formatter::Render(val->val.subnet_val); break; case TYPE_ADDR: - desc->AddRaw("\"", 1); - desc->Add(Render(val->val.addr_val)); - desc->AddRaw("\"", 1); + j = Formatter::Render(val->val.addr_val); break; case TYPE_DOUBLE: case TYPE_INTERVAL: - desc->Add(val->val.double_val); + j = val->val.double_val; break; case TYPE_TIME: @@ -110,15 +115,13 @@ bool JSON::Describe(ODesc* desc, Value* val, const string& name) const time_t the_time = time_t(floor(val->val.double_val)); struct tm t; - desc->AddRaw("\"", 1); - if ( ! gmtime_r(&the_time, &t) || ! strftime(buffer, sizeof(buffer), "%Y-%m-%dT%H:%M:%S", &t) ) { GetThread()->Error(GetThread()->Fmt("json formatter: failure getting time: (%lf)", val->val.double_val)); // This was a failure, doesn't really matter what gets put here // but it should probably stand out... - desc->Add("2000-01-01T00:00:00.000000"); + j = "2000-01-01T00:00:00.000000"; } else { @@ -129,20 +132,17 @@ bool JSON::Describe(ODesc* desc, Value* val, const string& name) const frac += 1; snprintf(buffer2, sizeof(buffer2), "%s.%06.0fZ", buffer, fabs(frac) * 1000000); - desc->Add(buffer2); + j = buffer2; } - - desc->AddRaw("\"", 1); } else if ( timestamps == TS_EPOCH ) - desc->Add(val->val.double_val); + j = val->val.double_val; else if ( timestamps == TS_MILLIS ) { // ElasticSearch uses milliseconds for timestamps - uint64_t ts = (uint64_t) (val->val.double_val * 1000); - desc->Add(ts); + j = (uint64_t) (val->val.double_val * 1000); } break; @@ -153,74 +153,40 @@ bool JSON::Describe(ODesc* desc, Value* val, const string& name) const case TYPE_FILE: case TYPE_FUNC: { - desc->AddRaw("\"", 1); - - for ( int i = 0; i < val->val.string_val.length; ++i ) - { - char c = val->val.string_val.data[i]; - - // 2byte Unicode escape special characters. - if ( c < 32 || c > 126 || c == '\n' || c == '"' || c == '\'' || c == '\\' || c == '&' ) - { - desc->AddRaw("\\u00", 4); - char hex[2] = {'0', '0'}; - bytetohex(c, hex); - desc->AddRaw(hex, 1); - desc->AddRaw(hex + 1, 1); - } - else - desc->AddRaw(&c, 1); - } - - desc->AddRaw("\"", 1); + j = json_escape_utf8(string(val->val.string_val.data, val->val.string_val.length)); break; } case TYPE_TABLE: { - desc->AddRaw("[", 1); + j = ZeekJson::array(); - for ( int j = 0; j < val->val.set_val.size; j++ ) - { - if ( j > 0 ) - desc->AddRaw(",", 1); + for ( int idx = 0; idx < val->val.set_val.size; idx++ ) + j.push_back(BuildJSON(val->val.set_val.vals[idx])); - Describe(desc, val->val.set_val.vals[j]); - } - - desc->AddRaw("]", 1); break; } case TYPE_VECTOR: { - desc->AddRaw("[", 1); + j = ZeekJson::array(); - for ( int j = 0; j < val->val.vector_val.size; j++ ) - { - if ( j > 0 ) - desc->AddRaw(",", 1); - Describe(desc, val->val.vector_val.vals[j]); - } + for ( int idx = 0; idx < val->val.vector_val.size; idx++ ) + j.push_back(BuildJSON(val->val.vector_val.vals[idx])); - desc->AddRaw("]", 1); break; } default: - return false; + break; } - return true; - } + if ( ! name.empty() && ! j.is_null() ) + { + ZeekJson j2 = ZeekJson::object(); + j2[name] = j; + return j2; + } -threading::Value* JSON::ParseValue(const string& s, const string& name, TypeTag type, TypeTag subtype) const - { - GetThread()->Error("JSON formatter does not support parsing yet."); - return NULL; - } - -void JSON::SurroundingBraces(bool use_braces) - { - surrounding_braces = use_braces; + return j; } diff --git a/src/threading/formatters/JSON.h b/src/threading/formatters/JSON.h index 4984f74067..8b9ab1ed6e 100644 --- a/src/threading/formatters/JSON.h +++ b/src/threading/formatters/JSON.h @@ -4,9 +4,19 @@ #define THREADING_FORMATTERS_JSON_H #include "../Formatter.h" +#include "3rdparty/json.hpp" +#include "3rdparty/fifo_map.hpp" + namespace threading { namespace formatter { +// Define a class for use with the json library that orders the keys in the same order that +// they were inserted. By default, the json library orders them alphabetically and we don't +// want it like that. +template +using json_fifo_map = nlohmann::fifo_map, A>; +using ZeekJson = nlohmann::basic_json; + /** * A thread-safe class for converting values into a JSON representation * and vice versa. @@ -27,9 +37,10 @@ public: threading::Value** vals) const override; threading::Value* ParseValue(const string& s, const string& name, TypeTag type, TypeTag subtype = TYPE_ERROR) const override; - void SurroundingBraces(bool use_braces); - private: + + ZeekJson BuildJSON(Value* val, const string& name = "") const; + TimeFormat timestamps; bool surrounding_braces; }; diff --git a/src/util.cc b/src/util.cc index 2a6a5c37c4..e7d989660b 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1842,24 +1842,24 @@ void bro_strerror_r(int bro_errno, char* buf, size_t buflen) strerror_r_helper(res, buf, buflen); } +static const std::map legacy_vars = { + { "ZEEKPATH", "BROPATH" }, + { "ZEEK_PLUGIN_PATH", "BRO_PLUGIN_PATH" }, + { "ZEEK_PLUGIN_ACTIVATE", "BRO_PLUGIN_ACTIVATE" }, + { "ZEEK_PREFIXES", "BRO_PREFIXES" }, + { "ZEEK_DNS_FAKE", "BRO_DNS_FAKE" }, + { "ZEEK_SEED_FILE", "BRO_SEED_FILE" }, + { "ZEEK_LOG_SUFFIX", "BRO_LOG_SUFFIX" }, + { "ZEEK_PROFILER_FILE", "BRO_PROFILER_FILE" }, + { "ZEEK_DISABLE_ZEEKYGEN", "BRO_DISABLE_BROXYGEN" }, + { "ZEEK_DEFAULT_CONNECT_RETRY", "BRO_DEFAULT_CONNECT_RETRY" }, + { "ZEEK_BROKER_MAX_THREADS", "BRO_BROKER_MAX_THREADS" }, + { "ZEEK_DEFAULT_LISTEN_ADDRESS", "BRO_DEFAULT_LISTEN_ADDRESS" }, + { "ZEEK_DEFAULT_LISTEN_RETRY", "BRO_DEFAULT_LISTEN_RETRY" }, +}; + char* zeekenv(const char* name) { - static std::map legacy_vars = { - { "ZEEKPATH", "BROPATH" }, - { "ZEEK_PLUGIN_PATH", "BRO_PLUGIN_PATH" }, - { "ZEEK_PLUGIN_ACTIVATE", "BRO_PLUGIN_ACTIVATE" }, - { "ZEEK_PREFIXES", "BRO_PREFIXES" }, - { "ZEEK_DNS_FAKE", "BRO_DNS_FAKE" }, - { "ZEEK_SEED_FILE", "BRO_SEED_FILE" }, - { "ZEEK_LOG_SUFFIX", "BRO_LOG_SUFFIX" }, - { "ZEEK_PROFILER_FILE", "BRO_PROFILER_FILE" }, - { "ZEEK_DISABLE_ZEEKYGEN", "BRO_DISABLE_BROXYGEN" }, - { "ZEEK_DEFAULT_CONNECT_RETRY", "BRO_DEFAULT_CONNECT_RETRY" }, - { "ZEEK_BROKER_MAX_THREADS", "BRO_BROKER_MAX_THREADS" }, - { "ZEEK_DEFAULT_LISTEN_ADDRESS", "BRO_DEFAULT_LISTEN_ADDRESS" }, - { "ZEEK_DEFAULT_LISTEN_RETRY", "BRO_DEFAULT_LISTEN_RETRY" }, - }; - auto rval = getenv(name); if ( rval ) @@ -1872,3 +1872,90 @@ char* zeekenv(const char* name) return getenv(it->second); } + +static string json_escape_byte(char c) + { + char hex[2] = {'0', '0'}; + bytetohex(c, hex); + + string result = "\\x"; + result.append(hex, 2); + + return result; + } + +string json_escape_utf8(const string& val) + { + string result; + result.reserve(val.length()); + + size_t char_start = 0; + size_t idx; + for ( idx = 0; idx < val.length(); ) + { + // Normal ASCII characters plus a few of the control characters can be inserted directly. The rest of + // the control characters should be escaped as regular bytes. + if ( ( val[idx] >= 32 && val[idx] <= 127 ) || + val[idx] == '\b' || val[idx] == '\f' || val[idx] == '\n' || val[idx] == '\r' || val[idx] == '\t' ) + { + result.push_back(val[idx]); + ++idx; + continue; + } + else if ( val[idx] >= 0 && val[idx] < 32 ) + { + result.append(json_escape_byte(val[idx])); + ++idx; + continue; + } + + // The next bit is based on the table at https://en.wikipedia.org/wiki/UTF-8#Description. + // If next character is 11110xxx, this is a 4-byte UTF-8 + unsigned int char_size = 0; + if ( (val[idx] & 0xF8) == 0xF0 ) char_size = 4; + + // If next character is 1110xxxx, this is a 3-byte UTF-8 + else if ( (val[idx] & 0xF0) == 0xE0 ) char_size = 3; + + // If next character is 110xxxxx, this is a 2-byte UTF-8 + else if ( (val[idx] & 0xE0) == 0xC0 ) char_size = 2; + + // This byte isn't a continuation byte, insert it as a byte and continue. + if ( char_size == 0) + { + result.append(json_escape_byte(val[idx])); + ++idx; + continue; + } + + // If we don't have enough bytes to get to the end of character, give up and insert all of the rest + // of them as escaped values. + if ( char_size > (val.length() - idx) ) + break; + + // Loop through the rest of the supposed character and see if this is a valid character. + size_t c_idx = idx + 1; + for ( ; c_idx < idx + char_size; c_idx++ ) + if ( (val[c_idx] & 0xC0) != 0x80 ) break; + + // if we didn't make it to the end of the character without finding an error, insert just this + // character and skip ahead. Otherwise insert all of the bytes for this character into the result. + if ( c_idx != idx + char_size ) + { + result.append(json_escape_byte(val[idx])); + ++idx; + continue; + } + else + { + for ( size_t step = 0; step < char_size; step++, idx++ ) + result.push_back(val[idx]); + } + } + + if ( idx != val.length() ) + for ( ; idx < val.length(); ++idx ) + result.append(json_escape_byte(val[idx])); + + return result; + } diff --git a/src/util.h b/src/util.h index f019f4cbc1..661e18a111 100644 --- a/src/util.h +++ b/src/util.h @@ -565,4 +565,12 @@ std::unique_ptr build_unique (Args&&... args) { return std::unique_ptr(new T(std::forward(args)...)); } +/** + * Escapes bytes in a string that are not valid UTF8 characters with \xYY format. Used + * by the JSON writer and BIF methods. + * @param val the input string to be escaped + * @return the escaped string + */ +std::string json_escape_utf8(const std::string& val); + #endif diff --git a/src/zeek.bif b/src/zeek.bif index da11a18681..8b7a047189 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -1981,68 +1981,7 @@ function lookup_ID%(id: string%) : any ## Returns: A table that describes the fields of a record. function record_fields%(rec: any%): record_field_table %{ - TableVal* fields = new TableVal(record_field_table); - - auto t = rec->Type(); - - if ( t->Tag() != TYPE_RECORD && t->Tag() != TYPE_TYPE ) - { - reporter->Error("non-record value/type passed to record_fields"); - return fields; - } - - RecordType* rt = nullptr; - RecordVal* rv = nullptr; - - if ( t->Tag() == TYPE_RECORD ) - { - rt = t->AsRecordType(); - rv = rec->AsRecordVal(); - } - else - { - t = t->AsTypeType()->Type(); - - if ( t->Tag() != TYPE_RECORD ) - { - reporter->Error("non-record value/type passed to record_fields"); - return fields; - } - - rt = t->AsRecordType(); - } - - for ( int i = 0; i < rt->NumFields(); ++i ) - { - BroType* ft = rt->FieldType(i); - TypeDecl* fd = rt->FieldDecl(i); - Val* fv = nullptr; - - if ( rv ) - fv = rv->Lookup(i); - - if ( fv ) - Ref(fv); - - bool logged = (fd->attrs && fd->FindAttr(ATTR_LOG) != 0); - - RecordVal* nr = new RecordVal(record_field); - - if ( ft->Tag() == TYPE_RECORD ) - nr->Assign(0, new StringVal("record " + ft->GetName())); - else - nr->Assign(0, new StringVal(type_name(ft->Tag()))); - - nr->Assign(1, val_mgr->GetBool(logged)); - nr->Assign(2, fv); - nr->Assign(3, rt->FieldDefault(i)); - - Val* field_name = new StringVal(rt->FieldName(i)); - fields->Assign(field_name, nr); - Unref(field_name); - } - - return fields; + return rec->GetRecordFields(); %} ## Enables detailed collection of profiling statistics. Statistics include @@ -5100,3 +5039,16 @@ function anonymize_addr%(a: addr, cl: IPAddrAnonymizationClass%): addr (enum ip_addr_anonymization_class_t) anon_class)); } %} + +## A function to convert arbitrary Zeek data into a JSON string. +## +## v: The value to convert to JSON. Typically a record. +## +## only_loggable: If the v value is a record this will only cause +## fields with the &log attribute to be included in the JSON. +## +## returns: a JSON formatted string. +function to_json%(val: any, only_loggable: bool &default=F, field_escape_pattern: pattern &default=/^_/%): string + %{ + return val->ToJSON(only_loggable, field_escape_pattern); + %} diff --git a/testing/btest/Baseline/coverage.bare-mode-errors/errors b/testing/btest/Baseline/coverage.bare-mode-errors/errors index e69de29bb2..c87b897c61 100644 --- a/testing/btest/Baseline/coverage.bare-mode-errors/errors +++ b/testing/btest/Baseline/coverage.bare-mode-errors/errors @@ -0,0 +1 @@ +warning in /Users/tim/Desktop/projects/zeek/testing/btest/../../scripts//base/utils/json.zeek, line 2: deprecated script loaded from command line arguments ="Remove in 3.1. to_json is now always available as a built-in function." diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 6fbc95e4ec..b78d8a2480 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -249,7 +249,6 @@ scripts/base/init-default.zeek scripts/base/frameworks/openflow/main.zeek scripts/base/frameworks/openflow/plugins/__load__.zeek scripts/base/frameworks/openflow/plugins/ryu.zeek - scripts/base/utils/json.zeek scripts/base/frameworks/openflow/plugins/log.zeek scripts/base/frameworks/openflow/plugins/broker.zeek scripts/base/frameworks/openflow/non-cluster.zeek diff --git a/testing/btest/Baseline/coverage.init-default/missing_loads b/testing/btest/Baseline/coverage.init-default/missing_loads index 893a603972..87361a686a 100644 --- a/testing/btest/Baseline/coverage.init-default/missing_loads +++ b/testing/btest/Baseline/coverage.init-default/missing_loads @@ -8,3 +8,4 @@ -./frameworks/openflow/cluster.zeek -./frameworks/packet-filter/cluster.zeek -./frameworks/sumstats/cluster.zeek +-./utils/json.zeek diff --git a/testing/btest/Baseline/language.attr-default-global-set-error/out b/testing/btest/Baseline/language.attr-default-global-set-error/out index 6f3fd63d4f..55c70a62ff 100644 --- a/testing/btest/Baseline/language.attr-default-global-set-error/out +++ b/testing/btest/Baseline/language.attr-default-global-set-error/out @@ -1,2 +1,5 @@ -error in /home/jon/pro/zeek/zeek/testing/btest/.tmp/language.attr-default-global-set-error/attr-default-global-set-error.zeek, line 4: arithmetic mixed with non-arithmetic (set[string] and 0) -error in /home/jon/pro/zeek/zeek/testing/btest/.tmp/language.attr-default-global-set-error/attr-default-global-set-error.zeek, line 4: &default value has inconsistent type (0 and set[string]) +error in /Users/tim/Desktop/projects/zeek/testing/btest/.tmp/language.attr-default-global-set-error/attr-default-global-set-error.zeek, line 4: arithmetic mixed with non-arithmetic (set[string] and 0) +error in /Users/tim/Desktop/projects/zeek/testing/btest/.tmp/language.attr-default-global-set-error/attr-default-global-set-error.zeek, line 4: &default value has inconsistent type (0 and set[string]) +error in /Users/tim/Desktop/projects/zeek/testing/btest/.tmp/language.attr-default-global-set-error/attr-default-global-set-error.zeek, line 9: &default is not valid for global variables (&default=10) +error in /Users/tim/Desktop/projects/zeek/testing/btest/.tmp/language.attr-default-global-set-error/attr-default-global-set-error.zeek, line 9: &default is not valid for global variables (&optional, &default=9) +error in /Users/tim/Desktop/projects/zeek/testing/btest/.tmp/language.attr-default-global-set-error/attr-default-global-set-error.zeek, line 9: &optional is not valid for global variables (&optional, &default=9, &optional) diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index ef7ba59161..9b70daf0c8 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -821,7 +821,6 @@ 0.000000 MetaHookPost LoadFile(0, base<...>/input.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/intel) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/irc) -> -1 -0.000000 MetaHookPost LoadFile(0, base<...>/json.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/krb) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/logging) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/logging.bif.zeek) -> -1 @@ -1712,7 +1711,6 @@ 0.000000 MetaHookPre LoadFile(0, base<...>/input.bif.zeek) 0.000000 MetaHookPre LoadFile(0, base<...>/intel) 0.000000 MetaHookPre LoadFile(0, base<...>/irc) -0.000000 MetaHookPre LoadFile(0, base<...>/json.zeek) 0.000000 MetaHookPre LoadFile(0, base<...>/krb) 0.000000 MetaHookPre LoadFile(0, base<...>/logging) 0.000000 MetaHookPre LoadFile(0, base<...>/logging.bif.zeek) @@ -2611,7 +2609,6 @@ 0.000000 | HookLoadFile base<...>/input.bif.zeek 0.000000 | HookLoadFile base<...>/intel 0.000000 | HookLoadFile base<...>/irc -0.000000 | HookLoadFile base<...>/json.zeek 0.000000 | HookLoadFile base<...>/krb 0.000000 | HookLoadFile base<...>/logging 0.000000 | HookLoadFile base<...>/logging.bif.zeek diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-double/json.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-double/json.log index 49b3c5d172..bb0f950b13 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-double/json.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-double/json.log @@ -1,22 +1,22 @@ -{"d":2.153226e+09} -{"d":2.153226e+09} -{"d":2.153226e+09} +{"d":2153226000.0} +{"d":2153226000.1} +{"d":2153226000.123457} {"d":1.0} {"d":1.1} -{"d":1.123457} -{"d":-1.123457} +{"d":1.123456789} +{"d":-1.123456789} {"d":1.1234} {"d":0.1234} {"d":50000.0} {"d":-50000.0} -{"d":3.140000e+15} -{"d":-3.140000e+15} -{"d":1.790000e+308} -{"d":-1.790000e+308} -{"d":0.000012} -{"d":0} -{"d":-0} -{"d":inf} -{"d":-inf} -{"d":0.0} -{"d":nan} +{"d":3.14e+15} +{"d":-3.14e+15} +{"d":1.79e+308} +{"d":-1.79e+308} +{"d":1.23456789e-05} +{"d":2.23e-308} +{"d":-2.23e-308} +{"d":null} +{"d":null} +{"d":-0.0} +{"d":null} diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-double/test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-double/test.log index 9d5dd6ecf0..21457f916c 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-double/test.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-double/test.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path test -#open 2017-11-06-19-58-08 +#open 2019-07-01-17-40-55 #fields d #types double 2153226000.0 @@ -28,4 +28,4 @@ inf -inf 0.0 nan -#close 2017-11-06-19-58-08 +#close 2019-07-01-17-40-55 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json-utf8/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json-utf8/ssh.log new file mode 100644 index 0000000000..b1a395906f --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json-utf8/ssh.log @@ -0,0 +1,12 @@ +{"s":"a"} +{"s":"\b\f\n\r\t\\x00\\x15"} +{"s":"ñ"} +{"s":"\\xc3("} +{"s":"\\xa0\\xa1"} +{"s":"₡"} +{"s":"\\xe2(\\xa1"} +{"s":"\\xe2\\x82("} +{"s":"𐌼"} +{"s":"\\xf0(\\x8c\\xbc"} +{"s":"\\xf0\\x90(\\xbc"} +{"s":"\\xf0(\\x8c("} diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json/ssh.log index 16ba5ddb60..52c5e4856c 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json/ssh.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-json/ssh.log @@ -1 +1 @@ -{"b":true,"i":-42,"e":"SSH::LOG","c":21,"p":123,"sn":"10.0.0.0/24","a":"1.2.3.4","d":3.14,"t":1215620010.54321,"iv":100.0,"s":"hurz","sc":[2,4,1,3],"ss":["BB","AA","CC"],"se":[],"vc":[10,20,30],"ve":[],"f":"SSH::foo\u000a{ \u000aif (0 < SSH::i) \u000a\u0009return (Foo);\u000aelse\u000a\u0009return (Bar);\u000a\u000a}"} +{"b":true,"i":-42,"e":"SSH::LOG","c":21,"p":123,"sn":"10.0.0.0/24","a":"1.2.3.4","d":3.14,"t":1215620010.54321,"iv":100.0,"s":"hurz","sc":[2,4,1,3],"ss":["BB","AA","CC"],"se":[],"vc":[10,20,30],"ve":[],"f":"SSH::foo\n{ \nif (0 < SSH::i) \n\treturn (Foo);\nelse\n\treturn (Bar);\n\n}"} diff --git a/testing/btest/Baseline/scripts.base.frameworks.openflow.ryu-basic/.stdout b/testing/btest/Baseline/scripts.base.frameworks.openflow.ryu-basic/.stdout index 31cb56f05a..20db16d9ff 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.openflow.ryu-basic/.stdout +++ b/testing/btest/Baseline/scripts.base.frameworks.openflow.ryu-basic/.stdout @@ -1,22 +1,22 @@ http://127.0.0.1:8080/stats/flowentry/clear/42 http://127.0.0.1:8080/stats/flowentry/add -{"priority": 0, "idle_timeout": 0, "hard_timeout": 0, "match": {}, "actions": [{"port": 3, "type": "OUTPUT"}, {"port": 7, "type": "OUTPUT"}], "cookie": 4398046511105, "flags": 0, "dpid": 42} +{"priority":0,"idle_timeout":0,"hard_timeout":0,"match":{},"actions":[{"port":3,"type":"OUTPUT"},{"port":7,"type":"OUTPUT"}],"cookie":4398046511105,"flags":0,"dpid":42} Flow_mod_success http://127.0.0.1:8080/stats/flowentry/add -{"priority": 5, "idle_timeout": 30, "hard_timeout": 0, "match": {"nw_dst": "74.53.140.153/32", "tp_dst": 25, "tp_src": 1470, "nw_proto": 6, "dl_type": 2048, "nw_src": "10.10.1.4/32"}, "actions": [], "cookie": 4398046511146, "flags": 0, "dpid": 42} +{"priority":5,"idle_timeout":30,"hard_timeout":0,"match":{"nw_dst":"74.53.140.153/32","tp_dst":25,"tp_src":1470,"nw_proto":6,"dl_type":2048,"nw_src":"10.10.1.4/32"},"actions":[],"cookie":4398046511146,"flags":0,"dpid":42} http://127.0.0.1:8080/stats/flowentry/add -{"priority": 5, "idle_timeout": 30, "hard_timeout": 0, "match": {"nw_dst": "10.10.1.4/32", "tp_dst": 1470, "tp_src": 25, "nw_proto": 6, "dl_type": 2048, "nw_src": "74.53.140.153/32"}, "actions": [], "cookie": 4398046511146, "flags": 0, "dpid": 42} +{"priority":5,"idle_timeout":30,"hard_timeout":0,"match":{"nw_dst":"10.10.1.4/32","tp_dst":1470,"tp_src":25,"nw_proto":6,"dl_type":2048,"nw_src":"74.53.140.153/32"},"actions":[],"cookie":4398046511146,"flags":0,"dpid":42} Flow_mod_success Flow_mod_success http://127.0.0.1:8080/stats/flowentry/add -{"priority": 5, "idle_timeout": 30, "hard_timeout": 0, "match": {"nw_dst": "192.168.133.102/32", "tp_dst": 25, "tp_src": 49648, "nw_proto": 6, "dl_type": 2048, "nw_src": "192.168.133.100/32"}, "actions": [], "cookie": 4398046511146, "flags": 0, "dpid": 42} +{"priority":5,"idle_timeout":30,"hard_timeout":0,"match":{"nw_dst":"192.168.133.102/32","tp_dst":25,"tp_src":49648,"nw_proto":6,"dl_type":2048,"nw_src":"192.168.133.100/32"},"actions":[],"cookie":4398046511146,"flags":0,"dpid":42} http://127.0.0.1:8080/stats/flowentry/add -{"priority": 5, "idle_timeout": 30, "hard_timeout": 0, "match": {"nw_dst": "192.168.133.100/32", "tp_dst": 49648, "tp_src": 25, "nw_proto": 6, "dl_type": 2048, "nw_src": "192.168.133.102/32"}, "actions": [], "cookie": 4398046511146, "flags": 0, "dpid": 42} +{"priority":5,"idle_timeout":30,"hard_timeout":0,"match":{"nw_dst":"192.168.133.100/32","tp_dst":49648,"tp_src":25,"nw_proto":6,"dl_type":2048,"nw_src":"192.168.133.102/32"},"actions":[],"cookie":4398046511146,"flags":0,"dpid":42} Flow_mod_success Flow_mod_success http://127.0.0.1:8080/stats/flowentry/add -{"priority": 5, "idle_timeout": 30, "hard_timeout": 0, "match": {"nw_dst": "17.167.150.73/32", "tp_dst": 443, "tp_src": 49655, "nw_proto": 6, "dl_type": 2048, "nw_src": "192.168.133.100/32"}, "actions": [], "cookie": 4398046511146, "flags": 0, "dpid": 42} +{"priority":5,"idle_timeout":30,"hard_timeout":0,"match":{"nw_dst":"17.167.150.73/32","tp_dst":443,"tp_src":49655,"nw_proto":6,"dl_type":2048,"nw_src":"192.168.133.100/32"},"actions":[],"cookie":4398046511146,"flags":0,"dpid":42} http://127.0.0.1:8080/stats/flowentry/add -{"priority": 5, "idle_timeout": 30, "hard_timeout": 0, "match": {"nw_dst": "192.168.133.100/32", "tp_dst": 49655, "tp_src": 443, "nw_proto": 6, "dl_type": 2048, "nw_src": "17.167.150.73/32"}, "actions": [], "cookie": 4398046511146, "flags": 0, "dpid": 42} +{"priority":5,"idle_timeout":30,"hard_timeout":0,"match":{"nw_dst":"192.168.133.100/32","tp_dst":49655,"tp_src":443,"nw_proto":6,"dl_type":2048,"nw_src":"17.167.150.73/32"},"actions":[],"cookie":4398046511146,"flags":0,"dpid":42} Flow_mod_success Flow_mod_success diff --git a/testing/btest/Baseline/scripts.base.utils.json/output b/testing/btest/Baseline/scripts.base.utils.json/output index 43513e43f9..2d2e56253f 100644 --- a/testing/btest/Baseline/scripts.base.utils.json/output +++ b/testing/btest/Baseline/scripts.base.utils.json/output @@ -8,32 +8,33 @@ true "-12.0 hrs" "hello" "" -65535 -1 -123 -0 +{"port":65535,"proto":"tcp"} +{"port":1,"proto":"udp"} +{"port":123,"proto":"icmp"} +{"port":0,"proto":"unknown"} "1.2.3.4" "ffff:1234::1" "123.123.123.123" "192.0.0.0/8" "fe80::/64" "Red" -{"s": "test", "c": 100} -{"s": "test"} -{"s": "test"} -{"m": {"s": "test"}} +"/^?(^abcd)$?/" +{"s":"test","c":100} +{"s":"test"} +{"s":"test"} +{"m":{"s":"test"}} [] -[2, 1] +[2,1] ["1.2.3.4"] -[[true, false]] -[{"s": "test"}] +[[true,false]] +[{"s":"test"}] [] -[2, 1] +[2,1] ["1.2.3.4"] -[{"s": "test"}] -[{"s": "test"}] +[{"s":"test"}] +[{"s":"test"}] {} -{"2": "10.2.2.2", "1": "10.1.1.1"} -{"10.1.1.1": {"a": 1}, "10.2.2.2": {"b": 2}} -{"10.1.1.1": [1, 2], "10.2.2.2": [3, 5]} -{"1": {"s": "test"}} +{"2":"10.2.2.2","1":"10.1.1.1"} +{"10.1.1.1":{"a":1},"10.2.2.2":{"b":2}} +{"10.1.1.1":[1,2],"10.2.2.2":[3,5]} +{"1":{"s":"test"}} diff --git a/testing/btest/language/attr-default-global-set-error.zeek b/testing/btest/language/attr-default-global-set-error.zeek index 515c71fc24..05ae6b1722 100644 --- a/testing/btest/language/attr-default-global-set-error.zeek +++ b/testing/btest/language/attr-default-global-set-error.zeek @@ -2,3 +2,8 @@ # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out global ss: set[string] &default=0; +global d: count &default = 10 + &default = 9 + &optional + &log + &add_func = function(): count { return 3; }; diff --git a/testing/btest/scripts/base/frameworks/logging/ascii-json-utf8.zeek b/testing/btest/scripts/base/frameworks/logging/ascii-json-utf8.zeek new file mode 100644 index 0000000000..dabe86073d --- /dev/null +++ b/testing/btest/scripts/base/frameworks/logging/ascii-json-utf8.zeek @@ -0,0 +1,59 @@ +# +# @TEST-EXEC: zeek -b %INPUT +# @TEST-EXEC: btest-diff ssh.log +# +# Testing all possible types. + +redef LogAscii::use_json = T; + + +module SSH; + +export { + redef enum Log::ID += { LOG }; + + type Log: record { + s: string; + } &log; +} + +event zeek_init() +{ + Log::create_stream(SSH::LOG, [$columns=Log]); + + # Strings taken from https://stackoverflow.com/a/3886015 + + # Valid ASCII and valid ASCII control characters + Log::write(SSH::LOG, [$s="a"]); + Log::write(SSH::LOG, [$s="\b\f\n\r\t\x00\x15"]); + + # Valid 2 Octet Sequence + Log::write(SSH::LOG, [$s="\xc3\xb1"]); + + # Invalid 2 Octet Sequence + Log::write(SSH::LOG, [$s="\xc3\x28"]); + + # Invalid Sequence Identifier + Log::write(SSH::LOG, [$s="\xa0\xa1"]); + + # Valid 3 Octet Sequence + Log::write(SSH::LOG, [$s="\xe2\x82\xa1"]); + + # Invalid 3 Octet Sequence (in 2nd Octet) + Log::write(SSH::LOG, [$s="\xe2\x28\xa1"]); + + # Invalid 3 Octet Sequence (in 3rd Octet) + Log::write(SSH::LOG, [$s="\xe2\x82\x28"]); + + # Valid 4 Octet Sequence + Log::write(SSH::LOG, [$s="\xf0\x90\x8c\xbc"]); + + # Invalid 4 Octet Sequence (in 2nd Octet) + Log::write(SSH::LOG, [$s="\xf0\x28\x8c\xbc"]); + + # Invalid 4 Octet Sequence (in 3rd Octet) + Log::write(SSH::LOG, [$s="\xf0\x90\x28\xbc"]); + + # Invalid 4 Octet Sequence (in 4th Octet) + Log::write(SSH::LOG, [$s="\xf0\x28\x8c\x28"]); +} diff --git a/testing/btest/scripts/base/frameworks/logging/field-extension-cluster-error.zeek b/testing/btest/scripts/base/frameworks/logging/field-extension-cluster-error.zeek index 6e66d56bb5..6c4f9af6e4 100644 --- a/testing/btest/scripts/base/frameworks/logging/field-extension-cluster-error.zeek +++ b/testing/btest/scripts/base/frameworks/logging/field-extension-cluster-error.zeek @@ -56,7 +56,7 @@ event slow_death() schedule 2sec { die() }; } -event kill_worker() +event ready() { Reporter::info("qux"); Broker::publish("death", slow_death); @@ -69,20 +69,30 @@ event zeek_init() Broker::subscribe("death"); suspend_processing(); } + + if ( Cluster::node == "manager-1" ) + { + Broker::subscribe("ready"); + } + } + +global conn_count = 0; + +event new_connection(c: connection) + { + ++conn_count; + + if ( conn_count == 30 ) + { + Reporter::info("qux"); + Broker::publish("ready", ready); + } } event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) { - if ( Cluster::node == "manager-1" ) - { - schedule 2sec { kill_worker() }; - } - if ( Cluster::node == "worker-1" ) - { continue_processing(); - Reporter::info("qux"); - } } event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) diff --git a/testing/btest/scripts/base/utils/json.test b/testing/btest/scripts/base/utils/json.test index 8d34ed98b1..6e7854b744 100644 --- a/testing/btest/scripts/base/utils/json.test +++ b/testing/btest/scripts/base/utils/json.test @@ -72,6 +72,9 @@ event zeek_init() local e: color = Red; print to_json(e); + local p: pattern = /^abcd/; + print to_json(p); + # ######################### # Test the container types: diff --git a/testing/scripts/diff-remove-timestamps b/testing/scripts/diff-remove-timestamps index 325268d832..8ee28c29b9 100755 --- a/testing/scripts/diff-remove-timestamps +++ b/testing/scripts/diff-remove-timestamps @@ -9,5 +9,5 @@ else sed="sed -E" fi -$sed 's/(0\.000000)|([0-9]{9,10}\.[0-9]{2,8})/XXXXXXXXXX.XXXXXX/g' | \ +$sed 's/(0\.000000)|([0-9]{9,10}\.[0-9]{1,8})/XXXXXXXXXX.XXXXXX/g' | \ $sed 's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g'