diff --git a/.github/workflows/coverity-scan.yml b/.github/workflows/coverity-scan.yml index d983e22256..9946ea9059 100644 --- a/.github/workflows/coverity-scan.yml +++ b/.github/workflows/coverity-scan.yml @@ -67,6 +67,7 @@ jobs: run: | export PATH=`pwd`/coverity-tools/bin:$PATH ( cd build && cov-build --dir cov-int make -j 3 ) + cat build/cov-int/build-log.txt - name: Submit env: diff --git a/CHANGES b/CHANGES index a82c7a446b..aa83e85437 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,59 @@ +3.2.0-dev.561 | 2020-05-21 13:05:39 -0700 + + * GH-983: Fix opaque Broker types lacking a Type after (de)serialization (Jon Siwek, Corelight) + +3.2.0-dev.559 | 2020-05-21 13:04:19 -0700 + + * Make SendEvent callable from all threads + + This commit refactors the SendEvent call and moves it from the Input + ReaderBackend to to MsgThread. This allows all other types of threads + to access this functionality. + + This necessitated a few more changes. Most importantly, one of the + ValueToVal methods was moved over to SerialTypes. Whereit arguably + belongs - there was nothing that was input-framework specific in + that method - and the functionality could come in useful in a number + of cases. (Johanna Amann, Corelight) + +3.2.0-dev.557 | 2020-05-21 11:41:12 -0700 + + * Speed up FuzzBuffer ChunkCount validity check (Justin Azoff, Corelight) + +3.2.0-dev.555 | 2020-05-20 11:19:08 -0700 + + * Disable output of Reporter messages to stderr in fuzz targets (Jon Siwek, Corelight) + +3.2.0-dev.554 | 2020-05-20 10:56:46 -0700 + + * Improve standalone fuzz driver timing output (Jon Siwek, Corelight) + + * Skip fuzz inputs that have more than 64 chunks (Justin Azoff, Corelight) + +3.2.0-dev.550 | 2020-05-19 10:50:42 -0700 + + * Upgrade to latest Broker changes for CAF 0.18 (Dominik Charousset, Corelight) + + * Include pcap.h instead of pcap/dlt.h in packet-fuzzer (Tim Wojtulewicz, Corelight) + +3.2.0-dev.547 | 2020-05-18 10:47:54 -0700 + + * add packet fuzzer (Justin Azoff) + + * Fix building fuzz targets on macOS (Jon Siwek, Corelight) + + * Highwayhash: small build fix (Johanna Amann, Corelight) + + Turns out that hh_neon should not be compiled on generic arm CPUs. + That one is only for aarch64. + +3.2.0-dev.539 | 2020-05-15 19:47:55 +0000 + + * Replace bzero() with memset() (Noah Treuhaft) + + * Change Coverity action to output build log (Jon Siwek, Corelight) + 3.2.0-dev.530 | 2020-05-13 15:05:31 -0700 * GH-906: Fix the regex in url.zeek to better match for find_all_urls (Tim Wojtulewicz, Corelight) diff --git a/VERSION b/VERSION index 560f02389d..47ec391cbf 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.2.0-dev.530 +3.2.0-dev.561 diff --git a/aux/binpac b/aux/binpac index cdc808b07b..4d3d10cd54 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit cdc808b07ba0caf05ed4bd83d906125b10feb695 +Subproject commit 4d3d10cd54b1aa64f30d2fd433252f353c6ea6e0 diff --git a/aux/broker b/aux/broker index d360a228c3..6974924007 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit d360a228c31a6c6d58afdc9827308471c28cd126 +Subproject commit 6974924007765f70d95e5cf123b6256048ae3af7 diff --git a/aux/zeek-aux b/aux/zeek-aux index 43fa1b7f01..8615abced8 160000 --- a/aux/zeek-aux +++ b/aux/zeek-aux @@ -1 +1 @@ -Subproject commit 43fa1b7f01b1d77ccc1d05cc50f1a1c3ea58b6fa +Subproject commit 8615abced86b5559fa3203264be55e664b887094 diff --git a/aux/zeekctl b/aux/zeekctl index 2bd666a0c1..bee11c6392 160000 --- a/aux/zeekctl +++ b/aux/zeekctl @@ -1 +1 @@ -Subproject commit 2bd666a0c13f8efa6dcbc2d1f125ccc73e333522 +Subproject commit bee11c63923b69a4468838d56067f15bc6ae3a52 diff --git a/cmake b/cmake index 2b3cd252b4..23f4b88f91 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 2b3cd252b4ef80b5d96c6f8aaac8d63eee86f5da +Subproject commit 23f4b88f91c537c59ef9a3ad56ec08f021ec2b2c diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f1b80e4b92..e8ccafb544 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -338,9 +338,6 @@ set(HH_SRCS if (${COMPILER_ARCHITECTURE} STREQUAL "arm") set_source_files_properties(${HH_SRCS} PROPERTIES COMPILE_FLAGS -mfloat-abi=hard -march=armv7-a -mfpu=neon) - list(APPEND HH_SRCS - ../aux/highwayhash/highwayhash/hh_neon.cc - ) elseif (${COMPILER_ARCHITECTURE} STREQUAL "aarch64") list(APPEND HH_SRCS ../aux/highwayhash/highwayhash/hh_neon.cc diff --git a/src/Conn.cc b/src/Conn.cc index 76495ddb57..560882dcad 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -80,12 +80,12 @@ Connection::Connection(NetSessions* s, const ConnIDKey& k, double t, const ConnI if ( pkt->l2_src ) memcpy(orig_l2_addr, pkt->l2_src, sizeof(orig_l2_addr)); else - bzero(orig_l2_addr, sizeof(orig_l2_addr)); + memset(orig_l2_addr, 0, sizeof(orig_l2_addr)); if ( pkt->l2_dst ) memcpy(resp_l2_addr, pkt->l2_dst, sizeof(resp_l2_addr)); else - bzero(resp_l2_addr, sizeof(resp_l2_addr)); + memset(resp_l2_addr, 0, sizeof(resp_l2_addr)); vlan = pkt->vlan; inner_vlan = pkt->inner_vlan; diff --git a/src/IntSet.h b/src/IntSet.h index 75bd977bf7..20b1d257d0 100644 --- a/src/IntSet.h +++ b/src/IntSet.h @@ -27,14 +27,11 @@ private: unsigned char* set; }; - -#define bzero(p, size) memset(p, 0, size) - inline IntSet::IntSet(unsigned int n) { size = n / 8 + 1; set = new unsigned char[size]; - bzero(set, size); + memset(set, 0, size); } inline IntSet::~IntSet() @@ -65,5 +62,5 @@ inline bool IntSet::Contains(unsigned int i) const inline void IntSet::Clear() { - bzero(set, size); + memset(set, 0, size); } diff --git a/src/broker/Data.h b/src/broker/Data.h index d511581ba1..fbafa49077 100644 --- a/src/broker/Data.h +++ b/src/broker/Data.h @@ -116,6 +116,7 @@ public: protected: DataVal() + : OpaqueVal(bro_broker::opaque_of_data_type) {} DECLARE_OPAQUE_VALUE(bro_broker::DataVal) @@ -236,7 +237,10 @@ public: broker::set::iterator it; protected: - SetIterator() {} + SetIterator() + : OpaqueVal(bro_broker::opaque_of_set_iterator) + {} + DECLARE_OPAQUE_VALUE(bro_broker::SetIterator) }; @@ -253,7 +257,10 @@ public: broker::table::iterator it; protected: - TableIterator() {} + TableIterator() + : OpaqueVal(bro_broker::opaque_of_table_iterator) + {} + DECLARE_OPAQUE_VALUE(bro_broker::TableIterator) }; @@ -270,7 +277,10 @@ public: broker::vector::iterator it; protected: - VectorIterator() {} + VectorIterator() + : OpaqueVal(bro_broker::opaque_of_vector_iterator) + {} + DECLARE_OPAQUE_VALUE(bro_broker::VectorIterator) }; @@ -287,7 +297,10 @@ public: broker::vector::iterator it; protected: - RecordIterator() {} + RecordIterator() + : OpaqueVal(bro_broker::opaque_of_record_iterator) + {} + DECLARE_OPAQUE_VALUE(bro_broker::RecordIterator) }; diff --git a/src/broker/Store.cc b/src/broker/Store.cc index 6ab0a408f3..7eb22c5948 100644 --- a/src/broker/Store.cc +++ b/src/broker/Store.cc @@ -86,13 +86,13 @@ broker::backend to_backend_type(BifEnum::Broker::BackendType type) { switch ( type ) { case BifEnum::Broker::MEMORY: - return broker::memory; + return broker::backend::memory; case BifEnum::Broker::SQLITE: - return broker::sqlite; + return broker::backend::sqlite; case BifEnum::Broker::ROCKSDB: - return broker::rocksdb; + return broker::backend::rocksdb; } throw std::runtime_error("unknown broker backend"); @@ -102,14 +102,14 @@ broker::backend_options to_backend_options(broker::backend backend, RecordVal* options) { switch ( backend ) { - case broker::sqlite: + case broker::backend::sqlite: { auto path = options->GetField(0)->AsRecordVal() ->GetField(0)->AsStringVal()->CheckString(); return {{"path", path}}; } - case broker::rocksdb: + case broker::backend::rocksdb: { auto path = options->GetField(1)->AsRecordVal() ->GetField(0)->AsStringVal()->CheckString(); diff --git a/src/broker/Store.h b/src/broker/Store.h index 7f329126bb..186bb519ba 100644 --- a/src/broker/Store.h +++ b/src/broker/Store.h @@ -103,7 +103,9 @@ public: broker::store::proxy proxy; protected: - StoreHandleVal() = default; + StoreHandleVal() + : OpaqueVal(bro_broker::opaque_of_store_handle) + {} DECLARE_OPAQUE_VALUE(StoreHandleVal) }; diff --git a/src/fuzzers/CMakeLists.txt b/src/fuzzers/CMakeLists.txt index 97a050d265..ac8b6ca345 100644 --- a/src/fuzzers/CMakeLists.txt +++ b/src/fuzzers/CMakeLists.txt @@ -22,16 +22,26 @@ if ( NOT DEFINED ZEEK_FUZZING_ENGINE AND DEFINED ENV{LIB_FUZZING_ENGINE} ) endif () endif () +# The bind library is handled a bit hack-ishly since it defaults to linking it +# as static library by default on Linux, but at least on one common distro, +# that static library wasn't compiled with -fPIC and so not usable in the +# shared library we're trying to build. So instead, the fuzzer executable, not +# the shared lib, links it. +string(REGEX MATCH ".*\\.a$" _have_static_bind_lib "${BIND_LIBRARY}") + macro(ADD_FUZZ_TARGET _name) set(_fuzz_target zeek-${_name}-fuzzer) set(_fuzz_source ${_name}-fuzzer.cc) add_executable(${_fuzz_target} ${_fuzz_source} ${ARGN}) - target_link_libraries(${_fuzz_target} - zeek_fuzzer_shared - ${BIND_LIBRARY} - ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) + target_link_libraries(${_fuzz_target} zeek_fuzzer_shared) + + if ( _have_static_bind_lib ) + target_link_libraries(${_fuzz_target} ${BIND_LIBRARY}) + endif () + + target_link_libraries(${_fuzz_target} ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) if ( DEFINED ZEEK_FUZZING_ENGINE ) target_link_libraries(${_fuzz_target} ${ZEEK_FUZZING_ENGINE}) @@ -55,12 +65,11 @@ add_library(zeek_fuzzer_shared SHARED set(zeek_fuzzer_shared_deps) foreach(_dep ${zeekdeps} ) - # The bind library is handled a bit hack-ishly since it defaults to - # linking it as static library by default on Linux, but at least - # on one common distro, that static library wasn't compiled with -fPIC - # and so not usable in the shared library we're trying to build. - # So instead, the fuzzer executable, not the shared lib, links it. - if ( NOT "${_dep}" STREQUAL "${BIND_LIBRARY}" ) + if ( "${_dep}" STREQUAL "${BIND_LIBRARY}" ) + if ( NOT _have_static_bind_lib ) + set(zeek_fuzzer_shared_deps ${zeek_fuzzer_shared_deps} ${_dep}) + endif () + else () set(zeek_fuzzer_shared_deps ${zeek_fuzzer_shared_deps} ${_dep}) endif () endforeach () @@ -70,3 +79,4 @@ target_link_libraries(zeek_fuzzer_shared ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) add_fuzz_target(pop3) +add_fuzz_target(packet) diff --git a/src/fuzzers/FuzzBuffer.cc b/src/fuzzers/FuzzBuffer.cc index ebae1a1db1..def5d34086 100644 --- a/src/fuzzers/FuzzBuffer.cc +++ b/src/fuzzers/FuzzBuffer.cc @@ -6,7 +6,7 @@ #include "FuzzBuffer.h" -bool zeek::detail::FuzzBuffer::Valid() const +bool zeek::detail::FuzzBuffer::Valid(int chunk_count_limit) const { if ( end - begin < PKT_MAGIC_LEN + 2 ) return false; @@ -14,9 +14,31 @@ bool zeek::detail::FuzzBuffer::Valid() const if ( memcmp(begin, PKT_MAGIC, PKT_MAGIC_LEN) != 0) return false; + if ( ExceedsChunkLimit(chunk_count_limit) ) + return false; + return true; } +int zeek::detail::FuzzBuffer::ChunkCount(int chunk_count_limit) const + { + auto pos = begin; + int chunks = 0; + + while ( pos < end && (chunks < chunk_count_limit || chunk_count_limit == 0) ) + { + pos = (const unsigned char*)memmem(pos, end - pos, + PKT_MAGIC, PKT_MAGIC_LEN); + if ( ! pos ) + break; + + pos += PKT_MAGIC_LEN + 1; + chunks++; + } + + return chunks; + } + std::optional zeek::detail::FuzzBuffer::Next() { if ( begin == end ) diff --git a/src/fuzzers/FuzzBuffer.h b/src/fuzzers/FuzzBuffer.h index 1b9cdb58a3..69ba821242 100644 --- a/src/fuzzers/FuzzBuffer.h +++ b/src/fuzzers/FuzzBuffer.h @@ -27,6 +27,7 @@ public: static constexpr int PKT_MAGIC_LEN = 4; static constexpr unsigned char PKT_MAGIC[PKT_MAGIC_LEN + 1] = "\1PKT"; + static constexpr int MAX_CHUNK_COUNT = 64; /** * Initialize fuzz buffer. @@ -39,9 +40,25 @@ public: /** * @return whether the fuzz buffer object is valid -- has enough bytes - * to Deliver to an analyzer and starts with a *PKT_MAGIC* bytestring. + * to Deliver to an analyzer, starts with a *PKT_MAGIC* bytestring, and + * contains less than the limiting number of chunk. + * . */ - bool Valid() const; + bool Valid(int chunk_count_limit = MAX_CHUNK_COUNT) const; + + /** + * @param chunk_count_limit Number of chunks to stop counting at (zero + * means "never stop"). + * @return the number of chunks in the fuzz buffer object + */ + int ChunkCount(int chunk_count_limit = 0) const; + + /** + * @param Maximum number of chunks to permit the FuzzBuffer to have. + * @return Whether the FuzzBuffer exceeds the desired chunk count limit. + */ + bool ExceedsChunkLimit(int chunk_count_limit) const + { return ChunkCount(chunk_count_limit + 1) > chunk_count_limit; } /** * @return the next chunk to deliver, if one could be extracted diff --git a/src/fuzzers/fuzzer-setup.h b/src/fuzzers/fuzzer-setup.h index e692ddd3c4..976aa9126e 100644 --- a/src/fuzzers/fuzzer-setup.h +++ b/src/fuzzers/fuzzer-setup.h @@ -32,6 +32,9 @@ extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv) options.scripts_to_load.emplace_back("local.zeek"); options.script_options_to_set.emplace_back("Site::local_nets={10.0.0.0/8}"); options.script_options_to_set.emplace_back("Log::default_writer=Log::WRITER_NONE"); + options.script_options_to_set.emplace_back("Reporter::info_to_stderr=F"); + options.script_options_to_set.emplace_back("Reporter::warnings_to_stderr=F"); + options.script_options_to_set.emplace_back("Reporter::errors_to_stderr=F"); options.deterministic_mode = true; options.ignore_checksums = true; options.abort_on_scripting_errors = true; diff --git a/src/fuzzers/packet-corpus.zip b/src/fuzzers/packet-corpus.zip new file mode 100644 index 0000000000..8ffc9d4162 Binary files /dev/null and b/src/fuzzers/packet-corpus.zip differ diff --git a/src/fuzzers/packet-fuzzer.cc b/src/fuzzers/packet-fuzzer.cc new file mode 100644 index 0000000000..d3bb28b806 --- /dev/null +++ b/src/fuzzers/packet-fuzzer.cc @@ -0,0 +1,47 @@ +#include "binpac.h" + +#include "iosource/Packet.h" +#include "Event.h" +#include "Sessions.h" + +#include "FuzzBuffer.h" +#include "fuzzer-setup.h" + +extern "C" { +#include +} + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) + { + zeek::detail::FuzzBuffer fb{data, size}; + + if ( ! fb.Valid() ) + return 0; + + for ( ; ; ) + { + auto chunk = fb.Next(); + + if ( ! chunk ) + break; + + Packet pkt; + auto timestamp = 42; + pkt_timeval ts = {timestamp, 0}; + pkt.Init(DLT_RAW, &ts, chunk->size, chunk->size, chunk->data.get(), false, ""); + + try + { + sessions->NextPacket(timestamp, &pkt); + } + catch ( binpac::Exception const &e ) + { + } + + chunk = {}; + mgr.Drain(); + } + + zeek::detail::fuzzer_cleanup_one_input(); + return 0; + } diff --git a/src/fuzzers/standalone-driver.cc b/src/fuzzers/standalone-driver.cc index 69ca7202cf..64cabaefc8 100644 --- a/src/fuzzers/standalone-driver.cc +++ b/src/fuzzers/standalone-driver.cc @@ -17,6 +17,7 @@ int main(int argc, char** argv) printf("Standalone fuzzer processing %d inputs\n", num_inputs); LLVMFuzzerInitialize(&argc, &argv); + auto fuzz_start = high_resolution_clock::now(); for ( auto i = 0; i < num_inputs; ++i ) { @@ -60,5 +61,7 @@ int main(int argc, char** argv) auto agg_stop = high_resolution_clock::now(); auto agg_dt = duration(agg_stop - agg_start).count(); - printf("Processed %d inputs in %fs\n", num_inputs, agg_dt); + auto fuzz_dt = duration(agg_stop - fuzz_start).count(); + printf("Processed %d inputs in %fs (%fs w/ initialization), avg = %fs\n", + num_inputs, fuzz_dt, agg_dt, fuzz_dt / num_inputs); } diff --git a/src/input/Manager.cc b/src/input/Manager.cc index c2d2f54b88..532f9f9e03 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -26,14 +26,6 @@ using namespace std; using threading::Value; using threading::Field; -static void delete_value_ptr_array(Value** vals, int num_fields) - { - for ( int i = 0; i < num_fields; ++i ) - delete vals[i]; - - delete [] vals; - } - /** * InputHashes are used as Dictionaries to store the value and index hashes * for all lines currently stored in a table. Index hash is stored as @@ -1091,7 +1083,7 @@ void Manager::SendEntry(ReaderFrontend* reader, Value* *vals) else assert(false); - delete_value_ptr_array(vals, readFields); + Value::delete_value_ptr_array(vals, readFields); } int Manager::SendEntryTable(Stream* i, const Value* const *vals) @@ -1458,7 +1450,7 @@ void Manager::Put(ReaderFrontend* reader, Value* *vals) else assert(false); - delete_value_ptr_array(vals, readFields); + Value::delete_value_ptr_array(vals, readFields); } int Manager::SendEventStreamEvent(Stream* i, EnumVal* type, const Value* const *vals) @@ -1762,7 +1754,7 @@ bool Manager::Delete(ReaderFrontend* reader, Value* *vals) return false; } - delete_value_ptr_array(vals, readVals); + Value::delete_value_ptr_array(vals, readVals); return success; } @@ -1787,68 +1779,6 @@ bool Manager::CallPred(Func* pred_func, const int numvals, ...) const return result; } -// Raise everything in here as warnings so it is passed to scriptland without -// looking "fatal". In addition to these warnings, ReaderBackend will queue -// one reporter message. -bool Manager::SendEvent(ReaderFrontend* reader, const string& name, const int num_vals, Value* *vals) const - { - Stream *i = FindStream(reader); - if ( i == nullptr ) - { - reporter->InternalWarning("Unknown reader %s in SendEvent for event %s", reader->Name(), name.c_str()); - delete_value_ptr_array(vals, num_vals); - return false; - } - - EventHandler* handler = event_registry->Lookup(name); - if ( handler == nullptr ) - { - Warning(i, "Event %s not found", name.c_str()); - delete_value_ptr_array(vals, num_vals); - return false; - } - -#ifdef DEBUG - DBG_LOG(DBG_INPUT, "SendEvent for event %s with %d vals", - name.c_str(), num_vals); -#endif - - const auto& type = handler->GetType()->Params(); - int num_event_vals = type->NumFields(); - if ( num_vals != num_event_vals ) - { - Warning(i, "Wrong number of values for event %s", name.c_str()); - delete_value_ptr_array(vals, num_vals); - return false; - } - - bool convert_error = false; - - zeek::Args vl; - vl.reserve(num_vals); - - for ( int j = 0; j < num_vals; j++) - { - Val* v = ValueToVal(i, vals[j], convert_error); - vl.emplace_back(AdoptRef{}, v); - - if ( v && ! convert_error && ! same_type(type->GetFieldType(j).get(), v->GetType().get()) ) - { - convert_error = true; - type->GetFieldType(j)->Error("SendEvent types do not match", v->GetType().get()); - } - } - - delete_value_ptr_array(vals, num_vals); - - if ( convert_error ) - return false; - else if ( handler ) - mgr.Enqueue(handler, std::move(vl), SOURCE_LOCAL); - - return true; -} - void Manager::SendEvent(EventHandlerPtr ev, const int numvals, ...) const { zeek::Args vl; @@ -2228,9 +2158,9 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) c } // convert threading value to Bro value -// have_error is a reference to a boolean which is set to true as soon as an error occured. +// have_error is a reference to a boolean which is set to true as soon as an error occurs. // When have_error is set to true at the beginning of the function, it is assumed that -// an error already occured in the past and processing is aborted. +// an error already occurred in the past and processing is aborted. Val* Manager::ValueToVal(const Stream* i, const Value* val, BroType* request_type, bool& have_error) const { if ( have_error ) @@ -2383,204 +2313,6 @@ Val* Manager::ValueToVal(const Stream* i, const Value* val, BroType* request_typ return nullptr; } -Val* Manager::ValueToVal(const Stream* i, const Value* val, bool& have_error) const - { - if ( have_error ) - return nullptr; - - if ( ! val->present ) - return nullptr; // unset field - - switch ( val->type ) { - case TYPE_BOOL: - return val_mgr->Bool(val->val.int_val)->Ref(); - - case TYPE_INT: - return val_mgr->Int(val->val.int_val).release(); - - case TYPE_COUNT: - case TYPE_COUNTER: - return val_mgr->Count(val->val.int_val).release(); - - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - return new Val(val->val.double_val, val->type); - - case TYPE_STRING: - { - BroString *s = new BroString((const u_char*)val->val.string_val.data, val->val.string_val.length, true); - return new StringVal(s); - } - - case TYPE_PORT: - return val_mgr->Port(val->val.port_val.port, val->val.port_val.proto)->Ref(); - - case TYPE_ADDR: - { - IPAddr* addr = nullptr; - switch ( val->val.addr_val.family ) { - case IPv4: - addr = new IPAddr(val->val.addr_val.in.in4); - break; - - case IPv6: - addr = new IPAddr(val->val.addr_val.in.in6); - break; - - default: - assert(false); - } - - AddrVal* addrval = new AddrVal(*addr); - delete addr; - return addrval; - } - - case TYPE_SUBNET: - { - IPAddr* addr = nullptr; - switch ( val->val.subnet_val.prefix.family ) { - case IPv4: - addr = new IPAddr(val->val.subnet_val.prefix.in.in4); - break; - - case IPv6: - addr = new IPAddr(val->val.subnet_val.prefix.in.in6); - break; - - default: - assert(false); - } - - SubNetVal* subnetval = new SubNetVal(*addr, val->val.subnet_val.length); - delete addr; - return subnetval; - } - - case TYPE_PATTERN: - { - RE_Matcher* re = new RE_Matcher(val->val.pattern_text_val); - re->Compile(); - return new PatternVal(re); - } - - case TYPE_TABLE: - { - IntrusivePtr set_index; - if ( val->val.set_val.size == 0 && val->subtype == TYPE_VOID ) - // don't know type - unspecified table. - set_index = make_intrusive(); - else - { - // all entries have to have the same type... - TypeTag stag = val->subtype; - if ( stag == TYPE_VOID ) - TypeTag stag = val->val.set_val.vals[0]->type; - - IntrusivePtr index_type; - - if ( stag == TYPE_ENUM ) - { - // Enums are not a base-type, so need to look it up. - const auto& sv = val->val.set_val.vals[0]->val.string_val; - std::string enum_name(sv.data, sv.length); - const auto& enum_id = global_scope()->Find(enum_name); - - if ( ! enum_id ) - { - Warning(i, "Value '%s' for stream '%s' is not a valid enum.", - enum_name.data(), i->name.c_str()); - - have_error = true; - return nullptr; - } - - index_type = enum_id->GetType(); - } - else - index_type = base_type(stag); - - set_index = make_intrusive(index_type); - set_index->Append(std::move(index_type)); - } - - auto s = make_intrusive(std::move(set_index), nullptr); - TableVal* t = new TableVal(std::move(s)); - for ( int j = 0; j < val->val.set_val.size; j++ ) - { - Val* assignval = ValueToVal(i, val->val.set_val.vals[j], have_error); - - t->Assign({AdoptRef{}, assignval}, nullptr); - } - - return t; - } - - case TYPE_VECTOR: - { - IntrusivePtr type; - - if ( val->val.vector_val.size == 0 && val->subtype == TYPE_VOID ) - // don't know type - unspecified table. - type = base_type(TYPE_ANY); - else - { - // all entries have to have the same type... - if ( val->subtype == TYPE_VOID ) - type = base_type(val->val.vector_val.vals[0]->type); - else - type = base_type(val->subtype); - } - - auto vt = make_intrusive(std::move(type)); - auto v = make_intrusive(std::move(vt)); - - for ( int j = 0; j < val->val.vector_val.size; j++ ) - v->Assign(j, ValueToVal(i, val->val.vector_val.vals[j], have_error)); - - return v.release(); - } - - case TYPE_ENUM: { - // Convert to string first to not have to deal with missing - // \0's... - string enum_string(val->val.string_val.data, val->val.string_val.length); - - // let's try looking it up by global ID. - auto id = lookup_ID(enum_string.c_str(), GLOBAL_MODULE_NAME); - if ( ! id || ! id->IsEnumConst() ) - { - Warning(i, "Value '%s' for stream '%s' is not a valid enum.", - enum_string.c_str(), i->name.c_str()); - - have_error = true; - return nullptr; - } - - EnumType* t = id->GetType()->AsEnumType(); - int intval = t->Lookup(id->ModuleName(), id->Name()); - if ( intval < 0 ) - { - Warning(i, "Enum value '%s' for stream '%s' not found.", - enum_string.c_str(), i->name.c_str()); - - have_error = true; - return nullptr; - } - - auto rval = t->GetVal(intval); - return rval.release(); - } - - default: - reporter->InternalError("Unsupported type for input_read in stream %s", i->name.c_str()); - } - - assert(false); - return nullptr; - } - Manager::Stream* Manager::FindStream(const string &name) const { for ( auto s = readers.begin(); s != readers.end(); ++s ) diff --git a/src/input/Manager.h b/src/input/Manager.h index 191692f9e6..c24b930d64 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -118,7 +118,6 @@ protected: friend class PutMessage; friend class DeleteMessage; friend class ClearMessage; - friend class SendEventMessage; friend class SendEntryMessage; friend class EndCurrentSendMessage; friend class ReaderClosedMessage; @@ -142,11 +141,6 @@ protected: void SendEntry(ReaderFrontend* reader, threading::Value* *vals); void EndCurrentSend(ReaderFrontend* reader); - // Allows readers to directly send Bro events. The num_vals and vals - // must be the same the named event expects. Takes ownership of - // threading::Value fields. - bool SendEvent(ReaderFrontend* reader, const std::string& name, const int num_vals, threading::Value* *vals) const; - // Instantiates a new ReaderBackend of the given type (note that // doing so creates a new thread!). ReaderBackend* CreateBackend(ReaderFrontend* frontend, EnumVal* tag); @@ -227,14 +221,9 @@ private: // startpos. int CopyValue(char *data, const int startpos, const threading::Value* val) const; - // Convert Threading::Value to an internal Bro Type (works also with - // Records). + // Convert Threading::Value to an internal Bro Type (works with Records). Val* ValueToVal(const Stream* i, const threading::Value* val, BroType* request_type, bool& have_error) const; - // Convert Threading::Value to an internal Bro type just using the information given in the threading::Value. - // This allows more flexibility, especially given structures in script-land that contain any types. - Val* ValueToVal(const Stream* i, const threading::Value* val, bool& have_error) const; - // Convert Threading::Value to an internal Bro list type. Val* ValueToIndexVal(const Stream* i, int num_fields, const RecordType* type, const threading::Value* const *vals, bool& have_error) const; diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index 90050a6438..7e0fbd7685 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -54,30 +54,6 @@ public: private: }; -class SendEventMessage final : public threading::OutputMessage { -public: - SendEventMessage(ReaderFrontend* reader, const char* name, const int num_vals, Value* *val) - : threading::OutputMessage("SendEvent", reader), - name(copy_string(name)), num_vals(num_vals), val(val) {} - - ~SendEventMessage() override { delete [] name; } - - bool Process() override - { - bool success = input_mgr->SendEvent(Object(), name, num_vals, val); - - if ( ! success ) - reporter->Error("SendEvent for event %s failed", name); - - return true; // We do not want to die if sendEvent fails because the event did not return. - } - -private: - const char* name; - const int num_vals; - Value* *val; -}; - class ReaderErrorMessage final : public threading::OutputMessage { public: @@ -229,11 +205,6 @@ void ReaderBackend::Clear() SendOut(new ClearMessage(frontend)); } -void ReaderBackend::SendEvent(const char* name, const int num_vals, Value* *vals) - { - SendOut(new SendEventMessage(frontend, name, num_vals, vals)); - } - void ReaderBackend::EndCurrentSend() { SendOut(new EndCurrentSendMessage(frontend)); diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index d158d0972c..2748d1e2d2 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -280,18 +280,6 @@ protected: */ virtual bool DoHeartbeat(double network_time, double current_time) = 0; - /** - * Method allowing a reader to send a specified Bro event. Vals must - * match the values expected by the bro event. - * - * @param name name of the bro event to send - * - * @param num_vals number of entries in \a vals - * - * @param vals the values to be given to the event - */ - void SendEvent(const char* name, const int num_vals, threading::Value* *vals); - // Content-sending-functions (simple mode). Include table-specific // functionality that simply is not used if we have no table. diff --git a/src/threading/BasicThread.h b/src/threading/BasicThread.h index fa1e097536..fa65e070ee 100644 --- a/src/threading/BasicThread.h +++ b/src/threading/BasicThread.h @@ -37,7 +37,7 @@ public: /** * Returns a descriptive name for the thread. If not set via - * SetName(). If not set, a default name is choosen automatically. + * SetName(), a default name is chosen automatically. * * This method is safe to call from any thread. */ diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index d7d3676f95..2c1491de1a 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -5,6 +5,7 @@ #include "NetVar.h" #include "iosource/Manager.h" +#include "Event.h" using namespace threading; @@ -128,6 +129,60 @@ void Manager::StartHeartbeatTimer() timer_mgr->Add(new HeartbeatTimer(network_time + zeek::BifConst::Threading::heartbeat_interval)); } +// Raise everything in here as warnings so it is passed to scriptland without +// looking "fatal". In addition to these warnings, ReaderBackend will queue +// one reporter message. +bool Manager::SendEvent(MsgThread* thread, const std::string& name, const int num_vals, Value* *vals) const + { + EventHandler* handler = event_registry->Lookup(name); + if ( handler == nullptr ) + { + reporter->Warning("Thread %s: Event %s not found", thread->Name(), name.c_str()); + Value::delete_value_ptr_array(vals, num_vals); + return false; + } + +#ifdef DEBUG + DBG_LOG(DBG_INPUT, "Thread %s: SendEvent for event %s with %d vals", + thread->Name(), name.c_str(), num_vals); +#endif + + const auto& type = handler->GetType()->Params(); + int num_event_vals = type->NumFields(); + if ( num_vals != num_event_vals ) + { + reporter->Warning("Thread %s: Wrong number of values for event %s", thread->Name(), name.c_str()); + Value::delete_value_ptr_array(vals, num_vals); + return false; + } + + bool convert_error = false; + + zeek::Args vl; + vl.reserve(num_vals); + + for ( int j = 0; j < num_vals; j++) + { + Val* v = Value::ValueToVal(std::string("thread ") + thread->Name(), vals[j], convert_error); + vl.emplace_back(AdoptRef{}, v); + + if ( v && ! convert_error && ! same_type(type->GetFieldType(j).get(), v->GetType().get()) ) + { + convert_error = true; + type->GetFieldType(j)->Error("SendEvent types do not match", v->GetType().get()); + } + } + + Value::delete_value_ptr_array(vals, num_vals); + + if ( convert_error ) + return false; + else if ( handler ) + mgr.Enqueue(handler, std::move(vl), SOURCE_LOCAL); + + return true; + } + void Manager::Flush() { bool do_beat = false; diff --git a/src/threading/Manager.h b/src/threading/Manager.h index 2c5b05d9b5..6429baf8a4 100644 --- a/src/threading/Manager.h +++ b/src/threading/Manager.h @@ -88,6 +88,18 @@ public: */ void KillThreads(); + /** + * Allows threads to directly send Zeek events. The num_vals and vals must be + * the same the named event expects. Takes ownership of threading::Value fields. + * + * @param thread Thread raising the event + * @param name Name of event being raised + * @param num_vals Number of values passed to the event + * @param vals Values passed to the event + * @returns True on success false on failure. + */ + bool SendEvent(MsgThread* thread, const std::string& name, const int num_vals, threading::Value* *vals) const; + protected: friend class BasicThread; friend class MsgThread; diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc index 3d2fa66271..f1fd2897d2 100644 --- a/src/threading/MsgThread.cc +++ b/src/threading/MsgThread.cc @@ -126,6 +126,31 @@ private: } +// An event that the child wants to pass into the main event queue +class SendEventMessage final : public OutputMessage { +public: + SendEventMessage(MsgThread* thread, const char* name, const int num_vals, Value* *val) + : OutputMessage("SendEvent", thread), + name(copy_string(name)), num_vals(num_vals), val(val) {} + + ~SendEventMessage() override { delete [] name; } + + bool Process() override + { + bool success = thread_mgr->SendEvent(Object(), name, num_vals, val); + + if ( ! success ) + reporter->Error("SendEvent for event %s failed", name); + + return true; // We do not want to die if sendEvent fails because the event did not return. + } + +private: + const char* name; + const int num_vals; + Value* *val; +}; + ////// Methods. Message::~Message() @@ -363,6 +388,11 @@ void MsgThread::SendOut(BasicOutputMessage* msg, bool force) flare.Fire(); } +void MsgThread::SendEvent(const char* name, const int num_vals, Value* *vals) + { + SendOut(new SendEventMessage(this, name, num_vals, vals)); + } + BasicOutputMessage* MsgThread::RetrieveOut() { BasicOutputMessage* msg = queue_out.Get(); diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h index e5c0e3a910..8bda2d40b8 100644 --- a/src/threading/MsgThread.h +++ b/src/threading/MsgThread.h @@ -60,6 +60,18 @@ public: */ void SendOut(BasicOutputMessage* msg) { return SendOut(msg, false); } + /** + * Allows the child thread to send a specified Zeek event. The given Vals + * must match the values expected by the event. + * + * @param name name of the bro event to send + * + * @param num_vals number of entries in \a vals + * + * @param vals the values to be given to the event + */ + void SendEvent(const char* name, const int num_vals, threading::Value* *vals); + /** * Reports an informational message from the child thread. The main * thread will pass this to the Reporter once received. @@ -393,7 +405,7 @@ protected: }; /** - * A paremeterized InputMessage that stores a pointer to an argument object. + * A parameterized InputMessage that stores a pointer to an argument object. * Normally, the objects will be used from the Process() callback. */ template diff --git a/src/threading/SerialTypes.cc b/src/threading/SerialTypes.cc index 189b40798a..1e48e78c17 100644 --- a/src/threading/SerialTypes.cc +++ b/src/threading/SerialTypes.cc @@ -4,6 +4,14 @@ #include "SerialTypes.h" #include "SerializationFormat.h" #include "Reporter.h" +// The following are required for ValueToVal. +#include "Val.h" +#include "BroString.h" +#include "RE.h" +#include "module_util.h" +#include "ID.h" +#include "Expr.h" +#include "Scope.h" using namespace threading; @@ -346,7 +354,6 @@ bool Value::Write(SerializationFormat* fmt) const case IPv6: return fmt->Write((char)6, "addr-family") && fmt->Write(val.addr_val.in.in6, "addr-in6"); - break; } // Can't be reached. @@ -366,7 +373,6 @@ bool Value::Write(SerializationFormat* fmt) const case IPv6: return fmt->Write((char)6, "subnet-family") && fmt->Write(val.subnet_val.prefix.in.in6, "subnet-in6"); - break; } // Can't be reached. @@ -417,5 +423,212 @@ bool Value::Write(SerializationFormat* fmt) const type_name(type)); } + // unreachable return false; } + +void Value::delete_value_ptr_array(Value** vals, int num_fields) + { + for ( int i = 0; i < num_fields; ++i ) + delete vals[i]; + + delete [] vals; + } + +Val* Value::ValueToVal(const std::string& source, const Value* val, bool& have_error) + { + if ( have_error ) + return nullptr; + + if ( ! val->present ) + return nullptr; // unset field + + switch ( val->type ) { + case TYPE_BOOL: + return val_mgr->Bool(val->val.int_val)->Ref(); + + case TYPE_INT: + return val_mgr->Int(val->val.int_val).release(); + + case TYPE_COUNT: + case TYPE_COUNTER: + return val_mgr->Count(val->val.int_val).release(); + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + return new Val(val->val.double_val, val->type); + + case TYPE_STRING: + { + BroString *s = new BroString((const u_char*)val->val.string_val.data, val->val.string_val.length, true); + return new StringVal(s); + } + + case TYPE_PORT: + return val_mgr->Port(val->val.port_val.port, val->val.port_val.proto)->Ref(); + + case TYPE_ADDR: + { + IPAddr* addr = nullptr; + switch ( val->val.addr_val.family ) { + case IPv4: + addr = new IPAddr(val->val.addr_val.in.in4); + break; + + case IPv6: + addr = new IPAddr(val->val.addr_val.in.in6); + break; + + default: + assert(false); + } + + AddrVal* addrval = new AddrVal(*addr); + delete addr; + return addrval; + } + + case TYPE_SUBNET: + { + IPAddr* addr = nullptr; + switch ( val->val.subnet_val.prefix.family ) { + case IPv4: + addr = new IPAddr(val->val.subnet_val.prefix.in.in4); + break; + + case IPv6: + addr = new IPAddr(val->val.subnet_val.prefix.in.in6); + break; + + default: + assert(false); + } + + SubNetVal* subnetval = new SubNetVal(*addr, val->val.subnet_val.length); + delete addr; + return subnetval; + } + + case TYPE_PATTERN: + { + RE_Matcher* re = new RE_Matcher(val->val.pattern_text_val); + re->Compile(); + return new PatternVal(re); + } + + case TYPE_TABLE: + { + IntrusivePtr set_index; + if ( val->val.set_val.size == 0 && val->subtype == TYPE_VOID ) + // don't know type - unspecified table. + set_index = make_intrusive(); + else + { + // all entries have to have the same type... + TypeTag stag = val->subtype; + if ( stag == TYPE_VOID ) + TypeTag stag = val->val.set_val.vals[0]->type; + + IntrusivePtr index_type; + + if ( stag == TYPE_ENUM ) + { + // Enums are not a base-type, so need to look it up. + const auto& sv = val->val.set_val.vals[0]->val.string_val; + std::string enum_name(sv.data, sv.length); + const auto& enum_id = global_scope()->Find(enum_name); + + if ( ! enum_id ) + { + reporter->Warning("Value '%s' of source '%s' is not a valid enum.", + enum_name.data(), source.c_str()); + + have_error = true; + return nullptr; + } + + index_type = enum_id->GetType(); + } + else + index_type = base_type(stag); + + set_index = make_intrusive(index_type); + set_index->Append(std::move(index_type)); + } + + auto s = make_intrusive(std::move(set_index), nullptr); + TableVal* t = new TableVal(std::move(s)); + for ( int j = 0; j < val->val.set_val.size; j++ ) + { + Val* assignval = ValueToVal(source, val->val.set_val.vals[j], have_error); + t->Assign({AdoptRef{}, assignval}, nullptr); + } + + return t; + } + + case TYPE_VECTOR: + { + IntrusivePtr type; + + if ( val->val.vector_val.size == 0 && val->subtype == TYPE_VOID ) + // don't know type - unspecified table. + type = base_type(TYPE_ANY); + else + { + // all entries have to have the same type... + if ( val->subtype == TYPE_VOID ) + type = base_type(val->val.vector_val.vals[0]->type); + else + type = base_type(val->subtype); + } + + auto vt = make_intrusive(std::move(type)); + auto v = make_intrusive(std::move(vt)); + + for ( int j = 0; j < val->val.vector_val.size; j++ ) + v->Assign(j, ValueToVal(source, val->val.vector_val.vals[j], have_error)); + + return v.release(); + } + + case TYPE_ENUM: { + // Convert to string first to not have to deal with missing + // \0's... + std::string enum_string(val->val.string_val.data, val->val.string_val.length); + + // let's try looking it up by global ID. + const auto& id = lookup_ID(enum_string.c_str(), GLOBAL_MODULE_NAME); + + if ( ! id || ! id->IsEnumConst() ) + { + reporter->Warning("Value '%s' for source '%s' is not a valid enum.", + enum_string.c_str(), source.c_str()); + + have_error = true; + return nullptr; + } + + EnumType* t = id->GetType()->AsEnumType(); + int intval = t->Lookup(id->ModuleName(), id->Name()); + if ( intval < 0 ) + { + reporter->Warning("Enum value '%s' for source '%s' not found.", + enum_string.c_str(), source.c_str()); + + have_error = true; + return nullptr; + } + + auto rval = t->GetVal(intval); + return rval.release(); + } + + default: + reporter->InternalError("Unsupported type in SerialTypes::ValueToVal from source %s", source.c_str()); + } + + assert(false); + return nullptr; + } diff --git a/src/threading/SerialTypes.h b/src/threading/SerialTypes.h index f424eb9143..0969e5b6db 100644 --- a/src/threading/SerialTypes.h +++ b/src/threading/SerialTypes.h @@ -187,6 +187,26 @@ struct Value { * method is thread-safe. */ static bool IsCompatibleType(BroType* t, bool atomic_only=false); + /** + * Convenience function to delete an array of value pointers. + * @param vals Array of values + * @param num_fields Number of members + */ + static void delete_value_ptr_array(Value** vals, int num_fields); + + /** + * Convert threading::Value to an internal Zeek type, just using the information given in the threading::Value. + * + * @param source Name of the source of this threading value. This is used for warnings that are raised + * in case an error occurs. + * @param val Threading Value to convert to a Zeek Val. + * @param have_error Reference to a boolean. This should be set to false when passed in and is set to true + * in case an error occurs. If this is set to false when the function is called, the function + * immediately aborts. + * @return Val representation of the threading::Value. nullptr on error. + */ + static Val* ValueToVal(const std::string& source, const threading::Value* val, bool& have_error); + private: friend class ::IPAddr; Value(const Value& other) { } // Disabled. diff --git a/testing/btest/Baseline/broker.opaque/out b/testing/btest/Baseline/broker.opaque/out index 35bf821c47..b4293c0100 100644 --- a/testing/btest/Baseline/broker.opaque/out +++ b/testing/btest/Baseline/broker.opaque/out @@ -1,53 +1,62 @@ ============ Topk +opaque of topk [b, a, c] [b, a, c] ============ HLL 3.000069 +opaque of cardinality 3.000069 3.000069 ============ Bloom 0 1 +opaque of bloomfilter 0 1 ============ Hashes +opaque of md5 5b9164ad6f496d9dee12ec7634ce253f 5b9164ad6f496d9dee12ec7634ce253f +opaque of sha1 30ae97492ce1da88d0e7117ace0a60a6f9e1e0bc 30ae97492ce1da88d0e7117ace0a60a6f9e1e0bc +opaque of sha256 25b6746d5172ed6352966a013d93ac846e1110d5a25e8f183b5931f4688842a1 25b6746d5172ed6352966a013d93ac846e1110d5a25e8f183b5931f4688842a1 ============ X509 +opaque of x509 [version=3, serial=040000000001154B5AC394, subject=CN=GlobalSign Root CA,OU=Root CA,O=GlobalSign nv-sa,C=BE, issuer=CN=GlobalSign Root CA,OU=Root CA,O=GlobalSign nv-sa,C=BE, cn=GlobalSign Root CA, not_valid_before=904651200.0, not_valid_after=1832673600.0, key_alg=rsaEncryption, sig_alg=sha1WithRSAEncryption, key_type=rsa, key_length=2048, exponent=65537, curve=] [version=3, serial=040000000001154B5AC394, subject=CN=GlobalSign Root CA,OU=Root CA,O=GlobalSign nv-sa,C=BE, issuer=CN=GlobalSign Root CA,OU=Root CA,O=GlobalSign nv-sa,C=BE, cn=GlobalSign Root CA, not_valid_before=904651200.0, not_valid_after=1832673600.0, key_alg=rsaEncryption, sig_alg=sha1WithRSAEncryption, key_type=rsa, key_length=2048, exponent=65537, curve=] ============ Entropy +opaque of entropy [entropy=4.715374, chi_square=591.981818, mean=75.472727, monte_carlo_pi=4.0, serial_correlation=-0.11027] [entropy=4.715374, chi_square=591.981818, mean=75.472727, monte_carlo_pi=4.0, serial_correlation=-0.11027] ============ broker::Data +opaque of Broker::Data broker::data{{hi, there}} broker::data{{hi, there}} T ============ broker::Set -| [data=broker::data{!}] | [data=broker::data{!}] +| [data=broker::data{!}] | [data=broker::data{!}] | opaque of Broker::SetIterator > [data=broker::data{hi}] | [data=broker::data{hi}] -| [data=broker::data{hi}] | [data=broker::data{hi}] +| [data=broker::data{hi}] | [data=broker::data{hi}] | opaque of Broker::SetIterator > [data=broker::data{there}] | [data=broker::data{there}] -| [data=broker::data{there}] | [data=broker::data{there}] +| [data=broker::data{there}] | [data=broker::data{there}] | opaque of Broker::SetIterator ============ broker::Table -| [key=[data=broker::data{!}], val=[data=broker::data{30}]] | [key=[data=broker::data{!}], val=[data=broker::data{30}]] +| [key=[data=broker::data{!}], val=[data=broker::data{30}]] | [key=[data=broker::data{!}], val=[data=broker::data{30}]] | opaque of Broker::TableIterator > [key=[data=broker::data{hi}], val=[data=broker::data{10}]] | [key=[data=broker::data{hi}], val=[data=broker::data{10}]] -| [key=[data=broker::data{hi}], val=[data=broker::data{10}]] | [key=[data=broker::data{hi}], val=[data=broker::data{10}]] +| [key=[data=broker::data{hi}], val=[data=broker::data{10}]] | [key=[data=broker::data{hi}], val=[data=broker::data{10}]] | opaque of Broker::TableIterator > [key=[data=broker::data{there}], val=[data=broker::data{20}]] | [key=[data=broker::data{there}], val=[data=broker::data{20}]] -| [key=[data=broker::data{there}], val=[data=broker::data{20}]] | [key=[data=broker::data{there}], val=[data=broker::data{20}]] +| [key=[data=broker::data{there}], val=[data=broker::data{20}]] | [key=[data=broker::data{there}], val=[data=broker::data{20}]] | opaque of Broker::TableIterator ============ broker::Vector -| [data=broker::data{hi}] | [data=broker::data{hi}] +| [data=broker::data{hi}] | [data=broker::data{hi}] | opaque of Broker::VectorIterator > [data=broker::data{there}] | [data=broker::data{there}] -| [data=broker::data{there}] | [data=broker::data{there}] +| [data=broker::data{there}] | [data=broker::data{there}] | opaque of Broker::VectorIterator > [data=broker::data{!}] | [data=broker::data{!}] -| [data=broker::data{!}] | [data=broker::data{!}] +| [data=broker::data{!}] | [data=broker::data{!}] | opaque of Broker::VectorIterator ============ broker::Record -| [data=broker::data{hi}] | [data=broker::data{hi}] +| [data=broker::data{hi}] | [data=broker::data{hi}] | opaque of Broker::RecordIterator > [data=broker::data{there}] | [data=broker::data{there}] -| [data=broker::data{there}] | [data=broker::data{there}] +| [data=broker::data{there}] | [data=broker::data{there}] | opaque of Broker::RecordIterator > [data=broker::data{!}] | [data=broker::data{!}] -| [data=broker::data{!}] | [data=broker::data{!}] +| [data=broker::data{!}] | [data=broker::data{!}] | opaque of Broker::RecordIterator diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.config.errors/errout b/testing/btest/Baseline/scripts.base.frameworks.input.config.errors/errout index 792f50d8c8..2b835abe07 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.config.errors/errout +++ b/testing/btest/Baseline/scripts.base.frameworks.input.config.errors/errout @@ -5,7 +5,7 @@ warning: ../configfile/Input::READER_CONFIG: Field: testbool Invalid value for b warning: ../configfile/Input::READER_CONFIG: Could not convert line 'testbool A B' to value. Ignoring line. warning: ../configfile/Input::READER_CONFIG: String 'A' contained no parseable number warning: ../configfile/Input::READER_CONFIG: Could not convert line 'testcount A' to value. Ignoring line. -warning: Value 'unknown' for stream 'configuration' is not a valid enum. +warning: Value 'unknown' for source 'thread ../configfile/Input::READER_CONFIG' is not a valid enum. error: SendEvent for event InputConfig::new_value failed warning: ../configfile/Input::READER_CONFIG: Option 'testbooool' does not exist. Ignoring line. warning: ../configfile/Input::READER_CONFIG: Option 'test_any' has type 'any', which is not supported for file input. Ignoring line. diff --git a/testing/btest/broker/opaque.zeek b/testing/btest/broker/opaque.zeek index e0a3bef6c7..41823a59ed 100644 --- a/testing/btest/broker/opaque.zeek +++ b/testing/btest/broker/opaque.zeek @@ -11,6 +11,7 @@ event zeek_init() topk_add(k1, "b"); topk_add(k1, "c"); local k2 = Broker::__opaque_clone_through_serialization(k1); + print type_name(k2); print topk_get_top(k1, 5); topk_add(k1, "shoulnotshowup"); print topk_get_top(k2, 5); @@ -23,6 +24,7 @@ event zeek_init() print hll_cardinality_estimate(c1); local c2 = Broker::__opaque_clone_through_serialization(c1); + print type_name(c2); hll_cardinality_add(c1, 2004); print hll_cardinality_estimate(c2); @@ -38,6 +40,7 @@ event zeek_init() print bloomfilter_lookup(bf_cnt, 0); print bloomfilter_lookup(bf_cnt, 42); local bf_copy = Broker::__opaque_clone_through_serialization(bf_cnt); + print type_name(bf_copy); bloomfilter_add(bf_cnt, 0); print bloomfilter_lookup(bf_copy, 0); print bloomfilter_lookup(bf_copy, 42); @@ -48,6 +51,7 @@ event zeek_init() local md5a = md5_hash_init(); md5_hash_update(md5a, "one"); local md5b = Broker::__opaque_clone_through_serialization(md5a); + print type_name(md5b); md5_hash_update(md5a, "two"); md5_hash_update(md5b, "two"); print md5_hash_finish(md5a); @@ -56,6 +60,7 @@ event zeek_init() local sha1a = sha1_hash_init(); sha1_hash_update(sha1a, "one"); local sha1b = Broker::__opaque_clone_through_serialization(sha1a); + print type_name(sha1b); sha1_hash_update(sha1a, "two"); sha1_hash_update(sha1b, "two"); print sha1_hash_finish(sha1a); @@ -64,6 +69,7 @@ event zeek_init() local sha256a = sha256_hash_init(); sha256_hash_update(sha256a, "one"); local sha256b = Broker::__opaque_clone_through_serialization(sha256a); + print type_name(sha256b); sha256_hash_update(sha256a, "two"); sha256_hash_update(sha256b, "two"); print sha256_hash_finish(sha256a); @@ -72,6 +78,7 @@ event zeek_init() print "============ X509"; local x509 = x509_from_der("\x30\x82\x03\x75\x30\x82\x02\x5D\xA0\x03\x02\x01\x02\x02\x0B\x04\x00\x00\x00\x00\x01\x15\x4B\x5A\xC3\x94\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x57\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x42\x45\x31\x19\x30\x17\x06\x03\x55\x04\x0A\x13\x10\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x6E\x76\x2D\x73\x61\x31\x10\x30\x0E\x06\x03\x55\x04\x0B\x13\x07\x52\x6F\x6F\x74\x20\x43\x41\x31\x1B\x30\x19\x06\x03\x55\x04\x03\x13\x12\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x1E\x17\x0D\x39\x38\x30\x39\x30\x31\x31\x32\x30\x30\x30\x30\x5A\x17\x0D\x32\x38\x30\x31\x32\x38\x31\x32\x30\x30\x30\x30\x5A\x30\x57\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x42\x45\x31\x19\x30\x17\x06\x03\x55\x04\x0A\x13\x10\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x6E\x76\x2D\x73\x61\x31\x10\x30\x0E\x06\x03\x55\x04\x0B\x13\x07\x52\x6F\x6F\x74\x20\x43\x41\x31\x1B\x30\x19\x06\x03\x55\x04\x03\x13\x12\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xDA\x0E\xE6\x99\x8D\xCE\xA3\xE3\x4F\x8A\x7E\xFB\xF1\x8B\x83\x25\x6B\xEA\x48\x1F\xF1\x2A\xB0\xB9\x95\x11\x04\xBD\xF0\x63\xD1\xE2\x67\x66\xCF\x1C\xDD\xCF\x1B\x48\x2B\xEE\x8D\x89\x8E\x9A\xAF\x29\x80\x65\xAB\xE9\xC7\x2D\x12\xCB\xAB\x1C\x4C\x70\x07\xA1\x3D\x0A\x30\xCD\x15\x8D\x4F\xF8\xDD\xD4\x8C\x50\x15\x1C\xEF\x50\xEE\xC4\x2E\xF7\xFC\xE9\x52\xF2\x91\x7D\xE0\x6D\xD5\x35\x30\x8E\x5E\x43\x73\xF2\x41\xE9\xD5\x6A\xE3\xB2\x89\x3A\x56\x39\x38\x6F\x06\x3C\x88\x69\x5B\x2A\x4D\xC5\xA7\x54\xB8\x6C\x89\xCC\x9B\xF9\x3C\xCA\xE5\xFD\x89\xF5\x12\x3C\x92\x78\x96\xD6\xDC\x74\x6E\x93\x44\x61\xD1\x8D\xC7\x46\xB2\x75\x0E\x86\xE8\x19\x8A\xD5\x6D\x6C\xD5\x78\x16\x95\xA2\xE9\xC8\x0A\x38\xEB\xF2\x24\x13\x4F\x73\x54\x93\x13\x85\x3A\x1B\xBC\x1E\x34\xB5\x8B\x05\x8C\xB9\x77\x8B\xB1\xDB\x1F\x20\x91\xAB\x09\x53\x6E\x90\xCE\x7B\x37\x74\xB9\x70\x47\x91\x22\x51\x63\x16\x79\xAE\xB1\xAE\x41\x26\x08\xC8\x19\x2B\xD1\x46\xAA\x48\xD6\x64\x2A\xD7\x83\x34\xFF\x2C\x2A\xC1\x6C\x19\x43\x4A\x07\x85\xE7\xD3\x7C\xF6\x21\x68\xEF\xEA\xF2\x52\x9F\x7F\x93\x90\xCF\x02\x03\x01\x00\x01\xA3\x42\x30\x40\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x60\x7B\x66\x1A\x45\x0D\x97\xCA\x89\x50\x2F\x7D\x04\xCD\x34\xA8\xFF\xFC\xFD\x4B\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\xD6\x73\xE7\x7C\x4F\x76\xD0\x8D\xBF\xEC\xBA\xA2\xBE\x34\xC5\x28\x32\xB5\x7C\xFC\x6C\x9C\x2C\x2B\xBD\x09\x9E\x53\xBF\x6B\x5E\xAA\x11\x48\xB6\xE5\x08\xA3\xB3\xCA\x3D\x61\x4D\xD3\x46\x09\xB3\x3E\xC3\xA0\xE3\x63\x55\x1B\xF2\xBA\xEF\xAD\x39\xE1\x43\xB9\x38\xA3\xE6\x2F\x8A\x26\x3B\xEF\xA0\x50\x56\xF9\xC6\x0A\xFD\x38\xCD\xC4\x0B\x70\x51\x94\x97\x98\x04\xDF\xC3\x5F\x94\xD5\x15\xC9\x14\x41\x9C\xC4\x5D\x75\x64\x15\x0D\xFF\x55\x30\xEC\x86\x8F\xFF\x0D\xEF\x2C\xB9\x63\x46\xF6\xAA\xFC\xDF\xBC\x69\xFD\x2E\x12\x48\x64\x9A\xE0\x95\xF0\xA6\xEF\x29\x8F\x01\xB1\x15\xB5\x0C\x1D\xA5\xFE\x69\x2C\x69\x24\x78\x1E\xB3\xA7\x1C\x71\x62\xEE\xCA\xC8\x97\xAC\x17\x5D\x8A\xC2\xF8\x47\x86\x6E\x2A\xC4\x56\x31\x95\xD0\x67\x89\x85\x2B\xF9\x6C\xA6\x5D\x46\x9D\x0C\xAA\x82\xE4\x99\x51\xDD\x70\xB7\xDB\x56\x3D\x61\xE4\x6A\xE1\x5C\xD6\xF6\xFE\x3D\xDE\x41\xCC\x07\xAE\x63\x52\xBF\x53\x53\xF4\x2B\xE9\xC7\xFD\xB6\xF7\x82\x5F\x85\xD2\x41\x18\xDB\x81\xB3\x04\x1C\xC5\x1F\xA4\x80\x6F\x15\x20\xC9\xDE\x0C\x88\x0A\x1D\xD6\x66\x55\xE2\xFC\x48\xC9\x29\x26\x69\xE0"); local x5092 = Broker::__opaque_clone_through_serialization(x509); + print type_name(x5092); print x509_parse(x509); print x509_parse(x5092); @@ -79,6 +86,7 @@ event zeek_init() local handle = entropy_test_init(); entropy_test_add(handle, "dh3Hie02uh^s#Sdf9L3frd243h$d78r2G4cM6*Q05d(7rh46f!0|4-f"); local handle2 = Broker::__opaque_clone_through_serialization(handle); + print type_name(handle2); print entropy_test_finish(handle); print entropy_test_finish(handle2); @@ -87,6 +95,7 @@ event zeek_init() Broker::set_insert(s1, "hi"); Broker::set_insert(s1, "there"); local d2 = Broker::__opaque_clone_through_serialization(s1$data); + print type_name(d2); print s1$data; print d2; print same_object(s1$data, d2) == F; @@ -101,7 +110,7 @@ event zeek_init() while ( ! Broker::set_iterator_last(i) ) { local ci = Broker::__opaque_clone_through_serialization(i); - print fmt("| %s | %s", Broker::set_iterator_value(i), Broker::set_iterator_value(ci)); + print fmt("| %s | %s | %s", Broker::set_iterator_value(i), Broker::set_iterator_value(ci), type_name(ci)); Broker::set_iterator_next(i); Broker::set_iterator_next(ci); if ( ! Broker::set_iterator_last(i) ) @@ -118,7 +127,7 @@ event zeek_init() while ( ! Broker::table_iterator_last(j) ) { local cj = Broker::__opaque_clone_through_serialization(j); - print fmt("| %s | %s", Broker::table_iterator_value(j), Broker::table_iterator_value(cj)); + print fmt("| %s | %s | %s", Broker::table_iterator_value(j), Broker::table_iterator_value(cj), type_name(cj)); Broker::table_iterator_next(j); Broker::table_iterator_next(cj); if ( ! Broker::table_iterator_last(j) ) @@ -135,7 +144,7 @@ event zeek_init() while ( ! Broker::vector_iterator_last(k) ) { local ck = Broker::__opaque_clone_through_serialization(k); - print fmt("| %s | %s", Broker::vector_iterator_value(k), Broker::vector_iterator_value(ck)); + print fmt("| %s | %s | %s", Broker::vector_iterator_value(k), Broker::vector_iterator_value(ck), type_name(ck)); Broker::vector_iterator_next(k); Broker::vector_iterator_next(ck); if ( ! Broker::vector_iterator_last(k) ) @@ -152,7 +161,7 @@ event zeek_init() while ( ! Broker::record_iterator_last(l) ) { local cl = Broker::__opaque_clone_through_serialization(l); - print fmt("| %s | %s", Broker::record_iterator_value(l), Broker::record_iterator_value(cl)); + print fmt("| %s | %s | %s", Broker::record_iterator_value(l), Broker::record_iterator_value(cl), type_name(cl)); Broker::record_iterator_next(l); Broker::record_iterator_next(cl); if ( ! Broker::record_iterator_last(l) )