From 2e5f96f5a96a179aa45a7bbcc28b93b3f62d35cb Mon Sep 17 00:00:00 2001 From: Noah Treuhaft Date: Thu, 14 May 2020 16:56:49 -0700 Subject: [PATCH 01/19] Replace bzero() with memset() --- src/Conn.cc | 4 ++-- src/IntSet.h | 7 ++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/Conn.cc b/src/Conn.cc index 8b017b5686..5796ad898c 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -80,12 +80,12 @@ Connection::Connection(NetSessions* s, const ConnIDKey& k, double t, const ConnI if ( pkt->l2_src ) memcpy(orig_l2_addr, pkt->l2_src, sizeof(orig_l2_addr)); else - bzero(orig_l2_addr, sizeof(orig_l2_addr)); + memset(orig_l2_addr, 0, sizeof(orig_l2_addr)); if ( pkt->l2_dst ) memcpy(resp_l2_addr, pkt->l2_dst, sizeof(resp_l2_addr)); else - bzero(resp_l2_addr, sizeof(resp_l2_addr)); + memset(resp_l2_addr, 0, sizeof(resp_l2_addr)); vlan = pkt->vlan; inner_vlan = pkt->inner_vlan; diff --git a/src/IntSet.h b/src/IntSet.h index 75bd977bf7..20b1d257d0 100644 --- a/src/IntSet.h +++ b/src/IntSet.h @@ -27,14 +27,11 @@ private: unsigned char* set; }; - -#define bzero(p, size) memset(p, 0, size) - inline IntSet::IntSet(unsigned int n) { size = n / 8 + 1; set = new unsigned char[size]; - bzero(set, size); + memset(set, 0, size); } inline IntSet::~IntSet() @@ -65,5 +62,5 @@ inline bool IntSet::Contains(unsigned int i) const inline void IntSet::Clear() { - bzero(set, size); + memset(set, 0, size); } From 5272079e790fa251757a6300a3a64fc2137bfd89 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 14 May 2020 22:49:32 -0700 Subject: [PATCH 02/19] Change Coverity action to output build log --- .github/workflows/coverity-scan.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/coverity-scan.yml b/.github/workflows/coverity-scan.yml index d983e22256..9946ea9059 100644 --- a/.github/workflows/coverity-scan.yml +++ b/.github/workflows/coverity-scan.yml @@ -67,6 +67,7 @@ jobs: run: | export PATH=`pwd`/coverity-tools/bin:$PATH ( cd build && cov-build --dir cov-int make -j 3 ) + cat build/cov-int/build-log.txt - name: Submit env: From 48ee85df0b0ebdd3fe53f3eee679215aa5241013 Mon Sep 17 00:00:00 2001 From: Justin Azoff Date: Sun, 3 May 2020 16:58:40 -0400 Subject: [PATCH 03/19] add packet fuzzer --- src/fuzzers/CMakeLists.txt | 1 + src/fuzzers/packet-fuzzer.cc | 50 ++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 src/fuzzers/packet-fuzzer.cc diff --git a/src/fuzzers/CMakeLists.txt b/src/fuzzers/CMakeLists.txt index 97a050d265..bd637979ac 100644 --- a/src/fuzzers/CMakeLists.txt +++ b/src/fuzzers/CMakeLists.txt @@ -70,3 +70,4 @@ target_link_libraries(zeek_fuzzer_shared ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) add_fuzz_target(pop3) +add_fuzz_target(packet) diff --git a/src/fuzzers/packet-fuzzer.cc b/src/fuzzers/packet-fuzzer.cc new file mode 100644 index 0000000000..f37110604a --- /dev/null +++ b/src/fuzzers/packet-fuzzer.cc @@ -0,0 +1,50 @@ +#include "binpac.h" + +#include "Net.h" +#include "Conn.h" +#include "Sessions.h" +#include "analyzer/Analyzer.h" +#include "analyzer/Manager.h" +#include "analyzer/protocol/pia/PIA.h" +#include "analyzer/protocol/tcp/TCP.h" + +#include "FuzzBuffer.h" +#include "fuzzer-setup.h" + +#include "pcap/dlt.h" + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) + { + + zeek::detail::FuzzBuffer fb{data, size}; + + if ( ! fb.Valid() ) + return 0; + + for ( ; ; ) + { + auto chunk = fb.Next(); + + if ( ! chunk ) + break; + + Packet pkt; + auto timestamp = current_time(true); + pkt_timeval ts = {long(timestamp), 0}; + pkt.Init(DLT_RAW, &ts, chunk->size, chunk->size, chunk->data.get(), false, ""); + + try + { + sessions->NextPacket(timestamp, &pkt); + } + catch ( binpac::Exception const &e ) + { + } + + chunk = {}; + mgr.Drain(); + } + + zeek::detail::fuzzer_cleanup_one_input(); + return 0; + } From dd458a810e1bdc4784bf96796b5c7d3389d1f0ad Mon Sep 17 00:00:00 2001 From: Justin Azoff Date: Fri, 15 May 2020 11:52:11 -0400 Subject: [PATCH 04/19] add initial packet corpus This contains converted files from the test suite: dns53.cap ipv6-fragmented-dns.trace.cap ipv6-hbh-routing0.trace.cap ipv6-http-atomic-frag.trace.cap ipv6_zero_len_ah.trace.cap and a new one I made: syn_packet.cap --- src/fuzzers/packet-corpus.zip | Bin 0 -> 2611 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/fuzzers/packet-corpus.zip diff --git a/src/fuzzers/packet-corpus.zip b/src/fuzzers/packet-corpus.zip new file mode 100644 index 0000000000000000000000000000000000000000..8ffc9d4162f6615378c64137fddf1119e83329b9 GIT binary patch literal 2611 zcmb7G3sh2D8a_OPloJo7Ld`NYH3T!r9(JeD%n~sPny%e4Kn1mw7@+c1B671r4U;lV zi_9`B^Kq5bO1Y+H5AEU1y))OYVK&;sN^^SPwVcyqT2`xDbN1TntaH}h|F`$IzyJR> zI|zX!0st@$Ag?Q9TLoLYKY_k(!T`V$m<&XUW!`kp2!Z6UD=h&8P*=Y$1?rpPF#rtt z3>x!xY6Lid6-b5r?~;1xh_q~d`Jg!gJ-f!A^lw`h)V44~mIubg@cEG@K0k^dMW%{8 z>9oG(mnfbyF0;tgz+gdp5LEzNrcW`!vO(}F6{Pe@9X3xJ+URv~ynpYZg1VP9T0>h^ ztwUxCpu4S^Hrqm%B5FA;j;udlcE`CN8=)i%BHBIXd{^FUv?3CmK|%ybpc4rTHS;Nf zCC$^XUNYB$_j(u=w+l`gMdau*l>CN`^1mB48gp!53>*PaHfP`~BUi zB^4=K_$HA8{`yjJ9f@TNPYD?8q|>~-loqyTCYfnv&E?RBZuVqmX6T-R^Ph5ptYzer zyFzI>YWn;=9CamE#_`Z`xlCIUk}*8nLg8c2s1q3IiI}{?&h3xwP#xT`j4r0b$n#Ri zPG8MVc_(gwaSLcdPgqQ&sVDe3`4M+@U6>;SB)OlVH67v9S zjO-VkzA@iD2VI)G6aMc1`vHuKhlH(FK83^4DE~>lRd8nm4*kg)uZHgmWk6ZMhr0d$ z&Nw zKktCSNjF}yz+;To-$f(>;Zxx~M|r-EheN4~0f4C=7A2QUr~>)MxR{8yBK)?6wdI6F z+xQY&ZlCJpM_@|geqpFjnT|HOmgX|guJkyL9+g*D%qRX<6dRc%zH@l+`Woc4DM^Da zN>ke^RX*~5ExK%7(T<9PPL%3rg;O$~1Z!ihD;At}$EwaLPjU zDAbotpL?bOOrBRiKgZ{Uwmr+VTl#B{Yvi&G>B}Bn2fTEL zAE%s3Us>bRha6uc_Q=p!{G~|SYsd$J@c0$XA#b3K(py@>KO+*|L~Yx$p*i-(WmZ9m zy<1iMk(Q459@5F9oh{^ssH|7#R2IKSPd4pC9B3?@vXr=@d%)dz6l3XKubmU#^k{sd zF7)8sTj@3)FO$w%C~tIz{p%Tyd~fTx%JVwCLbF7yOjhsVdTM8{qM{O?KVCoSA9ptl z?X$T{7FPfBg}QjRqYGn(eb?h#L`yVJ=8mIJ1nipZ}lDJ-bKJXvD0+^u}%zRfsPfxM&x<0*)rH7U_AswJ9O? zR?R2pHXaTY@wmO?RsC%D-bY#NnpG8P`qg&P%VR9l9D&7>r#ZLr1E)pVwM%7TGcf5! z*vke#0ZF-c|J;lH1XUOLk6R1)X{`a?mOzTc+IG-5|0;I z>uIBBD$C6t&-=O$*o-ULV?LDJ@6}MAM~1t_l<$4L4lMqaE?%o9X^o0i|25mZLumM2H8z(#PH zf{=Ob??z2khs&vOrL!GhWnZWejO$2oRi%!m))aK>-9l&N2{SS#gAe}Dy7th)3rifp zt|6OpQcF{7vmP7nm~agiMz1>-u-zZtwMN9czGM|U0CAx#iox6_fva*)3tC8ZlU9>< zs}V7VZad`4K=c0Xn(TVTjMqGbzqc2CWb~pUyHfX?t)X=mpAglw=gzFPDsenI?ZhyQ zvVCk`8^X%iGu!cISBM5G)Ugl^vIH?tB8b=^lz(s#x7t7T;p@=u(S|C_S`f@&TxDM| zI<^BAFct_(i)~3zX@kM1AiiYzBnb80Q~*n@$LffiTRHW_qiXA4M&4q@+kewM7)RiQgQw^fR9V-Sb&os0RE)VejCWgavuwljC!|=FNy!_@jutm h*!XE^;4@W0G2f`_Gv+OzwEzGLY81$82BDb%@LxPuujv2) literal 0 HcmV?d00001 From 48153ba12fadcf7fc832efa5b9e9a0dfb1ebc9f3 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 15 May 2020 10:38:17 -0700 Subject: [PATCH 05/19] Update submodule(s) [nomail] --- aux/broker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broker b/aux/broker index d360a228c3..c20d496d6b 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit d360a228c31a6c6d58afdc9827308471c28cd126 +Subproject commit c20d496d6bca8a6dfbbccf8744f1d3d004454ff2 From c4d41dcfbb4024b9ccb9fce73e89003fc511e914 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 15 May 2020 11:23:24 -0700 Subject: [PATCH 06/19] Fix building fuzz targets on macOS --- src/fuzzers/CMakeLists.txt | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/src/fuzzers/CMakeLists.txt b/src/fuzzers/CMakeLists.txt index 97a050d265..74097cb00e 100644 --- a/src/fuzzers/CMakeLists.txt +++ b/src/fuzzers/CMakeLists.txt @@ -22,16 +22,26 @@ if ( NOT DEFINED ZEEK_FUZZING_ENGINE AND DEFINED ENV{LIB_FUZZING_ENGINE} ) endif () endif () +# The bind library is handled a bit hack-ishly since it defaults to linking it +# as static library by default on Linux, but at least on one common distro, +# that static library wasn't compiled with -fPIC and so not usable in the +# shared library we're trying to build. So instead, the fuzzer executable, not +# the shared lib, links it. +string(REGEX MATCH ".*\\.a$" _have_static_bind_lib "${BIND_LIBRARY}") + macro(ADD_FUZZ_TARGET _name) set(_fuzz_target zeek-${_name}-fuzzer) set(_fuzz_source ${_name}-fuzzer.cc) add_executable(${_fuzz_target} ${_fuzz_source} ${ARGN}) - target_link_libraries(${_fuzz_target} - zeek_fuzzer_shared - ${BIND_LIBRARY} - ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) + target_link_libraries(${_fuzz_target} zeek_fuzzer_shared) + + if ( _have_static_bind_lib ) + target_link_libraries(${_fuzz_target} ${BIND_LIBRARY}) + endif () + + target_link_libraries(${_fuzz_target} ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}) if ( DEFINED ZEEK_FUZZING_ENGINE ) target_link_libraries(${_fuzz_target} ${ZEEK_FUZZING_ENGINE}) @@ -55,12 +65,11 @@ add_library(zeek_fuzzer_shared SHARED set(zeek_fuzzer_shared_deps) foreach(_dep ${zeekdeps} ) - # The bind library is handled a bit hack-ishly since it defaults to - # linking it as static library by default on Linux, but at least - # on one common distro, that static library wasn't compiled with -fPIC - # and so not usable in the shared library we're trying to build. - # So instead, the fuzzer executable, not the shared lib, links it. - if ( NOT "${_dep}" STREQUAL "${BIND_LIBRARY}" ) + if ( "${_dep}" STREQUAL "${BIND_LIBRARY}" ) + if ( NOT _have_static_bind_lib ) + set(zeek_fuzzer_shared_deps ${zeek_fuzzer_shared_deps} ${_dep}) + endif () + else () set(zeek_fuzzer_shared_deps ${zeek_fuzzer_shared_deps} ${_dep}) endif () endforeach () From 23a543f50965d99bc8958afbefdb560de90bb341 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 15 May 2020 12:39:03 -0700 Subject: [PATCH 07/19] Update submodule(s) [nomail] --- aux/bifcl | 2 +- aux/binpac | 2 +- aux/broker | 2 +- aux/zeek-aux | 2 +- aux/zeekctl | 2 +- cmake | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/aux/bifcl b/aux/bifcl index 9d6a296d28..abd57da8ba 160000 --- a/aux/bifcl +++ b/aux/bifcl @@ -1 +1 @@ -Subproject commit 9d6a296d283cfccf2c28a9181d9cee2760229fcb +Subproject commit abd57da8ba6c9337a5ed79cabbf962f71b38d62d diff --git a/aux/binpac b/aux/binpac index cdc808b07b..0730797611 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit cdc808b07ba0caf05ed4bd83d906125b10feb695 +Subproject commit 07307976115c90095af183ce950364bc1982027c diff --git a/aux/broker b/aux/broker index c20d496d6b..74d1689643 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit c20d496d6bca8a6dfbbccf8744f1d3d004454ff2 +Subproject commit 74d168964369f8b5c51238520d6ad35173791110 diff --git a/aux/zeek-aux b/aux/zeek-aux index 43fa1b7f01..f4a659ee89 160000 --- a/aux/zeek-aux +++ b/aux/zeek-aux @@ -1 +1 @@ -Subproject commit 43fa1b7f01b1d77ccc1d05cc50f1a1c3ea58b6fa +Subproject commit f4a659ee89e95f93ff2900cde7984f032a8113e9 diff --git a/aux/zeekctl b/aux/zeekctl index 2bd666a0c1..881fe0083a 160000 --- a/aux/zeekctl +++ b/aux/zeekctl @@ -1 +1 @@ -Subproject commit 2bd666a0c13f8efa6dcbc2d1f125ccc73e333522 +Subproject commit 881fe0083a512b3731738fb0272241e47b272e09 diff --git a/cmake b/cmake index 2b3cd252b4..7666cd4628 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 2b3cd252b4ef80b5d96c6f8aaac8d63eee86f5da +Subproject commit 7666cd462888b0eee173040c6888f41930a8be0b From 9dec370048191f5ff78cffaefcb4d525929ce582 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Fri, 15 May 2020 19:57:32 +0000 Subject: [PATCH 08/19] Highwayhash: small build fix Turns out that hh_neon should not be compiled on generic arm CPUs. That one is only for aarch64. --- src/CMakeLists.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index bc2ea56555..13be9f65e4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -334,9 +334,6 @@ set(HH_SRCS if (${COMPILER_ARCHITECTURE} STREQUAL "arm") set_source_files_properties(${HH_SRCS} PROPERTIES COMPILE_FLAGS -mfloat-abi=hard -march=armv7-a -mfpu=neon) - list(APPEND HH_SRCS - ../aux/highwayhash/highwayhash/hh_neon.cc - ) elseif (${COMPILER_ARCHITECTURE} STREQUAL "aarch64") list(APPEND HH_SRCS ../aux/highwayhash/highwayhash/hh_neon.cc From e78a5be17ddef36d8cf21274bf76068c23696fd8 Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 15 May 2020 15:59:46 -0400 Subject: [PATCH 09/19] Update src/fuzzers/packet-fuzzer.cc Remove unused #includes Co-authored-by: Tim Wojtulewicz --- src/fuzzers/packet-fuzzer.cc | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/fuzzers/packet-fuzzer.cc b/src/fuzzers/packet-fuzzer.cc index f37110604a..3401b00c19 100644 --- a/src/fuzzers/packet-fuzzer.cc +++ b/src/fuzzers/packet-fuzzer.cc @@ -1,12 +1,8 @@ #include "binpac.h" -#include "Net.h" -#include "Conn.h" +#include "iosource/Packet.h" +#include "Event.h" #include "Sessions.h" -#include "analyzer/Analyzer.h" -#include "analyzer/Manager.h" -#include "analyzer/protocol/pia/PIA.h" -#include "analyzer/protocol/tcp/TCP.h" #include "FuzzBuffer.h" #include "fuzzer-setup.h" From afde8a959685b184fbb9a4336eee3baf94c36ee1 Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 15 May 2020 16:05:21 -0400 Subject: [PATCH 10/19] Update src/fuzzers/packet-fuzzer.cc Use a constant timestamp for packets Co-authored-by: Jon Siwek --- src/fuzzers/packet-fuzzer.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fuzzers/packet-fuzzer.cc b/src/fuzzers/packet-fuzzer.cc index 3401b00c19..ca8e2cecf8 100644 --- a/src/fuzzers/packet-fuzzer.cc +++ b/src/fuzzers/packet-fuzzer.cc @@ -25,8 +25,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) break; Packet pkt; - auto timestamp = current_time(true); - pkt_timeval ts = {long(timestamp), 0}; + auto timestamp = 42; + pkt_timeval ts = {timestamp, 0}; pkt.Init(DLT_RAW, &ts, chunk->size, chunk->size, chunk->data.get(), false, ""); try From 034304b9d5c1529c156878cfd28289ffe2db237d Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Mon, 18 May 2020 14:38:10 -0700 Subject: [PATCH 11/19] Make SendEvent callable from all threads This commit refactors the SendEvent call and moves it from the Input ReaderBackend to to MsgThread. This allows all other types of threads to access this functionality. This necessitated a few more changes. Most importantly, one of the ValueToVal methods was moved over to SerialTypes. Whereit arguably belongs - there was nothing that was input-framework specific in that method - and the functionality could come in useful in a number of cases. --- src/input/Manager.cc | 278 +----------------- src/input/Manager.h | 13 +- src/input/ReaderBackend.cc | 29 -- src/input/ReaderBackend.h | 12 - src/threading/BasicThread.h | 2 +- src/threading/Manager.cc | 55 ++++ src/threading/Manager.h | 12 + src/threading/MsgThread.cc | 30 ++ src/threading/MsgThread.h | 14 +- src/threading/SerialTypes.cc | 217 +++++++++++++- src/threading/SerialTypes.h | 20 ++ .../errout | 2 +- 12 files changed, 353 insertions(+), 331 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index dcbc4f53c0..5a1edb3200 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -26,14 +26,6 @@ using namespace std; using threading::Value; using threading::Field; -static void delete_value_ptr_array(Value** vals, int num_fields) - { - for ( int i = 0; i < num_fields; ++i ) - delete vals[i]; - - delete [] vals; - } - /** * InputHashes are used as Dictionaries to store the value and index hashes * for all lines currently stored in a table. Index hash is stored as @@ -1087,7 +1079,7 @@ void Manager::SendEntry(ReaderFrontend* reader, Value* *vals) else assert(false); - delete_value_ptr_array(vals, readFields); + Value::delete_value_ptr_array(vals, readFields); } int Manager::SendEntryTable(Stream* i, const Value* const *vals) @@ -1469,7 +1461,7 @@ void Manager::Put(ReaderFrontend* reader, Value* *vals) else assert(false); - delete_value_ptr_array(vals, readFields); + Value::delete_value_ptr_array(vals, readFields); } int Manager::SendEventStreamEvent(Stream* i, EnumVal* type, const Value* const *vals) @@ -1774,7 +1766,7 @@ bool Manager::Delete(ReaderFrontend* reader, Value* *vals) return false; } - delete_value_ptr_array(vals, readVals); + Value::delete_value_ptr_array(vals, readVals); return success; } @@ -1799,68 +1791,6 @@ bool Manager::CallPred(Func* pred_func, const int numvals, ...) const return result; } -// Raise everything in here as warnings so it is passed to scriptland without -// looking "fatal". In addition to these warnings, ReaderBackend will queue -// one reporter message. -bool Manager::SendEvent(ReaderFrontend* reader, const string& name, const int num_vals, Value* *vals) const - { - Stream *i = FindStream(reader); - if ( i == nullptr ) - { - reporter->InternalWarning("Unknown reader %s in SendEvent for event %s", reader->Name(), name.c_str()); - delete_value_ptr_array(vals, num_vals); - return false; - } - - EventHandler* handler = event_registry->Lookup(name); - if ( handler == nullptr ) - { - Warning(i, "Event %s not found", name.c_str()); - delete_value_ptr_array(vals, num_vals); - return false; - } - -#ifdef DEBUG - DBG_LOG(DBG_INPUT, "SendEvent for event %s with %d vals", - name.c_str(), num_vals); -#endif - - RecordType *type = handler->FType()->Args(); - int num_event_vals = type->NumFields(); - if ( num_vals != num_event_vals ) - { - Warning(i, "Wrong number of values for event %s", name.c_str()); - delete_value_ptr_array(vals, num_vals); - return false; - } - - bool convert_error = false; - - zeek::Args vl; - vl.reserve(num_vals); - - for ( int j = 0; j < num_vals; j++) - { - Val* v = ValueToVal(i, vals[j], convert_error); - vl.emplace_back(AdoptRef{}, v); - - if ( v && ! convert_error && ! same_type(type->FieldType(j), v->Type()) ) - { - convert_error = true; - type->FieldType(j)->Error("SendEvent types do not match", v->Type()); - } - } - - delete_value_ptr_array(vals, num_vals); - - if ( convert_error ) - return false; - else if ( handler ) - mgr.Enqueue(handler, std::move(vl), SOURCE_LOCAL); - - return true; -} - void Manager::SendEvent(EventHandlerPtr ev, const int numvals, ...) const { zeek::Args vl; @@ -2240,9 +2170,9 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) c } // convert threading value to Bro value -// have_error is a reference to a boolean which is set to true as soon as an error occured. +// have_error is a reference to a boolean which is set to true as soon as an error occurs. // When have_error is set to true at the beginning of the function, it is assumed that -// an error already occured in the past and processing is aborted. +// an error already occurred in the past and processing is aborted. Val* Manager::ValueToVal(const Stream* i, const Value* val, BroType* request_type, bool& have_error) const { if ( have_error ) @@ -2397,204 +2327,6 @@ Val* Manager::ValueToVal(const Stream* i, const Value* val, BroType* request_typ return nullptr; } -Val* Manager::ValueToVal(const Stream* i, const Value* val, bool& have_error) const - { - if ( have_error ) - return nullptr; - - if ( ! val->present ) - return nullptr; // unset field - - switch ( val->type ) { - case TYPE_BOOL: - return val_mgr->Bool(val->val.int_val)->Ref(); - - case TYPE_INT: - return val_mgr->Int(val->val.int_val).release(); - - case TYPE_COUNT: - case TYPE_COUNTER: - return val_mgr->Count(val->val.int_val).release(); - - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - return new Val(val->val.double_val, val->type); - - case TYPE_STRING: - { - BroString *s = new BroString((const u_char*)val->val.string_val.data, val->val.string_val.length, true); - return new StringVal(s); - } - - case TYPE_PORT: - return val_mgr->Port(val->val.port_val.port, val->val.port_val.proto)->Ref(); - - case TYPE_ADDR: - { - IPAddr* addr = nullptr; - switch ( val->val.addr_val.family ) { - case IPv4: - addr = new IPAddr(val->val.addr_val.in.in4); - break; - - case IPv6: - addr = new IPAddr(val->val.addr_val.in.in6); - break; - - default: - assert(false); - } - - AddrVal* addrval = new AddrVal(*addr); - delete addr; - return addrval; - } - - case TYPE_SUBNET: - { - IPAddr* addr = nullptr; - switch ( val->val.subnet_val.prefix.family ) { - case IPv4: - addr = new IPAddr(val->val.subnet_val.prefix.in.in4); - break; - - case IPv6: - addr = new IPAddr(val->val.subnet_val.prefix.in.in6); - break; - - default: - assert(false); - } - - SubNetVal* subnetval = new SubNetVal(*addr, val->val.subnet_val.length); - delete addr; - return subnetval; - } - - case TYPE_PATTERN: - { - RE_Matcher* re = new RE_Matcher(val->val.pattern_text_val); - re->Compile(); - return new PatternVal(re); - } - - case TYPE_TABLE: - { - IntrusivePtr set_index; - if ( val->val.set_val.size == 0 && val->subtype == TYPE_VOID ) - // don't know type - unspecified table. - set_index = make_intrusive(); - else - { - // all entries have to have the same type... - TypeTag stag = val->subtype; - if ( stag == TYPE_VOID ) - TypeTag stag = val->val.set_val.vals[0]->type; - - IntrusivePtr index_type; - - if ( stag == TYPE_ENUM ) - { - // Enums are not a base-type, so need to look it up. - const auto& sv = val->val.set_val.vals[0]->val.string_val; - std::string enum_name(sv.data, sv.length); - auto enum_id = global_scope()->Lookup(enum_name); - - if ( ! enum_id ) - { - Warning(i, "Value '%s' for stream '%s' is not a valid enum.", - enum_name.data(), i->name.c_str()); - - have_error = true; - return nullptr; - } - - index_type = {NewRef{}, enum_id->Type()->AsEnumType()}; - } - else - index_type = base_type(stag); - - set_index = make_intrusive(index_type); - set_index->Append(std::move(index_type)); - } - - auto s = make_intrusive(std::move(set_index), nullptr); - TableVal* t = new TableVal(std::move(s)); - for ( int j = 0; j < val->val.set_val.size; j++ ) - { - Val* assignval = ValueToVal(i, val->val.set_val.vals[j], have_error); - - t->Assign(assignval, nullptr); - Unref(assignval); // index is not consumed by assign. - } - - return t; - } - - case TYPE_VECTOR: - { - IntrusivePtr type; - - if ( val->val.vector_val.size == 0 && val->subtype == TYPE_VOID ) - // don't know type - unspecified table. - type = base_type(TYPE_ANY); - else - { - // all entries have to have the same type... - if ( val->subtype == TYPE_VOID ) - type = base_type(val->val.vector_val.vals[0]->type); - else - type = base_type(val->subtype); - } - - auto vt = make_intrusive(std::move(type)); - VectorVal* v = new VectorVal(vt.get()); - - for ( int j = 0; j < val->val.vector_val.size; j++ ) - v->Assign(j, ValueToVal(i, val->val.vector_val.vals[j], have_error)); - - return v; - } - - case TYPE_ENUM: { - // Convert to string first to not have to deal with missing - // \0's... - string enum_string(val->val.string_val.data, val->val.string_val.length); - - // let's try looking it up by global ID. - auto id = lookup_ID(enum_string.c_str(), GLOBAL_MODULE_NAME); - if ( ! id || ! id->IsEnumConst() ) - { - Warning(i, "Value '%s' for stream '%s' is not a valid enum.", - enum_string.c_str(), i->name.c_str()); - - have_error = true; - return nullptr; - } - - EnumType* t = id->Type()->AsEnumType(); - int intval = t->Lookup(id->ModuleName(), id->Name()); - if ( intval < 0 ) - { - Warning(i, "Enum value '%s' for stream '%s' not found.", - enum_string.c_str(), i->name.c_str()); - - have_error = true; - return nullptr; - } - - return t->GetVal(intval).release(); - } - - default: - reporter->InternalError("Unsupported type for input_read in stream %s", i->name.c_str()); - } - - assert(false); - return nullptr; - } - Manager::Stream* Manager::FindStream(const string &name) const { for ( auto s = readers.begin(); s != readers.end(); ++s ) diff --git a/src/input/Manager.h b/src/input/Manager.h index 191692f9e6..c24b930d64 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -118,7 +118,6 @@ protected: friend class PutMessage; friend class DeleteMessage; friend class ClearMessage; - friend class SendEventMessage; friend class SendEntryMessage; friend class EndCurrentSendMessage; friend class ReaderClosedMessage; @@ -142,11 +141,6 @@ protected: void SendEntry(ReaderFrontend* reader, threading::Value* *vals); void EndCurrentSend(ReaderFrontend* reader); - // Allows readers to directly send Bro events. The num_vals and vals - // must be the same the named event expects. Takes ownership of - // threading::Value fields. - bool SendEvent(ReaderFrontend* reader, const std::string& name, const int num_vals, threading::Value* *vals) const; - // Instantiates a new ReaderBackend of the given type (note that // doing so creates a new thread!). ReaderBackend* CreateBackend(ReaderFrontend* frontend, EnumVal* tag); @@ -227,14 +221,9 @@ private: // startpos. int CopyValue(char *data, const int startpos, const threading::Value* val) const; - // Convert Threading::Value to an internal Bro Type (works also with - // Records). + // Convert Threading::Value to an internal Bro Type (works with Records). Val* ValueToVal(const Stream* i, const threading::Value* val, BroType* request_type, bool& have_error) const; - // Convert Threading::Value to an internal Bro type just using the information given in the threading::Value. - // This allows more flexibility, especially given structures in script-land that contain any types. - Val* ValueToVal(const Stream* i, const threading::Value* val, bool& have_error) const; - // Convert Threading::Value to an internal Bro list type. Val* ValueToIndexVal(const Stream* i, int num_fields, const RecordType* type, const threading::Value* const *vals, bool& have_error) const; diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index 90050a6438..7e0fbd7685 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -54,30 +54,6 @@ public: private: }; -class SendEventMessage final : public threading::OutputMessage { -public: - SendEventMessage(ReaderFrontend* reader, const char* name, const int num_vals, Value* *val) - : threading::OutputMessage("SendEvent", reader), - name(copy_string(name)), num_vals(num_vals), val(val) {} - - ~SendEventMessage() override { delete [] name; } - - bool Process() override - { - bool success = input_mgr->SendEvent(Object(), name, num_vals, val); - - if ( ! success ) - reporter->Error("SendEvent for event %s failed", name); - - return true; // We do not want to die if sendEvent fails because the event did not return. - } - -private: - const char* name; - const int num_vals; - Value* *val; -}; - class ReaderErrorMessage final : public threading::OutputMessage { public: @@ -229,11 +205,6 @@ void ReaderBackend::Clear() SendOut(new ClearMessage(frontend)); } -void ReaderBackend::SendEvent(const char* name, const int num_vals, Value* *vals) - { - SendOut(new SendEventMessage(frontend, name, num_vals, vals)); - } - void ReaderBackend::EndCurrentSend() { SendOut(new EndCurrentSendMessage(frontend)); diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index d158d0972c..2748d1e2d2 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -280,18 +280,6 @@ protected: */ virtual bool DoHeartbeat(double network_time, double current_time) = 0; - /** - * Method allowing a reader to send a specified Bro event. Vals must - * match the values expected by the bro event. - * - * @param name name of the bro event to send - * - * @param num_vals number of entries in \a vals - * - * @param vals the values to be given to the event - */ - void SendEvent(const char* name, const int num_vals, threading::Value* *vals); - // Content-sending-functions (simple mode). Include table-specific // functionality that simply is not used if we have no table. diff --git a/src/threading/BasicThread.h b/src/threading/BasicThread.h index fa1e097536..fa65e070ee 100644 --- a/src/threading/BasicThread.h +++ b/src/threading/BasicThread.h @@ -37,7 +37,7 @@ public: /** * Returns a descriptive name for the thread. If not set via - * SetName(). If not set, a default name is choosen automatically. + * SetName(), a default name is chosen automatically. * * This method is safe to call from any thread. */ diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index e5562a0ddd..666edff142 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -5,6 +5,7 @@ #include "NetVar.h" #include "iosource/Manager.h" +#include "Event.h" using namespace threading; @@ -128,6 +129,60 @@ void Manager::StartHeartbeatTimer() timer_mgr->Add(new HeartbeatTimer(network_time + BifConst::Threading::heartbeat_interval)); } +// Raise everything in here as warnings so it is passed to scriptland without +// looking "fatal". In addition to these warnings, ReaderBackend will queue +// one reporter message. +bool Manager::SendEvent(MsgThread* thread, const std::string& name, const int num_vals, Value* *vals) const + { + EventHandler* handler = event_registry->Lookup(name); + if ( handler == nullptr ) + { + reporter->Warning("Thread %s: Event %s not found", thread->Name(), name.c_str()); + Value::delete_value_ptr_array(vals, num_vals); + return false; + } + +#ifdef DEBUG + DBG_LOG(DBG_INPUT, "Thread %s: SendEvent for event %s with %d vals", + thread->Name(), name.c_str(), num_vals); +#endif + + RecordType *type = handler->FType()->Args(); + int num_event_vals = type->NumFields(); + if ( num_vals != num_event_vals ) + { + reporter->Warning("Thread %s: Wrong number of values for event %s", thread->Name(), name.c_str()); + Value::delete_value_ptr_array(vals, num_vals); + return false; + } + + bool convert_error = false; + + zeek::Args vl; + vl.reserve(num_vals); + + for ( int j = 0; j < num_vals; j++) + { + Val* v = Value::ValueToVal(std::string("thread ") + thread->Name(), vals[j], convert_error); + vl.emplace_back(AdoptRef{}, v); + + if ( v && ! convert_error && ! same_type(type->FieldType(j), v->Type()) ) + { + convert_error = true; + type->FieldType(j)->Error("SendEvent types do not match", v->Type()); + } + } + + Value::delete_value_ptr_array(vals, num_vals); + + if ( convert_error ) + return false; + else if ( handler ) + mgr.Enqueue(handler, std::move(vl), SOURCE_LOCAL); + + return true; + } + void Manager::Flush() { bool do_beat = false; diff --git a/src/threading/Manager.h b/src/threading/Manager.h index 2c5b05d9b5..6429baf8a4 100644 --- a/src/threading/Manager.h +++ b/src/threading/Manager.h @@ -88,6 +88,18 @@ public: */ void KillThreads(); + /** + * Allows threads to directly send Zeek events. The num_vals and vals must be + * the same the named event expects. Takes ownership of threading::Value fields. + * + * @param thread Thread raising the event + * @param name Name of event being raised + * @param num_vals Number of values passed to the event + * @param vals Values passed to the event + * @returns True on success false on failure. + */ + bool SendEvent(MsgThread* thread, const std::string& name, const int num_vals, threading::Value* *vals) const; + protected: friend class BasicThread; friend class MsgThread; diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc index 3d2fa66271..f1fd2897d2 100644 --- a/src/threading/MsgThread.cc +++ b/src/threading/MsgThread.cc @@ -126,6 +126,31 @@ private: } +// An event that the child wants to pass into the main event queue +class SendEventMessage final : public OutputMessage { +public: + SendEventMessage(MsgThread* thread, const char* name, const int num_vals, Value* *val) + : OutputMessage("SendEvent", thread), + name(copy_string(name)), num_vals(num_vals), val(val) {} + + ~SendEventMessage() override { delete [] name; } + + bool Process() override + { + bool success = thread_mgr->SendEvent(Object(), name, num_vals, val); + + if ( ! success ) + reporter->Error("SendEvent for event %s failed", name); + + return true; // We do not want to die if sendEvent fails because the event did not return. + } + +private: + const char* name; + const int num_vals; + Value* *val; +}; + ////// Methods. Message::~Message() @@ -363,6 +388,11 @@ void MsgThread::SendOut(BasicOutputMessage* msg, bool force) flare.Fire(); } +void MsgThread::SendEvent(const char* name, const int num_vals, Value* *vals) + { + SendOut(new SendEventMessage(this, name, num_vals, vals)); + } + BasicOutputMessage* MsgThread::RetrieveOut() { BasicOutputMessage* msg = queue_out.Get(); diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h index e5c0e3a910..8bda2d40b8 100644 --- a/src/threading/MsgThread.h +++ b/src/threading/MsgThread.h @@ -60,6 +60,18 @@ public: */ void SendOut(BasicOutputMessage* msg) { return SendOut(msg, false); } + /** + * Allows the child thread to send a specified Zeek event. The given Vals + * must match the values expected by the event. + * + * @param name name of the bro event to send + * + * @param num_vals number of entries in \a vals + * + * @param vals the values to be given to the event + */ + void SendEvent(const char* name, const int num_vals, threading::Value* *vals); + /** * Reports an informational message from the child thread. The main * thread will pass this to the Reporter once received. @@ -393,7 +405,7 @@ protected: }; /** - * A paremeterized InputMessage that stores a pointer to an argument object. + * A parameterized InputMessage that stores a pointer to an argument object. * Normally, the objects will be used from the Process() callback. */ template diff --git a/src/threading/SerialTypes.cc b/src/threading/SerialTypes.cc index a115c37351..7b686fc30d 100644 --- a/src/threading/SerialTypes.cc +++ b/src/threading/SerialTypes.cc @@ -4,6 +4,14 @@ #include "SerialTypes.h" #include "SerializationFormat.h" #include "Reporter.h" +// The following are required for ValueToVal. +#include "Val.h" +#include "BroString.h" +#include "RE.h" +#include "module_util.h" +#include "ID.h" +#include "Expr.h" +#include "Scope.h" using namespace threading; @@ -346,7 +354,6 @@ bool Value::Write(SerializationFormat* fmt) const case IPv6: return fmt->Write((char)6, "addr-family") && fmt->Write(val.addr_val.in.in6, "addr-in6"); - break; } // Can't be reached. @@ -366,7 +373,6 @@ bool Value::Write(SerializationFormat* fmt) const case IPv6: return fmt->Write((char)6, "subnet-family") && fmt->Write(val.subnet_val.prefix.in.in6, "subnet-in6"); - break; } // Can't be reached. @@ -417,5 +423,212 @@ bool Value::Write(SerializationFormat* fmt) const type_name(type)); } + // unreachable return false; } + +void Value::delete_value_ptr_array(Value** vals, int num_fields) + { + for ( int i = 0; i < num_fields; ++i ) + delete vals[i]; + + delete [] vals; + } + +Val* Value::ValueToVal(const std::string& source, const Value* val, bool& have_error) + { + if ( have_error ) + return nullptr; + + if ( ! val->present ) + return nullptr; // unset field + + switch ( val->type ) { + case TYPE_BOOL: + return val_mgr->Bool(val->val.int_val)->Ref(); + + case TYPE_INT: + return val_mgr->Int(val->val.int_val).release(); + + case TYPE_COUNT: + case TYPE_COUNTER: + return val_mgr->Count(val->val.int_val).release(); + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + return new Val(val->val.double_val, val->type); + + case TYPE_STRING: + { + BroString *s = new BroString((const u_char*)val->val.string_val.data, val->val.string_val.length, true); + return new StringVal(s); + } + + case TYPE_PORT: + return val_mgr->Port(val->val.port_val.port, val->val.port_val.proto)->Ref(); + + case TYPE_ADDR: + { + IPAddr* addr = nullptr; + switch ( val->val.addr_val.family ) { + case IPv4: + addr = new IPAddr(val->val.addr_val.in.in4); + break; + + case IPv6: + addr = new IPAddr(val->val.addr_val.in.in6); + break; + + default: + assert(false); + } + + AddrVal* addrval = new AddrVal(*addr); + delete addr; + return addrval; + } + + case TYPE_SUBNET: + { + IPAddr* addr = nullptr; + switch ( val->val.subnet_val.prefix.family ) { + case IPv4: + addr = new IPAddr(val->val.subnet_val.prefix.in.in4); + break; + + case IPv6: + addr = new IPAddr(val->val.subnet_val.prefix.in.in6); + break; + + default: + assert(false); + } + + SubNetVal* subnetval = new SubNetVal(*addr, val->val.subnet_val.length); + delete addr; + return subnetval; + } + + case TYPE_PATTERN: + { + RE_Matcher* re = new RE_Matcher(val->val.pattern_text_val); + re->Compile(); + return new PatternVal(re); + } + + case TYPE_TABLE: + { + IntrusivePtr set_index; + if ( val->val.set_val.size == 0 && val->subtype == TYPE_VOID ) + // don't know type - unspecified table. + set_index = make_intrusive(); + else + { + // all entries have to have the same type... + TypeTag stag = val->subtype; + if ( stag == TYPE_VOID ) + TypeTag stag = val->val.set_val.vals[0]->type; + + IntrusivePtr index_type; + + if ( stag == TYPE_ENUM ) + { + // Enums are not a base-type, so need to look it up. + const auto& sv = val->val.set_val.vals[0]->val.string_val; + std::string enum_name(sv.data, sv.length); + auto enum_id = global_scope()->Lookup(enum_name); + + if ( ! enum_id ) + { + reporter->Warning("Value '%s' of source '%s' is not a valid enum.", + enum_name.data(), source.c_str()); + + have_error = true; + return nullptr; + } + + index_type = {NewRef{}, enum_id->Type()->AsEnumType()}; + } + else + index_type = base_type(stag); + + set_index = make_intrusive(index_type); + set_index->Append(std::move(index_type)); + } + + auto s = make_intrusive(std::move(set_index), nullptr); + TableVal* t = new TableVal(std::move(s)); + for ( int j = 0; j < val->val.set_val.size; j++ ) + { + Val* assignval = ValueToVal(source, val->val.set_val.vals[j], have_error); + + t->Assign(assignval, nullptr); + Unref(assignval); // index is not consumed by assign. + } + + return t; + } + + case TYPE_VECTOR: + { + IntrusivePtr type; + + if ( val->val.vector_val.size == 0 && val->subtype == TYPE_VOID ) + // don't know type - unspecified table. + type = base_type(TYPE_ANY); + else + { + // all entries have to have the same type... + if ( val->subtype == TYPE_VOID ) + type = base_type(val->val.vector_val.vals[0]->type); + else + type = base_type(val->subtype); + } + + auto vt = make_intrusive(std::move(type)); + VectorVal* v = new VectorVal(vt.get()); + + for ( int j = 0; j < val->val.vector_val.size; j++ ) + v->Assign(j, ValueToVal(source, val->val.vector_val.vals[j], have_error)); + + return v; + } + + case TYPE_ENUM: { + // Convert to string first to not have to deal with missing + // \0's... + std::string enum_string(val->val.string_val.data, val->val.string_val.length); + + // let's try looking it up by global ID. + auto id = lookup_ID(enum_string.c_str(), GLOBAL_MODULE_NAME); + if ( ! id || ! id->IsEnumConst() ) + { + reporter->Warning("Value '%s' for source '%s' is not a valid enum.", + enum_string.c_str(), source.c_str()); + + have_error = true; + return nullptr; + } + + EnumType* t = id->Type()->AsEnumType(); + int intval = t->Lookup(id->ModuleName(), id->Name()); + if ( intval < 0 ) + { + reporter->Warning("Enum value '%s' for source '%s' not found.", + enum_string.c_str(), source.c_str()); + + have_error = true; + return nullptr; + } + + return t->GetVal(intval).release(); + } + + default: + reporter->InternalError("Unsupported type in SerialTypes::ValueToVal from source %s", source.c_str()); + } + + assert(false); + return nullptr; + } diff --git a/src/threading/SerialTypes.h b/src/threading/SerialTypes.h index f424eb9143..0969e5b6db 100644 --- a/src/threading/SerialTypes.h +++ b/src/threading/SerialTypes.h @@ -187,6 +187,26 @@ struct Value { * method is thread-safe. */ static bool IsCompatibleType(BroType* t, bool atomic_only=false); + /** + * Convenience function to delete an array of value pointers. + * @param vals Array of values + * @param num_fields Number of members + */ + static void delete_value_ptr_array(Value** vals, int num_fields); + + /** + * Convert threading::Value to an internal Zeek type, just using the information given in the threading::Value. + * + * @param source Name of the source of this threading value. This is used for warnings that are raised + * in case an error occurs. + * @param val Threading Value to convert to a Zeek Val. + * @param have_error Reference to a boolean. This should be set to false when passed in and is set to true + * in case an error occurs. If this is set to false when the function is called, the function + * immediately aborts. + * @return Val representation of the threading::Value. nullptr on error. + */ + static Val* ValueToVal(const std::string& source, const threading::Value* val, bool& have_error); + private: friend class ::IPAddr; Value(const Value& other) { } // Disabled. diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.config.errors/errout b/testing/btest/Baseline/scripts.base.frameworks.input.config.errors/errout index 792f50d8c8..2b835abe07 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.config.errors/errout +++ b/testing/btest/Baseline/scripts.base.frameworks.input.config.errors/errout @@ -5,7 +5,7 @@ warning: ../configfile/Input::READER_CONFIG: Field: testbool Invalid value for b warning: ../configfile/Input::READER_CONFIG: Could not convert line 'testbool A B' to value. Ignoring line. warning: ../configfile/Input::READER_CONFIG: String 'A' contained no parseable number warning: ../configfile/Input::READER_CONFIG: Could not convert line 'testcount A' to value. Ignoring line. -warning: Value 'unknown' for stream 'configuration' is not a valid enum. +warning: Value 'unknown' for source 'thread ../configfile/Input::READER_CONFIG' is not a valid enum. error: SendEvent for event InputConfig::new_value failed warning: ../configfile/Input::READER_CONFIG: Option 'testbooool' does not exist. Ignoring line. warning: ../configfile/Input::READER_CONFIG: Option 'test_any' has type 'any', which is not supported for file input. Ignoring line. From aafced6b196419b8b7fb6be8b94ddf51375d0590 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 18 May 2020 14:58:52 -0700 Subject: [PATCH 12/19] Include pcap.h instead of pcap/dlt.h in packet-fuzzer --- src/fuzzers/packet-fuzzer.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/fuzzers/packet-fuzzer.cc b/src/fuzzers/packet-fuzzer.cc index eab8202ce1..d3bb28b806 100644 --- a/src/fuzzers/packet-fuzzer.cc +++ b/src/fuzzers/packet-fuzzer.cc @@ -7,7 +7,9 @@ #include "FuzzBuffer.h" #include "fuzzer-setup.h" -#include "pcap/dlt.h" +extern "C" { +#include +} extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { From d7927622e93e7ff72bfa3782eef2b696fb6a162e Mon Sep 17 00:00:00 2001 From: Dominik Charousset Date: Thu, 14 May 2020 11:51:46 +0200 Subject: [PATCH 13/19] Upgrade to latest Broker changes for CAF 0.18 --- aux/broker | 2 +- src/broker/Store.cc | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/aux/broker b/aux/broker index 70fa315e8b..1abf27595e 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 70fa315e8b777056073a9e67f42b99573ef18231 +Subproject commit 1abf27595e0c6cd21ecc1b69412514983d966a5c diff --git a/src/broker/Store.cc b/src/broker/Store.cc index 81138edcd9..1d1ff7addb 100644 --- a/src/broker/Store.cc +++ b/src/broker/Store.cc @@ -85,13 +85,13 @@ broker::backend to_backend_type(BifEnum::Broker::BackendType type) { switch ( type ) { case BifEnum::Broker::MEMORY: - return broker::memory; + return broker::backend::memory; case BifEnum::Broker::SQLITE: - return broker::sqlite; + return broker::backend::sqlite; case BifEnum::Broker::ROCKSDB: - return broker::rocksdb; + return broker::backend::rocksdb; } throw std::runtime_error("unknown broker backend"); @@ -101,14 +101,14 @@ broker::backend_options to_backend_options(broker::backend backend, RecordVal* options) { switch ( backend ) { - case broker::sqlite: + case broker::backend::sqlite: { auto path = options->Lookup(0)->AsRecordVal() ->Lookup(0)->AsStringVal()->CheckString(); return {{"path", path}}; } - case broker::rocksdb: + case broker::backend::rocksdb: { auto path = options->Lookup(1)->AsRecordVal() ->Lookup(0)->AsStringVal()->CheckString(); From fda8b98ac783fe14830a8e1477e5f5f043382009 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 19 May 2020 11:36:03 -0700 Subject: [PATCH 14/19] Update submodule(s) [nomail] --- aux/bifcl | 2 +- aux/binpac | 2 +- aux/broker | 2 +- aux/zeek-aux | 2 +- aux/zeekctl | 2 +- cmake | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/aux/bifcl b/aux/bifcl index abd57da8ba..95fece382d 160000 --- a/aux/bifcl +++ b/aux/bifcl @@ -1 +1 @@ -Subproject commit abd57da8ba6c9337a5ed79cabbf962f71b38d62d +Subproject commit 95fece382d34bca72572cc863e1182b31a1b9945 diff --git a/aux/binpac b/aux/binpac index 0730797611..4d3d10cd54 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 07307976115c90095af183ce950364bc1982027c +Subproject commit 4d3d10cd54b1aa64f30d2fd433252f353c6ea6e0 diff --git a/aux/broker b/aux/broker index 5d5e4f02ef..6974924007 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 5d5e4f02ef3db2415c0272beaaa795ee3799272d +Subproject commit 6974924007765f70d95e5cf123b6256048ae3af7 diff --git a/aux/zeek-aux b/aux/zeek-aux index f4a659ee89..8615abced8 160000 --- a/aux/zeek-aux +++ b/aux/zeek-aux @@ -1 +1 @@ -Subproject commit f4a659ee89e95f93ff2900cde7984f032a8113e9 +Subproject commit 8615abced86b5559fa3203264be55e664b887094 diff --git a/aux/zeekctl b/aux/zeekctl index 881fe0083a..bee11c6392 160000 --- a/aux/zeekctl +++ b/aux/zeekctl @@ -1 +1 @@ -Subproject commit 881fe0083a512b3731738fb0272241e47b272e09 +Subproject commit bee11c63923b69a4468838d56067f15bc6ae3a52 diff --git a/cmake b/cmake index 7666cd4628..23f4b88f91 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 7666cd462888b0eee173040c6888f41930a8be0b +Subproject commit 23f4b88f91c537c59ef9a3ad56ec08f021ec2b2c From 1e4374bd2761ca42b01ffbb667b0cb0177071b84 Mon Sep 17 00:00:00 2001 From: Justin Azoff Date: Wed, 20 May 2020 09:44:06 -0400 Subject: [PATCH 15/19] Skip fuzz inputs that have more than 64 chunks --- src/fuzzers/FuzzBuffer.cc | 17 +++++++++++++++++ src/fuzzers/FuzzBuffer.h | 5 +++++ src/fuzzers/packet-fuzzer.cc | 3 +++ src/fuzzers/pop3-fuzzer.cc | 3 +++ 4 files changed, 28 insertions(+) diff --git a/src/fuzzers/FuzzBuffer.cc b/src/fuzzers/FuzzBuffer.cc index ebae1a1db1..098ce92f5c 100644 --- a/src/fuzzers/FuzzBuffer.cc +++ b/src/fuzzers/FuzzBuffer.cc @@ -17,6 +17,23 @@ bool zeek::detail::FuzzBuffer::Valid() const return true; } +int zeek::detail::FuzzBuffer::ChunkCount() const + { + auto pos = begin; + int chunks = 0; + while (pos < end) + { + pos = (const unsigned char*)memmem(pos, end - pos, + PKT_MAGIC, PKT_MAGIC_LEN); + if ( ! pos ) + break; + pos += PKT_MAGIC_LEN + 1; + chunks++; + } + + return chunks; + } + std::optional zeek::detail::FuzzBuffer::Next() { if ( begin == end ) diff --git a/src/fuzzers/FuzzBuffer.h b/src/fuzzers/FuzzBuffer.h index 1b9cdb58a3..f79f20091e 100644 --- a/src/fuzzers/FuzzBuffer.h +++ b/src/fuzzers/FuzzBuffer.h @@ -43,6 +43,11 @@ public: */ bool Valid() const; + /** + * @return the number of chunks in the fuzz buffer object + */ + int ChunkCount() const; + /** * @return the next chunk to deliver, if one could be extracted */ diff --git a/src/fuzzers/packet-fuzzer.cc b/src/fuzzers/packet-fuzzer.cc index d3bb28b806..59002f6c79 100644 --- a/src/fuzzers/packet-fuzzer.cc +++ b/src/fuzzers/packet-fuzzer.cc @@ -18,6 +18,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) if ( ! fb.Valid() ) return 0; + if ( fb.ChunkCount() > 64 ) + return 0; + for ( ; ; ) { auto chunk = fb.Next(); diff --git a/src/fuzzers/pop3-fuzzer.cc b/src/fuzzers/pop3-fuzzer.cc index 5cc4776aad..2c6e01fc5c 100644 --- a/src/fuzzers/pop3-fuzzer.cc +++ b/src/fuzzers/pop3-fuzzer.cc @@ -50,6 +50,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) if ( ! fb.Valid() ) return 0; + if ( fb.ChunkCount() > 64 ) + return 0; + auto conn = add_connection(); auto a = add_analyzer(conn); From 6d0599c5373566f3aed0f79d1faa1b4b845e0a63 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 20 May 2020 10:56:46 -0700 Subject: [PATCH 16/19] Improve standalone fuzz driver timing output --- CHANGES | 6 ++++++ VERSION | 2 +- src/fuzzers/standalone-driver.cc | 5 ++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index f20fcc5603..3f87234127 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,10 @@ +3.2.0-dev.554 | 2020-05-20 10:56:46 -0700 + + * Improve standalone fuzz driver timing output (Jon Siwek, Corelight) + + * Skip fuzz inputs that have more than 64 chunks (Justin Azoff, Corelight) + 3.2.0-dev.550 | 2020-05-19 10:50:42 -0700 * Upgrade to latest Broker changes for CAF 0.18 (Dominik Charousset, Corelight) diff --git a/VERSION b/VERSION index 9730e9d65d..ee2820da49 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.2.0-dev.550 +3.2.0-dev.554 diff --git a/src/fuzzers/standalone-driver.cc b/src/fuzzers/standalone-driver.cc index 69ca7202cf..64cabaefc8 100644 --- a/src/fuzzers/standalone-driver.cc +++ b/src/fuzzers/standalone-driver.cc @@ -17,6 +17,7 @@ int main(int argc, char** argv) printf("Standalone fuzzer processing %d inputs\n", num_inputs); LLVMFuzzerInitialize(&argc, &argv); + auto fuzz_start = high_resolution_clock::now(); for ( auto i = 0; i < num_inputs; ++i ) { @@ -60,5 +61,7 @@ int main(int argc, char** argv) auto agg_stop = high_resolution_clock::now(); auto agg_dt = duration(agg_stop - agg_start).count(); - printf("Processed %d inputs in %fs\n", num_inputs, agg_dt); + auto fuzz_dt = duration(agg_stop - fuzz_start).count(); + printf("Processed %d inputs in %fs (%fs w/ initialization), avg = %fs\n", + num_inputs, fuzz_dt, agg_dt, fuzz_dt / num_inputs); } From 7d98d816d83c276faaaddaf83b3991ec9f0b80d1 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 20 May 2020 11:19:08 -0700 Subject: [PATCH 17/19] Disable output of Reporter messages to stderr in fuzz targets --- CHANGES | 4 ++++ VERSION | 2 +- src/fuzzers/fuzzer-setup.h | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 3f87234127..f1e089d93b 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +3.2.0-dev.555 | 2020-05-20 11:19:08 -0700 + + * Disable output of Reporter messages to stderr in fuzz targets (Jon Siwek, Corelight) + 3.2.0-dev.554 | 2020-05-20 10:56:46 -0700 * Improve standalone fuzz driver timing output (Jon Siwek, Corelight) diff --git a/VERSION b/VERSION index ee2820da49..18a3b72a3c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.2.0-dev.554 +3.2.0-dev.555 diff --git a/src/fuzzers/fuzzer-setup.h b/src/fuzzers/fuzzer-setup.h index e692ddd3c4..976aa9126e 100644 --- a/src/fuzzers/fuzzer-setup.h +++ b/src/fuzzers/fuzzer-setup.h @@ -32,6 +32,9 @@ extern "C" int LLVMFuzzerInitialize(int* argc, char*** argv) options.scripts_to_load.emplace_back("local.zeek"); options.script_options_to_set.emplace_back("Site::local_nets={10.0.0.0/8}"); options.script_options_to_set.emplace_back("Log::default_writer=Log::WRITER_NONE"); + options.script_options_to_set.emplace_back("Reporter::info_to_stderr=F"); + options.script_options_to_set.emplace_back("Reporter::warnings_to_stderr=F"); + options.script_options_to_set.emplace_back("Reporter::errors_to_stderr=F"); options.deterministic_mode = true; options.ignore_checksums = true; options.abort_on_scripting_errors = true; From 6aa6eea7bc44993c00ee9d969b328f8347ec41b2 Mon Sep 17 00:00:00 2001 From: Justin Azoff Date: Thu, 21 May 2020 12:55:00 -0400 Subject: [PATCH 18/19] Speed up ChunkCount validity check When counting chunks for the purpose of a Valid check, only count up to chunk_count_limit + 1 chunks. This speeds up the skipping of the 70,000 chunk test file considerably. Before: Processed 1 inputs in 0.025517s After: Processed 1 inputs in 0.000620s --- src/fuzzers/FuzzBuffer.cc | 6 +++--- src/fuzzers/FuzzBuffer.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/fuzzers/FuzzBuffer.cc b/src/fuzzers/FuzzBuffer.cc index 0eb166c786..393e830d78 100644 --- a/src/fuzzers/FuzzBuffer.cc +++ b/src/fuzzers/FuzzBuffer.cc @@ -14,18 +14,18 @@ bool zeek::detail::FuzzBuffer::Valid(int chunk_count_limit) const if ( memcmp(begin, PKT_MAGIC, PKT_MAGIC_LEN) != 0) return false; - if ( ChunkCount() > chunk_count_limit ) + if ( ChunkCount(chunk_count_limit + 1) > chunk_count_limit ) return false; return true; } -int zeek::detail::FuzzBuffer::ChunkCount() const +int zeek::detail::FuzzBuffer::ChunkCount(int chunk_count_limit) const { auto pos = begin; int chunks = 0; - while ( pos < end ) + while ( pos < end && (chunk_count_limit == 0 || chunks < chunk_count_limit) ) { pos = (const unsigned char*)memmem(pos, end - pos, PKT_MAGIC, PKT_MAGIC_LEN); diff --git a/src/fuzzers/FuzzBuffer.h b/src/fuzzers/FuzzBuffer.h index 1959e6f18b..830ed65528 100644 --- a/src/fuzzers/FuzzBuffer.h +++ b/src/fuzzers/FuzzBuffer.h @@ -49,7 +49,7 @@ public: /** * @return the number of chunks in the fuzz buffer object */ - int ChunkCount() const; + int ChunkCount(int chunk_count_limit = 0) const; /** * @return the next chunk to deliver, if one could be extracted From d8327ee5266b9ee43b4bd99f2350e26214a90373 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 21 May 2020 12:36:15 -0700 Subject: [PATCH 19/19] GH-983: Fix opaque Broker types lacking a Type after (de)serialization --- src/broker/Data.h | 21 ++++++++++++--- src/broker/Store.h | 4 ++- testing/btest/Baseline/broker.opaque/out | 33 +++++++++++++++--------- testing/btest/broker/opaque.zeek | 17 +++++++++--- 4 files changed, 54 insertions(+), 21 deletions(-) diff --git a/src/broker/Data.h b/src/broker/Data.h index cf5f06b90d..5979de83ae 100644 --- a/src/broker/Data.h +++ b/src/broker/Data.h @@ -116,6 +116,7 @@ public: protected: DataVal() + : OpaqueVal(bro_broker::opaque_of_data_type) {} DECLARE_OPAQUE_VALUE(bro_broker::DataVal) @@ -236,7 +237,10 @@ public: broker::set::iterator it; protected: - SetIterator() {} + SetIterator() + : OpaqueVal(bro_broker::opaque_of_set_iterator) + {} + DECLARE_OPAQUE_VALUE(bro_broker::SetIterator) }; @@ -253,7 +257,10 @@ public: broker::table::iterator it; protected: - TableIterator() {} + TableIterator() + : OpaqueVal(bro_broker::opaque_of_table_iterator) + {} + DECLARE_OPAQUE_VALUE(bro_broker::TableIterator) }; @@ -270,7 +277,10 @@ public: broker::vector::iterator it; protected: - VectorIterator() {} + VectorIterator() + : OpaqueVal(bro_broker::opaque_of_vector_iterator) + {} + DECLARE_OPAQUE_VALUE(bro_broker::VectorIterator) }; @@ -287,7 +297,10 @@ public: broker::vector::iterator it; protected: - RecordIterator() {} + RecordIterator() + : OpaqueVal(bro_broker::opaque_of_record_iterator) + {} + DECLARE_OPAQUE_VALUE(bro_broker::RecordIterator) }; diff --git a/src/broker/Store.h b/src/broker/Store.h index 09799c8c68..e8e4fec03d 100644 --- a/src/broker/Store.h +++ b/src/broker/Store.h @@ -103,7 +103,9 @@ public: broker::store::proxy proxy; protected: - StoreHandleVal() = default; + StoreHandleVal() + : OpaqueVal(bro_broker::opaque_of_store_handle) + {} DECLARE_OPAQUE_VALUE(StoreHandleVal) }; diff --git a/testing/btest/Baseline/broker.opaque/out b/testing/btest/Baseline/broker.opaque/out index 35bf821c47..b4293c0100 100644 --- a/testing/btest/Baseline/broker.opaque/out +++ b/testing/btest/Baseline/broker.opaque/out @@ -1,53 +1,62 @@ ============ Topk +opaque of topk [b, a, c] [b, a, c] ============ HLL 3.000069 +opaque of cardinality 3.000069 3.000069 ============ Bloom 0 1 +opaque of bloomfilter 0 1 ============ Hashes +opaque of md5 5b9164ad6f496d9dee12ec7634ce253f 5b9164ad6f496d9dee12ec7634ce253f +opaque of sha1 30ae97492ce1da88d0e7117ace0a60a6f9e1e0bc 30ae97492ce1da88d0e7117ace0a60a6f9e1e0bc +opaque of sha256 25b6746d5172ed6352966a013d93ac846e1110d5a25e8f183b5931f4688842a1 25b6746d5172ed6352966a013d93ac846e1110d5a25e8f183b5931f4688842a1 ============ X509 +opaque of x509 [version=3, serial=040000000001154B5AC394, subject=CN=GlobalSign Root CA,OU=Root CA,O=GlobalSign nv-sa,C=BE, issuer=CN=GlobalSign Root CA,OU=Root CA,O=GlobalSign nv-sa,C=BE, cn=GlobalSign Root CA, not_valid_before=904651200.0, not_valid_after=1832673600.0, key_alg=rsaEncryption, sig_alg=sha1WithRSAEncryption, key_type=rsa, key_length=2048, exponent=65537, curve=] [version=3, serial=040000000001154B5AC394, subject=CN=GlobalSign Root CA,OU=Root CA,O=GlobalSign nv-sa,C=BE, issuer=CN=GlobalSign Root CA,OU=Root CA,O=GlobalSign nv-sa,C=BE, cn=GlobalSign Root CA, not_valid_before=904651200.0, not_valid_after=1832673600.0, key_alg=rsaEncryption, sig_alg=sha1WithRSAEncryption, key_type=rsa, key_length=2048, exponent=65537, curve=] ============ Entropy +opaque of entropy [entropy=4.715374, chi_square=591.981818, mean=75.472727, monte_carlo_pi=4.0, serial_correlation=-0.11027] [entropy=4.715374, chi_square=591.981818, mean=75.472727, monte_carlo_pi=4.0, serial_correlation=-0.11027] ============ broker::Data +opaque of Broker::Data broker::data{{hi, there}} broker::data{{hi, there}} T ============ broker::Set -| [data=broker::data{!}] | [data=broker::data{!}] +| [data=broker::data{!}] | [data=broker::data{!}] | opaque of Broker::SetIterator > [data=broker::data{hi}] | [data=broker::data{hi}] -| [data=broker::data{hi}] | [data=broker::data{hi}] +| [data=broker::data{hi}] | [data=broker::data{hi}] | opaque of Broker::SetIterator > [data=broker::data{there}] | [data=broker::data{there}] -| [data=broker::data{there}] | [data=broker::data{there}] +| [data=broker::data{there}] | [data=broker::data{there}] | opaque of Broker::SetIterator ============ broker::Table -| [key=[data=broker::data{!}], val=[data=broker::data{30}]] | [key=[data=broker::data{!}], val=[data=broker::data{30}]] +| [key=[data=broker::data{!}], val=[data=broker::data{30}]] | [key=[data=broker::data{!}], val=[data=broker::data{30}]] | opaque of Broker::TableIterator > [key=[data=broker::data{hi}], val=[data=broker::data{10}]] | [key=[data=broker::data{hi}], val=[data=broker::data{10}]] -| [key=[data=broker::data{hi}], val=[data=broker::data{10}]] | [key=[data=broker::data{hi}], val=[data=broker::data{10}]] +| [key=[data=broker::data{hi}], val=[data=broker::data{10}]] | [key=[data=broker::data{hi}], val=[data=broker::data{10}]] | opaque of Broker::TableIterator > [key=[data=broker::data{there}], val=[data=broker::data{20}]] | [key=[data=broker::data{there}], val=[data=broker::data{20}]] -| [key=[data=broker::data{there}], val=[data=broker::data{20}]] | [key=[data=broker::data{there}], val=[data=broker::data{20}]] +| [key=[data=broker::data{there}], val=[data=broker::data{20}]] | [key=[data=broker::data{there}], val=[data=broker::data{20}]] | opaque of Broker::TableIterator ============ broker::Vector -| [data=broker::data{hi}] | [data=broker::data{hi}] +| [data=broker::data{hi}] | [data=broker::data{hi}] | opaque of Broker::VectorIterator > [data=broker::data{there}] | [data=broker::data{there}] -| [data=broker::data{there}] | [data=broker::data{there}] +| [data=broker::data{there}] | [data=broker::data{there}] | opaque of Broker::VectorIterator > [data=broker::data{!}] | [data=broker::data{!}] -| [data=broker::data{!}] | [data=broker::data{!}] +| [data=broker::data{!}] | [data=broker::data{!}] | opaque of Broker::VectorIterator ============ broker::Record -| [data=broker::data{hi}] | [data=broker::data{hi}] +| [data=broker::data{hi}] | [data=broker::data{hi}] | opaque of Broker::RecordIterator > [data=broker::data{there}] | [data=broker::data{there}] -| [data=broker::data{there}] | [data=broker::data{there}] +| [data=broker::data{there}] | [data=broker::data{there}] | opaque of Broker::RecordIterator > [data=broker::data{!}] | [data=broker::data{!}] -| [data=broker::data{!}] | [data=broker::data{!}] +| [data=broker::data{!}] | [data=broker::data{!}] | opaque of Broker::RecordIterator diff --git a/testing/btest/broker/opaque.zeek b/testing/btest/broker/opaque.zeek index e0a3bef6c7..41823a59ed 100644 --- a/testing/btest/broker/opaque.zeek +++ b/testing/btest/broker/opaque.zeek @@ -11,6 +11,7 @@ event zeek_init() topk_add(k1, "b"); topk_add(k1, "c"); local k2 = Broker::__opaque_clone_through_serialization(k1); + print type_name(k2); print topk_get_top(k1, 5); topk_add(k1, "shoulnotshowup"); print topk_get_top(k2, 5); @@ -23,6 +24,7 @@ event zeek_init() print hll_cardinality_estimate(c1); local c2 = Broker::__opaque_clone_through_serialization(c1); + print type_name(c2); hll_cardinality_add(c1, 2004); print hll_cardinality_estimate(c2); @@ -38,6 +40,7 @@ event zeek_init() print bloomfilter_lookup(bf_cnt, 0); print bloomfilter_lookup(bf_cnt, 42); local bf_copy = Broker::__opaque_clone_through_serialization(bf_cnt); + print type_name(bf_copy); bloomfilter_add(bf_cnt, 0); print bloomfilter_lookup(bf_copy, 0); print bloomfilter_lookup(bf_copy, 42); @@ -48,6 +51,7 @@ event zeek_init() local md5a = md5_hash_init(); md5_hash_update(md5a, "one"); local md5b = Broker::__opaque_clone_through_serialization(md5a); + print type_name(md5b); md5_hash_update(md5a, "two"); md5_hash_update(md5b, "two"); print md5_hash_finish(md5a); @@ -56,6 +60,7 @@ event zeek_init() local sha1a = sha1_hash_init(); sha1_hash_update(sha1a, "one"); local sha1b = Broker::__opaque_clone_through_serialization(sha1a); + print type_name(sha1b); sha1_hash_update(sha1a, "two"); sha1_hash_update(sha1b, "two"); print sha1_hash_finish(sha1a); @@ -64,6 +69,7 @@ event zeek_init() local sha256a = sha256_hash_init(); sha256_hash_update(sha256a, "one"); local sha256b = Broker::__opaque_clone_through_serialization(sha256a); + print type_name(sha256b); sha256_hash_update(sha256a, "two"); sha256_hash_update(sha256b, "two"); print sha256_hash_finish(sha256a); @@ -72,6 +78,7 @@ event zeek_init() print "============ X509"; local x509 = x509_from_der("\x30\x82\x03\x75\x30\x82\x02\x5D\xA0\x03\x02\x01\x02\x02\x0B\x04\x00\x00\x00\x00\x01\x15\x4B\x5A\xC3\x94\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x57\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x42\x45\x31\x19\x30\x17\x06\x03\x55\x04\x0A\x13\x10\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x6E\x76\x2D\x73\x61\x31\x10\x30\x0E\x06\x03\x55\x04\x0B\x13\x07\x52\x6F\x6F\x74\x20\x43\x41\x31\x1B\x30\x19\x06\x03\x55\x04\x03\x13\x12\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x1E\x17\x0D\x39\x38\x30\x39\x30\x31\x31\x32\x30\x30\x30\x30\x5A\x17\x0D\x32\x38\x30\x31\x32\x38\x31\x32\x30\x30\x30\x30\x5A\x30\x57\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x42\x45\x31\x19\x30\x17\x06\x03\x55\x04\x0A\x13\x10\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x6E\x76\x2D\x73\x61\x31\x10\x30\x0E\x06\x03\x55\x04\x0B\x13\x07\x52\x6F\x6F\x74\x20\x43\x41\x31\x1B\x30\x19\x06\x03\x55\x04\x03\x13\x12\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xDA\x0E\xE6\x99\x8D\xCE\xA3\xE3\x4F\x8A\x7E\xFB\xF1\x8B\x83\x25\x6B\xEA\x48\x1F\xF1\x2A\xB0\xB9\x95\x11\x04\xBD\xF0\x63\xD1\xE2\x67\x66\xCF\x1C\xDD\xCF\x1B\x48\x2B\xEE\x8D\x89\x8E\x9A\xAF\x29\x80\x65\xAB\xE9\xC7\x2D\x12\xCB\xAB\x1C\x4C\x70\x07\xA1\x3D\x0A\x30\xCD\x15\x8D\x4F\xF8\xDD\xD4\x8C\x50\x15\x1C\xEF\x50\xEE\xC4\x2E\xF7\xFC\xE9\x52\xF2\x91\x7D\xE0\x6D\xD5\x35\x30\x8E\x5E\x43\x73\xF2\x41\xE9\xD5\x6A\xE3\xB2\x89\x3A\x56\x39\x38\x6F\x06\x3C\x88\x69\x5B\x2A\x4D\xC5\xA7\x54\xB8\x6C\x89\xCC\x9B\xF9\x3C\xCA\xE5\xFD\x89\xF5\x12\x3C\x92\x78\x96\xD6\xDC\x74\x6E\x93\x44\x61\xD1\x8D\xC7\x46\xB2\x75\x0E\x86\xE8\x19\x8A\xD5\x6D\x6C\xD5\x78\x16\x95\xA2\xE9\xC8\x0A\x38\xEB\xF2\x24\x13\x4F\x73\x54\x93\x13\x85\x3A\x1B\xBC\x1E\x34\xB5\x8B\x05\x8C\xB9\x77\x8B\xB1\xDB\x1F\x20\x91\xAB\x09\x53\x6E\x90\xCE\x7B\x37\x74\xB9\x70\x47\x91\x22\x51\x63\x16\x79\xAE\xB1\xAE\x41\x26\x08\xC8\x19\x2B\xD1\x46\xAA\x48\xD6\x64\x2A\xD7\x83\x34\xFF\x2C\x2A\xC1\x6C\x19\x43\x4A\x07\x85\xE7\xD3\x7C\xF6\x21\x68\xEF\xEA\xF2\x52\x9F\x7F\x93\x90\xCF\x02\x03\x01\x00\x01\xA3\x42\x30\x40\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x60\x7B\x66\x1A\x45\x0D\x97\xCA\x89\x50\x2F\x7D\x04\xCD\x34\xA8\xFF\xFC\xFD\x4B\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\xD6\x73\xE7\x7C\x4F\x76\xD0\x8D\xBF\xEC\xBA\xA2\xBE\x34\xC5\x28\x32\xB5\x7C\xFC\x6C\x9C\x2C\x2B\xBD\x09\x9E\x53\xBF\x6B\x5E\xAA\x11\x48\xB6\xE5\x08\xA3\xB3\xCA\x3D\x61\x4D\xD3\x46\x09\xB3\x3E\xC3\xA0\xE3\x63\x55\x1B\xF2\xBA\xEF\xAD\x39\xE1\x43\xB9\x38\xA3\xE6\x2F\x8A\x26\x3B\xEF\xA0\x50\x56\xF9\xC6\x0A\xFD\x38\xCD\xC4\x0B\x70\x51\x94\x97\x98\x04\xDF\xC3\x5F\x94\xD5\x15\xC9\x14\x41\x9C\xC4\x5D\x75\x64\x15\x0D\xFF\x55\x30\xEC\x86\x8F\xFF\x0D\xEF\x2C\xB9\x63\x46\xF6\xAA\xFC\xDF\xBC\x69\xFD\x2E\x12\x48\x64\x9A\xE0\x95\xF0\xA6\xEF\x29\x8F\x01\xB1\x15\xB5\x0C\x1D\xA5\xFE\x69\x2C\x69\x24\x78\x1E\xB3\xA7\x1C\x71\x62\xEE\xCA\xC8\x97\xAC\x17\x5D\x8A\xC2\xF8\x47\x86\x6E\x2A\xC4\x56\x31\x95\xD0\x67\x89\x85\x2B\xF9\x6C\xA6\x5D\x46\x9D\x0C\xAA\x82\xE4\x99\x51\xDD\x70\xB7\xDB\x56\x3D\x61\xE4\x6A\xE1\x5C\xD6\xF6\xFE\x3D\xDE\x41\xCC\x07\xAE\x63\x52\xBF\x53\x53\xF4\x2B\xE9\xC7\xFD\xB6\xF7\x82\x5F\x85\xD2\x41\x18\xDB\x81\xB3\x04\x1C\xC5\x1F\xA4\x80\x6F\x15\x20\xC9\xDE\x0C\x88\x0A\x1D\xD6\x66\x55\xE2\xFC\x48\xC9\x29\x26\x69\xE0"); local x5092 = Broker::__opaque_clone_through_serialization(x509); + print type_name(x5092); print x509_parse(x509); print x509_parse(x5092); @@ -79,6 +86,7 @@ event zeek_init() local handle = entropy_test_init(); entropy_test_add(handle, "dh3Hie02uh^s#Sdf9L3frd243h$d78r2G4cM6*Q05d(7rh46f!0|4-f"); local handle2 = Broker::__opaque_clone_through_serialization(handle); + print type_name(handle2); print entropy_test_finish(handle); print entropy_test_finish(handle2); @@ -87,6 +95,7 @@ event zeek_init() Broker::set_insert(s1, "hi"); Broker::set_insert(s1, "there"); local d2 = Broker::__opaque_clone_through_serialization(s1$data); + print type_name(d2); print s1$data; print d2; print same_object(s1$data, d2) == F; @@ -101,7 +110,7 @@ event zeek_init() while ( ! Broker::set_iterator_last(i) ) { local ci = Broker::__opaque_clone_through_serialization(i); - print fmt("| %s | %s", Broker::set_iterator_value(i), Broker::set_iterator_value(ci)); + print fmt("| %s | %s | %s", Broker::set_iterator_value(i), Broker::set_iterator_value(ci), type_name(ci)); Broker::set_iterator_next(i); Broker::set_iterator_next(ci); if ( ! Broker::set_iterator_last(i) ) @@ -118,7 +127,7 @@ event zeek_init() while ( ! Broker::table_iterator_last(j) ) { local cj = Broker::__opaque_clone_through_serialization(j); - print fmt("| %s | %s", Broker::table_iterator_value(j), Broker::table_iterator_value(cj)); + print fmt("| %s | %s | %s", Broker::table_iterator_value(j), Broker::table_iterator_value(cj), type_name(cj)); Broker::table_iterator_next(j); Broker::table_iterator_next(cj); if ( ! Broker::table_iterator_last(j) ) @@ -135,7 +144,7 @@ event zeek_init() while ( ! Broker::vector_iterator_last(k) ) { local ck = Broker::__opaque_clone_through_serialization(k); - print fmt("| %s | %s", Broker::vector_iterator_value(k), Broker::vector_iterator_value(ck)); + print fmt("| %s | %s | %s", Broker::vector_iterator_value(k), Broker::vector_iterator_value(ck), type_name(ck)); Broker::vector_iterator_next(k); Broker::vector_iterator_next(ck); if ( ! Broker::vector_iterator_last(k) ) @@ -152,7 +161,7 @@ event zeek_init() while ( ! Broker::record_iterator_last(l) ) { local cl = Broker::__opaque_clone_through_serialization(l); - print fmt("| %s | %s", Broker::record_iterator_value(l), Broker::record_iterator_value(cl)); + print fmt("| %s | %s | %s", Broker::record_iterator_value(l), Broker::record_iterator_value(cl), type_name(cl)); Broker::record_iterator_next(l); Broker::record_iterator_next(cl); if ( ! Broker::record_iterator_last(l) )