From b7877792c98c22f072f26ed5ef4870e598fcf672 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 5 Aug 2013 00:02:48 -0400 Subject: [PATCH 01/84] First commit of file entropy analyzer. - Code comments need cleaned up still. --- scripts/policy/frameworks/files/entropy.bro | 19 +++++ src/file_analysis/analyzer/CMakeLists.txt | 1 + .../analyzer/entropy/CMakeLists.txt | 9 ++ src/file_analysis/analyzer/entropy/Entropy.cc | 71 ++++++++++++++++ src/file_analysis/analyzer/entropy/Entropy.h | 84 +++++++++++++++++++ src/file_analysis/analyzer/entropy/Plugin.cc | 29 +++++++ src/file_analysis/analyzer/entropy/events.bif | 8 ++ 7 files changed, 221 insertions(+) create mode 100644 scripts/policy/frameworks/files/entropy.bro create mode 100644 src/file_analysis/analyzer/entropy/CMakeLists.txt create mode 100644 src/file_analysis/analyzer/entropy/Entropy.cc create mode 100644 src/file_analysis/analyzer/entropy/Entropy.h create mode 100644 src/file_analysis/analyzer/entropy/Plugin.cc create mode 100644 src/file_analysis/analyzer/entropy/events.bif diff --git a/scripts/policy/frameworks/files/entropy.bro b/scripts/policy/frameworks/files/entropy.bro new file mode 100644 index 0000000000..89dcead7d6 --- /dev/null +++ b/scripts/policy/frameworks/files/entropy.bro @@ -0,0 +1,19 @@ + +module Files; + +export { + redef record Files::Info += { + ## The information density of the contents of the file, expressed as a number of bits per character. + entropy: double &log &optional; + }; +} + +event file_new(f: fa_file) + { + Files::add_analyzer(f, Files::ANALYZER_ENTROPY); + } + +event file_entropy(f: fa_file, ent: entropy_test_result) + { + f$info$entropy = ent$entropy; + } \ No newline at end of file diff --git a/src/file_analysis/analyzer/CMakeLists.txt b/src/file_analysis/analyzer/CMakeLists.txt index bfafcd2894..ca93c4512c 100644 --- a/src/file_analysis/analyzer/CMakeLists.txt +++ b/src/file_analysis/analyzer/CMakeLists.txt @@ -1,3 +1,4 @@ add_subdirectory(data_event) +add_subdirectory(entropy) add_subdirectory(extract) add_subdirectory(hash) diff --git a/src/file_analysis/analyzer/entropy/CMakeLists.txt b/src/file_analysis/analyzer/entropy/CMakeLists.txt new file mode 100644 index 0000000000..38db5e726a --- /dev/null +++ b/src/file_analysis/analyzer/entropy/CMakeLists.txt @@ -0,0 +1,9 @@ +include(BroPlugin) + +include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR}) + +bro_plugin_begin(Bro FileEntropy) +bro_plugin_cc(Entropy.cc Plugin.cc ../../Analyzer.cc) +bro_plugin_bif(events.bif) +bro_plugin_end() diff --git a/src/file_analysis/analyzer/entropy/Entropy.cc b/src/file_analysis/analyzer/entropy/Entropy.cc new file mode 100644 index 0000000000..2a1bc72723 --- /dev/null +++ b/src/file_analysis/analyzer/entropy/Entropy.cc @@ -0,0 +1,71 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include + +#include "Entropy.h" +#include "util.h" +#include "Event.h" +#include "file_analysis/Manager.h" + +using namespace file_analysis; + +Entropy::Entropy(RecordVal* args, File* file) + : file_analysis::Analyzer(file_mgr->GetComponentTag("ENTROPY"), args, file) + { + //entropy->Init(); + entropy = new EntropyVal; + } + +Entropy::~Entropy() + { + Unref(entropy); + } + +file_analysis::Analyzer* Entropy::Instantiate(RecordVal* args, File* file) + { + return new Entropy(args, file); + } + +bool Entropy::DeliverStream(const u_char* data, uint64 len) + { + if ( ! fed ) + fed = len > 0; + + entropy->Feed(data, len); + return true; + } + +bool Entropy::EndOfFile() + { + Finalize(); + return false; + } + +bool Entropy::Undelivered(uint64 offset, uint64 len) + { + return false; + } + +void Entropy::Finalize() + { + //if ( ! entropy->IsValid() || ! fed ) + if ( ! fed ) + return; + + val_list* vl = new val_list(); + vl->append(GetFile()->GetVal()->Ref()); + + double montepi, scc, ent, mean, chisq; + montepi = scc = ent = mean = chisq = 0.0; + entropy->Get(&ent, &chisq, &mean, &montepi, &scc); + + RecordVal* ent_result = new RecordVal(entropy_test_result); + ent_result->Assign(0, new Val(ent, TYPE_DOUBLE)); + ent_result->Assign(1, new Val(chisq, TYPE_DOUBLE)); + ent_result->Assign(2, new Val(mean, TYPE_DOUBLE)); + ent_result->Assign(3, new Val(montepi, TYPE_DOUBLE)); + ent_result->Assign(4, new Val(scc, TYPE_DOUBLE)); + + vl->append(ent_result); + mgr.QueueEvent(file_entropy, vl); + } diff --git a/src/file_analysis/analyzer/entropy/Entropy.h b/src/file_analysis/analyzer/entropy/Entropy.h new file mode 100644 index 0000000000..6a5075263c --- /dev/null +++ b/src/file_analysis/analyzer/entropy/Entropy.h @@ -0,0 +1,84 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef FILE_ANALYSIS_ENTROPY_H +#define FILE_ANALYSIS_ENTROPY_H + +#include + +#include "Val.h" +#include "OpaqueVal.h" +#include "File.h" +#include "Analyzer.h" + +#include "events.bif.h" + +namespace file_analysis { + +/** + * An analyzer to produce a hash of file contents. + */ +class Entropy : public file_analysis::Analyzer { +public: + + /** + * Destructor. + */ + virtual ~Entropy(); + + /** + * Create a new instance of an Extract analyzer. + * @param args the \c AnalyzerArgs value which represents the analyzer. + * @param file the file to which the analyzer will be attached. + * @return the new Extract analyzer instance or a null pointer if the + * the "extraction_file" field of \a args wasn't set. + */ + static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file); + + /** + * Incrementally hash next chunk of file contents. + * @param data pointer to start of a chunk of a file data. + * @param len number of bytes in the data chunk. + * @return false if the digest is in an invalid state, else true. + */ + virtual bool DeliverStream(const u_char* data, uint64 len); + + /** + * Finalizes the hash and raises a "file_entropy_test" event. + * @return always false so analyze will be deteched from file. + */ + virtual bool EndOfFile(); + + /** + * Missing data can't be handled, so just indicate the this analyzer should + * be removed from receiving further data. The hash will not be finalized. + * @param offset byte offset in file at which missing chunk starts. + * @param len number of missing bytes. + * @return always false so analyzer will detach from file. + */ + virtual bool Undelivered(uint64 offset, uint64 len); + +protected: + + /** + * Constructor. + * @param args the \c AnalyzerArgs value which represents the analyzer. + * @param file the file to which the analyzer will be attached. + * @param hv specific hash calculator object. + * @param kind human readable name of the hash algorithm to use. + */ + Entropy(RecordVal* args, File* file); + + /** + * If some file contents have been seen, finalizes the hash of them and + * raises the "file_hash" event with the results. + */ + void Finalize(); + +private: + EntropyVal* entropy; + bool fed; +}; + +} // namespace file_analysis + +#endif diff --git a/src/file_analysis/analyzer/entropy/Plugin.cc b/src/file_analysis/analyzer/entropy/Plugin.cc new file mode 100644 index 0000000000..3eeae62480 --- /dev/null +++ b/src/file_analysis/analyzer/entropy/Plugin.cc @@ -0,0 +1,29 @@ +#include "plugin/Plugin.h" +#include "file_analysis/Component.h" + +#include "Entropy.h" + +namespace plugin { namespace Bro_FileEntropy { + +class Plugin : public plugin::Plugin { +protected: + void InitPreScript() + { + SetName("Bro::FileEntropy"); + SetVersion(-1); + SetAPIVersion(BRO_PLUGIN_API_VERSION); + SetDynamicPlugin(false); + + SetDescription("Entropy test file content"); + + AddComponent(new ::file_analysis::Component("ENTROPY", + ::file_analysis::Entropy::Instantiate)); + + extern std::list > __bif_events_init(); + AddBifInitFunction(&__bif_events_init); + } +}; + +Plugin __plugin; + +} } diff --git a/src/file_analysis/analyzer/entropy/events.bif b/src/file_analysis/analyzer/entropy/events.bif new file mode 100644 index 0000000000..a51bb3d39b --- /dev/null +++ b/src/file_analysis/analyzer/entropy/events.bif @@ -0,0 +1,8 @@ +## This event is generated each time file analysis performs +## entropy testing on a file. +## +## f: The file. +## +## ent: The results of the entropy testing. +## +event file_entropy%(f: fa_file, ent: entropy_test_result%); \ No newline at end of file From 9c692bad39c7d207b7682a87773806b9916fc9d8 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Tue, 3 Feb 2015 15:04:36 -0500 Subject: [PATCH 02/84] Update and clean up to file entropy measurement. - Updated to newer file analyzer api. --- ...entropy.bro => entropy-test-all-files.bro} | 3 +- src/file_analysis/analyzer/entropy/Plugin.cc | 33 ++++++++----------- 2 files changed, 16 insertions(+), 20 deletions(-) rename scripts/policy/frameworks/files/{entropy.bro => entropy-test-all-files.bro} (71%) diff --git a/scripts/policy/frameworks/files/entropy.bro b/scripts/policy/frameworks/files/entropy-test-all-files.bro similarity index 71% rename from scripts/policy/frameworks/files/entropy.bro rename to scripts/policy/frameworks/files/entropy-test-all-files.bro index 89dcead7d6..fd02b9ecaa 100644 --- a/scripts/policy/frameworks/files/entropy.bro +++ b/scripts/policy/frameworks/files/entropy-test-all-files.bro @@ -3,7 +3,8 @@ module Files; export { redef record Files::Info += { - ## The information density of the contents of the file, expressed as a number of bits per character. + ## The information density of the contents of the file, + ## expressed as a number of bits per character. entropy: double &log &optional; }; } diff --git a/src/file_analysis/analyzer/entropy/Plugin.cc b/src/file_analysis/analyzer/entropy/Plugin.cc index 3eeae62480..f1dd954cba 100644 --- a/src/file_analysis/analyzer/entropy/Plugin.cc +++ b/src/file_analysis/analyzer/entropy/Plugin.cc @@ -1,29 +1,24 @@ +// See the file in the main distribution directory for copyright. + #include "plugin/Plugin.h" -#include "file_analysis/Component.h" #include "Entropy.h" -namespace plugin { namespace Bro_FileEntropy { +namespace plugin { +namespace Bro_FileEntropy { class Plugin : public plugin::Plugin { -protected: - void InitPreScript() +public: + plugin::Configuration Configure() { - SetName("Bro::FileEntropy"); - SetVersion(-1); - SetAPIVersion(BRO_PLUGIN_API_VERSION); - SetDynamicPlugin(false); + AddComponent(new ::file_analysis::Component("ENTROPY", ::file_analysis::Entropy::Instantiate)); - SetDescription("Entropy test file content"); - - AddComponent(new ::file_analysis::Component("ENTROPY", - ::file_analysis::Entropy::Instantiate)); - - extern std::list > __bif_events_init(); - AddBifInitFunction(&__bif_events_init); + plugin::Configuration config; + config.name = "Bro::FileEntropy"; + config.description = "Entropy test file content"; + return config; } -}; +} plugin; -Plugin __plugin; - -} } +} +} From 39ebf8df791076f0c3b7e2f9ed83a833f11fe96d Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 26 Feb 2015 09:17:55 -0500 Subject: [PATCH 03/84] Updated tests for file entropy analyzer. --- scripts/test-all-policy.bro | 1 + .../canonified_loaded_scripts.log | 5 +++-- testing/btest/Baseline/plugins.hooks/output | 20 ++++++++++--------- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro index 0fb74f91cf..79641be788 100644 --- a/scripts/test-all-policy.bro +++ b/scripts/test-all-policy.bro @@ -28,6 +28,7 @@ @load frameworks/intel/seen/where-locations.bro @load frameworks/intel/seen/x509.bro @load frameworks/files/detect-MHR.bro +@load frameworks/files/entropy-test-all-files.bro @load frameworks/files/hash-all-files.bro @load frameworks/packet-filter/shunt.bro @load frameworks/software/version-changes.bro diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 67de0fc1dc..c3031cbb2d 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2014-10-31-20-38-48 +#open 2015-02-26-14-14-34 #fields name #types string scripts/base/init-bare.bro @@ -97,6 +97,7 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_Teredo.events.bif.bro build/scripts/base/bif/plugins/Bro_UDP.events.bif.bro build/scripts/base/bif/plugins/Bro_ZIP.events.bif.bro + build/scripts/base/bif/plugins/Bro_FileEntropy.events.bif.bro build/scripts/base/bif/plugins/Bro_FileExtract.events.bif.bro build/scripts/base/bif/plugins/Bro_FileExtract.functions.bif.bro build/scripts/base/bif/plugins/Bro_FileHash.events.bif.bro @@ -247,4 +248,4 @@ scripts/base/init-default.bro scripts/base/misc/find-checksum-offloading.bro scripts/base/misc/find-filtered-trace.bro scripts/policy/misc/loaded-scripts.bro -#close 2014-10-31-20-38-48 +#close 2015-02-26-14-14-34 diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 927a64692f..d9a87ec6ed 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -191,7 +191,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, (Weird::LOG, [columns=, ev=Weird::log_weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, (X509::LOG, [columns=, ev=X509::log_x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, (mysql::LOG, [columns=, ev=MySQL::log_mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, (PacketFilter::LOG, [ts=1421870896.278622, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, (PacketFilter::LOG, [ts=1424960167.277735, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, (Communication::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, (Conn::LOG)) -> @@ -285,8 +285,8 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, (Weird::LOG, [columns=, ev=Weird::log_weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, (X509::LOG, [columns=, ev=X509::log_x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, (mysql::LOG, [columns=, ev=MySQL::log_mysql])) -> -0.000000 MetaHookPost CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1421870896.278622, node=bro, filter=ip or not ip, init=T, success=T])) -> -0.000000 MetaHookPost CallFunction(Log::write, (PacketFilter::LOG, [ts=1421870896.278622, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1424960167.277735, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, (PacketFilter::LOG, [ts=1424960167.277735, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, ()) -> 0.000000 MetaHookPost CallFunction(PacketFilter::build, ()) -> 0.000000 MetaHookPost CallFunction(PacketFilter::combine_filters, (ip or not ip, and, )) -> @@ -344,6 +344,7 @@ 0.000000 MetaHookPost LoadFile(./Bro_FTP.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_FTP.functions.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_File.events.bif.bro) -> -1 +0.000000 MetaHookPost LoadFile(./Bro_FileEntropy.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_FileExtract.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_FileExtract.functions.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_FileHash.events.bif.bro) -> -1 @@ -730,7 +731,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, (Weird::LOG, [columns=, ev=Weird::log_weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, (X509::LOG, [columns=, ev=X509::log_x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, (mysql::LOG, [columns=, ev=MySQL::log_mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, (PacketFilter::LOG, [ts=1421870896.278622, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, (PacketFilter::LOG, [ts=1424960167.277735, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, (Communication::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, (Conn::LOG)) @@ -824,8 +825,8 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, (Weird::LOG, [columns=, ev=Weird::log_weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, (X509::LOG, [columns=, ev=X509::log_x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, (mysql::LOG, [columns=, ev=MySQL::log_mysql])) -0.000000 MetaHookPre CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1421870896.278622, node=bro, filter=ip or not ip, init=T, success=T])) -0.000000 MetaHookPre CallFunction(Log::write, (PacketFilter::LOG, [ts=1421870896.278622, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1424960167.277735, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, (PacketFilter::LOG, [ts=1424960167.277735, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Notice::want_pp, ()) 0.000000 MetaHookPre CallFunction(PacketFilter::build, ()) 0.000000 MetaHookPre CallFunction(PacketFilter::combine_filters, (ip or not ip, and, )) @@ -883,6 +884,7 @@ 0.000000 MetaHookPre LoadFile(./Bro_FTP.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_FTP.functions.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_File.events.bif.bro) +0.000000 MetaHookPre LoadFile(./Bro_FileEntropy.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_FileExtract.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_FileExtract.functions.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_FileHash.events.bif.bro) @@ -1269,7 +1271,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=, ev=Weird::log_weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=, ev=X509::log_x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1421870896.278622, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1424960167.277735, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG) @@ -1363,8 +1365,8 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=, ev=Weird::log_weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=, ev=X509::log_x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql]) -0.000000 | HookCallFunction Log::default_path_func(PacketFilter::LOG, , [ts=1421870896.278622, node=bro, filter=ip or not ip, init=T, success=T]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1421870896.278622, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::default_path_func(PacketFilter::LOG, , [ts=1424960167.277735, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1424960167.277735, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Notice::want_pp() 0.000000 | HookCallFunction PacketFilter::build() 0.000000 | HookCallFunction PacketFilter::combine_filters(ip or not ip, and, ) From 574bcb0a51b18d1e209f75f89bbe8ee4b9e6306a Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Tue, 21 Jul 2015 11:57:16 -0700 Subject: [PATCH 04/84] Add simple XMPP StartTLS analyzer. This is a very simple XMPP analyzer that basically only can parse the protocol until the client and server start negotiating a TLS session. At that point, the TLS analyzer is attached. While the basic case seems to be working, I fully expect that I missed something and that this might break in a lot of cases. --- scripts/base/init-default.bro | 1 + scripts/base/protocols/xmpp/README | 5 + scripts/base/protocols/xmpp/__load__.bro | 1 + scripts/base/protocols/xmpp/main.bro | 11 +++ src/analyzer/protocol/CMakeLists.txt | 1 + src/analyzer/protocol/xmpp/CMakeLists.txt | 11 +++ src/analyzer/protocol/xmpp/Plugin.cc | 26 ++++++ src/analyzer/protocol/xmpp/XMPP.cc | 87 ++++++++++++++++++ src/analyzer/protocol/xmpp/XMPP.h | 38 ++++++++ src/analyzer/protocol/xmpp/xmpp-analyzer.pac | 41 +++++++++ src/analyzer/protocol/xmpp/xmpp-protocol.pac | 17 ++++ src/analyzer/protocol/xmpp/xmpp.pac | 35 +++++++ .../conn.log | 10 ++ .../ssl.log | 10 ++ .../x509.log | 11 +++ testing/btest/Traces/tls/xmpp-starttls.pcap | Bin 0 -> 8174 bytes .../scripts/base/protocols/xmpp/starttls.test | 9 ++ 17 files changed, 314 insertions(+) create mode 100644 scripts/base/protocols/xmpp/README create mode 100644 scripts/base/protocols/xmpp/__load__.bro create mode 100644 scripts/base/protocols/xmpp/main.bro create mode 100644 src/analyzer/protocol/xmpp/CMakeLists.txt create mode 100644 src/analyzer/protocol/xmpp/Plugin.cc create mode 100644 src/analyzer/protocol/xmpp/XMPP.cc create mode 100644 src/analyzer/protocol/xmpp/XMPP.h create mode 100644 src/analyzer/protocol/xmpp/xmpp-analyzer.pac create mode 100644 src/analyzer/protocol/xmpp/xmpp-protocol.pac create mode 100644 src/analyzer/protocol/xmpp/xmpp.pac create mode 100644 testing/btest/Baseline/scripts.base.protocols.xmpp.starttls/conn.log create mode 100644 testing/btest/Baseline/scripts.base.protocols.xmpp.starttls/ssl.log create mode 100644 testing/btest/Baseline/scripts.base.protocols.xmpp.starttls/x509.log create mode 100644 testing/btest/Traces/tls/xmpp-starttls.pcap create mode 100644 testing/btest/scripts/base/protocols/xmpp/starttls.test diff --git a/scripts/base/init-default.bro b/scripts/base/init-default.bro index 473d94fc84..7e921a6831 100644 --- a/scripts/base/init-default.bro +++ b/scripts/base/init-default.bro @@ -59,6 +59,7 @@ @load base/protocols/ssl @load base/protocols/syslog @load base/protocols/tunnels +@load base/protocols/xmpp @load base/files/pe @load base/files/hash diff --git a/scripts/base/protocols/xmpp/README b/scripts/base/protocols/xmpp/README new file mode 100644 index 0000000000..3d2194ef3d --- /dev/null +++ b/scripts/base/protocols/xmpp/README @@ -0,0 +1,5 @@ +Support for the Extensible Messaging and Presence Protocol (XMPP). + +Note that currently the XMPP analyzer only supports analyzing XMPP sessions +until they do or do not switch to TLS using StartTLS. Hence, we do not get +actual chat information from XMPP sessions, only X509 certificates. diff --git a/scripts/base/protocols/xmpp/__load__.bro b/scripts/base/protocols/xmpp/__load__.bro new file mode 100644 index 0000000000..a10fe855df --- /dev/null +++ b/scripts/base/protocols/xmpp/__load__.bro @@ -0,0 +1 @@ +@load ./main diff --git a/scripts/base/protocols/xmpp/main.bro b/scripts/base/protocols/xmpp/main.bro new file mode 100644 index 0000000000..3d7a4cbc37 --- /dev/null +++ b/scripts/base/protocols/xmpp/main.bro @@ -0,0 +1,11 @@ + +module XMPP; + +const ports = { 5222/tcp, 5269/tcp }; +redef likely_server_ports += { ports }; + +event bro_init() &priority=5 + { + Analyzer::register_for_ports(Analyzer::ANALYZER_XMPP, ports); + } + diff --git a/src/analyzer/protocol/CMakeLists.txt b/src/analyzer/protocol/CMakeLists.txt index 467fce83ee..d19b2ac042 100644 --- a/src/analyzer/protocol/CMakeLists.txt +++ b/src/analyzer/protocol/CMakeLists.txt @@ -43,4 +43,5 @@ add_subdirectory(syslog) add_subdirectory(tcp) add_subdirectory(teredo) add_subdirectory(udp) +add_subdirectory(xmpp) add_subdirectory(zip) diff --git a/src/analyzer/protocol/xmpp/CMakeLists.txt b/src/analyzer/protocol/xmpp/CMakeLists.txt new file mode 100644 index 0000000000..408f01d47c --- /dev/null +++ b/src/analyzer/protocol/xmpp/CMakeLists.txt @@ -0,0 +1,11 @@ + +include(BroPlugin) + +include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) + +bro_plugin_begin(Bro XMPP) +bro_plugin_cc(Plugin.cc) +bro_plugin_cc(XMPP.cc) +bro_plugin_pac(xmpp.pac xmpp-analyzer.pac xmpp-protocol.pac) +bro_plugin_end() + diff --git a/src/analyzer/protocol/xmpp/Plugin.cc b/src/analyzer/protocol/xmpp/Plugin.cc new file mode 100644 index 0000000000..b4332b447b --- /dev/null +++ b/src/analyzer/protocol/xmpp/Plugin.cc @@ -0,0 +1,26 @@ +// See the file in the main distribution directory for copyright. + + +#include "plugin/Plugin.h" + +#include "XMPP.h" + +namespace plugin { +namespace Bro_XMPP { + +class Plugin : public plugin::Plugin { +public: + plugin::Configuration Configure() + { + AddComponent(new ::analyzer::Component("XMPP", ::analyzer::xmpp::XMPP_Analyzer::Instantiate)); + + + plugin::Configuration config; + config.name = "Bro::XMPP"; + config.description = "XMPP analyzer StartTLS only"; + return config; + } +} plugin; + +} +} diff --git a/src/analyzer/protocol/xmpp/XMPP.cc b/src/analyzer/protocol/xmpp/XMPP.cc new file mode 100644 index 0000000000..c84c372c4d --- /dev/null +++ b/src/analyzer/protocol/xmpp/XMPP.cc @@ -0,0 +1,87 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "XMPP.h" +#include "analyzer/protocol/tcp/TCP_Reassembler.h" +#include "analyzer/Manager.h" + +using namespace analyzer::xmpp; + +XMPP_Analyzer::XMPP_Analyzer(Connection* conn) + : tcp::TCP_ApplicationAnalyzer("XMPP", conn) + { + interp = new binpac::XMPP::XMPP_Conn(this); + had_gap = false; + tls_active = false; + } + +XMPP_Analyzer::~XMPP_Analyzer() + { + delete interp; + } + +void XMPP_Analyzer::Done() + { + tcp::TCP_ApplicationAnalyzer::Done(); + + interp->FlowEOF(true); + interp->FlowEOF(false); + } + +void XMPP_Analyzer::EndpointEOF(bool is_orig) + { + tcp::TCP_ApplicationAnalyzer::EndpointEOF(is_orig); + interp->FlowEOF(is_orig); + } + +void XMPP_Analyzer::DeliverStream(int len, const u_char* data, bool orig) + { + tcp::TCP_ApplicationAnalyzer::DeliverStream(len, data, orig); + + if ( tls_active ) + { + // If TLS has been initiated, forward to child and abort further + // processing + ForwardStream(len, data, orig); + return; + } + + assert(TCP()); + if ( TCP()->IsPartial() ) + return; + + if ( had_gap ) + // If only one side had a content gap, we could still try to + // deliver data to the other side if the script layer can + // handle this. + return; + + try + { + interp->NewData(orig, data, data + len); + } + catch ( const binpac::Exception& e ) + { + printf("BinPAC Exception: %s\n", e.c_msg()); + ProtocolViolation(fmt("Binpac exception: %s", e.c_msg())); + } + } + +void XMPP_Analyzer::Undelivered(uint64 seq, int len, bool orig) + { + tcp::TCP_ApplicationAnalyzer::Undelivered(seq, len, orig); + had_gap = true; + interp->NewGap(orig, len); + } + +void XMPP_Analyzer::StartTLS() + { + // StartTLS was called. This means we saw a client starttls followed + // by a server proceed. From here on, everything should be a binary + // TLS datastream. + + tls_active = true; + + Analyzer* ssl = analyzer_mgr->InstantiateAnalyzer("SSL", Conn()); + if ( ssl ) + AddChildAnalyzer(ssl); + } diff --git a/src/analyzer/protocol/xmpp/XMPP.h b/src/analyzer/protocol/xmpp/XMPP.h new file mode 100644 index 0000000000..628be7bb2d --- /dev/null +++ b/src/analyzer/protocol/xmpp/XMPP.h @@ -0,0 +1,38 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef ANALYZER_PROTOCOL_XMPP_XMPP_H +#define ANALYZER_PROTOCOL_XMPP_XMPP_H + +#include "analyzer/protocol/tcp/TCP.h" + +#include "xmpp_pac.h" + +namespace analyzer { namespace xmpp { + +class XMPP_Analyzer : public tcp::TCP_ApplicationAnalyzer { +public: + XMPP_Analyzer(Connection* conn); + virtual ~XMPP_Analyzer(); + + virtual void Done(); + virtual void DeliverStream(int len, const u_char* data, bool orig); + virtual void Undelivered(uint64 seq, int len, bool orig); + + // Overriden from tcp::TCP_ApplicationAnalyzer. + virtual void EndpointEOF(bool is_orig); + + void StartTLS(); + + static analyzer::Analyzer* Instantiate(Connection* conn) + { return new XMPP_Analyzer(conn); } + +protected: + binpac::XMPP::XMPP_Conn* interp; + bool had_gap; + + bool tls_active; +}; + +} } // namespace analyzer::* + +#endif /* ANALYZER_PROTOCOL_XMPP_XMPP_H */ diff --git a/src/analyzer/protocol/xmpp/xmpp-analyzer.pac b/src/analyzer/protocol/xmpp/xmpp-analyzer.pac new file mode 100644 index 0000000000..a4417e1601 --- /dev/null +++ b/src/analyzer/protocol/xmpp/xmpp-analyzer.pac @@ -0,0 +1,41 @@ +refine connection XMPP_Conn += { + + %member{ + bool client_starttls; + %} + + %init{ + client_starttls = false; + %} + + function proc_xmpp_token(is_orig: bool, name: bytestring, rest: bytestring): bool + %{ + string token = std_str(name); + + if ( is_orig && token == "stream:stream" ) + // Yup, looks like xmpp... + bro_analyzer()->ProtocolConfirmation(); + + if ( token == "success" || token == "message" ) + // Handshake has passed the phase where we should see StartTLS. Simply skip from hereon... + bro_analyzer()->SetSkip(true); + + if ( is_orig && token == "starttls" ) + client_starttls = true; + + if ( !is_orig && token == "proceed" && client_starttls ) + { + bro_analyzer()->StartTLS(); + } + + //printf("Processed: %d %s %s \n", is_orig, c_str(name), c_str(rest)); + + return true; + %} + +}; + +refine typeattr XMPP_TOKEN += &let { + proc: bool = $context.connection.proc_xmpp_token(is_orig, name, rest); +}; + diff --git a/src/analyzer/protocol/xmpp/xmpp-protocol.pac b/src/analyzer/protocol/xmpp/xmpp-protocol.pac new file mode 100644 index 0000000000..e05268fe32 --- /dev/null +++ b/src/analyzer/protocol/xmpp/xmpp-protocol.pac @@ -0,0 +1,17 @@ +type XML_START = RE//; +type XML_NAME = RE/\/?[?:[:alnum:]]+/; +type XML_REST = RE/[^<>]*/; +type SPACING = RE/[ \r\n]*/; + +type XMPP_PDU(is_orig: bool) = XMPP_TOKEN(is_orig)[] &until($input.length() == 0); + +type XMPP_TOKEN(is_orig: bool) = record { + : SPACING; + : XML_START; + name: XML_NAME; + rest: XML_REST; + : XML_END; + : SPACING; +}; + diff --git a/src/analyzer/protocol/xmpp/xmpp.pac b/src/analyzer/protocol/xmpp/xmpp.pac new file mode 100644 index 0000000000..42ec85f0cc --- /dev/null +++ b/src/analyzer/protocol/xmpp/xmpp.pac @@ -0,0 +1,35 @@ +# binpac file for the XMPP analyzer. +# Note that we currently do not even try to parse the protocol +# completely -- this is only supposed to be able to parse xmpp +# till StartTLS does (or does not) kick in. + +%include binpac.pac +%include bro.pac + +%extern{ +namespace analyzer { namespace xmpp { class XMPP_Analyzer; } } +namespace binpac { namespace XMPP { class XMPP_Conn; } } +typedef analyzer::xmpp::XMPP_Analyzer* XMPPAnalyzer; + +#include "XMPP.h" +%} + +extern type XMPPAnalyzer; + +analyzer XMPP withcontext { + connection: XMPP_Conn; + flow: XMPP_Flow; +}; + +connection XMPP_Conn(bro_analyzer: XMPPAnalyzer) { + upflow = XMPP_Flow(true); + downflow = XMPP_Flow(false); +}; + +%include xmpp-protocol.pac + +flow XMPP_Flow(is_orig: bool) { + datagram = XMPP_PDU(is_orig) withcontext(connection, this); +}; + +%include xmpp-analyzer.pac diff --git a/testing/btest/Baseline/scripts.base.protocols.xmpp.starttls/conn.log b/testing/btest/Baseline/scripts.base.protocols.xmpp.starttls/conn.log new file mode 100644 index 0000000000..2f5bd2f66d --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.xmpp.starttls/conn.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2015-07-21-18-55-16 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents +#types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] +1437091701.732171 CXWv6p3arKYeMETxOg 198.128.203.95 56048 146.255.57.229 5222 tcp ssl,xmpp 2.213218 676 4678 SF - - 0 ShADadfFr 19 1676 15 5442 (empty) +#close 2015-07-21-18-55-16 diff --git a/testing/btest/Baseline/scripts.base.protocols.xmpp.starttls/ssl.log b/testing/btest/Baseline/scripts.base.protocols.xmpp.starttls/ssl.log new file mode 100644 index 0000000000..f67ea92631 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.xmpp.starttls/ssl.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path ssl +#open 2015-07-21-18-55-16 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer +#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string +1437091702.232293 CXWv6p3arKYeMETxOg 198.128.203.95 56048 146.255.57.229 5222 TLSv12 TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 secp256r1 - F - - T F5Nz2G1vSZQ0QXM2s8,FUw8omi2keRxShDUa (empty) CN=jabber.ccc.de,O=Chaos Computer Club e.V.,L=Hamburg,ST=Hamburg,C=DE emailAddress=support@cacert.org,CN=CA Cert Signing Authority,OU=http://www.cacert.org,O=Root CA - - +#close 2015-07-21-18-55-16 diff --git a/testing/btest/Baseline/scripts.base.protocols.xmpp.starttls/x509.log b/testing/btest/Baseline/scripts.base.protocols.xmpp.starttls/x509.log new file mode 100644 index 0000000000..4a49298e8a --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.xmpp.starttls/x509.log @@ -0,0 +1,11 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path x509 +#open 2015-07-21-18-55-16 +#fields ts id certificate.version certificate.serial certificate.subject certificate.issuer certificate.not_valid_before certificate.not_valid_after certificate.key_alg certificate.sig_alg certificate.key_type certificate.key_length certificate.exponent certificate.curve san.dns san.uri san.email san.ip basic_constraints.ca basic_constraints.path_len +#types time string count string string string time time string string string count string string vector[string] vector[string] vector[string] vector[addr] bool count +1437091702.407347 F5Nz2G1vSZQ0QXM2s8 3 0DF4F2 CN=jabber.ccc.de,O=Chaos Computer Club e.V.,L=Hamburg,ST=Hamburg,C=DE emailAddress=support@cacert.org,CN=CA Cert Signing Authority,OU=http://www.cacert.org,O=Root CA 1382043019.000000 1445115019.000000 rsaEncryption sha1WithRSAEncryption rsa 2048 65537 - jabber.ccc.de,conference.jabber.ccc.de,jabberd.jabber.ccc.de,pubsub.jabber.ccc.de,vjud.jabber.ccc.de - - - F - +1437091702.407347 FUw8omi2keRxShDUa 3 00 emailAddress=support@cacert.org,CN=CA Cert Signing Authority,OU=http://www.cacert.org,O=Root CA emailAddress=support@cacert.org,CN=CA Cert Signing Authority,OU=http://www.cacert.org,O=Root CA 1049027389.000000 1995712189.000000 rsaEncryption md5WithRSAEncryption rsa 4096 65537 - - - - - T - +#close 2015-07-21-18-55-16 diff --git a/testing/btest/Traces/tls/xmpp-starttls.pcap b/testing/btest/Traces/tls/xmpp-starttls.pcap new file mode 100644 index 0000000000000000000000000000000000000000..b4a7ee61e10d771d4783cdc9f369b5d410c60f52 GIT binary patch literal 8174 zcmc&(c|4Tc|39-B3}fGlGDt%*^Vs)2kw`*myOuE++c1kFW%))$Dr=O=5|vb_v|de{ z7L}WP%ayKMT@o#p(53P_&sc8g-tX`G*VpTHUgkN^dEV#qd7t;^^LfsBns3$=V*wsa z{*8?RfPoi0r{msYi^aeS_#16eNkSPX!DhYd^)@#|fFl65HXgRXmQYwypV;7=HD=b) z+A9~t7%jMiB2{6>#*BLbz~Y6@VQ@Gh3`T^gYr1_1y@$t+qX*ZJfD61v+c@s{90GJp zI02v$-VLM3=nQf}t-T%P@0Y_F?iR_I?%N`ZqT{RZSP|lL)i*R09nK_-+T8|`?I9}K zX4!4HcspwMnJqKyz6;S^6Q!h-emMgf5ZIe?07XU-W6zSnMTmyBS%?)0VkqK2x|ZkU=5GFxVU>E7FRhM@1;sF{mHI4=s#kc{be+gSgIu1Q~~-2ZP$WCmJM3=9p;j7&@oO-(7}AT}#v67F{xfWfZN#=qW{#+iL!9l6?XLGsXoJkS!*pcQ;1~pSPV2p6M;7-}5Bdx2Bvvx;Hhkpv!O%z+wm{`b? zRqGd$4x_B{1)!FX@E7y7_zU=Qd?F|VrJxub0bw8%FhK|~1E#0QP)6zAj&zugRC-<3S;?1T>Hb?0_&|nlH>3=VO5o!1HDJB0vGi0x3Y^llVA5 z1oA)zNCI&n3dXRIDQ_SK@Hl|M!dog}E>Hw>;D2SH1b-_42|$AP(ZNKZ0o6hRWH6tA zAz%rxjtgOhaCi(Ji^t(GI4ll_#bB{m90bQ;(DlG;6F|2R5VV;p6KW$Wp(=YZqZ)i% z%NRX)2G*CjYA6o0^1{_U1y$CmfWrZ6ECA*>ocD33m|g$Qo}XJTf9&AN22b3w4epPM zetrE&xr?VoISMc>~1%078e5WT65i8JAQfFdRqv)Q^w| zqKi_UQ-?_yjF1pOV)djD2?EX=Pm~p2#bR;E4t9F;5f!vYOjdp(+o73@?oVg1xoBn9 zQ%A_?eVptZ2RpI@>?M0LgCm)d!DKrgH-yDza%1(>5Y?%PL@^3799~oui_NwD3kI35 zNYXRVL-b7a5hFy;Y#qEb`tA}5*AqiT(TNDM*d>mWc8bUfJNrij@Sq(NmJ9#YgU%}O zMadz=pT!|Nup**(Xs&b!=LL`%)YViy1w?M#c+x};AAfs5k}>MvbAu5AxMYkZfc->l zG6nS1I^*z1O)rV&k_}vT=Qw#Pc7ms4iUB4{O`&uZ|)=Z%SHyd2X}2>K|Hn zPqw)(Ut=cIFEzwzx*y+v_NgirtHq-aEo+_YWAvoL`Yv70Poe(KT)DUBSoZqQMJ`%~ z?`vCOOX+sE*J559blA3boxjWIg~zHNl%4_n9{;jqfrUZ%{T2F-DLPJwzOGI8PRX8N*`{AP~?zN%RU2gXB13h~Cx2>moV?5gh{5 znJ^AJ^$SA~^-Fceq<*%?V+cfe8;?mgA>wd|!Q>DO22VjW5sk@f1e2;bp+pv)6ZM@G zX^0*=l`4J_qJ=ChT2Q2(N}d==XNP}3kj;o;g)ykG7vZ3fNaMqi(B$bFNLHe=B7+!g zMkJjv`6fE9j%4|XB0Zy9E{Yex;RVd-m5B-E1y1k#URp5haLF()*=!cgXjP1pJFZ&y zawzP2SN3D8Dw7?E&ebq!za{M79&(~Bws-2rJ_-%*j7gE`c&~`f7BO+-@akCB$iC0p zwNs?>G<=6vHlKD)ao$>~690Hh=>ts3#nd`}a7oGB+8pytx3`UcFWmFaJ{cyz_cftlzNqYu+w@8D+1fM1pDifX9dB9ha~N zBiDu41#djdbSf&Rd?*Xwt&j6_wq7=D@5LRZouys%k2sr>Ql95=aMSrL>Q;mGef{fJ z>i+Wa9;MhhQ+DX0rBs7u^!7W_SJpTuonAw_9kgVdv9@%UZE)}V`qvwOS|ohDl&QCg z@2JLex;S>ppPGrv3zpXQ}X zIp@_YpGWvS(iFZ*>RHTs{B2o-PMi5#BU;>Z6Y7{g$1cfc%ps0Cwq!JH_=&f3#hvQP z?hkW30$0UGK2`ccx|0ip?T}<)E12~)$1^?#d@nrw|2*F#28e+HqNlHKW{Bo{_}f6= zY~u2NS&EXevy(p-P5xMz{9&j*Qs=)~y7is0mhj5F{vre0-OKwdf^76U!m!5ApT-aM z?cBR!ld+Ip?w_K)GL1`9Ier%+w{`J-0#84zI_0P+P9F&-c0FynX8k7rRQ z8^hIpJ52ud^1~o~6ZXik$l#raKKJ&#k@%_G`kP9z98o?zb^H3jpha;fmV3qy?kw5z zfkyP#$*9@pwz^FAU3Y8N;mRd;^L_jnLP4MTOMY(18MfahHp~dg_YmqG5X!{QQ&voV zcX!Q|n_+P~`>NOvG}n_e-zi{a_UgY?RC501jt~`#*k9Bv6biGcFhADg={(%kR4bg{ zrd)5BkZ?~WIZnt&M`ZngX~{{&jY0SFpBX;#)|9<9g!&?V`AMJ4w2bcDV?{T; zc9o4jPxnwiRCBXVx^8i1YI@%0Q{*RUw7E9*zj!cqs8!FwFcKV9b_~Y3Or9%7>(z^9J)TFbSFJFTlzNNqUts!dpyRL)LfdG;h1-d#bC4?K*(z zSWdonpnT2!On^x*=Bb&Sr;~!Mm%2xhgCE^~y7oEt_ui(4hnE8`?6d27LHOC3e1xeIMK?N5=;67js(MLN} z@j8e$LMhTL(nwW9D=7z_%}js&qkUIOk*Qja!Q(X){vS^m{`G_io}!@n!#7MCGcu5| znHm)mWdqMG^blnAUr&+n%M_fvSma;^mmJIDvB|7Wk!1AMlo`aN`*RuOAQqdvbd{qc zIfUUKNM^x9A$V3a`BpI9l^MxVP?fdrd?-xDUfz%>bKi(KFNw7)wBSL1h-AXCml-~} zXy$iB2>-CFyOEFX!R7={--8`I(@{QhB2h}R>6*pl9&ASz`reNs8ki73A`b5AV4Jy% zz_E#Q(OS}eYhxM-{c}Iz=Du~6=lzBX95AIjqY4iN96Gw=Ot23L zF}CPD<@}_9i%pHTex-v8=`#$lU25Ukt@WvLG$_|I&28gXi(mbo-H&BjR5P;5>ywf< z6h7>6!=#n9OM6$y-&Zx^C+A1`KM#%1E8Ro&vUXQHvF~GB^0l@u$5U?mI3kugj4FIV zOv04* z&40ONoi)b9B3-1aWFU9N+=hb>4^%&|iR#sLdEQd>n&s$hAT##OQh$-x@U7vr-klmZ zoW2FdMe`reFQ-=tZ#znRg4x@4M5nf1uSe|L25wmot*LtUg87)R$&k3b z>3-3eO8xG83U#+0=$nLad__h*hL?F73lVd%uKlrdrG{F}d4&V>`btPq!Kun;6$WV? zM|)yZ|Fjaj!4WGfsAPQ!zSuz%D_2^OvE{L|k+za;WwzhTywsKF;?}n8RTDeXn4DQ} zaV@M(IP=o})fyGG;W_or1j|1jt*H28h$hUDVk zf`*ZUN1~jlzsaRH*N{c?N&@>9sI(Ou70TB%U-hPLB@7lycz;^=G`ltClqSdZwgcSd zVcf(4W*H7B+Tp*+3>zf5EqyvcGf&S0^HrK|M_8YM$t1H(*9$Jqi`wb+{PB4+8i6H zubrBko7>vUHgGqKimS6_=K zfJdtN$#PNc#oob-_r$RTudmr@5Avf*FY7z#E2h}|sR|q-_VF#K-O5M*C1=(Whrhw~ zzWJoePp!2#{Dotj5|?FMxfeMvT~>SQvGegVbnA_mfV+P*9^+v=!mYWW%{*lRhQGc- zBOE(#c7%U@10%fiO4n3`XHQ1>@0J9>hlpsKg;`yrx|6sVJ#Ld-!CQDc`% zLHq3R`x_!M-T$rLdnG?-C3Eje-*55fTid;qzrJohxJYF&7GJDIS{VCkWC^vB)bJ;_ zH&F9)rCi(Kt6Q~=l4-piBZ_hmN^oibv{HrA{{4kQ%nk~;_Uz;O74J6d(mEDIUCuqE zvFv^Bu)1o&gLlojNzbZKhCJ_zOdavf%vd0psElT;SJl%q*4HM1GqKiF8AWyp}XcF#cH#`6;t?meM<^ub+f{z);J-rSm;jY{*c zg&8Ks*GPEJJD{F^w<`Lhjb6%@ethD`ty>ZX2AzV$O|dQ|r#F~i8VX6JX(@-rhGZ<| zy{%P!;~BkZ@cJ4tRc*VBxc8!dxn2Z~IRxwF%{_k9qO9FDMVDmrDcicvo-{kw8?>9; zQW#j~crHg~+n;Qyh1a`BKG5+B={)YsgwB`RH&=DoRa>4uHr6Pq6FnIB#qjcwnq$_B zhLMLT7n>m$@Pty(W^r+wI8VTZ#|$o-C%H)6Gn;vMKB3%ZE5SuXk$=L=jsH{HcC#-xk&J+ocZns71G@vRQl zZQatCUp3-me44I4&aqa$)8!KRp05%ipF(|QjZXgwPXG52BS}N`{ot1w`gVg6(+neK zq~^h-zBBedSA9!G5vd{ngjhX*A}$i2jX1`Dh=Vn@|1IKMxUse+^b0A%k)Jxf5kG>E&QH$^)9=(@Wb^_u-1;;cMj=vai> znCJTgue0I7KB$2Q#^WDcGxk9qX(-|?i0BVRfwozQo|(odqD#dO5n<->+&mpo9_E3$ z6PYKs+35E*+fVl$GF@EibFnGthGr2u;roAd!5)4YwIvkv%$P77(wv Date: Tue, 21 Jul 2015 13:20:35 -0700 Subject: [PATCH 05/84] Add xmpp dpd sig and fix a few parsing problems for connections that do not upgrade to TLS. --- scripts/base/protocols/xmpp/__load__.bro | 2 ++ scripts/base/protocols/xmpp/dpd.sig | 5 +++++ src/analyzer/protocol/xmpp/XMPP.cc | 1 - src/analyzer/protocol/xmpp/xmpp-analyzer.pac | 7 ++++--- src/analyzer/protocol/xmpp/xmpp-protocol.pac | 3 ++- .../ssl.log | 10 ++++++++++ .../ssl.log | 10 ++++++++++ .../Traces/tls/xmpp-dialback-starttls.pcap | Bin 0 -> 14673 bytes .../scripts/base/protocols/xmpp/client-dpd.test | 8 ++++++++ .../protocols/xmpp/server-dialback-dpd.test | 8 ++++++++ 10 files changed, 49 insertions(+), 5 deletions(-) create mode 100644 scripts/base/protocols/xmpp/dpd.sig create mode 100644 testing/btest/Baseline/scripts.base.protocols.xmpp.client-dpd/ssl.log create mode 100644 testing/btest/Baseline/scripts.base.protocols.xmpp.server-dialback-dpd/ssl.log create mode 100644 testing/btest/Traces/tls/xmpp-dialback-starttls.pcap create mode 100644 testing/btest/scripts/base/protocols/xmpp/client-dpd.test create mode 100644 testing/btest/scripts/base/protocols/xmpp/server-dialback-dpd.test diff --git a/scripts/base/protocols/xmpp/__load__.bro b/scripts/base/protocols/xmpp/__load__.bro index a10fe855df..0f41578f8a 100644 --- a/scripts/base/protocols/xmpp/__load__.bro +++ b/scripts/base/protocols/xmpp/__load__.bro @@ -1 +1,3 @@ @load ./main + +@load-sigs ./dpd.sig diff --git a/scripts/base/protocols/xmpp/dpd.sig b/scripts/base/protocols/xmpp/dpd.sig new file mode 100644 index 0000000000..50ae57a669 --- /dev/null +++ b/scripts/base/protocols/xmpp/dpd.sig @@ -0,0 +1,5 @@ +signature dpd_xmpp { + ip-proto == tcp + payload /^(<\?xml[^?>]*\?>)?[\n\r ]*]*xmlns='jabber:/ + enable "xmpp" +} diff --git a/src/analyzer/protocol/xmpp/XMPP.cc b/src/analyzer/protocol/xmpp/XMPP.cc index c84c372c4d..ee2667a276 100644 --- a/src/analyzer/protocol/xmpp/XMPP.cc +++ b/src/analyzer/protocol/xmpp/XMPP.cc @@ -61,7 +61,6 @@ void XMPP_Analyzer::DeliverStream(int len, const u_char* data, bool orig) } catch ( const binpac::Exception& e ) { - printf("BinPAC Exception: %s\n", e.c_msg()); ProtocolViolation(fmt("Binpac exception: %s", e.c_msg())); } } diff --git a/src/analyzer/protocol/xmpp/xmpp-analyzer.pac b/src/analyzer/protocol/xmpp/xmpp-analyzer.pac index a4417e1601..90b51ec183 100644 --- a/src/analyzer/protocol/xmpp/xmpp-analyzer.pac +++ b/src/analyzer/protocol/xmpp/xmpp-analyzer.pac @@ -16,7 +16,8 @@ refine connection XMPP_Conn += { // Yup, looks like xmpp... bro_analyzer()->ProtocolConfirmation(); - if ( token == "success" || token == "message" ) + if ( token == "success" || token == "message" || token == "db:result" + || token == "db:verify" || token == "presence" ) // Handshake has passed the phase where we should see StartTLS. Simply skip from hereon... bro_analyzer()->SetSkip(true); @@ -24,9 +25,9 @@ refine connection XMPP_Conn += { client_starttls = true; if ( !is_orig && token == "proceed" && client_starttls ) - { bro_analyzer()->StartTLS(); - } + else if ( !is_orig && token == "proceed" ) + reporter->Weird(bro_analyzer()->Conn(), "XMPP: proceed without starttls"); //printf("Processed: %d %s %s \n", is_orig, c_str(name), c_str(rest)); diff --git a/src/analyzer/protocol/xmpp/xmpp-protocol.pac b/src/analyzer/protocol/xmpp/xmpp-protocol.pac index e05268fe32..9b21679c30 100644 --- a/src/analyzer/protocol/xmpp/xmpp-protocol.pac +++ b/src/analyzer/protocol/xmpp/xmpp-protocol.pac @@ -3,6 +3,7 @@ type XML_END = RE/>/; type XML_NAME = RE/\/?[?:[:alnum:]]+/; type XML_REST = RE/[^<>]*/; type SPACING = RE/[ \r\n]*/; +type CONTENT = RE/[^<>]*/; type XMPP_PDU(is_orig: bool) = XMPP_TOKEN(is_orig)[] &until($input.length() == 0); @@ -12,6 +13,6 @@ type XMPP_TOKEN(is_orig: bool) = record { name: XML_NAME; rest: XML_REST; : XML_END; - : SPACING; + tagcontent: CONTENT; }; diff --git a/testing/btest/Baseline/scripts.base.protocols.xmpp.client-dpd/ssl.log b/testing/btest/Baseline/scripts.base.protocols.xmpp.client-dpd/ssl.log new file mode 100644 index 0000000000..0ce11b2e6f --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.xmpp.client-dpd/ssl.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path ssl +#open 2015-07-21-20-08-11 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer +#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string +1437091702.232293 CXWv6p3arKYeMETxOg 198.128.203.95 56048 146.255.57.229 5222 TLSv12 TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 secp256r1 - F - - T F5Nz2G1vSZQ0QXM2s8,FUw8omi2keRxShDUa (empty) CN=jabber.ccc.de,O=Chaos Computer Club e.V.,L=Hamburg,ST=Hamburg,C=DE emailAddress=support@cacert.org,CN=CA Cert Signing Authority,OU=http://www.cacert.org,O=Root CA - - +#close 2015-07-21-20-08-11 diff --git a/testing/btest/Baseline/scripts.base.protocols.xmpp.server-dialback-dpd/ssl.log b/testing/btest/Baseline/scripts.base.protocols.xmpp.server-dialback-dpd/ssl.log new file mode 100644 index 0000000000..15641ba5b0 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.xmpp.server-dialback-dpd/ssl.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path ssl +#open 2015-07-21-20-18-36 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer +#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string +1437506779.381295 CXWv6p3arKYeMETxOg 184.73.173.246 1193 104.236.167.107 5269 TLSv12 TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 secp384r1 - F - - T FLFr7Z1TXmFDv9FwC2,FydVem3ToAkEIAHD29,FK07OA1VxtQi69Irde F3D2e62Vxl7iTnwbA4,FUCD5w4ABMG5N0YvSi,FxWUEd3mgvThYO2uod,FGOrVE2laVCPsCLMF6 CN=www.0xxon.net,OU=Free SSL,OU=Domain Control Validated CN=COMODO RSA Domain Validation Secure Server CA,O=COMODO CA Limited,L=Salford,ST=Greater Manchester,C=GB CN=*.hosted.im,OU=Domain Control Validated CN=Go Daddy Secure Certificate Authority - G2,OU=http://certs.godaddy.com/repository/,O=GoDaddy.com\\, Inc.,L=Scottsdale,ST=Arizona,C=US +#close 2015-07-21-20-18-36 diff --git a/testing/btest/Traces/tls/xmpp-dialback-starttls.pcap b/testing/btest/Traces/tls/xmpp-dialback-starttls.pcap new file mode 100644 index 0000000000000000000000000000000000000000..ad55c6eceba70f34ebabaf3f8aa8f25f83fa0fe8 GIT binary patch literal 14673 zcmdUW2|QG7`~Mj;_I;Oi%9{1qvWx7Ih*C<~8QW-#wX%gulAWmRdv;0LrtoOB6d{yK zi#;iX@;hhHgQvIWUH#}5PZWR-0DyL%H2^U(Z}nP(k~!yzd{u zCkc*W*c|{!P^8re1d0TfR-FB(GypgbAe>iau1W&{4M2me1%xHR5?K;ag+?PN4oISi zClijr=gwwnV}l4l73v8te;J#I-7IlzI$Opj0!iFv!hmC_7l20zf8dcF8A1TBpli7N zMOK&H7g!cP?U%ARa11lt&?t8Mxx1l#9B^J3tcRMgv?M6JTV2%)k8`kbS6=->Lp40S z)^5~9y00z2-%U^uvsNEb<7X2?}LFyNRTKyZ-f;+BiLjKNr(}~u5WwYIpMx4qK zLT$8O>tqz*3=IM>30O`v{#%>aN6i9m2600cV{d9d{Az_knUA^T=^Xs@w<`S1em zoYs%5&R=1c)b5wm2r(kO5MGD^Lv3CP)(` z2s;Ua1Udo=xC+^p6p#dH5*P{O1R4SfKnlf`6%uHst^hTFLIMa9fC`XJ4BN(x-3 z28d;=jZnvgWWiDtB>$0=P)=A^NI;zg0F;r)7sgTDvS}1`>5c4;BlZqU`ILfuD;7PX`jg&`%98^Z*P`0a6_>lvEQ8MeWQWPMQ%&7EI(A zt4tDht+Y#-FE4u!(vne%M(9V((;^VmqyQKaBTWrbk|7VG$e2m=_DFMq+u$1tW+pu- zHsKv`=zTUGcFqo7(4{m3NC*E&&P;A*I5s4aqFA_S*q7O|&-F-3H@&N>ya zs2|KjBWr1|Ip(gg-SDx%k?*s91ssX2L)B4xwZj8DQhz9rotfzDix{Kz=b@jdm`ME^ zBfH=199Mkr%&URhIoJJIOW)MRG+FF?f(yOuvsY?&lda)#FL%gu07P~uq6b=$F5=4b z@S`|DwqTZ+G8g3pOX|LuF{VRaue7R_MP7UWd7wq+p+m8uSY%%-u$NyUw2c3tJaEYD zRx_ETNEgw9c<2#`6%-0dLYe{6z}1=IdlC>>Z7hg_RYf62h9D&+MaOZkG5(FC$%$~d;`s774E=fDAJJucxe@SVXn1K{zWKJP8 z2oeNn4h!T%?E?)#gABb4od~Tp-ere#TWe)!v)Ybm-;fEfn%tiTylTIz=BuSC2{i;m z!q>+Faa(`T7aI54Y(SC?J>dV{9`K;RPl1RCjT`CZ2}uOn}*_}hh6Pe(MxWkGZPu!%=DBiXF$GLu*&I@uSo~7 z$$C_$!!PbgzEl-9oMh4I$zh#zc}jvj{#xdphh0xjjkbuUifoVFhmG4WnEN?Een@Cg zH@EY~5&5VU<5(?|-ACei-AuPn9cC&1;ceolZdA@4!My02Qa`Mcz#D#G5E(DP;qge= zROh|tOF^w1+Shw?Cu;)lKXIRu(s$vZx$E%j z0Ku=#Citx@3nWon%$rdSi!PZ`OquAHk?Gz5zrRFhIDr3zUql4R1m6?kmjWb%P#9uk ze+R-`a1%6&1!Mxr$WYKFumMnw;2-%=I-nM)nXx-VEkfmQ0dvZ%Yj*go-Y@uiQ(>Ax=MT63;p2%yRpTYux2J+=QXS>JWUP(f4Mp0BdmLc&bxwN5;cdza zaN*Any>3%PX^D)oeeWmjw$c-}e!irr;)csgSlGk)Y7$EprE`xAC>r@V*nji2m*2!3 z|6tdUTQuk8+%fahNw=DR9E`qXaD3Umu0y>cX&m#UX;^>8Hk9EVjjqLK*%zJn?I!== zOCU|!%Ng4>Ki+ZqKq)Onzj{H2>FC{a_?`#z_+m-3rMiHn%$v*aGOHJjp4yP$oB~a*Gj+B3z#Of$5C`Gn|wlQoH~DGl5Xh6Q{Paz zL9#@WzL;+kV+uW-)}2o1-o8E5I%T2GFNYij6kyIz`ivB4M*+mLm93))AvrKxS6kl9 z))~=6&d=fRd1@~J#OzGirn^nBHn{gucZ7(GwxIac?JVefqtpycf$Eic(2D8z{y`EXiVJ|na zP&lYyH^dY(9cjqnAm@U(Hubt;qc!(P3qf3f$c1@+KGpGq$hwD(8`|951a0hq_r>B| zrFVd6xE_*u+h6NJr2POEi-)5J6gT8$L1|D^=7YH7DbsZ8wN2PYDo9Q-LeUK zU`WrNulBYaWiz$HnB4XyLY-<+{n)qo_i_RKEUq7}R)uGT2KbbnjOm^DPP=&3ZShrk z=ogm2Z;dT^Qu~9h1l?SA(!w>+0M9RsyNW-Xyqv<#QXKg1t;cd-!AP9I3oTdM+JoQ6 z?$*22T%4u5lsb~;hPkrfrw{7%l)R=@)v;cdNir3#cWQnpS>rmU+t!B8sNi#WsZVhy z$GGtGSzU#&@sE!~^tGIfaiVF*8jQF^B9HAjRb8CYKQ(N6wwY4b*j@d~Vj}yC)3O$t zs+UYBscVjviQ93$vbevye$=WZtxV_2n`}#g>N|yp)H<+5D+Y%fKNYNesVx~*Sh`dD zgw8zpk+1`E$sst+f}JLrbg3X?MR8&v1ZZqn{BA~1$b z98#Hkmi6Z7txD961Bt=$n2*s$4l}Vx2~Rx9*hMGSotOh8b-k}lTn_3sWn_!k1bKEg zXBCNWFVbrlxk{$v(-g`fT-{Sz{mA6K>ldSlli+CHE4GN?z7>t3rOrL)gR*6nC6DoH zGKSc-bDVS)o5+j2C4EfUMQ^y-|HyMRXW3_$lpl#Gvo^xGP5aQmNcXU$W;r?FSZDoPa_+K1s<7mZU~sajHg=zCcLd& z4Va`Llu}p$8Q}+L6cq>s(6w-=4L{;Tsevk>Qiei?e1z=JSmh?^4P|Qaa9AR4E7qNH z-DkBlVNWCmKl$}w2L->KpfD)7eh(rdwr%A6FuI+gCjVXPZh@F?)7YFh5*9ETVXZY` zNYIZkC8oKcKd`JiNY*lxm4M4uq$HUN$%T<}_mj;?$-0^yfsoSQ4*)jJaSK-_?lA0| z&d!`j4exwXo4M11_M_!|*QokuAC$+;0na*K)s=m;c$dc^p@PqvGeoV7zTUZ*>E2>a z>D~5Ln9eH9UqOS}_iObH?oO3rcBbdkavXtTqyEo|Ep%wgL>4;--kBtxuX`14fjJY` zci^E{NE7$$`&ti2M$#(Y1c{*7#JD3fM(x?LCyh?)(Wp>wo6D?li%>sS)FK)B=o@*` z9sPYVpA+sEdS#81o#MatGHZ6tX%@EZW zAAH1eSW|$f@-nOSVZDKA58dyqU;9s_Rt@ykj%ZXpNmeQiuj7fXA3T@Vl{Y=3f@M&| zLRlMxI}L!qAOQ{=)Z>T#=YvtQyu0JLHRQcH2a5EpRa8+ahm#)H@;*(uA#SSkpt{hL zevmxD%QRLkU%gU1N%85sCjCiYwkh6f(a+8U^A{#{8QR_OEERx0_w{5g{{BbfyK1*t zJK~lK>Ea+_2SkV$@40Am?{w-5ZnuNwOQ8&M5^oQgu_bu;w=bC9p-fI{I_OTubfUbb z#v$MVO*N0d0vF|Nwd^xx%^X4(x`XZ?w4kazSt`@;MYZ)PmlfU17K`BowjE4k?Sp4~ zH5s{v>fW`!9qY}hy;Tw}AX$v*calBg-JtI}ap#^@Gb!N1K(4((mWr=cf^xzU;fpFkFEB z(K1-=}qOKI~km??ZesR@35qb0aS1d$}^i?YxjVc;2*Er*(`bUo%Z1 zx(XZ1TVAp*U<@4LRGTC%00ZDVim52K7Y+Q7q3 zQd$BOgDC(tv%oqWK}h5!>4ddkuPB9c@Wgsy@K~I`lr%&G;OasA{XbH5Q%v$nJPDWM+l;0&M4JYrQJfO@jP)SZ+R^Ad+`e%}NYfJ}8LoF)ljD=E< z_L3O)zs=JXip0A{#V+~}r^GTn&+G#jFxxA)QPAH|3-V+<889w-!$pc!p)=X0;GTAl zU={8Wx8m2diE9xIf@LH$lM`p69r%t(U4MMo)qVSN_l0YJOqhK?9^Wiw)QYB}xz5h; z+G&ieb<}R#kw>@BD0r|uk0=PXN{;g?Q^=v_p4o|EcNsUODxyAXIr_LUZTrKPx3aBd zdGicXCznO*Z?b=|e;`;R^QviZu8q#d-|@x$gBKSL(sv=AJb$Kqig9e>54Xa~#{A={ zhFA3^MfrPr(&p=spS+)k-CfueQS zU-fJ39=SA=xw(m&*1ybZfxUbcv{5_{N?>%zQJ!4LO zro{1kaAMa}$8$4z9Y2->;vQw9-{kI_iZe5eJ*cW~SJx3WsGv4`T~40eS~@@1tcPKy z8+0n&<9?l5Fk2>k{}sa>uc;c%Yy>~iJzD6~W4d+RkQOv7`d0af(a_w$s_>P?^k`v4 zb~W3lIh?+roHFHm02(<96jfC(A{n$2ziO=LfH6mx=udwz=kS z(b6ILWjCm5rMH~QR^P0Uq-h!r+3SKxV`o(3Cs2`hwp?hzQP0l(tMM^&5l6r1oELAGI!@=G9s zPs7&FWRz3G$r+qPYNqEJNHsmscN1RFzl09r*3KUw2o(Y0l)?IW#FpcOO%N&%i{(Q} zg5scPhDe4`#82MvmneXPHZ2lPGQ=RD3pYcfc7xFVXKh0SQ;D@6A@nB(6Zmw-?sxp- zSKvqy=oZg#U$Hi+Fvf5t^TdACw43_X^JEW9F6Vj8TS#*?){jUHEZli^bh!G%dw*Lx znyg+IJ1^;*UkmheR462}tYUhrPYdp6JOsAXXkR-ee(l}y_tP{sMNK23TvK7!YtNp! ztJSWZ&+JJX%lG9zbIDoj{ytoRmjLh08$0M8vR|-0tj$$?K@d7fe$v8aJ78MWnZda zs7HYa02Km(;~5AX9ag~+$@PJC%R+&6o6&LYu#}3K!`#^c{gcY+;jrGGYs7F3{ivB& z(GSKgEjJr4FSN`C=z`B`f0Dhmg9d3RQ?wDcWT9hPB8-6{|8l|K$igl_L{n!VsOX{&oP{Aqs$s5+IhX zI|qtoyCpb&xM4QO51A-p{NPA04~hVQ9cP*ITG3hvm9Uxj;Rfl#V{>6##u?QEo+)iI zhze45a^tBli~FUHfIB`EZ#(4rCYXGAmd!g@$@J8_Z}K&GU3T!2bY(LF6gI}14OxPJ-bzAK86CXDfH%s19R|`X>5Ixn-h7JZazj$YZ}AM&DrJS!FrOml0uBH+^~qLp`kD zd@G*%K9zaP4J#HW1-FN}QqmL(MIP9w`m{F|Rstbq$;ns4O5AQJ7&*?`#4@5s8o~5{ zoSl^AujNkI`*LWQzKR0#$^aA!!b3Rm`*Dpd0vpkg6iTB1Z=oS52kw$ll2(BJ5?S2F z+y9+Ggd)%zC%^h03oZN}3kC5#mY3Ih$1FGuGLMe;p3sjNYaCh`nRep-;^16maWdn$ zZ{1{?!Ki1j+93`*3hax|k2`k8#uh0YoeYZnO+<>TuRo0L5!_+T>-d$pSJ^o0! z=M7U9vqWEX*$F1K3<^?;nq#>RiYT2cyxG>)_*TtaoGq7aD zlA9gwU<_2mw=S&Q#Kc?m+1XfVsK>SQ|Q zKFlZgDY~eb~weU$J~f)o&=E`>A8Rw^WM34(ua{*kK1hr5_G(7Ht9SN zxY-j*Qo`?#xl~)Tuf4a1AeLUddhPH7xHc6RdYlVlHy`f+2S8ZWVt zXLCkBgigM(VZ`rUvEV#M@2xM{VTxhTAYrme?CH{|WH6d)xxt{7mwTP6C($;Qw&Q$g zs|?MESh~)8+H{et6}Lq#ElL#V%XoqY8q(q1o+ASQ*xI0GBA+Tp@5w%7u}7-zfVzz; zxuK7^xmGdNn<4i78lTAS-bXugeNScaabl>umGJr#-H6-QV{?r-V=p}Srhaqd^2@nd z@*muRIXg>_sD++es-|_OON}h7IVdAkbRe1)lwRW4E&i2Gd;WPus2K~i?!-3`0g+Qf z_fB(vs*PyU!Xm?OwnY=Gx}X|Hg^5_6hI!8hwfcFTN@4((}}UkO9Z=r{Z-pUAIb z|H`jatl^yMI=@=PQ2czsckB^w{;^%K7W%Q7l?(2uP8+IyQ7xx#G@9K&BRW{8X|TT@bK-b zq0BPz1IxOKr2>|rJGg8miNKWI4_Fo+|4UiWBwQT<0D8?AxVyIc+P-z4W(y{l5+3Zw zN2f4}yi*u=VzJ4gd;28tAop$-HirDuN8cBtnyQpK+Z1}Uq<(yV=1V52w%3_cY=7^l z!o_w&J5|2Ul6^K#VlJwuBTGmpEfoXnFYRJF%A+$Hc-xaHC_^dJP9eSS#kpjZ2d6Wd z@BVSFDIr&Dj?+<@qUP$R-`Q#~?4$%&=ACURCe;@Z28@w(3Z~wQhda$t9Y^|>q86>5 zW;P?Or)z2sPU;0Ff2Wk~qwyj5^uTNJ9!~{KK`)=dWvfCF!y$#RLN$7~D74~Kj;fFe zM=^(=3vn$FRcoqz%ecdGW&Fi~s+L5J?R^behw67nT;N z@lOfzs0OZZ9xMIEyIuip@!u^I>gFC^5{M3J_6X}xV9m`IiQ?TpMnl6Tn2IyI==9mZ zm2Ys@R9Pu#z}YO48$LQykx|YVc75dgQPI4Z3e5ZR{tL%FQ1RFD1rHmd=<$5UJ^dCZVeOCIp#Gg#tq{FRX|pG&3K@-XVoVr{ zIS#hNHUocPS-PdvKoN8em%n(`ReONwRV}}i1zD0u=c-p>*Stz+-!Q0DP!e;ZBJZtN zvPX7uR%@`*Ig>8BJg(>CsOm#Py7^LzOU@U+=Jo+U@;t_sPV)FM->Pbq@QGI#f1}+t zgEvi4`s6dqTpxX@dHT5i5eMfl?8lZT_1;WG*f3Vpeac71+Rffvfk!S^K@DU=qlU{? z9{}P*iin`bzu5;kF^~_W!am^3vpRCeH6Iv>BL7ih%5$ti)n!PUCO{tl?y<2eha@#+ z>t5&b;n5zPnGRw9vz#fM(yM^p^nqnnf^+Z#X->zK2qzv3Q}c<8MO)IO9YfLDRpI%l zEDz(im>b-8`I$8G2D5_x=x~7t95Y7q8@1UZh-y>Vvec=N+PX~`i#g1vi1UR0z_O$# zXn=#zHC+BOdA*Qg;^aNHOuieMd>TA?pDqAkg}{(=ZSwC-3d6GX#B5Iy%5!8x_EPYW zeSkahhdTL9@VP Date: Wed, 22 Jul 2015 10:35:49 -0700 Subject: [PATCH 06/84] Basic IMAP StartTLS analyzer. Parses certificates out of imap connections using StartTLS. Aborts processing if StartTLS is not found. --- scripts/base/init-default.bro | 1 + scripts/base/protocols/imap/README | 5 + scripts/base/protocols/imap/__load__.bro | 2 + scripts/base/protocols/imap/main.bro | 11 +++ src/analyzer/protocol/CMakeLists.txt | 1 + src/analyzer/protocol/imap/CMakeLists.txt | 11 +++ src/analyzer/protocol/imap/IMAP.cc | 86 ++++++++++++++++++ src/analyzer/protocol/imap/IMAP.h | 38 ++++++++ src/analyzer/protocol/imap/Plugin.cc | 26 ++++++ src/analyzer/protocol/imap/imap-analyzer.pac | 57 ++++++++++++ src/analyzer/protocol/imap/imap-protocol.pac | 17 ++++ src/analyzer/protocol/imap/imap.pac | 35 +++++++ .../conn.log | 10 ++ .../ssl.log | 10 ++ .../x509.log | 12 +++ testing/btest/Traces/tls/imap-starttls.pcap | Bin 0 -> 8511 bytes .../scripts/base/protocols/imap/starttls.test | 9 ++ 17 files changed, 331 insertions(+) create mode 100644 scripts/base/protocols/imap/README create mode 100644 scripts/base/protocols/imap/__load__.bro create mode 100644 scripts/base/protocols/imap/main.bro create mode 100644 src/analyzer/protocol/imap/CMakeLists.txt create mode 100644 src/analyzer/protocol/imap/IMAP.cc create mode 100644 src/analyzer/protocol/imap/IMAP.h create mode 100644 src/analyzer/protocol/imap/Plugin.cc create mode 100644 src/analyzer/protocol/imap/imap-analyzer.pac create mode 100644 src/analyzer/protocol/imap/imap-protocol.pac create mode 100644 src/analyzer/protocol/imap/imap.pac create mode 100644 testing/btest/Baseline/scripts.base.protocols.imap.starttls/conn.log create mode 100644 testing/btest/Baseline/scripts.base.protocols.imap.starttls/ssl.log create mode 100644 testing/btest/Baseline/scripts.base.protocols.imap.starttls/x509.log create mode 100644 testing/btest/Traces/tls/imap-starttls.pcap create mode 100644 testing/btest/scripts/base/protocols/imap/starttls.test diff --git a/scripts/base/init-default.bro b/scripts/base/init-default.bro index 473d94fc84..58d2b4b2b9 100644 --- a/scripts/base/init-default.bro +++ b/scripts/base/init-default.bro @@ -44,6 +44,7 @@ @load base/protocols/dns @load base/protocols/ftp @load base/protocols/http +@load base/protocols/imap @load base/protocols/irc @load base/protocols/krb @load base/protocols/modbus diff --git a/scripts/base/protocols/imap/README b/scripts/base/protocols/imap/README new file mode 100644 index 0000000000..ba96748489 --- /dev/null +++ b/scripts/base/protocols/imap/README @@ -0,0 +1,5 @@ +Support for the Internet Message Access Protocol (IMAP). + +Note that currently the IMAP analyzer only supports analyzing IMAP sessions +until they do or do not switch to TLS using StartTLS. Hence, we do not get +mails from IMAP sessions, only X509 certificates. diff --git a/scripts/base/protocols/imap/__load__.bro b/scripts/base/protocols/imap/__load__.bro new file mode 100644 index 0000000000..aa3a41ef5e --- /dev/null +++ b/scripts/base/protocols/imap/__load__.bro @@ -0,0 +1,2 @@ +@load ./main + diff --git a/scripts/base/protocols/imap/main.bro b/scripts/base/protocols/imap/main.bro new file mode 100644 index 0000000000..9f0305c80c --- /dev/null +++ b/scripts/base/protocols/imap/main.bro @@ -0,0 +1,11 @@ + +module IMAP; + +const ports = { 143/tcp }; +redef likely_server_ports += { ports }; + +event bro_init() &priority=5 + { + Analyzer::register_for_ports(Analyzer::ANALYZER_IMAP, ports); + } + diff --git a/src/analyzer/protocol/CMakeLists.txt b/src/analyzer/protocol/CMakeLists.txt index 467fce83ee..9e824d42d2 100644 --- a/src/analyzer/protocol/CMakeLists.txt +++ b/src/analyzer/protocol/CMakeLists.txt @@ -16,6 +16,7 @@ add_subdirectory(gtpv1) add_subdirectory(http) add_subdirectory(icmp) add_subdirectory(ident) +add_subdirectory(imap) add_subdirectory(interconn) add_subdirectory(irc) add_subdirectory(krb) diff --git a/src/analyzer/protocol/imap/CMakeLists.txt b/src/analyzer/protocol/imap/CMakeLists.txt new file mode 100644 index 0000000000..755221b25a --- /dev/null +++ b/src/analyzer/protocol/imap/CMakeLists.txt @@ -0,0 +1,11 @@ + +include(BroPlugin) + +include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) + +bro_plugin_begin(Bro IMAP) +bro_plugin_cc(Plugin.cc) +bro_plugin_cc(IMAP.cc) +bro_plugin_pac(imap.pac imap-analyzer.pac imap-protocol.pac) +bro_plugin_end() + diff --git a/src/analyzer/protocol/imap/IMAP.cc b/src/analyzer/protocol/imap/IMAP.cc new file mode 100644 index 0000000000..ad38d598ac --- /dev/null +++ b/src/analyzer/protocol/imap/IMAP.cc @@ -0,0 +1,86 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "IMAP.h" +#include "analyzer/protocol/tcp/TCP_Reassembler.h" +#include "analyzer/Manager.h" + +using namespace analyzer::imap; + +IMAP_Analyzer::IMAP_Analyzer(Connection* conn) + : tcp::TCP_ApplicationAnalyzer("IMAP", conn) + { + interp = new binpac::IMAP::IMAP_Conn(this); + had_gap = false; + tls_active = false; + } + +IMAP_Analyzer::~IMAP_Analyzer() + { + delete interp; + } + +void IMAP_Analyzer::Done() + { + tcp::TCP_ApplicationAnalyzer::Done(); + + interp->FlowEOF(true); + interp->FlowEOF(false); + } + +void IMAP_Analyzer::EndpointEOF(bool is_orig) + { + tcp::TCP_ApplicationAnalyzer::EndpointEOF(is_orig); + interp->FlowEOF(is_orig); + } + +void IMAP_Analyzer::DeliverStream(int len, const u_char* data, bool orig) + { + tcp::TCP_ApplicationAnalyzer::DeliverStream(len, data, orig); + + if ( tls_active ) + { + // If TLS has been initiated, forward to child and abort further + // processing + ForwardStream(len, data, orig); + return; + } + + assert(TCP()); + if ( TCP()->IsPartial() ) + return; + + if ( had_gap ) + // If only one side had a content gap, we could still try to + // deliver data to the other side if the script layer can + // handle this. + return; + + try + { + interp->NewData(orig, data, data + len); + } + catch ( const binpac::Exception& e ) + { + ProtocolViolation(fmt("Binpac exception: %s", e.c_msg())); + } + } + +void IMAP_Analyzer::Undelivered(uint64 seq, int len, bool orig) + { + tcp::TCP_ApplicationAnalyzer::Undelivered(seq, len, orig); + had_gap = true; + interp->NewGap(orig, len); + } + +void IMAP_Analyzer::StartTLS() + { + // StartTLS was called. This means we saw a client starttls followed + // by a server proceed. From here on, everything should be a binary + // TLS datastream. + + tls_active = true; + + Analyzer* ssl = analyzer_mgr->InstantiateAnalyzer("SSL", Conn()); + if ( ssl ) + AddChildAnalyzer(ssl); + } diff --git a/src/analyzer/protocol/imap/IMAP.h b/src/analyzer/protocol/imap/IMAP.h new file mode 100644 index 0000000000..a1f59e5010 --- /dev/null +++ b/src/analyzer/protocol/imap/IMAP.h @@ -0,0 +1,38 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef ANALYZER_PROTOCOL_IMAP_IMAP_H +#define ANALYZER_PROTOCOL_IMAP_IMAP_H + +#include "analyzer/protocol/tcp/TCP.h" + +#include "imap_pac.h" + +namespace analyzer { namespace imap { + +class IMAP_Analyzer : public tcp::TCP_ApplicationAnalyzer { +public: + IMAP_Analyzer(Connection* conn); + virtual ~IMAP_Analyzer(); + + virtual void Done(); + virtual void DeliverStream(int len, const u_char* data, bool orig); + virtual void Undelivered(uint64 seq, int len, bool orig); + + // Overriden from tcp::TCP_ApplicationAnalyzer. + virtual void EndpointEOF(bool is_orig); + + void StartTLS(); + + static analyzer::Analyzer* Instantiate(Connection* conn) + { return new IMAP_Analyzer(conn); } + +protected: + binpac::IMAP::IMAP_Conn* interp; + bool had_gap; + + bool tls_active; +}; + +} } // namespace analyzer::* + +#endif /* ANALYZER_PROTOCOL_IMAP_IMAP_H */ diff --git a/src/analyzer/protocol/imap/Plugin.cc b/src/analyzer/protocol/imap/Plugin.cc new file mode 100644 index 0000000000..8660879bc3 --- /dev/null +++ b/src/analyzer/protocol/imap/Plugin.cc @@ -0,0 +1,26 @@ +// See the file in the main distribution directory for copyright. + + +#include "plugin/Plugin.h" + +#include "IMAP.h" + +namespace plugin { +namespace Bro_IMAP { + +class Plugin : public plugin::Plugin { +public: + plugin::Configuration Configure() + { + AddComponent(new ::analyzer::Component("IMAP", ::analyzer::imap::IMAP_Analyzer::Instantiate)); + + + plugin::Configuration config; + config.name = "Bro::IMAP"; + config.description = "IMAP analyzer StartTLS only"; + return config; + } +} plugin; + +} +} diff --git a/src/analyzer/protocol/imap/imap-analyzer.pac b/src/analyzer/protocol/imap/imap-analyzer.pac new file mode 100644 index 0000000000..918d339cfe --- /dev/null +++ b/src/analyzer/protocol/imap/imap-analyzer.pac @@ -0,0 +1,57 @@ +refine connection IMAP_Conn += { + + %member{ + string client_starttls_id; + %} + + %init{ + %} + + function proc_imap_token(is_orig: bool, tag: bytestring, command: bytestring): bool + %{ + string commands = std_str(command); + std::transform(commands.begin(), commands.end(), commands.begin(), ::tolower); + + string tags = std_str(tag); + + //printf("imap %s %s\n", commands.c_str(), tags.c_str()); + + if ( !is_orig && tags == "*" && commands == "ok" ) + bro_analyzer()->ProtocolConfirmation(); + + if ( is_orig && ( command == "capability" || commands == "starttls" ) ) + bro_analyzer()->ProtocolConfirmation(); + + if ( command == "authenticate" || command == "login" || command == "examine" || command == "create" || command == "list" || command == "fetch" ) + { + bro_analyzer()->ProtocolConfirmation(); + // Handshake has passed the phase where we should see StartTLS. Simply skip from hereon... + bro_analyzer()->SetSkip(true); + return true; + } + + if ( is_orig && commands == "starttls" ) + { + if ( !client_starttls_id.empty() ) + reporter->Weird(bro_analyzer()->Conn(), "IMAP: client sent duplicate StartTLS"); + + client_starttls_id = tags; + } + + if ( !is_orig && !client_starttls_id.empty() && tags == client_starttls_id ) + { + if ( commands == "ok" ) + bro_analyzer()->StartTLS(); + else + reporter->Weird(bro_analyzer()->Conn(), "IMAP: server refused StartTLS"); + } + + return true; + %} + +}; + +refine typeattr IMAP_TOKEN += &let { + proc: bool = $context.connection.proc_imap_token(is_orig, tag, command); +}; + diff --git a/src/analyzer/protocol/imap/imap-protocol.pac b/src/analyzer/protocol/imap/imap-protocol.pac new file mode 100644 index 0000000000..15bb753475 --- /dev/null +++ b/src/analyzer/protocol/imap/imap-protocol.pac @@ -0,0 +1,17 @@ +type TAG = RE/[[:alnum:][:punct:]]+/; +type CONTENT = RE/[^\r\n]*/; +type SPACING = RE/[ ]+/; +type OPTIONALSPACING = RE/[ ]*/; +type NEWLINE = RE/[\r\n]+/; + +type IMAP_PDU(is_orig: bool) = IMAP_TOKEN(is_orig)[] &until($input.length() == 0); + +type IMAP_TOKEN(is_orig: bool) = record { + tag : TAG; + : SPACING; + command: TAG; + : OPTIONALSPACING; + tagcontent: CONTENT; + : NEWLINE; +}; + diff --git a/src/analyzer/protocol/imap/imap.pac b/src/analyzer/protocol/imap/imap.pac new file mode 100644 index 0000000000..33382bc26d --- /dev/null +++ b/src/analyzer/protocol/imap/imap.pac @@ -0,0 +1,35 @@ +# binpac file for the IMAP analyzer. +# Note that we currently do not even try to parse the protocol +# completely -- this is only supposed to be able to parse imap +# till StartTLS does (or does not) kick in. + +%include binpac.pac +%include bro.pac + +%extern{ +namespace analyzer { namespace imap { class IMAP_Analyzer; } } +namespace binpac { namespace IMAP { class IMAP_Conn; } } +typedef analyzer::imap::IMAP_Analyzer* IMAPAnalyzer; + +#include "IMAP.h" +%} + +extern type IMAPAnalyzer; + +analyzer IMAP withcontext { + connection: IMAP_Conn; + flow: IMAP_Flow; +}; + +connection IMAP_Conn(bro_analyzer: IMAPAnalyzer) { + upflow = IMAP_Flow(true); + downflow = IMAP_Flow(false); +}; + +%include imap-protocol.pac + +flow IMAP_Flow(is_orig: bool) { + datagram = IMAP_PDU(is_orig) withcontext(connection, this); +}; + +%include imap-analyzer.pac diff --git a/testing/btest/Baseline/scripts.base.protocols.imap.starttls/conn.log b/testing/btest/Baseline/scripts.base.protocols.imap.starttls/conn.log new file mode 100644 index 0000000000..0ae19c2fda --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.imap.starttls/conn.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open 2015-07-22-17-31-02 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents +#types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] +1437584567.812552 CXWv6p3arKYeMETxOg 192.168.17.53 49640 212.227.17.186 143 tcp ssl,imap 2.827002 540 5653 SF - - 0 ShAdDafFr 18 1284 14 6225 (empty) +#close 2015-07-22-17-31-02 diff --git a/testing/btest/Baseline/scripts.base.protocols.imap.starttls/ssl.log b/testing/btest/Baseline/scripts.base.protocols.imap.starttls/ssl.log new file mode 100644 index 0000000000..aefbf3d41e --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.imap.starttls/ssl.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path ssl +#open 2015-07-22-17-31-02 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer +#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string +1437584568.570497 CXWv6p3arKYeMETxOg 192.168.17.53 49640 212.227.17.186 143 TLSv12 TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 secp256r1 - F - - T FOWmhO3rUj3SEB5RTb,FjH9n52SzEIJ9UoVK9,FisDHa396LIaZadgG9 (empty) CN=imap.gmx.net,emailAddress=server-certs@1und1.de,L=Montabaur,ST=Rhineland-Palatinate,O=1&1 Mail & Media GmbH,C=DE CN=TeleSec ServerPass DE-1,street=Untere Industriestr. 20,L=Netphen,postalCode=57250,ST=NRW,OU=T-Systems Trust Center,O=T-Systems International GmbH,C=DE - - +#close 2015-07-22-17-31-02 diff --git a/testing/btest/Baseline/scripts.base.protocols.imap.starttls/x509.log b/testing/btest/Baseline/scripts.base.protocols.imap.starttls/x509.log new file mode 100644 index 0000000000..6d1be68725 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.imap.starttls/x509.log @@ -0,0 +1,12 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path x509 +#open 2015-07-22-17-31-02 +#fields ts id certificate.version certificate.serial certificate.subject certificate.issuer certificate.not_valid_before certificate.not_valid_after certificate.key_alg certificate.sig_alg certificate.key_type certificate.key_length certificate.exponent certificate.curve san.dns san.uri san.email san.ip basic_constraints.ca basic_constraints.path_len +#types time string count string string string time time string string string count string string vector[string] vector[string] vector[string] vector[addr] bool count +1437584568.769690 FOWmhO3rUj3SEB5RTb 3 339D9ED8E73927C9 CN=imap.gmx.net,emailAddress=server-certs@1und1.de,L=Montabaur,ST=Rhineland-Palatinate,O=1&1 Mail & Media GmbH,C=DE CN=TeleSec ServerPass DE-1,street=Untere Industriestr. 20,L=Netphen,postalCode=57250,ST=NRW,OU=T-Systems Trust Center,O=T-Systems International GmbH,C=DE 1384251451.000000 1479427199.000000 rsaEncryption sha1WithRSAEncryption rsa 2048 65537 - imap.gmx.net,imap.gmx.de - - - F - +1437584568.769690 FjH9n52SzEIJ9UoVK9 3 21B6777E8CBD0EA8 CN=TeleSec ServerPass DE-1,street=Untere Industriestr. 20,L=Netphen,postalCode=57250,ST=NRW,OU=T-Systems Trust Center,O=T-Systems International GmbH,C=DE CN=Deutsche Telekom Root CA 2,OU=T-TeleSec Trust Center,O=Deutsche Telekom AG,C=DE 1362146309.000000 1562716740.000000 rsaEncryption sha1WithRSAEncryption rsa 2048 65537 - - - - - T 0 +1437584568.769690 FisDHa396LIaZadgG9 3 26 CN=Deutsche Telekom Root CA 2,OU=T-TeleSec Trust Center,O=Deutsche Telekom AG,C=DE CN=Deutsche Telekom Root CA 2,OU=T-TeleSec Trust Center,O=Deutsche Telekom AG,C=DE 931522260.000000 1562716740.000000 rsaEncryption sha1WithRSAEncryption rsa 2048 65537 - - - - - T 5 +#close 2015-07-22-17-31-02 diff --git a/testing/btest/Traces/tls/imap-starttls.pcap b/testing/btest/Traces/tls/imap-starttls.pcap new file mode 100644 index 0000000000000000000000000000000000000000..f6bfe5458d4c78e9c9023da6c7f835b3a78f6c54 GIT binary patch literal 8511 zcmds6XIN89x1OCw=p}SPf+$Mg0RnrD;#0Dtm*M07H?|w4b{atHTduPv7TsTxj03`5j>+S{sgD1ys z91u{aC;?x1jM~bO17TAtVoDNqV^yet1AtrFH8B$6n4qMOPC;117Gli|u_Ky*5=hXMdjS#5@p zkp!^fT(e&Z=M zznU9(mm@>D&`bavXadxF>xDTA>X2R|M)lI`G}6mo7Z{Rp0Lp*PF`^ zp>uuM{`RTD3PnKVR zCntd8$rHGb;dt=r03GrM@^@XrYFWmCr-l|c60`A&vY0reZ~s+!39YXR*aBLu3wsM z1aqtvj;C`N)07pEYl2z`{kZ1JS>&3NxqV$T>bk$0Ye(lonQI%btN)6F(pAL5=P04 zPOc{R6AqHUkR#76>E*~LekES}VL1H#Hdhu7g%IUi1?dwG_VG6niuSBuqQG#N_W^~& zxA>RoWq61R0--xm>+L`=T2L$Bc<^M|4_)?g;=QGq1ML+!wx=1Hn=^l zmkrvWCCi~Dzq}nUL&++m0uc;a)Ow>}Q--x7lwhl0qCiW=)BvDO#4sleU`!%WQ@3(p z;w{s1ezrl?D2tHShd4szi%Tw@ZX7%<n9euo*`NGv2nB__DdTR|Zx z0MU>#T){|S2W$ZY;9w)z0P?_kNLHS}1Gq~V5-W+BWQasXLIRth18l%Lzy@*>b%~rr zNkRZ*K$2)ks6ZRgfErMiC`*Vy0cZgYpbC_LJm@As>j;1%AQ1sZfVT{QKF|Sra6bU( z!eecq0+iu>^soXjfgVu?bTE*DQ3wQ zZipUJq-pq$fo&tfPbsEu4nlB+`^(6T?%iAy-=`2}QUV8}ze#Fft&D zF;xJsD-gsO2Cle@lpl-;YD`v)v6*r#wk@U9bB^|h3O->`LuFmV9Mx49ub--On>%o9 z>*Pn>?zwx-_F-XGqmH*O)lA9XT^zM>`bXW}_TlvI7p(Ri&$XtrXNfuba>k3Ao6bJE z6}MxcOULRWP1`r41VP2^8p--C%ELOV7F{!3rEjP+bljgD0#2n+qO+7>g@OJxM&L#yF{TSoG;vO$MlHVr7-JaPS&}EsBokf+WiJAGKuvS zYltoW0}Y-aJj>*C*7r<@*QB`)_p)raOzv^BN4fXGzQ?|mgq#j}cjQYvfhe~Z7^9zQ%L zt>K0=lu!g5;Db@js*_-m6!$1&V#V)8ElT)vJ%e-u>lp@ACJwK@S#rwkg1vVG>C;n1 z-In^YMvLKgdyR`Ku=iaa+b$HChWnlvw}X{ZkiML_|3ltQIq^Zkd-JPFzC*6zWBo#o zY}uB5#&-=U=6!Er_-W9l)2iXA@G6Ds0sY-yc_Is*G5zvTf_bQ=gNmitG--l z{F}PC)1*4W<3x9A`Y=Q5_Ac{XZ;GPt%hPr(blkt@;Ym)O(vX?6uD3nw4x(sec_efz zZ*SAuKKXNtX5g9!QRVd_`=&#Vr@y>bsJTt+9ILY4-%0t(-K9r|U@Hb647+|$b@D?~ zjRcbg3(hp)x=%OgFx2~Z4SAa<|#}UrA5XF=_qC6lN0G|uO5{AHIwN*DA@!@ zWd^d2exKI^l9M5Gvmwn&ITW|U84Nq-a9bzDp-!;IIUx?kamb$G_L5b`0me#SK8Dq4Qw*!OOi->pO3_WR;MS+P8Ez{)Jeak{IJY`v z>3OfYlV)tFGM@kby?UE!zVkvkLg|Dx5upZ&tbCIYRg3u6>;b{YuV;QJQmnN5Ywh~P zJkpoZ({0u{yHi}00?nAJyi9a!KLAvf{!v7#&YR_1*&+h9{r z6Mye3-x40jhI@#Qbs_h))$T$n!KWGC-|PTiszRyqj^hb=bS;TouZNUI4I zr@*2Dhp_*XB%9TN)}2AohsQp)s4j_MQ;zvP%-yRvGd7hOpENDAVouZK z+y?6{Se4Cy;#zwLb%Wbh0d0q=wNv$jPEV%HaAk$ATKV>k`J*E__SS(43HKGNpEqV? zXYI61CX`-d+FQ?c+kaba6Bx2BvIaq1y1=r~Wlx z?X~XWE0>(lbx-9N5iFHg zetP<&rE&hYKVNJS`UcDX#MiNW@r@EPx1e(!jRyW5jn#V5rfajOUHHP~29wHPwAE7rv8@47b2 z8Tp+w!tL4iv(6W|fRH_%)k>YVE*5+Jt1B$hpRjND@ik>P3)*%z$5{o~KXUFEpFXbo(HlnB%c*&`7x`MAN?rMa7yF**QVWZ7 z7IR@X%bRg<1i#Hm$7{C!a>Y+Zj1UItP`S<<%PSEM zf@g!O775DoM(>XLP-);$vmP#oq6|g$+J|K zeFfcfL!Rez%HEAAz3HTvXWP(f9KP~7!+5#O^Y3k{|t$&d!4hF zW~WSlYfZ@rsd~w&8ODC%wp3iG<5*|An8Tw|_fvmNT73`&zCY1wkb;bqNvFtRIji63&ae7-=N36IDhN(tVLnO#;}U@mn`G+lKsVmV zuS(NvO?h{^e8^|<-jAB4`gZSkt4+0-pAo{YHFv(9el6y5HU@U8K1+@u`#;zqaSR77 z?2=>2dFSDv0<3ucR!lOCtT4R#$He4}I_jH&&c(g~-Ct^esp*y-Gv7v;rcN_k;Z$+= z-uA@y{14Bj;p;0kUZjr-5{#=2)Y+joHDS$z=w*+zt+*`B%9Yz}J|;Im(vE+%roFXx z-Rxr@>{c(E=qxeW7*t*(_PAIzW7nzmxsU%*?hLndJ^Q7-p)rN3X;S4>G;iP8Js0n{ zsdv>rR+kuEx^lU+O)lWY@e{9RSbQW0W0t=A?iXrw2ose~EVYWcP#iKwkYAIu?m(Qp z*#oa`=HvBlr+o-p(ya!T@y=_xJ*PfAyuKn>FZn4m^v0siELYE=`cxOIk7Xm=SVvyg z7vB-P+su73q`ollnE{R_97%9KS2`1Y5XR!~XQH(wXe<-oYl#PqWrADL>f5ugD2iwM zLwU6Tp{%;ucSID1!x7P=Cr@O$?7yi9${-?Yz2~#Rjx-8FQMvO=6gZzfX@YDnb(9)h zon({l-S3v4lA@8veBAaRV4P$j<)Fftk@R=xqwV<_mG@k!LrY!+1vW046%p4Gov&f( znBbVTQa4jP)bIU(m-oE&FimJB3`4dWW_-0yzs4nZ=esg$6g^O5qn~_sb$*3>*M#MU z*=FwB79VuYTsG&?_01@^KkZh5)8RncdgnF>6SSb*cG2sZ+hI7&?VWHf!szLC0Dck- zaim*Q?QG+q0|yVznNjvW?(}@i^)oGJSG3;<9lyDF_AP!=Iug8kk4is|Ja3AO2dCtF zEc-U^^eYLAgf95>L>YjS5M}xU3V?Q$wmvTKD_e$8ZWjI$1-f8s3IICLVuQX~tVC() z8Bg-dET{O-ofAjSpWoU2gg>12&s_3a9umfc%K+I>7;1fNl_`Mh7jVt?Q!iUZLR*z1 zTa8te2^$07MjN(qSI1@TCbto>XM7lj%iH%Im>VhyZ41Uvy=u?hx~@Q^A3X2HnQgPu zmr<>&_LKbIHNW%BPI4S^R=ass=J4GUw#P>yrD~v30tkm%AEmqmZBhqX^it}OR4F}R zB&GtO1udYDG=dx9@52<n67frKb(^b+y{cKEKpx*MqF$yB|+O{t$H aI6dhS+_YXxaA9|mAN)xGEJ{O1X!{RQJcKm> literal 0 HcmV?d00001 diff --git a/testing/btest/scripts/base/protocols/imap/starttls.test b/testing/btest/scripts/base/protocols/imap/starttls.test new file mode 100644 index 0000000000..380a594c2b --- /dev/null +++ b/testing/btest/scripts/base/protocols/imap/starttls.test @@ -0,0 +1,9 @@ +# @TEST-EXEC: bro -b -C -r $TRACES/tls/imap-starttls.pcap %INPUT +# @TEST-EXEC: btest-diff conn.log +# @TEST-EXEC: btest-diff ssl.log +# @TEST-EXEC: btest-diff x509.log + +@load base/protocols/ssl +@load base/protocols/conn +@load base/frameworks/dpd +@load base/protocols/imap From 1933299543bce05776f7887cf36a388056cfbe32 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Thu, 23 Jul 2015 11:15:57 -0700 Subject: [PATCH 07/84] Add support of getting server capabilities to IMAP parser. --- src/analyzer/protocol/imap/CMakeLists.txt | 1 + src/analyzer/protocol/imap/events.bif | 10 ++++ src/analyzer/protocol/imap/imap-analyzer.pac | 20 ++++++- src/analyzer/protocol/imap/imap-protocol.pac | 57 ++++++++++++++++++- src/analyzer/protocol/imap/imap.pac | 2 + .../.stdout | 1 + .../base/protocols/imap/capabilities.test | 12 ++++ 7 files changed, 99 insertions(+), 4 deletions(-) create mode 100644 src/analyzer/protocol/imap/events.bif create mode 100644 testing/btest/Baseline/scripts.base.protocols.imap.capabilities/.stdout create mode 100644 testing/btest/scripts/base/protocols/imap/capabilities.test diff --git a/src/analyzer/protocol/imap/CMakeLists.txt b/src/analyzer/protocol/imap/CMakeLists.txt index 755221b25a..921dde2444 100644 --- a/src/analyzer/protocol/imap/CMakeLists.txt +++ b/src/analyzer/protocol/imap/CMakeLists.txt @@ -6,6 +6,7 @@ include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DI bro_plugin_begin(Bro IMAP) bro_plugin_cc(Plugin.cc) bro_plugin_cc(IMAP.cc) +bro_plugin_bif(events.bif) bro_plugin_pac(imap.pac imap-analyzer.pac imap-protocol.pac) bro_plugin_end() diff --git a/src/analyzer/protocol/imap/events.bif b/src/analyzer/protocol/imap/events.bif new file mode 100644 index 0000000000..903d1f0dff --- /dev/null +++ b/src/analyzer/protocol/imap/events.bif @@ -0,0 +1,10 @@ +## Generated for an SSL/TLS client's initial *hello* message. SSL/TLS sessions +## start with an unencrypted handshake, and Bro extracts as much information out +## of that as it can. This event provides access to the initial information +## sent by the client. +## +## c: The connection. +## +## capabilities: The list of IMAP capabilities as sent by the server. +event imap_capabilities%(c: connection, capabilities: string_vec%); + diff --git a/src/analyzer/protocol/imap/imap-analyzer.pac b/src/analyzer/protocol/imap/imap-analyzer.pac index 918d339cfe..67da283609 100644 --- a/src/analyzer/protocol/imap/imap-analyzer.pac +++ b/src/analyzer/protocol/imap/imap-analyzer.pac @@ -49,9 +49,25 @@ refine connection IMAP_Conn += { return true; %} + function proc_server_capability(capabilities: Capability[]): bool + %{ + VectorVal* capv = new VectorVal(internal_type("string_vec")->AsVectorType()); + for ( unsigned int i = 0; i< capabilities->size(); i++ ) + { + const bytestring& capability = (*capabilities)[i]->cap(); + capv ->Assign(i, new StringVal(capability.length(), (const char*)capability.data())); + } + + BifEvent::generate_imap_capabilities(bro_analyzer(), bro_analyzer()->Conn(), capv); + return true; + %} + }; -refine typeattr IMAP_TOKEN += &let { - proc: bool = $context.connection.proc_imap_token(is_orig, tag, command); +refine typeattr ImapToken += &let { + proc: bool = $context.connection.proc_imap_token(is_orig, tag, command); }; +refine typeattr ServerCapability += &let { + proc: bool = $context.connection.proc_server_capability(capabilities); +}; diff --git a/src/analyzer/protocol/imap/imap-protocol.pac b/src/analyzer/protocol/imap/imap-protocol.pac index 15bb753475..5fa1e34555 100644 --- a/src/analyzer/protocol/imap/imap-protocol.pac +++ b/src/analyzer/protocol/imap/imap-protocol.pac @@ -1,17 +1,70 @@ +# commands that we support parsing. The numbers do not really mean anything +# in this case +enum ImapCommand { + CMD_CAPABILITY, + CMD_UNKNOWN +} + type TAG = RE/[[:alnum:][:punct:]]+/; type CONTENT = RE/[^\r\n]*/; type SPACING = RE/[ ]+/; type OPTIONALSPACING = RE/[ ]*/; type NEWLINE = RE/[\r\n]+/; +type OPTIONALNEWLINE = RE/[\r\n]*/; -type IMAP_PDU(is_orig: bool) = IMAP_TOKEN(is_orig)[] &until($input.length() == 0); +type IMAP_PDU(is_orig: bool) = ImapToken(is_orig)[] &until($input.length() == 0); -type IMAP_TOKEN(is_orig: bool) = record { +type ImapToken(is_orig: bool) = record { tag : TAG; : SPACING; command: TAG; : OPTIONALSPACING; + client_or_server: case is_orig of { + true -> client: UnknownCommand(this) ; + false -> server: ServerContentText(this); + } &requires(pcommand) ; +} &let { + pcommand: int = $context.connection.determine_command(is_orig, tag, command); +}; + +type ServerContentText(rec: ImapToken) = case rec.pcommand of { + CMD_CAPABILITY -> capability: ServerCapability(rec); + default -> unknown: UnknownCommand(rec); +}; + +type Capability = record { + cap: TAG; + : OPTIONALSPACING; + nl: OPTIONALNEWLINE; +}; + +type ServerCapability(rec: ImapToken) = record { + capabilities: Capability[] &until($context.connection.strlen($element.nl) > 0); +}; + +type UnknownCommand(rec: ImapToken) = record { tagcontent: CONTENT; : NEWLINE; }; +refine connection IMAP_Conn += { + + function determine_command(is_orig: bool, tag: bytestring, command: bytestring): int + %{ + string cmdstr = std_str(command); + std::transform(cmdstr.begin(), cmdstr.end(), cmdstr.begin(), ::tolower); + string tagstr = std_str(tag); + + if ( !is_orig && cmdstr == "capability" && tag == "*" ) { + return CMD_CAPABILITY; + } + + return CMD_UNKNOWN; + %} + + function strlen(str: bytestring): int + %{ + return str.length(); + %} + +}; diff --git a/src/analyzer/protocol/imap/imap.pac b/src/analyzer/protocol/imap/imap.pac index 33382bc26d..f5c7559294 100644 --- a/src/analyzer/protocol/imap/imap.pac +++ b/src/analyzer/protocol/imap/imap.pac @@ -7,6 +7,8 @@ %include bro.pac %extern{ +#include "events.bif.h" + namespace analyzer { namespace imap { class IMAP_Analyzer; } } namespace binpac { namespace IMAP { class IMAP_Conn; } } typedef analyzer::imap::IMAP_Analyzer* IMAPAnalyzer; diff --git a/testing/btest/Baseline/scripts.base.protocols.imap.capabilities/.stdout b/testing/btest/Baseline/scripts.base.protocols.imap.capabilities/.stdout new file mode 100644 index 0000000000..bf69e13682 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.imap.capabilities/.stdout @@ -0,0 +1 @@ +[IMAP4rev1, CHILDREN, ENABLE, ID, IDLE, LIST-EXTENDED, LIST-STATUS, LITERAL+, MOVE, NAMESPACE, SASL-IR, SORT, SPECIAL-USE, THREAD=ORDEREDSUBJECT, UIDPLUS, UNSELECT, WITHIN, STARTTLS, AUTH=LOGIN, AUTH=PLAIN] diff --git a/testing/btest/scripts/base/protocols/imap/capabilities.test b/testing/btest/scripts/base/protocols/imap/capabilities.test new file mode 100644 index 0000000000..06bdb56b7d --- /dev/null +++ b/testing/btest/scripts/base/protocols/imap/capabilities.test @@ -0,0 +1,12 @@ +# @TEST-EXEC: bro -b -C -r $TRACES/tls/imap-starttls.pcap %INPUT +# @TEST-EXEC: btest-diff .stdout + +@load base/protocols/ssl +@load base/protocols/conn +@load base/frameworks/dpd +@load base/protocols/imap + +event imap_capabilities(c: connection, capabilities: string_vec) + { + print capabilities; + } From 7f2087af3433f60635bfb8288f51c312a068107b Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Thu, 23 Jul 2015 12:37:40 -0700 Subject: [PATCH 08/84] also generate an event when starttls is encounterd for imap. --- src/analyzer/protocol/imap/events.bif | 4 ++++ src/analyzer/protocol/imap/imap-analyzer.pac | 3 +++ .../Baseline/scripts.base.protocols.imap.starttls/.stdout | 1 + testing/btest/scripts/base/protocols/imap/starttls.test | 6 ++++++ 4 files changed, 14 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.protocols.imap.starttls/.stdout diff --git a/src/analyzer/protocol/imap/events.bif b/src/analyzer/protocol/imap/events.bif index 903d1f0dff..ba83791b13 100644 --- a/src/analyzer/protocol/imap/events.bif +++ b/src/analyzer/protocol/imap/events.bif @@ -8,3 +8,7 @@ ## capabilities: The list of IMAP capabilities as sent by the server. event imap_capabilities%(c: connection, capabilities: string_vec%); +## Generated when a IMAP connection goes encrypted +## +## c: The connection. +event imap_starttls%(c: connection%); diff --git a/src/analyzer/protocol/imap/imap-analyzer.pac b/src/analyzer/protocol/imap/imap-analyzer.pac index 67da283609..de6352d7b9 100644 --- a/src/analyzer/protocol/imap/imap-analyzer.pac +++ b/src/analyzer/protocol/imap/imap-analyzer.pac @@ -41,7 +41,10 @@ refine connection IMAP_Conn += { if ( !is_orig && !client_starttls_id.empty() && tags == client_starttls_id ) { if ( commands == "ok" ) + { bro_analyzer()->StartTLS(); + BifEvent::generate_imap_starttls(bro_analyzer(), bro_analyzer()->Conn()); + } else reporter->Weird(bro_analyzer()->Conn(), "IMAP: server refused StartTLS"); } diff --git a/testing/btest/Baseline/scripts.base.protocols.imap.starttls/.stdout b/testing/btest/Baseline/scripts.base.protocols.imap.starttls/.stdout new file mode 100644 index 0000000000..5fbafd3ab3 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.imap.starttls/.stdout @@ -0,0 +1 @@ +Tls started for connection diff --git a/testing/btest/scripts/base/protocols/imap/starttls.test b/testing/btest/scripts/base/protocols/imap/starttls.test index 380a594c2b..444c27688a 100644 --- a/testing/btest/scripts/base/protocols/imap/starttls.test +++ b/testing/btest/scripts/base/protocols/imap/starttls.test @@ -2,8 +2,14 @@ # @TEST-EXEC: btest-diff conn.log # @TEST-EXEC: btest-diff ssl.log # @TEST-EXEC: btest-diff x509.log +# @TEST-EXEC: btest-diff .stdout @load base/protocols/ssl @load base/protocols/conn @load base/frameworks/dpd @load base/protocols/imap + +event imap_starttls(c: connection) + { + print "Tls started for connection"; + } From 2b0a28686a6f5eebf7580618f196d9cfa90250a9 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 4 Jan 2016 00:55:52 -0500 Subject: [PATCH 09/84] Cleaned up stats collection. - Removed the gap_report event. It wasn't used anymore and functionally no more capable that scheduling events and using the get_gap_summary bif. - Added functionality to Dictionaries to count cumulative numbers of inserts performed. This is further used to measure the total number of connections of various types. Previously only the number of active connections was available. - The Reassembler base class now tracks active reassembly size for all subclasses (File/TCP/Frag & unknown). - Improvements to the stats.log. Mostly, more information. --- scripts/base/init-bare.bro | 81 +++++++++++--------- scripts/policy/misc/stats.bro | 37 +++++++-- src/Dict.cc | 2 + src/Dict.h | 7 ++ src/Event.cc | 4 +- src/Event.h | 4 +- src/Frag.cc | 2 +- src/Reassem.cc | 34 ++++++-- src/Reassem.h | 26 ++++++- src/Sessions.cc | 3 + src/Sessions.h | 23 +++--- src/analyzer/protocol/tcp/TCP_Reassembler.cc | 43 +---------- src/bro.bif | 38 ++++++--- src/file_analysis/FileReassembler.cc | 2 +- 14 files changed, 189 insertions(+), 117 deletions(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 94b6ed33e5..337052178d 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -462,34 +462,51 @@ type NetStats: record { ## .. note:: All process-level values refer to Bro's main process only, not to ## the child process it spawns for doing communication. type bro_resources: record { - version: string; ##< Bro version string. - debug: bool; ##< True if compiled with --enable-debug. - start_time: time; ##< Start time of process. - real_time: interval; ##< Elapsed real time since Bro started running. - user_time: interval; ##< User CPU seconds. - system_time: interval; ##< System CPU seconds. - mem: count; ##< Maximum memory consumed, in KB. - minor_faults: count; ##< Page faults not requiring actual I/O. - major_faults: count; ##< Page faults requiring actual I/O. - num_swap: count; ##< Times swapped out. - blocking_input: count; ##< Blocking input operations. - blocking_output: count; ##< Blocking output operations. - num_context: count; ##< Number of involuntary context switches. + version: string; ##< Bro version string. + debug: bool; ##< True if compiled with --enable-debug. + start_time: time; ##< Start time of process. + real_time: interval; ##< Elapsed real time since Bro started running. + user_time: interval; ##< User CPU seconds. + system_time: interval; ##< System CPU seconds. + mem: count; ##< Maximum memory consumed, in KB. + minor_faults: count; ##< Page faults not requiring actual I/O. + major_faults: count; ##< Page faults requiring actual I/O. + num_swap: count; ##< Times swapped out. + blocking_input: count; ##< Blocking input operations. + blocking_output: count; ##< Blocking output operations. + num_context: count; ##< Number of involuntary context switches. + + num_packets: count; ##< Total number of packets processed to date. + num_fragments: count; ##< Current number of fragments pending reassembly. + max_fragments: count; ##< Maximum number of concurrently buffered fragments so far. + + num_tcp_conns: count; ##< Current number of TCP connections in memory. + max_tcp_conns: count; ##< Maximum number of concurrent TCP connections so far. + cumulative_tcp_conns: count; ##< - num_TCP_conns: count; ##< Current number of TCP connections in memory. - num_UDP_conns: count; ##< Current number of UDP flows in memory. - num_ICMP_conns: count; ##< Current number of ICMP flows in memory. - num_fragments: count; ##< Current number of fragments pending reassembly. - num_packets: count; ##< Total number of packets processed to date. - num_timers: count; ##< Current number of pending timers. - num_events_queued: count; ##< Total number of events queued so far. - num_events_dispatched: count; ##< Total number of events dispatched so far. + num_udp_conns: count; ##< Current number of UDP flows in memory. + max_udp_conns: count; ##< Maximum number of concurrent UDP connections so far. + cumulative_udp_conns: count; ##< - max_TCP_conns: count; ##< Maximum number of concurrent TCP connections so far. - max_UDP_conns: count; ##< Maximum number of concurrent UDP connections so far. - max_ICMP_conns: count; ##< Maximum number of concurrent ICMP connections so far. - max_fragments: count; ##< Maximum number of concurrently buffered fragments so far. - max_timers: count; ##< Maximum number of concurrent timers pending so far. + num_icmp_conns: count; ##< Current number of ICMP flows in memory. + max_icmp_conns: count; ##< Maximum number of concurrent ICMP connections so far. + cumulative_icmp_conns: count; ##< + + num_timers: count; ##< Current number of pending timers. + max_timers: count; ##< Maximum number of concurrent timers pending so far. + + num_events_queued: count; ##< Total number of events queued so far. + num_events_dispatched: count; ##< Total number of events dispatched so far. + + total_conns: count; ##< + current_conns: count; ##< + current_conns_extern: count; ##< + sess_current_conns: count; ##< + + reassem_file_size: count; ##< Size of File reassembly tracking. + reassem_frag_size: count; ##< Size of Fragment reassembly tracking. + reassem_tcp_size: count; ##< Size of TCP reassembly tracking. + reassem_unknown_size: count; ##< Size of reassembly tracking for unknown purposes. }; ## Summary statistics of all regular expression matchers. @@ -507,7 +524,7 @@ type matcher_stats: record { ## Statistics about number of gaps in TCP connections. ## -## .. bro:see:: gap_report get_gap_summary +## .. bro:see:: get_gap_summary type gap_info: record { ack_events: count; ##< How many ack events *could* have had gaps. ack_bytes: count; ##< How many bytes those covered. @@ -3416,23 +3433,17 @@ global pkt_profile_file: file &redef; ## .. bro:see:: load_sample global load_sample_freq = 20 &redef; -## Rate at which to generate :bro:see:`gap_report` events assessing to what -## degree the measurement process appears to exhibit loss. -## -## .. bro:see:: gap_report -const gap_report_freq = 1.0 sec &redef; - ## Whether to attempt to automatically detect SYN/FIN/RST-filtered trace ## and not report missing segments for such connections. ## If this is enabled, then missing data at the end of connections may not ## be reported via :bro:see:`content_gap`. const detect_filtered_trace = F &redef; -## Whether we want :bro:see:`content_gap` and :bro:see:`gap_report` for partial +## Whether we want :bro:see:`content_gap` and :bro:see:`get_gap_summary` for partial ## connections. A connection is partial if it is missing a full handshake. Note ## that gap reports for partial connections might not be reliable. ## -## .. bro:see:: content_gap gap_report partial_connection +## .. bro:see:: content_gap get_gap_summary partial_connection const report_gaps_for_partial = F &redef; ## Flag to prevent Bro from exiting automatically when input is exhausted. diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index 215a3bb9de..484267898c 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -10,7 +10,7 @@ export { redef enum Log::ID += { LOG }; ## How often stats are reported. - const stats_report_interval = 1min &redef; + const stats_report_interval = 5min &redef; type Info: record { ## Timestamp for the measurement. @@ -27,6 +27,22 @@ export { ## interval. events_queued: count &log; + ## TCP connections seen since last stats interval. + tcp_conns: count &log; + ## UDP connections seen since last stats interval. + udp_conns: count &log; + ## ICMP connections seen since last stats interval. + icmp_conns: count &log; + + ## Current size of TCP data in reassembly. + reassem_tcp_size: count &log; + ## Current size of File data in reassembly. + reassem_file_size: count &log; + ## Current size of packet fragment data in reassembly. + reassem_frag_size: count &log; + ## Current size of unkown data in reassembly (this is only PIA buffer right now). + reassem_unknown_size: count &log; + ## Lag between the wall clock and packet timestamps if reading ## live traffic. lag: interval &log &optional; @@ -64,16 +80,27 @@ event check_stats(last_ts: time, last_ns: NetStats, last_res: bro_resources) # shutting down. return; - local info: Info = [$ts=now, $peer=peer_description, $mem=res$mem/1000000, + local info: Info = [$ts=now, + $peer=peer_description, + $mem=res$mem/1000000, $pkts_proc=res$num_packets - last_res$num_packets, $events_proc=res$num_events_dispatched - last_res$num_events_dispatched, - $events_queued=res$num_events_queued - last_res$num_events_queued]; + $events_queued=res$num_events_queued - last_res$num_events_queued, + $tcp_conns=res$cumulative_tcp_conns - last_res$cumulative_tcp_conns, + $udp_conns=res$cumulative_udp_conns - last_res$cumulative_udp_conns, + $icmp_conns=res$cumulative_icmp_conns - last_res$cumulative_icmp_conns, + $reassem_tcp_size=res$reassem_tcp_size, + $reassem_file_size=res$reassem_file_size, + $reassem_frag_size=res$reassem_frag_size, + $reassem_unknown_size=res$reassem_unknown_size + ]; + + # Someone's going to have to explain what this is and add a field to the Info record. + # info$util = 100.0*((res$user_time + res$system_time) - (last_res$user_time + last_res$system_time))/(now-last_ts); if ( reading_live_traffic() ) { info$lag = now - network_time(); - # Someone's going to have to explain what this is and add a field to the Info record. - # info$util = 100.0*((res$user_time + res$system_time) - (last_res$user_time + last_res$system_time))/(now-last_ts); info$pkts_recv = ns$pkts_recvd - last_ns$pkts_recvd; info$pkts_dropped = ns$pkts_dropped - last_ns$pkts_dropped; info$pkts_link = ns$pkts_link - last_ns$pkts_link; diff --git a/src/Dict.cc b/src/Dict.cc index 1d32eccde3..9e68d64089 100644 --- a/src/Dict.cc +++ b/src/Dict.cc @@ -66,6 +66,7 @@ Dictionary::Dictionary(dict_order ordering, int initial_size) delete_func = 0; tbl_next_ind = 0; + cumulative_entries = 0; num_buckets2 = num_entries2 = max_num_entries2 = thresh_entries2 = 0; den_thresh2 = 0; } @@ -444,6 +445,7 @@ void* Dictionary::Insert(DictEntry* new_entry, int copy_key) // on lists than prepending. chain->append(new_entry); + ++cumulative_entries; if ( *max_num_entries_ptr < ++*num_entries_ptr ) *max_num_entries_ptr = *num_entries_ptr; diff --git a/src/Dict.h b/src/Dict.h index 3a2239ef54..2def5ea28f 100644 --- a/src/Dict.h +++ b/src/Dict.h @@ -71,6 +71,12 @@ public: max_num_entries + max_num_entries2 : max_num_entries; } + // Total number of entries ever. + uint64 NumCumulativeInserts() const + { + return cumulative_entries; + } + // True if the dictionary is ordered, false otherwise. int IsOrdered() const { return order != 0; } @@ -166,6 +172,7 @@ private: int num_buckets; int num_entries; int max_num_entries; + uint64 cumulative_entries; double den_thresh; int thresh_entries; diff --git a/src/Event.cc b/src/Event.cc index 89e745361f..5d54752a5a 100644 --- a/src/Event.cc +++ b/src/Event.cc @@ -10,8 +10,8 @@ EventMgr mgr; -int num_events_queued = 0; -int num_events_dispatched = 0; +uint64 num_events_queued = 0; +uint64 num_events_dispatched = 0; Event::Event(EventHandlerPtr arg_handler, val_list* arg_args, SourceID arg_src, analyzer::ID arg_aid, TimerMgr* arg_mgr, diff --git a/src/Event.h b/src/Event.h index 6f9c9d10c3..0d004d526c 100644 --- a/src/Event.h +++ b/src/Event.h @@ -72,8 +72,8 @@ protected: Event* next_event; }; -extern int num_events_queued; -extern int num_events_dispatched; +extern uint64 num_events_queued; +extern uint64 num_events_dispatched; class EventMgr : public BroObj { public: diff --git a/src/Frag.cc b/src/Frag.cc index 6a8b901a73..842059e218 100644 --- a/src/Frag.cc +++ b/src/Frag.cc @@ -28,7 +28,7 @@ void FragTimer::Dispatch(double t, int /* is_expire */) FragReassembler::FragReassembler(NetSessions* arg_s, const IP_Hdr* ip, const u_char* pkt, HashKey* k, double t) - : Reassembler(0) + : Reassembler(0, REASSEM_FRAG) { s = arg_s; key = k; diff --git a/src/Reassem.cc b/src/Reassem.cc index 54f27bd895..35f491f8ed 100644 --- a/src/Reassem.cc +++ b/src/Reassem.cc @@ -1,6 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. #include +#include #include "bro-config.h" @@ -10,7 +11,8 @@ static const bool DEBUG_reassem = false; DataBlock::DataBlock(const u_char* data, uint64 size, uint64 arg_seq, - DataBlock* arg_prev, DataBlock* arg_next) + DataBlock* arg_prev, DataBlock* arg_next, + ReassemblerType reassem_type) { seq = arg_seq; upper = seq + size; @@ -26,17 +28,24 @@ DataBlock::DataBlock(const u_char* data, uint64 size, uint64 arg_seq, if ( next ) next->prev = this; + if ( Reassembler::sizes.size() == 0 ) + Reassembler::sizes.resize(REASSEM_TERM, 0); + + rtype = reassem_type; + Reassembler::sizes[rtype] += pad_size(size) + padded_sizeof(DataBlock); Reassembler::total_size += pad_size(size) + padded_sizeof(DataBlock); } uint64 Reassembler::total_size = 0; +std::vector Reassembler::sizes; -Reassembler::Reassembler(uint64 init_seq) +Reassembler::Reassembler(uint64 init_seq, ReassemblerType reassem_type) { blocks = last_block = 0; old_blocks = last_old_block = 0; total_old_blocks = max_old_blocks = 0; trim_seq = last_reassem_seq = init_seq; + rtype = reassem_type; } Reassembler::~Reassembler() @@ -110,7 +119,7 @@ void Reassembler::NewBlock(double t, uint64 seq, uint64 len, const u_char* data) if ( ! blocks ) blocks = last_block = start_block = - new DataBlock(data, len, seq, 0, 0); + new DataBlock(data, len, seq, 0, 0, rtype); else start_block = AddAndCheck(blocks, seq, upper_seq, data); @@ -275,7 +284,7 @@ DataBlock* Reassembler::AddAndCheck(DataBlock* b, uint64 seq, uint64 upper, if ( last_block && seq == last_block->upper ) { last_block = new DataBlock(data, upper - seq, seq, - last_block, 0); + last_block, 0, rtype); return last_block; } @@ -288,7 +297,7 @@ DataBlock* Reassembler::AddAndCheck(DataBlock* b, uint64 seq, uint64 upper, { // b is the last block, and it comes completely before // the new block. - last_block = new DataBlock(data, upper - seq, seq, b, 0); + last_block = new DataBlock(data, upper - seq, seq, b, 0, rtype); return last_block; } @@ -297,7 +306,7 @@ DataBlock* Reassembler::AddAndCheck(DataBlock* b, uint64 seq, uint64 upper, if ( upper <= b->seq ) { // The new block comes completely before b. - new_b = new DataBlock(data, upper - seq, seq, b->prev, b); + new_b = new DataBlock(data, upper - seq, seq, b->prev, b, rtype); if ( b == blocks ) blocks = new_b; return new_b; @@ -308,7 +317,7 @@ DataBlock* Reassembler::AddAndCheck(DataBlock* b, uint64 seq, uint64 upper, { // The new block has a prefix that comes before b. uint64 prefix_len = b->seq - seq; - new_b = new DataBlock(data, prefix_len, seq, b->prev, b); + new_b = new DataBlock(data, prefix_len, seq, b->prev, b, rtype); if ( b == blocks ) blocks = new_b; @@ -342,6 +351,17 @@ DataBlock* Reassembler::AddAndCheck(DataBlock* b, uint64 seq, uint64 upper, return new_b; } +uint64 Reassembler::MemoryAllocation(ReassemblerType rtype) + { + if (Reassembler::sizes.size() == 0 ) + Reassembler::sizes.resize(REASSEM_TERM, 0); + + if ( rtype < REASSEM_TERM ) + return Reassembler::sizes[rtype]; + else + return 0; + } + bool Reassembler::Serialize(SerialInfo* info) const { return SerialObj::Serialize(info); diff --git a/src/Reassem.h b/src/Reassem.h index e55c809990..d371b998bd 100644 --- a/src/Reassem.h +++ b/src/Reassem.h @@ -6,10 +6,23 @@ #include "Obj.h" #include "IPAddr.h" +// Whenever subclassing the Reassembler class +// you should add to this for known subclasses. +enum ReassemblerType { + REASSEM_UNKNOWN, + REASSEM_TCP, + REASSEM_FRAG, + REASSEM_FILE, + + // Terminal value. Add new above. + REASSEM_TERM, +}; + class DataBlock { public: DataBlock(const u_char* data, uint64 size, uint64 seq, - DataBlock* prev, DataBlock* next); + DataBlock* prev, DataBlock* next, + ReassemblerType reassem_type = REASSEM_UNKNOWN); ~DataBlock(); @@ -19,13 +32,12 @@ public: DataBlock* prev; // previous block with lower seq # uint64 seq, upper; u_char* block; + ReassemblerType rtype; }; - - class Reassembler : public BroObj { public: - Reassembler(uint64 init_seq); + Reassembler(uint64 init_seq, ReassemblerType reassem_type = REASSEM_UNKNOWN); virtual ~Reassembler(); void NewBlock(double t, uint64 seq, uint64 len, const u_char* data); @@ -51,6 +63,9 @@ public: // Sum over all data buffered in some reassembler. static uint64 TotalMemoryAllocation() { return total_size; } + // Data buffered by type of reassembler. + static uint64 MemoryAllocation(ReassemblerType rtype); + void SetMaxOldBlocks(uint32 count) { max_old_blocks = count; } protected: @@ -82,12 +97,15 @@ protected: uint32 max_old_blocks; uint32 total_old_blocks; + ReassemblerType rtype; static uint64 total_size; + static std::vector sizes; }; inline DataBlock::~DataBlock() { Reassembler::total_size -= pad_size(upper - seq) + padded_sizeof(DataBlock); + Reassembler::sizes[rtype] -= pad_size(upper - seq) + padded_sizeof(DataBlock); delete [] block; } diff --git a/src/Sessions.cc b/src/Sessions.cc index b8bfe82b34..3194985515 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -1156,8 +1156,11 @@ void NetSessions::Drain() void NetSessions::GetStats(SessionStats& s) const { s.num_TCP_conns = tcp_conns.Length(); + s.cumulative_TCP_conns = tcp_conns.NumCumulativeInserts(); s.num_UDP_conns = udp_conns.Length(); + s.cumulative_UDP_conns = udp_conns.NumCumulativeInserts(); s.num_ICMP_conns = icmp_conns.Length(); + s.cumulative_ICMP_conns = icmp_conns.NumCumulativeInserts(); s.num_fragments = fragments.Length(); s.num_packets = num_packets_processed; s.num_timers = timer_mgr->Size(); diff --git a/src/Sessions.h b/src/Sessions.h index 2aca292789..e8c53256ff 100644 --- a/src/Sessions.h +++ b/src/Sessions.h @@ -32,19 +32,24 @@ namespace analyzer { namespace arp { class ARP_Analyzer; } } struct SessionStats { int num_TCP_conns; - int num_UDP_conns; - int num_ICMP_conns; - int num_fragments; - int num_packets; - int num_timers; - int num_events_queued; - int num_events_dispatched; - int max_TCP_conns; + uint64 cumulative_TCP_conns; + + int num_UDP_conns; int max_UDP_conns; + uint64 cumulative_UDP_conns; + + int num_ICMP_conns; int max_ICMP_conns; + uint64 cumulative_ICMP_conns; + + int num_fragments; int max_fragments; + uint64 num_packets; + int num_timers; int max_timers; + uint64 num_events_queued; + uint64 num_events_dispatched; }; // Drains and deletes a timer manager if it hasn't seen any advances @@ -242,7 +247,7 @@ protected: OSFingerprint* SYN_OS_Fingerprinter; int build_backdoor_analyzer; int dump_this_packet; // if true, current packet should be recorded - int num_packets_processed; + uint64 num_packets_processed; PacketProfiler* pkt_profiler; // We may use independent timer managers for different sets of related diff --git a/src/analyzer/protocol/tcp/TCP_Reassembler.cc b/src/analyzer/protocol/tcp/TCP_Reassembler.cc index 5b88d2dafb..0095947071 100644 --- a/src/analyzer/protocol/tcp/TCP_Reassembler.cc +++ b/src/analyzer/protocol/tcp/TCP_Reassembler.cc @@ -5,9 +5,6 @@ #include "analyzer/protocol/tcp/TCP.h" #include "TCP_Endpoint.h" -// Only needed for gap_report events. -#include "Event.h" - #include "events.bif.h" using namespace analyzer::tcp; @@ -18,17 +15,11 @@ const bool DEBUG_tcp_contents = false; const bool DEBUG_tcp_connection_close = false; const bool DEBUG_tcp_match_undelivered = false; -static double last_gap_report = 0.0; -static uint64 last_ack_events = 0; -static uint64 last_ack_bytes = 0; -static uint64 last_gap_events = 0; -static uint64 last_gap_bytes = 0; - TCP_Reassembler::TCP_Reassembler(analyzer::Analyzer* arg_dst_analyzer, TCP_Analyzer* arg_tcp_analyzer, TCP_Reassembler::Type arg_type, TCP_Endpoint* arg_endp) - : Reassembler(1) + : Reassembler(1, REASSEM_TCP) { dst_analyzer = arg_dst_analyzer; tcp_analyzer = arg_tcp_analyzer; @@ -45,7 +36,7 @@ TCP_Reassembler::TCP_Reassembler(analyzer::Analyzer* arg_dst_analyzer, if ( tcp_max_old_segments ) SetMaxOldBlocks(tcp_max_old_segments); - if ( tcp_contents ) + if ( ::tcp_contents ) { // Val dst_port_val(ntohs(Conn()->RespPort()), TYPE_PORT); PortVal dst_port_val(ntohs(tcp_analyzer->Conn()->RespPort()), @@ -387,7 +378,6 @@ void TCP_Reassembler::BlockInserted(DataBlock* start_block) { // New stuff. uint64 len = b->Size(); uint64 seq = last_reassem_seq; - last_reassem_seq += len; if ( record_contents_file ) @@ -548,35 +538,6 @@ void TCP_Reassembler::AckReceived(uint64 seq) tot_gap_bytes += num_missing; tcp_analyzer->Event(ack_above_hole); } - - double dt = network_time - last_gap_report; - - if ( gap_report && gap_report_freq > 0.0 && - dt >= gap_report_freq ) - { - uint64 devents = tot_ack_events - last_ack_events; - uint64 dbytes = tot_ack_bytes - last_ack_bytes; - uint64 dgaps = tot_gap_events - last_gap_events; - uint64 dgap_bytes = tot_gap_bytes - last_gap_bytes; - - RecordVal* r = new RecordVal(gap_info); - r->Assign(0, new Val(devents, TYPE_COUNT)); - r->Assign(1, new Val(dbytes, TYPE_COUNT)); - r->Assign(2, new Val(dgaps, TYPE_COUNT)); - r->Assign(3, new Val(dgap_bytes, TYPE_COUNT)); - - val_list* vl = new val_list; - vl->append(new IntervalVal(dt, Seconds)); - vl->append(r); - - mgr.QueueEvent(gap_report, vl); - - last_gap_report = network_time; - last_ack_events = tot_ack_events; - last_ack_bytes = tot_ack_bytes; - last_gap_events = tot_gap_events; - last_gap_bytes = tot_gap_bytes; - } } // Check EOF here because t_reassem->LastReassemSeq() may have diff --git a/src/bro.bif b/src/bro.bif index b0465b9609..89e132ca24 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1763,20 +1763,38 @@ function resource_usage%(%): bro_resources #define ADD_STAT(x) \ res->Assign(n++, new Val(unsigned(sessions ? x : 0), TYPE_COUNT)); - ADD_STAT(s.num_TCP_conns); - ADD_STAT(s.num_UDP_conns); - ADD_STAT(s.num_ICMP_conns); - ADD_STAT(s.num_fragments); ADD_STAT(s.num_packets); - ADD_STAT(s.num_timers); - ADD_STAT(s.num_events_queued); - ADD_STAT(s.num_events_dispatched); - ADD_STAT(s.max_TCP_conns); - ADD_STAT(s.max_UDP_conns); - ADD_STAT(s.max_ICMP_conns); + ADD_STAT(s.num_fragments); ADD_STAT(s.max_fragments); + + ADD_STAT(s.num_TCP_conns); + ADD_STAT(s.max_TCP_conns); + ADD_STAT(s.cumulative_TCP_conns); + + ADD_STAT(s.num_UDP_conns); + ADD_STAT(s.max_UDP_conns); + ADD_STAT(s.cumulative_UDP_conns); + + ADD_STAT(s.num_ICMP_conns); + ADD_STAT(s.max_ICMP_conns); + ADD_STAT(s.cumulative_ICMP_conns); + + ADD_STAT(s.num_timers); ADD_STAT(s.max_timers); + ADD_STAT(s.mem); + ADD_STAT(s.num_events_dispatched); + + ADD_STAT(Connection::TotalConnections()); + ADD_STAT(Connection::CurrentConnections()); + ADD_STAT(Connection::CurrentExternalConnections()); + ADD_STAT(sessions->CurrentConnections()); + + ADD_STAT(Reassembler::MemoryAllocation(REASSEM_FILE)); + ADD_STAT(Reassembler::MemoryAllocation(REASSEM_FRAG)); + ADD_STAT(Reassembler::MemoryAllocation(REASSEM_TCP)); + ADD_STAT(Reassembler::MemoryAllocation(REASSEM_UNKNOWN)); + return res; %} diff --git a/src/file_analysis/FileReassembler.cc b/src/file_analysis/FileReassembler.cc index 8b678e5209..ba15086320 100644 --- a/src/file_analysis/FileReassembler.cc +++ b/src/file_analysis/FileReassembler.cc @@ -8,7 +8,7 @@ namespace file_analysis { class File; FileReassembler::FileReassembler(File *f, uint64 starting_offset) - : Reassembler(starting_offset), the_file(f), flushing(false) + : Reassembler(starting_offset, REASSEM_FILE), the_file(f), flushing(false) { } From 88517230b6e5b8239a476096f290040d335e6dea Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 4 Jan 2016 00:57:11 -0500 Subject: [PATCH 10/84] Fix memory usage collection on Mac OS X. - getrusage is broken on Mac OS X, but there is a Mach API available which can collect the same memory usage information. --- bro-config.h.in | 3 +++ src/util.cc | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/bro-config.h.in b/bro-config.h.in index 755a9eee98..0937950604 100644 --- a/bro-config.h.in +++ b/bro-config.h.in @@ -14,6 +14,9 @@ /* We are on a Linux system */ #cmakedefine HAVE_LINUX +/* We are on a Mac OS X (Darwin) system */ +#cmakedefine HAVE_DARWIN + /* Define if you have the `mallinfo' function. */ #cmakedefine HAVE_MALLINFO diff --git a/src/util.cc b/src/util.cc index 6a03859a3c..facbab295f 100644 --- a/src/util.cc +++ b/src/util.cc @@ -14,6 +14,11 @@ # endif #endif +#ifdef HAVE_DARWIN +#include +#include +#endif + #include #include #include @@ -1662,11 +1667,24 @@ void get_memory_usage(unsigned int* total, unsigned int* malloced) #endif +#ifdef HAVE_DARWIN + struct task_basic_info t_info; + mach_msg_type_number_t t_info_count = TASK_BASIC_INFO_COUNT; + + if ( KERN_SUCCESS != task_info(mach_task_self(), + TASK_BASIC_INFO, + (task_info_t)&t_info, + &t_info_count) ) + ret_total = 0; + else + ret_total = t_info.resident_size; +#else struct rusage r; getrusage(RUSAGE_SELF, &r); // In KB. ret_total = r.ru_maxrss * 1024; +#endif if ( total ) *total = ret_total; From 5a4859afe1f59321a354dd9b169d8931d8fb4de7 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 4 Jan 2016 00:59:26 -0500 Subject: [PATCH 11/84] Updating the cmake submodule for the stats updates. --- cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake b/cmake index 843cdf6a91..23773d7107 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 843cdf6a91f06e5407bffbc79a343bff3cf4c81f +Subproject commit 23773d7107e8d51e2b1bb0fd2e2d85fda50df743 From 13cf6e61122099c08aa6a156a629e6f8a6384514 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Tue, 5 Jan 2016 09:26:56 -0500 Subject: [PATCH 12/84] Fixing some small mistakes. --- scripts/base/init-bare.bro | 4 ++-- src/bro.bif | 25 ++++++++++--------------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 337052178d..f49bf89d18 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -475,11 +475,11 @@ type bro_resources: record { blocking_input: count; ##< Blocking input operations. blocking_output: count; ##< Blocking output operations. num_context: count; ##< Number of involuntary context switches. - + num_packets: count; ##< Total number of packets processed to date. num_fragments: count; ##< Current number of fragments pending reassembly. max_fragments: count; ##< Maximum number of concurrently buffered fragments so far. - + num_tcp_conns: count; ##< Current number of TCP connections in memory. max_tcp_conns: count; ##< Maximum number of concurrent TCP connections so far. cumulative_tcp_conns: count; ##< diff --git a/src/bro.bif b/src/bro.bif index 89e132ca24..948fc62684 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1766,34 +1766,29 @@ function resource_usage%(%): bro_resources ADD_STAT(s.num_packets); ADD_STAT(s.num_fragments); ADD_STAT(s.max_fragments); - ADD_STAT(s.num_TCP_conns); ADD_STAT(s.max_TCP_conns); ADD_STAT(s.cumulative_TCP_conns); - ADD_STAT(s.num_UDP_conns); ADD_STAT(s.max_UDP_conns); ADD_STAT(s.cumulative_UDP_conns); - ADD_STAT(s.num_ICMP_conns); ADD_STAT(s.max_ICMP_conns); ADD_STAT(s.cumulative_ICMP_conns); - ADD_STAT(s.num_timers); ADD_STAT(s.max_timers); - - ADD_STAT(s.mem); + ADD_STAT(s.num_events_queued); ADD_STAT(s.num_events_dispatched); - ADD_STAT(Connection::TotalConnections()); - ADD_STAT(Connection::CurrentConnections()); - ADD_STAT(Connection::CurrentExternalConnections()); - ADD_STAT(sessions->CurrentConnections()); - - ADD_STAT(Reassembler::MemoryAllocation(REASSEM_FILE)); - ADD_STAT(Reassembler::MemoryAllocation(REASSEM_FRAG)); - ADD_STAT(Reassembler::MemoryAllocation(REASSEM_TCP)); - ADD_STAT(Reassembler::MemoryAllocation(REASSEM_UNKNOWN)); + res->Assign(n++, new Val(unsigned(Connection::TotalConnections()), TYPE_COUNT)); + res->Assign(n++, new Val(unsigned(Connection::CurrentConnections()), TYPE_COUNT)); + res->Assign(n++, new Val(unsigned(Connection::CurrentExternalConnections()), TYPE_COUNT)); + res->Assign(n++, new Val(unsigned(sessions->CurrentConnections()), TYPE_COUNT)); + + res->Assign(n++, new Val(unsigned(Reassembler::MemoryAllocation(REASSEM_FILE)), TYPE_COUNT)); + res->Assign(n++, new Val(unsigned(Reassembler::MemoryAllocation(REASSEM_FRAG)), TYPE_COUNT)); + res->Assign(n++, new Val(unsigned(Reassembler::MemoryAllocation(REASSEM_TCP)), TYPE_COUNT)); + res->Assign(n++, new Val(unsigned(Reassembler::MemoryAllocation(REASSEM_UNKNOWN)), TYPE_COUNT)); return res; %} From 6aeeb94d760e9860b29eadcc52548721c9a3c630 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Wed, 6 Jan 2016 22:28:57 -0500 Subject: [PATCH 13/84] Slight change to Mach API for collecting memory usage. --- src/util.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/util.cc b/src/util.cc index facbab295f..9a4b4de9f6 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1664,15 +1664,14 @@ void get_memory_usage(unsigned int* total, unsigned int* malloced) if ( malloced ) *malloced = mi.uordblks; - #endif #ifdef HAVE_DARWIN - struct task_basic_info t_info; - mach_msg_type_number_t t_info_count = TASK_BASIC_INFO_COUNT; + struct mach_task_basic_info t_info; + mach_msg_type_number_t t_info_count = MACH_TASK_BASIC_INFO; if ( KERN_SUCCESS != task_info(mach_task_self(), - TASK_BASIC_INFO, + MACH_TASK_BASIC_INFO, (task_info_t)&t_info, &t_info_count) ) ret_total = 0; From 6d836b795648901558df09e3125fa40153f5c670 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 7 Jan 2016 16:20:24 -0500 Subject: [PATCH 14/84] More stats improvements Broke out the stats collection into a bunch of new Bifs in stats.bif. Scripts that use stats collection functions have also been updated. More work to do. --- .../frameworks/packet-filter/netstats.bro | 4 +- scripts/base/init-bare.bro | 123 +++++--- .../base/misc/find-checksum-offloading.bro | 2 +- scripts/policy/misc/capture-loss.bro | 2 +- scripts/policy/misc/stats.bro | 62 ++-- src/CMakeLists.txt | 1 + src/Conn.cc | 6 +- src/Conn.h | 12 +- src/DFA.cc | 17 +- src/DFA.h | 3 +- src/Func.cc | 20 +- src/NFA.cc | 5 - src/NFA.h | 1 - src/NetVar.cc | 1 - src/NetVar.h | 3 - src/Sessions.cc | 4 - src/Sessions.h | 4 - src/Stats.cc | 16 +- src/Stats.h | 8 +- src/analyzer/protocol/tcp/functions.bif | 20 -- src/bro.bif | 177 +---------- src/event.bif | 20 -- src/file_analysis/Manager.h | 9 + src/main.cc | 12 +- src/stats.bif | 293 ++++++++++++++++++ src/util.cc | 4 +- src/util.h | 3 +- 27 files changed, 479 insertions(+), 353 deletions(-) create mode 100644 src/stats.bif diff --git a/scripts/base/frameworks/packet-filter/netstats.bro b/scripts/base/frameworks/packet-filter/netstats.bro index b5ffe24f54..f1757d8d47 100644 --- a/scripts/base/frameworks/packet-filter/netstats.bro +++ b/scripts/base/frameworks/packet-filter/netstats.bro @@ -18,7 +18,7 @@ export { event net_stats_update(last_stat: NetStats) { - local ns = net_stats(); + local ns = get_net_stats(); local new_dropped = ns$pkts_dropped - last_stat$pkts_dropped; if ( new_dropped > 0 ) { @@ -38,5 +38,5 @@ event bro_init() # Since this currently only calculates packet drops, let's skip the stats # collection if reading traces. if ( ! reading_traces() ) - schedule stats_collection_interval { net_stats_update(net_stats()) }; + schedule stats_collection_interval { net_stats_update(get_net_stats()) }; } diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index f49bf89d18..fa9149c674 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -455,30 +455,15 @@ type NetStats: record { bytes_recvd: count &default=0; ##< Bytes received by Bro. }; -## Statistics about Bro's resource consumption. -## -## .. bro:see:: resource_usage -## -## .. note:: All process-level values refer to Bro's main process only, not to -## the child process it spawns for doing communication. -type bro_resources: record { - version: string; ##< Bro version string. - debug: bool; ##< True if compiled with --enable-debug. - start_time: time; ##< Start time of process. - real_time: interval; ##< Elapsed real time since Bro started running. - user_time: interval; ##< User CPU seconds. - system_time: interval; ##< System CPU seconds. - mem: count; ##< Maximum memory consumed, in KB. - minor_faults: count; ##< Page faults not requiring actual I/O. - major_faults: count; ##< Page faults requiring actual I/O. - num_swap: count; ##< Times swapped out. - blocking_input: count; ##< Blocking input operations. - blocking_output: count; ##< Blocking output operations. - num_context: count; ##< Number of involuntary context switches. +type ConnStats: record { + total_conns: count; ##< + current_conns: count; ##< + current_conns_extern: count; ##< + sess_current_conns: count; ##< - num_packets: count; ##< Total number of packets processed to date. - num_fragments: count; ##< Current number of fragments pending reassembly. - max_fragments: count; ##< Maximum number of concurrently buffered fragments so far. + num_packets: count; + num_fragments: count; + max_fragments: count; num_tcp_conns: count; ##< Current number of TCP connections in memory. max_tcp_conns: count; ##< Maximum number of concurrent TCP connections so far. @@ -492,46 +477,96 @@ type bro_resources: record { max_icmp_conns: count; ##< Maximum number of concurrent ICMP connections so far. cumulative_icmp_conns: count; ##< - num_timers: count; ##< Current number of pending timers. - max_timers: count; ##< Maximum number of concurrent timers pending so far. + killed_by_inactivity: count; +}; +## Statistics about Bro's process. +## +## .. bro:see:: get_proc_stats +## +## .. note:: All process-level values refer to Bro's main process only, not to +## the child process it spawns for doing communication. +type ProcStats: record { + debug: bool; ##< True if compiled with --enable-debug. + start_time: time; ##< Start time of process. + real_time: interval; ##< Elapsed real time since Bro started running. + user_time: interval; ##< User CPU seconds. + system_time: interval; ##< System CPU seconds. + mem: count; ##< Maximum memory consumed, in KB. + minor_faults: count; ##< Page faults not requiring actual I/O. + major_faults: count; ##< Page faults requiring actual I/O. + num_swap: count; ##< Times swapped out. + blocking_input: count; ##< Blocking input operations. + blocking_output: count; ##< Blocking output operations. + num_context: count; ##< Number of involuntary context switches. +}; + +type EventStats: record { num_events_queued: count; ##< Total number of events queued so far. num_events_dispatched: count; ##< Total number of events dispatched so far. +}; - total_conns: count; ##< - current_conns: count; ##< - current_conns_extern: count; ##< - sess_current_conns: count; ##< - - reassem_file_size: count; ##< Size of File reassembly tracking. - reassem_frag_size: count; ##< Size of Fragment reassembly tracking. - reassem_tcp_size: count; ##< Size of TCP reassembly tracking. - reassem_unknown_size: count; ##< Size of reassembly tracking for unknown purposes. +## Summary statistics of all regular expression matchers. +## +## .. bro:see:: get_reassembler_stats +type ReassemblerStats: record { + file_size: count; ##< Byte size of File reassembly tracking. + frag_size: count; ##< Byte size of Fragment reassembly tracking. + tcp_size: count; ##< Byte size of TCP reassembly tracking. + unknown_size: count; ##< Byte size of reassembly tracking for unknown purposes. }; ## Summary statistics of all regular expression matchers. ## ## .. bro:see:: get_matcher_stats -type matcher_stats: record { - matchers: count; ##< Number of distinct RE matchers. - dfa_states: count; ##< Number of DFA states across all matchers. - computed: count; ##< Number of computed DFA state transitions. - mem: count; ##< Number of bytes used by DFA states. - hits: count; ##< Number of cache hits. - misses: count; ##< Number of cache misses. - avg_nfa_states: count; ##< Average number of NFA states across all matchers. +type MatcherStats: record { + matchers: count; ##< Number of distinct RE matchers. + dfa_states: count; ##< Number of DFA states across all matchers. + computed: count; ##< Number of computed DFA state transitions. + mem: count; ##< Number of bytes used by DFA states. + hits: count; ##< Number of cache hits. + misses: count; ##< Number of cache misses. + avg_nfa_states: count; ##< Average number of NFA states across all matchers. +}; + +type TimerStats: record { + num_timers: count; ##< Current number of pending timers. + max_timers: count; ##< Maximum number of concurrent timers pending so far. +}; + +type FileAnalysisStats: record { + current: count; + max: count; + cumulative: count; +}; + +type DNSStats: record { + requests: count; + successful: count; + failed: count; + pending: count; + cached_hosts: count; + cached_addresses: count; }; ## Statistics about number of gaps in TCP connections. ## -## .. bro:see:: get_gap_summary -type gap_info: record { +## .. bro:see:: get_gap_stats +type GapStats: record { ack_events: count; ##< How many ack events *could* have had gaps. ack_bytes: count; ##< How many bytes those covered. gap_events: count; ##< How many *did* have gaps. gap_bytes: count; ##< How many bytes were missing in the gaps. }; +type PatternStats: record { + +}; + +type ThreadStats: record { + num_threads: count; +}; + ## Deprecated. ## ## .. todo:: Remove. It's still declared internally but doesn't seem used anywhere diff --git a/scripts/base/misc/find-checksum-offloading.bro b/scripts/base/misc/find-checksum-offloading.bro index fae017fff1..334cf4a2db 100644 --- a/scripts/base/misc/find-checksum-offloading.bro +++ b/scripts/base/misc/find-checksum-offloading.bro @@ -26,7 +26,7 @@ event ChecksumOffloading::check() if ( done ) return; - local pkts_recvd = net_stats()$pkts_recvd; + local pkts_recvd = get_net_stats()$pkts_recvd; local bad_ip_checksum_pct = (pkts_recvd != 0) ? (bad_ip_checksums*1.0 / pkts_recvd*1.0) : 0; local bad_tcp_checksum_pct = (pkts_recvd != 0) ? (bad_tcp_checksums*1.0 / pkts_recvd*1.0) : 0; local bad_udp_checksum_pct = (pkts_recvd != 0) ? (bad_udp_checksums*1.0 / pkts_recvd*1.0) : 0; diff --git a/scripts/policy/misc/capture-loss.bro b/scripts/policy/misc/capture-loss.bro index 28f468a1c8..648e3d6717 100644 --- a/scripts/policy/misc/capture-loss.bro +++ b/scripts/policy/misc/capture-loss.bro @@ -56,7 +56,7 @@ event CaptureLoss::take_measurement(last_ts: time, last_acks: count, last_gaps: } local now = network_time(); - local g = get_gap_summary(); + local g = get_gap_stats(); local acks = g$ack_events - last_acks; local gaps = g$gap_events - last_gaps; local pct_lost = (acks == 0) ? 0.0 : (100 * (1.0 * gaps) / (1.0 * acks)); diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index 484267898c..877d32130b 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -1,6 +1,4 @@ -##! Log memory/packet/lag statistics. Differs from -##! :doc:`/scripts/policy/misc/profiling.bro` in that this -##! is lighter-weight (much less info, and less load to generate). +##! Log memory/packet/lag statistics. @load base/frameworks/notice @@ -10,7 +8,7 @@ export { redef enum Log::ID += { LOG }; ## How often stats are reported. - const stats_report_interval = 5min &redef; + const stats_report_interval = 1sec &redef; type Info: record { ## Timestamp for the measurement. @@ -27,12 +25,19 @@ export { ## interval. events_queued: count &log; + ## TCP connections currently in memory. + active_tcp_conns: count &log; + ## UDP connections currently in memory. + active_udp_conns: count &log; + ## ICMP connections currently in memory. + active_icmp_conns: count &log; + ## TCP connections seen since last stats interval. - tcp_conns: count &log; + tcp_conns: count &log; ## UDP connections seen since last stats interval. - udp_conns: count &log; + udp_conns: count &log; ## ICMP connections seen since last stats interval. - icmp_conns: count &log; + icmp_conns: count &log; ## Current size of TCP data in reassembly. reassem_tcp_size: count &log; @@ -69,11 +74,14 @@ event bro_init() &priority=5 Log::create_stream(Stats::LOG, [$columns=Info, $ev=log_stats, $path="stats"]); } -event check_stats(last_ts: time, last_ns: NetStats, last_res: bro_resources) +event check_stats(last_ts: time, last_ns: NetStats, last_cs: ConnStats, last_ps: ProcStats, last_es: EventStats, last_rs: ReassemblerStats) { local now = current_time(); - local ns = net_stats(); - local res = resource_usage(); + local ns = get_net_stats(); + local cs = get_conn_stats(); + local ps = get_proc_stats(); + local es = get_event_stats(); + local rs = get_reassembler_stats(); if ( bro_is_terminating() ) # No more stats will be written or scheduled when Bro is @@ -82,21 +90,27 @@ event check_stats(last_ts: time, last_ns: NetStats, last_res: bro_resources) local info: Info = [$ts=now, $peer=peer_description, - $mem=res$mem/1000000, - $pkts_proc=res$num_packets - last_res$num_packets, - $events_proc=res$num_events_dispatched - last_res$num_events_dispatched, - $events_queued=res$num_events_queued - last_res$num_events_queued, - $tcp_conns=res$cumulative_tcp_conns - last_res$cumulative_tcp_conns, - $udp_conns=res$cumulative_udp_conns - last_res$cumulative_udp_conns, - $icmp_conns=res$cumulative_icmp_conns - last_res$cumulative_icmp_conns, - $reassem_tcp_size=res$reassem_tcp_size, - $reassem_file_size=res$reassem_file_size, - $reassem_frag_size=res$reassem_frag_size, - $reassem_unknown_size=res$reassem_unknown_size + $mem=ps$mem/1000000, + $pkts_proc=ns$pkts_recvd - last_ns$pkts_recvd, + + $active_tcp_conns=cs$num_tcp_conns, + $tcp_conns=cs$cumulative_tcp_conns - last_cs$cumulative_tcp_conns, + $active_udp_conns=cs$num_udp_conns, + $udp_conns=cs$cumulative_udp_conns - last_cs$cumulative_udp_conns, + $active_icmp_conns=cs$num_icmp_conns, + $icmp_conns=cs$cumulative_icmp_conns - last_cs$cumulative_icmp_conns, + + $reassem_tcp_size=rs$tcp_size, + $reassem_file_size=rs$file_size, + $reassem_frag_size=rs$frag_size, + $reassem_unknown_size=rs$unknown_size, + + $events_proc=es$num_events_dispatched - last_es$num_events_dispatched, + $events_queued=es$num_events_queued - last_es$num_events_queued ]; # Someone's going to have to explain what this is and add a field to the Info record. - # info$util = 100.0*((res$user_time + res$system_time) - (last_res$user_time + last_res$system_time))/(now-last_ts); + # info$util = 100.0*((ps$user_time + ps$system_time) - (last_ps$user_time + last_ps$system_time))/(now-last_ts); if ( reading_live_traffic() ) { @@ -108,10 +122,10 @@ event check_stats(last_ts: time, last_ns: NetStats, last_res: bro_resources) } Log::write(Stats::LOG, info); - schedule stats_report_interval { check_stats(now, ns, res) }; + schedule stats_report_interval { check_stats(now, ns, cs, ps, es, rs) }; } event bro_init() { - schedule stats_report_interval { check_stats(current_time(), net_stats(), resource_usage()) }; + schedule stats_report_interval { check_stats(current_time(), get_net_stats(), get_conn_stats(), get_proc_stats(), get_event_stats(), get_reassembler_stats()) }; } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9a807b3182..7b521125e4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -118,6 +118,7 @@ include(BifCl) set(BIF_SRCS bro.bif + stats.bif event.bif const.bif types.bif diff --git a/src/Conn.cc b/src/Conn.cc index 3f6757d89c..1082230869 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -108,9 +108,9 @@ bool ConnectionTimer::DoUnserialize(UnserialInfo* info) return true; } -unsigned int Connection::total_connections = 0; -unsigned int Connection::current_connections = 0; -unsigned int Connection::external_connections = 0; +uint64 Connection::total_connections = 0; +uint64 Connection::current_connections = 0; +uint64 Connection::external_connections = 0; IMPLEMENT_SERIAL(Connection, SER_CONNECTION); diff --git a/src/Conn.h b/src/Conn.h index 7a4331f91d..ffbc115e6e 100644 --- a/src/Conn.h +++ b/src/Conn.h @@ -220,11 +220,11 @@ public: unsigned int MemoryAllocation() const; unsigned int MemoryAllocationConnVal() const; - static unsigned int TotalConnections() + static uint64 TotalConnections() { return total_connections; } - static unsigned int CurrentConnections() + static uint64 CurrentConnections() { return current_connections; } - static unsigned int CurrentExternalConnections() + static uint64 CurrentExternalConnections() { return external_connections; } // Returns true if the history was already seen, false otherwise. @@ -315,9 +315,9 @@ protected: unsigned int saw_first_orig_packet:1, saw_first_resp_packet:1; // Count number of connections. - static unsigned int total_connections; - static unsigned int current_connections; - static unsigned int external_connections; + static uint64 total_connections; + static uint64 current_connections; + static uint64 external_connections; string history; uint32 hist_seen; diff --git a/src/DFA.cc b/src/DFA.cc index e7b2279ed5..9b8b3e5d31 100644 --- a/src/DFA.cc +++ b/src/DFA.cc @@ -9,6 +9,8 @@ unsigned int DFA_State::transition_counter = 0; +uint64 total_dfa_states = 0; + DFA_State::DFA_State(int arg_state_num, const EquivClass* ec, NFA_state_list* arg_nfa_states, AcceptingSet* arg_accept) @@ -20,6 +22,8 @@ DFA_State::DFA_State(int arg_state_num, const EquivClass* ec, mark = 0; centry = 0; + ++total_dfa_states; + SymPartition(ec); xtions = new DFA_State*[num_sym]; @@ -433,19 +437,6 @@ void DFA_Machine::Dump(FILE* f) start_state->ClearMarks(); } -void DFA_Machine::DumpStats(FILE* f) - { - DFA_State_Cache::Stats stats; - dfa_state_cache->GetStats(&stats); - - fprintf(f, "Computed dfa_states = %d; Classes = %d; Computed trans. = %d; Uncomputed trans. = %d\n", - stats.dfa_states, EC()->NumClasses(), - stats.computed, stats.uncomputed); - - fprintf(f, "DFA cache hits = %d; misses = %d\n", - stats.hits, stats.misses); - } - unsigned int DFA_Machine::MemoryAllocation() const { DFA_State_Cache::Stats s; diff --git a/src/DFA.h b/src/DFA.h index 00cfdc3d39..c329b929d4 100644 --- a/src/DFA.h +++ b/src/DFA.h @@ -19,6 +19,8 @@ class DFA_Machine; class DFA_State; struct CacheEntry; +extern uint64 total_dfa_states; + class DFA_State : public BroObj { public: DFA_State(int state_num, const EquivClass* ec, @@ -132,7 +134,6 @@ public: void Describe(ODesc* d) const; void Dump(FILE* f); - void DumpStats(FILE* f); unsigned int MemoryAllocation() const; diff --git a/src/Func.cc b/src/Func.cc index e1eadb8c9f..ac3cda6dd6 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -628,10 +628,12 @@ void builtin_error(const char* msg, BroObj* arg) } #include "bro.bif.func_h" +#include "stats.bif.func_h" #include "reporter.bif.func_h" #include "strings.bif.func_h" #include "bro.bif.func_def" +#include "stats.bif.func_def" #include "reporter.bif.func_def" #include "strings.bif.func_def" @@ -640,13 +642,23 @@ void builtin_error(const char* msg, BroObj* arg) void init_builtin_funcs() { - bro_resources = internal_type("bro_resources")->AsRecordType(); - net_stats = internal_type("NetStats")->AsRecordType(); - matcher_stats = internal_type("matcher_stats")->AsRecordType(); + ProcStats = internal_type("ProcStats")->AsRecordType(); + NetStats = internal_type("NetStats")->AsRecordType(); + MatcherStats = internal_type("MatcherStats")->AsRecordType(); + ConnStats = internal_type("ConnStats")->AsRecordType(); + ReassemblerStats = internal_type("ReassemblerStats")->AsRecordType(); + DNSStats = internal_type("DNSStats")->AsRecordType(); + GapStats = internal_type("GapStats")->AsRecordType(); + EventStats = internal_type("EventStats")->AsRecordType(); + TimerStats = internal_type("TimerStats")->AsRecordType(); + FileAnalysisStats = internal_type("FileAnalysisStats")->AsRecordType(); + ThreadStats = internal_type("ThreadStats")->AsRecordType(); + PatternStats = internal_type("PatternStats")->AsRecordType(); + var_sizes = internal_type("var_sizes")->AsTableType(); - gap_info = internal_type("gap_info")->AsRecordType(); #include "bro.bif.func_init" +#include "stats.bif.func_init" #include "reporter.bif.func_init" #include "strings.bif.func_init" diff --git a/src/NFA.cc b/src/NFA.cc index def04d79a1..4d18f75226 100644 --- a/src/NFA.cc +++ b/src/NFA.cc @@ -285,11 +285,6 @@ void NFA_Machine::Dump(FILE* f) first_state->ClearMarks(); } -void NFA_Machine::DumpStats(FILE* f) - { - fprintf(f, "highest NFA state ID is %d\n", nfa_state_id); - } - NFA_Machine* make_alternate(NFA_Machine* m1, NFA_Machine* m2) { if ( ! m1 ) diff --git a/src/NFA.h b/src/NFA.h index 9877b8787c..88ce3429c9 100644 --- a/src/NFA.h +++ b/src/NFA.h @@ -105,7 +105,6 @@ public: void Describe(ODesc* d) const; void Dump(FILE* f); - void DumpStats(FILE* f); unsigned int MemoryAllocation() const { return padded_sizeof(*this) + first_state->TotalMemoryAllocation(); } diff --git a/src/NetVar.cc b/src/NetVar.cc index 8a901842fd..457fcae0ce 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -197,7 +197,6 @@ Val* pkt_profile_file; int load_sample_freq; double gap_report_freq; -RecordType* gap_info; int packet_filter_default; diff --git a/src/NetVar.h b/src/NetVar.h index 97018121f9..582abffe65 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -200,9 +200,6 @@ extern Val* pkt_profile_file; extern int load_sample_freq; -extern double gap_report_freq; -extern RecordType* gap_info; - extern int packet_filter_default; extern int sig_max_group_size; diff --git a/src/Sessions.cc b/src/Sessions.cc index 3194985515..aae6712ef2 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -1163,15 +1163,11 @@ void NetSessions::GetStats(SessionStats& s) const s.cumulative_ICMP_conns = icmp_conns.NumCumulativeInserts(); s.num_fragments = fragments.Length(); s.num_packets = num_packets_processed; - s.num_timers = timer_mgr->Size(); - s.num_events_queued = num_events_queued; - s.num_events_dispatched = num_events_dispatched; s.max_TCP_conns = tcp_conns.MaxLength(); s.max_UDP_conns = udp_conns.MaxLength(); s.max_ICMP_conns = icmp_conns.MaxLength(); s.max_fragments = fragments.MaxLength(); - s.max_timers = timer_mgr->PeakSize(); } Connection* NetSessions::NewConn(HashKey* k, double t, const ConnID* id, diff --git a/src/Sessions.h b/src/Sessions.h index e8c53256ff..8da658633c 100644 --- a/src/Sessions.h +++ b/src/Sessions.h @@ -46,10 +46,6 @@ struct SessionStats { int num_fragments; int max_fragments; uint64 num_packets; - int num_timers; - int max_timers; - uint64 num_events_queued; - uint64 num_events_dispatched; }; // Drains and deletes a timer manager if it hasn't seen any advances diff --git a/src/Stats.cc b/src/Stats.cc index 00f603cba7..99e36625b8 100644 --- a/src/Stats.cc +++ b/src/Stats.cc @@ -14,7 +14,7 @@ #include "broker/Manager.h" #endif -int killed_by_inactivity = 0; +uint64 killed_by_inactivity = 0; uint64 tot_ack_events = 0; uint64 tot_ack_bytes = 0; @@ -82,7 +82,7 @@ void ProfileLogger::Log() struct timeval tv_utime = r.ru_utime; struct timeval tv_stime = r.ru_stime; - unsigned int total, malloced; + uint64 total, malloced; get_memory_usage(&total, &malloced); static unsigned int first_total = 0; @@ -110,7 +110,7 @@ void ProfileLogger::Log() file->Write(fmt("\n%.06f ------------------------\n", network_time)); } - file->Write(fmt("%.06f Memory: total=%dK total_adj=%dK malloced: %dK\n", + file->Write(fmt("%.06f Memory: total=%" PRId64 "K total_adj=%" PRId64 "K malloced: %" PRId64 "K\n", network_time, total / 1024, (total - first_total) / 1024, malloced / 1024)); @@ -120,7 +120,7 @@ void ProfileLogger::Log() int conn_mem_use = expensive ? sessions->ConnectionMemoryUsage() : 0; - file->Write(fmt("%.06f Conns: total=%d current=%d/%d ext=%d mem=%dK avg=%.1f table=%dK connvals=%dK\n", + file->Write(fmt("%.06f Conns: total=%" PRIu64 " current=%" PRIu64 "/%" PRIi32 " ext=%" PRIu64 " mem=%" PRIi32 "K avg=%.1f table=%" PRIu32 "K connvals=%" PRIu32 "K\n", network_time, Connection::TotalConnections(), Connection::CurrentConnections(), @@ -161,10 +161,10 @@ void ProfileLogger::Log() )); */ - file->Write(fmt("%.06f Connections expired due to inactivity: %d\n", + file->Write(fmt("%.06f Connections expired due to inactivity: %" PRIu64 "\n", network_time, killed_by_inactivity)); - file->Write(fmt("%.06f Total reassembler data: %" PRIu64"K\n", network_time, + file->Write(fmt("%.06f Total reassembler data: %" PRIu64 "K\n", network_time, Reassembler::TotalMemoryAllocation() / 1024)); // Signature engine. @@ -465,10 +465,10 @@ void PacketProfiler::ProfilePkt(double t, unsigned int bytes) double curr_Rtime = ptimestamp.tv_sec + ptimestamp.tv_usec / 1e6; - unsigned int curr_mem; + uint64 curr_mem; get_memory_usage(&curr_mem, 0); - file->Write(fmt("%.06f %.03f %d %d %.03f %.03f %.03f %d\n", + file->Write(fmt("%.06f %.03f %" PRIu64 " %" PRIu64 " %.03f %.03f %.03f %" PRIu64 "\n", t, time-last_timestamp, pkt_cnt, byte_cnt, curr_Rtime - last_Rtime, curr_Utime - last_Utime, diff --git a/src/Stats.h b/src/Stats.h index 1bcc2e18dc..7fbec8cab6 100644 --- a/src/Stats.h +++ b/src/Stats.h @@ -102,7 +102,7 @@ extern ProfileLogger* segment_logger; extern SampleLogger* sample_logger; // Connection statistics. -extern int killed_by_inactivity; +extern uint64 killed_by_inactivity; // Content gap statistics. extern uint64 tot_ack_events; @@ -127,9 +127,9 @@ protected: double update_freq; double last_Utime, last_Stime, last_Rtime; double last_timestamp, time; - unsigned int last_mem; - unsigned int pkt_cnt; - unsigned int byte_cnt; + uint64 last_mem; + uint64 pkt_cnt; + uint64 byte_cnt; }; #endif diff --git a/src/analyzer/protocol/tcp/functions.bif b/src/analyzer/protocol/tcp/functions.bif index 9fca05329a..75353180c6 100644 --- a/src/analyzer/protocol/tcp/functions.bif +++ b/src/analyzer/protocol/tcp/functions.bif @@ -63,26 +63,6 @@ function get_resp_seq%(cid: conn_id%): count } %} -## Returns statistics about TCP gaps. -## -## Returns: A record with TCP gap statistics. -## -## .. bro:see:: do_profiling -## net_stats -## resource_usage -## dump_rule_stats -## get_matcher_stats -function get_gap_summary%(%): gap_info - %{ - RecordVal* r = new RecordVal(gap_info); - r->Assign(0, new Val(tot_ack_events, TYPE_COUNT)); - r->Assign(1, new Val(tot_ack_bytes, TYPE_COUNT)); - r->Assign(2, new Val(tot_gap_events, TYPE_COUNT)); - r->Assign(3, new Val(tot_gap_bytes, TYPE_COUNT)); - - return r; - %} - ## Associates a file handle with a connection for writing TCP byte stream ## contents. ## diff --git a/src/bro.bif b/src/bro.bif index 948fc62684..ce16695afa 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -26,15 +26,8 @@ using namespace std; -RecordType* net_stats; -RecordType* bro_resources; -RecordType* matcher_stats; TableType* var_sizes; -// This one is extern, since it's used beyond just built-ins, -// and hence it's declared in NetVar.{h,cc}. -extern RecordType* gap_info; - static iosource::PktDumper* addl_pkt_dumper = 0; bro_int_t parse_int(const char*& fmt) @@ -1661,169 +1654,6 @@ function reading_traces%(%): bool return new Val(reading_traces, TYPE_BOOL); %} -## Returns packet capture statistics. Statistics include the number of -## packets *(i)* received by Bro, *(ii)* dropped, and *(iii)* seen on the -## link (not always available). -## -## Returns: A record of packet statistics. -## -## .. bro:see:: do_profiling -## resource_usage -## get_matcher_stats -## dump_rule_stats -## get_gap_summary -function net_stats%(%): NetStats - %{ - unsigned int recv = 0; - unsigned int drop = 0; - unsigned int link = 0; - unsigned int bytes_recv = 0; - - const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); - - for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); - i != pkt_srcs.end(); i++ ) - { - iosource::PktSrc* ps = *i; - - struct iosource::PktSrc::Stats stat; - ps->Statistics(&stat); - recv += stat.received; - drop += stat.dropped; - link += stat.link; - bytes_recv += stat.bytes_received; - } - - RecordVal* ns = new RecordVal(net_stats); - ns->Assign(0, new Val(recv, TYPE_COUNT)); - ns->Assign(1, new Val(drop, TYPE_COUNT)); - ns->Assign(2, new Val(link, TYPE_COUNT)); - ns->Assign(3, new Val(bytes_recv, TYPE_COUNT)); - - return ns; - %} - -## Returns Bro process statistics. Statistics include real/user/sys CPU time, -## memory usage, page faults, number of TCP/UDP/ICMP connections, timers, -## and events queued/dispatched. -## -## Returns: A record with resource usage statistics. -## -## .. bro:see:: do_profiling -## net_stats -## get_matcher_stats -## dump_rule_stats -## get_gap_summary -function resource_usage%(%): bro_resources - %{ - struct rusage r; - - if ( getrusage(RUSAGE_SELF, &r) < 0 ) - reporter->InternalError("getrusage() failed in bro_resource_usage()"); - - double elapsed_time = current_time() - bro_start_time; - - double user_time = - double(r.ru_utime.tv_sec) + double(r.ru_utime.tv_usec) / 1e6; - double system_time = - double(r.ru_stime.tv_sec) + double(r.ru_stime.tv_usec) / 1e6; - - RecordVal* res = new RecordVal(bro_resources); - int n = 0; - - res->Assign(n++, new StringVal(bro_version())); - -#ifdef DEBUG - res->Assign(n++, new Val(1, TYPE_COUNT)); -#else - res->Assign(n++, new Val(0, TYPE_COUNT)); -#endif - - res->Assign(n++, new Val(bro_start_time, TYPE_TIME)); - - res->Assign(n++, new IntervalVal(elapsed_time, Seconds)); - res->Assign(n++, new IntervalVal(user_time, Seconds)); - res->Assign(n++, new IntervalVal(system_time, Seconds)); - - unsigned int total_mem; - get_memory_usage(&total_mem, 0); - res->Assign(n++, new Val(unsigned(total_mem), TYPE_COUNT)); - - res->Assign(n++, new Val(unsigned(r.ru_minflt), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(r.ru_majflt), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(r.ru_nswap), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(r.ru_inblock), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(r.ru_oublock), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(r.ru_nivcsw), TYPE_COUNT)); - - SessionStats s; - if ( sessions ) - sessions->GetStats(s); - -#define ADD_STAT(x) \ - res->Assign(n++, new Val(unsigned(sessions ? x : 0), TYPE_COUNT)); - - ADD_STAT(s.num_packets); - ADD_STAT(s.num_fragments); - ADD_STAT(s.max_fragments); - ADD_STAT(s.num_TCP_conns); - ADD_STAT(s.max_TCP_conns); - ADD_STAT(s.cumulative_TCP_conns); - ADD_STAT(s.num_UDP_conns); - ADD_STAT(s.max_UDP_conns); - ADD_STAT(s.cumulative_UDP_conns); - ADD_STAT(s.num_ICMP_conns); - ADD_STAT(s.max_ICMP_conns); - ADD_STAT(s.cumulative_ICMP_conns); - ADD_STAT(s.num_timers); - ADD_STAT(s.max_timers); - ADD_STAT(s.num_events_queued); - ADD_STAT(s.num_events_dispatched); - - res->Assign(n++, new Val(unsigned(Connection::TotalConnections()), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(Connection::CurrentConnections()), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(Connection::CurrentExternalConnections()), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(sessions->CurrentConnections()), TYPE_COUNT)); - - res->Assign(n++, new Val(unsigned(Reassembler::MemoryAllocation(REASSEM_FILE)), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(Reassembler::MemoryAllocation(REASSEM_FRAG)), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(Reassembler::MemoryAllocation(REASSEM_TCP)), TYPE_COUNT)); - res->Assign(n++, new Val(unsigned(Reassembler::MemoryAllocation(REASSEM_UNKNOWN)), TYPE_COUNT)); - - return res; - %} - -## Returns statistics about the regular expression engine. Statistics include -## the number of distinct matchers, DFA states, DFA state transitions, memory -## usage of DFA states, cache hits/misses, and average number of NFA states -## across all matchers. -## -## Returns: A record with matcher statistics. -## -## .. bro:see:: do_profiling -## net_stats -## resource_usage -## dump_rule_stats -## get_gap_summary -function get_matcher_stats%(%): matcher_stats - %{ - RuleMatcher::Stats s; - memset(&s, 0, sizeof(s)); - - if ( rule_matcher ) - rule_matcher->GetStats(&s); - - RecordVal* r = new RecordVal(matcher_stats); - r->Assign(0, new Val(s.matchers, TYPE_COUNT)); - r->Assign(1, new Val(s.dfa_states, TYPE_COUNT)); - r->Assign(2, new Val(s.computed, TYPE_COUNT)); - r->Assign(3, new Val(s.mem, TYPE_COUNT)); - r->Assign(4, new Val(s.hits, TYPE_COUNT)); - r->Assign(5, new Val(s.misses, TYPE_COUNT)); - r->Assign(6, new Val(s.avg_nfa_states, TYPE_COUNT)); - - return r; - %} ## Generates a table of the size of all global variables. The table index is ## the variable name and the value is the variable size in bytes. @@ -1964,8 +1794,7 @@ function record_fields%(rec: any%): record_field_table ## .. bro:see:: net_stats ## resource_usage ## get_matcher_stats -## dump_rule_stats -## get_gap_summary +## get_gap_stats function do_profiling%(%) : any %{ if ( profiling_logger ) @@ -2030,8 +1859,8 @@ function is_local_interface%(ip: addr%) : bool ## .. bro:see:: do_profiling ## resource_usage ## get_matcher_stats -## net_stats -## get_gap_summary +## get_net_stats +## get_gap_stats ## ## .. todo:: The return value should be changed to any or check appropriately. function dump_rule_stats%(f: file%): bool diff --git a/src/event.bif b/src/event.bif index ff6ec059fb..aca1086e66 100644 --- a/src/event.bif +++ b/src/event.bif @@ -366,26 +366,6 @@ event ack_above_hole%(c: connection%); ## the two. event content_gap%(c: connection, is_orig: bool, seq: count, length: count%); -## Summarizes the amount of missing TCP payload at regular intervals. -## Internally, Bro tracks (1) the number of :bro:id:`ack_above_hole` events, -## including the number of bytes missing; and (2) the total number of TCP -## acks seen, with the total volume of bytes that have been acked. This event -## reports these statistics in :bro:id:`gap_report_freq` intervals for the -## purpose of determining packet loss. -## -## dt: The time that has passed since the last ``gap_report`` interval. -## -## info: The gap statistics. -## -## .. bro:see:: content_gap ack_above_hole -## -## .. note:: -## -## Bro comes with a script :doc:`/scripts/policy/misc/capture-loss.bro` that -## uses this event to estimate packet loss and report when a predefined -## threshold is exceeded. -event gap_report%(dt: interval, info: gap_info%); - ## Generated when a protocol analyzer confirms that a connection is indeed ## using that protocol. Bro's dynamic protocol detection heuristically activates ## analyzers as soon as it believes a connection *could* be using a particular diff --git a/src/file_analysis/Manager.h b/src/file_analysis/Manager.h index 93c8e7f613..bcc8ac5dd2 100644 --- a/src/file_analysis/Manager.h +++ b/src/file_analysis/Manager.h @@ -302,6 +302,15 @@ public: */ std::string DetectMIME(const u_char* data, uint64 len) const; + uint64 CurrentFiles() + { return id_map.Length(); } + + uint64 MaxFiles() + { return id_map.MaxLength(); } + + uint64 CumulativeFiles() + { return id_map.NumCumulativeInserts(); } + protected: friend class FileTimer; diff --git a/src/main.cc b/src/main.cc index 73181c82f2..a0615d75da 100644 --- a/src/main.cc +++ b/src/main.cc @@ -1172,8 +1172,8 @@ int main(int argc, char** argv) double time_net_start = current_time(true);; - unsigned int mem_net_start_total; - unsigned int mem_net_start_malloced; + uint64 mem_net_start_total; + uint64 mem_net_start_malloced; if ( time_bro ) { @@ -1181,7 +1181,7 @@ int main(int argc, char** argv) fprintf(stderr, "# initialization %.6f\n", time_net_start - time_start); - fprintf(stderr, "# initialization %uM/%uM\n", + fprintf(stderr, "# initialization %" PRIu64 "M/%" PRIu64 "M\n", mem_net_start_total / 1024 / 1024, mem_net_start_malloced / 1024 / 1024); } @@ -1190,8 +1190,8 @@ int main(int argc, char** argv) double time_net_done = current_time(true);; - unsigned int mem_net_done_total; - unsigned int mem_net_done_malloced; + uint64 mem_net_done_total; + uint64 mem_net_done_malloced; if ( time_bro ) { @@ -1200,7 +1200,7 @@ int main(int argc, char** argv) fprintf(stderr, "# total time %.6f, processing %.6f\n", time_net_done - time_start, time_net_done - time_net_start); - fprintf(stderr, "# total mem %uM/%uM, processing %uM/%uM\n", + fprintf(stderr, "# total mem %" PRId64 "M/%" PRId64 "M, processing %" PRId64 "M/%" PRId64 "M\n", mem_net_done_total / 1024 / 1024, mem_net_done_malloced / 1024 / 1024, (mem_net_done_total - mem_net_start_total) / 1024 / 1024, diff --git a/src/stats.bif b/src/stats.bif new file mode 100644 index 0000000000..d7e812df93 --- /dev/null +++ b/src/stats.bif @@ -0,0 +1,293 @@ + +%%{ // C segment +#include "util.h" +#include "threading/Manager.h" + +RecordType* ProcStats; +RecordType* NetStats; +RecordType* MatcherStats; +RecordType* ReassemblerStats; +RecordType* DNSStats; +RecordType* ConnStats; +RecordType* GapStats; +RecordType* EventStats; +RecordType* ThreadStats; +RecordType* PatternStats; +RecordType* TimerStats; +RecordType* FileAnalysisStats; +%%} + +## Returns packet capture statistics. Statistics include the number of +## packets *(i)* received by Bro, *(ii)* dropped, and *(iii)* seen on the +## link (not always available). +## +## Returns: A record of packet statistics. +## +## .. bro:see:: do_profiling +## get_proc_stats +## get_matcher_stats +## get_gap_stats +function get_net_stats%(%): NetStats + %{ + uint64 recv = 0; + uint64 drop = 0; + uint64 link = 0; + uint64 bytes_recv = 0; + + const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs()); + + for ( iosource::Manager::PktSrcList::const_iterator i = pkt_srcs.begin(); + i != pkt_srcs.end(); i++ ) + { + iosource::PktSrc* ps = *i; + + struct iosource::PktSrc::Stats stat; + ps->Statistics(&stat); + recv += stat.received; + drop += stat.dropped; + link += stat.link; + bytes_recv += stat.bytes_received; + } + + RecordVal* r = new RecordVal(NetStats); + int n = 0; + + r->Assign(n++, new Val(recv, TYPE_COUNT)); + r->Assign(n++, new Val(drop, TYPE_COUNT)); + r->Assign(n++, new Val(link, TYPE_COUNT)); + r->Assign(n++, new Val(bytes_recv, TYPE_COUNT)); + + return r; + %} + +function get_conn_stats%(%): ConnStats + %{ + RecordVal* r = new RecordVal(ConnStats); + int n = 0; + + r->Assign(n++, new Val(Connection::TotalConnections(), TYPE_COUNT)); + r->Assign(n++, new Val(Connection::CurrentConnections(), TYPE_COUNT)); + r->Assign(n++, new Val(Connection::CurrentExternalConnections(), TYPE_COUNT)); + r->Assign(n++, new Val(sessions->CurrentConnections(), TYPE_COUNT)); + + SessionStats s; + if ( sessions ) + sessions->GetStats(s); + +#define ADD_STAT(x) \ + r->Assign(n++, new Val(unsigned(sessions ? x : 0), TYPE_COUNT)); + + ADD_STAT(s.num_packets); + ADD_STAT(s.num_fragments); + ADD_STAT(s.max_fragments); + ADD_STAT(s.num_TCP_conns); + ADD_STAT(s.max_TCP_conns); + ADD_STAT(s.cumulative_TCP_conns); + ADD_STAT(s.num_UDP_conns); + ADD_STAT(s.max_UDP_conns); + ADD_STAT(s.cumulative_UDP_conns); + ADD_STAT(s.num_ICMP_conns); + ADD_STAT(s.max_ICMP_conns); + ADD_STAT(s.cumulative_ICMP_conns); + + r->Assign(n++, new Val(killed_by_inactivity, TYPE_COUNT)); + + return r; + %} + +## Returns Bro process statistics. Statistics include real/user/sys CPU time, +## memory usage, page faults, number of TCP/UDP/ICMP connections, timers, +## and events queued/dispatched. +## +## Returns: A record with resource usage statistics. +## +## .. bro:see:: do_profiling +## get_net_stats +## get_matcher_stats +## get_gap_stats +function get_proc_stats%(%): ProcStats + %{ + struct rusage ru; + if ( getrusage(RUSAGE_SELF, &ru) < 0 ) + reporter->InternalError("getrusage() failed in get_proc_stats()"); + + RecordVal* r = new RecordVal(ProcStats); + int n = 0; + + double elapsed_time = current_time() - bro_start_time; + double user_time = + double(ru.ru_utime.tv_sec) + double(ru.ru_utime.tv_usec) / 1e6; + double system_time = + double(ru.ru_stime.tv_sec) + double(ru.ru_stime.tv_usec) / 1e6; + +#ifdef DEBUG + r->Assign(n++, new Val(1, TYPE_COUNT)); +#else + r->Assign(n++, new Val(0, TYPE_COUNT)); +#endif + + r->Assign(n++, new Val(bro_start_time, TYPE_TIME)); + + r->Assign(n++, new IntervalVal(elapsed_time, Seconds)); + r->Assign(n++, new IntervalVal(user_time, Seconds)); + r->Assign(n++, new IntervalVal(system_time, Seconds)); + + uint64 total_mem; + get_memory_usage(&total_mem, NULL); + r->Assign(n++, new Val(unsigned(total_mem), TYPE_COUNT)); + + r->Assign(n++, new Val(unsigned(ru.ru_minflt), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(ru.ru_majflt), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(ru.ru_nswap), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(ru.ru_inblock), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(ru.ru_oublock), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(ru.ru_nivcsw), TYPE_COUNT)); + + return r; + %} + +function get_event_stats%(%): EventStats + %{ + RecordVal* r = new RecordVal(EventStats); + int n = 0; + + r->Assign(n++, new Val(num_events_queued, TYPE_COUNT)); + r->Assign(n++, new Val(num_events_dispatched, TYPE_COUNT)); + + return r; + %} + +function get_reassembler_stats%(%): ReassemblerStats + %{ + RecordVal* r = new RecordVal(ReassemblerStats); + int n = 0; + + r->Assign(n++, new Val(Reassembler::MemoryAllocation(REASSEM_FILE), TYPE_COUNT)); + r->Assign(n++, new Val(Reassembler::MemoryAllocation(REASSEM_FRAG), TYPE_COUNT)); + r->Assign(n++, new Val(Reassembler::MemoryAllocation(REASSEM_TCP), TYPE_COUNT)); + r->Assign(n++, new Val(Reassembler::MemoryAllocation(REASSEM_UNKNOWN), TYPE_COUNT)); + + return r; + %} + +function get_dns_stats%(%): DNSStats + %{ + RecordVal* r = new RecordVal(DNSStats); + int n = 0; + + DNS_Mgr::Stats dstats; + dns_mgr->GetStats(&dstats); + + r->Assign(n++, new Val(unsigned(dstats.requests), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(dstats.successful), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(dstats.failed), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(dstats.pending), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(dstats.cached_hosts), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(dstats.cached_addresses), TYPE_COUNT)); + + return r; + %} + +function get_pattern_stats%(%): PatternStats + %{ + RecordVal* r = new RecordVal(PatternStats); + int n = 0; + + //DFA_State_Cache::Stats stats; + //dfa_state_cache->GetStats(&stats); + + //fprintf(f, "Computed dfa_states = %d; Classes = %d; Computed trans. = %d; Uncomputed trans. = %d\n", + // stats.dfa_states, EC()->NumClasses(), + // stats.computed, stats.uncomputed); +// + //fprintf(f, "DFA cache hits = %d; misses = %d\n", + // stats.hits, stats.misses); + + return r; + + %} + +function get_timer_stats%(%): TimerStats + %{ + RecordVal* r = new RecordVal(TimerStats); + int n = 0; + + r->Assign(n++, new Val(unsigned(timer_mgr->Size()), TYPE_COUNT)); + r->Assign(n++, new Val(unsigned(timer_mgr->PeakSize()), TYPE_COUNT)); + + return r; + %} + +function get_file_analysis_stats%(%): FileAnalysisStats + %{ + RecordVal* r = new RecordVal(FileAnalysisStats); + int n = 0; + + r->Assign(n++, new Val(file_mgr->CurrentFiles(), TYPE_COUNT)); + r->Assign(n++, new Val(file_mgr->MaxFiles(), TYPE_COUNT)); + r->Assign(n++, new Val(file_mgr->CumulativeFiles(), TYPE_COUNT)); + + return r; + %} + +function get_thread_stats%(%): ThreadStats + %{ + RecordVal* r = new RecordVal(ThreadStats); + int n = 0; + + r->Assign(n++, new Val(thread_mgr->NumThreads(), TYPE_COUNT)); + + return r; + %} + +## Returns statistics about TCP gaps. +## +## Returns: A record with TCP gap statistics. +## +## .. bro:see:: do_profiling +## get_net_stats +## get_proc_stats +## get_matcher_stats +function get_gap_stats%(%): GapStats + %{ + RecordVal* r = new RecordVal(GapStats); + int n = 0; + + r->Assign(n++, new Val(tot_ack_events, TYPE_COUNT)); + r->Assign(n++, new Val(tot_ack_bytes, TYPE_COUNT)); + r->Assign(n++, new Val(tot_gap_events, TYPE_COUNT)); + r->Assign(n++, new Val(tot_gap_bytes, TYPE_COUNT)); + + return r; + %} + +## Returns statistics about the regular expression engine. Statistics include +## the number of distinct matchers, DFA states, DFA state transitions, memory +## usage of DFA states, cache hits/misses, and average number of NFA states +## across all matchers. +## +## Returns: A record with matcher statistics. +## +## .. bro:see:: get_net_stats +## get_proc_stats +## get_gap_summary +function get_matcher_stats%(%): MatcherStats + %{ + RecordVal* r = new RecordVal(MatcherStats); + int n = 0; + + RuleMatcher::Stats s; + memset(&s, 0, sizeof(s)); + if ( rule_matcher ) + rule_matcher->GetStats(&s); + + r->Assign(n++, new Val(s.matchers, TYPE_COUNT)); + r->Assign(n++, new Val(s.dfa_states, TYPE_COUNT)); + r->Assign(n++, new Val(s.computed, TYPE_COUNT)); + r->Assign(n++, new Val(s.mem, TYPE_COUNT)); + r->Assign(n++, new Val(s.hits, TYPE_COUNT)); + r->Assign(n++, new Val(s.misses, TYPE_COUNT)); + r->Assign(n++, new Val(s.avg_nfa_states, TYPE_COUNT)); + + return r; + %} diff --git a/src/util.cc b/src/util.cc index 9a4b4de9f6..a6ce473b6c 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1655,9 +1655,9 @@ extern "C" void out_of_memory(const char* where) abort(); } -void get_memory_usage(unsigned int* total, unsigned int* malloced) +void get_memory_usage(uint64* total, uint64* malloced) { - unsigned int ret_total; + uint64 ret_total; #ifdef HAVE_MALLINFO struct mallinfo mi = mallinfo(); diff --git a/src/util.h b/src/util.h index 901bb44d1c..191e5449e1 100644 --- a/src/util.h +++ b/src/util.h @@ -502,8 +502,7 @@ inline int safe_vsnprintf(char* str, size_t size, const char* format, va_list al // Returns total memory allocations and (if available) amount actually // handed out by malloc. -extern void get_memory_usage(unsigned int* total, - unsigned int* malloced); +extern void get_memory_usage(uint64* total, uint64* malloced); // Class to be used as a third argument for STL maps to be able to use // char*'s as keys. Otherwise the pointer values will be compared instead of From 3c71d4ffa8cc063915dd54c461395961368e3866 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Fri, 8 Jan 2016 17:03:16 -0500 Subject: [PATCH 15/84] More stats collection extensions. --- scripts/base/init-bare.bro | 5 +++-- scripts/policy/misc/stats.bro | 32 +++++++++++++++++++++++++------- src/PriorityQueue.cc | 3 ++- src/PriorityQueue.h | 3 +++ src/Timer.h | 3 +++ src/cq.c | 9 +++++++++ src/cq.h | 1 + src/stats.bif | 1 + 8 files changed, 47 insertions(+), 10 deletions(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index fa9149c674..3d870da38f 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -530,8 +530,9 @@ type MatcherStats: record { }; type TimerStats: record { - num_timers: count; ##< Current number of pending timers. - max_timers: count; ##< Maximum number of concurrent timers pending so far. + current: count; ##< Current number of pending timers. + max: count; ##< Maximum number of concurrent timers pending so far. + cumulative: count; }; type FileAnalysisStats: record { diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index 877d32130b..a49d377bae 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -39,6 +39,16 @@ export { ## ICMP connections seen since last stats interval. icmp_conns: count &log; + ## Number of timers scheduled since last stats interval. + timers: count &log; + ## Current number of scheduled timers. + active_timers: count &log; + + ## Number of files seen since last stats interval. + files: count &log; + ## Current number of files actively being seen. + active_files: count &log; + ## Current size of TCP data in reassembly. reassem_tcp_size: count &log; ## Current size of File data in reassembly. @@ -74,14 +84,16 @@ event bro_init() &priority=5 Log::create_stream(Stats::LOG, [$columns=Info, $ev=log_stats, $path="stats"]); } -event check_stats(last_ts: time, last_ns: NetStats, last_cs: ConnStats, last_ps: ProcStats, last_es: EventStats, last_rs: ReassemblerStats) +event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: ProcStats, last_es: EventStats, last_rs: ReassemblerStats, last_ts: TimerStats, last_fs: FileAnalysisStats) { - local now = current_time(); + local now = network_time(); local ns = get_net_stats(); local cs = get_conn_stats(); local ps = get_proc_stats(); local es = get_event_stats(); local rs = get_reassembler_stats(); + local ts = get_timer_stats(); + local fs = get_file_analysis_stats(); if ( bro_is_terminating() ) # No more stats will be written or scheduled when Bro is @@ -90,7 +102,7 @@ event check_stats(last_ts: time, last_ns: NetStats, last_cs: ConnStats, last_ps: local info: Info = [$ts=now, $peer=peer_description, - $mem=ps$mem/1000000, + $mem=ps$mem/1048576, $pkts_proc=ns$pkts_recvd - last_ns$pkts_recvd, $active_tcp_conns=cs$num_tcp_conns, @@ -106,11 +118,17 @@ event check_stats(last_ts: time, last_ns: NetStats, last_cs: ConnStats, last_ps: $reassem_unknown_size=rs$unknown_size, $events_proc=es$num_events_dispatched - last_es$num_events_dispatched, - $events_queued=es$num_events_queued - last_es$num_events_queued + $events_queued=es$num_events_queued - last_es$num_events_queued, + + $timers=ts$cumulative - last_ts$cumulative, + $active_timers=ts$current, + + $files=fs$cumulative - last_fs$cumulative, + $active_files=fs$current ]; # Someone's going to have to explain what this is and add a field to the Info record. - # info$util = 100.0*((ps$user_time + ps$system_time) - (last_ps$user_time + last_ps$system_time))/(now-last_ts); + # info$util = 100.0*((ps$user_time + ps$system_time) - (last_ps$user_time + last_ps$system_time))/(now-then); if ( reading_live_traffic() ) { @@ -122,10 +140,10 @@ event check_stats(last_ts: time, last_ns: NetStats, last_cs: ConnStats, last_ps: } Log::write(Stats::LOG, info); - schedule stats_report_interval { check_stats(now, ns, cs, ps, es, rs) }; + schedule stats_report_interval { check_stats(now, ns, cs, ps, es, rs, ts, fs) }; } event bro_init() { - schedule stats_report_interval { check_stats(current_time(), get_net_stats(), get_conn_stats(), get_proc_stats(), get_event_stats(), get_reassembler_stats()) }; + schedule stats_report_interval { check_stats(network_time(), get_net_stats(), get_conn_stats(), get_proc_stats(), get_event_stats(), get_reassembler_stats(), get_timer_stats(), get_file_analysis_stats()) }; } diff --git a/src/PriorityQueue.cc b/src/PriorityQueue.cc index 75b731142e..4f969c4830 100644 --- a/src/PriorityQueue.cc +++ b/src/PriorityQueue.cc @@ -13,7 +13,7 @@ PriorityQueue::PriorityQueue(int initial_size) { max_heap_size = initial_size; heap = new PQ_Element*[max_heap_size]; - peak_heap_size = heap_size = 0; + peak_heap_size = heap_size = cumulative_num = 0; } PriorityQueue::~PriorityQueue() @@ -62,6 +62,7 @@ int PriorityQueue::Add(PQ_Element* e) BubbleUp(heap_size); + ++cumulative_num; if ( ++heap_size > peak_heap_size ) peak_heap_size = heap_size; diff --git a/src/PriorityQueue.h b/src/PriorityQueue.h index 87e10aa7ac..bb1caad592 100644 --- a/src/PriorityQueue.h +++ b/src/PriorityQueue.h @@ -4,6 +4,7 @@ #define __PriorityQueue__ #include +#include "util.h" class PriorityQueue; @@ -53,6 +54,7 @@ public: int Size() const { return heap_size; } int PeakSize() const { return peak_heap_size; } + uint64 CumulativeNum() const { return cumulative_num; } protected: int Resize(int new_size); @@ -92,6 +94,7 @@ protected: int heap_size; int peak_heap_size; int max_heap_size; + uint64 cumulative_num; }; #endif diff --git a/src/Timer.h b/src/Timer.h index 615c8bf69a..12d849cac2 100644 --- a/src/Timer.h +++ b/src/Timer.h @@ -109,6 +109,7 @@ public: virtual int Size() const = 0; virtual int PeakSize() const = 0; + virtual uint64 CumulativeNum() const = 0; double LastTimestamp() const { return last_timestamp; } // Returns time of last advance in global network time. @@ -148,6 +149,7 @@ public: int Size() const { return q->Size(); } int PeakSize() const { return q->PeakSize(); } + uint64 CumulativeNum() const { return q->CumulativeNum(); } unsigned int MemoryUsage() const; protected: @@ -170,6 +172,7 @@ public: int Size() const { return cq_size(cq); } int PeakSize() const { return cq_max_size(cq); } + uint64 CumulativeNum() const { return cq_cumulative_num(cq); } unsigned int MemoryUsage() const; protected: diff --git a/src/cq.c b/src/cq.c index 8005544400..16153f0a39 100644 --- a/src/cq.c +++ b/src/cq.c @@ -42,6 +42,7 @@ struct cq_handle { int lowmark; /* low bucket threshold */ int nextbucket; /* next bucket to check */ int noresize; /* don't resize while we're resizing */ + uint64_t cumulative_num; /* cumulative entries ever enqueued */ double lastpri; /* last priority */ double ysize; /* length of a year */ double bwidth; /* width of each bucket */ @@ -175,6 +176,7 @@ cq_enqueue(register struct cq_handle *hp, register double pri, } bp->pri = pri; bp->cookie = cookie; + ++hp->cumulative_num; if (++hp->qlen > hp->max_qlen) hp->max_qlen = hp->qlen; #ifdef DEBUG @@ -414,6 +416,13 @@ cq_max_size(struct cq_handle *hp) return hp->max_qlen; } +uint64_t +cq_cumulative_num(struct cq_handle *hp) +{ + return hp->cumulative_num; +} + + /* Return without doing anything if we fail to allocate a new bucket array */ static int cq_resize(register struct cq_handle *hp, register int grow) diff --git a/src/cq.h b/src/cq.h index 540cccde74..c79eefc790 100644 --- a/src/cq.h +++ b/src/cq.h @@ -5,6 +5,7 @@ void *cq_dequeue(struct cq_handle *, double); void *cq_remove(struct cq_handle *, double, void *); int cq_size(struct cq_handle *); int cq_max_size(struct cq_handle *); +uint64_t cq_cumulative_num(struct cq_handle *); unsigned int cq_memory_allocation(void); #ifdef DEBUG void cq_debug(struct cq_handle *, int); diff --git a/src/stats.bif b/src/stats.bif index d7e812df93..3a975145b6 100644 --- a/src/stats.bif +++ b/src/stats.bif @@ -214,6 +214,7 @@ function get_timer_stats%(%): TimerStats r->Assign(n++, new Val(unsigned(timer_mgr->Size()), TYPE_COUNT)); r->Assign(n++, new Val(unsigned(timer_mgr->PeakSize()), TYPE_COUNT)); + r->Assign(n++, new Val(timer_mgr->CumulativeNum(), TYPE_COUNT)); return r; %} From cfdabb901fea7b904e5aaeedc2fc2617efb9de88 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Sat, 9 Jan 2016 01:14:13 -0500 Subject: [PATCH 16/84] Continued stats cleanup and extension. --- scripts/base/init-bare.bro | 22 ++++++------- scripts/policy/misc/stats.bro | 4 +-- src/DFA.cc | 5 +-- src/DFA.h | 7 ++-- src/Func.cc | 1 - src/RuleMatcher.cc | 9 ++---- src/RuleMatcher.h | 6 ++-- src/Stats.cc | 6 ++-- src/stats.bif | 60 +++++++++++++++++++++-------------- 9 files changed, 59 insertions(+), 61 deletions(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 3d870da38f..7b4f2c857f 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -502,8 +502,8 @@ type ProcStats: record { }; type EventStats: record { - num_events_queued: count; ##< Total number of events queued so far. - num_events_dispatched: count; ##< Total number of events dispatched so far. + queued: count; ##< Total number of events queued so far. + dispatched: count; ##< Total number of events dispatched so far. }; ## Summary statistics of all regular expression matchers. @@ -520,13 +520,13 @@ type ReassemblerStats: record { ## ## .. bro:see:: get_matcher_stats type MatcherStats: record { - matchers: count; ##< Number of distinct RE matchers. - dfa_states: count; ##< Number of DFA states across all matchers. - computed: count; ##< Number of computed DFA state transitions. - mem: count; ##< Number of bytes used by DFA states. - hits: count; ##< Number of cache hits. - misses: count; ##< Number of cache misses. - avg_nfa_states: count; ##< Average number of NFA states across all matchers. + matchers: count; ##< Number of distinct RE matchers. + nfa_states: count; ##< Number of NFA states across all matchers. + dfa_states: count; ##< Number of DFA states across all matchers. + computed: count; ##< Number of computed DFA state transitions. + mem: count; ##< Number of bytes used by DFA states. + hits: count; ##< Number of cache hits. + misses: count; ##< Number of cache misses. }; type TimerStats: record { @@ -560,10 +560,6 @@ type GapStats: record { gap_bytes: count; ##< How many bytes were missing in the gaps. }; -type PatternStats: record { - -}; - type ThreadStats: record { num_threads: count; }; diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index a49d377bae..a35ee4a90e 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -117,8 +117,8 @@ event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: Pr $reassem_frag_size=rs$frag_size, $reassem_unknown_size=rs$unknown_size, - $events_proc=es$num_events_dispatched - last_es$num_events_dispatched, - $events_queued=es$num_events_queued - last_es$num_events_queued, + $events_proc=es$dispatched - last_es$dispatched, + $events_queued=es$queued - last_es$queued, $timers=ts$cumulative - last_ts$cumulative, $active_timers=ts$current, diff --git a/src/DFA.cc b/src/DFA.cc index 9b8b3e5d31..5885a9bf3b 100644 --- a/src/DFA.cc +++ b/src/DFA.cc @@ -9,8 +9,6 @@ unsigned int DFA_State::transition_counter = 0; -uint64 total_dfa_states = 0; - DFA_State::DFA_State(int arg_state_num, const EquivClass* ec, NFA_state_list* arg_nfa_states, AcceptingSet* arg_accept) @@ -22,8 +20,6 @@ DFA_State::DFA_State(int arg_state_num, const EquivClass* ec, mark = 0; centry = 0; - ++total_dfa_states; - SymPartition(ec); xtions = new DFA_State*[num_sym]; @@ -350,6 +346,7 @@ DFA_State* DFA_State_Cache::Lookup(const NFA_state_list& nfas, ++misses; return 0; } + ++hits; delete *hash; *hash = 0; diff --git a/src/DFA.h b/src/DFA.h index c329b929d4..a63beca9ac 100644 --- a/src/DFA.h +++ b/src/DFA.h @@ -19,8 +19,6 @@ class DFA_Machine; class DFA_State; struct CacheEntry; -extern uint64 total_dfa_states; - class DFA_State : public BroObj { public: DFA_State(int state_num, const EquivClass* ec, @@ -91,10 +89,9 @@ public: int NumEntries() const { return states.Length(); } struct Stats { - unsigned int dfa_states; - - // Sum over all NFA states per DFA state. + // Sum of all NFA states unsigned int nfa_states; + unsigned int dfa_states; unsigned int computed; unsigned int uncomputed; unsigned int mem; diff --git a/src/Func.cc b/src/Func.cc index ac3cda6dd6..ccb2570f70 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -653,7 +653,6 @@ void init_builtin_funcs() TimerStats = internal_type("TimerStats")->AsRecordType(); FileAnalysisStats = internal_type("FileAnalysisStats")->AsRecordType(); ThreadStats = internal_type("ThreadStats")->AsRecordType(); - PatternStats = internal_type("PatternStats")->AsRecordType(); var_sizes = internal_type("var_sizes")->AsTableType(); diff --git a/src/RuleMatcher.cc b/src/RuleMatcher.cc index f40a5c4349..af4787086d 100644 --- a/src/RuleMatcher.cc +++ b/src/RuleMatcher.cc @@ -1174,7 +1174,7 @@ void RuleMatcher::GetStats(Stats* stats, RuleHdrTest* hdr_test) stats->mem = 0; stats->hits = 0; stats->misses = 0; - stats->avg_nfa_states = 0; + stats->nfa_states = 0; hdr_test = root; } @@ -1195,15 +1195,10 @@ void RuleMatcher::GetStats(Stats* stats, RuleHdrTest* hdr_test) stats->mem += cstats.mem; stats->hits += cstats.hits; stats->misses += cstats.misses; - stats->avg_nfa_states += cstats.nfa_states; + stats->nfa_states += cstats.nfa_states; } } - if ( stats->dfa_states ) - stats->avg_nfa_states /= stats->dfa_states; - else - stats->avg_nfa_states = 0; - for ( RuleHdrTest* h = hdr_test->child; h; h = h->sibling ) GetStats(stats, h); } diff --git a/src/RuleMatcher.h b/src/RuleMatcher.h index 6ffc971db1..b16a1556f9 100644 --- a/src/RuleMatcher.h +++ b/src/RuleMatcher.h @@ -297,6 +297,9 @@ public: struct Stats { unsigned int matchers; // # distinct RE matchers + // NFA states across all matchers. + unsigned int nfa_states; + // # DFA states across all matchers unsigned int dfa_states; unsigned int computed; // # computed DFA state transitions @@ -305,9 +308,6 @@ public: // # cache hits (sampled, multiply by MOVE_TO_FRONT_SAMPLE_SIZE) unsigned int hits; unsigned int misses; // # cache misses - - // Average # NFA states per DFA state. - unsigned int avg_nfa_states; }; Val* BuildRuleStateValue(const Rule* rule, diff --git a/src/Stats.cc b/src/Stats.cc index 99e36625b8..cf364d5747 100644 --- a/src/Stats.cc +++ b/src/Stats.cc @@ -173,9 +173,9 @@ void ProfileLogger::Log() RuleMatcher::Stats stats; rule_matcher->GetStats(&stats); - file->Write(fmt("%06f RuleMatcher: matchers=%d dfa_states=%d ncomputed=%d " - "mem=%dK avg_nfa_states=%d\n", network_time, stats.matchers, - stats.dfa_states, stats.computed, stats.mem / 1024, stats.avg_nfa_states)); + file->Write(fmt("%06f RuleMatcher: matchers=%d nfa_states=%d dfa_states=%d " + "ncomputed=%d mem=%dK\n", network_time, stats.matchers, + stats.nfa_states, stats.dfa_states, stats.computed, stats.mem / 1024)); } file->Write(fmt("%.06f Timers: current=%d max=%d mem=%dK lag=%.2fs\n", diff --git a/src/stats.bif b/src/stats.bif index 3a975145b6..ac8541182f 100644 --- a/src/stats.bif +++ b/src/stats.bif @@ -12,7 +12,6 @@ RecordType* ConnStats; RecordType* GapStats; RecordType* EventStats; RecordType* ThreadStats; -RecordType* PatternStats; RecordType* TimerStats; RecordType* FileAnalysisStats; %%} @@ -157,6 +156,13 @@ function get_event_stats%(%): EventStats return r; %} +## Returns statistics about reassembler usage. +## +## Returns: A record with reassembler statistics. +## +## .. bro:see:: get_net_stats +## get_proc_stats +## get_matcher_stats function get_reassembler_stats%(%): ReassemblerStats %{ RecordVal* r = new RecordVal(ReassemblerStats); @@ -170,6 +176,13 @@ function get_reassembler_stats%(%): ReassemblerStats return r; %} +## Returns statistics about DNS lookup activity. +## +## Returns: A record with DNS lookup statistics. +## +## .. bro:see:: get_net_stats +## get_proc_stats +## get_matcher_stats function get_dns_stats%(%): DNSStats %{ RecordVal* r = new RecordVal(DNSStats); @@ -188,25 +201,13 @@ function get_dns_stats%(%): DNSStats return r; %} -function get_pattern_stats%(%): PatternStats - %{ - RecordVal* r = new RecordVal(PatternStats); - int n = 0; - - //DFA_State_Cache::Stats stats; - //dfa_state_cache->GetStats(&stats); - - //fprintf(f, "Computed dfa_states = %d; Classes = %d; Computed trans. = %d; Uncomputed trans. = %d\n", - // stats.dfa_states, EC()->NumClasses(), - // stats.computed, stats.uncomputed); -// - //fprintf(f, "DFA cache hits = %d; misses = %d\n", - // stats.hits, stats.misses); - - return r; - - %} - +## Returns statistics about timer usage. +## +## Returns: A record with timer usage statistics. +## +## .. bro:see:: get_net_stats +## get_proc_stats +## get_matcher_stats function get_timer_stats%(%): TimerStats %{ RecordVal* r = new RecordVal(TimerStats); @@ -219,6 +220,13 @@ function get_timer_stats%(%): TimerStats return r; %} +## Returns statistics about file analysis. +## +## Returns: A record with file analysis statistics. +## +## .. bro:see:: get_net_stats +## get_proc_stats +## get_matcher_stats function get_file_analysis_stats%(%): FileAnalysisStats %{ RecordVal* r = new RecordVal(FileAnalysisStats); @@ -231,6 +239,13 @@ function get_file_analysis_stats%(%): FileAnalysisStats return r; %} +## Returns statistics about thread usage. +## +## Returns: A record with thread usage statistics. +## +## .. bro:see:: get_net_stats +## get_proc_stats +## get_matcher_stats function get_thread_stats%(%): ThreadStats %{ RecordVal* r = new RecordVal(ThreadStats); @@ -245,8 +260,7 @@ function get_thread_stats%(%): ThreadStats ## ## Returns: A record with TCP gap statistics. ## -## .. bro:see:: do_profiling -## get_net_stats +## .. bro:see:: get_net_stats ## get_proc_stats ## get_matcher_stats function get_gap_stats%(%): GapStats @@ -283,12 +297,12 @@ function get_matcher_stats%(%): MatcherStats rule_matcher->GetStats(&s); r->Assign(n++, new Val(s.matchers, TYPE_COUNT)); + r->Assign(n++, new Val(s.nfa_states, TYPE_COUNT)); r->Assign(n++, new Val(s.dfa_states, TYPE_COUNT)); r->Assign(n++, new Val(s.computed, TYPE_COUNT)); r->Assign(n++, new Val(s.mem, TYPE_COUNT)); r->Assign(n++, new Val(s.hits, TYPE_COUNT)); r->Assign(n++, new Val(s.misses, TYPE_COUNT)); - r->Assign(n++, new Val(s.avg_nfa_states, TYPE_COUNT)); return r; %} From 18a1e6f76b33732c84f54e3e4a07dc99bcce05ee Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 11 Jan 2016 09:25:36 -0500 Subject: [PATCH 17/84] Small stats script tweaks and beginning broker stats. --- scripts/policy/misc/stats.bro | 41 ++++++++++++++++------------------- src/stats.bif | 34 +++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 22 deletions(-) diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index a35ee4a90e..b43326e89d 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -19,6 +19,20 @@ export { mem: count &log; ## Number of packets processed since the last stats interval. pkts_proc: count &log; + ## Number of bytes received since the last stats interval if + ## reading live traffic. + bytes_recv: count &log; + + ## Number of packets dropped since the last stats interval if + ## reading live traffic. + pkts_dropped: count &log &optional; + ## Number of packets seen on the link since the last stats + ## interval if reading live traffic. + pkts_link: count &log &optional; + ## Lag between the wall clock and packet timestamps if reading + ## live traffic. + pkt_lag: interval &log &optional; + ## Number of events processed since the last stats interval. events_proc: count &log; ## Number of events that have been queued since the last stats @@ -57,22 +71,6 @@ export { reassem_frag_size: count &log; ## Current size of unkown data in reassembly (this is only PIA buffer right now). reassem_unknown_size: count &log; - - ## Lag between the wall clock and packet timestamps if reading - ## live traffic. - lag: interval &log &optional; - ## Number of packets received since the last stats interval if - ## reading live traffic. - pkts_recv: count &log &optional; - ## Number of packets dropped since the last stats interval if - ## reading live traffic. - pkts_dropped: count &log &optional; - ## Number of packets seen on the link since the last stats - ## interval if reading live traffic. - pkts_link: count &log &optional; - ## Number of bytes received since the last stats interval if - ## reading live traffic. - bytes_recv: count &log &optional; }; ## Event to catch stats as they are written to the logging stream. @@ -86,7 +84,7 @@ event bro_init() &priority=5 event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: ProcStats, last_es: EventStats, last_rs: ReassemblerStats, last_ts: TimerStats, last_fs: FileAnalysisStats) { - local now = network_time(); + local nettime = network_time(); local ns = get_net_stats(); local cs = get_conn_stats(); local ps = get_proc_stats(); @@ -100,10 +98,11 @@ event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: Pr # shutting down. return; - local info: Info = [$ts=now, + local info: Info = [$ts=nettime, $peer=peer_description, $mem=ps$mem/1048576, $pkts_proc=ns$pkts_recvd - last_ns$pkts_recvd, + $bytes_recv = ns$bytes_recvd - last_ns$bytes_recvd, $active_tcp_conns=cs$num_tcp_conns, $tcp_conns=cs$cumulative_tcp_conns - last_cs$cumulative_tcp_conns, @@ -132,15 +131,13 @@ event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: Pr if ( reading_live_traffic() ) { - info$lag = now - network_time(); - info$pkts_recv = ns$pkts_recvd - last_ns$pkts_recvd; + info$pkt_lag = current_time() - nettime; info$pkts_dropped = ns$pkts_dropped - last_ns$pkts_dropped; info$pkts_link = ns$pkts_link - last_ns$pkts_link; - info$bytes_recv = ns$bytes_recvd - last_ns$bytes_recvd; } Log::write(Stats::LOG, info); - schedule stats_report_interval { check_stats(now, ns, cs, ps, es, rs, ts, fs) }; + schedule stats_report_interval { check_stats(nettime, ns, cs, ps, es, rs, ts, fs) }; } event bro_init() diff --git a/src/stats.bif b/src/stats.bif index ac8541182f..2c5fd6151a 100644 --- a/src/stats.bif +++ b/src/stats.bif @@ -145,6 +145,13 @@ function get_proc_stats%(%): ProcStats return r; %} +## Returns statistics about the event engine. +## +## Returns: A record with event engine statistics. +## +## .. bro:see:: get_net_stats +## get_proc_stats +## get_matcher_stats function get_event_stats%(%): EventStats %{ RecordVal* r = new RecordVal(EventStats); @@ -306,3 +313,30 @@ function get_matcher_stats%(%): MatcherStats return r; %} + +function get_broker_stats%(%): BrokerStats + %{ + RecordVal* r = new RecordVal(CommunicationStats); + int n = 0; + +#ifdef ENABLE_BROKER + auto cs = broker_mgr->ConsumeStatistics(); + + r->Assign(n++, new Val(cs.outgoing_peer_count, TYPE_COUNT)); + r->Assign(n++, new Val(cs.data_store_count, TYPE_COUNT)); + r->Assign(n++, new Val(cs.pending_query_count, TYPE_COUNT)); + r->Assign(n++, new Val(cs.response_count, TYPE_COUNT)); + r->Assign(n++, new Val(cs.outgoing_conn_status_count, TYPE_COUNT)); + r->Assign(n++, new Val(cs.incoming_conn_status_count, TYPE_COUNT)); + r->Assign(n++, new Val(cs.report_count, TYPE_COUNT)); + + //for ( const auto& s : cs.print_count ) + // file->Write(fmt(" %-25s prints dequeued=%zu\n", s.first.data(), s.second)); + //for ( const auto& s : cs.event_count ) + // file->Write(fmt(" %-25s events dequeued=%zu\n", s.first.data(), s.second)); + //for ( const auto& s : cs.log_count ) + // file->Write(fmt(" %-25s logs dequeued=%zu\n", s.first.data(), s.second)); +#endif + + return r; + %} \ No newline at end of file From c1d7337a73ae600d75e632be000ec70defd9eef7 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Tue, 12 Jan 2016 15:35:29 -0600 Subject: [PATCH 18/84] Improve documentation of Bro script statements Added more documentation of the "delete" statement. Removed some other text that was probably more confusing than helpful. --- doc/script-reference/statements.rst | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/doc/script-reference/statements.rst b/doc/script-reference/statements.rst index e2f93a5627..47e82eb074 100644 --- a/doc/script-reference/statements.rst +++ b/doc/script-reference/statements.rst @@ -277,16 +277,25 @@ Here are the statements that the Bro scripting language supports. .. bro:keyword:: delete The "delete" statement is used to remove an element from a - :bro:type:`set` or :bro:type:`table`. Nothing happens if the - specified element does not exist in the set or table. + :bro:type:`set` or :bro:type:`table`, or to remove a value from + a :bro:type:`record` field that has the :bro:attr:`&optional` attribute. + When attempting to remove an element from a set or table, + nothing happens if the specified index does not exist. + When attempting to remove a value from an "&optional" record field, + nothing happens if that field doesn't have a value. Example:: local myset = set("this", "test"); local mytable = table(["key1"] = 80/tcp, ["key2"] = 53/udp); + local myrec = MyRecordType($a = 1, $b = 2); + delete myset["test"]; delete mytable["key1"]; + # In this example, "b" must have the "&optional" attribute + delete myrec$b; + .. bro:keyword:: event The "event" statement immediately queues invocation of an event handler. @@ -532,8 +541,6 @@ Here are the statements that the Bro scripting language supports. end with either a :bro:keyword:`break`, :bro:keyword:`fallthrough`, or :bro:keyword:`return` statement (although "return" is allowed only if the "switch" statement is inside a function, hook, or event handler). - If a "case" (or "default") block contain more than one statement, then - there is no need to wrap them in braces. Note that the braces in a "switch" statement are always required (these do not indicate the presence of a `compound statement`_), and that no @@ -604,12 +611,9 @@ Here are the statements that the Bro scripting language supports. if ( skip_ahead() ) next; - [...] - if ( finish_up ) break; - [...] } .. _compound statement: From 3550a2b2d360c0176afe505e4375c903862fdd42 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Tue, 12 Jan 2016 15:45:06 -0600 Subject: [PATCH 19/84] Update documentation for DNS "Z" field According to RFC 2535, RFC 3655, and RFC 4035, the Z field has been partitioned into three 1-bit fields. Therefore, we cannot claim in the documentation that it always has the value zero. --- scripts/base/protocols/dns/main.bro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/base/protocols/dns/main.bro b/scripts/base/protocols/dns/main.bro index 58a63293d0..05a44a0ba9 100644 --- a/scripts/base/protocols/dns/main.bro +++ b/scripts/base/protocols/dns/main.bro @@ -52,7 +52,7 @@ export { ## The Recursion Available bit in a response message indicates ## that the name server supports recursive queries. RA: bool &log &default=F; - ## A reserved field that is currently supposed to be zero in all + ## A reserved field that is usually zero in ## queries and responses. Z: count &log &default=0; ## The set of resource descriptions in the query answer. From 16adf2ff5aeeaff4140abf5c960c15c2ccc7e1b0 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 14 Jan 2016 14:05:23 -0500 Subject: [PATCH 20/84] Add DNS stats to the stats.log --- scripts/policy/misc/stats.bro | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index b43326e89d..be84c5f35f 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -63,6 +63,11 @@ export { ## Current number of files actively being seen. active_files: count &log; + ## Number of DNS requests seen since last stats interval. + dns_requests: count &log; + ## Current number of DNS requests awaiting a reply. + active_dns_requests: count &log; + ## Current size of TCP data in reassembly. reassem_tcp_size: count &log; ## Current size of File data in reassembly. @@ -82,7 +87,7 @@ event bro_init() &priority=5 Log::create_stream(Stats::LOG, [$columns=Info, $ev=log_stats, $path="stats"]); } -event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: ProcStats, last_es: EventStats, last_rs: ReassemblerStats, last_ts: TimerStats, last_fs: FileAnalysisStats) +event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: ProcStats, last_es: EventStats, last_rs: ReassemblerStats, last_ts: TimerStats, last_fs: FileAnalysisStats, last_ds: DNSStats) { local nettime = network_time(); local ns = get_net_stats(); @@ -92,6 +97,7 @@ event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: Pr local rs = get_reassembler_stats(); local ts = get_timer_stats(); local fs = get_file_analysis_stats(); + local ds = get_dns_stats(); if ( bro_is_terminating() ) # No more stats will be written or scheduled when Bro is @@ -123,7 +129,10 @@ event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: Pr $active_timers=ts$current, $files=fs$cumulative - last_fs$cumulative, - $active_files=fs$current + $active_files=fs$current, + + $dns_requests=ds$requests - last_ds$requests, + $active_dns_requests=ds$pending ]; # Someone's going to have to explain what this is and add a field to the Info record. @@ -137,10 +146,10 @@ event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: Pr } Log::write(Stats::LOG, info); - schedule stats_report_interval { check_stats(nettime, ns, cs, ps, es, rs, ts, fs) }; + schedule stats_report_interval { check_stats(nettime, ns, cs, ps, es, rs, ts, fs, ds) }; } event bro_init() { - schedule stats_report_interval { check_stats(network_time(), get_net_stats(), get_conn_stats(), get_proc_stats(), get_event_stats(), get_reassembler_stats(), get_timer_stats(), get_file_analysis_stats()) }; + schedule stats_report_interval { check_stats(network_time(), get_net_stats(), get_conn_stats(), get_proc_stats(), get_event_stats(), get_reassembler_stats(), get_timer_stats(), get_file_analysis_stats(), get_dns_stats()) }; } From ee763381b25b0456a01ca40f826fb9c9b9ca9ef8 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 14 Jan 2016 16:17:41 -0500 Subject: [PATCH 21/84] Fixing default stats collection interval to every 5 minutes. --- scripts/policy/misc/stats.bro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index be84c5f35f..d154da05e9 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -8,7 +8,7 @@ export { redef enum Log::ID += { LOG }; ## How often stats are reported. - const stats_report_interval = 1sec &redef; + const stats_report_interval = 5min &redef; type Info: record { ## Timestamp for the measurement. From 6064134119bb119095dff60c6644114571850104 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 14 Jan 2016 16:49:18 -0500 Subject: [PATCH 22/84] Removing Broker stats, it was broken and incomplete. --- src/stats.bif | 58 +++++++++++++++++++++++++-------------------------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/src/stats.bif b/src/stats.bif index 2c5fd6151a..f5c8ee4308 100644 --- a/src/stats.bif +++ b/src/stats.bif @@ -22,8 +22,7 @@ RecordType* FileAnalysisStats; ## ## Returns: A record of packet statistics. ## -## .. bro:see:: do_profiling -## get_proc_stats +## .. bro:see:: get_proc_stats ## get_matcher_stats ## get_gap_stats function get_net_stats%(%): NetStats @@ -100,8 +99,7 @@ function get_conn_stats%(%): ConnStats ## ## Returns: A record with resource usage statistics. ## -## .. bro:see:: do_profiling -## get_net_stats +## .. bro:see:: get_net_stats ## get_matcher_stats ## get_gap_stats function get_proc_stats%(%): ProcStats @@ -314,29 +312,29 @@ function get_matcher_stats%(%): MatcherStats return r; %} -function get_broker_stats%(%): BrokerStats - %{ - RecordVal* r = new RecordVal(CommunicationStats); - int n = 0; - -#ifdef ENABLE_BROKER - auto cs = broker_mgr->ConsumeStatistics(); - - r->Assign(n++, new Val(cs.outgoing_peer_count, TYPE_COUNT)); - r->Assign(n++, new Val(cs.data_store_count, TYPE_COUNT)); - r->Assign(n++, new Val(cs.pending_query_count, TYPE_COUNT)); - r->Assign(n++, new Val(cs.response_count, TYPE_COUNT)); - r->Assign(n++, new Val(cs.outgoing_conn_status_count, TYPE_COUNT)); - r->Assign(n++, new Val(cs.incoming_conn_status_count, TYPE_COUNT)); - r->Assign(n++, new Val(cs.report_count, TYPE_COUNT)); - - //for ( const auto& s : cs.print_count ) - // file->Write(fmt(" %-25s prints dequeued=%zu\n", s.first.data(), s.second)); - //for ( const auto& s : cs.event_count ) - // file->Write(fmt(" %-25s events dequeued=%zu\n", s.first.data(), s.second)); - //for ( const auto& s : cs.log_count ) - // file->Write(fmt(" %-25s logs dequeued=%zu\n", s.first.data(), s.second)); -#endif - - return r; - %} \ No newline at end of file +# function get_broker_stats%(%): BrokerStats +# %{ +# RecordVal* r = new RecordVal(CommunicationStats); +# int n = 0; +# +# #ifdef ENABLE_BROKER +# auto cs = broker_mgr->ConsumeStatistics(); +# +# r->Assign(n++, new Val(cs.outgoing_peer_count, TYPE_COUNT)); +# r->Assign(n++, new Val(cs.data_store_count, TYPE_COUNT)); +# r->Assign(n++, new Val(cs.pending_query_count, TYPE_COUNT)); +# r->Assign(n++, new Val(cs.response_count, TYPE_COUNT)); +# r->Assign(n++, new Val(cs.outgoing_conn_status_count, TYPE_COUNT)); +# r->Assign(n++, new Val(cs.incoming_conn_status_count, TYPE_COUNT)); +# r->Assign(n++, new Val(cs.report_count, TYPE_COUNT)); +# +# //for ( const auto& s : cs.print_count ) +# // file->Write(fmt(" %-25s prints dequeued=%zu\n", s.first.data(), s.second)); +# //for ( const auto& s : cs.event_count ) +# // file->Write(fmt(" %-25s events dequeued=%zu\n", s.first.data(), s.second)); +# //for ( const auto& s : cs.log_count ) +# // file->Write(fmt(" %-25s logs dequeued=%zu\n", s.first.data(), s.second)); +# #endif +# +# return r; +# %} \ No newline at end of file From 53db5d1711e2652596e8660d40789296013f9a0e Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 14 Jan 2016 17:09:55 -0500 Subject: [PATCH 23/84] Removing some references to resource_usage() --- scripts/policy/frameworks/control/controllee.bro | 12 ++++++------ src/bro.bif | 2 -- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/scripts/policy/frameworks/control/controllee.bro b/scripts/policy/frameworks/control/controllee.bro index b4769764f4..6e3b5499b6 100644 --- a/scripts/policy/frameworks/control/controllee.bro +++ b/scripts/policy/frameworks/control/controllee.bro @@ -29,12 +29,12 @@ event Control::peer_status_request() if ( ! peer$connected ) next; - local res = resource_usage(); - status += fmt("%.6f peer=%s host=%s events_in=%s events_out=%s ops_in=%s ops_out=%s bytes_in=? bytes_out=?\n", - network_time(), - peer$peer$descr, peer$host, - res$num_events_queued, res$num_events_dispatched, - res$blocking_input, res$blocking_output); + #local res = resource_usage(); + #status += fmt("%.6f peer=%s host=%s events_in=%s events_out=%s ops_in=%s ops_out=%s bytes_in=? bytes_out=?\n", + # network_time(), + # peer$peer$descr, peer$host, + # res$num_events_queued, res$num_events_dispatched, + # res$blocking_input, res$blocking_output); } event Control::peer_status_response(status); diff --git a/src/bro.bif b/src/bro.bif index ce16695afa..5385a0e22f 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1792,7 +1792,6 @@ function record_fields%(rec: any%): record_field_table ## holds the name of the file. ## ## .. bro:see:: net_stats -## resource_usage ## get_matcher_stats ## get_gap_stats function do_profiling%(%) : any @@ -1857,7 +1856,6 @@ function is_local_interface%(ip: addr%) : bool ## Returns: True (unconditionally). ## ## .. bro:see:: do_profiling -## resource_usage ## get_matcher_stats ## get_net_stats ## get_gap_stats From 41a181d98d7afe06ae47255986fea26bde55cafe Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 14 Jan 2016 21:22:09 -0500 Subject: [PATCH 24/84] Removing more broken functionality due to changed stats apis. --- .../policy/frameworks/control/controllee.bro | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/scripts/policy/frameworks/control/controllee.bro b/scripts/policy/frameworks/control/controllee.bro index 6e3b5499b6..1a62d294b7 100644 --- a/scripts/policy/frameworks/control/controllee.bro +++ b/scripts/policy/frameworks/control/controllee.bro @@ -22,30 +22,30 @@ event Control::id_value_request(id: string) event Control::peer_status_request() { - local status = ""; - for ( p in Communication::nodes ) - { - local peer = Communication::nodes[p]; - if ( ! peer$connected ) - next; - - #local res = resource_usage(); - #status += fmt("%.6f peer=%s host=%s events_in=%s events_out=%s ops_in=%s ops_out=%s bytes_in=? bytes_out=?\n", - # network_time(), - # peer$peer$descr, peer$host, - # res$num_events_queued, res$num_events_dispatched, - # res$blocking_input, res$blocking_output); - } - - event Control::peer_status_response(status); + #local status = ""; + #for ( p in Communication::nodes ) + # { + # local peer = Communication::nodes[p]; + # if ( ! peer$connected ) + # next; + # + # #local res = resource_usage(); + # #status += fmt("%.6f peer=%s host=%s events_in=%s events_out=%s ops_in=%s ops_out=%s bytes_in=? bytes_out=?\n", + # # network_time(), + # # peer$peer$descr, peer$host, + # # res$num_events_queued, res$num_events_dispatched, + # # res$blocking_input, res$blocking_output); + # } + # + #event Control::peer_status_response(status); } event Control::net_stats_request() { - local ns = net_stats(); - local reply = fmt("%.6f recvd=%d dropped=%d link=%d\n", network_time(), - ns$pkts_recvd, ns$pkts_dropped, ns$pkts_link); - event Control::net_stats_response(reply); + #local ns = net_stats(); + #local reply = fmt("%.6f recvd=%d dropped=%d link=%d\n", network_time(), + # ns$pkts_recvd, ns$pkts_dropped, ns$pkts_link); + #event Control::net_stats_response(reply); } event Control::configuration_update_request() From 6ef8a93dcaf5dc587207a1155f4eb2b4f22f1950 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 27 Jan 2016 14:56:42 -0600 Subject: [PATCH 25/84] Update traffic per core estimate in the cluster doc --- doc/cluster/index.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/cluster/index.rst b/doc/cluster/index.rst index 544ca5e0f8..6e426c005e 100644 --- a/doc/cluster/index.rst +++ b/doc/cluster/index.rst @@ -96,13 +96,13 @@ logging is done remotely to the manager, and normally very little is written to disk. The rule of thumb we have followed recently is to allocate approximately 1 -core for every 80Mbps of traffic that is being analyzed. However, this +core for every 250Mbps of traffic that is being analyzed. However, this estimate could be extremely traffic mix-specific. It has generally worked for mixed traffic with many users and servers. For example, if your traffic peaks around 2Gbps (combined) and you want to handle traffic at peak load, -you may want to have 26 cores available (2048 / 80 == 25.6). If the 80Mbps -estimate works for your traffic, this could be handled by 3 physical hosts -dedicated to being workers with each one containing dual 6-core processors. +you may want to have 8 cores available (2048 / 250 == 8.2). If the 250Mbps +estimate works for your traffic, this could be handled by 2 physical hosts +dedicated to being workers with each one containing a quad-core processor. Once a flow-based load balancer is put into place this model is extremely easy to scale. It is recommended that you estimate the amount of From 7ede9c65d2e40c24e6be1acfa815cbef991caa6e Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Tue, 1 Mar 2016 17:31:41 -0600 Subject: [PATCH 26/84] Add more documentation to sumstats framework scripts --- scripts/base/frameworks/sumstats/main.bro | 38 ++++++++++++------- .../frameworks/sumstats/plugins/average.bro | 4 +- .../sumstats/plugins/hll_unique.bro | 2 + .../base/frameworks/sumstats/plugins/last.bro | 2 + .../base/frameworks/sumstats/plugins/max.bro | 4 +- .../base/frameworks/sumstats/plugins/min.bro | 4 +- .../frameworks/sumstats/plugins/sample.bro | 4 +- .../frameworks/sumstats/plugins/std-dev.bro | 4 +- .../base/frameworks/sumstats/plugins/sum.bro | 6 ++- .../base/frameworks/sumstats/plugins/topk.bro | 5 +++ .../frameworks/sumstats/plugins/unique.bro | 6 ++- .../frameworks/sumstats/plugins/variance.bro | 6 ++- 12 files changed, 61 insertions(+), 24 deletions(-) diff --git a/scripts/base/frameworks/sumstats/main.bro b/scripts/base/frameworks/sumstats/main.bro index 8dbdb61edd..edd80ede0f 100644 --- a/scripts/base/frameworks/sumstats/main.bro +++ b/scripts/base/frameworks/sumstats/main.bro @@ -5,7 +5,8 @@ module SumStats; export { - ## The various calculations are all defined as plugins. + ## Type to represent the calculations that are available. The calculations + ## are all defined as plugins. type Calculation: enum { PLACEHOLDER }; @@ -39,6 +40,7 @@ export { str: string &optional; }; + ## Represents a reducer. type Reducer: record { ## Observation stream identifier for the reducer ## to attach to. @@ -56,7 +58,7 @@ export { normalize_key: function(key: SumStats::Key): Key &optional; }; - ## Value calculated for an observation stream fed into a reducer. + ## Result calculated for an observation stream fed into a reducer. ## Most of the fields are added by plugins. type ResultVal: record { ## The time when the first observation was added to @@ -71,14 +73,15 @@ export { num: count &default=0; }; - ## Type to store results for multiple reducers. + ## Type to store a table of results for multiple reducers indexed by + ## observation stream identifier. type Result: table[string] of ResultVal; ## Type to store a table of sumstats results indexed by keys. type ResultTable: table[Key] of Result; - ## SumStats represent an aggregation of reducers along with - ## mechanisms to handle various situations like the epoch ending + ## Represents a SumStat, which consists of an aggregation of reducers along + ## with mechanisms to handle various situations like the epoch ending ## or thresholds being crossed. ## ## It's best to not access any global state outside @@ -101,21 +104,28 @@ export { ## The reducers for the SumStat. reducers: set[Reducer]; - ## Provide a function to calculate a value from the - ## :bro:see:`SumStats::Result` structure which will be used - ## for thresholding. - ## This is required if a *threshold* value is given. + ## A function that will be called once for each observation in order + ## to calculate a value from the :bro:see:`SumStats::Result` structure + ## which will be used for thresholding. + ## This function is required if a *threshold* value or + ## a *threshold_series* is given. threshold_val: function(key: SumStats::Key, result: SumStats::Result): double &optional; - ## The threshold value for calling the - ## *threshold_crossed* callback. + ## The threshold value for calling the *threshold_crossed* callback. + ## If you need more than one threshold value, then use + ## *threshold_series* instead. threshold: double &optional; - ## A series of thresholds for calling the - ## *threshold_crossed* callback. + ## A series of thresholds for calling the *threshold_crossed* + ## callback. These thresholds must be listed in ascending order, + ## because a threshold is not checked until the preceding one has + ## been crossed. threshold_series: vector of double &optional; ## A callback that is called when a threshold is crossed. + ## A threshold is crossed when the value returned from *threshold_val* + ## is greater than or equal to the threshold value, but only the first + ## time this happens within an epoch. threshold_crossed: function(key: SumStats::Key, result: SumStats::Result) &optional; ## A callback that receives each of the results at the @@ -130,6 +140,8 @@ export { }; ## Create a summary statistic. + ## + ## ss: The SumStat to create. global create: function(ss: SumStats::SumStat); ## Add data into an observation stream. This should be diff --git a/scripts/base/frameworks/sumstats/plugins/average.bro b/scripts/base/frameworks/sumstats/plugins/average.bro index 8f7f7b568f..160ca64d78 100644 --- a/scripts/base/frameworks/sumstats/plugins/average.bro +++ b/scripts/base/frameworks/sumstats/plugins/average.bro @@ -1,3 +1,5 @@ +##! Calculate the average. + @load ../main module SumStats; @@ -9,7 +11,7 @@ export { }; redef record ResultVal += { - ## For numeric data, this calculates the average of all values. + ## For numeric data, this is the average of all values. average: double &optional; }; } diff --git a/scripts/base/frameworks/sumstats/plugins/hll_unique.bro b/scripts/base/frameworks/sumstats/plugins/hll_unique.bro index 494cbf4667..43cafcff7f 100644 --- a/scripts/base/frameworks/sumstats/plugins/hll_unique.bro +++ b/scripts/base/frameworks/sumstats/plugins/hll_unique.bro @@ -1,3 +1,5 @@ +##! Calculate the number of unique values (using the HyperLogLog algorithm). + @load base/frameworks/sumstats module SumStats; diff --git a/scripts/base/frameworks/sumstats/plugins/last.bro b/scripts/base/frameworks/sumstats/plugins/last.bro index 430c2e375b..ca04114f61 100644 --- a/scripts/base/frameworks/sumstats/plugins/last.bro +++ b/scripts/base/frameworks/sumstats/plugins/last.bro @@ -1,3 +1,5 @@ +##! Keep the last X observations. + @load base/frameworks/sumstats @load base/utils/queue diff --git a/scripts/base/frameworks/sumstats/plugins/max.bro b/scripts/base/frameworks/sumstats/plugins/max.bro index d43ad9dc38..adcc6ae113 100644 --- a/scripts/base/frameworks/sumstats/plugins/max.bro +++ b/scripts/base/frameworks/sumstats/plugins/max.bro @@ -1,3 +1,5 @@ +##! Find the maximum value. + @load ../main module SumStats; @@ -9,7 +11,7 @@ export { }; redef record ResultVal += { - ## For numeric data, this tracks the maximum value given. + ## For numeric data, this tracks the maximum value. max: double &optional; }; } diff --git a/scripts/base/frameworks/sumstats/plugins/min.bro b/scripts/base/frameworks/sumstats/plugins/min.bro index 014755cf32..22cab1009c 100644 --- a/scripts/base/frameworks/sumstats/plugins/min.bro +++ b/scripts/base/frameworks/sumstats/plugins/min.bro @@ -1,3 +1,5 @@ +##! Find the minimum value. + @load ../main module SumStats; @@ -9,7 +11,7 @@ export { }; redef record ResultVal += { - ## For numeric data, this tracks the minimum value given. + ## For numeric data, this tracks the minimum value. min: double &optional; }; } diff --git a/scripts/base/frameworks/sumstats/plugins/sample.bro b/scripts/base/frameworks/sumstats/plugins/sample.bro index 809d696896..0200e85949 100644 --- a/scripts/base/frameworks/sumstats/plugins/sample.bro +++ b/scripts/base/frameworks/sumstats/plugins/sample.bro @@ -1,3 +1,5 @@ +##! Keep a random sample of values. + @load base/frameworks/sumstats/main module SumStats; @@ -10,7 +12,7 @@ export { }; redef record Reducer += { - ## A number of sample Observations to collect. + ## The number of sample Observations to collect. num_samples: count &default=0; }; diff --git a/scripts/base/frameworks/sumstats/plugins/std-dev.bro b/scripts/base/frameworks/sumstats/plugins/std-dev.bro index 2e5b95b212..bfb02c82cc 100644 --- a/scripts/base/frameworks/sumstats/plugins/std-dev.bro +++ b/scripts/base/frameworks/sumstats/plugins/std-dev.bro @@ -1,3 +1,5 @@ +##! Calculate the standard deviation. + @load ./variance @load ../main @@ -5,7 +7,7 @@ module SumStats; export { redef enum Calculation += { - ## Find the standard deviation of the values. + ## Calculate the standard deviation of the values. STD_DEV }; diff --git a/scripts/base/frameworks/sumstats/plugins/sum.bro b/scripts/base/frameworks/sumstats/plugins/sum.bro index 074b4b72f3..fb1d96bcd4 100644 --- a/scripts/base/frameworks/sumstats/plugins/sum.bro +++ b/scripts/base/frameworks/sumstats/plugins/sum.bro @@ -1,11 +1,13 @@ +##! Calculate the sum. + @load ../main module SumStats; export { redef enum Calculation += { - ## Sums the values given. For string values, - ## this will be the number of strings given. + ## Calculate the sum of the values. For string values, + ## this will be the number of strings. SUM }; diff --git a/scripts/base/frameworks/sumstats/plugins/topk.bro b/scripts/base/frameworks/sumstats/plugins/topk.bro index 0ef0f01393..e7107cb4fb 100644 --- a/scripts/base/frameworks/sumstats/plugins/topk.bro +++ b/scripts/base/frameworks/sumstats/plugins/topk.bro @@ -1,3 +1,5 @@ +##! Keep the top-k (i.e., most frequently occurring) observations. + @load base/frameworks/sumstats module SumStats; @@ -9,10 +11,13 @@ export { }; redef enum Calculation += { + ## Keep a top-k list of values. TOPK }; redef record ResultVal += { + ## A handle which can be passed to some built-in functions to get + ## the top-k results. topk: opaque of topk &optional; }; diff --git a/scripts/base/frameworks/sumstats/plugins/unique.bro b/scripts/base/frameworks/sumstats/plugins/unique.bro index abfbe3669d..5fcaa1dc3c 100644 --- a/scripts/base/frameworks/sumstats/plugins/unique.bro +++ b/scripts/base/frameworks/sumstats/plugins/unique.bro @@ -1,10 +1,12 @@ +##! Calculate the number of unique values. + @load ../main module SumStats; export { redef record Reducer += { - ## Maximum number of unique elements to store. + ## Maximum number of unique values to store. unique_max: count &optional; }; @@ -15,7 +17,7 @@ export { redef record ResultVal += { ## If cardinality is being tracked, the number of unique - ## items is tracked here. + ## values is tracked here. unique: count &default=0; }; } diff --git a/scripts/base/frameworks/sumstats/plugins/variance.bro b/scripts/base/frameworks/sumstats/plugins/variance.bro index 12d30cc4fe..989bf07eaf 100644 --- a/scripts/base/frameworks/sumstats/plugins/variance.bro +++ b/scripts/base/frameworks/sumstats/plugins/variance.bro @@ -1,3 +1,5 @@ +##! Calculate the variance. + @load ./average @load ../main @@ -5,12 +7,12 @@ module SumStats; export { redef enum Calculation += { - ## Find the variance of the values. + ## Calculate the variance of the values. VARIANCE }; redef record ResultVal += { - ## For numeric data, this calculates the variance. + ## For numeric data, this is the variance. variance: double &optional; }; } From f4141bde6d20d66a0c705d7109fb80f918f41049 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Thu, 17 Mar 2016 14:23:18 -0500 Subject: [PATCH 27/84] Call ProtocolConfirmation in SIP only if we saw a response SIP packet --- src/analyzer/protocol/sip/sip-analyzer.pac | 1 - 1 file changed, 1 deletion(-) diff --git a/src/analyzer/protocol/sip/sip-analyzer.pac b/src/analyzer/protocol/sip/sip-analyzer.pac index 36a1dae7e2..829904aa3a 100644 --- a/src/analyzer/protocol/sip/sip-analyzer.pac +++ b/src/analyzer/protocol/sip/sip-analyzer.pac @@ -18,7 +18,6 @@ refine flow SIP_Flow += { function proc_sip_request(method: bytestring, uri: bytestring, vers: SIP_Version): bool %{ - connection()->bro_analyzer()->ProtocolConfirmation(); if ( sip_request ) { BifEvent::generate_sip_request(connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), From d731cb9a18d9304e5150b490970cfbe99d52bdfc Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Thu, 17 Mar 2016 14:25:15 -0500 Subject: [PATCH 28/84] Call ProtocolConfirmation in SNMP only if we saw a response SNMP packet --- src/analyzer/protocol/snmp/snmp-analyzer.pac | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/analyzer/protocol/snmp/snmp-analyzer.pac b/src/analyzer/protocol/snmp/snmp-analyzer.pac index 891531b292..44dce4dbf5 100644 --- a/src/analyzer/protocol/snmp/snmp-analyzer.pac +++ b/src/analyzer/protocol/snmp/snmp-analyzer.pac @@ -373,10 +373,12 @@ refine connection SNMP_Conn += { function proc_header(rec: Header): bool %{ + if ( ! ${rec.is_orig} ) + bro_analyzer()->ProtocolConfirmation(); + if ( rec->unknown() ) return false; - bro_analyzer()->ProtocolConfirmation(); return true; %} From 095e6c27876ebc467513af328417705e61100ca2 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 21 Mar 2016 12:08:02 -0400 Subject: [PATCH 29/84] Fixing a test. --- .../canonified_loaded_scripts.log | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 4d1f2037a4..a010a4fd3e 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2015-08-31-04-50-43 +#open 2016-03-21-16-06-31 #fields name #types string scripts/base/init-bare.bro @@ -109,6 +109,7 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_Teredo.events.bif.bro build/scripts/base/bif/plugins/Bro_UDP.events.bif.bro build/scripts/base/bif/plugins/Bro_ZIP.events.bif.bro + build/scripts/base/bif/plugins/Bro_FileEntropy.events.bif.bro build/scripts/base/bif/plugins/Bro_FileExtract.events.bif.bro build/scripts/base/bif/plugins/Bro_FileExtract.functions.bif.bro build/scripts/base/bif/plugins/Bro_FileHash.events.bif.bro @@ -128,4 +129,4 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro scripts/policy/misc/loaded-scripts.bro scripts/base/utils/paths.bro -#close 2015-08-31-04-50-43 +#close 2016-03-21-16-06-31 From 8650841bf553281bed7ea023354e83e8efea8970 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Thu, 24 Mar 2016 13:38:47 -0700 Subject: [PATCH 30/84] Only load openflow/netcontrol if compiled with broker. --- CHANGES | 6 ++++++ VERSION | 2 +- scripts/base/init-default.bro | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index e90e85f125..a527dcbcc2 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,10 @@ +2.4-424 | 2016-03-24 13:38:47 -0700 + + * Only load openflow/netcontrol if compiled with broker. (Johanna Amann) + + * Adding canonifier to test. (Robin Sommer) + 2.4-422 | 2016-03-21 19:48:30 -0700 * Adapt to recent change in CAF CMake script. (Matthias Vallentin) diff --git a/VERSION b/VERSION index 032d05a7ea..af797c6f72 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4-422 +2.4-424 diff --git a/scripts/base/init-default.bro b/scripts/base/init-default.bro index 7fefe0111d..609ed7200c 100644 --- a/scripts/base/init-default.bro +++ b/scripts/base/init-default.bro @@ -37,8 +37,10 @@ @load base/frameworks/reporter @load base/frameworks/sumstats @load base/frameworks/tunnels +@ifdef ( BrokerComm::enable ) @load base/frameworks/openflow @load base/frameworks/netcontrol +@endif @load base/protocols/conn @load base/protocols/dhcp From 35a4e428cfe4964bd6bbdff80ad0fff42c4ffa8f Mon Sep 17 00:00:00 2001 From: william Date: Sat, 26 Mar 2016 01:54:51 -0700 Subject: [PATCH 31/84] Wrong regex literal in scripting doc --- doc/scripting/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/scripting/index.rst b/doc/scripting/index.rst index 2b5cfbb49c..a776fc0ad3 100644 --- a/doc/scripting/index.rst +++ b/doc/scripting/index.rst @@ -776,7 +776,7 @@ string against which it will be tested to be on the right. In the sample above, two local variables are declared to hold our sample sentence and regular expression. Our regular expression in this case will return true if the string contains either the word -``quick`` or the word ``fox``. The ``if`` statement in the script uses +``quick`` or the word ``lazy``. The ``if`` statement in the script uses embedded matching and the ``in`` operator to check for the existence of the pattern within the string. If the statement resolves to true, :bro:id:`split` is called to break the string into separate pieces. From 9f5c820c7bed2d005bd5a9ed3d94ee3505f2fecf Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 30 Mar 2016 14:31:25 -0500 Subject: [PATCH 32/84] Rename the BrokerComm namespace to Broker --- doc/frameworks/broker.rst | 48 ++-- .../broker/connecting-connector.bro | 10 +- doc/frameworks/broker/connecting-listener.bro | 14 +- doc/frameworks/broker/events-connector.bro | 20 +- doc/frameworks/broker/events-listener.bro | 12 +- doc/frameworks/broker/logs-connector.bro | 14 +- doc/frameworks/broker/logs-listener.bro | 12 +- doc/frameworks/broker/printing-connector.bro | 18 +- doc/frameworks/broker/printing-listener.bro | 14 +- doc/frameworks/broker/stores-connector.bro | 34 +-- doc/frameworks/broker/stores-listener.bro | 16 +- doc/frameworks/broker/testlog.bro | 2 +- scripts/base/frameworks/broker/main.bro | 14 +- .../frameworks/netcontrol/plugins/acld.bro | 12 +- .../frameworks/netcontrol/plugins/broker.bro | 12 +- .../frameworks/openflow/plugins/broker.bro | 12 +- scripts/base/init-default.bro | 2 +- src/broker/Data.cc | 68 +++--- src/broker/Data.h | 22 +- src/broker/Manager.cc | 66 ++--- src/broker/Manager.h | 18 +- src/broker/Store.h | 2 +- src/broker/comm.bif | 64 ++--- src/broker/data.bif | 164 ++++++------- src/broker/messaging.bif | 56 ++--- src/broker/store.bif | 34 +-- .../broker.connection_updates/recv.recv.out | 4 +- .../broker.connection_updates/send.send.out | 2 +- testing/btest/Baseline/broker.data/out | 32 +-- .../broker.remote_event/send.send.out | 2 +- .../Baseline/broker.remote_log/send.send.out | 2 +- .../broker.remote_print/send.send.out | 2 +- .../core.leaks.broker.data/bro..stdout | 32 +-- .../send.send.out | 2 +- .../send.send.out | 2 +- .../send.send.out | 2 +- .../output | 10 +- .../output | 14 +- .../output | 20 +- .../output | 12 +- .../output | 14 +- .../output | 12 +- .../output | 18 +- .../output | 14 +- .../output | 34 +-- .../output | 16 +- .../output | 2 +- .../recv.recv.out | 2 +- .../send.send.out | 2 +- .../recv.recv.out | 2 +- .../send.send.out | 2 +- .../recv.recv.out | 2 +- .../send.send.out | 2 +- .../recv.recv.out | 2 +- .../send.send.out | 2 +- testing/btest/broker/clone_store.bro | 50 ++-- testing/btest/broker/connection_updates.bro | 24 +- testing/btest/broker/data.bro | 228 +++++++++--------- testing/btest/broker/enable-and-exit.bro | 2 +- testing/btest/broker/master_store.bro | 36 +-- testing/btest/broker/remote_event.test | 32 +-- testing/btest/broker/remote_log.test | 16 +- testing/btest/broker/remote_print.test | 28 +-- .../btest/core/leaks/broker/clone_store.bro | 50 ++-- testing/btest/core/leaks/broker/data.bro | 228 +++++++++--------- .../btest/core/leaks/broker/master_store.bro | 36 +-- .../btest/core/leaks/broker/remote_event.test | 32 +-- .../btest/core/leaks/broker/remote_log.test | 16 +- .../btest/core/leaks/broker/remote_print.test | 28 +-- ...orks_broker_connecting-connector_bro.btest | 10 +- ...works_broker_connecting-listener_bro.btest | 14 +- ...ameworks_broker_events-connector_bro.btest | 20 +- ...rameworks_broker_events-listener_bro.btest | 12 +- ...frameworks_broker_logs-connector_bro.btest | 14 +- ..._frameworks_broker_logs-listener_bro.btest | 12 +- ...eworks_broker_printing-connector_bro.btest | 18 +- ...meworks_broker_printing-listener_bro.btest | 14 +- ...ameworks_broker_stores-connector_bro.btest | 34 +-- ...rameworks_broker_stores-listener_bro.btest | 16 +- ...de-doc_frameworks_broker_testlog_bro.btest | 2 +- .../base/frameworks/netcontrol/acld-hook.bro | 20 +- .../base/frameworks/netcontrol/acld.bro | 20 +- .../base/frameworks/netcontrol/broker.bro | 22 +- .../base/frameworks/openflow/broker-basic.bro | 20 +- 84 files changed, 1039 insertions(+), 1039 deletions(-) diff --git a/doc/frameworks/broker.rst b/doc/frameworks/broker.rst index 8c5ed24e25..7b9174909f 100644 --- a/doc/frameworks/broker.rst +++ b/doc/frameworks/broker.rst @@ -17,20 +17,20 @@ Connecting to Peers =================== Communication via Broker must first be turned on via -:bro:see:`BrokerComm::enable`. +:bro:see:`Broker::enable`. -Bro can accept incoming connections by calling :bro:see:`BrokerComm::listen` +Bro can accept incoming connections by calling :bro:see:`Broker::listen` and then monitor connection status updates via the -:bro:see:`BrokerComm::incoming_connection_established` and -:bro:see:`BrokerComm::incoming_connection_broken` events. +:bro:see:`Broker::incoming_connection_established` and +:bro:see:`Broker::incoming_connection_broken` events. .. btest-include:: ${DOC_ROOT}/frameworks/broker/connecting-listener.bro -Bro can initiate outgoing connections by calling :bro:see:`BrokerComm::connect` +Bro can initiate outgoing connections by calling :bro:see:`Broker::connect` and then monitor connection status updates via the -:bro:see:`BrokerComm::outgoing_connection_established`, -:bro:see:`BrokerComm::outgoing_connection_broken`, and -:bro:see:`BrokerComm::outgoing_connection_incompatible` events. +:bro:see:`Broker::outgoing_connection_established`, +:bro:see:`Broker::outgoing_connection_broken`, and +:bro:see:`Broker::outgoing_connection_incompatible` events. .. btest-include:: ${DOC_ROOT}/frameworks/broker/connecting-connector.bro @@ -38,14 +38,14 @@ Remote Printing =============== To receive remote print messages, first use the -:bro:see:`BrokerComm::subscribe_to_prints` function to advertise to peers a +:bro:see:`Broker::subscribe_to_prints` function to advertise to peers a topic prefix of interest and then create an event handler for -:bro:see:`BrokerComm::print_handler` to handle any print messages that are +:bro:see:`Broker::print_handler` to handle any print messages that are received. .. btest-include:: ${DOC_ROOT}/frameworks/broker/printing-listener.bro -To send remote print messages, just call :bro:see:`BrokerComm::print`. +To send remote print messages, just call :bro:see:`Broker::print`. .. btest-include:: ${DOC_ROOT}/frameworks/broker/printing-connector.bro @@ -69,14 +69,14 @@ Remote Events ============= Receiving remote events is similar to remote prints. Just use the -:bro:see:`BrokerComm::subscribe_to_events` function and possibly define any +:bro:see:`Broker::subscribe_to_events` function and possibly define any new events along with handlers that peers may want to send. .. btest-include:: ${DOC_ROOT}/frameworks/broker/events-listener.bro There are two different ways to send events. The first is to call the -:bro:see:`BrokerComm::event` function directly. The second option is to call -the :bro:see:`BrokerComm::auto_event` function where you specify a +:bro:see:`Broker::event` function directly. The second option is to call +the :bro:see:`Broker::auto_event` function where you specify a particular event that will be automatically sent to peers whenever the event is called locally via the normal event invocation syntax. @@ -104,14 +104,14 @@ Remote Logging .. btest-include:: ${DOC_ROOT}/frameworks/broker/testlog.bro -Use the :bro:see:`BrokerComm::subscribe_to_logs` function to advertise interest +Use the :bro:see:`Broker::subscribe_to_logs` function to advertise interest in logs written by peers. The topic names that Bro uses are implicitly of the form "bro/log/". .. btest-include:: ${DOC_ROOT}/frameworks/broker/logs-listener.bro To send remote logs either redef :bro:see:`Log::enable_remote_logging` or -use the :bro:see:`BrokerComm::enable_remote_logs` function. The former +use the :bro:see:`Broker::enable_remote_logs` function. The former allows any log stream to be sent to peers while the latter enables remote logging for particular streams. @@ -137,24 +137,24 @@ Tuning Access Control By default, endpoints do not restrict the message topics that it sends to peers and do not restrict what message topics and data store identifiers get advertised to peers. These are the default -:bro:see:`BrokerComm::EndpointFlags` supplied to :bro:see:`BrokerComm::enable`. +:bro:see:`Broker::EndpointFlags` supplied to :bro:see:`Broker::enable`. If not using the ``auto_publish`` flag, one can use the -:bro:see:`BrokerComm::publish_topic` and :bro:see:`BrokerComm::unpublish_topic` +:bro:see:`Broker::publish_topic` and :bro:see:`Broker::unpublish_topic` functions to manipulate the set of message topics (must match exactly) that are allowed to be sent to peer endpoints. These settings take precedence over the per-message ``peers`` flag supplied to functions -that take a :bro:see:`BrokerComm::SendFlags` such as :bro:see:`BrokerComm::print`, -:bro:see:`BrokerComm::event`, :bro:see:`BrokerComm::auto_event` or -:bro:see:`BrokerComm::enable_remote_logs`. +that take a :bro:see:`Broker::SendFlags` such as :bro:see:`Broker::print`, +:bro:see:`Broker::event`, :bro:see:`Broker::auto_event` or +:bro:see:`Broker::enable_remote_logs`. If not using the ``auto_advertise`` flag, one can use the -:bro:see:`BrokerComm::advertise_topic` and -:bro:see:`BrokerComm::unadvertise_topic` functions +:bro:see:`Broker::advertise_topic` and +:bro:see:`Broker::unadvertise_topic` functions to manipulate the set of topic prefixes that are allowed to be advertised to peers. If an endpoint does not advertise a topic prefix, then the only way peers can send messages to it is via the ``unsolicited`` -flag of :bro:see:`BrokerComm::SendFlags` and choosing a topic with a matching +flag of :bro:see:`Broker::SendFlags` and choosing a topic with a matching prefix (i.e. full topic may be longer than receivers prefix, just the prefix needs to match). diff --git a/doc/frameworks/broker/connecting-connector.bro b/doc/frameworks/broker/connecting-connector.bro index cd5c74add8..adf901ea6a 100644 --- a/doc/frameworks/broker/connecting-connector.bro +++ b/doc/frameworks/broker/connecting-connector.bro @@ -1,18 +1,18 @@ const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -redef BrokerComm::endpoint_name = "connector"; +redef Broker::endpoint_name = "connector"; event bro_init() { - BrokerComm::enable(); - BrokerComm::connect("127.0.0.1", broker_port, 1sec); + Broker::enable(); + Broker::connect("127.0.0.1", broker_port, 1sec); } -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { - print "BrokerComm::outgoing_connection_established", + print "Broker::outgoing_connection_established", peer_address, peer_port, peer_name; terminate(); } diff --git a/doc/frameworks/broker/connecting-listener.bro b/doc/frameworks/broker/connecting-listener.bro index 21c67f9696..aa2b945dbe 100644 --- a/doc/frameworks/broker/connecting-listener.bro +++ b/doc/frameworks/broker/connecting-listener.bro @@ -1,20 +1,20 @@ const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -redef BrokerComm::endpoint_name = "listener"; +redef Broker::endpoint_name = "listener"; event bro_init() { - BrokerComm::enable(); - BrokerComm::listen(broker_port, "127.0.0.1"); + Broker::enable(); + Broker::listen(broker_port, "127.0.0.1"); } -event BrokerComm::incoming_connection_established(peer_name: string) +event Broker::incoming_connection_established(peer_name: string) { - print "BrokerComm::incoming_connection_established", peer_name; + print "Broker::incoming_connection_established", peer_name; } -event BrokerComm::incoming_connection_broken(peer_name: string) +event Broker::incoming_connection_broken(peer_name: string) { - print "BrokerComm::incoming_connection_broken", peer_name; + print "Broker::incoming_connection_broken", peer_name; terminate(); } diff --git a/doc/frameworks/broker/events-connector.bro b/doc/frameworks/broker/events-connector.bro index 1ad458c245..19a617c9cd 100644 --- a/doc/frameworks/broker/events-connector.bro +++ b/doc/frameworks/broker/events-connector.bro @@ -1,30 +1,30 @@ const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -redef BrokerComm::endpoint_name = "connector"; +redef Broker::endpoint_name = "connector"; global my_event: event(msg: string, c: count); global my_auto_event: event(msg: string, c: count); event bro_init() { - BrokerComm::enable(); - BrokerComm::connect("127.0.0.1", broker_port, 1sec); - BrokerComm::auto_event("bro/event/my_auto_event", my_auto_event); + Broker::enable(); + Broker::connect("127.0.0.1", broker_port, 1sec); + Broker::auto_event("bro/event/my_auto_event", my_auto_event); } -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { - print "BrokerComm::outgoing_connection_established", + print "Broker::outgoing_connection_established", peer_address, peer_port, peer_name; - BrokerComm::event("bro/event/my_event", BrokerComm::event_args(my_event, "hi", 0)); + Broker::event("bro/event/my_event", Broker::event_args(my_event, "hi", 0)); event my_auto_event("stuff", 88); - BrokerComm::event("bro/event/my_event", BrokerComm::event_args(my_event, "...", 1)); + Broker::event("bro/event/my_event", Broker::event_args(my_event, "...", 1)); event my_auto_event("more stuff", 51); - BrokerComm::event("bro/event/my_event", BrokerComm::event_args(my_event, "bye", 2)); + Broker::event("bro/event/my_event", Broker::event_args(my_event, "bye", 2)); } -event BrokerComm::outgoing_connection_broken(peer_address: string, +event Broker::outgoing_connection_broken(peer_address: string, peer_port: port) { terminate(); diff --git a/doc/frameworks/broker/events-listener.bro b/doc/frameworks/broker/events-listener.bro index dc18795903..b803e646ec 100644 --- a/doc/frameworks/broker/events-listener.bro +++ b/doc/frameworks/broker/events-listener.bro @@ -1,20 +1,20 @@ const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -redef BrokerComm::endpoint_name = "listener"; +redef Broker::endpoint_name = "listener"; global msg_count = 0; global my_event: event(msg: string, c: count); global my_auto_event: event(msg: string, c: count); event bro_init() { - BrokerComm::enable(); - BrokerComm::subscribe_to_events("bro/event/"); - BrokerComm::listen(broker_port, "127.0.0.1"); + Broker::enable(); + Broker::subscribe_to_events("bro/event/"); + Broker::listen(broker_port, "127.0.0.1"); } -event BrokerComm::incoming_connection_established(peer_name: string) +event Broker::incoming_connection_established(peer_name: string) { - print "BrokerComm::incoming_connection_established", peer_name; + print "Broker::incoming_connection_established", peer_name; } event my_event(msg: string, c: count) diff --git a/doc/frameworks/broker/logs-connector.bro b/doc/frameworks/broker/logs-connector.bro index 6089419cab..9c5df335b9 100644 --- a/doc/frameworks/broker/logs-connector.bro +++ b/doc/frameworks/broker/logs-connector.bro @@ -2,16 +2,16 @@ const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -redef BrokerComm::endpoint_name = "connector"; +redef Broker::endpoint_name = "connector"; redef Log::enable_local_logging = F; redef Log::enable_remote_logging = F; global n = 0; event bro_init() { - BrokerComm::enable(); - BrokerComm::enable_remote_logs(Test::LOG); - BrokerComm::connect("127.0.0.1", broker_port, 1sec); + Broker::enable(); + Broker::enable_remote_logs(Test::LOG); + Broker::connect("127.0.0.1", broker_port, 1sec); } event do_write() @@ -24,16 +24,16 @@ event do_write() event do_write(); } -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { - print "BrokerComm::outgoing_connection_established", + print "Broker::outgoing_connection_established", peer_address, peer_port, peer_name; event do_write(); } -event BrokerComm::outgoing_connection_broken(peer_address: string, +event Broker::outgoing_connection_broken(peer_address: string, peer_port: port) { terminate(); diff --git a/doc/frameworks/broker/logs-listener.bro b/doc/frameworks/broker/logs-listener.bro index 5c807f08b7..34d475512a 100644 --- a/doc/frameworks/broker/logs-listener.bro +++ b/doc/frameworks/broker/logs-listener.bro @@ -2,18 +2,18 @@ const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -redef BrokerComm::endpoint_name = "listener"; +redef Broker::endpoint_name = "listener"; event bro_init() { - BrokerComm::enable(); - BrokerComm::subscribe_to_logs("bro/log/Test::LOG"); - BrokerComm::listen(broker_port, "127.0.0.1"); + Broker::enable(); + Broker::subscribe_to_logs("bro/log/Test::LOG"); + Broker::listen(broker_port, "127.0.0.1"); } -event BrokerComm::incoming_connection_established(peer_name: string) +event Broker::incoming_connection_established(peer_name: string) { - print "BrokerComm::incoming_connection_established", peer_name; + print "Broker::incoming_connection_established", peer_name; } event Test::log_test(rec: Test::Info) diff --git a/doc/frameworks/broker/printing-connector.bro b/doc/frameworks/broker/printing-connector.bro index 2a504ffba0..0ab14d926b 100644 --- a/doc/frameworks/broker/printing-connector.bro +++ b/doc/frameworks/broker/printing-connector.bro @@ -1,25 +1,25 @@ const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -redef BrokerComm::endpoint_name = "connector"; +redef Broker::endpoint_name = "connector"; event bro_init() { - BrokerComm::enable(); - BrokerComm::connect("127.0.0.1", broker_port, 1sec); + Broker::enable(); + Broker::connect("127.0.0.1", broker_port, 1sec); } -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { - print "BrokerComm::outgoing_connection_established", + print "Broker::outgoing_connection_established", peer_address, peer_port, peer_name; - BrokerComm::print("bro/print/hi", "hello"); - BrokerComm::print("bro/print/stuff", "..."); - BrokerComm::print("bro/print/bye", "goodbye"); + Broker::print("bro/print/hi", "hello"); + Broker::print("bro/print/stuff", "..."); + Broker::print("bro/print/bye", "goodbye"); } -event BrokerComm::outgoing_connection_broken(peer_address: string, +event Broker::outgoing_connection_broken(peer_address: string, peer_port: port) { terminate(); diff --git a/doc/frameworks/broker/printing-listener.bro b/doc/frameworks/broker/printing-listener.bro index f55c5b9bad..4630a7e6d7 100644 --- a/doc/frameworks/broker/printing-listener.bro +++ b/doc/frameworks/broker/printing-listener.bro @@ -1,21 +1,21 @@ const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -redef BrokerComm::endpoint_name = "listener"; +redef Broker::endpoint_name = "listener"; global msg_count = 0; event bro_init() { - BrokerComm::enable(); - BrokerComm::subscribe_to_prints("bro/print/"); - BrokerComm::listen(broker_port, "127.0.0.1"); + Broker::enable(); + Broker::subscribe_to_prints("bro/print/"); + Broker::listen(broker_port, "127.0.0.1"); } -event BrokerComm::incoming_connection_established(peer_name: string) +event Broker::incoming_connection_established(peer_name: string) { - print "BrokerComm::incoming_connection_established", peer_name; + print "Broker::incoming_connection_established", peer_name; } -event BrokerComm::print_handler(msg: string) +event Broker::print_handler(msg: string) { ++msg_count; print "got print message", msg; diff --git a/doc/frameworks/broker/stores-connector.bro b/doc/frameworks/broker/stores-connector.bro index 5db8657a68..b9e9f782fb 100644 --- a/doc/frameworks/broker/stores-connector.bro +++ b/doc/frameworks/broker/stores-connector.bro @@ -3,38 +3,38 @@ redef exit_only_after_terminate = T; global h: opaque of BrokerStore::Handle; -function dv(d: BrokerComm::Data): BrokerComm::DataVector +function dv(d: Broker::Data): Broker::DataVector { - local rval: BrokerComm::DataVector; + local rval: Broker::DataVector; rval[0] = d; return rval; } global ready: event(); -event BrokerComm::outgoing_connection_broken(peer_address: string, +event Broker::outgoing_connection_broken(peer_address: string, peer_port: port) { terminate(); } -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { local myset: set[string] = {"a", "b", "c"}; local myvec: vector of string = {"alpha", "beta", "gamma"}; h = BrokerStore::create_master("mystore"); - BrokerStore::insert(h, BrokerComm::data("one"), BrokerComm::data(110)); - BrokerStore::insert(h, BrokerComm::data("two"), BrokerComm::data(223)); - BrokerStore::insert(h, BrokerComm::data("myset"), BrokerComm::data(myset)); - BrokerStore::insert(h, BrokerComm::data("myvec"), BrokerComm::data(myvec)); - BrokerStore::increment(h, BrokerComm::data("one")); - BrokerStore::decrement(h, BrokerComm::data("two")); - BrokerStore::add_to_set(h, BrokerComm::data("myset"), BrokerComm::data("d")); - BrokerStore::remove_from_set(h, BrokerComm::data("myset"), BrokerComm::data("b")); - BrokerStore::push_left(h, BrokerComm::data("myvec"), dv(BrokerComm::data("delta"))); - BrokerStore::push_right(h, BrokerComm::data("myvec"), dv(BrokerComm::data("omega"))); + BrokerStore::insert(h, Broker::data("one"), Broker::data(110)); + BrokerStore::insert(h, Broker::data("two"), Broker::data(223)); + BrokerStore::insert(h, Broker::data("myset"), Broker::data(myset)); + BrokerStore::insert(h, Broker::data("myvec"), Broker::data(myvec)); + BrokerStore::increment(h, Broker::data("one")); + BrokerStore::decrement(h, Broker::data("two")); + BrokerStore::add_to_set(h, Broker::data("myset"), Broker::data("d")); + BrokerStore::remove_from_set(h, Broker::data("myset"), Broker::data("b")); + BrokerStore::push_left(h, Broker::data("myvec"), dv(Broker::data("delta"))); + BrokerStore::push_right(h, Broker::data("myvec"), dv(Broker::data("omega"))); when ( local res = BrokerStore::size(h) ) { @@ -47,7 +47,7 @@ event BrokerComm::outgoing_connection_established(peer_address: string, event bro_init() { - BrokerComm::enable(); - BrokerComm::connect("127.0.0.1", broker_port, 1secs); - BrokerComm::auto_event("bro/event/ready", ready); + Broker::enable(); + Broker::connect("127.0.0.1", broker_port, 1secs); + Broker::auto_event("bro/event/ready", ready); } diff --git a/doc/frameworks/broker/stores-listener.bro b/doc/frameworks/broker/stores-listener.bro index 454e41a8c2..b0dc720868 100644 --- a/doc/frameworks/broker/stores-listener.bro +++ b/doc/frameworks/broker/stores-listener.bro @@ -7,7 +7,7 @@ global key_count = 0; function do_lookup(key: string) { - when ( local res = BrokerStore::lookup(h, BrokerComm::data(key)) ) + when ( local res = BrokerStore::lookup(h, Broker::data(key)) ) { ++key_count; print "lookup", key, res; @@ -26,10 +26,10 @@ event ready() when ( local res = BrokerStore::keys(h) ) { print "clone keys", res; - do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 0))); - do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 1))); - do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 2))); - do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 3))); + do_lookup(Broker::refine_to_string(Broker::vector_lookup(res$result, 0))); + do_lookup(Broker::refine_to_string(Broker::vector_lookup(res$result, 1))); + do_lookup(Broker::refine_to_string(Broker::vector_lookup(res$result, 2))); + do_lookup(Broker::refine_to_string(Broker::vector_lookup(res$result, 3))); } timeout 10sec { print "timeout"; } @@ -37,7 +37,7 @@ event ready() event bro_init() { - BrokerComm::enable(); - BrokerComm::subscribe_to_events("bro/event/ready"); - BrokerComm::listen(broker_port, "127.0.0.1"); + Broker::enable(); + Broker::subscribe_to_events("bro/event/ready"); + Broker::listen(broker_port, "127.0.0.1"); } diff --git a/doc/frameworks/broker/testlog.bro b/doc/frameworks/broker/testlog.bro index 506d359bb7..0099671e6d 100644 --- a/doc/frameworks/broker/testlog.bro +++ b/doc/frameworks/broker/testlog.bro @@ -13,6 +13,6 @@ export { event bro_init() &priority=5 { - BrokerComm::enable(); + Broker::enable(); Log::create_stream(Test::LOG, [$columns=Test::Info, $ev=log_test, $path="test"]); } diff --git a/scripts/base/frameworks/broker/main.bro b/scripts/base/frameworks/broker/main.bro index e8b57d57d9..ac388b78a1 100644 --- a/scripts/base/frameworks/broker/main.bro +++ b/scripts/base/frameworks/broker/main.bro @@ -1,11 +1,11 @@ ##! Various data structure definitions for use with Bro's communication system. -module BrokerComm; +module Broker; export { ## A name used to identify this endpoint to peers. - ## .. bro:see:: BrokerComm::connect BrokerComm::listen + ## .. bro:see:: Broker::connect Broker::listen const endpoint_name = "" &redef; ## Change communication behavior. @@ -32,11 +32,11 @@ export { ## Opaque communication data. type Data: record { - d: opaque of BrokerComm::Data &optional; + d: opaque of Broker::Data &optional; }; ## Opaque communication data. - type DataVector: vector of BrokerComm::Data; + type DataVector: vector of Broker::Data; ## Opaque event communication data. type EventArgs: record { @@ -49,8 +49,8 @@ export { ## Opaque communication data used as a convenient way to wrap key-value ## pairs that comprise table entries. type TableItem : record { - key: BrokerComm::Data; - val: BrokerComm::Data; + key: Broker::Data; + val: Broker::Data; }; } @@ -80,7 +80,7 @@ export { ## The result of the query. Certain queries may use a particular ## data type (e.g. querying store size always returns a count, but ## a lookup may return various data types). - result: BrokerComm::Data; + result: Broker::Data; }; ## Options to tune the SQLite storage backend. diff --git a/scripts/base/frameworks/netcontrol/plugins/acld.bro b/scripts/base/frameworks/netcontrol/plugins/acld.bro index 76661bc857..13802f2e21 100644 --- a/scripts/base/frameworks/netcontrol/plugins/acld.bro +++ b/scripts/base/frameworks/netcontrol/plugins/acld.bro @@ -227,7 +227,7 @@ function acld_add_rule_fun(p: PluginState, r: Rule) : bool if ( ar$command == "" ) return F; - BrokerComm::event(p$acld_config$acld_topic, BrokerComm::event_args(acld_add_rule, p$acld_id, r, ar)); + Broker::event(p$acld_config$acld_topic, Broker::event_args(acld_add_rule, p$acld_id, r, ar)); return T; } @@ -242,18 +242,18 @@ function acld_remove_rule_fun(p: PluginState, r: Rule) : bool else return F; - BrokerComm::event(p$acld_config$acld_topic, BrokerComm::event_args(acld_remove_rule, p$acld_id, r, ar)); + Broker::event(p$acld_config$acld_topic, Broker::event_args(acld_remove_rule, p$acld_id, r, ar)); return T; } function acld_init(p: PluginState) { - BrokerComm::enable(); - BrokerComm::connect(cat(p$acld_config$acld_host), p$acld_config$acld_port, 1sec); - BrokerComm::subscribe_to_events(p$acld_config$acld_topic); + Broker::enable(); + Broker::connect(cat(p$acld_config$acld_host), p$acld_config$acld_port, 1sec); + Broker::subscribe_to_events(p$acld_config$acld_topic); } -event BrokerComm::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { if ( [peer_port, peer_address] !in netcontrol_acld_peers ) # ok, this one was none of ours... diff --git a/scripts/base/frameworks/netcontrol/plugins/broker.bro b/scripts/base/frameworks/netcontrol/plugins/broker.bro index 619b6b607b..2af3724db7 100644 --- a/scripts/base/frameworks/netcontrol/plugins/broker.bro +++ b/scripts/base/frameworks/netcontrol/plugins/broker.bro @@ -96,24 +96,24 @@ function broker_name(p: PluginState) : string function broker_add_rule_fun(p: PluginState, r: Rule) : bool { - BrokerComm::event(p$broker_topic, BrokerComm::event_args(broker_add_rule, p$broker_id, r)); + Broker::event(p$broker_topic, Broker::event_args(broker_add_rule, p$broker_id, r)); return T; } function broker_remove_rule_fun(p: PluginState, r: Rule) : bool { - BrokerComm::event(p$broker_topic, BrokerComm::event_args(broker_remove_rule, p$broker_id, r)); + Broker::event(p$broker_topic, Broker::event_args(broker_remove_rule, p$broker_id, r)); return T; } function broker_init(p: PluginState) { - BrokerComm::enable(); - BrokerComm::connect(cat(p$broker_host), p$broker_port, 1sec); - BrokerComm::subscribe_to_events(p$broker_topic); + Broker::enable(); + Broker::connect(cat(p$broker_host), p$broker_port, 1sec); + Broker::subscribe_to_events(p$broker_topic); } -event BrokerComm::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { if ( [peer_port, peer_address] !in netcontrol_broker_peers ) return; diff --git a/scripts/base/frameworks/openflow/plugins/broker.bro b/scripts/base/frameworks/openflow/plugins/broker.bro index d6cf52a92c..93a627a8f4 100644 --- a/scripts/base/frameworks/openflow/plugins/broker.bro +++ b/scripts/base/frameworks/openflow/plugins/broker.bro @@ -47,26 +47,26 @@ function broker_describe(state: ControllerState): string function broker_flow_mod_fun(state: ControllerState, match: ofp_match, flow_mod: OpenFlow::ofp_flow_mod): bool { - BrokerComm::event(state$broker_topic, BrokerComm::event_args(broker_flow_mod, state$_name, state$broker_dpid, match, flow_mod)); + Broker::event(state$broker_topic, Broker::event_args(broker_flow_mod, state$_name, state$broker_dpid, match, flow_mod)); return T; } function broker_flow_clear_fun(state: OpenFlow::ControllerState): bool { - BrokerComm::event(state$broker_topic, BrokerComm::event_args(broker_flow_clear, state$_name, state$broker_dpid)); + Broker::event(state$broker_topic, Broker::event_args(broker_flow_clear, state$_name, state$broker_dpid)); return T; } function broker_init(state: OpenFlow::ControllerState) { - BrokerComm::enable(); - BrokerComm::connect(cat(state$broker_host), state$broker_port, 1sec); - BrokerComm::subscribe_to_events(state$broker_topic); # openflow success and failure events are directly sent back via the other plugin via broker. + Broker::enable(); + Broker::connect(cat(state$broker_host), state$broker_port, 1sec); + Broker::subscribe_to_events(state$broker_topic); # openflow success and failure events are directly sent back via the other plugin via broker. } -event BrokerComm::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { if ( [peer_port, peer_address] !in broker_peers ) # ok, this one was none of ours... diff --git a/scripts/base/init-default.bro b/scripts/base/init-default.bro index 609ed7200c..9acc59479c 100644 --- a/scripts/base/init-default.bro +++ b/scripts/base/init-default.bro @@ -37,7 +37,7 @@ @load base/frameworks/reporter @load base/frameworks/sumstats @load base/frameworks/tunnels -@ifdef ( BrokerComm::enable ) +@ifdef ( Broker::enable ) @load base/frameworks/openflow @load base/frameworks/netcontrol @endif diff --git a/src/broker/Data.cc b/src/broker/Data.cc index 8f66427bb5..fe3f271c49 100644 --- a/src/broker/Data.cc +++ b/src/broker/Data.cc @@ -539,7 +539,7 @@ broker::util::optional bro_broker::val_to_data(Val* v) return {rval}; } default: - reporter->Error("unsupported BrokerComm::Data type: %s", + reporter->Error("unsupported Broker::Data type: %s", type_name(v->Type()->Tag())); break; } @@ -549,7 +549,7 @@ broker::util::optional bro_broker::val_to_data(Val* v) RecordVal* bro_broker::make_data_val(Val* v) { - auto rval = new RecordVal(BifType::Record::BrokerComm::Data); + auto rval = new RecordVal(BifType::Record::Broker::Data); auto data = val_to_data(v); if ( data ) @@ -560,7 +560,7 @@ RecordVal* bro_broker::make_data_val(Val* v) RecordVal* bro_broker::make_data_val(broker::data d) { - auto rval = new RecordVal(BifType::Record::BrokerComm::Data); + auto rval = new RecordVal(BifType::Record::Broker::Data); rval->Assign(0, new DataVal(move(d))); return rval; } @@ -570,92 +570,92 @@ struct data_type_getter { result_type operator()(bool a) { - return new EnumVal(BifEnum::BrokerComm::BOOL, - BifType::Enum::BrokerComm::DataType); + return new EnumVal(BifEnum::Broker::BOOL, + BifType::Enum::Broker::DataType); } result_type operator()(uint64_t a) { - return new EnumVal(BifEnum::BrokerComm::COUNT, - BifType::Enum::BrokerComm::DataType); + return new EnumVal(BifEnum::Broker::COUNT, + BifType::Enum::Broker::DataType); } result_type operator()(int64_t a) { - return new EnumVal(BifEnum::BrokerComm::INT, - BifType::Enum::BrokerComm::DataType); + return new EnumVal(BifEnum::Broker::INT, + BifType::Enum::Broker::DataType); } result_type operator()(double a) { - return new EnumVal(BifEnum::BrokerComm::DOUBLE, - BifType::Enum::BrokerComm::DataType); + return new EnumVal(BifEnum::Broker::DOUBLE, + BifType::Enum::Broker::DataType); } result_type operator()(const std::string& a) { - return new EnumVal(BifEnum::BrokerComm::STRING, - BifType::Enum::BrokerComm::DataType); + return new EnumVal(BifEnum::Broker::STRING, + BifType::Enum::Broker::DataType); } result_type operator()(const broker::address& a) { - return new EnumVal(BifEnum::BrokerComm::ADDR, - BifType::Enum::BrokerComm::DataType); + return new EnumVal(BifEnum::Broker::ADDR, + BifType::Enum::Broker::DataType); } result_type operator()(const broker::subnet& a) { - return new EnumVal(BifEnum::BrokerComm::SUBNET, - BifType::Enum::BrokerComm::DataType); + return new EnumVal(BifEnum::Broker::SUBNET, + BifType::Enum::Broker::DataType); } result_type operator()(const broker::port& a) { - return new EnumVal(BifEnum::BrokerComm::PORT, - BifType::Enum::BrokerComm::DataType); + return new EnumVal(BifEnum::Broker::PORT, + BifType::Enum::Broker::DataType); } result_type operator()(const broker::time_point& a) { - return new EnumVal(BifEnum::BrokerComm::TIME, - BifType::Enum::BrokerComm::DataType); + return new EnumVal(BifEnum::Broker::TIME, + BifType::Enum::Broker::DataType); } result_type operator()(const broker::time_duration& a) { - return new EnumVal(BifEnum::BrokerComm::INTERVAL, - BifType::Enum::BrokerComm::DataType); + return new EnumVal(BifEnum::Broker::INTERVAL, + BifType::Enum::Broker::DataType); } result_type operator()(const broker::enum_value& a) { - return new EnumVal(BifEnum::BrokerComm::ENUM, - BifType::Enum::BrokerComm::DataType); + return new EnumVal(BifEnum::Broker::ENUM, + BifType::Enum::Broker::DataType); } result_type operator()(const broker::set& a) { - return new EnumVal(BifEnum::BrokerComm::SET, - BifType::Enum::BrokerComm::DataType); + return new EnumVal(BifEnum::Broker::SET, + BifType::Enum::Broker::DataType); } result_type operator()(const broker::table& a) { - return new EnumVal(BifEnum::BrokerComm::TABLE, - BifType::Enum::BrokerComm::DataType); + return new EnumVal(BifEnum::Broker::TABLE, + BifType::Enum::Broker::DataType); } result_type operator()(const broker::vector& a) { - return new EnumVal(BifEnum::BrokerComm::VECTOR, - BifType::Enum::BrokerComm::DataType); + return new EnumVal(BifEnum::Broker::VECTOR, + BifType::Enum::Broker::DataType); } result_type operator()(const broker::record& a) { - return new EnumVal(BifEnum::BrokerComm::RECORD, - BifType::Enum::BrokerComm::DataType); + return new EnumVal(BifEnum::Broker::RECORD, + BifType::Enum::Broker::DataType); } }; @@ -670,7 +670,7 @@ broker::data& bro_broker::opaque_field_to_data(RecordVal* v, Frame* f) if ( ! d ) reporter->RuntimeError(f->GetCall()->GetLocationInfo(), - "BrokerComm::Data's opaque field is not set"); + "Broker::Data's opaque field is not set"); return static_cast(d)->data; } diff --git a/src/broker/Data.h b/src/broker/Data.h index 84495056be..f212979853 100644 --- a/src/broker/Data.h +++ b/src/broker/Data.h @@ -21,25 +21,25 @@ extern OpaqueType* opaque_of_record_iterator; TransportProto to_bro_port_proto(broker::port::protocol tp); /** - * Create a BrokerComm::Data value from a Bro value. + * Create a Broker::Data value from a Bro value. * @param v the Bro value to convert to a Broker data value. - * @return a BrokerComm::Data value, where the optional field is set if the conversion + * @return a Broker::Data value, where the optional field is set if the conversion * was possible, else it is unset. */ RecordVal* make_data_val(Val* v); /** - * Create a BrokerComm::Data value from a Broker data value. + * Create a Broker::Data value from a Broker data value. * @param d the Broker value to wrap in an opaque type. - * @return a BrokerComm::Data value that wraps the Broker value. + * @return a Broker::Data value that wraps the Broker value. */ RecordVal* make_data_val(broker::data d); /** - * Get the type of Broker data that BrokerComm::Data wraps. - * @param v a BrokerComm::Data value. + * Get the type of Broker data that Broker::Data wraps. + * @param v a Broker::Data value. * @param frame used to get location info upon error. - * @return a BrokerComm::DataType value. + * @return a Broker::DataType value. */ EnumVal* get_data_type(RecordVal* v, Frame* frame); @@ -141,8 +141,8 @@ struct type_name_getter { }; /** - * Retrieve Broker data value associated with a BrokerComm::Data Bro value. - * @param v a BrokerComm::Data value. + * Retrieve Broker data value associated with a Broker::Data Bro value. + * @param v a Broker::Data value. * @param f used to get location information on error. * @return a reference to the wrapped Broker data value. A runtime interpreter * exception is thrown if the the optional opaque value of \a v is not set. @@ -183,9 +183,9 @@ inline T& require_data_type(RecordVal* v, TypeTag tag, Frame* f) } /** - * Convert a BrokerComm::Data Bro value to a Bro value of a given type. + * Convert a Broker::Data Bro value to a Bro value of a given type. * @tparam a type that a Broker data variant may contain. - * @param v a BrokerComm::Data value. + * @param v a Broker::Data value. * @param tag a Bro type to convert to. * @param f used to get location information on error. * A runtime interpret exception is thrown if trying to access a type which diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index 06ece6d6c1..62007c8ebb 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -77,20 +77,20 @@ bool bro_broker::Manager::Enable(Val* broker_endpoint_flags) if ( endpoint != nullptr ) return true; - auto send_flags_type = internal_type("BrokerComm::SendFlags")->AsRecordType(); + auto send_flags_type = internal_type("Broker::SendFlags")->AsRecordType(); send_flags_self_idx = require_field(send_flags_type, "self"); send_flags_peers_idx = require_field(send_flags_type, "peers"); send_flags_unsolicited_idx = require_field(send_flags_type, "unsolicited"); log_id_type = internal_type("Log::ID")->AsEnumType(); - bro_broker::opaque_of_data_type = new OpaqueType("BrokerComm::Data"); - bro_broker::opaque_of_set_iterator = new OpaqueType("BrokerComm::SetIterator"); - bro_broker::opaque_of_table_iterator = new OpaqueType("BrokerComm::TableIterator"); - bro_broker::opaque_of_vector_iterator = new OpaqueType("BrokerComm::VectorIterator"); - bro_broker::opaque_of_record_iterator = new OpaqueType("BrokerComm::RecordIterator"); + bro_broker::opaque_of_data_type = new OpaqueType("Broker::Data"); + bro_broker::opaque_of_set_iterator = new OpaqueType("Broker::SetIterator"); + bro_broker::opaque_of_table_iterator = new OpaqueType("Broker::TableIterator"); + bro_broker::opaque_of_vector_iterator = new OpaqueType("Broker::VectorIterator"); + bro_broker::opaque_of_record_iterator = new OpaqueType("Broker::RecordIterator"); bro_broker::opaque_of_store_handle = new OpaqueType("BrokerStore::Handle"); - vector_of_data_type = new VectorType(internal_type("BrokerComm::Data")->Ref()); + vector_of_data_type = new VectorType(internal_type("Broker::Data")->Ref()); auto res = broker::init(); @@ -110,7 +110,7 @@ bool bro_broker::Manager::Enable(Val* broker_endpoint_flags) } const char* name; - auto name_from_script = internal_val("BrokerComm::endpoint_name")->AsString(); + auto name_from_script = internal_val("Broker::endpoint_name")->AsString(); if ( name_from_script->Len() ) name = name_from_script->CheckString(); @@ -290,7 +290,7 @@ bool bro_broker::Manager::AutoEvent(string topic, Val* event, Val* flags) if ( event->Type()->Tag() != TYPE_FUNC ) { - reporter->Error("BrokerComm::auto_event must operate on an event"); + reporter->Error("Broker::auto_event must operate on an event"); return false; } @@ -298,7 +298,7 @@ bool bro_broker::Manager::AutoEvent(string topic, Val* event, Val* flags) if ( event_val->Flavor() != FUNC_FLAVOR_EVENT ) { - reporter->Error("BrokerComm::auto_event must operate on an event"); + reporter->Error("Broker::auto_event must operate on an event"); return false; } @@ -306,7 +306,7 @@ bool bro_broker::Manager::AutoEvent(string topic, Val* event, Val* flags) if ( ! handler ) { - reporter->Error("BrokerComm::auto_event failed to lookup event '%s'", + reporter->Error("Broker::auto_event failed to lookup event '%s'", event_val->Name()); return false; } @@ -322,7 +322,7 @@ bool bro_broker::Manager::AutoEventStop(const string& topic, Val* event) if ( event->Type()->Tag() != TYPE_FUNC ) { - reporter->Error("BrokerComm::auto_event_stop must operate on an event"); + reporter->Error("Broker::auto_event_stop must operate on an event"); return false; } @@ -330,7 +330,7 @@ bool bro_broker::Manager::AutoEventStop(const string& topic, Val* event) if ( event_val->Flavor() != FUNC_FLAVOR_EVENT ) { - reporter->Error("BrokerComm::auto_event_stop must operate on an event"); + reporter->Error("Broker::auto_event_stop must operate on an event"); return false; } @@ -338,7 +338,7 @@ bool bro_broker::Manager::AutoEventStop(const string& topic, Val* event) if ( ! handler ) { - reporter->Error("BrokerComm::auto_event_stop failed to lookup event '%s'", + reporter->Error("Broker::auto_event_stop failed to lookup event '%s'", event_val->Name()); return false; } @@ -353,7 +353,7 @@ RecordVal* bro_broker::Manager::MakeEventArgs(val_list* args) if ( ! Enabled() ) return nullptr; - auto rval = new RecordVal(BifType::Record::BrokerComm::EventArgs); + auto rval = new RecordVal(BifType::Record::Broker::EventArgs); auto arg_vec = new VectorVal(vector_of_data_type); rval->Assign(1, arg_vec); Func* func = 0; @@ -368,7 +368,7 @@ RecordVal* bro_broker::Manager::MakeEventArgs(val_list* args) if ( arg_val->Type()->Tag() != TYPE_FUNC ) { - reporter->Error("1st param of BrokerComm::event_args must be event"); + reporter->Error("1st param of Broker::event_args must be event"); return rval; } @@ -376,7 +376,7 @@ RecordVal* bro_broker::Manager::MakeEventArgs(val_list* args) if ( func->Flavor() != FUNC_FLAVOR_EVENT ) { - reporter->Error("1st param of BrokerComm::event_args must be event"); + reporter->Error("1st param of Broker::event_args must be event"); return rval; } @@ -384,7 +384,7 @@ RecordVal* bro_broker::Manager::MakeEventArgs(val_list* args) if ( num_args != args->length() - 1 ) { - reporter->Error("bad # of BrokerComm::event_args: got %d, expect %d", + reporter->Error("bad # of Broker::event_args: got %d, expect %d", args->length(), num_args + 1); return rval; } @@ -398,7 +398,7 @@ RecordVal* bro_broker::Manager::MakeEventArgs(val_list* args) if ( ! same_type((*args)[i]->Type(), expected_type) ) { rval->Assign(0, 0); - reporter->Error("BrokerComm::event_args param %d type mismatch", i); + reporter->Error("Broker::event_args param %d type mismatch", i); return rval; } @@ -408,7 +408,7 @@ RecordVal* bro_broker::Manager::MakeEventArgs(val_list* args) { Unref(data_val); rval->Assign(0, 0); - reporter->Error("BrokerComm::event_args unsupported event/params"); + reporter->Error("Broker::event_args unsupported event/params"); return rval; } @@ -584,7 +584,7 @@ struct response_converter { case broker::store::query::tag::lookup: // A boolean result means the key doesn't exist (if it did, then // the result would contain the broker::data value, not a bool). - return new RecordVal(BifType::Record::BrokerComm::Data); + return new RecordVal(BifType::Record::Broker::Data); default: return bro_broker::make_data_val(broker::data{d}); } @@ -639,36 +639,36 @@ void bro_broker::Manager::Process() { switch ( u.status ) { case broker::outgoing_connection_status::tag::established: - if ( BrokerComm::outgoing_connection_established ) + if ( Broker::outgoing_connection_established ) { val_list* vl = new val_list; vl->append(new StringVal(u.relation.remote_tuple().first)); vl->append(new PortVal(u.relation.remote_tuple().second, TRANSPORT_TCP)); vl->append(new StringVal(u.peer_name)); - mgr.QueueEvent(BrokerComm::outgoing_connection_established, vl); + mgr.QueueEvent(Broker::outgoing_connection_established, vl); } break; case broker::outgoing_connection_status::tag::disconnected: - if ( BrokerComm::outgoing_connection_broken ) + if ( Broker::outgoing_connection_broken ) { val_list* vl = new val_list; vl->append(new StringVal(u.relation.remote_tuple().first)); vl->append(new PortVal(u.relation.remote_tuple().second, TRANSPORT_TCP)); - mgr.QueueEvent(BrokerComm::outgoing_connection_broken, vl); + mgr.QueueEvent(Broker::outgoing_connection_broken, vl); } break; case broker::outgoing_connection_status::tag::incompatible: - if ( BrokerComm::outgoing_connection_incompatible ) + if ( Broker::outgoing_connection_incompatible ) { val_list* vl = new val_list; vl->append(new StringVal(u.relation.remote_tuple().first)); vl->append(new PortVal(u.relation.remote_tuple().second, TRANSPORT_TCP)); - mgr.QueueEvent(BrokerComm::outgoing_connection_incompatible, vl); + mgr.QueueEvent(Broker::outgoing_connection_incompatible, vl); } break; @@ -684,20 +684,20 @@ void bro_broker::Manager::Process() { switch ( u.status ) { case broker::incoming_connection_status::tag::established: - if ( BrokerComm::incoming_connection_established ) + if ( Broker::incoming_connection_established ) { val_list* vl = new val_list; vl->append(new StringVal(u.peer_name)); - mgr.QueueEvent(BrokerComm::incoming_connection_established, vl); + mgr.QueueEvent(Broker::incoming_connection_established, vl); } break; case broker::incoming_connection_status::tag::disconnected: - if ( BrokerComm::incoming_connection_broken ) + if ( Broker::incoming_connection_broken ) { val_list* vl = new val_list; vl->append(new StringVal(u.peer_name)); - mgr.QueueEvent(BrokerComm::incoming_connection_broken, vl); + mgr.QueueEvent(Broker::incoming_connection_broken, vl); } break; @@ -718,7 +718,7 @@ void bro_broker::Manager::Process() ps.second.received += print_messages.size(); - if ( ! BrokerComm::print_handler ) + if ( ! Broker::print_handler ) continue; for ( auto& pm : print_messages ) @@ -741,7 +741,7 @@ void bro_broker::Manager::Process() val_list* vl = new val_list; vl->append(new StringVal(move(*msg))); - mgr.QueueEvent(BrokerComm::print_handler, vl); + mgr.QueueEvent(Broker::print_handler, vl); } } diff --git a/src/broker/Manager.h b/src/broker/Manager.h index 9e1ac7a70b..9fb7b9e328 100644 --- a/src/broker/Manager.h +++ b/src/broker/Manager.h @@ -63,7 +63,7 @@ public: /** * Enable use of communication. * @param flags used to tune the local Broker endpoint's behavior. - * See the BrokerComm::EndpointFlags record type. + * See the Broker::EndpointFlags record type. * @return true if communication is successfully initialized. */ bool Enable(Val* flags); @@ -122,7 +122,7 @@ public: * of this topic name. * @param msg the string to send to peers. * @param flags tune the behavior of how the message is send. - * See the BrokerComm::SendFlags record type. + * See the Broker::SendFlags record type. * @return true if the message is sent successfully. */ bool Print(std::string topic, std::string msg, Val* flags); @@ -135,7 +135,7 @@ public: * @param msg the event to send to peers, which is the name of the event * as a string followed by all of its arguments. * @param flags tune the behavior of how the message is send. - * See the BrokerComm::SendFlags record type. + * See the Broker::SendFlags record type. * @return true if the message is sent successfully. */ bool Event(std::string topic, broker::message msg, int flags); @@ -146,9 +146,9 @@ public: * Peers advertise interest by registering a subscription to some prefix * of this topic name. * @param args the event and its arguments to send to peers. See the - * BrokerComm::EventArgs record type. + * Broker::EventArgs record type. * @param flags tune the behavior of how the message is send. - * See the BrokerComm::SendFlags record type. + * See the Broker::SendFlags record type. * @return true if the message is sent successfully. */ bool Event(std::string topic, RecordVal* args, Val* flags); @@ -160,7 +160,7 @@ public: * @param columns the data which comprises the log entry. * @param info the record type corresponding to the log's columns. * @param flags tune the behavior of how the message is send. - * See the BrokerComm::SendFlags record type. + * See the Broker::SendFlags record type. * @return true if the message is sent successfully. */ bool Log(EnumVal* stream_id, RecordVal* columns, RecordType* info, @@ -174,7 +174,7 @@ public: * of this topic name. * @param event a Bro event value. * @param flags tune the behavior of how the message is send. - * See the BrokerComm::SendFlags record type. + * See the Broker::SendFlags record type. * @return true if automatic event sending is now enabled. */ bool AutoEvent(std::string topic, Val* event, Val* flags); @@ -320,7 +320,7 @@ public: Stats ConsumeStatistics(); /** - * Convert BrokerComm::SendFlags to int flags for use with broker::send(). + * Convert Broker::SendFlags to int flags for use with broker::send(). */ static int send_flags_to_int(Val* flags); @@ -335,7 +335,7 @@ private: void Process() override; const char* Tag() override - { return "BrokerComm::Manager"; } + { return "Broker::Manager"; } broker::endpoint& Endpoint() { return *endpoint; } diff --git a/src/broker/Store.h b/src/broker/Store.h index 5823e0c3f8..6f31381768 100644 --- a/src/broker/Store.h +++ b/src/broker/Store.h @@ -53,7 +53,7 @@ inline RecordVal* query_result() { auto rval = new RecordVal(BifType::Record::BrokerStore::QueryResult); rval->Assign(0, query_status(false)); - rval->Assign(1, new RecordVal(BifType::Record::BrokerComm::Data)); + rval->Assign(1, new RecordVal(BifType::Record::Broker::Data)); return rval; } diff --git a/src/broker/comm.bif b/src/broker/comm.bif index f8dd546965..4caa1f8859 100644 --- a/src/broker/comm.bif +++ b/src/broker/comm.bif @@ -5,124 +5,124 @@ #include "broker/Manager.h" %%} -module BrokerComm; +module Broker; -type BrokerComm::EndpointFlags: record; +type Broker::EndpointFlags: record; ## Enable use of communication. ## ## flags: used to tune the local Broker endpoint behavior. ## ## Returns: true if communication is successfully initialized. -function BrokerComm::enable%(flags: EndpointFlags &default = EndpointFlags()%): bool +function Broker::enable%(flags: EndpointFlags &default = EndpointFlags()%): bool %{ return new Val(broker_mgr->Enable(flags), TYPE_BOOL); %} -## Changes endpoint flags originally supplied to :bro:see:`BrokerComm::enable`. +## Changes endpoint flags originally supplied to :bro:see:`Broker::enable`. ## ## flags: the new endpoint behavior flags to use. ## ## Returns: true if flags were changed. -function BrokerComm::set_endpoint_flags%(flags: EndpointFlags &default = EndpointFlags()%): bool +function Broker::set_endpoint_flags%(flags: EndpointFlags &default = EndpointFlags()%): bool %{ return new Val(broker_mgr->SetEndpointFlags(flags), TYPE_BOOL); %} ## Allow sending messages to peers if associated with the given topic. ## This has no effect if auto publication behavior is enabled via the flags -## supplied to :bro:see:`BrokerComm::enable` or :bro:see:`BrokerComm::set_endpoint_flags`. +## supplied to :bro:see:`Broker::enable` or :bro:see:`Broker::set_endpoint_flags`. ## ## topic: a topic to allow messages to be published under. ## ## Returns: true if successful. -function BrokerComm::publish_topic%(topic: string%): bool +function Broker::publish_topic%(topic: string%): bool %{ return new Val(broker_mgr->PublishTopic(topic->CheckString()), TYPE_BOOL); %} ## Disallow sending messages to peers if associated with the given topic. ## This has no effect if auto publication behavior is enabled via the flags -## supplied to :bro:see:`BrokerComm::enable` or :bro:see:`BrokerComm::set_endpoint_flags`. +## supplied to :bro:see:`Broker::enable` or :bro:see:`Broker::set_endpoint_flags`. ## ## topic: a topic to disallow messages to be published under. ## ## Returns: true if successful. -function BrokerComm::unpublish_topic%(topic: string%): bool +function Broker::unpublish_topic%(topic: string%): bool %{ return new Val(broker_mgr->UnpublishTopic(topic->CheckString()), TYPE_BOOL); %} ## Allow advertising interest in the given topic to peers. ## This has no effect if auto advertise behavior is enabled via the flags -## supplied to :bro:see:`BrokerComm::enable` or :bro:see:`BrokerComm::set_endpoint_flags`. +## supplied to :bro:see:`Broker::enable` or :bro:see:`Broker::set_endpoint_flags`. ## ## topic: a topic to allow advertising interest/subscription to peers. ## ## Returns: true if successful. -function BrokerComm::advertise_topic%(topic: string%): bool +function Broker::advertise_topic%(topic: string%): bool %{ return new Val(broker_mgr->AdvertiseTopic(topic->CheckString()), TYPE_BOOL); %} ## Disallow advertising interest in the given topic to peers. ## This has no effect if auto advertise behavior is enabled via the flags -## supplied to :bro:see:`BrokerComm::enable` or :bro:see:`BrokerComm::set_endpoint_flags`. +## supplied to :bro:see:`Broker::enable` or :bro:see:`Broker::set_endpoint_flags`. ## ## topic: a topic to disallow advertising interest/subscription to peers. ## ## Returns: true if successful. -function BrokerComm::unadvertise_topic%(topic: string%): bool +function Broker::unadvertise_topic%(topic: string%): bool %{ return new Val(broker_mgr->UnadvertiseTopic(topic->CheckString()), TYPE_BOOL); %} ## Generated when a connection has been established due to a previous call -## to :bro:see:`BrokerComm::connect`. +## to :bro:see:`Broker::connect`. ## ## peer_address: the address used to connect to the peer. ## ## peer_port: the port used to connect to the peer. ## ## peer_name: the name by which the peer identified itself. -event BrokerComm::outgoing_connection_established%(peer_address: string, +event Broker::outgoing_connection_established%(peer_address: string, peer_port: port, peer_name: string%); ## Generated when a previously established connection becomes broken. ## Reconnection will automatically be attempted at a frequency given -## by the original call to :bro:see:`BrokerComm::connect`. +## by the original call to :bro:see:`Broker::connect`. ## ## peer_address: the address used to connect to the peer. ## ## peer_port: the port used to connect to the peer. ## -## .. bro:see:: BrokerComm::outgoing_connection_established -event BrokerComm::outgoing_connection_broken%(peer_address: string, +## .. bro:see:: Broker::outgoing_connection_established +event Broker::outgoing_connection_broken%(peer_address: string, peer_port: port%); -## Generated when a connection via :bro:see:`BrokerComm::connect` has failed +## Generated when a connection via :bro:see:`Broker::connect` has failed ## because the remote side is incompatible. ## ## peer_address: the address used to connect to the peer. ## ## peer_port: the port used to connect to the peer. -event BrokerComm::outgoing_connection_incompatible%(peer_address: string, +event Broker::outgoing_connection_incompatible%(peer_address: string, peer_port: port%); ## Generated when a peer has established a connection with this process -## as a result of previously performing a :bro:see:`BrokerComm::listen`. +## as a result of previously performing a :bro:see:`Broker::listen`. ## ## peer_name: the name by which the peer identified itself. -event BrokerComm::incoming_connection_established%(peer_name: string%); +event Broker::incoming_connection_established%(peer_name: string%); ## Generated when a peer that previously established a connection with this ## process becomes disconnected. ## ## peer_name: the name by which the peer identified itself. ## -## .. bro:see:: BrokerComm::incoming_connection_established -event BrokerComm::incoming_connection_broken%(peer_name: string%); +## .. bro:see:: Broker::incoming_connection_established +event Broker::incoming_connection_broken%(peer_name: string%); ## Listen for remote connections. ## @@ -135,8 +135,8 @@ event BrokerComm::incoming_connection_broken%(peer_name: string%); ## ## Returns: true if the local endpoint is now listening for connections. ## -## .. bro:see:: BrokerComm::incoming_connection_established -function BrokerComm::listen%(p: port, a: string &default = "", +## .. bro:see:: Broker::incoming_connection_established +function Broker::listen%(p: port, a: string &default = "", reuse: bool &default = T%): bool %{ if ( ! p->IsTCP() ) @@ -164,8 +164,8 @@ function BrokerComm::listen%(p: port, a: string &default = "", ## it's a new peer. The actual connection may not be established ## until a later point in time. ## -## .. bro:see:: BrokerComm::outgoing_connection_established -function BrokerComm::connect%(a: string, p: port, retry: interval%): bool +## .. bro:see:: Broker::outgoing_connection_established +function Broker::connect%(a: string, p: port, retry: interval%): bool %{ if ( ! p->IsTCP() ) { @@ -180,13 +180,13 @@ function BrokerComm::connect%(a: string, p: port, retry: interval%): bool ## Remove a remote connection. ## -## a: the address used in previous successful call to :bro:see:`BrokerComm::connect`. +## a: the address used in previous successful call to :bro:see:`Broker::connect`. ## -## p: the port used in previous successful call to :bro:see:`BrokerComm::connect`. +## p: the port used in previous successful call to :bro:see:`Broker::connect`. ## ## Returns: true if the arguments match a previously successful call to -## :bro:see:`BrokerComm::connect`. -function BrokerComm::disconnect%(a: string, p: port%): bool +## :bro:see:`Broker::connect`. +function Broker::disconnect%(a: string, p: port%): bool %{ if ( ! p->IsTCP() ) { diff --git a/src/broker/data.bif b/src/broker/data.bif index 9ea1ca1e86..d4744f07c6 100644 --- a/src/broker/data.bif +++ b/src/broker/data.bif @@ -5,9 +5,9 @@ #include "broker/Data.h" %%} -module BrokerComm; +module Broker; -## Enumerates the possible types that :bro:see:`BrokerComm::Data` may be in +## Enumerates the possible types that :bro:see:`Broker::Data` may be in ## terms of Bro data types. enum DataType %{ BOOL, @@ -27,9 +27,9 @@ enum DataType %{ RECORD, %} -type BrokerComm::Data: record; +type Broker::Data: record; -type BrokerComm::TableItem: record; +type Broker::TableItem: record; ## Convert any Bro value to communication data. ## @@ -39,7 +39,7 @@ type BrokerComm::TableItem: record; ## field will not be set if the conversion was not possible (this can ## happen if the Bro data type does not support being converted to ## communication data). -function BrokerComm::data%(d: any%): BrokerComm::Data +function Broker::data%(d: any%): Broker::Data %{ return bro_broker::make_data_val(d); %} @@ -49,75 +49,75 @@ function BrokerComm::data%(d: any%): BrokerComm::Data ## d: the communication data. ## ## Returns: the data type associated with the communication data. -function BrokerComm::data_type%(d: BrokerComm::Data%): BrokerComm::DataType +function Broker::data_type%(d: Broker::Data%): Broker::DataType %{ return bro_broker::get_data_type(d->AsRecordVal(), frame); %} -## Convert communication data with a type of :bro:see:`BrokerComm::BOOL` to +## Convert communication data with a type of :bro:see:`Broker::BOOL` to ## an actual Bro value. ## ## d: the communication data to convert. ## ## Returns: the value retrieved from the communication data. -function BrokerComm::refine_to_bool%(d: BrokerComm::Data%): bool +function Broker::refine_to_bool%(d: Broker::Data%): bool %{ return bro_broker::refine(d->AsRecordVal(), TYPE_BOOL, frame); %} -## Convert communication data with a type of :bro:see:`BrokerComm::INT` to +## Convert communication data with a type of :bro:see:`Broker::INT` to ## an actual Bro value. ## ## d: the communication data to convert. ## ## Returns: the value retrieved from the communication data. -function BrokerComm::refine_to_int%(d: BrokerComm::Data%): int +function Broker::refine_to_int%(d: Broker::Data%): int %{ return bro_broker::refine(d->AsRecordVal(), TYPE_INT, frame); %} -## Convert communication data with a type of :bro:see:`BrokerComm::COUNT` to +## Convert communication data with a type of :bro:see:`Broker::COUNT` to ## an actual Bro value. ## ## d: the communication data to convert. ## ## Returns: the value retrieved from the communication data. -function BrokerComm::refine_to_count%(d: BrokerComm::Data%): count +function Broker::refine_to_count%(d: Broker::Data%): count %{ return bro_broker::refine(d->AsRecordVal(), TYPE_COUNT, frame); %} -## Convert communication data with a type of :bro:see:`BrokerComm::DOUBLE` to +## Convert communication data with a type of :bro:see:`Broker::DOUBLE` to ## an actual Bro value. ## ## d: the communication data to convert. ## ## Returns: the value retrieved from the communication data. -function BrokerComm::refine_to_double%(d: BrokerComm::Data%): double +function Broker::refine_to_double%(d: Broker::Data%): double %{ return bro_broker::refine(d->AsRecordVal(), TYPE_DOUBLE, frame); %} -## Convert communication data with a type of :bro:see:`BrokerComm::STRING` to +## Convert communication data with a type of :bro:see:`Broker::STRING` to ## an actual Bro value. ## ## d: the communication data to convert. ## ## Returns: the value retrieved from the communication data. -function BrokerComm::refine_to_string%(d: BrokerComm::Data%): string +function Broker::refine_to_string%(d: Broker::Data%): string %{ return new StringVal(bro_broker::require_data_type(d->AsRecordVal(), TYPE_STRING, frame)); %} -## Convert communication data with a type of :bro:see:`BrokerComm::ADDR` to +## Convert communication data with a type of :bro:see:`Broker::ADDR` to ## an actual Bro value. ## ## d: the communication data to convert. ## ## Returns: the value retrieved from the communication data. -function BrokerComm::refine_to_addr%(d: BrokerComm::Data%): addr +function Broker::refine_to_addr%(d: Broker::Data%): addr %{ auto& a = bro_broker::require_data_type(d->AsRecordVal(), TYPE_ADDR, frame); @@ -125,13 +125,13 @@ function BrokerComm::refine_to_addr%(d: BrokerComm::Data%): addr return new AddrVal(IPAddr(*bits)); %} -## Convert communication data with a type of :bro:see:`BrokerComm::SUBNET` to +## Convert communication data with a type of :bro:see:`Broker::SUBNET` to ## an actual Bro value. ## ## d: the communication data to convert. ## ## Returns: the value retrieved from the communication data. -function BrokerComm::refine_to_subnet%(d: BrokerComm::Data%): subnet +function Broker::refine_to_subnet%(d: Broker::Data%): subnet %{ auto& a = bro_broker::require_data_type(d->AsRecordVal(), TYPE_SUBNET, frame); @@ -139,53 +139,53 @@ function BrokerComm::refine_to_subnet%(d: BrokerComm::Data%): subnet return new SubNetVal(IPPrefix(IPAddr(*bits), a.length())); %} -## Convert communication data with a type of :bro:see:`BrokerComm::PORT` to +## Convert communication data with a type of :bro:see:`Broker::PORT` to ## an actual Bro value. ## ## d: the communication data to convert. ## ## Returns: the value retrieved from the communication data. -function BrokerComm::refine_to_port%(d: BrokerComm::Data%): port +function Broker::refine_to_port%(d: Broker::Data%): port %{ auto& a = bro_broker::require_data_type(d->AsRecordVal(), TYPE_SUBNET, frame); return new PortVal(a.number(), bro_broker::to_bro_port_proto(a.type())); %} -## Convert communication data with a type of :bro:see:`BrokerComm::TIME` to +## Convert communication data with a type of :bro:see:`Broker::TIME` to ## an actual Bro value. ## ## d: the communication data to convert. ## ## Returns: the value retrieved from the communication data. -function BrokerComm::refine_to_time%(d: BrokerComm::Data%): time +function Broker::refine_to_time%(d: Broker::Data%): time %{ auto v = bro_broker::require_data_type(d->AsRecordVal(), TYPE_TIME, frame).value; return new Val(v, TYPE_TIME); %} -## Convert communication data with a type of :bro:see:`BrokerComm::INTERVAL` to +## Convert communication data with a type of :bro:see:`Broker::INTERVAL` to ## an actual Bro value. ## ## d: the communication data to convert. ## ## Returns: the value retrieved from the communication data. -function BrokerComm::refine_to_interval%(d: BrokerComm::Data%): interval +function Broker::refine_to_interval%(d: Broker::Data%): interval %{ auto v = bro_broker::require_data_type(d->AsRecordVal(), TYPE_TIME, frame).value; return new Val(v, TYPE_INTERVAL); %} -## Convert communication data with a type of :bro:see:`BrokerComm::ENUM` to +## Convert communication data with a type of :bro:see:`Broker::ENUM` to ## the name of the enum value. :bro:see:`lookup_ID` may be used to convert ## the name to the actual enum value. ## ## d: the communication data to convert. ## ## Returns: the enum name retrieved from the communication data. -function BrokerComm::refine_to_enum_name%(d: BrokerComm::Data%): string +function Broker::refine_to_enum_name%(d: Broker::Data%): string %{ auto& v = bro_broker::require_data_type(d->AsRecordVal(), TYPE_ENUM, frame).name; @@ -193,7 +193,7 @@ function BrokerComm::refine_to_enum_name%(d: BrokerComm::Data%): string %} ## Create communication data of type "set". -function BrokerComm::set_create%(%): BrokerComm::Data +function Broker::set_create%(%): Broker::Data %{ return bro_broker::make_data_val(broker::set()); %} @@ -203,7 +203,7 @@ function BrokerComm::set_create%(%): BrokerComm::Data ## s: the set to clear. ## ## Returns: always true. -function BrokerComm::set_clear%(s: BrokerComm::Data%): bool +function Broker::set_clear%(s: Broker::Data%): bool %{ auto& v = bro_broker::require_data_type(s->AsRecordVal(), TYPE_TABLE, frame); @@ -216,7 +216,7 @@ function BrokerComm::set_clear%(s: BrokerComm::Data%): bool ## s: the set to query. ## ## Returns: the number of elements in the set. -function BrokerComm::set_size%(s: BrokerComm::Data%): count +function Broker::set_size%(s: Broker::Data%): count %{ auto& v = bro_broker::require_data_type(s->AsRecordVal(), TYPE_TABLE, frame); @@ -230,7 +230,7 @@ function BrokerComm::set_size%(s: BrokerComm::Data%): count ## key: the element to check for existence. ## ## Returns: true if the key exists in the set. -function BrokerComm::set_contains%(s: BrokerComm::Data, key: BrokerComm::Data%): bool +function Broker::set_contains%(s: Broker::Data, key: Broker::Data%): bool %{ auto& v = bro_broker::require_data_type(s->AsRecordVal(), TYPE_TABLE, frame); @@ -245,7 +245,7 @@ function BrokerComm::set_contains%(s: BrokerComm::Data, key: BrokerComm::Data%): ## key: the element to insert. ## ## Returns: true if the key was inserted, or false if it already existed. -function BrokerComm::set_insert%(s: BrokerComm::Data, key: BrokerComm::Data%): bool +function Broker::set_insert%(s: Broker::Data, key: Broker::Data%): bool %{ auto& v = bro_broker::require_data_type(s->AsRecordVal(), TYPE_TABLE, frame); @@ -260,7 +260,7 @@ function BrokerComm::set_insert%(s: BrokerComm::Data, key: BrokerComm::Data%): b ## key: the element to remove. ## ## Returns: true if the element existed in the set and is now removed. -function BrokerComm::set_remove%(s: BrokerComm::Data, key: BrokerComm::Data%): bool +function Broker::set_remove%(s: Broker::Data, key: Broker::Data%): bool %{ auto& v = bro_broker::require_data_type(s->AsRecordVal(), TYPE_TABLE, frame); @@ -274,7 +274,7 @@ function BrokerComm::set_remove%(s: BrokerComm::Data, key: BrokerComm::Data%): b ## s: the set to iterate over. ## ## Returns: an iterator. -function BrokerComm::set_iterator%(s: BrokerComm::Data%): opaque of BrokerComm::SetIterator +function Broker::set_iterator%(s: Broker::Data%): opaque of Broker::SetIterator %{ return new bro_broker::SetIterator(s->AsRecordVal(), TYPE_TABLE, frame); %} @@ -285,7 +285,7 @@ function BrokerComm::set_iterator%(s: BrokerComm::Data%): opaque of BrokerComm:: ## ## Returns: true if there are no more elements to iterator over, i.e. ## the iterator is one-past-the-final-element. -function BrokerComm::set_iterator_last%(it: opaque of BrokerComm::SetIterator%): bool +function Broker::set_iterator_last%(it: opaque of Broker::SetIterator%): bool %{ auto set_it = static_cast(it); return new Val(set_it->it == set_it->dat.end(), TYPE_BOOL); @@ -298,7 +298,7 @@ function BrokerComm::set_iterator_last%(it: opaque of BrokerComm::SetIterator%): ## Returns: true if the iterator, after advancing, still references an element ## in the collection. False if the iterator, after advancing, is ## one-past-the-final-element. -function BrokerComm::set_iterator_next%(it: opaque of BrokerComm::SetIterator%): bool +function Broker::set_iterator_next%(it: opaque of Broker::SetIterator%): bool %{ auto set_it = static_cast(it); @@ -314,10 +314,10 @@ function BrokerComm::set_iterator_next%(it: opaque of BrokerComm::SetIterator%): ## it: an iterator. ## ## Returns: element in the collection that the iterator currently references. -function BrokerComm::set_iterator_value%(it: opaque of BrokerComm::SetIterator%): BrokerComm::Data +function Broker::set_iterator_value%(it: opaque of Broker::SetIterator%): Broker::Data %{ auto set_it = static_cast(it); - auto rval = new RecordVal(BifType::Record::BrokerComm::Data); + auto rval = new RecordVal(BifType::Record::Broker::Data); if ( set_it->it == set_it->dat.end() ) { @@ -332,7 +332,7 @@ function BrokerComm::set_iterator_value%(it: opaque of BrokerComm::SetIterator%) %} ## Create communication data of type "table". -function BrokerComm::table_create%(%): BrokerComm::Data +function Broker::table_create%(%): Broker::Data %{ return bro_broker::make_data_val(broker::table()); %} @@ -342,7 +342,7 @@ function BrokerComm::table_create%(%): BrokerComm::Data ## t: the table to clear. ## ## Returns: always true. -function BrokerComm::table_clear%(t: BrokerComm::Data%): bool +function Broker::table_clear%(t: Broker::Data%): bool %{ auto& v = bro_broker::require_data_type(t->AsRecordVal(), TYPE_TABLE, frame); @@ -355,7 +355,7 @@ function BrokerComm::table_clear%(t: BrokerComm::Data%): bool ## t: the table to query. ## ## Returns: the number of elements in the table. -function BrokerComm::table_size%(t: BrokerComm::Data%): count +function Broker::table_size%(t: Broker::Data%): count %{ auto& v = bro_broker::require_data_type(t->AsRecordVal(), TYPE_TABLE, frame); @@ -369,7 +369,7 @@ function BrokerComm::table_size%(t: BrokerComm::Data%): count ## key: the key to check for existence. ## ## Returns: true if the key exists in the table. -function BrokerComm::table_contains%(t: BrokerComm::Data, key: BrokerComm::Data%): bool +function Broker::table_contains%(t: Broker::Data, key: Broker::Data%): bool %{ auto& v = bro_broker::require_data_type(t->AsRecordVal(), TYPE_TABLE, frame); @@ -387,7 +387,7 @@ function BrokerComm::table_contains%(t: BrokerComm::Data, key: BrokerComm::Data% ## ## Returns: true if the key-value pair was inserted, or false if the key ## already existed in the table. -function BrokerComm::table_insert%(t: BrokerComm::Data, key: BrokerComm::Data, val: BrokerComm::Data%): BrokerComm::Data +function Broker::table_insert%(t: Broker::Data, key: Broker::Data, val: Broker::Data%): Broker::Data %{ auto& table = bro_broker::require_data_type(t->AsRecordVal(), TYPE_TABLE, frame); @@ -404,7 +404,7 @@ function BrokerComm::table_insert%(t: BrokerComm::Data, key: BrokerComm::Data, v catch (const std::out_of_range&) { table[k] = v; - return new RecordVal(BifType::Record::BrokerComm::Data); + return new RecordVal(BifType::Record::Broker::Data); } %} @@ -416,7 +416,7 @@ function BrokerComm::table_insert%(t: BrokerComm::Data, key: BrokerComm::Data, v ## ## Returns: the value associated with the key. If the key did not exist, then ## the optional field of the returned record is not set. -function BrokerComm::table_remove%(t: BrokerComm::Data, key: BrokerComm::Data%): BrokerComm::Data +function Broker::table_remove%(t: Broker::Data, key: Broker::Data%): Broker::Data %{ auto& table = bro_broker::require_data_type(t->AsRecordVal(), TYPE_TABLE, frame); @@ -424,7 +424,7 @@ function BrokerComm::table_remove%(t: BrokerComm::Data, key: BrokerComm::Data%): auto it = table.find(k); if ( it == table.end() ) - return new RecordVal(BifType::Record::BrokerComm::Data); + return new RecordVal(BifType::Record::Broker::Data); else { auto rval = bro_broker::make_data_val(move(it->second)); @@ -441,7 +441,7 @@ function BrokerComm::table_remove%(t: BrokerComm::Data, key: BrokerComm::Data%): ## ## Returns: the value associated with the key. If the key did not exist, then ## the optional field of the returned record is not set. -function BrokerComm::table_lookup%(t: BrokerComm::Data, key: BrokerComm::Data%): BrokerComm::Data +function Broker::table_lookup%(t: Broker::Data, key: Broker::Data%): Broker::Data %{ auto& table = bro_broker::require_data_type(t->AsRecordVal(), TYPE_TABLE, frame); @@ -449,7 +449,7 @@ function BrokerComm::table_lookup%(t: BrokerComm::Data, key: BrokerComm::Data%): auto it = table.find(k); if ( it == table.end() ) - return new RecordVal(BifType::Record::BrokerComm::Data); + return new RecordVal(BifType::Record::Broker::Data); else return bro_broker::make_data_val(it->second); %} @@ -460,7 +460,7 @@ function BrokerComm::table_lookup%(t: BrokerComm::Data, key: BrokerComm::Data%): ## t: the table to iterate over. ## ## Returns: an iterator. -function BrokerComm::table_iterator%(t: BrokerComm::Data%): opaque of BrokerComm::TableIterator +function Broker::table_iterator%(t: Broker::Data%): opaque of Broker::TableIterator %{ return new bro_broker::TableIterator(t->AsRecordVal(), TYPE_TABLE, frame); %} @@ -471,7 +471,7 @@ function BrokerComm::table_iterator%(t: BrokerComm::Data%): opaque of BrokerComm ## ## Returns: true if there are no more elements to iterator over, i.e. ## the iterator is one-past-the-final-element. -function BrokerComm::table_iterator_last%(it: opaque of BrokerComm::TableIterator%): bool +function Broker::table_iterator_last%(it: opaque of Broker::TableIterator%): bool %{ auto ti = static_cast(it); return new Val(ti->it == ti->dat.end(), TYPE_BOOL); @@ -484,7 +484,7 @@ function BrokerComm::table_iterator_last%(it: opaque of BrokerComm::TableIterato ## Returns: true if the iterator, after advancing, still references an element ## in the collection. False if the iterator, after advancing, is ## one-past-the-final-element. -function BrokerComm::table_iterator_next%(it: opaque of BrokerComm::TableIterator%): bool +function Broker::table_iterator_next%(it: opaque of Broker::TableIterator%): bool %{ auto ti = static_cast(it); @@ -500,12 +500,12 @@ function BrokerComm::table_iterator_next%(it: opaque of BrokerComm::TableIterato ## it: an iterator. ## ## Returns: element in the collection that the iterator currently references. -function BrokerComm::table_iterator_value%(it: opaque of BrokerComm::TableIterator%): BrokerComm::TableItem +function Broker::table_iterator_value%(it: opaque of Broker::TableIterator%): Broker::TableItem %{ auto ti = static_cast(it); - auto rval = new RecordVal(BifType::Record::BrokerComm::TableItem); - auto key_val = new RecordVal(BifType::Record::BrokerComm::Data); - auto val_val = new RecordVal(BifType::Record::BrokerComm::Data); + auto rval = new RecordVal(BifType::Record::Broker::TableItem); + auto key_val = new RecordVal(BifType::Record::Broker::Data); + auto val_val = new RecordVal(BifType::Record::Broker::Data); rval->Assign(0, key_val); rval->Assign(1, val_val); @@ -523,7 +523,7 @@ function BrokerComm::table_iterator_value%(it: opaque of BrokerComm::TableIterat %} ## Create communication data of type "vector". -function BrokerComm::vector_create%(%): BrokerComm::Data +function Broker::vector_create%(%): Broker::Data %{ return bro_broker::make_data_val(broker::vector()); %} @@ -533,7 +533,7 @@ function BrokerComm::vector_create%(%): BrokerComm::Data ## v: the vector to clear. ## ## Returns: always true. -function BrokerComm::vector_clear%(v: BrokerComm::Data%): bool +function Broker::vector_clear%(v: Broker::Data%): bool %{ auto& vec = bro_broker::require_data_type(v->AsRecordVal(), TYPE_VECTOR, frame); @@ -546,7 +546,7 @@ function BrokerComm::vector_clear%(v: BrokerComm::Data%): bool ## v: the vector to query. ## ## Returns: the number of elements in the vector. -function BrokerComm::vector_size%(v: BrokerComm::Data%): count +function Broker::vector_size%(v: Broker::Data%): count %{ auto& vec = bro_broker::require_data_type(v->AsRecordVal(), TYPE_VECTOR, frame); @@ -564,7 +564,7 @@ function BrokerComm::vector_size%(v: BrokerComm::Data%): count ## current size of the vector, the element is inserted at the end. ## ## Returns: always true. -function BrokerComm::vector_insert%(v: BrokerComm::Data, d: BrokerComm::Data, idx: count%): bool +function Broker::vector_insert%(v: Broker::Data, d: Broker::Data, idx: count%): bool %{ auto& vec = bro_broker::require_data_type(v->AsRecordVal(), TYPE_VECTOR, frame); @@ -584,14 +584,14 @@ function BrokerComm::vector_insert%(v: BrokerComm::Data, d: BrokerComm::Data, id ## ## Returns: the value that was just evicted. If the index was larger than any ## valid index, the optional field of the returned record is not set. -function BrokerComm::vector_replace%(v: BrokerComm::Data, d: BrokerComm::Data, idx: count%): BrokerComm::Data +function Broker::vector_replace%(v: Broker::Data, d: Broker::Data, idx: count%): Broker::Data %{ auto& vec = bro_broker::require_data_type(v->AsRecordVal(), TYPE_VECTOR, frame); auto& item = bro_broker::opaque_field_to_data(d->AsRecordVal(), frame); if ( idx >= vec.size() ) - return new RecordVal(BifType::Record::BrokerComm::Data); + return new RecordVal(BifType::Record::Broker::Data); auto rval = bro_broker::make_data_val(move(vec[idx])); vec[idx] = item; @@ -606,13 +606,13 @@ function BrokerComm::vector_replace%(v: BrokerComm::Data, d: BrokerComm::Data, i ## ## Returns: the value that was just evicted. If the index was larger than any ## valid index, the optional field of the returned record is not set. -function BrokerComm::vector_remove%(v: BrokerComm::Data, idx: count%): BrokerComm::Data +function Broker::vector_remove%(v: Broker::Data, idx: count%): Broker::Data %{ auto& vec = bro_broker::require_data_type(v->AsRecordVal(), TYPE_VECTOR, frame); if ( idx >= vec.size() ) - return new RecordVal(BifType::Record::BrokerComm::Data); + return new RecordVal(BifType::Record::Broker::Data); auto rval = bro_broker::make_data_val(move(vec[idx])); vec.erase(vec.begin() + idx); @@ -627,13 +627,13 @@ function BrokerComm::vector_remove%(v: BrokerComm::Data, idx: count%): BrokerCom ## ## Returns: the value at the index. If the index was larger than any ## valid index, the optional field of the returned record is not set. -function BrokerComm::vector_lookup%(v: BrokerComm::Data, idx: count%): BrokerComm::Data +function Broker::vector_lookup%(v: Broker::Data, idx: count%): Broker::Data %{ auto& vec = bro_broker::require_data_type(v->AsRecordVal(), TYPE_VECTOR, frame); if ( idx >= vec.size() ) - return new RecordVal(BifType::Record::BrokerComm::Data); + return new RecordVal(BifType::Record::Broker::Data); return bro_broker::make_data_val(vec[idx]); %} @@ -644,7 +644,7 @@ function BrokerComm::vector_lookup%(v: BrokerComm::Data, idx: count%): BrokerCom ## v: the vector to iterate over. ## ## Returns: an iterator. -function BrokerComm::vector_iterator%(v: BrokerComm::Data%): opaque of BrokerComm::VectorIterator +function Broker::vector_iterator%(v: Broker::Data%): opaque of Broker::VectorIterator %{ return new bro_broker::VectorIterator(v->AsRecordVal(), TYPE_VECTOR, frame); %} @@ -655,7 +655,7 @@ function BrokerComm::vector_iterator%(v: BrokerComm::Data%): opaque of BrokerCom ## ## Returns: true if there are no more elements to iterator over, i.e. ## the iterator is one-past-the-final-element. -function BrokerComm::vector_iterator_last%(it: opaque of BrokerComm::VectorIterator%): bool +function Broker::vector_iterator_last%(it: opaque of Broker::VectorIterator%): bool %{ auto vi = static_cast(it); return new Val(vi->it == vi->dat.end(), TYPE_BOOL); @@ -668,7 +668,7 @@ function BrokerComm::vector_iterator_last%(it: opaque of BrokerComm::VectorItera ## Returns: true if the iterator, after advancing, still references an element ## in the collection. False if the iterator, after advancing, is ## one-past-the-final-element. -function BrokerComm::vector_iterator_next%(it: opaque of BrokerComm::VectorIterator%): bool +function Broker::vector_iterator_next%(it: opaque of Broker::VectorIterator%): bool %{ auto vi = static_cast(it); @@ -684,10 +684,10 @@ function BrokerComm::vector_iterator_next%(it: opaque of BrokerComm::VectorItera ## it: an iterator. ## ## Returns: element in the collection that the iterator currently references. -function BrokerComm::vector_iterator_value%(it: opaque of BrokerComm::VectorIterator%): BrokerComm::Data +function Broker::vector_iterator_value%(it: opaque of Broker::VectorIterator%): Broker::Data %{ auto vi = static_cast(it); - auto rval = new RecordVal(BifType::Record::BrokerComm::Data); + auto rval = new RecordVal(BifType::Record::Broker::Data); if ( vi->it == vi->dat.end() ) { @@ -706,7 +706,7 @@ function BrokerComm::vector_iterator_value%(it: opaque of BrokerComm::VectorIter ## sz: the number of fields in the record. ## ## Returns: record data, with all fields uninitialized. -function BrokerComm::record_create%(sz: count%): BrokerComm::Data +function Broker::record_create%(sz: count%): Broker::Data %{ return bro_broker::make_data_val(broker::record(std::vector(sz))); %} @@ -716,7 +716,7 @@ function BrokerComm::record_create%(sz: count%): BrokerComm::Data ## r: the record to query. ## ## Returns: the number of fields in the record. -function BrokerComm::record_size%(r: BrokerComm::Data%): count +function Broker::record_size%(r: Broker::Data%): count %{ auto& v = bro_broker::require_data_type(r->AsRecordVal(), TYPE_RECORD, frame); @@ -732,7 +732,7 @@ function BrokerComm::record_size%(r: BrokerComm::Data%): count ## idx: the index to replace. ## ## Returns: false if the index was larger than any valid index, else true. -function BrokerComm::record_assign%(r: BrokerComm::Data, d: BrokerComm::Data, idx: count%): bool +function Broker::record_assign%(r: Broker::Data, d: Broker::Data, idx: count%): bool %{ auto& v = bro_broker::require_data_type(r->AsRecordVal(), TYPE_RECORD, frame); @@ -754,16 +754,16 @@ function BrokerComm::record_assign%(r: BrokerComm::Data, d: BrokerComm::Data, id ## Returns: the value at the index. The optional field of the returned record ## may not be set if the field of the record has no value or if the ## index was not valid. -function BrokerComm::record_lookup%(r: BrokerComm::Data, idx: count%): BrokerComm::Data +function Broker::record_lookup%(r: Broker::Data, idx: count%): Broker::Data %{ auto& v = bro_broker::require_data_type(r->AsRecordVal(), TYPE_RECORD, frame); if ( idx >= v.size() ) - return new RecordVal(BifType::Record::BrokerComm::Data); + return new RecordVal(BifType::Record::Broker::Data); if ( ! v.fields[idx] ) - return new RecordVal(BifType::Record::BrokerComm::Data); + return new RecordVal(BifType::Record::Broker::Data); return bro_broker::make_data_val(*v.fields[idx]); %} @@ -774,7 +774,7 @@ function BrokerComm::record_lookup%(r: BrokerComm::Data, idx: count%): BrokerCom ## r: the record to iterate over. ## ## Returns: an iterator. -function BrokerComm::record_iterator%(r: BrokerComm::Data%): opaque of BrokerComm::RecordIterator +function Broker::record_iterator%(r: Broker::Data%): opaque of Broker::RecordIterator %{ return new bro_broker::RecordIterator(r->AsRecordVal(), TYPE_RECORD, frame); %} @@ -785,7 +785,7 @@ function BrokerComm::record_iterator%(r: BrokerComm::Data%): opaque of BrokerCom ## ## Returns: true if there are no more elements to iterator over, i.e. ## the iterator is one-past-the-final-element. -function BrokerComm::record_iterator_last%(it: opaque of BrokerComm::RecordIterator%): bool +function Broker::record_iterator_last%(it: opaque of Broker::RecordIterator%): bool %{ auto ri = static_cast(it); return new Val(ri->it == ri->dat.fields.end(), TYPE_BOOL); @@ -798,7 +798,7 @@ function BrokerComm::record_iterator_last%(it: opaque of BrokerComm::RecordItera ## Returns: true if the iterator, after advancing, still references an element ## in the collection. False if the iterator, after advancing, is ## one-past-the-final-element. -function BrokerComm::record_iterator_next%(it: opaque of BrokerComm::RecordIterator%): bool +function Broker::record_iterator_next%(it: opaque of Broker::RecordIterator%): bool %{ auto ri = static_cast(it); @@ -814,10 +814,10 @@ function BrokerComm::record_iterator_next%(it: opaque of BrokerComm::RecordItera ## it: an iterator. ## ## Returns: element in the collection that the iterator currently references. -function BrokerComm::record_iterator_value%(it: opaque of BrokerComm::RecordIterator%): BrokerComm::Data +function Broker::record_iterator_value%(it: opaque of Broker::RecordIterator%): Broker::Data %{ auto ri = static_cast(it); - auto rval = new RecordVal(BifType::Record::BrokerComm::Data); + auto rval = new RecordVal(BifType::Record::Broker::Data); if ( ri->it == ri->dat.fields.end() ) { diff --git a/src/broker/messaging.bif b/src/broker/messaging.bif index 97b794b50e..3c3240ff16 100644 --- a/src/broker/messaging.bif +++ b/src/broker/messaging.bif @@ -6,18 +6,18 @@ #include "logging/Manager.h" %%} -module BrokerComm; +module Broker; -type BrokerComm::SendFlags: record; +type Broker::SendFlags: record; -type BrokerComm::EventArgs: record; +type Broker::EventArgs: record; ## Used to handle remote print messages from peers that call -## :bro:see:`BrokerComm::print`. -event BrokerComm::print_handler%(msg: string%); +## :bro:see:`Broker::print`. +event Broker::print_handler%(msg: string%); ## Print a simple message to any interested peers. The receiver can use -## :bro:see:`BrokerComm::print_handler` to handle messages. +## :bro:see:`Broker::print_handler` to handle messages. ## ## topic: a topic associated with the printed message. ## @@ -26,7 +26,7 @@ event BrokerComm::print_handler%(msg: string%); ## flags: tune the behavior of how the message is sent. ## ## Returns: true if the message is sent. -function BrokerComm::print%(topic: string, msg: string, +function Broker::print%(topic: string, msg: string, flags: SendFlags &default = SendFlags()%): bool %{ auto rval = broker_mgr->Print(topic->CheckString(), msg->CheckString(), @@ -35,14 +35,14 @@ function BrokerComm::print%(topic: string, msg: string, %} ## Register interest in all peer print messages that use a certain topic prefix. -## Use :bro:see:`BrokerComm::print_handler` to handle received messages. +## Use :bro:see:`Broker::print_handler` to handle received messages. ## ## topic_prefix: a prefix to match against remote message topics. ## e.g. an empty prefix matches everything and "a" matches ## "alice" and "amy" but not "bob". ## ## Returns: true if it's a new print subscription and it is now registered. -function BrokerComm::subscribe_to_prints%(topic_prefix: string%): bool +function Broker::subscribe_to_prints%(topic_prefix: string%): bool %{ auto rval = broker_mgr->SubscribeToPrints(topic_prefix->CheckString()); return new Val(rval, TYPE_BOOL); @@ -51,23 +51,23 @@ function BrokerComm::subscribe_to_prints%(topic_prefix: string%): bool ## Unregister interest in all peer print messages that use a topic prefix. ## ## topic_prefix: a prefix previously supplied to a successful call to -## :bro:see:`BrokerComm::subscribe_to_prints`. +## :bro:see:`Broker::subscribe_to_prints`. ## ## Returns: true if interest in the topic prefix is no longer advertised. -function BrokerComm::unsubscribe_to_prints%(topic_prefix: string%): bool +function Broker::unsubscribe_to_prints%(topic_prefix: string%): bool %{ auto rval = broker_mgr->UnsubscribeToPrints(topic_prefix->CheckString()); return new Val(rval, TYPE_BOOL); %} ## Create a data structure that may be used to send a remote event via -## :bro:see:`BrokerComm::event`. +## :bro:see:`Broker::event`. ## ## args: an event, followed by a list of argument values that may be used ## to call it. ## ## Returns: opaque communication data that may be used to send a remote event. -function BrokerComm::event_args%(...%): BrokerComm::EventArgs +function Broker::event_args%(...%): Broker::EventArgs %{ auto rval = broker_mgr->MakeEventArgs(@ARGS@); return rval; @@ -77,12 +77,12 @@ function BrokerComm::event_args%(...%): BrokerComm::EventArgs ## ## topic: a topic associated with the event message. ## -## args: event arguments as made by :bro:see:`BrokerComm::event_args`. +## args: event arguments as made by :bro:see:`Broker::event_args`. ## ## flags: tune the behavior of how the message is sent. ## ## Returns: true if the message is sent. -function BrokerComm::event%(topic: string, args: BrokerComm::EventArgs, +function Broker::event%(topic: string, args: Broker::EventArgs, flags: SendFlags &default = SendFlags()%): bool %{ auto rval = broker_mgr->Event(topic->CheckString(), args->AsRecordVal(), @@ -102,7 +102,7 @@ function BrokerComm::event%(topic: string, args: BrokerComm::EventArgs, ## flags: tune the behavior of how the message is sent. ## ## Returns: true if automatic event sending is now enabled. -function BrokerComm::auto_event%(topic: string, ev: any, +function Broker::auto_event%(topic: string, ev: any, flags: SendFlags &default = SendFlags()%): bool %{ auto rval = broker_mgr->AutoEvent(topic->CheckString(), ev, flags); @@ -111,12 +111,12 @@ function BrokerComm::auto_event%(topic: string, ev: any, ## Stop automatically sending an event to peers upon local dispatch. ## -## topic: a topic originally given to :bro:see:`BrokerComm::auto_event`. +## topic: a topic originally given to :bro:see:`Broker::auto_event`. ## -## ev: an event originally given to :bro:see:`BrokerComm::auto_event`. +## ev: an event originally given to :bro:see:`Broker::auto_event`. ## ## Returns: true if automatic events will not occur for the topic/event pair. -function BrokerComm::auto_event_stop%(topic: string, ev: any%): bool +function Broker::auto_event_stop%(topic: string, ev: any%): bool %{ auto rval = broker_mgr->AutoEventStop(topic->CheckString(), ev); return new Val(rval, TYPE_BOOL); @@ -129,7 +129,7 @@ function BrokerComm::auto_event_stop%(topic: string, ev: any%): bool ## "alice" and "amy" but not "bob". ## ## Returns: true if it's a new event subscription and it is now registered. -function BrokerComm::subscribe_to_events%(topic_prefix: string%): bool +function Broker::subscribe_to_events%(topic_prefix: string%): bool %{ auto rval = broker_mgr->SubscribeToEvents(topic_prefix->CheckString()); return new Val(rval, TYPE_BOOL); @@ -138,10 +138,10 @@ function BrokerComm::subscribe_to_events%(topic_prefix: string%): bool ## Unregister interest in all peer event messages that use a topic prefix. ## ## topic_prefix: a prefix previously supplied to a successful call to -## :bro:see:`BrokerComm::subscribe_to_events`. +## :bro:see:`Broker::subscribe_to_events`. ## ## Returns: true if interest in the topic prefix is no longer advertised. -function BrokerComm::unsubscribe_to_events%(topic_prefix: string%): bool +function Broker::unsubscribe_to_events%(topic_prefix: string%): bool %{ auto rval = broker_mgr->UnsubscribeToEvents(topic_prefix->CheckString()); return new Val(rval, TYPE_BOOL); @@ -155,7 +155,7 @@ function BrokerComm::unsubscribe_to_events%(topic_prefix: string%): bool ## ## Returns: true if remote logs are enabled for the stream. function -BrokerComm::enable_remote_logs%(id: Log::ID, +Broker::enable_remote_logs%(id: Log::ID, flags: SendFlags &default = SendFlags()%): bool %{ auto rval = log_mgr->EnableRemoteLogs(id->AsEnumVal(), @@ -168,7 +168,7 @@ BrokerComm::enable_remote_logs%(id: Log::ID, ## id: the log stream to disable remote logs for. ## ## Returns: true if remote logs are disabled for the stream. -function BrokerComm::disable_remote_logs%(id: Log::ID%): bool +function Broker::disable_remote_logs%(id: Log::ID%): bool %{ auto rval = log_mgr->DisableRemoteLogs(id->AsEnumVal()); return new Val(rval, TYPE_BOOL); @@ -179,7 +179,7 @@ function BrokerComm::disable_remote_logs%(id: Log::ID%): bool ## id: the log stream to check. ## ## Returns: true if remote logs are enabled for the given stream. -function BrokerComm::remote_logs_enabled%(id: Log::ID%): bool +function Broker::remote_logs_enabled%(id: Log::ID%): bool %{ auto rval = log_mgr->RemoteLogsAreEnabled(id->AsEnumVal()); return new Val(rval, TYPE_BOOL); @@ -194,7 +194,7 @@ function BrokerComm::remote_logs_enabled%(id: Log::ID%): bool ## "alice" and "amy" but not "bob". ## ## Returns: true if it's a new log subscription and it is now registered. -function BrokerComm::subscribe_to_logs%(topic_prefix: string%): bool +function Broker::subscribe_to_logs%(topic_prefix: string%): bool %{ auto rval = broker_mgr->SubscribeToLogs(topic_prefix->CheckString()); return new Val(rval, TYPE_BOOL); @@ -205,10 +205,10 @@ function BrokerComm::subscribe_to_logs%(topic_prefix: string%): bool ## receiving side processes them through the logging framework as usual. ## ## topic_prefix: a prefix previously supplied to a successful call to -## :bro:see:`BrokerComm::subscribe_to_logs`. +## :bro:see:`Broker::subscribe_to_logs`. ## ## Returns: true if interest in the topic prefix is no longer advertised. -function BrokerComm::unsubscribe_to_logs%(topic_prefix: string%): bool +function Broker::unsubscribe_to_logs%(topic_prefix: string%): bool %{ auto rval = broker_mgr->UnsubscribeToLogs(topic_prefix->CheckString()); return new Val(rval, TYPE_BOOL); diff --git a/src/broker/store.bif b/src/broker/store.bif index 853bd1f2d7..565dee9e30 100644 --- a/src/broker/store.bif +++ b/src/broker/store.bif @@ -155,7 +155,7 @@ function BrokerStore::close_by_handle%(h: opaque of BrokerStore::Handle%): bool ## ## Returns: false if the store handle was not valid. function BrokerStore::insert%(h: opaque of BrokerStore::Handle, - k: BrokerComm::Data, v: BrokerComm::Data, + k: Broker::Data, v: Broker::Data, e: BrokerStore::ExpiryTime &default = BrokerStore::ExpiryTime()%): bool %{ auto handle = static_cast(h); @@ -198,7 +198,7 @@ function BrokerStore::insert%(h: opaque of BrokerStore::Handle, ## k: the key to remove. ## ## Returns: false if the store handle was not valid. -function BrokerStore::erase%(h: opaque of BrokerStore::Handle, k: BrokerComm::Data%): bool +function BrokerStore::erase%(h: opaque of BrokerStore::Handle, k: Broker::Data%): bool %{ auto handle = static_cast(h); @@ -237,7 +237,7 @@ function BrokerStore::clear%(h: opaque of BrokerStore::Handle%): bool ## ## Returns: false if the store handle was not valid. function BrokerStore::increment%(h: opaque of BrokerStore::Handle, - k: BrokerComm::Data, by: int &default = +1%): bool + k: Broker::Data, by: int &default = +1%): bool %{ auto handle = static_cast(h); @@ -260,7 +260,7 @@ function BrokerStore::increment%(h: opaque of BrokerStore::Handle, ## ## Returns: false if the store handle was not valid. function BrokerStore::decrement%(h: opaque of BrokerStore::Handle, - k: BrokerComm::Data, by: int &default = +1%): bool + k: Broker::Data, by: int &default = +1%): bool %{ auto handle = static_cast(h); @@ -283,7 +283,7 @@ function BrokerStore::decrement%(h: opaque of BrokerStore::Handle, ## ## Returns: false if the store handle was not valid. function BrokerStore::add_to_set%(h: opaque of BrokerStore::Handle, - k: BrokerComm::Data, element: BrokerComm::Data%): bool + k: Broker::Data, element: Broker::Data%): bool %{ auto handle = static_cast(h); @@ -307,7 +307,7 @@ function BrokerStore::add_to_set%(h: opaque of BrokerStore::Handle, ## ## Returns: false if the store handle was not valid. function BrokerStore::remove_from_set%(h: opaque of BrokerStore::Handle, - k: BrokerComm::Data, element: BrokerComm::Data%): bool + k: Broker::Data, element: Broker::Data%): bool %{ auto handle = static_cast(h); @@ -330,8 +330,8 @@ function BrokerStore::remove_from_set%(h: opaque of BrokerStore::Handle, ## create an empty vector value before modifying. ## ## Returns: false if the store handle was not valid. -function BrokerStore::push_left%(h: opaque of BrokerStore::Handle, k: BrokerComm::Data, - items: BrokerComm::DataVector%): bool +function BrokerStore::push_left%(h: opaque of BrokerStore::Handle, k: Broker::Data, + items: Broker::DataVector%): bool %{ auto handle = static_cast(h); @@ -363,8 +363,8 @@ function BrokerStore::push_left%(h: opaque of BrokerStore::Handle, k: BrokerComm ## create an empty vector value before modifying. ## ## Returns: false if the store handle was not valid. -function BrokerStore::push_right%(h: opaque of BrokerStore::Handle, k: BrokerComm::Data, - items: BrokerComm::DataVector%): bool +function BrokerStore::push_right%(h: opaque of BrokerStore::Handle, k: Broker::Data, + items: Broker::DataVector%): bool %{ auto handle = static_cast(h); @@ -445,7 +445,7 @@ static bool prepare_for_query(Val* opaque, Frame* frame, ## ## Returns: the result of the query. function BrokerStore::pop_left%(h: opaque of BrokerStore::Handle, - k: BrokerComm::Data%): BrokerStore::QueryResult + k: Broker::Data%): BrokerStore::QueryResult %{ if ( ! broker_mgr->Enabled() ) return bro_broker::query_result(); @@ -475,7 +475,7 @@ function BrokerStore::pop_left%(h: opaque of BrokerStore::Handle, ## ## Returns: the result of the query. function BrokerStore::pop_right%(h: opaque of BrokerStore::Handle, - k: BrokerComm::Data%): BrokerStore::QueryResult + k: Broker::Data%): BrokerStore::QueryResult %{ if ( ! broker_mgr->Enabled() ) return bro_broker::query_result(); @@ -505,7 +505,7 @@ function BrokerStore::pop_right%(h: opaque of BrokerStore::Handle, ## ## Returns: the result of the query. function BrokerStore::lookup%(h: opaque of BrokerStore::Handle, - k: BrokerComm::Data%): BrokerStore::QueryResult + k: Broker::Data%): BrokerStore::QueryResult %{ if ( ! broker_mgr->Enabled() ) return bro_broker::query_result(); @@ -533,9 +533,9 @@ function BrokerStore::lookup%(h: opaque of BrokerStore::Handle, ## ## k: the key to check for existence. ## -## Returns: the result of the query (uses :bro:see:`BrokerComm::BOOL`). +## Returns: the result of the query (uses :bro:see:`Broker::BOOL`). function BrokerStore::exists%(h: opaque of BrokerStore::Handle, - k: BrokerComm::Data%): BrokerStore::QueryResult + k: Broker::Data%): BrokerStore::QueryResult %{ if ( ! broker_mgr->Enabled() ) return bro_broker::query_result(); @@ -561,7 +561,7 @@ function BrokerStore::exists%(h: opaque of BrokerStore::Handle, ## ## h: the handle of the store to query. ## -## Returns: the result of the query (uses :bro:see:`BrokerComm::VECTOR`). +## Returns: the result of the query (uses :bro:see:`Broker::VECTOR`). function BrokerStore::keys%(h: opaque of BrokerStore::Handle%): BrokerStore::QueryResult %{ double timeout; @@ -579,7 +579,7 @@ function BrokerStore::keys%(h: opaque of BrokerStore::Handle%): BrokerStore::Que ## ## h: the handle of the store to query. ## -## Returns: the result of the query (uses :bro:see:`BrokerComm::COUNT`). +## Returns: the result of the query (uses :bro:see:`Broker::COUNT`). function BrokerStore::size%(h: opaque of BrokerStore::Handle%): BrokerStore::QueryResult %{ if ( ! broker_mgr->Enabled() ) diff --git a/testing/btest/Baseline/broker.connection_updates/recv.recv.out b/testing/btest/Baseline/broker.connection_updates/recv.recv.out index 714cbfbac4..d246bf153f 100644 --- a/testing/btest/Baseline/broker.connection_updates/recv.recv.out +++ b/testing/btest/Baseline/broker.connection_updates/recv.recv.out @@ -1,2 +1,2 @@ -BrokerComm::incoming_connection_established, connector -BrokerComm::incoming_connection_broken, connector +Broker::incoming_connection_established, connector +Broker::incoming_connection_broken, connector diff --git a/testing/btest/Baseline/broker.connection_updates/send.send.out b/testing/btest/Baseline/broker.connection_updates/send.send.out index 61c988d1c8..205782c8f0 100644 --- a/testing/btest/Baseline/broker.connection_updates/send.send.out +++ b/testing/btest/Baseline/broker.connection_updates/send.send.out @@ -1 +1 @@ -BrokerComm::outgoing_connection_established, 127.0.0.1, 9999/tcp, listener +Broker::outgoing_connection_established, 127.0.0.1, 9999/tcp, listener diff --git a/testing/btest/Baseline/broker.data/out b/testing/btest/Baseline/broker.data/out index 628870144a..281eb9b316 100644 --- a/testing/btest/Baseline/broker.data/out +++ b/testing/btest/Baseline/broker.data/out @@ -1,18 +1,18 @@ -BrokerComm::BOOL -BrokerComm::INT -BrokerComm::COUNT -BrokerComm::DOUBLE -BrokerComm::STRING -BrokerComm::ADDR -BrokerComm::SUBNET -BrokerComm::PORT -BrokerComm::TIME -BrokerComm::INTERVAL -BrokerComm::ENUM -BrokerComm::SET -BrokerComm::TABLE -BrokerComm::VECTOR -BrokerComm::RECORD +Broker::BOOL +Broker::INT +Broker::COUNT +Broker::DOUBLE +Broker::STRING +Broker::ADDR +Broker::SUBNET +Broker::PORT +Broker::TIME +Broker::INTERVAL +Broker::ENUM +Broker::SET +Broker::TABLE +Broker::VECTOR +Broker::RECORD *************************** T F @@ -29,7 +29,7 @@ hello 22/tcp 42.0 180.0 -BrokerComm::BOOL +Broker::BOOL *************************** { two, diff --git a/testing/btest/Baseline/broker.remote_event/send.send.out b/testing/btest/Baseline/broker.remote_event/send.send.out index a29c1ecd1e..2d61135abe 100644 --- a/testing/btest/Baseline/broker.remote_event/send.send.out +++ b/testing/btest/Baseline/broker.remote_event/send.send.out @@ -1,4 +1,4 @@ -BrokerComm::outgoing_connection_established, 127.0.0.1, 9999/tcp +Broker::outgoing_connection_established, 127.0.0.1, 9999/tcp got event msg, pong, 0 got auto event msg, ping, 0 got event msg, pong, 1 diff --git a/testing/btest/Baseline/broker.remote_log/send.send.out b/testing/btest/Baseline/broker.remote_log/send.send.out index d97ef33af1..632279e697 100644 --- a/testing/btest/Baseline/broker.remote_log/send.send.out +++ b/testing/btest/Baseline/broker.remote_log/send.send.out @@ -1 +1 @@ -BrokerComm::outgoing_connection_established, 127.0.0.1, 9999/tcp +Broker::outgoing_connection_established, 127.0.0.1, 9999/tcp diff --git a/testing/btest/Baseline/broker.remote_print/send.send.out b/testing/btest/Baseline/broker.remote_print/send.send.out index 65d8ee79b7..861dd64a8a 100644 --- a/testing/btest/Baseline/broker.remote_print/send.send.out +++ b/testing/btest/Baseline/broker.remote_print/send.send.out @@ -1,4 +1,4 @@ -BrokerComm::outgoing_connection_established, 127.0.0.1, 9999/tcp +Broker::outgoing_connection_established, 127.0.0.1, 9999/tcp got print msg, pong 0 got print msg, pong 1 got print msg, pong 2 diff --git a/testing/btest/Baseline/core.leaks.broker.data/bro..stdout b/testing/btest/Baseline/core.leaks.broker.data/bro..stdout index 628870144a..281eb9b316 100644 --- a/testing/btest/Baseline/core.leaks.broker.data/bro..stdout +++ b/testing/btest/Baseline/core.leaks.broker.data/bro..stdout @@ -1,18 +1,18 @@ -BrokerComm::BOOL -BrokerComm::INT -BrokerComm::COUNT -BrokerComm::DOUBLE -BrokerComm::STRING -BrokerComm::ADDR -BrokerComm::SUBNET -BrokerComm::PORT -BrokerComm::TIME -BrokerComm::INTERVAL -BrokerComm::ENUM -BrokerComm::SET -BrokerComm::TABLE -BrokerComm::VECTOR -BrokerComm::RECORD +Broker::BOOL +Broker::INT +Broker::COUNT +Broker::DOUBLE +Broker::STRING +Broker::ADDR +Broker::SUBNET +Broker::PORT +Broker::TIME +Broker::INTERVAL +Broker::ENUM +Broker::SET +Broker::TABLE +Broker::VECTOR +Broker::RECORD *************************** T F @@ -29,7 +29,7 @@ hello 22/tcp 42.0 180.0 -BrokerComm::BOOL +Broker::BOOL *************************** { two, diff --git a/testing/btest/Baseline/core.leaks.broker.remote_event/send.send.out b/testing/btest/Baseline/core.leaks.broker.remote_event/send.send.out index a29c1ecd1e..2d61135abe 100644 --- a/testing/btest/Baseline/core.leaks.broker.remote_event/send.send.out +++ b/testing/btest/Baseline/core.leaks.broker.remote_event/send.send.out @@ -1,4 +1,4 @@ -BrokerComm::outgoing_connection_established, 127.0.0.1, 9999/tcp +Broker::outgoing_connection_established, 127.0.0.1, 9999/tcp got event msg, pong, 0 got auto event msg, ping, 0 got event msg, pong, 1 diff --git a/testing/btest/Baseline/core.leaks.broker.remote_log/send.send.out b/testing/btest/Baseline/core.leaks.broker.remote_log/send.send.out index d97ef33af1..632279e697 100644 --- a/testing/btest/Baseline/core.leaks.broker.remote_log/send.send.out +++ b/testing/btest/Baseline/core.leaks.broker.remote_log/send.send.out @@ -1 +1 @@ -BrokerComm::outgoing_connection_established, 127.0.0.1, 9999/tcp +Broker::outgoing_connection_established, 127.0.0.1, 9999/tcp diff --git a/testing/btest/Baseline/core.leaks.broker.remote_print/send.send.out b/testing/btest/Baseline/core.leaks.broker.remote_print/send.send.out index 65d8ee79b7..861dd64a8a 100644 --- a/testing/btest/Baseline/core.leaks.broker.remote_print/send.send.out +++ b/testing/btest/Baseline/core.leaks.broker.remote_print/send.send.out @@ -1,4 +1,4 @@ -BrokerComm::outgoing_connection_established, 127.0.0.1, 9999/tcp +Broker::outgoing_connection_established, 127.0.0.1, 9999/tcp got print msg, pong 0 got print msg, pong 1 got print msg, pong 2 diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_connecting-connector_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_connecting-connector_bro/output index 042b8999f3..c4cbde045c 100644 --- a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_connecting-connector_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_connecting-connector_bro/output @@ -4,19 +4,19 @@ connecting-connector.bro const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -redef BrokerComm::endpoint_name = "connector"; +redef Broker::endpoint_name = "connector"; event bro_init() { - BrokerComm::enable(); - BrokerComm::connect("127.0.0.1", broker_port, 1sec); + Broker::enable(); + Broker::connect("127.0.0.1", broker_port, 1sec); } -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { - print "BrokerComm::outgoing_connection_established", + print "Broker::outgoing_connection_established", peer_address, peer_port, peer_name; terminate(); } diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_connecting-listener_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_connecting-listener_bro/output index 33e3df2330..8ea85569c9 100644 --- a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_connecting-listener_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_connecting-listener_bro/output @@ -4,21 +4,21 @@ connecting-listener.bro const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -redef BrokerComm::endpoint_name = "listener"; +redef Broker::endpoint_name = "listener"; event bro_init() { - BrokerComm::enable(); - BrokerComm::listen(broker_port, "127.0.0.1"); + Broker::enable(); + Broker::listen(broker_port, "127.0.0.1"); } -event BrokerComm::incoming_connection_established(peer_name: string) +event Broker::incoming_connection_established(peer_name: string) { - print "BrokerComm::incoming_connection_established", peer_name; + print "Broker::incoming_connection_established", peer_name; } -event BrokerComm::incoming_connection_broken(peer_name: string) +event Broker::incoming_connection_broken(peer_name: string) { - print "BrokerComm::incoming_connection_broken", peer_name; + print "Broker::incoming_connection_broken", peer_name; terminate(); } diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_events-connector_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_events-connector_bro/output index fe97fdb4ce..8a88bde1c2 100644 --- a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_events-connector_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_events-connector_bro/output @@ -4,31 +4,31 @@ events-connector.bro const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -redef BrokerComm::endpoint_name = "connector"; +redef Broker::endpoint_name = "connector"; global my_event: event(msg: string, c: count); global my_auto_event: event(msg: string, c: count); event bro_init() { - BrokerComm::enable(); - BrokerComm::connect("127.0.0.1", broker_port, 1sec); - BrokerComm::auto_event("bro/event/my_auto_event", my_auto_event); + Broker::enable(); + Broker::connect("127.0.0.1", broker_port, 1sec); + Broker::auto_event("bro/event/my_auto_event", my_auto_event); } -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { - print "BrokerComm::outgoing_connection_established", + print "Broker::outgoing_connection_established", peer_address, peer_port, peer_name; - BrokerComm::event("bro/event/my_event", BrokerComm::event_args(my_event, "hi", 0)); + Broker::event("bro/event/my_event", Broker::event_args(my_event, "hi", 0)); event my_auto_event("stuff", 88); - BrokerComm::event("bro/event/my_event", BrokerComm::event_args(my_event, "...", 1)); + Broker::event("bro/event/my_event", Broker::event_args(my_event, "...", 1)); event my_auto_event("more stuff", 51); - BrokerComm::event("bro/event/my_event", BrokerComm::event_args(my_event, "bye", 2)); + Broker::event("bro/event/my_event", Broker::event_args(my_event, "bye", 2)); } -event BrokerComm::outgoing_connection_broken(peer_address: string, +event Broker::outgoing_connection_broken(peer_address: string, peer_port: port) { terminate(); diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_events-listener_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_events-listener_bro/output index 9f004692cb..640722cac0 100644 --- a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_events-listener_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_events-listener_bro/output @@ -4,21 +4,21 @@ events-listener.bro const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -redef BrokerComm::endpoint_name = "listener"; +redef Broker::endpoint_name = "listener"; global msg_count = 0; global my_event: event(msg: string, c: count); global my_auto_event: event(msg: string, c: count); event bro_init() { - BrokerComm::enable(); - BrokerComm::subscribe_to_events("bro/event/"); - BrokerComm::listen(broker_port, "127.0.0.1"); + Broker::enable(); + Broker::subscribe_to_events("bro/event/"); + Broker::listen(broker_port, "127.0.0.1"); } -event BrokerComm::incoming_connection_established(peer_name: string) +event Broker::incoming_connection_established(peer_name: string) { - print "BrokerComm::incoming_connection_established", peer_name; + print "Broker::incoming_connection_established", peer_name; } event my_event(msg: string, c: count) diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_logs-connector_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_logs-connector_bro/output index 6884d5e4d6..907d712c88 100644 --- a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_logs-connector_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_logs-connector_bro/output @@ -6,16 +6,16 @@ logs-connector.bro const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -redef BrokerComm::endpoint_name = "connector"; +redef Broker::endpoint_name = "connector"; redef Log::enable_local_logging = F; redef Log::enable_remote_logging = F; global n = 0; event bro_init() { - BrokerComm::enable(); - BrokerComm::enable_remote_logs(Test::LOG); - BrokerComm::connect("127.0.0.1", broker_port, 1sec); + Broker::enable(); + Broker::enable_remote_logs(Test::LOG); + Broker::connect("127.0.0.1", broker_port, 1sec); } event do_write() @@ -28,16 +28,16 @@ event do_write() event do_write(); } -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { - print "BrokerComm::outgoing_connection_established", + print "Broker::outgoing_connection_established", peer_address, peer_port, peer_name; event do_write(); } -event BrokerComm::outgoing_connection_broken(peer_address: string, +event Broker::outgoing_connection_broken(peer_address: string, peer_port: port) { terminate(); diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_logs-listener_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_logs-listener_bro/output index 1610bde502..de6abbf5a0 100644 --- a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_logs-listener_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_logs-listener_bro/output @@ -6,18 +6,18 @@ logs-listener.bro const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -redef BrokerComm::endpoint_name = "listener"; +redef Broker::endpoint_name = "listener"; event bro_init() { - BrokerComm::enable(); - BrokerComm::subscribe_to_logs("bro/log/Test::LOG"); - BrokerComm::listen(broker_port, "127.0.0.1"); + Broker::enable(); + Broker::subscribe_to_logs("bro/log/Test::LOG"); + Broker::listen(broker_port, "127.0.0.1"); } -event BrokerComm::incoming_connection_established(peer_name: string) +event Broker::incoming_connection_established(peer_name: string) { - print "BrokerComm::incoming_connection_established", peer_name; + print "Broker::incoming_connection_established", peer_name; } event Test::log_test(rec: Test::Info) diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_printing-connector_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_printing-connector_bro/output index 86ad4f459f..f332f6e4ca 100644 --- a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_printing-connector_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_printing-connector_bro/output @@ -4,26 +4,26 @@ printing-connector.bro const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -redef BrokerComm::endpoint_name = "connector"; +redef Broker::endpoint_name = "connector"; event bro_init() { - BrokerComm::enable(); - BrokerComm::connect("127.0.0.1", broker_port, 1sec); + Broker::enable(); + Broker::connect("127.0.0.1", broker_port, 1sec); } -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { - print "BrokerComm::outgoing_connection_established", + print "Broker::outgoing_connection_established", peer_address, peer_port, peer_name; - BrokerComm::print("bro/print/hi", "hello"); - BrokerComm::print("bro/print/stuff", "..."); - BrokerComm::print("bro/print/bye", "goodbye"); + Broker::print("bro/print/hi", "hello"); + Broker::print("bro/print/stuff", "..."); + Broker::print("bro/print/bye", "goodbye"); } -event BrokerComm::outgoing_connection_broken(peer_address: string, +event Broker::outgoing_connection_broken(peer_address: string, peer_port: port) { terminate(); diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_printing-listener_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_printing-listener_bro/output index fb416612ab..37e4d0eae9 100644 --- a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_printing-listener_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_printing-listener_bro/output @@ -4,22 +4,22 @@ printing-listener.bro const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -redef BrokerComm::endpoint_name = "listener"; +redef Broker::endpoint_name = "listener"; global msg_count = 0; event bro_init() { - BrokerComm::enable(); - BrokerComm::subscribe_to_prints("bro/print/"); - BrokerComm::listen(broker_port, "127.0.0.1"); + Broker::enable(); + Broker::subscribe_to_prints("bro/print/"); + Broker::listen(broker_port, "127.0.0.1"); } -event BrokerComm::incoming_connection_established(peer_name: string) +event Broker::incoming_connection_established(peer_name: string) { - print "BrokerComm::incoming_connection_established", peer_name; + print "Broker::incoming_connection_established", peer_name; } -event BrokerComm::print_handler(msg: string) +event Broker::print_handler(msg: string) { ++msg_count; print "got print message", msg; diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_stores-connector_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_stores-connector_bro/output index 6ca9e3b49b..9671878eef 100644 --- a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_stores-connector_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_stores-connector_bro/output @@ -7,38 +7,38 @@ redef exit_only_after_terminate = T; global h: opaque of BrokerStore::Handle; -function dv(d: BrokerComm::Data): BrokerComm::DataVector +function dv(d: Broker::Data): Broker::DataVector { - local rval: BrokerComm::DataVector; + local rval: Broker::DataVector; rval[0] = d; return rval; } global ready: event(); -event BrokerComm::outgoing_connection_broken(peer_address: string, +event Broker::outgoing_connection_broken(peer_address: string, peer_port: port) { terminate(); } -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { local myset: set[string] = {"a", "b", "c"}; local myvec: vector of string = {"alpha", "beta", "gamma"}; h = BrokerStore::create_master("mystore"); - BrokerStore::insert(h, BrokerComm::data("one"), BrokerComm::data(110)); - BrokerStore::insert(h, BrokerComm::data("two"), BrokerComm::data(223)); - BrokerStore::insert(h, BrokerComm::data("myset"), BrokerComm::data(myset)); - BrokerStore::insert(h, BrokerComm::data("myvec"), BrokerComm::data(myvec)); - BrokerStore::increment(h, BrokerComm::data("one")); - BrokerStore::decrement(h, BrokerComm::data("two")); - BrokerStore::add_to_set(h, BrokerComm::data("myset"), BrokerComm::data("d")); - BrokerStore::remove_from_set(h, BrokerComm::data("myset"), BrokerComm::data("b")); - BrokerStore::push_left(h, BrokerComm::data("myvec"), dv(BrokerComm::data("delta"))); - BrokerStore::push_right(h, BrokerComm::data("myvec"), dv(BrokerComm::data("omega"))); + BrokerStore::insert(h, Broker::data("one"), Broker::data(110)); + BrokerStore::insert(h, Broker::data("two"), Broker::data(223)); + BrokerStore::insert(h, Broker::data("myset"), Broker::data(myset)); + BrokerStore::insert(h, Broker::data("myvec"), Broker::data(myvec)); + BrokerStore::increment(h, Broker::data("one")); + BrokerStore::decrement(h, Broker::data("two")); + BrokerStore::add_to_set(h, Broker::data("myset"), Broker::data("d")); + BrokerStore::remove_from_set(h, Broker::data("myset"), Broker::data("b")); + BrokerStore::push_left(h, Broker::data("myvec"), dv(Broker::data("delta"))); + BrokerStore::push_right(h, Broker::data("myvec"), dv(Broker::data("omega"))); when ( local res = BrokerStore::size(h) ) { @@ -51,7 +51,7 @@ event BrokerComm::outgoing_connection_established(peer_address: string, event bro_init() { - BrokerComm::enable(); - BrokerComm::connect("127.0.0.1", broker_port, 1secs); - BrokerComm::auto_event("bro/event/ready", ready); + Broker::enable(); + Broker::connect("127.0.0.1", broker_port, 1secs); + Broker::auto_event("bro/event/ready", ready); } diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_stores-listener_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_stores-listener_bro/output index 6942ec17d2..35ff3dc41a 100644 --- a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_stores-listener_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_stores-listener_bro/output @@ -11,7 +11,7 @@ global key_count = 0; function do_lookup(key: string) { - when ( local res = BrokerStore::lookup(h, BrokerComm::data(key)) ) + when ( local res = BrokerStore::lookup(h, Broker::data(key)) ) { ++key_count; print "lookup", key, res; @@ -30,10 +30,10 @@ event ready() when ( local res = BrokerStore::keys(h) ) { print "clone keys", res; - do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 0))); - do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 1))); - do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 2))); - do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 3))); + do_lookup(Broker::refine_to_string(Broker::vector_lookup(res$result, 0))); + do_lookup(Broker::refine_to_string(Broker::vector_lookup(res$result, 1))); + do_lookup(Broker::refine_to_string(Broker::vector_lookup(res$result, 2))); + do_lookup(Broker::refine_to_string(Broker::vector_lookup(res$result, 3))); } timeout 10sec { print "timeout"; } @@ -41,7 +41,7 @@ event ready() event bro_init() { - BrokerComm::enable(); - BrokerComm::subscribe_to_events("bro/event/ready"); - BrokerComm::listen(broker_port, "127.0.0.1"); + Broker::enable(); + Broker::subscribe_to_events("bro/event/ready"); + Broker::listen(broker_port, "127.0.0.1"); } diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_testlog_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_testlog_bro/output index c87fc3cd6f..d5a92417dc 100644 --- a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_testlog_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_testlog_bro/output @@ -17,6 +17,6 @@ export { event bro_init() &priority=5 { - BrokerComm::enable(); + Broker::enable(); Log::create_stream(Test::LOG, [$columns=Test::Info, $ev=log_test, $path="test"]); } diff --git a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.acld-hook/recv.recv.out b/testing/btest/Baseline/scripts.base.frameworks.netcontrol.acld-hook/recv.recv.out index d36130b29b..d6d5c32fb2 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.acld-hook/recv.recv.out +++ b/testing/btest/Baseline/scripts.base.frameworks.netcontrol.acld-hook/recv.recv.out @@ -1,4 +1,4 @@ -BrokerComm::incoming_connection_established +Broker::incoming_connection_established add_rule, 0, [ty=NetControl::FLOW, conn=, flow=[src_h=192.168.18.50/32, src_p=, dst_h=74.125.239.97/32, dst_p=, src_m=, dst_m=], ip=, mac=], NetControl::DROP, [command=blockhosthost, cookie=2, arg=192.168.18.50 74.125.239.97, comment=here] add_rule, 0, [ty=NetControl::FLOW, conn=, flow=[src_h=, src_p=, dst_h=, dst_p=443/tcp, src_m=, dst_m=], ip=, mac=], NetControl::DROP, [command=droptcpport, cookie=3, arg=443, comment=there] add_rule, 0, [ty=NetControl::ADDRESS, conn=, flow=, ip=192.168.18.50/32, mac=], NetControl::DROP, [command=nullzero, cookie=4, arg=192.168.18.50/32, comment=] diff --git a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.acld-hook/send.send.out b/testing/btest/Baseline/scripts.base.frameworks.netcontrol.acld-hook/send.send.out index fd7f00bb7c..5d8cb431f4 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.acld-hook/send.send.out +++ b/testing/btest/Baseline/scripts.base.frameworks.netcontrol.acld-hook/send.send.out @@ -1,4 +1,4 @@ -BrokerComm::outgoing_connection_established, 127.0.0.1, 9999/tcp +Broker::outgoing_connection_established, 127.0.0.1, 9999/tcp rule added, [ty=NetControl::FLOW, conn=, flow=[src_h=192.168.18.50/32, src_p=, dst_h=74.125.239.97/32, dst_p=, src_m=, dst_m=], ip=, mac=], NetControl::DROP rule added, [ty=NetControl::FLOW, conn=, flow=[src_h=, src_p=, dst_h=, dst_p=443/tcp, src_m=, dst_m=], ip=, mac=], NetControl::DROP rule added, [ty=NetControl::ADDRESS, conn=, flow=, ip=192.168.18.50/32, mac=], NetControl::DROP diff --git a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.acld/recv.recv.out b/testing/btest/Baseline/scripts.base.frameworks.netcontrol.acld/recv.recv.out index 6890484529..f75f20ea28 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.acld/recv.recv.out +++ b/testing/btest/Baseline/scripts.base.frameworks.netcontrol.acld/recv.recv.out @@ -1,4 +1,4 @@ -BrokerComm::incoming_connection_established +Broker::incoming_connection_established add_rule, 0, [ty=NetControl::FLOW, conn=, flow=[src_h=192.168.18.50/32, src_p=, dst_h=74.125.239.97/32, dst_p=, src_m=, dst_m=], ip=, mac=], NetControl::DROP, [command=blockhosthost, cookie=2, arg=192.168.18.50 74.125.239.97, comment=here] add_rule, 0, [ty=NetControl::FLOW, conn=, flow=[src_h=, src_p=, dst_h=, dst_p=443/tcp, src_m=, dst_m=], ip=, mac=], NetControl::DROP, [command=droptcpport, cookie=3, arg=443, comment=there] add_rule, 0, [ty=NetControl::ADDRESS, conn=, flow=, ip=192.168.18.50/32, mac=], NetControl::DROP, [command=drop, cookie=4, arg=192.168.18.50/32, comment=] diff --git a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.acld/send.send.out b/testing/btest/Baseline/scripts.base.frameworks.netcontrol.acld/send.send.out index fd7f00bb7c..5d8cb431f4 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.acld/send.send.out +++ b/testing/btest/Baseline/scripts.base.frameworks.netcontrol.acld/send.send.out @@ -1,4 +1,4 @@ -BrokerComm::outgoing_connection_established, 127.0.0.1, 9999/tcp +Broker::outgoing_connection_established, 127.0.0.1, 9999/tcp rule added, [ty=NetControl::FLOW, conn=, flow=[src_h=192.168.18.50/32, src_p=, dst_h=74.125.239.97/32, dst_p=, src_m=, dst_m=], ip=, mac=], NetControl::DROP rule added, [ty=NetControl::FLOW, conn=, flow=[src_h=, src_p=, dst_h=, dst_p=443/tcp, src_m=, dst_m=], ip=, mac=], NetControl::DROP rule added, [ty=NetControl::ADDRESS, conn=, flow=, ip=192.168.18.50/32, mac=], NetControl::DROP diff --git a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.broker/recv.recv.out b/testing/btest/Baseline/scripts.base.frameworks.netcontrol.broker/recv.recv.out index 3b02eef7c7..74c5f3499c 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.broker/recv.recv.out +++ b/testing/btest/Baseline/scripts.base.frameworks.netcontrol.broker/recv.recv.out @@ -1,4 +1,4 @@ -BrokerComm::incoming_connection_established +Broker::incoming_connection_established add_rule, 0, [ty=NetControl::FLOW, conn=, flow=[src_h=10.10.1.4/32, src_p=1470/tcp, dst_h=74.53.140.153/32, dst_p=25/tcp, src_m=, dst_m=], ip=, mac=], NetControl::DROP add_rule, 0, [ty=NetControl::ADDRESS, conn=, flow=, ip=10.10.1.4/32, mac=], NetControl::DROP remove_rule, 0, [ty=NetControl::FLOW, conn=, flow=[src_h=10.10.1.4/32, src_p=1470/tcp, dst_h=74.53.140.153/32, dst_p=25/tcp, src_m=, dst_m=], ip=, mac=], NetControl::DROP diff --git a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.broker/send.send.out b/testing/btest/Baseline/scripts.base.frameworks.netcontrol.broker/send.send.out index 31d94be31e..fb086ee0e7 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.broker/send.send.out +++ b/testing/btest/Baseline/scripts.base.frameworks.netcontrol.broker/send.send.out @@ -1,4 +1,4 @@ -BrokerComm::outgoing_connection_established, 127.0.0.1, 9999/tcp +Broker::outgoing_connection_established, 127.0.0.1, 9999/tcp rule added, [ty=NetControl::FLOW, conn=, flow=[src_h=10.10.1.4/32, src_p=1470/tcp, dst_h=74.53.140.153/32, dst_p=25/tcp, src_m=, dst_m=], ip=, mac=], NetControl::DROP rule added, [ty=NetControl::ADDRESS, conn=, flow=, ip=10.10.1.4/32, mac=], NetControl::DROP rule timeout, [ty=NetControl::FLOW, conn=, flow=[src_h=10.10.1.4/32, src_p=1470/tcp, dst_h=74.53.140.153/32, dst_p=25/tcp, src_m=, dst_m=], ip=, mac=], NetControl::DROP, [duration=, packet_count=, byte_count=] diff --git a/testing/btest/Baseline/scripts.base.frameworks.openflow.broker-basic/recv.recv.out b/testing/btest/Baseline/scripts.base.frameworks.openflow.broker-basic/recv.recv.out index ec3b038bd9..b1c2ed5050 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.openflow.broker-basic/recv.recv.out +++ b/testing/btest/Baseline/scripts.base.frameworks.openflow.broker-basic/recv.recv.out @@ -1,4 +1,4 @@ -BrokerComm::incoming_connection_established +Broker::incoming_connection_established flow_clear, 42 got flow_mod, 42, [in_port=, dl_src=, dl_dst=, dl_vlan=, dl_vlan_pcp=, dl_type=, nw_tos=, nw_proto=, nw_src=, nw_dst=, tp_src=, tp_dst=], [cookie=4398046511105, table_id=, command=OpenFlow::OFPFC_ADD, idle_timeout=0, hard_timeout=0, priority=0, out_port=, out_group=, flags=0, actions=[out_ports=[3, 7], vlan_vid=, vlan_pcp=, vlan_strip=F, dl_src=, dl_dst=, nw_tos=, nw_src=, nw_dst=, tp_src=, tp_dst=]] got flow_mod, 42, [in_port=, dl_src=, dl_dst=, dl_vlan=, dl_vlan_pcp=, dl_type=2048, nw_tos=, nw_proto=6, nw_src=10.10.1.4/32, nw_dst=74.53.140.153/32, tp_src=1470, tp_dst=25], [cookie=4398046511146, table_id=, command=OpenFlow::OFPFC_ADD, idle_timeout=30, hard_timeout=0, priority=5, out_port=, out_group=, flags=0, actions=[out_ports=[], vlan_vid=, vlan_pcp=, vlan_strip=F, dl_src=, dl_dst=, nw_tos=, nw_src=, nw_dst=, tp_src=, tp_dst=]] diff --git a/testing/btest/Baseline/scripts.base.frameworks.openflow.broker-basic/send.send.out b/testing/btest/Baseline/scripts.base.frameworks.openflow.broker-basic/send.send.out index d81ed49aee..5f4fadfb81 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.openflow.broker-basic/send.send.out +++ b/testing/btest/Baseline/scripts.base.frameworks.openflow.broker-basic/send.send.out @@ -1,4 +1,4 @@ -BrokerComm::outgoing_connection_established, 127.0.0.1, 9999/tcp +Broker::outgoing_connection_established, 127.0.0.1, 9999/tcp Flow_mod_success Flow_mod_failure connection established diff --git a/testing/btest/broker/clone_store.bro b/testing/btest/broker/clone_store.bro index 1973595bab..cfcbc025f1 100644 --- a/testing/btest/broker/clone_store.bro +++ b/testing/btest/broker/clone_store.bro @@ -21,7 +21,7 @@ global query_timeout = 30sec; function do_lookup(key: string) { - when ( local res = BrokerStore::lookup(h, BrokerComm::data(key)) ) + when ( local res = BrokerStore::lookup(h, Broker::data(key)) ) { ++key_count; print "lookup", key, res; @@ -43,10 +43,10 @@ event ready() when ( local res = BrokerStore::keys(h) ) { print "clone keys", res; - do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 0))); - do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 1))); - do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 2))); - do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 3))); + do_lookup(Broker::refine_to_string(Broker::vector_lookup(res$result, 0))); + do_lookup(Broker::refine_to_string(Broker::vector_lookup(res$result, 1))); + do_lookup(Broker::refine_to_string(Broker::vector_lookup(res$result, 2))); + do_lookup(Broker::refine_to_string(Broker::vector_lookup(res$result, 3))); } timeout query_timeout { @@ -57,9 +57,9 @@ event ready() event bro_init() { - BrokerComm::enable(); - BrokerComm::subscribe_to_events("bro/event/ready"); - BrokerComm::listen(broker_port, "127.0.0.1"); + Broker::enable(); + Broker::subscribe_to_events("bro/event/ready"); + Broker::listen(broker_port, "127.0.0.1"); } @TEST-END-FILE @@ -73,38 +73,38 @@ redef exit_only_after_terminate = T; global h: opaque of BrokerStore::Handle; -function dv(d: BrokerComm::Data): BrokerComm::DataVector +function dv(d: Broker::Data): Broker::DataVector { - local rval: BrokerComm::DataVector; + local rval: Broker::DataVector; rval[0] = d; return rval; } global ready: event(); -event BrokerComm::outgoing_connection_broken(peer_address: string, +event Broker::outgoing_connection_broken(peer_address: string, peer_port: port) { terminate(); } -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { local myset: set[string] = {"a", "b", "c"}; local myvec: vector of string = {"alpha", "beta", "gamma"}; h = BrokerStore::create_master("mystore"); - BrokerStore::insert(h, BrokerComm::data("one"), BrokerComm::data(110)); - BrokerStore::insert(h, BrokerComm::data("two"), BrokerComm::data(223)); - BrokerStore::insert(h, BrokerComm::data("myset"), BrokerComm::data(myset)); - BrokerStore::insert(h, BrokerComm::data("myvec"), BrokerComm::data(myvec)); - BrokerStore::increment(h, BrokerComm::data("one")); - BrokerStore::decrement(h, BrokerComm::data("two")); - BrokerStore::add_to_set(h, BrokerComm::data("myset"), BrokerComm::data("d")); - BrokerStore::remove_from_set(h, BrokerComm::data("myset"), BrokerComm::data("b")); - BrokerStore::push_left(h, BrokerComm::data("myvec"), dv(BrokerComm::data("delta"))); - BrokerStore::push_right(h, BrokerComm::data("myvec"), dv(BrokerComm::data("omega"))); + BrokerStore::insert(h, Broker::data("one"), Broker::data(110)); + BrokerStore::insert(h, Broker::data("two"), Broker::data(223)); + BrokerStore::insert(h, Broker::data("myset"), Broker::data(myset)); + BrokerStore::insert(h, Broker::data("myvec"), Broker::data(myvec)); + BrokerStore::increment(h, Broker::data("one")); + BrokerStore::decrement(h, Broker::data("two")); + BrokerStore::add_to_set(h, Broker::data("myset"), Broker::data("d")); + BrokerStore::remove_from_set(h, Broker::data("myset"), Broker::data("b")); + BrokerStore::push_left(h, Broker::data("myvec"), dv(Broker::data("delta"))); + BrokerStore::push_right(h, Broker::data("myvec"), dv(Broker::data("omega"))); when ( local res = BrokerStore::size(h) ) { event ready(); } @@ -117,9 +117,9 @@ event BrokerComm::outgoing_connection_established(peer_address: string, event bro_init() { - BrokerComm::enable(); - BrokerComm::auto_event("bro/event/ready", ready); - BrokerComm::connect("127.0.0.1", broker_port, 1secs); + Broker::enable(); + Broker::auto_event("bro/event/ready", ready); + Broker::connect("127.0.0.1", broker_port, 1secs); } @TEST-END-FILE diff --git a/testing/btest/broker/connection_updates.bro b/testing/btest/broker/connection_updates.bro index 1bbe90ccb5..032049e5ef 100644 --- a/testing/btest/broker/connection_updates.bro +++ b/testing/btest/broker/connection_updates.bro @@ -12,22 +12,22 @@ const broker_port: port &redef; redef exit_only_after_terminate = T; -redef BrokerComm::endpoint_name = "listener"; +redef Broker::endpoint_name = "listener"; event bro_init() { - BrokerComm::enable(); - BrokerComm::listen(broker_port, "127.0.0.1"); + Broker::enable(); + Broker::listen(broker_port, "127.0.0.1"); } -event BrokerComm::incoming_connection_established(peer_name: string) +event Broker::incoming_connection_established(peer_name: string) { - print "BrokerComm::incoming_connection_established", peer_name;; + print "Broker::incoming_connection_established", peer_name;; } -event BrokerComm::incoming_connection_broken(peer_name: string) +event Broker::incoming_connection_broken(peer_name: string) { - print "BrokerComm::incoming_connection_broken", peer_name;; + print "Broker::incoming_connection_broken", peer_name;; terminate(); } @@ -37,19 +37,19 @@ event BrokerComm::incoming_connection_broken(peer_name: string) const broker_port: port &redef; redef exit_only_after_terminate = T; -redef BrokerComm::endpoint_name = "connector"; +redef Broker::endpoint_name = "connector"; event bro_init() { - BrokerComm::enable(); - BrokerComm::connect("127.0.0.1", broker_port, 1sec); + Broker::enable(); + Broker::connect("127.0.0.1", broker_port, 1sec); } -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { - print "BrokerComm::outgoing_connection_established", + print "Broker::outgoing_connection_established", peer_address, peer_port, peer_name;; terminate(); } diff --git a/testing/btest/broker/data.bro b/testing/btest/broker/data.bro index bac7242c85..d8f4c2f8b5 100644 --- a/testing/btest/broker/data.bro +++ b/testing/btest/broker/data.bro @@ -13,210 +13,210 @@ type bro_record : record { c: count; }; -function comm_record_to_bro_record_recurse(it: opaque of BrokerComm::RecordIterator, +function comm_record_to_bro_record_recurse(it: opaque of Broker::RecordIterator, rval: bro_record, idx: count): bro_record { - if ( BrokerComm::record_iterator_last(it) ) + if ( Broker::record_iterator_last(it) ) return rval; - local field_value = BrokerComm::record_iterator_value(it); + local field_value = Broker::record_iterator_value(it); if ( field_value?$d ) switch ( idx ) { case 0: - rval$a = BrokerComm::refine_to_string(field_value); + rval$a = Broker::refine_to_string(field_value); break; case 1: - rval$b = BrokerComm::refine_to_string(field_value); + rval$b = Broker::refine_to_string(field_value); break; case 2: - rval$c = BrokerComm::refine_to_count(field_value); + rval$c = Broker::refine_to_count(field_value); break; }; ++idx; - BrokerComm::record_iterator_next(it); + Broker::record_iterator_next(it); return comm_record_to_bro_record_recurse(it, rval, idx); } -function comm_record_to_bro_record(d: BrokerComm::Data): bro_record +function comm_record_to_bro_record(d: Broker::Data): bro_record { - return comm_record_to_bro_record_recurse(BrokerComm::record_iterator(d), + return comm_record_to_bro_record_recurse(Broker::record_iterator(d), bro_record($c = 0), 0); } function -comm_set_to_bro_set_recurse(it: opaque of BrokerComm::SetIterator, +comm_set_to_bro_set_recurse(it: opaque of Broker::SetIterator, rval: bro_set): bro_set { - if ( BrokerComm::set_iterator_last(it) ) + if ( Broker::set_iterator_last(it) ) return rval; - add rval[BrokerComm::refine_to_string(BrokerComm::set_iterator_value(it))]; - BrokerComm::set_iterator_next(it); + add rval[Broker::refine_to_string(Broker::set_iterator_value(it))]; + Broker::set_iterator_next(it); return comm_set_to_bro_set_recurse(it, rval); } -function comm_set_to_bro_set(d: BrokerComm::Data): bro_set +function comm_set_to_bro_set(d: Broker::Data): bro_set { - return comm_set_to_bro_set_recurse(BrokerComm::set_iterator(d), bro_set()); + return comm_set_to_bro_set_recurse(Broker::set_iterator(d), bro_set()); } function -comm_table_to_bro_table_recurse(it: opaque of BrokerComm::TableIterator, +comm_table_to_bro_table_recurse(it: opaque of Broker::TableIterator, rval: bro_table): bro_table { - if ( BrokerComm::table_iterator_last(it) ) + if ( Broker::table_iterator_last(it) ) return rval; - local item = BrokerComm::table_iterator_value(it); - rval[BrokerComm::refine_to_string(item$key)] = BrokerComm::refine_to_count(item$val); - BrokerComm::table_iterator_next(it); + local item = Broker::table_iterator_value(it); + rval[Broker::refine_to_string(item$key)] = Broker::refine_to_count(item$val); + Broker::table_iterator_next(it); return comm_table_to_bro_table_recurse(it, rval); } -function comm_table_to_bro_table(d: BrokerComm::Data): bro_table +function comm_table_to_bro_table(d: Broker::Data): bro_table { - return comm_table_to_bro_table_recurse(BrokerComm::table_iterator(d), + return comm_table_to_bro_table_recurse(Broker::table_iterator(d), bro_table()); } -function comm_vector_to_bro_vector_recurse(it: opaque of BrokerComm::VectorIterator, +function comm_vector_to_bro_vector_recurse(it: opaque of Broker::VectorIterator, rval: bro_vector): bro_vector { - if ( BrokerComm::vector_iterator_last(it) ) + if ( Broker::vector_iterator_last(it) ) return rval; - rval[|rval|] = BrokerComm::refine_to_string(BrokerComm::vector_iterator_value(it)); - BrokerComm::vector_iterator_next(it); + rval[|rval|] = Broker::refine_to_string(Broker::vector_iterator_value(it)); + Broker::vector_iterator_next(it); return comm_vector_to_bro_vector_recurse(it, rval); } -function comm_vector_to_bro_vector(d: BrokerComm::Data): bro_vector +function comm_vector_to_bro_vector(d: Broker::Data): bro_vector { - return comm_vector_to_bro_vector_recurse(BrokerComm::vector_iterator(d), + return comm_vector_to_bro_vector_recurse(Broker::vector_iterator(d), bro_vector()); } event bro_init() { -BrokerComm::enable(); -print BrokerComm::data_type(BrokerComm::data(T)); -print BrokerComm::data_type(BrokerComm::data(+1)); -print BrokerComm::data_type(BrokerComm::data(1)); -print BrokerComm::data_type(BrokerComm::data(1.1)); -print BrokerComm::data_type(BrokerComm::data("1 (how creative)")); -print BrokerComm::data_type(BrokerComm::data(1.1.1.1)); -print BrokerComm::data_type(BrokerComm::data(1.1.1.1/1)); -print BrokerComm::data_type(BrokerComm::data(1/udp)); -print BrokerComm::data_type(BrokerComm::data(double_to_time(1))); -print BrokerComm::data_type(BrokerComm::data(1sec)); -print BrokerComm::data_type(BrokerComm::data(BrokerComm::BOOL)); +Broker::enable(); +print Broker::data_type(Broker::data(T)); +print Broker::data_type(Broker::data(+1)); +print Broker::data_type(Broker::data(1)); +print Broker::data_type(Broker::data(1.1)); +print Broker::data_type(Broker::data("1 (how creative)")); +print Broker::data_type(Broker::data(1.1.1.1)); +print Broker::data_type(Broker::data(1.1.1.1/1)); +print Broker::data_type(Broker::data(1/udp)); +print Broker::data_type(Broker::data(double_to_time(1))); +print Broker::data_type(Broker::data(1sec)); +print Broker::data_type(Broker::data(Broker::BOOL)); local s: bro_set = bro_set("one", "two", "three"); local t: bro_table = bro_table(["one"] = 1, ["two"] = 2, ["three"] = 3); local v: bro_vector = bro_vector("zero", "one", "two"); local r: bro_record = bro_record($c = 1); -print BrokerComm::data_type(BrokerComm::data(s)); -print BrokerComm::data_type(BrokerComm::data(t)); -print BrokerComm::data_type(BrokerComm::data(v)); -print BrokerComm::data_type(BrokerComm::data(r)); +print Broker::data_type(Broker::data(s)); +print Broker::data_type(Broker::data(t)); +print Broker::data_type(Broker::data(v)); +print Broker::data_type(Broker::data(r)); print "***************************"; -print BrokerComm::refine_to_bool(BrokerComm::data(T)); -print BrokerComm::refine_to_bool(BrokerComm::data(F)); -print BrokerComm::refine_to_int(BrokerComm::data(+1)); -print BrokerComm::refine_to_int(BrokerComm::data(+0)); -print BrokerComm::refine_to_int(BrokerComm::data(-1)); -print BrokerComm::refine_to_count(BrokerComm::data(1)); -print BrokerComm::refine_to_count(BrokerComm::data(0)); -print BrokerComm::refine_to_double(BrokerComm::data(1.1)); -print BrokerComm::refine_to_double(BrokerComm::data(-11.1)); -print BrokerComm::refine_to_string(BrokerComm::data("hello")); -print BrokerComm::refine_to_addr(BrokerComm::data(1.2.3.4)); -print BrokerComm::refine_to_subnet(BrokerComm::data(192.168.1.1/16)); -print BrokerComm::refine_to_port(BrokerComm::data(22/tcp)); -print BrokerComm::refine_to_time(BrokerComm::data(double_to_time(42))); -print BrokerComm::refine_to_interval(BrokerComm::data(3min)); -print BrokerComm::refine_to_enum_name(BrokerComm::data(BrokerComm::BOOL)); +print Broker::refine_to_bool(Broker::data(T)); +print Broker::refine_to_bool(Broker::data(F)); +print Broker::refine_to_int(Broker::data(+1)); +print Broker::refine_to_int(Broker::data(+0)); +print Broker::refine_to_int(Broker::data(-1)); +print Broker::refine_to_count(Broker::data(1)); +print Broker::refine_to_count(Broker::data(0)); +print Broker::refine_to_double(Broker::data(1.1)); +print Broker::refine_to_double(Broker::data(-11.1)); +print Broker::refine_to_string(Broker::data("hello")); +print Broker::refine_to_addr(Broker::data(1.2.3.4)); +print Broker::refine_to_subnet(Broker::data(192.168.1.1/16)); +print Broker::refine_to_port(Broker::data(22/tcp)); +print Broker::refine_to_time(Broker::data(double_to_time(42))); +print Broker::refine_to_interval(Broker::data(3min)); +print Broker::refine_to_enum_name(Broker::data(Broker::BOOL)); print "***************************"; -local cs = BrokerComm::data(s); +local cs = Broker::data(s); print comm_set_to_bro_set(cs); -cs = BrokerComm::set_create(); -print BrokerComm::set_size(cs); -print BrokerComm::set_insert(cs, BrokerComm::data("hi")); -print BrokerComm::set_size(cs); -print BrokerComm::set_contains(cs, BrokerComm::data("hi")); -print BrokerComm::set_contains(cs, BrokerComm::data("bye")); -print BrokerComm::set_insert(cs, BrokerComm::data("bye")); -print BrokerComm::set_size(cs); -print BrokerComm::set_remove(cs, BrokerComm::data("hi")); -print BrokerComm::set_size(cs); -print BrokerComm::set_remove(cs, BrokerComm::data("hi")); +cs = Broker::set_create(); +print Broker::set_size(cs); +print Broker::set_insert(cs, Broker::data("hi")); +print Broker::set_size(cs); +print Broker::set_contains(cs, Broker::data("hi")); +print Broker::set_contains(cs, Broker::data("bye")); +print Broker::set_insert(cs, Broker::data("bye")); +print Broker::set_size(cs); +print Broker::set_remove(cs, Broker::data("hi")); +print Broker::set_size(cs); +print Broker::set_remove(cs, Broker::data("hi")); print comm_set_to_bro_set(cs); -BrokerComm::set_clear(cs); -print BrokerComm::set_size(cs); +Broker::set_clear(cs); +print Broker::set_size(cs); print "***************************"; -local ct = BrokerComm::data(t); +local ct = Broker::data(t); print comm_table_to_bro_table(ct); -ct = BrokerComm::table_create(); -print BrokerComm::table_size(ct); -print BrokerComm::table_insert(ct, BrokerComm::data("hi"), BrokerComm::data(42)); -print BrokerComm::table_size(ct); -print BrokerComm::table_contains(ct, BrokerComm::data("hi")); -print BrokerComm::refine_to_count(BrokerComm::table_lookup(ct, BrokerComm::data("hi"))); -print BrokerComm::table_contains(ct, BrokerComm::data("bye")); -print BrokerComm::table_insert(ct, BrokerComm::data("bye"), BrokerComm::data(7)); -print BrokerComm::table_size(ct); -print BrokerComm::table_insert(ct, BrokerComm::data("bye"), BrokerComm::data(37)); -print BrokerComm::table_size(ct); -print BrokerComm::refine_to_count(BrokerComm::table_lookup(ct, BrokerComm::data("bye"))); -print BrokerComm::table_remove(ct, BrokerComm::data("hi")); -print BrokerComm::table_size(ct); +ct = Broker::table_create(); +print Broker::table_size(ct); +print Broker::table_insert(ct, Broker::data("hi"), Broker::data(42)); +print Broker::table_size(ct); +print Broker::table_contains(ct, Broker::data("hi")); +print Broker::refine_to_count(Broker::table_lookup(ct, Broker::data("hi"))); +print Broker::table_contains(ct, Broker::data("bye")); +print Broker::table_insert(ct, Broker::data("bye"), Broker::data(7)); +print Broker::table_size(ct); +print Broker::table_insert(ct, Broker::data("bye"), Broker::data(37)); +print Broker::table_size(ct); +print Broker::refine_to_count(Broker::table_lookup(ct, Broker::data("bye"))); +print Broker::table_remove(ct, Broker::data("hi")); +print Broker::table_size(ct); print "***************************"; -local cv = BrokerComm::data(v); +local cv = Broker::data(v); print comm_vector_to_bro_vector(cv); -cv = BrokerComm::vector_create(); -print BrokerComm::vector_size(cv); -print BrokerComm::vector_insert(cv, BrokerComm::data("hi"), 0); -print BrokerComm::vector_insert(cv, BrokerComm::data("hello"), 1); -print BrokerComm::vector_insert(cv, BrokerComm::data("greetings"), 2); -print BrokerComm::vector_insert(cv, BrokerComm::data("salutations"), 1); +cv = Broker::vector_create(); +print Broker::vector_size(cv); +print Broker::vector_insert(cv, Broker::data("hi"), 0); +print Broker::vector_insert(cv, Broker::data("hello"), 1); +print Broker::vector_insert(cv, Broker::data("greetings"), 2); +print Broker::vector_insert(cv, Broker::data("salutations"), 1); print comm_vector_to_bro_vector(cv); -print BrokerComm::vector_size(cv); -print BrokerComm::vector_replace(cv, BrokerComm::data("bah"), 2); -print BrokerComm::vector_lookup(cv, 2); -print BrokerComm::vector_lookup(cv, 0); +print Broker::vector_size(cv); +print Broker::vector_replace(cv, Broker::data("bah"), 2); +print Broker::vector_lookup(cv, 2); +print Broker::vector_lookup(cv, 0); print comm_vector_to_bro_vector(cv); -print BrokerComm::vector_remove(cv, 2); +print Broker::vector_remove(cv, 2); print comm_vector_to_bro_vector(cv); -print BrokerComm::vector_size(cv); +print Broker::vector_size(cv); print "***************************"; -local cr = BrokerComm::data(r); +local cr = Broker::data(r); print comm_record_to_bro_record(cr); r$a = "test"; -cr = BrokerComm::data(r); +cr = Broker::data(r); print comm_record_to_bro_record(cr); r$b = "testagain"; -cr = BrokerComm::data(r); +cr = Broker::data(r); print comm_record_to_bro_record(cr); -cr = BrokerComm::record_create(3); -print BrokerComm::record_size(cr); -print BrokerComm::record_assign(cr, BrokerComm::data("hi"), 0); -print BrokerComm::record_assign(cr, BrokerComm::data("hello"), 1); -print BrokerComm::record_assign(cr, BrokerComm::data(37), 2); -print BrokerComm::record_lookup(cr, 0); -print BrokerComm::record_lookup(cr, 1); -print BrokerComm::record_lookup(cr, 2); -print BrokerComm::record_size(cr); +cr = Broker::record_create(3); +print Broker::record_size(cr); +print Broker::record_assign(cr, Broker::data("hi"), 0); +print Broker::record_assign(cr, Broker::data("hello"), 1); +print Broker::record_assign(cr, Broker::data(37), 2); +print Broker::record_lookup(cr, 0); +print Broker::record_lookup(cr, 1); +print Broker::record_lookup(cr, 2); +print Broker::record_size(cr); } diff --git a/testing/btest/broker/enable-and-exit.bro b/testing/btest/broker/enable-and-exit.bro index 9f45672bb6..5a73a71c30 100644 --- a/testing/btest/broker/enable-and-exit.bro +++ b/testing/btest/broker/enable-and-exit.bro @@ -11,7 +11,7 @@ event terminate_me() { } event bro_init() { - BrokerComm::enable(); + Broker::enable(); print "1"; schedule 1sec { terminate_me() }; diff --git a/testing/btest/broker/master_store.bro b/testing/btest/broker/master_store.bro index 2536addc0f..810d303aa1 100644 --- a/testing/btest/broker/master_store.bro +++ b/testing/btest/broker/master_store.bro @@ -66,7 +66,7 @@ event test_keys() event test_pop(key: string) { - when ( local lres = BrokerStore::pop_left(h, BrokerComm::data(key)) ) + when ( local lres = BrokerStore::pop_left(h, Broker::data(key)) ) { print fmt("pop_left(%s): %s", key, lres); ++pop_count; @@ -83,7 +83,7 @@ event test_pop(key: string) event test_keys(); } - when ( local rres = BrokerStore::pop_right(h, BrokerComm::data(key)) ) + when ( local rres = BrokerStore::pop_right(h, Broker::data(key)) ) { print fmt("pop_right(%s): %s", key, rres); ++pop_count; @@ -103,7 +103,7 @@ event test_pop(key: string) function do_exists(key: string) { - when ( local res = BrokerStore::exists(h, BrokerComm::data(key)) ) + when ( local res = BrokerStore::exists(h, Broker::data(key)) ) { print fmt("exists(%s): %s", key, res); ++exists_count; @@ -123,7 +123,7 @@ function do_exists(key: string) event test_erase() { - BrokerStore::erase(h, BrokerComm::data("two")); + BrokerStore::erase(h, Broker::data("two")); do_exists("one"); do_exists("two"); do_exists("myset"); @@ -132,7 +132,7 @@ event test_erase() function do_lookup(key: string) { - when ( local res = BrokerStore::lookup(h, BrokerComm::data(key)) ) + when ( local res = BrokerStore::lookup(h, Broker::data(key)) ) { print fmt("lookup(%s): %s", key, res); ++lookup_count; @@ -150,29 +150,29 @@ function do_lookup(key: string) } } -function dv(d: BrokerComm::Data): BrokerComm::DataVector +function dv(d: Broker::Data): Broker::DataVector { - local rval: BrokerComm::DataVector; + local rval: Broker::DataVector; rval[0] = d; return rval; } event bro_init() { - BrokerComm::enable(); + Broker::enable(); local myset: set[string] = {"a", "b", "c"}; local myvec: vector of string = {"alpha", "beta", "gamma"}; h = BrokerStore::create_master("master"); - BrokerStore::insert(h, BrokerComm::data("one"), BrokerComm::data(110)); - BrokerStore::insert(h, BrokerComm::data("two"), BrokerComm::data(223)); - BrokerStore::insert(h, BrokerComm::data("myset"), BrokerComm::data(myset)); - BrokerStore::insert(h, BrokerComm::data("myvec"), BrokerComm::data(myvec)); - BrokerStore::increment(h, BrokerComm::data("one")); - BrokerStore::decrement(h, BrokerComm::data("two")); - BrokerStore::add_to_set(h, BrokerComm::data("myset"), BrokerComm::data("d")); - BrokerStore::remove_from_set(h, BrokerComm::data("myset"), BrokerComm::data("b")); - BrokerStore::push_left(h, BrokerComm::data("myvec"), dv(BrokerComm::data("delta"))); - BrokerStore::push_right(h, BrokerComm::data("myvec"), dv(BrokerComm::data("omega"))); + BrokerStore::insert(h, Broker::data("one"), Broker::data(110)); + BrokerStore::insert(h, Broker::data("two"), Broker::data(223)); + BrokerStore::insert(h, Broker::data("myset"), Broker::data(myset)); + BrokerStore::insert(h, Broker::data("myvec"), Broker::data(myvec)); + BrokerStore::increment(h, Broker::data("one")); + BrokerStore::decrement(h, Broker::data("two")); + BrokerStore::add_to_set(h, Broker::data("myset"), Broker::data("d")); + BrokerStore::remove_from_set(h, Broker::data("myset"), Broker::data("b")); + BrokerStore::push_left(h, Broker::data("myvec"), dv(Broker::data("delta"))); + BrokerStore::push_right(h, Broker::data("myvec"), dv(Broker::data("omega"))); do_lookup("one"); do_lookup("two"); do_lookup("myset"); diff --git a/testing/btest/broker/remote_event.test b/testing/btest/broker/remote_event.test index 6dbf8e77a0..e18fc3715f 100644 --- a/testing/btest/broker/remote_event.test +++ b/testing/btest/broker/remote_event.test @@ -18,10 +18,10 @@ global auto_event_handler: event(msg: string, c: count); event bro_init() { - BrokerComm::enable(); - BrokerComm::subscribe_to_events("bro/event/"); - BrokerComm::auto_event("bro/event/my_topic", auto_event_handler); - BrokerComm::listen(broker_port, "127.0.0.1"); + Broker::enable(); + Broker::subscribe_to_events("bro/event/"); + Broker::auto_event("bro/event/my_topic", auto_event_handler); + Broker::listen(broker_port, "127.0.0.1"); } global event_count = 0; @@ -39,8 +39,8 @@ event event_handler(msg: string, n: count) } event auto_event_handler(msg, n); - local args = BrokerComm::event_args(event_handler, "pong", n); - BrokerComm::event("bro/event/my_topic", args); + local args = Broker::event_args(event_handler, "pong", n); + Broker::event("bro/event/my_topic", args); } @TEST-END-FILE @@ -55,24 +55,24 @@ global auto_event_handler: event(msg: string, c: count); event bro_init() { - BrokerComm::enable(); - BrokerComm::subscribe_to_events("bro/event/my_topic"); - BrokerComm::connect("127.0.0.1", broker_port, 1secs); + Broker::enable(); + Broker::subscribe_to_events("bro/event/my_topic"); + Broker::connect("127.0.0.1", broker_port, 1secs); } global event_count = 0; -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { - print "BrokerComm::outgoing_connection_established", peer_address, peer_port; - local args = BrokerComm::event_args(event_handler, "ping", event_count); - BrokerComm::event("bro/event/hi", args); + print "Broker::outgoing_connection_established", peer_address, peer_port; + local args = Broker::event_args(event_handler, "ping", event_count); + Broker::event("bro/event/hi", args); ++event_count; } -event BrokerComm::outgoing_connection_broken(peer_address: string, +event Broker::outgoing_connection_broken(peer_address: string, peer_port: port) { terminate(); @@ -81,8 +81,8 @@ event BrokerComm::outgoing_connection_broken(peer_address: string, event event_handler(msg: string, n: count) { print "got event msg", msg, n; - local args = BrokerComm::event_args(event_handler, "ping", event_count); - BrokerComm::event("bro/event/hi", args); + local args = Broker::event_args(event_handler, "ping", event_count); + Broker::event("bro/event/hi", args); ++event_count; } diff --git a/testing/btest/broker/remote_log.test b/testing/btest/broker/remote_log.test index d481f0ae25..52a534c8f9 100644 --- a/testing/btest/broker/remote_log.test +++ b/testing/btest/broker/remote_log.test @@ -28,7 +28,7 @@ export { event bro_init() &priority=5 { - BrokerComm::enable(); + Broker::enable(); Log::create_stream(Test::LOG, [$columns=Test::Info, $ev=log_test]); } @@ -41,8 +41,8 @@ redef exit_only_after_terminate = T; event bro_init() { - BrokerComm::subscribe_to_logs("bro/log/"); - BrokerComm::listen(broker_port, "127.0.0.1"); + Broker::subscribe_to_logs("bro/log/"); + Broker::listen(broker_port, "127.0.0.1"); } event Test::log_test(rec: Test::Info) @@ -62,8 +62,8 @@ redef exit_only_after_terminate = T; event bro_init() { - BrokerComm::enable_remote_logs(Test::LOG); - BrokerComm::connect("127.0.0.1", broker_port, 1secs); + Broker::enable_remote_logs(Test::LOG); + Broker::connect("127.0.0.1", broker_port, 1secs); } global n = 0; @@ -80,15 +80,15 @@ event do_write() } } -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { - print "BrokerComm::outgoing_connection_established", peer_address, peer_port; + print "Broker::outgoing_connection_established", peer_address, peer_port; event do_write(); } -event BrokerComm::outgoing_connection_broken(peer_address: string, +event Broker::outgoing_connection_broken(peer_address: string, peer_port: port) { terminate(); diff --git a/testing/btest/broker/remote_print.test b/testing/btest/broker/remote_print.test index b6430ec3be..a3c06599ae 100644 --- a/testing/btest/broker/remote_print.test +++ b/testing/btest/broker/remote_print.test @@ -15,16 +15,16 @@ redef exit_only_after_terminate = T; event bro_init() { - BrokerComm::enable(); - BrokerComm::listen(broker_port, "127.0.0.1"); - BrokerComm::subscribe_to_prints("bro/print/"); + Broker::enable(); + Broker::listen(broker_port, "127.0.0.1"); + Broker::subscribe_to_prints("bro/print/"); } global messages_to_recv = 6; global messages_sent = 0; global messages_recv = 0; -event BrokerComm::print_handler(msg: string) +event Broker::print_handler(msg: string) { ++messages_recv; print "got print msg", msg; @@ -35,7 +35,7 @@ event BrokerComm::print_handler(msg: string) return; } - BrokerComm::print("bro/print/my_topic", fmt("pong %d", messages_sent)); + Broker::print("bro/print/my_topic", fmt("pong %d", messages_sent)); ++messages_sent; } @@ -48,35 +48,35 @@ redef exit_only_after_terminate = T; event bro_init() { - BrokerComm::enable(); - BrokerComm::subscribe_to_prints("bro/print/my_topic"); - BrokerComm::connect("127.0.0.1", broker_port, 1secs); + Broker::enable(); + Broker::subscribe_to_prints("bro/print/my_topic"); + Broker::connect("127.0.0.1", broker_port, 1secs); } global messages_sent = 0; global messages_recv = 0; global peer_disconnected = F; -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { - print "BrokerComm::outgoing_connection_established", peer_address, peer_port; - BrokerComm::print("bro/print/hi", fmt("ping %d", messages_sent)); + print "Broker::outgoing_connection_established", peer_address, peer_port; + Broker::print("bro/print/hi", fmt("ping %d", messages_sent)); ++messages_sent; } -event BrokerComm::outgoing_connection_broken(peer_address: string, +event Broker::outgoing_connection_broken(peer_address: string, peer_port: port) { terminate(); } -event BrokerComm::print_handler(msg: string) +event Broker::print_handler(msg: string) { ++messages_recv; print "got print msg", msg; - BrokerComm::print("bro/print/hi", fmt("ping %d", messages_sent)); + Broker::print("bro/print/hi", fmt("ping %d", messages_sent)); ++messages_sent; } diff --git a/testing/btest/core/leaks/broker/clone_store.bro b/testing/btest/core/leaks/broker/clone_store.bro index 06df81e1d5..7bed6e43a6 100644 --- a/testing/btest/core/leaks/broker/clone_store.bro +++ b/testing/btest/core/leaks/broker/clone_store.bro @@ -20,7 +20,7 @@ global key_count = 0; function do_lookup(key: string) { - when ( local res = BrokerStore::lookup(h, BrokerComm::data(key)) ) + when ( local res = BrokerStore::lookup(h, Broker::data(key)) ) { ++key_count; print "lookup", key, res; @@ -39,10 +39,10 @@ event ready() when ( local res = BrokerStore::keys(h) ) { print "clone keys", res; - do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 0))); - do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 1))); - do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 2))); - do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 3))); + do_lookup(Broker::refine_to_string(Broker::vector_lookup(res$result, 0))); + do_lookup(Broker::refine_to_string(Broker::vector_lookup(res$result, 1))); + do_lookup(Broker::refine_to_string(Broker::vector_lookup(res$result, 2))); + do_lookup(Broker::refine_to_string(Broker::vector_lookup(res$result, 3))); } timeout 10sec { print "timeout"; } @@ -50,9 +50,9 @@ event ready() event bro_init() { - BrokerComm::enable(); - BrokerComm::listen(broker_port, "127.0.0.1"); - BrokerComm::subscribe_to_events("bro/event/ready"); + Broker::enable(); + Broker::listen(broker_port, "127.0.0.1"); + Broker::subscribe_to_events("bro/event/ready"); } @TEST-END-FILE @@ -64,37 +64,37 @@ redef exit_only_after_terminate = T; global h: opaque of BrokerStore::Handle; -function dv(d: BrokerComm::Data): BrokerComm::DataVector +function dv(d: Broker::Data): Broker::DataVector { - local rval: BrokerComm::DataVector; + local rval: Broker::DataVector; rval[0] = d; return rval; } global ready: event(); -event BrokerComm::outgoing_connection_broken(peer_address: string, +event Broker::outgoing_connection_broken(peer_address: string, peer_port: port) { terminate(); } -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { local myset: set[string] = {"a", "b", "c"}; local myvec: vector of string = {"alpha", "beta", "gamma"}; - BrokerStore::insert(h, BrokerComm::data("one"), BrokerComm::data(110)); - BrokerStore::insert(h, BrokerComm::data("two"), BrokerComm::data(223)); - BrokerStore::insert(h, BrokerComm::data("myset"), BrokerComm::data(myset)); - BrokerStore::insert(h, BrokerComm::data("myvec"), BrokerComm::data(myvec)); - BrokerStore::increment(h, BrokerComm::data("one")); - BrokerStore::decrement(h, BrokerComm::data("two")); - BrokerStore::add_to_set(h, BrokerComm::data("myset"), BrokerComm::data("d")); - BrokerStore::remove_from_set(h, BrokerComm::data("myset"), BrokerComm::data("b")); - BrokerStore::push_left(h, BrokerComm::data("myvec"), dv(BrokerComm::data("delta"))); - BrokerStore::push_right(h, BrokerComm::data("myvec"), dv(BrokerComm::data("omega"))); + BrokerStore::insert(h, Broker::data("one"), Broker::data(110)); + BrokerStore::insert(h, Broker::data("two"), Broker::data(223)); + BrokerStore::insert(h, Broker::data("myset"), Broker::data(myset)); + BrokerStore::insert(h, Broker::data("myvec"), Broker::data(myvec)); + BrokerStore::increment(h, Broker::data("one")); + BrokerStore::decrement(h, Broker::data("two")); + BrokerStore::add_to_set(h, Broker::data("myset"), Broker::data("d")); + BrokerStore::remove_from_set(h, Broker::data("myset"), Broker::data("b")); + BrokerStore::push_left(h, Broker::data("myvec"), dv(Broker::data("delta"))); + BrokerStore::push_right(h, Broker::data("myvec"), dv(Broker::data("omega"))); when ( local res = BrokerStore::size(h) ) { event ready(); } @@ -104,10 +104,10 @@ event BrokerComm::outgoing_connection_established(peer_address: string, event bro_init() { - BrokerComm::enable(); + Broker::enable(); h = BrokerStore::create_master("mystore"); - BrokerComm::connect("127.0.0.1", broker_port, 1secs); - BrokerComm::auto_event("bro/event/ready", ready); + Broker::connect("127.0.0.1", broker_port, 1secs); + Broker::auto_event("bro/event/ready", ready); } @TEST-END-FILE diff --git a/testing/btest/core/leaks/broker/data.bro b/testing/btest/core/leaks/broker/data.bro index d4f6402ae3..0902f6e862 100644 --- a/testing/btest/core/leaks/broker/data.bro +++ b/testing/btest/core/leaks/broker/data.bro @@ -16,96 +16,96 @@ type bro_record : record { c: count; }; -function comm_record_to_bro_record_recurse(it: opaque of BrokerComm::RecordIterator, +function comm_record_to_bro_record_recurse(it: opaque of Broker::RecordIterator, rval: bro_record, idx: count): bro_record { - if ( BrokerComm::record_iterator_last(it) ) + if ( Broker::record_iterator_last(it) ) return rval; - local field_value = BrokerComm::record_iterator_value(it); + local field_value = Broker::record_iterator_value(it); if ( field_value?$d ) switch ( idx ) { case 0: - rval$a = BrokerComm::refine_to_string(field_value); + rval$a = Broker::refine_to_string(field_value); break; case 1: - rval$b = BrokerComm::refine_to_string(field_value); + rval$b = Broker::refine_to_string(field_value); break; case 2: - rval$c = BrokerComm::refine_to_count(field_value); + rval$c = Broker::refine_to_count(field_value); break; }; ++idx; - BrokerComm::record_iterator_next(it); + Broker::record_iterator_next(it); return comm_record_to_bro_record_recurse(it, rval, idx); } -function comm_record_to_bro_record(d: BrokerComm::Data): bro_record +function comm_record_to_bro_record(d: Broker::Data): bro_record { - return comm_record_to_bro_record_recurse(BrokerComm::record_iterator(d), + return comm_record_to_bro_record_recurse(Broker::record_iterator(d), bro_record($c = 0), 0); } function -comm_set_to_bro_set_recurse(it: opaque of BrokerComm::SetIterator, +comm_set_to_bro_set_recurse(it: opaque of Broker::SetIterator, rval: bro_set): bro_set { - if ( BrokerComm::set_iterator_last(it) ) + if ( Broker::set_iterator_last(it) ) return rval; - add rval[BrokerComm::refine_to_string(BrokerComm::set_iterator_value(it))]; - BrokerComm::set_iterator_next(it); + add rval[Broker::refine_to_string(Broker::set_iterator_value(it))]; + Broker::set_iterator_next(it); return comm_set_to_bro_set_recurse(it, rval); } -function comm_set_to_bro_set(d: BrokerComm::Data): bro_set +function comm_set_to_bro_set(d: Broker::Data): bro_set { - return comm_set_to_bro_set_recurse(BrokerComm::set_iterator(d), bro_set()); + return comm_set_to_bro_set_recurse(Broker::set_iterator(d), bro_set()); } function -comm_table_to_bro_table_recurse(it: opaque of BrokerComm::TableIterator, +comm_table_to_bro_table_recurse(it: opaque of Broker::TableIterator, rval: bro_table): bro_table { - if ( BrokerComm::table_iterator_last(it) ) + if ( Broker::table_iterator_last(it) ) return rval; - local item = BrokerComm::table_iterator_value(it); - rval[BrokerComm::refine_to_string(item$key)] = BrokerComm::refine_to_count(item$val); - BrokerComm::table_iterator_next(it); + local item = Broker::table_iterator_value(it); + rval[Broker::refine_to_string(item$key)] = Broker::refine_to_count(item$val); + Broker::table_iterator_next(it); return comm_table_to_bro_table_recurse(it, rval); } -function comm_table_to_bro_table(d: BrokerComm::Data): bro_table +function comm_table_to_bro_table(d: Broker::Data): bro_table { - return comm_table_to_bro_table_recurse(BrokerComm::table_iterator(d), + return comm_table_to_bro_table_recurse(Broker::table_iterator(d), bro_table()); } -function comm_vector_to_bro_vector_recurse(it: opaque of BrokerComm::VectorIterator, +function comm_vector_to_bro_vector_recurse(it: opaque of Broker::VectorIterator, rval: bro_vector): bro_vector { - if ( BrokerComm::vector_iterator_last(it) ) + if ( Broker::vector_iterator_last(it) ) return rval; - rval[|rval|] = BrokerComm::refine_to_string(BrokerComm::vector_iterator_value(it)); - BrokerComm::vector_iterator_next(it); + rval[|rval|] = Broker::refine_to_string(Broker::vector_iterator_value(it)); + Broker::vector_iterator_next(it); return comm_vector_to_bro_vector_recurse(it, rval); } -function comm_vector_to_bro_vector(d: BrokerComm::Data): bro_vector +function comm_vector_to_bro_vector(d: Broker::Data): bro_vector { - return comm_vector_to_bro_vector_recurse(BrokerComm::vector_iterator(d), + return comm_vector_to_bro_vector_recurse(Broker::vector_iterator(d), bro_vector()); } event bro_init() { -BrokerComm::enable(); +Broker::enable(); } global did_it = F; @@ -114,120 +114,120 @@ event new_connection(c: connection) { if ( did_it ) return; did_it = T; -print BrokerComm::data_type(BrokerComm::data(T)); -print BrokerComm::data_type(BrokerComm::data(+1)); -print BrokerComm::data_type(BrokerComm::data(1)); -print BrokerComm::data_type(BrokerComm::data(1.1)); -print BrokerComm::data_type(BrokerComm::data("1 (how creative)")); -print BrokerComm::data_type(BrokerComm::data(1.1.1.1)); -print BrokerComm::data_type(BrokerComm::data(1.1.1.1/1)); -print BrokerComm::data_type(BrokerComm::data(1/udp)); -print BrokerComm::data_type(BrokerComm::data(double_to_time(1))); -print BrokerComm::data_type(BrokerComm::data(1sec)); -print BrokerComm::data_type(BrokerComm::data(BrokerComm::BOOL)); +print Broker::data_type(Broker::data(T)); +print Broker::data_type(Broker::data(+1)); +print Broker::data_type(Broker::data(1)); +print Broker::data_type(Broker::data(1.1)); +print Broker::data_type(Broker::data("1 (how creative)")); +print Broker::data_type(Broker::data(1.1.1.1)); +print Broker::data_type(Broker::data(1.1.1.1/1)); +print Broker::data_type(Broker::data(1/udp)); +print Broker::data_type(Broker::data(double_to_time(1))); +print Broker::data_type(Broker::data(1sec)); +print Broker::data_type(Broker::data(Broker::BOOL)); local s: bro_set = bro_set("one", "two", "three"); local t: bro_table = bro_table(["one"] = 1, ["two"] = 2, ["three"] = 3); local v: bro_vector = bro_vector("zero", "one", "two"); local r: bro_record = bro_record($c = 1); -print BrokerComm::data_type(BrokerComm::data(s)); -print BrokerComm::data_type(BrokerComm::data(t)); -print BrokerComm::data_type(BrokerComm::data(v)); -print BrokerComm::data_type(BrokerComm::data(r)); +print Broker::data_type(Broker::data(s)); +print Broker::data_type(Broker::data(t)); +print Broker::data_type(Broker::data(v)); +print Broker::data_type(Broker::data(r)); print "***************************"; -print BrokerComm::refine_to_bool(BrokerComm::data(T)); -print BrokerComm::refine_to_bool(BrokerComm::data(F)); -print BrokerComm::refine_to_int(BrokerComm::data(+1)); -print BrokerComm::refine_to_int(BrokerComm::data(+0)); -print BrokerComm::refine_to_int(BrokerComm::data(-1)); -print BrokerComm::refine_to_count(BrokerComm::data(1)); -print BrokerComm::refine_to_count(BrokerComm::data(0)); -print BrokerComm::refine_to_double(BrokerComm::data(1.1)); -print BrokerComm::refine_to_double(BrokerComm::data(-11.1)); -print BrokerComm::refine_to_string(BrokerComm::data("hello")); -print BrokerComm::refine_to_addr(BrokerComm::data(1.2.3.4)); -print BrokerComm::refine_to_subnet(BrokerComm::data(192.168.1.1/16)); -print BrokerComm::refine_to_port(BrokerComm::data(22/tcp)); -print BrokerComm::refine_to_time(BrokerComm::data(double_to_time(42))); -print BrokerComm::refine_to_interval(BrokerComm::data(3min)); -print BrokerComm::refine_to_enum_name(BrokerComm::data(BrokerComm::BOOL)); +print Broker::refine_to_bool(Broker::data(T)); +print Broker::refine_to_bool(Broker::data(F)); +print Broker::refine_to_int(Broker::data(+1)); +print Broker::refine_to_int(Broker::data(+0)); +print Broker::refine_to_int(Broker::data(-1)); +print Broker::refine_to_count(Broker::data(1)); +print Broker::refine_to_count(Broker::data(0)); +print Broker::refine_to_double(Broker::data(1.1)); +print Broker::refine_to_double(Broker::data(-11.1)); +print Broker::refine_to_string(Broker::data("hello")); +print Broker::refine_to_addr(Broker::data(1.2.3.4)); +print Broker::refine_to_subnet(Broker::data(192.168.1.1/16)); +print Broker::refine_to_port(Broker::data(22/tcp)); +print Broker::refine_to_time(Broker::data(double_to_time(42))); +print Broker::refine_to_interval(Broker::data(3min)); +print Broker::refine_to_enum_name(Broker::data(Broker::BOOL)); print "***************************"; -local cs = BrokerComm::data(s); +local cs = Broker::data(s); print comm_set_to_bro_set(cs); -cs = BrokerComm::set_create(); -print BrokerComm::set_size(cs); -print BrokerComm::set_insert(cs, BrokerComm::data("hi")); -print BrokerComm::set_size(cs); -print BrokerComm::set_contains(cs, BrokerComm::data("hi")); -print BrokerComm::set_contains(cs, BrokerComm::data("bye")); -print BrokerComm::set_insert(cs, BrokerComm::data("bye")); -print BrokerComm::set_size(cs); -print BrokerComm::set_remove(cs, BrokerComm::data("hi")); -print BrokerComm::set_size(cs); -print BrokerComm::set_remove(cs, BrokerComm::data("hi")); +cs = Broker::set_create(); +print Broker::set_size(cs); +print Broker::set_insert(cs, Broker::data("hi")); +print Broker::set_size(cs); +print Broker::set_contains(cs, Broker::data("hi")); +print Broker::set_contains(cs, Broker::data("bye")); +print Broker::set_insert(cs, Broker::data("bye")); +print Broker::set_size(cs); +print Broker::set_remove(cs, Broker::data("hi")); +print Broker::set_size(cs); +print Broker::set_remove(cs, Broker::data("hi")); print comm_set_to_bro_set(cs); -BrokerComm::set_clear(cs); -print BrokerComm::set_size(cs); +Broker::set_clear(cs); +print Broker::set_size(cs); print "***************************"; -local ct = BrokerComm::data(t); +local ct = Broker::data(t); print comm_table_to_bro_table(ct); -ct = BrokerComm::table_create(); -print BrokerComm::table_size(ct); -print BrokerComm::table_insert(ct, BrokerComm::data("hi"), BrokerComm::data(42)); -print BrokerComm::table_size(ct); -print BrokerComm::table_contains(ct, BrokerComm::data("hi")); -print BrokerComm::refine_to_count(BrokerComm::table_lookup(ct, BrokerComm::data("hi"))); -print BrokerComm::table_contains(ct, BrokerComm::data("bye")); -print BrokerComm::table_insert(ct, BrokerComm::data("bye"), BrokerComm::data(7)); -print BrokerComm::table_size(ct); -print BrokerComm::table_insert(ct, BrokerComm::data("bye"), BrokerComm::data(37)); -print BrokerComm::table_size(ct); -print BrokerComm::refine_to_count(BrokerComm::table_lookup(ct, BrokerComm::data("bye"))); -print BrokerComm::table_remove(ct, BrokerComm::data("hi")); -print BrokerComm::table_size(ct); +ct = Broker::table_create(); +print Broker::table_size(ct); +print Broker::table_insert(ct, Broker::data("hi"), Broker::data(42)); +print Broker::table_size(ct); +print Broker::table_contains(ct, Broker::data("hi")); +print Broker::refine_to_count(Broker::table_lookup(ct, Broker::data("hi"))); +print Broker::table_contains(ct, Broker::data("bye")); +print Broker::table_insert(ct, Broker::data("bye"), Broker::data(7)); +print Broker::table_size(ct); +print Broker::table_insert(ct, Broker::data("bye"), Broker::data(37)); +print Broker::table_size(ct); +print Broker::refine_to_count(Broker::table_lookup(ct, Broker::data("bye"))); +print Broker::table_remove(ct, Broker::data("hi")); +print Broker::table_size(ct); print "***************************"; -local cv = BrokerComm::data(v); +local cv = Broker::data(v); print comm_vector_to_bro_vector(cv); -cv = BrokerComm::vector_create(); -print BrokerComm::vector_size(cv); -print BrokerComm::vector_insert(cv, BrokerComm::data("hi"), 0); -print BrokerComm::vector_insert(cv, BrokerComm::data("hello"), 1); -print BrokerComm::vector_insert(cv, BrokerComm::data("greetings"), 2); -print BrokerComm::vector_insert(cv, BrokerComm::data("salutations"), 1); +cv = Broker::vector_create(); +print Broker::vector_size(cv); +print Broker::vector_insert(cv, Broker::data("hi"), 0); +print Broker::vector_insert(cv, Broker::data("hello"), 1); +print Broker::vector_insert(cv, Broker::data("greetings"), 2); +print Broker::vector_insert(cv, Broker::data("salutations"), 1); print comm_vector_to_bro_vector(cv); -print BrokerComm::vector_size(cv); -print BrokerComm::vector_replace(cv, BrokerComm::data("bah"), 2); -print BrokerComm::vector_lookup(cv, 2); -print BrokerComm::vector_lookup(cv, 0); +print Broker::vector_size(cv); +print Broker::vector_replace(cv, Broker::data("bah"), 2); +print Broker::vector_lookup(cv, 2); +print Broker::vector_lookup(cv, 0); print comm_vector_to_bro_vector(cv); -print BrokerComm::vector_remove(cv, 2); +print Broker::vector_remove(cv, 2); print comm_vector_to_bro_vector(cv); -print BrokerComm::vector_size(cv); +print Broker::vector_size(cv); print "***************************"; -local cr = BrokerComm::data(r); +local cr = Broker::data(r); print comm_record_to_bro_record(cr); r$a = "test"; -cr = BrokerComm::data(r); +cr = Broker::data(r); print comm_record_to_bro_record(cr); r$b = "testagain"; -cr = BrokerComm::data(r); +cr = Broker::data(r); print comm_record_to_bro_record(cr); -cr = BrokerComm::record_create(3); -print BrokerComm::record_size(cr); -print BrokerComm::record_assign(cr, BrokerComm::data("hi"), 0); -print BrokerComm::record_assign(cr, BrokerComm::data("hello"), 1); -print BrokerComm::record_assign(cr, BrokerComm::data(37), 2); -print BrokerComm::record_lookup(cr, 0); -print BrokerComm::record_lookup(cr, 1); -print BrokerComm::record_lookup(cr, 2); -print BrokerComm::record_size(cr); +cr = Broker::record_create(3); +print Broker::record_size(cr); +print Broker::record_assign(cr, Broker::data("hi"), 0); +print Broker::record_assign(cr, Broker::data("hello"), 1); +print Broker::record_assign(cr, Broker::data(37), 2); +print Broker::record_lookup(cr, 0); +print Broker::record_lookup(cr, 1); +print Broker::record_lookup(cr, 2); +print Broker::record_size(cr); } diff --git a/testing/btest/core/leaks/broker/master_store.bro b/testing/btest/core/leaks/broker/master_store.bro index 19c63236f5..b18d75bede 100644 --- a/testing/btest/core/leaks/broker/master_store.bro +++ b/testing/btest/core/leaks/broker/master_store.bro @@ -56,7 +56,7 @@ event test_keys() event test_pop(key: string) { - when ( local lres = BrokerStore::pop_left(h, BrokerComm::data(key)) ) + when ( local lres = BrokerStore::pop_left(h, Broker::data(key)) ) { print fmt("pop_left(%s): %s", key, lres); ++pop_count; @@ -67,7 +67,7 @@ event test_pop(key: string) timeout 10sec { print "timeout"; } - when ( local rres = BrokerStore::pop_right(h, BrokerComm::data(key)) ) + when ( local rres = BrokerStore::pop_right(h, Broker::data(key)) ) { print fmt("pop_right(%s): %s", key, rres); ++pop_count; @@ -81,7 +81,7 @@ event test_pop(key: string) function do_exists(key: string) { - when ( local res = BrokerStore::exists(h, BrokerComm::data(key)) ) + when ( local res = BrokerStore::exists(h, Broker::data(key)) ) { print fmt("exists(%s): %s", key, res); ++exists_count; @@ -95,7 +95,7 @@ function do_exists(key: string) event test_erase() { - BrokerStore::erase(h, BrokerComm::data("two")); + BrokerStore::erase(h, Broker::data("two")); do_exists("one"); do_exists("two"); do_exists("myset"); @@ -104,7 +104,7 @@ event test_erase() function do_lookup(key: string) { - when ( local res = BrokerStore::lookup(h, BrokerComm::data(key)) ) + when ( local res = BrokerStore::lookup(h, Broker::data(key)) ) { print fmt("lookup(%s): %s", key, res); ++lookup_count; @@ -116,9 +116,9 @@ function do_lookup(key: string) { print "timeout"; } } -function dv(d: BrokerComm::Data): BrokerComm::DataVector +function dv(d: Broker::Data): Broker::DataVector { - local rval: BrokerComm::DataVector; + local rval: Broker::DataVector; rval[0] = d; return rval; } @@ -127,7 +127,7 @@ global did_it = F; event bro_init() { - BrokerComm::enable(); + Broker::enable(); h = BrokerStore::create_master("master"); } @@ -137,16 +137,16 @@ event new_connection(c: connection) did_it = T; local myset: set[string] = {"a", "b", "c"}; local myvec: vector of string = {"alpha", "beta", "gamma"}; - BrokerStore::insert(h, BrokerComm::data("one"), BrokerComm::data(110)); - BrokerStore::insert(h, BrokerComm::data("two"), BrokerComm::data(223)); - BrokerStore::insert(h, BrokerComm::data("myset"), BrokerComm::data(myset)); - BrokerStore::insert(h, BrokerComm::data("myvec"), BrokerComm::data(myvec)); - BrokerStore::increment(h, BrokerComm::data("one")); - BrokerStore::decrement(h, BrokerComm::data("two")); - BrokerStore::add_to_set(h, BrokerComm::data("myset"), BrokerComm::data("d")); - BrokerStore::remove_from_set(h, BrokerComm::data("myset"), BrokerComm::data("b")); - BrokerStore::push_left(h, BrokerComm::data("myvec"), dv(BrokerComm::data("delta"))); - BrokerStore::push_right(h, BrokerComm::data("myvec"), dv(BrokerComm::data("omega"))); + BrokerStore::insert(h, Broker::data("one"), Broker::data(110)); + BrokerStore::insert(h, Broker::data("two"), Broker::data(223)); + BrokerStore::insert(h, Broker::data("myset"), Broker::data(myset)); + BrokerStore::insert(h, Broker::data("myvec"), Broker::data(myvec)); + BrokerStore::increment(h, Broker::data("one")); + BrokerStore::decrement(h, Broker::data("two")); + BrokerStore::add_to_set(h, Broker::data("myset"), Broker::data("d")); + BrokerStore::remove_from_set(h, Broker::data("myset"), Broker::data("b")); + BrokerStore::push_left(h, Broker::data("myvec"), dv(Broker::data("delta"))); + BrokerStore::push_right(h, Broker::data("myvec"), dv(Broker::data("omega"))); do_lookup("one"); do_lookup("two"); do_lookup("myset"); diff --git a/testing/btest/core/leaks/broker/remote_event.test b/testing/btest/core/leaks/broker/remote_event.test index 243d3b04d3..731f001f8f 100644 --- a/testing/btest/core/leaks/broker/remote_event.test +++ b/testing/btest/core/leaks/broker/remote_event.test @@ -20,10 +20,10 @@ global auto_event_handler: event(msg: string, c: count); event bro_init() { - BrokerComm::enable(); - BrokerComm::listen(broker_port, "127.0.0.1"); - BrokerComm::subscribe_to_events("bro/event/"); - BrokerComm::auto_event("bro/event/my_topic", auto_event_handler); + Broker::enable(); + Broker::listen(broker_port, "127.0.0.1"); + Broker::subscribe_to_events("bro/event/"); + Broker::auto_event("bro/event/my_topic", auto_event_handler); } global event_count = 0; @@ -41,8 +41,8 @@ event event_handler(msg: string, n: count) } event auto_event_handler(msg, n); - local args = BrokerComm::event_args(event_handler, "pong", n); - BrokerComm::event("bro/event/my_topic", args); + local args = Broker::event_args(event_handler, "pong", n); + Broker::event("bro/event/my_topic", args); } @TEST-END-FILE @@ -57,24 +57,24 @@ global auto_event_handler: event(msg: string, c: count); event bro_init() { - BrokerComm::enable(); - BrokerComm::subscribe_to_events("bro/event/my_topic"); - BrokerComm::connect("127.0.0.1", broker_port, 1secs); + Broker::enable(); + Broker::subscribe_to_events("bro/event/my_topic"); + Broker::connect("127.0.0.1", broker_port, 1secs); } global event_count = 0; -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { - print "BrokerComm::outgoing_connection_established", peer_address, peer_port; - local args = BrokerComm::event_args(event_handler, "ping", event_count); - BrokerComm::event("bro/event/hi", args); + print "Broker::outgoing_connection_established", peer_address, peer_port; + local args = Broker::event_args(event_handler, "ping", event_count); + Broker::event("bro/event/hi", args); ++event_count; } -event BrokerComm::outgoing_connection_broken(peer_address: string, +event Broker::outgoing_connection_broken(peer_address: string, peer_port: port) { terminate(); @@ -83,8 +83,8 @@ event BrokerComm::outgoing_connection_broken(peer_address: string, event event_handler(msg: string, n: count) { print "got event msg", msg, n; - local args = BrokerComm::event_args(event_handler, "ping", event_count); - BrokerComm::event("bro/event/hi", args); + local args = Broker::event_args(event_handler, "ping", event_count); + Broker::event("bro/event/hi", args); ++event_count; } diff --git a/testing/btest/core/leaks/broker/remote_log.test b/testing/btest/core/leaks/broker/remote_log.test index f6c0c41fda..12602115a4 100644 --- a/testing/btest/core/leaks/broker/remote_log.test +++ b/testing/btest/core/leaks/broker/remote_log.test @@ -29,7 +29,7 @@ export { event bro_init() &priority=5 { - BrokerComm::enable(); + Broker::enable(); Log::create_stream(Test::LOG, [$columns=Test::Info, $ev=log_test]); } @@ -42,8 +42,8 @@ redef exit_only_after_terminate = T; event bro_init() { - BrokerComm::listen(broker_port, "127.0.0.1"); - BrokerComm::subscribe_to_logs("bro/log/"); + Broker::listen(broker_port, "127.0.0.1"); + Broker::subscribe_to_logs("bro/log/"); } event Test::log_test(rec: Test::Info) @@ -63,8 +63,8 @@ redef exit_only_after_terminate = T; event bro_init() { - BrokerComm::enable_remote_logs(Test::LOG); - BrokerComm::connect("127.0.0.1", broker_port, 1secs); + Broker::enable_remote_logs(Test::LOG); + Broker::connect("127.0.0.1", broker_port, 1secs); } global n = 0; @@ -81,15 +81,15 @@ event do_write() } } -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { - print "BrokerComm::outgoing_connection_established", peer_address, peer_port; + print "Broker::outgoing_connection_established", peer_address, peer_port; event do_write(); } -event BrokerComm::outgoing_connection_broken(peer_address: string, +event Broker::outgoing_connection_broken(peer_address: string, peer_port: port) { terminate(); diff --git a/testing/btest/core/leaks/broker/remote_print.test b/testing/btest/core/leaks/broker/remote_print.test index e77881c694..623097b091 100644 --- a/testing/btest/core/leaks/broker/remote_print.test +++ b/testing/btest/core/leaks/broker/remote_print.test @@ -17,16 +17,16 @@ redef exit_only_after_terminate = T; event bro_init() { - BrokerComm::enable(); - BrokerComm::listen(broker_port, "127.0.0.1"); - BrokerComm::subscribe_to_prints("bro/print/"); + Broker::enable(); + Broker::listen(broker_port, "127.0.0.1"); + Broker::subscribe_to_prints("bro/print/"); } global messages_to_recv = 6; global messages_sent = 0; global messages_recv = 0; -event BrokerComm::print_handler(msg: string) +event Broker::print_handler(msg: string) { ++messages_recv; print "got print msg", msg; @@ -37,7 +37,7 @@ event BrokerComm::print_handler(msg: string) return; } - BrokerComm::print("bro/print/my_topic", fmt("pong %d", messages_sent)); + Broker::print("bro/print/my_topic", fmt("pong %d", messages_sent)); ++messages_sent; } @@ -50,35 +50,35 @@ redef exit_only_after_terminate = T; event bro_init() { - BrokerComm::enable(); - BrokerComm::subscribe_to_prints("bro/print/my_topic"); - BrokerComm::connect("127.0.0.1", broker_port, 1secs); + Broker::enable(); + Broker::subscribe_to_prints("bro/print/my_topic"); + Broker::connect("127.0.0.1", broker_port, 1secs); } global messages_sent = 0; global messages_recv = 0; global peer_disconnected = F; -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { - print "BrokerComm::outgoing_connection_established", peer_address, peer_port; - BrokerComm::print("bro/print/hi", fmt("ping %d", messages_sent)); + print "Broker::outgoing_connection_established", peer_address, peer_port; + Broker::print("bro/print/hi", fmt("ping %d", messages_sent)); ++messages_sent; } -event BrokerComm::outgoing_connection_broken(peer_address: string, +event Broker::outgoing_connection_broken(peer_address: string, peer_port: port) { terminate(); } -event BrokerComm::print_handler(msg: string) +event Broker::print_handler(msg: string) { ++messages_recv; print "got print msg", msg; - BrokerComm::print("bro/print/hi", fmt("ping %d", messages_sent)); + Broker::print("bro/print/hi", fmt("ping %d", messages_sent)); ++messages_sent; } diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_broker_connecting-connector_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_broker_connecting-connector_bro.btest index 042b8999f3..c4cbde045c 100644 --- a/testing/btest/doc/sphinx/include-doc_frameworks_broker_connecting-connector_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_frameworks_broker_connecting-connector_bro.btest @@ -4,19 +4,19 @@ connecting-connector.bro const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -redef BrokerComm::endpoint_name = "connector"; +redef Broker::endpoint_name = "connector"; event bro_init() { - BrokerComm::enable(); - BrokerComm::connect("127.0.0.1", broker_port, 1sec); + Broker::enable(); + Broker::connect("127.0.0.1", broker_port, 1sec); } -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { - print "BrokerComm::outgoing_connection_established", + print "Broker::outgoing_connection_established", peer_address, peer_port, peer_name; terminate(); } diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_broker_connecting-listener_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_broker_connecting-listener_bro.btest index 33e3df2330..8ea85569c9 100644 --- a/testing/btest/doc/sphinx/include-doc_frameworks_broker_connecting-listener_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_frameworks_broker_connecting-listener_bro.btest @@ -4,21 +4,21 @@ connecting-listener.bro const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -redef BrokerComm::endpoint_name = "listener"; +redef Broker::endpoint_name = "listener"; event bro_init() { - BrokerComm::enable(); - BrokerComm::listen(broker_port, "127.0.0.1"); + Broker::enable(); + Broker::listen(broker_port, "127.0.0.1"); } -event BrokerComm::incoming_connection_established(peer_name: string) +event Broker::incoming_connection_established(peer_name: string) { - print "BrokerComm::incoming_connection_established", peer_name; + print "Broker::incoming_connection_established", peer_name; } -event BrokerComm::incoming_connection_broken(peer_name: string) +event Broker::incoming_connection_broken(peer_name: string) { - print "BrokerComm::incoming_connection_broken", peer_name; + print "Broker::incoming_connection_broken", peer_name; terminate(); } diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_broker_events-connector_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_broker_events-connector_bro.btest index fe97fdb4ce..8a88bde1c2 100644 --- a/testing/btest/doc/sphinx/include-doc_frameworks_broker_events-connector_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_frameworks_broker_events-connector_bro.btest @@ -4,31 +4,31 @@ events-connector.bro const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -redef BrokerComm::endpoint_name = "connector"; +redef Broker::endpoint_name = "connector"; global my_event: event(msg: string, c: count); global my_auto_event: event(msg: string, c: count); event bro_init() { - BrokerComm::enable(); - BrokerComm::connect("127.0.0.1", broker_port, 1sec); - BrokerComm::auto_event("bro/event/my_auto_event", my_auto_event); + Broker::enable(); + Broker::connect("127.0.0.1", broker_port, 1sec); + Broker::auto_event("bro/event/my_auto_event", my_auto_event); } -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { - print "BrokerComm::outgoing_connection_established", + print "Broker::outgoing_connection_established", peer_address, peer_port, peer_name; - BrokerComm::event("bro/event/my_event", BrokerComm::event_args(my_event, "hi", 0)); + Broker::event("bro/event/my_event", Broker::event_args(my_event, "hi", 0)); event my_auto_event("stuff", 88); - BrokerComm::event("bro/event/my_event", BrokerComm::event_args(my_event, "...", 1)); + Broker::event("bro/event/my_event", Broker::event_args(my_event, "...", 1)); event my_auto_event("more stuff", 51); - BrokerComm::event("bro/event/my_event", BrokerComm::event_args(my_event, "bye", 2)); + Broker::event("bro/event/my_event", Broker::event_args(my_event, "bye", 2)); } -event BrokerComm::outgoing_connection_broken(peer_address: string, +event Broker::outgoing_connection_broken(peer_address: string, peer_port: port) { terminate(); diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_broker_events-listener_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_broker_events-listener_bro.btest index 9f004692cb..640722cac0 100644 --- a/testing/btest/doc/sphinx/include-doc_frameworks_broker_events-listener_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_frameworks_broker_events-listener_bro.btest @@ -4,21 +4,21 @@ events-listener.bro const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -redef BrokerComm::endpoint_name = "listener"; +redef Broker::endpoint_name = "listener"; global msg_count = 0; global my_event: event(msg: string, c: count); global my_auto_event: event(msg: string, c: count); event bro_init() { - BrokerComm::enable(); - BrokerComm::subscribe_to_events("bro/event/"); - BrokerComm::listen(broker_port, "127.0.0.1"); + Broker::enable(); + Broker::subscribe_to_events("bro/event/"); + Broker::listen(broker_port, "127.0.0.1"); } -event BrokerComm::incoming_connection_established(peer_name: string) +event Broker::incoming_connection_established(peer_name: string) { - print "BrokerComm::incoming_connection_established", peer_name; + print "Broker::incoming_connection_established", peer_name; } event my_event(msg: string, c: count) diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_broker_logs-connector_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_broker_logs-connector_bro.btest index 6884d5e4d6..907d712c88 100644 --- a/testing/btest/doc/sphinx/include-doc_frameworks_broker_logs-connector_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_frameworks_broker_logs-connector_bro.btest @@ -6,16 +6,16 @@ logs-connector.bro const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -redef BrokerComm::endpoint_name = "connector"; +redef Broker::endpoint_name = "connector"; redef Log::enable_local_logging = F; redef Log::enable_remote_logging = F; global n = 0; event bro_init() { - BrokerComm::enable(); - BrokerComm::enable_remote_logs(Test::LOG); - BrokerComm::connect("127.0.0.1", broker_port, 1sec); + Broker::enable(); + Broker::enable_remote_logs(Test::LOG); + Broker::connect("127.0.0.1", broker_port, 1sec); } event do_write() @@ -28,16 +28,16 @@ event do_write() event do_write(); } -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { - print "BrokerComm::outgoing_connection_established", + print "Broker::outgoing_connection_established", peer_address, peer_port, peer_name; event do_write(); } -event BrokerComm::outgoing_connection_broken(peer_address: string, +event Broker::outgoing_connection_broken(peer_address: string, peer_port: port) { terminate(); diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_broker_logs-listener_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_broker_logs-listener_bro.btest index 1610bde502..de6abbf5a0 100644 --- a/testing/btest/doc/sphinx/include-doc_frameworks_broker_logs-listener_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_frameworks_broker_logs-listener_bro.btest @@ -6,18 +6,18 @@ logs-listener.bro const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -redef BrokerComm::endpoint_name = "listener"; +redef Broker::endpoint_name = "listener"; event bro_init() { - BrokerComm::enable(); - BrokerComm::subscribe_to_logs("bro/log/Test::LOG"); - BrokerComm::listen(broker_port, "127.0.0.1"); + Broker::enable(); + Broker::subscribe_to_logs("bro/log/Test::LOG"); + Broker::listen(broker_port, "127.0.0.1"); } -event BrokerComm::incoming_connection_established(peer_name: string) +event Broker::incoming_connection_established(peer_name: string) { - print "BrokerComm::incoming_connection_established", peer_name; + print "Broker::incoming_connection_established", peer_name; } event Test::log_test(rec: Test::Info) diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_broker_printing-connector_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_broker_printing-connector_bro.btest index 86ad4f459f..f332f6e4ca 100644 --- a/testing/btest/doc/sphinx/include-doc_frameworks_broker_printing-connector_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_frameworks_broker_printing-connector_bro.btest @@ -4,26 +4,26 @@ printing-connector.bro const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -redef BrokerComm::endpoint_name = "connector"; +redef Broker::endpoint_name = "connector"; event bro_init() { - BrokerComm::enable(); - BrokerComm::connect("127.0.0.1", broker_port, 1sec); + Broker::enable(); + Broker::connect("127.0.0.1", broker_port, 1sec); } -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { - print "BrokerComm::outgoing_connection_established", + print "Broker::outgoing_connection_established", peer_address, peer_port, peer_name; - BrokerComm::print("bro/print/hi", "hello"); - BrokerComm::print("bro/print/stuff", "..."); - BrokerComm::print("bro/print/bye", "goodbye"); + Broker::print("bro/print/hi", "hello"); + Broker::print("bro/print/stuff", "..."); + Broker::print("bro/print/bye", "goodbye"); } -event BrokerComm::outgoing_connection_broken(peer_address: string, +event Broker::outgoing_connection_broken(peer_address: string, peer_port: port) { terminate(); diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_broker_printing-listener_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_broker_printing-listener_bro.btest index fb416612ab..37e4d0eae9 100644 --- a/testing/btest/doc/sphinx/include-doc_frameworks_broker_printing-listener_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_frameworks_broker_printing-listener_bro.btest @@ -4,22 +4,22 @@ printing-listener.bro const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -redef BrokerComm::endpoint_name = "listener"; +redef Broker::endpoint_name = "listener"; global msg_count = 0; event bro_init() { - BrokerComm::enable(); - BrokerComm::subscribe_to_prints("bro/print/"); - BrokerComm::listen(broker_port, "127.0.0.1"); + Broker::enable(); + Broker::subscribe_to_prints("bro/print/"); + Broker::listen(broker_port, "127.0.0.1"); } -event BrokerComm::incoming_connection_established(peer_name: string) +event Broker::incoming_connection_established(peer_name: string) { - print "BrokerComm::incoming_connection_established", peer_name; + print "Broker::incoming_connection_established", peer_name; } -event BrokerComm::print_handler(msg: string) +event Broker::print_handler(msg: string) { ++msg_count; print "got print message", msg; diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_broker_stores-connector_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_broker_stores-connector_bro.btest index 6ca9e3b49b..9671878eef 100644 --- a/testing/btest/doc/sphinx/include-doc_frameworks_broker_stores-connector_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_frameworks_broker_stores-connector_bro.btest @@ -7,38 +7,38 @@ redef exit_only_after_terminate = T; global h: opaque of BrokerStore::Handle; -function dv(d: BrokerComm::Data): BrokerComm::DataVector +function dv(d: Broker::Data): Broker::DataVector { - local rval: BrokerComm::DataVector; + local rval: Broker::DataVector; rval[0] = d; return rval; } global ready: event(); -event BrokerComm::outgoing_connection_broken(peer_address: string, +event Broker::outgoing_connection_broken(peer_address: string, peer_port: port) { terminate(); } -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { local myset: set[string] = {"a", "b", "c"}; local myvec: vector of string = {"alpha", "beta", "gamma"}; h = BrokerStore::create_master("mystore"); - BrokerStore::insert(h, BrokerComm::data("one"), BrokerComm::data(110)); - BrokerStore::insert(h, BrokerComm::data("two"), BrokerComm::data(223)); - BrokerStore::insert(h, BrokerComm::data("myset"), BrokerComm::data(myset)); - BrokerStore::insert(h, BrokerComm::data("myvec"), BrokerComm::data(myvec)); - BrokerStore::increment(h, BrokerComm::data("one")); - BrokerStore::decrement(h, BrokerComm::data("two")); - BrokerStore::add_to_set(h, BrokerComm::data("myset"), BrokerComm::data("d")); - BrokerStore::remove_from_set(h, BrokerComm::data("myset"), BrokerComm::data("b")); - BrokerStore::push_left(h, BrokerComm::data("myvec"), dv(BrokerComm::data("delta"))); - BrokerStore::push_right(h, BrokerComm::data("myvec"), dv(BrokerComm::data("omega"))); + BrokerStore::insert(h, Broker::data("one"), Broker::data(110)); + BrokerStore::insert(h, Broker::data("two"), Broker::data(223)); + BrokerStore::insert(h, Broker::data("myset"), Broker::data(myset)); + BrokerStore::insert(h, Broker::data("myvec"), Broker::data(myvec)); + BrokerStore::increment(h, Broker::data("one")); + BrokerStore::decrement(h, Broker::data("two")); + BrokerStore::add_to_set(h, Broker::data("myset"), Broker::data("d")); + BrokerStore::remove_from_set(h, Broker::data("myset"), Broker::data("b")); + BrokerStore::push_left(h, Broker::data("myvec"), dv(Broker::data("delta"))); + BrokerStore::push_right(h, Broker::data("myvec"), dv(Broker::data("omega"))); when ( local res = BrokerStore::size(h) ) { @@ -51,7 +51,7 @@ event BrokerComm::outgoing_connection_established(peer_address: string, event bro_init() { - BrokerComm::enable(); - BrokerComm::connect("127.0.0.1", broker_port, 1secs); - BrokerComm::auto_event("bro/event/ready", ready); + Broker::enable(); + Broker::connect("127.0.0.1", broker_port, 1secs); + Broker::auto_event("bro/event/ready", ready); } diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_broker_stores-listener_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_broker_stores-listener_bro.btest index 6942ec17d2..35ff3dc41a 100644 --- a/testing/btest/doc/sphinx/include-doc_frameworks_broker_stores-listener_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_frameworks_broker_stores-listener_bro.btest @@ -11,7 +11,7 @@ global key_count = 0; function do_lookup(key: string) { - when ( local res = BrokerStore::lookup(h, BrokerComm::data(key)) ) + when ( local res = BrokerStore::lookup(h, Broker::data(key)) ) { ++key_count; print "lookup", key, res; @@ -30,10 +30,10 @@ event ready() when ( local res = BrokerStore::keys(h) ) { print "clone keys", res; - do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 0))); - do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 1))); - do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 2))); - do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 3))); + do_lookup(Broker::refine_to_string(Broker::vector_lookup(res$result, 0))); + do_lookup(Broker::refine_to_string(Broker::vector_lookup(res$result, 1))); + do_lookup(Broker::refine_to_string(Broker::vector_lookup(res$result, 2))); + do_lookup(Broker::refine_to_string(Broker::vector_lookup(res$result, 3))); } timeout 10sec { print "timeout"; } @@ -41,7 +41,7 @@ event ready() event bro_init() { - BrokerComm::enable(); - BrokerComm::subscribe_to_events("bro/event/ready"); - BrokerComm::listen(broker_port, "127.0.0.1"); + Broker::enable(); + Broker::subscribe_to_events("bro/event/ready"); + Broker::listen(broker_port, "127.0.0.1"); } diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_broker_testlog_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_broker_testlog_bro.btest index c87fc3cd6f..d5a92417dc 100644 --- a/testing/btest/doc/sphinx/include-doc_frameworks_broker_testlog_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_frameworks_broker_testlog_bro.btest @@ -17,6 +17,6 @@ export { event bro_init() &priority=5 { - BrokerComm::enable(); + Broker::enable(); Log::create_stream(Test::LOG, [$columns=Test::Info, $ev=log_test, $path="test"]); } diff --git a/testing/btest/scripts/base/frameworks/netcontrol/acld-hook.bro b/testing/btest/scripts/base/frameworks/netcontrol/acld-hook.bro index 566739c0b7..31d5f4df96 100644 --- a/testing/btest/scripts/base/frameworks/netcontrol/acld-hook.bro +++ b/testing/btest/scripts/base/frameworks/netcontrol/acld-hook.bro @@ -26,14 +26,14 @@ event NetControl::init_done() continue_processing(); } -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { - print "BrokerComm::outgoing_connection_established", peer_address, peer_port; + print "Broker::outgoing_connection_established", peer_address, peer_port; } -event BrokerComm::outgoing_connection_broken(peer_address: string, +event Broker::outgoing_connection_broken(peer_address: string, peer_port: port) { terminate(); @@ -91,28 +91,28 @@ redef exit_only_after_terminate = T; event bro_init() { - BrokerComm::enable(); - BrokerComm::subscribe_to_events("bro/event/netcontroltest"); - BrokerComm::listen(broker_port, "127.0.0.1"); + Broker::enable(); + Broker::subscribe_to_events("bro/event/netcontroltest"); + Broker::listen(broker_port, "127.0.0.1"); } -event BrokerComm::incoming_connection_established(peer_name: string) +event Broker::incoming_connection_established(peer_name: string) { - print "BrokerComm::incoming_connection_established"; + print "Broker::incoming_connection_established"; } event NetControl::acld_add_rule(id: count, r: NetControl::Rule, ar: NetControl::AclRule) { print "add_rule", id, r$entity, r$ty, ar; - BrokerComm::event("bro/event/netcontroltest", BrokerComm::event_args(NetControl::acld_rule_added, id, r, ar$command)); + Broker::event("bro/event/netcontroltest", Broker::event_args(NetControl::acld_rule_added, id, r, ar$command)); } event NetControl::acld_remove_rule(id: count, r: NetControl::Rule, ar: NetControl::AclRule) { print "remove_rule", id, r$entity, r$ty, ar; - BrokerComm::event("bro/event/netcontroltest", BrokerComm::event_args(NetControl::acld_rule_removed, id, r, ar$command)); + Broker::event("bro/event/netcontroltest", Broker::event_args(NetControl::acld_rule_removed, id, r, ar$command)); if ( r$cid == 4 ) terminate(); diff --git a/testing/btest/scripts/base/frameworks/netcontrol/acld.bro b/testing/btest/scripts/base/frameworks/netcontrol/acld.bro index dfeaee1055..89743296b1 100644 --- a/testing/btest/scripts/base/frameworks/netcontrol/acld.bro +++ b/testing/btest/scripts/base/frameworks/netcontrol/acld.bro @@ -21,11 +21,11 @@ event NetControl::init() NetControl::activate(netcontrol_acld, 0); } -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { - print "BrokerComm::outgoing_connection_established", peer_address, peer_port; + print "Broker::outgoing_connection_established", peer_address, peer_port; } event NetControl::init_done() @@ -33,7 +33,7 @@ event NetControl::init_done() continue_processing(); } -event BrokerComm::outgoing_connection_broken(peer_address: string, +event Broker::outgoing_connection_broken(peer_address: string, peer_port: port) { terminate(); @@ -84,28 +84,28 @@ redef exit_only_after_terminate = T; event bro_init() { - BrokerComm::enable(); - BrokerComm::subscribe_to_events("bro/event/netcontroltest"); - BrokerComm::listen(broker_port, "127.0.0.1"); + Broker::enable(); + Broker::subscribe_to_events("bro/event/netcontroltest"); + Broker::listen(broker_port, "127.0.0.1"); } -event BrokerComm::incoming_connection_established(peer_name: string) +event Broker::incoming_connection_established(peer_name: string) { - print "BrokerComm::incoming_connection_established"; + print "Broker::incoming_connection_established"; } event NetControl::acld_add_rule(id: count, r: NetControl::Rule, ar: NetControl::AclRule) { print "add_rule", id, r$entity, r$ty, ar; - BrokerComm::event("bro/event/netcontroltest", BrokerComm::event_args(NetControl::acld_rule_added, id, r, ar$command)); + Broker::event("bro/event/netcontroltest", Broker::event_args(NetControl::acld_rule_added, id, r, ar$command)); } event NetControl::acld_remove_rule(id: count, r: NetControl::Rule, ar: NetControl::AclRule) { print "remove_rule", id, r$entity, r$ty, ar; - BrokerComm::event("bro/event/netcontroltest", BrokerComm::event_args(NetControl::acld_rule_removed, id, r, ar$command)); + Broker::event("bro/event/netcontroltest", Broker::event_args(NetControl::acld_rule_removed, id, r, ar$command)); if ( r$cid == 4 ) terminate(); diff --git a/testing/btest/scripts/base/frameworks/netcontrol/broker.bro b/testing/btest/scripts/base/frameworks/netcontrol/broker.bro index 56a76433f2..652f89f4a5 100644 --- a/testing/btest/scripts/base/frameworks/netcontrol/broker.bro +++ b/testing/btest/scripts/base/frameworks/netcontrol/broker.bro @@ -27,14 +27,14 @@ event NetControl::init_done() continue_processing(); } -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { - print "BrokerComm::outgoing_connection_established", peer_address, peer_port; + print "Broker::outgoing_connection_established", peer_address, peer_port; } -event BrokerComm::outgoing_connection_broken(peer_address: string, +event Broker::outgoing_connection_broken(peer_address: string, peer_port: port) { terminate(); @@ -75,29 +75,29 @@ redef exit_only_after_terminate = T; event bro_init() { - BrokerComm::enable(); - BrokerComm::subscribe_to_events("bro/event/netcontroltest"); - BrokerComm::listen(broker_port, "127.0.0.1"); + Broker::enable(); + Broker::subscribe_to_events("bro/event/netcontroltest"); + Broker::listen(broker_port, "127.0.0.1"); } -event BrokerComm::incoming_connection_established(peer_name: string) +event Broker::incoming_connection_established(peer_name: string) { - print "BrokerComm::incoming_connection_established"; + print "Broker::incoming_connection_established"; } event NetControl::broker_add_rule(id: count, r: NetControl::Rule) { print "add_rule", id, r$entity, r$ty; - BrokerComm::event("bro/event/netcontroltest", BrokerComm::event_args(NetControl::broker_rule_added, id, r, "")); + Broker::event("bro/event/netcontroltest", Broker::event_args(NetControl::broker_rule_added, id, r, "")); } event NetControl::broker_remove_rule(id: count, r: NetControl::Rule) { print "remove_rule", id, r$entity, r$ty; - BrokerComm::event("bro/event/netcontroltest", BrokerComm::event_args(NetControl::broker_rule_timeout, id, r, NetControl::FlowInfo())); - BrokerComm::event("bro/event/netcontroltest", BrokerComm::event_args(NetControl::broker_rule_removed, id, r, "")); + Broker::event("bro/event/netcontroltest", Broker::event_args(NetControl::broker_rule_timeout, id, r, NetControl::FlowInfo())); + Broker::event("bro/event/netcontroltest", Broker::event_args(NetControl::broker_rule_removed, id, r, "")); if ( r$cid == 3 ) terminate(); diff --git a/testing/btest/scripts/base/frameworks/openflow/broker-basic.bro b/testing/btest/scripts/base/frameworks/openflow/broker-basic.bro index e973517d44..ed1afb4c3e 100644 --- a/testing/btest/scripts/base/frameworks/openflow/broker-basic.bro +++ b/testing/btest/scripts/base/frameworks/openflow/broker-basic.bro @@ -23,11 +23,11 @@ event bro_init() of_controller = OpenFlow::broker_new("broker1", 127.0.0.1, broker_port, "bro/event/openflow", 42); } -event BrokerComm::outgoing_connection_established(peer_address: string, +event Broker::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { - print "BrokerComm::outgoing_connection_established", peer_address, peer_port; + print "Broker::outgoing_connection_established", peer_address, peer_port; } event OpenFlow::controller_activated(name: string, controller: OpenFlow::Controller) @@ -37,7 +37,7 @@ event OpenFlow::controller_activated(name: string, controller: OpenFlow::Control OpenFlow::flow_mod(of_controller, [], [$cookie=OpenFlow::generate_cookie(1), $command=OpenFlow::OFPFC_ADD, $actions=[$out_ports=vector(3, 7)]]); } -event BrokerComm::outgoing_connection_broken(peer_address: string, +event Broker::outgoing_connection_broken(peer_address: string, peer_port: port) { terminate(); @@ -83,14 +83,14 @@ global msg_count: count = 0; event bro_init() { - BrokerComm::enable(); - BrokerComm::subscribe_to_events("bro/event/openflow"); - BrokerComm::listen(broker_port, "127.0.0.1"); + Broker::enable(); + Broker::subscribe_to_events("bro/event/openflow"); + Broker::listen(broker_port, "127.0.0.1"); } -event BrokerComm::incoming_connection_established(peer_name: string) +event Broker::incoming_connection_established(peer_name: string) { - print "BrokerComm::incoming_connection_established"; + print "Broker::incoming_connection_established"; } function got_message() @@ -104,8 +104,8 @@ function got_message() event OpenFlow::broker_flow_mod(name: string, dpid: count, match: OpenFlow::ofp_match, flow_mod: OpenFlow::ofp_flow_mod) { print "got flow_mod", dpid, match, flow_mod; - BrokerComm::event("bro/event/openflow", BrokerComm::event_args(OpenFlow::flow_mod_success, name, match, flow_mod, "")); - BrokerComm::event("bro/event/openflow", BrokerComm::event_args(OpenFlow::flow_mod_failure, name, match, flow_mod, "")); + Broker::event("bro/event/openflow", Broker::event_args(OpenFlow::flow_mod_success, name, match, flow_mod, "")); + Broker::event("bro/event/openflow", Broker::event_args(OpenFlow::flow_mod_failure, name, match, flow_mod, "")); got_message(); } From f46dfac63ae91327b6c3104de371de13a54d0aa1 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 30 Mar 2016 16:39:19 -0500 Subject: [PATCH 33/84] Rename the BrokerStore namespace to Broker --- doc/frameworks/broker.rst | 4 +- doc/frameworks/broker/stores-connector.bro | 26 +++---- doc/frameworks/broker/stores-listener.bro | 8 +-- scripts/base/frameworks/broker/main.bro | 4 +- src/broker/Manager.cc | 2 +- src/broker/Store.cc | 6 +- src/broker/Store.h | 22 +++--- src/broker/store.bif | 70 +++++++++---------- .../broker.clone_store/clone.clone.out | 10 +-- .../Baseline/broker.master_store/master.out | 28 ++++---- .../clone.clone.out | 10 +-- .../bro..stdout | 28 ++++---- .../output | 26 +++---- .../output | 8 +-- testing/btest/broker/clone_store.bro | 34 ++++----- testing/btest/broker/master_store.bro | 40 +++++------ .../btest/core/leaks/broker/clone_store.bro | 34 ++++----- .../btest/core/leaks/broker/master_store.bro | 40 +++++------ ...ameworks_broker_stores-connector_bro.btest | 26 +++---- ...rameworks_broker_stores-listener_bro.btest | 8 +-- 20 files changed, 217 insertions(+), 217 deletions(-) diff --git a/doc/frameworks/broker.rst b/doc/frameworks/broker.rst index 7b9174909f..328c465c18 100644 --- a/doc/frameworks/broker.rst +++ b/doc/frameworks/broker.rst @@ -192,8 +192,8 @@ last modification time. .. btest-include:: ${DOC_ROOT}/frameworks/broker/stores-connector.bro In the above example, if a local copy of the store contents isn't -needed, just replace the :bro:see:`BrokerStore::create_clone` call with -:bro:see:`BrokerStore::create_frontend`. Queries will then be made against +needed, just replace the :bro:see:`Broker::create_clone` call with +:bro:see:`Broker::create_frontend`. Queries will then be made against the remote master store instead of the local clone. Note that all data store queries must be made within Bro's asynchronous diff --git a/doc/frameworks/broker/stores-connector.bro b/doc/frameworks/broker/stores-connector.bro index b9e9f782fb..d50807cc89 100644 --- a/doc/frameworks/broker/stores-connector.bro +++ b/doc/frameworks/broker/stores-connector.bro @@ -1,7 +1,7 @@ const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -global h: opaque of BrokerStore::Handle; +global h: opaque of Broker::Handle; function dv(d: Broker::Data): Broker::DataVector { @@ -24,19 +24,19 @@ event Broker::outgoing_connection_established(peer_address: string, { local myset: set[string] = {"a", "b", "c"}; local myvec: vector of string = {"alpha", "beta", "gamma"}; - h = BrokerStore::create_master("mystore"); - BrokerStore::insert(h, Broker::data("one"), Broker::data(110)); - BrokerStore::insert(h, Broker::data("two"), Broker::data(223)); - BrokerStore::insert(h, Broker::data("myset"), Broker::data(myset)); - BrokerStore::insert(h, Broker::data("myvec"), Broker::data(myvec)); - BrokerStore::increment(h, Broker::data("one")); - BrokerStore::decrement(h, Broker::data("two")); - BrokerStore::add_to_set(h, Broker::data("myset"), Broker::data("d")); - BrokerStore::remove_from_set(h, Broker::data("myset"), Broker::data("b")); - BrokerStore::push_left(h, Broker::data("myvec"), dv(Broker::data("delta"))); - BrokerStore::push_right(h, Broker::data("myvec"), dv(Broker::data("omega"))); + h = Broker::create_master("mystore"); + Broker::insert(h, Broker::data("one"), Broker::data(110)); + Broker::insert(h, Broker::data("two"), Broker::data(223)); + Broker::insert(h, Broker::data("myset"), Broker::data(myset)); + Broker::insert(h, Broker::data("myvec"), Broker::data(myvec)); + Broker::increment(h, Broker::data("one")); + Broker::decrement(h, Broker::data("two")); + Broker::add_to_set(h, Broker::data("myset"), Broker::data("d")); + Broker::remove_from_set(h, Broker::data("myset"), Broker::data("b")); + Broker::push_left(h, Broker::data("myvec"), dv(Broker::data("delta"))); + Broker::push_right(h, Broker::data("myvec"), dv(Broker::data("omega"))); - when ( local res = BrokerStore::size(h) ) + when ( local res = Broker::size(h) ) { print "master size", res; event ready(); diff --git a/doc/frameworks/broker/stores-listener.bro b/doc/frameworks/broker/stores-listener.bro index b0dc720868..3dac30deca 100644 --- a/doc/frameworks/broker/stores-listener.bro +++ b/doc/frameworks/broker/stores-listener.bro @@ -1,13 +1,13 @@ const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -global h: opaque of BrokerStore::Handle; +global h: opaque of Broker::Handle; global expected_key_count = 4; global key_count = 0; function do_lookup(key: string) { - when ( local res = BrokerStore::lookup(h, Broker::data(key)) ) + when ( local res = Broker::lookup(h, Broker::data(key)) ) { ++key_count; print "lookup", key, res; @@ -21,9 +21,9 @@ function do_lookup(key: string) event ready() { - h = BrokerStore::create_clone("mystore"); + h = Broker::create_clone("mystore"); - when ( local res = BrokerStore::keys(h) ) + when ( local res = Broker::keys(h) ) { print "clone keys", res; do_lookup(Broker::refine_to_string(Broker::vector_lookup(res$result, 0))); diff --git a/scripts/base/frameworks/broker/main.bro b/scripts/base/frameworks/broker/main.bro index ac388b78a1..0270e6a6ba 100644 --- a/scripts/base/frameworks/broker/main.bro +++ b/scripts/base/frameworks/broker/main.bro @@ -54,7 +54,7 @@ export { }; } -module BrokerStore; +module Broker; export { @@ -76,7 +76,7 @@ export { ## The result of a data store query. type QueryResult: record { ## Whether the query completed or not. - status: BrokerStore::QueryStatus; + status: Broker::QueryStatus; ## The result of the query. Certain queries may use a particular ## data type (e.g. querying store size always returns a count, but ## a lookup may return various data types). diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index 62007c8ebb..334b7f84f5 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -89,7 +89,7 @@ bool bro_broker::Manager::Enable(Val* broker_endpoint_flags) bro_broker::opaque_of_table_iterator = new OpaqueType("Broker::TableIterator"); bro_broker::opaque_of_vector_iterator = new OpaqueType("Broker::VectorIterator"); bro_broker::opaque_of_record_iterator = new OpaqueType("Broker::RecordIterator"); - bro_broker::opaque_of_store_handle = new OpaqueType("BrokerStore::Handle"); + bro_broker::opaque_of_store_handle = new OpaqueType("Broker::Handle"); vector_of_data_type = new VectorType(internal_type("Broker::Data")->Ref()); auto res = broker::init(); diff --git a/src/broker/Store.cc b/src/broker/Store.cc index f9effa6d9e..97954bb328 100644 --- a/src/broker/Store.cc +++ b/src/broker/Store.cc @@ -14,12 +14,12 @@ OpaqueType* bro_broker::opaque_of_store_handle; bro_broker::StoreHandleVal::StoreHandleVal(broker::store::identifier id, bro_broker::StoreType arg_type, - broker::util::optional arg_back, + broker::util::optional arg_back, RecordVal* backend_options, std::chrono::duration resync) : OpaqueVal(opaque_of_store_handle), store(), store_type(arg_type), backend_type(arg_back) { - using BifEnum::BrokerStore::BackendType; + using BifEnum::Broker::BackendType; std::unique_ptr backend; if ( backend_type ) @@ -91,7 +91,7 @@ bro_broker::StoreHandleVal::StoreHandleVal(broker::store::identifier id, void bro_broker::StoreHandleVal::ValDescribe(ODesc* d) const { - using BifEnum::BrokerStore::BackendType; + using BifEnum::Broker::BackendType; d->Add("broker::store::"); switch ( store_type ) { diff --git a/src/broker/Store.h b/src/broker/Store.h index 6f31381768..4b673e70dc 100644 --- a/src/broker/Store.h +++ b/src/broker/Store.h @@ -25,9 +25,9 @@ enum StoreType { }; /** - * Create a BrokerStore::QueryStatus value. + * Create a Broker::QueryStatus value. * @param success whether the query status should be set to success or failure. - * @return a BrokerStore::QueryStatus value. + * @return a Broker::QueryStatus value. */ inline EnumVal* query_status(bool success) { @@ -37,21 +37,21 @@ inline EnumVal* query_status(bool success) if ( ! store_query_status ) { - store_query_status = internal_type("BrokerStore::QueryStatus")->AsEnumType(); - success_val = store_query_status->Lookup("BrokerStore", "SUCCESS"); - failure_val = store_query_status->Lookup("BrokerStore", "FAILURE"); + store_query_status = internal_type("Broker::QueryStatus")->AsEnumType(); + success_val = store_query_status->Lookup("Broker", "SUCCESS"); + failure_val = store_query_status->Lookup("Broker", "FAILURE"); } return new EnumVal(success ? success_val : failure_val, store_query_status); } /** - * @return a BrokerStore::QueryResult value that has a BrokerStore::QueryStatus indicating + * @return a Broker::QueryResult value that has a Broker::QueryStatus indicating * a failure. */ inline RecordVal* query_result() { - auto rval = new RecordVal(BifType::Record::BrokerStore::QueryResult); + auto rval = new RecordVal(BifType::Record::Broker::QueryResult); rval->Assign(0, query_status(false)); rval->Assign(1, new RecordVal(BifType::Record::Broker::Data)); return rval; @@ -59,12 +59,12 @@ inline RecordVal* query_result() /** * @param data the result of the query. - * @return a BrokerStore::QueryResult value that has a BrokerStore::QueryStatus indicating + * @return a Broker::QueryResult value that has a Broker::QueryStatus indicating * a success. */ inline RecordVal* query_result(RecordVal* data) { - auto rval = new RecordVal(BifType::Record::BrokerStore::QueryResult); + auto rval = new RecordVal(BifType::Record::Broker::QueryResult); rval->Assign(0, query_status(true)); rval->Assign(1, data); return rval; @@ -130,7 +130,7 @@ public: StoreHandleVal(broker::store::identifier id, bro_broker::StoreType arg_type, - broker::util::optional arg_back, + broker::util::optional arg_back, RecordVal* backend_options, std::chrono::duration resync = std::chrono::seconds(1)); @@ -140,7 +140,7 @@ public: broker::store::frontend* store; bro_broker::StoreType store_type; - broker::util::optional backend_type; + broker::util::optional backend_type; protected: diff --git a/src/broker/store.bif b/src/broker/store.bif index 565dee9e30..57bddd3da7 100644 --- a/src/broker/store.bif +++ b/src/broker/store.bif @@ -8,13 +8,13 @@ #include "Trigger.h" %%} -module BrokerStore; +module Broker; -type BrokerStore::ExpiryTime: record; +type Broker::ExpiryTime: record; -type BrokerStore::QueryResult: record; +type Broker::QueryResult: record; -type BrokerStore::BackendOptions: record; +type Broker::BackendOptions: record; ## Enumerates the possible storage backends. enum BackendType %{ @@ -32,8 +32,8 @@ enum BackendType %{ ## options: tunes how some storage backends operate. ## ## Returns: a handle to the data store. -function BrokerStore::create_master%(id: string, b: BackendType &default = MEMORY, - options: BackendOptions &default = BackendOptions()%): opaque of BrokerStore::Handle +function Broker::create_master%(id: string, b: BackendType &default = MEMORY, + options: BackendOptions &default = BackendOptions()%): opaque of Broker::Handle %{ auto id_str = id->CheckString(); auto type = bro_broker::StoreType::MASTER; @@ -46,7 +46,7 @@ function BrokerStore::create_master%(id: string, b: BackendType &default = MEMOR } rval = new bro_broker::StoreHandleVal(id_str, type, - static_cast(b->AsEnum()), + static_cast(b->AsEnum()), options->AsRecordVal()); auto added = broker_mgr->AddStore(rval); assert(added); @@ -75,9 +75,9 @@ function BrokerStore::create_master%(id: string, b: BackendType &default = MEMOR ## but updates will be lost until the master is once again available. ## ## Returns: a handle to the data store. -function BrokerStore::create_clone%(id: string, b: BackendType &default = MEMORY, +function Broker::create_clone%(id: string, b: BackendType &default = MEMORY, options: BackendOptions &default = BackendOptions(), - resync: interval &default = 1sec%): opaque of BrokerStore::Handle + resync: interval &default = 1sec%): opaque of Broker::Handle %{ auto id_str = id->CheckString(); auto type = bro_broker::StoreType::CLONE; @@ -90,7 +90,7 @@ function BrokerStore::create_clone%(id: string, b: BackendType &default = MEMORY } rval = new bro_broker::StoreHandleVal(id_str, type, - static_cast(b->AsEnum()), + static_cast(b->AsEnum()), options->AsRecordVal(), std::chrono::duration(resync)); auto added = broker_mgr->AddStore(rval); @@ -104,7 +104,7 @@ function BrokerStore::create_clone%(id: string, b: BackendType &default = MEMORY ## id: the unique name which identifies the master data store. ## ## Returns: a handle to the data store. -function BrokerStore::create_frontend%(id: string%): opaque of BrokerStore::Handle +function Broker::create_frontend%(id: string%): opaque of Broker::Handle %{ auto id_str = id->CheckString(); auto type = bro_broker::StoreType::FRONTEND; @@ -128,7 +128,7 @@ function BrokerStore::create_frontend%(id: string%): opaque of BrokerStore::Hand ## ## Returns: true if store was valid and is now closed. The handle can no ## longer be used for data store operations. -function BrokerStore::close_by_handle%(h: opaque of BrokerStore::Handle%): bool +function Broker::close_by_handle%(h: opaque of Broker::Handle%): bool %{ auto handle = static_cast(h); @@ -154,9 +154,9 @@ function BrokerStore::close_by_handle%(h: opaque of BrokerStore::Handle%): bool ## e: the expiration time of the key-value pair. ## ## Returns: false if the store handle was not valid. -function BrokerStore::insert%(h: opaque of BrokerStore::Handle, +function Broker::insert%(h: opaque of Broker::Handle, k: Broker::Data, v: Broker::Data, - e: BrokerStore::ExpiryTime &default = BrokerStore::ExpiryTime()%): bool + e: Broker::ExpiryTime &default = Broker::ExpiryTime()%): bool %{ auto handle = static_cast(h); @@ -198,7 +198,7 @@ function BrokerStore::insert%(h: opaque of BrokerStore::Handle, ## k: the key to remove. ## ## Returns: false if the store handle was not valid. -function BrokerStore::erase%(h: opaque of BrokerStore::Handle, k: Broker::Data%): bool +function Broker::erase%(h: opaque of Broker::Handle, k: Broker::Data%): bool %{ auto handle = static_cast(h); @@ -215,7 +215,7 @@ function BrokerStore::erase%(h: opaque of BrokerStore::Handle, k: Broker::Data%) ## h: the handle of the store to modify. ## ## Returns: false if the store handle was not valid. -function BrokerStore::clear%(h: opaque of BrokerStore::Handle%): bool +function Broker::clear%(h: opaque of Broker::Handle%): bool %{ auto handle = static_cast(h); @@ -236,7 +236,7 @@ function BrokerStore::clear%(h: opaque of BrokerStore::Handle%): bool ## create it with an implicit value of zero before incrementing. ## ## Returns: false if the store handle was not valid. -function BrokerStore::increment%(h: opaque of BrokerStore::Handle, +function Broker::increment%(h: opaque of Broker::Handle, k: Broker::Data, by: int &default = +1%): bool %{ auto handle = static_cast(h); @@ -259,7 +259,7 @@ function BrokerStore::increment%(h: opaque of BrokerStore::Handle, ## create it with an implicit value of zero before decrementing. ## ## Returns: false if the store handle was not valid. -function BrokerStore::decrement%(h: opaque of BrokerStore::Handle, +function Broker::decrement%(h: opaque of Broker::Handle, k: Broker::Data, by: int &default = +1%): bool %{ auto handle = static_cast(h); @@ -282,7 +282,7 @@ function BrokerStore::decrement%(h: opaque of BrokerStore::Handle, ## create it with an implicit empty set value before modifying. ## ## Returns: false if the store handle was not valid. -function BrokerStore::add_to_set%(h: opaque of BrokerStore::Handle, +function Broker::add_to_set%(h: opaque of Broker::Handle, k: Broker::Data, element: Broker::Data%): bool %{ auto handle = static_cast(h); @@ -306,7 +306,7 @@ function BrokerStore::add_to_set%(h: opaque of BrokerStore::Handle, ## implicitly create an empty set value associated with the key. ## ## Returns: false if the store handle was not valid. -function BrokerStore::remove_from_set%(h: opaque of BrokerStore::Handle, +function Broker::remove_from_set%(h: opaque of Broker::Handle, k: Broker::Data, element: Broker::Data%): bool %{ auto handle = static_cast(h); @@ -330,7 +330,7 @@ function BrokerStore::remove_from_set%(h: opaque of BrokerStore::Handle, ## create an empty vector value before modifying. ## ## Returns: false if the store handle was not valid. -function BrokerStore::push_left%(h: opaque of BrokerStore::Handle, k: Broker::Data, +function Broker::push_left%(h: opaque of Broker::Handle, k: Broker::Data, items: Broker::DataVector%): bool %{ auto handle = static_cast(h); @@ -363,7 +363,7 @@ function BrokerStore::push_left%(h: opaque of BrokerStore::Handle, k: Broker::Da ## create an empty vector value before modifying. ## ## Returns: false if the store handle was not valid. -function BrokerStore::push_right%(h: opaque of BrokerStore::Handle, k: Broker::Data, +function Broker::push_right%(h: opaque of Broker::Handle, k: Broker::Data, items: Broker::DataVector%): bool %{ auto handle = static_cast(h); @@ -401,7 +401,7 @@ static bool prepare_for_query(Val* opaque, Frame* frame, if ( ! (*handle)->store ) { reporter->PushLocation(frame->GetCall()->GetLocationInfo()); - reporter->Error("BrokerStore query has an invalid data store"); + reporter->Error("Broker query has an invalid data store"); reporter->PopLocation(); return false; } @@ -411,7 +411,7 @@ static bool prepare_for_query(Val* opaque, Frame* frame, if ( ! trigger ) { reporter->PushLocation(frame->GetCall()->GetLocationInfo()); - reporter->Error("BrokerStore queries can only be called inside when-condition"); + reporter->Error("Broker queries can only be called inside when-condition"); reporter->PopLocation(); return false; } @@ -421,7 +421,7 @@ static bool prepare_for_query(Val* opaque, Frame* frame, if ( *timeout < 0 ) { reporter->PushLocation(frame->GetCall()->GetLocationInfo()); - reporter->Error("BrokerStore queries must specify a timeout block"); + reporter->Error("Broker queries must specify a timeout block"); reporter->PopLocation(); return false; } @@ -444,8 +444,8 @@ static bool prepare_for_query(Val* opaque, Frame* frame, ## k: the key associated with the vector to modify. ## ## Returns: the result of the query. -function BrokerStore::pop_left%(h: opaque of BrokerStore::Handle, - k: Broker::Data%): BrokerStore::QueryResult +function Broker::pop_left%(h: opaque of Broker::Handle, + k: Broker::Data%): Broker::QueryResult %{ if ( ! broker_mgr->Enabled() ) return bro_broker::query_result(); @@ -474,8 +474,8 @@ function BrokerStore::pop_left%(h: opaque of BrokerStore::Handle, ## k: the key associated with the vector to modify. ## ## Returns: the result of the query. -function BrokerStore::pop_right%(h: opaque of BrokerStore::Handle, - k: Broker::Data%): BrokerStore::QueryResult +function Broker::pop_right%(h: opaque of Broker::Handle, + k: Broker::Data%): Broker::QueryResult %{ if ( ! broker_mgr->Enabled() ) return bro_broker::query_result(); @@ -504,8 +504,8 @@ function BrokerStore::pop_right%(h: opaque of BrokerStore::Handle, ## k: the key to lookup. ## ## Returns: the result of the query. -function BrokerStore::lookup%(h: opaque of BrokerStore::Handle, - k: Broker::Data%): BrokerStore::QueryResult +function Broker::lookup%(h: opaque of Broker::Handle, + k: Broker::Data%): Broker::QueryResult %{ if ( ! broker_mgr->Enabled() ) return bro_broker::query_result(); @@ -534,8 +534,8 @@ function BrokerStore::lookup%(h: opaque of BrokerStore::Handle, ## k: the key to check for existence. ## ## Returns: the result of the query (uses :bro:see:`Broker::BOOL`). -function BrokerStore::exists%(h: opaque of BrokerStore::Handle, - k: Broker::Data%): BrokerStore::QueryResult +function Broker::exists%(h: opaque of Broker::Handle, + k: Broker::Data%): Broker::QueryResult %{ if ( ! broker_mgr->Enabled() ) return bro_broker::query_result(); @@ -562,7 +562,7 @@ function BrokerStore::exists%(h: opaque of BrokerStore::Handle, ## h: the handle of the store to query. ## ## Returns: the result of the query (uses :bro:see:`Broker::VECTOR`). -function BrokerStore::keys%(h: opaque of BrokerStore::Handle%): BrokerStore::QueryResult +function Broker::keys%(h: opaque of Broker::Handle%): Broker::QueryResult %{ double timeout; bro_broker::StoreQueryCallback* cb; @@ -580,7 +580,7 @@ function BrokerStore::keys%(h: opaque of BrokerStore::Handle%): BrokerStore::Que ## h: the handle of the store to query. ## ## Returns: the result of the query (uses :bro:see:`Broker::COUNT`). -function BrokerStore::size%(h: opaque of BrokerStore::Handle%): BrokerStore::QueryResult +function Broker::size%(h: opaque of Broker::Handle%): Broker::QueryResult %{ if ( ! broker_mgr->Enabled() ) return bro_broker::query_result(); diff --git a/testing/btest/Baseline/broker.clone_store/clone.clone.out b/testing/btest/Baseline/broker.clone_store/clone.clone.out index 570f3f25ca..3db1dd4e00 100644 --- a/testing/btest/Baseline/broker.clone_store/clone.clone.out +++ b/testing/btest/Baseline/broker.clone_store/clone.clone.out @@ -1,5 +1,5 @@ -clone keys, [status=BrokerStore::SUCCESS, result=[d=broker::data{[one, two, myset, myvec]}]] -lookup, one, [status=BrokerStore::SUCCESS, result=[d=broker::data{111}]] -lookup, myset, [status=BrokerStore::SUCCESS, result=[d=broker::data{{a, c, d}}]] -lookup, two, [status=BrokerStore::SUCCESS, result=[d=broker::data{222}]] -lookup, myvec, [status=BrokerStore::SUCCESS, result=[d=broker::data{[delta, alpha, beta, gamma, omega]}]] +clone keys, [status=Broker::SUCCESS, result=[d=broker::data{[one, two, myset, myvec]}]] +lookup, two, [status=Broker::SUCCESS, result=[d=broker::data{222}]] +lookup, one, [status=Broker::SUCCESS, result=[d=broker::data{111}]] +lookup, myvec, [status=Broker::SUCCESS, result=[d=broker::data{[delta, alpha, beta, gamma, omega]}]] +lookup, myset, [status=Broker::SUCCESS, result=[d=broker::data{{a, c, d}}]] diff --git a/testing/btest/Baseline/broker.master_store/master.out b/testing/btest/Baseline/broker.master_store/master.out index 4208503151..1983d0bccc 100644 --- a/testing/btest/Baseline/broker.master_store/master.out +++ b/testing/btest/Baseline/broker.master_store/master.out @@ -1,14 +1,14 @@ -lookup(two): [status=BrokerStore::SUCCESS, result=[d=broker::data{222}]] -lookup(four): [status=BrokerStore::SUCCESS, result=[d=]] -lookup(myset): [status=BrokerStore::SUCCESS, result=[d=broker::data{{a, c, d}}]] -lookup(one): [status=BrokerStore::SUCCESS, result=[d=broker::data{111}]] -lookup(myvec): [status=BrokerStore::SUCCESS, result=[d=broker::data{[delta, alpha, beta, gamma, omega]}]] -exists(one): [status=BrokerStore::SUCCESS, result=[d=broker::data{1}]] -exists(two): [status=BrokerStore::SUCCESS, result=[d=broker::data{0}]] -exists(myset): [status=BrokerStore::SUCCESS, result=[d=broker::data{1}]] -exists(four): [status=BrokerStore::SUCCESS, result=[d=broker::data{0}]] -pop_right(myvec): [status=BrokerStore::SUCCESS, result=[d=broker::data{omega}]] -pop_left(myvec): [status=BrokerStore::SUCCESS, result=[d=broker::data{delta}]] -keys: [status=BrokerStore::SUCCESS, result=[d=broker::data{[myvec, myset, one]}]] -size: [status=BrokerStore::SUCCESS, result=[d=broker::data{3}]] -size (after clear): [status=BrokerStore::SUCCESS, result=[d=broker::data{0}]] +lookup(two): [status=Broker::SUCCESS, result=[d=broker::data{222}]] +lookup(myset): [status=Broker::SUCCESS, result=[d=broker::data{{a, c, d}}]] +lookup(one): [status=Broker::SUCCESS, result=[d=broker::data{111}]] +lookup(myvec): [status=Broker::SUCCESS, result=[d=broker::data{[delta, alpha, beta, gamma, omega]}]] +lookup(four): [status=Broker::SUCCESS, result=[d=]] +exists(two): [status=Broker::SUCCESS, result=[d=broker::data{0}]] +exists(myset): [status=Broker::SUCCESS, result=[d=broker::data{1}]] +exists(one): [status=Broker::SUCCESS, result=[d=broker::data{1}]] +exists(four): [status=Broker::SUCCESS, result=[d=broker::data{0}]] +pop_left(myvec): [status=Broker::SUCCESS, result=[d=broker::data{delta}]] +pop_right(myvec): [status=Broker::SUCCESS, result=[d=broker::data{omega}]] +keys: [status=Broker::SUCCESS, result=[d=broker::data{[myvec, myset, one]}]] +size: [status=Broker::SUCCESS, result=[d=broker::data{3}]] +size (after clear): [status=Broker::SUCCESS, result=[d=broker::data{0}]] diff --git a/testing/btest/Baseline/core.leaks.broker.clone_store/clone.clone.out b/testing/btest/Baseline/core.leaks.broker.clone_store/clone.clone.out index 017537fea9..ef997abeb8 100644 --- a/testing/btest/Baseline/core.leaks.broker.clone_store/clone.clone.out +++ b/testing/btest/Baseline/core.leaks.broker.clone_store/clone.clone.out @@ -1,5 +1,5 @@ -clone keys, [status=BrokerStore::SUCCESS, result=[d=broker::data{[one, two, myset, myvec]}]] -lookup, one, [status=BrokerStore::SUCCESS, result=[d=broker::data{111}]] -lookup, two, [status=BrokerStore::SUCCESS, result=[d=broker::data{222}]] -lookup, myset, [status=BrokerStore::SUCCESS, result=[d=broker::data{{a, c, d}}]] -lookup, myvec, [status=BrokerStore::SUCCESS, result=[d=broker::data{[delta, alpha, beta, gamma, omega]}]] +clone keys, [status=Broker::SUCCESS, result=[d=broker::data{[one, two, myset, myvec]}]] +lookup, one, [status=Broker::SUCCESS, result=[d=broker::data{111}]] +lookup, two, [status=Broker::SUCCESS, result=[d=broker::data{222}]] +lookup, myset, [status=Broker::SUCCESS, result=[d=broker::data{{a, c, d}}]] +lookup, myvec, [status=Broker::SUCCESS, result=[d=broker::data{[delta, alpha, beta, gamma, omega]}]] diff --git a/testing/btest/Baseline/core.leaks.broker.master_store/bro..stdout b/testing/btest/Baseline/core.leaks.broker.master_store/bro..stdout index 4208503151..9eebc797e5 100644 --- a/testing/btest/Baseline/core.leaks.broker.master_store/bro..stdout +++ b/testing/btest/Baseline/core.leaks.broker.master_store/bro..stdout @@ -1,14 +1,14 @@ -lookup(two): [status=BrokerStore::SUCCESS, result=[d=broker::data{222}]] -lookup(four): [status=BrokerStore::SUCCESS, result=[d=]] -lookup(myset): [status=BrokerStore::SUCCESS, result=[d=broker::data{{a, c, d}}]] -lookup(one): [status=BrokerStore::SUCCESS, result=[d=broker::data{111}]] -lookup(myvec): [status=BrokerStore::SUCCESS, result=[d=broker::data{[delta, alpha, beta, gamma, omega]}]] -exists(one): [status=BrokerStore::SUCCESS, result=[d=broker::data{1}]] -exists(two): [status=BrokerStore::SUCCESS, result=[d=broker::data{0}]] -exists(myset): [status=BrokerStore::SUCCESS, result=[d=broker::data{1}]] -exists(four): [status=BrokerStore::SUCCESS, result=[d=broker::data{0}]] -pop_right(myvec): [status=BrokerStore::SUCCESS, result=[d=broker::data{omega}]] -pop_left(myvec): [status=BrokerStore::SUCCESS, result=[d=broker::data{delta}]] -keys: [status=BrokerStore::SUCCESS, result=[d=broker::data{[myvec, myset, one]}]] -size: [status=BrokerStore::SUCCESS, result=[d=broker::data{3}]] -size (after clear): [status=BrokerStore::SUCCESS, result=[d=broker::data{0}]] +lookup(two): [status=Broker::SUCCESS, result=[d=broker::data{222}]] +lookup(four): [status=Broker::SUCCESS, result=[d=]] +lookup(myset): [status=Broker::SUCCESS, result=[d=broker::data{{a, c, d}}]] +lookup(one): [status=Broker::SUCCESS, result=[d=broker::data{111}]] +lookup(myvec): [status=Broker::SUCCESS, result=[d=broker::data{[delta, alpha, beta, gamma, omega]}]] +exists(one): [status=Broker::SUCCESS, result=[d=broker::data{1}]] +exists(two): [status=Broker::SUCCESS, result=[d=broker::data{0}]] +exists(myset): [status=Broker::SUCCESS, result=[d=broker::data{1}]] +exists(four): [status=Broker::SUCCESS, result=[d=broker::data{0}]] +pop_right(myvec): [status=Broker::SUCCESS, result=[d=broker::data{omega}]] +pop_left(myvec): [status=Broker::SUCCESS, result=[d=broker::data{delta}]] +keys: [status=Broker::SUCCESS, result=[d=broker::data{[myvec, myset, one]}]] +size: [status=Broker::SUCCESS, result=[d=broker::data{3}]] +size (after clear): [status=Broker::SUCCESS, result=[d=broker::data{0}]] diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_stores-connector_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_stores-connector_bro/output index 9671878eef..74b59467e7 100644 --- a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_stores-connector_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_stores-connector_bro/output @@ -5,7 +5,7 @@ stores-connector.bro const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -global h: opaque of BrokerStore::Handle; +global h: opaque of Broker::Handle; function dv(d: Broker::Data): Broker::DataVector { @@ -28,19 +28,19 @@ event Broker::outgoing_connection_established(peer_address: string, { local myset: set[string] = {"a", "b", "c"}; local myvec: vector of string = {"alpha", "beta", "gamma"}; - h = BrokerStore::create_master("mystore"); - BrokerStore::insert(h, Broker::data("one"), Broker::data(110)); - BrokerStore::insert(h, Broker::data("two"), Broker::data(223)); - BrokerStore::insert(h, Broker::data("myset"), Broker::data(myset)); - BrokerStore::insert(h, Broker::data("myvec"), Broker::data(myvec)); - BrokerStore::increment(h, Broker::data("one")); - BrokerStore::decrement(h, Broker::data("two")); - BrokerStore::add_to_set(h, Broker::data("myset"), Broker::data("d")); - BrokerStore::remove_from_set(h, Broker::data("myset"), Broker::data("b")); - BrokerStore::push_left(h, Broker::data("myvec"), dv(Broker::data("delta"))); - BrokerStore::push_right(h, Broker::data("myvec"), dv(Broker::data("omega"))); + h = Broker::create_master("mystore"); + Broker::insert(h, Broker::data("one"), Broker::data(110)); + Broker::insert(h, Broker::data("two"), Broker::data(223)); + Broker::insert(h, Broker::data("myset"), Broker::data(myset)); + Broker::insert(h, Broker::data("myvec"), Broker::data(myvec)); + Broker::increment(h, Broker::data("one")); + Broker::decrement(h, Broker::data("two")); + Broker::add_to_set(h, Broker::data("myset"), Broker::data("d")); + Broker::remove_from_set(h, Broker::data("myset"), Broker::data("b")); + Broker::push_left(h, Broker::data("myvec"), dv(Broker::data("delta"))); + Broker::push_right(h, Broker::data("myvec"), dv(Broker::data("omega"))); - when ( local res = BrokerStore::size(h) ) + when ( local res = Broker::size(h) ) { print "master size", res; event ready(); diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_stores-listener_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_stores-listener_bro/output index 35ff3dc41a..8dadbc803c 100644 --- a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_stores-listener_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_stores-listener_bro/output @@ -5,13 +5,13 @@ stores-listener.bro const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -global h: opaque of BrokerStore::Handle; +global h: opaque of Broker::Handle; global expected_key_count = 4; global key_count = 0; function do_lookup(key: string) { - when ( local res = BrokerStore::lookup(h, Broker::data(key)) ) + when ( local res = Broker::lookup(h, Broker::data(key)) ) { ++key_count; print "lookup", key, res; @@ -25,9 +25,9 @@ function do_lookup(key: string) event ready() { - h = BrokerStore::create_clone("mystore"); + h = Broker::create_clone("mystore"); - when ( local res = BrokerStore::keys(h) ) + when ( local res = Broker::keys(h) ) { print "clone keys", res; do_lookup(Broker::refine_to_string(Broker::vector_lookup(res$result, 0))); diff --git a/testing/btest/broker/clone_store.bro b/testing/btest/broker/clone_store.bro index cfcbc025f1..b761fc56ad 100644 --- a/testing/btest/broker/clone_store.bro +++ b/testing/btest/broker/clone_store.bro @@ -13,7 +13,7 @@ const broker_port: port &redef; redef exit_only_after_terminate = T; -global h: opaque of BrokerStore::Handle; +global h: opaque of Broker::Handle; global expected_key_count = 4; global key_count = 0; @@ -21,7 +21,7 @@ global query_timeout = 30sec; function do_lookup(key: string) { - when ( local res = BrokerStore::lookup(h, Broker::data(key)) ) + when ( local res = Broker::lookup(h, Broker::data(key)) ) { ++key_count; print "lookup", key, res; @@ -38,9 +38,9 @@ function do_lookup(key: string) event ready() { - h = BrokerStore::create_clone("mystore"); + h = Broker::create_clone("mystore"); - when ( local res = BrokerStore::keys(h) ) + when ( local res = Broker::keys(h) ) { print "clone keys", res; do_lookup(Broker::refine_to_string(Broker::vector_lookup(res$result, 0))); @@ -71,7 +71,7 @@ global query_timeout = 15sec; const broker_port: port &redef; redef exit_only_after_terminate = T; -global h: opaque of BrokerStore::Handle; +global h: opaque of Broker::Handle; function dv(d: Broker::Data): Broker::DataVector { @@ -94,19 +94,19 @@ event Broker::outgoing_connection_established(peer_address: string, { local myset: set[string] = {"a", "b", "c"}; local myvec: vector of string = {"alpha", "beta", "gamma"}; - h = BrokerStore::create_master("mystore"); - BrokerStore::insert(h, Broker::data("one"), Broker::data(110)); - BrokerStore::insert(h, Broker::data("two"), Broker::data(223)); - BrokerStore::insert(h, Broker::data("myset"), Broker::data(myset)); - BrokerStore::insert(h, Broker::data("myvec"), Broker::data(myvec)); - BrokerStore::increment(h, Broker::data("one")); - BrokerStore::decrement(h, Broker::data("two")); - BrokerStore::add_to_set(h, Broker::data("myset"), Broker::data("d")); - BrokerStore::remove_from_set(h, Broker::data("myset"), Broker::data("b")); - BrokerStore::push_left(h, Broker::data("myvec"), dv(Broker::data("delta"))); - BrokerStore::push_right(h, Broker::data("myvec"), dv(Broker::data("omega"))); + h = Broker::create_master("mystore"); + Broker::insert(h, Broker::data("one"), Broker::data(110)); + Broker::insert(h, Broker::data("two"), Broker::data(223)); + Broker::insert(h, Broker::data("myset"), Broker::data(myset)); + Broker::insert(h, Broker::data("myvec"), Broker::data(myvec)); + Broker::increment(h, Broker::data("one")); + Broker::decrement(h, Broker::data("two")); + Broker::add_to_set(h, Broker::data("myset"), Broker::data("d")); + Broker::remove_from_set(h, Broker::data("myset"), Broker::data("b")); + Broker::push_left(h, Broker::data("myvec"), dv(Broker::data("delta"))); + Broker::push_right(h, Broker::data("myvec"), dv(Broker::data("omega"))); - when ( local res = BrokerStore::size(h) ) + when ( local res = Broker::size(h) ) { event ready(); } timeout query_timeout { diff --git a/testing/btest/broker/master_store.bro b/testing/btest/broker/master_store.bro index 810d303aa1..a8cc8d3ad2 100644 --- a/testing/btest/broker/master_store.bro +++ b/testing/btest/broker/master_store.bro @@ -6,7 +6,7 @@ redef exit_only_after_terminate = T; -global h: opaque of BrokerStore::Handle; +global h: opaque of Broker::Handle; global lookup_count = 0; const lookup_expect_count = 5; global exists_count = 0; @@ -20,13 +20,13 @@ global query_timeout = 30sec; event test_clear() { - BrokerStore::clear(h); + Broker::clear(h); event test_size("after clear"); } event test_size(where: string) { - when ( local res = BrokerStore::size(h) ) + when ( local res = Broker::size(h) ) { if ( where == "" ) { @@ -52,7 +52,7 @@ event test_size(where: string) event test_keys() { - when ( local res = BrokerStore::keys(h) ) + when ( local res = Broker::keys(h) ) { print fmt("keys: %s", res); event test_size(); @@ -66,7 +66,7 @@ event test_keys() event test_pop(key: string) { - when ( local lres = BrokerStore::pop_left(h, Broker::data(key)) ) + when ( local lres = Broker::pop_left(h, Broker::data(key)) ) { print fmt("pop_left(%s): %s", key, lres); ++pop_count; @@ -83,7 +83,7 @@ event test_pop(key: string) event test_keys(); } - when ( local rres = BrokerStore::pop_right(h, Broker::data(key)) ) + when ( local rres = Broker::pop_right(h, Broker::data(key)) ) { print fmt("pop_right(%s): %s", key, rres); ++pop_count; @@ -103,7 +103,7 @@ event test_pop(key: string) function do_exists(key: string) { - when ( local res = BrokerStore::exists(h, Broker::data(key)) ) + when ( local res = Broker::exists(h, Broker::data(key)) ) { print fmt("exists(%s): %s", key, res); ++exists_count; @@ -123,7 +123,7 @@ function do_exists(key: string) event test_erase() { - BrokerStore::erase(h, Broker::data("two")); + Broker::erase(h, Broker::data("two")); do_exists("one"); do_exists("two"); do_exists("myset"); @@ -132,7 +132,7 @@ event test_erase() function do_lookup(key: string) { - when ( local res = BrokerStore::lookup(h, Broker::data(key)) ) + when ( local res = Broker::lookup(h, Broker::data(key)) ) { print fmt("lookup(%s): %s", key, res); ++lookup_count; @@ -162,17 +162,17 @@ event bro_init() Broker::enable(); local myset: set[string] = {"a", "b", "c"}; local myvec: vector of string = {"alpha", "beta", "gamma"}; - h = BrokerStore::create_master("master"); - BrokerStore::insert(h, Broker::data("one"), Broker::data(110)); - BrokerStore::insert(h, Broker::data("two"), Broker::data(223)); - BrokerStore::insert(h, Broker::data("myset"), Broker::data(myset)); - BrokerStore::insert(h, Broker::data("myvec"), Broker::data(myvec)); - BrokerStore::increment(h, Broker::data("one")); - BrokerStore::decrement(h, Broker::data("two")); - BrokerStore::add_to_set(h, Broker::data("myset"), Broker::data("d")); - BrokerStore::remove_from_set(h, Broker::data("myset"), Broker::data("b")); - BrokerStore::push_left(h, Broker::data("myvec"), dv(Broker::data("delta"))); - BrokerStore::push_right(h, Broker::data("myvec"), dv(Broker::data("omega"))); + h = Broker::create_master("master"); + Broker::insert(h, Broker::data("one"), Broker::data(110)); + Broker::insert(h, Broker::data("two"), Broker::data(223)); + Broker::insert(h, Broker::data("myset"), Broker::data(myset)); + Broker::insert(h, Broker::data("myvec"), Broker::data(myvec)); + Broker::increment(h, Broker::data("one")); + Broker::decrement(h, Broker::data("two")); + Broker::add_to_set(h, Broker::data("myset"), Broker::data("d")); + Broker::remove_from_set(h, Broker::data("myset"), Broker::data("b")); + Broker::push_left(h, Broker::data("myvec"), dv(Broker::data("delta"))); + Broker::push_right(h, Broker::data("myvec"), dv(Broker::data("omega"))); do_lookup("one"); do_lookup("two"); do_lookup("myset"); diff --git a/testing/btest/core/leaks/broker/clone_store.bro b/testing/btest/core/leaks/broker/clone_store.bro index 7bed6e43a6..09308eb42e 100644 --- a/testing/btest/core/leaks/broker/clone_store.bro +++ b/testing/btest/core/leaks/broker/clone_store.bro @@ -14,13 +14,13 @@ const broker_port: port &redef; redef exit_only_after_terminate = T; -global h: opaque of BrokerStore::Handle; +global h: opaque of Broker::Handle; global expected_key_count = 4; global key_count = 0; function do_lookup(key: string) { - when ( local res = BrokerStore::lookup(h, Broker::data(key)) ) + when ( local res = Broker::lookup(h, Broker::data(key)) ) { ++key_count; print "lookup", key, res; @@ -34,9 +34,9 @@ function do_lookup(key: string) event ready() { - h = BrokerStore::create_clone("mystore"); + h = Broker::create_clone("mystore"); - when ( local res = BrokerStore::keys(h) ) + when ( local res = Broker::keys(h) ) { print "clone keys", res; do_lookup(Broker::refine_to_string(Broker::vector_lookup(res$result, 0))); @@ -62,7 +62,7 @@ event bro_init() const broker_port: port &redef; redef exit_only_after_terminate = T; -global h: opaque of BrokerStore::Handle; +global h: opaque of Broker::Handle; function dv(d: Broker::Data): Broker::DataVector { @@ -85,18 +85,18 @@ event Broker::outgoing_connection_established(peer_address: string, { local myset: set[string] = {"a", "b", "c"}; local myvec: vector of string = {"alpha", "beta", "gamma"}; - BrokerStore::insert(h, Broker::data("one"), Broker::data(110)); - BrokerStore::insert(h, Broker::data("two"), Broker::data(223)); - BrokerStore::insert(h, Broker::data("myset"), Broker::data(myset)); - BrokerStore::insert(h, Broker::data("myvec"), Broker::data(myvec)); - BrokerStore::increment(h, Broker::data("one")); - BrokerStore::decrement(h, Broker::data("two")); - BrokerStore::add_to_set(h, Broker::data("myset"), Broker::data("d")); - BrokerStore::remove_from_set(h, Broker::data("myset"), Broker::data("b")); - BrokerStore::push_left(h, Broker::data("myvec"), dv(Broker::data("delta"))); - BrokerStore::push_right(h, Broker::data("myvec"), dv(Broker::data("omega"))); + Broker::insert(h, Broker::data("one"), Broker::data(110)); + Broker::insert(h, Broker::data("two"), Broker::data(223)); + Broker::insert(h, Broker::data("myset"), Broker::data(myset)); + Broker::insert(h, Broker::data("myvec"), Broker::data(myvec)); + Broker::increment(h, Broker::data("one")); + Broker::decrement(h, Broker::data("two")); + Broker::add_to_set(h, Broker::data("myset"), Broker::data("d")); + Broker::remove_from_set(h, Broker::data("myset"), Broker::data("b")); + Broker::push_left(h, Broker::data("myvec"), dv(Broker::data("delta"))); + Broker::push_right(h, Broker::data("myvec"), dv(Broker::data("omega"))); - when ( local res = BrokerStore::size(h) ) + when ( local res = Broker::size(h) ) { event ready(); } timeout 10sec { print "timeout"; } @@ -105,7 +105,7 @@ event Broker::outgoing_connection_established(peer_address: string, event bro_init() { Broker::enable(); - h = BrokerStore::create_master("mystore"); + h = Broker::create_master("mystore"); Broker::connect("127.0.0.1", broker_port, 1secs); Broker::auto_event("bro/event/ready", ready); } diff --git a/testing/btest/core/leaks/broker/master_store.bro b/testing/btest/core/leaks/broker/master_store.bro index b18d75bede..8f4286ef3e 100644 --- a/testing/btest/core/leaks/broker/master_store.bro +++ b/testing/btest/core/leaks/broker/master_store.bro @@ -8,7 +8,7 @@ redef exit_only_after_terminate = T; -global h: opaque of BrokerStore::Handle; +global h: opaque of Broker::Handle; global lookup_count = 0; const lookup_expect_count = 5; global exists_count = 0; @@ -20,13 +20,13 @@ global test_size: event(where: string &default = ""); event test_clear() { - BrokerStore::clear(h); + Broker::clear(h); event test_size("after clear"); } event test_size(where: string) { - when ( local res = BrokerStore::size(h) ) + when ( local res = Broker::size(h) ) { if ( where == "" ) { @@ -45,7 +45,7 @@ event test_size(where: string) event test_keys() { - when ( local res = BrokerStore::keys(h) ) + when ( local res = Broker::keys(h) ) { print fmt("keys: %s", res); event test_size(); @@ -56,7 +56,7 @@ event test_keys() event test_pop(key: string) { - when ( local lres = BrokerStore::pop_left(h, Broker::data(key)) ) + when ( local lres = Broker::pop_left(h, Broker::data(key)) ) { print fmt("pop_left(%s): %s", key, lres); ++pop_count; @@ -67,7 +67,7 @@ event test_pop(key: string) timeout 10sec { print "timeout"; } - when ( local rres = BrokerStore::pop_right(h, Broker::data(key)) ) + when ( local rres = Broker::pop_right(h, Broker::data(key)) ) { print fmt("pop_right(%s): %s", key, rres); ++pop_count; @@ -81,7 +81,7 @@ event test_pop(key: string) function do_exists(key: string) { - when ( local res = BrokerStore::exists(h, Broker::data(key)) ) + when ( local res = Broker::exists(h, Broker::data(key)) ) { print fmt("exists(%s): %s", key, res); ++exists_count; @@ -95,7 +95,7 @@ function do_exists(key: string) event test_erase() { - BrokerStore::erase(h, Broker::data("two")); + Broker::erase(h, Broker::data("two")); do_exists("one"); do_exists("two"); do_exists("myset"); @@ -104,7 +104,7 @@ event test_erase() function do_lookup(key: string) { - when ( local res = BrokerStore::lookup(h, Broker::data(key)) ) + when ( local res = Broker::lookup(h, Broker::data(key)) ) { print fmt("lookup(%s): %s", key, res); ++lookup_count; @@ -128,7 +128,7 @@ global did_it = F; event bro_init() { Broker::enable(); - h = BrokerStore::create_master("master"); + h = Broker::create_master("master"); } event new_connection(c: connection) @@ -137,16 +137,16 @@ event new_connection(c: connection) did_it = T; local myset: set[string] = {"a", "b", "c"}; local myvec: vector of string = {"alpha", "beta", "gamma"}; - BrokerStore::insert(h, Broker::data("one"), Broker::data(110)); - BrokerStore::insert(h, Broker::data("two"), Broker::data(223)); - BrokerStore::insert(h, Broker::data("myset"), Broker::data(myset)); - BrokerStore::insert(h, Broker::data("myvec"), Broker::data(myvec)); - BrokerStore::increment(h, Broker::data("one")); - BrokerStore::decrement(h, Broker::data("two")); - BrokerStore::add_to_set(h, Broker::data("myset"), Broker::data("d")); - BrokerStore::remove_from_set(h, Broker::data("myset"), Broker::data("b")); - BrokerStore::push_left(h, Broker::data("myvec"), dv(Broker::data("delta"))); - BrokerStore::push_right(h, Broker::data("myvec"), dv(Broker::data("omega"))); + Broker::insert(h, Broker::data("one"), Broker::data(110)); + Broker::insert(h, Broker::data("two"), Broker::data(223)); + Broker::insert(h, Broker::data("myset"), Broker::data(myset)); + Broker::insert(h, Broker::data("myvec"), Broker::data(myvec)); + Broker::increment(h, Broker::data("one")); + Broker::decrement(h, Broker::data("two")); + Broker::add_to_set(h, Broker::data("myset"), Broker::data("d")); + Broker::remove_from_set(h, Broker::data("myset"), Broker::data("b")); + Broker::push_left(h, Broker::data("myvec"), dv(Broker::data("delta"))); + Broker::push_right(h, Broker::data("myvec"), dv(Broker::data("omega"))); do_lookup("one"); do_lookup("two"); do_lookup("myset"); diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_broker_stores-connector_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_broker_stores-connector_bro.btest index 9671878eef..74b59467e7 100644 --- a/testing/btest/doc/sphinx/include-doc_frameworks_broker_stores-connector_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_frameworks_broker_stores-connector_bro.btest @@ -5,7 +5,7 @@ stores-connector.bro const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -global h: opaque of BrokerStore::Handle; +global h: opaque of Broker::Handle; function dv(d: Broker::Data): Broker::DataVector { @@ -28,19 +28,19 @@ event Broker::outgoing_connection_established(peer_address: string, { local myset: set[string] = {"a", "b", "c"}; local myvec: vector of string = {"alpha", "beta", "gamma"}; - h = BrokerStore::create_master("mystore"); - BrokerStore::insert(h, Broker::data("one"), Broker::data(110)); - BrokerStore::insert(h, Broker::data("two"), Broker::data(223)); - BrokerStore::insert(h, Broker::data("myset"), Broker::data(myset)); - BrokerStore::insert(h, Broker::data("myvec"), Broker::data(myvec)); - BrokerStore::increment(h, Broker::data("one")); - BrokerStore::decrement(h, Broker::data("two")); - BrokerStore::add_to_set(h, Broker::data("myset"), Broker::data("d")); - BrokerStore::remove_from_set(h, Broker::data("myset"), Broker::data("b")); - BrokerStore::push_left(h, Broker::data("myvec"), dv(Broker::data("delta"))); - BrokerStore::push_right(h, Broker::data("myvec"), dv(Broker::data("omega"))); + h = Broker::create_master("mystore"); + Broker::insert(h, Broker::data("one"), Broker::data(110)); + Broker::insert(h, Broker::data("two"), Broker::data(223)); + Broker::insert(h, Broker::data("myset"), Broker::data(myset)); + Broker::insert(h, Broker::data("myvec"), Broker::data(myvec)); + Broker::increment(h, Broker::data("one")); + Broker::decrement(h, Broker::data("two")); + Broker::add_to_set(h, Broker::data("myset"), Broker::data("d")); + Broker::remove_from_set(h, Broker::data("myset"), Broker::data("b")); + Broker::push_left(h, Broker::data("myvec"), dv(Broker::data("delta"))); + Broker::push_right(h, Broker::data("myvec"), dv(Broker::data("omega"))); - when ( local res = BrokerStore::size(h) ) + when ( local res = Broker::size(h) ) { print "master size", res; event ready(); diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_broker_stores-listener_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_broker_stores-listener_bro.btest index 35ff3dc41a..8dadbc803c 100644 --- a/testing/btest/doc/sphinx/include-doc_frameworks_broker_stores-listener_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_frameworks_broker_stores-listener_bro.btest @@ -5,13 +5,13 @@ stores-listener.bro const broker_port: port = 9999/tcp &redef; redef exit_only_after_terminate = T; -global h: opaque of BrokerStore::Handle; +global h: opaque of Broker::Handle; global expected_key_count = 4; global key_count = 0; function do_lookup(key: string) { - when ( local res = BrokerStore::lookup(h, Broker::data(key)) ) + when ( local res = Broker::lookup(h, Broker::data(key)) ) { ++key_count; print "lookup", key, res; @@ -25,9 +25,9 @@ function do_lookup(key: string) event ready() { - h = BrokerStore::create_clone("mystore"); + h = Broker::create_clone("mystore"); - when ( local res = BrokerStore::keys(h) ) + when ( local res = Broker::keys(h) ) { print "clone keys", res; do_lookup(Broker::refine_to_string(Broker::vector_lookup(res$result, 0))); From cca9a6616e2d97e16c6a168c296ac19beeb57219 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 30 Mar 2016 20:32:36 -0500 Subject: [PATCH 34/84] Split the broker main.bro into two scripts Separate the former BrokerComm and BrokerStore portions of the script because these files will be much larger when script wrappers for BIFs are written. --- scripts/base/frameworks/broker/__load__.bro | 1 + scripts/base/frameworks/broker/main.bro | 48 ----------------- scripts/base/frameworks/broker/store.bro | 51 +++++++++++++++++++ .../canonified_loaded_scripts.log | 5 +- .../canonified_loaded_scripts.log | 5 +- testing/btest/Baseline/plugins.hooks/output | 14 ++--- 6 files changed, 66 insertions(+), 58 deletions(-) create mode 100644 scripts/base/frameworks/broker/store.bro diff --git a/scripts/base/frameworks/broker/__load__.bro b/scripts/base/frameworks/broker/__load__.bro index a10fe855df..018d772f4f 100644 --- a/scripts/base/frameworks/broker/__load__.bro +++ b/scripts/base/frameworks/broker/__load__.bro @@ -1 +1,2 @@ @load ./main +@load ./store diff --git a/scripts/base/frameworks/broker/main.bro b/scripts/base/frameworks/broker/main.bro index 0270e6a6ba..d8b4a208a2 100644 --- a/scripts/base/frameworks/broker/main.bro +++ b/scripts/base/frameworks/broker/main.bro @@ -53,51 +53,3 @@ export { val: Broker::Data; }; } - -module Broker; - -export { - - ## Whether a data store query could be completed or not. - type QueryStatus: enum { - SUCCESS, - FAILURE, - }; - - ## An expiry time for a key-value pair inserted in to a data store. - type ExpiryTime: record { - ## Absolute point in time at which to expire the entry. - absolute: time &optional; - ## A point in time relative to the last modification time at which - ## to expire the entry. New modifications will delay the expiration. - since_last_modification: interval &optional; - }; - - ## The result of a data store query. - type QueryResult: record { - ## Whether the query completed or not. - status: Broker::QueryStatus; - ## The result of the query. Certain queries may use a particular - ## data type (e.g. querying store size always returns a count, but - ## a lookup may return various data types). - result: Broker::Data; - }; - - ## Options to tune the SQLite storage backend. - type SQLiteOptions: record { - ## File system path of the database. - path: string &default = "store.sqlite"; - }; - - ## Options to tune the RocksDB storage backend. - type RocksDBOptions: record { - ## File system path of the database. - path: string &default = "store.rocksdb"; - }; - - ## Options to tune the particular storage backends. - type BackendOptions: record { - sqlite: SQLiteOptions &default = SQLiteOptions(); - rocksdb: RocksDBOptions &default = RocksDBOptions(); - }; -} diff --git a/scripts/base/frameworks/broker/store.bro b/scripts/base/frameworks/broker/store.bro new file mode 100644 index 0000000000..e6468f2b2c --- /dev/null +++ b/scripts/base/frameworks/broker/store.bro @@ -0,0 +1,51 @@ +##! Various data structure definitions for use with Bro's communication system. + +@load ./main + +module Broker; + +export { + + ## Whether a data store query could be completed or not. + type QueryStatus: enum { + SUCCESS, + FAILURE, + }; + + ## An expiry time for a key-value pair inserted in to a data store. + type ExpiryTime: record { + ## Absolute point in time at which to expire the entry. + absolute: time &optional; + ## A point in time relative to the last modification time at which + ## to expire the entry. New modifications will delay the expiration. + since_last_modification: interval &optional; + }; + + ## The result of a data store query. + type QueryResult: record { + ## Whether the query completed or not. + status: Broker::QueryStatus; + ## The result of the query. Certain queries may use a particular + ## data type (e.g. querying store size always returns a count, but + ## a lookup may return various data types). + result: Broker::Data; + }; + + ## Options to tune the SQLite storage backend. + type SQLiteOptions: record { + ## File system path of the database. + path: string &default = "store.sqlite"; + }; + + ## Options to tune the RocksDB storage backend. + type RocksDBOptions: record { + ## File system path of the database. + path: string &default = "store.rocksdb"; + }; + + ## Options to tune the particular storage backends. + type BackendOptions: record { + sqlite: SQLiteOptions &default = SQLiteOptions(); + rocksdb: RocksDBOptions &default = RocksDBOptions(); + }; +} diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 4d1f2037a4..44b199bc51 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2015-08-31-04-50-43 +#open 2016-03-30-22-54-40 #fields name #types string scripts/base/init-bare.bro @@ -17,6 +17,7 @@ scripts/base/init-bare.bro build/scripts/base/bif/event.bif.bro scripts/base/frameworks/broker/__load__.bro scripts/base/frameworks/broker/main.bro + scripts/base/frameworks/broker/store.bro scripts/base/frameworks/logging/__load__.bro scripts/base/frameworks/logging/main.bro build/scripts/base/bif/logging.bif.bro @@ -128,4 +129,4 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro scripts/policy/misc/loaded-scripts.bro scripts/base/utils/paths.bro -#close 2015-08-31-04-50-43 +#close 2016-03-30-22-54-40 diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 85fe19eb96..cf15f484c8 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2016-02-17-20-30-50 +#open 2016-03-30-22-54-53 #fields name #types string scripts/base/init-bare.bro @@ -17,6 +17,7 @@ scripts/base/init-bare.bro build/scripts/base/bif/event.bif.bro scripts/base/frameworks/broker/__load__.bro scripts/base/frameworks/broker/main.bro + scripts/base/frameworks/broker/store.bro scripts/base/frameworks/logging/__load__.bro scripts/base/frameworks/logging/main.bro build/scripts/base/bif/logging.bif.bro @@ -297,4 +298,4 @@ scripts/base/init-default.bro scripts/base/misc/find-checksum-offloading.bro scripts/base/misc/find-filtered-trace.bro scripts/policy/misc/loaded-scripts.bro -#close 2016-02-17-20-30-50 +#close 2016-03-30-22-54-53 diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 25808a20d8..a66fe365c6 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -228,7 +228,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1457718658.75999, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1459378506.94705, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Communication::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Conn::LOG)) -> @@ -346,7 +346,7 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1457718658.75999, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1459378506.94705, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(NetControl::check_plugins, , ()) -> 0.000000 MetaHookPost CallFunction(NetControl::init, , ()) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, , ()) -> @@ -517,6 +517,7 @@ 0.000000 MetaHookPost LoadFile(./shunt) -> -1 0.000000 MetaHookPost LoadFile(./site) -> -1 0.000000 MetaHookPost LoadFile(./std-dev) -> -1 +0.000000 MetaHookPost LoadFile(./store) -> -1 0.000000 MetaHookPost LoadFile(./store.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./strings.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./sum) -> -1 @@ -859,7 +860,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1457718658.75999, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1459378506.94705, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Communication::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Conn::LOG)) @@ -977,7 +978,7 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1457718658.75999, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1459378506.94705, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(NetControl::check_plugins, , ()) 0.000000 MetaHookPre CallFunction(NetControl::init, , ()) 0.000000 MetaHookPre CallFunction(Notice::want_pp, , ()) @@ -1148,6 +1149,7 @@ 0.000000 MetaHookPre LoadFile(./shunt) 0.000000 MetaHookPre LoadFile(./site) 0.000000 MetaHookPre LoadFile(./std-dev) +0.000000 MetaHookPre LoadFile(./store) 0.000000 MetaHookPre LoadFile(./store.bif.bro) 0.000000 MetaHookPre LoadFile(./strings.bif.bro) 0.000000 MetaHookPre LoadFile(./sum) @@ -1489,7 +1491,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1457718658.75999, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1459378506.94705, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG) @@ -1607,7 +1609,7 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1457718658.75999, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1459378506.94705, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction NetControl::check_plugins() 0.000000 | HookCallFunction NetControl::init() 0.000000 | HookCallFunction Notice::want_pp() From b5f1fb33fac78d305653feb799bd936a16083e3b Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 7 Apr 2016 13:40:31 -0700 Subject: [PATCH 35/84] Updating submodule(s). [nomail] --- aux/broccoli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broccoli b/aux/broccoli index 6ded82da49..f83038b17f 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 6ded82da498d805def6aa129cd7691d3b7287c37 +Subproject commit f83038b17fc83788415a58d77f75ad182ca6a9b7 From 849875e8be73d0e0b5a6ebca74ed56fdabba464b Mon Sep 17 00:00:00 2001 From: Martin van Hensbergen Date: Mon, 11 Apr 2016 10:35:00 +0200 Subject: [PATCH 36/84] Analyzer and bro script for RFB protocol (VNC) This analyzer parses the Remote Frame Buffer protocol, usually referred to as the 'VNC protocol'. It supports several dialects (3.3, 3.7, 3.8) and also handles the Apple Remote Desktop variant. It will log such facts as client/server versions, authentication method used, authentication result, height, width and name of the shared screen. It also includes two testcases. Todo: Apple Remote Desktop seems to have some bytes prepended to the screen name. This is not interepreted correctly. --- scripts/base/init-default.bro | 1 + scripts/base/protocols/rfb/__load__.bro | 3 + scripts/base/protocols/rfb/dpd.sig | 12 ++ scripts/base/protocols/rfb/main.bro | 143 +++++++++++++ src/analyzer/protocol/CMakeLists.txt | 1 + src/analyzer/protocol/rfb/CMakeLists.txt | 11 + src/analyzer/protocol/rfb/Plugin.cc | 25 +++ src/analyzer/protocol/rfb/RFB.cc | 69 +++++++ src/analyzer/protocol/rfb/RFB.h | 45 ++++ src/analyzer/protocol/rfb/events.bif | 50 +++++ src/analyzer/protocol/rfb/rfb-analyzer.pac | 193 ++++++++++++++++++ src/analyzer/protocol/rfb/rfb-protocol.pac | 139 +++++++++++++ src/analyzer/protocol/rfb/rfb.pac | 42 ++++ .../rfb.log | 12 ++ .../rfb.log | 12 ++ .../btest/Traces/rfb/vnc-mac-to-linux.pcap | Bin 0 -> 40255 bytes testing/btest/Traces/rfb/vncmac.pcap | Bin 0 -> 8848 bytes .../rfb/rfb-apple-remote-desktop.test | 4 + .../base/protocols/rfb/vnc-mac-to-linux.test | 4 + 19 files changed, 766 insertions(+) create mode 100644 scripts/base/protocols/rfb/__load__.bro create mode 100644 scripts/base/protocols/rfb/dpd.sig create mode 100644 scripts/base/protocols/rfb/main.bro create mode 100644 src/analyzer/protocol/rfb/CMakeLists.txt create mode 100644 src/analyzer/protocol/rfb/Plugin.cc create mode 100644 src/analyzer/protocol/rfb/RFB.cc create mode 100644 src/analyzer/protocol/rfb/RFB.h create mode 100644 src/analyzer/protocol/rfb/events.bif create mode 100644 src/analyzer/protocol/rfb/rfb-analyzer.pac create mode 100644 src/analyzer/protocol/rfb/rfb-protocol.pac create mode 100644 src/analyzer/protocol/rfb/rfb.pac create mode 100644 testing/btest/Baseline/scripts.base.protocols.rfb.rfb-apple-remote-desktop/rfb.log create mode 100644 testing/btest/Baseline/scripts.base.protocols.rfb.vnc-mac-to-linux/rfb.log create mode 100644 testing/btest/Traces/rfb/vnc-mac-to-linux.pcap create mode 100644 testing/btest/Traces/rfb/vncmac.pcap create mode 100644 testing/btest/scripts/base/protocols/rfb/rfb-apple-remote-desktop.test create mode 100644 testing/btest/scripts/base/protocols/rfb/vnc-mac-to-linux.test diff --git a/scripts/base/init-default.bro b/scripts/base/init-default.bro index 609ed7200c..418ccbb43e 100644 --- a/scripts/base/init-default.bro +++ b/scripts/base/init-default.bro @@ -55,6 +55,7 @@ @load base/protocols/pop3 @load base/protocols/radius @load base/protocols/rdp +@load base/protocols/rfb @load base/protocols/sip @load base/protocols/snmp @load base/protocols/smtp diff --git a/scripts/base/protocols/rfb/__load__.bro b/scripts/base/protocols/rfb/__load__.bro new file mode 100644 index 0000000000..9e43682d13 --- /dev/null +++ b/scripts/base/protocols/rfb/__load__.bro @@ -0,0 +1,3 @@ +# Generated by binpac_quickstart +@load ./main +@load-sigs ./dpd.sig \ No newline at end of file diff --git a/scripts/base/protocols/rfb/dpd.sig b/scripts/base/protocols/rfb/dpd.sig new file mode 100644 index 0000000000..40793ad590 --- /dev/null +++ b/scripts/base/protocols/rfb/dpd.sig @@ -0,0 +1,12 @@ +signature dpd_rfb_server { + ip-proto == tcp + payload /^RFB/ + requires-reverse-signature dpd_rfb_client + enable "rfb" +} + +signature dpd_rfb_client { + ip-proto == tcp + payload /^RFB/ + tcp-state originator +} \ No newline at end of file diff --git a/scripts/base/protocols/rfb/main.bro b/scripts/base/protocols/rfb/main.bro new file mode 100644 index 0000000000..97f194b789 --- /dev/null +++ b/scripts/base/protocols/rfb/main.bro @@ -0,0 +1,143 @@ +module Rfb; + +export { + redef enum Log::ID += { LOG }; + + type Info: record { + ## Timestamp for when the event happened. + ts: time &log; + ## Unique ID for the connection. + uid: string &log; + ## The connection's 4-tuple of endpoint addresses/ports. + id: conn_id &log; + + client_major_version: string &log &optional; + client_minor_version: string &log &optional; + server_major_version: string &log &optional; + server_minor_version: string &log &optional; + + authentication_method: string &log &optional; + auth: bool &log &optional; + + share_flag: bool &log &optional; + desktop_name: string &log &optional; + width: count &log &optional; + height: count &log &optional; + + done: bool &default=F; + }; + + global log_rfb: event(rec: Info); +} + +function friendly_auth_name(auth: count): string { + switch (auth) { + case 0: + return "Invalid"; + case 1: + return "None"; + case 2: + return "VNC"; + case 16: + return "Tight"; + case 17: + return "Ultra"; + case 18: + return "TLS"; + case 19: + return "VeNCrypt"; + case 20: + return "GTK-VNC SASL"; + case 21: + return "MD5 hash authentication"; + case 22: + return "Colin Dean xvp"; + case 30: + return "Apple Remote Desktop"; + } + return "RealVNC"; + +} + + +redef record connection += { + rfb_state: Info &optional; +}; + +event bro_init() &priority=5 + { + Log::create_stream(Rfb::LOG, [$columns=Info, $ev=log_rfb, $path="rfb"]); + } + +function write_log(c:connection) { + local state = c$rfb_state; + if ( state?$done && state$done == T) { + return; + } + Log::write(Rfb::LOG, c$rfb_state); + c$rfb_state$done = T; +} + +function set_session(c: connection) { + if ( ! c?$rfb_state ) { + local info: Info; + info$ts = network_time(); + info$uid = c$uid; + info$id = c$id; + + c$rfb_state = info; + } + } + +event rfb_event(c: connection) + { + set_session(c); + } + +event rfb_client_version(c: connection, major_version: string, minor_version: string) + { + set_session(c); + c$rfb_state$client_major_version = major_version; + c$rfb_state$client_minor_version = minor_version; + } + +event rfb_server_version(c: connection, major_version: string, minor_version: string) + { + set_session(c); + c$rfb_state$server_major_version = major_version; + c$rfb_state$server_minor_version = minor_version; + add c$service["rfb"]; + } + +event rfb_authentication_type(c: connection, authtype: count) + { + c$rfb_state$authentication_method = friendly_auth_name(authtype); + } + +event rfb_server_parameters(c: connection, name: string, width: count, height: count) + { + c$rfb_state$desktop_name = name; + c$rfb_state$width = width; + c$rfb_state$height = height; + write_log(c); + } + +event rfb_auth_result(c: connection, result: count) + { + if ( result ==0 ) { + c$rfb_state$auth = T; + } else { + c$rfb_state$auth = F; + } + } + +event rfb_share_flag(c: connection, flag: bool) + { + c$rfb_state$share_flag = flag; + } + +event connection_state_remove(c: connection) { + if ( c?$rfb_state ) { + write_log(c); + } +} diff --git a/src/analyzer/protocol/CMakeLists.txt b/src/analyzer/protocol/CMakeLists.txt index 467fce83ee..8c7a3f002e 100644 --- a/src/analyzer/protocol/CMakeLists.txt +++ b/src/analyzer/protocol/CMakeLists.txt @@ -30,6 +30,7 @@ add_subdirectory(pia) add_subdirectory(pop3) add_subdirectory(radius) add_subdirectory(rdp) +add_subdirectory(rfb) add_subdirectory(rpc) add_subdirectory(sip) add_subdirectory(snmp) diff --git a/src/analyzer/protocol/rfb/CMakeLists.txt b/src/analyzer/protocol/rfb/CMakeLists.txt new file mode 100644 index 0000000000..8131ca7362 --- /dev/null +++ b/src/analyzer/protocol/rfb/CMakeLists.txt @@ -0,0 +1,11 @@ +# Generated by binpac_quickstart + +include(BroPlugin) + +include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) + +bro_plugin_begin(Bro RFB) + bro_plugin_cc(RFB.cc Plugin.cc) + bro_plugin_bif(events.bif) + bro_plugin_pac(rfb.pac rfb-analyzer.pac rfb-protocol.pac) +bro_plugin_end() \ No newline at end of file diff --git a/src/analyzer/protocol/rfb/Plugin.cc b/src/analyzer/protocol/rfb/Plugin.cc new file mode 100644 index 0000000000..55704497e9 --- /dev/null +++ b/src/analyzer/protocol/rfb/Plugin.cc @@ -0,0 +1,25 @@ +// Generated by binpac_quickstart + +#include "plugin/Plugin.h" + +#include "RFB.h" + +namespace plugin { +namespace Bro_RFB { + +class Plugin : public plugin::Plugin { +public: + plugin::Configuration Configure() + { + AddComponent(new ::analyzer::Component("RFB", + ::analyzer::rfb::RFB_Analyzer::InstantiateAnalyzer)); + + plugin::Configuration config; + config.name = "Bro::RFB"; + config.description = "Parser for rfb (VNC) analyzer"; + return config; + } +} plugin; + +} +} \ No newline at end of file diff --git a/src/analyzer/protocol/rfb/RFB.cc b/src/analyzer/protocol/rfb/RFB.cc new file mode 100644 index 0000000000..c761d0bf0f --- /dev/null +++ b/src/analyzer/protocol/rfb/RFB.cc @@ -0,0 +1,69 @@ +// Generated by binpac_quickstart + +#include "RFB.h" + +#include "analyzer/protocol/tcp/TCP_Reassembler.h" + +#include "Reporter.h" + +#include "events.bif.h" + +using namespace analyzer::rfb; + +RFB_Analyzer::RFB_Analyzer(Connection* c) + +: tcp::TCP_ApplicationAnalyzer("RFB", c) + + { + interp = new binpac::RFB::RFB_Conn(this); + had_gap = false; + } + +RFB_Analyzer::~RFB_Analyzer() + { + delete interp; + } + +void RFB_Analyzer::Done() + { + tcp::TCP_ApplicationAnalyzer::Done(); + + interp->FlowEOF(true); + interp->FlowEOF(false); + + } + +void RFB_Analyzer::EndpointEOF(bool is_orig) + { + tcp::TCP_ApplicationAnalyzer::EndpointEOF(is_orig); + interp->FlowEOF(is_orig); + } + +void RFB_Analyzer::DeliverStream(int len, const u_char* data, bool orig) + { + tcp::TCP_ApplicationAnalyzer::DeliverStream(len, data, orig); + assert(TCP()); + if ( TCP()->IsPartial() ) + return; + + if ( had_gap ) + // If only one side had a content gap, we could still try to + // deliver data to the other side if the script layer can handle this. + return; + + try + { + interp->NewData(orig, data, data + len); + } + catch ( const binpac::Exception& e ) + { + ProtocolViolation(fmt("Binpac exception: %s", e.c_msg())); + } + } + +void RFB_Analyzer::Undelivered(uint64 seq, int len, bool orig) + { + tcp::TCP_ApplicationAnalyzer::Undelivered(seq, len, orig); + had_gap = true; + interp->NewGap(orig, len); + } diff --git a/src/analyzer/protocol/rfb/RFB.h b/src/analyzer/protocol/rfb/RFB.h new file mode 100644 index 0000000000..cd6e7348d0 --- /dev/null +++ b/src/analyzer/protocol/rfb/RFB.h @@ -0,0 +1,45 @@ +// Generated by binpac_quickstart + +#ifndef ANALYZER_PROTOCOL_RFB_RFB_H +#define ANALYZER_PROTOCOL_RFB_RFB_H + +#include "events.bif.h" + + +#include "analyzer/protocol/tcp/TCP.h" + +#include "rfb_pac.h" + +namespace analyzer { namespace rfb { + +class RFB_Analyzer + +: public tcp::TCP_ApplicationAnalyzer { + +public: + RFB_Analyzer(Connection* conn); + virtual ~RFB_Analyzer(); + + // Overriden from Analyzer. + virtual void Done(); + + virtual void DeliverStream(int len, const u_char* data, bool orig); + virtual void Undelivered(uint64 seq, int len, bool orig); + + // Overriden from tcp::TCP_ApplicationAnalyzer. + virtual void EndpointEOF(bool is_orig); + + + static analyzer::Analyzer* InstantiateAnalyzer(Connection* conn) + { return new RFB_Analyzer(conn); } + +protected: + binpac::RFB::RFB_Conn* interp; + + bool had_gap; + +}; + +} } // namespace analyzer::* + +#endif diff --git a/src/analyzer/protocol/rfb/events.bif b/src/analyzer/protocol/rfb/events.bif new file mode 100644 index 0000000000..a3cf5f7ad8 --- /dev/null +++ b/src/analyzer/protocol/rfb/events.bif @@ -0,0 +1,50 @@ +## Generated for RFB event +## +## c: The connection record for the underlying transport-layer session/flow. +event rfb_event%(c: connection%); + +## Generated for RFB event authentication mechanism selection +## +## c: The connection record for the underlying transport-layer session/flow. +## +## authtype: the value of the chosen authentication mechanism +event rfb_authentication_type%(c: connection, authtype: count%); + +## Generated for RFB event authentication result message +## +## c: The connection record for the underlying transport-layer session/flow. +## +## result: whether or not authentication was succesful +event rfb_auth_result%(c: connection, result: count%); + +## Generated for RFB event share flag messages +## +## c: The connection record for the underlying transport-layer session/flow. +## +## flag: whether or not the share flag was set +event rfb_share_flag%(c: connection, flag: bool%); + +## Generated for RFB event client banner message +## +## c: The connection record for the underlying transport-layer session/flow. +## +## version: of the client's rfb library +event rfb_client_version%(c: connection, major_version: string, minor_version: string%); + +## Generated for RFB event server banner message +## +## c: The connection record for the underlying transport-layer session/flow. +## +## version: of the server's rfb library +event rfb_server_version%(c: connection, major_version: string, minor_version: string%); + +## Generated for RFB event server parameter message +## +## c: The connection record for the underlying transport-layer session/flow. +## +## name: name of the shared screen +## +## width: width of the shared screen +## +## height: height of the shared screen +event rfb_server_parameters%(c: connection, name: string, width: count, height: count%); \ No newline at end of file diff --git a/src/analyzer/protocol/rfb/rfb-analyzer.pac b/src/analyzer/protocol/rfb/rfb-analyzer.pac new file mode 100644 index 0000000000..4233a423f7 --- /dev/null +++ b/src/analyzer/protocol/rfb/rfb-analyzer.pac @@ -0,0 +1,193 @@ +# Generated by binpac_quickstart + +refine flow RFB_Flow += { + function proc_rfb_message(msg: RFB_PDU): bool + %{ + BifEvent::generate_rfb_event(connection()->bro_analyzer(), connection()->bro_analyzer()->Conn()); + return true; + %} + + function proc_rfb_client_version(major: bytestring, minor: bytestring) : bool + %{ + BifEvent::generate_rfb_client_version(connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), bytestring_to_val(major), bytestring_to_val(minor)); + return true; + %} + + function proc_rfb_version(client: bool, major: bytestring, minor: bytestring) : bool + %{ + if (client) { + BifEvent::generate_rfb_client_version(connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), bytestring_to_val(major), bytestring_to_val(minor)); + } else { + BifEvent::generate_rfb_server_version(connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), bytestring_to_val(major), bytestring_to_val(minor)); + } + return true; + %} + + function proc_rfb_share_flag(shared: bool) : bool + %{ + BifEvent::generate_rfb_share_flag(connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), shared); + return true; + %} + + function proc_security_types(msg: RFBSecurityTypes) : bool + %{ + BifEvent::generate_rfb_authentication_type(connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), ${msg.sectype}); + return true; + %} + + function proc_security_types37(msg: RFBAuthTypeSelected) : bool + %{ + BifEvent::generate_rfb_authentication_type(connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), ${msg.type}); + return true; + %} + + function proc_handle_server_params(msg:RFBServerInit) : bool + %{ + BifEvent::generate_rfb_server_parameters(connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), bytestring_to_val(${msg.name}), ${msg.width}, ${msg.height}); + return true; + %} + + function proc_handle_security_result(result : uint32) : bool + %{ + BifEvent::generate_rfb_auth_result(connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), result); + return true; + %} +}; + +refine connection RFB_Conn += { + %member{ + enum states { + AWAITING_SERVER_BANNER = 0, + AWAITING_CLIENT_BANNER = 1, + AWAITING_SERVER_AUTH_TYPES = 2, + AWAITING_SERVER_CHALLENGE = 3, + AWAITING_CLIENT_RESPONSE = 4, + AWAITING_SERVER_AUTH_RESULT = 5, + AWAITING_CLIENT_SHARE_FLAG = 6, + AWAITING_SERVER_PARAMS = 7, + AWAITING_CLIENT_AUTH_METHOD = 8, + AWAITING_SERVER_ARD_CHALLENGE = 9, + AWAITING_CLIENT_ARD_RESPONSE = 10, + AWAITING_SERVER_AUTH_TYPES37 = 11, + AWAITING_CLIENT_AUTH_TYPE_SELECTED37 = 12, + RFB_MESSAGE = 13 + }; + %} + + function get_state(client: bool) : int + %{ + return state; + %} + + function handle_banners(client: bool, msg: RFBProtocolVersion) : bool + %{ + if ( client ) { + // Set protocol version on client's version + int minor_version = bytestring_to_int(${msg.minor},10); + + // Apple specifies minor version "889" but talks v37 + if ( minor_version >= 7 ) { + state = AWAITING_SERVER_AUTH_TYPES37; + } else { + state = AWAITING_SERVER_AUTH_TYPES; + } + } else { + if ( !client ) { + state = AWAITING_CLIENT_BANNER; + } + } + return true; + %} + + function handle_ard_challenge() : bool + %{ + state = AWAITING_CLIENT_ARD_RESPONSE; + return true; + %} + + function handle_ard_response() : bool + %{ + state = AWAITING_SERVER_AUTH_RESULT; + return true; + %} + + function handle_auth_request() : bool + %{ + state = AWAITING_CLIENT_RESPONSE; + return true; + %} + + function handle_auth_response() : bool + %{ + state = AWAITING_SERVER_AUTH_RESULT; + return true; + %} + + function handle_security_result(msg: RFBSecurityResult) : bool + %{ + if ( ${msg.result} == 0 ) //FIXME + { + state = AWAITING_CLIENT_SHARE_FLAG; + } + return true; + %} + + function handle_client_init(msg: RFBClientInit) : bool + %{ + state = AWAITING_SERVER_PARAMS; + + return true; + %} + + function handle_server_init(msg: RFBServerInit) : bool + %{ + state = RFB_MESSAGE; + return true; + %} + + function handle_security_types(msg: RFBSecurityTypes): bool + %{ + if ( msg->sectype() == 0 ) { // No auth + state = AWAITING_CLIENT_SHARE_FLAG; + return true; + } + if ( msg->sectype() == 2 ) { //VNC + state = AWAITING_SERVER_CHALLENGE; + } + return false; + %} + + function handle_security_types37(msg: RFBSecurityTypes37): bool + %{ + if ( ${msg.count} == 0 ) { // No auth + state = AWAITING_CLIENT_SHARE_FLAG; + return true; + } + state = AWAITING_CLIENT_AUTH_TYPE_SELECTED37; + return true; + %} + + function handle_auth_type_selected(msg: RFBAuthTypeSelected): bool + %{ + if ( ${msg.type} == 30 ) { // Apple Remote Desktop + state = AWAITING_SERVER_ARD_CHALLENGE; + return true; + } + + if ( ${msg.type} == 1 ) { // No Auth + state = AWAITING_SERVER_AUTH_RESULT; + } else { + // Assume VNC + state = AWAITING_SERVER_CHALLENGE; + } + return true; + %} + + %member{ + uint8 state = AWAITING_SERVER_BANNER; + %} +}; + +refine typeattr RFB_PDU += &let { + proc: bool = $context.flow.proc_rfb_message(this); +}; diff --git a/src/analyzer/protocol/rfb/rfb-protocol.pac b/src/analyzer/protocol/rfb/rfb-protocol.pac new file mode 100644 index 0000000000..0eb5542001 --- /dev/null +++ b/src/analyzer/protocol/rfb/rfb-protocol.pac @@ -0,0 +1,139 @@ +enum states { + AWAITING_SERVER_BANNER = 0, + AWAITING_CLIENT_BANNER = 1, + AWAITING_SERVER_AUTH_TYPES = 2, + AWAITING_SERVER_CHALLENGE = 3, + AWAITING_CLIENT_RESPONSE = 4, + AWAITING_SERVER_AUTH_RESULT = 5, + AWAITING_CLIENT_SHARE_FLAG = 6, + AWAITING_SERVER_PARAMS = 7, + AWAITING_CLIENT_AUTH_METHOD = 8, + AWAITING_SERVER_ARD_CHALLENGE = 9, + AWAITING_CLIENT_ARD_RESPONSE = 10, + AWAITING_SERVER_AUTH_TYPES37 = 11, + AWAITING_CLIENT_AUTH_TYPE_SELECTED37 = 12, + RFB_MESSAGE = 13 + }; + +type RFBProtocolVersion (client: bool) = record { + header : "RFB "; + major :bytestring &length=3; + dot: "."; + minor: bytestring &length=3; + pad: uint8; +} &let { + proc: bool = $context.connection.handle_banners(client, this); + proc2: bool = $context.flow.proc_rfb_version(client, major, minor); +} + +type RFBSecurityTypes = record { + sectype: uint32; +} &let { + proc: bool = $context.connection.handle_security_types(this); + proc2: bool = $context.flow.proc_security_types(this); +}; + +type RFBSecurityTypes37 = record { + count: uint8; + types: uint8[count]; +} &let { + proc: bool = $context.connection.handle_security_types37(this); +}; + +type RFBAuthTypeSelected = record { + type: uint8; +} &let { + proc: bool = $context.connection.handle_auth_type_selected(this); + proc2: bool = $context.flow.proc_security_types37(this); +}; + +type RFBSecurityResult = record { + result: uint32; +} &let { + proc: bool = $context.connection.handle_security_result(this); + proc2: bool = $context.flow.proc_handle_security_result(result); +}; + +type RFBSecurityResultReason = record { + len: uint32; + reason: bytestring &length=len; +}; + +type RFBVNCAuthenticationRequest = record { + challenge: bytestring &length=16; +} &let { + proc: bool = $context.connection.handle_auth_request(); +}; + +type RFBVNCAuthenticationResponse = record { + response: bytestring &length= 16; +} &let { + proc: bool = $context.connection.handle_auth_response(); +}; + +type RFBSecurityARDChallenge = record { + challenge: bytestring &restofdata; +} &let { + proc: bool = $context.connection.handle_ard_challenge(); +} + +type RFBSecurityARDResponse = record { + response: bytestring &restofdata; +} &let { + proc: bool = $context.connection.handle_ard_response(); +} + +type RFBClientInit = record { + shared_flag: uint8; +} &let { + proc: bool = $context.connection.handle_client_init(this); + proc2: bool = $context.flow.proc_rfb_share_flag(shared_flag); +} + +type RFBServerInit = record { + width: uint16; + height: uint16; + pixel_format: bytestring &length= 16; + len : uint32; + name: bytestring &length = len; +} &let { + proc: bool = $context.connection.handle_server_init(this); + proc2: bool = $context.flow.proc_handle_server_params(this); +}; + +type RFB_PDU_request = record { + request: case state of { + AWAITING_CLIENT_BANNER -> version: RFBProtocolVersion(true); + AWAITING_CLIENT_RESPONSE -> response: RFBVNCAuthenticationResponse; + AWAITING_CLIENT_SHARE_FLAG -> shareflag: RFBClientInit; + AWAITING_CLIENT_AUTH_TYPE_SELECTED37 -> authtype: RFBAuthTypeSelected; + AWAITING_CLIENT_ARD_RESPONSE -> ard_response: RFBSecurityARDResponse; + RFB_MESSAGE -> ignore: bytestring &restofdata; + default -> data: bytestring &restofdata; + } &requires(state); + } &let { + state: uint8 = $context.connection.get_state(true); +}; + +type RFB_PDU_response = record { + request: case rstate of { + AWAITING_SERVER_BANNER -> version: RFBProtocolVersion(false); + AWAITING_SERVER_AUTH_TYPES -> auth_types: RFBSecurityTypes; + AWAITING_SERVER_AUTH_TYPES37 -> auth_types37: RFBSecurityTypes37; + AWAITING_SERVER_CHALLENGE -> challenge: RFBVNCAuthenticationRequest; + AWAITING_SERVER_AUTH_RESULT -> authresult : RFBSecurityResult; + AWAITING_SERVER_ARD_CHALLENGE -> ard_challenge: RFBSecurityARDChallenge; + AWAITING_SERVER_PARAMS -> serverinit: RFBServerInit; + RFB_MESSAGE -> ignore: bytestring &restofdata; + default -> data: bytestring &restofdata; + } &requires(rstate); + } &let { + rstate: uint8 = $context.connection.get_state(false); +}; + +type RFB_PDU(is_orig: bool) = record { + payload: case is_orig of { + true -> request: RFB_PDU_request; + false -> response: RFB_PDU_response; + }; +} &byteorder = bigendian; diff --git a/src/analyzer/protocol/rfb/rfb.pac b/src/analyzer/protocol/rfb/rfb.pac new file mode 100644 index 0000000000..310ad38893 --- /dev/null +++ b/src/analyzer/protocol/rfb/rfb.pac @@ -0,0 +1,42 @@ +# Generated by binpac_quickstart + +# Analyzer for Parser for rfb (VNC) +# - rfb-protocol.pac: describes the rfb protocol messages +# - rfb-analyzer.pac: describes the rfb analyzer code + +%include binpac.pac +%include bro.pac + +%extern{ + #include "events.bif.h" +%} + +analyzer RFB withcontext { + connection: RFB_Conn; + flow: RFB_Flow; +}; + +# Our connection consists of two flows, one in each direction. +connection RFB_Conn(bro_analyzer: BroAnalyzer) { + upflow = RFB_Flow(true); + downflow = RFB_Flow(false); +}; + +%include rfb-protocol.pac + +# Now we define the flow: +flow RFB_Flow(is_orig: bool) { + + # ## TODO: Determine if you want flowunit or datagram parsing: + + # Using flowunit will cause the anlayzer to buffer incremental input. + # This is needed for &oneline and &length. If you don't need this, you'll + # get better performance with datagram. + + # flowunit = RFB_PDU(is_orig) withcontext(connection, this); + + datagram = RFB_PDU(is_orig) withcontext(connection, this); + +}; + +%include rfb-analyzer.pac \ No newline at end of file diff --git a/testing/btest/Baseline/scripts.base.protocols.rfb.rfb-apple-remote-desktop/rfb.log b/testing/btest/Baseline/scripts.base.protocols.rfb.rfb-apple-remote-desktop/rfb.log new file mode 100644 index 0000000000..6f8a2e987d --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.rfb.rfb-apple-remote-desktop/rfb.log @@ -0,0 +1,12 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path rfb +#open 2016-04-11-08-25-48 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p client_major_version client_minor_version server_major_version server_minor_version authentication_method auth share_flag desktop_name width height +#types time string addr port addr port string string string string string bool bool string count count +1459148054.031382 CCvvfg3TEfuqmmG4bh 192.168.2.115 52353 192.168.2.16 5900 003 889 003 889 Apple Remote Desktop T T \x00\x00\x00\x00\x00\x02\xbf\xfe\xe7\x03\xe0\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00MacMini SSD 1920 1080 +1459148050.685932 CjhGID4nQcgTWjvg4c 192.168.2.115 52352 192.168.2.16 5900 003 889 003 889 Apple Remote Desktop F - - - - +1459148047.738043 CXWv6p3arKYeMETxOg 192.168.2.115 52351 192.168.2.16 5900 003 889 003 889 - - - - - - +#close 2016-04-11-08-25-48 diff --git a/testing/btest/Baseline/scripts.base.protocols.rfb.vnc-mac-to-linux/rfb.log b/testing/btest/Baseline/scripts.base.protocols.rfb.vnc-mac-to-linux/rfb.log new file mode 100644 index 0000000000..de1d70ec63 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.rfb.vnc-mac-to-linux/rfb.log @@ -0,0 +1,12 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path rfb +#open 2016-04-06-13-48-56 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p client_major_version client_minor_version server_major_version server_minor_version authentication_method auth share_flag desktop_name width height +#types time string addr port addr port string string string string string bool bool string count count +1459093553.334734 CsRx2w45OKnoww6xl4 192.168.2.115 49259 192.168.2.125 5901 003 003 003 008 VNC T T root's X desktop (martin-VirtualBox:1) 1024 768 +1459093548.745805 CjhGID4nQcgTWjvg4c 192.168.2.115 49256 192.168.2.125 5901 003 003 003 008 VNC - - - - - +1459093551.559391 CCvvfg3TEfuqmmG4bh 192.168.2.115 49257 192.168.2.125 5901 003 003 003 008 VNC F - - - - +#close 2016-04-06-13-48-56 diff --git a/testing/btest/Traces/rfb/vnc-mac-to-linux.pcap b/testing/btest/Traces/rfb/vnc-mac-to-linux.pcap new file mode 100644 index 0000000000000000000000000000000000000000..3856b94caebf0764b7f7eab92fea8a8e24caffa9 GIT binary patch literal 40255 zcmdU&2UHbT`>zKOu%KY?Jr)#HEU1VjMzMe)Hms;vP?|(VjPzK*ib_X75Ct^~*igZO zC3cVqA_fqQ0*ZhrAV@v;eK(oIna!O2{lE2J>)tC_vk2`s&wgglyJz<7b9`~{;&DZ; z8Q1vHj8lLwiX@dLqp~>eXZU~Insb9ZS5^fqEj(aR-JF}waZ~1KPT{7gtpEA;X~iwD zIp1E>x%C*6fh#fTyrM+%_!h@0HdDT=proX%pwPUfPV^2<{8=-_hVt_xq|#^>Tb zNc!N&iJ0DI9!Z8hbZ&in4gNLkVbw1kIIdy?$7yljOvbEi#bDf;fhm(hq>LLnx9*W( zI3JUc@mUeY81riY{tOv0eHNt8hOcpJ!l(}tApNqV4C99B*B+gOJuC=ox2cc2Z9{r9 z#pW#~5`#55kPbFfG5I#)qArX@?aoMUy z6HQpBcs{_y0T(FN2xN8Z!o73f(vZk=&NGpJRoIjl81>?>rgaCN@TUJ|s47P&O`rja6evi4dZQscTlVyk7Cf)VdPN!Yb$ovBc{& zp#@B|g2Z_ck6RPYX`YV>=N}YjENMA%ZWD4w?9D%1GWKGicib<9qqJIv;2u=gu$bp} z!abPTh%ny=<_>4h2$_dN;tTiYjI$v@9}~YCh$gnx=nx7ur{QdHc#FLY6FYb`XY{Uq z-!Sort2EJ~bL+0^8{w!!UAQZeoOGt^g2P+<8P)|%9|GxE6L4$dSgbBs?5=3527A~L zHjl0gv-;`8IK0KWfXOkC%-4k`$)i1=kfRqz9liT-^vvAa%J$oaR80IRli0M63)pKXHU{{LG7B=AI_=Mg}x8Qx<^vbX!bRTP2#=F`+AM_&z`?C{~tvM+{58O=FuBPtjwmYXoHnq3|F*wo{N z3QSZyNO>Fy9(Uu}TJyFcQ6ib0trO$+6%)@wqMYi&aZ@sh$5&sZt+wjiI`L*eHXK(n zcHM@9D!A^3RQwr^U6_6q(*LWm%gVFNcn0htr|H<$z>H%TCKq&NxgC{@Cfq_P?i@&; ziS^*2m#kYzhQ#;X8ck4c6}c(OewY}YBjXlXXCkX`q-o%6th3`Z%O!rb1 z;axy@rDrbog$7jc`k&$v6vjWtZ*{~GrLc>VYa26vYTqS9A*cKv9 z{DzE6j@B*T(&otL*DBqfoc?wGHni{z_Fw>xackmvobplP`5onX19+|xjtjGMpvQfK z5KGC=sDQUGJ?GKRF?qNG&6EQJlyfU{LAaH9UZSC*Fu!W=gizar_Xm_TMFcE5hD~?kt zgY6>(s=%fpkx1IYu<#3PM)3cA;qx5W=EC+f5{ye~Y!l@e%MH14h{hWS{xTGc?;nXb z1GT7f^AR0mVl5;dhF#;DZ=ApM-4+)FPIkO;*40ZTh`0964+U*Qby0sQtI? zRKgIL{!$`cztuBli$h4S$`@&|L(>-j-WOY(o1)A?)g~L|P*J(H@-a;Oqorsfl*0W} z8d{94)rt8Xju6+CQ-%LD8jcrysI^L;wGO%AkK=>}z=$V!tc)z5mVM%lljY`(pd!zsLX1fBMY- zzU*hQzu|w*`WgO?*a!c&{Abqh&_Cn4x_4Tf`lfU);td9F@21OD+VZ{&pZAH;Nh1MLWb}y`^^Fo_Y zjhUy1E&PA_kp?$?P)e{=ZcS}0zSQ6V!%a^|ES3Jvslma)vuwdmHx2c|J5kcXp&u!i zkZAovnKOcX+^9rFZqmf3m}q}aG_i{YcXhWgQAq(}q(jBgbY(6NzQ&C%>|DBXC>bhb zXDVQDT(A=cM{-Z@(Na*@`$0$_nFHlrbWf)ReD8Kxeblz7-8=sHfBOObsNoPbd!@{o z02gjegw9+&78^ZJmlC?mvuxo&w=U|{=Tf9X`{N;M6x#TI*bz2nDRT~L9NB2KYSzkO zxR>xV!ZCBg1{X(L?Cs!Vuv&AvnU}$H<)Qtf1`jv= zrmo*Rc|uO&_Gg8akDpg-y995^Ps|HkpFc6t)%@9Z(obNwf{#_TaUa6>bDaA?hb!a< zxBvcW^et6xLw|<{SBySeOrP|k6tdyb7EKmKd@^MGDlSgYrk zun=%u^?+`vN}OF^hnM8bDGAfoR9^~gHT=eixz{Hx8*yEK?$LSGgR>=jx7;7J{>if| zleR`aIUpN0X;6Ci0X;{W+{`pFUlguywQkxM%L==L6V^TP88K&Bb!u#t^&7k5q|~in zopTQOhs>=@JLQxxq^++`FWCl`oUv#XGr1^hRHxP{Ci%IKz1y33nyR00bWfYTy{BP8 zX>H=LO?Q+#tG(*-_rkO3BXld178iYW8CgJzhQ; z98osuueuu>V=s0Xwk$E)aQIETKl88H#hFHoK9Uqy99vQTC8M;1EGn^d1AhyZ*_Ry6UfIw0wsSoO9I2?w4qZ;kvwkBBLhNe|wkz;rWTE zrH_j)Y@QfVt(jPOp&<3tp+74fqRmgGoZT2PI)2fm@yRhS{31pg)}ML(^j>D^9;f9` z7kzVh(s_2Fb9mYEW&WjQTW!y-Px8ESE#mWy;u|4juJ=n=Qs*3cRM#eBX4mRD-*SGh z4yxRK=8&KDcvD&Dl}Be)o4tm=ic_JmmhfzFifes zn&UDd+;q0p`^YV&j(XYlnKnr|u9DY9S#g)*KK^m`s8w`aWL(*^3997@&wLX+@1Clx z$}<}93I2#K~-S^vlFI#J_cwC<`vG=ie z?;e?31(_8m)oHI>+`Z=f((bucX&*ytUcX1? zosMHgQGU|c9BY@luj%LYJ?x{}>ezwx3D$ghX^|qWj z=~uHI3f;^fk8zClj=uJGpLtn+Qj%>#-OHlPtHYC~53a4N%(3FB* zgV`HnVz-rKzjpe3N43AhoZP(X?XB;pWvk{z)wMi&{quH1?>vX1CqrX;tUFbDqaZTq zX!2x^;L(iUuj`QI6Op=DjNY2LGT6*qOitX$8*s4$0OAIo8F^jO*{u;4t zV_Zar{l)RgnK5^s`kU67T(>RjZ)34l0v*}-efmmQXFV5?^lqf8Y}L9Ms&x40=hC*L0gL}p)S>Ds;b_Hw%1z} z@M+dI%};RB>Ysn`aqIEYXJd{e8E(j)GQ;A>7?1M$9iAy>aqc7MB&<(9Hd^(U)Dx4U zuT^ANd52o(+^esvQ7!dLa9Wxcv-Z@do6x%omL5$>c~PG5cVFn#qn|vym{gcJFL3UJ z7b*IZ+OmS``kE43^Q5B-(?TQSpJvayWo~=s*zEV28QGJM)K`?vji}W!PCD}9qeayp z$yOUr7+$Zw^+We;mzrX`;>>61E1j3*rIouT*|to6AHO*vf6K3NCAr2)hX!kfzPqs9 z>%$VadQ~jEq#?uba2Lp6g|(SLo_n2c7D`^8N1~9#a1>WI}q@?(|tzlPpr}Bfh#GO0k%@ zq(l$uNN>#|5Zvw2^^*@R?LUUtJWcGtw#tujBK zVGqV;$@*nRdeKWRsnn56&d-zkbW8Owm$Wz5KVrAD!m;q0eQ`$ku=!3QR>R|?Bc^YS z`fA)WqNjGqXuk;4xXpPpuT`uVYGie0h5w8~iLPzf7z@YF@sS7ZqDwk{%z9aN{ex@F znO((|x2iv$J)8IKvWv^T%rn2HrVYtS$*s=Gw04NkeYae)EYRQ7{_mOF^HV)~Ih=mg zCi~}@pJFB1Peye=xihNg>`4`69pkiDbYJHBG&QX}#&%n1Xu;AP538@2N0fM*Wh{Ij zab;+MBr9^5OLr5AztgSXzP#-aIZN`7> zPuK;%wJ^T+?CYd>yT>qq9**w4J@<&+prKfY?+?D z@%5$H6*p~*w;mc^_;^sLQN{wt%Vkw9T7=te2RfTl-u9lMndi=X9jdFo`mAhZtjCIfs)tR9 z>S<|z>&t`gn#&z0uHAXJJbTyO{N&J;UoVtzf7~hJar(zHJC`;CUALX^3r)|+kc6hs zzfx0gW^;DUi|Fm$GbePp1vNi*{N8Anr-io8mjB${eIozPZ0BhO>b)lg-(Bcxx@=|d zr@h={uT09ubE!iz)V1rvY&`-^+Eq&?N3A)gw{?hfp;1nY%O$l1%j`0*+8Wm=*cZKe zo)|QC`0j-A6hqD7#R})0xt61%E;@Yu=o52mLfGc9)~^nKDp>fDJgsD#IKVzV(qP;CYQbl zo_XfjibXXO+t0n8ue+gFUTpe`?i(ibewgvx`r`I6QTMie35s#eK2@6Y5ud zubvlL5$i{fmYq&wtXtbknh0xg}QP4TZ@m@g{O+c)%N+2pER?{di=~POAvlTF7%w6^pX5)QRJ~Q#!WK696>Ay)_@oJFy?2p{pA2#JF2UFE6ydJwomFc}c=ErRljzKi>I$ zcymRL%bshjK4NYZ?9)CiRmo@p-0--Wel6$Pr@@`ksznng-kufWM>lbWv?m$ZD70TE0*N18nq09ZKkJ&$6{0 zQRu02QlW!DXpyk@&-~03dWr}gD-^o6BM5yWB6JW4ee)d%JqeALFZ78LHu_`Zf-LfI zIPeJwy@o;~>!m^mHx)XuoGJ7)5sLGbGWB`wH4yqjMCf4H6Zahm^@m2w7kY*~zi)Yy z>Y*<@%hoL;Lg8`nzco~|snAj1m_pAGp<{$X*DV8~*&;$UVNZR&1EJ;6X!$~u{jt%x zxClvRAxB^~2(3q<5#dsyLz)WptYr%Qg9sf>39SdAA1I;4P(z2no~C>ULhGQ>@`Y}H zjg8i8T#wsu%75@IGe3?(Pr@RGe|zXq5Lzs(p?|}ZF2-}2vqb19p-}VVAhc9O=up_x zs_#JPYy}mD&|bT+(Y|F=4=wd9TR)8moh}us)l}$1Wv0+`MCeGN(Dla3q!)l#Xh6mRV>Mp?XrG!$IgLVGXrt#}pbv zgbo)9wa^Bk-9&^AhdsUd4urmkM#~p^hm6FfiImW8AoMR3di;P?=!m95w{~I*Jx_!V zqlEqiLVpkuIs#VNa^Hc_=g?^RLKADS(eLrKld?VZ2hTFgEhscBTPjqisn84Em_jcQ zq1r;BmRmsRFiL0%oboy#w9j`Sv~72m&~0z9(fah>K7^%~3)T1r-#-ASi}8i7y=5jsFBbWBsB z8?~51V~J2rN~i>cnovSNLrH1XJi0B6P4&s7*KsT_qw^7lhva4un32M#~qvcOf=5utb&CNn5)`~X6!7Z%U1hEskVK69M%fJHmM025bv zQnYprv>IH{8!7E#psg03Pce`^m1}rHDnc7aXrGACx=^piK-$Gf`*x_R^rer~Ggn1U zKVTCk_Mny+A9dkr!!Jr{Hz6%_CisxF#^kwLsw`GA=^BkMnxf6R3t0_0VBItl5*4~O z<7PrWZcSMA<{Uy+b9}L%>{#l~vmS+Ou!erHYmG!=$SHG*q1+eD9SOM%tJjIuM;stnC!6*Lp?xSq>nD%4=NgvR zH)QfLHH4z|<7opIN@=%(*GJ&htHJ9pkjwCTlaM|TAq}QT6T$0NYi<|knke)-0J0ghw+QWh5!y(3w0(}Tye0==V(wdtHj<|eoF=8+0bUcq>rU=7 zS{Mbn4AKNbic@j)nRSjL{R_PAz(l1OmRB9}#A&A;MG6m@vBGL_{-*_`e+j*6KsJN+ zHle*ILVJaxea_SF$zgf@=7ovZKTx#r1h;`UaH5p<2zY%CUN>-`&_W>OGDzD zTBBadjQV;~6B9M)dzA+~Exe$Uudkut6<%nfID;!h3o9X)L3)pn-WDNEq)1UX#N$Uy)TUl_aWs)9^W{wU zX#;0VY0raKc)f{Y5AHo$NQGPm>3u?aON2CwB7F^B&tu~L)hw@-Co%Cx3M9&oTUk7* zf8&{jq_2fuk3u$s_5q>2DMFhskM_z5mec3sqnOcGo-Yy!7I2_xW#3oh4nBG#2|f2NUt^Z8l*;%7J%2+n3xBN^2aSj zQdurs6nYI(<4OHq2uKTrUUgvthe6xWyJX)qj`!M09u>EfjP_&(R+Cb?n z@U9~8S^!=zbHu_R3npn2@p?tXYY&PPo}%q4La(uqDBo)=xs$D@-u5A=2T$sEUg{M| zWgbysc3Vp(?K46fD?&R^9_?#*Wt{x`trIgbvF#|z>p-41u#1$o4r#&bX)XgvO|CLY zlL_f%5mGIRv=qG7VdBC#me9aE;c0inI%2~z@EXSbjk(9aGD)8kQao#^`WmE9kyhx$ z>~4vPd9^IBlgT~29gf*D*8@TNJgJ|Bl(ZF)RtUYC)-h>Q2(9d!bMke-fTFDg+Ez&G zr>n~7uTNAk@gsE?6J)^C2INa=y8vw^csBTc;~?wRh$r=%E+y>-q%}gXlO{81Ul3Xxz3EmW=_a$$M(A@#L4HB%V0oG10` zCnbf2eufgES7%oyExDJAjuN3Y7opWe+U9XAuj%B0SQgIY$o2wro;Ki=lon>*4JF{! zmwSP^9pjm#>BQ?f5mIX*sgV+R)y2fKkSKrLN|}O*Z|VAK&6D~m$dVc<3B9UaXVShR zv}Z+V?d8$-tz&teM~;+P1m)G9rwwqI(i%u4MoQq-n@dFtf50mg8CNcU6Vg9KNS!Fs z=HS%;6IEdzQ@+=}_L%6cK#@A}q`sL_(iuS7Tv6QudK#1T9U;Y8Xu7{9QlveA^e`s+LZW=HyNoe$#9)dvktg-V6{Td|)ovv1A@o|$ zrp+a^p(3;|<H%JTxInbf=OEK-9w9v@LYhU9 z_64tGUi-Nx%WE@BOg!J3@|wky`nHggh5>0`q1UQIOjkPBPQv4;x$-=w1^@d0A5LD`2!N=_t(2UF|l+B zMOws@`h1XjMbZI6ul7%vv>ymgw#)jvLWs`MUm=)SMr2+VG7G@{%uT5xv0 zu@$4fwur>UKhh{#U7psjNJ?7+UUk9iiw3V6of`iU7?IUkUkIs}2&q0rss~6qv^k|NdTNqxo$NcDtX)jBh2O9|}}5n2Ozv}PtOuQ$kAx=LI>Av2#9Y{1j{ z-QsE8VP4xv54`3yc>Myoj2=L);-e3XkQz~>lXPO-$-K7fB9_-Al9<1fA~oVkecA~~ zCked(cd^m`rHQ{Ofq)#{8yMotA;PnG{0DD)T04C{I zLW(OD>58?SBK--xcE!XGkSM>uhU~?}PSg|5;N?82_a~`WB>hR~bxBr`>-*K)V3Ew&VU3GF!qP7=59RFuRK|N6KS< zmdol3;}>J1BV7?9d1hbflUaA1iACm)Tr#@on8zfoC#1VYNY7EEi@`3LGrkCk^2Y?% z)0j9qnX-G1C-t5rwTq;SgigNLPVZvKIP?CClqZCrq4PLy;!(q+Z_yq^pEpb0C{Rt3+tsL}*{iqn#GT^7^AG zCT{6P(Z1wqee$HVuol{A6{%gt=yeC=GDw>dQoQ%1`)d|Ox(>XOwa~qySzcRi#Kb>Z zP^4Krsn-hu={lj;*^tekRVK79BDDGPX!F0Yyw+!7Vl2+>%3c}d^RzypJk- zlKYBYhm|r($r_btbV_?IqDU>kE1BI7fkgRrpwkm14Z(E@vZO^ksn>Y{sfEz%eKxHs z@#-W(TOp6OU}PIceck&5CJtRh(N^%ZJ|R-t3*glPymA~_Q`L7=8&(f!PDmX^NNXul z8}NDok5mXG%J*tZp3P6Ep3Mi>@}yn|1f(`XuPJQW7KGNJDQ$>~JlacJSzZtH!^FYV z40nhMPun=N<$ede+Q2n;g@Q6#FtcZRZAnPCi;${Oq?^F&9ZcNd#PT|GAtqY)6ZU`* zHJ;RKgVZaMZW4OUf^0?|Xhmp$6QS)y(Qf8x+h1jQeN5)cKH}Yx%-I#viKq2hFQxqp zXg7n`w+dtiVjko&NYx1GHWAVu6sbLs{)Jv!USoOfOP=<7QLEuYdhn!P(*&gULa!?z zn?c)}(Ata84wOf0_>JXtqYfq-%%;2!JUJ70v!0QQx)|l(jy5Uu)NZWL7Q$>T2Hk<=+E_7~v z``^`?A^H@lD|ju%MB_FM{|pV**d}*flg4erL?;!BRG%k3QYH0@q^^x#F?&B`GrYDX zw3|h|8pxxy)n|F_br%zT<0x7Kp4Mlhl(rUW!RuB9RV0mrTm~swOA)z}hNaU+aA^F*cu~wdZMl-biU-g|X3oWsZBxC7^{m$YqdrA*5C!q)rs+ zA@Hh+iF=wUF#2oNSxk(7N|8G8q(?RjNDm3UUW04~ZC65TDMITmk9PEQmeE{YXErN#+jpq0Z*8uy$I<#5z=!MX)t(Qgo$q; zQU18)M8=kJt0}MNc+w-P0@7fiSM{e%+TMf~uQlki>xw+unwE-;`udrS27glXSs_<= zT3@)D`nSKDgV$j2x{jNN7Jh1_$m#)o2J22y_i9V#o~PlxrtI~= z4W9IHu7EU5=(U#`leRCR#bZW$eISoE!ieSd1$oAkL#-tVdBD^9&XLmMTIewFYQ&90 zuU{dTac1=+q*%9U(nN~%BzU#M#DF<0uLWV4_~a$!HIXMhEPdALf%DoYg)TdJ8x3ABg4e;E8hTv_xeU@FgcRp>sUC1l zjUtT&uhE!T0EzPZYbo)1m-2c{jVC=kRzMmn^xAellXfVfT`WS|NgnM;UzXQ&TTFc2 zit^ftr}Zh5(&Ad^SnxUs{%tj|-~_o0uUdo@pF_}Idr+it;PnP3cJX6*byC8JJ*LHz9{M04O%QsW0oe@N;e>X9 z2(7L>T8(C`_f8qR0~4L9C|X^f)@LhEyQdv^O(0%dpaoaRWsr^_r1M2c^(j(#7p*<* zFtNQd>%CL*%rG&zBSosulOFOAkUkfBoekLxS{*_=PlVP$9_`OPS?`&!fSj+Zj44_J zp4O+mly*LNh4=h|{@NTZwCKfp7Xp=$gcM)fPS*h=iu4_LosWr&AW?pO9kdz~57J(Z zc+x{Z2uR-vy}n}8jv}D%*+RLMT%&}hEuVsvh zd$cH8d!E)iOiJqkUW>tNA5H;DTPUzz+pj|Ij-m}kNS!FsFW}V!6Zb=+e6L#-kaRLW zGnMt~#FKhn6p(%q)`6#N+VR9IK0T$+E_ZpfHB(t$CzfEM{Q-*Bou~DN=NAq0E_-~z z>lfmcLklZ@VtUmhq_adw4^X7=o-})WF)P-c~2V5 zR-4A8oj_=3iqQIr(E5#r#39!jXUM3hobdj_DOH&G53T~1?FD{>wyg%|H-e|#htCZw zFwtKoQRGRiK4F|8!szeTR`z`#=lJTx?5o8@wRjez(jH7aUqdna17m@khPuS}h%DnA G-~R*9_Hix% literal 0 HcmV?d00001 diff --git a/testing/btest/Traces/rfb/vncmac.pcap b/testing/btest/Traces/rfb/vncmac.pcap new file mode 100644 index 0000000000000000000000000000000000000000..026078185de32f2f9651a6f265dc966e06795324 GIT binary patch literal 8848 zcmd6scUTnX7RHBVK}1$iKu{A|Q3NF+7?vtc#EOz2(GUx|AOxg{6r;r8VicnnOc01- z7aO7oKGsB3P}E>RgGP;oTr5~0QOmVjzparj8-G>%oN+g7ZCPprZl}3cg=Y_>fL!uXiM!{bpL_`l_v5V6jCmBI# z6TJyvdHffVvC@%_b}~DA_*_gl5Q(92^O9oXq&Cr!A#tJdIO(FW_yoM!NIOS62kHCL z(Q$FnVf4F>vf+4Vf#r77DFO1G(IdWRvH)~kB^M~ zocwD9ey$5=I(13(`#^`-ZrNMz57&4utv>hLkvkh^G^9s%>JeiJ!uxzLZ^B#m*>9!! zq69dNDwmlVI2<4T3ErSs)YTPT12nPD5iMrTCdxgg>0I#m-c{Aef81A) zZ*F9&^OEKtLjKfoK2m#o2Rj!RSAES|glZ8ATGZD#&-wcKds)TIe> zAt$|nTVgK3hb!m@+Y5EMSOFg4fqfCT!&DA41I`WvkMCS?4Eg%bf_(5|!gM8Hb{F}g zK||m^%z1*V0omN%*a{ld=V81AHN-sN)I$!v`5Zln!cmjGS-aF;Pj;8h=hi^c&0h?! zc%TPfj_w-x!DYyw{(&IBkCABz42R;^&!G4?mvgzu!*MfvQpd@i?n!g}C28o72m8RJ&e&TcpW`BUK9eB%l`d&;`=v&Zd!q3jv^Y4YIaSE!{X zp7wz11JqN99_XggwGXB|U}le-eFfB=+lNVWGE_!{g@`9I9*_j_vORFQNE~HK!iw&>JP_fzAO+ z1MpAV>v@UU9&l@*^Z;M+!1-}}rH6bB{EjCt4~};#k$=llkPjX_&SUcjy;U9+z`_Ih zF4tA#=Vw6oYvdn{;nbi6GLS)ze;-`~UG#uGezh6E54!os+CAjs;2KXZU(01m74pL^ z1o<%j)*&=sLh?m~Lb>~_Fua{NIO43V{6fgT6nEdL9xn3NTl0BkqNvyDJ)?{_Sq4nf zTe4Oru6dTz|8SMtn1S}T`Z0gZF>(Iro?}c*z}c}6R~}sy5}b0OWwOuHJ1nm^*UZ*r^7;fJ4I z{o{%LsQa%T#?>tmZ_3Mz$~7;1C*9)U{>g{ZPbEkKwJIKTF8#^3zsdIacCFn#q>D-~ zHy-)3;ZAJI#-kOv=5v}O4ffrevCVLV_WDCk`?f}$J^J)d7spNIb$ybTU*3O)Sgvo? zcwN86*Xq@K&I3+CgE8kMZVgtz8#q5JumEsi25O)N>U+Td6j=iYP6jh-XE&BY1|(wo zdA+^@dO*VFtL*_4*3GXQ;3@ZkYdpCM{YFhQwws7!@!_~ER>-O)~8{ux6 z)lYZvrNo3w&R@yDTe-3KsvoMq&&{jLO|5A5Hwhk-(c5K}Pw}H+FP|mY9K1Mv#r8Rl zIR>?fQ}?(P97&jy@BGSIdw5An=r0>G)9)^EtFA~3csB5K{K=OIK~*=h4DLp3c7Gk6 zBys7gTJ4vS=inds!b76(`bp2_WyZ%>P5S=cOWk=>Ez1_(NuJoc*IHT|LzFb7*S)*L zd93g92m4D+q@AY^PHUE&8M-;Vz$2w7+v|GD`fW9zUU^vAV^Sc8D zTqB>k4t%8TEguK(;mPB>)a*t6OUx0rd~W>l0uB7WU$;f8LgY& z$4Jr!x1#n)?co$(=y?Qr=si9Q;DO`b1zh#tXPr2Q<9;Kh$DL|s(9@{375_DUpLJ*> z`#IGryFDB~<3FczJrH^xSsbkLfR_VV`fSugPLJCKvL6XojC~0=Zwm05oBbV8 znzH|e6Y@O{GJIF@;W%b*3CNzESZ=3&%Q<4)H= zmBBR9i>GuAq9FS&Y%kPh5Ov^z{C?3YFL)X7c3X)2WXwOe2hwoAD(=o8)Pdxy_kycM z$P2GTl^6V6ocJ2Gyou)2C9S7Xn~3z{4qX>F)Wi0ITMGq@A9KN6o6k4LY1pH^a!3Bj zVnIH5F(HuV=OX`~?a|smgZeytdKWd!HeuJJO$7b1n)JYo(g6G_$OGrQeQphu9^fk; zn1_z%aT@k$;u7Q+I|%Z@gO7T%`TzYM&9756etrfn*&@HboKu6L&Z^!YOdu_@se*J9jXq!d1@?@5!|5lV7d-9~{ zXk|~W<>rT6#Q}YSHy;hY+2g^6sB+g#vy*iRMP4IJi1VJkURxV>N{n}f4ZU-}tIG5G zwUok|lQ$-rt~R**i`fRvH4xgP1wswf_khDbtU>B{P6oBZzRe<^(W84%H30u_Ko9I> z^VRkMBLjBMz=vz$J(~XwY-p<_g7mN>fRcjer44J$CXpA>l~1*E&8T5d0BMF1-aw#zTcmyj;$Y- zI&j!T?KMBXsZ7i4eJky%U(S%SIpe3lxH;%#V|lq}@}8U|*^K_q{y%$M%-I=WxYztb zgK2iMZF1_zuBTp496t5ec&GBd0aMGae&TfICnLA|E5+}|ZJuCgU$>mogFOGiXPk!> zWCv1n(Cx{p`t#?2C~|)=k@I|zmC@6v9p?ba{1(c8@V*E#0RId9R&xM@@9&wQVUHHD z9_t@EjXei+^P&FppE-_-?(NDrC>`QSl*1DjvQ8vonyD<`VP&(A>g8stw1=hPq* zG9V_$|AOMf*hg4pcg$svA6s2$pMC<~#{<_vkFW&!m+(8DJbp~s66C-15ah%7R}ZB5 zBaz=jA$F0P!(P1${}|~Z1&+bwPtlqGJl3uyf2Dp*k%vx+jE$5A2Kusm Date: Mon, 11 Apr 2016 11:28:22 +0200 Subject: [PATCH 37/84] Implement protocol confirmation Do not set the service field in the bro script but use the protocol confirmation paradigm. Protocol is considered confirmed if both a succesful client and server banner have been parsed. --- scripts/base/protocols/rfb/main.bro | 1 - src/analyzer/protocol/rfb/rfb-analyzer.pac | 11 +++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/scripts/base/protocols/rfb/main.bro b/scripts/base/protocols/rfb/main.bro index 97f194b789..60dcd17b03 100644 --- a/scripts/base/protocols/rfb/main.bro +++ b/scripts/base/protocols/rfb/main.bro @@ -106,7 +106,6 @@ event rfb_server_version(c: connection, major_version: string, minor_version: st set_session(c); c$rfb_state$server_major_version = major_version; c$rfb_state$server_minor_version = minor_version; - add c$service["rfb"]; } event rfb_authentication_type(c: connection, authtype: count) diff --git a/src/analyzer/protocol/rfb/rfb-analyzer.pac b/src/analyzer/protocol/rfb/rfb-analyzer.pac index 4233a423f7..d357ddee28 100644 --- a/src/analyzer/protocol/rfb/rfb-analyzer.pac +++ b/src/analyzer/protocol/rfb/rfb-analyzer.pac @@ -1,5 +1,3 @@ -# Generated by binpac_quickstart - refine flow RFB_Flow += { function proc_rfb_message(msg: RFB_PDU): bool %{ @@ -7,16 +5,13 @@ refine flow RFB_Flow += { return true; %} - function proc_rfb_client_version(major: bytestring, minor: bytestring) : bool - %{ - BifEvent::generate_rfb_client_version(connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), bytestring_to_val(major), bytestring_to_val(minor)); - return true; - %} - function proc_rfb_version(client: bool, major: bytestring, minor: bytestring) : bool %{ if (client) { BifEvent::generate_rfb_client_version(connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), bytestring_to_val(major), bytestring_to_val(minor)); + + connection()->bro_analyzer()->ProtocolConfirmation(); + } else { BifEvent::generate_rfb_server_version(connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), bytestring_to_val(major), bytestring_to_val(minor)); } From 034f725f3f45bda74dadb86bb977fa0540ffb246 Mon Sep 17 00:00:00 2001 From: Martin van Hensbergen Date: Mon, 11 Apr 2016 11:35:36 +0200 Subject: [PATCH 38/84] Some styling tweaks - used transient declarations where appropriate - fixed brackets - cleaned up some comments --- scripts/base/protocols/rfb/main.bro | 60 ++++++---- src/analyzer/protocol/rfb/CMakeLists.txt | 2 - src/analyzer/protocol/rfb/Plugin.cc | 2 - src/analyzer/protocol/rfb/RFB.cc | 2 - src/analyzer/protocol/rfb/RFB.h | 2 - src/analyzer/protocol/rfb/events.bif | 2 +- src/analyzer/protocol/rfb/rfb-analyzer.pac | 128 +++++++++++---------- src/analyzer/protocol/rfb/rfb-protocol.pac | 12 +- src/analyzer/protocol/rfb/rfb.pac | 12 -- 9 files changed, 109 insertions(+), 113 deletions(-) diff --git a/scripts/base/protocols/rfb/main.bro b/scripts/base/protocols/rfb/main.bro index 60dcd17b03..50673d6514 100644 --- a/scripts/base/protocols/rfb/main.bro +++ b/scripts/base/protocols/rfb/main.bro @@ -1,4 +1,4 @@ -module Rfb; +module RFB; export { redef enum Log::ID += { LOG }; @@ -11,17 +11,27 @@ export { ## The connection's 4-tuple of endpoint addresses/ports. id: conn_id &log; + ## Major version of the client. client_major_version: string &log &optional; + ## Minor version of the client. client_minor_version: string &log &optional; + ## Major version of the server. server_major_version: string &log &optional; + ## Major version of the client. server_minor_version: string &log &optional; + ## Identifier of authentication method used. authentication_method: string &log &optional; + ## Whether or not authentication was succesful. auth: bool &log &optional; + ## Whether the client has an exclusive or a shared session. share_flag: bool &log &optional; + ## Name of the screen that is being shared. desktop_name: string &log &optional; + ## Width of the screen that is being shared. width: count &log &optional; + ## Height of the screen that is being shared. height: count &log &optional; done: bool &default=F; @@ -30,7 +40,8 @@ export { global log_rfb: event(rec: Info); } -function friendly_auth_name(auth: count): string { +function friendly_auth_name(auth: count): string + { switch (auth) { case 0: return "Invalid"; @@ -56,37 +67,40 @@ function friendly_auth_name(auth: count): string { return "Apple Remote Desktop"; } return "RealVNC"; - } - redef record connection += { rfb_state: Info &optional; }; event bro_init() &priority=5 { - Log::create_stream(Rfb::LOG, [$columns=Info, $ev=log_rfb, $path="rfb"]); + Log::create_stream(RFB::LOG, [$columns=Info, $ev=log_rfb, $path="rfb"]); } -function write_log(c:connection) { +function write_log(c:connection) + { local state = c$rfb_state; - if ( state?$done && state$done == T) { + if ( state?$done && state$done == T ) + { return; - } - Log::write(Rfb::LOG, c$rfb_state); - c$rfb_state$done = T; -} + } -function set_session(c: connection) { - if ( ! c?$rfb_state ) { + Log::write(RFB::LOG, c$rfb_state); + c$rfb_state$done = T; + } + +function set_session(c: connection) + { + if ( ! c?$rfb_state ) + { local info: Info; info$ts = network_time(); info$uid = c$uid; info$id = c$id; c$rfb_state = info; - } + } } event rfb_event(c: connection) @@ -121,13 +135,9 @@ event rfb_server_parameters(c: connection, name: string, width: count, height: c write_log(c); } -event rfb_auth_result(c: connection, result: count) +event rfb_auth_result(c: connection, result: bool) { - if ( result ==0 ) { - c$rfb_state$auth = T; - } else { - c$rfb_state$auth = F; - } + c$rfb_state$auth = !result; } event rfb_share_flag(c: connection, flag: bool) @@ -135,8 +145,10 @@ event rfb_share_flag(c: connection, flag: bool) c$rfb_state$share_flag = flag; } -event connection_state_remove(c: connection) { - if ( c?$rfb_state ) { - write_log(c); +event connection_state_remove(c: connection) + { + if ( c?$rfb_state ) + { + write_log(c); + } } -} diff --git a/src/analyzer/protocol/rfb/CMakeLists.txt b/src/analyzer/protocol/rfb/CMakeLists.txt index 8131ca7362..28523bfe2d 100644 --- a/src/analyzer/protocol/rfb/CMakeLists.txt +++ b/src/analyzer/protocol/rfb/CMakeLists.txt @@ -1,5 +1,3 @@ -# Generated by binpac_quickstart - include(BroPlugin) include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/src/analyzer/protocol/rfb/Plugin.cc b/src/analyzer/protocol/rfb/Plugin.cc index 55704497e9..b3bed0f093 100644 --- a/src/analyzer/protocol/rfb/Plugin.cc +++ b/src/analyzer/protocol/rfb/Plugin.cc @@ -1,5 +1,3 @@ -// Generated by binpac_quickstart - #include "plugin/Plugin.h" #include "RFB.h" diff --git a/src/analyzer/protocol/rfb/RFB.cc b/src/analyzer/protocol/rfb/RFB.cc index c761d0bf0f..2669d6ed56 100644 --- a/src/analyzer/protocol/rfb/RFB.cc +++ b/src/analyzer/protocol/rfb/RFB.cc @@ -1,5 +1,3 @@ -// Generated by binpac_quickstart - #include "RFB.h" #include "analyzer/protocol/tcp/TCP_Reassembler.h" diff --git a/src/analyzer/protocol/rfb/RFB.h b/src/analyzer/protocol/rfb/RFB.h index cd6e7348d0..88a17eea5a 100644 --- a/src/analyzer/protocol/rfb/RFB.h +++ b/src/analyzer/protocol/rfb/RFB.h @@ -1,5 +1,3 @@ -// Generated by binpac_quickstart - #ifndef ANALYZER_PROTOCOL_RFB_RFB_H #define ANALYZER_PROTOCOL_RFB_RFB_H diff --git a/src/analyzer/protocol/rfb/events.bif b/src/analyzer/protocol/rfb/events.bif index a3cf5f7ad8..4a5bb40121 100644 --- a/src/analyzer/protocol/rfb/events.bif +++ b/src/analyzer/protocol/rfb/events.bif @@ -15,7 +15,7 @@ event rfb_authentication_type%(c: connection, authtype: count%); ## c: The connection record for the underlying transport-layer session/flow. ## ## result: whether or not authentication was succesful -event rfb_auth_result%(c: connection, result: count%); +event rfb_auth_result%(c: connection, result: bool%); ## Generated for RFB event share flag messages ## diff --git a/src/analyzer/protocol/rfb/rfb-analyzer.pac b/src/analyzer/protocol/rfb/rfb-analyzer.pac index d357ddee28..69e8e7a99a 100644 --- a/src/analyzer/protocol/rfb/rfb-analyzer.pac +++ b/src/analyzer/protocol/rfb/rfb-analyzer.pac @@ -7,14 +7,16 @@ refine flow RFB_Flow += { function proc_rfb_version(client: bool, major: bytestring, minor: bytestring) : bool %{ - if (client) { + if (client) + { BifEvent::generate_rfb_client_version(connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), bytestring_to_val(major), bytestring_to_val(minor)); connection()->bro_analyzer()->ProtocolConfirmation(); - - } else { + } + else + { BifEvent::generate_rfb_server_version(connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), bytestring_to_val(major), bytestring_to_val(minor)); - } + } return true; %} @@ -25,28 +27,28 @@ refine flow RFB_Flow += { %} function proc_security_types(msg: RFBSecurityTypes) : bool - %{ + %{ BifEvent::generate_rfb_authentication_type(connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), ${msg.sectype}); return true; - %} + %} function proc_security_types37(msg: RFBAuthTypeSelected) : bool - %{ + %{ BifEvent::generate_rfb_authentication_type(connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), ${msg.type}); return true; - %} + %} function proc_handle_server_params(msg:RFBServerInit) : bool - %{ + %{ BifEvent::generate_rfb_server_parameters(connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), bytestring_to_val(${msg.name}), ${msg.width}, ${msg.height}); return true; - %} + %} function proc_handle_security_result(result : uint32) : bool - %{ + %{ BifEvent::generate_rfb_auth_result(connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), result); return true; - %} + %} }; refine connection RFB_Conn += { @@ -70,113 +72,115 @@ refine connection RFB_Conn += { %} function get_state(client: bool) : int - %{ + %{ return state; - %} + %} function handle_banners(client: bool, msg: RFBProtocolVersion) : bool - %{ - if ( client ) { + %{ + if ( client ) + { // Set protocol version on client's version int minor_version = bytestring_to_int(${msg.minor},10); // Apple specifies minor version "889" but talks v37 - if ( minor_version >= 7 ) { + if ( minor_version >= 7 ) state = AWAITING_SERVER_AUTH_TYPES37; - } else { + else state = AWAITING_SERVER_AUTH_TYPES; } - } else { - if ( !client ) { + else state = AWAITING_CLIENT_BANNER; - } - } + return true; - %} + %} function handle_ard_challenge() : bool - %{ + %{ state = AWAITING_CLIENT_ARD_RESPONSE; return true; - %} + %} function handle_ard_response() : bool - %{ + %{ state = AWAITING_SERVER_AUTH_RESULT; return true; - %} + %} function handle_auth_request() : bool - %{ + %{ state = AWAITING_CLIENT_RESPONSE; return true; - %} + %} function handle_auth_response() : bool - %{ + %{ state = AWAITING_SERVER_AUTH_RESULT; return true; - %} + %} function handle_security_result(msg: RFBSecurityResult) : bool - %{ - if ( ${msg.result} == 0 ) //FIXME - { + %{ + if ( ${msg.result} == 0 ) + { state = AWAITING_CLIENT_SHARE_FLAG; - } + } return true; - %} + %} function handle_client_init(msg: RFBClientInit) : bool - %{ + %{ state = AWAITING_SERVER_PARAMS; - return true; - %} + %} function handle_server_init(msg: RFBServerInit) : bool - %{ + %{ state = RFB_MESSAGE; return true; - %} + %} function handle_security_types(msg: RFBSecurityTypes): bool - %{ - if ( msg->sectype() == 0 ) { // No auth + %{ + if ( msg->sectype() == 0 ) + { // No auth state = AWAITING_CLIENT_SHARE_FLAG; return true; - } - if ( msg->sectype() == 2 ) { //VNC + } + + if ( msg->sectype() == 2 ) + { //VNC state = AWAITING_SERVER_CHALLENGE; - } - return false; - %} + } + return true; + %} function handle_security_types37(msg: RFBSecurityTypes37): bool - %{ - if ( ${msg.count} == 0 ) { // No auth + %{ + if ( ${msg.count} == 0 ) + { // No auth state = AWAITING_CLIENT_SHARE_FLAG; return true; - } + } state = AWAITING_CLIENT_AUTH_TYPE_SELECTED37; return true; - %} + %} function handle_auth_type_selected(msg: RFBAuthTypeSelected): bool - %{ - if ( ${msg.type} == 30 ) { // Apple Remote Desktop - state = AWAITING_SERVER_ARD_CHALLENGE; - return true; - } + %{ + if ( ${msg.type} == 30 ) + { // Apple Remote Desktop + state = AWAITING_SERVER_ARD_CHALLENGE; + return true; + } - if ( ${msg.type} == 1 ) { // No Auth + if ( ${msg.type} == 1 ) state = AWAITING_SERVER_AUTH_RESULT; - } else { - // Assume VNC + else state = AWAITING_SERVER_CHALLENGE; - } + return true; - %} + %} %member{ uint8 state = AWAITING_SERVER_BANNER; diff --git a/src/analyzer/protocol/rfb/rfb-protocol.pac b/src/analyzer/protocol/rfb/rfb-protocol.pac index 0eb5542001..764046e747 100644 --- a/src/analyzer/protocol/rfb/rfb-protocol.pac +++ b/src/analyzer/protocol/rfb/rfb-protocol.pac @@ -16,8 +16,8 @@ enum states { }; type RFBProtocolVersion (client: bool) = record { - header : "RFB "; - major :bytestring &length=3; + header: "RFB "; + major: bytestring &length=3; dot: "."; minor: bytestring &length=3; pad: uint8; @@ -108,8 +108,8 @@ type RFB_PDU_request = record { AWAITING_CLIENT_SHARE_FLAG -> shareflag: RFBClientInit; AWAITING_CLIENT_AUTH_TYPE_SELECTED37 -> authtype: RFBAuthTypeSelected; AWAITING_CLIENT_ARD_RESPONSE -> ard_response: RFBSecurityARDResponse; - RFB_MESSAGE -> ignore: bytestring &restofdata; - default -> data: bytestring &restofdata; + RFB_MESSAGE -> ignore: bytestring &restofdata &transient; + default -> data: bytestring &restofdata &transient; } &requires(state); } &let { state: uint8 = $context.connection.get_state(true); @@ -124,8 +124,8 @@ type RFB_PDU_response = record { AWAITING_SERVER_AUTH_RESULT -> authresult : RFBSecurityResult; AWAITING_SERVER_ARD_CHALLENGE -> ard_challenge: RFBSecurityARDChallenge; AWAITING_SERVER_PARAMS -> serverinit: RFBServerInit; - RFB_MESSAGE -> ignore: bytestring &restofdata; - default -> data: bytestring &restofdata; + RFB_MESSAGE -> ignore: bytestring &restofdata &transient; + default -> data: bytestring &restofdata &transient; } &requires(rstate); } &let { rstate: uint8 = $context.connection.get_state(false); diff --git a/src/analyzer/protocol/rfb/rfb.pac b/src/analyzer/protocol/rfb/rfb.pac index 310ad38893..2e88f8e5bb 100644 --- a/src/analyzer/protocol/rfb/rfb.pac +++ b/src/analyzer/protocol/rfb/rfb.pac @@ -1,5 +1,3 @@ -# Generated by binpac_quickstart - # Analyzer for Parser for rfb (VNC) # - rfb-protocol.pac: describes the rfb protocol messages # - rfb-analyzer.pac: describes the rfb analyzer code @@ -26,17 +24,7 @@ connection RFB_Conn(bro_analyzer: BroAnalyzer) { # Now we define the flow: flow RFB_Flow(is_orig: bool) { - - # ## TODO: Determine if you want flowunit or datagram parsing: - - # Using flowunit will cause the anlayzer to buffer incremental input. - # This is needed for &oneline and &length. If you don't need this, you'll - # get better performance with datagram. - - # flowunit = RFB_PDU(is_orig) withcontext(connection, this); - datagram = RFB_PDU(is_orig) withcontext(connection, this); - }; %include rfb-analyzer.pac \ No newline at end of file From 000540645dfc406074d2a2098418711348b98079 Mon Sep 17 00:00:00 2001 From: Martin van Hensbergen Date: Mon, 11 Apr 2016 11:37:50 +0200 Subject: [PATCH 39/84] Fixed issue in state machine There is a slight difference in the message sequence between version 3.7 and 3.8. Version 3.8 will always send a Authentication Result message when authentication type 'None' is selected while 3.7 does not. --- src/analyzer/protocol/rfb/rfb-analyzer.pac | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/analyzer/protocol/rfb/rfb-analyzer.pac b/src/analyzer/protocol/rfb/rfb-analyzer.pac index 69e8e7a99a..b63b9f4085 100644 --- a/src/analyzer/protocol/rfb/rfb-analyzer.pac +++ b/src/analyzer/protocol/rfb/rfb-analyzer.pac @@ -82,6 +82,7 @@ refine connection RFB_Conn += { { // Set protocol version on client's version int minor_version = bytestring_to_int(${msg.minor},10); + version = minor_version; // Apple specifies minor version "889" but talks v37 if ( minor_version >= 7 ) @@ -175,7 +176,12 @@ refine connection RFB_Conn += { } if ( ${msg.type} == 1 ) - state = AWAITING_SERVER_AUTH_RESULT; + { + if ( version > 7 ) + state = AWAITING_SERVER_AUTH_RESULT; + else + state = AWAITING_CLIENT_SHARE_FLAG; + } else state = AWAITING_SERVER_CHALLENGE; @@ -184,6 +190,7 @@ refine connection RFB_Conn += { %member{ uint8 state = AWAITING_SERVER_BANNER; + int version = 0; %} }; From 00e759b44c36750666a2db545a2137ecf4ef5d53 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Mon, 11 Apr 2016 15:50:02 +0200 Subject: [PATCH 40/84] Intel: CERT_HASH indicator type was never checked Hence, when people specify data of type CERT_HASH in their intel source files, it will never trigger an alert. --- scripts/policy/frameworks/intel/seen/x509.bro | 11 +++++++++++ .../intel-all.log | 11 +++++++---- .../scripts/policy/frameworks/intel/seen/certs.bro | 1 + 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/scripts/policy/frameworks/intel/seen/x509.bro b/scripts/policy/frameworks/intel/seen/x509.bro index 3a2859b6d5..9dcbc3edb9 100644 --- a/scripts/policy/frameworks/intel/seen/x509.bro +++ b/scripts/policy/frameworks/intel/seen/x509.bro @@ -26,3 +26,14 @@ event x509_certificate(f: fa_file, cert_ref: opaque of x509, cert: X509::Certifi $where=X509::IN_CERT]); } } + +event file_hash(f: fa_file, kind: string, hash: string) + { + if ( ! f?$info || ! f$info?$x509 || kind != "sha1" ) + return; + + Intel::seen([$indicator=hash, + $indicator_type=Intel::CERT_HASH, + $f=f, + $where=X509::IN_CERT]); + } diff --git a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log index ba1afe4239..4b5786e00d 100644 --- a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log +++ b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log @@ -3,20 +3,23 @@ #empty_field (empty) #unset_field - #path intel -#open 2015-03-14-01-47-46 +#open 2016-04-11-13-48-49 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node sources #types time string addr port addr port string string string string enum enum string set[string] 1416942644.593119 CXWv6p3arKYeMETxOg 192.168.4.149 49422 23.92.19.75 443 F0txuw2pvrkZOn04a8 application/pkix-cert 23.92.19.75:443/tcp www.pantz.org Intel::DOMAIN X509::IN_CERT bro source1 -#close 2015-03-14-01-47-46 +#close 2016-04-11-13-48-49 #separator \x09 #set_separator , #empty_field (empty) #unset_field - #path intel -#open 2015-03-14-01-47-46 +#open 2016-04-11-13-48-49 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node sources #types time string addr port addr port string string string string enum enum string set[string] +1170717505.735416 CXWv6p3arKYeMETxOg 192.150.187.164 58868 194.127.84.106 443 FeCwNK3rzqPnZ7eBQ5 application/pkix-cert 194.127.84.106:443/tcp 2c322ae2b7fe91391345e070b63668978bb1c9da Intel::CERT_HASH X509::IN_CERT bro source1 1170717505.934612 CXWv6p3arKYeMETxOg 192.150.187.164 58868 194.127.84.106 443 - - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro source1 +1170717508.883051 CjhGID4nQcgTWjvg4c 192.150.187.164 58869 194.127.84.106 443 FjkLnG4s34DVZlaBNc application/pkix-cert 194.127.84.106:443/tcp 2c322ae2b7fe91391345e070b63668978bb1c9da Intel::CERT_HASH X509::IN_CERT bro source1 1170717509.082241 CjhGID4nQcgTWjvg4c 192.150.187.164 58869 194.127.84.106 443 - - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro source1 +1170717511.909717 CCvvfg3TEfuqmmG4bh 192.150.187.164 58870 194.127.84.106 443 FQXAWgI2FB5STbrff application/pkix-cert 194.127.84.106:443/tcp 2c322ae2b7fe91391345e070b63668978bb1c9da Intel::CERT_HASH X509::IN_CERT bro source1 1170717512.108799 CCvvfg3TEfuqmmG4bh 192.150.187.164 58870 194.127.84.106 443 - - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro source1 -#close 2015-03-14-01-47-46 +#close 2016-04-11-13-48-49 diff --git a/testing/btest/scripts/policy/frameworks/intel/seen/certs.bro b/testing/btest/scripts/policy/frameworks/intel/seen/certs.bro index 2ab4c6a50a..859e3a6b9f 100644 --- a/testing/btest/scripts/policy/frameworks/intel/seen/certs.bro +++ b/testing/btest/scripts/policy/frameworks/intel/seen/certs.bro @@ -8,6 +8,7 @@ #fields indicator indicator_type meta.source meta.desc meta.url www.pantz.org Intel::DOMAIN source1 test entry http://some-data-distributor.com/100000 www.dresdner-privat.de Intel::DOMAIN source1 test entry http://some-data-distributor.com/100000 +2c322ae2b7fe91391345e070b63668978bb1c9da Intel::CERT_HASH source1 test entry http://some-data-distributor.com/100000 @TEST-END-FILE @load base/frameworks/intel From f54a5b52e5d62c2394a1aa16e1176b829e54f152 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Tue, 12 Apr 2016 15:40:18 -0500 Subject: [PATCH 41/84] Improve documentation of the "for" statement --- doc/script-reference/statements.rst | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/doc/script-reference/statements.rst b/doc/script-reference/statements.rst index 47e82eb074..14e0cc3c32 100644 --- a/doc/script-reference/statements.rst +++ b/doc/script-reference/statements.rst @@ -315,30 +315,33 @@ Here are the statements that the Bro scripting language supports. .. bro:keyword:: for A "for" loop iterates over each element in a string, set, vector, or - table and executes a statement for each iteration. Currently, - modifying a container's membership while iterating over it may - result in undefined behavior, so avoid adding or removing elements - inside the loop. + table and executes a statement for each iteration (note that the order + in which the loop iterates over the elements in a set or a table is + nondeterministic). However, no loop iterations occur if the string, + set, vector, or table is empty. For each iteration of the loop, a loop variable will be assigned to an element if the expression evaluates to a string or set, or an index if the expression evaluates to a vector or table. Then the statement - is executed. However, the statement will not be executed if the expression - evaluates to an object with no elements. + is executed. If the expression is a table or a set with more than one index, then the loop variable must be specified as a comma-separated list of different loop variables (one for each index), enclosed in brackets. - A :bro:keyword:`break` statement can be used at any time to immediately - terminate the "for" loop, and a :bro:keyword:`next` statement can be - used to skip to the next loop iteration. - Note that the loop variable in a "for" statement is not allowed to be a global variable, and it does not need to be declared prior to the "for" statement. The type will be inferred from the elements of the expression. + Currently, modifying a container's membership while iterating over it may + result in undefined behavior, so do not add or remove elements + inside the loop. + + A :bro:keyword:`break` statement will immediately terminate the "for" + loop, and a :bro:keyword:`next` statement will skip to the next loop + iteration. + Example:: local myset = set(80/tcp, 81/tcp); From adcc978f14faa1c3a2df555da8ad5ff2c15bf6c8 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Wed, 13 Apr 2016 00:44:02 -0400 Subject: [PATCH 42/84] Add a file entropy test. --- .../scripts.base.files.entropy.basic/.stdout | 1 + testing/btest/scripts/base/files/entropy/basic.test | 13 +++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.files.entropy.basic/.stdout create mode 100644 testing/btest/scripts/base/files/entropy/basic.test diff --git a/testing/btest/Baseline/scripts.base.files.entropy.basic/.stdout b/testing/btest/Baseline/scripts.base.files.entropy.basic/.stdout new file mode 100644 index 0000000000..0682a357e8 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.files.entropy.basic/.stdout @@ -0,0 +1 @@ +[entropy=4.950189, chi_square=63750.814665, mean=80.496493, monte_carlo_pi=4.0, serial_correlation=0.395907] diff --git a/testing/btest/scripts/base/files/entropy/basic.test b/testing/btest/scripts/base/files/entropy/basic.test new file mode 100644 index 0000000000..2b867eb8cb --- /dev/null +++ b/testing/btest/scripts/base/files/entropy/basic.test @@ -0,0 +1,13 @@ +# @TEST-EXEC: bro -r $TRACES/http/get.trace %INPUT +# @TEST-EXEC: btest-diff .stdout + + +event file_new(f: fa_file) + { + Files::add_analyzer(f, Files::ANALYZER_ENTROPY); + } + +event file_entropy(f: fa_file, ent: entropy_test_result) + { + print ent; + } \ No newline at end of file From 16c0707b1d804ccfcc671fb9642a0c21ffd7219f Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 13 Apr 2016 14:16:31 -0500 Subject: [PATCH 43/84] Fix RFB analyzer to build on FreeBSD The auto-generated header rfb_pac.h had class member functions "major" and "minor" which were clashing with macros of the same name defined in /usr/include/sys/types.h on FreeBSD. Fixed by renaming the fields. --- src/analyzer/protocol/rfb/rfb-analyzer.pac | 2 +- src/analyzer/protocol/rfb/rfb-protocol.pac | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/analyzer/protocol/rfb/rfb-analyzer.pac b/src/analyzer/protocol/rfb/rfb-analyzer.pac index b63b9f4085..47b87cf5ef 100644 --- a/src/analyzer/protocol/rfb/rfb-analyzer.pac +++ b/src/analyzer/protocol/rfb/rfb-analyzer.pac @@ -81,7 +81,7 @@ refine connection RFB_Conn += { if ( client ) { // Set protocol version on client's version - int minor_version = bytestring_to_int(${msg.minor},10); + int minor_version = bytestring_to_int(${msg.versionminor},10); version = minor_version; // Apple specifies minor version "889" but talks v37 diff --git a/src/analyzer/protocol/rfb/rfb-protocol.pac b/src/analyzer/protocol/rfb/rfb-protocol.pac index 764046e747..8f795c1751 100644 --- a/src/analyzer/protocol/rfb/rfb-protocol.pac +++ b/src/analyzer/protocol/rfb/rfb-protocol.pac @@ -17,13 +17,13 @@ enum states { type RFBProtocolVersion (client: bool) = record { header: "RFB "; - major: bytestring &length=3; + versionmajor: bytestring &length=3; dot: "."; - minor: bytestring &length=3; + versionminor: bytestring &length=3; pad: uint8; } &let { proc: bool = $context.connection.handle_banners(client, this); - proc2: bool = $context.flow.proc_rfb_version(client, major, minor); + proc2: bool = $context.flow.proc_rfb_version(client, versionmajor, versionminor); } type RFBSecurityTypes = record { From 61eea09395e7a915ccb4ad741cc0cf9fbd0f393b Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Wed, 13 Apr 2016 16:33:15 -0400 Subject: [PATCH 44/84] Avoid a macro name conflict on FreeBSD. --- src/analyzer/protocol/rfb/rfb-analyzer.pac | 2 +- src/analyzer/protocol/rfb/rfb-protocol.pac | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/analyzer/protocol/rfb/rfb-analyzer.pac b/src/analyzer/protocol/rfb/rfb-analyzer.pac index b63b9f4085..cd24ea0ced 100644 --- a/src/analyzer/protocol/rfb/rfb-analyzer.pac +++ b/src/analyzer/protocol/rfb/rfb-analyzer.pac @@ -81,7 +81,7 @@ refine connection RFB_Conn += { if ( client ) { // Set protocol version on client's version - int minor_version = bytestring_to_int(${msg.minor},10); + int minor_version = bytestring_to_int(${msg.minor_ver},10); version = minor_version; // Apple specifies minor version "889" but talks v37 diff --git a/src/analyzer/protocol/rfb/rfb-protocol.pac b/src/analyzer/protocol/rfb/rfb-protocol.pac index 764046e747..d80416664b 100644 --- a/src/analyzer/protocol/rfb/rfb-protocol.pac +++ b/src/analyzer/protocol/rfb/rfb-protocol.pac @@ -17,13 +17,13 @@ enum states { type RFBProtocolVersion (client: bool) = record { header: "RFB "; - major: bytestring &length=3; + major_ver: bytestring &length=3; dot: "."; - minor: bytestring &length=3; + minor_ver: bytestring &length=3; pad: uint8; } &let { proc: bool = $context.connection.handle_banners(client, this); - proc2: bool = $context.flow.proc_rfb_version(client, major, minor); + proc2: bool = $context.flow.proc_rfb_version(client, major_ver, minor_ver); } type RFBSecurityTypes = record { From 23d25628ad9473f2a0faecafb1d6eb157a141673 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Wed, 13 Apr 2016 16:55:28 -0400 Subject: [PATCH 45/84] Revert "Fix RFB analyzer to build on FreeBSD" This reverts commit 16c0707b1d804ccfcc671fb9642a0c21ffd7219f. --- src/analyzer/protocol/rfb/rfb-analyzer.pac | 2 +- src/analyzer/protocol/rfb/rfb-protocol.pac | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/analyzer/protocol/rfb/rfb-analyzer.pac b/src/analyzer/protocol/rfb/rfb-analyzer.pac index 47b87cf5ef..b63b9f4085 100644 --- a/src/analyzer/protocol/rfb/rfb-analyzer.pac +++ b/src/analyzer/protocol/rfb/rfb-analyzer.pac @@ -81,7 +81,7 @@ refine connection RFB_Conn += { if ( client ) { // Set protocol version on client's version - int minor_version = bytestring_to_int(${msg.versionminor},10); + int minor_version = bytestring_to_int(${msg.minor},10); version = minor_version; // Apple specifies minor version "889" but talks v37 diff --git a/src/analyzer/protocol/rfb/rfb-protocol.pac b/src/analyzer/protocol/rfb/rfb-protocol.pac index 8f795c1751..764046e747 100644 --- a/src/analyzer/protocol/rfb/rfb-protocol.pac +++ b/src/analyzer/protocol/rfb/rfb-protocol.pac @@ -17,13 +17,13 @@ enum states { type RFBProtocolVersion (client: bool) = record { header: "RFB "; - versionmajor: bytestring &length=3; + major: bytestring &length=3; dot: "."; - versionminor: bytestring &length=3; + minor: bytestring &length=3; pad: uint8; } &let { proc: bool = $context.connection.handle_banners(client, this); - proc2: bool = $context.flow.proc_rfb_version(client, versionmajor, versionminor); + proc2: bool = $context.flow.proc_rfb_version(client, major, minor); } type RFBSecurityTypes = record { From 9aa9618473e47a4adc17a4a966207d54a305da0e Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 14 Apr 2016 10:06:58 -0400 Subject: [PATCH 46/84] Additional mime types for file identification and a few fixes. Some of the existing mime types received extended matchers to fix problems with UTF-16 BOMs. New file mime types: - .ini files - MS Registry policy files - MS Registry files - MS Registry format files (e.g. DESKTOP.DAT) - MS Outlook PST files - Apple AFPInfo files Mime type fixes: - MP3 files with ID3 tags. - JSON and XML matchers were extended --- scripts/base/frameworks/files/magic/audio.sig | 2 +- .../base/frameworks/files/magic/general.sig | 65 +++++++++++++++---- 2 files changed, 54 insertions(+), 13 deletions(-) diff --git a/scripts/base/frameworks/files/magic/audio.sig b/scripts/base/frameworks/files/magic/audio.sig index efba99ed0d..9b4d7da66b 100644 --- a/scripts/base/frameworks/files/magic/audio.sig +++ b/scripts/base/frameworks/files/magic/audio.sig @@ -2,7 +2,7 @@ # MPEG v3 audio signature file-mpeg-audio { file-mime "audio/mpeg", 20 - file-magic /^\xff[\xe2\xe3\xf2\xf3\xf6\xf7\xfa\xfb\xfc\xfd]/ + file-magic /^(ID3|\xff[\xe2\xe3\xf2\xf3\xf6\xf7\xfa\xfb\xfc\xfd])/ } # MPEG v4 audio diff --git a/scripts/base/frameworks/files/magic/general.sig b/scripts/base/frameworks/files/magic/general.sig index 268412ff05..bea6ae9ece 100644 --- a/scripts/base/frameworks/files/magic/general.sig +++ b/scripts/base/frameworks/files/magic/general.sig @@ -9,53 +9,53 @@ signature file-plaintext { signature file-json { file-mime "text/json", 1 - file-magic /^(\xef\xbb\xbf)?[\x0d\x0a[:blank:]]*\{[\x0d\x0a[:blank:]]*(["][^"]{1,}["]|[a-zA-Z][a-zA-Z0-9\\_]*)[\x0d\x0a[:blank:]]*:[\x0d\x0a[:blank:]]*(["]|\[|\{|[0-9]|true|false)/ + file-magic /^(\xef\xbb\xbf|\xff\xfe|\xfe\xff)?[\x0d\x0a[:blank:]]*\{[\x0d\x0a[:blank:]]*(["][^"]{1,}["]|[a-zA-Z][a-zA-Z0-9\\_]*)[\x0d\x0a[:blank:]]*:[\x0d\x0a[:blank:]]*(["]|\[|\{|[0-9]|true|false)/ } signature file-json2 { file-mime "text/json", 1 - file-magic /^(\xef\xbb\xbf)?[\x0d\x0a[:blank:]]*\[[\x0d\x0a[:blank:]]*(((["][^"]{1,}["]|[0-9]{1,}(\.[0-9]{1,})?|true|false)[\x0d\x0a[:blank:]]*,)|\{|\[)[\x0d\x0a[:blank:]]*/ + file-magic /^(\xef\xbb\xbf|\xff\xfe|\xfe\xff)?[\x0d\x0a[:blank:]]*\[[\x0d\x0a[:blank:]]*(((["][^"]{1,}["]|[0-9]{1,}(\.[0-9]{1,})?|true|false)[\x0d\x0a[:blank:]]*,)|\{|\[)[\x0d\x0a[:blank:]]*/ } # Match empty JSON documents. signature file-json3 { file-mime "text/json", 0 - file-magic /^(\xef\xbb\xbf)?[\x0d\x0a[:blank:]]*(\[\]|\{\})[\x0d\x0a[:blank:]]*$/ + file-magic /^(\xef\xbb\xbf|\xff\xfe|\xfe\xff)?[\x0d\x0a[:blank:]]*(\[\]|\{\})[\x0d\x0a[:blank:]]*$/ } signature file-xml { file-mime "application/xml", 10 - file-magic /^(\xef\xbb\xbf)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*<\?xml / + file-magic /^(\xef\xbb\xbf|\xff\xfe|\xfe\xff)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*\x00?<\x00?\?\x00?x\x00?m\x00?l\x00? \x00?/ } signature file-xhtml { file-mime "text/html", 100 - file-magic /^(\xef\xbb\xbf)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*<(![dD][oO][cC][tT][yY][pP][eE] {1,}[hH][tT][mM][lL]|[hH][tT][mM][lL]|[mM][eE][tT][aA] {1,}[hH][tT][tT][pP]-[eE][qQ][uU][iI][vV])/ + file-magic /^(\xef\xbb\xbf|\xff\xfe|\xfe\xff)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*<(![dD][oO][cC][tT][yY][pP][eE] {1,}[hH][tT][mM][lL]|[hH][tT][mM][lL]|[mM][eE][tT][aA] {1,}[hH][tT][tT][pP]-[eE][qQ][uU][iI][vV])/ } signature file-html { file-mime "text/html", 49 - file-magic /^(\xef\xbb\xbf)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*)?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*)?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*<([hH][eE][aA][dD]|[hH][tT][mM][lL]|[tT][iI][tT][lL][eE]|[bB][oO][dD][yY])/ + file-magic /^(\xef\xbb\xbf|\xff\xfe|\xfe\xff)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*<([hH][eE][aA][dD]|[hH][tT][mM][lL]|[tT][iI][tT][lL][eE]|[bB][oO][dD][yY])/ } signature file-rss { file-mime "text/rss", 90 - file-magic /^(\xef\xbb\xbf)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*<[rR][sS][sS]/ + file-magic /^(\xef\xbb\xbf|\xff\xfe|\xfe\xff)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*<[rR][sS][sS]/ } signature file-atom { file-mime "text/atom", 100 - file-magic /^(\xef\xbb\xbf)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*<([rR][sS][sS][^>]*xmlns:atom|[fF][eE][eE][dD][^>]*xmlns=["']?http:\/\/www.w3.org\/2005\/Atom["']?)/ + file-magic /^(\xef\xbb\xbf|\xff\xfe|\xfe\xff)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*<([rR][sS][sS][^>]*xmlns:atom|[fF][eE][eE][dD][^>]*xmlns=["']?http:\/\/www.w3.org\/2005\/Atom["']?)/ } signature file-soap { file-mime "application/soap+xml", 49 - file-magic /^(\xef\xbb\xbf)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*<[sS][oO][aA][pP](-[eE][nN][vV])?:[eE][nN][vV][eE][lL][oO][pP][eE]/ + file-magic /^(\xef\xbb\xbf|\xff\xfe|\xfe\xff)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*<[sS][oO][aA][pP](-[eE][nN][vV])?:[eE][nN][vV][eE][lL][oO][pP][eE]/ } signature file-cross-domain-policy { @@ -70,7 +70,7 @@ signature file-cross-domain-policy2 { signature file-xmlrpc { file-mime "application/xml-rpc", 49 - file-magic /^(\xef\xbb\xbf)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*<[mM][eE][tT][hH][oO][dD][rR][eE][sS][pP][oO][nN][sS][eE]>/ + file-magic /^(\xef\xbb\xbf|\xff\xfe|\xfe\xff)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*<[mM][eE][tT][hH][oO][dD][rR][eE][sS][pP][oO][nN][sS][eE]>/ } signature file-coldfusion { @@ -81,7 +81,13 @@ signature file-coldfusion { # Adobe Flash Media Manifest signature file-f4m { file-mime "application/f4m", 49 - file-magic /^(\xef\xbb\xbf)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*<[mM][aA][nN][iI][fF][eE][sS][tT][\x0d\x0a[:blank:]]{1,}xmlns=\"http:\/\/ns\.adobe\.com\/f4m\// + file-magic /^(\xef\xbb\xbf|\xff\xfe|\xfe\xff)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*<[mM][aA][nN][iI][fF][eE][sS][tT][\x0d\x0a[:blank:]]{1,}xmlns=\"http:\/\/ns\.adobe\.com\/f4m\// +} + +# .ini style files +signature file-ini { + file-mime "text/ini", 20 + file-magic /^(\xef\xbb\xbf|\xff\xfe|\xfe\xff)?[\x00\x0d\x0a[:blank:]]*\[[^\x0d\x0a]+\][[:blank:]\x00]*[\x0d\x0a]/ } # Microsoft LNK files @@ -90,6 +96,41 @@ signature file-lnk { file-magic /^\x4C\x00\x00\x00\x01\x14\x02\x00\x00\x00\x00\x00\xC0\x00\x00\x00\x00\x10\x00\x00\x00\x46/ } +# Microsoft Registry policies +signature file-pol { + file-mime "application/vnd.ms-pol", 49 + file-magic /^PReg/ +} + +# Old style Windows registry file +signature file-reg { + file-mime "application/vnd.ms-reg", 49 + file-magic /^REGEDIT4/ +} + +# Newer Windows registry file +signature file-reg-utf16 { + file-mime "application/vnd.ms-reg", 49 + file-magic /^\xFF\xFEW\x00i\x00n\x00d\x00o\x00w\x00s\x00 \x00R\x00e\x00g\x00i\x00s\x00t\x00r\x00y\x00 \x00E\x00d\x00i\x00t\x00o\x00r\x00 \x00V\x00e\x00r\x00s\x00i\x00o\x00n\x00 \x005\x00\.\x000\x000/ +} + +# Microsoft Registry format (typically DESKTOP.DAT) +signature file-regf { + file-mime "application vnd.ms-regf", 49 + file-magic /^\x72\x65\x67\x66/ +} + +# Microsoft Outlook PST files +signature file-pst { + file-mime "application/vnd.ms-outlook", 49 + file-magic /!BDN......[\x0e\x0f\x15\x17][\x00-\x02]/ +} + +signature file-afpinfo { + file-mime "application/vnd.apple-afpinfo" + file-magic /^AFP/ +} + signature file-jar { file-mime "application/java-archive", 100 file-magic /^PK\x03\x04.{1,200}\x14\x00..META-INF\/MANIFEST\.MF/ From 2fc8ef232a82e3aa8ea40f7619fa34b799fcaad8 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Thu, 14 Apr 2016 10:08:26 -0400 Subject: [PATCH 47/84] Updating CHANGES and VERSION. --- CHANGES | 18 ++++++++++++++++++ VERSION | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 890368dc53..38e25cd07d 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,22 @@ +2.4-454 | 2016-04-14 10:06:58 -0400 + + * Additional mime types for file identification and a few fixes. (Seth Hall) + + New file mime types: + - .ini files + - MS Registry policy files + - MS Registry files + - MS Registry format files (e.g. DESKTOP.DAT) + - MS Outlook PST files + - Apple AFPInfo files + + Mime type fixes: + - MP3 files with ID3 tags. + - JSON and XML matchers were extended + + * Avoid a macro name conflict on FreeBSD. (Seth Hall, Daniel Thayer) + 2.4-452 | 2016-04-13 01:15:20 -0400 * Add a simple file entropy analyzer. (Seth Hall) diff --git a/VERSION b/VERSION index 030385b8f8..532e871b23 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4-452 +2.4-454 From c0bf1b3c6793e3769c7e1988c2c2d677945f9acc Mon Sep 17 00:00:00 2001 From: Jan Grashoefer Date: Wed, 20 Apr 2016 00:00:47 +0200 Subject: [PATCH 48/84] Added get_current_packet_header bif. --- scripts/base/init-bare.bro | 130 ++++++++++++++++++------------------- src/IP.cc | 1 + src/NetVar.cc | 4 ++ src/NetVar.h | 2 + src/bro.bif | 20 ++++++ src/iosource/Packet.cc | 9 --- 6 files changed, 92 insertions(+), 74 deletions(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 9c60d76746..a2cb3e4c5e 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -793,71 +793,6 @@ type entropy_test_result: record { serial_correlation: double; ##< Serial correlation coefficient. }; -# Prototypes of Bro built-in functions. -@load base/bif/strings.bif -@load base/bif/bro.bif -@load base/bif/reporter.bif - -## Deprecated. This is superseded by the new logging framework. -global log_file_name: function(tag: string): string &redef; - -## Deprecated. This is superseded by the new logging framework. -global open_log_file: function(tag: string): file &redef; - -## Specifies a directory for Bro to store its persistent state. All globals can -## be declared persistent via the :bro:attr:`&persistent` attribute. -const state_dir = ".state" &redef; - -## Length of the delays inserted when storing state incrementally. To avoid -## dropping packets when serializing larger volumes of persistent state to -## disk, Bro interleaves the operation with continued packet processing. -const state_write_delay = 0.01 secs &redef; - -global done_with_network = F; -event net_done(t: time) { done_with_network = T; } - -function log_file_name(tag: string): string - { - local suffix = getenv("BRO_LOG_SUFFIX") == "" ? "log" : getenv("BRO_LOG_SUFFIX"); - return fmt("%s.%s", tag, suffix); - } - -function open_log_file(tag: string): file - { - return open(log_file_name(tag)); - } - -## Internal function. -function add_interface(iold: string, inew: string): string - { - if ( iold == "" ) - return inew; - else - return fmt("%s %s", iold, inew); - } - -## Network interfaces to listen on. Use ``redef interfaces += "eth0"`` to -## extend. -global interfaces = "" &add_func = add_interface; - -## Internal function. -function add_signature_file(sold: string, snew: string): string - { - if ( sold == "" ) - return snew; - else - return cat(sold, " ", snew); - } - -## Signature files to read. Use ``redef signature_files += "foo.sig"`` to -## extend. Signature files added this way will be searched relative to -## ``BROPATH``. Using the ``@load-sigs`` directive instead is preferred -## since that can search paths relative to the current script. -global signature_files = "" &add_func = add_signature_file; - -## ``p0f`` fingerprint file to use. Will be searched relative to ``BROPATH``. -const passive_fingerprint_file = "base/misc/p0f.fp" &redef; - # TCP values for :bro:see:`endpoint` *state* field. # todo:: these should go into an enum to make them autodoc'able. const TCP_INACTIVE = 0; ##< Endpoint is still inactive. @@ -1768,6 +1703,71 @@ type gtp_delete_pdp_ctx_response_elements: record { ext: gtp_private_extension &optional; }; +# Prototypes of Bro built-in functions. +@load base/bif/strings.bif +@load base/bif/bro.bif +@load base/bif/reporter.bif + +## Deprecated. This is superseded by the new logging framework. +global log_file_name: function(tag: string): string &redef; + +## Deprecated. This is superseded by the new logging framework. +global open_log_file: function(tag: string): file &redef; + +## Specifies a directory for Bro to store its persistent state. All globals can +## be declared persistent via the :bro:attr:`&persistent` attribute. +const state_dir = ".state" &redef; + +## Length of the delays inserted when storing state incrementally. To avoid +## dropping packets when serializing larger volumes of persistent state to +## disk, Bro interleaves the operation with continued packet processing. +const state_write_delay = 0.01 secs &redef; + +global done_with_network = F; +event net_done(t: time) { done_with_network = T; } + +function log_file_name(tag: string): string + { + local suffix = getenv("BRO_LOG_SUFFIX") == "" ? "log" : getenv("BRO_LOG_SUFFIX"); + return fmt("%s.%s", tag, suffix); + } + +function open_log_file(tag: string): file + { + return open(log_file_name(tag)); + } + +## Internal function. +function add_interface(iold: string, inew: string): string + { + if ( iold == "" ) + return inew; + else + return fmt("%s %s", iold, inew); + } + +## Network interfaces to listen on. Use ``redef interfaces += "eth0"`` to +## extend. +global interfaces = "" &add_func = add_interface; + +## Internal function. +function add_signature_file(sold: string, snew: string): string + { + if ( sold == "" ) + return snew; + else + return cat(sold, " ", snew); + } + +## Signature files to read. Use ``redef signature_files += "foo.sig"`` to +## extend. Signature files added this way will be searched relative to +## ``BROPATH``. Using the ``@load-sigs`` directive instead is preferred +## since that can search paths relative to the current script. +global signature_files = "" &add_func = add_signature_file; + +## ``p0f`` fingerprint file to use. Will be searched relative to ``BROPATH``. +const passive_fingerprint_file = "base/misc/p0f.fp" &redef; + ## Definition of "secondary filters". A secondary filter is a BPF filter given ## as index in this table. For each such filter, the corresponding event is ## raised for all matching packets. diff --git a/src/IP.cc b/src/IP.cc index 3a19f02d23..a36a3cf6fb 100644 --- a/src/IP.cc +++ b/src/IP.cc @@ -393,6 +393,7 @@ RecordVal* IP_Hdr::BuildPktHdrVal(RecordVal* pkt_hdr, int sindex) const } case IPPROTO_ICMP: + case IPPROTO_ICMPV6: { const struct icmp* icmpp = (const struct icmp *) data; RecordVal* icmp_hdr = new RecordVal(icmp_hdr_type); diff --git a/src/NetVar.cc b/src/NetVar.cc index 8a901842fd..ccc94c97a6 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -15,6 +15,8 @@ RecordType* icmp_conn; RecordType* icmp_context; RecordType* SYN_packet; RecordType* pcap_packet; +RecordType* raw_pkt_hdr_type; +RecordType* l2_hdr_type; RecordType* signature_state; EnumType* transport_proto; TableType* string_set; @@ -324,6 +326,8 @@ void init_net_var() signature_state = internal_type("signature_state")->AsRecordType(); SYN_packet = internal_type("SYN_packet")->AsRecordType(); pcap_packet = internal_type("pcap_packet")->AsRecordType(); + raw_pkt_hdr_type = internal_type("raw_pkt_hdr")->AsRecordType(); + l2_hdr_type = internal_type("l2_hdr")->AsRecordType(); transport_proto = internal_type("transport_proto")->AsEnumType(); string_set = internal_type("string_set")->AsTableType(); string_array = internal_type("string_array")->AsTableType(); diff --git a/src/NetVar.h b/src/NetVar.h index 97018121f9..909a2a4c1c 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -19,6 +19,8 @@ extern RecordType* icmp_context; extern RecordType* signature_state; extern RecordType* SYN_packet; extern RecordType* pcap_packet; +extern RecordType* raw_pkt_hdr_type; +extern RecordType* l2_hdr_type; extern EnumType* transport_proto; extern TableType* string_set; extern TableType* string_array; diff --git a/src/bro.bif b/src/bro.bif index 2c55c2bc95..fb8db5e23a 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -3458,6 +3458,26 @@ function get_current_packet%(%) : pcap_packet return pkt; %} +## Function to get the raw headers of the currently processed packet. +## +## Returns: A record containing the Layer 2, 3 and 4 headers of the +## currently processed packet. +## +## .. bro:see:: raw_pkt_hdr get_current_packet +function get_current_packet_header%(%) : raw_pkt_hdr + %{ + const Packet* p; + + if ( current_pktsrc && + current_pktsrc->GetCurrentPacket(&p) ) + { + return p->BuildPktHdrVal(); + } + + RecordVal* hdr = new RecordVal(raw_pkt_hdr_type); + return hdr; + %} + ## Writes a given packet to a file. ## ## pkt: The PCAP packet. diff --git a/src/iosource/Packet.cc b/src/iosource/Packet.cc index 2aa7fa58c7..c75b62a832 100644 --- a/src/iosource/Packet.cc +++ b/src/iosource/Packet.cc @@ -428,15 +428,6 @@ void Packet::ProcessLayer2() RecordVal* Packet::BuildPktHdrVal() const { - static RecordType* l2_hdr_type = 0; - static RecordType* raw_pkt_hdr_type = 0; - - if ( ! raw_pkt_hdr_type ) - { - raw_pkt_hdr_type = internal_type("raw_pkt_hdr")->AsRecordType(); - l2_hdr_type = internal_type("l2_hdr")->AsRecordType(); - } - RecordVal* pkt_hdr = new RecordVal(raw_pkt_hdr_type); RecordVal* l2_hdr = new RecordVal(l2_hdr_type); From 8ac92cf7ff0dfb7fa20e1ad78fcf9f7eecf1c189 Mon Sep 17 00:00:00 2001 From: Jan Grashoefer Date: Wed, 20 Apr 2016 00:05:33 +0200 Subject: [PATCH 49/84] Added test case for get_current_packet_header bif. --- .../btest/Baseline/bifs.get_current_packet_header/output | 1 + testing/btest/bifs/get_current_packet_header.bro | 8 ++++++++ 2 files changed, 9 insertions(+) create mode 100644 testing/btest/Baseline/bifs.get_current_packet_header/output create mode 100644 testing/btest/bifs/get_current_packet_header.bro diff --git a/testing/btest/Baseline/bifs.get_current_packet_header/output b/testing/btest/Baseline/bifs.get_current_packet_header/output new file mode 100644 index 0000000000..761a248077 --- /dev/null +++ b/testing/btest/Baseline/bifs.get_current_packet_header/output @@ -0,0 +1 @@ +[l2=[encap=LINK_ETHERNET, len=78, cap_len=78, src=00:00:00:00:00:00, dst=ff:ff:ff:ff:ff:ff, vlan=, inner_vlan=, eth_type=34525, proto=L3_IPV6], ip=, ip6=[class=0, flow=0, len=24, nxt=58, hlim=255, src=fe80::dead, dst=fe80::beef, exts=[]], tcp=, udp=, icmp=[icmp_type=135]] diff --git a/testing/btest/bifs/get_current_packet_header.bro b/testing/btest/bifs/get_current_packet_header.bro new file mode 100644 index 0000000000..24144545ef --- /dev/null +++ b/testing/btest/bifs/get_current_packet_header.bro @@ -0,0 +1,8 @@ +# @TEST-EXEC: bro -C -r $TRACES/icmp/icmp6-neighbor-solicit.pcap %INPUT > output +# @TEST-EXEC: btest-diff output + +event icmp_neighbor_solicitation(c: connection, icmp: icmp_conn, tgt: addr, options: icmp6_nd_options) + { + local hdr: raw_pkt_hdr = get_current_packet_header(); + print fmt("%s", hdr); + } \ No newline at end of file From 3665f745adaf8d254daa30525ddbd73b91749e68 Mon Sep 17 00:00:00 2001 From: Jan Grashoefer Date: Wed, 20 Apr 2016 00:23:11 +0200 Subject: [PATCH 50/84] Updated affected test case. Fixing IP_Hdr::BuildPktHdrVal to generate an icmp_hdr record for ICMPv6 packets slightly changed the output of core/ipv6_zero_len_ah.test. --- testing/btest/Baseline/core.ipv6_zero_len_ah/output | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/btest/Baseline/core.ipv6_zero_len_ah/output b/testing/btest/Baseline/core.ipv6_zero_len_ah/output index d8db6a4c48..585011acbe 100644 --- a/testing/btest/Baseline/core.ipv6_zero_len_ah/output +++ b/testing/btest/Baseline/core.ipv6_zero_len_ah/output @@ -1,2 +1,2 @@ [orig_h=2000:1300::1, orig_p=128/icmp, resp_h=2000:1300::2, resp_p=129/icmp] -[ip=, ip6=[class=0, flow=0, len=166, nxt=51, hlim=255, src=2000:1300::1, dst=2000:1300::2, exts=[[id=51, hopopts=, dstopts=, routing=, fragment=, ah=[nxt=58, len=0, rsv=0, spi=0, seq=, data=], esp=, mobility=]]], tcp=, udp=, icmp=] +[ip=, ip6=[class=0, flow=0, len=166, nxt=51, hlim=255, src=2000:1300::1, dst=2000:1300::2, exts=[[id=51, hopopts=, dstopts=, routing=, fragment=, ah=[nxt=58, len=0, rsv=0, spi=0, seq=, data=], esp=, mobility=]]], tcp=, udp=, icmp=[icmp_type=128]] From cdd687979ef29913225153a7526184a5b61aff4c Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Fri, 22 Apr 2016 10:36:02 -0700 Subject: [PATCH 51/84] Update submodule [nomail] --- cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake b/cmake index 537e45afe1..0a2b36874a 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 537e45afe1006a10f73847fab5f13d28ce43fc4d +Subproject commit 0a2b36874ad5c1a22829135f8aeeac534469053f From 59bf2f8a1e6b50b43db6a4ff8aae849829685d33 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Fri, 22 Apr 2016 15:03:29 -0400 Subject: [PATCH 52/84] DNS TTL responses are to be unsigned. --- src/analyzer/protocol/dns/DNS.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/analyzer/protocol/dns/DNS.h b/src/analyzer/protocol/dns/DNS.h index 59f51812ca..54b5d291cc 100644 --- a/src/analyzer/protocol/dns/DNS.h +++ b/src/analyzer/protocol/dns/DNS.h @@ -132,7 +132,7 @@ public: StringVal* query_name; RR_Type atype; int aclass; ///< normally = 1, inet - int ttl; + uint32 ttl; DNS_AnswerType answer_type; int skip_event; ///< if true, don't generate corresponding events From a14de582a235d7ef7b6b9e761fdb6c3d699d494c Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Fri, 22 Apr 2016 15:26:34 -0400 Subject: [PATCH 53/84] Add DNS "CAA" RR type and event. --- scripts/base/protocols/dns/consts.bro | 1 + src/analyzer/protocol/dns/DNS.cc | 47 +++++++++++++++++++++++++++ src/analyzer/protocol/dns/DNS.h | 4 +++ src/analyzer/protocol/dns/events.bif | 6 ++++ 4 files changed, 58 insertions(+) diff --git a/scripts/base/protocols/dns/consts.bro b/scripts/base/protocols/dns/consts.bro index 13af6c3e81..026588f777 100644 --- a/scripts/base/protocols/dns/consts.bro +++ b/scripts/base/protocols/dns/consts.bro @@ -26,6 +26,7 @@ export { [49] = "DHCID", [99] = "SPF", [100] = "DINFO", [101] = "UID", [102] = "GID", [103] = "UNSPEC", [249] = "TKEY", [250] = "TSIG", [251] = "IXFR", [252] = "AXFR", [253] = "MAILB", [254] = "MAILA", + [257] = "CAA", [32768] = "TA", [32769] = "DLV", [ANY] = "*", } &default = function(n: count): string { return fmt("query-%d", n); }; diff --git a/src/analyzer/protocol/dns/DNS.cc b/src/analyzer/protocol/dns/DNS.cc index b449589e6c..ff7ccc83a5 100644 --- a/src/analyzer/protocol/dns/DNS.cc +++ b/src/analyzer/protocol/dns/DNS.cc @@ -282,6 +282,10 @@ int DNS_Interpreter::ParseAnswer(DNS_MsgInfo* msg, status = ParseRR_TXT(msg, data, len, rdlength, msg_start); break; + case TYPE_CAA: + status = ParseRR_CAA(msg, data, len, rdlength, msg_start); + break; + case TYPE_NBS: status = ParseRR_NBS(msg, data, len, rdlength, msg_start); break; @@ -904,6 +908,49 @@ int DNS_Interpreter::ParseRR_TXT(DNS_MsgInfo* msg, return rdlength == 0; } +int DNS_Interpreter::ParseRR_CAA(DNS_MsgInfo* msg, + const u_char*& data, int& len, int rdlength, + const u_char* msg_start) + { + if ( ! dns_CAA_reply || msg->skip_event ) + { + data += rdlength; + len -= rdlength; + return 1; + } + + unsigned int flags = ExtractShort(data, len); + unsigned int tagLen = flags & 0xff; + flags = flags >> 8; + if ( tagLen >= (unsigned int) rdlength - 2 ) + { + analyzer->Weird("DNS_CAA_char_str_past_rdlen"); + return 0; + } + BroString* tag = new BroString(data, tagLen, 0); + len -= tagLen; + data += tagLen; + BroString* value = new BroString(data, rdlength-2-tagLen, 0); + + val_list* vl = new val_list; + + vl->append(analyzer->BuildConnVal()); + vl->append(msg->BuildHdrVal()); + vl->append(msg->BuildAnswerVal()); + vl->append(new Val(flags, TYPE_COUNT)); + vl->append(new StringVal(tag)); + vl->append(new StringVal(value)); + + analyzer->ConnectionEvent(dns_CAA_reply, vl); + + len -= value->Len(); + data += value->Len(); + rdlength -= 2 + tagLen + value->Len(); + + return rdlength == 0; + } + + void DNS_Interpreter::SendReplyOrRejectEvent(DNS_MsgInfo* msg, EventHandlerPtr event, const u_char*& data, int& len, diff --git a/src/analyzer/protocol/dns/DNS.h b/src/analyzer/protocol/dns/DNS.h index 59f51812ca..c081f5172a 100644 --- a/src/analyzer/protocol/dns/DNS.h +++ b/src/analyzer/protocol/dns/DNS.h @@ -56,6 +56,7 @@ typedef enum { TYPE_EDNS = 41, ///< OPT pseudo-RR (RFC 2671) TYPE_TKEY = 249, ///< Transaction Key (RFC 2930) TYPE_TSIG = 250, ///< Transaction Signature (RFC 2845) + TYPE_CAA = 257, ///< Certification Authority Authorization (RFC 6844) // The following are only valid in queries. TYPE_AXFR = 252, @@ -211,6 +212,9 @@ protected: int ParseRR_TXT(DNS_MsgInfo* msg, const u_char*& data, int& len, int rdlength, const u_char* msg_start); + int ParseRR_CAA(DNS_MsgInfo* msg, + const u_char*& data, int& len, int rdlength, + const u_char* msg_start); int ParseRR_TSIG(DNS_MsgInfo* msg, const u_char*& data, int& len, int rdlength, const u_char* msg_start); diff --git a/src/analyzer/protocol/dns/events.bif b/src/analyzer/protocol/dns/events.bif index 9350939a2e..a9a924bf13 100644 --- a/src/analyzer/protocol/dns/events.bif +++ b/src/analyzer/protocol/dns/events.bif @@ -378,6 +378,12 @@ event dns_MX_reply%(c: connection, msg: dns_msg, ans: dns_answer, name: string, ## dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth event dns_TXT_reply%(c: connection, msg: dns_msg, ans: dns_answer, strs: string_vec%); + +## https://tools.ietf.org/html/rfc6844 +## Certification Authority Authorization +event dns_CAA_reply%(c: connection, msg: dns_msg, ans: dns_answer, flags: count, tag: string, value: string%); + + ## Generated for DNS replies of type *SRV*. For replies with multiple answers, ## an individual event of the corresponding type is raised for each. ## From 6dddd35d218583014938c2ee732cb6a1dfdee0f2 Mon Sep 17 00:00:00 2001 From: Jeannette Dopheide Date: Mon, 25 Apr 2016 11:49:04 -0500 Subject: [PATCH 54/84] Correcting spelling errors found under bro 2.4.1+dfsg-2 here: https://lintian.debian.org/full/bengen@debian.org.html#bro_2.4.1_x2bdfsg-2 --- src/RuleCondition.cc | 2 +- src/RuleMatcher.cc | 2 +- src/Serializer.cc | 2 +- src/StateAccess.cc | 2 +- src/broxygen/Configuration.cc | 2 +- src/nb_dns.c | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/RuleCondition.cc b/src/RuleCondition.cc index 68eb13121f..40ef5f0ad1 100644 --- a/src/RuleCondition.cc +++ b/src/RuleCondition.cc @@ -111,7 +111,7 @@ bool RuleConditionPayloadSize::DoMatch(Rule* rule, RuleEndpointState* state, return payload_size >= val; default: - reporter->InternalError("unknown comparision type"); + reporter->InternalError("unknown comparison type"); } // Should not be reached diff --git a/src/RuleMatcher.cc b/src/RuleMatcher.cc index f40a5c4349..f5b5b82517 100644 --- a/src/RuleMatcher.cc +++ b/src/RuleMatcher.cc @@ -21,7 +21,7 @@ // it may fail to match. Work-around: Insert an always // matching "payload" pattern (not done in snort2bro yet) // - tcp-state always evaluates to true -// (implemented but deactivated for comparision to Snort) +// (implemented but deactivated for comparison to Snort) uint32 RuleHdrTest::idcounter = 0; diff --git a/src/Serializer.cc b/src/Serializer.cc index 49e57c0216..5c1ae6077c 100644 --- a/src/Serializer.cc +++ b/src/Serializer.cc @@ -437,7 +437,7 @@ bool Serializer::UnserializeCall(UnserialInfo* info) bool Serializer::UnserializeStateAccess(UnserialInfo* info) { - SetErrorDescr("unserializing state acess"); + SetErrorDescr("unserializing state access"); StateAccess* s = StateAccess::Unserialize(info); diff --git a/src/StateAccess.cc b/src/StateAccess.cc index aa4a1f36d2..6e73c8cf61 100644 --- a/src/StateAccess.cc +++ b/src/StateAccess.cc @@ -150,7 +150,7 @@ bool StateAccess::CheckOld(const char* op, ID* id, Val* index, if ( should && is ) { - // There's no general comparision for non-atomic vals currently. + // There's no general comparison for non-atomic vals currently. if ( ! (is_atomic_val(is) && is_atomic_val(should)) ) return true; diff --git a/src/broxygen/Configuration.cc b/src/broxygen/Configuration.cc index 264e8e6fcb..4780e6ad99 100644 --- a/src/broxygen/Configuration.cc +++ b/src/broxygen/Configuration.cc @@ -65,7 +65,7 @@ Config::Config(const string& arg_file, const string& delim) Target* target = target_factory.Create(tokens[0], tokens[2], tokens[1]); if ( ! target ) - reporter->FatalError("unkown Broxygen target type: %s", + reporter->FatalError("unknown Broxygen target type: %s", tokens[0].c_str()); targets.push_back(target); diff --git a/src/nb_dns.c b/src/nb_dns.c index 1e5d427924..35059ab4f0 100644 --- a/src/nb_dns.c +++ b/src/nb_dns.c @@ -389,7 +389,7 @@ nb_dns_addr_request2(register struct nb_dns_info *nd, char *addrp, default: snprintf(errstr, NB_DNS_ERRSIZE, - "nb_dns_addr_request2(): uknown address family %d", af); + "nb_dns_addr_request2(): unknown address family %d", af); return (-1); } From a705b2c08dbd8f14dd54163d978ddbf2e9561d94 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Mon, 25 Apr 2016 15:37:15 -0700 Subject: [PATCH 55/84] Add DNS tests for huge TLL and CAA --- CHANGES | 8 ++++++++ NEWS | 9 ++++++--- VERSION | 2 +- .../scripts.base.protocols.dns.caa/.stdout | 1 + .../scripts.base.protocols.dns.huge-ttl/.stdout | 8 ++++++++ testing/btest/Traces/dns-caa.pcap | Bin 0 -> 227 bytes testing/btest/Traces/dns-huge-ttl.pcap | Bin 0 -> 993 bytes testing/btest/scripts/base/protocols/dns/caa.bro | 7 +++++++ .../btest/scripts/base/protocols/dns/huge-ttl.bro | 7 +++++++ 9 files changed, 38 insertions(+), 4 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.protocols.dns.caa/.stdout create mode 100644 testing/btest/Baseline/scripts.base.protocols.dns.huge-ttl/.stdout create mode 100644 testing/btest/Traces/dns-caa.pcap create mode 100644 testing/btest/Traces/dns-huge-ttl.pcap create mode 100644 testing/btest/scripts/base/protocols/dns/caa.bro create mode 100644 testing/btest/scripts/base/protocols/dns/huge-ttl.bro diff --git a/CHANGES b/CHANGES index 1ecbf765e3..af063f122d 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,12 @@ +2.4-471 | 2016-04-25 15:37:15 -0700 + + * Add DNS tests for huge TLLs and CAA. (Johanna Amann) + + * Add DNS "CAA" RR type and event. (Mark Taylor) + + * Fix DNS response parsing: TTLs are unsigned. (Mark Taylor) + 2.4-466 | 2016-04-22 16:25:33 -0700 * Rename BrokerStore and BrokerComm to Broker. Also split broker main.bro diff --git a/NEWS b/NEWS index 2858023439..4f1a84b7b6 100644 --- a/NEWS +++ b/NEWS @@ -36,6 +36,9 @@ New Functionality - Bro now tracks VLAN IDs. To record them inside the connection log, load protocols/conn/vlan-logging.bro. +- A new dns_CAA_reply event gives access to DNS Certification Authority + Authorization replies. + - A new per-packet event raw_packet() provides access to layer 2 information. Use with care, generating events per packet is expensive. @@ -45,8 +48,8 @@ New Functionality argument that will be used for decoding errors into weird.log (instead of reporter.log). -- A new get_current_packet_header bif returning the headers of the current - packet +- A new get_current_packet_header bif returns the headers of the current + packet. - Two new built-in functions for handling set[subnet] and table[subnet]: @@ -87,7 +90,7 @@ New Functionality Changed Functionality --------------------- -- The BrokerComm and BrokerStore namespaces were renamed to Broker +- The BrokerComm and BrokerStore namespaces were renamed to Broker. - ``SSH::skip_processing_after_detection`` was removed. The functionality was replaced by ``SSH::disable_analyzer_after_detection``. diff --git a/VERSION b/VERSION index 4d856a68a0..33a6cae723 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4-466 +2.4-471 diff --git a/testing/btest/Baseline/scripts.base.protocols.dns.caa/.stdout b/testing/btest/Baseline/scripts.base.protocols.dns.caa/.stdout new file mode 100644 index 0000000000..4ba72f24b4 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.dns.caa/.stdout @@ -0,0 +1 @@ +0, issue, symantec.com diff --git a/testing/btest/Baseline/scripts.base.protocols.dns.huge-ttl/.stdout b/testing/btest/Baseline/scripts.base.protocols.dns.huge-ttl/.stdout new file mode 100644 index 0000000000..99f7325c23 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.dns.huge-ttl/.stdout @@ -0,0 +1,8 @@ +[answer_type=1, query=us.v27.distributed.net, qtype=1, qclass=1, TTL=49710.0 days 6.0 hrs 28.0 mins 15.0 secs] +[answer_type=1, query=us.v27.distributed.net, qtype=1, qclass=1, TTL=15.0 mins] +[answer_type=1, query=us.v27.distributed.net, qtype=1, qclass=1, TTL=15.0 mins] +[answer_type=1, query=us.v27.distributed.net, qtype=1, qclass=1, TTL=15.0 mins] +[answer_type=1, query=us.v27.distributed.net, qtype=1, qclass=1, TTL=15.0 mins] +[answer_type=1, query=us.v27.distributed.net, qtype=1, qclass=1, TTL=15.0 mins] +[answer_type=1, query=us.v27.distributed.net, qtype=1, qclass=1, TTL=15.0 mins] +[answer_type=1, query=us.v27.distributed.net, qtype=1, qclass=1, TTL=15.0 mins] diff --git a/testing/btest/Traces/dns-caa.pcap b/testing/btest/Traces/dns-caa.pcap new file mode 100644 index 0000000000000000000000000000000000000000..7409c0347b598e0906a463e2a2d3e16893f10007 GIT binary patch literal 227 zcmca|c+)~A1{MYw`2U}Qff2~znk5&W@|=so4af%J36blg)2>t)9z14$h=akEfx$v~ zJ_Cb;V5!Q16@peAK=8Dg!IVMe?PJq8pni}MK){xspP!zS%AA~^%fQIUzz8xmSDu?8 z6(|S75c42r15H(62;Y*&z+fu)UKC^^$Ycgnpt;VIDoo=V8-S*Q%|$o$01wD)28Iw% Y24M!)%;MtG)Z)tA#JrN!WIdqG0Q(U*+W-In literal 0 HcmV?d00001 diff --git a/testing/btest/Traces/dns-huge-ttl.pcap b/testing/btest/Traces/dns-huge-ttl.pcap new file mode 100644 index 0000000000000000000000000000000000000000..27849b904b047d24923e3fa480b8c75e72f22f05 GIT binary patch literal 993 zcmca|c+)~A1{MY+z{m*X{9|shEZ)t+u#=eqj2Q&(6#2w6T-YwRPl1EMm4SKnjn51W zMuLK&=jR>J;1Xe!XE0?jWuAWYZc1uP0|O%i2Ln3;J5y;fbD5DjcS>e)Nl|7}X-R4d zb6#o*15ln3pWJ^SU|>0y>#z%-Dh3AT76z6ZjAsK0t2#S%@nyoQ+Dx0V@!q~GZOXif&9aglv-TE zoSdJ_fZrn^#YUtkHbzp6E!;rqlYzm)fq~^>k^ Date: Mon, 25 Apr 2016 16:54:47 -0700 Subject: [PATCH 56/84] Intel: Allow to provide uid/fuid instead of conn/f. This patch allows users to provide the fuid or the connection id directly, in case they do not have access to either in the event that they handle. An example for this is the handling of certificates in SSL, where the fa_file record cannot be retained because this would create a cyclic data structure. This patch also provides file IDs for hostname matches in certificates, which was not possible with the previous API. --- scripts/base/frameworks/intel/main.bro | 33 +++++++++++++++---- scripts/policy/frameworks/intel/seen/ssl.bro | 1 + .../intel-all.log | 14 ++++---- 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro index eba27ca56a..d334210db6 100644 --- a/scripts/base/frameworks/intel/main.bro +++ b/scripts/base/frameworks/intel/main.bro @@ -77,23 +77,35 @@ export { ## The type of data that the indicator represents. indicator_type: Type &log &optional; - ## If the indicator type was :bro:enum:`Intel::ADDR`, then this + ## If the indicator type was :bro:enum:`Intel::ADDR`, then this ## field will be present. host: addr &optional; ## Where the data was discovered. where: Where &log; - + ## The name of the node where the match was discovered. node: string &optional &log; - ## If the data was discovered within a connection, the + ## If the data was discovered within a connection, the ## connection record should go here to give context to the data. conn: connection &optional; + ## If the data was discovered within a connection, the + ## connection uid should go here to give context to the data. + ## If the *conn* field is provided, this will be automatically + ## filled out. + uid: string &optional; + + ## If the data was discovered within a file, the file record ## should go here to provide context to the data. f: fa_file &optional; + + ## If the data was discovered within a file, the file uid should + ## go here to provide context to the data. If the *f* field is + ## provided, this will be automatically filled out. + fuid: string &optional; }; ## Record used for the logging framework representing a positive @@ -112,7 +124,8 @@ export { ## If a file was associated with this intelligence hit, ## this is the uid for the file. fuid: string &log &optional; - ## A mime type if the intelligence hit is related to a file. + + ## A mime type if the intelligence hit is related to a file. ## If the $f field is provided this will be automatically filled ## out. file_mime_type: string &log &optional; @@ -283,14 +296,14 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5 if ( s?$f ) { + s$fuid = s$f$id; + if ( s$f?$conns && |s$f$conns| == 1 ) { for ( cid in s$f$conns ) s$conn = s$f$conns[cid]; } - if ( ! info?$fuid ) - info$fuid = s$f$id; if ( ! info?$file_mime_type && s$f?$info && s$f$info?$mime_type ) info$file_mime_type = s$f$info$mime_type; @@ -299,12 +312,18 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5 info$file_desc = Files::describe(s$f); } + if ( s?$fuid ) + info$fuid = s$fuid; + if ( s?$conn ) { - info$uid = s$conn$uid; + s$uid = s$conn$uid; info$id = s$conn$id; } + if ( s?$uid ) + info$uid = s$uid; + for ( item in items ) add info$sources[item$meta$source]; diff --git a/scripts/policy/frameworks/intel/seen/ssl.bro b/scripts/policy/frameworks/intel/seen/ssl.bro index 7bfbef4e9b..89aebc1891 100644 --- a/scripts/policy/frameworks/intel/seen/ssl.bro +++ b/scripts/policy/frameworks/intel/seen/ssl.bro @@ -20,6 +20,7 @@ event ssl_established(c: connection) if ( c$ssl$cert_chain[0]$x509?$certificate && c$ssl$cert_chain[0]$x509$certificate?$cn ) Intel::seen([$indicator=c$ssl$cert_chain[0]$x509$certificate$cn, $indicator_type=Intel::DOMAIN, + $fuid=c$ssl$cert_chain_fuids[0], $conn=c, $where=X509::IN_CERT]); } diff --git a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log index 4b5786e00d..0cac337cf3 100644 --- a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log +++ b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log @@ -3,23 +3,23 @@ #empty_field (empty) #unset_field - #path intel -#open 2016-04-11-13-48-49 +#open 2016-04-25-23-53-37 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node sources #types time string addr port addr port string string string string enum enum string set[string] 1416942644.593119 CXWv6p3arKYeMETxOg 192.168.4.149 49422 23.92.19.75 443 F0txuw2pvrkZOn04a8 application/pkix-cert 23.92.19.75:443/tcp www.pantz.org Intel::DOMAIN X509::IN_CERT bro source1 -#close 2016-04-11-13-48-49 +#close 2016-04-25-23-53-37 #separator \x09 #set_separator , #empty_field (empty) #unset_field - #path intel -#open 2016-04-11-13-48-49 +#open 2016-04-25-23-53-38 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node sources #types time string addr port addr port string string string string enum enum string set[string] 1170717505.735416 CXWv6p3arKYeMETxOg 192.150.187.164 58868 194.127.84.106 443 FeCwNK3rzqPnZ7eBQ5 application/pkix-cert 194.127.84.106:443/tcp 2c322ae2b7fe91391345e070b63668978bb1c9da Intel::CERT_HASH X509::IN_CERT bro source1 -1170717505.934612 CXWv6p3arKYeMETxOg 192.150.187.164 58868 194.127.84.106 443 - - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro source1 +1170717505.934612 CXWv6p3arKYeMETxOg 192.150.187.164 58868 194.127.84.106 443 FeCwNK3rzqPnZ7eBQ5 - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro source1 1170717508.883051 CjhGID4nQcgTWjvg4c 192.150.187.164 58869 194.127.84.106 443 FjkLnG4s34DVZlaBNc application/pkix-cert 194.127.84.106:443/tcp 2c322ae2b7fe91391345e070b63668978bb1c9da Intel::CERT_HASH X509::IN_CERT bro source1 -1170717509.082241 CjhGID4nQcgTWjvg4c 192.150.187.164 58869 194.127.84.106 443 - - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro source1 +1170717509.082241 CjhGID4nQcgTWjvg4c 192.150.187.164 58869 194.127.84.106 443 FjkLnG4s34DVZlaBNc - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro source1 1170717511.909717 CCvvfg3TEfuqmmG4bh 192.150.187.164 58870 194.127.84.106 443 FQXAWgI2FB5STbrff application/pkix-cert 194.127.84.106:443/tcp 2c322ae2b7fe91391345e070b63668978bb1c9da Intel::CERT_HASH X509::IN_CERT bro source1 -1170717512.108799 CCvvfg3TEfuqmmG4bh 192.150.187.164 58870 194.127.84.106 443 - - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro source1 -#close 2016-04-11-13-48-49 +1170717512.108799 CCvvfg3TEfuqmmG4bh 192.150.187.164 58870 194.127.84.106 443 FQXAWgI2FB5STbrff - - www.dresdner-privat.de Intel::DOMAIN X509::IN_CERT bro source1 +#close 2016-04-25-23-53-38 From d93186881dfaf4647019acc17dbe93dff041c33b Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Tue, 26 Apr 2016 08:32:55 -0700 Subject: [PATCH 57/84] Fix small error in bif documentation. --- src/bro.bif | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bro.bif b/src/bro.bif index 6360d326a1..5d097734a4 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -3460,7 +3460,7 @@ function get_current_packet%(%) : pcap_packet ## Function to get the raw headers of the currently processed packet. ## -## Returns: The :bro:type:`connection` record containing the Layer 2, 3 and +## Returns: The :bro:type:`raw_pkt_hdr` record containing the Layer 2, 3 and ## 4 headers of the currently processed packet. ## ## .. bro:see:: raw_pkt_hdr get_current_packet From 25f8993b57138b03dd2d0d665bbf5ba9b37b0f7f Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Tue, 26 Apr 2016 11:15:41 -0700 Subject: [PATCH 58/84] IMAP: documentation and test updates --- src/analyzer/protocol/imap/IMAP.cc | 1 - src/analyzer/protocol/imap/Plugin.cc | 6 +---- src/analyzer/protocol/imap/events.bif | 9 +++---- .../Baseline/core.print-bpf-filters/output2 | 9 ++++--- .../canonified_loaded_scripts.log | 5 ++-- .../canonified_loaded_scripts.log | 7 ++++-- testing/btest/Baseline/plugins.hooks/output | 25 ++++++++++++++----- 7 files changed, 37 insertions(+), 25 deletions(-) diff --git a/src/analyzer/protocol/imap/IMAP.cc b/src/analyzer/protocol/imap/IMAP.cc index ad38d598ac..ea09a66717 100644 --- a/src/analyzer/protocol/imap/IMAP.cc +++ b/src/analyzer/protocol/imap/IMAP.cc @@ -77,7 +77,6 @@ void IMAP_Analyzer::StartTLS() // StartTLS was called. This means we saw a client starttls followed // by a server proceed. From here on, everything should be a binary // TLS datastream. - tls_active = true; Analyzer* ssl = analyzer_mgr->InstantiateAnalyzer("SSL", Conn()); diff --git a/src/analyzer/protocol/imap/Plugin.cc b/src/analyzer/protocol/imap/Plugin.cc index 8660879bc3..63358f1aeb 100644 --- a/src/analyzer/protocol/imap/Plugin.cc +++ b/src/analyzer/protocol/imap/Plugin.cc @@ -1,8 +1,5 @@ // See the file in the main distribution directory for copyright. - - #include "plugin/Plugin.h" - #include "IMAP.h" namespace plugin { @@ -14,10 +11,9 @@ public: { AddComponent(new ::analyzer::Component("IMAP", ::analyzer::imap::IMAP_Analyzer::Instantiate)); - plugin::Configuration config; config.name = "Bro::IMAP"; - config.description = "IMAP analyzer StartTLS only"; + config.description = "IMAP analyzer (StartTLS only)"; return config; } } plugin; diff --git a/src/analyzer/protocol/imap/events.bif b/src/analyzer/protocol/imap/events.bif index ba83791b13..8d70dda26f 100644 --- a/src/analyzer/protocol/imap/events.bif +++ b/src/analyzer/protocol/imap/events.bif @@ -1,14 +1,13 @@ -## Generated for an SSL/TLS client's initial *hello* message. SSL/TLS sessions -## start with an unencrypted handshake, and Bro extracts as much information out -## of that as it can. This event provides access to the initial information -## sent by the client. +## Generated when a server sends a capability list to the client, +## after being queried using the CAPABILITY command. ## ## c: The connection. ## ## capabilities: The list of IMAP capabilities as sent by the server. event imap_capabilities%(c: connection, capabilities: string_vec%); -## Generated when a IMAP connection goes encrypted +## Generated when a IMAP connection goes encrypted after a successful +## StartTLS exchange between the client and the server. ## ## c: The connection. event imap_starttls%(c: connection%); diff --git a/testing/btest/Baseline/core.print-bpf-filters/output2 b/testing/btest/Baseline/core.print-bpf-filters/output2 index ac140925fc..d0f448441b 100644 --- a/testing/btest/Baseline/core.print-bpf-filters/output2 +++ b/testing/btest/Baseline/core.print-bpf-filters/output2 @@ -1,5 +1,6 @@ 2 1080 1 137 +1 143 1 1434 1 161 1 162 @@ -47,8 +48,8 @@ 1 992 1 993 1 995 -54 and -53 or -54 port -36 tcp +55 and +54 or +55 port +37 tcp 18 udp diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index b5107374d1..0427e043e1 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2016-04-22-23-21-01 +#open 2016-04-26-18-11-39 #fields name #types string scripts/base/init-bare.bro @@ -76,6 +76,7 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_HTTP.functions.bif.bro build/scripts/base/bif/plugins/Bro_ICMP.events.bif.bro build/scripts/base/bif/plugins/Bro_Ident.events.bif.bro + build/scripts/base/bif/plugins/Bro_IMAP.events.bif.bro build/scripts/base/bif/plugins/Bro_InterConn.events.bif.bro build/scripts/base/bif/plugins/Bro_IRC.events.bif.bro build/scripts/base/bif/plugins/Bro_KRB.events.bif.bro @@ -131,4 +132,4 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro scripts/policy/misc/loaded-scripts.bro scripts/base/utils/paths.bro -#close 2016-04-22-23-21-01 +#close 2016-04-26-18-11-39 diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index c2db0ad12e..806f1c6b9b 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2016-04-22-23-21-18 +#open 2016-04-26-18-11-49 #fields name #types string scripts/base/init-bare.bro @@ -76,6 +76,7 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_HTTP.functions.bif.bro build/scripts/base/bif/plugins/Bro_ICMP.events.bif.bro build/scripts/base/bif/plugins/Bro_Ident.events.bif.bro + build/scripts/base/bif/plugins/Bro_IMAP.events.bif.bro build/scripts/base/bif/plugins/Bro_InterConn.events.bif.bro build/scripts/base/bif/plugins/Bro_IRC.events.bif.bro build/scripts/base/bif/plugins/Bro_KRB.events.bif.bro @@ -252,6 +253,8 @@ scripts/base/init-default.bro scripts/base/protocols/http/entities.bro scripts/base/protocols/http/utils.bro scripts/base/protocols/http/files.bro + scripts/base/protocols/imap/__load__.bro + scripts/base/protocols/imap/main.bro scripts/base/protocols/irc/__load__.bro scripts/base/protocols/irc/main.bro scripts/base/protocols/irc/dcc-send.bro @@ -302,4 +305,4 @@ scripts/base/init-default.bro scripts/base/misc/find-checksum-offloading.bro scripts/base/misc/find-filtered-trace.bro scripts/policy/misc/loaded-scripts.bro -#close 2016-04-22-23-21-18 +#close 2016-04-26-18-11-49 diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index d4bd063e12..a30a37bf95 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -25,6 +25,7 @@ 0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_HTTP, 8080/tcp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_HTTP, 81/tcp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_HTTP, 8888/tcp)) -> +0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_IMAP, 143/tcp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_IRC, 6666/tcp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_IRC, 6667/tcp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_IRC, 6668/tcp)) -> @@ -83,6 +84,7 @@ 0.000000 MetaHookPost CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_HTTP, 8080/tcp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_HTTP, 81/tcp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_HTTP, 8888/tcp)) -> +0.000000 MetaHookPost CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_IMAP, 143/tcp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_IRC, 6666/tcp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_IRC, 6667/tcp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_IRC, 6668/tcp)) -> @@ -122,6 +124,7 @@ 0.000000 MetaHookPost CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_FTP, {2811<...>/tcp})) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_GTPV1, {2152<...>/udp})) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_HTTP, {631<...>/tcp})) -> +0.000000 MetaHookPost CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_IMAP, {143/tcp})) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_IRC, {6669<...>/tcp})) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_KRB, {88/udp})) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_KRB_TCP, {88/tcp})) -> @@ -230,7 +233,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461367323.154279, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461694342.200388, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Communication::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Conn::LOG)) -> @@ -351,7 +354,7 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461367323.154279, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461694342.200388, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(NetControl::check_plugins, , ()) -> 0.000000 MetaHookPost CallFunction(NetControl::init, , ()) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, , ()) -> @@ -416,6 +419,7 @@ 0.000000 MetaHookPost LoadFile(./Bro_HTTP.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_HTTP.functions.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_ICMP.events.bif.bro) -> -1 +0.000000 MetaHookPost LoadFile(./Bro_IMAP.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_IRC.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_Ident.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_InterConn.events.bif.bro) -> -1 @@ -587,6 +591,7 @@ 0.000000 MetaHookPost LoadFile(base<...>/ftp) -> -1 0.000000 MetaHookPost LoadFile(base<...>/hash) -> -1 0.000000 MetaHookPost LoadFile(base<...>/http) -> -1 +0.000000 MetaHookPost LoadFile(base<...>/imap) -> -1 0.000000 MetaHookPost LoadFile(base<...>/input) -> -1 0.000000 MetaHookPost LoadFile(base<...>/input.bif) -> -1 0.000000 MetaHookPost LoadFile(base<...>/intel) -> -1 @@ -665,6 +670,7 @@ 0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_HTTP, 8080/tcp)) 0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_HTTP, 81/tcp)) 0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_HTTP, 8888/tcp)) +0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_IMAP, 143/tcp)) 0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_IRC, 6666/tcp)) 0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_IRC, 6667/tcp)) 0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_IRC, 6668/tcp)) @@ -723,6 +729,7 @@ 0.000000 MetaHookPre CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_HTTP, 8080/tcp)) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_HTTP, 81/tcp)) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_HTTP, 8888/tcp)) +0.000000 MetaHookPre CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_IMAP, 143/tcp)) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_IRC, 6666/tcp)) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_IRC, 6667/tcp)) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_IRC, 6668/tcp)) @@ -762,6 +769,7 @@ 0.000000 MetaHookPre CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_FTP, {2811<...>/tcp})) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_GTPV1, {2152<...>/udp})) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_HTTP, {631<...>/tcp})) +0.000000 MetaHookPre CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_IMAP, {143/tcp})) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_IRC, {6669<...>/tcp})) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_KRB, {88/udp})) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_KRB_TCP, {88/tcp})) @@ -870,7 +878,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461367323.154279, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461694342.200388, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Communication::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Conn::LOG)) @@ -991,7 +999,7 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461367323.154279, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461694342.200388, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(NetControl::check_plugins, , ()) 0.000000 MetaHookPre CallFunction(NetControl::init, , ()) 0.000000 MetaHookPre CallFunction(Notice::want_pp, , ()) @@ -1056,6 +1064,7 @@ 0.000000 MetaHookPre LoadFile(./Bro_HTTP.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_HTTP.functions.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_ICMP.events.bif.bro) +0.000000 MetaHookPre LoadFile(./Bro_IMAP.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_IRC.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_Ident.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_InterConn.events.bif.bro) @@ -1227,6 +1236,7 @@ 0.000000 MetaHookPre LoadFile(base<...>/ftp) 0.000000 MetaHookPre LoadFile(base<...>/hash) 0.000000 MetaHookPre LoadFile(base<...>/http) +0.000000 MetaHookPre LoadFile(base<...>/imap) 0.000000 MetaHookPre LoadFile(base<...>/input) 0.000000 MetaHookPre LoadFile(base<...>/input.bif) 0.000000 MetaHookPre LoadFile(base<...>/intel) @@ -1305,6 +1315,7 @@ 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_HTTP, 8080/tcp) 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_HTTP, 81/tcp) 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_HTTP, 8888/tcp) +0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_IMAP, 143/tcp) 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_IRC, 6666/tcp) 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_IRC, 6667/tcp) 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_IRC, 6668/tcp) @@ -1363,6 +1374,7 @@ 0.000000 | HookCallFunction Analyzer::register_for_port(Analyzer::ANALYZER_HTTP, 8080/tcp) 0.000000 | HookCallFunction Analyzer::register_for_port(Analyzer::ANALYZER_HTTP, 81/tcp) 0.000000 | HookCallFunction Analyzer::register_for_port(Analyzer::ANALYZER_HTTP, 8888/tcp) +0.000000 | HookCallFunction Analyzer::register_for_port(Analyzer::ANALYZER_IMAP, 143/tcp) 0.000000 | HookCallFunction Analyzer::register_for_port(Analyzer::ANALYZER_IRC, 6666/tcp) 0.000000 | HookCallFunction Analyzer::register_for_port(Analyzer::ANALYZER_IRC, 6667/tcp) 0.000000 | HookCallFunction Analyzer::register_for_port(Analyzer::ANALYZER_IRC, 6668/tcp) @@ -1402,6 +1414,7 @@ 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_FTP, {2811<...>/tcp}) 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_GTPV1, {2152<...>/udp}) 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_HTTP, {631<...>/tcp}) +0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_IMAP, {143/tcp}) 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_IRC, {6669<...>/tcp}) 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_KRB, {88/udp}) 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_KRB_TCP, {88/tcp}) @@ -1509,7 +1522,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1461367323.154279, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1461694342.200388, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG) @@ -1630,7 +1643,7 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1461367323.154279, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1461694342.200388, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction NetControl::check_plugins() 0.000000 | HookCallFunction NetControl::init() 0.000000 | HookCallFunction Notice::want_pp() From e9a87566ef9d5e1e88e54d9cfea97d4fef19a46f Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Tue, 26 Apr 2016 12:30:28 -0700 Subject: [PATCH 59/84] Fix parsing of x509 pre-y2k dates There was a bug in the new parsing code, introduced in 708ede22c6781e854739c67332ac18a391f4782f which parses validity times incorrectly if they are before the year 2000. What happens in this case is that the 2-digit year will be interpreted to be in the 21st century (1999 will be parsed as 2099, e.g.). --- src/file_analysis/analyzer/x509/X509.cc | 2 +- .../scripts.base.files.x509.1999/x509.log | 12 ++++++++++++ testing/btest/Traces/tls/telesec.pcap | Bin 0 -> 7636 bytes testing/btest/scripts/base/files/x509/1999.test | 5 +++++ 4 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 testing/btest/Baseline/scripts.base.files.x509.1999/x509.log create mode 100644 testing/btest/Traces/tls/telesec.pcap create mode 100644 testing/btest/scripts/base/files/x509/1999.test diff --git a/src/file_analysis/analyzer/x509/X509.cc b/src/file_analysis/analyzer/x509/X509.cc index e8ea5cb7b4..ebf7b1d04f 100644 --- a/src/file_analysis/analyzer/x509/X509.cc +++ b/src/file_analysis/analyzer/x509/X509.cc @@ -543,7 +543,7 @@ double file_analysis::X509::GetTimeFromAsn1(const ASN1_TIME* atime, const char* } // year is first two digits in YY format. Buffer expects YYYY format. - if ( pString[0] - '0' < 50 ) // RFC 2459 4.1.2.5.1 + if ( pString[0] < '5' ) // RFC 2459 4.1.2.5.1 { *(pBuffer++) = '2'; *(pBuffer++) = '0'; diff --git a/testing/btest/Baseline/scripts.base.files.x509.1999/x509.log b/testing/btest/Baseline/scripts.base.files.x509.1999/x509.log new file mode 100644 index 0000000000..60bd109b5d --- /dev/null +++ b/testing/btest/Baseline/scripts.base.files.x509.1999/x509.log @@ -0,0 +1,12 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path x509 +#open 2016-04-26-19-27-59 +#fields ts id certificate.version certificate.serial certificate.subject certificate.issuer certificate.not_valid_before certificate.not_valid_after certificate.key_alg certificate.sig_alg certificate.key_type certificate.key_length certificate.exponent certificate.curve san.dns san.uri san.email san.ip basic_constraints.ca basic_constraints.path_len +#types time string count string string string time time string string string count string string vector[string] vector[string] vector[string] vector[addr] bool count +1461697070.246986 Feyr3x4h8S7yqikqYd 3 339D9ED8E73927C9 CN=imap.gmx.net,emailAddress=server-certs@1und1.de,L=Montabaur,ST=Rhineland-Palatinate,O=1&1 Mail & Media GmbH,C=DE CN=TeleSec ServerPass DE-1,street=Untere Industriestr. 20,L=Netphen,postalCode=57250,ST=NRW,OU=T-Systems Trust Center,O=T-Systems International GmbH,C=DE 1384251451.000000 1479427199.000000 rsaEncryption sha1WithRSAEncryption rsa 2048 65537 - imap.gmx.net,imap.gmx.de - - - F - +1461697070.246986 FdSwvBrmfL9It607b 3 21B6777E8CBD0EA8 CN=TeleSec ServerPass DE-1,street=Untere Industriestr. 20,L=Netphen,postalCode=57250,ST=NRW,OU=T-Systems Trust Center,O=T-Systems International GmbH,C=DE CN=Deutsche Telekom Root CA 2,OU=T-TeleSec Trust Center,O=Deutsche Telekom AG,C=DE 1362146309.000000 1562716740.000000 rsaEncryption sha1WithRSAEncryption rsa 2048 65537 - - - - - T 0 +1461697070.246986 F7YtKFoAux1T0Ycb3 3 26 CN=Deutsche Telekom Root CA 2,OU=T-TeleSec Trust Center,O=Deutsche Telekom AG,C=DE CN=Deutsche Telekom Root CA 2,OU=T-TeleSec Trust Center,O=Deutsche Telekom AG,C=DE 931522260.000000 1562716740.000000 rsaEncryption sha1WithRSAEncryption rsa 2048 65537 - - - - - T 5 +#close 2016-04-26-19-27-59 diff --git a/testing/btest/Traces/tls/telesec.pcap b/testing/btest/Traces/tls/telesec.pcap new file mode 100644 index 0000000000000000000000000000000000000000..0f27b68d594910f6bedc6749be233d09e7a87b03 GIT binary patch literal 7636 zcmds6cRbX8{C|J$&Yiu^N;Z+K&z-%pLxnOj$~f+?tRVmM|a}@0gQb{MgTzIfkv&Y zJ5b>Tj&L6vKslppQ_2cT-6gYsIDriS=J;*qz?_(w$=XMBZhj$DJ7R`^*7fw(0Cw{c z+{g%55CAv=$8iLY=RgonZsUZE4D1>KH;O**AAT?w9%Ew^chm+O-2@r{8pIMM3>mY4 ze>P$k*rMqOEew_jFO!qZ8AT`HIJuou%fDgh&=MCm??Q-d2~n}}%e-~IXJGTrDw{Iz zL5S{@Bq%7lzXnEtJ>N`23>ibbTF(!b!?0juDk2iR#uh{5tNtybgrKPMm&u4?cm&Zx z0HTJ+)2-HRo1=GAX!Pzp*w5eS0eH3PGOHlP9uK*3J118fJ|U}CL-6|iJcSQ;!<)(jRuivV^36QBk1 zfdwG3gjggNFAE1a0D&dU;sj!V3-b-h?5W{2Ia zjBUp>i{uu>58nTj6B%)!baB8*_u2i8#^qNiRMXHkxVbv_r+aQ{91|(NBg|q-oAsUphW5lUlSk&5OsB#76BXZ43vrFseLl6!QfF@s|@St2o zyc>Z?##!4?W}u4L0XJF7Rl_AdhRFzzk+Y9rGNL1B%#g?kT9}+|xW64m7L~?s;~|Uv zyv;Q_Hiju@#lY^S@S@z<4V+}Wqq94OALYf4gvcDa`cz$%B8c*1$0Rbzk->}#W<*fr zP#NqD7g^j5d!7M}(ji=Q2m^j;%2Cm=JMd(2S4J4ag%Kd_R( zA|@Kco}xwoBbpgwPKk}6Q#9!eiYQ9Pp2U-RL&9lMnnB?!G$R;HGzC$Zm>z@!;8PF* z0MB#dQV;|*nlbtA8@s=Zs*_HLCGG4wY`{qclu|x9d`pcrvTc^KTsMY*6`>sQe*F7>LuRHTHTV$zE z!`-e$F#=~)-;uj3^b$T?d+9IB7%j)CA1G1XuuGs;LyD=schlL^Z- zh7kZ0t~FW)QLs|A0-|CxgPGxBC?}B^l*|kH3+7iI1D7SFAQJFE1hpx#DzQj2`+*e@ z9j5scU?OaEnAVS8@cr2fzF044!iUi0t0l>eCCLqG$#r3L8a6F#^9QVe2-6=}GpOiK zGw}QMkO0^|&Wz=ea0@vl zEF^|$=1+_H%VpD9FfN8;aYhgrDK&UT1=SuqhazbzKjIY`5c4M|u}jhD{PBlyY39EV zGGu;6!(W*{kj?xulbK(;e@si1Ur%QKiD?UnX8t*EjA%4@_k;;1Z~3p}&cf`ZCRHv_Zwf7%_?!bRSnp?@gvL9NYp-Wdt2uJt@0aVH8VTZN7A?d=iW z>-W7|WKrII`pp+iovx$$XNNy>KfXooo5#OxfgxYxosB1FAX!S)i?7`k@Oz*v9Hy9_ zey$Uhc(!92<*msEX|5L4qwh@=oo46yl;ggdUohpOk!?r~fLIJgx&Te#So=e$j`@-Pum?d zpT10TlOQQ|Lyo+&Jn!((;&w5jbX&1E-;i6}Mu7cu72drFQendZrFZS&mtk8P6g z${nrp>-zhJdISoLGDx^1-g*9BQsJfrik<>$OJAo>_c(nmZg4lxJ?$%7wuNse4BN$P zW{|7+7jbf00ny^lEwLsZ+eY-ZL+m2U`|h4tHE@`}uI;_4mDJ ziQ$z#Ek*f9DUV`_!ks4&JKY&PLUiI7y_>+Jug z;3oHC_GAgkU1)L#YE)uSqL-#KDg@X&f6(8VKZwoz(vz9ruy~C5d8;Qgzt&&%+uGIF z*~!V(*UG||Ix5jvM^?2jb?gM8QD>g;#j(X#9oJXnoI`O~b2 zC(?8@7jcKR@U(TjT%MX-p&p4la#dGHGs*nWEx}!2#-6D6rro{G{b#76N}N?Z`K-nARPzw!|AyEZvega#e562in<{qy2m1SOnskt-Ot*c5sz|j+>R}VX4#P-Y@41H_ zzou>4YX3QsUMA@mxM#n)!ny~|dplR;@dqPSH4aORZr(k;&(i#9Q(0|_XK3#9N~xV~ zoN6;v20|2VKeYF#uPo>z<8H0dS@lnm3Gh7F*T&mNHBqZv&~~r>$=WSf_wnwa@^jiP zS{ypm_RPZrvGyj!-N+Tbc` z%{x%w_O|M&1ZPQE+IstUE*x3Y=C?fYh%dMkDrI@_VoOC+ut0vtGurm$w97fG?#%|z za~7mOE98Dzwe61dC8O2X&bHxoS~f36`a(&k#7;)cO&rxEOVoUV{`^*61laOI7}FMx zU-S#>Qub;OWd2IHK7);^t39})%wUZ9MZ6|6znLwY`6Ym{PugVW|GCV={+-!<@X|Js5`TK?S=f5RT0hgroBZ;5oc77Z(mt$?Al`}Zo5Wu zGtW0!iYJ9|^rrYH^+ta^x91A2nQq6o+Z~_}-jmX6wO--nrD){7t#2jjbf`FV{yeqK zCHNP!+xIu$Nw{-t-}7jbs-lr3&yE6{<6n%A+%TNBo!a?Yo|fG~k>9LS-LIuPH&w8$ zYrVewdC!3MkEt0Hmu6Ry)TsE+n#4GjSS{Jp>j#D}*0$<(tbNnPICUYw>(iXW$y@ZB zEjHd$yeHGt7d+iMpwz3jLf3P5XB0;PzoPq&E!^R?h5<9vaz9C|YaP1X#IcC(4i~5p z2VQ{K-`d+J@6KdrSORb6j?PkPESo&=@yU&ILI!&|NL zf|iwBm@$vCzbaSM;OKiUkIPGyr5!jb*S+l+-akN4c|7;+x2+Oi+*0~`Ube4Ie68wG z->VbwkI>%IV6(f<`Au8tIPY202 z6wX#z+tuKsaWb)~!)v>@Jh`P>toYp9yY%U6DC&!55|w-hW}gihD)sCw(u&HL)%@79 zA;{RmAnH@>$@Eekv-6KB&MGazwC|6&F68?9Rbb1B1b(z^9`jwtPiAx2*R717KVu9Aa%uhDLp zh{$*GZ*vUp#}Jite?_#YhI4$g1^x*!y6*mSj+d_C0Sh{ejbFSzI9Lkq$U13xskV%G^dh2Bz?xl;$$=*3V*wwXWOJn*1 zl`iVbCUug&P*QJb*t2x|`|UZ84US}CF*oxjf>`JZHl{KV2+9n>7)UI8G6PvuLk6l$ zf(*n@A7`L&825z}CV|BV`;=0zG_|`vAzknVsuvZ$JI6V8xD<1r*)z+X^V&J??>^Wh zHp5B$xs<}GSWsmLrv$Y!u*ISSrfE3Wb3CYDVNk;{#Eu3Yzb$dxLwARe z8!I}VqduEj+kEU@p3oic;6i&%xohuOee_$^|0yjfTnl~CvF2E^gS*s2Tl@D4%q1;) zAx|HZ4{w!iyq$GuhS^Yg+lbjrn|4+xzt5+$J2N6?3HGN26$iFCoN@nlKdq0%h`eH@ zQ+HdHx953mRX$~*r~*3NCP5)aXXWp#akn$KFW1Xl+pwnXsJ>uNHNL)=r${wAs^r;` zEZ39kx^K%Xk0{NynYWs{%E9Q};I60X`jmiwd^cF38BHdUG6mAkD#nFa(B&@j{3VS| z3RoJ?bN!k|D-ulOm#kxBY24@q)0nRX5f85+02&NDHh$?gmxA3G;)kWbMTFf}cWn%D zvnz%u3Y$anM{~H$?>*4f{P4k&)%+w|5u+Mno%xu-KUX5_C}!|_{1k)N0$ZR9LV{pZ z)?|Yi;$9xoxC>J&5x8V7Wj9#q_YAcI;b0PsHofQnTB literal 0 HcmV?d00001 diff --git a/testing/btest/scripts/base/files/x509/1999.test b/testing/btest/scripts/base/files/x509/1999.test new file mode 100644 index 0000000000..7c1ab7971f --- /dev/null +++ b/testing/btest/scripts/base/files/x509/1999.test @@ -0,0 +1,5 @@ +# Test that the timestamp of a pre-y-2000 certificate is correctly parsed + +# @TEST-EXEC: bro -r $TRACES/tls/telesec.pcap +# @TEST-EXEC: btest-diff x509.log + From 59573bad33a68a363b9fefed306445d1edd3c750 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Tue, 26 Apr 2016 13:06:48 -0700 Subject: [PATCH 60/84] IMAP: add c++11 header file that gcc complains about. --- src/analyzer/protocol/imap/IMAP.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/analyzer/protocol/imap/IMAP.h b/src/analyzer/protocol/imap/IMAP.h index a1f59e5010..e71770d360 100644 --- a/src/analyzer/protocol/imap/IMAP.h +++ b/src/analyzer/protocol/imap/IMAP.h @@ -3,6 +3,8 @@ #ifndef ANALYZER_PROTOCOL_IMAP_IMAP_H #define ANALYZER_PROTOCOL_IMAP_IMAP_H +// for std::transform +#include #include "analyzer/protocol/tcp/TCP.h" #include "imap_pac.h" From f44bb4d9b887237cf7e417c47bd21ddf6a1c1d80 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Tue, 26 Apr 2016 16:24:10 -0500 Subject: [PATCH 61/84] Add script wrapper functions for broker BIFs Also renamed the "print" function to "send_print" and the "event" function to "send_event" because Bro shows a syntax error when a Bro script function is named "event" or "print". --- scripts/base/frameworks/broker/main.bro | 317 ++++++++++++++++ scripts/base/frameworks/broker/store.bro | 344 ++++++++++++++++++ .../frameworks/netcontrol/plugins/acld.bro | 4 +- .../frameworks/netcontrol/plugins/broker.bro | 4 +- .../frameworks/openflow/plugins/broker.bro | 4 +- src/broker/comm.bif | 74 +--- src/broker/messaging.bif | 137 +------ src/broker/store.bif | 202 +--------- .../canonified_loaded_scripts.log | 10 +- .../canonified_loaded_scripts.log | 10 +- testing/btest/Baseline/plugins.hooks/output | 18 +- testing/btest/broker/remote_event.test | 6 +- testing/btest/broker/remote_print.test | 6 +- .../base/frameworks/netcontrol/acld-hook.bro | 4 +- .../base/frameworks/netcontrol/acld.bro | 4 +- .../base/frameworks/netcontrol/broker.bro | 6 +- .../base/frameworks/openflow/broker-basic.bro | 4 +- 17 files changed, 747 insertions(+), 407 deletions(-) diff --git a/scripts/base/frameworks/broker/main.bro b/scripts/base/frameworks/broker/main.bro index d8b4a208a2..a0024055a7 100644 --- a/scripts/base/frameworks/broker/main.bro +++ b/scripts/base/frameworks/broker/main.bro @@ -1,5 +1,14 @@ ##! Various data structure definitions for use with Bro's communication system. +module Log; + +export { + type Log::ID: enum { + ## Dummy place-holder. + UNKNOWN + }; +} + module Broker; export { @@ -52,4 +61,312 @@ export { key: Broker::Data; val: Broker::Data; }; + + ## Enable use of communication. + ## + ## flags: used to tune the local Broker endpoint behavior. + ## + ## Returns: true if communication is successfully initialized. + global enable: function(flags: EndpointFlags &default = EndpointFlags()): bool; + + ## Changes endpoint flags originally supplied to :bro:see:`Broker::enable`. + ## + ## flags: the new endpoint behavior flags to use. + ## + ## Returns: true if flags were changed. + global set_endpoint_flags: function(flags: EndpointFlags &default = EndpointFlags()): bool; + + ## Allow sending messages to peers if associated with the given topic. + ## This has no effect if auto publication behavior is enabled via the flags + ## supplied to :bro:see:`Broker::enable` or :bro:see:`Broker::set_endpoint_flags`. + ## + ## topic: a topic to allow messages to be published under. + ## + ## Returns: true if successful. + global publish_topic: function(topic: string): bool; + + ## Disallow sending messages to peers if associated with the given topic. + ## This has no effect if auto publication behavior is enabled via the flags + ## supplied to :bro:see:`Broker::enable` or :bro:see:`Broker::set_endpoint_flags`. + ## + ## topic: a topic to disallow messages to be published under. + ## + ## Returns: true if successful. + global unpublish_topic: function(topic: string): bool; + + ## Listen for remote connections. + ## + ## p: the TCP port to listen on. + ## + ## a: an address string on which to accept connections, e.g. + ## "127.0.0.1". An empty string refers to @p INADDR_ANY. + ## + ## reuse: equivalent to behavior of SO_REUSEADDR. + ## + ## Returns: true if the local endpoint is now listening for connections. + ## + ## .. bro:see:: Broker::incoming_connection_established + global listen: function(p: port, a: string &default = "", reuse: bool &default = T): bool; + + ## Initiate a remote connection. + ## + ## a: an address to connect to, e.g. "localhost" or "127.0.0.1". + ## + ## p: the TCP port on which the remote side is listening. + ## + ## retry: an interval at which to retry establishing the + ## connection with the remote peer if it cannot be made initially, or + ## if it ever becomes disconnected. + ## + ## Returns: true if it's possible to try connecting with the peer and + ## it's a new peer. The actual connection may not be established + ## until a later point in time. + ## + ## .. bro:see:: Broker::outgoing_connection_established + global connect: function(a: string, p: port, retry: interval): bool; + + ## Remove a remote connection. + ## + ## a: the address used in previous successful call to :bro:see:`Broker::connect`. + ## + ## p: the port used in previous successful call to :bro:see:`Broker::connect`. + ## + ## Returns: true if the arguments match a previously successful call to + ## :bro:see:`Broker::connect`. + global disconnect: function(a: string, p: port): bool; + + ## Print a simple message to any interested peers. The receiver can use + ## :bro:see:`Broker::print_handler` to handle messages. + ## + ## topic: a topic associated with the printed message. + ## + ## msg: the print message to send to peers. + ## + ## flags: tune the behavior of how the message is sent. + ## + ## Returns: true if the message is sent. + global send_print: function(topic: string, msg: string, flags: SendFlags &default = SendFlags()): bool; + + ## Register interest in all peer print messages that use a certain topic + ## prefix. Use :bro:see:`Broker::print_handler` to handle received + ## messages. + ## + ## topic_prefix: a prefix to match against remote message topics. + ## e.g. an empty prefix matches everything and "a" matches + ## "alice" and "amy" but not "bob". + ## + ## Returns: true if it's a new print subscription and it is now registered. + global subscribe_to_prints: function(topic_prefix: string): bool; + + ## Unregister interest in all peer print messages that use a topic prefix. + ## + ## topic_prefix: a prefix previously supplied to a successful call to + ## :bro:see:`Broker::subscribe_to_prints`. + ## + ## Returns: true if interest in the topic prefix is no longer advertised. + global unsubscribe_to_prints: function(topic_prefix: string): bool; + + ## Send an event to any interested peers. + ## + ## topic: a topic associated with the event message. + ## + ## args: event arguments as made by :bro:see:`Broker::event_args`. + ## + ## flags: tune the behavior of how the message is sent. + ## + ## Returns: true if the message is sent. + global send_event: function(topic: string, args: EventArgs, flags: SendFlags &default = SendFlags()): bool; + + ## Automatically send an event to any interested peers whenever it is + ## locally dispatched (e.g. using "event my_event(...);" in a script). + ## + ## topic: a topic string associated with the event message. + ## Peers advertise interest by registering a subscription to some + ## prefix of this topic name. + ## + ## ev: a Bro event value. + ## + ## flags: tune the behavior of how the message is sent. + ## + ## Returns: true if automatic event sending is now enabled. + global auto_event: function(topic: string, ev: any, flags: SendFlags &default = SendFlags()): bool; + + ## Stop automatically sending an event to peers upon local dispatch. + ## + ## topic: a topic originally given to :bro:see:`Broker::auto_event`. + ## + ## ev: an event originally given to :bro:see:`Broker::auto_event`. + ## + ## Returns: true if automatic events will not occur for the topic/event + ## pair. + global auto_event_stop: function(topic: string, ev: any): bool; + + ## Register interest in all peer event messages that use a certain topic + ## prefix. + ## + ## topic_prefix: a prefix to match against remote message topics. + ## e.g. an empty prefix matches everything and "a" matches + ## "alice" and "amy" but not "bob". + ## + ## Returns: true if it's a new event subscription and it is now registered. + global subscribe_to_events: function(topic_prefix: string): bool; + + ## Unregister interest in all peer event messages that use a topic prefix. + ## + ## topic_prefix: a prefix previously supplied to a successful call to + ## :bro:see:`Broker::subscribe_to_events`. + ## + ## Returns: true if interest in the topic prefix is no longer advertised. + global unsubscribe_to_events: function(topic_prefix: string): bool; + + ## Enable remote logs for a given log stream. + ## + ## id: the log stream to enable remote logs for. + ## + ## flags: tune the behavior of how log entry messages are sent. + ## + ## Returns: true if remote logs are enabled for the stream. + global enable_remote_logs: function(id: Log::ID, flags: SendFlags &default = SendFlags()): bool; + + ## Disable remote logs for a given log stream. + ## + ## id: the log stream to disable remote logs for. + ## + ## Returns: true if remote logs are disabled for the stream. + global disable_remote_logs: function(id: Log::ID): bool; + + ## Check if remote logs are enabled for a given log stream. + ## + ## id: the log stream to check. + ## + ## Returns: true if remote logs are enabled for the given stream. + global remote_logs_enabled: function(id: Log::ID): bool; + + ## Register interest in all peer log messages that use a certain topic + ## prefix. Logs are implicitly sent with topic "bro/log/" and + ## the receiving side processes them through the logging framework as usual. + ## + ## topic_prefix: a prefix to match against remote message topics. + ## e.g. an empty prefix matches everything and "a" matches + ## "alice" and "amy" but not "bob". + ## + ## Returns: true if it's a new log subscription and it is now registered. + global subscribe_to_logs: function(topic_prefix: string): bool; + + ## Unregister interest in all peer log messages that use a topic prefix. + ## Logs are implicitly sent with topic "bro/log/" and the + ## receiving side processes them through the logging framework as usual. + ## + ## topic_prefix: a prefix previously supplied to a successful call to + ## :bro:see:`Broker::subscribe_to_logs`. + ## + ## Returns: true if interest in the topic prefix is no longer advertised. + global unsubscribe_to_logs: function(topic_prefix: string): bool; + } + +@load base/bif/comm.bif +@load base/bif/messaging.bif + +module Broker; + +function enable(flags: EndpointFlags &default = EndpointFlags()) : bool + { + return __enable(flags); + } + +function set_endpoint_flags(flags: EndpointFlags &default = EndpointFlags()): bool + { + return __set_endpoint_flags(flags); + } + +function publish_topic(topic: string): bool + { + return __publish_topic(topic); + } + +function unpublish_topic(topic: string): bool + { + return __unpublish_topic(topic); + } + +function listen(p: port, a: string &default = "", reuse: bool &default = T): bool + { + return __listen(p, a, reuse); + } + +function connect(a: string, p: port, retry: interval): bool + { + return __connect(a, p, retry); + } + +function disconnect(a: string, p: port): bool + { + return __disconnect(a, p); + } + +function send_print(topic: string, msg: string, flags: SendFlags &default = SendFlags()): bool + { + return __send_print(topic, msg, flags); + } + +function subscribe_to_prints(topic_prefix: string): bool + { + return __subscribe_to_prints(topic_prefix); + } + +function unsubscribe_to_prints(topic_prefix: string): bool + { + return __unsubscribe_to_prints(topic_prefix); + } + +function send_event(topic: string, args: EventArgs, flags: SendFlags &default = SendFlags()): bool + { + return __event(topic, args, flags); + } + +function auto_event(topic: string, ev: any, flags: SendFlags &default = SendFlags()): bool + { + return __auto_event(topic, ev, flags); + } + +function auto_event_stop(topic: string, ev: any): bool + { + return __auto_event_stop(topic, ev); + } + +function subscribe_to_events(topic_prefix: string): bool + { + return __subscribe_to_events(topic_prefix); + } + +function unsubscribe_to_events(topic_prefix: string): bool + { + return __unsubscribe_to_events(topic_prefix); + } + +function enable_remote_logs(id: Log::ID, flags: SendFlags &default = SendFlags()): bool + { + return __enable_remote_logs(id, flags); + } + +function disable_remote_logs(id: Log::ID): bool + { + return __disable_remote_logs(id); + } + +function remote_logs_enabled(id: Log::ID): bool + { + return __remote_logs_enabled(id); + } + +function subscribe_to_logs(topic_prefix: string): bool + { + return __subscribe_to_logs(topic_prefix); + } + +function unsubscribe_to_logs(topic_prefix: string): bool + { + return __unsubscribe_to_logs(topic_prefix); + } + diff --git a/scripts/base/frameworks/broker/store.bro b/scripts/base/frameworks/broker/store.bro index e6468f2b2c..36565d72aa 100644 --- a/scripts/base/frameworks/broker/store.bro +++ b/scripts/base/frameworks/broker/store.bro @@ -31,6 +31,13 @@ export { result: Broker::Data; }; + ## Enumerates the possible storage backends. + type BackendType: enum { + MEMORY, + SQLITE, + ROCKSDB, + }; + ## Options to tune the SQLite storage backend. type SQLiteOptions: record { ## File system path of the database. @@ -48,4 +55,341 @@ export { sqlite: SQLiteOptions &default = SQLiteOptions(); rocksdb: RocksDBOptions &default = RocksDBOptions(); }; + + ## Create a master data store which contains key-value pairs. + ## + ## id: a unique name for the data store. + ## + ## b: the storage backend to use. + ## + ## options: tunes how some storage backends operate. + ## + ## Returns: a handle to the data store. + global create_master: function(id: string, b: BackendType &default = MEMORY, + options: BackendOptions &default = BackendOptions()): opaque of Broker::Handle; + + ## Create a clone of a master data store which may live with a remote peer. + ## A clone automatically synchronizes to the master by automatically + ## receiving modifications and applying them locally. Direct modifications + ## are not possible, they must be sent through the master store, which then + ## automatically broadcasts the changes out to clones. But queries may be + ## made directly against the local cloned copy, which may be resolved + ## quicker than reaching out to a remote master store. + ## + ## id: the unique name which identifies the master data store. + ## + ## b: the storage backend to use. + ## + ## options: tunes how some storage backends operate. + ## + ## resync: the interval at which to re-attempt synchronizing with the master + ## store should the connection be lost. If the clone has not yet + ## synchronized for the first time, updates and queries queue up + ## until the synchronization completes. After, if the connection + ## to the master store is lost, queries continue to use the clone's + ## version, but updates will be lost until the master is once again + ## available. + ## + ## Returns: a handle to the data store. + global create_clone: function(id: string, b: BackendType &default = MEMORY, + options: BackendOptions &default = BackendOptions(), + resync: interval &default = 1sec): opaque of Broker::Handle; + + ## Create a frontend interface to an existing master data store that allows + ## querying and updating its contents. + ## + ## id: the unique name which identifies the master data store. + ## + ## Returns: a handle to the data store. + global create_frontend: function(id: string): opaque of Broker::Handle; + + ## Close a data store. + ## + ## h: a data store handle. + ## + ## Returns: true if store was valid and is now closed. The handle can no + ## longer be used for data store operations. + global close_by_handle: function(h: opaque of Broker::Handle): bool; + + ########################### + # non-blocking update API # + ########################### + + ## Insert a key-value pair in to the store. + ## + ## h: the handle of the store to modify. + ## + ## k: the key to insert. + ## + ## v: the value to insert. + ## + ## e: the expiration time of the key-value pair. + ## + ## Returns: false if the store handle was not valid. + global insert: function(h: opaque of Broker::Handle, + k: Broker::Data, v: Broker::Data, + e: Broker::ExpiryTime &default = Broker::ExpiryTime()): bool; + + ## Remove a key-value pair from the store. + ## + ## h: the handle of the store to modify. + ## + ## k: the key to remove. + ## + ## Returns: false if the store handle was not valid. + global erase: function(h: opaque of Broker::Handle, k: Broker::Data): bool; + + ## Remove all key-value pairs from the store. + ## + ## h: the handle of the store to modify. + ## + ## Returns: false if the store handle was not valid. + global clear: function(h: opaque of Broker::Handle): bool; + + ## Increment an integer value in a data store. + ## + ## h: the handle of the store to modify. + ## + ## k: the key whose associated value is to be modified. + ## + ## by: the amount to increment the value by. A non-existent key will first + ## create it with an implicit value of zero before incrementing. + ## + ## Returns: false if the store handle was not valid. + global increment: function(h: opaque of Broker::Handle, + k: Broker::Data, by: int &default = +1): bool; + + ## Decrement an integer value in a data store. + ## + ## h: the handle of the store to modify. + ## + ## k: the key whose associated value is to be modified. + ## + ## by: the amount to decrement the value by. A non-existent key will first + ## create it with an implicit value of zero before decrementing. + ## + ## Returns: false if the store handle was not valid. + global decrement: function(h: opaque of Broker::Handle, + k: Broker::Data, by: int &default = +1): bool; + + ## Add an element to a set value in a data store. + ## + ## h: the handle of the store to modify. + ## + ## k: the key whose associated value is to be modified. + ## + ## element: the element to add to the set. A non-existent key will first + ## create it with an implicit empty set value before modifying. + ## + ## Returns: false if the store handle was not valid. + global add_to_set: function(h: opaque of Broker::Handle, + k: Broker::Data, element: Broker::Data): bool; + + ## Remove an element from a set value in a data store. + ## + ## h: the handle of the store to modify. + ## + ## k: the key whose associated value is to be modified. + ## + ## element: the element to remove from the set. A non-existent key will + ## implicitly create an empty set value associated with the key. + ## + ## Returns: false if the store handle was not valid. + global remove_from_set: function(h: opaque of Broker::Handle, + k: Broker::Data, element: Broker::Data): bool; + + ## Add a new item to the head of a vector value in a data store. + ## + ## h: the handle of store to modify. + ## + ## k: the key whose associated value is to be modified. + ## + ## items: the element to insert in to the vector. A non-existent key will + ## first create an empty vector value before modifying. + ## + ## Returns: false if the store handle was not valid. + global push_left: function(h: opaque of Broker::Handle, k: Broker::Data, + items: Broker::DataVector): bool; + + ## Add a new item to the tail of a vector value in a data store. + ## + ## h: the handle of store to modify. + ## + ## k: the key whose associated value is to be modified. + ## + ## items: the element to insert in to the vector. A non-existent key will + ## first create an empty vector value before modifying. + ## + ## Returns: false if the store handle was not valid. + global push_right: function(h: opaque of Broker::Handle, k: Broker::Data, + items: Broker::DataVector): bool; + + ########################## + # non-blocking query API # + ########################## + + ## Pop the head of a data store vector value. + ## + ## h: the handle of the store to query. + ## + ## k: the key associated with the vector to modify. + ## + ## Returns: the result of the query. + global pop_left: function(h: opaque of Broker::Handle, + k: Broker::Data): QueryResult; + + ## Pop the tail of a data store vector value. + ## + ## h: the handle of the store to query. + ## + ## k: the key associated with the vector to modify. + ## + ## Returns: the result of the query. + global pop_right: function(h: opaque of Broker::Handle, + k: Broker::Data): QueryResult; + + ## Lookup the value associated with a key in a data store. + ## + ## h: the handle of the store to query. + ## + ## k: the key to lookup. + ## + ## Returns: the result of the query. + global lookup: function(h: opaque of Broker::Handle, + k: Broker::Data): QueryResult; + + ## Check if a data store contains a given key. + ## + ## h: the handle of the store to query. + ## + ## k: the key to check for existence. + ## + ## Returns: the result of the query (uses :bro:see:`Broker::BOOL`). + global exists: function(h: opaque of Broker::Handle, + k: Broker::Data): QueryResult; + + ## Retrieve all keys in a data store. + ## + ## h: the handle of the store to query. + ## + ## Returns: the result of the query (uses :bro:see:`Broker::VECTOR`). + global keys: function(h: opaque of Broker::Handle): QueryResult; + + ## Get the number of key-value pairs in a data store. + ## + ## h: the handle of the store to query. + ## + ## Returns: the result of the query (uses :bro:see:`Broker::COUNT`). + global size: function(h: opaque of Broker::Handle): QueryResult; + } + +@load base/bif/store.bif + +module Broker; + +function create_master(id: string, b: BackendType &default = MEMORY, + options: BackendOptions &default = BackendOptions()): opaque of Broker::Handle + { + return __create_master(id, b, options); + } + +function create_clone(id: string, b: BackendType &default = MEMORY, + options: BackendOptions &default = BackendOptions(), + resync: interval &default = 1sec): opaque of Broker::Handle + { + return __create_clone(id, b, options, resync); + } + +function create_frontend(id: string): opaque of Broker::Handle + { + return __create_frontend(id); + } + +function close_by_handle(h: opaque of Broker::Handle): bool + { + return __close_by_handle(h); + } + +function insert(h: opaque of Broker::Handle, k: Broker::Data, v: Broker::Data, + e: Broker::ExpiryTime &default = Broker::ExpiryTime()): bool + { + return __insert(h, k, v, e); + } + +function erase(h: opaque of Broker::Handle, k: Broker::Data): bool + { + return __erase(h, k); + } + +function clear(h: opaque of Broker::Handle): bool + { + return __clear(h); + } + +function increment(h: opaque of Broker::Handle, + k: Broker::Data, by: int &default = +1): bool + { + return __increment(h, k, by); + } + +function decrement(h: opaque of Broker::Handle, + k: Broker::Data, by: int &default = +1): bool + { + return __decrement(h, k, by); + } + +function add_to_set(h: opaque of Broker::Handle, + k: Broker::Data, element: Broker::Data): bool + { + return __add_to_set(h, k, element); + } + +function remove_from_set(h: opaque of Broker::Handle, + k: Broker::Data, element: Broker::Data): bool + { + return __remove_from_set(h, k, element); + } + +function push_left(h: opaque of Broker::Handle, k: Broker::Data, + items: Broker::DataVector): bool + { + return __push_left(h, k, items); + } + +function push_right(h: opaque of Broker::Handle, k: Broker::Data, + items: Broker::DataVector): bool + { + return __push_right(h, k, items); + } + +function pop_left(h: opaque of Broker::Handle, k: Broker::Data): QueryResult + { + return __pop_left(h, k); + } + +function pop_right(h: opaque of Broker::Handle, k: Broker::Data): QueryResult + { + return __pop_right(h, k); + } + +function lookup(h: opaque of Broker::Handle, k: Broker::Data): QueryResult + { + return __lookup(h, k); + } + +function exists(h: opaque of Broker::Handle, k: Broker::Data): QueryResult + { + return __exists(h, k); + } + +function keys(h: opaque of Broker::Handle): QueryResult + { + return __keys(h); + } + +function size(h: opaque of Broker::Handle): QueryResult + { + return __size(h); + } + diff --git a/scripts/base/frameworks/netcontrol/plugins/acld.bro b/scripts/base/frameworks/netcontrol/plugins/acld.bro index 13802f2e21..ba50558d9a 100644 --- a/scripts/base/frameworks/netcontrol/plugins/acld.bro +++ b/scripts/base/frameworks/netcontrol/plugins/acld.bro @@ -227,7 +227,7 @@ function acld_add_rule_fun(p: PluginState, r: Rule) : bool if ( ar$command == "" ) return F; - Broker::event(p$acld_config$acld_topic, Broker::event_args(acld_add_rule, p$acld_id, r, ar)); + Broker::send_event(p$acld_config$acld_topic, Broker::event_args(acld_add_rule, p$acld_id, r, ar)); return T; } @@ -242,7 +242,7 @@ function acld_remove_rule_fun(p: PluginState, r: Rule) : bool else return F; - Broker::event(p$acld_config$acld_topic, Broker::event_args(acld_remove_rule, p$acld_id, r, ar)); + Broker::send_event(p$acld_config$acld_topic, Broker::event_args(acld_remove_rule, p$acld_id, r, ar)); return T; } diff --git a/scripts/base/frameworks/netcontrol/plugins/broker.bro b/scripts/base/frameworks/netcontrol/plugins/broker.bro index 2af3724db7..82e1d20f07 100644 --- a/scripts/base/frameworks/netcontrol/plugins/broker.bro +++ b/scripts/base/frameworks/netcontrol/plugins/broker.bro @@ -96,13 +96,13 @@ function broker_name(p: PluginState) : string function broker_add_rule_fun(p: PluginState, r: Rule) : bool { - Broker::event(p$broker_topic, Broker::event_args(broker_add_rule, p$broker_id, r)); + Broker::send_event(p$broker_topic, Broker::event_args(broker_add_rule, p$broker_id, r)); return T; } function broker_remove_rule_fun(p: PluginState, r: Rule) : bool { - Broker::event(p$broker_topic, Broker::event_args(broker_remove_rule, p$broker_id, r)); + Broker::send_event(p$broker_topic, Broker::event_args(broker_remove_rule, p$broker_id, r)); return T; } diff --git a/scripts/base/frameworks/openflow/plugins/broker.bro b/scripts/base/frameworks/openflow/plugins/broker.bro index 93a627a8f4..ba15cc6ad1 100644 --- a/scripts/base/frameworks/openflow/plugins/broker.bro +++ b/scripts/base/frameworks/openflow/plugins/broker.bro @@ -47,14 +47,14 @@ function broker_describe(state: ControllerState): string function broker_flow_mod_fun(state: ControllerState, match: ofp_match, flow_mod: OpenFlow::ofp_flow_mod): bool { - Broker::event(state$broker_topic, Broker::event_args(broker_flow_mod, state$_name, state$broker_dpid, match, flow_mod)); + Broker::send_event(state$broker_topic, Broker::event_args(broker_flow_mod, state$_name, state$broker_dpid, match, flow_mod)); return T; } function broker_flow_clear_fun(state: OpenFlow::ControllerState): bool { - Broker::event(state$broker_topic, Broker::event_args(broker_flow_clear, state$_name, state$broker_dpid)); + Broker::send_event(state$broker_topic, Broker::event_args(broker_flow_clear, state$_name, state$broker_dpid)); return T; } diff --git a/src/broker/comm.bif b/src/broker/comm.bif index 4caa1f8859..3bc8fa7dff 100644 --- a/src/broker/comm.bif +++ b/src/broker/comm.bif @@ -9,46 +9,22 @@ module Broker; type Broker::EndpointFlags: record; -## Enable use of communication. -## -## flags: used to tune the local Broker endpoint behavior. -## -## Returns: true if communication is successfully initialized. -function Broker::enable%(flags: EndpointFlags &default = EndpointFlags()%): bool +function Broker::__enable%(flags: EndpointFlags%): bool %{ return new Val(broker_mgr->Enable(flags), TYPE_BOOL); %} -## Changes endpoint flags originally supplied to :bro:see:`Broker::enable`. -## -## flags: the new endpoint behavior flags to use. -## -## Returns: true if flags were changed. -function Broker::set_endpoint_flags%(flags: EndpointFlags &default = EndpointFlags()%): bool +function Broker::__set_endpoint_flags%(flags: EndpointFlags%): bool %{ return new Val(broker_mgr->SetEndpointFlags(flags), TYPE_BOOL); %} -## Allow sending messages to peers if associated with the given topic. -## This has no effect if auto publication behavior is enabled via the flags -## supplied to :bro:see:`Broker::enable` or :bro:see:`Broker::set_endpoint_flags`. -## -## topic: a topic to allow messages to be published under. -## -## Returns: true if successful. -function Broker::publish_topic%(topic: string%): bool +function Broker::__publish_topic%(topic: string%): bool %{ return new Val(broker_mgr->PublishTopic(topic->CheckString()), TYPE_BOOL); %} -## Disallow sending messages to peers if associated with the given topic. -## This has no effect if auto publication behavior is enabled via the flags -## supplied to :bro:see:`Broker::enable` or :bro:see:`Broker::set_endpoint_flags`. -## -## topic: a topic to disallow messages to be published under. -## -## Returns: true if successful. -function Broker::unpublish_topic%(topic: string%): bool +function Broker::__unpublish_topic%(topic: string%): bool %{ return new Val(broker_mgr->UnpublishTopic(topic->CheckString()), TYPE_BOOL); %} @@ -124,20 +100,7 @@ event Broker::incoming_connection_established%(peer_name: string%); ## .. bro:see:: Broker::incoming_connection_established event Broker::incoming_connection_broken%(peer_name: string%); -## Listen for remote connections. -## -## p: the TCP port to listen on. -## -## a: an address string on which to accept connections, e.g. -## "127.0.0.1". An empty string refers to @p INADDR_ANY. -## -## reuse: equivalent to behavior of SO_REUSEADDR. -## -## Returns: true if the local endpoint is now listening for connections. -## -## .. bro:see:: Broker::incoming_connection_established -function Broker::listen%(p: port, a: string &default = "", - reuse: bool &default = T%): bool +function Broker::__listen%(p: port, a: string, reuse: bool%): bool %{ if ( ! p->IsTCP() ) { @@ -150,22 +113,7 @@ function Broker::listen%(p: port, a: string &default = "", return new Val(rval, TYPE_BOOL); %} -## Initiate a remote connection. -## -## a: an address to connect to, e.g. "localhost" or "127.0.0.1". -## -## p: the TCP port on which the remote side is listening. -## -## retry: an interval at which to retry establishing the -## connection with the remote peer if it cannot be made initially, or -## if it ever becomes disconnected. -## -## Returns: true if it's possible to try connecting with the peer and -## it's a new peer. The actual connection may not be established -## until a later point in time. -## -## .. bro:see:: Broker::outgoing_connection_established -function Broker::connect%(a: string, p: port, retry: interval%): bool +function Broker::__connect%(a: string, p: port, retry: interval%): bool %{ if ( ! p->IsTCP() ) { @@ -178,15 +126,7 @@ function Broker::connect%(a: string, p: port, retry: interval%): bool return new Val(rval, TYPE_BOOL); %} -## Remove a remote connection. -## -## a: the address used in previous successful call to :bro:see:`Broker::connect`. -## -## p: the port used in previous successful call to :bro:see:`Broker::connect`. -## -## Returns: true if the arguments match a previously successful call to -## :bro:see:`Broker::connect`. -function Broker::disconnect%(a: string, p: port%): bool +function Broker::__disconnect%(a: string, p: port%): bool %{ if ( ! p->IsTCP() ) { diff --git a/src/broker/messaging.bif b/src/broker/messaging.bif index 3c3240ff16..dadece9681 100644 --- a/src/broker/messaging.bif +++ b/src/broker/messaging.bif @@ -13,202 +13,99 @@ type Broker::SendFlags: record; type Broker::EventArgs: record; ## Used to handle remote print messages from peers that call -## :bro:see:`Broker::print`. +## :bro:see:`Broker::send_print`. event Broker::print_handler%(msg: string%); -## Print a simple message to any interested peers. The receiver can use -## :bro:see:`Broker::print_handler` to handle messages. -## -## topic: a topic associated with the printed message. -## -## msg: the print message to send to peers. -## -## flags: tune the behavior of how the message is sent. -## -## Returns: true if the message is sent. -function Broker::print%(topic: string, msg: string, - flags: SendFlags &default = SendFlags()%): bool +function Broker::__send_print%(topic: string, msg: string, flags: Broker::SendFlags%): bool %{ auto rval = broker_mgr->Print(topic->CheckString(), msg->CheckString(), flags); return new Val(rval, TYPE_BOOL); %} -## Register interest in all peer print messages that use a certain topic prefix. -## Use :bro:see:`Broker::print_handler` to handle received messages. -## -## topic_prefix: a prefix to match against remote message topics. -## e.g. an empty prefix matches everything and "a" matches -## "alice" and "amy" but not "bob". -## -## Returns: true if it's a new print subscription and it is now registered. -function Broker::subscribe_to_prints%(topic_prefix: string%): bool +function Broker::__subscribe_to_prints%(topic_prefix: string%): bool %{ auto rval = broker_mgr->SubscribeToPrints(topic_prefix->CheckString()); return new Val(rval, TYPE_BOOL); %} -## Unregister interest in all peer print messages that use a topic prefix. -## -## topic_prefix: a prefix previously supplied to a successful call to -## :bro:see:`Broker::subscribe_to_prints`. -## -## Returns: true if interest in the topic prefix is no longer advertised. -function Broker::unsubscribe_to_prints%(topic_prefix: string%): bool +function Broker::__unsubscribe_to_prints%(topic_prefix: string%): bool %{ auto rval = broker_mgr->UnsubscribeToPrints(topic_prefix->CheckString()); return new Val(rval, TYPE_BOOL); %} ## Create a data structure that may be used to send a remote event via -## :bro:see:`Broker::event`. +## :bro:see:`Broker::send_event`. ## ## args: an event, followed by a list of argument values that may be used ## to call it. ## -## Returns: opaque communication data that may be used to send a remote event. +## Returns: opaque communication data that may be used to send a remote +## event. function Broker::event_args%(...%): Broker::EventArgs %{ auto rval = broker_mgr->MakeEventArgs(@ARGS@); return rval; %} -## Send an event to any interested peers. -## -## topic: a topic associated with the event message. -## -## args: event arguments as made by :bro:see:`Broker::event_args`. -## -## flags: tune the behavior of how the message is sent. -## -## Returns: true if the message is sent. -function Broker::event%(topic: string, args: Broker::EventArgs, - flags: SendFlags &default = SendFlags()%): bool +function Broker::__event%(topic: string, args: Broker::EventArgs, flags: Broker::SendFlags%): bool %{ auto rval = broker_mgr->Event(topic->CheckString(), args->AsRecordVal(), flags); return new Val(rval, TYPE_BOOL); %} -## Automatically send an event to any interested peers whenever it is -## locally dispatched (e.g. using "event my_event(...);" in a script). -## -## topic: a topic string associated with the event message. -## Peers advertise interest by registering a subscription to some prefix -## of this topic name. -## -## ev: a Bro event value. -## -## flags: tune the behavior of how the message is sent. -## -## Returns: true if automatic event sending is now enabled. -function Broker::auto_event%(topic: string, ev: any, - flags: SendFlags &default = SendFlags()%): bool +function Broker::__auto_event%(topic: string, ev: any, flags: Broker::SendFlags%): bool %{ auto rval = broker_mgr->AutoEvent(topic->CheckString(), ev, flags); return new Val(rval, TYPE_BOOL); %} -## Stop automatically sending an event to peers upon local dispatch. -## -## topic: a topic originally given to :bro:see:`Broker::auto_event`. -## -## ev: an event originally given to :bro:see:`Broker::auto_event`. -## -## Returns: true if automatic events will not occur for the topic/event pair. -function Broker::auto_event_stop%(topic: string, ev: any%): bool +function Broker::__auto_event_stop%(topic: string, ev: any%): bool %{ auto rval = broker_mgr->AutoEventStop(topic->CheckString(), ev); return new Val(rval, TYPE_BOOL); %} -## Register interest in all peer event messages that use a certain topic prefix. -## -## topic_prefix: a prefix to match against remote message topics. -## e.g. an empty prefix matches everything and "a" matches -## "alice" and "amy" but not "bob". -## -## Returns: true if it's a new event subscription and it is now registered. -function Broker::subscribe_to_events%(topic_prefix: string%): bool +function Broker::__subscribe_to_events%(topic_prefix: string%): bool %{ auto rval = broker_mgr->SubscribeToEvents(topic_prefix->CheckString()); return new Val(rval, TYPE_BOOL); %} -## Unregister interest in all peer event messages that use a topic prefix. -## -## topic_prefix: a prefix previously supplied to a successful call to -## :bro:see:`Broker::subscribe_to_events`. -## -## Returns: true if interest in the topic prefix is no longer advertised. -function Broker::unsubscribe_to_events%(topic_prefix: string%): bool +function Broker::__unsubscribe_to_events%(topic_prefix: string%): bool %{ auto rval = broker_mgr->UnsubscribeToEvents(topic_prefix->CheckString()); return new Val(rval, TYPE_BOOL); %} -## Enable remote logs for a given log stream. -## -## id: the log stream to enable remote logs for. -## -## flags: tune the behavior of how log entry messages are sent. -## -## Returns: true if remote logs are enabled for the stream. -function -Broker::enable_remote_logs%(id: Log::ID, - flags: SendFlags &default = SendFlags()%): bool +function Broker::__enable_remote_logs%(id: Log::ID, flags: Broker::SendFlags%): bool %{ auto rval = log_mgr->EnableRemoteLogs(id->AsEnumVal(), bro_broker::Manager::send_flags_to_int(flags)); return new Val(rval, TYPE_BOOL); %} -## Disable remote logs for a given log stream. -## -## id: the log stream to disable remote logs for. -## -## Returns: true if remote logs are disabled for the stream. -function Broker::disable_remote_logs%(id: Log::ID%): bool +function Broker::__disable_remote_logs%(id: Log::ID%): bool %{ auto rval = log_mgr->DisableRemoteLogs(id->AsEnumVal()); return new Val(rval, TYPE_BOOL); %} -## Check if remote logs are enabled for a given log stream. -## -## id: the log stream to check. -## -## Returns: true if remote logs are enabled for the given stream. -function Broker::remote_logs_enabled%(id: Log::ID%): bool +function Broker::__remote_logs_enabled%(id: Log::ID%): bool %{ auto rval = log_mgr->RemoteLogsAreEnabled(id->AsEnumVal()); return new Val(rval, TYPE_BOOL); %} -## Register interest in all peer log messages that use a certain topic prefix. -## Logs are implicitly sent with topic "bro/log/" and the -## receiving side processes them through the logging framework as usual. -## -## topic_prefix: a prefix to match against remote message topics. -## e.g. an empty prefix matches everything and "a" matches -## "alice" and "amy" but not "bob". -## -## Returns: true if it's a new log subscription and it is now registered. -function Broker::subscribe_to_logs%(topic_prefix: string%): bool +function Broker::__subscribe_to_logs%(topic_prefix: string%): bool %{ auto rval = broker_mgr->SubscribeToLogs(topic_prefix->CheckString()); return new Val(rval, TYPE_BOOL); %} -## Unregister interest in all peer log messages that use a topic prefix. -## Logs are implicitly sent with topic "bro/log/" and the -## receiving side processes them through the logging framework as usual. -## -## topic_prefix: a prefix previously supplied to a successful call to -## :bro:see:`Broker::subscribe_to_logs`. -## -## Returns: true if interest in the topic prefix is no longer advertised. -function Broker::unsubscribe_to_logs%(topic_prefix: string%): bool +function Broker::__unsubscribe_to_logs%(topic_prefix: string%): bool %{ auto rval = broker_mgr->UnsubscribeToLogs(topic_prefix->CheckString()); return new Val(rval, TYPE_BOOL); diff --git a/src/broker/store.bif b/src/broker/store.bif index 57bddd3da7..6d7ddea6af 100644 --- a/src/broker/store.bif +++ b/src/broker/store.bif @@ -23,16 +23,7 @@ enum BackendType %{ ROCKSDB, %} -## Create a master data store which contains key-value pairs. -## -## id: a unique name for the data store. -## -## b: the storage backend to use. -## -## options: tunes how some storage backends operate. -## -## Returns: a handle to the data store. -function Broker::create_master%(id: string, b: BackendType &default = MEMORY, +function Broker::__create_master%(id: string, b: BackendType, options: BackendOptions &default = BackendOptions()%): opaque of Broker::Handle %{ auto id_str = id->CheckString(); @@ -53,29 +44,7 @@ function Broker::create_master%(id: string, b: BackendType &default = MEMORY, return rval; %} -## Create a clone of a master data store which may live with a remote peer. -## A clone automatically synchronizes to the master by automatically receiving -## modifications and applying them locally. Direct modifications are not -## possible, they must be sent through the master store, which then -## automatically broadcasts the changes out to clones. But queries may be made -## directly against the local cloned copy, which may be resolved quicker than -## reaching out to a remote master store. -## -## id: the unique name which identifies the master data store. -## -## b: the storage backend to use. -## -## options: tunes how some storage backends operate. -## -## resync: the interval at which to re-attempt synchronizing with the master -## store should the connection be lost. If the clone has not yet -## synchronized for the first time, updates and queries queue up until -## the synchronization completes. After, if the connection to the -## master store is lost, queries continue to use the clone's version, -## but updates will be lost until the master is once again available. -## -## Returns: a handle to the data store. -function Broker::create_clone%(id: string, b: BackendType &default = MEMORY, +function Broker::__create_clone%(id: string, b: BackendType, options: BackendOptions &default = BackendOptions(), resync: interval &default = 1sec%): opaque of Broker::Handle %{ @@ -98,13 +67,7 @@ function Broker::create_clone%(id: string, b: BackendType &default = MEMORY, return rval; %} -## Create a frontend interface to an existing master data store that allows -## querying and updating its contents. -## -## id: the unique name which identifies the master data store. -## -## Returns: a handle to the data store. -function Broker::create_frontend%(id: string%): opaque of Broker::Handle +function Broker::__create_frontend%(id: string%): opaque of Broker::Handle %{ auto id_str = id->CheckString(); auto type = bro_broker::StoreType::FRONTEND; @@ -122,13 +85,7 @@ function Broker::create_frontend%(id: string%): opaque of Broker::Handle return rval; %} -## Close a data store. -## -## h: a data store handle. -## -## Returns: true if store was valid and is now closed. The handle can no -## longer be used for data store operations. -function Broker::close_by_handle%(h: opaque of Broker::Handle%): bool +function Broker::__close_by_handle%(h: opaque of Broker::Handle%): bool %{ auto handle = static_cast(h); @@ -143,18 +100,7 @@ function Broker::close_by_handle%(h: opaque of Broker::Handle%): bool # non-blocking update API # ########################### -## Insert a key-value pair in to the store. -## -## h: the handle of the store to modify. -## -## k: the key to insert. -## -## v: the value to insert. -## -## e: the expiration time of the key-value pair. -## -## Returns: false if the store handle was not valid. -function Broker::insert%(h: opaque of Broker::Handle, +function Broker::__insert%(h: opaque of Broker::Handle, k: Broker::Data, v: Broker::Data, e: Broker::ExpiryTime &default = Broker::ExpiryTime()%): bool %{ @@ -191,14 +137,7 @@ function Broker::insert%(h: opaque of Broker::Handle, return new Val(true, TYPE_BOOL); %} -## Remove a key-value pair from the store. -## -## h: the handle of the store to modify. -## -## k: the key to remove. -## -## Returns: false if the store handle was not valid. -function Broker::erase%(h: opaque of Broker::Handle, k: Broker::Data%): bool +function Broker::__erase%(h: opaque of Broker::Handle, k: Broker::Data%): bool %{ auto handle = static_cast(h); @@ -210,12 +149,7 @@ function Broker::erase%(h: opaque of Broker::Handle, k: Broker::Data%): bool return new Val(true, TYPE_BOOL); %} -## Remove all key-value pairs from the store. -## -## h: the handle of the store to modify. -## -## Returns: false if the store handle was not valid. -function Broker::clear%(h: opaque of Broker::Handle%): bool +function Broker::__clear%(h: opaque of Broker::Handle%): bool %{ auto handle = static_cast(h); @@ -226,17 +160,7 @@ function Broker::clear%(h: opaque of Broker::Handle%): bool return new Val(true, TYPE_BOOL); %} -## Increment an integer value in a data store. -## -## h: the handle of the store to modify. -## -## k: the key whose associated value is to be modified. -## -## by: the amount to increment the value by. A non-existent key will first -## create it with an implicit value of zero before incrementing. -## -## Returns: false if the store handle was not valid. -function Broker::increment%(h: opaque of Broker::Handle, +function Broker::__increment%(h: opaque of Broker::Handle, k: Broker::Data, by: int &default = +1%): bool %{ auto handle = static_cast(h); @@ -249,17 +173,7 @@ function Broker::increment%(h: opaque of Broker::Handle, return new Val(true, TYPE_BOOL); %} -## Decrement an integer value in a data store. -## -## h: the handle of the store to modify. -## -## k: the key whose associated value is to be modified. -## -## by: the amount to decrement the value by. A non-existent key will first -## create it with an implicit value of zero before decrementing. -## -## Returns: false if the store handle was not valid. -function Broker::decrement%(h: opaque of Broker::Handle, +function Broker::__decrement%(h: opaque of Broker::Handle, k: Broker::Data, by: int &default = +1%): bool %{ auto handle = static_cast(h); @@ -272,17 +186,7 @@ function Broker::decrement%(h: opaque of Broker::Handle, return new Val(true, TYPE_BOOL); %} -## Add an element to a set value in a data store. -## -## h: the handle of the store to modify. -## -## k: the key whose associated value is to be modified. -## -## element: the element to add to the set. A non-existent key will first -## create it with an implicit empty set value before modifying. -## -## Returns: false if the store handle was not valid. -function Broker::add_to_set%(h: opaque of Broker::Handle, +function Broker::__add_to_set%(h: opaque of Broker::Handle, k: Broker::Data, element: Broker::Data%): bool %{ auto handle = static_cast(h); @@ -296,17 +200,7 @@ function Broker::add_to_set%(h: opaque of Broker::Handle, return new Val(true, TYPE_BOOL); %} -## Remove an element from a set value in a data store. -## -## h: the handle of the store to modify. -## -## k: the key whose associated value is to be modified. -## -## element: the element to remove from the set. A non-existent key will -## implicitly create an empty set value associated with the key. -## -## Returns: false if the store handle was not valid. -function Broker::remove_from_set%(h: opaque of Broker::Handle, +function Broker::__remove_from_set%(h: opaque of Broker::Handle, k: Broker::Data, element: Broker::Data%): bool %{ auto handle = static_cast(h); @@ -320,17 +214,7 @@ function Broker::remove_from_set%(h: opaque of Broker::Handle, return new Val(true, TYPE_BOOL); %} -## Add a new item to the head of a vector value in a data store. -## -## h: the handle of store to modify. -## -## k: the key whose associated value is to be modified. -## -## items: the element to insert in to the vector. A non-existent key will first -## create an empty vector value before modifying. -## -## Returns: false if the store handle was not valid. -function Broker::push_left%(h: opaque of Broker::Handle, k: Broker::Data, +function Broker::__push_left%(h: opaque of Broker::Handle, k: Broker::Data, items: Broker::DataVector%): bool %{ auto handle = static_cast(h); @@ -353,17 +237,7 @@ function Broker::push_left%(h: opaque of Broker::Handle, k: Broker::Data, return new Val(true, TYPE_BOOL); %} -## Add a new item to the tail of a vector value in a data store. -## -## h: the handle of store to modify. -## -## k: the key whose associated value is to be modified. -## -## items: the element to insert in to the vector. A non-existent key will first -## create an empty vector value before modifying. -## -## Returns: false if the store handle was not valid. -function Broker::push_right%(h: opaque of Broker::Handle, k: Broker::Data, +function Broker::__push_right%(h: opaque of Broker::Handle, k: Broker::Data, items: Broker::DataVector%): bool %{ auto handle = static_cast(h); @@ -437,14 +311,7 @@ static bool prepare_for_query(Val* opaque, Frame* frame, %%} -## Pop the head of a data store vector value. -## -## h: the handle of the store to query. -## -## k: the key associated with the vector to modify. -## -## Returns: the result of the query. -function Broker::pop_left%(h: opaque of Broker::Handle, +function Broker::__pop_left%(h: opaque of Broker::Handle, k: Broker::Data%): Broker::QueryResult %{ if ( ! broker_mgr->Enabled() ) @@ -467,14 +334,7 @@ function Broker::pop_left%(h: opaque of Broker::Handle, return 0; %} -## Pop the tail of a data store vector value. -## -## h: the handle of the store to query. -## -## k: the key associated with the vector to modify. -## -## Returns: the result of the query. -function Broker::pop_right%(h: opaque of Broker::Handle, +function Broker::__pop_right%(h: opaque of Broker::Handle, k: Broker::Data%): Broker::QueryResult %{ if ( ! broker_mgr->Enabled() ) @@ -497,14 +357,7 @@ function Broker::pop_right%(h: opaque of Broker::Handle, return 0; %} -## Lookup the value associated with a key in a data store. -## -## h: the handle of the store to query. -## -## k: the key to lookup. -## -## Returns: the result of the query. -function Broker::lookup%(h: opaque of Broker::Handle, +function Broker::__lookup%(h: opaque of Broker::Handle, k: Broker::Data%): Broker::QueryResult %{ if ( ! broker_mgr->Enabled() ) @@ -527,14 +380,7 @@ function Broker::lookup%(h: opaque of Broker::Handle, return 0; %} -## Check if a data store contains a given key. -## -## h: the handle of the store to query. -## -## k: the key to check for existence. -## -## Returns: the result of the query (uses :bro:see:`Broker::BOOL`). -function Broker::exists%(h: opaque of Broker::Handle, +function Broker::__exists%(h: opaque of Broker::Handle, k: Broker::Data%): Broker::QueryResult %{ if ( ! broker_mgr->Enabled() ) @@ -557,12 +403,7 @@ function Broker::exists%(h: opaque of Broker::Handle, return 0; %} -## Retrieve all keys in a data store. -## -## h: the handle of the store to query. -## -## Returns: the result of the query (uses :bro:see:`Broker::VECTOR`). -function Broker::keys%(h: opaque of Broker::Handle%): Broker::QueryResult +function Broker::__keys%(h: opaque of Broker::Handle%): Broker::QueryResult %{ double timeout; bro_broker::StoreQueryCallback* cb; @@ -575,12 +416,7 @@ function Broker::keys%(h: opaque of Broker::Handle%): Broker::QueryResult return 0; %} -## Get the number of key-value pairs in a data store. -## -## h: the handle of the store to query. -## -## Returns: the result of the query (uses :bro:see:`Broker::COUNT`). -function Broker::size%(h: opaque of Broker::Handle%): Broker::QueryResult +function Broker::__size%(h: opaque of Broker::Handle%): Broker::QueryResult %{ if ( ! broker_mgr->Enabled() ) return bro_broker::query_result(); diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index b5107374d1..7e55509a86 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2016-04-22-23-21-01 +#open 2016-04-26-21-21-19 #fields name #types string scripts/base/init-bare.bro @@ -17,7 +17,10 @@ scripts/base/init-bare.bro build/scripts/base/bif/event.bif.bro scripts/base/frameworks/broker/__load__.bro scripts/base/frameworks/broker/main.bro + build/scripts/base/bif/comm.bif.bro + build/scripts/base/bif/messaging.bif.bro scripts/base/frameworks/broker/store.bro + build/scripts/base/bif/store.bif.bro scripts/base/frameworks/logging/__load__.bro scripts/base/frameworks/logging/main.bro build/scripts/base/bif/logging.bif.bro @@ -51,10 +54,7 @@ scripts/base/init-bare.bro build/scripts/base/bif/bloom-filter.bif.bro build/scripts/base/bif/cardinality-counter.bif.bro build/scripts/base/bif/top-k.bif.bro - build/scripts/base/bif/comm.bif.bro build/scripts/base/bif/data.bif.bro - build/scripts/base/bif/messaging.bif.bro - build/scripts/base/bif/store.bif.bro build/scripts/base/bif/plugins/__load__.bro build/scripts/base/bif/plugins/Bro_ARP.events.bif.bro build/scripts/base/bif/plugins/Bro_AYIYA.events.bif.bro @@ -131,4 +131,4 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro scripts/policy/misc/loaded-scripts.bro scripts/base/utils/paths.bro -#close 2016-04-22-23-21-01 +#close 2016-04-26-21-21-19 diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index c2db0ad12e..075a7c1389 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2016-04-22-23-21-18 +#open 2016-04-26-21-21-31 #fields name #types string scripts/base/init-bare.bro @@ -17,7 +17,10 @@ scripts/base/init-bare.bro build/scripts/base/bif/event.bif.bro scripts/base/frameworks/broker/__load__.bro scripts/base/frameworks/broker/main.bro + build/scripts/base/bif/comm.bif.bro + build/scripts/base/bif/messaging.bif.bro scripts/base/frameworks/broker/store.bro + build/scripts/base/bif/store.bif.bro scripts/base/frameworks/logging/__load__.bro scripts/base/frameworks/logging/main.bro build/scripts/base/bif/logging.bif.bro @@ -51,10 +54,7 @@ scripts/base/init-bare.bro build/scripts/base/bif/bloom-filter.bif.bro build/scripts/base/bif/cardinality-counter.bif.bro build/scripts/base/bif/top-k.bif.bro - build/scripts/base/bif/comm.bif.bro build/scripts/base/bif/data.bif.bro - build/scripts/base/bif/messaging.bif.bro - build/scripts/base/bif/store.bif.bro build/scripts/base/bif/plugins/__load__.bro build/scripts/base/bif/plugins/Bro_ARP.events.bif.bro build/scripts/base/bif/plugins/Bro_AYIYA.events.bif.bro @@ -302,4 +302,4 @@ scripts/base/init-default.bro scripts/base/misc/find-checksum-offloading.bro scripts/base/misc/find-filtered-trace.bro scripts/policy/misc/loaded-scripts.bro -#close 2016-04-22-23-21-18 +#close 2016-04-26-21-21-31 diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index d4bd063e12..a39e6d8dd8 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -230,7 +230,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461367323.154279, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461705704.402549, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Communication::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Conn::LOG)) -> @@ -351,7 +351,7 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461367323.154279, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461705704.402549, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(NetControl::check_plugins, , ()) -> 0.000000 MetaHookPost CallFunction(NetControl::init, , ()) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, , ()) -> @@ -566,6 +566,7 @@ 0.000000 MetaHookPost LoadFile(base<...>/bro.bif) -> -1 0.000000 MetaHookPost LoadFile(base<...>/broker) -> -1 0.000000 MetaHookPost LoadFile(base<...>/cluster) -> -1 +0.000000 MetaHookPost LoadFile(base<...>/comm.bif) -> -1 0.000000 MetaHookPost LoadFile(base<...>/communication) -> -1 0.000000 MetaHookPost LoadFile(base<...>/conn) -> -1 0.000000 MetaHookPost LoadFile(base<...>/conn-ids) -> -1 @@ -596,6 +597,7 @@ 0.000000 MetaHookPost LoadFile(base<...>/logging) -> -1 0.000000 MetaHookPost LoadFile(base<...>/logging.bif) -> -1 0.000000 MetaHookPost LoadFile(base<...>/main) -> -1 +0.000000 MetaHookPost LoadFile(base<...>/messaging.bif) -> -1 0.000000 MetaHookPost LoadFile(base<...>/modbus) -> -1 0.000000 MetaHookPost LoadFile(base<...>/mysql) -> -1 0.000000 MetaHookPost LoadFile(base<...>/netcontrol) -> -1 @@ -623,6 +625,7 @@ 0.000000 MetaHookPost LoadFile(base<...>/software) -> -1 0.000000 MetaHookPost LoadFile(base<...>/ssh) -> -1 0.000000 MetaHookPost LoadFile(base<...>/ssl) -> -1 +0.000000 MetaHookPost LoadFile(base<...>/store.bif) -> -1 0.000000 MetaHookPost LoadFile(base<...>/strings) -> -1 0.000000 MetaHookPost LoadFile(base<...>/strings.bif) -> -1 0.000000 MetaHookPost LoadFile(base<...>/sumstats) -> -1 @@ -870,7 +873,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461367323.154279, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461705704.402549, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Communication::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Conn::LOG)) @@ -991,7 +994,7 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461367323.154279, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461705704.402549, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(NetControl::check_plugins, , ()) 0.000000 MetaHookPre CallFunction(NetControl::init, , ()) 0.000000 MetaHookPre CallFunction(Notice::want_pp, , ()) @@ -1206,6 +1209,7 @@ 0.000000 MetaHookPre LoadFile(base<...>/bro.bif) 0.000000 MetaHookPre LoadFile(base<...>/broker) 0.000000 MetaHookPre LoadFile(base<...>/cluster) +0.000000 MetaHookPre LoadFile(base<...>/comm.bif) 0.000000 MetaHookPre LoadFile(base<...>/communication) 0.000000 MetaHookPre LoadFile(base<...>/conn) 0.000000 MetaHookPre LoadFile(base<...>/conn-ids) @@ -1236,6 +1240,7 @@ 0.000000 MetaHookPre LoadFile(base<...>/logging) 0.000000 MetaHookPre LoadFile(base<...>/logging.bif) 0.000000 MetaHookPre LoadFile(base<...>/main) +0.000000 MetaHookPre LoadFile(base<...>/messaging.bif) 0.000000 MetaHookPre LoadFile(base<...>/modbus) 0.000000 MetaHookPre LoadFile(base<...>/mysql) 0.000000 MetaHookPre LoadFile(base<...>/netcontrol) @@ -1263,6 +1268,7 @@ 0.000000 MetaHookPre LoadFile(base<...>/software) 0.000000 MetaHookPre LoadFile(base<...>/ssh) 0.000000 MetaHookPre LoadFile(base<...>/ssl) +0.000000 MetaHookPre LoadFile(base<...>/store.bif) 0.000000 MetaHookPre LoadFile(base<...>/strings) 0.000000 MetaHookPre LoadFile(base<...>/strings.bif) 0.000000 MetaHookPre LoadFile(base<...>/sumstats) @@ -1509,7 +1515,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1461367323.154279, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1461705704.402549, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG) @@ -1630,7 +1636,7 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1461367323.154279, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1461705704.402549, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction NetControl::check_plugins() 0.000000 | HookCallFunction NetControl::init() 0.000000 | HookCallFunction Notice::want_pp() diff --git a/testing/btest/broker/remote_event.test b/testing/btest/broker/remote_event.test index e18fc3715f..bd3c087d9a 100644 --- a/testing/btest/broker/remote_event.test +++ b/testing/btest/broker/remote_event.test @@ -40,7 +40,7 @@ event event_handler(msg: string, n: count) event auto_event_handler(msg, n); local args = Broker::event_args(event_handler, "pong", n); - Broker::event("bro/event/my_topic", args); + Broker::send_event("bro/event/my_topic", args); } @TEST-END-FILE @@ -68,7 +68,7 @@ event Broker::outgoing_connection_established(peer_address: string, { print "Broker::outgoing_connection_established", peer_address, peer_port; local args = Broker::event_args(event_handler, "ping", event_count); - Broker::event("bro/event/hi", args); + Broker::send_event("bro/event/hi", args); ++event_count; } @@ -82,7 +82,7 @@ event event_handler(msg: string, n: count) { print "got event msg", msg, n; local args = Broker::event_args(event_handler, "ping", event_count); - Broker::event("bro/event/hi", args); + Broker::send_event("bro/event/hi", args); ++event_count; } diff --git a/testing/btest/broker/remote_print.test b/testing/btest/broker/remote_print.test index a3c06599ae..9cfcc44ca9 100644 --- a/testing/btest/broker/remote_print.test +++ b/testing/btest/broker/remote_print.test @@ -35,7 +35,7 @@ event Broker::print_handler(msg: string) return; } - Broker::print("bro/print/my_topic", fmt("pong %d", messages_sent)); + Broker::send_print("bro/print/my_topic", fmt("pong %d", messages_sent)); ++messages_sent; } @@ -62,7 +62,7 @@ event Broker::outgoing_connection_established(peer_address: string, peer_name: string) { print "Broker::outgoing_connection_established", peer_address, peer_port; - Broker::print("bro/print/hi", fmt("ping %d", messages_sent)); + Broker::send_print("bro/print/hi", fmt("ping %d", messages_sent)); ++messages_sent; } @@ -76,7 +76,7 @@ event Broker::print_handler(msg: string) { ++messages_recv; print "got print msg", msg; - Broker::print("bro/print/hi", fmt("ping %d", messages_sent)); + Broker::send_print("bro/print/hi", fmt("ping %d", messages_sent)); ++messages_sent; } diff --git a/testing/btest/scripts/base/frameworks/netcontrol/acld-hook.bro b/testing/btest/scripts/base/frameworks/netcontrol/acld-hook.bro index 31d5f4df96..779799ab4f 100644 --- a/testing/btest/scripts/base/frameworks/netcontrol/acld-hook.bro +++ b/testing/btest/scripts/base/frameworks/netcontrol/acld-hook.bro @@ -105,14 +105,14 @@ event NetControl::acld_add_rule(id: count, r: NetControl::Rule, ar: NetControl:: { print "add_rule", id, r$entity, r$ty, ar; - Broker::event("bro/event/netcontroltest", Broker::event_args(NetControl::acld_rule_added, id, r, ar$command)); + Broker::send_event("bro/event/netcontroltest", Broker::event_args(NetControl::acld_rule_added, id, r, ar$command)); } event NetControl::acld_remove_rule(id: count, r: NetControl::Rule, ar: NetControl::AclRule) { print "remove_rule", id, r$entity, r$ty, ar; - Broker::event("bro/event/netcontroltest", Broker::event_args(NetControl::acld_rule_removed, id, r, ar$command)); + Broker::send_event("bro/event/netcontroltest", Broker::event_args(NetControl::acld_rule_removed, id, r, ar$command)); if ( r$cid == 4 ) terminate(); diff --git a/testing/btest/scripts/base/frameworks/netcontrol/acld.bro b/testing/btest/scripts/base/frameworks/netcontrol/acld.bro index 89743296b1..83a9cfc1af 100644 --- a/testing/btest/scripts/base/frameworks/netcontrol/acld.bro +++ b/testing/btest/scripts/base/frameworks/netcontrol/acld.bro @@ -98,14 +98,14 @@ event NetControl::acld_add_rule(id: count, r: NetControl::Rule, ar: NetControl:: { print "add_rule", id, r$entity, r$ty, ar; - Broker::event("bro/event/netcontroltest", Broker::event_args(NetControl::acld_rule_added, id, r, ar$command)); + Broker::send_event("bro/event/netcontroltest", Broker::event_args(NetControl::acld_rule_added, id, r, ar$command)); } event NetControl::acld_remove_rule(id: count, r: NetControl::Rule, ar: NetControl::AclRule) { print "remove_rule", id, r$entity, r$ty, ar; - Broker::event("bro/event/netcontroltest", Broker::event_args(NetControl::acld_rule_removed, id, r, ar$command)); + Broker::send_event("bro/event/netcontroltest", Broker::event_args(NetControl::acld_rule_removed, id, r, ar$command)); if ( r$cid == 4 ) terminate(); diff --git a/testing/btest/scripts/base/frameworks/netcontrol/broker.bro b/testing/btest/scripts/base/frameworks/netcontrol/broker.bro index 652f89f4a5..4dbf3a09d2 100644 --- a/testing/btest/scripts/base/frameworks/netcontrol/broker.bro +++ b/testing/btest/scripts/base/frameworks/netcontrol/broker.bro @@ -89,15 +89,15 @@ event NetControl::broker_add_rule(id: count, r: NetControl::Rule) { print "add_rule", id, r$entity, r$ty; - Broker::event("bro/event/netcontroltest", Broker::event_args(NetControl::broker_rule_added, id, r, "")); + Broker::send_event("bro/event/netcontroltest", Broker::event_args(NetControl::broker_rule_added, id, r, "")); } event NetControl::broker_remove_rule(id: count, r: NetControl::Rule) { print "remove_rule", id, r$entity, r$ty; - Broker::event("bro/event/netcontroltest", Broker::event_args(NetControl::broker_rule_timeout, id, r, NetControl::FlowInfo())); - Broker::event("bro/event/netcontroltest", Broker::event_args(NetControl::broker_rule_removed, id, r, "")); + Broker::send_event("bro/event/netcontroltest", Broker::event_args(NetControl::broker_rule_timeout, id, r, NetControl::FlowInfo())); + Broker::send_event("bro/event/netcontroltest", Broker::event_args(NetControl::broker_rule_removed, id, r, "")); if ( r$cid == 3 ) terminate(); diff --git a/testing/btest/scripts/base/frameworks/openflow/broker-basic.bro b/testing/btest/scripts/base/frameworks/openflow/broker-basic.bro index ed1afb4c3e..014f07390b 100644 --- a/testing/btest/scripts/base/frameworks/openflow/broker-basic.bro +++ b/testing/btest/scripts/base/frameworks/openflow/broker-basic.bro @@ -104,8 +104,8 @@ function got_message() event OpenFlow::broker_flow_mod(name: string, dpid: count, match: OpenFlow::ofp_match, flow_mod: OpenFlow::ofp_flow_mod) { print "got flow_mod", dpid, match, flow_mod; - Broker::event("bro/event/openflow", Broker::event_args(OpenFlow::flow_mod_success, name, match, flow_mod, "")); - Broker::event("bro/event/openflow", Broker::event_args(OpenFlow::flow_mod_failure, name, match, flow_mod, "")); + Broker::send_event("bro/event/openflow", Broker::event_args(OpenFlow::flow_mod_success, name, match, flow_mod, "")); + Broker::send_event("bro/event/openflow", Broker::event_args(OpenFlow::flow_mod_failure, name, match, flow_mod, "")); got_message(); } From 4df948f3c8ec7c6967ef3d5a8621107d77543f8e Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Tue, 26 Apr 2016 22:01:09 -0500 Subject: [PATCH 62/84] Add script wrapper functions for broker data BIFs --- scripts/base/frameworks/broker/store.bro | 702 ++++++++++++++++++ src/broker/data.bif | 434 ++--------- .../canonified_loaded_scripts.log | 6 +- .../canonified_loaded_scripts.log | 6 +- testing/btest/Baseline/plugins.hooks/output | 14 +- 5 files changed, 769 insertions(+), 393 deletions(-) diff --git a/scripts/base/frameworks/broker/store.bro b/scripts/base/frameworks/broker/store.bro index 36565d72aa..f93b701d1c 100644 --- a/scripts/base/frameworks/broker/store.bro +++ b/scripts/base/frameworks/broker/store.bro @@ -1,6 +1,7 @@ ##! Various data structure definitions for use with Bro's communication system. @load ./main +@load base/bif/data.bif module Broker; @@ -282,6 +283,443 @@ export { ## Returns: the result of the query (uses :bro:see:`Broker::COUNT`). global size: function(h: opaque of Broker::Handle): QueryResult; + ########################## + # data API # + ########################## + + ## Convert any Bro value to communication data. + ## + ## d: any Bro value to attempt to convert (not all types are supported). + ## + ## Returns: the converted communication data. The returned record's optional + ## field will not be set if the conversion was not possible (this can + ## happen if the Bro data type does not support being converted to + ## communication data). + global data: function(d: any): Broker::Data; + + ## Retrieve the type of data associated with communication data. + ## + ## d: the communication data. + ## + ## Returns: the data type associated with the communication data. + global data_type: function(d: Broker::Data): Broker::DataType; + + ## Convert communication data with a type of :bro:see:`Broker::BOOL` to + ## an actual Bro value. + ## + ## d: the communication data to convert. + ## + ## Returns: the value retrieved from the communication data. + global refine_to_bool: function(d: Broker::Data): bool; + + ## Convert communication data with a type of :bro:see:`Broker::INT` to + ## an actual Bro value. + ## + ## d: the communication data to convert. + ## + ## Returns: the value retrieved from the communication data. + global refine_to_int: function(d: Broker::Data): int; + + ## Convert communication data with a type of :bro:see:`Broker::COUNT` to + ## an actual Bro value. + ## + ## d: the communication data to convert. + ## + ## Returns: the value retrieved from the communication data. + global refine_to_count: function(d: Broker::Data): count; + + ## Convert communication data with a type of :bro:see:`Broker::DOUBLE` to + ## an actual Bro value. + ## + ## d: the communication data to convert. + ## + ## Returns: the value retrieved from the communication data. + global refine_to_double: function(d: Broker::Data): double; + + ## Convert communication data with a type of :bro:see:`Broker::STRING` to + ## an actual Bro value. + ## + ## d: the communication data to convert. + ## + ## Returns: the value retrieved from the communication data. + global refine_to_string: function(d: Broker::Data): string; + + ## Convert communication data with a type of :bro:see:`Broker::ADDR` to + ## an actual Bro value. + ## + ## d: the communication data to convert. + ## + ## Returns: the value retrieved from the communication data. + global refine_to_addr: function(d: Broker::Data): addr; + + ## Convert communication data with a type of :bro:see:`Broker::SUBNET` to + ## an actual Bro value. + ## + ## d: the communication data to convert. + ## + ## Returns: the value retrieved from the communication data. + global refine_to_subnet: function(d: Broker::Data): subnet; + + ## Convert communication data with a type of :bro:see:`Broker::PORT` to + ## an actual Bro value. + ## + ## d: the communication data to convert. + ## + ## Returns: the value retrieved from the communication data. + global refine_to_port: function(d: Broker::Data): port; + + ## Convert communication data with a type of :bro:see:`Broker::TIME` to + ## an actual Bro value. + ## + ## d: the communication data to convert. + ## + ## Returns: the value retrieved from the communication data. + global refine_to_time: function(d: Broker::Data): time; + + ## Convert communication data with a type of :bro:see:`Broker::INTERVAL` to + ## an actual Bro value. + ## + ## d: the communication data to convert. + ## + ## Returns: the value retrieved from the communication data. + global refine_to_interval: function(d: Broker::Data): interval; + + ## Convert communication data with a type of :bro:see:`Broker::ENUM` to + ## the name of the enum value. :bro:see:`lookup_ID` may be used to convert + ## the name to the actual enum value. + ## + ## d: the communication data to convert. + ## + ## Returns: the enum name retrieved from the communication data. + global refine_to_enum_name: function(d: Broker::Data): string; + + ## Create communication data of type "set". + global set_create: function(): Broker::Data; + + ## Remove all elements within a set. + ## + ## s: the set to clear. + ## + ## Returns: always true. + global set_clear: function(s: Broker::Data): bool; + + ## Get the number of elements within a set. + ## + ## s: the set to query. + ## + ## Returns: the number of elements in the set. + global set_size: function(s: Broker::Data): count; + + ## Check if a set contains a particular element. + ## + ## s: the set to query. + ## + ## key: the element to check for existence. + ## + ## Returns: true if the key exists in the set. + global set_contains: function(s: Broker::Data, key: Broker::Data): bool; + + ## Insert an element into a set. + ## + ## s: the set to modify. + ## + ## key: the element to insert. + ## + ## Returns: true if the key was inserted, or false if it already existed. + global set_insert: function(s: Broker::Data, key: Broker::Data): bool; + + ## Remove an element from a set. + ## + ## s: the set to modify. + ## + ## key: the element to remove. + ## + ## Returns: true if the element existed in the set and is now removed. + global set_remove: function(s: Broker::Data, key: Broker::Data): bool; + + ## Create an iterator for a set. Note that this makes a copy of the set + ## internally to ensure the iterator is always valid. + ## + ## s: the set to iterate over. + ## + ## Returns: an iterator. + global set_iterator: function(s: Broker::Data): opaque of Broker::SetIterator; + + ## Check if there are no more elements to iterate over. + ## + ## it: an iterator. + ## + ## Returns: true if there are no more elements to iterator over, i.e. + ## the iterator is one-past-the-final-element. + global set_iterator_last: function(it: opaque of Broker::SetIterator): bool; + + ## Advance an iterator. + ## + ## it: an iterator. + ## + ## Returns: true if the iterator, after advancing, still references an element + ## in the collection. False if the iterator, after advancing, is + ## one-past-the-final-element. + global set_iterator_next: function(it: opaque of Broker::SetIterator): bool; + + ## Retrieve the data at an iterator's current position. + ## + ## it: an iterator. + ## + ## Returns: element in the collection that the iterator currently references. + global set_iterator_value: function(it: opaque of Broker::SetIterator): Broker::Data; + + ## Create communication data of type "table". + global table_create: function(): Broker::Data; + + ## Remove all elements within a table. + ## + ## t: the table to clear. + ## + ## Returns: always true. + global table_clear: function(t: Broker::Data): bool; + + ## Get the number of elements within a table. + ## + ## t: the table to query. + ## + ## Returns: the number of elements in the table. + global table_size: function(t: Broker::Data): count; + + ## Check if a table contains a particular key. + ## + ## t: the table to query. + ## + ## key: the key to check for existence. + ## + ## Returns: true if the key exists in the table. + global table_contains: function(t: Broker::Data, key: Broker::Data): bool; + + ## Insert a key-value pair into a table. + ## + ## t: the table to modify. + ## + ## key: the key at which to insert the value. + ## + ## val: the value to insert. + ## + ## Returns: true if the key-value pair was inserted, or false if the key + ## already existed in the table. + global table_insert: function(t: Broker::Data, key: Broker::Data, val: Broker::Data): Broker::Data; + + ## Remove a key-value pair from a table. + ## + ## t: the table to modify. + ## + ## key: the key to remove from the table. + ## + ## Returns: the value associated with the key. If the key did not exist, then + ## the optional field of the returned record is not set. + global table_remove: function(t: Broker::Data, key: Broker::Data): Broker::Data; + + ## Retrieve a value from a table. + ## + ## t: the table to query. + ## + ## key: the key to lookup. + ## + ## Returns: the value associated with the key. If the key did not exist, then + ## the optional field of the returned record is not set. + global table_lookup: function(t: Broker::Data, key: Broker::Data): Broker::Data; + + ## Create an iterator for a table. Note that this makes a copy of the table + ## internally to ensure the iterator is always valid. + ## + ## t: the table to iterate over. + ## + ## Returns: an iterator. + global table_iterator: function(t: Broker::Data): opaque of Broker::TableIterator; + + ## Check if there are no more elements to iterate over. + ## + ## it: an iterator. + ## + ## Returns: true if there are no more elements to iterator over, i.e. + ## the iterator is one-past-the-final-element. + global table_iterator_last: function(it: opaque of Broker::TableIterator): bool; + + ## Advance an iterator. + ## + ## it: an iterator. + ## + ## Returns: true if the iterator, after advancing, still references an element + ## in the collection. False if the iterator, after advancing, is + ## one-past-the-final-element. + global table_iterator_next: function(it: opaque of Broker::TableIterator): bool; + + ## Retrieve the data at an iterator's current position. + ## + ## it: an iterator. + ## + ## Returns: element in the collection that the iterator currently references. + global table_iterator_value: function(it: opaque of Broker::TableIterator): Broker::TableItem; + + ## Create communication data of type "vector". + global vector_create: function(): Broker::Data; + + ## Remove all elements within a vector. + ## + ## v: the vector to clear. + ## + ## Returns: always true. + global vector_clear: function(v: Broker::Data): bool; + + ## Get the number of elements within a vector. + ## + ## v: the vector to query. + ## + ## Returns: the number of elements in the vector. + global vector_size: function(v: Broker::Data): count; + + ## Insert an element into a vector at a particular position, possibly displacing + ## existing elements (insertion always grows the size of the vector by one). + ## + ## v: the vector to modify. + ## + ## d: the element to insert. + ## + ## idx: the index at which to insert the data. If it is greater than the + ## current size of the vector, the element is inserted at the end. + ## + ## Returns: always true. + global vector_insert: function(v: Broker::Data, d: Broker::Data, idx: count): bool; + + ## Replace an element in a vector at a particular position. + ## + ## v: the vector to modify. + ## + ## d: the element to insert. + ## + ## idx: the index to replace. + ## + ## Returns: the value that was just evicted. If the index was larger than any + ## valid index, the optional field of the returned record is not set. + global vector_replace: function(v: Broker::Data, d: Broker::Data, idx: count): Broker::Data; + + ## Remove an element from a vector at a particular position. + ## + ## v: the vector to modify. + ## + ## idx: the index to remove. + ## + ## Returns: the value that was just evicted. If the index was larger than any + ## valid index, the optional field of the returned record is not set. + global vector_remove: function(v: Broker::Data, idx: count): Broker::Data; + + ## Lookup an element in a vector at a particular position. + ## + ## v: the vector to query. + ## + ## idx: the index to lookup. + ## + ## Returns: the value at the index. If the index was larger than any + ## valid index, the optional field of the returned record is not set. + global vector_lookup: function(v: Broker::Data, idx: count): Broker::Data; + + ## Create an iterator for a vector. Note that this makes a copy of the vector + ## internally to ensure the iterator is always valid. + ## + ## v: the vector to iterate over. + ## + ## Returns: an iterator. + global vector_iterator: function(v: Broker::Data): opaque of Broker::VectorIterator; + + ## Check if there are no more elements to iterate over. + ## + ## it: an iterator. + ## + ## Returns: true if there are no more elements to iterator over, i.e. + ## the iterator is one-past-the-final-element. + global vector_iterator_last: function(it: opaque of Broker::VectorIterator): bool; + + ## Advance an iterator. + ## + ## it: an iterator. + ## + ## Returns: true if the iterator, after advancing, still references an element + ## in the collection. False if the iterator, after advancing, is + ## one-past-the-final-element. + global vector_iterator_next: function(it: opaque of Broker::VectorIterator): bool; + + ## Retrieve the data at an iterator's current position. + ## + ## it: an iterator. + ## + ## Returns: element in the collection that the iterator currently references. + global vector_iterator_value: function(it: opaque of Broker::VectorIterator): Broker::Data; + + ## Create communication data of type "record". + ## + ## sz: the number of fields in the record. + ## + ## Returns: record data, with all fields uninitialized. + global record_create: function(sz: count): Broker::Data; + + ## Get the number of fields within a record. + ## + ## r: the record to query. + ## + ## Returns: the number of fields in the record. + global record_size: function(r: Broker::Data): count; + + ## Replace a field in a record at a particular position. + ## + ## r: the record to modify. + ## + ## d: the new field value to assign. + ## + ## idx: the index to replace. + ## + ## Returns: false if the index was larger than any valid index, else true. + global record_assign: function(r: Broker::Data, d: Broker::Data, idx: count): bool; + + ## Lookup a field in a record at a particular position. + ## + ## r: the record to query. + ## + ## idx: the index to lookup. + ## + ## Returns: the value at the index. The optional field of the returned record + ## may not be set if the field of the record has no value or if the + ## index was not valid. + global record_lookup: function(r: Broker::Data, idx: count): Broker::Data; + + ## Create an iterator for a record. Note that this makes a copy of the record + ## internally to ensure the iterator is always valid. + ## + ## r: the record to iterate over. + ## + ## Returns: an iterator. + global record_iterator: function(r: Broker::Data): opaque of Broker::RecordIterator; + + ## Check if there are no more elements to iterate over. + ## + ## it: an iterator. + ## + ## Returns: true if there are no more elements to iterator over, i.e. + ## the iterator is one-past-the-final-element. + global record_iterator_last: function(it: opaque of Broker::RecordIterator): bool; + + ## Advance an iterator. + ## + ## it: an iterator. + ## + ## Returns: true if the iterator, after advancing, still references an element + ## in the collection. False if the iterator, after advancing, is + ## one-past-the-final-element. + global record_iterator_next: function(it: opaque of Broker::RecordIterator): bool; + + ## Retrieve the data at an iterator's current position. + ## + ## it: an iterator. + ## + ## Returns: element in the collection that the iterator currently references. + global record_iterator_value: function(it: opaque of Broker::RecordIterator): Broker::Data; } @load base/bif/store.bif @@ -393,3 +831,267 @@ function size(h: opaque of Broker::Handle): QueryResult return __size(h); } +function data(d: any): Broker::Data + { + return __data(d); + } + +function data_type(d: Broker::Data): Broker::DataType + { + return __data_type(d); + } + +function refine_to_bool(d: Broker::Data): bool + { + return __refine_to_bool(d); + } + +function refine_to_int(d: Broker::Data): int + { + return __refine_to_int(d); + } + +function refine_to_count(d: Broker::Data): count + { + return __refine_to_count(d); + } + +function refine_to_double(d: Broker::Data): double + { + return __refine_to_double(d); + } + +function refine_to_string(d: Broker::Data): string + { + return __refine_to_string(d); + } + +function refine_to_addr(d: Broker::Data): addr + { + return __refine_to_addr(d); + } + +function refine_to_subnet(d: Broker::Data): subnet + { + return __refine_to_subnet(d); + } + +function refine_to_port(d: Broker::Data): port + { + return __refine_to_port(d); + } + +function refine_to_time(d: Broker::Data): time + { + return __refine_to_time(d); + } + +function refine_to_interval(d: Broker::Data): interval + { + return __refine_to_interval(d); + } + +function refine_to_enum_name(d: Broker::Data): string + { + return __refine_to_enum_name(d); + } + +function set_create(): Broker::Data + { + return __set_create(); + } + +function set_clear(s: Broker::Data): bool + { + return __set_clear(s); + } + +function set_size(s: Broker::Data): count + { + return __set_size(s); + } + +function set_contains(s: Broker::Data, key: Broker::Data): bool + { + return __set_contains(s, key); + } + +function set_insert(s: Broker::Data, key: Broker::Data): bool + { + return __set_insert(s, key); + } + +function set_remove(s: Broker::Data, key: Broker::Data): bool + { + return __set_remove(s, key); + } + +function set_iterator(s: Broker::Data): opaque of Broker::SetIterator + { + return __set_iterator(s); + } + +function set_iterator_last(it: opaque of Broker::SetIterator): bool + { + return __set_iterator_last(it); + } + +function set_iterator_next(it: opaque of Broker::SetIterator): bool + { + return __set_iterator_next(it); + } + +function set_iterator_value(it: opaque of Broker::SetIterator): Broker::Data + { + return __set_iterator_value(it); + } + +function table_create(): Broker::Data + { + return __table_create(); + } + +function table_clear(t: Broker::Data): bool + { + return __table_clear(t); + } + +function table_size(t: Broker::Data): count + { + return __table_size(t); + } + +function table_contains(t: Broker::Data, key: Broker::Data): bool + { + return __table_contains(t, key); + } + +function table_insert(t: Broker::Data, key: Broker::Data, val: Broker::Data): Broker::Data + { + return __table_insert(t, key, val); + } + +function table_remove(t: Broker::Data, key: Broker::Data): Broker::Data + { + return __table_remove(t, key); + } + +function table_lookup(t: Broker::Data, key: Broker::Data): Broker::Data + { + return __table_lookup(t, key); + } + +function table_iterator(t: Broker::Data): opaque of Broker::TableIterator + { + return __table_iterator(t); + } + +function table_iterator_last(it: opaque of Broker::TableIterator): bool + { + return __table_iterator_last(it); + } + +function table_iterator_next(it: opaque of Broker::TableIterator): bool + { + return __table_iterator_next(it); + } + +function table_iterator_value(it: opaque of Broker::TableIterator): Broker::TableItem + { + return __table_iterator_value(it); + } + +function vector_create(): Broker::Data + { + return __vector_create(); + } + +function vector_clear(v: Broker::Data): bool + { + return __vector_clear(v); + } + +function vector_size(v: Broker::Data): count + { + return __vector_size(v); + } + +function vector_insert(v: Broker::Data, d: Broker::Data, idx: count): bool + { + return __vector_insert(v, d, idx); + } + +function vector_replace(v: Broker::Data, d: Broker::Data, idx: count): Broker::Data + { + return __vector_replace(v, d, idx); + } + +function vector_remove(v: Broker::Data, idx: count): Broker::Data + { + return __vector_remove(v, idx); + } + +function vector_lookup(v: Broker::Data, idx: count): Broker::Data + { + return __vector_lookup(v, idx); + } + +function vector_iterator(v: Broker::Data): opaque of Broker::VectorIterator + { + return __vector_iterator(v); + } + +function vector_iterator_last(it: opaque of Broker::VectorIterator): bool + { + return __vector_iterator_last(it); + } + +function vector_iterator_next(it: opaque of Broker::VectorIterator): bool + { + return __vector_iterator_next(it); + } + +function vector_iterator_value(it: opaque of Broker::VectorIterator): Broker::Data + { + return __vector_iterator_value(it); + } + +function record_create(sz: count): Broker::Data + { + return __record_create(sz); + } + +function record_size(r: Broker::Data): count + { + return __record_size(r); + } + +function record_assign(r: Broker::Data, d: Broker::Data, idx: count): bool + { + return __record_assign(r, d, idx); + } + +function record_lookup(r: Broker::Data, idx: count): Broker::Data + { + return __record_lookup(r, idx); + } + +function record_iterator(r: Broker::Data): opaque of Broker::RecordIterator + { + return __record_iterator(r); + } + +function record_iterator_last(it: opaque of Broker::RecordIterator): bool + { + return __record_iterator_last(it); + } + +function record_iterator_next(it: opaque of Broker::RecordIterator): bool + { + return __record_iterator_next(it); + } + +function record_iterator_value(it: opaque of Broker::RecordIterator): Broker::Data + { + return __record_iterator_value(it); + } diff --git a/src/broker/data.bif b/src/broker/data.bif index d4744f07c6..1788931d86 100644 --- a/src/broker/data.bif +++ b/src/broker/data.bif @@ -31,93 +31,44 @@ type Broker::Data: record; type Broker::TableItem: record; -## Convert any Bro value to communication data. -## -## d: any Bro value to attempt to convert (not all types are supported). -## -## Returns: the converted communication data. The returned record's optional -## field will not be set if the conversion was not possible (this can -## happen if the Bro data type does not support being converted to -## communication data). -function Broker::data%(d: any%): Broker::Data +function Broker::__data%(d: any%): Broker::Data %{ return bro_broker::make_data_val(d); %} -## Retrieve the type of data associated with communication data. -## -## d: the communication data. -## -## Returns: the data type associated with the communication data. -function Broker::data_type%(d: Broker::Data%): Broker::DataType +function Broker::__data_type%(d: Broker::Data%): Broker::DataType %{ return bro_broker::get_data_type(d->AsRecordVal(), frame); %} -## Convert communication data with a type of :bro:see:`Broker::BOOL` to -## an actual Bro value. -## -## d: the communication data to convert. -## -## Returns: the value retrieved from the communication data. -function Broker::refine_to_bool%(d: Broker::Data%): bool +function Broker::__refine_to_bool%(d: Broker::Data%): bool %{ return bro_broker::refine(d->AsRecordVal(), TYPE_BOOL, frame); %} -## Convert communication data with a type of :bro:see:`Broker::INT` to -## an actual Bro value. -## -## d: the communication data to convert. -## -## Returns: the value retrieved from the communication data. -function Broker::refine_to_int%(d: Broker::Data%): int +function Broker::__refine_to_int%(d: Broker::Data%): int %{ return bro_broker::refine(d->AsRecordVal(), TYPE_INT, frame); %} -## Convert communication data with a type of :bro:see:`Broker::COUNT` to -## an actual Bro value. -## -## d: the communication data to convert. -## -## Returns: the value retrieved from the communication data. -function Broker::refine_to_count%(d: Broker::Data%): count +function Broker::__refine_to_count%(d: Broker::Data%): count %{ return bro_broker::refine(d->AsRecordVal(), TYPE_COUNT, frame); %} -## Convert communication data with a type of :bro:see:`Broker::DOUBLE` to -## an actual Bro value. -## -## d: the communication data to convert. -## -## Returns: the value retrieved from the communication data. -function Broker::refine_to_double%(d: Broker::Data%): double +function Broker::__refine_to_double%(d: Broker::Data%): double %{ return bro_broker::refine(d->AsRecordVal(), TYPE_DOUBLE, frame); %} -## Convert communication data with a type of :bro:see:`Broker::STRING` to -## an actual Bro value. -## -## d: the communication data to convert. -## -## Returns: the value retrieved from the communication data. -function Broker::refine_to_string%(d: Broker::Data%): string +function Broker::__refine_to_string%(d: Broker::Data%): string %{ return new StringVal(bro_broker::require_data_type(d->AsRecordVal(), TYPE_STRING, frame)); %} -## Convert communication data with a type of :bro:see:`Broker::ADDR` to -## an actual Bro value. -## -## d: the communication data to convert. -## -## Returns: the value retrieved from the communication data. -function Broker::refine_to_addr%(d: Broker::Data%): addr +function Broker::__refine_to_addr%(d: Broker::Data%): addr %{ auto& a = bro_broker::require_data_type(d->AsRecordVal(), TYPE_ADDR, frame); @@ -125,13 +76,7 @@ function Broker::refine_to_addr%(d: Broker::Data%): addr return new AddrVal(IPAddr(*bits)); %} -## Convert communication data with a type of :bro:see:`Broker::SUBNET` to -## an actual Bro value. -## -## d: the communication data to convert. -## -## Returns: the value retrieved from the communication data. -function Broker::refine_to_subnet%(d: Broker::Data%): subnet +function Broker::__refine_to_subnet%(d: Broker::Data%): subnet %{ auto& a = bro_broker::require_data_type(d->AsRecordVal(), TYPE_SUBNET, frame); @@ -139,71 +84,40 @@ function Broker::refine_to_subnet%(d: Broker::Data%): subnet return new SubNetVal(IPPrefix(IPAddr(*bits), a.length())); %} -## Convert communication data with a type of :bro:see:`Broker::PORT` to -## an actual Bro value. -## -## d: the communication data to convert. -## -## Returns: the value retrieved from the communication data. -function Broker::refine_to_port%(d: Broker::Data%): port +function Broker::__refine_to_port%(d: Broker::Data%): port %{ auto& a = bro_broker::require_data_type(d->AsRecordVal(), TYPE_SUBNET, frame); return new PortVal(a.number(), bro_broker::to_bro_port_proto(a.type())); %} -## Convert communication data with a type of :bro:see:`Broker::TIME` to -## an actual Bro value. -## -## d: the communication data to convert. -## -## Returns: the value retrieved from the communication data. -function Broker::refine_to_time%(d: Broker::Data%): time +function Broker::__refine_to_time%(d: Broker::Data%): time %{ auto v = bro_broker::require_data_type(d->AsRecordVal(), TYPE_TIME, frame).value; return new Val(v, TYPE_TIME); %} -## Convert communication data with a type of :bro:see:`Broker::INTERVAL` to -## an actual Bro value. -## -## d: the communication data to convert. -## -## Returns: the value retrieved from the communication data. -function Broker::refine_to_interval%(d: Broker::Data%): interval +function Broker::__refine_to_interval%(d: Broker::Data%): interval %{ auto v = bro_broker::require_data_type(d->AsRecordVal(), TYPE_TIME, frame).value; return new Val(v, TYPE_INTERVAL); %} -## Convert communication data with a type of :bro:see:`Broker::ENUM` to -## the name of the enum value. :bro:see:`lookup_ID` may be used to convert -## the name to the actual enum value. -## -## d: the communication data to convert. -## -## Returns: the enum name retrieved from the communication data. -function Broker::refine_to_enum_name%(d: Broker::Data%): string +function Broker::__refine_to_enum_name%(d: Broker::Data%): string %{ auto& v = bro_broker::require_data_type(d->AsRecordVal(), TYPE_ENUM, frame).name; return new StringVal(v); %} -## Create communication data of type "set". -function Broker::set_create%(%): Broker::Data +function Broker::__set_create%(%): Broker::Data %{ return bro_broker::make_data_val(broker::set()); %} -## Remove all elements within a set. -## -## s: the set to clear. -## -## Returns: always true. -function Broker::set_clear%(s: Broker::Data%): bool +function Broker::__set_clear%(s: Broker::Data%): bool %{ auto& v = bro_broker::require_data_type(s->AsRecordVal(), TYPE_TABLE, frame); @@ -211,26 +125,14 @@ function Broker::set_clear%(s: Broker::Data%): bool return new Val(true, TYPE_BOOL); %} -## Get the number of elements within a set. -## -## s: the set to query. -## -## Returns: the number of elements in the set. -function Broker::set_size%(s: Broker::Data%): count +function Broker::__set_size%(s: Broker::Data%): count %{ auto& v = bro_broker::require_data_type(s->AsRecordVal(), TYPE_TABLE, frame); return new Val(static_cast(v.size()), TYPE_COUNT); %} -## Check if a set contains a particular element. -## -## s: the set to query. -## -## key: the element to check for existence. -## -## Returns: true if the key exists in the set. -function Broker::set_contains%(s: Broker::Data, key: Broker::Data%): bool +function Broker::__set_contains%(s: Broker::Data, key: Broker::Data%): bool %{ auto& v = bro_broker::require_data_type(s->AsRecordVal(), TYPE_TABLE, frame); @@ -238,14 +140,7 @@ function Broker::set_contains%(s: Broker::Data, key: Broker::Data%): bool return new Val(v.find(k) != v.end(), TYPE_BOOL); %} -## Insert an element into a set. -## -## s: the set to modify. -## -## key: the element to insert. -## -## Returns: true if the key was inserted, or false if it already existed. -function Broker::set_insert%(s: Broker::Data, key: Broker::Data%): bool +function Broker::__set_insert%(s: Broker::Data, key: Broker::Data%): bool %{ auto& v = bro_broker::require_data_type(s->AsRecordVal(), TYPE_TABLE, frame); @@ -253,14 +148,7 @@ function Broker::set_insert%(s: Broker::Data, key: Broker::Data%): bool return new Val(v.insert(k).second, TYPE_BOOL); %} -## Remove an element from a set. -## -## s: the set to modify. -## -## key: the element to remove. -## -## Returns: true if the element existed in the set and is now removed. -function Broker::set_remove%(s: Broker::Data, key: Broker::Data%): bool +function Broker::__set_remove%(s: Broker::Data, key: Broker::Data%): bool %{ auto& v = bro_broker::require_data_type(s->AsRecordVal(), TYPE_TABLE, frame); @@ -268,37 +156,18 @@ function Broker::set_remove%(s: Broker::Data, key: Broker::Data%): bool return new Val(v.erase(k) > 0, TYPE_BOOL); %} -## Create an iterator for a set. Note that this makes a copy of the set -## internally to ensure the iterator is always valid. -## -## s: the set to iterate over. -## -## Returns: an iterator. -function Broker::set_iterator%(s: Broker::Data%): opaque of Broker::SetIterator +function Broker::__set_iterator%(s: Broker::Data%): opaque of Broker::SetIterator %{ return new bro_broker::SetIterator(s->AsRecordVal(), TYPE_TABLE, frame); %} -## Check if there are no more elements to iterate over. -## -## it: an iterator. -## -## Returns: true if there are no more elements to iterator over, i.e. -## the iterator is one-past-the-final-element. -function Broker::set_iterator_last%(it: opaque of Broker::SetIterator%): bool +function Broker::__set_iterator_last%(it: opaque of Broker::SetIterator%): bool %{ auto set_it = static_cast(it); return new Val(set_it->it == set_it->dat.end(), TYPE_BOOL); %} -## Advance an iterator. -## -## it: an iterator. -## -## Returns: true if the iterator, after advancing, still references an element -## in the collection. False if the iterator, after advancing, is -## one-past-the-final-element. -function Broker::set_iterator_next%(it: opaque of Broker::SetIterator%): bool +function Broker::__set_iterator_next%(it: opaque of Broker::SetIterator%): bool %{ auto set_it = static_cast(it); @@ -309,12 +178,7 @@ function Broker::set_iterator_next%(it: opaque of Broker::SetIterator%): bool return new Val(set_it->it != set_it->dat.end(), TYPE_BOOL); %} -## Retrieve the data at an iterator's current position. -## -## it: an iterator. -## -## Returns: element in the collection that the iterator currently references. -function Broker::set_iterator_value%(it: opaque of Broker::SetIterator%): Broker::Data +function Broker::__set_iterator_value%(it: opaque of Broker::SetIterator%): Broker::Data %{ auto set_it = static_cast(it); auto rval = new RecordVal(BifType::Record::Broker::Data); @@ -331,18 +195,12 @@ function Broker::set_iterator_value%(it: opaque of Broker::SetIterator%): Broker return rval; %} -## Create communication data of type "table". -function Broker::table_create%(%): Broker::Data +function Broker::__table_create%(%): Broker::Data %{ return bro_broker::make_data_val(broker::table()); %} -## Remove all elements within a table. -## -## t: the table to clear. -## -## Returns: always true. -function Broker::table_clear%(t: Broker::Data%): bool +function Broker::__table_clear%(t: Broker::Data%): bool %{ auto& v = bro_broker::require_data_type(t->AsRecordVal(), TYPE_TABLE, frame); @@ -350,26 +208,14 @@ function Broker::table_clear%(t: Broker::Data%): bool return new Val(true, TYPE_BOOL); %} -## Get the number of elements within a table. -## -## t: the table to query. -## -## Returns: the number of elements in the table. -function Broker::table_size%(t: Broker::Data%): count +function Broker::__table_size%(t: Broker::Data%): count %{ auto& v = bro_broker::require_data_type(t->AsRecordVal(), TYPE_TABLE, frame); return new Val(static_cast(v.size()), TYPE_COUNT); %} -## Check if a table contains a particular key. -## -## t: the table to query. -## -## key: the key to check for existence. -## -## Returns: true if the key exists in the table. -function Broker::table_contains%(t: Broker::Data, key: Broker::Data%): bool +function Broker::__table_contains%(t: Broker::Data, key: Broker::Data%): bool %{ auto& v = bro_broker::require_data_type(t->AsRecordVal(), TYPE_TABLE, frame); @@ -377,17 +223,7 @@ function Broker::table_contains%(t: Broker::Data, key: Broker::Data%): bool return new Val(v.find(k) != v.end(), TYPE_BOOL); %} -## Insert a key-value pair into a table. -## -## t: the table to modify. -## -## key: the key at which to insert the value. -## -## val: the value to insert. -## -## Returns: true if the key-value pair was inserted, or false if the key -## already existed in the table. -function Broker::table_insert%(t: Broker::Data, key: Broker::Data, val: Broker::Data%): Broker::Data +function Broker::__table_insert%(t: Broker::Data, key: Broker::Data, val: Broker::Data%): Broker::Data %{ auto& table = bro_broker::require_data_type(t->AsRecordVal(), TYPE_TABLE, frame); @@ -408,15 +244,7 @@ function Broker::table_insert%(t: Broker::Data, key: Broker::Data, val: Broker:: } %} -## Remove a key-value pair from a table. -## -## t: the table to modify. -## -## key: the key to remove from the table. -## -## Returns: the value associated with the key. If the key did not exist, then -## the optional field of the returned record is not set. -function Broker::table_remove%(t: Broker::Data, key: Broker::Data%): Broker::Data +function Broker::__table_remove%(t: Broker::Data, key: Broker::Data%): Broker::Data %{ auto& table = bro_broker::require_data_type(t->AsRecordVal(), TYPE_TABLE, frame); @@ -433,15 +261,7 @@ function Broker::table_remove%(t: Broker::Data, key: Broker::Data%): Broker::Dat } %} -## Retrieve a value from a table. -## -## t: the table to query. -## -## key: the key to lookup. -## -## Returns: the value associated with the key. If the key did not exist, then -## the optional field of the returned record is not set. -function Broker::table_lookup%(t: Broker::Data, key: Broker::Data%): Broker::Data +function Broker::__table_lookup%(t: Broker::Data, key: Broker::Data%): Broker::Data %{ auto& table = bro_broker::require_data_type(t->AsRecordVal(), TYPE_TABLE, frame); @@ -454,37 +274,18 @@ function Broker::table_lookup%(t: Broker::Data, key: Broker::Data%): Broker::Dat return bro_broker::make_data_val(it->second); %} -## Create an iterator for a table. Note that this makes a copy of the table -## internally to ensure the iterator is always valid. -## -## t: the table to iterate over. -## -## Returns: an iterator. -function Broker::table_iterator%(t: Broker::Data%): opaque of Broker::TableIterator +function Broker::__table_iterator%(t: Broker::Data%): opaque of Broker::TableIterator %{ return new bro_broker::TableIterator(t->AsRecordVal(), TYPE_TABLE, frame); %} -## Check if there are no more elements to iterate over. -## -## it: an iterator. -## -## Returns: true if there are no more elements to iterator over, i.e. -## the iterator is one-past-the-final-element. -function Broker::table_iterator_last%(it: opaque of Broker::TableIterator%): bool +function Broker::__table_iterator_last%(it: opaque of Broker::TableIterator%): bool %{ auto ti = static_cast(it); return new Val(ti->it == ti->dat.end(), TYPE_BOOL); %} -## Advance an iterator. -## -## it: an iterator. -## -## Returns: true if the iterator, after advancing, still references an element -## in the collection. False if the iterator, after advancing, is -## one-past-the-final-element. -function Broker::table_iterator_next%(it: opaque of Broker::TableIterator%): bool +function Broker::__table_iterator_next%(it: opaque of Broker::TableIterator%): bool %{ auto ti = static_cast(it); @@ -495,12 +296,7 @@ function Broker::table_iterator_next%(it: opaque of Broker::TableIterator%): boo return new Val(ti->it != ti->dat.end(), TYPE_BOOL); %} -## Retrieve the data at an iterator's current position. -## -## it: an iterator. -## -## Returns: element in the collection that the iterator currently references. -function Broker::table_iterator_value%(it: opaque of Broker::TableIterator%): Broker::TableItem +function Broker::__table_iterator_value%(it: opaque of Broker::TableIterator%): Broker::TableItem %{ auto ti = static_cast(it); auto rval = new RecordVal(BifType::Record::Broker::TableItem); @@ -522,18 +318,12 @@ function Broker::table_iterator_value%(it: opaque of Broker::TableIterator%): Br return rval; %} -## Create communication data of type "vector". -function Broker::vector_create%(%): Broker::Data +function Broker::__vector_create%(%): Broker::Data %{ return bro_broker::make_data_val(broker::vector()); %} -## Remove all elements within a vector. -## -## v: the vector to clear. -## -## Returns: always true. -function Broker::vector_clear%(v: Broker::Data%): bool +function Broker::__vector_clear%(v: Broker::Data%): bool %{ auto& vec = bro_broker::require_data_type(v->AsRecordVal(), TYPE_VECTOR, frame); @@ -541,30 +331,14 @@ function Broker::vector_clear%(v: Broker::Data%): bool return new Val(true, TYPE_BOOL); %} -## Get the number of elements within a vector. -## -## v: the vector to query. -## -## Returns: the number of elements in the vector. -function Broker::vector_size%(v: Broker::Data%): count +function Broker::__vector_size%(v: Broker::Data%): count %{ auto& vec = bro_broker::require_data_type(v->AsRecordVal(), TYPE_VECTOR, frame); return new Val(static_cast(vec.size()), TYPE_COUNT); %} -## Insert an element into a vector at a particular position, possibly displacing -## existing elements (insertion always grows the size of the vector by one). -## -## v: the vector to modify. -## -## d: the element to insert. -## -## idx: the index at which to insert the data. If it is greater than the -## current size of the vector, the element is inserted at the end. -## -## Returns: always true. -function Broker::vector_insert%(v: Broker::Data, d: Broker::Data, idx: count%): bool +function Broker::__vector_insert%(v: Broker::Data, d: Broker::Data, idx: count%): bool %{ auto& vec = bro_broker::require_data_type(v->AsRecordVal(), TYPE_VECTOR, frame); @@ -574,17 +348,7 @@ function Broker::vector_insert%(v: Broker::Data, d: Broker::Data, idx: count%): return new Val(true, TYPE_BOOL); %} -## Replace an element in a vector at a particular position. -## -## v: the vector to modify. -## -## d: the element to insert. -## -## idx: the index to replace. -## -## Returns: the value that was just evicted. If the index was larger than any -## valid index, the optional field of the returned record is not set. -function Broker::vector_replace%(v: Broker::Data, d: Broker::Data, idx: count%): Broker::Data +function Broker::__vector_replace%(v: Broker::Data, d: Broker::Data, idx: count%): Broker::Data %{ auto& vec = bro_broker::require_data_type(v->AsRecordVal(), TYPE_VECTOR, frame); @@ -598,15 +362,7 @@ function Broker::vector_replace%(v: Broker::Data, d: Broker::Data, idx: count%): return rval; %} -## Remove an element from a vector at a particular position. -## -## v: the vector to modify. -## -## idx: the index to remove. -## -## Returns: the value that was just evicted. If the index was larger than any -## valid index, the optional field of the returned record is not set. -function Broker::vector_remove%(v: Broker::Data, idx: count%): Broker::Data +function Broker::__vector_remove%(v: Broker::Data, idx: count%): Broker::Data %{ auto& vec = bro_broker::require_data_type(v->AsRecordVal(), TYPE_VECTOR, frame); @@ -619,15 +375,7 @@ function Broker::vector_remove%(v: Broker::Data, idx: count%): Broker::Data return rval; %} -## Lookup an element in a vector at a particular position. -## -## v: the vector to query. -## -## idx: the index to lookup. -## -## Returns: the value at the index. If the index was larger than any -## valid index, the optional field of the returned record is not set. -function Broker::vector_lookup%(v: Broker::Data, idx: count%): Broker::Data +function Broker::__vector_lookup%(v: Broker::Data, idx: count%): Broker::Data %{ auto& vec = bro_broker::require_data_type(v->AsRecordVal(), TYPE_VECTOR, frame); @@ -638,37 +386,18 @@ function Broker::vector_lookup%(v: Broker::Data, idx: count%): Broker::Data return bro_broker::make_data_val(vec[idx]); %} -## Create an iterator for a vector. Note that this makes a copy of the vector -## internally to ensure the iterator is always valid. -## -## v: the vector to iterate over. -## -## Returns: an iterator. -function Broker::vector_iterator%(v: Broker::Data%): opaque of Broker::VectorIterator +function Broker::__vector_iterator%(v: Broker::Data%): opaque of Broker::VectorIterator %{ return new bro_broker::VectorIterator(v->AsRecordVal(), TYPE_VECTOR, frame); %} -## Check if there are no more elements to iterate over. -## -## it: an iterator. -## -## Returns: true if there are no more elements to iterator over, i.e. -## the iterator is one-past-the-final-element. -function Broker::vector_iterator_last%(it: opaque of Broker::VectorIterator%): bool +function Broker::__vector_iterator_last%(it: opaque of Broker::VectorIterator%): bool %{ auto vi = static_cast(it); return new Val(vi->it == vi->dat.end(), TYPE_BOOL); %} -## Advance an iterator. -## -## it: an iterator. -## -## Returns: true if the iterator, after advancing, still references an element -## in the collection. False if the iterator, after advancing, is -## one-past-the-final-element. -function Broker::vector_iterator_next%(it: opaque of Broker::VectorIterator%): bool +function Broker::__vector_iterator_next%(it: opaque of Broker::VectorIterator%): bool %{ auto vi = static_cast(it); @@ -679,12 +408,7 @@ function Broker::vector_iterator_next%(it: opaque of Broker::VectorIterator%): b return new Val(vi->it != vi->dat.end(), TYPE_BOOL); %} -## Retrieve the data at an iterator's current position. -## -## it: an iterator. -## -## Returns: element in the collection that the iterator currently references. -function Broker::vector_iterator_value%(it: opaque of Broker::VectorIterator%): Broker::Data +function Broker::__vector_iterator_value%(it: opaque of Broker::VectorIterator%): Broker::Data %{ auto vi = static_cast(it); auto rval = new RecordVal(BifType::Record::Broker::Data); @@ -701,38 +425,19 @@ function Broker::vector_iterator_value%(it: opaque of Broker::VectorIterator%): return rval; %} -## Create communication data of type "record". -## -## sz: the number of fields in the record. -## -## Returns: record data, with all fields uninitialized. -function Broker::record_create%(sz: count%): Broker::Data +function Broker::__record_create%(sz: count%): Broker::Data %{ return bro_broker::make_data_val(broker::record(std::vector(sz))); %} -## Get the number of fields within a record. -## -## r: the record to query. -## -## Returns: the number of fields in the record. -function Broker::record_size%(r: Broker::Data%): count +function Broker::__record_size%(r: Broker::Data%): count %{ auto& v = bro_broker::require_data_type(r->AsRecordVal(), TYPE_RECORD, frame); return new Val(static_cast(v.fields.size()), TYPE_COUNT); %} -## Replace a field in a record at a particular position. -## -## r: the record to modify. -## -## d: the new field value to assign. -## -## idx: the index to replace. -## -## Returns: false if the index was larger than any valid index, else true. -function Broker::record_assign%(r: Broker::Data, d: Broker::Data, idx: count%): bool +function Broker::__record_assign%(r: Broker::Data, d: Broker::Data, idx: count%): bool %{ auto& v = bro_broker::require_data_type(r->AsRecordVal(), TYPE_RECORD, frame); @@ -745,16 +450,7 @@ function Broker::record_assign%(r: Broker::Data, d: Broker::Data, idx: count%): return new Val(true, TYPE_BOOL); %} -## Lookup a field in a record at a particular position. -## -## r: the record to query. -## -## idx: the index to lookup. -## -## Returns: the value at the index. The optional field of the returned record -## may not be set if the field of the record has no value or if the -## index was not valid. -function Broker::record_lookup%(r: Broker::Data, idx: count%): Broker::Data +function Broker::__record_lookup%(r: Broker::Data, idx: count%): Broker::Data %{ auto& v = bro_broker::require_data_type(r->AsRecordVal(), TYPE_RECORD, frame); @@ -768,37 +464,18 @@ function Broker::record_lookup%(r: Broker::Data, idx: count%): Broker::Data return bro_broker::make_data_val(*v.fields[idx]); %} -## Create an iterator for a record. Note that this makes a copy of the record -## internally to ensure the iterator is always valid. -## -## r: the record to iterate over. -## -## Returns: an iterator. -function Broker::record_iterator%(r: Broker::Data%): opaque of Broker::RecordIterator +function Broker::__record_iterator%(r: Broker::Data%): opaque of Broker::RecordIterator %{ return new bro_broker::RecordIterator(r->AsRecordVal(), TYPE_RECORD, frame); %} -## Check if there are no more elements to iterate over. -## -## it: an iterator. -## -## Returns: true if there are no more elements to iterator over, i.e. -## the iterator is one-past-the-final-element. -function Broker::record_iterator_last%(it: opaque of Broker::RecordIterator%): bool +function Broker::__record_iterator_last%(it: opaque of Broker::RecordIterator%): bool %{ auto ri = static_cast(it); return new Val(ri->it == ri->dat.fields.end(), TYPE_BOOL); %} -## Advance an iterator. -## -## it: an iterator. -## -## Returns: true if the iterator, after advancing, still references an element -## in the collection. False if the iterator, after advancing, is -## one-past-the-final-element. -function Broker::record_iterator_next%(it: opaque of Broker::RecordIterator%): bool +function Broker::__record_iterator_next%(it: opaque of Broker::RecordIterator%): bool %{ auto ri = static_cast(it); @@ -809,12 +486,7 @@ function Broker::record_iterator_next%(it: opaque of Broker::RecordIterator%): b return new Val(ri->it != ri->dat.fields.end(), TYPE_BOOL); %} -## Retrieve the data at an iterator's current position. -## -## it: an iterator. -## -## Returns: element in the collection that the iterator currently references. -function Broker::record_iterator_value%(it: opaque of Broker::RecordIterator%): Broker::Data +function Broker::__record_iterator_value%(it: opaque of Broker::RecordIterator%): Broker::Data %{ auto ri = static_cast(it); auto rval = new RecordVal(BifType::Record::Broker::Data); diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 7e55509a86..acb451c71f 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2016-04-26-21-21-19 +#open 2016-04-27-02-37-50 #fields name #types string scripts/base/init-bare.bro @@ -20,6 +20,7 @@ scripts/base/init-bare.bro build/scripts/base/bif/comm.bif.bro build/scripts/base/bif/messaging.bif.bro scripts/base/frameworks/broker/store.bro + build/scripts/base/bif/data.bif.bro build/scripts/base/bif/store.bif.bro scripts/base/frameworks/logging/__load__.bro scripts/base/frameworks/logging/main.bro @@ -54,7 +55,6 @@ scripts/base/init-bare.bro build/scripts/base/bif/bloom-filter.bif.bro build/scripts/base/bif/cardinality-counter.bif.bro build/scripts/base/bif/top-k.bif.bro - build/scripts/base/bif/data.bif.bro build/scripts/base/bif/plugins/__load__.bro build/scripts/base/bif/plugins/Bro_ARP.events.bif.bro build/scripts/base/bif/plugins/Bro_AYIYA.events.bif.bro @@ -131,4 +131,4 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro scripts/policy/misc/loaded-scripts.bro scripts/base/utils/paths.bro -#close 2016-04-26-21-21-19 +#close 2016-04-27-02-37-50 diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 075a7c1389..3a1a811a8d 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2016-04-26-21-21-31 +#open 2016-04-27-02-38-00 #fields name #types string scripts/base/init-bare.bro @@ -20,6 +20,7 @@ scripts/base/init-bare.bro build/scripts/base/bif/comm.bif.bro build/scripts/base/bif/messaging.bif.bro scripts/base/frameworks/broker/store.bro + build/scripts/base/bif/data.bif.bro build/scripts/base/bif/store.bif.bro scripts/base/frameworks/logging/__load__.bro scripts/base/frameworks/logging/main.bro @@ -54,7 +55,6 @@ scripts/base/init-bare.bro build/scripts/base/bif/bloom-filter.bif.bro build/scripts/base/bif/cardinality-counter.bif.bro build/scripts/base/bif/top-k.bif.bro - build/scripts/base/bif/data.bif.bro build/scripts/base/bif/plugins/__load__.bro build/scripts/base/bif/plugins/Bro_ARP.events.bif.bro build/scripts/base/bif/plugins/Bro_AYIYA.events.bif.bro @@ -302,4 +302,4 @@ scripts/base/init-default.bro scripts/base/misc/find-checksum-offloading.bro scripts/base/misc/find-filtered-trace.bro scripts/policy/misc/loaded-scripts.bro -#close 2016-04-26-21-21-31 +#close 2016-04-27-02-38-00 diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index a39e6d8dd8..9c12980f9e 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -230,7 +230,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461705704.402549, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461724691.655146, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Communication::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Conn::LOG)) -> @@ -351,7 +351,7 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461705704.402549, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461724691.655146, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(NetControl::check_plugins, , ()) -> 0.000000 MetaHookPost CallFunction(NetControl::init, , ()) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, , ()) -> @@ -572,6 +572,7 @@ 0.000000 MetaHookPost LoadFile(base<...>/conn-ids) -> -1 0.000000 MetaHookPost LoadFile(base<...>/const.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(base<...>/control) -> -1 +0.000000 MetaHookPost LoadFile(base<...>/data.bif) -> -1 0.000000 MetaHookPost LoadFile(base<...>/dhcp) -> -1 0.000000 MetaHookPost LoadFile(base<...>/dir) -> -1 0.000000 MetaHookPost LoadFile(base<...>/directions-and-hosts) -> -1 @@ -873,7 +874,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461705704.402549, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461724691.655146, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Communication::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Conn::LOG)) @@ -994,7 +995,7 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461705704.402549, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461724691.655146, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(NetControl::check_plugins, , ()) 0.000000 MetaHookPre CallFunction(NetControl::init, , ()) 0.000000 MetaHookPre CallFunction(Notice::want_pp, , ()) @@ -1215,6 +1216,7 @@ 0.000000 MetaHookPre LoadFile(base<...>/conn-ids) 0.000000 MetaHookPre LoadFile(base<...>/const.bif.bro) 0.000000 MetaHookPre LoadFile(base<...>/control) +0.000000 MetaHookPre LoadFile(base<...>/data.bif) 0.000000 MetaHookPre LoadFile(base<...>/dhcp) 0.000000 MetaHookPre LoadFile(base<...>/dir) 0.000000 MetaHookPre LoadFile(base<...>/directions-and-hosts) @@ -1515,7 +1517,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1461705704.402549, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1461724691.655146, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG) @@ -1636,7 +1638,7 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1461705704.402549, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1461724691.655146, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction NetControl::check_plugins() 0.000000 | HookCallFunction NetControl::init() 0.000000 | HookCallFunction Notice::want_pp() From b1876bf744fafb6fb723716ef64103a952014be4 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Tue, 26 Apr 2016 22:30:12 -0500 Subject: [PATCH 63/84] Code cleanup for some broker tests Simplified some function names, fixed some names of broker script wrappers, reorder some broker function calls to avoid potential race conditions, and don't have bro read a trace file when it will not be used. --- testing/btest/broker/clone_store.bro | 4 +- testing/btest/broker/connection_updates.bro | 6 +-- testing/btest/broker/data.bro | 52 +++++++++---------- testing/btest/broker/remote_print.test | 2 +- .../btest/core/leaks/broker/clone_store.bro | 4 +- testing/btest/core/leaks/broker/data.bro | 52 +++++++++---------- .../btest/core/leaks/broker/remote_event.test | 8 +-- .../btest/core/leaks/broker/remote_log.test | 2 +- .../btest/core/leaks/broker/remote_print.test | 8 +-- 9 files changed, 69 insertions(+), 69 deletions(-) diff --git a/testing/btest/broker/clone_store.bro b/testing/btest/broker/clone_store.bro index b761fc56ad..c810a0d209 100644 --- a/testing/btest/broker/clone_store.bro +++ b/testing/btest/broker/clone_store.bro @@ -1,8 +1,8 @@ # @TEST-SERIALIZE: brokercomm # @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt -# @TEST-EXEC: btest-bg-run clone "bro -b -r $TRACES/wikipedia.trace ../clone.bro broker_port=$BROKER_PORT >clone.out" -# @TEST-EXEC: btest-bg-run master "bro -b -r $TRACES/wikipedia.trace ../master.bro broker_port=$BROKER_PORT >master.out" +# @TEST-EXEC: btest-bg-run clone "bro -b ../clone.bro broker_port=$BROKER_PORT >clone.out" +# @TEST-EXEC: btest-bg-run master "bro -b ../master.bro broker_port=$BROKER_PORT >master.out" # @TEST-EXEC: btest-bg-wait 60 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff clone/clone.out diff --git a/testing/btest/broker/connection_updates.bro b/testing/btest/broker/connection_updates.bro index 032049e5ef..bd08fff924 100644 --- a/testing/btest/broker/connection_updates.bro +++ b/testing/btest/broker/connection_updates.bro @@ -22,12 +22,12 @@ event bro_init() event Broker::incoming_connection_established(peer_name: string) { - print "Broker::incoming_connection_established", peer_name;; + print "Broker::incoming_connection_established", peer_name; } event Broker::incoming_connection_broken(peer_name: string) { - print "Broker::incoming_connection_broken", peer_name;; + print "Broker::incoming_connection_broken", peer_name; terminate(); } @@ -50,7 +50,7 @@ event Broker::outgoing_connection_established(peer_address: string, peer_name: string) { print "Broker::outgoing_connection_established", - peer_address, peer_port, peer_name;; + peer_address, peer_port, peer_name; terminate(); } diff --git a/testing/btest/broker/data.bro b/testing/btest/broker/data.bro index d8f4c2f8b5..dc05d4142d 100644 --- a/testing/btest/broker/data.bro +++ b/testing/btest/broker/data.bro @@ -13,7 +13,7 @@ type bro_record : record { c: count; }; -function comm_record_to_bro_record_recurse(it: opaque of Broker::RecordIterator, +function broker_to_bro_record_recurse(it: opaque of Broker::RecordIterator, rval: bro_record, idx: count): bro_record { @@ -37,17 +37,17 @@ function comm_record_to_bro_record_recurse(it: opaque of Broker::RecordIterator, ++idx; Broker::record_iterator_next(it); - return comm_record_to_bro_record_recurse(it, rval, idx); + return broker_to_bro_record_recurse(it, rval, idx); } -function comm_record_to_bro_record(d: Broker::Data): bro_record +function broker_to_bro_record(d: Broker::Data): bro_record { - return comm_record_to_bro_record_recurse(Broker::record_iterator(d), + return broker_to_bro_record_recurse(Broker::record_iterator(d), bro_record($c = 0), 0); } function -comm_set_to_bro_set_recurse(it: opaque of Broker::SetIterator, +broker_to_bro_set_recurse(it: opaque of Broker::SetIterator, rval: bro_set): bro_set { if ( Broker::set_iterator_last(it) ) @@ -55,17 +55,17 @@ comm_set_to_bro_set_recurse(it: opaque of Broker::SetIterator, add rval[Broker::refine_to_string(Broker::set_iterator_value(it))]; Broker::set_iterator_next(it); - return comm_set_to_bro_set_recurse(it, rval); + return broker_to_bro_set_recurse(it, rval); } -function comm_set_to_bro_set(d: Broker::Data): bro_set +function broker_to_bro_set(d: Broker::Data): bro_set { - return comm_set_to_bro_set_recurse(Broker::set_iterator(d), bro_set()); + return broker_to_bro_set_recurse(Broker::set_iterator(d), bro_set()); } function -comm_table_to_bro_table_recurse(it: opaque of Broker::TableIterator, +broker_to_bro_table_recurse(it: opaque of Broker::TableIterator, rval: bro_table): bro_table { if ( Broker::table_iterator_last(it) ) @@ -74,16 +74,16 @@ comm_table_to_bro_table_recurse(it: opaque of Broker::TableIterator, local item = Broker::table_iterator_value(it); rval[Broker::refine_to_string(item$key)] = Broker::refine_to_count(item$val); Broker::table_iterator_next(it); - return comm_table_to_bro_table_recurse(it, rval); + return broker_to_bro_table_recurse(it, rval); } -function comm_table_to_bro_table(d: Broker::Data): bro_table +function broker_to_bro_table(d: Broker::Data): bro_table { - return comm_table_to_bro_table_recurse(Broker::table_iterator(d), + return broker_to_bro_table_recurse(Broker::table_iterator(d), bro_table()); } -function comm_vector_to_bro_vector_recurse(it: opaque of Broker::VectorIterator, +function broker_to_bro_vector_recurse(it: opaque of Broker::VectorIterator, rval: bro_vector): bro_vector { if ( Broker::vector_iterator_last(it) ) @@ -91,12 +91,12 @@ function comm_vector_to_bro_vector_recurse(it: opaque of Broker::VectorIterator, rval[|rval|] = Broker::refine_to_string(Broker::vector_iterator_value(it)); Broker::vector_iterator_next(it); - return comm_vector_to_bro_vector_recurse(it, rval); + return broker_to_bro_vector_recurse(it, rval); } -function comm_vector_to_bro_vector(d: Broker::Data): bro_vector +function broker_to_bro_vector(d: Broker::Data): bro_vector { - return comm_vector_to_bro_vector_recurse(Broker::vector_iterator(d), + return broker_to_bro_vector_recurse(Broker::vector_iterator(d), bro_vector()); } @@ -145,7 +145,7 @@ print Broker::refine_to_enum_name(Broker::data(Broker::BOOL)); print "***************************"; local cs = Broker::data(s); -print comm_set_to_bro_set(cs); +print broker_to_bro_set(cs); cs = Broker::set_create(); print Broker::set_size(cs); print Broker::set_insert(cs, Broker::data("hi")); @@ -157,14 +157,14 @@ print Broker::set_size(cs); print Broker::set_remove(cs, Broker::data("hi")); print Broker::set_size(cs); print Broker::set_remove(cs, Broker::data("hi")); -print comm_set_to_bro_set(cs); +print broker_to_bro_set(cs); Broker::set_clear(cs); print Broker::set_size(cs); print "***************************"; local ct = Broker::data(t); -print comm_table_to_bro_table(ct); +print broker_to_bro_table(ct); ct = Broker::table_create(); print Broker::table_size(ct); print Broker::table_insert(ct, Broker::data("hi"), Broker::data(42)); @@ -183,33 +183,33 @@ print Broker::table_size(ct); print "***************************"; local cv = Broker::data(v); -print comm_vector_to_bro_vector(cv); +print broker_to_bro_vector(cv); cv = Broker::vector_create(); print Broker::vector_size(cv); print Broker::vector_insert(cv, Broker::data("hi"), 0); print Broker::vector_insert(cv, Broker::data("hello"), 1); print Broker::vector_insert(cv, Broker::data("greetings"), 2); print Broker::vector_insert(cv, Broker::data("salutations"), 1); -print comm_vector_to_bro_vector(cv); +print broker_to_bro_vector(cv); print Broker::vector_size(cv); print Broker::vector_replace(cv, Broker::data("bah"), 2); print Broker::vector_lookup(cv, 2); print Broker::vector_lookup(cv, 0); -print comm_vector_to_bro_vector(cv); +print broker_to_bro_vector(cv); print Broker::vector_remove(cv, 2); -print comm_vector_to_bro_vector(cv); +print broker_to_bro_vector(cv); print Broker::vector_size(cv); print "***************************"; local cr = Broker::data(r); -print comm_record_to_bro_record(cr); +print broker_to_bro_record(cr); r$a = "test"; cr = Broker::data(r); -print comm_record_to_bro_record(cr); +print broker_to_bro_record(cr); r$b = "testagain"; cr = Broker::data(r); -print comm_record_to_bro_record(cr); +print broker_to_bro_record(cr); cr = Broker::record_create(3); print Broker::record_size(cr); print Broker::record_assign(cr, Broker::data("hi"), 0); diff --git a/testing/btest/broker/remote_print.test b/testing/btest/broker/remote_print.test index 9cfcc44ca9..e8e9e0f71d 100644 --- a/testing/btest/broker/remote_print.test +++ b/testing/btest/broker/remote_print.test @@ -16,8 +16,8 @@ redef exit_only_after_terminate = T; event bro_init() { Broker::enable(); - Broker::listen(broker_port, "127.0.0.1"); Broker::subscribe_to_prints("bro/print/"); + Broker::listen(broker_port, "127.0.0.1"); } global messages_to_recv = 6; diff --git a/testing/btest/core/leaks/broker/clone_store.bro b/testing/btest/core/leaks/broker/clone_store.bro index 09308eb42e..a02e3b2880 100644 --- a/testing/btest/core/leaks/broker/clone_store.bro +++ b/testing/btest/core/leaks/broker/clone_store.bro @@ -51,8 +51,8 @@ event ready() event bro_init() { Broker::enable(); - Broker::listen(broker_port, "127.0.0.1"); Broker::subscribe_to_events("bro/event/ready"); + Broker::listen(broker_port, "127.0.0.1"); } @TEST-END-FILE @@ -105,9 +105,9 @@ event Broker::outgoing_connection_established(peer_address: string, event bro_init() { Broker::enable(); + Broker::auto_event("bro/event/ready", ready); h = Broker::create_master("mystore"); Broker::connect("127.0.0.1", broker_port, 1secs); - Broker::auto_event("bro/event/ready", ready); } @TEST-END-FILE diff --git a/testing/btest/core/leaks/broker/data.bro b/testing/btest/core/leaks/broker/data.bro index 0902f6e862..146554c879 100644 --- a/testing/btest/core/leaks/broker/data.bro +++ b/testing/btest/core/leaks/broker/data.bro @@ -16,7 +16,7 @@ type bro_record : record { c: count; }; -function comm_record_to_bro_record_recurse(it: opaque of Broker::RecordIterator, +function broker_to_bro_record_recurse(it: opaque of Broker::RecordIterator, rval: bro_record, idx: count): bro_record { @@ -40,17 +40,17 @@ function comm_record_to_bro_record_recurse(it: opaque of Broker::RecordIterator, ++idx; Broker::record_iterator_next(it); - return comm_record_to_bro_record_recurse(it, rval, idx); + return broker_to_bro_record_recurse(it, rval, idx); } -function comm_record_to_bro_record(d: Broker::Data): bro_record +function broker_to_bro_record(d: Broker::Data): bro_record { - return comm_record_to_bro_record_recurse(Broker::record_iterator(d), + return broker_to_bro_record_recurse(Broker::record_iterator(d), bro_record($c = 0), 0); } function -comm_set_to_bro_set_recurse(it: opaque of Broker::SetIterator, +broker_to_bro_set_recurse(it: opaque of Broker::SetIterator, rval: bro_set): bro_set { if ( Broker::set_iterator_last(it) ) @@ -58,17 +58,17 @@ comm_set_to_bro_set_recurse(it: opaque of Broker::SetIterator, add rval[Broker::refine_to_string(Broker::set_iterator_value(it))]; Broker::set_iterator_next(it); - return comm_set_to_bro_set_recurse(it, rval); + return broker_to_bro_set_recurse(it, rval); } -function comm_set_to_bro_set(d: Broker::Data): bro_set +function broker_to_bro_set(d: Broker::Data): bro_set { - return comm_set_to_bro_set_recurse(Broker::set_iterator(d), bro_set()); + return broker_to_bro_set_recurse(Broker::set_iterator(d), bro_set()); } function -comm_table_to_bro_table_recurse(it: opaque of Broker::TableIterator, +broker_to_bro_table_recurse(it: opaque of Broker::TableIterator, rval: bro_table): bro_table { if ( Broker::table_iterator_last(it) ) @@ -77,16 +77,16 @@ comm_table_to_bro_table_recurse(it: opaque of Broker::TableIterator, local item = Broker::table_iterator_value(it); rval[Broker::refine_to_string(item$key)] = Broker::refine_to_count(item$val); Broker::table_iterator_next(it); - return comm_table_to_bro_table_recurse(it, rval); + return broker_to_bro_table_recurse(it, rval); } -function comm_table_to_bro_table(d: Broker::Data): bro_table +function broker_to_bro_table(d: Broker::Data): bro_table { - return comm_table_to_bro_table_recurse(Broker::table_iterator(d), + return broker_to_bro_table_recurse(Broker::table_iterator(d), bro_table()); } -function comm_vector_to_bro_vector_recurse(it: opaque of Broker::VectorIterator, +function broker_to_bro_vector_recurse(it: opaque of Broker::VectorIterator, rval: bro_vector): bro_vector { if ( Broker::vector_iterator_last(it) ) @@ -94,12 +94,12 @@ function comm_vector_to_bro_vector_recurse(it: opaque of Broker::VectorIterator, rval[|rval|] = Broker::refine_to_string(Broker::vector_iterator_value(it)); Broker::vector_iterator_next(it); - return comm_vector_to_bro_vector_recurse(it, rval); + return broker_to_bro_vector_recurse(it, rval); } -function comm_vector_to_bro_vector(d: Broker::Data): bro_vector +function broker_to_bro_vector(d: Broker::Data): bro_vector { - return comm_vector_to_bro_vector_recurse(Broker::vector_iterator(d), + return broker_to_bro_vector_recurse(Broker::vector_iterator(d), bro_vector()); } @@ -156,7 +156,7 @@ print Broker::refine_to_enum_name(Broker::data(Broker::BOOL)); print "***************************"; local cs = Broker::data(s); -print comm_set_to_bro_set(cs); +print broker_to_bro_set(cs); cs = Broker::set_create(); print Broker::set_size(cs); print Broker::set_insert(cs, Broker::data("hi")); @@ -168,14 +168,14 @@ print Broker::set_size(cs); print Broker::set_remove(cs, Broker::data("hi")); print Broker::set_size(cs); print Broker::set_remove(cs, Broker::data("hi")); -print comm_set_to_bro_set(cs); +print broker_to_bro_set(cs); Broker::set_clear(cs); print Broker::set_size(cs); print "***************************"; local ct = Broker::data(t); -print comm_table_to_bro_table(ct); +print broker_to_bro_table(ct); ct = Broker::table_create(); print Broker::table_size(ct); print Broker::table_insert(ct, Broker::data("hi"), Broker::data(42)); @@ -194,33 +194,33 @@ print Broker::table_size(ct); print "***************************"; local cv = Broker::data(v); -print comm_vector_to_bro_vector(cv); +print broker_to_bro_vector(cv); cv = Broker::vector_create(); print Broker::vector_size(cv); print Broker::vector_insert(cv, Broker::data("hi"), 0); print Broker::vector_insert(cv, Broker::data("hello"), 1); print Broker::vector_insert(cv, Broker::data("greetings"), 2); print Broker::vector_insert(cv, Broker::data("salutations"), 1); -print comm_vector_to_bro_vector(cv); +print broker_to_bro_vector(cv); print Broker::vector_size(cv); print Broker::vector_replace(cv, Broker::data("bah"), 2); print Broker::vector_lookup(cv, 2); print Broker::vector_lookup(cv, 0); -print comm_vector_to_bro_vector(cv); +print broker_to_bro_vector(cv); print Broker::vector_remove(cv, 2); -print comm_vector_to_bro_vector(cv); +print broker_to_bro_vector(cv); print Broker::vector_size(cv); print "***************************"; local cr = Broker::data(r); -print comm_record_to_bro_record(cr); +print broker_to_bro_record(cr); r$a = "test"; cr = Broker::data(r); -print comm_record_to_bro_record(cr); +print broker_to_bro_record(cr); r$b = "testagain"; cr = Broker::data(r); -print comm_record_to_bro_record(cr); +print broker_to_bro_record(cr); cr = Broker::record_create(3); print Broker::record_size(cr); print Broker::record_assign(cr, Broker::data("hi"), 0); diff --git a/testing/btest/core/leaks/broker/remote_event.test b/testing/btest/core/leaks/broker/remote_event.test index 731f001f8f..c68a9e5beb 100644 --- a/testing/btest/core/leaks/broker/remote_event.test +++ b/testing/btest/core/leaks/broker/remote_event.test @@ -21,9 +21,9 @@ global auto_event_handler: event(msg: string, c: count); event bro_init() { Broker::enable(); - Broker::listen(broker_port, "127.0.0.1"); Broker::subscribe_to_events("bro/event/"); Broker::auto_event("bro/event/my_topic", auto_event_handler); + Broker::listen(broker_port, "127.0.0.1"); } global event_count = 0; @@ -42,7 +42,7 @@ event event_handler(msg: string, n: count) event auto_event_handler(msg, n); local args = Broker::event_args(event_handler, "pong", n); - Broker::event("bro/event/my_topic", args); + Broker::send_event("bro/event/my_topic", args); } @TEST-END-FILE @@ -70,7 +70,7 @@ event Broker::outgoing_connection_established(peer_address: string, { print "Broker::outgoing_connection_established", peer_address, peer_port; local args = Broker::event_args(event_handler, "ping", event_count); - Broker::event("bro/event/hi", args); + Broker::send_event("bro/event/hi", args); ++event_count; } @@ -84,7 +84,7 @@ event event_handler(msg: string, n: count) { print "got event msg", msg, n; local args = Broker::event_args(event_handler, "ping", event_count); - Broker::event("bro/event/hi", args); + Broker::send_event("bro/event/hi", args); ++event_count; } diff --git a/testing/btest/core/leaks/broker/remote_log.test b/testing/btest/core/leaks/broker/remote_log.test index 12602115a4..bf608dd459 100644 --- a/testing/btest/core/leaks/broker/remote_log.test +++ b/testing/btest/core/leaks/broker/remote_log.test @@ -42,8 +42,8 @@ redef exit_only_after_terminate = T; event bro_init() { - Broker::listen(broker_port, "127.0.0.1"); Broker::subscribe_to_logs("bro/log/"); + Broker::listen(broker_port, "127.0.0.1"); } event Test::log_test(rec: Test::Info) diff --git a/testing/btest/core/leaks/broker/remote_print.test b/testing/btest/core/leaks/broker/remote_print.test index 623097b091..34266ebf4c 100644 --- a/testing/btest/core/leaks/broker/remote_print.test +++ b/testing/btest/core/leaks/broker/remote_print.test @@ -18,8 +18,8 @@ redef exit_only_after_terminate = T; event bro_init() { Broker::enable(); - Broker::listen(broker_port, "127.0.0.1"); Broker::subscribe_to_prints("bro/print/"); + Broker::listen(broker_port, "127.0.0.1"); } global messages_to_recv = 6; @@ -37,7 +37,7 @@ event Broker::print_handler(msg: string) return; } - Broker::print("bro/print/my_topic", fmt("pong %d", messages_sent)); + Broker::send_print("bro/print/my_topic", fmt("pong %d", messages_sent)); ++messages_sent; } @@ -64,7 +64,7 @@ event Broker::outgoing_connection_established(peer_address: string, peer_name: string) { print "Broker::outgoing_connection_established", peer_address, peer_port; - Broker::print("bro/print/hi", fmt("ping %d", messages_sent)); + Broker::send_print("bro/print/hi", fmt("ping %d", messages_sent)); ++messages_sent; } @@ -78,7 +78,7 @@ event Broker::print_handler(msg: string) { ++messages_recv; print "got print msg", msg; - Broker::print("bro/print/hi", fmt("ping %d", messages_sent)); + Broker::send_print("bro/print/hi", fmt("ping %d", messages_sent)); ++messages_sent; } From fbab6490ec56ecd24f51e315a8a99acf63a9a70e Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Tue, 26 Apr 2016 23:03:28 -0500 Subject: [PATCH 64/84] Add missing tests for broker data BIFs Added tests for the table_clear and vector_clear BIFs, and added more tests for container types (e.g. adding the same element twice to a set or table, or overwriting a record field value, etc.). Also reorganized several test cases. --- testing/btest/Baseline/broker.data/out | 37 +++++++++++---- testing/btest/broker/data.bro | 63 ++++++++++++++++++++------ 2 files changed, 76 insertions(+), 24 deletions(-) diff --git a/testing/btest/Baseline/broker.data/out b/testing/btest/Baseline/broker.data/out index 281eb9b316..8703ca6a0c 100644 --- a/testing/btest/Baseline/broker.data/out +++ b/testing/btest/Baseline/broker.data/out @@ -30,12 +30,21 @@ hello 42.0 180.0 Broker::BOOL -*************************** { two, one, three } +{ +[two] = 2, +[one] = 1, +[three] = 3 +} +[zero, one, two] +[a=, b=bee, c=1] +[a=test, b=bee, c=1] +[a=test, b=testagain, c=1] +*************************** 0 T 1 @@ -43,19 +52,20 @@ T F T 2 +F +2 T 1 F { bye } +T 0 -*************************** { -[two] = 2, -[one] = 1, -[three] = 3 + } +*************************** 0 [d=] 1 @@ -69,8 +79,14 @@ F 37 [d=broker::data{42}] 1 +[d=] +1 +T +0 +{ + +} *************************** -[zero, one, two] 0 T T @@ -85,10 +101,10 @@ T [d=broker::data{bah}] [hi, salutations, greetings] 3 +T +0 +[] *************************** -[a=, b=bee, c=1] -[a=test, b=bee, c=1] -[a=test, b=testagain, c=1] 3 T T @@ -97,3 +113,6 @@ T [d=broker::data{hello}] [d=broker::data{37}] 3 +T +3 +[d=broker::data{goodbye}] diff --git a/testing/btest/broker/data.bro b/testing/btest/broker/data.bro index dc05d4142d..ab51caf68d 100644 --- a/testing/btest/broker/data.bro +++ b/testing/btest/broker/data.bro @@ -103,6 +103,9 @@ function broker_to_bro_vector(d: Broker::Data): bro_vector event bro_init() { Broker::enable(); + +### Print every broker data type + print Broker::data_type(Broker::data(T)); print Broker::data_type(Broker::data(+1)); print Broker::data_type(Broker::data(1)); @@ -125,6 +128,8 @@ print Broker::data_type(Broker::data(r)); print "***************************"; +### Convert a Bro value to a broker value, then print the result + print Broker::refine_to_bool(Broker::data(T)); print Broker::refine_to_bool(Broker::data(F)); print Broker::refine_to_int(Broker::data(+1)); @@ -142,10 +147,30 @@ print Broker::refine_to_time(Broker::data(double_to_time(42))); print Broker::refine_to_interval(Broker::data(3min)); print Broker::refine_to_enum_name(Broker::data(Broker::BOOL)); -print "***************************"; - local cs = Broker::data(s); print broker_to_bro_set(cs); + +local ct = Broker::data(t); +print broker_to_bro_table(ct); + +local cv = Broker::data(v); +print broker_to_bro_vector(cv); + +local cr = Broker::data(r); +print broker_to_bro_record(cr); + +r$a = "test"; +cr = Broker::data(r); +print broker_to_bro_record(cr); + +r$b = "testagain"; +cr = Broker::data(r); +print broker_to_bro_record(cr); + +print "***************************"; + +### Test the broker set BIFs + cs = Broker::set_create(); print Broker::set_size(cs); print Broker::set_insert(cs, Broker::data("hi")); @@ -154,17 +179,20 @@ print Broker::set_contains(cs, Broker::data("hi")); print Broker::set_contains(cs, Broker::data("bye")); print Broker::set_insert(cs, Broker::data("bye")); print Broker::set_size(cs); +print Broker::set_insert(cs, Broker::data("bye")); +print Broker::set_size(cs); print Broker::set_remove(cs, Broker::data("hi")); print Broker::set_size(cs); print Broker::set_remove(cs, Broker::data("hi")); print broker_to_bro_set(cs); -Broker::set_clear(cs); +print Broker::set_clear(cs); print Broker::set_size(cs); +print broker_to_bro_set(cs); print "***************************"; -local ct = Broker::data(t); -print broker_to_bro_table(ct); +### Test the broker table BIFs + ct = Broker::table_create(); print Broker::table_size(ct); print Broker::table_insert(ct, Broker::data("hi"), Broker::data(42)); @@ -179,11 +207,16 @@ print Broker::table_size(ct); print Broker::refine_to_count(Broker::table_lookup(ct, Broker::data("bye"))); print Broker::table_remove(ct, Broker::data("hi")); print Broker::table_size(ct); +print Broker::table_remove(ct, Broker::data("hi")); +print Broker::table_size(ct); +print Broker::table_clear(ct); +print Broker::table_size(ct); +print broker_to_bro_table(ct); print "***************************"; -local cv = Broker::data(v); -print broker_to_bro_vector(cv); +### Test the broker vector BIFs + cv = Broker::vector_create(); print Broker::vector_size(cv); print Broker::vector_insert(cv, Broker::data("hi"), 0); @@ -199,17 +232,14 @@ print broker_to_bro_vector(cv); print Broker::vector_remove(cv, 2); print broker_to_bro_vector(cv); print Broker::vector_size(cv); +print Broker::vector_clear(cv); +print Broker::vector_size(cv); +print broker_to_bro_vector(cv); print "***************************"; -local cr = Broker::data(r); -print broker_to_bro_record(cr); -r$a = "test"; -cr = Broker::data(r); -print broker_to_bro_record(cr); -r$b = "testagain"; -cr = Broker::data(r); -print broker_to_bro_record(cr); +### Test the broker record BIFs + cr = Broker::record_create(3); print Broker::record_size(cr); print Broker::record_assign(cr, Broker::data("hi"), 0); @@ -219,4 +249,7 @@ print Broker::record_lookup(cr, 0); print Broker::record_lookup(cr, 1); print Broker::record_lookup(cr, 2); print Broker::record_size(cr); +print Broker::record_assign(cr, Broker::data("goodbye"), 1); +print Broker::record_size(cr); +print Broker::record_lookup(cr, 1); } From f5361fb27c8b98249d50146c37be34787ecdba01 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Tue, 26 Apr 2016 23:34:39 -0500 Subject: [PATCH 65/84] Sync the core/leaks/broker/data.bro test with broker/data.bro --- .../core.leaks.broker.data/bro..stdout | 37 +++++++--- testing/btest/core/leaks/broker/data.bro | 69 ++++++++++++++----- 2 files changed, 79 insertions(+), 27 deletions(-) diff --git a/testing/btest/Baseline/core.leaks.broker.data/bro..stdout b/testing/btest/Baseline/core.leaks.broker.data/bro..stdout index 281eb9b316..8703ca6a0c 100644 --- a/testing/btest/Baseline/core.leaks.broker.data/bro..stdout +++ b/testing/btest/Baseline/core.leaks.broker.data/bro..stdout @@ -30,12 +30,21 @@ hello 42.0 180.0 Broker::BOOL -*************************** { two, one, three } +{ +[two] = 2, +[one] = 1, +[three] = 3 +} +[zero, one, two] +[a=, b=bee, c=1] +[a=test, b=bee, c=1] +[a=test, b=testagain, c=1] +*************************** 0 T 1 @@ -43,19 +52,20 @@ T F T 2 +F +2 T 1 F { bye } +T 0 -*************************** { -[two] = 2, -[one] = 1, -[three] = 3 + } +*************************** 0 [d=] 1 @@ -69,8 +79,14 @@ F 37 [d=broker::data{42}] 1 +[d=] +1 +T +0 +{ + +} *************************** -[zero, one, two] 0 T T @@ -85,10 +101,10 @@ T [d=broker::data{bah}] [hi, salutations, greetings] 3 +T +0 +[] *************************** -[a=, b=bee, c=1] -[a=test, b=bee, c=1] -[a=test, b=testagain, c=1] 3 T T @@ -97,3 +113,6 @@ T [d=broker::data{hello}] [d=broker::data{37}] 3 +T +3 +[d=broker::data{goodbye}] diff --git a/testing/btest/core/leaks/broker/data.bro b/testing/btest/core/leaks/broker/data.bro index 146554c879..5ce53b93dd 100644 --- a/testing/btest/core/leaks/broker/data.bro +++ b/testing/btest/core/leaks/broker/data.bro @@ -104,16 +104,19 @@ function broker_to_bro_vector(d: Broker::Data): bro_vector } event bro_init() - { +{ Broker::enable(); - } +} global did_it = F; event new_connection(c: connection) - { +{ if ( did_it ) return; did_it = T; + +### Print every broker data type + print Broker::data_type(Broker::data(T)); print Broker::data_type(Broker::data(+1)); print Broker::data_type(Broker::data(1)); @@ -136,6 +139,8 @@ print Broker::data_type(Broker::data(r)); print "***************************"; +### Convert a Bro value to a broker value, then print the result + print Broker::refine_to_bool(Broker::data(T)); print Broker::refine_to_bool(Broker::data(F)); print Broker::refine_to_int(Broker::data(+1)); @@ -153,10 +158,30 @@ print Broker::refine_to_time(Broker::data(double_to_time(42))); print Broker::refine_to_interval(Broker::data(3min)); print Broker::refine_to_enum_name(Broker::data(Broker::BOOL)); -print "***************************"; - local cs = Broker::data(s); print broker_to_bro_set(cs); + +local ct = Broker::data(t); +print broker_to_bro_table(ct); + +local cv = Broker::data(v); +print broker_to_bro_vector(cv); + +local cr = Broker::data(r); +print broker_to_bro_record(cr); + +r$a = "test"; +cr = Broker::data(r); +print broker_to_bro_record(cr); + +r$b = "testagain"; +cr = Broker::data(r); +print broker_to_bro_record(cr); + +print "***************************"; + +### Test the broker set BIFs + cs = Broker::set_create(); print Broker::set_size(cs); print Broker::set_insert(cs, Broker::data("hi")); @@ -165,17 +190,20 @@ print Broker::set_contains(cs, Broker::data("hi")); print Broker::set_contains(cs, Broker::data("bye")); print Broker::set_insert(cs, Broker::data("bye")); print Broker::set_size(cs); +print Broker::set_insert(cs, Broker::data("bye")); +print Broker::set_size(cs); print Broker::set_remove(cs, Broker::data("hi")); print Broker::set_size(cs); print Broker::set_remove(cs, Broker::data("hi")); print broker_to_bro_set(cs); -Broker::set_clear(cs); +print Broker::set_clear(cs); print Broker::set_size(cs); +print broker_to_bro_set(cs); print "***************************"; -local ct = Broker::data(t); -print broker_to_bro_table(ct); +### Test the broker table BIFs + ct = Broker::table_create(); print Broker::table_size(ct); print Broker::table_insert(ct, Broker::data("hi"), Broker::data(42)); @@ -190,11 +218,16 @@ print Broker::table_size(ct); print Broker::refine_to_count(Broker::table_lookup(ct, Broker::data("bye"))); print Broker::table_remove(ct, Broker::data("hi")); print Broker::table_size(ct); +print Broker::table_remove(ct, Broker::data("hi")); +print Broker::table_size(ct); +print Broker::table_clear(ct); +print Broker::table_size(ct); +print broker_to_bro_table(ct); print "***************************"; -local cv = Broker::data(v); -print broker_to_bro_vector(cv); +### Test the broker vector BIFs + cv = Broker::vector_create(); print Broker::vector_size(cv); print Broker::vector_insert(cv, Broker::data("hi"), 0); @@ -210,17 +243,14 @@ print broker_to_bro_vector(cv); print Broker::vector_remove(cv, 2); print broker_to_bro_vector(cv); print Broker::vector_size(cv); +print Broker::vector_clear(cv); +print Broker::vector_size(cv); +print broker_to_bro_vector(cv); print "***************************"; -local cr = Broker::data(r); -print broker_to_bro_record(cr); -r$a = "test"; -cr = Broker::data(r); -print broker_to_bro_record(cr); -r$b = "testagain"; -cr = Broker::data(r); -print broker_to_bro_record(cr); +### Test the broker record BIFs + cr = Broker::record_create(3); print Broker::record_size(cr); print Broker::record_assign(cr, Broker::data("hi"), 0); @@ -230,4 +260,7 @@ print Broker::record_lookup(cr, 0); print Broker::record_lookup(cr, 1); print Broker::record_lookup(cr, 2); print Broker::record_size(cr); +print Broker::record_assign(cr, Broker::data("goodbye"), 1); +print Broker::record_size(cr); +print Broker::record_lookup(cr, 1); } From 12eb7a380ddf723125712f176bf91d3ba133d3fa Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 27 Apr 2016 00:47:48 -0500 Subject: [PATCH 66/84] Rename broker BIF wrapper functions in a few more places --- doc/frameworks/broker.rst | 8 ++++---- doc/frameworks/broker/events-connector.bro | 6 +++--- doc/frameworks/broker/printing-connector.bro | 6 +++--- .../output | 6 +++--- .../output | 6 +++--- ...clude-doc_frameworks_broker_events-connector_bro.btest | 6 +++--- ...ude-doc_frameworks_broker_printing-connector_bro.btest | 6 +++--- 7 files changed, 22 insertions(+), 22 deletions(-) diff --git a/doc/frameworks/broker.rst b/doc/frameworks/broker.rst index 328c465c18..9c9ed89514 100644 --- a/doc/frameworks/broker.rst +++ b/doc/frameworks/broker.rst @@ -45,7 +45,7 @@ received. .. btest-include:: ${DOC_ROOT}/frameworks/broker/printing-listener.bro -To send remote print messages, just call :bro:see:`Broker::print`. +To send remote print messages, just call :bro:see:`Broker::send_print`. .. btest-include:: ${DOC_ROOT}/frameworks/broker/printing-connector.bro @@ -75,7 +75,7 @@ new events along with handlers that peers may want to send. .. btest-include:: ${DOC_ROOT}/frameworks/broker/events-listener.bro There are two different ways to send events. The first is to call the -:bro:see:`Broker::event` function directly. The second option is to call +:bro:see:`Broker::send_event` function directly. The second option is to call the :bro:see:`Broker::auto_event` function where you specify a particular event that will be automatically sent to peers whenever the event is called locally via the normal event invocation syntax. @@ -144,8 +144,8 @@ If not using the ``auto_publish`` flag, one can use the functions to manipulate the set of message topics (must match exactly) that are allowed to be sent to peer endpoints. These settings take precedence over the per-message ``peers`` flag supplied to functions -that take a :bro:see:`Broker::SendFlags` such as :bro:see:`Broker::print`, -:bro:see:`Broker::event`, :bro:see:`Broker::auto_event` or +that take a :bro:see:`Broker::SendFlags` such as :bro:see:`Broker::send_print`, +:bro:see:`Broker::send_event`, :bro:see:`Broker::auto_event` or :bro:see:`Broker::enable_remote_logs`. If not using the ``auto_advertise`` flag, one can use the diff --git a/doc/frameworks/broker/events-connector.bro b/doc/frameworks/broker/events-connector.bro index 19a617c9cd..437e197925 100644 --- a/doc/frameworks/broker/events-connector.bro +++ b/doc/frameworks/broker/events-connector.bro @@ -17,11 +17,11 @@ event Broker::outgoing_connection_established(peer_address: string, { print "Broker::outgoing_connection_established", peer_address, peer_port, peer_name; - Broker::event("bro/event/my_event", Broker::event_args(my_event, "hi", 0)); + Broker::send_event("bro/event/my_event", Broker::event_args(my_event, "hi", 0)); event my_auto_event("stuff", 88); - Broker::event("bro/event/my_event", Broker::event_args(my_event, "...", 1)); + Broker::send_event("bro/event/my_event", Broker::event_args(my_event, "...", 1)); event my_auto_event("more stuff", 51); - Broker::event("bro/event/my_event", Broker::event_args(my_event, "bye", 2)); + Broker::send_event("bro/event/my_event", Broker::event_args(my_event, "bye", 2)); } event Broker::outgoing_connection_broken(peer_address: string, diff --git a/doc/frameworks/broker/printing-connector.bro b/doc/frameworks/broker/printing-connector.bro index 0ab14d926b..42d961669a 100644 --- a/doc/frameworks/broker/printing-connector.bro +++ b/doc/frameworks/broker/printing-connector.bro @@ -14,9 +14,9 @@ event Broker::outgoing_connection_established(peer_address: string, { print "Broker::outgoing_connection_established", peer_address, peer_port, peer_name; - Broker::print("bro/print/hi", "hello"); - Broker::print("bro/print/stuff", "..."); - Broker::print("bro/print/bye", "goodbye"); + Broker::send_print("bro/print/hi", "hello"); + Broker::send_print("bro/print/stuff", "..."); + Broker::send_print("bro/print/bye", "goodbye"); } event Broker::outgoing_connection_broken(peer_address: string, diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_events-connector_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_events-connector_bro/output index 8a88bde1c2..d7a0e64be2 100644 --- a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_events-connector_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_events-connector_bro/output @@ -21,11 +21,11 @@ event Broker::outgoing_connection_established(peer_address: string, { print "Broker::outgoing_connection_established", peer_address, peer_port, peer_name; - Broker::event("bro/event/my_event", Broker::event_args(my_event, "hi", 0)); + Broker::send_event("bro/event/my_event", Broker::event_args(my_event, "hi", 0)); event my_auto_event("stuff", 88); - Broker::event("bro/event/my_event", Broker::event_args(my_event, "...", 1)); + Broker::send_event("bro/event/my_event", Broker::event_args(my_event, "...", 1)); event my_auto_event("more stuff", 51); - Broker::event("bro/event/my_event", Broker::event_args(my_event, "bye", 2)); + Broker::send_event("bro/event/my_event", Broker::event_args(my_event, "bye", 2)); } event Broker::outgoing_connection_broken(peer_address: string, diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_printing-connector_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_printing-connector_bro/output index f332f6e4ca..91ee179fe6 100644 --- a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_printing-connector_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_broker_printing-connector_bro/output @@ -18,9 +18,9 @@ event Broker::outgoing_connection_established(peer_address: string, { print "Broker::outgoing_connection_established", peer_address, peer_port, peer_name; - Broker::print("bro/print/hi", "hello"); - Broker::print("bro/print/stuff", "..."); - Broker::print("bro/print/bye", "goodbye"); + Broker::send_print("bro/print/hi", "hello"); + Broker::send_print("bro/print/stuff", "..."); + Broker::send_print("bro/print/bye", "goodbye"); } event Broker::outgoing_connection_broken(peer_address: string, diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_broker_events-connector_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_broker_events-connector_bro.btest index 8a88bde1c2..d7a0e64be2 100644 --- a/testing/btest/doc/sphinx/include-doc_frameworks_broker_events-connector_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_frameworks_broker_events-connector_bro.btest @@ -21,11 +21,11 @@ event Broker::outgoing_connection_established(peer_address: string, { print "Broker::outgoing_connection_established", peer_address, peer_port, peer_name; - Broker::event("bro/event/my_event", Broker::event_args(my_event, "hi", 0)); + Broker::send_event("bro/event/my_event", Broker::event_args(my_event, "hi", 0)); event my_auto_event("stuff", 88); - Broker::event("bro/event/my_event", Broker::event_args(my_event, "...", 1)); + Broker::send_event("bro/event/my_event", Broker::event_args(my_event, "...", 1)); event my_auto_event("more stuff", 51); - Broker::event("bro/event/my_event", Broker::event_args(my_event, "bye", 2)); + Broker::send_event("bro/event/my_event", Broker::event_args(my_event, "bye", 2)); } event Broker::outgoing_connection_broken(peer_address: string, diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_broker_printing-connector_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_broker_printing-connector_bro.btest index f332f6e4ca..91ee179fe6 100644 --- a/testing/btest/doc/sphinx/include-doc_frameworks_broker_printing-connector_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_frameworks_broker_printing-connector_bro.btest @@ -18,9 +18,9 @@ event Broker::outgoing_connection_established(peer_address: string, { print "Broker::outgoing_connection_established", peer_address, peer_port, peer_name; - Broker::print("bro/print/hi", "hello"); - Broker::print("bro/print/stuff", "..."); - Broker::print("bro/print/bye", "goodbye"); + Broker::send_print("bro/print/hi", "hello"); + Broker::send_print("bro/print/stuff", "..."); + Broker::send_print("bro/print/bye", "goodbye"); } event Broker::outgoing_connection_broken(peer_address: string, From 3a70289e91b09640cda77a0534aa997a15fff40f Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Wed, 27 Apr 2016 06:51:04 -0700 Subject: [PATCH 67/84] ARP: remove unnecessary variables and add testcase BIT-1573 #close --- src/analyzer/protocol/arp/ARP.cc | 3 --- src/analyzer/protocol/arp/ARP.h | 4 ---- .../scripts.base.protocols.arp.basic/.stdout | 2 ++ testing/btest/Traces/arp-who-has.pcap | Bin 0 -> 158 bytes .../btest/scripts/base/protocols/arp/basic.test | 13 +++++++++++++ 5 files changed, 15 insertions(+), 7 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.protocols.arp.basic/.stdout create mode 100644 testing/btest/Traces/arp-who-has.pcap create mode 100644 testing/btest/scripts/base/protocols/arp/basic.test diff --git a/src/analyzer/protocol/arp/ARP.cc b/src/analyzer/protocol/arp/ARP.cc index 5cbb25451b..b9af26ecfa 100644 --- a/src/analyzer/protocol/arp/ARP.cc +++ b/src/analyzer/protocol/arp/ARP.cc @@ -10,9 +10,6 @@ using namespace analyzer::arp; ARP_Analyzer::ARP_Analyzer() { - bad_arp = internal_handler("bad_arp"); - arp_request = internal_handler("arp_request"); - arp_reply = internal_handler("arp_reply"); } ARP_Analyzer::~ARP_Analyzer() diff --git a/src/analyzer/protocol/arp/ARP.h b/src/analyzer/protocol/arp/ARP.h index c4deddee03..1bdd382714 100644 --- a/src/analyzer/protocol/arp/ARP.h +++ b/src/analyzer/protocol/arp/ARP.h @@ -50,10 +50,6 @@ protected: StringVal* EthAddrToStr(const u_char* addr); void BadARP(const struct arp_pkthdr* hdr, const char* string); void Corrupted(const char* string); - - EventHandlerPtr arp_corrupted_packet; - EventHandlerPtr arp_request; - EventHandlerPtr arp_reply; }; } } // namespace analyzer::* diff --git a/testing/btest/Baseline/scripts.base.protocols.arp.basic/.stdout b/testing/btest/Baseline/scripts.base.protocols.arp.basic/.stdout new file mode 100644 index 0000000000..d45f9ba0d7 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.arp.basic/.stdout @@ -0,0 +1,2 @@ +78:31:c1:c6:3f:c2, ff:ff:ff:ff:ff:ff, 10.0.0.2, 78:31:c1:c6:3f:c2, 10.0.0.1, 00:00:00:00:00:00 +f8:ed:a5:c0:a4:f1, 78:31:c1:c6:3f:c2, 10.0.0.1, f8:ed:a5:c0:a4:f1, 10.0.0.2, 78:31:c1:c6:3f:c2 diff --git a/testing/btest/Traces/arp-who-has.pcap b/testing/btest/Traces/arp-who-has.pcap new file mode 100644 index 0000000000000000000000000000000000000000..085dddf1fe6dce2a581b1d8d2e413ca3e069cc4a GIT binary patch literal 158 zcmca|c+)~A1{MYw`2U}Qff2|#cSs@p)F*ZZEg&0&|ARq=;lX3}hd9_67&#c&SQr=~ yd@cqCCWw9@ixH$#oP&eG1}Fr=5H&yEE Date: Wed, 27 Apr 2016 15:34:47 -0500 Subject: [PATCH 68/84] Update docs and tests of the fmt() function Removed tests and documentation of the "%A" format specifier, which was removed in commit 7344052b. --- src/bro.bif | 2 -- testing/btest/Baseline/bifs.fmt/out | 5 ----- testing/btest/bifs/fmt.bro | 22 ++++++---------------- 3 files changed, 6 insertions(+), 23 deletions(-) diff --git a/src/bro.bif b/src/bro.bif index 5d097734a4..f21f927f92 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1480,8 +1480,6 @@ function cat_sep%(sep: string, def: string, ...%): string ## ## - ``.``: Precision of floating point specifiers ``[efg]`` (< 128) ## -## - ``A``: Escape only NUL bytes (each one replaced with ``\0``) in a string -## ## - ``[DTdxsefg]``: Format specifier ## ## - ``[DT]``: ISO timestamp with microsecond precision diff --git a/testing/btest/Baseline/bifs.fmt/out b/testing/btest/Baseline/bifs.fmt/out index 5f380c1b22..2572f924fb 100644 --- a/testing/btest/Baseline/bifs.fmt/out +++ b/testing/btest/Baseline/bifs.fmt/out @@ -45,11 +45,6 @@ test 310 310 2 -1 2 2 -1 -2 -2 -1 2 diff --git a/testing/btest/bifs/fmt.bro b/testing/btest/bifs/fmt.bro index 93607c2740..7fc4dc38d7 100644 --- a/testing/btest/bifs/fmt.bro +++ b/testing/btest/bifs/fmt.bro @@ -65,26 +65,16 @@ event bro_init() print fmt("%.3g", 3.1e+2); print fmt("%.7g", 3.1e+2); - # Tests comparing "%As" and "%s" (the string length is printed instead - # of the string itself because the print command does its own escaping) - local s0 = "\x00\x07"; - local s1 = fmt("%As", s0); # expands \x00 to "\0" - local s2 = fmt("%s", s0); # expands \x00 to "\0", and \x07 to "^G" + # Tests of "%s" with non-printable characters (the string length is printed + # instead of the string itself because the print command does its own + # escaping) + local s0 = "\x00\x1f"; + local s1 = fmt("%s", s0); print |s0|; print |s1|; - print |s2|; - - s0 = "\x07\x1f"; - s1 = fmt("%As", s0); - s2 = fmt("%s", s0); # expands \x07 to "^G", and \x1f to "\x1f" - print |s0|; - print |s1|; - print |s2|; s0 = "\x7f\xff"; - s1 = fmt("%As", s0); - s2 = fmt("%s", s0); # expands \x7f to "^?", and \xff to "\xff" + s1 = fmt("%s", s0); print |s0|; print |s1|; - print |s2|; } From cd2ec7c49522629f2013f2ada5d8ad54dedf1103 Mon Sep 17 00:00:00 2001 From: Vitaly Repin Date: Thu, 28 Apr 2016 11:10:52 +0300 Subject: [PATCH 69/84] Unknown data link type error message printed out props.link_type instead of arg_props.link_type. It lead to the meaningless and misleading output (E.g.: 'unknown data link type 0xffffffff') --- src/iosource/PktSrc.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/iosource/PktSrc.cc b/src/iosource/PktSrc.cc index 8db9db6ef1..025432eba3 100644 --- a/src/iosource/PktSrc.cc +++ b/src/iosource/PktSrc.cc @@ -91,7 +91,7 @@ void PktSrc::Opened(const Properties& arg_props) { char buf[512]; safe_snprintf(buf, sizeof(buf), - "unknown data link type 0x%x", props.link_type); + "unknown data link type 0x%x", arg_props.link_type); Error(buf); Close(); return; From 380963b5063d53953462fa36452f12ac81bdc376 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 28 Apr 2016 10:06:01 -0700 Subject: [PATCH 70/84] Updating submodule(s). [nomail] --- aux/binpac | 2 +- aux/bro-aux | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- aux/broker | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aux/binpac b/aux/binpac index 424d40c1e8..edbbe445d9 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 424d40c1e8d5888311b50c0e5a9dfc9c5f818b66 +Subproject commit edbbe445d92cc6a5c2557661195f486b784769db diff --git a/aux/bro-aux b/aux/bro-aux index 105dfe4ad6..cb771a3cf5 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 105dfe4ad6c4ae4563b21cb0466ee350f0af0d43 +Subproject commit cb771a3cf592d46643eea35d206b9f3e1a0758f7 diff --git a/aux/broccoli b/aux/broccoli index f83038b17f..b4d1686cdd 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit f83038b17fc83788415a58d77f75ad182ca6a9b7 +Subproject commit b4d1686cdd3f5505e405667b1083e8335cae6928 diff --git a/aux/broctl b/aux/broctl index 583f3a3ff1..6583b0a84b 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 583f3a3ff1847cf96a87f865d5cf0f36fae9dd67 +Subproject commit 6583b0a84b59a90e671d6405613c35f8502ce023 diff --git a/aux/broker b/aux/broker index 6684ab5109..bb3f55f198 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 6684ab5109f526fb535013760f17a4c8dff093ae +Subproject commit bb3f55f198f9cfd5e545345dd6425dd08ca1d45e From f98561b85c88191a7cf4fdef0124caad89fdb48b Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 28 Apr 2016 11:29:00 -0700 Subject: [PATCH 71/84] Updating NEWS and a test baseline after merges. --- CHANGES | 11 +++++++++++ NEWS | 2 ++ VERSION | 2 +- testing/btest/Baseline/plugins.hooks/output | 20 ++++++++++++++------ 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/CHANGES b/CHANGES index 5f88f92cf5..d2ead001df 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,15 @@ +2.4-493 | 2016-04-28 11:29:00 -0700 + + * Rename Broker::print to Broker::send_print and Broker::event to + Broker::send_event to avoid using reserved keywords as function + names. (Daniel Thayer) + + * Add script wrapper functions for Broker BIFs. This faciliates + documenting them through Broxygen. (Daniel Thayer) + + * Extend, update, and clean up Broker tests. (Daniel Thayer) + 2.4-485 | 2016-04-28 10:18:46 -0700 * Intel: Allow to provide uid/fuid instead of conn/file. (Johanna diff --git a/NEWS b/NEWS index 0ee0eb6670..e87c884e72 100644 --- a/NEWS +++ b/NEWS @@ -96,6 +96,8 @@ Changed Functionality --------------------- - The BrokerComm and BrokerStore namespaces were renamed to Broker. + The Broker "print" function was renamed to Broker::send_print, and + "event" to "Broker::send_event". - ``SSH::skip_processing_after_detection`` was removed. The functionality was replaced by ``SSH::disable_analyzer_after_detection``. diff --git a/VERSION b/VERSION index f2db26a997..ca50394da0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4-485 +2.4-493 diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index a30a37bf95..61099efaf9 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -233,7 +233,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461694342.200388, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Communication::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Conn::LOG)) -> @@ -354,7 +354,7 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461694342.200388, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(NetControl::check_plugins, , ()) -> 0.000000 MetaHookPost CallFunction(NetControl::init, , ()) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, , ()) -> @@ -570,11 +570,13 @@ 0.000000 MetaHookPost LoadFile(base<...>/bro.bif) -> -1 0.000000 MetaHookPost LoadFile(base<...>/broker) -> -1 0.000000 MetaHookPost LoadFile(base<...>/cluster) -> -1 +0.000000 MetaHookPost LoadFile(base<...>/comm.bif) -> -1 0.000000 MetaHookPost LoadFile(base<...>/communication) -> -1 0.000000 MetaHookPost LoadFile(base<...>/conn) -> -1 0.000000 MetaHookPost LoadFile(base<...>/conn-ids) -> -1 0.000000 MetaHookPost LoadFile(base<...>/const.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(base<...>/control) -> -1 +0.000000 MetaHookPost LoadFile(base<...>/data.bif) -> -1 0.000000 MetaHookPost LoadFile(base<...>/dhcp) -> -1 0.000000 MetaHookPost LoadFile(base<...>/dir) -> -1 0.000000 MetaHookPost LoadFile(base<...>/directions-and-hosts) -> -1 @@ -601,6 +603,7 @@ 0.000000 MetaHookPost LoadFile(base<...>/logging) -> -1 0.000000 MetaHookPost LoadFile(base<...>/logging.bif) -> -1 0.000000 MetaHookPost LoadFile(base<...>/main) -> -1 +0.000000 MetaHookPost LoadFile(base<...>/messaging.bif) -> -1 0.000000 MetaHookPost LoadFile(base<...>/modbus) -> -1 0.000000 MetaHookPost LoadFile(base<...>/mysql) -> -1 0.000000 MetaHookPost LoadFile(base<...>/netcontrol) -> -1 @@ -628,6 +631,7 @@ 0.000000 MetaHookPost LoadFile(base<...>/software) -> -1 0.000000 MetaHookPost LoadFile(base<...>/ssh) -> -1 0.000000 MetaHookPost LoadFile(base<...>/ssl) -> -1 +0.000000 MetaHookPost LoadFile(base<...>/store.bif) -> -1 0.000000 MetaHookPost LoadFile(base<...>/strings) -> -1 0.000000 MetaHookPost LoadFile(base<...>/strings.bif) -> -1 0.000000 MetaHookPost LoadFile(base<...>/sumstats) -> -1 @@ -878,7 +882,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461694342.200388, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Communication::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Conn::LOG)) @@ -999,7 +1003,7 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461694342.200388, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(NetControl::check_plugins, , ()) 0.000000 MetaHookPre CallFunction(NetControl::init, , ()) 0.000000 MetaHookPre CallFunction(Notice::want_pp, , ()) @@ -1215,11 +1219,13 @@ 0.000000 MetaHookPre LoadFile(base<...>/bro.bif) 0.000000 MetaHookPre LoadFile(base<...>/broker) 0.000000 MetaHookPre LoadFile(base<...>/cluster) +0.000000 MetaHookPre LoadFile(base<...>/comm.bif) 0.000000 MetaHookPre LoadFile(base<...>/communication) 0.000000 MetaHookPre LoadFile(base<...>/conn) 0.000000 MetaHookPre LoadFile(base<...>/conn-ids) 0.000000 MetaHookPre LoadFile(base<...>/const.bif.bro) 0.000000 MetaHookPre LoadFile(base<...>/control) +0.000000 MetaHookPre LoadFile(base<...>/data.bif) 0.000000 MetaHookPre LoadFile(base<...>/dhcp) 0.000000 MetaHookPre LoadFile(base<...>/dir) 0.000000 MetaHookPre LoadFile(base<...>/directions-and-hosts) @@ -1246,6 +1252,7 @@ 0.000000 MetaHookPre LoadFile(base<...>/logging) 0.000000 MetaHookPre LoadFile(base<...>/logging.bif) 0.000000 MetaHookPre LoadFile(base<...>/main) +0.000000 MetaHookPre LoadFile(base<...>/messaging.bif) 0.000000 MetaHookPre LoadFile(base<...>/modbus) 0.000000 MetaHookPre LoadFile(base<...>/mysql) 0.000000 MetaHookPre LoadFile(base<...>/netcontrol) @@ -1273,6 +1280,7 @@ 0.000000 MetaHookPre LoadFile(base<...>/software) 0.000000 MetaHookPre LoadFile(base<...>/ssh) 0.000000 MetaHookPre LoadFile(base<...>/ssl) +0.000000 MetaHookPre LoadFile(base<...>/store.bif) 0.000000 MetaHookPre LoadFile(base<...>/strings) 0.000000 MetaHookPre LoadFile(base<...>/strings.bif) 0.000000 MetaHookPre LoadFile(base<...>/sumstats) @@ -1522,7 +1530,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1461694342.200388, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG) @@ -1643,7 +1651,7 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1461694342.200388, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction NetControl::check_plugins() 0.000000 | HookCallFunction NetControl::init() 0.000000 | HookCallFunction Notice::want_pp() From cc54b3772a08071bc24aebf53f3e03c3b64bedbb Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 28 Apr 2016 20:13:44 -0700 Subject: [PATCH 72/84] Updating submodule(s). [nomail] --- aux/broctl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broctl b/aux/broctl index 6583b0a84b..7df7878abf 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 6583b0a84b59a90e671d6405613c35f8502ce023 +Subproject commit 7df7878abfd864f9ae5609918c0f04f58b5f5e2d From 373c872e939f97c498b029cd08d4b24c0ab71c70 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Fri, 29 Apr 2016 01:45:59 -0500 Subject: [PATCH 73/84] Fix a few incorrect type tags in Bro broker source code These are just used for error reporting. --- src/broker/Data.h | 2 +- src/broker/data.bif | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/broker/Data.h b/src/broker/Data.h index f212979853..0045ad58ad 100644 --- a/src/broker/Data.h +++ b/src/broker/Data.h @@ -243,7 +243,7 @@ public: RecordIterator(RecordVal* v, TypeTag tag, Frame* f) : OpaqueVal(bro_broker::opaque_of_record_iterator), - dat(require_data_type(v, TYPE_VECTOR, f)), + dat(require_data_type(v, TYPE_RECORD, f)), it(dat.fields.begin()) {} diff --git a/src/broker/data.bif b/src/broker/data.bif index 1788931d86..d526d0a779 100644 --- a/src/broker/data.bif +++ b/src/broker/data.bif @@ -87,7 +87,7 @@ function Broker::__refine_to_subnet%(d: Broker::Data%): subnet function Broker::__refine_to_port%(d: Broker::Data%): port %{ auto& a = bro_broker::require_data_type(d->AsRecordVal(), - TYPE_SUBNET, frame); + TYPE_PORT, frame); return new PortVal(a.number(), bro_broker::to_bro_port_proto(a.type())); %} @@ -101,7 +101,7 @@ function Broker::__refine_to_time%(d: Broker::Data%): time function Broker::__refine_to_interval%(d: Broker::Data%): interval %{ auto v = bro_broker::require_data_type(d->AsRecordVal(), - TYPE_TIME, frame).value; + TYPE_INTERVAL, frame).value; return new Val(v, TYPE_INTERVAL); %} From f2acaec9b7512418f7b71947da90810ab082486e Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Fri, 29 Apr 2016 13:50:52 -0700 Subject: [PATCH 74/84] XMPP: Add StartTLS event and update tests Also tiny cleanyp to the code. --- src/analyzer/protocol/xmpp/CMakeLists.txt | 1 + src/analyzer/protocol/xmpp/Plugin.cc | 5 +-- src/analyzer/protocol/xmpp/XMPP.cc | 3 +- src/analyzer/protocol/xmpp/XMPP.h | 10 +++--- src/analyzer/protocol/xmpp/events.bif | 5 +++ src/analyzer/protocol/xmpp/xmpp-analyzer.pac | 5 ++- src/analyzer/protocol/xmpp/xmpp.pac | 3 ++ .../Baseline/core.print-bpf-filters/output2 | 10 +++--- .../canonified_loaded_scripts.log | 5 +-- .../canonified_loaded_scripts.log | 7 +++-- testing/btest/Baseline/plugins.hooks/output | 31 +++++++++++++++---- 11 files changed, 59 insertions(+), 26 deletions(-) create mode 100644 src/analyzer/protocol/xmpp/events.bif diff --git a/src/analyzer/protocol/xmpp/CMakeLists.txt b/src/analyzer/protocol/xmpp/CMakeLists.txt index 408f01d47c..ec5bb84837 100644 --- a/src/analyzer/protocol/xmpp/CMakeLists.txt +++ b/src/analyzer/protocol/xmpp/CMakeLists.txt @@ -6,6 +6,7 @@ include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DI bro_plugin_begin(Bro XMPP) bro_plugin_cc(Plugin.cc) bro_plugin_cc(XMPP.cc) +bro_plugin_bif(events.bif) bro_plugin_pac(xmpp.pac xmpp-analyzer.pac xmpp-protocol.pac) bro_plugin_end() diff --git a/src/analyzer/protocol/xmpp/Plugin.cc b/src/analyzer/protocol/xmpp/Plugin.cc index b4332b447b..d3bfcc5b10 100644 --- a/src/analyzer/protocol/xmpp/Plugin.cc +++ b/src/analyzer/protocol/xmpp/Plugin.cc @@ -1,6 +1,4 @@ // See the file in the main distribution directory for copyright. - - #include "plugin/Plugin.h" #include "XMPP.h" @@ -14,10 +12,9 @@ public: { AddComponent(new ::analyzer::Component("XMPP", ::analyzer::xmpp::XMPP_Analyzer::Instantiate)); - plugin::Configuration config; config.name = "Bro::XMPP"; - config.description = "XMPP analyzer StartTLS only"; + config.description = "XMPP analyzer (StartTLS only)"; return config; } } plugin; diff --git a/src/analyzer/protocol/xmpp/XMPP.cc b/src/analyzer/protocol/xmpp/XMPP.cc index ee2667a276..72229aeaba 100644 --- a/src/analyzer/protocol/xmpp/XMPP.cc +++ b/src/analyzer/protocol/xmpp/XMPP.cc @@ -9,14 +9,13 @@ using namespace analyzer::xmpp; XMPP_Analyzer::XMPP_Analyzer(Connection* conn) : tcp::TCP_ApplicationAnalyzer("XMPP", conn) { - interp = new binpac::XMPP::XMPP_Conn(this); + interp = unique_ptr(new binpac::XMPP::XMPP_Conn(this)); had_gap = false; tls_active = false; } XMPP_Analyzer::~XMPP_Analyzer() { - delete interp; } void XMPP_Analyzer::Done() diff --git a/src/analyzer/protocol/xmpp/XMPP.h b/src/analyzer/protocol/xmpp/XMPP.h index 628be7bb2d..202403748a 100644 --- a/src/analyzer/protocol/xmpp/XMPP.h +++ b/src/analyzer/protocol/xmpp/XMPP.h @@ -14,12 +14,12 @@ public: XMPP_Analyzer(Connection* conn); virtual ~XMPP_Analyzer(); - virtual void Done(); - virtual void DeliverStream(int len, const u_char* data, bool orig); - virtual void Undelivered(uint64 seq, int len, bool orig); + void Done() override; + void DeliverStream(int len, const u_char* data, bool orig) override; + void Undelivered(uint64 seq, int len, bool orig) override; // Overriden from tcp::TCP_ApplicationAnalyzer. - virtual void EndpointEOF(bool is_orig); + void EndpointEOF(bool is_orig) override; void StartTLS(); @@ -27,7 +27,7 @@ public: { return new XMPP_Analyzer(conn); } protected: - binpac::XMPP::XMPP_Conn* interp; + std::unique_ptr interp; bool had_gap; bool tls_active; diff --git a/src/analyzer/protocol/xmpp/events.bif b/src/analyzer/protocol/xmpp/events.bif new file mode 100644 index 0000000000..ee36bd5333 --- /dev/null +++ b/src/analyzer/protocol/xmpp/events.bif @@ -0,0 +1,5 @@ +## Generated when a XMPP connection goes encrypted after a successful +## StartTLS exchange between the client and the server. +## +## c: The connection. +event xmpp_starttls%(c: connection%); diff --git a/src/analyzer/protocol/xmpp/xmpp-analyzer.pac b/src/analyzer/protocol/xmpp/xmpp-analyzer.pac index 90b51ec183..3240b57bb3 100644 --- a/src/analyzer/protocol/xmpp/xmpp-analyzer.pac +++ b/src/analyzer/protocol/xmpp/xmpp-analyzer.pac @@ -25,7 +25,10 @@ refine connection XMPP_Conn += { client_starttls = true; if ( !is_orig && token == "proceed" && client_starttls ) + { bro_analyzer()->StartTLS(); + BifEvent::generate_xmpp_starttls(bro_analyzer(), bro_analyzer()->Conn()); + } else if ( !is_orig && token == "proceed" ) reporter->Weird(bro_analyzer()->Conn(), "XMPP: proceed without starttls"); @@ -37,6 +40,6 @@ refine connection XMPP_Conn += { }; refine typeattr XMPP_TOKEN += &let { - proc: bool = $context.connection.proc_xmpp_token(is_orig, name, rest); + proc: bool = $context.connection.proc_xmpp_token(is_orig, name, rest); }; diff --git a/src/analyzer/protocol/xmpp/xmpp.pac b/src/analyzer/protocol/xmpp/xmpp.pac index 42ec85f0cc..e6b5f4bba0 100644 --- a/src/analyzer/protocol/xmpp/xmpp.pac +++ b/src/analyzer/protocol/xmpp/xmpp.pac @@ -6,7 +6,10 @@ %include binpac.pac %include bro.pac + %extern{ +#include "events.bif.h" + namespace analyzer { namespace xmpp { class XMPP_Analyzer; } } namespace binpac { namespace XMPP { class XMPP_Conn; } } typedef analyzer::xmpp::XMPP_Analyzer* XMPPAnalyzer; diff --git a/testing/btest/Baseline/core.print-bpf-filters/output2 b/testing/btest/Baseline/core.print-bpf-filters/output2 index d0f448441b..3321684b43 100644 --- a/testing/btest/Baseline/core.print-bpf-filters/output2 +++ b/testing/btest/Baseline/core.print-bpf-filters/output2 @@ -21,7 +21,9 @@ 1 5060 1 5072 1 514 +1 5222 1 5223 +1 5269 2 53 1 5353 1 5355 @@ -48,8 +50,8 @@ 1 992 1 993 1 995 -55 and -54 or -55 port -37 tcp +57 and +56 or +57 port +39 tcp 18 udp diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 703db6ea63..65f93aa51d 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2016-04-26-18-11-39 +#open 2016-04-29-20-49-16 #fields name #types string scripts/base/init-bare.bro @@ -111,6 +111,7 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_TCP.functions.bif.bro build/scripts/base/bif/plugins/Bro_Teredo.events.bif.bro build/scripts/base/bif/plugins/Bro_UDP.events.bif.bro + build/scripts/base/bif/plugins/Bro_XMPP.events.bif.bro build/scripts/base/bif/plugins/Bro_ZIP.events.bif.bro build/scripts/base/bif/plugins/Bro_FileEntropy.events.bif.bro build/scripts/base/bif/plugins/Bro_FileExtract.events.bif.bro @@ -132,4 +133,4 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro scripts/policy/misc/loaded-scripts.bro scripts/base/utils/paths.bro -#close 2016-04-26-18-11-39 +#close 2016-04-29-20-49-16 diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index c7a3c03d09..6ea7dd5d17 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2016-04-26-18-11-49 +#open 2016-04-29-20-49-25 #fields name #types string scripts/base/init-bare.bro @@ -111,6 +111,7 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_TCP.functions.bif.bro build/scripts/base/bif/plugins/Bro_Teredo.events.bif.bro build/scripts/base/bif/plugins/Bro_UDP.events.bif.bro + build/scripts/base/bif/plugins/Bro_XMPP.events.bif.bro build/scripts/base/bif/plugins/Bro_ZIP.events.bif.bro build/scripts/base/bif/plugins/Bro_FileEntropy.events.bif.bro build/scripts/base/bif/plugins/Bro_FileExtract.events.bif.bro @@ -295,6 +296,8 @@ scripts/base/init-default.bro scripts/base/protocols/syslog/consts.bro scripts/base/protocols/syslog/main.bro scripts/base/protocols/tunnels/__load__.bro + scripts/base/protocols/xmpp/__load__.bro + scripts/base/protocols/xmpp/main.bro scripts/base/files/pe/__load__.bro scripts/base/files/pe/consts.bro scripts/base/files/pe/main.bro @@ -305,4 +308,4 @@ scripts/base/init-default.bro scripts/base/misc/find-checksum-offloading.bro scripts/base/misc/find-filtered-trace.bro scripts/policy/misc/loaded-scripts.bro -#close 2016-04-26-18-11-49 +#close 2016-04-29-20-49-25 diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 61099efaf9..186f3a4a2a 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -57,6 +57,8 @@ 0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_SSL, 995/tcp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_SYSLOG, 514/udp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_TEREDO, 3544/udp)) -> +0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_XMPP, 5222/tcp)) -> +0.000000 MetaHookPost CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_XMPP, 5269/tcp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_BACKDOOR)) -> 0.000000 MetaHookPost CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_INTERCONN)) -> 0.000000 MetaHookPost CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_STEPPINGSTONE)) -> @@ -116,6 +118,8 @@ 0.000000 MetaHookPost CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_SSL, 995/tcp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_SYSLOG, 514/udp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_TEREDO, 3544/udp)) -> +0.000000 MetaHookPost CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_XMPP, 5222/tcp)) -> +0.000000 MetaHookPost CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_XMPP, 5269/tcp)) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_AYIYA, {5072/udp})) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_DHCP, {67<...>/udp})) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_DNP3_TCP, {20000<...>/tcp})) -> @@ -140,6 +144,7 @@ 0.000000 MetaHookPost CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_SSL, {5223<...>/tcp})) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_SYSLOG, {514/udp})) -> 0.000000 MetaHookPost CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_TEREDO, {3544/udp})) -> +0.000000 MetaHookPost CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_XMPP, {5222<...>/tcp})) -> 0.000000 MetaHookPost CallFunction(Cluster::is_enabled, , ()) -> 0.000000 MetaHookPost CallFunction(Cluster::is_enabled, , ()) -> 0.000000 MetaHookPost CallFunction(Files::register_analyzer_add_callback, , (Files::ANALYZER_EXTRACT, FileExtract::on_add{ if (!FileExtract::args?$extract_filename) FileExtract::args$extract_filename = cat(extract-, FileExtract::f$last_active, -, FileExtract::f$source, -, FileExtract::f$id)FileExtract::f$info$extracted = FileExtract::args$extract_filenameFileExtract::args$extract_filename = build_path_compressed(FileExtract::prefix, FileExtract::args$extract_filename)mkdir(FileExtract::prefix)})) -> @@ -233,7 +238,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461962978.799805, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Communication::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Conn::LOG)) -> @@ -354,7 +359,7 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461962978.799805, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(NetControl::check_plugins, , ()) -> 0.000000 MetaHookPost CallFunction(NetControl::init, , ()) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, , ()) -> @@ -467,6 +472,7 @@ 0.000000 MetaHookPost LoadFile(./Bro_X509.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_X509.functions.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_X509.types.bif.bro) -> -1 +0.000000 MetaHookPost LoadFile(./Bro_XMPP.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_ZIP.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./acld) -> -1 0.000000 MetaHookPost LoadFile(./addrs) -> -1 @@ -644,6 +650,7 @@ 0.000000 MetaHookPost LoadFile(base<...>/urls) -> -1 0.000000 MetaHookPost LoadFile(base<...>/utils) -> -1 0.000000 MetaHookPost LoadFile(base<...>/x509) -> -1 +0.000000 MetaHookPost LoadFile(base<...>/xmpp) -> -1 0.000000 MetaHookPost QueueEvent(NetControl::init()) -> false 0.000000 MetaHookPost QueueEvent(bro_init()) -> false 0.000000 MetaHookPost QueueEvent(filter_change_tracking()) -> false @@ -706,6 +713,8 @@ 0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_SSL, 995/tcp)) 0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_SYSLOG, 514/udp)) 0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_TEREDO, 3544/udp)) +0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_XMPP, 5222/tcp)) +0.000000 MetaHookPre CallFunction(Analyzer::__register_for_port, , (Analyzer::ANALYZER_XMPP, 5269/tcp)) 0.000000 MetaHookPre CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_BACKDOOR)) 0.000000 MetaHookPre CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_INTERCONN)) 0.000000 MetaHookPre CallFunction(Analyzer::disable_analyzer, , (Analyzer::ANALYZER_STEPPINGSTONE)) @@ -765,6 +774,8 @@ 0.000000 MetaHookPre CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_SSL, 995/tcp)) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_SYSLOG, 514/udp)) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_TEREDO, 3544/udp)) +0.000000 MetaHookPre CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_XMPP, 5222/tcp)) +0.000000 MetaHookPre CallFunction(Analyzer::register_for_port, , (Analyzer::ANALYZER_XMPP, 5269/tcp)) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_AYIYA, {5072/udp})) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_DHCP, {67<...>/udp})) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_DNP3_TCP, {20000<...>/tcp})) @@ -789,6 +800,7 @@ 0.000000 MetaHookPre CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_SSL, {5223<...>/tcp})) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_SYSLOG, {514/udp})) 0.000000 MetaHookPre CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_TEREDO, {3544/udp})) +0.000000 MetaHookPre CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_XMPP, {5222<...>/tcp})) 0.000000 MetaHookPre CallFunction(Cluster::is_enabled, , ()) 0.000000 MetaHookPre CallFunction(Cluster::is_enabled, , ()) 0.000000 MetaHookPre CallFunction(Files::register_analyzer_add_callback, , (Files::ANALYZER_EXTRACT, FileExtract::on_add{ if (!FileExtract::args?$extract_filename) FileExtract::args$extract_filename = cat(extract-, FileExtract::f$last_active, -, FileExtract::f$source, -, FileExtract::f$id)FileExtract::f$info$extracted = FileExtract::args$extract_filenameFileExtract::args$extract_filename = build_path_compressed(FileExtract::prefix, FileExtract::args$extract_filename)mkdir(FileExtract::prefix)})) @@ -882,7 +894,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461962978.799805, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Communication::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Conn::LOG)) @@ -1003,7 +1015,7 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461962978.799805, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(NetControl::check_plugins, , ()) 0.000000 MetaHookPre CallFunction(NetControl::init, , ()) 0.000000 MetaHookPre CallFunction(Notice::want_pp, , ()) @@ -1116,6 +1128,7 @@ 0.000000 MetaHookPre LoadFile(./Bro_X509.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_X509.functions.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_X509.types.bif.bro) +0.000000 MetaHookPre LoadFile(./Bro_XMPP.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_ZIP.events.bif.bro) 0.000000 MetaHookPre LoadFile(./acld) 0.000000 MetaHookPre LoadFile(./addrs) @@ -1293,6 +1306,7 @@ 0.000000 MetaHookPre LoadFile(base<...>/urls) 0.000000 MetaHookPre LoadFile(base<...>/utils) 0.000000 MetaHookPre LoadFile(base<...>/x509) +0.000000 MetaHookPre LoadFile(base<...>/xmpp) 0.000000 MetaHookPre QueueEvent(NetControl::init()) 0.000000 MetaHookPre QueueEvent(bro_init()) 0.000000 MetaHookPre QueueEvent(filter_change_tracking()) @@ -1355,6 +1369,8 @@ 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_SSL, 995/tcp) 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_SYSLOG, 514/udp) 0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_TEREDO, 3544/udp) +0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_XMPP, 5222/tcp) +0.000000 | HookCallFunction Analyzer::__register_for_port(Analyzer::ANALYZER_XMPP, 5269/tcp) 0.000000 | HookCallFunction Analyzer::disable_analyzer(Analyzer::ANALYZER_BACKDOOR) 0.000000 | HookCallFunction Analyzer::disable_analyzer(Analyzer::ANALYZER_INTERCONN) 0.000000 | HookCallFunction Analyzer::disable_analyzer(Analyzer::ANALYZER_STEPPINGSTONE) @@ -1414,6 +1430,8 @@ 0.000000 | HookCallFunction Analyzer::register_for_port(Analyzer::ANALYZER_SSL, 995/tcp) 0.000000 | HookCallFunction Analyzer::register_for_port(Analyzer::ANALYZER_SYSLOG, 514/udp) 0.000000 | HookCallFunction Analyzer::register_for_port(Analyzer::ANALYZER_TEREDO, 3544/udp) +0.000000 | HookCallFunction Analyzer::register_for_port(Analyzer::ANALYZER_XMPP, 5222/tcp) +0.000000 | HookCallFunction Analyzer::register_for_port(Analyzer::ANALYZER_XMPP, 5269/tcp) 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_AYIYA, {5072/udp}) 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_DHCP, {67<...>/udp}) 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_DNP3_TCP, {20000<...>/tcp}) @@ -1438,6 +1456,7 @@ 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_SSL, {5223<...>/tcp}) 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_SYSLOG, {514/udp}) 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_TEREDO, {3544/udp}) +0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_XMPP, {5222<...>/tcp}) 0.000000 | HookCallFunction Cluster::is_enabled() 0.000000 | HookCallFunction Files::register_analyzer_add_callback(Files::ANALYZER_EXTRACT, FileExtract::on_add{ if (!FileExtract::args?$extract_filename) FileExtract::args$extract_filename = cat(extract-, FileExtract::f$last_active, -, FileExtract::f$source, -, FileExtract::f$id)FileExtract::f$info$extracted = FileExtract::args$extract_filenameFileExtract::args$extract_filename = build_path_compressed(FileExtract::prefix, FileExtract::args$extract_filename)mkdir(FileExtract::prefix)}) 0.000000 | HookCallFunction Files::register_for_mime_type(Files::ANALYZER_PE, application/x-dosexec) @@ -1530,7 +1549,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1461962978.799805, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG) @@ -1651,7 +1670,7 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1461962978.799805, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction NetControl::check_plugins() 0.000000 | HookCallFunction NetControl::init() 0.000000 | HookCallFunction Notice::want_pp() From da014e1eca6136ff729eb11aacdf11688bcbb64d Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 2 May 2016 16:20:53 -0400 Subject: [PATCH 75/84] Rename the reporting interval variable for stats. --- scripts/policy/misc/stats.bro | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index d154da05e9..50032f6ec4 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -8,7 +8,7 @@ export { redef enum Log::ID += { LOG }; ## How often stats are reported. - const stats_report_interval = 5min &redef; + const report_interval = 5min &redef; type Info: record { ## Timestamp for the measurement. @@ -146,10 +146,10 @@ event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: Pr } Log::write(Stats::LOG, info); - schedule stats_report_interval { check_stats(nettime, ns, cs, ps, es, rs, ts, fs, ds) }; + schedule report_interval { check_stats(nettime, ns, cs, ps, es, rs, ts, fs, ds) }; } event bro_init() { - schedule stats_report_interval { check_stats(network_time(), get_net_stats(), get_conn_stats(), get_proc_stats(), get_event_stats(), get_reassembler_stats(), get_timer_stats(), get_file_analysis_stats(), get_dns_stats()) }; + schedule report_interval { check_stats(network_time(), get_net_stats(), get_conn_stats(), get_proc_stats(), get_event_stats(), get_reassembler_stats(), get_timer_stats(), get_file_analysis_stats(), get_dns_stats()) }; } From f8f599832832e027b8019554eae2d430f2193251 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 2 May 2016 16:43:08 -0400 Subject: [PATCH 76/84] Fixing tests for stats improvements --- scripts/base/init-bare.bro | 44 ++++++++++++------- .../canonified_loaded_scripts.log | 5 ++- .../canonified_loaded_scripts.log | 5 ++- testing/btest/Baseline/plugins.hooks/output | 26 ++++++----- testing/btest/bifs/net_stats_trace.test | 2 +- testing/btest/bifs/resource_usage.bro | 9 ---- 6 files changed, 50 insertions(+), 41 deletions(-) delete mode 100644 testing/btest/bifs/resource_usage.bro diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index c433aae503..5430d52ba4 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -535,7 +535,7 @@ type ReassemblerStats: record { unknown_size: count; ##< Byte size of reassembly tracking for unknown purposes. }; -## Summary statistics of all regular expression matchers. +## Statistics of all regular expression matchers. ## ## .. bro:see:: get_matcher_stats type MatcherStats: record { @@ -548,37 +548,51 @@ type MatcherStats: record { misses: count; ##< Number of cache misses. }; +## Statistics of timers. +## +## .. bro:see:: get_timer_stats type TimerStats: record { current: count; ##< Current number of pending timers. max: count; ##< Maximum number of concurrent timers pending so far. - cumulative: count; + cumulative: count; ##< Cumulative number of timers scheduled. }; +## Statistics of file analysis. +## +## .. bro:see:: get_file_analysis_stats type FileAnalysisStats: record { - current: count; - max: count; - cumulative: count; + current: count; ##< Current number of files being analyzed. + max: count; ##< Maximum number of concurrent files so far. + cumulative: count; ##< Cumulative number of files analyzed. }; +## Statistics related to Bro's active use of DNS. These numbers are +## about Bro performing DNS queries on it's own, not traffic +## being seen. +## +## .. bro:see:: get_dns_stats type DNSStats: record { - requests: count; - successful: count; - failed: count; - pending: count; - cached_hosts: count; - cached_addresses: count; + requests: count; ##< Number of DNS requests made + successful: count; ##< Number of successful DNS replies. + failed: count; ##< Number of DNS reply failures. + pending: count; ##< Current pending queries. + cached_hosts: count; ##< Number of cached hosts. + cached_addresses: count; ##< Number of cached addresses. }; ## Statistics about number of gaps in TCP connections. ## ## .. bro:see:: get_gap_stats type GapStats: record { - ack_events: count; ##< How many ack events *could* have had gaps. - ack_bytes: count; ##< How many bytes those covered. - gap_events: count; ##< How many *did* have gaps. - gap_bytes: count; ##< How many bytes were missing in the gaps. + ack_events: count; ##< How many ack events *could* have had gaps. + ack_bytes: count; ##< How many bytes those covered. + gap_events: count; ##< How many *did* have gaps. + gap_bytes: count; ##< How many bytes were missing in the gaps. }; +## Statistics about threads. +## +## .. bro:see:: get_thread_stats type ThreadStats: record { num_threads: count; }; diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 703db6ea63..f3fbccdd52 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2016-04-26-18-11-39 +#open 2016-05-02-20-39-26 #fields name #types string scripts/base/init-bare.bro @@ -50,6 +50,7 @@ scripts/base/init-bare.bro scripts/base/utils/patterns.bro scripts/base/frameworks/files/magic/__load__.bro build/scripts/base/bif/__load__.bro + build/scripts/base/bif/stats.bif.bro build/scripts/base/bif/broxygen.bif.bro build/scripts/base/bif/functions.bif.bro build/scripts/base/bif/bloom-filter.bif.bro @@ -132,4 +133,4 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro scripts/policy/misc/loaded-scripts.bro scripts/base/utils/paths.bro -#close 2016-04-26-18-11-39 +#close 2016-05-02-20-39-26 diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index c7a3c03d09..37cfa6ff28 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2016-04-26-18-11-49 +#open 2016-05-02-20-39-35 #fields name #types string scripts/base/init-bare.bro @@ -50,6 +50,7 @@ scripts/base/init-bare.bro scripts/base/utils/patterns.bro scripts/base/frameworks/files/magic/__load__.bro build/scripts/base/bif/__load__.bro + build/scripts/base/bif/stats.bif.bro build/scripts/base/bif/broxygen.bif.bro build/scripts/base/bif/functions.bif.bro build/scripts/base/bif/bloom-filter.bif.bro @@ -305,4 +306,4 @@ scripts/base/init-default.bro scripts/base/misc/find-checksum-offloading.bro scripts/base/misc/find-filtered-trace.bro scripts/policy/misc/loaded-scripts.bro -#close 2016-04-26-18-11-49 +#close 2016-05-02-20-39-35 diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 61099efaf9..186fc55040 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -233,7 +233,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1462221741.258723, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Communication::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Conn::LOG)) -> @@ -354,7 +354,7 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1462221741.258723, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(NetControl::check_plugins, , ()) -> 0.000000 MetaHookPost CallFunction(NetControl::init, , ()) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, , ()) -> @@ -527,6 +527,7 @@ 0.000000 MetaHookPost LoadFile(./sftp) -> -1 0.000000 MetaHookPost LoadFile(./shunt) -> -1 0.000000 MetaHookPost LoadFile(./site) -> -1 +0.000000 MetaHookPost LoadFile(./stats.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./std-dev) -> -1 0.000000 MetaHookPost LoadFile(./store) -> -1 0.000000 MetaHookPost LoadFile(./store.bif.bro) -> -1 @@ -882,7 +883,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1462221741.258723, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Communication::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Conn::LOG)) @@ -1003,7 +1004,7 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1462221741.258723, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(NetControl::check_plugins, , ()) 0.000000 MetaHookPre CallFunction(NetControl::init, , ()) 0.000000 MetaHookPre CallFunction(Notice::want_pp, , ()) @@ -1176,6 +1177,7 @@ 0.000000 MetaHookPre LoadFile(./sftp) 0.000000 MetaHookPre LoadFile(./shunt) 0.000000 MetaHookPre LoadFile(./site) +0.000000 MetaHookPre LoadFile(./stats.bif.bro) 0.000000 MetaHookPre LoadFile(./std-dev) 0.000000 MetaHookPre LoadFile(./store) 0.000000 MetaHookPre LoadFile(./store.bif.bro) @@ -1530,7 +1532,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1462221741.258723, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG) @@ -1651,7 +1653,7 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1461868125.285894, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1462221741.258723, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction NetControl::check_plugins() 0.000000 | HookCallFunction NetControl::init() 0.000000 | HookCallFunction Notice::want_pp() @@ -1699,7 +1701,7 @@ 1362692526.869344 MetaHookPost CallFunction(ChecksumOffloading::check, , ()) -> 1362692526.869344 MetaHookPost CallFunction(NetControl::check_conn, , (141.142.228.5)) -> 1362692526.869344 MetaHookPost CallFunction(filter_change_tracking, , ()) -> -1362692526.869344 MetaHookPost CallFunction(net_stats, , ()) -> +1362692526.869344 MetaHookPost CallFunction(get_net_stats, , ()) -> 1362692526.869344 MetaHookPost CallFunction(new_connection, , ([id=[orig_h=141.142.228.5, orig_p=59856<...>/tcp], orig=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0], resp=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0], start_time=1362692526.869344, duration=0.0, service={}, history=, uid=CXWv6p3arKYeMETxOg, tunnel=, vlan=, inner_vlan=, dpd=, conn=, extract_orig=F, extract_resp=F, thresholds=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=])) -> 1362692526.869344 MetaHookPost DrainEvents() -> 1362692526.869344 MetaHookPost QueueEvent(ChecksumOffloading::check()) -> false @@ -1710,7 +1712,7 @@ 1362692526.869344 MetaHookPre CallFunction(ChecksumOffloading::check, , ()) 1362692526.869344 MetaHookPre CallFunction(NetControl::check_conn, , (141.142.228.5)) 1362692526.869344 MetaHookPre CallFunction(filter_change_tracking, , ()) -1362692526.869344 MetaHookPre CallFunction(net_stats, , ()) +1362692526.869344 MetaHookPre CallFunction(get_net_stats, , ()) 1362692526.869344 MetaHookPre CallFunction(new_connection, , ([id=[orig_h=141.142.228.5, orig_p=59856<...>/tcp], orig=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0], resp=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0], start_time=1362692526.869344, duration=0.0, service={}, history=, uid=CXWv6p3arKYeMETxOg, tunnel=, vlan=, inner_vlan=, dpd=, conn=, extract_orig=F, extract_resp=F, thresholds=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=])) 1362692526.869344 MetaHookPre DrainEvents() 1362692526.869344 MetaHookPre QueueEvent(ChecksumOffloading::check()) @@ -1722,7 +1724,7 @@ 1362692526.869344 | HookCallFunction ChecksumOffloading::check() 1362692526.869344 | HookCallFunction NetControl::check_conn(141.142.228.5) 1362692526.869344 | HookCallFunction filter_change_tracking() -1362692526.869344 | HookCallFunction net_stats() +1362692526.869344 | HookCallFunction get_net_stats() 1362692526.869344 | HookCallFunction new_connection([id=[orig_h=141.142.228.5, orig_p=59856<...>/tcp], orig=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0], resp=[size=0, state=0, num_pkts=0, num_bytes_ip=0, flow_label=0], start_time=1362692526.869344, duration=0.0, service={}, history=, uid=CXWv6p3arKYeMETxOg, tunnel=, vlan=, inner_vlan=, dpd=, conn=, extract_orig=F, extract_resp=F, thresholds=, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, krb=, modbus=, mysql=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=]) 1362692526.869344 | HookDrainEvents 1362692526.869344 | HookQueueEvent ChecksumOffloading::check() @@ -2127,11 +2129,11 @@ 1362692527.080972 MetaHookPost CallFunction(filter_change_tracking, , ()) -> 1362692527.080972 MetaHookPost CallFunction(fmt, , (%s:%d > %s:%d, 141.142.228.5, 59856<...>/tcp)) -> 1362692527.080972 MetaHookPost CallFunction(get_file_handle, , (Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], T)) -> +1362692527.080972 MetaHookPost CallFunction(get_net_stats, , ()) -> 1362692527.080972 MetaHookPost CallFunction(get_port_transport_proto, , (80/tcp)) -> 1362692527.080972 MetaHookPost CallFunction(id_string, , ([orig_h=141.142.228.5, orig_p=59856<...>/tcp])) -> 1362692527.080972 MetaHookPost CallFunction(is_tcp_port, , (59856/tcp)) -> 1362692527.080972 MetaHookPost CallFunction(net_done, , (1362692527.080972)) -> -1362692527.080972 MetaHookPost CallFunction(net_stats, , ()) -> 1362692527.080972 MetaHookPost CallFunction(reading_traces, , ()) -> 1362692527.080972 MetaHookPost CallFunction(set_file_handle, , (Analyzer::ANALYZER_HTTP1362692526.869344T11141.142.228.5:59856 > 192.150.187.43:80)) -> 1362692527.080972 MetaHookPost CallFunction(sub_bytes, , (HTTP, 0, 1)) -> @@ -2157,11 +2159,11 @@ 1362692527.080972 MetaHookPre CallFunction(filter_change_tracking, , ()) 1362692527.080972 MetaHookPre CallFunction(fmt, , (%s:%d > %s:%d, 141.142.228.5, 59856<...>/tcp)) 1362692527.080972 MetaHookPre CallFunction(get_file_handle, , (Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], T)) +1362692527.080972 MetaHookPre CallFunction(get_net_stats, , ()) 1362692527.080972 MetaHookPre CallFunction(get_port_transport_proto, , (80/tcp)) 1362692527.080972 MetaHookPre CallFunction(id_string, , ([orig_h=141.142.228.5, orig_p=59856<...>/tcp])) 1362692527.080972 MetaHookPre CallFunction(is_tcp_port, , (59856/tcp)) 1362692527.080972 MetaHookPre CallFunction(net_done, , (1362692527.080972)) -1362692527.080972 MetaHookPre CallFunction(net_stats, , ()) 1362692527.080972 MetaHookPre CallFunction(reading_traces, , ()) 1362692527.080972 MetaHookPre CallFunction(set_file_handle, , (Analyzer::ANALYZER_HTTP1362692526.869344T11141.142.228.5:59856 > 192.150.187.43:80)) 1362692527.080972 MetaHookPre CallFunction(sub_bytes, , (HTTP, 0, 1)) @@ -2188,11 +2190,11 @@ 1362692527.080972 | HookCallFunction filter_change_tracking() 1362692527.080972 | HookCallFunction fmt(%s:%d > %s:%d, 141.142.228.5, 59856<...>/tcp) 1362692527.080972 | HookCallFunction get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], T) +1362692527.080972 | HookCallFunction get_net_stats() 1362692527.080972 | HookCallFunction get_port_transport_proto(80/tcp) 1362692527.080972 | HookCallFunction id_string([orig_h=141.142.228.5, orig_p=59856<...>/tcp]) 1362692527.080972 | HookCallFunction is_tcp_port(59856/tcp) 1362692527.080972 | HookCallFunction net_done(1362692527.080972) -1362692527.080972 | HookCallFunction net_stats() 1362692527.080972 | HookCallFunction reading_traces() 1362692527.080972 | HookCallFunction set_file_handle(Analyzer::ANALYZER_HTTP1362692526.869344T11141.142.228.5:59856 > 192.150.187.43:80) 1362692527.080972 | HookCallFunction sub_bytes(HTTP, 0, 1) diff --git a/testing/btest/bifs/net_stats_trace.test b/testing/btest/bifs/net_stats_trace.test index fcf3e9ba0d..cd9ee52a27 100644 --- a/testing/btest/bifs/net_stats_trace.test +++ b/testing/btest/bifs/net_stats_trace.test @@ -4,5 +4,5 @@ event bro_done() { - print net_stats(); + print get_net_stats(); } diff --git a/testing/btest/bifs/resource_usage.bro b/testing/btest/bifs/resource_usage.bro deleted file mode 100644 index 5cf3f0f962..0000000000 --- a/testing/btest/bifs/resource_usage.bro +++ /dev/null @@ -1,9 +0,0 @@ -# -# @TEST-EXEC: bro -b %INPUT - -event bro_init() - { - local a = resource_usage(); - if ( a$version != bro_version() ) - exit(1); - } From 8a6ca053bf3ff6a3a27cd679d5487b5ae098e13e Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 3 May 2016 11:16:50 -0700 Subject: [PATCH 77/84] Updating submodule(s). [nomail] --- CHANGES | 4 ++++ VERSION | 2 +- aux/broctl | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 8844dc7522..f62f544a44 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.4-500 | 2016-05-03 11:16:50 -0700 + + * Updating submodule(s). + 2.4-498 | 2016-04-28 11:34:52 -0700 * Rename Broker::print to Broker::send_print and Broker::event to diff --git a/VERSION b/VERSION index ada78ab155..e7d45626ab 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4-498 +2.4-500 diff --git a/aux/broctl b/aux/broctl index 7df7878abf..6f12b4da74 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 7df7878abfd864f9ae5609918c0f04f58b5f5e2d +Subproject commit 6f12b4da74e9e0885e1bd8cb67c2eda2b33c93a5 From 75e69d8c098dd79b89b4ac383ddaae75088e5d32 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 5 May 2016 12:47:21 -0500 Subject: [PATCH 78/84] Fix some "make doc" warnings --- scripts/base/frameworks/netcontrol/main.bro | 25 +++++++++++-------- .../frameworks/netcontrol/plugins/debug.bro | 2 +- src/event.bif | 8 +++--- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/scripts/base/frameworks/netcontrol/main.bro b/scripts/base/frameworks/netcontrol/main.bro index 563188921d..f3ff97b79b 100644 --- a/scripts/base/frameworks/netcontrol/main.bro +++ b/scripts/base/frameworks/netcontrol/main.bro @@ -109,21 +109,24 @@ export { ## ## r: The rule to install. ## - ## Returns: If succesful, returns an ID string unique to the rule that can later - ## be used to refer to it. If unsuccessful, returns an empty string. The ID is also - ## assigned to ``r$id``. Note that "successful" means "a plugin knew how to handle - ## the rule", it doesn't necessarily mean that it was indeed successfully put in - ## place, because that might happen asynchronously and thus fail only later. + ## Returns: If succesful, returns an ID string unique to the rule that can + ## later be used to refer to it. If unsuccessful, returns an empty + ## string. The ID is also assigned to ``r$id``. Note that + ## "successful" means "a plugin knew how to handle the rule", it + ## doesn't necessarily mean that it was indeed successfully put in + ## place, because that might happen asynchronously and thus fail + ## only later. global add_rule: function(r: Rule) : string; ## Removes a rule. ## ## id: The rule to remove, specified as the ID returned by :bro:id:`add_rule` . ## - ## Returns: True if succesful, the relevant plugin indicated that it knew how - ## to handle the removal. Note that again "success" means the plugin accepted the - ## removal. They might still fail to put it into effect, as that might happen - ## asynchronously and thus go wrong at that point. + ## Returns: True if succesful, the relevant plugin indicated that it knew + ## how to handle the removal. Note that again "success" means the + ## plugin accepted the removal. They might still fail to put it + ## into effect, as that might happen asynchronously and thus go + ## wrong at that point. global remove_rule: function(id: string) : bool; ## Searches all rules affecting a certain IP address. @@ -156,7 +159,7 @@ export { ## r: The rule now removed. ## ## p: The state for the plugin that had the rule in place and now - ## removed it. + ## removed it. ## ## msg: An optional informational message by the plugin. global rule_removed: event(r: Rule, p: PluginState, msg: string &default=""); @@ -168,7 +171,7 @@ export { ## i: Additional flow information, if supported by the protocol. ## ## p: The state for the plugin that had the rule in place and now - ## removed it. + ## removed it. ## ## msg: An optional informational message by the plugin. global rule_timeout: event(r: Rule, i: FlowInfo, p: PluginState); diff --git a/scripts/base/frameworks/netcontrol/plugins/debug.bro b/scripts/base/frameworks/netcontrol/plugins/debug.bro index f421dc55e3..a26a151400 100644 --- a/scripts/base/frameworks/netcontrol/plugins/debug.bro +++ b/scripts/base/frameworks/netcontrol/plugins/debug.bro @@ -11,7 +11,7 @@ export { ## plugin simply logs the operations it receives. ## ## do_something: If true, the plugin will claim it supports all operations; if - ## false, it will indicate it doesn't support any. + ## false, it will indicate it doesn't support any. global create_debug: function(do_something: bool) : PluginState; } diff --git a/src/event.bif b/src/event.bif index ff6ec059fb..b6227af9ad 100644 --- a/src/event.bif +++ b/src/event.bif @@ -306,10 +306,10 @@ event packet_contents%(c: connection, contents: string%); ## t2: The new payload. ## ## tcp_flags: A string with the TCP flags of the packet triggering the -## inconsistency. In the string, each character corresponds to one set flag, -## as follows: ``S`` -> SYN; ``F`` -> FIN; ``R`` -> RST; ``A`` -> ACK; ``P`` -> -## PUSH. This string will not always be set, only if the information is available; -## it's "best effort". +## inconsistency. In the string, each character corresponds to one +## set flag, as follows: ``S`` -> SYN; ``F`` -> FIN; ``R`` -> RST; +## ``A`` -> ACK; ``P`` -> PUSH. This string will not always be set, +## only if the information is available; it's "best effort". ## ## .. bro:see:: tcp_rexmit tcp_contents event rexmit_inconsistency%(c: connection, t1: string, t2: string, tcp_flags: string%); From 28125e367ef99859f1a79dd9a828d622bda5dd2c Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 5 May 2016 15:18:50 -0500 Subject: [PATCH 79/84] Fix more "make doc" warnings --- doc/components/bro-plugins/kafka/README.rst | 1 + scripts/base/frameworks/netcontrol/main.bro | 2 +- scripts/base/frameworks/netcontrol/types.bro | 16 ++++++++-------- src/analyzer/protocol/ssh/events.bif | 2 +- src/bro.bif | 6 +++--- 5 files changed, 14 insertions(+), 13 deletions(-) create mode 120000 doc/components/bro-plugins/kafka/README.rst diff --git a/doc/components/bro-plugins/kafka/README.rst b/doc/components/bro-plugins/kafka/README.rst new file mode 120000 index 0000000000..6ca2195f17 --- /dev/null +++ b/doc/components/bro-plugins/kafka/README.rst @@ -0,0 +1 @@ +../../../../aux/plugins/kafka/README \ No newline at end of file diff --git a/scripts/base/frameworks/netcontrol/main.bro b/scripts/base/frameworks/netcontrol/main.bro index f3ff97b79b..0acd4d0661 100644 --- a/scripts/base/frameworks/netcontrol/main.bro +++ b/scripts/base/frameworks/netcontrol/main.bro @@ -120,7 +120,7 @@ export { ## Removes a rule. ## - ## id: The rule to remove, specified as the ID returned by :bro:id:`add_rule` . + ## id: The rule to remove, specified as the ID returned by :bro:id:`NetControl::add_rule`. ## ## Returns: True if succesful, the relevant plugin indicated that it knew ## how to handle the removal. Note that again "success" means the diff --git a/scripts/base/frameworks/netcontrol/types.bro b/scripts/base/frameworks/netcontrol/types.bro index 440d63d8bc..3147420c99 100644 --- a/scripts/base/frameworks/netcontrol/types.bro +++ b/scripts/base/frameworks/netcontrol/types.bro @@ -14,7 +14,7 @@ export { MAC, ##< Activity involving a MAC address. }; - ## Type of a :bro:id:`Flow` for defining a flow. + ## Type for defining a flow. type Flow: record { src_h: subnet &optional; ##< The source IP address/subnet. src_p: port &optional; ##< The source port number. @@ -27,10 +27,10 @@ export { ## Type defining the enity an :bro:id:`Rule` is operating on. type Entity: record { ty: EntityType; ##< Type of entity. - conn: conn_id &optional; ##< Used with :bro:id:`CONNECTION` . - flow: Flow &optional; ##< Used with :bro:id:`FLOW` . - ip: subnet &optional; ##< Used with bro:id:`ADDRESS`; can specifiy a CIDR subnet. - mac: string &optional; ##< Used with :bro:id:`MAC`. + conn: conn_id &optional; ##< Used with :bro:enum:`NetControl::CONNECTION`. + flow: Flow &optional; ##< Used with :bro:enum:`NetControl::FLOW`. + ip: subnet &optional; ##< Used with :bro:enum:`NetControl::ADDRESS` to specifiy a CIDR subnet. + mac: string &optional; ##< Used with :bro:enum:`NetControl::MAC`. }; ## Target of :bro:id:`Rule` action. @@ -68,7 +68,7 @@ export { WHITELIST, }; - ## Type of a :bro:id:`FlowMod` for defining a flow modification action. + ## Type for defining a flow modification action. type FlowMod: record { src_h: addr &optional; ##< The source IP address. src_p: count &optional; ##< The source port number. @@ -90,8 +90,8 @@ export { priority: int &default=default_priority; ##< Priority if multiple rules match an entity (larger value is higher priority). location: string &optional; ##< Optional string describing where/what installed the rule. - out_port: count &optional; ##< Argument for bro:id:`REDIRECT` rules. - mod: FlowMod &optional; ##< Argument for :bro:id:`MODIFY` rules. + out_port: count &optional; ##< Argument for :bro:enum:`NetControl::REDIRECT` rules. + mod: FlowMod &optional; ##< Argument for :bro:enum:`NetControl::MODIFY` rules. id: string &default=""; ##< Internally determined unique ID for this rule. Will be set when added. cid: count &default=0; ##< Internally determined unique numeric ID for this rule. Set when added. diff --git a/src/analyzer/protocol/ssh/events.bif b/src/analyzer/protocol/ssh/events.bif index 57b736ac85..2c8079d9b7 100644 --- a/src/analyzer/protocol/ssh/events.bif +++ b/src/analyzer/protocol/ssh/events.bif @@ -120,7 +120,7 @@ event ssh1_server_host_key%(c: connection, p: string, e: string%); ## This event is generated when an :abbr:`SSH (Secure Shell)` ## encrypted packet is seen. This event is not handled by default, but ## is provided for heuristic analysis scripts. Note that you have to set -## :bro:id:`SSH::skip_processing_after_detection` to false to use this +## :bro:id:`SSH::disable_analyzer_after_detection` to false to use this ## event. This carries a performance penalty. ## ## c: The connection over which the :abbr:`SSH (Secure Shell)` diff --git a/src/bro.bif b/src/bro.bif index f21f927f92..5c3228eecc 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -2465,7 +2465,7 @@ function to_subnet%(sn: string%): subnet ## ## Returns: The *a* address as a :bro:type:`subnet`. ## -## .. bro:see:: to_subset +## .. bro:see:: to_subnet function addr_to_subnet%(a: addr%): subnet %{ int width = (a->AsAddr().GetFamily() == IPv4 ? 32 : 128); @@ -2479,7 +2479,7 @@ function addr_to_subnet%(a: addr%): subnet ## ## Returns: The *s* subnet as a :bro:type:`addr`. ## -## .. bro:see:: to_subset +## .. bro:see:: to_subnet function subnet_to_addr%(sn: subnet%): addr %{ return new AddrVal(sn->Prefix()); @@ -2491,7 +2491,7 @@ function subnet_to_addr%(sn: subnet%): addr ## ## Returns: The width of the subnet. ## -## .. bro:see:: to_subset +## .. bro:see:: to_subnet function subnet_width%(sn: subnet%): count %{ return new Val(sn->Width(), TYPE_COUNT); From 2d9127888ffe2a2cbe0b8eaea0ccaf601801e92d Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 5 May 2016 16:35:31 -0500 Subject: [PATCH 80/84] Add some missing Bro script documentation Also fixed a few reST formatting issues. --- doc/script-reference/log-files.rst | 2 ++ scripts/base/files/x509/main.bro | 1 + scripts/base/frameworks/cluster/main.bro | 2 +- scripts/base/frameworks/netcontrol/main.bro | 20 ++++++++++---------- scripts/base/frameworks/notice/main.bro | 2 ++ scripts/base/protocols/http/main.bro | 1 + scripts/base/protocols/rfb/main.bro | 1 + scripts/base/protocols/sip/main.bro | 1 + scripts/base/protocols/smtp/main.bro | 1 + scripts/base/protocols/socks/main.bro | 1 + scripts/base/protocols/ssh/main.bro | 1 + scripts/base/protocols/ssl/main.bro | 1 + scripts/base/protocols/syslog/main.bro | 3 ++- 13 files changed, 25 insertions(+), 12 deletions(-) diff --git a/doc/script-reference/log-files.rst b/doc/script-reference/log-files.rst index c3fbca95a0..3c1720afd1 100644 --- a/doc/script-reference/log-files.rst +++ b/doc/script-reference/log-files.rst @@ -39,6 +39,8 @@ Network Protocols +----------------------------+---------------------------------------+---------------------------------+ | rdp.log | RDP | :bro:type:`RDP::Info` | +----------------------------+---------------------------------------+---------------------------------+ +| rfb.log | Remote Framebuffer (RFB) | :bro:type:`RFB::Info` | ++----------------------------+---------------------------------------+---------------------------------+ | sip.log | SIP | :bro:type:`SIP::Info` | +----------------------------+---------------------------------------+---------------------------------+ | smtp.log | SMTP transactions | :bro:type:`SMTP::Info` | diff --git a/scripts/base/files/x509/main.bro b/scripts/base/files/x509/main.bro index c097b84560..bbf99f6a4d 100644 --- a/scripts/base/files/x509/main.bro +++ b/scripts/base/files/x509/main.bro @@ -6,6 +6,7 @@ module X509; export { redef enum Log::ID += { LOG }; + ## The record type which contains the fields of the X.509 log. type Info: record { ## Current timestamp. ts: time &log; diff --git a/scripts/base/frameworks/cluster/main.bro b/scripts/base/frameworks/cluster/main.bro index 3451cb4169..55fc084641 100644 --- a/scripts/base/frameworks/cluster/main.bro +++ b/scripts/base/frameworks/cluster/main.bro @@ -68,7 +68,7 @@ export { ## Events raised by TimeMachine instances and handled by workers. const tm2worker_events = /EMPTY/ &redef; - ## Events sent by the control host (i.e. BroControl) when dynamically + ## Events sent by the control host (i.e., BroControl) when dynamically ## connecting to a running instance to update settings or request data. const control_events = Control::controller_events &redef; diff --git a/scripts/base/frameworks/netcontrol/main.bro b/scripts/base/frameworks/netcontrol/main.bro index 0acd4d0661..65537ed9cf 100644 --- a/scripts/base/frameworks/netcontrol/main.bro +++ b/scripts/base/frameworks/netcontrol/main.bro @@ -23,20 +23,20 @@ export { # ### Generic functions and events. # ### - # Activates a plugin. - # - # p: The plugin to acticate. - # - # priority: The higher the priority, the earlier this plugin will be checked - # whether it supports an operation, relative to other plugins. + ## Activates a plugin. + ## + ## p: The plugin to acticate. + ## + ## priority: The higher the priority, the earlier this plugin will be checked + ## whether it supports an operation, relative to other plugins. global activate: function(p: PluginState, priority: int); - # Event that is used to initialize plugins. Place all plugin initialization - # related functionality in this event. + ## Event that is used to initialize plugins. Place all plugin initialization + ## related functionality in this event. global NetControl::init: event(); - # Event that is raised once all plugins activated in ``NetControl::init`` have finished - # their initialization. + ## Event that is raised once all plugins activated in ``NetControl::init`` + ## have finished their initialization. global NetControl::init_done: event(); # ### diff --git a/scripts/base/frameworks/notice/main.bro b/scripts/base/frameworks/notice/main.bro index 2418b499e5..a203f6a772 100644 --- a/scripts/base/frameworks/notice/main.bro +++ b/scripts/base/frameworks/notice/main.bro @@ -44,6 +44,7 @@ export { ACTION_ALARM, }; + ## Type that represents a set of actions. type ActionSet: set[Notice::Action]; ## The notice framework is able to do automatic notice suppression by @@ -52,6 +53,7 @@ export { ## suppression. const default_suppression_interval = 1hrs &redef; + ## The record type that is used for representing and logging notices. type Info: record { ## An absolute time indicating when the notice occurred, ## defaults to the current network time. diff --git a/scripts/base/protocols/http/main.bro b/scripts/base/protocols/http/main.bro index e70d166f11..2988a1a646 100644 --- a/scripts/base/protocols/http/main.bro +++ b/scripts/base/protocols/http/main.bro @@ -21,6 +21,7 @@ export { ## not. const default_capture_password = F &redef; + ## The record type which contains the fields of the HTTP log. type Info: record { ## Timestamp for when the request happened. ts: time &log; diff --git a/scripts/base/protocols/rfb/main.bro b/scripts/base/protocols/rfb/main.bro index 03e39a40f9..3bcb86890b 100644 --- a/scripts/base/protocols/rfb/main.bro +++ b/scripts/base/protocols/rfb/main.bro @@ -3,6 +3,7 @@ module RFB; export { redef enum Log::ID += { LOG }; + ## The record type which contains the fields of the RFB log. type Info: record { ## Timestamp for when the event happened. ts: time &log; diff --git a/scripts/base/protocols/sip/main.bro b/scripts/base/protocols/sip/main.bro index dc790ad560..f629049928 100644 --- a/scripts/base/protocols/sip/main.bro +++ b/scripts/base/protocols/sip/main.bro @@ -10,6 +10,7 @@ module SIP; export { redef enum Log::ID += { LOG }; + ## The record type which contains the fields of the SIP log. type Info: record { ## Timestamp for when the request happened. ts: time &log; diff --git a/scripts/base/protocols/smtp/main.bro b/scripts/base/protocols/smtp/main.bro index 6df9bddb54..766c0850bc 100644 --- a/scripts/base/protocols/smtp/main.bro +++ b/scripts/base/protocols/smtp/main.bro @@ -7,6 +7,7 @@ module SMTP; export { redef enum Log::ID += { LOG }; + ## The record type which contains the fields of the SMTP log. type Info: record { ## Time when the message was first seen. ts: time &log; diff --git a/scripts/base/protocols/socks/main.bro b/scripts/base/protocols/socks/main.bro index c63092f609..e22ed718c6 100644 --- a/scripts/base/protocols/socks/main.bro +++ b/scripts/base/protocols/socks/main.bro @@ -6,6 +6,7 @@ module SOCKS; export { redef enum Log::ID += { LOG }; + ## The record type which contains the fields of the SOCKS log. type Info: record { ## Time when the proxy connection was first detected. ts: time &log; diff --git a/scripts/base/protocols/ssh/main.bro b/scripts/base/protocols/ssh/main.bro index fad2da0b8e..d547e92e8f 100644 --- a/scripts/base/protocols/ssh/main.bro +++ b/scripts/base/protocols/ssh/main.bro @@ -8,6 +8,7 @@ export { ## The SSH protocol logging stream identifier. redef enum Log::ID += { LOG }; + ## The record type which contains the fields of the SSH log. type Info: record { ## Time when the SSH connection began. ts: time &log; diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index 8483f473f4..4c61df916a 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -8,6 +8,7 @@ module SSL; export { redef enum Log::ID += { LOG }; + ## The record type which contains the fields of the SSL log. type Info: record { ## Time when the SSL connection was first detected. ts: time &log; diff --git a/scripts/base/protocols/syslog/main.bro b/scripts/base/protocols/syslog/main.bro index 593c8ab9a2..6e74760225 100644 --- a/scripts/base/protocols/syslog/main.bro +++ b/scripts/base/protocols/syslog/main.bro @@ -7,7 +7,8 @@ module Syslog; export { redef enum Log::ID += { LOG }; - + + ## The record type which contains the fields of the syslog log. type Info: record { ## Timestamp when the syslog message was seen. ts: time &log; From f596d30386e980b92fa33bdbee8222d54733e047 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 5 May 2016 17:23:15 -0500 Subject: [PATCH 81/84] Fix some scripting tutorial examples Some of the examples in the scripting tutorial were regularly getting out of sync with the base scripts (because the line numbering would need to be updated). Fixed this maintenance burden by using small example scripts instead of actual Bro scripts. These small example scripts do not need to be kept in sync with the bro base scripts. --- doc/scripting/data_type_record.bro | 25 +++++++++++++++++++ doc/scripting/http_main.bro | 7 ++++++ doc/scripting/index.rst | 6 ++--- .../output | 2 +- .../output | 2 +- ...-doc_scripting_data_type_record_bro.btest} | 2 +- ...include-doc_scripting_http_main_bro.btest} | 2 +- 7 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 doc/scripting/data_type_record.bro create mode 100644 doc/scripting/http_main.bro rename testing/btest/Baseline/{doc.sphinx.include-scripts_base_protocols_conn_main_bro => doc.sphinx.include-doc_scripting_data_type_record_bro}/output (97%) rename testing/btest/Baseline/{doc.sphinx.include-scripts_base_protocols_http_main_bro => doc.sphinx.include-doc_scripting_http_main_bro}/output (93%) rename testing/btest/doc/sphinx/{include-scripts_base_protocols_conn_main_bro.btest => include-doc_scripting_data_type_record_bro.btest} (97%) rename testing/btest/doc/sphinx/{include-scripts_base_protocols_http_main_bro.btest => include-doc_scripting_http_main_bro.btest} (93%) diff --git a/doc/scripting/data_type_record.bro b/doc/scripting/data_type_record.bro new file mode 100644 index 0000000000..2380137cac --- /dev/null +++ b/doc/scripting/data_type_record.bro @@ -0,0 +1,25 @@ +module Conn; + +export { + ## The record type which contains column fields of the connection log. + type Info: record { + ts: time &log; + uid: string &log; + id: conn_id &log; + proto: transport_proto &log; + service: string &log &optional; + duration: interval &log &optional; + orig_bytes: count &log &optional; + resp_bytes: count &log &optional; + conn_state: string &log &optional; + local_orig: bool &log &optional; + local_resp: bool &log &optional; + missed_bytes: count &log &default=0; + history: string &log &optional; + orig_pkts: count &log &optional; + orig_ip_bytes: count &log &optional; + resp_pkts: count &log &optional; + resp_ip_bytes: count &log &optional; + tunnel_parents: set[string] &log; + }; +} diff --git a/doc/scripting/http_main.bro b/doc/scripting/http_main.bro new file mode 100644 index 0000000000..5182accb35 --- /dev/null +++ b/doc/scripting/http_main.bro @@ -0,0 +1,7 @@ +module HTTP; + +export { + ## This setting changes if passwords used in Basic-Auth are captured or + ## not. + const default_capture_password = F &redef; +} diff --git a/doc/scripting/index.rst b/doc/scripting/index.rst index a776fc0ad3..597d8ec41a 100644 --- a/doc/scripting/index.rst +++ b/doc/scripting/index.rst @@ -362,8 +362,7 @@ decrypted from HTTP streams is stored in :bro:see:`HTTP::default_capture_password` as shown in the stripped down excerpt from :doc:`/scripts/base/protocols/http/main.bro` below. -.. btest-include:: ${BRO_SRC_ROOT}/scripts/base/protocols/http/main.bro - :lines: 9-11,20-22,125 +.. btest-include:: ${DOC_ROOT}/scripting/http_main.bro Because the constant was declared with the ``&redef`` attribute, if we needed to turn this option on globally, we could do so by adding the @@ -825,8 +824,7 @@ example of the ``record`` data type in the earlier sections, the :bro:type:`Conn::Info`, which corresponds to the fields logged into ``conn.log``, is shown by the excerpt below. -.. btest-include:: ${BRO_SRC_ROOT}/scripts/base/protocols/conn/main.bro - :lines: 10-12,16-17,19,21,23,25,28,31,35,38,57,63,69,75,98,101,105,108,112,116-117,122 +.. btest-include:: ${DOC_ROOT}/scripting/data_type_record.bro Looking at the structure of the definition, a new collection of data types is being defined as a type called ``Info``. Since this type diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_base_protocols_conn_main_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_scripting_data_type_record_bro/output similarity index 97% rename from testing/btest/Baseline/doc.sphinx.include-scripts_base_protocols_conn_main_bro/output rename to testing/btest/Baseline/doc.sphinx.include-doc_scripting_data_type_record_bro/output index 83e9d5bea1..6d8760700a 100644 --- a/testing/btest/Baseline/doc.sphinx.include-scripts_base_protocols_conn_main_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_scripting_data_type_record_bro/output @@ -1,6 +1,6 @@ # @TEST-EXEC: cat %INPUT >output && btest-diff output -main.bro +data_type_record.bro module Conn; diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_base_protocols_http_main_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_scripting_http_main_bro/output similarity index 93% rename from testing/btest/Baseline/doc.sphinx.include-scripts_base_protocols_http_main_bro/output rename to testing/btest/Baseline/doc.sphinx.include-doc_scripting_http_main_bro/output index e3f7a39429..9f49450799 100644 --- a/testing/btest/Baseline/doc.sphinx.include-scripts_base_protocols_http_main_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_scripting_http_main_bro/output @@ -1,6 +1,6 @@ # @TEST-EXEC: cat %INPUT >output && btest-diff output -main.bro +http_main.bro module HTTP; diff --git a/testing/btest/doc/sphinx/include-scripts_base_protocols_conn_main_bro.btest b/testing/btest/doc/sphinx/include-doc_scripting_data_type_record_bro.btest similarity index 97% rename from testing/btest/doc/sphinx/include-scripts_base_protocols_conn_main_bro.btest rename to testing/btest/doc/sphinx/include-doc_scripting_data_type_record_bro.btest index 83e9d5bea1..6d8760700a 100644 --- a/testing/btest/doc/sphinx/include-scripts_base_protocols_conn_main_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_scripting_data_type_record_bro.btest @@ -1,6 +1,6 @@ # @TEST-EXEC: cat %INPUT >output && btest-diff output -main.bro +data_type_record.bro module Conn; diff --git a/testing/btest/doc/sphinx/include-scripts_base_protocols_http_main_bro.btest b/testing/btest/doc/sphinx/include-doc_scripting_http_main_bro.btest similarity index 93% rename from testing/btest/doc/sphinx/include-scripts_base_protocols_http_main_bro.btest rename to testing/btest/doc/sphinx/include-doc_scripting_http_main_bro.btest index e3f7a39429..9f49450799 100644 --- a/testing/btest/doc/sphinx/include-scripts_base_protocols_http_main_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_scripting_http_main_bro.btest @@ -1,6 +1,6 @@ # @TEST-EXEC: cat %INPUT >output && btest-diff output -main.bro +http_main.bro module HTTP; From 40e9724de723249aec9cccedf489c7c15b6d6879 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Sat, 7 May 2016 01:22:38 -0400 Subject: [PATCH 82/84] Switching all use of gmtime and localtime to use reentrant variants. This was causing occasional problems with the time on processes running lots of threads. The use of gmtime in the json formatter is the likely culprit due to the fact that the json formatter runs in threads. More evidence for this is that the problem only appears to exhibit when logs are being written as JSON. --- src/bro.bif | 18 +++++++++++++----- src/threading/formatters/JSON.cc | 21 ++++++++++++++------- src/util.cc | 9 ++++++++- 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/src/bro.bif b/src/bro.bif index f21f927f92..e2baf62550 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -145,12 +145,17 @@ static void do_fmt(const char*& fmt, Val* v, ODesc* d) } time_t time = time_t(v->InternalDouble()); + struct tm t; + int is_time_fmt = *fmt == 'T'; + if ( ! localtime_r(&time, &t) ) + s.AddSP(""); + if ( ! strftime(out_buf, sizeof(out_buf), is_time_fmt ? "%Y-%m-%d-%H:%M" : "%Y-%m-%d-%H:%M:%S", - localtime(&time)) ) + &t) ) s.AddSP(""); else @@ -3140,9 +3145,11 @@ function strftime%(fmt: string, d: time%) : string %{ static char buffer[128]; - time_t t = time_t(d); + time_t timeval = time_t(d); + struct tm t; - if ( strftime(buffer, 128, fmt->CheckString(), localtime(&t)) == 0 ) + if ( ! localtime_r(&timeval, &t) || + ! strftime(buffer, 128, fmt->CheckString(), &t) ) return new StringVal(""); return new StringVal(buffer); @@ -3160,9 +3167,10 @@ function strftime%(fmt: string, d: time%) : string function strptime%(fmt: string, d: string%) : time %{ const time_t timeval = time_t(); - struct tm t = *localtime(&timeval); + struct tm t; - if ( strptime(d->CheckString(), fmt->CheckString(), &t) == NULL ) + if ( ! localtime_r(&timeval, &t) || + ! strptime(d->CheckString(), fmt->CheckString(), &t) ) { reporter->Warning("strptime conversion failed: fmt:%s d:%s", fmt->CheckString(), d->CheckString()); return new Val(0.0, TYPE_TIME); diff --git a/src/threading/formatters/JSON.cc b/src/threading/formatters/JSON.cc index 3558baee5c..45c7be3e93 100644 --- a/src/threading/formatters/JSON.cc +++ b/src/threading/formatters/JSON.cc @@ -116,21 +116,28 @@ bool JSON::Describe(ODesc* desc, Value* val, const string& name) const { char buffer[40]; char buffer2[40]; - time_t t = time_t(val->val.double_val); + time_t the_time = time_t(val->val.double_val); + struct tm t; - if ( strftime(buffer, sizeof(buffer), "%Y-%m-%dT%H:%M:%S", gmtime(&t)) > 0 ) + desc->AddRaw("\"", 1); + + if ( ! gmtime_r(&the_time, &t) || + ! strftime(buffer, sizeof(buffer), "%Y-%m-%dT%H:%M:%S", &t) ) + { + GetThread()->Error(GetThread()->Fmt("json formatter: failure getting time: (%" PRIu64 ")", val->val.double_val)); + // This was a failure, doesn't really matter what gets put here + // but it should probably stand out... + desc->Add("2000-01-01T00:00:00.000000"); + } + else { double integ; double frac = modf(val->val.double_val, &integ); snprintf(buffer2, sizeof(buffer2), "%s.%06.0fZ", buffer, frac * 1000000); - desc->AddRaw("\"", 1); desc->Add(buffer2); - desc->AddRaw("\"", 1); } - else - GetThread()->Error(GetThread()->Fmt("strftime error for JSON: %" PRIu64)); - + desc->AddRaw("\"", 1); } else if ( timestamps == TS_EPOCH ) diff --git a/src/util.cc b/src/util.cc index 0ea89beb90..1f10d7446d 100644 --- a/src/util.cc +++ b/src/util.cc @@ -571,7 +571,14 @@ const char* fmt_access_time(double t) { static char buf[256]; time_t time = (time_t) t; - strftime(buf, sizeof(buf), "%d/%m-%H:%M", localtime(&time)); + struct tm ts; + + if ( ! localtime_r(&time, &ts) ) + { + reporter->InternalError("unable to get time"); + } + + strftime(buf, sizeof(buf), "%d/%m-%H:%M", &ts); return buf; } From b23ed77819b228cdc4e118e2f72e823e78a27e06 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sat, 7 May 2016 12:19:07 -0700 Subject: [PATCH 83/84] Updating submodule(s). [nomail] --- CHANGES | 5 +++++ VERSION | 2 +- aux/plugins | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index ed7e16ca1b..9217c29793 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,9 @@ +2.4-544 | 2016-05-07 12:19:07 -0700 + + * Switching all use of gmtime and localtime to use reentrant + variants. (Seth Hall) + 2.4-541 | 2016-05-06 17:58:45 -0700 * A set of new built-in function for gathering execution statistics: diff --git a/VERSION b/VERSION index a4706ae7f1..9851c2a833 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4-541 +2.4-544 diff --git a/aux/plugins b/aux/plugins index ab61be0c4f..bacbf297e3 160000 --- a/aux/plugins +++ b/aux/plugins @@ -1 +1 @@ -Subproject commit ab61be0c4f128c976f72dfa5a09a87cd842f387a +Subproject commit bacbf297e37f92e1a00f91e293a4e059a5b6aedd From 90223fe4285313858dadfad18b59ca579b236a60 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 9 May 2016 09:45:21 -0700 Subject: [PATCH 84/84] Updating submodule(s). [nomail] --- aux/plugins | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/plugins b/aux/plugins index bacbf297e3..6bd2ac4846 160000 --- a/aux/plugins +++ b/aux/plugins @@ -1 +1 @@ -Subproject commit bacbf297e37f92e1a00f91e293a4e059a5b6aedd +Subproject commit 6bd2ac48466b57cdda84a593faebc25a59d98a51