From cca9a6616e2d97e16c6a168c296ac19beeb57219 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 30 Mar 2016 20:32:36 -0500 Subject: [PATCH] Split the broker main.bro into two scripts Separate the former BrokerComm and BrokerStore portions of the script because these files will be much larger when script wrappers for BIFs are written. --- scripts/base/frameworks/broker/__load__.bro | 1 + scripts/base/frameworks/broker/main.bro | 48 ----------------- scripts/base/frameworks/broker/store.bro | 51 +++++++++++++++++++ .../canonified_loaded_scripts.log | 5 +- .../canonified_loaded_scripts.log | 5 +- testing/btest/Baseline/plugins.hooks/output | 14 ++--- 6 files changed, 66 insertions(+), 58 deletions(-) create mode 100644 scripts/base/frameworks/broker/store.bro diff --git a/scripts/base/frameworks/broker/__load__.bro b/scripts/base/frameworks/broker/__load__.bro index a10fe855df..018d772f4f 100644 --- a/scripts/base/frameworks/broker/__load__.bro +++ b/scripts/base/frameworks/broker/__load__.bro @@ -1 +1,2 @@ @load ./main +@load ./store diff --git a/scripts/base/frameworks/broker/main.bro b/scripts/base/frameworks/broker/main.bro index 0270e6a6ba..d8b4a208a2 100644 --- a/scripts/base/frameworks/broker/main.bro +++ b/scripts/base/frameworks/broker/main.bro @@ -53,51 +53,3 @@ export { val: Broker::Data; }; } - -module Broker; - -export { - - ## Whether a data store query could be completed or not. - type QueryStatus: enum { - SUCCESS, - FAILURE, - }; - - ## An expiry time for a key-value pair inserted in to a data store. - type ExpiryTime: record { - ## Absolute point in time at which to expire the entry. - absolute: time &optional; - ## A point in time relative to the last modification time at which - ## to expire the entry. New modifications will delay the expiration. - since_last_modification: interval &optional; - }; - - ## The result of a data store query. - type QueryResult: record { - ## Whether the query completed or not. - status: Broker::QueryStatus; - ## The result of the query. Certain queries may use a particular - ## data type (e.g. querying store size always returns a count, but - ## a lookup may return various data types). - result: Broker::Data; - }; - - ## Options to tune the SQLite storage backend. - type SQLiteOptions: record { - ## File system path of the database. - path: string &default = "store.sqlite"; - }; - - ## Options to tune the RocksDB storage backend. - type RocksDBOptions: record { - ## File system path of the database. - path: string &default = "store.rocksdb"; - }; - - ## Options to tune the particular storage backends. - type BackendOptions: record { - sqlite: SQLiteOptions &default = SQLiteOptions(); - rocksdb: RocksDBOptions &default = RocksDBOptions(); - }; -} diff --git a/scripts/base/frameworks/broker/store.bro b/scripts/base/frameworks/broker/store.bro new file mode 100644 index 0000000000..e6468f2b2c --- /dev/null +++ b/scripts/base/frameworks/broker/store.bro @@ -0,0 +1,51 @@ +##! Various data structure definitions for use with Bro's communication system. + +@load ./main + +module Broker; + +export { + + ## Whether a data store query could be completed or not. + type QueryStatus: enum { + SUCCESS, + FAILURE, + }; + + ## An expiry time for a key-value pair inserted in to a data store. + type ExpiryTime: record { + ## Absolute point in time at which to expire the entry. + absolute: time &optional; + ## A point in time relative to the last modification time at which + ## to expire the entry. New modifications will delay the expiration. + since_last_modification: interval &optional; + }; + + ## The result of a data store query. + type QueryResult: record { + ## Whether the query completed or not. + status: Broker::QueryStatus; + ## The result of the query. Certain queries may use a particular + ## data type (e.g. querying store size always returns a count, but + ## a lookup may return various data types). + result: Broker::Data; + }; + + ## Options to tune the SQLite storage backend. + type SQLiteOptions: record { + ## File system path of the database. + path: string &default = "store.sqlite"; + }; + + ## Options to tune the RocksDB storage backend. + type RocksDBOptions: record { + ## File system path of the database. + path: string &default = "store.rocksdb"; + }; + + ## Options to tune the particular storage backends. + type BackendOptions: record { + sqlite: SQLiteOptions &default = SQLiteOptions(); + rocksdb: RocksDBOptions &default = RocksDBOptions(); + }; +} diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 4d1f2037a4..44b199bc51 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2015-08-31-04-50-43 +#open 2016-03-30-22-54-40 #fields name #types string scripts/base/init-bare.bro @@ -17,6 +17,7 @@ scripts/base/init-bare.bro build/scripts/base/bif/event.bif.bro scripts/base/frameworks/broker/__load__.bro scripts/base/frameworks/broker/main.bro + scripts/base/frameworks/broker/store.bro scripts/base/frameworks/logging/__load__.bro scripts/base/frameworks/logging/main.bro build/scripts/base/bif/logging.bif.bro @@ -128,4 +129,4 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro scripts/policy/misc/loaded-scripts.bro scripts/base/utils/paths.bro -#close 2015-08-31-04-50-43 +#close 2016-03-30-22-54-40 diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 85fe19eb96..cf15f484c8 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2016-02-17-20-30-50 +#open 2016-03-30-22-54-53 #fields name #types string scripts/base/init-bare.bro @@ -17,6 +17,7 @@ scripts/base/init-bare.bro build/scripts/base/bif/event.bif.bro scripts/base/frameworks/broker/__load__.bro scripts/base/frameworks/broker/main.bro + scripts/base/frameworks/broker/store.bro scripts/base/frameworks/logging/__load__.bro scripts/base/frameworks/logging/main.bro build/scripts/base/bif/logging.bif.bro @@ -297,4 +298,4 @@ scripts/base/init-default.bro scripts/base/misc/find-checksum-offloading.bro scripts/base/misc/find-filtered-trace.bro scripts/policy/misc/loaded-scripts.bro -#close 2016-02-17-20-30-50 +#close 2016-03-30-22-54-53 diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 25808a20d8..a66fe365c6 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -228,7 +228,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1457718658.75999, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1459378506.94705, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Communication::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Conn::LOG)) -> @@ -346,7 +346,7 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1457718658.75999, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1459378506.94705, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(NetControl::check_plugins, , ()) -> 0.000000 MetaHookPost CallFunction(NetControl::init, , ()) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, , ()) -> @@ -517,6 +517,7 @@ 0.000000 MetaHookPost LoadFile(./shunt) -> -1 0.000000 MetaHookPost LoadFile(./site) -> -1 0.000000 MetaHookPost LoadFile(./std-dev) -> -1 +0.000000 MetaHookPost LoadFile(./store) -> -1 0.000000 MetaHookPost LoadFile(./store.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./strings.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./sum) -> -1 @@ -859,7 +860,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1457718658.75999, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1459378506.94705, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Communication::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Conn::LOG)) @@ -977,7 +978,7 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1457718658.75999, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1459378506.94705, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(NetControl::check_plugins, , ()) 0.000000 MetaHookPre CallFunction(NetControl::init, , ()) 0.000000 MetaHookPre CallFunction(Notice::want_pp, , ()) @@ -1148,6 +1149,7 @@ 0.000000 MetaHookPre LoadFile(./shunt) 0.000000 MetaHookPre LoadFile(./site) 0.000000 MetaHookPre LoadFile(./std-dev) +0.000000 MetaHookPre LoadFile(./store) 0.000000 MetaHookPre LoadFile(./store.bif.bro) 0.000000 MetaHookPre LoadFile(./strings.bif.bro) 0.000000 MetaHookPre LoadFile(./sum) @@ -1489,7 +1491,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1457718658.75999, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1459378506.94705, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG) @@ -1607,7 +1609,7 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1457718658.75999, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1459378506.94705, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction NetControl::check_plugins() 0.000000 | HookCallFunction NetControl::init() 0.000000 | HookCallFunction Notice::want_pp()