diff --git a/.gitmodules b/.gitmodules index 5efc3b0fb8..c7a9313543 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,9 +4,6 @@ [submodule "aux/binpac"] path = aux/binpac url = https://github.com/zeek/binpac -[submodule "aux/broccoli"] - path = aux/broccoli - url = https://github.com/zeek/broccoli [submodule "aux/broctl"] path = aux/broctl url = https://github.com/zeek/broctl diff --git a/CHANGES b/CHANGES index 7ac58d9f4a..3ab9e1f4b8 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,18 @@ +2.6-272 | 2019-05-06 18:43:13 -0700 + + * Remove support for using && and || with patterns. (Johanna Amann, Corelight) + + This was never documented and previously deprecated. + + * Remove RemoteSerializer and related code/types. (Johanna Amann, Corelight) + + Also removes broccoli from the source tree. + + * Remove PersistenceSerializer. (Johanna Amann, Corelight) + + * Remove &synchronized and &persistent attributes. (Johanna Amann, Corelight) + 2.6-264 | 2019-05-03 11:16:38 -0700 * Fix sporadic openflow/broker test failure (Jon Siwek, Corelight) diff --git a/CMakeLists.txt b/CMakeLists.txt index cfe0b29ed9..c2110d3da8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -326,7 +326,6 @@ include(CheckOptionalBuildSources) CheckOptionalBuildSources(aux/broctl Broctl INSTALL_BROCTL) CheckOptionalBuildSources(aux/bro-aux Bro-Aux INSTALL_AUX_TOOLS) -CheckOptionalBuildSources(aux/broccoli Broccoli INSTALL_BROCCOLI) ######################################################################## ## Packaging Setup @@ -366,7 +365,6 @@ message( "\nCXXFLAGS: ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${BuildType}}" "\nCPP: ${CMAKE_CXX_COMPILER}" "\n" - "\nBroccoli: ${INSTALL_BROCCOLI}" "\nBroctl: ${INSTALL_BROCTL}" "\nAux. Tools: ${INSTALL_AUX_TOOLS}" "\n" diff --git a/NEWS b/NEWS index 082ad782b1..ac8a02c8d4 100644 --- a/NEWS +++ b/NEWS @@ -244,11 +244,55 @@ Removed Functionality - ``dhcp_offer`` - ``dhcp_release`` - ``dhcp_request`` + - ``remote_state_access_performed`` + - ``remote_state_inconsistency`` + - ``remote_connection_established`` + - ``remote_connection_closed`` + - ``remote_connection_handshake_done`` + - ``remote_event_registered`` + - ``remote_connection_error`` + - ``remote_capture_filter`` + - ``remote_log_peer`` + - ``remote_log`` - ``finished_send_state`` + - ``remote_pong`` + +- The following types/records were deprecated in version 2.6 or below and are + removed from this release: + + - ``peer_id`` + - ``event_peer`` + +- The following configuration options were deprecated in version 2.6 or below and are + removed from this release: + + - ``max_remote_events_processed`` + - ``forward_remote_events`` + - ``forward_remote_state_changes`` + - ``enable_syslog`` + - ``remote_trace_sync_interval`` + - ``remote_trace_sync_peers`` + - ``remote_check_sync_consistency`` + +- The following constants were used as part of deprecated functionality in version 2.6 + or below and are removed from this release: + + - ``PEER_ID_NONE`` + - ``REMOTE_LOG_INFO`` + - ``REMOTE_SRC_CHILD`` + - ``REMOTE_SRC_PARENT`` + - ``REMOTE_SRC_SCRIPT`` - The deprecated script ``policy/protocols/smb/__load__.bro`` was removed. Instead of ``@load policy/protocols/smb`` use ``@load base/protocols/smb``. +- Broccoli, which had been deprecated in version 2.6 and was no longer built by default + was removed from the source tree. + +- Support for the &persistent and the &synchronized attributes, which were deprecated + in Bro 2.6, was removed. The ``-g`` command-line option (dump-config) which relied on + this functionality was also removed. + Deprecated Functionality ------------------------ @@ -263,6 +307,11 @@ Deprecated Functionality such that existing code will not break, but will emit a deprecation warning. +- The ``rotate_file``, ``rotate_file_by_name`` and ``calc_next_rotate`` functions + were marked as deprecated. These functions were used with the old pre-2.0 logging + framework and are no longer used. They also were marked as deprecated in their + documentation, however the functions themselves did not carry the deprecation marker. + Bro 2.6 ======= @@ -634,7 +683,7 @@ New Functionality Each has the same form, e.g.:: event tcp_multiple_retransmissions(c: connection, is_orig: bool, - threshold: count); + threshold: count); - Added support for set union, intersection, difference, and comparison operations. The corresponding operators for the first three are diff --git a/VERSION b/VERSION index 70f4699737..76a811396c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.6-264 +2.6-272 diff --git a/aux/broccoli b/aux/broccoli deleted file mode 160000 index 8668422406..0000000000 --- a/aux/broccoli +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8668422406cb74f4f0c574a0c9b6365a21f3e81a diff --git a/configure b/configure index 98bfc5308d..12ef158d9b 100755 --- a/configure +++ b/configure @@ -51,7 +51,6 @@ Usage: $0 [OPTION]... [VAR=VALUE]... (automatically on when perftools is present on Linux) --enable-perftools-debug use Google's perftools for debugging --enable-jemalloc link against jemalloc - --enable-broccoli build or install the Broccoli library (deprecated) --enable-static-broker build broker statically (ignored if --with-broker is specified) --enable-static-binpac build binpac statically (ignored if --with-binpac is specified) --disable-broctl don't install Broctl @@ -140,7 +139,6 @@ append_cache_entry ENABLE_PERFTOOLS BOOL false append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL false append_cache_entry ENABLE_JEMALLOC BOOL false append_cache_entry BUILD_SHARED_LIBS BOOL true -append_cache_entry INSTALL_BROCCOLI BOOL false append_cache_entry INSTALL_AUX_TOOLS BOOL true append_cache_entry INSTALL_BROCTL BOOL true append_cache_entry CPACK_SOURCE_IGNORE_FILES STRING @@ -221,10 +219,6 @@ while [ $# -ne 0 ]; do --enable-jemalloc) append_cache_entry ENABLE_JEMALLOC BOOL true ;; - --enable-broccoli) - append_cache_entry DISABLE_RUBY_BINDINGS BOOL true - append_cache_entry INSTALL_BROCCOLI BOOL yes - ;; --enable-static-broker) append_cache_entry BUILD_STATIC_BROKER BOOL true ;; diff --git a/doc b/doc index 8aa690e20d..736323fe8a 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 8aa690e20d19f79805d7f680e454e4ea10231add +Subproject commit 736323fe8a78fd8478325c134afe38c075e297a7 diff --git a/man/bro.8 b/man/bro.8 index 37c20bf0c5..9dffbe2a27 100644 --- a/man/bro.8 +++ b/man/bro.8 @@ -36,9 +36,6 @@ augment loaded policies by given code \fB\-f\fR,\ \-\-filter tcpdump filter .TP -\fB\-g\fR,\ \-\-dump\-config -dump current config into .state dir -.TP \fB\-h\fR,\ \-\-help|\-? command line help .TP diff --git a/scripts/base/frameworks/cluster/nodes/logger.zeek b/scripts/base/frameworks/cluster/nodes/logger.zeek index 39dcb751df..03a422e460 100644 --- a/scripts/base/frameworks/cluster/nodes/logger.zeek +++ b/scripts/base/frameworks/cluster/nodes/logger.zeek @@ -24,6 +24,3 @@ redef Log::default_mail_alarms_interval = 24 hrs; ## Use the cluster's archive logging script. redef Log::default_rotation_postprocessor_cmd = "archive-log"; - -## We're processing essentially *only* remote events. -redef max_remote_events_processed = 10000; diff --git a/scripts/base/frameworks/cluster/nodes/manager.zeek b/scripts/base/frameworks/cluster/nodes/manager.zeek index e54b090522..8858025a25 100644 --- a/scripts/base/frameworks/cluster/nodes/manager.zeek +++ b/scripts/base/frameworks/cluster/nodes/manager.zeek @@ -21,6 +21,3 @@ redef Log::default_rotation_interval = 24 hrs; ## Use the cluster's delete-log script. redef Log::default_rotation_postprocessor_cmd = "delete-log"; - -## We're processing essentially *only* remote events. -redef max_remote_events_processed = 10000; diff --git a/scripts/base/frameworks/cluster/nodes/proxy.zeek b/scripts/base/frameworks/cluster/nodes/proxy.zeek index e38a5e9109..df2a7c552b 100644 --- a/scripts/base/frameworks/cluster/nodes/proxy.zeek +++ b/scripts/base/frameworks/cluster/nodes/proxy.zeek @@ -5,10 +5,6 @@ @prefixes += cluster-proxy -## The proxy only syncs state; does not forward events. -redef forward_remote_events = F; -redef forward_remote_state_changes = T; - ## Don't do any local logging. redef Log::enable_local_logging = F; diff --git a/scripts/base/frameworks/packet-filter/cluster.zeek b/scripts/base/frameworks/packet-filter/cluster.zeek index 6e41a6045f..b1e1ceaddf 100644 --- a/scripts/base/frameworks/packet-filter/cluster.zeek +++ b/scripts/base/frameworks/packet-filter/cluster.zeek @@ -4,11 +4,11 @@ module PacketFilter; -event remote_connection_handshake_done(p: event_peer) &priority=3 +event Cluster::hello(name: string, id: string) &priority=-3 { - if ( Cluster::local_node_type() == Cluster::WORKER && - p$descr in Cluster::nodes && - Cluster::nodes[p$descr]$node_type == Cluster::MANAGER ) + if ( Cluster::local_node_type() == Cluster::WORKER && + name in Cluster::nodes && + Cluster::nodes[name]$node_type == Cluster::MANAGER ) { # This ensures that a packet filter is installed and logged # after the manager connects to us. diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index 1863a51a26..228128b486 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -775,29 +775,6 @@ type IPAddrAnonymizationClass: enum { OTHER_ADDR, }; -## A locally unique ID identifying a communication peer. -## -type peer_id: count; - -## A communication peer. -## -## .. zeek:see:: remote_capture_filter -## remote_connection_closed remote_connection_error -## remote_connection_established remote_connection_handshake_done -## remote_event_registered remote_log_peer remote_pong -## -## .. todo::The type's name is too narrow these days, should rename. -type event_peer: record { - id: peer_id; ##< Locally unique ID of peer - host: addr; ##< The IP address of the peer. - ## Either the port we connected to at the peer; or our port the peer - ## connected to if the session is remotely initiated. - p: port; - is_local: bool; ##< True if this record describes the local process. - descr: string; ##< The peer's :zeek:see:`peer_description`. - class: string &optional; ##< The self-assigned *class* of the peer. -}; - ## Deprecated. ## ## .. zeek:see:: rotate_file rotate_file_by_name rotate_interval @@ -1967,10 +1944,6 @@ const watchdog_interval = 10 sec &redef; ## "process all expired timers with each new packet". const max_timer_expires = 300 &redef; -## With a similar trade-off, this gives the number of remote events -## to process in a batch before interleaving other activity. -const max_remote_events_processed = 10 &redef; - # These need to match the definitions in Login.h. # # .. zeek:see:: get_login_state @@ -4737,70 +4710,14 @@ const packet_filter_default = F &redef; ## Maximum size of regular expression groups for signature matching. const sig_max_group_size = 50 &redef; -## Deprecated. No longer functional. -const enable_syslog = F &redef; - ## Description transmitted to remote communication peers for identification. const peer_description = "bro" &redef; -## If true, broadcast events received from one peer to all other peers. -## -## .. zeek:see:: forward_remote_state_changes -## -## .. note:: This option is only temporary and will disappear once we get a -## more sophisticated script-level communication framework. -const forward_remote_events = F &redef; - -## If true, broadcast state updates received from one peer to all other peers. -## -## .. zeek:see:: forward_remote_events -## -## .. note:: This option is only temporary and will disappear once we get a -## more sophisticated script-level communication framework. -const forward_remote_state_changes = F &redef; - ## The number of IO chunks allowed to be buffered between the child ## and parent process of remote communication before Bro starts dropping ## connections to remote peers in an attempt to catch up. const chunked_io_buffer_soft_cap = 800000 &redef; -## Place-holder constant indicating "no peer". -const PEER_ID_NONE = 0; - -# Signature payload pattern types. -# todo:: use enum to help autodoc -# todo:: Still used? -#const SIG_PATTERN_PAYLOAD = 0; -#const SIG_PATTERN_HTTP = 1; -#const SIG_PATTERN_FTP = 2; -#const SIG_PATTERN_FINGER = 3; - -# Deprecated. -# todo::Should use the new logging framework directly. -const REMOTE_LOG_INFO = 1; ##< Deprecated. -const REMOTE_LOG_ERROR = 2; ##< Deprecated. - -# Source of logging messages from the communication framework. -# todo:: these should go into an enum to make them autodoc'able. -const REMOTE_SRC_CHILD = 1; ##< Message from the child process. -const REMOTE_SRC_PARENT = 2; ##< Message from the parent process. -const REMOTE_SRC_SCRIPT = 3; ##< Message from a policy script. - -## Synchronize trace processing at a regular basis in pseudo-realtime mode. -## -## .. zeek:see:: remote_trace_sync_peers -const remote_trace_sync_interval = 0 secs &redef; - -## Number of peers across which to synchronize trace processing in -## pseudo-realtime mode. -## -## .. zeek:see:: remote_trace_sync_interval -const remote_trace_sync_peers = 0 &redef; - -## Whether for :zeek:attr:`&synchronized` state to send the old value as a -## consistency check. -const remote_check_sync_consistency = F &redef; - ## Reassemble the beginning of all TCP connections before doing ## signature matching. Enabling this provides more accurate matching at the ## expense of CPU cycles. diff --git a/src/Attr.cc b/src/Attr.cc index d3a347e8d1..a3d8d15e20 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -14,7 +14,6 @@ const char* attr_name(attr_tag t) "&rotate_interval", "&rotate_size", "&add_func", "&delete_func", "&expire_func", "&read_expire", "&write_expire", "&create_expire", - "&persistent", "&synchronized", "&encrypt", "&raw_output", "&mergeable", "&priority", "&group", "&log", "&error_handler", "&type_column", @@ -438,8 +437,6 @@ void Attributes::CheckAttr(Attr* a) } break; - case ATTR_PERSISTENT: - case ATTR_SYNCHRONIZED: case ATTR_TRACKED: // FIXME: Check here for global ID? break; @@ -559,8 +556,7 @@ bool Attributes::DoSerialize(SerialInfo* info) const { Attr* a = (*attrs)[i]; - // Broccoli doesn't support expressions. - Expr* e = (! info->broccoli_peer) ? a->AttrExpr() : 0; + Expr* e = a->AttrExpr(); SERIALIZE_OPTIONAL(e); if ( ! SERIALIZE(char(a->Tag())) ) diff --git a/src/Attr.h b/src/Attr.h index bfb7c4803c..4a1110bc04 100644 --- a/src/Attr.h +++ b/src/Attr.h @@ -23,8 +23,6 @@ typedef enum { ATTR_EXPIRE_READ, ATTR_EXPIRE_WRITE, ATTR_EXPIRE_CREATE, - ATTR_PERSISTENT, - ATTR_SYNCHRONIZED, ATTR_ENCRYPT, ATTR_RAW_OUTPUT, ATTR_MERGEABLE, diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 262aaf07a5..f09d0d224e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -287,7 +287,6 @@ set(bro_SRCS OpaqueVal.cc OSFinger.cc PacketFilter.cc - PersistenceSerializer.cc Pipe.cc PolicyFile.cc PrefixTable.cc @@ -296,7 +295,6 @@ set(bro_SRCS RandTest.cc RE.cc Reassem.cc - RemoteSerializer.cc Rule.cc RuleAction.cc RuleCondition.cc diff --git a/src/ChunkedIO.cc b/src/ChunkedIO.cc index d2cdbc6425..c57ab34dc8 100644 --- a/src/ChunkedIO.cc +++ b/src/ChunkedIO.cc @@ -14,7 +14,6 @@ #include "bro-config.h" #include "ChunkedIO.h" #include "NetVar.h" -#include "RemoteSerializer.h" ChunkedIO::ChunkedIO() : stats(), tag(), pure() { @@ -377,7 +376,7 @@ ChunkedIO::Chunk* ChunkedIOFd::ConcatChunks(Chunk* c1, Chunk* c2) void ChunkedIO::Log(const char* str) { - RemoteSerializer::Log(RemoteSerializer::LogError, str); + //RemoteSerializer::Log(RemoteSerializer::LogError, str); } bool ChunkedIOFd::Read(Chunk** chunk, bool may_block) diff --git a/src/Conn.cc b/src/Conn.cc index 83ad6c08f6..d607550e8a 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -151,7 +151,6 @@ Connection::Connection(NetSessions* s, HashKey* k, double t, const ConnID* id, is_active = 1; skip = 0; weird = 0; - persistent = 0; suppress_event = 0; @@ -951,15 +950,11 @@ bool Connection::DoSerialize(SerialInfo* info) const SERIALIZE_BIT(weird) && SERIALIZE_BIT(finished) && SERIALIZE_BIT(record_packets) && - SERIALIZE_BIT(record_contents) && - SERIALIZE_BIT(persistent); + SERIALIZE_BIT(record_contents); } bool Connection::DoUnserialize(UnserialInfo* info) { - // Make sure this is initialized for the condition in Unserialize(). - persistent = 0; - DO_UNSERIALIZE(BroObj); // Build the hash key first. Some of the recursive *::Unserialize() @@ -1022,7 +1017,6 @@ bool Connection::DoUnserialize(UnserialInfo* info) UNSERIALIZE_BIT(finished); UNSERIALIZE_BIT(record_packets); UNSERIALIZE_BIT(record_contents); - UNSERIALIZE_BIT(persistent); // Hmm... Why does each connection store a sessions ptr? sessions = ::sessions; diff --git a/src/Conn.h b/src/Conn.h index fc1baf4b07..fb7f5be0b4 100644 --- a/src/Conn.h +++ b/src/Conn.h @@ -12,7 +12,6 @@ #include "Val.h" #include "Timer.h" #include "Serializer.h" -#include "PersistenceSerializer.h" #include "RuleMatcher.h" #include "IPAddr.h" #include "TunnelEncapsulation.h" @@ -228,14 +227,6 @@ public: return 1; } - void MakePersistent() - { - persistent = 1; - persistence_serializer->Register(this); - } - - bool IsPersistent() { return persistent; } - void Describe(ODesc* d) const override; void IDString(ODesc* d) const; @@ -315,7 +306,7 @@ public: protected: - Connection() { persistent = 0; } + Connection() { } // Add the given timer to expire at time t. If do_expire // is true, then the timer is also evaluated when Bro terminates, @@ -361,7 +352,6 @@ protected: unsigned int weird:1; unsigned int finished:1; unsigned int record_packets:1, record_contents:1; - unsigned int persistent:1; unsigned int record_current_packet:1, record_current_content:1; unsigned int saw_first_orig_packet:1, saw_first_resp_packet:1; diff --git a/src/DebugLogger.cc b/src/DebugLogger.cc index 8df6a5ef55..7adc7aa65a 100644 --- a/src/DebugLogger.cc +++ b/src/DebugLogger.cc @@ -11,9 +11,9 @@ DebugLogger debug_logger; // Same order here as in DebugStream. DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = { - { "serial", 0, false }, { "rules", 0, false }, { "comm", 0, false }, + { "serial", 0, false }, { "rules", 0, false }, { "state", 0, false }, { "chunkedio", 0, false }, - { "compressor", 0, false }, {"string", 0, false }, + {"string", 0, false }, { "notifiers", 0, false }, { "main-loop", 0, false }, { "dpd", 0, false }, { "tm", 0, false }, { "logging", 0, false }, {"input", 0, false }, diff --git a/src/DebugLogger.h b/src/DebugLogger.h index dab9fd9758..db646bd0cf 100644 --- a/src/DebugLogger.h +++ b/src/DebugLogger.h @@ -16,10 +16,8 @@ enum DebugStream { DBG_SERIAL, // Serialization DBG_RULES, // Signature matching - DBG_COMM, // Remote communication DBG_STATE, // StateAccess logging DBG_CHUNKEDIO, // ChunkedIO logging - DBG_COMPRESSOR, // Connection compressor DBG_STRING, // String code DBG_NOTIFIERS, // Notifiers (see StateAccess.h) DBG_MAINLOOP, // Main IOSource loop diff --git a/src/Event.cc b/src/Event.cc index 8b87caa9b1..f033a01e40 100644 --- a/src/Event.cc +++ b/src/Event.cc @@ -189,21 +189,3 @@ void EventMgr::Describe(ODesc* d) const d->NL(); } } - -RecordVal* EventMgr::GetLocalPeerVal() - { - if ( ! src_val ) - { - src_val = new RecordVal(peer); - src_val->Assign(0, val_mgr->GetCount(0)); - src_val->Assign(1, new AddrVal("127.0.0.1")); - src_val->Assign(2, val_mgr->GetPort(0)); - src_val->Assign(3, val_mgr->GetTrue()); - - Ref(peer_description); - src_val->Assign(4, peer_description); - src_val->Assign(5, 0); // class (optional). - } - - return src_val; - } diff --git a/src/Event.h b/src/Event.h index 1b23f304f2..cafe0057d6 100644 --- a/src/Event.h +++ b/src/Event.h @@ -129,9 +129,6 @@ public: int Size() const { return num_events_queued - num_events_dispatched; } - // Returns a peer record describing the local Bro. - RecordVal* GetLocalPeerVal(); - void Describe(ODesc* d) const override; protected: diff --git a/src/EventHandler.cc b/src/EventHandler.cc index 08e8728d6f..718e6d6ae8 100644 --- a/src/EventHandler.cc +++ b/src/EventHandler.cc @@ -2,7 +2,6 @@ #include "EventHandler.h" #include "Func.h" #include "Scope.h" -#include "RemoteSerializer.h" #include "NetVar.h" #include "broker/Manager.h" @@ -28,7 +27,6 @@ EventHandler::~EventHandler() EventHandler::operator bool() const { return enabled && ((local && local->HasBodies()) - || receivers.length() || generate_always || ! auto_publish.empty()); } @@ -73,12 +71,6 @@ void EventHandler::Call(val_list* vl, bool no_remote) if ( ! no_remote ) { - loop_over_list(receivers, i) - { - SerialInfo info(remote_serializer); - remote_serializer->SendCall(&info, receivers[i], name, vl); - } - if ( ! auto_publish.empty() ) { // Send event in form [name, xs...] where xs represent the arguments. @@ -179,16 +171,6 @@ void EventHandler::NewEvent(val_list* vl) mgr.Dispatch(ev); } -void EventHandler::AddRemoteHandler(SourceID peer) - { - receivers.append(peer); - } - -void EventHandler::RemoveRemoteHandler(SourceID peer) - { - receivers.remove(peer); - } - bool EventHandler::Serialize(SerialInfo* info) const { return SERIALIZE(name); diff --git a/src/EventHandler.h b/src/EventHandler.h index bad3d278fa..216badee4b 100644 --- a/src/EventHandler.h +++ b/src/EventHandler.h @@ -26,9 +26,6 @@ public: void SetLocalHandler(Func* f); - void AddRemoteHandler(SourceID peer); - void RemoveRemoteHandler(SourceID peer); - void AutoPublish(std::string topic) { auto_publish.insert(std::move(topic)); @@ -75,10 +72,6 @@ private: bool error_handler; // this handler reports error messages. bool generate_always; - declare(List, SourceID); - typedef List(SourceID) receiver_list; - receiver_list receivers; - std::unordered_set auto_publish; }; diff --git a/src/EventRegistry.cc b/src/EventRegistry.cc index e28c7b4176..be3cf13799 100644 --- a/src/EventRegistry.cc +++ b/src/EventRegistry.cc @@ -1,6 +1,6 @@ #include "EventRegistry.h" #include "RE.h" -#include "RemoteSerializer.h" +#include "Reporter.h" void EventRegistry::Register(EventHandlerPtr handler) { diff --git a/src/Expr.cc b/src/Expr.cc index ff039ece35..e6cb9937c4 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -10,7 +10,6 @@ #include "Scope.h" #include "Stmt.h" #include "EventRegistry.h" -#include "RemoteSerializer.h" #include "Net.h" #include "Traverse.h" #include "Trigger.h" @@ -1884,13 +1883,6 @@ BoolExpr::BoolExpr(BroExprTag arg_tag, Expr* arg_op1, Expr* arg_op2) else SetType(base_type(TYPE_BOOL)); } - - else if ( bt1 == TYPE_PATTERN && bt2 == bt1 ) - { - reporter->Warning("&& and || operators deprecated for pattern operands"); - SetType(base_type(TYPE_PATTERN)); - } - else ExprError("requires boolean operands"); } diff --git a/src/Func.cc b/src/Func.cc index cbbbef6fa5..d34f97c197 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -42,7 +42,6 @@ #include "Sessions.h" #include "RE.h" #include "Serializer.h" -#include "RemoteSerializer.h" #include "Event.h" #include "Traverse.h" #include "Reporter.h" diff --git a/src/ID.cc b/src/ID.cc index e11625667a..b9b0a5a624 100644 --- a/src/ID.cc +++ b/src/ID.cc @@ -10,8 +10,6 @@ #include "Scope.h" #include "File.h" #include "Serializer.h" -#include "RemoteSerializer.h" -#include "PersistenceSerializer.h" #include "Scope.h" #include "Traverse.h" #include "zeekygen/Manager.h" @@ -78,12 +76,6 @@ void ID::SetVal(Val* v, Opcode op, bool arg_weak_ref) MutableVal::Properties props = 0; - if ( attrs && attrs->FindAttr(ATTR_SYNCHRONIZED) ) - props |= MutableVal::SYNCHRONIZED; - - if ( attrs && attrs->FindAttr(ATTR_PERSISTENT) ) - props |= MutableVal::PERSISTENT; - if ( attrs && attrs->FindAttr(ATTR_TRACKED) ) props |= MutableVal::TRACKED; @@ -198,27 +190,12 @@ void ID::UpdateValAttrs() if ( val && val->IsMutableVal() ) { - if ( attrs->FindAttr(ATTR_SYNCHRONIZED) ) - props |= MutableVal::SYNCHRONIZED; - - if ( attrs->FindAttr(ATTR_PERSISTENT) ) - props |= MutableVal::PERSISTENT; - if ( attrs->FindAttr(ATTR_TRACKED) ) props |= MutableVal::TRACKED; val->AsMutableVal()->AddProperties(props); } - if ( ! IsInternalGlobal() ) - { - if ( attrs->FindAttr(ATTR_SYNCHRONIZED) ) - remote_serializer->Register(this); - - if ( attrs->FindAttr(ATTR_PERSISTENT) ) - persistence_serializer->Register(this); - } - if ( val && val->Type()->Tag() == TYPE_TABLE ) val->AsTableVal()->SetAttrs(attrs); @@ -281,12 +258,6 @@ void ID::RemoveAttr(attr_tag a) { MutableVal::Properties props = 0; - if ( a == ATTR_SYNCHRONIZED ) - props |= MutableVal::SYNCHRONIZED; - - if ( a == ATTR_PERSISTENT ) - props |= MutableVal::PERSISTENT; - if ( a == ATTR_TRACKED ) props |= MutableVal::TRACKED; @@ -337,9 +308,6 @@ void ID::CopyFrom(const ID* id) offset = id->offset ; infer_return_type = id->infer_return_type; - if ( FindAttr(ATTR_PERSISTENT) ) - persistence_serializer->Unregister(this); - if ( id->type ) Ref(id->type); if ( id->val && ! id->weak_ref ) @@ -360,10 +328,6 @@ void ID::CopyFrom(const ID* id) #ifdef DEBUG UpdateValID(); #endif - - if ( FindAttr(ATTR_PERSISTENT) ) - persistence_serializer->Unregister(this); - } #endif ID* ID::Unserialize(UnserialInfo* info) @@ -396,12 +360,6 @@ ID* ID::Unserialize(UnserialInfo* info) else { - if ( info->id_policy != UnserialInfo::InstantiateNew ) - { - persistence_serializer->Unregister(current); - remote_serializer->Unregister(current); - } - switch ( info->id_policy ) { case UnserialInfo::Keep: @@ -473,12 +431,6 @@ ID* ID::Unserialize(UnserialInfo* info) } } - if ( id->FindAttr(ATTR_PERSISTENT) ) - persistence_serializer->Register(id); - - if ( id->FindAttr(ATTR_SYNCHRONIZED) ) - remote_serializer->Register(id); - return id; } diff --git a/src/Net.cc b/src/Net.cc index b61d365a2a..79b66e1a80 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -49,8 +49,6 @@ int reading_live = 0; int reading_traces = 0; int have_pending_timers = 0; double pseudo_realtime = 0.0; -bool using_communication = false; - double network_time = 0.0; // time according to last packet timestamp // (or current time) double processing_start_time = 0.0; // time started working on current pkt @@ -309,7 +307,7 @@ void net_run() } #endif current_iosrc = src; - auto communication_enabled = using_communication || broker_mgr->Active(); + auto communication_enabled = broker_mgr->Active(); if ( src ) src->Process(); // which will call net_packet_dispatch() @@ -372,11 +370,6 @@ void net_run() // current packet and its related events. termination_signal(); -#ifdef DEBUG_COMMUNICATION - if ( signal_val == SIGPROF && remote_serializer ) - remote_serializer->DumpDebugData(); -#endif - if ( ! reading_traces ) // Check whether we have timers scheduled for // the future on which we need to wait. diff --git a/src/Net.h b/src/Net.h index caea61c436..26a3d0f883 100644 --- a/src/Net.h +++ b/src/Net.h @@ -7,7 +7,6 @@ #include "util.h" #include "List.h" #include "Func.h" -#include "RemoteSerializer.h" #include "iosource/IOSource.h" #include "iosource/PktSrc.h" #include "iosource/PktDumper.h" @@ -67,9 +66,6 @@ extern double bro_start_network_time; // True if we're a in the process of cleaning-up just before termination. extern bool terminating; -// True if the remote serializer is to be activated. -extern bool using_communication; - // True if Bro is currently parsing scripts. extern bool is_parsing; diff --git a/src/NetVar.cc b/src/NetVar.cc index 57a5452123..9a6a20043f 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -30,7 +30,6 @@ RecordType* mime_match; int watchdog_interval; int max_timer_expires; -int max_remote_events_processed; int ignore_checksums; int partial_connection_ok; @@ -173,10 +172,6 @@ StringVal* log_encryption_key; StringVal* log_rotate_base_time; StringVal* peer_description; -RecordType* peer; -int forward_remote_state_changes; -int forward_remote_events; -int remote_check_sync_consistency; bro_uint_t chunked_io_buffer_soft_cap; StringVal* ssl_ca_certificate; @@ -199,8 +194,6 @@ int packet_filter_default; int sig_max_group_size; -int enable_syslog; - TableType* irc_join_list; RecordType* irc_join_info; TableVal* irc_servers; @@ -212,9 +205,6 @@ int dpd_ignore_ports; TableVal* likely_server_ports; -double remote_trace_sync_interval; -int remote_trace_sync_peers; - int check_for_unused_event_handlers; int dump_used_event_handlers; @@ -267,12 +257,6 @@ void init_general_global_var() peer_description = internal_val("peer_description")->AsStringVal(); - peer = internal_type("event_peer")->AsRecordType(); - forward_remote_state_changes = - opt_internal_int("forward_remote_state_changes"); - forward_remote_events = opt_internal_int("forward_remote_events"); - remote_check_sync_consistency = - opt_internal_int("remote_check_sync_consistency"); chunked_io_buffer_soft_cap = opt_internal_unsigned("chunked_io_buffer_soft_cap"); ssl_ca_certificate = internal_val("ssl_ca_certificate")->AsStringVal(); @@ -282,7 +266,6 @@ void init_general_global_var() packet_filter_default = opt_internal_int("packet_filter_default"); sig_max_group_size = opt_internal_int("sig_max_group_size"); - enable_syslog = opt_internal_int("enable_syslog"); check_for_unused_event_handlers = opt_internal_int("check_for_unused_event_handlers"); @@ -395,8 +378,6 @@ void init_net_var() watchdog_interval = int(opt_internal_double("watchdog_interval")); max_timer_expires = opt_internal_int("max_timer_expires"); - max_remote_events_processed = - opt_internal_int("max_remote_events_processed"); skip_authentication = internal_list_val("skip_authentication"); direct_login_prompts = internal_list_val("direct_login_prompts"); @@ -498,10 +479,6 @@ void init_net_var() irc_join_list = internal_type("irc_join_list")->AsTableType(); irc_servers = internal_val("irc_servers")->AsTableVal(); - remote_trace_sync_interval = - opt_internal_double("remote_trace_sync_interval"); - remote_trace_sync_peers = opt_internal_int("remote_trace_sync_peers"); - dpd_reassemble_first_packets = opt_internal_int("dpd_reassemble_first_packets"); dpd_buffer_size = opt_internal_int("dpd_buffer_size"); diff --git a/src/NetVar.h b/src/NetVar.h index 1dee27f372..92d717f50a 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -33,7 +33,6 @@ extern RecordType* mime_match; extern int watchdog_interval; extern int max_timer_expires; -extern int max_remote_events_processed; extern int ignore_checksums; extern int partial_connection_ok; @@ -176,10 +175,6 @@ extern StringVal* log_encryption_key; extern StringVal* log_rotate_base_time; extern StringVal* peer_description; -extern RecordType* peer; -extern int forward_remote_state_changes; -extern int forward_remote_events; -extern int remote_check_sync_consistency; extern bro_uint_t chunked_io_buffer_soft_cap; extern StringVal* ssl_ca_certificate; @@ -201,8 +196,6 @@ extern int packet_filter_default; extern int sig_max_group_size; -extern int enable_syslog; - extern TableType* irc_join_list; extern RecordType* irc_join_info; extern TableVal* irc_servers; @@ -214,9 +207,6 @@ extern int dpd_ignore_ports; extern TableVal* likely_server_ports; -extern double remote_trace_sync_interval; -extern int remote_trace_sync_peers; - extern int check_for_unused_event_handlers; extern int dump_used_event_handlers; diff --git a/src/PersistenceSerializer.cc b/src/PersistenceSerializer.cc deleted file mode 100644 index 6f4082314f..0000000000 --- a/src/PersistenceSerializer.cc +++ /dev/null @@ -1,577 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -#include "PersistenceSerializer.h" -#include "RemoteSerializer.h" -#include "Conn.h" -#include "Event.h" -#include "Reporter.h" -#include "Net.h" - -static void persistence_serializer_delete_func(void* val) - { - time_t* t = reinterpret_cast(val); - free(t); - } - -class IncrementalWriteTimer : public Timer { -public: - IncrementalWriteTimer(double t, PersistenceSerializer::SerialStatus* s) - : Timer(t, TIMER_INCREMENTAL_WRITE), status(s) {} - - void Dispatch(double t, int is_expire); - - PersistenceSerializer::SerialStatus* status; -}; - -void IncrementalWriteTimer::Dispatch(double t, int is_expire) - { - // Never suspend when we're finishing up. - if ( terminating ) - status->info.may_suspend = false; - - persistence_serializer->RunSerialization(status); - } - -PersistenceSerializer::PersistenceSerializer() - { - dir = 0; - files.SetDeleteFunc(persistence_serializer_delete_func); - } - -PersistenceSerializer::~PersistenceSerializer() - { - } - -void PersistenceSerializer::Register(ID* id) - { - if ( id->Type()->Tag() == TYPE_FUNC ) - { - Error("can't register functions as persistent ID"); - return; - } - - DBG_LOG(DBG_STATE, "&persistent %s", id->Name()); - - HashKey key(id->Name()); - if ( persistent_ids.Lookup(&key) ) - return; - - Ref(id); - persistent_ids.Insert(&key, id); - } - -void PersistenceSerializer::Unregister(ID* id) - { - HashKey key(id->Name()); - Unref((ID*) persistent_ids.Remove(&key)); - } - -void PersistenceSerializer::Register(Connection* conn) - { - if ( persistent_conns.Lookup(conn->Key()) ) - return; - - Ref(conn); - HashKey* k = conn->Key(); - HashKey* new_key = new HashKey(k->Key(), k->Size(), k->Hash()); - persistent_conns.Insert(new_key, conn); - delete new_key; - } - -void PersistenceSerializer::Unregister(Connection* conn) - { - Unref(persistent_conns.RemoveEntry(conn->Key())); - } - -bool PersistenceSerializer::CheckTimestamp(const char* file) - { - struct stat s; - if ( stat(file, &s) < 0 ) - return false; - - if ( ! S_ISREG(s.st_mode) ) - return false; - - bool changed = true; - - HashKey* key = new HashKey(file, strlen(file)); - time_t* t = files.Lookup(key); - - if ( ! t ) - { - t = (time_t*) malloc(sizeof(time_t)); - if ( ! t ) - out_of_memory("saving file timestamp"); - files.Insert(key, t); - } - - else if ( *t >= s.st_mtime ) - changed = false; - - *t = s.st_mtime; - - delete key; - return changed; - } - -bool PersistenceSerializer::CheckForFile(UnserialInfo* info, const char* file, - bool delete_file) - { - bool ret = true; - if ( CheckTimestamp(file) ) - { - // Need to copy the filename here, as it may be passed - // in via fmt(). - const char* f = copy_string(file); - - bool ret = Read(info, f); - - if ( delete_file && unlink(f) < 0 ) - Error(fmt("can't delete file %s: %s", f, strerror(errno))); - - delete [] f; - } - - return ret; - } - -bool PersistenceSerializer::ReadAll(bool is_init, bool delete_files) - { -#ifdef USE_PERFTOOLS_DEBUG - HeapLeakChecker::Disabler disabler; -#endif - - assert(dir); - - UnserialInfo config_info(this); - config_info.id_policy = is_init ? - UnserialInfo::Replace : UnserialInfo::CopyCurrentToNew; - - if ( ! CheckForFile(&config_info, fmt("%s/config.bst", dir), - delete_files) ) - return false; - - UnserialInfo state_info(this); - state_info.id_policy = UnserialInfo::CopyNewToCurrent; - if ( ! CheckForFile(&state_info, fmt("%s/state.bst", dir), - delete_files) ) - return false; - - return true; - } - -bool PersistenceSerializer::MoveFileUp(const char* dir, const char* file) - { - char oldname[PATH_MAX]; - char newname[PATH_MAX]; - - safe_snprintf(oldname, PATH_MAX, "%s/.tmp/%s", dir, file ); - safe_snprintf(newname, PATH_MAX, "%s/%s", dir, file ); - - if ( rename(oldname, newname) < 0 ) - { - Error(fmt("can't move %s to %s: %s", oldname, newname, - strerror(errno))); - return false; - } - - CheckTimestamp(newname); - return true; - } - -#if 0 -void PersistenceSerializer::RaiseFinishedSendState() - { - val_list* vl = new val_list; - vl->append(new AddrVal(htonl(remote_host))); - vl->append(val_mgr->GetPort(remote_port)); - - mgr.QueueEvent(finished_send_state, vl); - reporter->Log("Serialization done."); - } -#endif - -void PersistenceSerializer::GotEvent(const char* name, double time, - EventHandlerPtr event, val_list* args) - { - mgr.QueueEvent(event, std::move(*args)); - delete args; - } - -void PersistenceSerializer::GotFunctionCall(const char* name, double time, - Func* func, val_list* args) - { - try - { - func->Call(args); - } - - catch ( InterpreterException& e ) - { /* Already reported. */ } - } - -void PersistenceSerializer::GotStateAccess(StateAccess* s) - { - s->Replay(); - delete s; - } - -void PersistenceSerializer::GotTimer(Timer* s) - { - reporter->Error("PersistenceSerializer::GotTimer not implemented"); - } - -void PersistenceSerializer::GotConnection(Connection* c) - { - Unref(c); - } - -void PersistenceSerializer::GotID(ID* id, Val* /* val */) - { - Unref(id); - } - -void PersistenceSerializer::GotPacket(Packet* p) - { - reporter->Error("PersistenceSerializer::GotPacket not implemented"); - } - -bool PersistenceSerializer::LogAccess(const StateAccess& s) - { - if ( ! IsSerializationRunning() ) - return true; - - loop_over_list(running, i) - { - running[i]->accesses.append(new StateAccess(s)); - } - - return true; - } - -bool PersistenceSerializer::WriteState(bool may_suspend) - { - SerialStatus* status = - new SerialStatus(this, SerialStatus::WritingState); - - status->info.may_suspend = may_suspend; - - status->ids = &persistent_ids; - status->conns = &persistent_conns; - status->filename = "state.bst"; - - return RunSerialization(status); - } - -bool PersistenceSerializer::WriteConfig(bool may_suspend) - { - if ( mgr.IsDraining() && may_suspend ) - // Events which trigger checkpoint are flushed. Ignore; we'll - // checkpoint at termination in any case. - return true; - - SerialStatus* status = - new SerialStatus(this, SerialStatus::WritingConfig); - - status->info.may_suspend = may_suspend; - status->info.clear_containers = true; - status->ids = global_scope()->GetIDs(); - status->filename = "config.bst"; - - return RunSerialization(status); - } - -bool PersistenceSerializer::SendState(SourceID peer, bool may_suspend) - { - SerialStatus* status = - new SerialStatus(remote_serializer, SerialStatus::SendingState); - - status->info.may_suspend = may_suspend; - status->ids = &persistent_ids; - status->conns = &persistent_conns; - status->peer = peer; - - reporter->Info("Sending state..."); - - return RunSerialization(status); - } - -bool PersistenceSerializer::SendConfig(SourceID peer, bool may_suspend) - { - SerialStatus* status = - new SerialStatus(remote_serializer, SerialStatus::SendingConfig); - - status->info.may_suspend = may_suspend; - status->info.clear_containers = true; - status->ids = global_scope()->GetIDs(); - status->peer = peer; - - reporter->Info("Sending config..."); - - return RunSerialization(status); - } - -bool PersistenceSerializer::RunSerialization(SerialStatus* status) - { - Continuation* cont = &status->info.cont; - - if ( cont->NewInstance() ) - { - // Serialization is starting. Initialize. - - // See if there is already a serialization of this type running. - loop_over_list(running, i) - { - if ( running[i]->type == status->type ) - { - reporter->Warning("Serialization of type %d already running.", status->type); - return false; - } - } - - running.append(status); - - // Initialize. - if ( ! (ensure_dir(dir) && ensure_dir(fmt("%s/.tmp", dir))) ) - return false; - - if ( ! OpenFile(fmt("%s/.tmp/%s", dir, status->filename), false) ) - return false; - - if ( ! PrepareForWriting() ) - return false; - - if ( status->ids ) - { - status->id_cookie = status->ids->InitForIteration(); - status->ids->MakeRobustCookie(status->id_cookie); - } - - if ( status->conns ) - { - status->conn_cookie = status->conns->InitForIteration(); - status->conns->MakeRobustCookie(status->conn_cookie); - } - } - - else if ( cont->ChildSuspended() ) - { - // One of our former Serialize() calls suspended itself. - // We have to call it again. - - if ( status->id_cookie ) - { - if ( ! DoIDSerialization(status, status->current.id) ) - return false; - - if ( cont->ChildSuspended() ) - { - // Oops, it did it again. - timer_mgr->Add(new IncrementalWriteTimer(network_time + state_write_delay, status)); - return true; - } - } - - else if ( status->conn_cookie ) - { - if ( ! DoConnSerialization(status, status->current.conn) ) - return false; - - if ( cont->ChildSuspended() ) - { - // Oops, it did it again. - timer_mgr->Add(new IncrementalWriteTimer(network_time + state_write_delay, status)); - return true; - } - } - - else - reporter->InternalError("unknown suspend state"); - } - - else if ( cont->Resuming() ) - cont->Resume(); - - else - reporter->InternalError("unknown continuation state"); - - if ( status->id_cookie ) - { - ID* id; - - while ( (id = status->ids->NextEntry(status->id_cookie)) ) - { - if ( ! DoIDSerialization(status, id) ) - return false; - - if ( cont->ChildSuspended() ) - { - timer_mgr->Add(new IncrementalWriteTimer(network_time + state_write_delay, status)); - return true; - } - - if ( status->info.may_suspend ) - { - timer_mgr->Add(new IncrementalWriteTimer(network_time + state_write_delay, status)); - cont->Suspend(); - return true; - } - } - - // Cookie has been set to 0 by NextEntry(). - } - - if ( status->conn_cookie ) - { - Connection* conn; - while ( (conn = status->conns->NextEntry(status->conn_cookie)) ) - { - if ( ! DoConnSerialization(status, conn) ) - return false; - - if ( cont->ChildSuspended() ) - { - timer_mgr->Add(new IncrementalWriteTimer(network_time + state_write_delay, status)); - return true; - } - - if ( status->info.may_suspend ) - { - timer_mgr->Add(new IncrementalWriteTimer(network_time + state_write_delay, status)); - cont->Suspend(); - return true; - } - - } - - // Cookie has been set to 0 by NextEntry(). - } - - DBG_LOG(DBG_STATE, "finished serialization; %d accesses pending", - status->accesses.length()); - - if ( status->accesses.length() ) - { - // Serialize pending state accesses. - // FIXME: Does this need to suspend? - StateAccess* access; - loop_over_list(status->accesses, i) - { - // Serializing a StateAccess will not suspend. - if ( ! DoAccessSerialization(status, status->accesses[i]) ) - return false; - - delete status->accesses[i]; - } - } - - // Finalize. - CloseFile(); - - bool ret = MoveFileUp(dir, status->filename); - - loop_over_list(running, i) - { - if ( running[i]->type == status->type ) - { - running.remove_nth(i); - break; - } - } - - delete status; - return ret; - } - -bool PersistenceSerializer::DoIDSerialization(SerialStatus* status, ID* id) - { - bool success = false; - Continuation* cont = &status->info.cont; - - status->current.id = id; - - switch ( status->type ) { - case SerialStatus::WritingState: - case SerialStatus::WritingConfig: - cont->SaveContext(); - success = Serialize(&status->info, *id); - cont->RestoreContext(); - break; - - case SerialStatus::SendingState: - case SerialStatus::SendingConfig: - cont->SaveContext(); - success = remote_serializer->SendID(&status->info, - status->peer, *id); - cont->RestoreContext(); - break; - - default: - reporter->InternalError("unknown serialization type"); - } - - return success; - } - -bool PersistenceSerializer::DoConnSerialization(SerialStatus* status, - Connection* conn) - { - bool success = false; - Continuation* cont = &status->info.cont; - - status->current.conn = conn; - - switch ( status->type ) { - case SerialStatus::WritingState: - case SerialStatus::WritingConfig: - cont->SaveContext(); - success = Serialize(&status->info, *conn); - cont->RestoreContext(); - break; - - case SerialStatus::SendingState: - case SerialStatus::SendingConfig: - cont->SaveContext(); - success = remote_serializer->SendConnection(&status->info, - status->peer, *conn); - cont->RestoreContext(); - break; - - default: - reporter->InternalError("unknown serialization type"); - } - - return success; - } - -bool PersistenceSerializer::DoAccessSerialization(SerialStatus* status, - StateAccess* access) - { - bool success = false; - DisableSuspend suspend(&status->info); - - switch ( status->type ) { - case SerialStatus::WritingState: - case SerialStatus::WritingConfig: - success = Serialize(&status->info, *access); - break; - - case SerialStatus::SendingState: - case SerialStatus::SendingConfig: - success = remote_serializer->SendAccess(&status->info, - status->peer, *access); - break; - - default: - reporter->InternalError("unknown serialization type"); - } - - return success; - } diff --git a/src/PersistenceSerializer.h b/src/PersistenceSerializer.h deleted file mode 100644 index 99d8da88c4..0000000000 --- a/src/PersistenceSerializer.h +++ /dev/null @@ -1,165 +0,0 @@ -// Implements persistance for Bro's data structures. - -#ifndef persistence_serializer_h -#define persistence_serializer_h - -#include "Serializer.h" -#include "List.h" - -class StateAccess; - -class PersistenceSerializer : public FileSerializer { -public: - PersistenceSerializer(); - - ~PersistenceSerializer() override; - - // Define the directory where to store the data. - void SetDir(const char* arg_dir) { dir = copy_string(arg_dir); } - - // Register/unregister the ID/connection to be saved by WriteAll(). - void Register(ID* id); - void Unregister(ID* id); - void Register(Connection* conn); - void Unregister(Connection* conn); - - // Read all data that has been changed since last scan of directory. - // is_init should be true for the first read upon start-up. All existing - // state will be cleared. If delete_files is true, file which have been - // read are removed (even if the read was unsuccessful!). - bool ReadAll(bool is_init, bool delete_files); - - // Each of the following four methods may suspend operation. - // If they do, they install a Timer which resumes after some - // amount of time. If a function is called again before it - // has completely finished its task, it will do nothing and - // return false. - - bool WriteState(bool may_suspend); - - // Writes Bro's configuration (w/o dynamic state). - bool WriteConfig(bool may_suspend); - - // Sends all registered state to remote host - // (by leveraging the remote_serializer). - bool SendState(SourceID peer, bool may_suspend); - - // Sends Bro's config to remote host - // (by leveraging the remote_serializer). - bool SendConfig(SourceID peer, bool may_suspend); - - // Returns true if a serialization is currently running. - bool IsSerializationRunning() const { return running.length(); } - - // Tells the serializer that this access was performed. If a - // serialization is going on, it may store it. (Need only be called if - // IsSerializationRunning() returns true.) - bool LogAccess(const StateAccess& s); - -protected: - friend class RemoteSerializer; - friend class IncrementalWriteTimer; - - void GotID(ID* id, Val* val) override; - void GotEvent(const char* name, double time, - EventHandlerPtr event, val_list* args) override; - void GotFunctionCall(const char* name, double time, - Func* func, val_list* args) override; - void GotStateAccess(StateAccess* s) override; - void GotTimer(Timer* t) override; - void GotConnection(Connection* c) override; - void GotPacket(Packet* packet) override; - - // If file has changed since last check, read it. - bool CheckForFile(UnserialInfo* info, const char* file, - bool delete_file); - - // Returns true if it's a regular file and has a more recent timestamp - // than last time we checked it. - bool CheckTimestamp(const char* file); - - // Move file from /tmp/ to /. Afterwards, call - // CheckTimestamp() with /. - bool MoveFileUp(const char* dir, const char* file); - - // Generates an error message, terminates current serialization, - // and returns false. - bool SerialError(const char* msg); - - // Start a new serialization. - struct SerialStatus; - bool RunSerialization(SerialStatus* status); - - // Helpers for RunSerialization. - bool DoIDSerialization(SerialStatus* status, ID* id); - bool DoConnSerialization(SerialStatus* status, Connection* conn); - bool DoAccessSerialization(SerialStatus* status, StateAccess* access); - - typedef PDict(ID) id_map; - - declare(PDict, Connection); - typedef PDict(Connection) conn_map; - - struct SerialStatus { - enum Type { - WritingState, WritingConfig, - SendingState, SendingConfig, - }; - - SerialStatus(Serializer* s, Type arg_type) : info(s) - { - type = arg_type; - ids = 0; - id_cookie = 0; - conns = 0; - conn_cookie = 0; - peer = SOURCE_LOCAL; - filename = 0; - } - - Type type; - SerialInfo info; - - // IDs to serialize. - id_map* ids; - IterCookie* id_cookie; - - // Connections to serialize. - conn_map* conns; - IterCookie* conn_cookie; - - // Accesses performed while we're serializing. - declare(PList,StateAccess); - typedef PList(StateAccess) state_access_list; - state_access_list accesses; - - // The ID/Conn we're currently serializing. - union { - ID* id; - Connection* conn; - } current; - - // Only set if type is Writing{State,Config}. - const char* filename; - - // Only set if type is Sending{State,Config}. - SourceID peer; - }; - - const char* dir; - - declare(PList, SerialStatus); - PList(SerialStatus) running; - - id_map persistent_ids; - conn_map persistent_conns; - - // To keep track of files' modification times. - declare(PDict, time_t); - typedef PDict(time_t) file_map; - file_map files; -}; - -extern PersistenceSerializer* persistence_serializer; - -#endif diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc deleted file mode 100644 index 3abd8e6423..0000000000 --- a/src/RemoteSerializer.cc +++ /dev/null @@ -1,4585 +0,0 @@ -// Processes involved in the communication: -// -// (Local-Parent) <-> (Local-Child) <-> (Remote-Child) <-> (Remote-Parent) -// -// Message types (for parent<->child communication the CMsg's peer indicates -// about whom we're talking). -// -// Communication protocol version -// VERSION -// [] -// -// Send serialization -// SERIAL -// -// Terminate(d) connection -// CLOSE -// -// Close(d) all connections -// CLOSE_ALL -// -// Connect to remote side -// CONNECT_TO -// -// Connected to remote side -// CONNECTED -// -// Request events from remote side -// REQUEST_EVENTS -// -// Request synchronization of IDs with remote side -// REQUEST_SYNC -// -// Listen for connection on ip/port (ip may be INADDR_ANY) -// LISTEN -// -// Close listen ports. -// LISTEN_STOP -// -// Error caused by host -// ERROR -// -// Some statistics about the given peer connection -// STATS -// -// Requests to set a new capture_filter -// CAPTURE_FILTER -// -// Ping to peer -// PING -// -// Pong from peer -// PONG -// -// Announce our capabilities -// CAPS -// -// Activate compression (parent->child) -// COMPRESS -// -// Indicate that all following blocks are compressed (child->child) -// COMPRESS -// -// Synchronize for pseudo-realtime processing. -// Signals that we have reached sync-point number . -// SYNC_POINT -// -// Signals the child that we want to terminate. Anything sent after this may -// get lost. When the child answers with another TERMINATE it is safe to -// shutdown. -// TERMINATE -// -// Debug-only: tell child to dump recently received/sent data to disk. -// DEBUG_DUMP -// -// Valid messages between processes: -// -// Main -> Child -// CONNECT_TO -// REQUEST_EVENTS -// SERIAL -// CLOSE -// CLOSE_ALL -// LISTEN -// LISTEN_STOP -// CAPTURE_FILTER -// VERSION -// REQUEST_SYNC -// PHASE_DONE -// PING -// PONG -// CAPS -// COMPRESS -// SYNC_POINT -// DEBUG_DUMP -// REMOTE_PRINT -// -// Child -> Main -// CONNECTED -// REQUEST_EVENTS -// SERIAL -// CLOSE -// ERROR -// STATS -// VERSION -// CAPTURE_FILTER -// REQUEST_SYNC -// PHASE_DONE -// PING -// PONG -// CAPS -// LOG -// SYNC_POINT -// REMOTE_PRINT -// -// Child <-> Child -// VERSION -// SERIAL -// REQUEST_EVENTS -// CAPTURE_FILTER -// REQUEST_SYNC -// PHASE_DONE -// PING -// PONG -// CAPS -// COMPRESS -// SYNC_POINT -// REMOTE_PRINT -// -// A connection between two peers has four phases: -// -// Setup: -// Initial phase. -// VERSION messages must be exchanged. -// Ends when both peers have sent VERSION. -// Handshake: -// REQUEST_EVENTS/REQUEST_SYNC/CAPTURE_FILTER/CAPS/selected SERIALs -// may be exchanged. -// Phase ends when both peers have sent PHASE_DONE. -// State synchronization: -// Entered iff at least one of the peers has sent REQUEST_SYNC. -// The peer with the smallest runtime (incl. in VERSION msg) sends -// SERIAL messages compromising all of its state. -// Phase ends when peer sends another PHASE_DONE. -// Running: -// Peers exchange SERIAL (and PING/PONG) messages. -// Phase ends with connection tear-down by one of the peers. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "bro-config.h" -#ifdef TIME_WITH_SYS_TIME -# include -# include -#else -# ifdef HAVE_SYS_TIME_H -# include -# else -# include -# endif -#endif -#include - -#include -#include -#include -#include - -#include "RemoteSerializer.h" -#include "Func.h" -#include "EventRegistry.h" -#include "Event.h" -#include "Net.h" -#include "NetVar.h" -#include "Scope.h" -#include "Sessions.h" -#include "File.h" -#include "Conn.h" -#include "Reporter.h" -#include "IPAddr.h" -#include "bro_inet_ntop.h" -#include "iosource/Manager.h" -#include "logging/Manager.h" -#include "logging/logging.bif.h" - -extern "C" { -#include "setsignal.h" -}; - -// Gets incremented each time there's an incompatible change -// to the communication internals. -static const unsigned short PROTOCOL_VERSION = 0x07; - -static const char MSG_NONE = 0x00; -static const char MSG_VERSION = 0x01; -static const char MSG_SERIAL = 0x02; -static const char MSG_CLOSE = 0x03; -static const char MSG_CLOSE_ALL = 0x04; -static const char MSG_ERROR = 0x05; -static const char MSG_CONNECT_TO = 0x06; -static const char MSG_CONNECTED = 0x07; -static const char MSG_REQUEST_EVENTS = 0x08; -static const char MSG_LISTEN = 0x09; -static const char MSG_LISTEN_STOP = 0x0a; -static const char MSG_STATS = 0x0b; -static const char MSG_CAPTURE_FILTER = 0x0c; -static const char MSG_REQUEST_SYNC = 0x0d; -static const char MSG_PHASE_DONE = 0x0e; -static const char MSG_PING = 0x0f; -static const char MSG_PONG = 0x10; -static const char MSG_CAPS = 0x11; -static const char MSG_COMPRESS = 0x12; -static const char MSG_LOG = 0x13; -static const char MSG_SYNC_POINT = 0x14; -static const char MSG_TERMINATE = 0x15; -static const char MSG_DEBUG_DUMP = 0x16; -static const char MSG_REMOTE_PRINT = 0x17; -static const char MSG_LOG_CREATE_WRITER = 0x18; -static const char MSG_LOG_WRITE = 0x19; -static const char MSG_REQUEST_LOGS = 0x20; - -// Update this one whenever adding a new ID: -static const char MSG_ID_MAX = MSG_REQUEST_LOGS; - -static const uint32 FINAL_SYNC_POINT = /* UINT32_MAX */ 4294967295U; - -// Buffer size for remote-print data -static const int PRINT_BUFFER_SIZE = 10 * 1024; -static const int SOCKBUF_SIZE = 1024 * 1024; - -// Buffer size for remote-log data. -static const int LOG_BUFFER_SIZE = 50 * 1024; - -struct ping_args { - uint32 seq; - double time1; // Round-trip time parent1<->parent2 - double time2; // Round-trip time child1<->parent2 - double time3; // Round-trip time child2<->parent2 -}; - -#ifdef DEBUG -# define DEBUG_COMM(msg) DBG_LOG(DBG_COMM, "%s", msg) -#else -# define DEBUG_COMM(msg) -#endif - -#define READ_CHUNK(i, c, do_if_eof, kill_me) \ - { \ - if ( ! i->Read(&c) ) \ - { \ - if ( i->Eof() ) \ - { \ - do_if_eof; \ - } \ - else \ - Error(fmt("can't read data chunk: %s", io->Error()), kill_me); \ - return false; \ - } \ - \ - if ( ! c ) \ - return true; \ - } - -#define READ_CHUNK_FROM_CHILD(c) \ - { \ - if ( ! io->Read(&c) ) \ - { \ - if ( io->Eof() ) \ - ChildDied(); \ - else \ - Error(fmt("can't read data chunk: %s", io->Error())); \ - return false; \ - } \ - \ - if ( ! c ) \ - { \ - SetIdle(io->IsIdle());\ - return true; \ - } \ - SetIdle(false); \ - } - -static const char* msgToStr(int msg) - { -# define MSG_STR(x) case x: return #x; - switch ( msg ) { - MSG_STR(MSG_VERSION) - MSG_STR(MSG_NONE) - MSG_STR(MSG_SERIAL) - MSG_STR(MSG_CLOSE) - MSG_STR(MSG_CLOSE_ALL) - MSG_STR(MSG_ERROR) - MSG_STR(MSG_CONNECT_TO) - MSG_STR(MSG_CONNECTED) - MSG_STR(MSG_REQUEST_EVENTS) - MSG_STR(MSG_LISTEN) - MSG_STR(MSG_LISTEN_STOP) - MSG_STR(MSG_STATS) - MSG_STR(MSG_CAPTURE_FILTER) - MSG_STR(MSG_REQUEST_SYNC) - MSG_STR(MSG_PHASE_DONE) - MSG_STR(MSG_PING) - MSG_STR(MSG_PONG) - MSG_STR(MSG_CAPS) - MSG_STR(MSG_COMPRESS) - MSG_STR(MSG_LOG) - MSG_STR(MSG_SYNC_POINT) - MSG_STR(MSG_TERMINATE) - MSG_STR(MSG_DEBUG_DUMP) - MSG_STR(MSG_REMOTE_PRINT) - MSG_STR(MSG_LOG_CREATE_WRITER) - MSG_STR(MSG_LOG_WRITE) - MSG_STR(MSG_REQUEST_LOGS) - default: - return "UNKNOWN_MSG"; - } - } - -static vector tokenize(const string& s, char delim) - { - vector tokens; - stringstream ss(s); - string token; - - while ( std::getline(ss, token, delim) ) - tokens.push_back(token); - - return tokens; - } - -// Start of every message between two processes. We do the low-level work -// ourselves to make this 64-bit safe. (The actual layout is an artifact of -// an earlier design that depended on how a 32-bit GCC lays out its structs ...) -class CMsg { -public: - CMsg(char type, RemoteSerializer::PeerID peer) - { - buffer[0] = type; - uint32 tmp = htonl(peer); - memcpy(buffer + 4, &tmp, sizeof(tmp)); - } - - char Type() { return buffer[0]; } - - RemoteSerializer::PeerID Peer() - { - uint32 tmp; - memcpy(&tmp, buffer + 4, sizeof(tmp)); - return ntohl(tmp); - } - - const char* Raw() { return buffer; } - -private: - char buffer[8]; -}; - -static bool sendCMsg(ChunkedIO* io, char msg_type, RemoteSerializer::PeerID id) - { - // We use the new[] operator here to avoid mismatches - // when deleting the data. - CMsg* msg = (CMsg*) new char[sizeof(CMsg)]; - new (msg) CMsg(msg_type, id); - - ChunkedIO::Chunk* c = new ChunkedIO::Chunk((char*)msg, sizeof(CMsg)); - return io->Write(c); - } - -static ChunkedIO::Chunk* makeSerialMsg(RemoteSerializer::PeerID id) - { - // We use the new[] operator here to avoid mismatches - // when deleting the data. - CMsg* msg = (CMsg*) new char[sizeof(CMsg)]; - new (msg) CMsg(MSG_SERIAL, id); - - ChunkedIO::Chunk* c = new ChunkedIO::Chunk((char*)msg, sizeof(CMsg)); - return c; - } - -inline void RemoteSerializer::SetupSerialInfo(SerialInfo* info, Peer* peer) - { - info->chunk = makeSerialMsg(peer->id); - if ( peer->caps & Peer::NO_CACHING ) - info->cache = false; - - if ( ! (peer->caps & Peer::PID_64BIT) || peer->phase != Peer::RUNNING ) - info->pid_32bit = true; - - if ( (peer->caps & Peer::NEW_CACHE_STRATEGY) && - peer->phase == Peer::RUNNING ) - info->new_cache_strategy = true; - - if ( (peer->caps & Peer::BROCCOLI_PEER) ) - info->broccoli_peer = true; - - info->include_locations = false; - } - -static bool sendToIO(ChunkedIO* io, ChunkedIO::Chunk* c) - { - if ( ! io->Write(c) ) - { - reporter->Warning("can't send chunk: %s", io->Error()); - return false; - } - - return true; - } - -static bool sendToIO(ChunkedIO* io, char msg_type, RemoteSerializer::PeerID id, - const char* str, int len = -1, bool delete_with_free = false) - { - if ( ! sendCMsg(io, msg_type, id) ) - { - reporter->Warning("can't send message of type %d: %s", msg_type, io->Error()); - return false; - } - - uint32 sz = len >= 0 ? len : strlen(str) + 1; - ChunkedIO::Chunk* c = new ChunkedIO::Chunk(const_cast(str), sz); - - if ( delete_with_free ) - c->free_func = ChunkedIO::Chunk::free_func_free; - else - c->free_func = ChunkedIO::Chunk::free_func_delete; - - return sendToIO(io, c); - } - -static bool sendToIO(ChunkedIO* io, char msg_type, RemoteSerializer::PeerID id, - int nargs, va_list ap) - { - if ( ! sendCMsg(io, msg_type, id) ) - { - reporter->Warning("can't send message of type %d: %s", msg_type, io->Error()); - return false; - } - - if ( nargs == 0 ) - return true; - - uint32* args = new uint32[nargs]; - - for ( int i = 0; i < nargs; i++ ) - args[i] = htonl(va_arg(ap, uint32)); - - ChunkedIO::Chunk* c = new ChunkedIO::Chunk((char*)args, - sizeof(uint32) * nargs); - return sendToIO(io, c); - } - -#ifdef DEBUG -static inline char* fmt_uint32s(int nargs, va_list ap) - { - static char buf[512]; - char* p = buf; - *p = '\0'; - for ( int i = 0; i < nargs; i++ ) - p += snprintf(p, sizeof(buf) - (p - buf), - " 0x%08x", va_arg(ap, uint32)); - buf[511] = '\0'; - return buf; - } -#endif - -static pid_t child_pid = 0; - -// Return true if message type is sent by a peer (rather than the child -// process itself). -static inline bool is_peer_msg(int msg) - { - return msg == MSG_VERSION || - msg == MSG_SERIAL || - msg == MSG_REQUEST_EVENTS || - msg == MSG_REQUEST_SYNC || - msg == MSG_CAPTURE_FILTER || - msg == MSG_PHASE_DONE || - msg == MSG_PING || - msg == MSG_PONG || - msg == MSG_CAPS || - msg == MSG_COMPRESS || - msg == MSG_SYNC_POINT || - msg == MSG_REMOTE_PRINT || - msg == MSG_LOG_CREATE_WRITER || - msg == MSG_LOG_WRITE || - msg == MSG_REQUEST_LOGS; - } - -bool RemoteSerializer::IsConnectedPeer(PeerID id) - { - if ( id == PEER_NONE ) - return true; - - return LookupPeer(id, true) != 0; - } - -class IncrementalSendTimer : public Timer { -public: - IncrementalSendTimer(double t, RemoteSerializer::Peer* p, SerialInfo* i) - : Timer(t, TIMER_INCREMENTAL_SEND), info(i), peer(p) {} - virtual void Dispatch(double t, int is_expire) - { - // Never suspend when we're finishing up. - if ( terminating ) - info->may_suspend = false; - - remote_serializer->SendAllSynchronized(peer, info); - } - - SerialInfo* info; - RemoteSerializer::Peer* peer; -}; - -RemoteSerializer::RemoteSerializer() - { - initialized = false; - current_peer = 0; - msgstate = TYPE; - id_counter = 1; - listening = false; - ignore_accesses = false; - propagate_accesses = 1; - current_sync_point = 0; - syncing_times = false; - io = 0; - terminating = false; - in_sync = 0; - last_flush = 0; - received_logs = 0; - current_id = 0; - current_msgtype = 0; - current_args = 0; - source_peer = 0; - - // Register as a "dont-count" source first, we may change that later. - iosource_mgr->Register(this, true); - } - -RemoteSerializer::~RemoteSerializer() - { - if ( child_pid ) - { - if ( kill(child_pid, SIGKILL) < 0 ) - reporter->Warning("warning: cannot kill child (pid %d), %s", child_pid, strerror(errno)); - - else if ( waitpid(child_pid, 0, 0) < 0 ) - reporter->Warning("warning: error encountered during waitpid(%d), %s", child_pid, strerror(errno)); - } - - delete io; - } - -void RemoteSerializer::Enable() - { - if ( initialized ) - return; - - if ( reading_traces && ! pseudo_realtime ) - { - using_communication = 0; - return; - } - - Fork(); - - Log(LogInfo, fmt("communication started, parent pid is %d, child pid is %d", getpid(), child_pid)); - initialized = 1; - } - -void RemoteSerializer::SetSocketBufferSize(int fd, int opt, const char *what, int size, int verbose) - { - int defsize = 0; - socklen_t len = sizeof(defsize); - - if ( getsockopt(fd, SOL_SOCKET, opt, (void *)&defsize, &len) < 0 ) - { - if ( verbose ) - Log(LogInfo, fmt("warning: cannot get socket buffer size (%s): %s", what, strerror(errno))); - return; - } - - for ( int trysize = size; trysize > defsize; trysize -= 1024 ) - { - if ( setsockopt(fd, SOL_SOCKET, opt, &trysize, sizeof(trysize)) >= 0 ) - { - if ( verbose ) - { - if ( trysize == size ) - Log(LogInfo, fmt("raised pipe's socket buffer size from %dK to %dK", defsize / 1024, trysize / 1024)); - else - Log(LogInfo, fmt("raised pipe's socket buffer size from %dK to %dK (%dK was requested)", defsize / 1024, trysize / 1024, size / 1024)); - } - return; - } - } - - Log(LogInfo, fmt("warning: cannot increase %s socket buffer size from %dK (%dK was requested)", what, defsize / 1024, size / 1024)); - } - -void RemoteSerializer::Fork() - { - if ( child_pid ) - return; - - // Register as a "does-count" source now. - iosource_mgr->Register(this, false); - - // If we are re-forking, remove old entries - loop_over_list(peers, i) - RemovePeer(peers[i]); - - // Create pipe for communication between parent and child. - int pipe[2]; - - if ( socketpair(AF_UNIX, SOCK_STREAM, 0, pipe) < 0 ) - { - Error(fmt("can't create pipe: %s", strerror(errno))); - return; - } - - // Try to increase the size of the socket send and receive buffers. - SetSocketBufferSize(pipe[0], SO_SNDBUF, "SO_SNDBUF", SOCKBUF_SIZE, 1); - SetSocketBufferSize(pipe[0], SO_RCVBUF, "SO_RCVBUF", SOCKBUF_SIZE, 0); - SetSocketBufferSize(pipe[1], SO_SNDBUF, "SO_SNDBUF", SOCKBUF_SIZE, 0); - SetSocketBufferSize(pipe[1], SO_RCVBUF, "SO_RCVBUF", SOCKBUF_SIZE, 0); - - child_pid = 0; - - int pid = fork(); - - if ( pid < 0 ) - { - Error(fmt("can't fork: %s", strerror(errno))); - return; - } - - if ( pid > 0 ) - { - // Parent - child_pid = pid; - - io = new ChunkedIOFd(pipe[0], "parent->child", child_pid); - if ( ! io->Init() ) - { - Error(fmt("can't init child io: %s", io->Error())); - exit(1); // FIXME: Better way to handle this? - } - - safe_close(pipe[1]); - - return; - } - else - { // child - SocketComm child; - - ChunkedIOFd* io = - new ChunkedIOFd(pipe[1], "child->parent", getppid()); - if ( ! io->Init() ) - { - Error(fmt("can't init parent io: %s", io->Error())); - exit(1); - } - - child.SetParentIO(io); - safe_close(pipe[0]); - - // Close file descriptors. - safe_close(0); - safe_close(1); - safe_close(2); - - // Be nice. - setpriority(PRIO_PROCESS, 0, 5); - - child.Run(); - reporter->InternalError("cannot be reached"); - } - } - -RemoteSerializer::PeerID RemoteSerializer::Connect(const IPAddr& ip, - const string& zone_id, uint16 port, const char* our_class, double retry, - bool use_ssl) - { - if ( ! using_communication ) - return true; - - if ( ! initialized ) - reporter->InternalError("remote serializer not initialized"); - - if ( ! child_pid ) - Fork(); - - Peer* p = AddPeer(ip, port); - p->orig = true; - - if ( our_class ) - p->our_class = our_class; - - const size_t BUFSIZE = 1024; - char* data = new char[BUFSIZE]; - snprintf(data, BUFSIZE, - "%" PRI_PTR_COMPAT_UINT",%s,%s,%" PRIu16",%" PRIu32",%d", p->id, - ip.AsString().c_str(), zone_id.c_str(), port, uint32(retry), - use_ssl); - - if ( ! SendToChild(MSG_CONNECT_TO, p, data) ) - { - RemovePeer(p); - return false; - } - - p->state = Peer::PENDING; - return p->id; - } - -bool RemoteSerializer::CloseConnection(PeerID id) - { - if ( ! using_communication ) - return true; - - Peer* peer = LookupPeer(id, true); - if ( ! peer ) - { - reporter->Error("unknown peer id %d for closing connection", int(id)); - return false; - } - - return CloseConnection(peer); - } - -bool RemoteSerializer::CloseConnection(Peer* peer) - { - if ( peer->suspended_processing ) - { - net_continue_processing(); - peer->suspended_processing = false; - } - - if ( peer->state == Peer::CLOSING ) - return true; - - FlushPrintBuffer(peer); - FlushLogBuffer(peer); - - Log(LogInfo, "closing connection", peer); - - peer->state = Peer::CLOSING; - return SendToChild(MSG_CLOSE, peer, 0); - } - -bool RemoteSerializer::RequestSync(PeerID id, bool auth) - { - if ( ! using_communication ) - return true; - - Peer* peer = LookupPeer(id, true); - if ( ! peer ) - { - reporter->Error("unknown peer id %d for request sync", int(id)); - return false; - } - - if ( peer->phase != Peer::HANDSHAKE ) - { - reporter->Error("can't request sync from peer; wrong phase %d", - peer->phase); - return false; - } - - if ( ! SendToChild(MSG_REQUEST_SYNC, peer, 1, auth ? 1 : 0) ) - return false; - - peer->sync_requested |= Peer::WE | (auth ? Peer::AUTH_WE : 0); - - return true; - } - -bool RemoteSerializer::RequestLogs(PeerID id) - { - if ( ! using_communication ) - return true; - - Peer* peer = LookupPeer(id, true); - if ( ! peer ) - { - reporter->Error("unknown peer id %d for request logs", int(id)); - return false; - } - - if ( peer->phase != Peer::HANDSHAKE ) - { - reporter->Error("can't request logs from peer; wrong phase %d", - peer->phase); - return false; - } - - if ( ! SendToChild(MSG_REQUEST_LOGS, peer, 0) ) - return false; - - return true; - } - -bool RemoteSerializer::RequestEvents(PeerID id, RE_Matcher* pattern) - { - if ( ! using_communication ) - return true; - - Peer* peer = LookupPeer(id, true); - if ( ! peer ) - { - reporter->Error("unknown peer id %d for request sync", int(id)); - return false; - } - - if ( peer->phase != Peer::HANDSHAKE ) - { - reporter->Error("can't request events from peer; wrong phase %d", - peer->phase); - return false; - } - - EventRegistry::string_list* handlers = event_registry->Match(pattern); - - // Concat the handlers' names. - int len = 0; - loop_over_list(*handlers, i) - len += strlen((*handlers)[i]) + 1; - - if ( ! len ) - { - Log(LogInfo, "warning: no events to request"); - delete handlers; - return true; - } - - char* data = new char[len]; - char* d = data; - loop_over_list(*handlers, j) - { - for ( const char* p = (*handlers)[j]; *p; *d++ = *p++ ) - ; - *d++ = '\0'; - } - - delete handlers; - - return SendToChild(MSG_REQUEST_EVENTS, peer, data, len); - } - -bool RemoteSerializer::SetAcceptState(PeerID id, bool accept) - { - Peer* p = LookupPeer(id, false); - if ( ! p ) - return true; - - p->accept_state = accept; - return true; - } - -bool RemoteSerializer::SetCompressionLevel(PeerID id, int level) - { - Peer* p = LookupPeer(id, false); - if ( ! p ) - return true; - - p->comp_level = level; - return true; - } - -bool RemoteSerializer::CompleteHandshake(PeerID id) - { - Peer* p = LookupPeer(id, false); - if ( ! p ) - return true; - - if ( p->phase != Peer::HANDSHAKE ) - { - reporter->Error("can't complete handshake; wrong phase %d", - p->phase); - return false; - } - - p->handshake_done |= Peer::WE; - - if ( ! SendToChild(MSG_PHASE_DONE, p, 0) ) - return false; - - if ( p->handshake_done == Peer::BOTH ) - HandshakeDone(p); - - return true; - } - -bool RemoteSerializer::SendCall(SerialInfo* info, PeerID id, - const char* name, val_list* vl) - { - if ( ! using_communication || terminating ) - return true; - - Peer* peer = LookupPeer(id, true); - if ( ! peer ) - return false; - - return SendCall(info, peer, name, vl); - } - -bool RemoteSerializer::SendCall(SerialInfo* info, Peer* peer, - const char* name, val_list* vl) - { - if ( peer->phase != Peer::RUNNING || terminating ) - return false; - - ++stats.events.out; - SetCache(peer->cache_out); - SetupSerialInfo(info, peer); - - if ( ! Serialize(info, name, vl) ) - { - FatalError(io->Error()); - return false; - } - - return true; - } - -bool RemoteSerializer::SendCall(SerialInfo* info, const char* name, - val_list* vl) - { - if ( ! IsOpen() || ! PropagateAccesses() || terminating ) - return true; - - loop_over_list(peers, i) - { - // Do not send event back to originating peer. - if ( peers[i] == current_peer ) - continue; - - SerialInfo new_info(*info); - if ( ! SendCall(&new_info, peers[i], name, vl) ) - return false; - } - - return true; - } - -bool RemoteSerializer::SendAccess(SerialInfo* info, Peer* peer, - const StateAccess& access) - { - if ( ! (peer->sync_requested & Peer::PEER) || terminating ) - return true; - -#ifdef DEBUG - ODesc desc; - access.Describe(&desc); - DBG_LOG(DBG_COMM, "Sending %s", desc.Description()); -#endif - - ++stats.accesses.out; - SetCache(peer->cache_out); - SetupSerialInfo(info, peer); - info->globals_as_names = true; - - if ( ! Serialize(info, access) ) - { - FatalError(io->Error()); - return false; - } - - return true; - } - -bool RemoteSerializer::SendAccess(SerialInfo* info, PeerID pid, - const StateAccess& access) - { - Peer* p = LookupPeer(pid, false); - if ( ! p ) - return true; - - return SendAccess(info, p, access); - } - -bool RemoteSerializer::SendAccess(SerialInfo* info, const StateAccess& access) - { - if ( ! IsOpen() || ! PropagateAccesses() || terminating ) - return true; - - // A real broadcast would be nice here. But the different peers have - // different serialization caches, so we cannot simply send the same - // serialization to all of them ... - loop_over_list(peers, i) - { - // Do not send access back to originating peer. - if ( peers[i] == source_peer ) - continue; - - // Only sent accesses for fully setup peers. - if ( peers[i]->phase != Peer::RUNNING ) - continue; - - SerialInfo new_info(*info); - if ( ! SendAccess(&new_info, peers[i], access) ) - return false; - } - - return true; - } - -bool RemoteSerializer::SendAllSynchronized(Peer* peer, SerialInfo* info) - { - // FIXME: When suspending ID serialization works, remove! - DisableSuspend suspend(info); - - current_peer = peer; - - Continuation* cont = &info->cont; - ptr_compat_int index; - - if ( info->cont.NewInstance() ) - { - Log(LogInfo, "starting to send full state", peer); - index = 0; - } - - else - { - index = int(ptr_compat_int(cont->RestoreState())); - if ( ! cont->ChildSuspended() ) - cont->Resume(); - } - - for ( ; index < sync_ids.length(); ++index ) - { - if ( ! sync_ids[index]->ID_Val() ) - { -#ifdef DEBUG - DBG_LOG(DBG_COMM, "Skip sync of ID with null value: %s\n", - sync_ids[index]->Name()); -#endif - continue; - } - cont->SaveContext(); - - StateAccess sa(OP_ASSIGN, sync_ids[index], - sync_ids[index]->ID_Val()); - // FIXME: When suspending ID serialization works, we need to - // addsupport to StateAccesses, too. - bool result = SendAccess(info, peer, sa); - cont->RestoreContext(); - - if ( ! result ) - return false; - - if ( cont->ChildSuspended() || info->may_suspend ) - { - double t = network_time + state_write_delay; - timer_mgr->Add(new IncrementalSendTimer(t, peer, info)); - - cont->SaveState((void*) index); - if ( info->may_suspend ) - cont->Suspend(); - - return true; - } - } - - if ( ! SendToChild(MSG_PHASE_DONE, peer, 0) ) - return false; - - suspend.Release(); - delete info; - - Log(LogInfo, "done sending full state", peer); - - return EnterPhaseRunning(peer); - } - -bool RemoteSerializer::SendID(SerialInfo* info, Peer* peer, const ID& id) - { - if ( terminating ) - return true; - - // FIXME: When suspending ID serialization works, remove! - DisableSuspend suspend(info); - - if ( info->cont.NewInstance() ) - ++stats.ids.out; - - SetCache(peer->cache_out); - SetupSerialInfo(info, peer); - info->cont.SaveContext(); - bool result = Serialize(info, id); - info->cont.RestoreContext(); - - if ( ! result ) - { - FatalError(io->Error()); - return false; - } - - return true; - } - -bool RemoteSerializer::SendID(SerialInfo* info, PeerID pid, const ID& id) - { - if ( ! using_communication || terminating ) - return true; - - Peer* peer = LookupPeer(pid, true); - if ( ! peer ) - return false; - - if ( peer->phase != Peer::RUNNING ) - return false; - - return SendID(info, peer, id); - } - -bool RemoteSerializer::SendConnection(SerialInfo* info, PeerID id, - const Connection& c) - { - if ( ! using_communication || terminating ) - return true; - - Peer* peer = LookupPeer(id, true); - if ( ! peer ) - return false; - - if ( peer->phase != Peer::RUNNING ) - return false; - - ++stats.conns.out; - SetCache(peer->cache_out); - SetupSerialInfo(info, peer); - - if ( ! Serialize(info, c) ) - { - FatalError(io->Error()); - return false; - } - - return true; - } - -bool RemoteSerializer::SendCaptureFilter(PeerID id, const char* filter) - { - if ( ! using_communication || terminating ) - return true; - - Peer* peer = LookupPeer(id, true); - if ( ! peer ) - return false; - - if ( peer->phase != Peer::HANDSHAKE ) - { - reporter->Error("can't sent capture filter to peer; wrong phase %d", peer->phase); - return false; - } - - return SendToChild(MSG_CAPTURE_FILTER, peer, copy_string(filter)); - } - -bool RemoteSerializer::SendPacket(SerialInfo* info, const Packet& p) - { - if ( ! IsOpen() || !PropagateAccesses() || terminating ) - return true; - - loop_over_list(peers, i) - { - // Only sent packet for fully setup peers. - if ( peers[i]->phase != Peer::RUNNING ) - continue; - - SerialInfo new_info(*info); - if ( ! SendPacket(&new_info, peers[i], p) ) - return false; - } - - return true; - } - -bool RemoteSerializer::SendPacket(SerialInfo* info, PeerID id, const Packet& p) - { - if ( ! using_communication || terminating ) - return true; - - Peer* peer = LookupPeer(id, true); - if ( ! peer ) - return false; - - return SendPacket(info, peer, p); - } - -bool RemoteSerializer::SendPacket(SerialInfo* info, Peer* peer, const Packet& p) - { - ++stats.packets.out; - SetCache(peer->cache_out); - SetupSerialInfo(info, peer); - - if ( ! Serialize(info, p) ) - { - FatalError(io->Error()); - return false; - } - - return true; - } - -bool RemoteSerializer::SendPing(PeerID id, uint32 seq) - { - if ( ! using_communication || terminating ) - return true; - - Peer* peer = LookupPeer(id, true); - if ( ! peer ) - return false; - - char* data = new char[sizeof(ping_args)]; - - ping_args* args = (ping_args*) data; - args->seq = htonl(seq); - args->time1 = htond(current_time(true)); - args->time2 = 0; - args->time3 = 0; - - return SendToChild(MSG_PING, peer, data, sizeof(ping_args)); - } - -bool RemoteSerializer::SendCapabilities(Peer* peer) - { - if ( peer->phase != Peer::HANDSHAKE ) - { - reporter->Error("can't sent capabilties to peer; wrong phase %d", - peer->phase); - return false; - } - - uint32 caps = 0; - - caps |= Peer::COMPRESSION; - caps |= Peer::PID_64BIT; - caps |= Peer::NEW_CACHE_STRATEGY; - - return SendToChild(MSG_CAPS, peer, 3, caps, 0, 0); - } - -bool RemoteSerializer::Listen(const IPAddr& ip, uint16 port, bool expect_ssl, - bool ipv6, const string& zone_id, double retry) - { - if ( ! using_communication ) - return true; - - if ( ! initialized ) - reporter->InternalError("remote serializer not initialized"); - - if ( ! ipv6 && ip.GetFamily() == IPv6 && - ip != IPAddr("0.0.0.0") && ip != IPAddr("::") ) - reporter->FatalError("Attempt to listen on address %s, but IPv6 " - "communication disabled", ip.AsString().c_str()); - - const size_t BUFSIZE = 1024; - char* data = new char[BUFSIZE]; - snprintf(data, BUFSIZE, "%s,%" PRIu16",%d,%d,%s,%" PRIu32, - ip.AsString().c_str(), port, expect_ssl, ipv6, zone_id.c_str(), - (uint32) retry); - - if ( ! SendToChild(MSG_LISTEN, 0, data) ) - return false; - - listening = true; - SetClosed(false); - return true; - } - -void RemoteSerializer::SendSyncPoint(uint32 point) - { - if ( ! (remote_trace_sync_interval && pseudo_realtime) || terminating ) - return; - - current_sync_point = point; - - loop_over_list(peers, i) - if ( peers[i]->phase == Peer::RUNNING && - ! SendToChild(MSG_SYNC_POINT, peers[i], - 1, current_sync_point) ) - return; - - if ( ! syncing_times ) - { - Log(LogInfo, "waiting for peers"); - syncing_times = true; - - loop_over_list(peers, i) - { - // Need to do this once per peer to correctly - // track the number of suspend calls. - net_suspend_processing(); - peers[i]->suspended_processing = true; - } - } - - CheckSyncPoints(); - } - -uint32 RemoteSerializer::SendSyncPoint() - { - Log(LogInfo, fmt("reached sync-point %u", current_sync_point)); - SendSyncPoint(current_sync_point + 1); - return current_sync_point; - } - -void RemoteSerializer::SendFinalSyncPoint() - { - Log(LogInfo, fmt("reached end of trace, sending final sync point")); - SendSyncPoint(FINAL_SYNC_POINT); - } - -bool RemoteSerializer::Terminate() - { - loop_over_list(peers, i) - { - FlushPrintBuffer(peers[i]); - FlushLogBuffer(peers[i]); - } - - Log(LogInfo, fmt("terminating...")); - - return terminating = SendToChild(MSG_TERMINATE, 0, 0); - } - -bool RemoteSerializer::StopListening() - { - if ( ! listening ) - return true; - - if ( ! SendToChild(MSG_LISTEN_STOP, 0, 0) ) - return false; - - listening = false; - SetClosed(! IsActive()); - return true; - } - -void RemoteSerializer::Register(ID* id) - { - DBG_LOG(DBG_STATE, "&synchronized %s", id->Name()); - Unregister(id); - Ref(id); - sync_ids.append(id); - } - -void RemoteSerializer::Unregister(ID* id) - { - loop_over_list(sync_ids, i) - if ( streq(sync_ids[i]->Name(), id->Name()) ) - { - Unref(sync_ids[i]); - sync_ids.remove_nth(i); - break; - } - } - -void RemoteSerializer::GetFds(iosource::FD_Set* read, iosource::FD_Set* write, - iosource::FD_Set* except) - { - read->Insert(io->Fd()); - read->Insert(io->ExtraReadFDs()); - - if ( io->CanWrite() ) - write->Insert(io->Fd()); - } - -double RemoteSerializer::NextTimestamp(double* local_network_time) - { - Poll(false); - - if ( received_logs > 0 ) - { - // If we processed logs last time, assume there's more. - SetIdle(false); - received_logs = 0; - return timer_mgr->Time(); - } - - double et = events.length() ? events[0]->time : -1; - double pt = packets.length() ? packets[0]->time : -1; - - if ( ! et ) - et = timer_mgr->Time(); - - if ( ! pt ) - pt = timer_mgr->Time(); - - if ( packets.length() ) - SetIdle(false); - - if ( et >= 0 && (et < pt || pt < 0) ) - return et; - - if ( pt >= 0 ) - { - // Return packet time as network time. - *local_network_time = packets[0]->p->time; - return pt; - } - - return -1; - } - -TimerMgr::Tag* RemoteSerializer::GetCurrentTag() - { - return packets.length() ? &packets[0]->p->tag : 0; - } - -void RemoteSerializer::Process() - { - Poll(false); - - int i = 0; - while ( events.length() ) - { - if ( max_remote_events_processed && - ++i > max_remote_events_processed ) - break; - - BufferedEvent* be = events[0]; - ::Event* event = new ::Event(be->handler, std::move(*be->args), be->src); - delete be->args; - be->args = nullptr; - - Peer* old_current_peer = current_peer; - // Prevent the source peer from getting the event back. - current_peer = LookupPeer(be->src, true); // may be null. - mgr.Dispatch(event, ! forward_remote_events); - current_peer = old_current_peer; - - assert(events[0] == be); - delete be; - events.remove_nth(0); - } - - // We shouldn't pass along more than one packet, as otherwise the - // timer mgr will not advance. - if ( packets.length() ) - { - BufferedPacket* bp = packets[0]; - const Packet* p = bp->p; - - // FIXME: The following chunk of code is copied from - // net_packet_dispatch(). We should change that function - // to accept an IOSource instead of the PktSrc. - net_update_time(p->time); - - SegmentProfiler(segment_logger, "expiring-timers"); - TimerMgr* tmgr = sessions->LookupTimerMgr(GetCurrentTag()); - current_dispatched = - tmgr->Advance(network_time, max_timer_expires); - - current_pkt = p; - current_pktsrc = 0; - current_iosrc = this; - sessions->NextPacket(p->time, p); - mgr.Drain(); - - current_pkt = 0; - current_iosrc = 0; - - delete p; - delete bp; - packets.remove_nth(0); - } - - if ( packets.length() ) - SetIdle(false); - } - -void RemoteSerializer::Finish() - { - if ( ! using_communication ) - return; - - do - Poll(true); - while ( io->CanWrite() ); - - loop_over_list(peers, i) - { - CloseConnection(peers[i]); - } - } - -bool RemoteSerializer::Poll(bool may_block) - { - if ( ! child_pid ) - return true; - - // See if there's any peer waiting for initial state synchronization. - if ( sync_pending.length() && ! in_sync ) - { - Peer* p = sync_pending[0]; - sync_pending.remove_nth(0); - HandshakeDone(p); - } - - io->Flush(); - SetIdle(false); - - switch ( msgstate ) { - case TYPE: - { - current_peer = 0; - current_msgtype = MSG_NONE; - - // CMsg follows - ChunkedIO::Chunk* c; - READ_CHUNK_FROM_CHILD(c); - - CMsg* msg = (CMsg*) c->data; - current_peer = LookupPeer(msg->Peer(), false); - current_id = msg->Peer(); - current_msgtype = msg->Type(); - current_args = 0; - - delete c; - - switch ( current_msgtype ) { - case MSG_CLOSE: - case MSG_CLOSE_ALL: - case MSG_LISTEN_STOP: - case MSG_PHASE_DONE: - case MSG_TERMINATE: - case MSG_DEBUG_DUMP: - case MSG_REQUEST_LOGS: - { - // No further argument chunk. - msgstate = TYPE; - return DoMessage(); - } - case MSG_VERSION: - case MSG_SERIAL: - case MSG_ERROR: - case MSG_CONNECT_TO: - case MSG_CONNECTED: - case MSG_REQUEST_EVENTS: - case MSG_REQUEST_SYNC: - case MSG_LISTEN: - case MSG_STATS: - case MSG_CAPTURE_FILTER: - case MSG_PING: - case MSG_PONG: - case MSG_CAPS: - case MSG_COMPRESS: - case MSG_LOG: - case MSG_SYNC_POINT: - case MSG_REMOTE_PRINT: - case MSG_LOG_CREATE_WRITER: - case MSG_LOG_WRITE: - { - // One further argument chunk. - msgstate = ARGS; - return Poll(may_block); - } - - case MSG_NONE: - InternalCommError(fmt("unexpected msg type %d", - current_msgtype)); - return true; - - default: - InternalCommError(fmt("unknown msg type %d in Poll()", - current_msgtype)); - return true; - } - } - - case ARGS: - { - // Argument chunk follows. - ChunkedIO::Chunk* c; - READ_CHUNK_FROM_CHILD(c); - - current_args = c; - msgstate = TYPE; - bool result = DoMessage(); - - delete current_args; - current_args = 0; - - return result; - } - - default: - reporter->InternalError("unknown msgstate"); - } - - reporter->InternalError("cannot be reached"); - return false; - } - -bool RemoteSerializer::DoMessage() - { - if ( current_peer && - (current_peer->state == Peer::CLOSING || - current_peer->state == Peer::CLOSED) && - is_peer_msg(current_msgtype) ) - { - // We shut the connection to this peer down, - // so we ignore all further messages. - DEBUG_COMM(fmt("parent: ignoring %s due to shutdown of peer #%" PRI_SOURCE_ID, - msgToStr(current_msgtype), - current_peer ? current_peer->id : 0)); - return true; - } - - DEBUG_COMM(fmt("parent: %s from child; peer is #%" PRI_SOURCE_ID, - msgToStr(current_msgtype), - current_peer ? current_peer->id : 0)); - - if ( current_peer && - (current_msgtype < 0 || current_msgtype > MSG_ID_MAX) ) - { - Log(LogError, "garbage message from peer, shutting down", - current_peer); - CloseConnection(current_peer); - return true; - } - - // As long as we haven't finished the version - // handshake, no other messages than MSG_VERSION - // are allowed from peer. - if ( current_peer && current_peer->phase == Peer::SETUP && - is_peer_msg(current_msgtype) && current_msgtype != MSG_VERSION ) - { - Log(LogError, "peer did not send version", current_peer); - CloseConnection(current_peer); - return true; - } - - switch ( current_msgtype ) { - case MSG_CLOSE: - PeerDisconnected(current_peer); - return true; - - case MSG_CONNECTED: - return ProcessConnected(); - - case MSG_SERIAL: - return ProcessSerialization(); - - case MSG_REQUEST_EVENTS: - return ProcessRequestEventsMsg(); - - case MSG_REQUEST_SYNC: - return ProcessRequestSyncMsg(); - - case MSG_PHASE_DONE: - return ProcessPhaseDone(); - - case MSG_ERROR: - return ProcessLogMsg(true); - - case MSG_LOG: - return ProcessLogMsg(false); - - case MSG_STATS: - return ProcessStatsMsg(); - - case MSG_CAPTURE_FILTER: - return ProcessCaptureFilterMsg(); - - case MSG_VERSION: - return ProcessVersionMsg(); - - case MSG_PING: - return ProcessPingMsg(); - - case MSG_PONG: - return ProcessPongMsg(); - - case MSG_CAPS: - return ProcessCapsMsg(); - - case MSG_SYNC_POINT: - return ProcessSyncPointMsg(); - - case MSG_TERMINATE: - assert(terminating); - iosource_mgr->Terminate(); - return true; - - case MSG_REMOTE_PRINT: - return ProcessRemotePrint(); - - case MSG_LOG_CREATE_WRITER: - return ProcessLogCreateWriter(); - - case MSG_LOG_WRITE: - return ProcessLogWrite(); - - case MSG_REQUEST_LOGS: - return ProcessRequestLogs(); - - default: - DEBUG_COMM(fmt("unexpected msg type: %d", - int(current_msgtype))); - InternalCommError(fmt("unexpected msg type in DoMessage(): %d", - int(current_msgtype))); - return true; // keep going - } - - reporter->InternalError("cannot be reached"); - return false; - } - -void RemoteSerializer::PeerDisconnected(Peer* peer) - { - assert(peer); - - if ( peer->suspended_processing ) - { - net_continue_processing(); - peer->suspended_processing = false; - } - - if ( peer->state == Peer::CLOSED || peer->state == Peer::INIT ) - return; - - if ( peer->state == Peer::PENDING ) - { - peer->state = Peer::CLOSED; - Log(LogError, "could not connect", peer); - return; - } - - Log(LogInfo, "peer disconnected", peer); - - if ( peer->phase != Peer::SETUP ) - RaiseEvent(remote_connection_closed, peer); - - if ( in_sync == peer ) - in_sync = 0; - - peer->state = Peer::CLOSED; - peer->phase = Peer::UNKNOWN; - peer->cache_in->Clear(); - peer->cache_out->Clear(); - UnregisterHandlers(peer); - } - -void RemoteSerializer::PeerConnected(Peer* peer) - { - if ( peer->state == Peer::CONNECTED ) - return; - - peer->state = Peer::CONNECTED; - peer->phase = Peer::SETUP; - peer->sent_version = Peer::NONE; - peer->sync_requested = Peer::NONE; - peer->handshake_done = Peer::NONE; - - peer->cache_in->Clear(); - peer->cache_out->Clear(); - peer->our_runtime = int(current_time(true) - bro_start_time); - peer->sync_point = 0; - peer->logs_requested = false; - - if ( ! SendCMsgToChild(MSG_VERSION, peer) ) - return; - - int len = 4 * sizeof(uint32) + peer->our_class.size() + 1; - char* data = new char[len]; - uint32* args = (uint32*) data; - - *args++ = htonl(PROTOCOL_VERSION); - *args++ = htonl(peer->cache_out->GetMaxCacheSize()); - *args++ = htonl(DATA_FORMAT_VERSION); - *args++ = htonl(peer->our_runtime); - strcpy((char*) args, peer->our_class.c_str()); - - ChunkedIO::Chunk* c = new ChunkedIO::Chunk(data, len); - - if ( peer->our_class.size() ) - Log(LogInfo, fmt("sending class \"%s\"", peer->our_class.c_str()), peer); - - if ( ! SendToChild(c) ) - { - Log(LogError, "can't send version message"); - CloseConnection(peer); - return; - } - - peer->sent_version |= Peer::WE; - Log(LogInfo, "peer connected", peer); - Log(LogInfo, "phase: version", peer); - } - -RecordVal* RemoteSerializer::MakePeerVal(Peer* peer) - { - RecordVal* v = new RecordVal(::peer); - v->Assign(0, val_mgr->GetCount(uint32(peer->id))); - // Sic! Network order for AddrVal, host order for PortVal. - v->Assign(1, new AddrVal(peer->ip)); - v->Assign(2, val_mgr->GetPort(peer->port, TRANSPORT_TCP)); - v->Assign(3, val_mgr->GetFalse()); - v->Assign(4, val_mgr->GetEmptyString()); // set when received - v->Assign(5, peer->peer_class.size() ? - new StringVal(peer->peer_class.c_str()) : 0); - return v; - } - -RemoteSerializer::Peer* RemoteSerializer::AddPeer(const IPAddr& ip, uint16 port, - PeerID id) - { - Peer* peer = new Peer; - peer->id = id != PEER_NONE ? id : id_counter++; - peer->ip = ip; - peer->port = port; - peer->state = Peer::INIT; - peer->phase = Peer::UNKNOWN; - peer->sent_version = Peer::NONE; - peer->sync_requested = Peer::NONE; - peer->handshake_done = Peer::NONE; - peer->orig = false; - peer->accept_state = false; - peer->send_state = false; - peer->logs_requested = false; - peer->caps = 0; - peer->comp_level = 0; - peer->suspended_processing = false; - peer->caps = 0; - peer->val = MakePeerVal(peer); - peer->cache_in = new SerializationCache(MAX_CACHE_SIZE); - peer->cache_out = new SerializationCache(MAX_CACHE_SIZE); - peer->sync_point = 0; - peer->print_buffer = 0; - peer->print_buffer_used = 0; - peer->log_buffer = new char[LOG_BUFFER_SIZE]; - peer->log_buffer_used = 0; - - peers.append(peer); - Log(LogInfo, "added peer", peer); - - return peer; - } - -void RemoteSerializer::UnregisterHandlers(Peer* peer) - { - // Unregister the peers for the EventHandlers. - loop_over_list(peer->handlers, i) - { - peer->handlers[i]->RemoveRemoteHandler(peer->id); - } - } - -void RemoteSerializer::RemovePeer(Peer* peer) - { - if ( peer->suspended_processing ) - { - net_continue_processing(); - peer->suspended_processing = false; - } - - peers.remove(peer); - UnregisterHandlers(peer); - - Log(LogInfo, "removed peer", peer); - - int id = peer->id; - Unref(peer->val); - delete [] peer->print_buffer; - delete [] peer->log_buffer; - delete peer->cache_in; - delete peer->cache_out; - delete peer; - - SetClosed(! IsActive()); - - if ( in_sync == peer ) - in_sync = 0; - } - -RemoteSerializer::Peer* RemoteSerializer::LookupPeer(PeerID id, - bool only_if_connected) - { - Peer* peer = 0; - loop_over_list(peers, i) - if ( peers[i]->id == id ) - { - peer = peers[i]; - break; - } - - if ( ! only_if_connected || (peer && peer->state == Peer::CONNECTED) ) - return peer; - else - return 0; - } - -bool RemoteSerializer::ProcessVersionMsg() - { - uint32* args = (uint32*) current_args->data; - uint32 version = ntohl(args[0]); - uint32 data_version = ntohl(args[2]); - - if ( PROTOCOL_VERSION != version ) - { - Log(LogError, fmt("remote protocol version mismatch: got %d, but expected %d", - version, PROTOCOL_VERSION), current_peer); - CloseConnection(current_peer); - return true; - } - - // For backwards compatibility, data_version may be null. - if ( data_version && DATA_FORMAT_VERSION != data_version ) - { - Log(LogError, fmt("remote data version mismatch: got %d, but expected %d", - data_version, DATA_FORMAT_VERSION), - current_peer); - CloseConnection(current_peer); - return true; - } - - uint32 cache_size = ntohl(args[1]); - current_peer->cache_in->SetMaxCacheSize(cache_size); - current_peer->runtime = ntohl(args[3]); - - current_peer->sent_version |= Peer::PEER; - - if ( current_args->len > 4 * sizeof(uint32) ) - { - // The peer sends us a class string. - const char* pclass = (const char*) &args[4]; - current_peer->peer_class = pclass; - if ( *pclass ) - Log(LogInfo, fmt("peer sent class \"%s\"", pclass), current_peer); - if ( current_peer->val ) - current_peer->val->Assign(5, new StringVal(pclass)); - } - - assert(current_peer->sent_version == Peer::BOTH); - current_peer->phase = Peer::HANDSHAKE; - Log(LogInfo, "phase: handshake", current_peer); - - if ( ! SendCapabilities(current_peer) ) - return false; - - RaiseEvent(remote_connection_established, current_peer); - - return true; - } - -bool RemoteSerializer::EnterPhaseRunning(Peer* peer) - { - if ( in_sync == peer ) - in_sync = 0; - - peer->phase = Peer::RUNNING; - Log(LogInfo, "phase: running", peer); - RaiseEvent(remote_connection_handshake_done, peer); - - if ( remote_trace_sync_interval ) - { - loop_over_list(peers, i) - { - if ( ! SendToChild(MSG_SYNC_POINT, peers[i], - 1, current_sync_point) ) - return false; - } - } - - return true; - } - -bool RemoteSerializer::ProcessConnected() - { - // IP and port follow. - vector args = tokenize(current_args->data, ','); - - if ( args.size() != 2 ) - { - InternalCommError("ProcessConnected() bad number of arguments"); - return false; - } - - IPAddr host = IPAddr(args[0]); - uint16 port; - - if ( ! atoi_n(args[1].size(), args[1].c_str(), 0, 10, port) ) - { - InternalCommError("ProcessConnected() bad peer port string"); - return false; - } - - if ( ! current_peer ) - { - // The other side connected to one of our listening ports. - current_peer = AddPeer(host, port, current_id); - current_peer->orig = false; - } - else if ( current_peer->orig ) - { - // It's a successful retry. - current_peer->port = port; - current_peer->accept_state = false; - Unref(current_peer->val); - current_peer->val = MakePeerVal(current_peer); - } - - PeerConnected(current_peer); - - ID* descr = global_scope()->Lookup("peer_description"); - if ( ! descr ) - reporter->InternalError("peer_description not defined"); - - SerialInfo info(this); - SendID(&info, current_peer, *descr); - - return true; - } - -bool RemoteSerializer::ProcessRequestEventsMsg() - { - if ( ! current_peer ) - return false; - - // Register new handlers. - char* p = current_args->data; - while ( p < current_args->data + current_args->len ) - { - EventHandler* handler = event_registry->Lookup(p); - if ( handler ) - { - handler->AddRemoteHandler(current_peer->id); - current_peer->handlers.append(handler); - RaiseEvent(remote_event_registered, current_peer, p); - Log(LogInfo, fmt("registered for event %s", p), - current_peer); - - // If the other side requested the print_hook event, - // we initialize the buffer. - if ( current_peer->print_buffer == 0 && - streq(p, "print_hook") ) - { - current_peer->print_buffer = - new char[PRINT_BUFFER_SIZE]; - current_peer->print_buffer_used = 0; - Log(LogInfo, "initialized print buffer", - current_peer); - } - } - else - Log(LogInfo, fmt("request for unknown event %s", p), - current_peer); - - p += strlen(p) + 1; - } - - return true; - } - -bool RemoteSerializer::ProcessRequestSyncMsg() - { - if ( ! current_peer ) - return false; - - int auth = 0; - uint32* args = (uint32*) current_args->data; - if ( ntohl(args[0]) != 0 ) - { - Log(LogInfo, "peer considers its state authoritative", current_peer); - auth = Peer::AUTH_PEER; - } - - current_peer->sync_requested |= Peer::PEER | auth; - return true; - } - -bool RemoteSerializer::ProcessRequestLogs() - { - if ( ! current_peer ) - return false; - - Log(LogInfo, "peer requested logs", current_peer); - - current_peer->logs_requested = true; - return true; - } - -bool RemoteSerializer::ProcessPhaseDone() - { - switch ( current_peer->phase ) { - case Peer::HANDSHAKE: - { - current_peer->handshake_done |= Peer::PEER; - - if ( current_peer->handshake_done == Peer::BOTH ) - HandshakeDone(current_peer); - break; - } - - case Peer::SYNC: - { - // Make sure that the other side is supposed to sent us this. - if ( current_peer->send_state ) - { - Log(LogError, "unexpected phase_done in sync phase from peer", current_peer); - CloseConnection(current_peer); - return false; - } - - if ( ! EnterPhaseRunning(current_peer) ) - { - if ( current_peer->suspended_processing ) - { - net_continue_processing(); - current_peer->suspended_processing = false; - } - - return false; - } - - if ( current_peer->suspended_processing ) - { - net_continue_processing(); - current_peer->suspended_processing = false; - } - - break; - } - - default: - Log(LogError, "unexpected phase_done", current_peer); - CloseConnection(current_peer); - } - - return true; - } - -bool RemoteSerializer::HandshakeDone(Peer* peer) - { - if ( peer->caps & Peer::COMPRESSION && peer->comp_level > 0 ) - if ( ! SendToChild(MSG_COMPRESS, peer, 1, peer->comp_level) ) - return false; - - if ( ! (peer->caps & Peer::PID_64BIT) ) - Log(LogInfo, "peer does not support 64bit PIDs; using compatibility mode", peer); - - if ( (peer->caps & Peer::NEW_CACHE_STRATEGY) ) - Log(LogInfo, "peer supports keep-in-cache; using that", peer); - - if ( (peer->caps & Peer::BROCCOLI_PEER) ) - Log(LogInfo, "peer is a Broccoli", peer); - - if ( peer->logs_requested ) - log_mgr->SendAllWritersTo(peer->id); - - if ( peer->sync_requested != Peer::NONE ) - { - if ( in_sync ) - { - Log(LogInfo, "another sync in progress, waiting...", - peer); - sync_pending.append(peer); - return true; - } - - if ( (peer->sync_requested & Peer::AUTH_PEER) && - (peer->sync_requested & Peer::AUTH_WE) ) - { - Log(LogError, "misconfiguration: authoritative state on both sides", - current_peer); - CloseConnection(peer); - return false; - } - - in_sync = peer; - peer->phase = Peer::SYNC; - - // If only one side has requested state synchronization, - // it will get all the state from the peer. - // - // If both sides have shown interest, the one considering - // itself authoritative will send the state. If none is - // authoritative, the peer which is running longest sends - // its state. - // - if ( (peer->sync_requested & Peer::BOTH) != Peer::BOTH ) - { - // One side. - if ( peer->sync_requested & Peer::PEER ) - peer->send_state = true; - else if ( peer->sync_requested & Peer::WE ) - peer->send_state = false; - else - reporter->InternalError("illegal sync_requested value"); - } - else - { - // Both. - if ( peer->sync_requested & Peer::AUTH_WE ) - peer->send_state = true; - else if ( peer->sync_requested & Peer::AUTH_PEER ) - peer->send_state = false; - else - { - if ( peer->our_runtime == peer->runtime ) - peer->send_state = peer->orig; - else - peer->send_state = (peer->our_runtime > - peer->runtime); - } - } - - Log(LogInfo, fmt("phase: sync (%s)", (peer->send_state ? "sender" : "receiver")), peer); - - if ( peer->send_state ) - { - SerialInfo* info = new SerialInfo(this); - SendAllSynchronized(peer, info); - } - - else - { - // Suspend until we got everything. - net_suspend_processing(); - peer->suspended_processing = true; - } - } - else - return EnterPhaseRunning(peer); - - return true; - } - -bool RemoteSerializer::ProcessPingMsg() - { - if ( ! current_peer ) - return false; - - if ( ! SendToChild(MSG_PONG, current_peer, - current_args->data, current_args->len) ) - return false; - - return true; - } - -bool RemoteSerializer::ProcessPongMsg() - { - if ( ! current_peer ) - return false; - - ping_args* args = (ping_args*) current_args->data; - - mgr.QueueEvent(remote_pong, { - current_peer->val->Ref(), - val_mgr->GetCount((unsigned int) ntohl(args->seq)), - new Val(current_time(true) - ntohd(args->time1), - TYPE_INTERVAL), - new Val(ntohd(args->time2), TYPE_INTERVAL), - new Val(ntohd(args->time3), TYPE_INTERVAL) - }); - return true; - } - -bool RemoteSerializer::ProcessCapsMsg() - { - if ( ! current_peer ) - return false; - - uint32* args = (uint32*) current_args->data; - current_peer->caps = ntohl(args[0]); - return true; - } - -bool RemoteSerializer::ProcessLogMsg(bool is_error) - { - Log(is_error ? LogError : LogInfo, current_args->data, 0, LogChild); - return true; - } - -bool RemoteSerializer::ProcessStatsMsg() - { - // Take the opportunity to log our stats, too. - LogStats(); - - // Split the concatenated child stats into indiviual log messages. - int count = 0; - for ( char* p = current_args->data; - p < current_args->data + current_args->len; p += strlen(p) + 1 ) - Log(LogInfo, fmt("child statistics: [%d] %s", count++, p), - current_peer); - - return true; - } - -bool RemoteSerializer::ProcessCaptureFilterMsg() - { - if ( ! current_peer ) - return false; - - RaiseEvent(remote_capture_filter, current_peer, current_args->data); - return true; - } - -bool RemoteSerializer::CheckSyncPoints() - { - if ( ! current_sync_point ) - return false; - - int ready = 0; - - loop_over_list(peers, i) - if ( peers[i]->sync_point >= current_sync_point ) - ready++; - - if ( ready < remote_trace_sync_peers ) - return false; - - if ( current_sync_point == FINAL_SYNC_POINT ) - { - Log(LogInfo, fmt("all peers reached final sync-point, going to finish")); - Terminate(); - } - else - Log(LogInfo, fmt("all peers reached sync-point %u", - current_sync_point)); - - if ( syncing_times ) - { - loop_over_list(peers, i) - { - if ( peers[i]->suspended_processing ) - { - net_continue_processing(); - peers[i]->suspended_processing = false; - } - } - - syncing_times = false; - } - - return true; - } - -bool RemoteSerializer::ProcessSyncPointMsg() - { - if ( ! current_peer ) - return false; - - uint32* args = (uint32*) current_args->data; - uint32 count = ntohl(args[0]); - - current_peer->sync_point = max(current_peer->sync_point, count); - - if ( current_peer->sync_point == FINAL_SYNC_POINT ) - Log(LogInfo, fmt("reached final sync-point"), current_peer); - else - Log(LogInfo, fmt("reached sync-point %u", current_peer->sync_point), current_peer); - - if ( syncing_times ) - CheckSyncPoints(); - - return true; - } - -bool RemoteSerializer::ProcessSerialization() - { - if ( current_peer->state == Peer::CLOSING ) - return false; - - SetCache(current_peer->cache_in); - UnserialInfo info(this); - - bool accept_state = current_peer->accept_state; - -#if 0 - // If processing is suspended, we unserialize the data but throw - // it away. - if ( current_peer->phase == Peer::RUNNING && - net_is_processing_suspended() ) - accept_state = false; -#endif - - assert(current_args); - info.chunk = current_args; - - info.install_globals = accept_state; - info.install_conns = accept_state; - info.ignore_callbacks = ! accept_state; - - if ( current_peer->phase != Peer::RUNNING ) - info.id_policy = UnserialInfo::InstantiateNew; - else - info.id_policy = accept_state ? - UnserialInfo::CopyNewToCurrent : - UnserialInfo::Keep; - - if ( ! (current_peer->caps & Peer::PID_64BIT) || - current_peer->phase != Peer::RUNNING ) - info.pid_32bit = true; - - if ( (current_peer->caps & Peer::NEW_CACHE_STRATEGY) && - current_peer->phase == Peer::RUNNING ) - info.new_cache_strategy = true; - - if ( current_peer->caps & Peer::BROCCOLI_PEER ) - info.broccoli_peer = true; - - if ( ! forward_remote_state_changes ) - ignore_accesses = true; - - source_peer = current_peer; - int i = Unserialize(&info); - source_peer = 0; - - if ( ! forward_remote_state_changes ) - ignore_accesses = false; - - if ( i < 0 ) - { - Log(LogError, "unserialization error", current_peer); - CloseConnection(current_peer); - // Error - return false; - } - - return true; - } - -bool RemoteSerializer::FlushPrintBuffer(Peer* p) - { - if ( p->state == Peer::CLOSING ) - return false; - - if ( ! (p->print_buffer && p->print_buffer_used) ) - return true; - - SendToChild(MSG_REMOTE_PRINT, p, p->print_buffer, p->print_buffer_used); - - p->print_buffer = new char[PRINT_BUFFER_SIZE]; - p->print_buffer_used = 0; - return true; - } - -bool RemoteSerializer::SendPrintHookEvent(BroFile* f, const char* txt, size_t len) - { - loop_over_list(peers, i) - { - Peer* p = peers[i]; - - if ( ! p->print_buffer ) - continue; - - const char* fname = f->Name(); - if ( ! fname ) - continue; // not a managed file. - - // We cut off everything after the max buffer size. That - // makes the code a bit easier, and we shouldn't have such - // long lines anyway. - len = min(len, PRINT_BUFFER_SIZE - strlen(fname) - 2); - - // If there's not enough space in the buffer, flush it. - - int need = strlen(fname) + 1 + len + 1; - if ( p->print_buffer_used + need > PRINT_BUFFER_SIZE ) - { - if ( ! FlushPrintBuffer(p) ) - return false; - } - - assert(p->print_buffer_used + need <= PRINT_BUFFER_SIZE); - - char* dst = p->print_buffer + p->print_buffer_used; - strcpy(dst, fname); - dst += strlen(fname) + 1; - memcpy(dst, txt, len); - dst += len; - *dst++ = '\0'; - - p->print_buffer_used = dst - p->print_buffer; - } - - return true; - } - -bool RemoteSerializer::ProcessRemotePrint() - { - if ( current_peer->state == Peer::CLOSING ) - return false; - - const char* p = current_args->data; - while ( p < current_args->data + current_args->len ) - { - const char* fname = p; - p += strlen(p) + 1; - const char* txt = p; - p += strlen(p) + 1; - - val_list* vl = new val_list(2); - BroFile* f = BroFile::GetFile(fname); - Ref(f); - vl->append(new Val(f)); - vl->append(new StringVal(txt)); - GotEvent("print_hook", -1.0, print_hook, vl); - } - - return true; - } - -bool RemoteSerializer::SendLogCreateWriter(EnumVal* id, EnumVal* writer, const logging::WriterBackend::WriterInfo& info, int num_fields, const threading::Field* const * fields) - { - loop_over_list(peers, i) - { - SendLogCreateWriter(peers[i]->id, id, writer, info, num_fields, fields); - } - - return true; - } - -bool RemoteSerializer::SendLogCreateWriter(PeerID peer_id, EnumVal* id, EnumVal* writer, const logging::WriterBackend::WriterInfo& info, int num_fields, const threading::Field* const * fields) - { - SetErrorDescr("logging"); - - ChunkedIO::Chunk* c = 0; - - Peer* peer = LookupPeer(peer_id, true); - if ( ! peer ) - return false; - - if ( peer->phase != Peer::HANDSHAKE && peer->phase != Peer::RUNNING ) - return false; - - if ( ! peer->logs_requested ) - return false; - - BinarySerializationFormat fmt; - - fmt.StartWrite(); - - bool success = fmt.Write(id->AsEnum(), "id") && - fmt.Write(writer->AsEnum(), "writer") && - fmt.Write(num_fields, "num_fields") && - info.Write(&fmt); - - if ( ! success ) - goto error; - - for ( int i = 0; i < num_fields; i++ ) - { - if ( ! fields[i]->Write(&fmt) ) - goto error; - } - - if ( ! SendToChild(MSG_LOG_CREATE_WRITER, peer, 0) ) - goto error; - - c = new ChunkedIO::Chunk; - c->len = fmt.EndWrite(&c->data); - c->free_func = ChunkedIO::Chunk::free_func_free; - - if ( ! SendToChild(c) ) - goto error; - - return true; - -error: - delete c; - - FatalError(io->Error()); - return false; - } - -bool RemoteSerializer::SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Value* const * vals) - { - loop_over_list(peers, i) - { - SendLogWrite(peers[i], id, writer, path, num_fields, vals); - } - - return true; - } - -bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Value* const * vals) - { - if ( peer->phase != Peer::HANDSHAKE && peer->phase != Peer::RUNNING ) - return false; - - if ( ! peer->logs_requested ) - return false; - - if ( ! peer->log_buffer ) - // Peer shutting down. - return false; - - // Serialize the log record entry. - - BinarySerializationFormat fmt; - - fmt.StartWrite(); - - bool success = fmt.Write(id->AsEnum(), "id") && - fmt.Write(writer->AsEnum(), "writer") && - fmt.Write(path, "path") && - fmt.Write(num_fields, "num_fields"); - - if ( ! success ) - goto error; - - for ( int i = 0; i < num_fields; i++ ) - { - if ( ! vals[i]->Write(&fmt) ) - goto error; - } - - // Ok, we have the binary data now. - char* data; - int len; - - len = fmt.EndWrite(&data); - - assert(len > 10); - - // Do we have not enough space in the buffer, or was the last flush a - // while ago? If so, flush first. - if ( len > (LOG_BUFFER_SIZE - peer->log_buffer_used) || (network_time - last_flush > 1.0) ) - { - if ( ! FlushLogBuffer(peer) ) - { - free(data); - return false; - } - } - - // If the data is actually larger than our complete buffer, just send it out. - if ( len > LOG_BUFFER_SIZE ) - return SendToChild(MSG_LOG_WRITE, peer, data, len, true); - - // Now we have space in the buffer, copy it into there. - memcpy(peer->log_buffer + peer->log_buffer_used, data, len); - peer->log_buffer_used += len; - assert(peer->log_buffer_used <= LOG_BUFFER_SIZE); - - free(data); - - return true; - -error: - FatalError(io->Error()); - return false; - } - -bool RemoteSerializer::FlushLogBuffer(Peer* p) - { - if ( ! p->logs_requested ) - return false; - - last_flush = network_time; - - if ( p->state == Peer::CLOSING ) - return false; - - if ( ! (p->log_buffer && p->log_buffer_used) ) - return true; - - char* data = new char[p->log_buffer_used]; - memcpy(data, p->log_buffer, p->log_buffer_used); - SendToChild(MSG_LOG_WRITE, p, data, p->log_buffer_used); - - p->log_buffer_used = 0; - return true; - } - -bool RemoteSerializer::ProcessLogCreateWriter() - { - if ( current_peer->state == Peer::CLOSING ) - return false; - -#ifdef USE_PERFTOOLS_DEBUG - // Don't track allocations here, they'll be released only after the - // main loop exists. And it's just a tiny amount anyway. - HeapLeakChecker::Disabler disabler; -#endif - - assert(current_args); - - EnumVal* id_val = 0; - EnumVal* writer_val = 0; - threading::Field** fields = 0; - int delete_fields_up_to = -1; - - BinarySerializationFormat fmt; - fmt.StartRead(current_args->data, current_args->len); - - int id, writer; - int num_fields; - logging::WriterBackend::WriterInfo* info = new logging::WriterBackend::WriterInfo(); - - bool success = fmt.Read(&id, "id") && - fmt.Read(&writer, "writer") && - fmt.Read(&num_fields, "num_fields") && - info->Read(&fmt); - - if ( ! success ) - goto error; - - fields = new threading::Field* [num_fields]; - - for ( int i = 0; i < num_fields; i++ ) - { - fields[i] = new threading::Field; - if ( ! fields[i]->Read(&fmt) ) - { - delete_fields_up_to = i + 1; - goto error; - } - } - - fmt.EndRead(); - - id_val = internal_type("Log::ID")->AsEnumType()->GetVal(id); - writer_val = internal_type("Log::Writer")->AsEnumType()->GetVal(writer); - - if ( ! log_mgr->CreateWriterForRemoteLog(id_val, writer_val, info, num_fields, fields) ) - { - info = 0; - fields = 0; - goto error; - } - - Unref(id_val); - Unref(writer_val); - - return true; - -error: - Unref(id_val); - Unref(writer_val); - delete info; - - for ( int i = 0; i < delete_fields_up_to; ++i ) - delete fields[i]; - - delete [] fields; - Error("write error for creating writer"); - return false; - } - -bool RemoteSerializer::ProcessLogWrite() - { - if ( current_peer->state == Peer::CLOSING ) - return false; - - assert(current_args); - - BinarySerializationFormat fmt; - fmt.StartRead(current_args->data, current_args->len); - - while ( fmt.BytesRead() != (int)current_args->len ) - { - // Unserialize one entry. - EnumVal* id_val = 0; - EnumVal* writer_val = 0; - threading::Value** vals = 0; - - int id, writer; - string path; - int num_fields; - - bool success = fmt.Read(&id, "id") && - fmt.Read(&writer, "writer") && - fmt.Read(&path, "path") && - fmt.Read(&num_fields, "num_fields"); - - if ( ! success ) - goto error; - - vals = new threading::Value* [num_fields]; - - for ( int i = 0; i < num_fields; i++ ) - { - vals[i] = new threading::Value; - - if ( ! vals[i]->Read(&fmt) ) - { - for ( int j = 0; j <= i; ++j ) - delete vals[j]; - - delete [] vals; - goto error; - } - } - - id_val = internal_type("Log::ID")->AsEnumType()->GetVal(id); - writer_val = internal_type("Log::Writer")->AsEnumType()->GetVal(writer); - - success = log_mgr->WriteFromRemote(id_val, writer_val, path, num_fields, vals); - - Unref(id_val); - Unref(writer_val); - - if ( ! success ) - goto error; - - } - - fmt.EndRead(); - - ++received_logs; - - return true; - -error: - Error("write error for log entry"); - return false; - } - -void RemoteSerializer::GotEvent(const char* name, double time, - EventHandlerPtr event, val_list* args) - { - if ( time >= 0 ) - { - // Marker for being called from ProcessRemotePrint(). - DEBUG_COMM("parent: got event"); - ++stats.events.in; - } - - if ( ! current_peer ) - { - Error("unserialized event from unknown peer"); - delete_vals(args); - return; - } - - BufferedEvent* e = new BufferedEvent; - - // Our time, not the time when the event was generated. - e->time = iosource_mgr->GetPktSrcs().size() ? - time_t(network_time) : time_t(timer_mgr->Time()); - - e->src = current_peer->id; - e->handler = event; - e->args = args; - - // If needed, coerce received record arguments to the expected record type. - if ( e->handler->FType() ) - { - const type_list* arg_types = e->handler->FType()->ArgTypes()->Types(); - loop_over_list(*args, i) - { - Val* v = (*args)[i]; - BroType* v_t = v->Type(); - BroType* arg_t = (*arg_types)[i]; - if ( v_t->Tag() == TYPE_RECORD && arg_t->Tag() == TYPE_RECORD ) - { - if ( ! same_type(v_t, arg_t) ) - { - Val* nv = v->AsRecordVal()->CoerceTo(arg_t->AsRecordType()); - if ( nv ) - { - args->replace(i, nv); - Unref(v); - } - } - } - } - } - - events.append(e); - } - -void RemoteSerializer::GotFunctionCall(const char* name, double time, - Func* function, val_list* args) - { - DEBUG_COMM("parent: got function call"); - ++stats.events.in; - - if ( ! current_peer ) - { - Error("unserialized function from unknown peer"); - delete_vals(args); - return; - } - - try - { - function->Call(args); - } - - catch ( InterpreterException& e ) - { /* Already reported. */ } - } - -void RemoteSerializer::GotID(ID* id, Val* val) - { - ++stats.ids.in; - - if ( ! current_peer ) - { - Error("unserialized id from unknown peer"); - Unref(id); - return; - } - - if ( current_peer->phase == Peer::HANDSHAKE && - streq(id->Name(), "peer_description") ) - { - if ( val->Type()->Tag() != TYPE_STRING ) - { - Error("peer_description not a string"); - Unref(id); - return; - } - - const char* desc = val->AsString()->CheckString(); - current_peer->val->Assign(4, new StringVal(desc)); - - Log(LogInfo, fmt("peer_description is %s", *desc ? desc : "not set"), - current_peer); - - Unref(id); - return; - } - - if ( id->Name()[0] == '#' ) - { - // This is a globally unique, non-user-visible ID. - - // Only MutableVals can be bound to names starting with '#'. - assert(val->IsMutableVal()); - - // It must be already installed in the global namespace: - // either we saw it before, or MutableVal::Unserialize() - // installed it. - assert(global_scope()->Lookup(id->Name())); - - // Only synchronized values can arrive here. - assert(((MutableVal*) val)->GetProperties() & MutableVal::SYNCHRONIZED); - - DBG_LOG(DBG_COMM, "got ID %s from peer\n", id->Name()); - } - - Unref(id); - } - -void RemoteSerializer::GotConnection(Connection* c) - { - ++stats.conns.in; - - // Nothing else to-do. Connection will be installed automatically - // (if allowed). - - Unref(c); - } - -void RemoteSerializer::GotStateAccess(StateAccess* s) - { - ++stats.accesses.in; - - ODesc d; - DBG_LOG(DBG_COMM, "got StateAccess: %s", (s->Describe(&d), d.Description())); - - if ( ! current_peer ) - { - Error("unserialized function from unknown peer"); - return; - } - - if ( current_peer->sync_requested & Peer::WE ) - s->Replay(); - - delete s; - } - -void RemoteSerializer::GotTimer(Timer* s) - { - reporter->Error("RemoteSerializer::GotTimer not implemented"); - } - -void RemoteSerializer::GotPacket(Packet* p) - { - ++stats.packets.in; - - BufferedPacket* bp = new BufferedPacket; - bp->time = time_t(timer_mgr->Time()); - bp->p = p; - packets.append(bp); - } - -void RemoteSerializer::Log(LogLevel level, const char* msg) - { - Log(level, msg, 0, LogParent); - } - -void RemoteSerializer::Log(LogLevel level, const char* msg, Peer* peer, - LogSrc src) - { - if ( peer ) - { - mgr.QueueEvent(remote_log_peer, { - peer->val->Ref(), - val_mgr->GetCount(level), - val_mgr->GetCount(src), - new StringVal(msg) - }); - } - else - { - mgr.QueueEvent(remote_log, { - val_mgr->GetCount(level), - val_mgr->GetCount(src), - new StringVal(msg) - }); - } - -#ifdef DEBUG - const int BUFSIZE = 1024; - char buffer[BUFSIZE]; - int len = 0; - - if ( peer ) - len += snprintf(buffer + len, sizeof(buffer) - len, "[#%d/%s:%d] ", - int(peer->id), peer->ip.AsURIString().c_str(), - peer->port); - - len += safe_snprintf(buffer + len, sizeof(buffer) - len, "%s", msg); - - DEBUG_COMM(fmt("parent: %.6f %s", current_time(), buffer)); -#endif - } - -void RemoteSerializer::RaiseEvent(EventHandlerPtr event, Peer* peer, - const char* arg) - { - val_list vl(1 + (bool)arg); - - if ( peer ) - { - Ref(peer->val); - vl.append(peer->val); - } - else - { - Val* v = mgr.GetLocalPeerVal(); - v->Ref(); - vl.append(v); - } - - if ( arg ) - vl.append(new StringVal(arg)); - - // If we only have remote sources, the network time - // will not increase as long as no peers are connected. - // Therefore, we send these events immediately. - mgr.Dispatch(new Event(event, std::move(vl), PEER_LOCAL)); - } - -void RemoteSerializer::LogStats() - { - if ( ! io ) - return; - - char buffer[512]; - io->Stats(buffer, 512); - Log(LogInfo, fmt("parent statistics: %s events=%lu/%lu operations=%lu/%lu", - buffer, stats.events.in, stats.events.out, - stats.accesses.in, stats.accesses.out)); - } - -RecordVal* RemoteSerializer::GetPeerVal(PeerID id) - { - Peer* peer = LookupPeer(id, true); - if ( ! peer ) - return 0; - - Ref(peer->val); - return peer->val; - } - -void RemoteSerializer::ChildDied() - { - Log(LogError, "child died"); - SetClosed(true); - child_pid = 0; - - // Shut down the main process as well. - terminate_processing(); - } - -bool RemoteSerializer::SendCMsgToChild(char msg_type, Peer* peer) - { - if ( ! sendCMsg(io, msg_type, peer ? peer->id : PEER_NONE) ) - { - reporter->Warning("can't send message of type %d: %s", - msg_type, io->Error()); - return false; - } - return true; - } - -bool RemoteSerializer::SendToChild(char type, Peer* peer, char* str, int len, - bool delete_with_free) - { - DEBUG_COMM(fmt("parent: (->child) %s (#%" PRI_SOURCE_ID ", %s)", msgToStr(type), peer ? peer->id : PEER_NONE, str)); - - if ( child_pid && sendToIO(io, type, peer ? peer->id : PEER_NONE, str, len, - delete_with_free) ) - return true; - - if ( delete_with_free ) - free(str); - else - delete [] str; - - if ( ! child_pid ) - return false; - - if ( io->Eof() ) - ChildDied(); - - FatalError(io->Error()); - return false; - } - -bool RemoteSerializer::SendToChild(char type, Peer* peer, int nargs, ...) - { - va_list ap; - -#ifdef DEBUG - va_start(ap, nargs); - DEBUG_COMM(fmt("parent: (->child) %s (#%" PRI_SOURCE_ID ",%s)", - msgToStr(type), peer ? peer->id : PEER_NONE, fmt_uint32s(nargs, ap))); - va_end(ap); -#endif - - if ( child_pid ) - { - va_start(ap, nargs); - bool ret = sendToIO(io, type, peer ? peer->id : PEER_NONE, nargs, ap); - va_end(ap); - - if ( ret ) - return true; - } - - if ( ! child_pid ) - return false; - - if ( io->Eof() ) - ChildDied(); - - FatalError(io->Error()); - return false; - } - -bool RemoteSerializer::SendToChild(ChunkedIO::Chunk* c) - { - DEBUG_COMM(fmt("parent: (->child) chunk of size %d", c->len)); - - if ( child_pid && sendToIO(io, c) ) - return true; - - c->free_func(c->data); - c->data = 0; - - if ( ! child_pid ) - return false; - - if ( io->Eof() ) - ChildDied(); - - FatalError(io->Error()); - return false; - } - -void RemoteSerializer::FatalError(const char* msg) - { - msg = fmt("fatal error, shutting down communication: %s", msg); - Log(LogError, msg); - reporter->Error("%s", msg); - - SetClosed(true); - - if ( kill(child_pid, SIGQUIT) < 0 ) - reporter->Warning("warning: cannot kill child pid %d, %s", child_pid, strerror(errno)); - - child_pid = 0; - using_communication = false; - io->Clear(); - - loop_over_list(peers, i) - { - // Make perftools happy. - Peer* p = peers[i]; - delete [] p->log_buffer; - delete [] p->print_buffer; - p->log_buffer = p->print_buffer = 0; - } - } - -bool RemoteSerializer::IsActive() - { - if ( listening ) - return true; - - loop_over_list(peers, i) - if ( peers[i]->state == Peer::PENDING || - peers[i]->state == Peer::CONNECTED ) - return true; - - return false; - } - -void RemoteSerializer::ReportError(const char* msg) - { - if ( current_peer && current_peer->phase != Peer::SETUP ) - RaiseEvent(remote_connection_error, current_peer, msg); - Log(LogError, msg, current_peer); - } - -void RemoteSerializer::InternalCommError(const char* msg) - { -#ifdef DEBUG_COMMUNICATION - DumpDebugData(); -#else - reporter->InternalError("%s", msg); -#endif - } - -#ifdef DEBUG_COMMUNICATION - -void RemoteSerializer::DumpDebugData() - { - Log(LogError, "dumping debug data and terminating ..."); - io->DumpDebugData("comm-dump.parent", true); - io->DumpDebugData("comm-dump.parent", false); - SendToChild(MSG_DEBUG_DUMP, 0, 0); - Terminate(); - } - -static ChunkedIO* openDump(const char* file) - { - int fd = open(file, O_RDONLY, 0600); - - if ( fd < 0 ) - { - reporter->Error("cannot open %s: %s\n", file, strerror(errno)); - return 0; - } - - return new ChunkedIOFd(fd, "dump-file"); - } - -void RemoteSerializer::ReadDumpAsMessageType(const char* file) - { - ChunkedIO* io = openDump(file); - if ( ! io ) - return; - - ChunkedIO::Chunk* chunk; - - if ( ! io->Read(&chunk, true ) ) - { - reporter->Error("cannot read %s: %s\n", file, strerror(errno)); - return; - } - - CMsg* msg = (CMsg*) chunk->data; - - delete [] chunk->data; - delete io; - } - -void RemoteSerializer::ReadDumpAsSerialization(const char* file) - { - FileSerializer s; - UnserialInfo info(&s); - info.print = stdout; - info.install_uniques = info.ignore_callbacks = true; - s.Read(&info, file, false); - } - -#endif - -//////////////////////////// - -// If true (set by signal handler), we will log some stats to parent. -static bool log_stats = false; -static bool log_prof = false; - -// How often stats are sent (in seconds). -// Perhaps we should make this configurable... -const int STATS_INTERVAL = 60; - -static RETSIGTYPE sig_handler_log(int signo) - { - // SIGALRM is the only one we get. - log_stats = true; - } - -static RETSIGTYPE sig_handler_prof(int signo) - { - log_prof = true; - } - -SocketComm::SocketComm() - { - io = 0; - - // We start the ID counter high so that IDs assigned by us - // (hopefully) don't conflict with those of our parent. - id_counter = 10000; - parent_peer = 0; - parent_msgstate = TYPE; - parent_id = RemoteSerializer::PEER_NONE; - parent_msgtype = 0; - parent_args = 0; - shutting_conns_down = false; - terminating = false; - killing = false; - - listen_port = 0; - listen_ssl = false; - enable_ipv6 = false; - bind_retry_interval = 0; - listen_next_try = 0; - - // We don't want to use the signal handlers of our parent. - (void) setsignal(SIGTERM, SIG_DFL); - (void) setsignal(SIGINT, SIG_DFL); - (void) setsignal(SIGUSR1, SIG_DFL); - (void) setsignal(SIGUSR2, SIG_DFL); - (void) setsignal(SIGCONT, SIG_DFL); - (void) setsignal(SIGCHLD, SIG_DFL); - - // Raping SIGPROF for profiling - (void) setsignal(SIGPROF, sig_handler_prof); - (void) setsignal(SIGALRM, sig_handler_log); - alarm(STATS_INTERVAL); - } - -SocketComm::~SocketComm() - { - loop_over_list(peers, i) - delete peers[i]->io; - - delete io; - CloseListenFDs(); - } - -static unsigned int first_rtime = 0; - -static void fd_vector_set(const std::vector& fds, fd_set* set, int* max) - { - for ( size_t i = 0; i < fds.size(); ++i ) - { - FD_SET(fds[i], set); - *max = ::max(fds[i], *max); - } - } - -void SocketComm::Run() - { - first_rtime = (unsigned int) current_time(true); - - while ( true ) - { - // Logging signaled? - if ( log_stats ) - LogStats(); - - // Termination signaled - if ( terminating ) - CheckFinished(); - - // Build FDSets for select. - fd_set fd_read, fd_write, fd_except; - - FD_ZERO(&fd_read); - FD_ZERO(&fd_write); - FD_ZERO(&fd_except); - - int max_fd = io->Fd(); - FD_SET(io->Fd(), &fd_read); - max_fd = std::max(max_fd, io->ExtraReadFDs().Set(&fd_read)); - - loop_over_list(peers, i) - { - if ( peers[i]->connected ) - { - FD_SET(peers[i]->io->Fd(), &fd_read); - if ( peers[i]->io->Fd() > max_fd ) - max_fd = peers[i]->io->Fd(); - max_fd = std::max(max_fd, - peers[i]->io->ExtraReadFDs().Set(&fd_read)); - } - else - { - if ( peers[i]->next_try > 0 && - time(0) > peers[i]->next_try ) - // Try reconnect. - Connect(peers[i]); - } - } - - if ( listen_next_try && time(0) > listen_next_try ) - Listen(); - - for ( size_t i = 0; i < listen_fds.size(); ++i ) - { - FD_SET(listen_fds[i], &fd_read); - if ( listen_fds[i] > max_fd ) - max_fd = listen_fds[i]; - } - - if ( io->IsFillingUp() && ! shutting_conns_down ) - { - Error("queue to parent filling up; shutting down heaviest connection"); - - const ChunkedIO::Statistics* stats = 0; - unsigned long max = 0; - Peer* max_peer = 0; - - loop_over_list(peers, i) - { - if ( ! peers[i]->connected ) - continue; - - stats = peers[i]->io->Stats(); - if ( stats->bytes_read > max ) - { - max = stats->bytes_read; - max_peer = peers[i]; - } - } - - if ( max_peer ) - CloseConnection(max_peer, true); - - shutting_conns_down = true; - } - - if ( ! io->IsFillingUp() && shutting_conns_down ) - shutting_conns_down = false; - - static long selects = 0; - static long canwrites = 0; - - ++selects; - if ( io->CanWrite() ) - ++canwrites; - - struct timeval timeout; - timeout.tv_sec = 1; - timeout.tv_usec = 0; - - int a = select(max_fd + 1, &fd_read, &fd_write, &fd_except, &timeout); - - if ( selects % 100000 == 0 ) - Log(fmt("selects=%ld canwrites=%ld pending=%lu", - selects, canwrites, io->Stats()->pending)); - - if ( a < 0 ) - // Ignore errors for now. - continue; - - if ( io->CanRead() ) - ProcessParentMessage(); - - io->Flush(); - - loop_over_list(peers, j) - { - // We have to be careful here as the peer may - // be removed when an error occurs. - Peer* current = peers[j]; - int round = 0; - while ( ++round <= 10 && j < peers.length() && - peers[j] == current && current->connected && - current->io->CanRead() ) - { - ProcessRemoteMessage(current); - } - } - - for ( size_t i = 0; i < listen_fds.size(); ++i ) - if ( FD_ISSET(listen_fds[i], &fd_read) ) - AcceptConnection(listen_fds[i]); - - // Hack to display CPU usage of the child, triggered via - // SIGPROF. - static unsigned int first_rtime = 0; - if ( first_rtime == 0 ) - first_rtime = (unsigned int) current_time(true); - - if ( log_prof ) - { - LogProf(); - log_prof = false; - } - } - } - -bool SocketComm::ProcessParentMessage() - { - switch ( parent_msgstate ) { - case TYPE: - { - parent_peer = 0; - parent_msgtype = MSG_NONE; - - // CMsg follows - ChunkedIO::Chunk* c; - if ( ! io->Read(&c) ) - { - if ( io->Eof() ) - Error("parent died", true); - - Error(fmt("can't read parent's cmsg: %s", - io->Error()), true); - return false; - } - - if ( ! c ) - return true; - - CMsg* msg = (CMsg*) c->data; - parent_peer = LookupPeer(msg->Peer(), false); - parent_id = msg->Peer(); - parent_msgtype = msg->Type(); - parent_args = 0; - - delete c; - - switch ( parent_msgtype ) { - case MSG_LISTEN_STOP: - case MSG_CLOSE: - case MSG_CLOSE_ALL: - case MSG_TERMINATE: - case MSG_PHASE_DONE: - case MSG_DEBUG_DUMP: - case MSG_REQUEST_LOGS: - { - // No further argument chunk. - parent_msgstate = TYPE; - return DoParentMessage(); - } - - case MSG_LISTEN: - case MSG_CONNECT_TO: - case MSG_COMPRESS: - case MSG_PING: - case MSG_PONG: - case MSG_REQUEST_EVENTS: - case MSG_REQUEST_SYNC: - case MSG_SERIAL: - case MSG_CAPTURE_FILTER: - case MSG_VERSION: - case MSG_CAPS: - case MSG_SYNC_POINT: - case MSG_REMOTE_PRINT: - case MSG_LOG_CREATE_WRITER: - case MSG_LOG_WRITE: - { - // One further argument chunk. - parent_msgstate = ARGS; - return ProcessParentMessage(); - } - - default: - InternalError(fmt("unknown msg type %d", parent_msgtype)); - return true; - } - } - - case ARGS: - { - // Argument chunk follows. - ChunkedIO::Chunk* c = 0; - READ_CHUNK(io, c, Error("parent died", true), true); - parent_args = c; - parent_msgstate = TYPE; - bool result = DoParentMessage(); - - if ( parent_args ) - { - delete parent_args; - parent_args = 0; - } - - return result; - } - - default: - InternalError("unknown msgstate"); - } - - // Cannot be reached. - return false; - } - -bool SocketComm::DoParentMessage() - { - switch ( parent_msgtype ) { - - case MSG_LISTEN_STOP: - { - CloseListenFDs(); - - Log("stopped listening"); - - return true; - } - - case MSG_CLOSE: - { - if ( parent_peer && parent_peer->connected ) - CloseConnection(parent_peer, false); - return true; - } - - case MSG_CLOSE_ALL: - { - loop_over_list(peers, i) - { - if ( peers[i]->connected ) - CloseConnection(peers[i], false); - } - return true; - } - - case MSG_TERMINATE: - { - terminating = true; - CheckFinished(); - return true; - } - - case MSG_DEBUG_DUMP: - { -#ifdef DEBUG_COMMUNICATION - io->DumpDebugData("comm-dump.child.pipe", true); - io->DumpDebugData("comm-dump.child.pipe", false); - - loop_over_list(peers, j) - { - RemoteSerializer::PeerID id = peers[j]->id; - peers[j]->io->DumpDebugData(fmt("comm-dump.child.peer.%d", id), true); - peers[j]->io->DumpDebugData(fmt("comm-dump.child.peer.%d", id), false); - } -#else - InternalError("DEBUG_DUMP support not compiled in"); -#endif - return true; - } - - case MSG_LISTEN: - return ProcessListen(); - - case MSG_CONNECT_TO: - return ProcessConnectTo(); - - case MSG_COMPRESS: - return ProcessParentCompress(); - - case MSG_PING: - { - // Set time2. - assert(parent_args); - ping_args* args = (ping_args*) parent_args->data; - args->time2 = htond(current_time(true)); - return ForwardChunkToPeer(); - } - - case MSG_PONG: - { - assert(parent_args); - // Calculate time delta. - ping_args* args = (ping_args*) parent_args->data; - args->time3 = htond(current_time(true) - ntohd(args->time3)); - return ForwardChunkToPeer(); - } - - case MSG_PHASE_DONE: - case MSG_REQUEST_LOGS: - { - // No argument block follows. - if ( parent_peer && parent_peer->connected ) - { - DEBUG_COMM(fmt("child: forwarding %s to peer", msgToStr(parent_msgtype))); - if ( ! SendToPeer(parent_peer, parent_msgtype, 0) ) - return false; - } - - return true; - } - - case MSG_REQUEST_EVENTS: - case MSG_REQUEST_SYNC: - case MSG_SERIAL: - case MSG_CAPTURE_FILTER: - case MSG_VERSION: - case MSG_CAPS: - case MSG_SYNC_POINT: - case MSG_REMOTE_PRINT: - case MSG_LOG_CREATE_WRITER: - case MSG_LOG_WRITE: - assert(parent_args); - return ForwardChunkToPeer(); - - default: - InternalError("ProcessParentMessage: unexpected state"); - } - - InternalError("cannot be reached"); - return false; - } - -bool SocketComm::ForwardChunkToPeer() - { - char state = parent_msgtype; - - if ( parent_peer && parent_peer->connected ) - { - DEBUG_COMM("child: forwarding with 1 arg to peer"); - - if ( ! SendToPeer(parent_peer, state, 0) ) - return false; - - if ( ! SendToPeer(parent_peer, parent_args) ) - return false; - - parent_args = 0; - } - else - { -#ifdef DEBUG - if ( parent_peer ) - DEBUG_COMM(fmt("child: not connected to #%" PRI_SOURCE_ID, parent_id)); -#endif - } - - return true; - } - -bool SocketComm::ProcessConnectTo() - { - assert(parent_args); - vector args = tokenize(parent_args->data, ','); - - if ( args.size() != 6 ) - { - Error(fmt("ProcessConnectTo() bad number of arguments")); - return false; - } - - Peer* peer = new Peer; - - if ( ! atoi_n(args[0].size(), args[0].c_str(), 0, 10, peer->id) ) - { - Error(fmt("ProccessConnectTo() bad peer id string")); - delete peer; - return false; - } - - peer->ip = IPAddr(args[1]); - peer->zone_id = args[2]; - - if ( ! atoi_n(args[3].size(), args[3].c_str(), 0, 10, peer->port) ) - { - Error(fmt("ProcessConnectTo() bad peer port string")); - delete peer; - return false; - } - - if ( ! atoi_n(args[4].size(), args[4].c_str(), 0, 10, peer->retry) ) - { - Error(fmt("ProcessConnectTo() bad peer retry string")); - delete peer; - return false; - } - - peer->ssl = false; - if ( args[5] != "0" ) - peer->ssl = true; - - return Connect(peer); - } - -bool SocketComm::ProcessListen() - { - assert(parent_args); - vector args = tokenize(parent_args->data, ','); - - if ( args.size() != 6 ) - { - Error(fmt("ProcessListen() bad number of arguments")); - return false; - } - - listen_if = args[0]; - - if ( ! atoi_n(args[1].size(), args[1].c_str(), 0, 10, listen_port) ) - { - Error(fmt("ProcessListen() bad peer port string")); - return false; - } - - listen_ssl = false; - if ( args[2] != "0" ) - listen_ssl = true; - - enable_ipv6 = false; - if ( args[3] != "0" ) - enable_ipv6 = true; - - listen_zone_id = args[4]; - - if ( ! atoi_n(args[5].size(), args[5].c_str(), 0, 10, bind_retry_interval) ) - { - Error(fmt("ProcessListen() bad peer port string")); - return false; - } - - return Listen(); - } - -bool SocketComm::ProcessParentCompress() - { - assert(parent_args); - uint32* args = (uint32*) parent_args->data; - - uint32 level = ntohl(args[0]); - - if ( ! parent_peer->compressor ) - { - parent_peer->io = new CompressedChunkedIO(parent_peer->io); - parent_peer->io->Init(); - parent_peer->compressor = true; - } - - // Signal compression to peer. - if ( ! SendToPeer(parent_peer, MSG_COMPRESS, 0) ) - return false; - - // This cast is safe. - CompressedChunkedIO* comp_io = (CompressedChunkedIO*) parent_peer->io; - comp_io->EnableCompression(level); - - Log(fmt("enabling compression (level %d)", level), parent_peer); - - return true; - } - -bool SocketComm::ProcessRemoteMessage(SocketComm::Peer* peer) - { - assert(peer); - - peer->io->Flush(); - - switch ( peer->state ) { - case MSG_NONE: - { // CMsg follows - ChunkedIO::Chunk* c; - READ_CHUNK(peer->io, c, - (CloseConnection(peer, true), peer), false) - - CMsg* msg = (CMsg*) c->data; - - DEBUG_COMM(fmt("child: %s from peer #%" PRI_SOURCE_ID, - msgToStr(msg->Type()), peer->id)); - - switch ( msg->Type() ) { - case MSG_PHASE_DONE: - case MSG_REQUEST_LOGS: - // No further argument block. - DEBUG_COMM("child: forwarding with 0 args to parent"); - if ( ! SendToParent(msg->Type(), peer, 0) ) - return false; - break; - - default: - peer->state = msg->Type(); - } - - delete c; - - break; - } - - case MSG_COMPRESS: - ProcessPeerCompress(peer); - break; - - case MSG_PING: - { - // Messages with one further argument block which we simply - // forward to our parent. - ChunkedIO::Chunk* c; - READ_CHUNK(peer->io, c, - (CloseConnection(peer, true), peer), false) - - // Set time3. - ping_args* args = (ping_args*) c->data; - args->time3 = htond(current_time(true)); - return ForwardChunkToParent(peer, c); - } - - case MSG_PONG: - { - // Messages with one further argument block which we simply - // forward to our parent. - ChunkedIO::Chunk* c; - READ_CHUNK(peer->io, c, - (CloseConnection(peer, true), peer), false) - - // Calculate time delta. - ping_args* args = (ping_args*) c->data; - args->time2 = htond(current_time(true) - ntohd(args->time2)); - return ForwardChunkToParent(peer, c); - } - - case MSG_REQUEST_EVENTS: - case MSG_REQUEST_SYNC: - case MSG_SERIAL: - case MSG_CAPTURE_FILTER: - case MSG_VERSION: - case MSG_CAPS: - case MSG_SYNC_POINT: - case MSG_REMOTE_PRINT: - case MSG_LOG_CREATE_WRITER: - case MSG_LOG_WRITE: - { - // Messages with one further argument block which we simply - // forward to our parent. - ChunkedIO::Chunk* c; - READ_CHUNK(peer->io, c, - (CloseConnection(peer, true), peer), false) - - return ForwardChunkToParent(peer, c); - } - - default: - InternalError("ProcessRemoteMessage: unexpected state"); - } - - return true; - } - -bool SocketComm::ForwardChunkToParent(Peer* peer, ChunkedIO::Chunk* c) - { - char state = peer->state; - peer->state = MSG_NONE; - - DEBUG_COMM("child: forwarding message with 1 arg to parent"); - - if ( ! SendToParent(state, peer, 0) ) - return false; - - if ( ! SendToParent(c) ) - return false; - - io->Flush(); // FIXME: Needed? - return true; - } - -bool SocketComm::ProcessPeerCompress(Peer* peer) - { - peer->state = MSG_NONE; - - if ( ! parent_peer->compressor ) - { - parent_peer->io = new CompressedChunkedIO(parent_peer->io); - parent_peer->io->Init(); - parent_peer->compressor = true; - } - - // This cast is safe here. - ((CompressedChunkedIO*) peer->io)->EnableDecompression(); - Log("enabling decompression", peer); - return true; - } - -bool SocketComm::Connect(Peer* peer) - { - int status; - addrinfo hints, *res, *res0; - memset(&hints, 0, sizeof(hints)); - - hints.ai_family = PF_UNSPEC; - hints.ai_protocol = IPPROTO_TCP; - hints.ai_socktype = SOCK_STREAM; - hints.ai_flags = AI_NUMERICHOST; - - char port_str[16]; - modp_uitoa10(peer->port, port_str); - - string gaihostname(peer->ip.AsString()); - if ( peer->zone_id != "" ) - gaihostname.append("%").append(peer->zone_id); - - status = getaddrinfo(gaihostname.c_str(), port_str, &hints, &res0); - if ( status != 0 ) - { - Error(fmt("getaddrinfo error: %s", gai_strerror(status))); - return false; - } - - int sockfd = -1; - for ( res = res0; res; res = res->ai_next ) - { - sockfd = socket(res->ai_family, res->ai_socktype, res->ai_protocol); - if ( sockfd < 0 ) - { - Error(fmt("can't create connect socket, %s", strerror(errno))); - continue; - } - - if ( connect(sockfd, res->ai_addr, res->ai_addrlen) < 0 ) - { - Error(fmt("connect failed: %s", strerror(errno)), peer); - safe_close(sockfd); - sockfd = -1; - continue; - } - - break; - } - - freeaddrinfo(res0); - - bool connected = sockfd != -1; - - if ( ! (connected || peer->retry) ) - { - CloseConnection(peer, false); - return false; - } - - Peer* existing_peer = LookupPeer(peer->id, false); - if ( existing_peer ) - { - *existing_peer = *peer; - peer = existing_peer; - } - else - peers.append(peer); - - peer->connected = connected; - peer->next_try = connected ? 0 : time(0) + peer->retry; - peer->state = MSG_NONE; - peer->io = 0; - peer->compressor = false; - - if ( connected ) - { - if ( peer->ssl ) - peer->io = new ChunkedIOSSL(sockfd, false); - else - peer->io = new ChunkedIOFd(sockfd, "child->peer"); - - if ( ! peer->io->Init() ) - { - Error(fmt("can't init peer io: %s", - peer->io->Error()), false); - return 0; - } - } - - if ( connected ) - { - Log("connected", peer); - - const size_t BUFSIZE = 1024; - char* data = new char[BUFSIZE]; - snprintf(data, BUFSIZE, "%s,%" PRIu32, peer->ip.AsString().c_str(), - peer->port); - - if ( ! SendToParent(MSG_CONNECTED, peer, data) ) - return false; - } - - return connected; - } - -bool SocketComm::CloseConnection(Peer* peer, bool reconnect) - { - if ( ! SendToParent(MSG_CLOSE, peer, 0) ) - return false; - - Log("connection closed", peer); - - if ( ! peer->retry || ! reconnect ) - { - peers.remove(peer); - delete peer->io; // This will close the fd. - delete peer; - } - else - { - delete peer->io; // This will close the fd. - peer->io = 0; - peer->connected = false; - peer->next_try = time(0) + peer->retry; - } - - if ( parent_peer == peer ) - { - parent_peer = 0; - parent_id = RemoteSerializer::PEER_NONE; - } - - return true; - } - -bool SocketComm::Listen() - { - int status, on = 1; - addrinfo hints, *res, *res0; - memset(&hints, 0, sizeof(hints)); - - IPAddr listen_ip(listen_if); - - if ( enable_ipv6 ) - { - if ( listen_ip == IPAddr("0.0.0.0") || listen_ip == IPAddr("::") ) - hints.ai_family = PF_UNSPEC; - else - hints.ai_family = (listen_ip.GetFamily() == IPv4 ? PF_INET : PF_INET6); - } - else - hints.ai_family = PF_INET; - - hints.ai_protocol = IPPROTO_TCP; - hints.ai_socktype = SOCK_STREAM; - hints.ai_flags = AI_PASSIVE | AI_NUMERICHOST; - - char port_str[16]; - modp_uitoa10(listen_port, port_str); - - string scoped_addr(listen_if); - if ( listen_zone_id != "" ) - scoped_addr.append("%").append(listen_zone_id); - - const char* addr_str = 0; - if ( listen_ip != IPAddr("0.0.0.0") && listen_ip != IPAddr("::") ) - addr_str = scoped_addr.c_str(); - - CloseListenFDs(); - - if ( (status = getaddrinfo(addr_str, port_str, &hints, &res0)) != 0 ) - { - Error(fmt("getaddrinfo error: %s", gai_strerror(status))); - return false; - } - - for ( res = res0; res; res = res->ai_next ) - { - if ( res->ai_family != AF_INET && res->ai_family != AF_INET6 ) - { - Error(fmt("can't create listen socket: unknown address family, %d", - res->ai_family)); - continue; - } - - IPAddr a = (res->ai_family == AF_INET) ? - IPAddr(((sockaddr_in*)res->ai_addr)->sin_addr) : - IPAddr(((sockaddr_in6*)res->ai_addr)->sin6_addr); - - string l_addr_str(a.AsURIString()); - if ( listen_zone_id != "") - l_addr_str.append("%").append(listen_zone_id); - - int fd = socket(res->ai_family, res->ai_socktype, res->ai_protocol); - if ( fd < 0 ) - { - Error(fmt("can't create listen socket, %s", strerror(errno))); - continue; - } - - if ( setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) < 0 ) - Error(fmt("can't set SO_REUSEADDR, %s", strerror(errno))); - - // For IPv6 listening sockets, we don't want do dual binding to also - // get IPv4-mapped addresses because that's not as portable. e.g. - // many BSDs don't allow that. - if ( res->ai_family == AF_INET6 && - setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &on, sizeof(on)) < 0 ) - Error(fmt("can't set IPV6_V6ONLY, %s", strerror(errno))); - - if ( ::bind(fd, res->ai_addr, res->ai_addrlen) < 0 ) - { - Error(fmt("can't bind to %s:%s, %s", l_addr_str.c_str(), - port_str, strerror(errno))); - - if ( errno == EADDRINUSE ) - { - // Abandon completely this attempt to set up listening sockets, - // try again later. - safe_close(fd); - CloseListenFDs(); - listen_next_try = time(0) + bind_retry_interval; - freeaddrinfo(res0); - return false; - } - - safe_close(fd); - continue; - } - - if ( listen(fd, 50) < 0 ) - { - Error(fmt("can't listen on %s:%s, %s", l_addr_str.c_str(), - port_str, strerror(errno))); - safe_close(fd); - continue; - } - - listen_fds.push_back(fd); - Log(fmt("listening on %s:%s (%s)", l_addr_str.c_str(), port_str, - listen_ssl ? "ssl" : "clear")); - } - - freeaddrinfo(res0); - - listen_next_try = 0; - return listen_fds.size() > 0; - } - -bool SocketComm::AcceptConnection(int fd) - { - union { - sockaddr_storage ss; - sockaddr_in s4; - sockaddr_in6 s6; - } client; - - socklen_t len = sizeof(client.ss); - - int clientfd = accept(fd, (sockaddr*) &client.ss, &len); - if ( clientfd < 0 ) - { - Error(fmt("accept failed, %s %d", strerror(errno), errno)); - return false; - } - - if ( client.ss.ss_family != AF_INET && client.ss.ss_family != AF_INET6 ) - { - Error(fmt("accept fail, unknown address family %d", - client.ss.ss_family)); - safe_close(clientfd); - return false; - } - - Peer* peer = new Peer; - peer->id = id_counter++; - peer->ip = client.ss.ss_family == AF_INET ? - IPAddr(client.s4.sin_addr) : - IPAddr(client.s6.sin6_addr); - - peer->port = client.ss.ss_family == AF_INET ? - ntohs(client.s4.sin_port) : - ntohs(client.s6.sin6_port); - - peer->connected = true; - peer->ssl = listen_ssl; - peer->compressor = false; - - if ( peer->ssl ) - peer->io = new ChunkedIOSSL(clientfd, true); - else - peer->io = new ChunkedIOFd(clientfd, "child->peer"); - - if ( ! peer->io->Init() ) - { - Error(fmt("can't init peer io: %s", peer->io->Error()), false); - delete peer->io; - delete peer; - return false; - } - - peers.append(peer); - - Log(fmt("accepted %s connection", peer->ssl ? "SSL" : "clear"), peer); - - const size_t BUFSIZE = 1024; - char* data = new char[BUFSIZE]; - snprintf(data, BUFSIZE, "%s,%" PRIu32, peer->ip.AsString().c_str(), - peer->port); - - if ( ! SendToParent(MSG_CONNECTED, peer, data) ) - return false; - - return true; - } - -const char* SocketComm::MakeLogString(const char* msg, Peer* peer) - { - const int BUFSIZE = 1024; - static char* buffer = 0; - - if ( ! buffer ) - buffer = new char[BUFSIZE]; - - int len = 0; - - if ( peer ) - { - string scoped_addr(peer->ip.AsURIString()); - if ( peer->zone_id != "" ) - scoped_addr.append("%").append(peer->zone_id); - - len = snprintf(buffer, BUFSIZE, "[#%d/%s:%d] ", int(peer->id), - scoped_addr.c_str(), peer->port); - } - - len += safe_snprintf(buffer + len, BUFSIZE - len, "%s", msg); - return buffer; - } - -void SocketComm::CloseListenFDs() - { - for ( size_t i = 0; i < listen_fds.size(); ++i ) - safe_close(listen_fds[i]); - - listen_fds.clear(); - } - -void SocketComm::Error(const char* msg, bool kill_me) - { - if ( kill_me ) - { - fprintf(stderr, "fatal error in child: %s\n", msg); - Kill(); - } - else - { - if ( io->Eof() ) - // Can't send to parent, so fall back to stderr. - fprintf(stderr, "error in child: %s", msg); - else - SendToParent(MSG_ERROR, 0, copy_string(msg)); - } - - DEBUG_COMM(fmt("child: %s", msg)); - } - -bool SocketComm::Error(const char* msg, Peer* peer) - { - const char* buffer = MakeLogString(msg, peer); - Error(buffer); - - // If a remote peer causes an error, we shutdown the connection - // as resynchronizing is in general not possible. But we may - // try again later. - if ( peer->connected ) - CloseConnection(peer, true); - - return true; - } - -void SocketComm::Log(const char* msg, Peer* peer) - { - const char* buffer = MakeLogString(msg, peer); - SendToParent(MSG_LOG, 0, copy_string(buffer)); - DEBUG_COMM(fmt("child: %s", buffer)); - } - -void SocketComm::InternalError(const char* msg) - { - fprintf(stderr, "internal error in child: %s\n", msg); - Kill(); - } - -void SocketComm::Kill() - { - if ( killing ) - // Ignore recursive calls. - return; - - killing = true; - - LogProf(); - Log("terminating"); - - CloseListenFDs(); - - if ( kill(getpid(), SIGTERM) < 0 ) - Log(fmt("warning: cannot kill SocketComm pid %d, %s", getpid(), strerror(errno))); - - while ( 1 ) - ; // loop until killed - } - -SocketComm::Peer* SocketComm::LookupPeer(RemoteSerializer::PeerID id, - bool only_if_connected) - { - loop_over_list(peers, i) - if ( peers[i]->id == id ) - return ! only_if_connected || - peers[i]->connected ? peers[i] : 0; - return 0; - } - -bool SocketComm::LogStats() - { - if ( ! peers.length() ) - return true; - - // Concat stats of all peers into single buffer. - char* buffer = new char[peers.length() * 512]; - int pos = 0; - - loop_over_list(peers, i) - { - if ( peers[i]->connected ) - peers[i]->io->Stats(buffer+pos, 512); - else - strcpy(buffer+pos, "not connected"); - pos += strlen(buffer+pos) + 1; - } - - // Send it. - if ( ! SendToParent(MSG_STATS, 0, buffer, pos) ) - return false; - - log_stats = false; - alarm(STATS_INTERVAL); - return true; - } - -bool SocketComm::LogProf() - { - static struct rusage cld_res; - getrusage(RUSAGE_SELF, &cld_res); - - double Utime = cld_res.ru_utime.tv_sec + cld_res.ru_utime.tv_usec / 1e6; - double Stime = cld_res.ru_stime.tv_sec + cld_res.ru_stime.tv_usec / 1e6; - double Rtime = current_time(true); - - SocketComm::Log(fmt("CPU usage: user %.03f sys %.03f real %0.03f", - Utime, Stime, Rtime - first_rtime)); - - return true; - } - -void SocketComm::CheckFinished() - { - assert(terminating); - - loop_over_list(peers, i) - { - if ( ! peers[i]->connected ) - continue; - if ( ! peers[i]->io->IsIdle() ) - return; - } - - LogProf(); - Log("terminating"); - - // All done. - SendToParent(MSG_TERMINATE, 0, 0); - } - -bool SocketComm::SendToParent(char type, Peer* peer, const char* str, int len) - { -#ifdef DEBUG - // str may already by constructed with fmt() - const char* tmp = copy_string(str); - DEBUG_COMM(fmt("child: (->parent) %s (#%" PRI_SOURCE_ID ", %s)", msgToStr(type), peer ? peer->id : RemoteSerializer::PEER_NONE, tmp)); - delete [] tmp; -#endif - if ( sendToIO(io, type, peer ? peer->id : RemoteSerializer::PEER_NONE, - str, len) ) - return true; - - if ( io->Eof() ) - Error("parent died", true); - - return false; - } - -bool SocketComm::SendToParent(char type, Peer* peer, int nargs, ...) - { - va_list ap; - -#ifdef DEBUG - va_start(ap,nargs); - DEBUG_COMM(fmt("child: (->parent) %s (#%" PRI_SOURCE_ID ",%s)", msgToStr(type), peer ? peer->id : RemoteSerializer::PEER_NONE, fmt_uint32s(nargs, ap))); - va_end(ap); -#endif - - va_start(ap, nargs); - bool ret = sendToIO(io, type, - peer ? peer->id : RemoteSerializer::PEER_NONE, - nargs, ap); - va_end(ap); - - if ( ret ) - return true; - - if ( io->Eof() ) - Error("parent died", true); - - return false; - } - -bool SocketComm::SocketComm::SendToParent(ChunkedIO::Chunk* c) - { - DEBUG_COMM(fmt("child: (->parent) chunk of size %d", c->len)); - if ( sendToIO(io, c) ) - return true; - - if ( io->Eof() ) - Error("parent died", true); - - return false; - } - -bool SocketComm::SendToPeer(Peer* peer, char type, const char* str, int len) - { -#ifdef DEBUG - // str may already by constructed with fmt() - const char* tmp = copy_string(str); - DEBUG_COMM(fmt("child: (->peer) %s to #%" PRI_SOURCE_ID " (%s)", msgToStr(type), peer->id, tmp)); - delete [] tmp; -#endif - - if ( ! sendToIO(peer->io, type, RemoteSerializer::PEER_NONE, str, len) ) - { - Error(fmt("child: write error %s", io->Error()), peer); - return false; - } - - return true; - } - -bool SocketComm::SendToPeer(Peer* peer, char type, int nargs, ...) - { - va_list ap; - -#ifdef DEBUG - va_start(ap,nargs); - DEBUG_COMM(fmt("child: (->peer) %s to #%" PRI_SOURCE_ID " (%s)", - msgToStr(type), peer->id, fmt_uint32s(nargs, ap))); - va_end(ap); -#endif - - va_start(ap, nargs); - bool ret = sendToIO(peer->io, type, RemoteSerializer::PEER_NONE, - nargs, ap); - va_end(ap); - - if ( ! ret ) - { - Error(fmt("child: write error %s", io->Error()), peer); - return false; - } - - return true; - } - -bool SocketComm::SendToPeer(Peer* peer, ChunkedIO::Chunk* c) - { - DEBUG_COMM(fmt("child: (->peer) chunk of size %d to #%" PRI_SOURCE_ID, c->len, peer->id)); - if ( ! sendToIO(peer->io, c) ) - { - Error(fmt("child: write error %s", io->Error()), peer); - return false; - } - - return true; - } diff --git a/src/RemoteSerializer.h b/src/RemoteSerializer.h deleted file mode 100644 index 28ca495f17..0000000000 --- a/src/RemoteSerializer.h +++ /dev/null @@ -1,525 +0,0 @@ -// Communication between two Bro's. - -#ifndef REMOTE_SERIALIZER -#define REMOTE_SERIALIZER - -#include "Dict.h" -#include "List.h" -#include "Serializer.h" -#include "iosource/IOSource.h" -#include "Stats.h" -#include "File.h" -#include "logging/WriterBackend.h" - -#include -#include - -class IncrementalSendTimer; - -namespace threading { - struct Field; - struct Value; -} - -// This class handles the communication done in Bro's main loop. -class RemoteSerializer : public Serializer, public iosource::IOSource { -public: - RemoteSerializer(); - ~RemoteSerializer() override; - - // Initialize the remote serializer (calling this will fork). - void Enable(); - - // FIXME: Use SourceID directly (or rename everything to Peer*). - typedef SourceID PeerID; - static const PeerID PEER_LOCAL = SOURCE_LOCAL; - static const PeerID PEER_NONE = SOURCE_LOCAL; - - // Connect to host (returns PEER_NONE on error). - PeerID Connect(const IPAddr& ip, const string& zone_id, uint16 port, - const char* our_class, double retry, bool use_ssl); - - // Close connection to host. - bool CloseConnection(PeerID peer); - - // Request all events matching pattern from remote side. - bool RequestEvents(PeerID peer, RE_Matcher* pattern); - - // Request synchronization of IDs with remote side. If auth is true, - // we consider our current state to authoritative and send it to - // the peer right after the handshake. - bool RequestSync(PeerID peer, bool auth); - - // Requests logs from the remote side. - bool RequestLogs(PeerID id); - - // Sets flag whether we're accepting state from this peer - // (default: yes). - bool SetAcceptState(PeerID peer, bool accept); - - // Sets compression level (0-9, 0 is defaults and means no compression) - bool SetCompressionLevel(PeerID peer, int level); - - // Signal the other side that we have finished our part of - // the initial handshake. - bool CompleteHandshake(PeerID peer); - - // Start to listen. - bool Listen(const IPAddr& ip, uint16 port, bool expect_ssl, bool ipv6, - const string& zone_id, double retry); - - // Stop it. - bool StopListening(); - - // Broadcast the event/function call. - bool SendCall(SerialInfo* info, const char* name, val_list* vl); - - // Send the event/function call (only if handshake completed). - bool SendCall(SerialInfo* info, PeerID peer, const char* name, val_list* vl); - - // Broadcasts the access (only if handshake completed). - bool SendAccess(SerialInfo* info, const StateAccess& access); - - // Send the access. - bool SendAccess(SerialInfo* info, PeerID pid, const StateAccess& access); - - // Sends ID. - bool SendID(SerialInfo* info, PeerID peer, const ID& id); - - // Sends the internal connection state. - bool SendConnection(SerialInfo* info, PeerID peer, const Connection& c); - - // Send capture filter. - bool SendCaptureFilter(PeerID peer, const char* filter); - - // Send packet. - bool SendPacket(SerialInfo* info, PeerID peer, const Packet& p); - - // Broadcast packet. - bool SendPacket(SerialInfo* info, const Packet& p); - - // Broadcast ping. - bool SendPing(PeerID peer, uint32 seq); - - // Broadcast remote print. - bool SendPrintHookEvent(BroFile* f, const char* txt, size_t len); - - // Send a request to create a writer on a remote side. - bool SendLogCreateWriter(PeerID peer, EnumVal* id, EnumVal* writer, const logging::WriterBackend::WriterInfo& info, int num_fields, const threading::Field* const * fields); - - // Broadcasts a request to create a writer. - bool SendLogCreateWriter(EnumVal* id, EnumVal* writer, const logging::WriterBackend::WriterInfo& info, int num_fields, const threading::Field* const * fields); - - // Broadcast a log entry to everybody interested. - bool SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Value* const * vals); - - // Synchronzizes time with all connected peers. Returns number of - // current sync-point, or -1 on error. - uint32 SendSyncPoint(); - void SendFinalSyncPoint(); - - // Registers the ID to be &synchronized. - void Register(ID* id); - void Unregister(ID* id); - - // Stop/restart propagating state updates. - void SuspendStateUpdates() { --propagate_accesses; } - void ResumeStateUpdates() { ++propagate_accesses; } - - // Check for incoming events and queue them. - bool Poll(bool may_block); - - // Returns the corresponding record (already ref'ed). - RecordVal* GetPeerVal(PeerID id); - - // Log some statistics. - void LogStats(); - - // Tries to sent out all remaining data. - // FIXME: Do we still need this? - void Finish(); - - // Overidden from IOSource: - void GetFds(iosource::FD_Set* read, iosource::FD_Set* write, - iosource::FD_Set* except) override; - double NextTimestamp(double* local_network_time) override; - void Process() override; - TimerMgr::Tag* GetCurrentTag() override; - const char* Tag() override { return "RemoteSerializer"; } - - // Gracefully finishes communication by first making sure that all - // remaining data (parent & child) has been sent out. - virtual bool Terminate(); - -#ifdef DEBUG_COMMUNICATION - // Dump data recently read/written into files. - void DumpDebugData(); - - // Read dump file and interpret as message block. - void ReadDumpAsMessageType(const char* file); - - // Read dump file and interpret as serialization. - void ReadDumpAsSerialization(const char* file); -#endif - - enum LogLevel { LogInfo = 1, LogError = 2, }; - static void Log(LogLevel level, const char* msg); - -protected: - friend class PersistenceSerializer; - friend class IncrementalSendTimer; - - // Maximum size of serialization caches. - static const unsigned int MAX_CACHE_SIZE = 3000; - - // When syncing traces in pseudo-realtime mode, we wait this many - // seconds after the final sync-point to make sure that all - // remaining I/O gets propagated. - static const unsigned int FINAL_SYNC_POINT_DELAY = 5; - - declare(PList, EventHandler); - typedef PList(EventHandler) handler_list; - - struct Peer { - PeerID id; // Unique ID (non-zero) per peer. - - IPAddr ip; - - uint16 port; - handler_list handlers; - RecordVal* val; // Record of type event_source. - SerializationCache* cache_in; // One cache for each direction. - SerializationCache* cache_out; - - // TCP-level state of the connection to the peer. - // State of the connection to the peer. - enum { INIT, PENDING, CONNECTED, CLOSING, CLOSED } state; - - // Current protocol phase of the connection (see RemoteSerializer.cc) - enum { UNKNOWN, SETUP, HANDSHAKE, SYNC, RUNNING } phase; - - // Capabilities. - static const int COMPRESSION = 1; - static const int NO_CACHING = 2; - static const int PID_64BIT = 4; - static const int NEW_CACHE_STRATEGY = 8; - static const int BROCCOLI_PEER = 16; - - // Constants to remember to who did something. - static const int NONE = 0; - static const int WE = 1; - static const int PEER = 2; - static const int BOTH = WE | PEER; - - static const int AUTH_WE = 4; - static const int AUTH_PEER = 8; - - int sent_version; // Who has sent the VERSION. - int handshake_done; // Who finished its handshake phase. - int sync_requested; // Who requested sync'ed state. - - bool orig; // True if we connected to the peer. - bool accept_state; // True if we accept state from peer. - bool send_state; // True if we're supposed to initially sent our state. - int comp_level; // Compression level. - bool logs_requested; // True if the peer has requested logs. - - // True if this peer triggered a net_suspend_processing(). - bool suspended_processing; - - uint32 caps; // Capabilities announced by peer. - int runtime; // Runtime we got from the peer. - int our_runtime; // Our runtime as we told it to this peer. - string peer_class; // Class from peer ("" = no class). - string our_class; // Class we send the peer. - uint32 sync_point; // Highest sync-point received so far - char* print_buffer; // Buffer for remote print or null. - int print_buffer_used; // Number of bytes used in buffer. - char* log_buffer; // Buffer for remote log or null. - int log_buffer_used; // Number of bytes used in buffer. - }; - - // Shuts down remote serializer. - void FatalError(const char* msg); - - enum LogSrc { LogChild = 1, LogParent = 2, LogScript = 3, }; - - static void Log(LogLevel level, const char* msg, Peer* peer, LogSrc src = LogParent); - - void ReportError(const char* msg) override; - - void GotEvent(const char* name, double time, - EventHandlerPtr event, val_list* args) override; - void GotFunctionCall(const char* name, double time, - Func* func, val_list* args) override; - void GotID(ID* id, Val* val) override; - void GotStateAccess(StateAccess* s) override; - void GotTimer(Timer* t) override; - void GotConnection(Connection* c) override; - void GotPacket(Packet* packet) override; - - void Fork(); - - bool DoMessage(); - bool ProcessConnected(); - bool ProcessSerialization(); - bool ProcessRequestEventsMsg(); - bool ProcessRequestSyncMsg(); - bool ProcessVersionMsg(); - bool ProcessLogMsg(bool is_error); - bool ProcessStatsMsg(); - bool ProcessCaptureFilterMsg(); - bool ProcessPhaseDone(); - bool ProcessPingMsg(); - bool ProcessPongMsg(); - bool ProcessCapsMsg(); - bool ProcessSyncPointMsg(); - bool ProcessRemotePrint(); - bool ProcessLogCreateWriter(); - bool ProcessLogWrite(); - bool ProcessRequestLogs(); - - Peer* AddPeer(const IPAddr& ip, uint16 port, PeerID id = PEER_NONE); - Peer* LookupPeer(PeerID id, bool only_if_connected); - void RemovePeer(Peer* peer); - bool IsConnectedPeer(PeerID id); - void PeerDisconnected(Peer* peer); - void PeerConnected(Peer* peer); - RecordVal* MakePeerVal(Peer* peer); - bool HandshakeDone(Peer* peer); - bool IsActive(); - void SetupSerialInfo(SerialInfo* info, Peer* peer); - bool CheckSyncPoints(); - void SendSyncPoint(uint32 syncpoint); - bool PropagateAccesses() - { - return ignore_accesses ? - propagate_accesses > 1 : propagate_accesses > 0; - } - - bool CloseConnection(Peer* peer); - - bool SendAllSynchronized(Peer* peer, SerialInfo* info); - bool SendCall(SerialInfo* info, Peer* peer, const char* name, val_list* vl); - bool SendAccess(SerialInfo* info, Peer* peer, const StateAccess& access); - bool SendID(SerialInfo* info, Peer* peer, const ID& id); - bool SendCapabilities(Peer* peer); - bool SendPacket(SerialInfo* info, Peer* peer, const Packet& p); - bool SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Value* const * vals); - - void UnregisterHandlers(Peer* peer); - void RaiseEvent(EventHandlerPtr event, Peer* peer, const char* arg = 0); - bool EnterPhaseRunning(Peer* peer); - bool FlushPrintBuffer(Peer* p); - bool FlushLogBuffer(Peer* p); - - void ChildDied(); - void InternalCommError(const char* msg); - - // Communication helpers - bool SendCMsgToChild(char msg_type, Peer* peer); - bool SendToChild(char type, Peer* peer, char* str, int len = -1, - bool delete_with_free = false); - bool SendToChild(char type, Peer* peer, int nargs, ...); // can send uints32 only - bool SendToChild(ChunkedIO::Chunk* c); - - void SetSocketBufferSize(int fd, int opt, const char *what, int size, int verbose); - -private: - enum { TYPE, ARGS } msgstate; // current state of reading comm. - Peer* current_peer; - PeerID current_id; - char current_msgtype; - ChunkedIO::Chunk* current_args; - double last_flush; - - id_list sync_ids; - - // FIXME: Check which of these are necessary... - bool initialized; - bool listening; - int propagate_accesses; - bool ignore_accesses; - bool terminating; - int received_logs; - Peer* source_peer; - PeerID id_counter; // Keeps track of assigned IDs. - uint32 current_sync_point; - bool syncing_times; - - declare(PList, Peer); - typedef PList(Peer) peer_list; - peer_list peers; - - Peer* in_sync; // Peer we're currently syncing state with. - peer_list sync_pending; // List of peers waiting to sync state. - - // Event buffer - struct BufferedEvent { - time_t time; - PeerID src; - EventHandlerPtr handler; - val_list* args; - }; - - declare(PList, BufferedEvent); - typedef PList(BufferedEvent) EventQueue; - EventQueue events; - - // Packet buffer - struct BufferedPacket { - time_t time; - Packet* p; - }; - - declare(PList, BufferedPacket); - typedef PList(BufferedPacket) PacketQueue; - PacketQueue packets; - - // Some stats - struct Statistics { - struct Pair { - Pair() : in(0), out(0) {} - unsigned long in; - unsigned long out; - }; - - Pair events; // actually events and function calls - Pair accesses; - Pair conns; - Pair packets; - Pair ids; - } stats; - -}; - -// This class handles the communication done in the forked child. -class SocketComm { -public: - SocketComm(); - ~SocketComm(); - - void SetParentIO(ChunkedIO* arg_io) { io = arg_io; } - - void Run(); // does not return - - // Log some statistics (via pipe to parent). - bool LogStats(); - - // Log CPU usage (again via pipe to parent). - bool LogProf(); - -protected: - struct Peer { - Peer() - { - id = 0; - io = 0; - port = 0; - state = 0; - connected = false; - ssl = false; - retry = 0; - next_try = 0; - compressor = false; - } - - RemoteSerializer::PeerID id; - ChunkedIO* io; - IPAddr ip; - string zone_id; - uint16 port; - char state; - bool connected; - bool ssl; - // If we get disconnected, reconnect after this many seconds. - int retry; - // Time of next connection attempt (0 if none). - time_t next_try; - // True if io is a CompressedChunkedIO. - bool compressor; - }; - - bool Listen(); - bool AcceptConnection(int listen_fd); - bool Connect(Peer* peer); - bool CloseConnection(Peer* peer, bool reconnect); - - Peer* LookupPeer(RemoteSerializer::PeerID id, bool only_if_connected); - - bool ProcessRemoteMessage(Peer* peer); - bool ProcessParentMessage(); - bool DoParentMessage(); - - bool ProcessListen(); - bool ProcessConnectTo(); - bool ProcessCompress(); - - void Log(const char* msg, Peer* peer = 0); - - // The connection to the peer will be closed. - bool Error(const char* msg, Peer* peer); - - // If kill is true, this is a fatal error and we kill ourselves. - void Error(const char* msg, bool kill = false); - - // Kill the current process. - void Kill(); - - // Check whether everything has been sent out. - void CheckFinished(); - - // Reports the error and terminates the process. - void InternalError(const char* msg); - - // Communication helpers. - bool SendToParent(char type, Peer* peer, const char* str, int len = -1); - bool SendToParent(char type, Peer* peer, int nargs, ...); // can send uints32 only - bool SendToParent(ChunkedIO::Chunk* c); - bool SendToPeer(Peer* peer, char type, const char* str, int len = -1); - bool SendToPeer(Peer* peer, char type, int nargs, ...); // can send uints32 only - bool SendToPeer(Peer* peer, ChunkedIO::Chunk* c); - bool ProcessParentCompress(); - bool ProcessPeerCompress(Peer* peer); - bool ForwardChunkToParent(Peer* p, ChunkedIO::Chunk* c); - bool ForwardChunkToPeer(); - const char* MakeLogString(const char* msg, Peer *peer); - - // Closes all file descriptors associated with listening sockets. - void CloseListenFDs(); - - // Peers we are communicating with: - declare(PList, Peer); - typedef PList(Peer) peer_list; - - RemoteSerializer::PeerID id_counter; - peer_list peers; - - ChunkedIO* io; // I/O to parent - - // Current state of reading from parent. - enum { TYPE, ARGS } parent_msgstate; - Peer* parent_peer; - RemoteSerializer::PeerID parent_id; - char parent_msgtype; - ChunkedIO::Chunk* parent_args; - - vector listen_fds; - - // If the port we're trying to bind to is already in use, we will retry - // it regularly. - string listen_if; - string listen_zone_id; // RFC 4007 IPv6 zone_id - uint16 listen_port; - bool listen_ssl; // use SSL for IO - bool enable_ipv6; // allow IPv6 listen sockets - uint32 bind_retry_interval; // retry interval for already-in-use sockets - time_t listen_next_try; // time at which to try another bind - bool shutting_conns_down; - bool terminating; - bool killing; -}; - -extern RemoteSerializer* remote_serializer; - -#endif diff --git a/src/SerialInfo.h b/src/SerialInfo.h index de2d9eeb61..294c5747ba 100644 --- a/src/SerialInfo.h +++ b/src/SerialInfo.h @@ -3,6 +3,8 @@ #ifndef serialinfo_h #define serialinfo_h +#include "ChunkedIO.h" + class SerialInfo { public: SerialInfo(Serializer* arg_s) @@ -15,7 +17,6 @@ public: pid_32bit = false; include_locations = true; new_cache_strategy = false; - broccoli_peer = false; } SerialInfo(const SerialInfo& info) @@ -30,7 +31,6 @@ public: pid_32bit = info.pid_32bit; include_locations = info.include_locations; new_cache_strategy = info.new_cache_strategy; - broccoli_peer = info.broccoli_peer; } // Parameters that control serialization. @@ -49,11 +49,6 @@ public: // If true, we support keeping objs in cache permanently. bool new_cache_strategy; - // If true, we're connecting to a Broccoli. If so, serialization - // specifics may be adapted for functionality Broccoli does not - // support. - bool broccoli_peer; - ChunkedIO::Chunk* chunk; // chunk written right before the serialization // Attributes set during serialization. @@ -78,7 +73,6 @@ public: print = 0; pid_32bit = false; new_cache_strategy = false; - broccoli_peer = false; } UnserialInfo(const UnserialInfo& info) @@ -95,7 +89,6 @@ public: print = info.print; pid_32bit = info.pid_32bit; new_cache_strategy = info.new_cache_strategy; - broccoli_peer = info.broccoli_peer; } // Parameters that control unserialization. @@ -116,11 +109,6 @@ public: // If true, we support keeping objs in cache permanently. bool new_cache_strategy; - // If true, we're connecting to a Broccoli. If so, serialization - // specifics may be adapted for functionality Broccoli does not - // support. - bool broccoli_peer; - // If a global ID already exits, of these policies is used. enum { Keep, // keep the old ID and ignore the new diff --git a/src/Serializer.cc b/src/Serializer.cc index 2c32283c56..28dc6bbd01 100644 --- a/src/Serializer.cc +++ b/src/Serializer.cc @@ -18,7 +18,6 @@ #include "NetVar.h" #include "Conn.h" #include "Timer.h" -#include "RemoteSerializer.h" #include "iosource/Manager.h" Serializer::Serializer(SerializationFormat* arg_format) @@ -508,8 +507,6 @@ bool Serializer::UnserializeConnection(UnserialInfo* info) if ( info->install_conns ) { - if ( c->IsPersistent() && c->Key() ) - persistence_serializer->Register(c); Ref(c); sessions->Insert(c); } diff --git a/src/Sessions.cc b/src/Sessions.cc index 3507c46e53..f2d6e27219 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -1101,9 +1101,6 @@ void NetSessions::Remove(Connection* c) tcp_stats.StateLeft(to->state, tr->state); } - if ( c->IsPersistent() ) - persistence_serializer->Unregister(c); - c->Done(); if ( connection_state_remove ) @@ -1194,8 +1191,6 @@ void NetSessions::Insert(Connection* c) // Some clean-ups similar to those in Remove() (but invisible // to the script layer). old->CancelTimers(); - if ( old->IsPersistent() ) - persistence_serializer->Unregister(old); delete old->Key(); old->ClearKey(); Unref(old); diff --git a/src/Sessions.h b/src/Sessions.h index b237428d25..880182c7cd 100644 --- a/src/Sessions.h +++ b/src/Sessions.h @@ -180,7 +180,6 @@ public: analyzer::tcp::TCPStateStats tcp_stats; // keeps statistics on TCP states protected: - friend class RemoteSerializer; friend class ConnCompressor; friend class TimerMgrExpireTimer; friend class IPTunnelTimer; diff --git a/src/StateAccess.cc b/src/StateAccess.cc index 72ed9ef236..134cca5db5 100644 --- a/src/StateAccess.cc +++ b/src/StateAccess.cc @@ -4,8 +4,6 @@ #include "Event.h" #include "NetVar.h" #include "DebugLogger.h" -#include "RemoteSerializer.h" -#include "PersistenceSerializer.h" int StateAccess::replaying = 0; @@ -135,100 +133,6 @@ void StateAccess::RefThem() Ref(op3); } -bool StateAccess::CheckOld(const char* op, ID* id, Val* index, - Val* should, Val* is) - { - if ( ! remote_check_sync_consistency ) - return true; - - if ( ! should && ! is ) - return true; - - // 'should == index' means that 'is' should be non-nil. - if ( should == index && is ) - return true; - - if ( should && is ) - { - // There's no general comparison for non-atomic vals currently. - if ( ! (is_atomic_val(is) && is_atomic_val(should)) ) - return true; - - if ( same_atomic_val(should, is) ) - return true; - } - - Val* arg1; - Val* arg2; - Val* arg3; - - if ( index ) - { - ODesc d; - d.SetShort(); - index->Describe(&d); - arg1 = new StringVal(fmt("%s[%s]", id->Name(), d.Description())); - } - else - arg1 = new StringVal(id->Name()); - - if ( should ) - { - ODesc d; - d.SetShort(); - should->Describe(&d); - arg2 = new StringVal(d.Description()); - } - else - arg2 = new StringVal(""); - - if ( is ) - { - ODesc d; - d.SetShort(); - is->Describe(&d); - arg3 = new StringVal(d.Description()); - } - else - arg3 = new StringVal(""); - - mgr.QueueEvent(remote_state_inconsistency, { - new StringVal(op), - arg1, - arg2, - arg3, - }); - - return false; - } - -bool StateAccess::CheckOldSet(const char* op, ID* id, Val* index, - bool should, bool is) - { - if ( ! remote_check_sync_consistency ) - return true; - - if ( should == is ) - return true; - - ODesc d; - d.SetShort(); - index->Describe(&d); - - Val* arg1 = new StringVal(fmt("%s[%s]", id->Name(), d.Description())); - Val* arg2 = new StringVal(should ? "set" : "not set"); - Val* arg3 = new StringVal(is ? "set" : "not set"); - - mgr.QueueEvent(remote_state_inconsistency, { - new StringVal(op), - arg1, - arg2, - arg3, - }); - - return false; - } - bool StateAccess::MergeTables(TableVal* dst, Val* src) { if ( src->Type()->Tag() != TYPE_TABLE ) @@ -287,7 +191,6 @@ void StateAccess::Replay() assert(op1.val); // There mustn't be a direct assignment to a unique ID. assert(target.id->Name()[0] != '#'); - CheckOld("assign", target.id, 0, op2, v); if ( t == TYPE_TABLE && v && v->AsTableVal()->FindAttr(ATTR_MERGEABLE) ) @@ -329,9 +232,6 @@ void StateAccess::Replay() break; } - CheckOld("index assign", target.id, op1.val, op3, - v->AsTableVal()->Lookup(op1.val)); - v->AsTableVal()->Assign(op1.val, op2 ? op2->Ref() : 0); } @@ -353,8 +253,6 @@ void StateAccess::Replay() break; } - CheckOld("index assign", target.id, op1.val, op3, - v->AsRecordVal()->Lookup(idx)); v->AsRecordVal()->Assign(idx, op2 ? op2->Ref() : 0); } else @@ -377,8 +275,6 @@ void StateAccess::Replay() break; } - CheckOld("index assign", target.id, op1.val, op3, - v->AsVectorVal()->Lookup(index)); v->AsVectorVal()->Assign(index, op2 ? op2->Ref() : 0); } @@ -442,8 +338,6 @@ void StateAccess::Replay() assert(op1.val); if ( t == TYPE_TABLE ) { - CheckOldSet("add", target.id, op1.val, op2 != 0, - v->AsTableVal()->Lookup(op1.val) != 0); v->AsTableVal()->Assign(op1.val, 0); } break; @@ -452,13 +346,6 @@ void StateAccess::Replay() assert(op1.val); if ( t == TYPE_TABLE ) { - if ( v->Type()->AsTableType()->IsSet() ) - CheckOldSet("delete", target.id, op1.val, op2 != 0, - v->AsTableVal()->Lookup(op1.val) != 0); - else - CheckOld("delete", target.id, op1.val, op2, - v->AsTableVal()->Lookup(op1.val)); - Unref(v->AsTableVal()->Delete(op1.val)); } break; @@ -477,14 +364,8 @@ void StateAccess::Replay() // are performed in the expire_func. StateAccess::ResumeReplay(); - if ( remote_serializer ) - remote_serializer->ResumeStateUpdates(); - tv->CallExpireFunc(op1.val->Ref()); - if ( remote_serializer ) - remote_serializer->SuspendStateUpdates(); - StateAccess::SuspendReplay(); Unref(tv->AsTableVal()->Delete(op1.val)); @@ -507,20 +388,7 @@ void StateAccess::Replay() // Update the timestamp if we have a read_expire. if ( tv->FindAttr(ATTR_EXPIRE_READ) ) { - if ( ! tv->UpdateTimestamp(op1.val) && - remote_check_sync_consistency ) - { - ODesc d; - d.SetShort(); - op1.val->Describe(&d); - - mgr.QueueEvent(remote_state_inconsistency, { - new StringVal("read"), - new StringVal(fmt("%s[%s]", target.id->Name(), d.Description())), - new StringVal("existent"), - new StringVal("not existent"), - }); - } + tv->UpdateTimestamp(op1.val); } } else @@ -533,14 +401,6 @@ void StateAccess::Replay() } --replaying; - - if ( remote_state_access_performed ) - { - mgr.QueueEventFast(remote_state_access_performed, { - new StringVal(target.id->Name()), - target.id->ID_Val()->Ref(), - }); - } } ID* StateAccess::Target() const @@ -597,50 +457,41 @@ bool StateAccess::DoSerialize(SerialInfo* info) const const Val* null = 0; - if ( remote_check_sync_consistency ) - { + switch ( opcode ) { + case OP_PRINT: + case OP_EXPIRE: + case OP_READ_IDX: + // No old. + SERIALIZE_OPTIONAL(null); + SERIALIZE_OPTIONAL(null); + break; + + case OP_INCR: + case OP_INCR_IDX: + // Always need old. SERIALIZE_OPTIONAL(op2); SERIALIZE_OPTIONAL(op3); - } + break; - else - { - switch ( opcode ) { - case OP_PRINT: - case OP_EXPIRE: - case OP_READ_IDX: - // No old. - SERIALIZE_OPTIONAL(null); - SERIALIZE_OPTIONAL(null); - break; + case OP_ASSIGN: + case OP_ADD: + case OP_DEL: + // Op2 is old. + SERIALIZE_OPTIONAL(null); + SERIALIZE_OPTIONAL(null); + break; - case OP_INCR: - case OP_INCR_IDX: - // Always need old. - SERIALIZE_OPTIONAL(op2); - SERIALIZE_OPTIONAL(op3); - break; + case OP_ASSIGN_IDX: + // Op3 is old. + SERIALIZE_OPTIONAL(op2); + SERIALIZE_OPTIONAL(null); + break; - case OP_ASSIGN: - case OP_ADD: - case OP_DEL: - // Op2 is old. - SERIALIZE_OPTIONAL(null); - SERIALIZE_OPTIONAL(null); - break; + default: + reporter->InternalError("StateAccess::DoSerialize: unknown opcode"); + } - case OP_ASSIGN_IDX: - // Op3 is old. - SERIALIZE_OPTIONAL(op2); - SERIALIZE_OPTIONAL(null); - break; - - default: - reporter->InternalError("StateAccess::DoSerialize: unknown opcode"); - } - } - - return true; + return true; } bool StateAccess::DoUnserialize(UnserialInfo* info) @@ -870,57 +721,27 @@ void StateAccess::Describe(ODesc* d) const void StateAccess::Log(StateAccess* access) { - bool synchronized = false; - bool persistent = false; bool tracked = false; if ( access->target_type == TYPE_ID ) { - if ( access->target.id->FindAttr(ATTR_SYNCHRONIZED) ) - synchronized = true; - - if ( access->target.id->FindAttr(ATTR_PERSISTENT) ) - persistent = true; - if ( access->target.id->FindAttr(ATTR_TRACKED) ) tracked = true; } else { - if ( access->target.val->GetProperties() & MutableVal::SYNCHRONIZED ) - synchronized = true; - - if ( access->target.val->GetProperties() & MutableVal::PERSISTENT ) - persistent = true; - if ( access->target.val->GetProperties() & MutableVal::TRACKED ) tracked = true; } - if ( synchronized ) - { - if ( state_serializer ) - { - SerialInfo info(state_serializer); - state_serializer->Serialize(&info, *access); - } - - SerialInfo info(remote_serializer); - remote_serializer->SendAccess(&info, *access); - } - - if ( persistent && persistence_serializer->IsSerializationRunning() ) - persistence_serializer->LogAccess(*access); - if ( tracked ) notifiers.AccessPerformed(*access); #ifdef DEBUG ODesc desc; access->Describe(&desc); - DBG_LOG(DBG_STATE, "operation: %s%s [%s%s]", - desc.Description(), replaying > 0 ? " (replay)" : "", - persistent ? "P" : "", synchronized ? "S" : ""); + DBG_LOG(DBG_STATE, "operation: %s%s", + desc.Description(), replaying > 0 ? " (replay)" : ""); #endif delete access; diff --git a/src/StateAccess.h b/src/StateAccess.h index 1e84430956..8530ec1d91 100644 --- a/src/StateAccess.h +++ b/src/StateAccess.h @@ -74,8 +74,6 @@ private: StateAccess() { target.id = 0; op1.val = op2 = op3 = 0; } void RefThem(); - bool CheckOld(const char* op, ID* id, Val* index, Val* should, Val* is); - bool CheckOldSet(const char* op, ID* id, Val* index, bool should, bool is); bool MergeTables(TableVal* dst, Val* src); DECLARE_SERIAL(StateAccess); diff --git a/src/Stmt.cc b/src/Stmt.cc index 6dba9eb251..5bf7c47d75 100644 --- a/src/Stmt.cc +++ b/src/Stmt.cc @@ -14,7 +14,6 @@ #include "Debug.h" #include "Traverse.h" #include "Trigger.h" -#include "RemoteSerializer.h" const char* stmt_name(BroStmtTag t) { @@ -301,9 +300,6 @@ Val* PrintStmt::DoExec(val_list* vals, stmt_flow_type& /* flow */) const {new Val(f), new StringVal(d.Len(), d.Description())}), true); } - - if ( remote_serializer ) - remote_serializer->SendPrintHookEvent(f, d.Description(), d.Len()); } return 0; diff --git a/src/Timer.cc b/src/Timer.cc index 101733028c..519ceaae1e 100644 --- a/src/Timer.cc +++ b/src/Timer.cc @@ -20,8 +20,6 @@ const char* TimerNames[] = { "FileAnalysisInactivityTimer", "FlowWeirdTimer", "FragTimer", - "IncrementalSendTimer", - "IncrementalWriteTimer", "InterconnTimer", "IPTunnelInactivityTimer", "NetbiosExpireTimer", diff --git a/src/Timer.h b/src/Timer.h index 8d6de857a0..2ce9f56e0b 100644 --- a/src/Timer.h +++ b/src/Timer.h @@ -25,8 +25,6 @@ enum TimerType { TIMER_FILE_ANALYSIS_INACTIVITY, TIMER_FLOW_WEIRD_EXPIRE, TIMER_FRAG, - TIMER_INCREMENTAL_SEND, - TIMER_INCREMENTAL_WRITE, TIMER_INTERCONN, TIMER_IP_TUNNEL_INACTIVITY, TIMER_NB_EXPIRE, diff --git a/src/Val.cc b/src/Val.cc index 9bc53665fc..07ae251fc2 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -21,7 +21,6 @@ #include "NetVar.h" #include "Expr.h" #include "Serializer.h" -#include "RemoteSerializer.h" #include "PrefixTable.h" #include "Conn.h" #include "Reporter.h" @@ -1562,18 +1561,9 @@ int TableVal::Assign(Val* index, HashKey* k, Val* new_val, Opcode op) else { // A set. - if ( old_entry_val && remote_check_sync_consistency ) - { - Val* has_old_val = val_mgr->GetInt(1); - StateAccess::Log( - new StateAccess(OP_ADD, this, index, - has_old_val)); - Unref(has_old_val); - } - else - StateAccess::Log( - new StateAccess(OP_ADD, this, - index, 0, 0)); + StateAccess::Log( + new StateAccess(OP_ADD, this, + index, 0, 0)); } if ( rec_index ) @@ -2057,20 +2047,12 @@ Val* TableVal::Delete(const Val* index) { if ( v ) { - if ( v->Value() && remote_check_sync_consistency ) - // A table. - StateAccess::Log( - new StateAccess(OP_DEL, this, - index, v->Value())); - else - { - // A set. - Val* has_old_val = val_mgr->GetInt(1); - StateAccess::Log( - new StateAccess(OP_DEL, this, index, - has_old_val)); - Unref(has_old_val); - } + // A set. + Val* has_old_val = val_mgr->GetInt(1); + StateAccess::Log( + new StateAccess(OP_DEL, this, index, + has_old_val)); + Unref(has_old_val); } else StateAccess::Log( diff --git a/src/Val.h b/src/Val.h index 63e790848d..2d915bcc6f 100644 --- a/src/Val.h +++ b/src/Val.h @@ -524,9 +524,6 @@ public: // values. (In any case, don't forget to call the parent's method.) typedef char Properties; - static const int PERSISTENT = 0x01; - static const int SYNCHRONIZED = 0x02; - // Tracked by NotifierRegistry, not recursive. static const int TRACKED = 0x04; @@ -540,10 +537,10 @@ public: bool LoggingAccess() const { #ifndef DEBUG - return props & (SYNCHRONIZED|PERSISTENT|TRACKED); + return props & TRACKED; #else return debug_logger.IsVerbose() || - (props & (SYNCHRONIZED|PERSISTENT|TRACKED)); + (props & TRACKED); #endif } diff --git a/src/Var.cc b/src/Var.cc index fb27b7261f..3dd3d2702b 100644 --- a/src/Var.cc +++ b/src/Var.cc @@ -7,7 +7,6 @@ #include "Stmt.h" #include "Scope.h" #include "Serializer.h" -#include "RemoteSerializer.h" #include "EventRegistry.h" #include "Traverse.h" @@ -142,26 +141,6 @@ static void make_var(ID* id, BroType* t, init_class c, Expr* init, } } - if ( id->FindAttr(ATTR_PERSISTENT) || id->FindAttr(ATTR_SYNCHRONIZED) ) - { - if ( dt == VAR_CONST ) - { - id->Error("&persistent/synchronized with constant"); - return; - } - else if ( dt == VAR_OPTION ) - { - id->Error("&persistent/synchronized with option"); - return; - } - - if ( ! id->IsGlobal() ) - { - id->Error("&persistant/synchronized with non-global"); - return; - } - } - if ( do_init ) { if ( c == INIT_NONE && dt == VAR_REDEF && t->IsTable() && diff --git a/src/analyzer/protocol/krb/KRB.h b/src/analyzer/protocol/krb/KRB.h index 7eee46d838..6a6af93c45 100644 --- a/src/analyzer/protocol/krb/KRB.h +++ b/src/analyzer/protocol/krb/KRB.h @@ -9,6 +9,8 @@ #include #endif +#include + namespace analyzer { namespace krb { class KRB_Analyzer : public analyzer::Analyzer { diff --git a/src/analyzer/protocol/tcp/TCP_Reassembler.cc b/src/analyzer/protocol/tcp/TCP_Reassembler.cc index 3a4654295d..fcd8237c55 100644 --- a/src/analyzer/protocol/tcp/TCP_Reassembler.cc +++ b/src/analyzer/protocol/tcp/TCP_Reassembler.cc @@ -1,5 +1,6 @@ #include +#include "File.h" #include "analyzer/Analyzer.h" #include "TCP_Reassembler.h" #include "analyzer/protocol/tcp/TCP.h" diff --git a/src/analyzer/protocol/tcp/functions.bif b/src/analyzer/protocol/tcp/functions.bif index 4aa218991e..c74c7ef9b5 100644 --- a/src/analyzer/protocol/tcp/functions.bif +++ b/src/analyzer/protocol/tcp/functions.bif @@ -1,5 +1,6 @@ %%{ +#include "File.h" #include "analyzer/protocol/tcp/TCP.h" %%} diff --git a/src/bro.bif b/src/bro.bif index f684378557..c3a9f13d56 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -4540,7 +4540,7 @@ function get_file_name%(f: file%): string ## after the rotation, and the time when *f* was opened/closed. ## ## .. zeek:see:: rotate_file_by_name calc_next_rotate -function rotate_file%(f: file%): rotate_info +function rotate_file%(f: file%): rotate_info &deprecated %{ RecordVal* info = f->Rotate(); if ( info ) @@ -4564,7 +4564,7 @@ function rotate_file%(f: file%): rotate_info ## after the rotation, and the time when *f* was opened/closed. ## ## .. zeek:see:: rotate_file calc_next_rotate -function rotate_file_by_name%(f: string%): rotate_info +function rotate_file_by_name%(f: string%): rotate_info &deprecated %{ RecordVal* info = new RecordVal(rotate_info); @@ -4618,7 +4618,7 @@ function rotate_file_by_name%(f: string%): rotate_info ## Returns: The duration until the next file rotation time. ## ## .. zeek:see:: rotate_file rotate_file_by_name -function calc_next_rotate%(i: interval%) : interval +function calc_next_rotate%(i: interval%) : interval &deprecated %{ const char* base_time = log_rotate_base_time ? log_rotate_base_time->AsString()->CheckString() : 0; diff --git a/src/broker/Data.cc b/src/broker/Data.cc index 754a51390b..849bad5d9b 100644 --- a/src/broker/Data.cc +++ b/src/broker/Data.cc @@ -1,4 +1,5 @@ #include "Data.h" +#include "File.h" #include "broker/data.bif.h" #include #include diff --git a/src/broker/Manager.h b/src/broker/Manager.h index a0520698da..6c1040f989 100644 --- a/src/broker/Manager.h +++ b/src/broker/Manager.h @@ -13,6 +13,7 @@ #include "Reporter.h" #include "iosource/IOSource.h" #include "Val.h" +#include "logging/WriterBackend.h" namespace bro_broker { diff --git a/src/event.bif b/src/event.bif index bb6962e5a5..fd432feb84 100644 --- a/src/event.bif +++ b/src/event.bif @@ -600,188 +600,6 @@ event software_unparsed_version_found%(c: connection, host: addr, str: string%); ## generate_OS_version_event event OS_version_found%(c: connection, host: addr, OS: OS_version%); -## Generated when a connection to a remote Bro has been established. This event -## is intended primarily for use by Bro's communication framework, but it can -## also trigger additional code if helpful. -## -## p: A record describing the peer. -## -## .. zeek:see:: remote_capture_filter remote_connection_closed remote_connection_error -## remote_connection_handshake_done remote_event_registered remote_log remote_pong -## remote_state_access_performed remote_state_inconsistency print_hook -event remote_connection_established%(p: event_peer%); - -## Generated when a connection to a remote Bro has been closed. This event is -## intended primarily for use by Bro's communication framework, but it can -## also trigger additional code if helpful. -## -## p: A record describing the peer. -## -## .. zeek:see:: remote_capture_filter remote_connection_error -## remote_connection_established remote_connection_handshake_done -## remote_event_registered remote_log remote_pong remote_state_access_performed -## remote_state_inconsistency print_hook -event remote_connection_closed%(p: event_peer%); - -## Generated when a remote connection's initial handshake has been completed. -## This event is intended primarily for use by Bro's communication framework, -## but it can also trigger additional code if helpful. -## -## p: A record describing the peer. -## -## .. zeek:see:: remote_capture_filter remote_connection_closed remote_connection_error -## remote_connection_established remote_event_registered remote_log remote_pong -## remote_state_access_performed remote_state_inconsistency print_hook -event remote_connection_handshake_done%(p: event_peer%); - -## Generated for each event registered by a remote peer. This event is intended -## primarily for use by Bro's communication framework, but it can also trigger -## additional code if helpful. -## -## p: A record describing the peer. -## -## name: TODO. -## -## .. zeek:see:: remote_capture_filter remote_connection_closed -## remote_connection_error remote_connection_established -## remote_connection_handshake_done remote_log remote_pong -## remote_state_access_performed remote_state_inconsistency print_hook -event remote_event_registered%(p: event_peer, name: string%); - -## Generated when a connection to a remote Bro encountered an error. This event -## is intended primarily for use by Bro's communication framework, but it can -## also trigger additional code if helpful. -## -## p: A record describing the peer. -## -## reason: A textual description of the error. -## -## .. zeek:see:: remote_capture_filter remote_connection_closed -## remote_connection_established remote_connection_handshake_done -## remote_event_registered remote_log remote_pong remote_state_access_performed -## remote_state_inconsistency print_hook -event remote_connection_error%(p: event_peer, reason: string%); - -## Generated when a remote peer sent us a capture filter. While this event is -## intended primarily for use by Bro's communication framework, it can also -## trigger additional code if helpful. -## -## p: A record describing the peer. -## -## filter: The filter string sent by the peer. -## -## .. zeek:see:: remote_connection_closed remote_connection_error -## remote_connection_established remote_connection_handshake_done -## remote_event_registered remote_log remote_pong remote_state_access_performed -## remote_state_inconsistency print_hook -event remote_capture_filter%(p: event_peer, filter: string%); - -## Generated if state synchronization detects an inconsistency. While this -## event is intended primarily for use by Bro's communication framework, it can -## also trigger additional code if helpful. This event is only raised if -## :zeek:id:`remote_check_sync_consistency` is false. -## -## operation: The textual description of the state operation performed. -## -## id: The name of the Bro script identifier that was operated on. -## -## expected_old: A textual representation of the value of *id* that was -## expected to be found before the operation was carried out. -## -## real_old: A textual representation of the value of *id* that was actually -## found before the operation was carried out. The difference between -## *real_old* and *expected_old* is the inconsistency being reported. -## -## .. zeek:see:: remote_capture_filter remote_connection_closed -## remote_connection_error remote_connection_established -## remote_connection_handshake_done remote_event_registered remote_log remote_pong -## remote_state_access_performed print_hook remote_check_sync_consistency -event remote_state_inconsistency%(operation: string, id: string, - expected_old: string, real_old: string%); - -## Generated for communication log messages. While this event is -## intended primarily for use by Bro's communication framework, it can also -## trigger additional code if helpful. -## -## level: The log level, which is either :zeek:id:`REMOTE_LOG_INFO` or -## :zeek:id:`REMOTE_LOG_ERROR`. -## -## src: The component of the communication system that logged the message. -## Currently, this will be one of :zeek:id:`REMOTE_SRC_CHILD` (Bro's -## child process), :zeek:id:`REMOTE_SRC_PARENT` (Bro's main process), or -## :zeek:id:`REMOTE_SRC_SCRIPT` (the script level). -## -## msg: The message logged. -## -## .. zeek:see:: remote_capture_filter remote_connection_closed remote_connection_error -## remote_connection_established remote_connection_handshake_done -## remote_event_registered remote_pong remote_state_access_performed -## remote_state_inconsistency print_hook remote_log_peer -event remote_log%(level: count, src: count, msg: string%); - -## Generated for communication log messages. While this event is -## intended primarily for use by Bro's communication framework, it can also -## trigger additional code if helpful. This event is equivalent to -## :zeek:see:`remote_log` except the message is with respect to a certain peer. -## -## p: A record describing the remote peer. -## -## level: The log level, which is either :zeek:id:`REMOTE_LOG_INFO` or -## :zeek:id:`REMOTE_LOG_ERROR`. -## -## src: The component of the communication system that logged the message. -## Currently, this will be one of :zeek:id:`REMOTE_SRC_CHILD` (Bro's -## child process), :zeek:id:`REMOTE_SRC_PARENT` (Bro's main process), or -## :zeek:id:`REMOTE_SRC_SCRIPT` (the script level). -## -## msg: The message logged. -## -## .. zeek:see:: remote_capture_filter remote_connection_closed remote_connection_error -## remote_connection_established remote_connection_handshake_done -## remote_event_registered remote_pong remote_state_access_performed -## remote_state_inconsistency print_hook remote_log -event remote_log_peer%(p: event_peer, level: count, src: count, msg: string%); - -## Generated when a remote peer has answered to our ping. This event is part of -## Bro's infrastructure for measuring communication latency. One can send a ping -## by calling ``send_ping`` and when a corresponding reply is received, -## this event will be raised. -## -## p: The peer sending us the pong. -## -## seq: The sequence number passed to the original ``send_ping`` call. -## The number is sent back by the peer in its response. -## -## d1: The time interval between sending the ping and receiving the pong. This -## is the latency of the complete path. -## -## d2: The time interval between sending out the ping to the network and its -## reception at the peer. This is the network latency. -## -## d3: The time interval between when the peer's child process received the -## ping and when its parent process sent the pong. This is the -## processing latency at the peer. -## -## .. zeek:see:: remote_capture_filter remote_connection_closed remote_connection_error -## remote_connection_established remote_connection_handshake_done -## remote_event_registered remote_log remote_state_access_performed -## remote_state_inconsistency print_hook -event remote_pong%(p: event_peer, seq: count, - d1: interval, d2: interval, d3: interval%); - -## Generated each time a remote state access has been replayed locally. This -## event is primarily intended for debugging. -## -## id: The name of the Bro script variable that's being operated on. -## -## v: The new value of the variable. -## -## .. zeek:see:: remote_capture_filter remote_connection_closed remote_connection_error -## remote_connection_established remote_connection_handshake_done -## remote_event_registered remote_log remote_pong remote_state_inconsistency -## print_hook -event remote_state_access_performed%(id: string, v: any%); - ## Generated each time Bro's internal profiling log is updated. The file is ## defined by :zeek:id:`profiling_file`, and its update frequency by ## :zeek:id:`profiling_interval` and :zeek:id:`expensive_profiling_multiple`. diff --git a/src/file_analysis/analyzer/extract/Extract.cc b/src/file_analysis/analyzer/extract/Extract.cc index e7aca5bcf3..8761c8493c 100644 --- a/src/file_analysis/analyzer/extract/Extract.cc +++ b/src/file_analysis/analyzer/extract/Extract.cc @@ -1,6 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. #include +#include #include "Extract.h" #include "util.h" diff --git a/src/input/Manager.h b/src/input/Manager.h index abbf8793b5..6b48f69ee4 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -7,7 +7,6 @@ #include "BroString.h" #include "EventHandler.h" -#include "RemoteSerializer.h" #include "Val.h" #include "Component.h" diff --git a/src/iosource/Packet.cc b/src/iosource/Packet.cc index 3bb6e34e50..54d1cc6f27 100644 --- a/src/iosource/Packet.cc +++ b/src/iosource/Packet.cc @@ -2,6 +2,8 @@ #include "Packet.h" #include "Sessions.h" #include "iosource/Manager.h" +#include "SerialInfo.h" +#include "Serializer.h" extern "C" { #ifdef HAVE_NET_ETHERNET_H diff --git a/src/iosource/PktSrc.cc b/src/iosource/PktSrc.cc index 343801ab7d..5f7d180cde 100644 --- a/src/iosource/PktSrc.cc +++ b/src/iosource/PktSrc.cc @@ -160,21 +160,6 @@ double PktSrc::CheckPseudoTime() if ( ! ExtractNextPacketInternal() ) return 0; - if ( remote_trace_sync_interval ) - { - if ( next_sync_point == 0 || current_packet.time >= next_sync_point ) - { - int n = remote_serializer->SendSyncPoint(); - next_sync_point = first_timestamp + - n * remote_trace_sync_interval; - remote_serializer->Log(RemoteSerializer::LogInfo, - fmt("stopping at packet %.6f, next sync-point at %.6f", - current_packet.time, next_sync_point)); - - return 0; - } - } - double pseudo_time = current_packet.time - first_timestamp; double ct = (current_time(true) - first_wallclock) * pseudo_realtime; @@ -308,15 +293,6 @@ bool PktSrc::ExtractNextPacketInternal() if ( pseudo_realtime && ! IsOpen() ) { - if ( using_communication ) - { - // Source has gone dry, we're done. - if ( remote_trace_sync_interval ) - remote_serializer->SendFinalSyncPoint(); - else - remote_serializer->Terminate(); - } - if ( broker_mgr->Active() ) iosource_mgr->Terminate(); } diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index 39496671a2..0fe75b91db 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -2,11 +2,12 @@ #include -#include "../Event.h" -#include "../EventHandler.h" -#include "../NetVar.h" -#include "../Net.h" -#include "../Type.h" +#include "Event.h" +#include "EventHandler.h" +#include "NetVar.h" +#include "Net.h" +#include "Type.h" +#include "File.h" #include "broker/Manager.h" #include "threading/Manager.h" @@ -16,8 +17,8 @@ #include "WriterFrontend.h" #include "WriterBackend.h" #include "logging.bif.h" -#include "../plugin/Plugin.h" -#include "../plugin/Manager.h" +#include "plugin/Plugin.h" +#include "plugin/Manager.h" using namespace logging; @@ -1300,32 +1301,6 @@ bool Manager::WriteFromRemote(EnumVal* id, EnumVal* writer, string path, int num return true; } -void Manager::SendAllWritersTo(RemoteSerializer::PeerID peer) - { - auto et = internal_type("Log::Writer")->AsEnumType(); - - for ( vector::iterator s = streams.begin(); s != streams.end(); ++s ) - { - Stream* stream = (*s); - - if ( ! (stream && stream->enable_remote) ) - continue; - - for ( Stream::WriterMap::iterator i = stream->writers.begin(); - i != stream->writers.end(); i++ ) - { - WriterFrontend* writer = i->second->writer; - auto writer_val = et->GetVal(i->first.first); - remote_serializer->SendLogCreateWriter(peer, (*s)->id, - writer_val, - *i->second->info, - writer->NumFields(), - writer->Fields()); - Unref(writer_val); - } - } - } - void Manager::SendAllWritersTo(const broker::endpoint_info& ei) { auto et = internal_type("Log::Writer")->AsEnumType(); diff --git a/src/logging/Manager.h b/src/logging/Manager.h index d04def7938..96ff2ea0c9 100644 --- a/src/logging/Manager.h +++ b/src/logging/Manager.h @@ -10,14 +10,12 @@ #include "../Val.h" #include "../Tag.h" #include "../EventHandler.h" -#include "../RemoteSerializer.h" #include "../plugin/ComponentManager.h" #include "Component.h" #include "WriterBackend.h" class SerializationFormat; -class RemoteSerializer; class RotationTimer; namespace logging { @@ -234,7 +232,6 @@ protected: friend class WriterFrontend; friend class RotationFinishedMessage; friend class RotationFailedMessage; - friend class ::RemoteSerializer; friend class ::RotationTimer; // Instantiates a new WriterBackend of the given type (note that @@ -248,9 +245,6 @@ protected: int num_fields, const threading::Field* const* fields, bool local, bool remote, bool from_remote, const string& instantiating_filter=""); - // Announces all instantiated writers to peer. - void SendAllWritersTo(RemoteSerializer::PeerID peer); - // Signals that a file has been rotated. bool FinishedRotation(WriterFrontend* writer, const char* new_name, const char* old_name, double open, double close, bool success, bool terminating); diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc index 4416e41d17..7bede8f6e6 100644 --- a/src/logging/WriterBackend.cc +++ b/src/logging/WriterBackend.cc @@ -4,6 +4,7 @@ #include "util.h" #include "threading/SerialTypes.h" +#include "SerializationFormat.h" #include "Manager.h" #include "WriterBackend.h" diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h index 74541d8586..187a1957d7 100644 --- a/src/logging/WriterBackend.h +++ b/src/logging/WriterBackend.h @@ -9,8 +9,6 @@ #include "Component.h" -class RemoteSerializer; - namespace broker { class data; } namespace logging { diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc index 56bbf68161..fdc4a7a97b 100644 --- a/src/logging/WriterFrontend.cc +++ b/src/logging/WriterFrontend.cc @@ -169,12 +169,6 @@ void WriterFrontend::Init(int arg_num_fields, const Field* const * arg_fields) if ( remote ) { - remote_serializer->SendLogCreateWriter(stream, - writer, - *info, - arg_num_fields, - arg_fields); - broker_mgr->PublishLogCreate(stream, writer, *info, @@ -201,12 +195,6 @@ void WriterFrontend::Write(int arg_num_fields, Value** vals) if ( remote ) { - remote_serializer->SendLogWrite(stream, - writer, - info->path, - num_fields, - vals); - broker_mgr->PublishLogWrite(stream, writer, info->path, diff --git a/src/main.cc b/src/main.cc index 160e6fd1d3..c9e32ab4b4 100644 --- a/src/main.cc +++ b/src/main.cc @@ -39,8 +39,6 @@ extern "C" { #include "RuleMatcher.h" #include "Anon.h" #include "Serializer.h" -#include "RemoteSerializer.h" -#include "PersistenceSerializer.h" #include "EventRegistry.h" #include "Stats.h" #include "Brofiler.h" @@ -101,10 +99,8 @@ name_list prefixes; Stmt* stmts; EventHandlerPtr net_done = 0; RuleMatcher* rule_matcher = 0; -PersistenceSerializer* persistence_serializer = 0; FileSerializer* event_serializer = 0; FileSerializer* state_serializer = 0; -RemoteSerializer* remote_serializer = 0; EventPlayer* event_player = 0; EventRegistry* event_registry = 0; ProfileLogger* profiling_logger = 0; @@ -167,7 +163,6 @@ void usage(int code = 1) fprintf(stderr, " -d|--debug-policy | activate policy file debugging\n"); fprintf(stderr, " -e|--exec | augment loaded policies by given code\n"); fprintf(stderr, " -f|--filter | tcpdump filter\n"); - fprintf(stderr, " -g|--dump-config | dump current config into .state dir\n"); fprintf(stderr, " -h|--help | command line help\n"); fprintf(stderr, " -i|--iface | read from given interface\n"); fprintf(stderr, " -p|--prefix | add given prefix to policy file resolution\n"); @@ -275,10 +270,6 @@ void done_with_network() { set_processing_status("TERMINATING", "done_with_network"); - // Release the port, which is important for checkpointing Bro. - if ( remote_serializer ) - remote_serializer->StopListening(); - // Cancel any pending alarms (watchdog, in particular). (void) alarm(0); @@ -291,9 +282,6 @@ void done_with_network() true); } - // Save state before expiring the remaining events/timers. - persistence_serializer->WriteState(false); - if ( profiling_logger ) profiling_logger->Log(); @@ -305,9 +293,6 @@ void done_with_network() mgr.Drain(); mgr.Drain(); - if ( remote_serializer ) - remote_serializer->Finish(); - net_finish(1); #ifdef USE_PERFTOOLS_DEBUG @@ -355,9 +340,6 @@ void terminate_bro() delete profiling_logger; } - if ( remote_serializer ) - remote_serializer->LogStats(); - mgr.Drain(); log_mgr->Terminate(); @@ -371,7 +353,6 @@ void terminate_bro() delete zeekygen_mgr; delete timer_mgr; - delete persistence_serializer; delete event_serializer; delete state_serializer; delete event_registry; @@ -452,7 +433,6 @@ int main(int argc, char** argv) char* debug_streams = 0; int parse_only = false; int bare_mode = false; - int dump_cfg = false; int do_watchdog = 0; int override_ignore_checksums = 0; int rule_debug = 0; @@ -464,7 +444,6 @@ int main(int argc, char** argv) {"parse-only", no_argument, 0, 'a'}, {"bare-mode", no_argument, 0, 'b'}, {"debug-policy", no_argument, 0, 'd'}, - {"dump-config", no_argument, 0, 'g'}, {"exec", required_argument, 0, 'e'}, {"filter", required_argument, 0, 'f'}, {"help", no_argument, 0, 'h'}, @@ -565,10 +544,6 @@ int main(int argc, char** argv) user_pcap_filter = optarg; break; - case 'g': - dump_cfg = true; - break; - case 'h': usage(0); break; @@ -795,8 +770,6 @@ int main(int argc, char** argv) dns_mgr->SetDir(".state"); iosource_mgr = new iosource::Manager(); - persistence_serializer = new PersistenceSerializer(); - remote_serializer = new RemoteSerializer(); event_registry = new EventRegistry(); analyzer_mgr = new analyzer::Manager(); log_mgr = new logging::Manager(); @@ -1012,13 +985,9 @@ int main(int argc, char** argv) exit(0); } - persistence_serializer->SetDir((const char *)state_dir->AsString()->CheckString()); - // Print the ID. if ( id_name ) { - persistence_serializer->ReadAll(true, false); - ID* id = global_scope()->Lookup(id_name); if ( ! id ) reporter->FatalError("No such ID: %s\n", id_name); @@ -1032,14 +1001,6 @@ int main(int argc, char** argv) exit(0); } - persistence_serializer->ReadAll(true, true); - - if ( dump_cfg ) - { - persistence_serializer->WriteConfig(false); - exit(0); - } - if ( profiling_interval > 0 ) { profiling_logger = new ProfileLogger(profiling_file->AsFile(), @@ -1205,7 +1166,6 @@ int main(int argc, char** argv) } else { - persistence_serializer->WriteState(false); terminate_bro(); } diff --git a/src/parse.y b/src/parse.y index 076e73f53e..13c15cad8f 100644 --- a/src/parse.y +++ b/src/parse.y @@ -5,7 +5,7 @@ // Switching parser table type fixes ambiguity problems. %define lr.type ielr -%expect 141 +%expect 129 %token TOK_ADD TOK_ADD_TO TOK_ADDR TOK_ANY %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF @@ -25,7 +25,6 @@ %token TOK_ATTR_OPTIONAL TOK_ATTR_REDEF TOK_ATTR_ROTATE_INTERVAL %token TOK_ATTR_ROTATE_SIZE TOK_ATTR_DEL_FUNC TOK_ATTR_EXPIRE_FUNC %token TOK_ATTR_EXPIRE_CREATE TOK_ATTR_EXPIRE_READ TOK_ATTR_EXPIRE_WRITE -%token TOK_ATTR_PERSISTENT TOK_ATTR_SYNCHRONIZED %token TOK_ATTR_RAW_OUTPUT TOK_ATTR_MERGEABLE %token TOK_ATTR_PRIORITY TOK_ATTR_LOG TOK_ATTR_ERROR_HANDLER %token TOK_ATTR_TYPE_COLUMN TOK_ATTR_DEPRECATED @@ -1308,10 +1307,6 @@ attr: { $$ = new Attr(ATTR_EXPIRE_READ, $3); } | TOK_ATTR_EXPIRE_WRITE '=' expr { $$ = new Attr(ATTR_EXPIRE_WRITE, $3); } - | TOK_ATTR_PERSISTENT - { $$ = new Attr(ATTR_PERSISTENT); } - | TOK_ATTR_SYNCHRONIZED - { $$ = new Attr(ATTR_SYNCHRONIZED); } | TOK_ATTR_ENCRYPT { $$ = new Attr(ATTR_ENCRYPT); } | TOK_ATTR_ENCRYPT '=' expr diff --git a/src/scan.l b/src/scan.l index 6b2610ee3f..21b3e0dcf4 100644 --- a/src/scan.l +++ b/src/scan.l @@ -310,11 +310,6 @@ when return TOK_WHEN; return TOK_ATTR_MERGEABLE; } -&persistent { - deprecated_attr(yytext); - return TOK_ATTR_PERSISTENT; - } - &rotate_interval { deprecated_attr(yytext); return TOK_ATTR_ROTATE_INTERVAL; @@ -325,11 +320,6 @@ when return TOK_WHEN; return TOK_ATTR_ROTATE_SIZE; } -&synchronized { - deprecated_attr(yytext); - return TOK_ATTR_SYNCHRONIZED; - } - @deprecated.* { auto num_files = file_stack.length(); auto comment = skip_whitespace(yytext + 11); diff --git a/src/threading/SerialTypes.cc b/src/threading/SerialTypes.cc index 8468d19ea8..dcc35f793c 100644 --- a/src/threading/SerialTypes.cc +++ b/src/threading/SerialTypes.cc @@ -2,8 +2,8 @@ #include "SerialTypes.h" -#include "../RemoteSerializer.h" - +#include "SerializationFormat.h" +#include "Reporter.h" using namespace threading; diff --git a/src/threading/SerialTypes.h b/src/threading/SerialTypes.h index 5a8361feba..65bb79b659 100644 --- a/src/threading/SerialTypes.h +++ b/src/threading/SerialTypes.h @@ -13,7 +13,6 @@ using namespace std; class SerializationFormat; -class RemoteSerializer; namespace threading { @@ -78,8 +77,6 @@ struct Field { string TypeName() const; private: - friend class ::RemoteSerializer; - // Force usage of constructor above. Field() {} }; diff --git a/testing/btest/Baseline/coverage.bare-mode-errors/errors b/testing/btest/Baseline/coverage.bare-mode-errors/errors index e69de29bb2..a13c8849a1 100644 --- a/testing/btest/Baseline/coverage.bare-mode-errors/errors +++ b/testing/btest/Baseline/coverage.bare-mode-errors/errors @@ -0,0 +1,4 @@ +warning in /Users/johanna/bro/master/scripts/policy/misc/trim-trace-file.zeek, line 25: deprecated (rotate_file_by_name) +warning in /Users/johanna/bro/master/scripts/policy/misc/trim-trace-file.zeek, line 25: deprecated (rotate_file_by_name) +warning in /Users/johanna/bro/master/scripts/policy/misc/trim-trace-file.zeek, line 25: deprecated (rotate_file_by_name) +warning in /Users/johanna/bro/master/testing/btest/../../scripts//policy/misc/trim-trace-file.zeek, line 25: deprecated (rotate_file_by_name)