Merge remote-tracking branch 'upstream/master' into paraglob

.gitignore

@@ -1,3 +1,7 @@
 build
 tmp
 *.gcov
+
+# Configuration and build directories for CLion
+.idea
+cmake-build-debug

CHANGES

@@ -1,4 +1,234 @@
 
+2.6-476 | 2019-06-20 13:23:22 -0700
+
+  * Remove opaque of ocsp_resp. (Johanna Amann, Corelight)
+
+    Only used in one event, without any way to use the opaque for anything
+    else. At this point this just seems like a complication that has no
+    reason to be there.
+
+  * Remove remnants of event serializer. (Johanna Amann, Corelight)
+
+  * Reimplement serialization infrastructure for OpaqueVals.
+    (Robin Sommer, Corelight & Johanna Amann, Corelight)
+
+    We need this to sender through Broker, and we also leverage it for
+    cloning opaques. The serialization methods now produce Broker data
+    instances directly, and no longer go through the binary formatter.
+
+    Summary of the new API for types derived from OpaqueVal:
+
+        - Add DECLARE_OPAQUE_VALUE(<class>) to the class declaration
+        - Add IMPLEMENT_OPAQUE_VALUE(<class>) to the class' implementation file
+        - Implement these two methods (which are declated by the 1st macro):
+            - broker::data DoSerialize() const
+            - bool DoUnserialize(const broker::data& data)
+
+    This machinery should work correctly from dynamic plugins as well.
+
+    OpaqueVal provides a default implementation of DoClone() as well that
+    goes through serialization. Derived classes can provide a more
+    efficient version if they want.
+
+    The declaration of the "OpaqueVal" class has moved into the header
+    file "OpaqueVal.h", along with the new serialization infrastructure.
+    This is breaking existing code that relies on the location, but
+    because the API is changing anyways that seems fine.
+
+  * Implement a Shallow Clone operation for types. (Johanna Amann, Corelight)
+
+    This is needed to track name changes for the documentation.
+
+  * Remove old serialization infrastrucutre. (Johanna Amann, Corelight)
+
+2.6-454 | 2019-06-19 09:39:06 -0700
+
+  * GH-393: Add slice notation for vectors (Tim Wojtulewicz, Corelight & Jon Siwek, Corelight)
+
+    Example Syntax:
+
+    local v = vector(1, 2, 3, 4, 5);
+    v[2:4] = vector(6, 7, 8);        # v is now [1, 2, 6, 7, 8, 5]
+    print v[:4];                     # prints [1, 2, 6, 7]
+
+2.6-446 | 2019-06-17 20:26:49 -0700
+
+  * Rename bro to zeek in error messages (Daniel Thayer)
+
+2.6-444 | 2019-06-15 19:09:03 -0700
+
+  * Add/rewrite NTP support (Vlad Grigorescu and Mauro Palumbo)
+
+2.6-416 | 2019-06-14 20:57:57 -0700
+
+  * DNS: Add support for SPF response records (Vlad Grigorescu)
+
+2.6-413 | 2019-06-14 19:51:28 -0700
+
+  * GH-406: rename bro.bif to zeek.bif (Jon Siwek, Corelight)
+
+2.6-412 | 2019-06-14 19:26:21 -0700
+
+  * GH-387: update Broker topic names to use "zeek/" prefix (Jon Siwek, Corelight)
+
+  * GH-323: change builtin plugin namespaces to Zeek (Jon Siwek, Corelight)
+
+2.6-408 | 2019-06-13 11:19:50 -0700
+
+  * Fix potential null-dereference in current_time() (Tim Wojtulewicz, Corelight)
+
+  * Add --sanitizers configure script to enable Clang sanitizers (Tim Wojtulewicz, Corelight)
+
+2.6-404 | 2019-06-12 15:10:19 -0700
+
+  * Rename directories from bro to zeek (Daniel Thayer)
+
+    The new default installation prefix is /usr/local/zeek
+
+2.6-400 | 2019-06-07 20:06:33 -0700
+
+  * Adapt bro_plugin CMake macros to use zeek_plugin (Jon Siwek, Corelight)
+
+2.6-399 | 2019-06-07 14:02:18 -0700
+
+  * Update SSL documentation. (Johanna Amann)
+
+  * Support the newer TLS 1.3 key_share extension. (Johanna Amann)
+
+  * Include all data of the server-hello random (Johanna Amann)
+
+    Before we cut the first 4 bytes, which makes it impossible to recognize
+    several newer packets (like the hello retry).
+
+  * Parse TLS 1.3 pre-shared-key extension. (Johanna Amann)
+
+
+    Adds new events:
+
+    - ssl_extension_pre_shared_key_client_hello
+    - ssl_extension_pre_shared_key_server_hello
+
+2.6-391 | 2019-06-07 17:29:28 +1000
+
+  * GH-209: replace "remote_ip" field of radius.log with "tunnel_client".
+    Also changes type from addr to string. (Jon Siwek, Corelight)
+
+2.6-389 | 2019-06-06 20:02:19 -0700
+
+  * Update plugin unit tests to use --zeek-dist (Jon Siwek, Corelight)
+
+2.6-388 | 2019-06-06 19:48:55 -0700
+
+  * Change default value of peer_description "zeek" (Jon Siwek, Corelight)
+
+2.6-387 | 2019-06-06 18:51:09 -0700
+
+  * Rename Bro to Zeek in Zeekygen-generated documentation (Jon Siwek, Corelight)
+
+2.6-386 | 2019-06-06 17:17:55 -0700
+
+  * Add new RDP event: rdp_native_encrytped_data (Anthony Kasza, Corelight)
+
+2.6-384 | 2019-06-06 16:49:14 -0700
+
+  * Add new RDP event: rdp_client_security_data (Jeff Atkinson)
+
+2.6-379 | 2019-06-06 11:56:58 -0700
+
+  * Improve sqlite logging unit tests (Jon Siwek, Corelight)
+
+2.6-378 | 2019-06-05 16:23:04 -0700
+
+  * Rename BRO_DEPRECATED macro to ZEEK_DEPRECATED (Jon Siwek, Corelight)
+
+2.6-377 | 2019-06-05 16:15:58 -0700
+
+  * Deprecate functions with "bro" in them. (Jon Siwek, Corelight)
+
+    * "bro_is_terminating" is now "zeek_is_terminating"
+
+    * "bro_version" is now "zeek_version"
+
+    The old functions still exist for now, but are deprecated.
+
+2.6-376 | 2019-06-05 13:29:57 -0700
+
+  * GH-379: move catch-and-release and unified2 scripts to policy/ (Jon Siwek, Corelight)
+
+    These are no longer loaded by default due to the performance impact they
+    cause simply by being loaded (they have event handlers for commonly
+    generated events) and they aren't generally useful enough to justify it.
+
+2.6-375 | 2019-06-04 19:28:06 -0700
+
+  * Simplify threading::Value destructor (Jon Siwek, Corelight)
+
+  * Add pattern support to input framework. (Zeke Medley, Corelight)
+
+2.6-369 | 2019-06-04 17:53:10 -0700
+
+  * GH-155: Improve coercion of expression lists to vector types (Tim Wojtulewicz, Corelight)
+
+  * GH-159: Allow coercion of numeric record field values to other types (Tim Wojtulewicz, Corelight)
+
+  * Allow passing a location to BroObj::Warning and BroObj::Error. (Tim Wojtulewicz, Corelight)
+
+    This allows callers (such as check_and_promote) to pass an expression
+    location to be logged if the location doesn't exist in the value being
+    promoted.
+
+  * Add CLion directories to gitignore (Tim Wojtulewicz, Corelight)
+
+  * Move #define outside of max_type for clarity (Tim Wojtulewicz, Corelight)
+
+2.6-361 | 2019-06-04 10:30:21 -0700
+
+  * GH-293: Protect copy() against reference cycles. (Robin Sommer, Corelight)
+
+    Reference cycles shouldn't occur but there's nothing really preventing
+    people from creating them, so may just as well be safe and deal with
+    them when cloning values.
+
+2.6-359 | 2019-05-31 13:37:17 -0700
+
+  * Remove old documentation reference to rotate_interval (Jon Siwek, Corelight)
+
+2.6-357 | 2019-05-30 10:57:54 -0700
+
+  * Tweak to ASCII reader warning suppression (Christian Kreibich, Corelight)
+
+    Warnings in the ASCII reader so far remained suppressed even
+    when an input file changed. It's helpful to learn about problems
+    in the data when putting in place new data files, so this change
+    maintains the existing warning suppression while processing a file,
+    but re-enables warnings after updates to a file.
+
+2.6-354 | 2019-05-29 09:46:19 -0700
+
+  * Add weird: "RDP_channels_requested_exceeds_max" (Vlad Grigorescu)
+
+2.6-352 | 2019-05-28 17:57:36 -0700
+
+  * Reduce data copying in Broker message processing (Jon Siwek, Corelight)
+
+  * Improve Broker I/O loop integration: less mutex locking (Jon Siwek, Corelight)
+
+    Checking a subscriber for available messages required locking a mutex,
+    but we should never actually need to do that in the main-loop to check
+    for Broker readiness since we can rely on file descriptor polling.
+
+  * Improve processing of broker data store responses (Jon Siwek, Corelight)
+
+    Now retrieves and processes all N available responses at once instead
+    of one-by-one-until-empty.
+
+2.6-345 | 2019-05-28 11:32:16 -0700
+
+  * RDP: Add parsing and logging of channels requested by the client. (Vlad Grigorescu)
+
+    Can determine capabilities requested by the client, as well as attacks such
+    as CVE-2019-0708.
+
 2.6-342 | 2019-05-28 10:48:37 -0700
 
   * GH-168: Improve type-checking for table/set list assignment. (Zeke Medley and Jon Siwek, Corelight)

CMakeLists.txt

@@ -24,7 +24,7 @@ endif ()
 set(ZEEK_ROOT_DIR ${CMAKE_INSTALL_PREFIX})
 if (NOT ZEEK_SCRIPT_INSTALL_PATH)
     # set the default Zeek script installation path (user did not specify one)
-    set(ZEEK_SCRIPT_INSTALL_PATH ${ZEEK_ROOT_DIR}/share/bro)
+    set(ZEEK_SCRIPT_INSTALL_PATH ${ZEEK_ROOT_DIR}/share/zeek)
 endif ()
 
 if (NOT ZEEK_MAN_INSTALL_PATH)
@@ -37,7 +37,7 @@ endif ()
 get_filename_component(ZEEK_SCRIPT_INSTALL_PATH ${ZEEK_SCRIPT_INSTALL_PATH}
     ABSOLUTE)
 
-set(BRO_PLUGIN_INSTALL_PATH ${ZEEK_ROOT_DIR}/lib/bro/plugins CACHE STRING "Installation path for plugins" FORCE)
+set(BRO_PLUGIN_INSTALL_PATH ${ZEEK_ROOT_DIR}/lib/zeek/plugins CACHE STRING "Installation path for plugins" FORCE)
 
 configure_file(zeek-path-dev.in ${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev)
 
@@ -72,6 +72,40 @@ if(${ENABLE_DEBUG})
     set(VERSION_C_IDENT "${VERSION_C_IDENT}_debug")
 endif()
 
+if ( NOT BINARY_PACKAGING_MODE )
+    macro(_make_install_dir_symlink _target _link)
+      install(CODE "
+        if ( \"\$ENV{DESTDIR}\" STREQUAL \"\" )
+          if ( EXISTS \"${_target}\" AND NOT EXISTS \"${_link}\" )
+            message(STATUS \"WARNING: installed ${_link} as symlink to ${_target}\")
+            execute_process(COMMAND \"${CMAKE_COMMAND}\" -E create_symlink
+              \"${_target}\" \"${_link}\")
+          endif ()
+        endif ()
+      ")
+    endmacro()
+
+    if ( "${CMAKE_INSTALL_PREFIX}" STREQUAL "/usr/local/zeek" )
+        # If we're installing into the default prefix, check if the
+        # old default prefix already exists and symlink to it.
+        # This is done to help keep custom user configuration/installation
+        # if they're upgrading from a version before Zeek 3.0.
+        _make_install_dir_symlink("/usr/local/bro" "/usr/local/zeek")
+    endif ()
+
+    # Check whether we need to symlink directories used by versions
+    # before Zeek 3.0.
+    _make_install_dir_symlink("${CMAKE_INSTALL_PREFIX}/include/bro" "${CMAKE_INSTALL_PREFIX}/include/zeek")
+    _make_install_dir_symlink("${CMAKE_INSTALL_PREFIX}/share/bro" "${CMAKE_INSTALL_PREFIX}/share/zeek")
+    _make_install_dir_symlink("${CMAKE_INSTALL_PREFIX}/lib/bro" "${CMAKE_INSTALL_PREFIX}/lib/zeek")
+endif ()
+
+if ( SANITIZERS )
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=${SANITIZERS} -fno-omit-frame-pointer")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=${SANITIZERS} -fno-omit-frame-pointer")
+    set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -fsanitize=${SANITIZERS} -fno-omit-frame-pointer")
+endif()
+
 ########################################################################
 ## Dependency Configuration
 
@@ -257,7 +291,7 @@ string(TOLOWER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_LOWER)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/zeek-config.h.in
                ${CMAKE_CURRENT_BINARY_DIR}/zeek-config.h)
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
-install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zeek-config.h DESTINATION include/bro)
+install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zeek-config.h DESTINATION include/zeek)
 
 if ( CAF_ROOT_DIR )
   set(ZEEK_CONFIG_CAF_ROOT_DIR ${CAF_ROOT_DIR})
@@ -281,7 +315,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/zeek-config.in
                ${CMAKE_CURRENT_BINARY_DIR}/zeek-config @ONLY)
 install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/zeek-config DESTINATION bin)
 
-install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/cmake DESTINATION share/bro
+install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/cmake DESTINATION share/zeek
         USE_SOURCE_PERMISSIONS)
 
 # Install wrapper script for Bro-to-Zeek renaming.
@@ -289,7 +323,7 @@ include(InstallShellScript)
 include(InstallSymlink)
 InstallShellScript("bin" "zeek-wrapper.in" "zeek-wrapper")
 InstallSymlink("${CMAKE_INSTALL_PREFIX}/bin/zeek-wrapper" "${CMAKE_INSTALL_PREFIX}/bin/bro-config")
-InstallSymlink("${CMAKE_INSTALL_PREFIX}/include/bro/zeek-config.h" "${CMAKE_INSTALL_PREFIX}/include/bro/bro-config.h")
+InstallSymlink("${CMAKE_INSTALL_PREFIX}/include/zeek/zeek-config.h" "${CMAKE_INSTALL_PREFIX}/include/zeek/bro-config.h")
 
 ########################################################################
 ## Recurse on sub-directories

NEWS

@@ -18,6 +18,9 @@ New Functionality
   - dns_NSEC
   - dns_NSEC3
 
+- Added support for parsing and logging DNS SPF resource records.
+  A new ``dns_SPF_reply`` event is also available.
+
 - Zeek's Plugin framework now allows a patch version.  If a patch version is not
   provided, it will default to 0.  To specify this, modify the plugin
   Configuration class in your ``src/Plugin.cc`` and set
@@ -79,6 +82,35 @@ New Functionality
 - The ``/<re>/i`` convenience syntax for case-insensitive patterns is now
   also allowed when specifying patterns used in signature files.
 
+- New RDP functionality.
+
+  - New events:
+    - rdp_client_network_data
+    - rdp_client_security_data
+    - rdp_native_encrypted_data
+
+  - Add a new "client_channels" field to rdp.log based on data parsed from
+    the Client Network Data (TS_UD_CS_NET) packet.  The channel list is also
+    available in the new ``rdp_client_network_data`` event.
+
+- Add parsing support for TLS 1.3 pre-shared key extension.  This info
+  is available in the events: ``ssl_extension_pre_shared_key_client_hello``
+  and ``ssl_extension_pre_shared_key_server_hello``.
+
+- Added/re-wrote support for NTP.
+
+  - Parsing support for modes 1-7, with parsed structures available in
+    the ``ntp_message`` event.
+
+  - An ntp.log is produced by default, containing data extracted from
+    NTP messages with modes 1-5.
+
+- Add support for vector slicing operations.  For example::
+
+    local v = vector(1, 2, 3, 4, 5);
+    v[2:4] = vector(6, 7, 8);        # v is now [1, 2, 6, 7, 8, 5]
+    print v[:4];                     # prints [1, 2, 6, 7]
+
 Changed Functionality
 ---------------------
 
@@ -101,6 +133,13 @@ Changed Functionality
   installation, installing the newer Zeek version over it will retain that
   file and even symlink the new ``zeekctl.cfg`` to it.
 
+- The default install prefix is now ``/usr/local/zeek`` instead of
+  ``/usr/local/bro``.  If you have an existing installation that used
+  the previous default and are still using the new default when upgrading,
+  we'll crate ``/usr/local/zeek`` as a symlink to ``/usr/local/bro``.
+  Certain subdirectories will also get similar treatment: ``share/bro``,
+  ``include/bro``, and ``lib/bro``.
+
 - ``$prefix/share/bro/site/local.bro`` has been renamed to
   ``local.zeek``.  If you have a ``local.bro`` file from a previous
   installation, possibly with customizations made to it, the new
@@ -212,6 +251,85 @@ Changed Functionality
   in scripts has also been updated to replace Sphinx cross-referencing roles
   and directives like ":bro:see:" with ":zeek:zee:".
 
+- The catch-and-release and unified2 scripts are no longer loaded by
+  default.  Because there was a performance impact simply from loading
+  them and it's unlikely a majority of user make use of their features,
+  they've been moved from the scripts/base/ directory into
+  scripts/policy/ and must be manually loaded to use their
+  functionality.  The "drop" action for the notice framework is likewise
+  moved since it was implemented via catch-and-release.  As a result,
+  the default notice.log no longer contains a "dropped" field.
+
+  If you previously used the catch-and-release functionality add this:
+
+      @load policy/frameworks/netcontrol/catch-and-release
+
+  If you previously used Notice::ACTION_DROP add:
+
+      @load policy/frameworks/notice/actions/drop
+
+  If you previously used the Unified2 file analysis support add:
+
+      @load policy/files/unified2
+
+- The default value of ``peer_description`` has changed from "bro"
+  to "zeek".  This won't effect most users, except for the fact that
+  this value may appear in several log files, so any external plugins
+  that have written unit tests that compare baselines of such log
+  files may need to be updated.
+
+- The "remote_ip" field of "addr" type was removed from radius.log and
+  replaced with a field named "tunnel_client" of "string" type.  The
+  reason for this is that the Tunnel-Client-Endpoint RADIUS attribute
+  this data is derived from may also be a FQDN, not just an IP address.
+
+- The ``ssl_server_hello`` event's ``server_random`` parameter has been
+  changed to always include the full 32-byte field from the
+  ServerHello.  Previously a 4-byte timestamp and 28-byte random data
+  were parsed separately as some TLS protocol versions specified a
+  separate timestamp field as part of the full 32-byte random sequence.
+
+- The namespace used by all the builtin plugins that ship with Zeek have
+  changed to use "Zeek::" instead of "Bro::".
+
+- Any Broker topic names used in scripts shipped with Zeek that
+  previously were prefixed with "bro/" are now prefixed with "zeek/"
+  instead.
+
+  In the case where external applications were using a "bro/" topic
+  to send data into a Bro process, a Zeek process still subscribes
+  to those topics in addition to the equivalently named "zeek/" topic.
+
+  In the case where external applications were using a "bro/" topic
+  to subscribe to remote messages or query data stores, there's no
+  backwards compatibility and external applications must be changed
+  to use the new "zeek/" topic.  The thought is this change will have
+  low impact since most data published under "bro/" topic names is
+  intended for use only as a detail of implementing cluster-enabled
+  versions of various scripts.
+
+  A list of the most relevant/common topic names that could potentially
+  be used in external applications to consume/query remote data that
+  one may need to change:
+
+  - store names
+    - bro/known/services
+    - bro/known/hosts
+    - bro/known/certs
+
+  - cluster nodes
+    - bro/cluster/<node type>
+    - bro/cluster/node/<name>
+    - bro/cluster/nodeid/<id>
+
+  - logging
+    - bro/logs/<stream>
+
+- The ``resp_ref`` argument was removed from the ``ocsp_response_bytes``
+  event. ``resp_ref`` was not used by anything in the codebase and could not be
+  passed to any other functions for further processing. The remainder of the
+  ``ocsp_response_bytes`` is unchanged.
+
 Removed Functionality
 ---------------------
 
@@ -320,8 +438,18 @@ Removed Functionality
   and &rotate_size attributes, which were deprecated in Bro 2.6, was removed. The ``-g``
   command-line option (dump-config) which relied on this functionality was also removed.
 
+- Functionality for writing state updates for variables with the
+  &synchronized attribute was removed. This entails the
+  ``-x`` command-line option (print-state) as well as the
+  ``capture_state_updates`` function.
+
 - Removed the BroControl ``update`` command, which was deprecated in Bro 2.6.
 
+- Functionality for writing/reading binary event streams was
+  removed. This functionality relied on the old communication code
+  anc was basically untested. The ``-R`` command-line option (replay)
+  as well as the ``capture_events`` function were removed.
+
 Deprecated Functionality
 ------------------------
 
@@ -336,6 +464,9 @@ Deprecated Functionality
   such that existing code will not break, but will emit a deprecation
   warning.
 
+- The ``bro_is_terminating`` and ``bro_version`` function are deprecated and
+  replaced by functions named ``zeek_is_terminating`` and ``zeek_version``.
+
 - The ``rotate_file``, ``rotate_file_by_name`` and ``calc_next_rotate`` functions
   were marked as deprecated. These functions were used with the old pre-2.0 logging
   framework and are no longer used. They also were marked as deprecated in their

VERSION

@@ -1 +1 @@
-2.6-342
+2.6-476

aux/broker

@@ -1 +1 @@
-Subproject commit 474a6d902b1c41912b216c06a74085ff3b102bad
+Subproject commit bfefac3a882b382bfb7ad749974930884da954a6

configure

@@ -31,9 +31,9 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
                            (useful for cross-compiling)
 
   Installation Directories:
-    --prefix=PREFIX        installation directory [/usr/local/bro]
+    --prefix=PREFIX        installation directory [/usr/local/zeek]
     --scriptdir=PATH       root installation directory for Zeek scripts
-                           [PREFIX/share/bro]
+                           [PREFIX/share/zeek]
     --localstatedir=PATH   when using ZeekControl, path to store log files
                            and run-time data (within log/ and spool/ subdirs)
                            [PREFIX]
@@ -58,6 +58,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --disable-perftools    don't try to build with Google Perftools
     --disable-python       don't try to build python bindings for Broker
     --disable-broker-tests don't try to build Broker unit tests
+    --sanitizers=SANITIZERS comma-separated list of Clang sanitizers to enable
 
   Required Packages in Non-Standard Locations:
     --with-openssl=PATH    path to OpenSSL install root
@@ -127,12 +128,12 @@ remove_cache_entry () {
 
 # set defaults
 builddir=build
-prefix=/usr/local/bro
+prefix=/usr/local/zeek
 CMakeCacheEntries=""
 append_cache_entry CMAKE_INSTALL_PREFIX PATH   $prefix
 append_cache_entry ZEEK_ROOT_DIR        PATH   $prefix
 append_cache_entry PY_MOD_INSTALL_DIR   PATH   $prefix/lib/zeekctl
-append_cache_entry ZEEK_SCRIPT_INSTALL_PATH STRING $prefix/share/bro
+append_cache_entry ZEEK_SCRIPT_INSTALL_PATH STRING $prefix/share/zeek
 append_cache_entry ZEEK_ETC_INSTALL_DIR PATH   $prefix/etc
 append_cache_entry ENABLE_DEBUG         BOOL   false
 append_cache_entry ENABLE_PERFTOOLS     BOOL   false
@@ -144,6 +145,7 @@ append_cache_entry INSTALL_ZEEKCTL      BOOL   true
 append_cache_entry CPACK_SOURCE_IGNORE_FILES STRING
 append_cache_entry ENABLE_MOBILE_IPV6   BOOL   false
 append_cache_entry DISABLE_PERFTOOLS    BOOL   false
+append_cache_entry SANITIZERS           STRING ""
 
 # parse arguments
 while [ $# -ne 0 ]; do
@@ -216,6 +218,9 @@ while [ $# -ne 0 ]; do
             append_cache_entry ENABLE_PERFTOOLS     BOOL   true
             append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL   true
             ;;
+        --sanitizers=*)
+            append_cache_entry SANITIZERS    STRING    $optarg
+            ;;
         --enable-jemalloc)
             append_cache_entry ENABLE_JEMALLOC     BOOL   true
             ;;
@@ -321,7 +326,7 @@ while [ $# -ne 0 ]; do
 done
 
 if [ "$user_set_scriptdir" != "true" ]; then
-    append_cache_entry ZEEK_SCRIPT_INSTALL_PATH STRING $prefix/share/bro
+    append_cache_entry ZEEK_SCRIPT_INSTALL_PATH STRING $prefix/share/zeek
 fi
 
 if [ "$user_set_conffilesdir" != "true" ]; then

scripts/base/frameworks/broker/main.zeek

@@ -113,7 +113,7 @@ export {
 
 	## The default topic prefix where logs will be published.  The log's stream
 	## id is appended when writing to a particular stream.
-	const default_log_topic_prefix = "bro/logs/" &redef;
+	const default_log_topic_prefix = "zeek/logs/" &redef;
 
 	## The default implementation for :zeek:see:`Broker::log_topic`.
 	function default_log_topic(id: Log::ID, path: string): string

scripts/base/frameworks/cluster/main.zeek

@@ -17,31 +17,31 @@ export {
 
 	## The topic name used for exchanging messages that are relevant to
 	## logger nodes in a cluster.  Used with broker-enabled cluster communication.
-	const logger_topic = "bro/cluster/logger" &redef;
+	const logger_topic = "zeek/cluster/logger" &redef;
 
 	## The topic name used for exchanging messages that are relevant to
 	## manager nodes in a cluster.  Used with broker-enabled cluster communication.
-	const manager_topic = "bro/cluster/manager" &redef;
+	const manager_topic = "zeek/cluster/manager" &redef;
 
 	## The topic name used for exchanging messages that are relevant to
 	## proxy nodes in a cluster.  Used with broker-enabled cluster communication.
-	const proxy_topic = "bro/cluster/proxy" &redef;
+	const proxy_topic = "zeek/cluster/proxy" &redef;
 
 	## The topic name used for exchanging messages that are relevant to
 	## worker nodes in a cluster.  Used with broker-enabled cluster communication.
-	const worker_topic = "bro/cluster/worker" &redef;
+	const worker_topic = "zeek/cluster/worker" &redef;
 
 	## The topic name used for exchanging messages that are relevant to
 	## time machine nodes in a cluster.  Used with broker-enabled cluster communication.
-	const time_machine_topic = "bro/cluster/time_machine" &redef;
+	const time_machine_topic = "zeek/cluster/time_machine" &redef;
 
 	## The topic prefix used for exchanging messages that are relevant to
 	## a named node in a cluster.  Used with broker-enabled cluster communication.
-	const node_topic_prefix = "bro/cluster/node/" &redef;
+	const node_topic_prefix = "zeek/cluster/node/" &redef;
 
 	## The topic prefix used for exchanging messages that are relevant to
 	## a unique node in a cluster.  Used with broker-enabled cluster communication.
-	const nodeid_topic_prefix = "bro/cluster/nodeid/" &redef;
+	const nodeid_topic_prefix = "zeek/cluster/nodeid/" &redef;
 
 	## Name of the node on which master data stores will be created if no other
 	## has already been specified by the user in :zeek:see:`Cluster::stores`.

scripts/base/frameworks/cluster/pools.zeek

@@ -60,17 +60,17 @@ export {
 
 	## The specification for :zeek:see:`Cluster::proxy_pool`.
 	global proxy_pool_spec: PoolSpec =
-		PoolSpec($topic = "bro/cluster/pool/proxy",
+		PoolSpec($topic = "zeek/cluster/pool/proxy",
 				 $node_type = Cluster::PROXY) &redef;
 
 	## The specification for :zeek:see:`Cluster::worker_pool`.
 	global worker_pool_spec: PoolSpec =
-		PoolSpec($topic = "bro/cluster/pool/worker",
+		PoolSpec($topic = "zeek/cluster/pool/worker",
 				 $node_type = Cluster::WORKER) &redef;
 
 	## The specification for :zeek:see:`Cluster::logger_pool`.
 	global logger_pool_spec: PoolSpec =
-		PoolSpec($topic = "bro/cluster/pool/logger",
+		PoolSpec($topic = "zeek/cluster/pool/logger",
 				 $node_type = Cluster::LOGGER) &redef;
 
 	## A pool containing all the proxy nodes of a cluster.

scripts/base/frameworks/netcontrol/__load__.zeek

@@ -3,7 +3,6 @@
 @load ./plugins
 @load ./drop
 @load ./shunt
-@load ./catch-and-release
 
 # The cluster framework must be loaded first.
 @load base/frameworks/cluster

scripts/base/frameworks/netcontrol/drop.zeek

@@ -1,9 +1,9 @@
 ##! Implementation of the drop functionality for NetControl.
 
-module NetControl;
-
 @load ./main
 
+module NetControl;
+
 export {
 	redef enum Log::ID += { DROP };
 

scripts/base/frameworks/netcontrol/main.zeek

@@ -10,11 +10,11 @@
 ##! provides convenience functions for a set of common operations. The
 ##! low-level API provides full flexibility.
 
-module NetControl;
-
 @load ./plugin
 @load ./types
 
+module NetControl;
+
 export {
 	## The framework's logging stream identifier.
 	redef enum Log::ID += { LOG };
@@ -889,7 +889,7 @@ function remove_rule_impl(id: string, reason: string) : bool
 function rule_expire_impl(r: Rule, p: PluginState) &priority=-5
 	{
 	# do not emit timeout events on shutdown
-	if ( bro_is_terminating() )
+	if ( zeek_is_terminating() )
 		return;
 
 	if ( r$id !in rules )

scripts/base/frameworks/netcontrol/non-cluster.zeek

@@ -1,7 +1,8 @@
-module NetControl;
 
 @load ./main
 
+module NetControl;
+
 function activate(p: PluginState, priority: int)
 	{
 	activate_impl(p, priority);

scripts/base/frameworks/netcontrol/plugin.zeek

@@ -1,9 +1,9 @@
 ##! This file defines the plugin interface for NetControl.
 
-module NetControl;
-
 @load ./types
 
+module NetControl;
+
 export {
 	## This record keeps the per instance state of a plugin.
 	##

scripts/base/frameworks/netcontrol/plugins/acld.zeek

@@ -1,11 +1,11 @@
 ##! Acld plugin for the netcontrol framework.
 
-module NetControl;
-
 @load ../main
 @load ../plugin
 @load base/frameworks/broker
 
+module NetControl;
+
 export {
 	type AclRule : record {
 		command: string;

scripts/base/frameworks/netcontrol/plugins/broker.zeek

@@ -2,12 +2,12 @@
 ##! used in NetControl on to Broker to allow for easy handling, e.g., of
 ##! command-line scripts.
 
-module NetControl;
-
 @load ../main
 @load ../plugin
 @load base/frameworks/broker
 
+module NetControl;
+
 export {
 	## This record specifies the configuration that is passed to :zeek:see:`NetControl::create_broker`.
 	type BrokerConfig: record {

scripts/base/frameworks/netcontrol/plugins/packetfilter.zeek

@@ -3,10 +3,10 @@
 ##! and can only add/remove filters for addresses, this is quite
 ##! limited in scope at the moment. 
 
-module NetControl;
-
 @load ../plugin
 
+module NetControl;
+
 export {
 	## Instantiates the packetfilter plugin.
 	global create_packetfilter: function() : PluginState;

scripts/base/frameworks/notice/__load__.zeek

@@ -3,7 +3,6 @@
 
 # There should be no overhead imposed by loading notice actions so we
 # load them all.
-@load ./actions/drop
 @load ./actions/email_admin
 @load ./actions/page
 @load ./actions/add-geodata

scripts/base/frameworks/notice/main.zeek

@@ -405,7 +405,7 @@ function email_headers(subject_desc: string, dest: string): string
 		"From: ", mail_from, "\n",
 		"Subject: ", mail_subject_prefix, " ", subject_desc, "\n",
 		"To: ", dest, "\n",
-		"User-Agent: Bro-IDS/", bro_version(), "\n");
+		"User-Agent: Bro-IDS/", zeek_version(), "\n");
 	if ( reply_to != "" )
 		header_text = string_cat(header_text, "Reply-To: ", reply_to, "\n");
 	return header_text;

scripts/base/frameworks/openflow/consts.zeek

@@ -11,7 +11,7 @@ const COOKIE_BID_SIZE = 16777216;
 # start at bit 40 (1 << 40)
 const COOKIE_BID_START = 1099511627776;
 # Zeek specific cookie ID shall have the 42 bit set (1 << 42)
-const BRO_COOKIE_ID = 4;
+const ZEEK_COOKIE_ID = 4;
 # 8 bits group identifier
 const COOKIE_GID_SIZE = 256;
 # start at bit 32 (1 << 32)

scripts/base/frameworks/openflow/main.zeek

@@ -198,7 +198,7 @@ function match_conn(id: conn_id, reverse: bool &default=F): ofp_match
 # 42 bit of the cookie set.
 function generate_cookie(cookie: count &default=0): count
 	{
-	local c = BRO_COOKIE_ID * COOKIE_BID_START;
+	local c = ZEEK_COOKIE_ID * COOKIE_BID_START;
 
 	if ( cookie >= COOKIE_UID_SIZE )
 		Reporter::warning(fmt("The given cookie uid '%d' is > 32bit and will be discarded", cookie));
@@ -211,7 +211,7 @@ function generate_cookie(cookie: count &default=0): count
 # local function to check if a given flow_mod cookie is forged from this framework.
 function is_valid_cookie(cookie: count): bool
 	{
-	if ( cookie / COOKIE_BID_START == BRO_COOKIE_ID )
+	if ( cookie / COOKIE_BID_START == ZEEK_COOKIE_ID )
 		return T;
 
 	Reporter::warning(fmt("The given Openflow cookie '%d' is not valid", cookie));
@@ -231,7 +231,7 @@ function get_cookie_gid(cookie: count): count
 	{
 	if( is_valid_cookie(cookie) )
 		return (
-			(cookie	- (COOKIE_BID_START * BRO_COOKIE_ID) -
+			(cookie	- (COOKIE_BID_START * ZEEK_COOKIE_ID) -
 			(cookie - ((cookie / COOKIE_GID_START) * COOKIE_GID_START))) /
 			COOKIE_GID_START
 		);

scripts/base/frameworks/openflow/types.zeek

@@ -89,7 +89,7 @@ export {
 		## Opaque controller-issued identifier.
 		# This is optional in the specification - but let's force
 		# it so we always can identify our flows...
-		cookie: count; # &default=BRO_COOKIE_ID * COOKIE_BID_START;
+		cookie: count; # &default=ZEEK_COOKIE_ID * COOKIE_BID_START;
 		# Flow actions
 		## Table to put the flow in. OFPTT_ALL can be used for delete,
 		## to delete flows from all matching tables.

scripts/base/frameworks/sumstats/non-cluster.zeek

@@ -35,7 +35,7 @@ event SumStats::finish_epoch(ss: SumStat)
 			{
 			local data = result_store[ss$name];
 			local now = network_time();
-			if ( bro_is_terminating() )
+			if ( zeek_is_terminating() )
 				{
 				for ( key, val in data )
 					ss$epoch_result(now, key, val);

scripts/base/init-bare.zeek

@@ -777,7 +777,7 @@ type IPAddrAnonymizationClass: enum {
 
 ## Deprecated.
 ##
-## .. zeek:see:: rotate_file rotate_file_by_name rotate_interval
+## .. zeek:see:: rotate_file rotate_file_by_name
 type rotate_info: record {
 	old_name: string;	##< Original filename.
 	new_name: string;	##< File name after rotation.
@@ -1113,9 +1113,6 @@ const table_expire_delay = 0.01 secs &redef;
 ## Time to wait before timing out a DNS request.
 const dns_session_timeout = 10 sec &redef;
 
-## Time to wait before timing out an NTP request.
-const ntp_session_timeout = 300 sec &redef;
-
 ## Time to wait before timing out an RPC request.
 const rpc_timeout = 24 sec &redef;
 
@@ -1789,7 +1786,7 @@ type gtp_delete_pdp_ctx_response_elements: record {
 };
 
 # Prototypes of Zeek built-in functions.
-@load base/bif/bro.bif
+@load base/bif/zeek.bif
 @load base/bif/stats.bif
 @load base/bif/reporter.bif
 @load base/bif/strings.bif
@@ -2529,26 +2526,6 @@ export {
 	};
 }
 
-module GLOBAL;
-
-## An NTP message.
-##
-## .. zeek:see:: ntp_message
-type ntp_msg: record {
-	id: count;	##< Message ID.
-	code: count;	##< Message code.
-	stratum: count;	##< Stratum.
-	poll: count;	##< Poll.
-	precision: int;	##< Precision.
-	distance: interval;	##< Distance.
-	dispersion: interval;	##< Dispersion.
-	ref_t: time;	##< Reference time.
-	originate_t: time;	##< Originating time.
-	receive_t: time;	##< Receive time.
-	xmit_t: time;	##< Send time.
-};
-
-
 module NTLM;
 
 export {
@@ -4140,6 +4117,10 @@ export {
 		SignatureAlgorithm: count; ##< Signature algorithm number
 	};
 
+	type PSKIdentity: record {
+		identity: string; ##< PSK identity
+		obfuscated_ticket_age: count;
+	};
 
 ## Number of non-DTLS frames that can occur in a DTLS connection before
 ## parsing of the connection is suspended.
@@ -4161,6 +4142,8 @@ module GLOBAL;
 ##    directly and then remove this alias.
 type signature_and_hashalgorithm_vec: vector of SSL::SignatureAndHashAlgorithm;
 
+type psk_identity_vec: vector of SSL::PSKIdentity;
+
 module X509;
 export {
 	type Certificate: record {
@@ -4275,9 +4258,57 @@ export {
 		ec_flags:               RDP::EarlyCapabilityFlags &optional;
 		dig_product_id:         string &optional;
 	};
+
+	## The TS_UD_CS_SEC data block contains security-related information used
+	## to advertise client cryptographic support.
+	type RDP::ClientSecurityData: record {
+		## Cryptographic encryption methods supported by the client and used in
+		## conjunction with Standard RDP Security.  Known flags:
+		##
+		## - 0x00000001: support for 40-bit session encryption keys
+		## - 0x00000002: support for 128-bit session encryption keys
+		## - 0x00000008: support for 56-bit session encryption keys
+		## - 0x00000010: support for FIPS compliant encryption and MAC methods
+		encryption_methods:	count;
+		## Only used in French locale and designates the encryption method.  If
+		## non-zero, then encryption_methods should be set to 0.
+		ext_encryption_methods:	count;
+	};
+
+	## Name and flags for a single channel requested by the client.
+	type RDP::ClientChannelDef: record {
+		## A unique name for the channel
+		name:           string;
+		## Absence of this flag indicates that this channel is
+		## a placeholder and that the server MUST NOT set it up.
+		initialized:    bool;
+		## Unused, must be ignored by the server.
+		encrypt_rdp:    bool;
+		## Unused, must be ignored by the server.
+		encrypt_sc:     bool;
+		## Unused, must be ignored by the server.
+		encrypt_cs:     bool;
+		## Channel data must be sent with high MCS priority.
+		pri_high:       bool;
+		## Channel data must be sent with medium MCS priority.
+		pri_med:        bool;
+		## Channel data must be sent with low MCS priority.
+		pri_low:        bool;
+		## Virtual channel data must be compressed if RDP data is being compressed.
+		compress_rdp:   bool;
+		## Virtual channel data must be compressed.
+		compress:       bool;
+		## Ignored by the server.
+		show_protocol:  bool;
+		## Channel must be persistent across remote control transactions.
+		persistent:     bool;
+	};
+
+	## The list of channels requested by the client.
+	type RDP::ClientChannelList: vector of ClientChannelDef;
 }
 
-@load base/bif/plugins/Bro_SNMP.types.bif
+@load base/bif/plugins/Zeek_SNMP.types.bif
 
 module SNMP;
 export {
@@ -4399,7 +4430,7 @@ export {
 	};
 }
 
-@load base/bif/plugins/Bro_KRB.types.bif
+@load base/bif/plugins/Zeek_KRB.types.bif
 
 module KRB;
 export {
@@ -4711,7 +4742,7 @@ const packet_filter_default = F &redef;
 const sig_max_group_size = 50 &redef;
 
 ## Description transmitted to remote communication peers for identification.
-const peer_description = "bro" &redef;
+const peer_description = "zeek" &redef;
 
 ## The number of IO chunks allowed to be buffered between the child
 ## and parent process of remote communication before Zeek starts dropping
@@ -4921,6 +4952,180 @@ export {
 	const max_frame_size = 65536 &redef;
 }
 
+module NTP;
+export {
+	## NTP standard message as defined in :rfc:`5905` for modes 1-5
+	## This record contains the standard fields used by the NTP protocol
+	## for standard syncronization operations.
+	type NTP::StandardMessage: record {
+		## This value mainly identifies the type of server (primary server,
+		## secondary server, etc.). Possible values, as in :rfc:`5905`, are:
+		##
+		##   * 0 -> unspecified or invalid
+		##   * 1 -> primary server (e.g., equipped with a GPS receiver)
+		##   * 2-15 -> secondary server (via NTP)
+		##   * 16 -> unsynchronized
+		##   * 17-255 -> reserved
+		##
+		## For stratum 0, a *kiss_code* can be given for debugging and
+		## monitoring.
+		stratum:            count;
+		## The maximum interval between successive messages.
+		poll:               interval;
+		## The precision of the system clock.
+		precision:          interval;
+		## Root delay. The total round-trip delay to the reference clock.
+		root_delay:         interval;
+		## Root Dispersion. The total dispersion to the reference clock.
+		root_disp:          interval;
+		## For stratum 0, four-character ASCII string used for debugging and
+		## monitoring. Values are defined in :rfc:`1345`.
+		kiss_code:          string &optional;
+		## Reference ID. For stratum 1, this is the ID assigned to the
+		## reference clock by IANA.
+		## For example: GOES, GPS, GAL, etc. (see :rfc:`5905`)
+		ref_id:             string &optional;
+		## Above stratum 1, when using IPv4, the IP address of the reference
+		## clock.  Note that the NTP protocol did not originally specify a
+		## large enough field to represent IPv6 addresses, so they use
+		## the first four bytes of the MD5 hash of the reference clock's
+		## IPv6 address (i.e. an IPv4 address here is not necessarily IPv4).
+		ref_addr:           addr &optional;
+		## Reference timestamp. Time when the system clock was last set or
+		## correct.
+		ref_time:           time;
+		## Origin timestamp. Time at the client when the request departed for
+		## the NTP server.
+		org_time:           time;
+		## Receive timestamp. Time at the server when the request arrived from
+		## the NTP client.
+		rec_time:           time;
+		## Transmit timestamp. Time at the server when the response departed
+		# for the NTP client.
+		xmt_time:           time;
+		## Key used to designate a secret MD5 key.
+		key_id:             count &optional;
+		## MD5 hash computed over the key followed by the NTP packet header and
+		## extension fields.
+		digest:             string &optional;
+		## Number of extension fields (which are not currently parsed).
+		num_exts:           count &default=0;
+	};
+
+	## NTP control message as defined in :rfc:`1119` for mode=6
+	## This record contains the fields used by the NTP protocol
+	## for control operations.
+	type NTP::ControlMessage: record {
+		## An integer specifying the command function. Values currently defined:
+		##
+		## * 1 read status command/response
+		## * 2 read variables command/response
+		## * 3 write variables command/response
+		## * 4 read clock variables command/response
+		## * 5 write clock variables command/response
+		## * 6 set trap address/port command/response
+		## * 7 trap response
+		##
+		## Other values are reserved.
+		op_code:            count;
+		## The response bit. Set to zero for commands, one for responses.
+		resp_bit:           bool;
+		## The error bit. Set to zero for normal response, one for error
+		## response.
+		err_bit:            bool;
+		## The more bit. Set to zero for last fragment, one for all others.
+		more_bit:           bool;
+		## The sequence number of the command or response.
+		sequence:           count;
+		## The current status of the system, peer or clock.
+		#TODO: this can be further parsed internally
+		status:             count;
+		## A 16-bit integer identifying a valid association.
+		association_id:     count;
+		## Message data for the command or response + Authenticator (optional).
+		data:               string &optional;
+		## This is an integer identifying the cryptographic
+		## key used to generate the message-authentication code.
+		key_id:             count &optional;
+		## This is a crypto-checksum computed by the encryption procedure.
+		crypto_checksum:    string &optional;
+	};
+
+	## NTP mode 7 message. Note that this is not defined in any RFC and is
+	## implementation dependent. We used the official implementation from the
+	## `NTP official project <www.ntp.org>`_.  A mode 7 packet is used
+	## exchanging data between an NTP server and a client for purposes other
+	## than time synchronization, e.g.  monitoring, statistics gathering and
+	## configuration.  For details see the documentation from the `NTP official
+	## project <www.ntp.org>`_, code v. ntp-4.2.8p13, in include/ntp_request.h.
+	type NTP::Mode7Message: record {
+		## An implementation-specific code which specifies the
+		## operation to be (which has been) performed and/or the
+		## format and semantics of the data included in the packet.
+		req_code:       count;
+		## The authenticated bit. If set, this packet is authenticated.
+		auth_bit:       bool;
+		## For a multipacket response, contains the sequence
+		## number of this packet.  0 is the first in the sequence,
+		## 127 (or less) is the last.  The More Bit must be set in
+		## all packets but the last.
+		sequence:       count;
+		## The number of the implementation this request code
+		## is defined by.  An implementation number of zero is used
+		## for requst codes/data formats which all implementations
+		## agree on.  Implementation number 255 is reserved (for
+		## extensions, in case we run out).
+		implementation: count;
+		## Must be 0 for a request.  For a response, holds an error
+		## code relating to the request.  If nonzero, the operation
+		## requested wasn't performed.
+		##
+		##   * 0 - no error
+		##   * 1 - incompatible implementation number
+		##   * 2 - unimplemented request code
+		##   * 3 - format error (wrong data items, data size, packet size etc.)
+		##   * 4 - no data available (e.g. request for details on unknown peer)
+		##   * 5 - unknown
+		##   * 6 - unknown
+		##   * 7 - authentication failure (i.e. permission denied)
+		err:            count;
+		## Rest of data
+		data:           string &optional;
+	};
+
+	## NTP message as defined in :rfc:`5905`.  Does include fields for mode 7,
+	## reserved for private use in :rfc:`5905`, but used in some implementation
+	## for commands such as "monlist".
+	type NTP::Message: record {
+		## The NTP version number (1, 2, 3, 4).
+		version:        count;
+		## The NTP mode being used. Possible values are:
+		##
+		##   * 1 - symmetric active
+		##   * 2 - symmetric passive
+		##   * 3 - client
+		##   * 4 - server
+		##   * 5 - broadcast
+		##   * 6 - NTP control message
+		##   * 7 - reserved for private use
+		mode:           count;
+		## If mode 1-5, the standard fields for syncronization operations are
+		## here.  See :rfc:`5905`
+		std_msg:        NTP::StandardMessage &optional;
+		## If mode 6, the fields for control operations are here.
+		## See :rfc:`1119`
+		control_msg:    NTP::ControlMessage &optional;
+		## If mode 7, the fields for extra operations are here.
+		## Note that this is not defined in any RFC
+		## and is implementation dependent. We used the official implementation
+		## from the `NTP official project <www.ntp.org>`_.
+		## A mode 7 packet is used exchanging data between an NTP server
+		## and a client for purposes other than time synchronization, e.g.
+		## monitoring, statistics gathering and configuration.
+		mode7_msg: NTP::Mode7Message &optional;
+	};
+}
+
 module Cluster;
 export {
 	type Cluster::Pool: record {};

scripts/base/init-default.zeek

@@ -56,6 +56,7 @@
 @load base/protocols/modbus
 @load base/protocols/mysql
 @load base/protocols/ntlm
+@load base/protocols/ntp
 @load base/protocols/pop3
 @load base/protocols/radius
 @load base/protocols/rdp
@@ -74,7 +75,6 @@
 @load base/files/pe
 @load base/files/hash
 @load base/files/extract
-@load base/files/unified2
 @load base/files/x509
 
 @load base/misc/find-checksum-offloading

scripts/base/misc/version.zeek

@@ -78,10 +78,10 @@ export {
 	## The format of the number is ABBCC with A being the major version,
 	## bb being the minor version (2 digits) and CC being the patchlevel (2 digits).
 	## As an example, Zeek 2.4.1 results in the number 20401
-	const number = Version::parse(bro_version())$version_number;
+	const number = Version::parse(zeek_version())$version_number;
 
 	## `VersionDescription` record pertaining to the currently running version of Zeek.
-	const info = Version::parse(bro_version());
+	const info = Version::parse(zeek_version());
 }
 
 function at_least(version_string: string): bool

scripts/base/protocols/dhcp/main.zeek

@@ -141,7 +141,7 @@ function join_data_expiration(t: table[count] of Info, idx: count): interval
 	# Also, if Zeek is shutting down.
 	if ( (now - info$last_message_ts) > 5sec ||
 	     (now - info$ts) > max_txid_watch_time ||
-	     bro_is_terminating() )
+	     zeek_is_terminating() )
 		{
 		Log::write(LOG, info);
 

scripts/base/protocols/dns/main.zeek

@@ -456,6 +456,21 @@ event dns_TXT_reply(c: connection, msg: dns_msg, ans: dns_answer, strs: string_v
 	hook DNS::do_reply(c, msg, ans, txt_strings);
 	}
 
+event dns_SPF_reply(c: connection, msg: dns_msg, ans: dns_answer, strs: string_vec) &priority=5
+	{
+	local spf_strings: string = "";
+
+	for ( i in strs )
+		{
+		if ( i > 0 )
+			spf_strings += " ";
+
+		spf_strings += fmt("SPF %d %s", |strs[i]|, strs[i]);
+		}
+
+	hook DNS::do_reply(c, msg, ans, spf_strings);
+	}
+
 event dns_AAAA_reply(c: connection, msg: dns_msg, ans: dns_answer, a: addr) &priority=5
 	{
 	hook DNS::do_reply(c, msg, ans, fmt("%s", a));

scripts/base/protocols/ntp/__load__.zeek

@@ -0,0 +1,2 @@
+@load ./main
+@load ./consts

scripts/base/protocols/ntp/consts.zeek

@@ -0,0 +1,15 @@
+module NTP;
+
+export {
+	## The descriptions of the NTP mode value, as described
+	## in :rfc:`5905`, Figure 1
+	const modes: table[count] of string = {
+		[1] = "symmetric active",
+		[2] = "symmetric passive",
+		[3] = "client",
+		[4] = "server",
+		[5] = "broadcast server",
+		[6] = "broadcast client",
+		[7] = "reserved",
+	} &default=function(i: count):string { return fmt("unknown-%d", i); } &redef;
+}

scripts/base/protocols/ntp/main.zeek

@@ -0,0 +1,107 @@
+module NTP;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		## Timestamp for when the event happened.
+		ts:         time	&log;
+		## Unique ID for the connection.
+		uid:        string  &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
+		id:         conn_id &log;
+		## The NTP version number (1, 2, 3, 4).
+		version:    count &log;
+		## The NTP mode being used.
+		mode:       count &log;
+		## The stratum (primary server, secondary server, etc.).
+		stratum:    count &log;
+		## The maximum interval between successive messages.
+		poll:       interval &log;
+		## The precision of the system clock.
+		precision:  interval &log;
+		## Total round-trip delay to the reference clock.
+		root_delay: interval &log;
+		## Total dispersion to the reference clock.
+		root_disp:  interval &log;
+		## For stratum 0, 4 character string used for debugging.
+		## For stratum 1, ID assigned to the reference clock by IANA.
+		## Above stratum 1, when using IPv4, the IP address of the reference
+		## clock.  Note that the NTP protocol did not originally specify a
+		## large enough field to represent IPv6 addresses, so they use
+		## the first four bytes of the MD5 hash of the reference clock's
+		## IPv6 address (i.e. an IPv4 address here is not necessarily IPv4).
+		ref_id:     string &log;
+		## Time when the system clock was last set or correct.
+		ref_time:   time &log;
+		## Time at the client when the request departed for the NTP server.
+		org_time:   time &log;
+		## Time at the server when the request arrived from the NTP client.
+		rec_time:   time &log;
+		## Time at the server when the response departed for the NTP client.
+		xmt_time:   time &log;
+		## Number of extension fields (which are not currently parsed).
+		num_exts:   count &default=0 &log;
+	};
+
+	## Event that can be handled to access the NTP record as it is sent on
+	## to the logging framework.
+	global log_ntp: event(rec: Info);
+}
+
+redef record connection += {
+	ntp: Info &optional;
+};
+
+const ports = { 123/udp };
+redef likely_server_ports += { ports };
+
+event ntp_message(c: connection, is_orig: bool, msg: NTP::Message) &priority=5
+	{
+	local info: Info;
+	info$ts  = network_time();
+	info$uid = c$uid;
+	info$id  = c$id;
+	info$version = msg$version;
+	info$mode = msg$mode;
+
+	if ( msg$mode < 6 )
+		{
+		info$stratum = msg$std_msg$stratum;
+		info$poll = msg$std_msg$poll;
+		info$precision = msg$std_msg$precision;
+		info$root_delay = msg$std_msg$root_delay;
+		info$root_disp = msg$std_msg$root_disp;
+
+		if ( msg$std_msg?$kiss_code )
+			info$ref_id = msg$std_msg$kiss_code;
+		else if ( msg$std_msg?$ref_id )
+			info$ref_id = msg$std_msg$ref_id;
+		else if ( msg$std_msg?$ref_addr )
+			info$ref_id= cat(msg$std_msg$ref_addr);
+
+		info$ref_time = msg$std_msg$ref_time;
+		info$org_time = msg$std_msg$org_time;
+		info$rec_time = msg$std_msg$rec_time;
+		info$xmt_time = msg$std_msg$xmt_time;
+
+		info$num_exts = msg$std_msg$num_exts;
+		}
+
+	# Copy the present packet info into the connection record
+	# If more ntp packets are sent on the same connection, the newest one
+	# will overwrite the previous
+	c$ntp = info;
+	}
+
+event ntp_message(c: connection, is_orig: bool, msg: NTP::Message) &priority=-5
+	{
+	if ( msg$mode < 6 )
+		Log::write(NTP::LOG, c$ntp);
+	}
+
+event zeek_init() &priority=5
+	{
+	Analyzer::register_for_ports(Analyzer::ANALYZER_NTP, ports);
+	Log::create_stream(NTP::LOG, [$columns = Info, $ev = log_ntp]);
+	}

scripts/base/protocols/radius/main.zeek

@@ -24,9 +24,10 @@ export {
 		## and the network access server is not required to honor 
 		## the address.
 		framed_addr  : addr     &log &optional;
-		## Remote IP address, if present.  This is collected
+		## Address (IPv4, IPv6, or FQDN) of the initiator end of the tunnel,
-		## from the Tunnel-Client-Endpoint attribute.
+		## if present.  This is collected from the Tunnel-Client-Endpoint
-		remote_ip    : addr     &log &optional;
+		## attribute.
+		tunnel_client: string   &log &optional;
 		## Connect info, if present.
 		connect_info : string   &log &optional;
 		## Reply message from the server challenge. This is 
@@ -85,8 +86,8 @@ event radius_message(c: connection, result: RADIUS::Message) &priority=5
 					c$radius$mac = normalize_mac(result$attributes[31][0]);
 
 				# Tunnel-Client-EndPoint (useful for VPNs)
-				if ( ! c$radius?$remote_ip && 66 in result$attributes )
+				if ( ! c$radius?$tunnel_client && 66 in result$attributes )
-					c$radius$remote_ip = to_addr(result$attributes[66][0]);
+					c$radius$tunnel_client = result$attributes[66][0];
 
 				# Connect-Info
 				if ( ! c$radius?$connect_info && 77 in result$attributes )

scripts/base/protocols/rdp/main.zeek

@@ -23,6 +23,8 @@ export {
 		result:                string  &log &optional;
 		## Security protocol chosen by the server.
 		security_protocol:     string &log &optional;
+		## The channels requested by the client
+		client_channels:       vector of string &log &optional;
 
 		## Keyboard layout (language) of the client machine.
 		keyboard_layout:       string  &log &optional;
@@ -189,6 +191,21 @@ event rdp_client_core_data(c: connection, data: RDP::ClientCoreData) &priority=5
 		c$rdp$requested_color_depth = RDP::high_color_depths[data$high_color_depth];
 	}
 
+event rdp_client_network_data(c: connection, channels: ClientChannelList)
+	{
+	set_session(c);
+
+	if ( ! c$rdp?$client_channels )
+		c$rdp$client_channels = vector();
+
+	for ( i in channels )
+		# Remove the NULs at the end
+		c$rdp$client_channels[i] = gsub(channels[i]$name, /\x00+$/, "");
+
+	if ( |channels| > 31 )
+		Reporter::conn_weird("RDP_channels_requested_exceeds_max", c, fmt("%s", |channels|));
+	}
+
 event rdp_gcc_server_create_response(c: connection, result: count) &priority=5
 	{
 	set_session(c);

scripts/policy/frameworks/netcontrol/catch-and-release.zeek

@@ -1,10 +1,9 @@
 ##! Implementation of catch-and-release functionality for NetControl.
 
-module NetControl;
+@load base/frameworks/netcontrol
-
 @load base/frameworks/cluster
-@load ./main
+
-@load ./drop
+module NetControl;
 
 export {
 

scripts/policy/frameworks/notice/actions/drop.zeek

@@ -1,8 +1,9 @@
 ##! This script extends the built in notice code to implement the IP address
 ##! dropping functionality.
 
-@load ../main
+@load base/frameworks/notice/main
 @load base/frameworks/netcontrol
+@load policy/frameworks/netcontrol/catch-and-release
 
 module Notice;
 

scripts/policy/misc/stats.zeek

@@ -99,7 +99,7 @@ event check_stats(then: time, last_ns: NetStats, last_cs: ConnStats, last_ps: Pr
 	local fs = get_file_analysis_stats();
 	local ds = get_dns_stats();
 
-	if ( bro_is_terminating() )
+	if ( zeek_is_terminating() )
 		# No more stats will be written or scheduled when Zeek is
 		# shutting down.
 		return;

scripts/policy/misc/trim-trace-file.zeek

@@ -17,7 +17,7 @@ export {
 
 event TrimTraceFile::go(first_trim: bool)
 	{
-	if ( bro_is_terminating() || trace_output_file == "" )
+	if ( zeek_is_terminating() || trace_output_file == "" )
 		return;
 	
 	if ( ! first_trim )

scripts/policy/protocols/conn/known-hosts.zeek

@@ -36,7 +36,7 @@ export {
 	global host_store: Cluster::StoreInfo;
 
 	## The Broker topic name to use for :zeek:see:`Known::host_store`.
-	const host_store_name = "bro/known/hosts" &redef;
+	const host_store_name = "zeek/known/hosts" &redef;
 
 	## The expiry interval of new entries in :zeek:see:`Known::host_store`.
 	## This also changes the interval at which hosts get logged.

scripts/policy/protocols/conn/known-services.zeek

@@ -48,7 +48,7 @@ export {
 	global service_store: Cluster::StoreInfo;
 
 	## The Broker topic name to use for :zeek:see:`Known::service_store`.
-	const service_store_name = "bro/known/services" &redef;
+	const service_store_name = "zeek/known/services" &redef;
 
 	## The expiry interval of new entries in :zeek:see:`Known::service_store`.
 	## This also changes the interval at which services get logged.

scripts/policy/protocols/ssl/known-certs.zeek

@@ -48,7 +48,7 @@ export {
 	global cert_store: Cluster::StoreInfo;
 
 	## The Broker topic name to use for :zeek:see:`Known::cert_store`.
-	const cert_store_name = "bro/known/certs" &redef;
+	const cert_store_name = "zeek/known/certs" &redef;
 
 	## The expiry interval of new entries in :zeek:see:`Known::cert_store`.
 	## This also changes the interval at which certs get logged.

scripts/test-all-policy.zeek

@@ -31,12 +31,16 @@
 @load frameworks/intel/seen/ssl.zeek
 @load frameworks/intel/seen/where-locations.zeek
 @load frameworks/intel/seen/x509.zeek
+@load frameworks/netcontrol/catch-and-release.zeek
 @load frameworks/files/detect-MHR.zeek
 @load frameworks/files/entropy-test-all-files.zeek
 #@load frameworks/files/extract-all-files.zeek
 @load frameworks/files/hash-all-files.zeek
 @load frameworks/notice/__load__.zeek
+@load frameworks/notice/actions/drop.zeek
 @load frameworks/notice/extend-email/hostnames.zeek
+@load files/unified2/__load__.zeek
+@load files/unified2/main.zeek
 @load files/x509/log-ocsp.zeek
 @load frameworks/packet-filter/shunt.zeek
 @load frameworks/software/version-changes.zeek

src/#OpaqueVal.h#

@@ -0,0 +1,213 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef OPAQUEVAL_H
+#define OPAQUEVAL_H
+
+#include <typeinfo>
+#include <memory> // std::unique_ptr
+
+#include "RandTest.h"
+#include "Val.h"
+#include "digest.h"
+#include "src/paraglob.h"
+
+namespace probabilistic {
+	class BloomFilter;
+	class CardinalityCounter;
+}
+
+class HashVal : public OpaqueVal {
+public:
+	virtual bool IsValid() const;
+	virtual bool Init();
+	virtual bool Feed(const void* data, size_t size);
+	virtual StringVal* Get();
+
+protected:
+	HashVal() { };
+	explicit HashVal(OpaqueType* t);
+
+	virtual bool DoInit();
+	virtual bool DoFeed(const void* data, size_t size);
+	virtual StringVal* DoGet();
+
+	DECLARE_SERIAL(HashVal);
+
+private:
+	// This flag exists because Get() can only be called once.
+	bool valid;
+};
+
+class MD5Val : public HashVal {
+public:
+	static void digest(val_list& vlist, u_char result[MD5_DIGEST_LENGTH]);
+
+	static void hmac(val_list& vlist,
+			 u_char key[MD5_DIGEST_LENGTH],
+			 u_char result[MD5_DIGEST_LENGTH]);
+
+	MD5Val();
+	~MD5Val();
+
+	Val* DoClone(CloneState* state) override;
+
+protected:
+	friend class Val;
+
+	bool DoInit() override;
+	bool DoFeed(const void* data, size_t size) override;
+	StringVal* DoGet() override;
+
+	DECLARE_SERIAL(MD5Val);
+
+private:
+	EVP_MD_CTX* ctx;
+};
+
+class SHA1Val : public HashVal {
+public:
+	static void digest(val_list& vlist, u_char result[SHA_DIGEST_LENGTH]);
+
+	SHA1Val();
+	~SHA1Val();
+
+	Val* DoClone(CloneState* state) override;
+
+protected:
+	friend class Val;
+
+	bool DoInit() override;
+	bool DoFeed(const void* data, size_t size) override;
+	StringVal* DoGet() override;
+
+	DECLARE_SERIAL(SHA1Val);
+
+private:
+	EVP_MD_CTX* ctx;
+};
+
+class SHA256Val : public HashVal {
+public:
+	static void digest(val_list& vlist, u_char result[SHA256_DIGEST_LENGTH]);
+
+	SHA256Val();
+	~SHA256Val();
+
+	Val* DoClone(CloneState* state) override;
+
+protected:
+	friend class Val;
+
+	bool DoInit() override;
+	bool DoFeed(const void* data, size_t size) override;
+	StringVal* DoGet() override;
+
+	DECLARE_SERIAL(SHA256Val);
+
+private:
+	EVP_MD_CTX* ctx;
+};
+
+class EntropyVal : public OpaqueVal {
+public:
+	EntropyVal();
+
+	Val* DoClone(CloneState* state) override;
+
+	bool Feed(const void* data, size_t size);
+	bool Get(double *r_ent, double *r_chisq, double *r_mean,
+		 double *r_montepicalc, double *r_scc);
+
+protected:
+	friend class Val;
+
+	DECLARE_SERIAL(EntropyVal);
+
+private:
+	RandTest state;
+};
+
+class BloomFilterVal : public OpaqueVal {
+public:
+	explicit BloomFilterVal(probabilistic::BloomFilter* bf);
+	~BloomFilterVal() override;
+
+	Val* DoClone(CloneState* state) override;
+
+	BroType* Type() const;
+	bool Typify(BroType* type);
+
+	void Add(const Val* val);
+	size_t Count(const Val* val) const;
+	void Clear();
+	bool Empty() const;
+	string InternalState() const;
+
+	static BloomFilterVal* Merge(const BloomFilterVal* x,
+				     const BloomFilterVal* y);
+
+protected:
+	friend class Val;
+	BloomFilterVal();
+	explicit BloomFilterVal(OpaqueType* t);
+
+	DECLARE_SERIAL(BloomFilterVal);
+
+private:
+	// Disable.
+	BloomFilterVal(const BloomFilterVal&);
+	BloomFilterVal& operator=(const BloomFilterVal&);
+
+	BroType* type;
+	CompositeHash* hash;
+	probabilistic::BloomFilter* bloom_filter;
+};
+
+
+class CardinalityVal: public OpaqueVal {
+public:
+	explicit CardinalityVal(probabilistic::CardinalityCounter*);
+	~CardinalityVal() override;
+
+	Val* DoClone(CloneState* state) override;
+
+	void Add(const Val* val);
+
+	BroType* Type() const;
+	bool Typify(BroType* type);
+
+
+	probabilistic::CardinalityCounter* Get()	{ return c; };
+
+protected:
+	CardinalityVal();
+
+private:
+	BroType* type;
+	CompositeHash* hash;
+	probabilistic::CardinalityCounter* c;
+
+	DECLARE_SERIAL(CardinalityVal);
+};
+
+class ParaglobVal : public OpaqueVal {
+public:
+	explicit ParaglobVal(std::unique_ptr<paraglob::Paraglob> p);
+	VectorVal* get(StringVal* &pattern);
+	Val* DoClone(CloneState* state) override;
+	bool operator==(const ParaglobVal& other) const;
+
+protected:
+	ParaglobVal() : OpaqueVal(paraglob_type) {}
+
+private:
+	std::unique_ptr<paraglob::Paraglob> internal_paraglob;
+	// Small convenience function. Does what std::make_unique does in C++14. Wont
+	// work on arrays.
+	template <typename T, typename ... Args>
+	std::unique_ptr<T> build_unique (Args&&... args) {
+		return std::unique_ptr<T>(new T(std::forward<Args>(args)...));
+	}
+
+	DECLARE_SERIAL(ParaglobVal)
+};

src/.#OpaqueVal.h

@@ -0,0 +1 @@
+zekemedley@zeke.6738:1560935202

src/Attr.cc

@@ -4,7 +4,6 @@
 
 #include "Attr.h"
 #include "Expr.h"
-#include "Serializer.h"
 #include "threading/SerialTypes.h"
 
 const char* attr_name(attr_tag t)
@@ -509,70 +508,3 @@ bool Attributes::operator==(const Attributes& other) const
 	return true;
 	}
 
-bool Attributes::Serialize(SerialInfo* info) const
-	{
-	return SerialObj::Serialize(info);
-	}
-
-Attributes* Attributes::Unserialize(UnserialInfo* info)
-	{
-	return (Attributes*) SerialObj::Unserialize(info, SER_ATTRIBUTES);
-	}
-
-IMPLEMENT_SERIAL(Attributes, SER_ATTRIBUTES);
-
-bool Attributes::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_ATTRIBUTES, BroObj);
-
-	info->s->WriteOpenTag("Attributes");
-	assert(type);
-	if ( ! (type->Serialize(info) && SERIALIZE(attrs->length())) )
-		return false;
-
-	loop_over_list((*attrs), i)
-		{
-		Attr* a = (*attrs)[i];
-
-		Expr* e = a->AttrExpr();
-		SERIALIZE_OPTIONAL(e);
-
-		if ( ! SERIALIZE(char(a->Tag())) )
-			return false;
-		}
-
-	info->s->WriteCloseTag("Attributes");
-	return true;
-	}
-
-bool Attributes::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(BroObj);
-
-	type = BroType::Unserialize(info);
-	if ( ! type )
-		return false;
-
-	int len;
-	if ( ! UNSERIALIZE(&len) )
-		return false;
-
-	attrs = new attr_list(len);
-	while ( len-- )
-		{
-		Expr* e;
-		UNSERIALIZE_OPTIONAL(e, Expr::Unserialize(info))
-
-		char tag;
-		if ( ! UNSERIALIZE(&tag) )
-			{
-			delete e;
-			return false;
-			}
-
-		attrs->append(new Attr((attr_tag)tag, e));
-		}
-
-	return true;
-	}
-

src/Attr.h

@@ -92,17 +92,12 @@ public:
 
 	attr_list* Attrs()	{ return attrs; }
 
-	bool Serialize(SerialInfo* info) const;
-	static Attributes* Unserialize(UnserialInfo* info);
-
 	bool operator==(const Attributes& other) const;
 
 protected:
 	Attributes() : type(), attrs(), in_record()	{ }
 	void CheckAttr(Attr* attr);
 
-	DECLARE_SERIAL(Attributes);
-
 	BroType* type;
 	attr_list* attrs;
 	bool in_record;

src/BroString.cc

@@ -288,7 +288,7 @@ void BroString::ToUpper()
 
 BroString* BroString::GetSubstring(int start, int len) const
 	{
-	// This code used to live in bro.bif's sub_bytes() routine.
+	// This code used to live in zeek.bif's sub_bytes() routine.
 	if ( start < 0 || start > n )
 		return 0;
 

src/CMakeLists.txt

@@ -10,8 +10,8 @@ set(bro_ALL_GENERATED_OUTPUTS  CACHE INTERNAL "automatically generated files" FO
 set(bro_AUTO_BIFS     CACHE INTERNAL "BIFs for automatic inclusion" FORCE)
 set(bro_REGISTER_BIFS CACHE INTERNAL "BIFs for automatic registering" FORCE)
 
-set(bro_BASE_BIF_SCRIPTS CACHE INTERNAL "Bro script stubs for BIFs in base distribution of Bro" FORCE)
+set(bro_BASE_BIF_SCRIPTS CACHE INTERNAL "Zeek script stubs for BIFs in base distribution of Zeek" FORCE)
-set(bro_PLUGIN_BIF_SCRIPTS CACHE INTERNAL "Bro script stubs for BIFs in Bro plugins" FORCE)
+set(bro_PLUGIN_BIF_SCRIPTS CACHE INTERNAL "Zeek script stubs for BIFs in Zeek plugins" FORCE)
 
 # If TRUE, use CMake's object libraries for sub-directories instead of
 # static libraries. This requires CMake >= 2.8.8.
@@ -103,7 +103,7 @@ set_property(SOURCE scan.cc APPEND_STRING PROPERTY COMPILE_FLAGS "-Wno-sign-comp
 include(BifCl)
 
 set(BIF_SRCS
-    bro.bif
+    zeek.bif
     stats.bif
     event.bif
     const.bif
@@ -247,7 +247,6 @@ set(bro_SRCS
     Brofiler.cc
     BroString.cc
     CCL.cc
-    ChunkedIO.cc
     CompHash.cc
     Conn.cc
     ConvertUTF.c
@@ -302,8 +301,6 @@ set(bro_SRCS
     SmithWaterman.cc
     Scope.cc
     SerializationFormat.cc
-    SerialObj.cc
-    Serializer.cc
     Sessions.cc
     StateAccess.cc
     Stats.cc
@@ -418,7 +415,7 @@ install(CODE "
 ")
 
 install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/
-        DESTINATION include/bro
+        DESTINATION include/zeek
         FILES_MATCHING
         PATTERN "*.h"
         PATTERN "*.pac"
@@ -426,7 +423,7 @@ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/
 )
 
 install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/
-        DESTINATION include/bro
+        DESTINATION include/zeek
         FILES_MATCHING
         PATTERN "*.bif.func_h"
         PATTERN "*.bif.netvar_h"
@@ -435,5 +432,5 @@ install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/
 )
 
 install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/sqlite3.h
-        DESTINATION include/bro/3rdparty
+        DESTINATION include/zeek/3rdparty
 )

src/ChunkedIO.h

@@ -1,362 +0,0 @@
-// Implements non-blocking chunk-wise I/O.
-
-#ifndef CHUNKEDIO_H
-#define CHUNKEDIO_H
-
-#include "zeek-config.h"
-#include "List.h"
-#include "util.h"
-#include "Flare.h"
-#include "iosource/FD_Set.h"
-#include <list>
-
-#ifdef NEED_KRB5_H
-# include <krb5.h>
-#endif
-
-class CompressedChunkedIO;
-
-// #define DEBUG_COMMUNICATION 10
-
-// Abstract base class.
-class ChunkedIO {
-public:
-	ChunkedIO();
-	virtual ~ChunkedIO()	{ }
-
-	struct Chunk {
-		typedef void (*FreeFunc)(char*);
-		static void free_func_free(char* data)	{ free(data); }
-		static void free_func_delete(char* data)	{ delete [] data; }
-
-		Chunk()
-		    : data(), len(), free_func(free_func_delete)
-			{ }
-
-		// Takes ownership of data.
-		Chunk(char* arg_data, uint32 arg_len,
-		      FreeFunc arg_ff = free_func_delete)
-			: data(arg_data), len(arg_len), free_func(arg_ff)
-			{ }
-
-		~Chunk()
-			{ free_func(data); }
-
-		char* data;
-		uint32 len;
-		FreeFunc free_func;
-	};
-
-	// Initialization before any I/O operation is performed. Returns false
-	// on any form of error.
-	virtual bool Init()	{ return true; }
-
-	// Tries to read the next chunk of data. If it can be read completely,
-	// a pointer to it is returned in 'chunk' (ownership of chunk is
-	// passed).  If not, 'chunk' is set to nil.  Returns false if any
-	// I/O error occurred (use Eof() to see if it's an end-of-file).
-	// If 'may_block' is true, we explicitly allow blocking.
-	virtual bool Read(Chunk** chunk, bool may_block = false) = 0;
-
-	// Puts the chunk into the write queue and writes as much data
-	// as possible (takes ownership of chunk).
-	// Returns false on any I/O error.
-	virtual bool Write(Chunk* chunk) = 0;
-
-	// Tries to write as much as currently possible.
-	// Returns false on any I/O error.
-	virtual bool Flush() = 0;
-
-	// If an I/O error has been encountered, returns a string describing it.
-	virtual const char* Error() = 0;
-
-	// Return true if there is currently at least one chunk available
-	// for reading.
-	virtual bool CanRead() = 0;
-
-	// Return true if there is currently at least one chunk waiting to be
-	// written.
-	virtual bool CanWrite() = 0;
-
-	// Returns true if source believes that there won't be much data soon.
-	virtual bool IsIdle() = 0;
-
-	// Returns true if internal write buffers are about to fill up.
-	virtual bool IsFillingUp() = 0;
-
-	// Throws away buffered data.
-	virtual void Clear() = 0;
-
-	// Returns true,if end-of-file has been reached.
-	virtual bool Eof() = 0;
-
-	// Returns underlying fd if available, -1 otherwise.
-	virtual int Fd()	{ return -1; }
-
-	// Returns supplementary file descriptors that become read-ready in order
-	// to signal that there is some work that can be performed.
-	virtual iosource::FD_Set ExtraReadFDs() const
-		{ return iosource::FD_Set(); }
-
-	// Makes sure that no additional protocol data is written into
-	// the output stream.  If this is activated, the output cannot
-	// be read again by any of these classes!
-	void MakePure()	{ pure = true; }
-	bool IsPure()	{ return pure; }
-
-	// Writes a log message to the error_fd.
-	void Log(const char* str);
-
-	struct Statistics {
-		Statistics()
-			{
-			bytes_read = 0;
-			bytes_written = 0;
-			chunks_read = 0;
-			chunks_written = 0;
-			reads = 0;
-			writes = 0;
-			pending = 0;
-			}
-
-		unsigned long bytes_read;
-		unsigned long bytes_written;
-		unsigned long chunks_read;
-		unsigned long chunks_written;
-		unsigned long reads;	// # calls which transferred > 0 bytes
-		unsigned long writes;
-		unsigned long pending;
-		};
-
-	// Returns raw statistics.
-	const Statistics* Stats() const 	{ return &stats; }
-
-	// Puts a formatted string containing statistics into buffer.
-	virtual void Stats(char* buffer, int length);
-
-#ifdef DEBUG_COMMUNICATION
-	void DumpDebugData(const char* basefnname, bool want_reads);
-#endif
-
-protected:
-	void InternalError(const char* msg)
-		// We can't use the reporter here as we might be running in a
-		// sub-process.
-		{ fprintf(stderr, "%s", msg); abort(); }
-
-	Statistics stats;
-	const char* tag;
-
-#ifdef DEBUG_COMMUNICATION
-	void AddToBuffer(char* data, bool is_read)
-		{ AddToBuffer(strlen(data), data, is_read); }
-	void AddToBuffer(uint32 len, char* data, bool is_read);
-	void AddToBuffer(Chunk* chunk, bool is_read);
-	std::list<Chunk*> data_read;
-	std::list<Chunk*> data_written;
-#endif
-
-private:
-	bool pure;
-};
-
-// Chunked I/O using a file descriptor.
-class ChunkedIOFd : public ChunkedIO {
-public:
-	// fd is an open bidirectional file descriptor, tag is used in error
-	// messages, and pid gives a pid to monitor (if the process dies, we
-	// return EOF).
-	ChunkedIOFd(int fd, const char* tag, pid_t pid = 0);
-	~ChunkedIOFd() override;
-
-	bool Read(Chunk** chunk, bool may_block = false) override;
-	bool Write(Chunk* chunk) override;
-	bool Flush() override;
-	const char* Error() override;
-	bool CanRead() override;
-	bool CanWrite() override;
-	bool IsIdle() override;
-	bool IsFillingUp() override;
-	void Clear() override;
-	bool Eof() override { return eof; }
-	int Fd() override { return fd; }
-	iosource::FD_Set ExtraReadFDs() const override;
-	void Stats(char* buffer, int length) override;
-
-private:
-
-	bool PutIntoWriteBuffer(Chunk* chunk);
-	bool FlushWriteBuffer();
-	Chunk* ExtractChunk();
-
-	// Returns size of next chunk in buffer or 0 if none.
-	uint32 ChunkAvailable();
-
-	// Flushes if it thinks it is time to.
-	bool OptionalFlush();
-
-	// Concatenates the the data of the two chunks forming a new one.
-	// The old chunkds are deleted.
-	Chunk* ConcatChunks(Chunk* c1, Chunk* c2);
-
-	// Reads/writes on chunk of upto BUFFER_SIZE bytes.
-	bool WriteChunk(Chunk* chunk, bool partial);
-	bool ReadChunk(Chunk** chunk, bool may_block);
-
-	int fd;
-	bool eof;
-	double last_flush;
-	int failed_reads;
-
-	// Optimally, this should match the file descriptor's
-	// buffer size (for sockets, it may be helpful to
-	// increase the send/receive buffers).
-	static const unsigned int BUFFER_SIZE = 1024 * 1024 * 1;
-
-	// We 'or' this to the length of a data chunk to mark
-	// that it's part of a larger one. This has to be larger
-	// than BUFFER_SIZE.
-	static const uint32 FLAG_PARTIAL = 0x80000000;
-
-	char* read_buffer;
-	uint32 read_len;
-	uint32 read_pos;
-	Chunk* partial;	// when we read an oversized chunk, we store it here
-
-	char* write_buffer;
-	uint32 write_len;
-	uint32 write_pos;
-
-	struct ChunkQueue {
-		Chunk* chunk;
-		ChunkQueue* next;
-	};
-
-	// Chunks that don't fit into our write buffer.
-	ChunkQueue* pending_head;
-	ChunkQueue* pending_tail;
-
-	pid_t pid;
-	bro::Flare write_flare;
-	bro::Flare read_flare;
-};
-
-// From OpenSSL. We forward-declare these here to avoid introducing a
-// dependency on OpenSSL headers just for this header file.
-typedef struct ssl_ctx_st SSL_CTX;
-typedef struct ssl_st SSL;
-
-// Chunked I/O using an SSL connection.
-class ChunkedIOSSL : public ChunkedIO {
-public:
-	// Argument is an open socket and a flag indicating whether we are the
-	// server side of the connection.
-	ChunkedIOSSL(int socket, bool server);
-	~ChunkedIOSSL() override;
-
-	bool Init() override;
-	bool Read(Chunk** chunk, bool mayblock = false) override;
-	bool Write(Chunk* chunk) override;
-	bool Flush() override;
-	const char* Error() override;
-	bool CanRead() override;
-	bool CanWrite() override;
-	bool IsIdle() override;
-	bool IsFillingUp() override;
-	void Clear() override;
-	bool Eof() override { return eof; }
-	int Fd() override { return socket; }
-	iosource::FD_Set ExtraReadFDs() const override;
-	void Stats(char* buffer, int length) override;
-
-private:
-
-	// Only returns true if all data has been read. If not, call
-	// it again with the same parameters as long as error is not
-	// set to true.
-	bool ReadData(char* p, uint32 len, bool* error);
-	// Same for writing.
-	bool WriteData(char* p, uint32 len, bool* error);
-
-	int socket;
-	int last_ret;	// last error code
-	bool eof;
-
-	bool server;	// are we the server?
-	bool setup;	// has the connection been setup successfully?
-
-	SSL* ssl;
-
-	// Write queue.
-	struct Queue {
-		Chunk* chunk;
-		Queue* next;
-	};
-
-	// The chunk part we are reading/writing
-	enum State { LEN, DATA };
-
-	State write_state;
-	Queue* write_head;
-	Queue* write_tail;
-
-	State read_state;
-	Chunk* read_chunk;
-	char* read_ptr;
-
-	// One SSL for all connections.
-	static SSL_CTX* ctx;
-
-	bro::Flare write_flare;
-};
-
-#include <zlib.h>
-
-// Wrapper class around a another ChunkedIO which the (un-)compresses data.
-class CompressedChunkedIO : public ChunkedIO {
-public:
-	explicit CompressedChunkedIO(ChunkedIO* arg_io) // takes ownership
-	    : io(arg_io), zin(), zout(), error(), compress(), uncompress(),
-	      uncompressed_bytes_read(), uncompressed_bytes_written() {}
-	~CompressedChunkedIO() override { delete io; }
-
-	bool Init() override; // does *not* call arg_io->Init()
-	bool Read(Chunk** chunk, bool may_block = false) override;
-	bool Write(Chunk* chunk) override;
-	bool Flush() override { return io->Flush(); }
-	const char* Error() override { return error ? error : io->Error(); }
-	bool CanRead() override { return io->CanRead(); }
-	bool CanWrite() override { return io->CanWrite(); }
-	bool IsIdle() override { return io->IsIdle(); }
-	bool IsFillingUp() override { return io->IsFillingUp(); }
-	void Clear() override { return io->Clear(); }
-	bool Eof() override { return io->Eof(); }
-
-	int Fd() override { return io->Fd(); }
-	iosource::FD_Set ExtraReadFDs() const override
-		{ return io->ExtraReadFDs(); }
-	void Stats(char* buffer, int length) override;
-
-	void EnableCompression(int level)
-		{ deflateInit(&zout, level); compress = true; }
-	void EnableDecompression()
-		{ inflateInit(&zin); uncompress = true; }
-
-protected:
-	// Only compress block with size >= this.
-	static const unsigned int MIN_COMPRESS_SIZE = 30;
-
-	ChunkedIO* io;
-	z_stream zin;
-	z_stream zout;
-	const char* error;
-
-	bool compress;
-	bool uncompress;
-
-	// Keep some statistics.
-	unsigned long uncompressed_bytes_read;
-	unsigned long uncompressed_bytes_written;
-};
-
-#endif

src/Conn.cc

@@ -50,70 +50,10 @@ void ConnectionTimer::Dispatch(double t, int is_expire)
 		reporter->InternalError("reference count inconsistency in ConnectionTimer::Dispatch");
 	}
 
-IMPLEMENT_SERIAL(ConnectionTimer, SER_CONNECTION_TIMER);
-
-bool ConnectionTimer::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_CONNECTION_TIMER, Timer);
-
-	// We enumerate all the possible timer functions here ... This
-	// has to match the list is DoUnserialize()!
-	char type = 0;
-
-	if ( timer == timer_func(&Connection::DeleteTimer) )
-		type = 1;
-	else if ( timer == timer_func(&Connection::InactivityTimer) )
-		type = 2;
-	else if ( timer == timer_func(&Connection::StatusUpdateTimer) )
-		type = 3;
-	else if ( timer == timer_func(&Connection::RemoveConnectionTimer) )
-		type = 4;
-	else
-		reporter->InternalError("unknown function in ConnectionTimer::DoSerialize()");
-
-	return conn->Serialize(info) && SERIALIZE(type) && SERIALIZE(do_expire);
-	}
-
-bool ConnectionTimer::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(Timer);
-
-	conn = Connection::Unserialize(info);
-	if ( ! conn )
-		return false;
-
-	char type;
-
-	if ( ! UNSERIALIZE(&type) || ! UNSERIALIZE(&do_expire) )
-		return false;
-
-	switch ( type ) {
-	case 1:
-		timer = timer_func(&Connection::DeleteTimer);
-		break;
-	case 2:
-		timer = timer_func(&Connection::InactivityTimer);
-		break;
-	case 3:
-		timer = timer_func(&Connection::StatusUpdateTimer);
-		break;
-	case 4:
-		timer = timer_func(&Connection::RemoveConnectionTimer);
-		break;
-	default:
-		info->s->Error("unknown connection timer function");
-		return false;
-	}
-
-	return true;
-	}
-
 uint64 Connection::total_connections = 0;
 uint64 Connection::current_connections = 0;
 uint64 Connection::external_connections = 0;
 
-IMPLEMENT_SERIAL(Connection, SER_CONNECTION);
-
 Connection::Connection(NetSessions* s, HashKey* k, double t, const ConnID* id,
                        uint32 flow, const Packet* pkt,
 		       const EncapsulationStack* arg_encap)
@@ -900,139 +840,6 @@ void Connection::IDString(ODesc* d) const
 	d->Add(ntohs(resp_port));
 	}
 
-bool Connection::Serialize(SerialInfo* info) const
-	{
-	return SerialObj::Serialize(info);
-	}
-
-Connection* Connection::Unserialize(UnserialInfo* info)
-	{
-	return (Connection*) SerialObj::Unserialize(info, SER_CONNECTION);
-	}
-
-bool Connection::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_CONNECTION, BroObj);
-
-	// First we write the members which are needed to
-	// create the HashKey.
-	if ( ! SERIALIZE(orig_addr) || ! SERIALIZE(resp_addr) )
-		return false;
-
-	if ( ! SERIALIZE(orig_port) || ! SERIALIZE(resp_port) )
-		return false;
-
-	if ( ! SERIALIZE(timers.length()) )
-		return false;
-
-	loop_over_list(timers, i)
-		if ( ! timers[i]->Serialize(info) )
-			return false;
-
-	SERIALIZE_OPTIONAL(conn_val);
-
-	// FIXME: RuleEndpointState not yet serializable.
-	// FIXME: Analyzers not yet serializable.
-
-	return
-		SERIALIZE(int(proto)) &&
-		SERIALIZE(history) &&
-		SERIALIZE(hist_seen) &&
-		SERIALIZE(start_time) &&
-		SERIALIZE(last_time) &&
-		SERIALIZE(inactivity_timeout) &&
-		SERIALIZE(suppress_event) &&
-		SERIALIZE(login_conn != 0) &&
-		SERIALIZE_BIT(installed_status_timer) &&
-		SERIALIZE_BIT(timers_canceled) &&
-		SERIALIZE_BIT(is_active) &&
-		SERIALIZE_BIT(skip) &&
-		SERIALIZE_BIT(weird) &&
-		SERIALIZE_BIT(finished) &&
-		SERIALIZE_BIT(record_packets) &&
-		SERIALIZE_BIT(record_contents);
-	}
-
-bool Connection::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(BroObj);
-
-	// Build the hash key first. Some of the recursive *::Unserialize()
-	// functions may need it.
-	ConnID id;
-
-	if ( ! UNSERIALIZE(&orig_addr) || ! UNSERIALIZE(&resp_addr) )
-		goto error;
-
-	if ( ! UNSERIALIZE(&orig_port) || ! UNSERIALIZE(&resp_port) )
-		goto error;
-
-	id.src_addr = orig_addr;
-	id.dst_addr = resp_addr;
-	// This doesn't work for ICMP. But I guess this is not really important.
-	id.src_port = orig_port;
-	id.dst_port = resp_port;
-	id.is_one_way = 0;	// ### incorrect for ICMP
-	key = BuildConnIDHashKey(id);
-
-	int len;
-	if ( ! UNSERIALIZE(&len) )
-		goto error;
-
-	while ( len-- )
-		{
-		Timer* t = Timer::Unserialize(info);
-		if ( ! t )
-			goto error;
-		timers.append(t);
-		}
-
-	UNSERIALIZE_OPTIONAL(conn_val,
-			(RecordVal*) Val::Unserialize(info, connection_type));
-
-	int iproto;
-
-	if ( ! (UNSERIALIZE(&iproto) &&
-		UNSERIALIZE(&history) &&
-		UNSERIALIZE(&hist_seen) &&
-		UNSERIALIZE(&start_time) &&
-		UNSERIALIZE(&last_time) &&
-		UNSERIALIZE(&inactivity_timeout) &&
-		UNSERIALIZE(&suppress_event)) )
-		goto error;
-
-	proto = static_cast<TransportProto>(iproto);
-
-	bool has_login_conn;
-	if ( ! UNSERIALIZE(&has_login_conn) )
-		goto error;
-
-	login_conn = has_login_conn ? (LoginConn*) this : 0;
-
-	UNSERIALIZE_BIT(installed_status_timer);
-	UNSERIALIZE_BIT(timers_canceled);
-	UNSERIALIZE_BIT(is_active);
-	UNSERIALIZE_BIT(skip);
-	UNSERIALIZE_BIT(weird);
-	UNSERIALIZE_BIT(finished);
-	UNSERIALIZE_BIT(record_packets);
-	UNSERIALIZE_BIT(record_contents);
-
-	// Hmm... Why does each connection store a sessions ptr?
-	sessions = ::sessions;
-
-	root_analyzer = 0;
-	primary_PIA = 0;
-	conn_timer_mgr = 0;
-
-	return true;
-
-error:
-	abort();
-	CancelTimers();
-	return false;
-	}
-
 void Connection::SetRootAnalyzer(analyzer::TransportLayerAnalyzer* analyzer, analyzer::pia::PIA* pia)
 	{
 	root_analyzer = analyzer;

src/Conn.h

@@ -11,7 +11,6 @@
 #include "Dict.h"
 #include "Val.h"
 #include "Timer.h"
-#include "Serializer.h"
 #include "RuleMatcher.h"
 #include "IPAddr.h"
 #include "TunnelEncapsulation.h"
@@ -235,11 +234,6 @@ public:
 	// Returns true if connection has been received externally.
 	bool IsExternal() const	{ return conn_timer_mgr != 0; }
 
-	bool Serialize(SerialInfo* info) const;
-	static Connection* Unserialize(UnserialInfo* info);
-
-	DECLARE_SERIAL(Connection);
-
 	// Statistics.
 
 	// Just a lower bound.
@@ -385,8 +379,6 @@ protected:
 
 	void Init(Connection* conn, timer_func timer, int do_expire);
 
-	DECLARE_SERIAL(ConnectionTimer);
-
 	Connection* conn;
 	timer_func timer;
 	int do_expire;

src/Debug.cc

@@ -721,7 +721,7 @@ static char* get_prompt(bool reset_counter = false)
 	if ( reset_counter )
 		counter = 0;
 
-	safe_snprintf(prompt, sizeof(prompt), "(Bro [%d]) ", counter++);
+	safe_snprintf(prompt, sizeof(prompt), "(Zeek [%d]) ", counter++);
 
 	return prompt;
 	}

src/DebugCmdInfoConstants.cc

@@ -35,7 +35,7 @@ void init_global_dbg_constants () {
 	"quit"
       };
 
-      info = new DebugCmdInfo (dcQuit, names, 1, false, "Exit Bro",
+      info = new DebugCmdInfo (dcQuit, names, 1, false, "Exit Zeek",
                                false);
       g_DebugCmdInfos.push_back(info);
    }

src/DebugCmdInfoConstants.in

@@ -13,7 +13,7 @@ help: Get help with debugger commands
 cmd: dcQuit
 names: quit
 resume: false
-help: Exit Bro
+help: Exit Zeek
 
 cmd: dcNext
 names: next

src/DebugLogger.cc

@@ -12,8 +12,7 @@ DebugLogger debug_logger;
 // Same order here as in DebugStream.
 DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = {
 	{ "serial", 0, false }, { "rules", 0, false },
-	{ "state", 0, false }, { "chunkedio", 0, false },
+	{ "state", 0, false }, {"string", 0, false },
-	{"string", 0, false },
 	{ "notifiers", 0, false },  { "main-loop", 0, false },
 	{ "dpd", 0, false }, { "tm", 0, false },
 	{ "logging", 0, false }, {"input", 0, false },
@@ -72,7 +71,7 @@ void DebugLogger::ShowStreamsHelp()
 		fprintf(stderr,"  %s\n", streams[i].prefix);
 
 	fprintf(stderr, "\n");
-	fprintf(stderr, "  plugin-<plugin-name>   (replace '::' in name with '-'; e.g., '-B plugin-Bro-Netmap')\n");
+	fprintf(stderr, "  plugin-<plugin-name>   (replace '::' in name with '-'; e.g., '-B plugin-Zeek-Netmap')\n");
 	fprintf(stderr, "\n");
 	fprintf(stderr, "Pseudo streams\n");
 	fprintf(stderr, "  verbose  Increase verbosity.\n");

src/DebugLogger.h

@@ -17,7 +17,6 @@ enum DebugStream {
 	DBG_SERIAL, // Serialization
 	DBG_RULES,	// Signature matching
 	DBG_STATE,	// StateAccess logging
-	DBG_CHUNKEDIO,	// ChunkedIO logging
 	DBG_STRING,	// String code
 	DBG_NOTIFIERS,	// Notifiers (see StateAccess.h)
 	DBG_MAINLOOP,	// Main IOSource loop

src/Event.cc

@@ -58,12 +58,6 @@ void Event::Dispatch(bool no_remote)
 	if ( src == SOURCE_BROKER )
 		no_remote = true;
 
-	if ( event_serializer )
-		{
-		SerialInfo info(event_serializer);
-		event_serializer->Serialize(&info, handler->Name(), &args);
-		}
-
 	if ( handler->ErrorHandler() )
 		reporter->BeginErrorHandler();
 

src/Event.h

@@ -4,7 +4,6 @@
 #define event_h
 
 #include "EventRegistry.h"
-#include "Serializer.h"
 
 #include "analyzer/Tag.h"
 #include "analyzer/Analyzer.h"

src/EventHandler.cc

@@ -171,23 +171,3 @@ void EventHandler::NewEvent(val_list* vl)
 	mgr.Dispatch(ev);
 	}
 
-bool EventHandler::Serialize(SerialInfo* info) const
-	{
-	return SERIALIZE(name);
-	}
-
-EventHandler* EventHandler::Unserialize(UnserialInfo* info)
-	{
-	char* name;
-	if ( ! UNSERIALIZE_STR(&name, 0) )
-		return 0;
-
-	EventHandler* h = event_registry->Lookup(name);
-	if ( ! h )
-		{
-		h = new EventHandler(name);
-		event_registry->Register(h);
-		}
-
-	return h;
-	}

src/EventHandler.h

@@ -11,9 +11,6 @@
 
 class Func;
 class FuncType;
-class Serializer;
-class SerialInfo;
-class UnserialInfo;
 
 class EventHandler {
 public:
@@ -56,11 +53,6 @@ public:
 	void SetGenerateAlways()	{ generate_always = true; }
 	bool GenerateAlways()	{ return generate_always; }
 
-	// We don't serialize the handler(s) itself here, but
-	// just the reference to it.
-	bool Serialize(SerialInfo* info) const;
-	static EventHandler* Unserialize(UnserialInfo* info);
-
 private:
 	void NewEvent(val_list* vl);	// Raise new_event() meta event.
 

src/Expr.h

@@ -49,7 +49,8 @@ typedef enum {
 	EXPR_FLATTEN,
 	EXPR_CAST,
 	EXPR_IS,
-#define NUM_EXPRS (int(EXPR_IS) + 1)
+	EXPR_INDEX_SLICE_ASSIGN,
+#define NUM_EXPRS (int(EXPR_INDEX_SLICE_ASSIGN) + 1)
 } BroExprTag;
 
 extern const char* expr_name(BroExprTag t);
@@ -58,6 +59,7 @@ class Stmt;
 class Frame;
 class ListExpr;
 class NameExpr;
+class IndexExpr;
 class AssignExpr;
 class CallExpr;
 class EventExpr;
@@ -187,10 +189,19 @@ public:
 		return (AssignExpr*) this;
 		}
 
-	void Describe(ODesc* d) const override;
+	const IndexExpr* AsIndexExpr() const
+		{
+		CHECK_TAG(tag, EXPR_INDEX, "ExprVal::AsIndexExpr", expr_name)
+		return (const IndexExpr*) this;
+		}
 
-	bool Serialize(SerialInfo* info) const;
+	IndexExpr* AsIndexExpr()
-	static Expr* Unserialize(UnserialInfo* info, BroExprTag want = EXPR_ANY);
+		{
+		CHECK_TAG(tag, EXPR_INDEX, "ExprVal::AsIndexExpr", expr_name)
+		return (IndexExpr*) this;
+		}
+
+	void Describe(ODesc* d) const override;
 
 	virtual TraversalCode Traverse(TraversalCallback* cb) const = 0;
 
@@ -214,8 +225,6 @@ protected:
 
 	void RuntimeErrorWithCallStack(const std::string& msg) const;
 
-	DECLARE_ABSTRACT_SERIAL(Expr);
-
 	BroExprTag tag;
 	BroType* type;
 
@@ -242,8 +251,6 @@ protected:
 
 	void ExprDescribe(ODesc* d) const override;
 
-	DECLARE_SERIAL(NameExpr);
-
 	ID* id;
 	bool in_const_init;
 };
@@ -264,8 +271,6 @@ protected:
 	ConstExpr()	{ val = 0; }
 
 	void ExprDescribe(ODesc* d) const override;
-	DECLARE_SERIAL(ConstExpr);
-
 	Val* val;
 };
 
@@ -294,8 +299,6 @@ protected:
 	// Returns the expression folded using the given constant.
 	virtual Val* Fold(Val* v) const;
 
-	DECLARE_SERIAL(UnaryExpr);
-
 	Expr* op;
 };
 
@@ -357,8 +360,6 @@ protected:
 
 	void ExprDescribe(ODesc* d) const override;
 
-	DECLARE_SERIAL(BinaryExpr);
-
 	Expr* op1;
 	Expr* op2;
 };
@@ -373,8 +374,6 @@ protected:
 	CloneExpr()	{ }
 
 	Val* Fold(Val* v) const override;
-
-	DECLARE_SERIAL(CloneExpr);
 };
 
 class IncrExpr : public UnaryExpr {
@@ -388,8 +387,6 @@ public:
 protected:
 	friend class Expr;
 	IncrExpr()	{ }
-
-	DECLARE_SERIAL(IncrExpr);
 };
 
 class ComplementExpr : public UnaryExpr {
@@ -401,8 +398,6 @@ protected:
 	ComplementExpr()	{ }
 
 	Val* Fold(Val* v) const override;
-
-	DECLARE_SERIAL(ComplementExpr);
 };
 
 class NotExpr : public UnaryExpr {
@@ -414,8 +409,6 @@ protected:
 	NotExpr()	{ }
 
 	Val* Fold(Val* v) const override;
-
-	DECLARE_SERIAL(NotExpr);
 };
 
 class PosExpr : public UnaryExpr {
@@ -427,8 +420,6 @@ protected:
 	PosExpr()	{ }
 
 	Val* Fold(Val* v) const override;
-
-	DECLARE_SERIAL(PosExpr);
 };
 
 class NegExpr : public UnaryExpr {
@@ -440,8 +431,6 @@ protected:
 	NegExpr()	{ }
 
 	Val* Fold(Val* v) const override;
-
-	DECLARE_SERIAL(NegExpr);
 };
 
 class SizeExpr : public UnaryExpr {
@@ -454,7 +443,6 @@ protected:
 	SizeExpr()	{ }
 
 	Val* Fold(Val* v) const override;
-	DECLARE_SERIAL(SizeExpr);
 };
 
 class AddExpr : public BinaryExpr {
@@ -465,9 +453,6 @@ public:
 protected:
 	friend class Expr;
 	AddExpr()	{ }
-
-	DECLARE_SERIAL(AddExpr);
-
 };
 
 class AddToExpr : public BinaryExpr {
@@ -478,8 +463,6 @@ public:
 protected:
 	friend class Expr;
 	AddToExpr()	{ }
-
-	DECLARE_SERIAL(AddToExpr);
 };
 
 class RemoveFromExpr : public BinaryExpr {
@@ -490,8 +473,6 @@ public:
 protected:
 	friend class Expr;
 	RemoveFromExpr()	{ }
-
-	DECLARE_SERIAL(RemoveFromExpr);
 };
 
 class SubExpr : public BinaryExpr {
@@ -501,9 +482,6 @@ public:
 protected:
 	friend class Expr;
 	SubExpr()	{ }
-
-	DECLARE_SERIAL(SubExpr);
-
 };
 
 class TimesExpr : public BinaryExpr {
@@ -514,9 +492,6 @@ public:
 protected:
 	friend class Expr;
 	TimesExpr()	{ }
-
-	DECLARE_SERIAL(TimesExpr);
-
 };
 
 class DivideExpr : public BinaryExpr {
@@ -528,9 +503,6 @@ protected:
 	DivideExpr()	{ }
 
 	Val* AddrFold(Val* v1, Val* v2) const override;
-
-	DECLARE_SERIAL(DivideExpr);
-
 };
 
 class ModExpr : public BinaryExpr {
@@ -540,8 +512,6 @@ public:
 protected:
 	friend class Expr;
 	ModExpr()	{ }
-
-	DECLARE_SERIAL(ModExpr);
 };
 
 class BoolExpr : public BinaryExpr {
@@ -554,8 +524,6 @@ public:
 protected:
 	friend class Expr;
 	BoolExpr()	{ }
-
-	DECLARE_SERIAL(BoolExpr);
 };
 
 class BitExpr : public BinaryExpr {
@@ -565,8 +533,6 @@ public:
 protected:
 	friend class Expr;
 	BitExpr()	{ }
-
-	DECLARE_SERIAL(BitExpr);
 };
 
 class EqExpr : public BinaryExpr {
@@ -579,8 +545,6 @@ protected:
 	EqExpr()	{ }
 
 	Val* Fold(Val* v1, Val* v2) const override;
-
-	DECLARE_SERIAL(EqExpr);
 };
 
 class RelExpr : public BinaryExpr {
@@ -591,8 +555,6 @@ public:
 protected:
 	friend class Expr;
 	RelExpr()	{ }
-
-	DECLARE_SERIAL(RelExpr);
 };
 
 class CondExpr : public Expr {
@@ -615,8 +577,6 @@ protected:
 
 	void ExprDescribe(ODesc* d) const override;
 
-	DECLARE_SERIAL(CondExpr);
-
 	Expr* op1;
 	Expr* op2;
 	Expr* op3;
@@ -632,8 +592,6 @@ public:
 protected:
 	friend class Expr;
 	RefExpr()	{ }
-
-	DECLARE_SERIAL(RefExpr);
 };
 
 class AssignExpr : public BinaryExpr {
@@ -657,12 +615,20 @@ protected:
 	bool TypeCheck(attr_list* attrs = 0);
 	bool TypeCheckArithmetics(TypeTag bt1, TypeTag bt2);
 
-	DECLARE_SERIAL(AssignExpr);
-
 	int is_init;
 	Val* val;	// optional
 };
 
+class IndexSliceAssignExpr : public AssignExpr {
+public:
+	IndexSliceAssignExpr(Expr* op1, Expr* op2, int is_init);
+	Val* Eval(Frame* f) const override;
+
+protected:
+	friend class Expr;
+	IndexSliceAssignExpr() {}
+};
+
 class IndexExpr : public BinaryExpr {
 public:
 	IndexExpr(Expr* op1, ListExpr* op2, bool is_slice = false);
@@ -682,6 +648,8 @@ public:
 
 	TraversalCode Traverse(TraversalCallback* cb) const override;
 
+	bool IsSlice() const { return is_slice; }
+
 protected:
 	friend class Expr;
 	IndexExpr()	{ }
@@ -690,7 +658,7 @@ protected:
 
 	void ExprDescribe(ODesc* d) const override;
 
-	DECLARE_SERIAL(IndexExpr);
+	bool is_slice;
 };
 
 class FieldExpr : public UnaryExpr {
@@ -716,8 +684,6 @@ protected:
 
 	void ExprDescribe(ODesc* d) const override;
 
-	DECLARE_SERIAL(FieldExpr);
-
 	const char* field_name;
 	const TypeDecl* td;
 	int field; // -1 = attributes
@@ -740,8 +706,6 @@ protected:
 
 	void ExprDescribe(ODesc* d) const override;
 
-	DECLARE_SERIAL(HasFieldExpr);
-
 	const char* field_name;
 	int field;
 };
@@ -759,8 +723,6 @@ protected:
 	Val* Fold(Val* v) const override;
 
 	void ExprDescribe(ODesc* d) const override;
-
-	DECLARE_SERIAL(RecordConstructorExpr);
 };
 
 class TableConstructorExpr : public UnaryExpr {
@@ -781,8 +743,6 @@ protected:
 
 	void ExprDescribe(ODesc* d) const override;
 
-	DECLARE_SERIAL(TableConstructorExpr);
-
 	Attributes* attrs;
 };
 
@@ -804,8 +764,6 @@ protected:
 
 	void ExprDescribe(ODesc* d) const override;
 
-	DECLARE_SERIAL(SetConstructorExpr);
-
 	Attributes* attrs;
 };
 
@@ -822,8 +780,6 @@ protected:
 	Val* InitVal(const BroType* t, Val* aggr) const override;
 
 	void ExprDescribe(ODesc* d) const override;
-
-	DECLARE_SERIAL(VectorConstructorExpr);
 };
 
 class FieldAssignExpr : public UnaryExpr {
@@ -841,8 +797,6 @@ protected:
 
 	void ExprDescribe(ODesc* d) const override;
 
-	DECLARE_SERIAL(FieldAssignExpr);
-
 	string field_name;
 };
 
@@ -856,8 +810,6 @@ protected:
 
 	Val* FoldSingleVal(Val* v, InternalTypeTag t) const;
 	Val* Fold(Val* v) const override;
-
-	DECLARE_SERIAL(ArithCoerceExpr);
 };
 
 class RecordCoerceExpr : public UnaryExpr {
@@ -872,8 +824,6 @@ protected:
 	Val* InitVal(const BroType* t, Val* aggr) const override;
 	Val* Fold(Val* v) const override;
 
-	DECLARE_SERIAL(RecordCoerceExpr);
-
 	// For each super-record slot, gives subrecord slot with which to
 	// fill it.
 	int* map;
@@ -890,8 +840,6 @@ protected:
 	TableCoerceExpr()	{ }
 
 	Val* Fold(Val* v) const override;
-
-	DECLARE_SERIAL(TableCoerceExpr);
 };
 
 class VectorCoerceExpr : public UnaryExpr {
@@ -904,8 +852,6 @@ protected:
 	VectorCoerceExpr()	{ }
 
 	Val* Fold(Val* v) const override;
-
-	DECLARE_SERIAL(VectorCoerceExpr);
 };
 
 // An internal operator for flattening array indices that are records
@@ -920,8 +866,6 @@ protected:
 
 	Val* Fold(Val* v) const override;
 
-	DECLARE_SERIAL(FlattenExpr);
-
 	int num_fields;
 };
 
@@ -961,8 +905,6 @@ protected:
 
 	void ExprDescribe(ODesc* d) const override;
 
-	DECLARE_SERIAL(ScheduleExpr);
-
 	Expr* when;
 	EventExpr* event;
 };
@@ -977,8 +919,6 @@ protected:
 
 	Val* Fold(Val* v1, Val* v2) const override;
 
-	DECLARE_SERIAL(InExpr);
-
 };
 
 class CallExpr : public Expr {
@@ -1001,8 +941,6 @@ protected:
 
 	void ExprDescribe(ODesc* d) const override;
 
-	DECLARE_SERIAL(CallExpr);
-
 	Expr* func;
 	ListExpr* args;
 };
@@ -1026,8 +964,6 @@ protected:
 
 	void ExprDescribe(ODesc* d) const override;
 
-	DECLARE_SERIAL(EventExpr);
-
 	string name;
 	EventHandlerPtr handler;
 	ListExpr* args;
@@ -1064,8 +1000,6 @@ protected:
 
 	void ExprDescribe(ODesc* d) const override;
 
-	DECLARE_SERIAL(ListExpr);
-
 	expr_list exprs;
 };
 
@@ -1079,8 +1013,6 @@ public:
 protected:
 	friend class Expr;
 	RecordAssignExpr()	{ }
-
-	DECLARE_SERIAL(RecordAssignExpr);
 };
 
 class CastExpr : public UnaryExpr {
@@ -1093,8 +1025,6 @@ protected:
 
 	Val* Eval(Frame* f) const override;
 	void ExprDescribe(ODesc* d) const override;
-
-	DECLARE_SERIAL(CastExpr);
 };
 
 class IsExpr : public UnaryExpr {
@@ -1108,7 +1038,6 @@ protected:
 
 	Val* Fold(Val* v) const override;
 	void ExprDescribe(ODesc* d) const override;
-	DECLARE_SERIAL(IsExpr);
 
 private:
 	BroType* t;

src/File.cc

@@ -25,7 +25,6 @@
 #include "Expr.h"
 #include "NetVar.h"
 #include "Net.h"
-#include "Serializer.h"
 #include "Event.h"
 #include "Reporter.h"
 
@@ -518,11 +517,6 @@ double BroFile::Size()
 	return s.st_size;
 	}
 
-bool BroFile::Serialize(SerialInfo* info) const
-	{
-	return SerialObj::Serialize(info);
-	}
-
 BroFile* BroFile::GetFile(const char* name)
 	{
 	for ( BroFile* f = head; f; f = f->next )
@@ -534,135 +528,3 @@ BroFile* BroFile::GetFile(const char* name)
 	return new BroFile(name, "w", 0);
 	}
 
-BroFile* BroFile::Unserialize(UnserialInfo* info)
-	{
-	BroFile* file = (BroFile*) SerialObj::Unserialize(info, SER_BRO_FILE);
-
-	if ( ! file )
-		return 0;
-
-	if ( file->is_open )
-		return file;
-
-	// If there is already an object for this file, return it.
-	if ( file->name )
-		{
-		for ( BroFile* f = head; f; f = f->next )
-			{
-			if ( f->name && streq(file->name, f->name) )
-				{
-				Unref(file);
-				Ref(f);
-				return f;
-				}
-			}
-		}
-
-	// Otherwise, open, but don't clobber.
-	if ( ! file->Open(0, "a") )
-		{
-		info->s->Error(fmt("cannot open %s: %s",
-					file->name, strerror(errno)));
-		return 0;
-		}
-
-	// Here comes a hack.  This method will return a pointer to a newly
-	// instantiated file object.  As soon as this pointer is Unref'ed, the
-	// file will be closed.  That means that when we unserialize the same
-	// file next time, we will re-open it and thereby delete the first one,
-	// i.e., we will be keeping to delete what we've written just before.
-	//
-	// To avoid this loop, we do an extra Ref here, i.e., this file will
-	// *never* be closed anymore (as long the file cache does not overflow).
-	Ref(file);
-
-	file->SetBuf(file->buffered);
-
-	return file;
-	}
-
-IMPLEMENT_SERIAL(BroFile, SER_BRO_FILE);
-
-bool BroFile::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_BRO_FILE, BroObj);
-
-	const char* s = name;
-
-	if ( ! okay_to_manage )
-		{
-		// We can handle stdin/stdout/stderr but no others.
-		if ( f == stdin )
-			s = "/dev/stdin";
-		else if ( f == stdout )
-			s = "/dev/stdout";
-		else if ( f == stderr )
-			s = "/dev/stderr";
-		else
-			{
-			// We don't manage the file, and therefore don't
-			// really know how to pass it on to the other side.
-			// However, in order to not abort communication
-			// when this happens, we still send the name if we
-			// have one; or if we don't, we create a special
-			// "dont-have-a-file" file to be created on the
-			// receiver side.
-			if ( ! s )
-				s = "unmanaged-bro-output-file.log";
-			}
-		}
-
-	if ( ! (SERIALIZE(s) && SERIALIZE(buffered)) )
-		return false;
-
-	SERIALIZE_OPTIONAL_STR(access);
-
-	if ( ! t->Serialize(info) )
-		return false;
-
-	SERIALIZE_OPTIONAL(attrs);
-	return true;
-	}
-
-bool BroFile::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(BroObj);
-
-	if ( ! (UNSERIALIZE_STR(&name, 0) && UNSERIALIZE(&buffered)) )
-		return false;
-
-	UNSERIALIZE_OPTIONAL_STR(access);
-
-	t = BroType::Unserialize(info);
-	if ( ! t )
-		return false;
-
-	UNSERIALIZE_OPTIONAL(attrs, Attributes::Unserialize(info));
-
-	// Parse attributes.
-	SetAttrs(attrs);
-	// SetAttrs() has ref'ed attrs again.
-	Unref(attrs);
-
-	// Bind stdin/stdout/stderr.
-	FILE* file = 0;
-	is_open = false;
-	f = 0;
-
-	if ( streq(name, "/dev/stdin") )
-		file = stdin;
-	else if ( streq(name, "/dev/stdout") )
-		file = stdout;
-	else if ( streq(name, "/dev/stderr") )
-		file = stderr;
-
-	if ( file )
-		{
-		delete [] name;
-		name = 0;
-		f = file;
-		is_open = true;
-		}
-
-	return true;
-	}

src/File.h

@@ -66,9 +66,6 @@ public:
 	void EnableRawOutput()		{ raw_output = true; }
 	bool IsRawOutput() const	{ return raw_output; }
 
-	bool Serialize(SerialInfo* info) const;
-	static BroFile* Unserialize(UnserialInfo* info);
-
 protected:
 	BroFile()	{ Init(); }
 	void Init();
@@ -100,8 +97,6 @@ protected:
 	// Raises a file_opened event.
 	void RaiseOpenEvent();
 
-	DECLARE_SERIAL(BroFile);
-
 	FILE* f;
 	BroType* t;
 	char* name;

src/Func.cc

@@ -41,7 +41,6 @@
 #include "analyzer/protocol/login/Login.h"
 #include "Sessions.h"
 #include "RE.h"
-#include "Serializer.h"
 #include "Event.h"
 #include "Traverse.h"
 #include "Reporter.h"
@@ -127,110 +126,6 @@ void Func::AddBody(Stmt* /* new_body */, id_list* /* new_inits */,
 	Internal("Func::AddBody called");
 	}
 
-bool Func::Serialize(SerialInfo* info) const
-	{
-	return SerialObj::Serialize(info);
-	}
-
-Func* Func::Unserialize(UnserialInfo* info)
-	{
-	Func* f = (Func*) SerialObj::Unserialize(info, SER_FUNC);
-
-	// For builtins, we return a reference to the (hopefully) already
-	// existing function.
-	if ( f && f->kind == BUILTIN_FUNC )
-		{
-		const char* name = ((BuiltinFunc*) f)->Name();
-		ID* id = global_scope()->Lookup(name);
-		if ( ! id )
-			{
-			info->s->Error(fmt("can't find built-in %s", name));
-			return 0;
-			}
-
-		if ( ! (id->HasVal() && id->ID_Val()->Type()->Tag() == TYPE_FUNC) )
-			{
-			info->s->Error(fmt("ID %s is not a built-in", name));
-			return 0;
-			}
-
-		Unref(f);
-		f = id->ID_Val()->AsFunc();
-		Ref(f);
-		}
-
-	return f;
-	}
-
-bool Func::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_FUNC, BroObj);
-
-	if ( ! SERIALIZE(int(bodies.size())) )
-		return false;
-
-	for ( unsigned int i = 0; i < bodies.size(); ++i )
-		{
-		if ( ! bodies[i].stmts->Serialize(info) )
-			return false;
-		if ( ! SERIALIZE(bodies[i].priority) )
-			return false;
-		}
-
-	if ( ! SERIALIZE(char(kind) ) )
-		return false;
-
-	if ( ! type->Serialize(info) )
-		return false;
-
-	if ( ! SERIALIZE(Name()) )
-		return false;
-
-	// We don't serialize scope as only global functions are considered here
-	// anyway.
-	return true;
-	}
-
-bool Func::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(BroObj);
-
-	int len;
-	if ( ! UNSERIALIZE(&len) )
-		return false;
-
-	while ( len-- )
-		{
-		Body b;
-		b.stmts = Stmt::Unserialize(info);
-		if ( ! b.stmts )
-			return false;
-
-		if ( ! UNSERIALIZE(&b.priority) )
-			return false;
-
-		bodies.push_back(b);
-		}
-
-	char c;
-	if ( ! UNSERIALIZE(&c) )
-		return false;
-
-	kind = (Kind) c;
-
-	type = BroType::Unserialize(info);
-	if ( ! type )
-		return false;
-
-	const char* n;
-	if ( ! UNSERIALIZE_STR(&n, 0) )
-		return false;
-
-	name = n;
-	delete [] n;
-
-	return true;
-	}
 
 void Func::DescribeDebug(ODesc* d, const val_list* args) const
 	{
@@ -584,21 +479,6 @@ Stmt* BroFunc::AddInits(Stmt* body, id_list* inits)
 	return stmt_series;
 	}
 
-IMPLEMENT_SERIAL(BroFunc, SER_BRO_FUNC);
-
-bool BroFunc::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_BRO_FUNC, Func);
-	return SERIALIZE(frame_size);
-	}
-
-bool BroFunc::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(Func);
-
-	return UNSERIALIZE(&frame_size);
-	}
-
 BuiltinFunc::BuiltinFunc(built_in_func arg_func, const char* arg_name,
 			int arg_is_pure)
 : Func(BUILTIN_FUNC)
@@ -681,20 +561,6 @@ void BuiltinFunc::Describe(ODesc* d) const
 	d->AddCount(is_pure);
 	}
 
-IMPLEMENT_SERIAL(BuiltinFunc, SER_BUILTIN_FUNC);
-
-bool BuiltinFunc::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_BUILTIN_FUNC, Func);
-	return true;
-	}
-
-bool BuiltinFunc::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(Func);
-	return true;
-	}
-
 void builtin_error(const char* msg, BroObj* arg)
 	{
 	auto emit = [=](const CallExpr* ce)
@@ -761,13 +627,13 @@ void builtin_error(const char* msg, BroObj* arg)
 		emit(last_call.call);
 	}
 
-#include "bro.bif.func_h"
+#include "zeek.bif.func_h"
 #include "stats.bif.func_h"
 #include "reporter.bif.func_h"
 #include "strings.bif.func_h"
 #include "option.bif.func_h"
 
-#include "bro.bif.func_def"
+#include "zeek.bif.func_def"
 #include "stats.bif.func_def"
 #include "reporter.bif.func_def"
 #include "strings.bif.func_def"
@@ -794,7 +660,7 @@ void init_builtin_funcs()
 
 	var_sizes = internal_type("var_sizes")->AsTableType();
 
-#include "bro.bif.func_init"
+#include "zeek.bif.func_init"
 #include "stats.bif.func_init"
 #include "reporter.bif.func_init"
 #include "strings.bif.func_init"

src/Func.h

@@ -59,10 +59,6 @@ public:
 	void Describe(ODesc* d) const override = 0;
 	virtual void DescribeDebug(ODesc* d, const val_list* args) const;
 
-	// This (un-)serializes only a single body (as given in SerialInfo).
-	bool Serialize(SerialInfo* info) const;
-	static Func* Unserialize(UnserialInfo* info);
-
 	virtual TraversalCode Traverse(TraversalCallback* cb) const;
 
 	uint32 GetUniqueFuncID() const { return unique_id; }
@@ -75,8 +71,6 @@ protected:
 	// Helper function for handling result of plugin hook.
 	std::pair<bool, Val*> HandlePluginResult(std::pair<bool, Val*> plugin_result, val_list* args, function_flavor flavor) const;
 
-	DECLARE_ABSTRACT_SERIAL(Func);
-
 	vector<Body> bodies;
 	Scope* scope;
 	Kind kind;
@@ -106,8 +100,6 @@ protected:
 	BroFunc() : Func(BRO_FUNC)	{}
 	Stmt* AddInits(Stmt* body, id_list* inits);
 
-	DECLARE_SERIAL(BroFunc);
-
 	int frame_size;
 };
 
@@ -127,8 +119,6 @@ public:
 protected:
 	BuiltinFunc()	{ func = 0; is_pure = 0; }
 
-	DECLARE_SERIAL(BuiltinFunc);
-
 	built_in_func func;
 	int is_pure;
 };

src/Hash.cc

@@ -26,7 +26,7 @@ void init_hash_function()
 	{
 	// Make sure we have already called init_random_seed().
 	if ( ! (hmac_key_set && siphash_key_set) )
-		reporter->InternalError("Bro's hash functions aren't fully initialized");
+		reporter->InternalError("Zeek's hash functions aren't fully initialized");
 	}
 
 HashKey::HashKey(bro_int_t i)

src/ID.cc

@@ -9,7 +9,6 @@
 #include "Func.h"
 #include "Scope.h"
 #include "File.h"
-#include "Serializer.h"
 #include "Scope.h"
 #include "Traverse.h"
 #include "zeekygen/Manager.h"
@@ -283,11 +282,6 @@ void ID::EvalFunc(Expr* ef, Expr* ev)
 	Unref(ce);
 	}
 
-bool ID::Serialize(SerialInfo* info) const
-	{
-	return (ID*) SerialObj::Serialize(info);
-	}
-
 #if 0
 void ID::CopyFrom(const ID* id)
 	{
@@ -320,205 +314,6 @@ void ID::CopyFrom(const ID* id)
 #endif
 #endif
 
-ID* ID::Unserialize(UnserialInfo* info)
-	{
-	ID* id = (ID*) SerialObj::Unserialize(info, SER_ID);
-	if ( ! id )
-		return 0;
-
-	if ( ! id->IsGlobal() )
-		return id;
-
-	// Globals.
-	ID* current = global_scope()->Lookup(id->name);
-
-	if ( ! current )
-		{
-		if ( ! info->install_globals )
-			{
-			info->s->Error("undefined");
-			Unref(id);
-			return 0;
-			}
-
-		Ref(id);
-		global_scope()->Insert(id->Name(), id);
-#ifdef USE_PERFTOOLS_DEBUG
-		heap_checker->IgnoreObject(id);
-#endif
-		}
-
-	else
-		{
-		switch ( info->id_policy ) {
-
-		case UnserialInfo::Keep:
-			Unref(id);
-			Ref(current);
-			id = current;
-			break;
-
-		case UnserialInfo::Replace:
-			Unref(current);
-			Ref(id);
-			global_scope()->Insert(id->Name(), id);
-			break;
-
-		case UnserialInfo::CopyNewToCurrent:
-			if ( ! same_type(current->type, id->type) )
-				{
-				info->s->Error("type mismatch");
-				Unref(id);
-				return 0;
-				}
-
-			if ( ! current->weak_ref )
-				Unref(current->val);
-
-			current->val = id->val;
-			current->weak_ref = id->weak_ref;
-			if ( current->val && ! current->weak_ref )
-				Ref(current->val);
-
-#ifdef DEBUG
-			current->UpdateValID();
-#endif
-
-			Unref(id);
-			Ref(current);
-			id = current;
-
-		break;
-
-		case UnserialInfo::CopyCurrentToNew:
-			if ( ! same_type(current->type, id->type) )
-				{
-				info->s->Error("type mismatch");
-				return 0;
-				}
-			if ( ! id->weak_ref )
-				Unref(id->val);
-			id->val = current->val;
-			id->weak_ref = current->weak_ref;
-			if ( id->val && ! id->weak_ref )
-				Ref(id->val);
-
-#ifdef DEBUG
-			id->UpdateValID();
-#endif
-
-			Unref(current);
-			Ref(id);
-			global_scope()->Insert(id->Name(), id);
-			break;
-
-		case UnserialInfo::InstantiateNew:
-			// Do nothing.
-			break;
-
-		default:
-			reporter->InternalError("unknown type for UnserialInfo::id_policy");
-		}
-		}
-
-	return id;
-
-	}
-
-IMPLEMENT_SERIAL(ID, SER_ID);
-
-bool ID::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE_WITH_SUSPEND(SER_ID, BroObj);
-
-	if ( info->cont.NewInstance() )
-		{
-		DisableSuspend suspend(info);
-
-		info->s->WriteOpenTag("ID");
-
-		if ( ! (SERIALIZE(name) &&
-			SERIALIZE(char(scope)) &&
-			SERIALIZE(is_export) &&
-			SERIALIZE(is_const) &&
-			SERIALIZE(is_enum_const) &&
-			SERIALIZE(is_type) &&
-			SERIALIZE(offset) &&
-			SERIALIZE(infer_return_type) &&
-			SERIALIZE(weak_ref) &&
-			type->Serialize(info)) )
-			return false;
-
-		SERIALIZE_OPTIONAL(attrs);
-		}
-
-	SERIALIZE_OPTIONAL(val);
-
-	return true;
-	}
-
-bool ID::DoUnserialize(UnserialInfo* info)
-	{
-	bool installed_tmp = false;
-
-	DO_UNSERIALIZE(BroObj);
-
-	char id_scope;
-
-	if ( ! (UNSERIALIZE_STR(&name, 0) &&
-		UNSERIALIZE(&id_scope) &&
-		UNSERIALIZE(&is_export) &&
-		UNSERIALIZE(&is_const) &&
-		UNSERIALIZE(&is_enum_const) &&
-		UNSERIALIZE(&is_type) &&
-		UNSERIALIZE(&offset) &&
-		UNSERIALIZE(&infer_return_type) &&
-		UNSERIALIZE(&weak_ref)
-	       ) )
-		return false;
-
-	scope = IDScope(id_scope);
-
-	info->s->SetErrorDescr(fmt("unserializing ID %s", name));
-
-	type = BroType::Unserialize(info);
-	if ( ! type )
-		return false;
-
-	UNSERIALIZE_OPTIONAL(attrs, Attributes::Unserialize(info));
-
-	// If it's a global function not currently known,
-	// we temporarily install it in global scope.
-	// This is necessary for recursive functions.
-	if ( IsGlobal() && Type()->Tag() == TYPE_FUNC )
-		{
-		ID* current = global_scope()->Lookup(name);
-		if ( ! current )
-			{
-			installed_tmp = true;
-			global_scope()->Insert(Name(), this);
-			}
-		}
-
-	UNSERIALIZE_OPTIONAL(val, Val::Unserialize(info));
-#ifdef DEBUG
-	UpdateValID();
-#endif
-
-	if ( weak_ref )
-		{
-		// At this point at least the serialization cache will hold a
-		// reference so this will not delete the val.
-		assert(val->RefCnt() > 1);
-		Unref(val);
-		}
-
-	if ( installed_tmp && ! global_scope()->Remove(name) )
-		reporter->InternalWarning("missing tmp ID in %s unserialization", name);
-
-	return true;
-	}
-
 TraversalCode ID::Traverse(TraversalCallback* cb) const
 	{
 	TraversalCode tc = cb->PreID(this);

src/ID.h

@@ -10,7 +10,6 @@
 #include <string>
 
 class Val;
-class SerialInfo;
 class Func;
 
 typedef enum { INIT_NONE, INIT_FULL, INIT_EXTRA, INIT_REMOVE, } init_class;
@@ -98,9 +97,6 @@ public:
 	void DescribeReST(ODesc* d, bool roles_only = false) const;
 	void DescribeReSTShort(ODesc* d) const;
 
-	bool Serialize(SerialInfo* info) const;
-	static ID* Unserialize(UnserialInfo* info);
-
 	bool DoInferReturnType() const
 		{ return infer_return_type; }
 	void SetInferReturnType(bool infer)
@@ -124,8 +120,6 @@ protected:
 	void UpdateValID();
 #endif
 
-	DECLARE_SERIAL(ID);
-
 	const char* name;
 	IDScope scope;
 	bool is_export;

src/Net.cc

@@ -27,7 +27,6 @@
 #include "Reporter.h"
 #include "Net.h"
 #include "Anon.h"
-#include "Serializer.h"
 #include "PacketDumper.h"
 #include "iosource/Manager.h"
 #include "iosource/PktSrc.h"

src/NetVar.cc

@@ -77,7 +77,6 @@ bool udp_content_deliver_all_orig;
 bool udp_content_deliver_all_resp;
 
 double dns_session_timeout;
-double ntp_session_timeout;
 double rpc_timeout;
 
 ListVal* skip_authentication;
@@ -103,8 +102,6 @@ TableType* pm_mappings;
 RecordType* pm_port_request;
 RecordType* pm_callit_request;
 
-RecordType* ntp_msg;
-
 RecordType* geo_location;
 
 RecordType* entropy_test_result;
@@ -360,7 +357,6 @@ void init_net_var()
 		bool(internal_val("udp_content_deliver_all_resp")->AsBool());
 
 	dns_session_timeout = opt_internal_double("dns_session_timeout");
-	ntp_session_timeout = opt_internal_double("ntp_session_timeout");
 	rpc_timeout = opt_internal_double("rpc_timeout");
 
 	watchdog_interval = int(opt_internal_double("watchdog_interval"));
@@ -390,8 +386,6 @@ void init_net_var()
 	pm_port_request = internal_type("pm_port_request")->AsRecordType();
 	pm_callit_request = internal_type("pm_callit_request")->AsRecordType();
 
-	ntp_msg = internal_type("ntp_msg")->AsRecordType();
-
 	geo_location = internal_type("geo_location")->AsRecordType();
 
 	entropy_test_result = internal_type("entropy_test_result")->AsRecordType();

src/NetVar.h

@@ -80,7 +80,6 @@ extern bool udp_content_deliver_all_orig;
 extern bool udp_content_deliver_all_resp;
 
 extern double dns_session_timeout;
-extern double ntp_session_timeout;
 extern double rpc_timeout;
 
 extern ListVal* skip_authentication;
@@ -106,8 +105,6 @@ extern TableType* pm_mappings;
 extern RecordType* pm_port_request;
 extern RecordType* pm_callit_request;
 
-extern RecordType* ntp_msg;
-
 extern RecordType* geo_location;
 
 extern RecordType* entropy_test_result;

src/Obj.cc

@@ -5,7 +5,6 @@
 #include <stdlib.h>
 
 #include "Obj.h"
-#include "Serializer.h"
 #include "Func.h"
 #include "File.h"
 #include "plugin/Manager.h"
@@ -14,47 +13,6 @@ Location no_location("<no location>", 0, 0, 0, 0);
 Location start_location("<start uninitialized>", 0, 0, 0, 0);
 Location end_location("<end uninitialized>", 0, 0, 0, 0);
 
-bool Location::Serialize(SerialInfo* info) const
-	{
-	return SerialObj::Serialize(info);
-	}
-
-Location* Location::Unserialize(UnserialInfo* info)
-	{
-	return (Location*) SerialObj::Unserialize(info, SER_LOCATION);
-	}
-
-IMPLEMENT_SERIAL(Location, SER_LOCATION);
-
-bool Location::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_LOCATION, SerialObj);
-	info->s->WriteOpenTag("Location");
-
-	if ( ! (SERIALIZE(filename) &&
-	        SERIALIZE(first_line) &&
-	        SERIALIZE(last_line) &&
-	        SERIALIZE(first_column) &&
-	        SERIALIZE(last_column)) )
-		return false;
-
-	info->s->WriteCloseTag("Location");
-	return true;
-	}
-
-bool Location::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(SerialObj);
-
-	delete_data = true;
-
-	return UNSERIALIZE_STR(&filename, 0)
-		&& UNSERIALIZE(&first_line)
-		&& UNSERIALIZE(&last_line)
-		&& UNSERIALIZE(&first_column)
-		&& UNSERIALIZE(&last_column);
-	}
-
 void Location::Describe(ODesc* d) const
 	{
 	if ( filename )
@@ -100,21 +58,21 @@ BroObj::~BroObj()
 	delete location;
 	}
 
-void BroObj::Warn(const char* msg, const BroObj* obj2, int pinpoint_only) const
+void BroObj::Warn(const char* msg, const BroObj* obj2, int pinpoint_only, const Location* expr_location) const
 	{
 	ODesc d;
-	DoMsg(&d, msg, obj2, pinpoint_only);
+	DoMsg(&d, msg, obj2, pinpoint_only, expr_location);
 	reporter->Warning("%s", d.Description());
 	reporter->PopLocation();
 	}
 
-void BroObj::Error(const char* msg, const BroObj* obj2, int pinpoint_only) const
+void BroObj::Error(const char* msg, const BroObj* obj2, int pinpoint_only, const Location* expr_location) const
 	{
 	if ( suppress_errors )
 		return;
 
 	ODesc d;
-	DoMsg(&d, msg, obj2, pinpoint_only);
+	DoMsg(&d, msg, obj2, pinpoint_only, expr_location);
 	reporter->Error("%s", d.Description());
 	reporter->PopLocation();
 	}
@@ -200,7 +158,7 @@ void BroObj::UpdateLocationEndInfo(const Location& end)
 	}
 
 void BroObj::DoMsg(ODesc* d, const char s1[], const BroObj* obj2,
-			int pinpoint_only) const
+			int pinpoint_only, const Location* expr_location) const
 	{
 	d->SetShort();
 
@@ -211,6 +169,8 @@ void BroObj::DoMsg(ODesc* d, const char s1[], const BroObj* obj2,
 	if ( obj2 && obj2->GetLocationInfo() != &no_location &&
 		 *obj2->GetLocationInfo() != *GetLocationInfo() )
 		loc2 = obj2->GetLocationInfo();
+	else if ( expr_location )
+		loc2 = expr_location;
 
 	reporter->PushLocation(GetLocationInfo(), loc2);
 	}
@@ -228,29 +188,6 @@ void BroObj::PinPoint(ODesc* d, const BroObj* obj2, int pinpoint_only) const
 	d->Add(")");
 	}
 
-bool BroObj::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_BRO_OBJ, SerialObj);
-
-	info->s->WriteOpenTag("Object");
-
-	Location* loc = info->include_locations ? location : 0;
-	SERIALIZE_OPTIONAL(loc);
-	info->s->WriteCloseTag("Object");
-
-	return true;
-	}
-
-bool BroObj::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(SerialObj);
-
-	delete location;
-
-	UNSERIALIZE_OPTIONAL(location, Location::Unserialize(info));
-	return true;
-	}
-
 void print(const BroObj* obj)
 	{
 	static BroFile fstderr(stderr);

src/Obj.h

@@ -7,12 +7,8 @@
 
 #include "input.h"
 #include "Desc.h"
-#include "SerialObj.h"
 
-class Serializer;
+class Location {
-class SerialInfo;
-
-class Location : SerialObj {
 public:
 	Location(const char* fname, int line_f, int line_l, int col_f, int col_l)
 		{
@@ -36,7 +32,7 @@ public:
 		text = 0;
 		}
 
-	~Location() override
+	virtual ~Location()
 		{
 		if ( delete_data )
 			delete [] filename;
@@ -44,9 +40,6 @@ public:
 
 	void Describe(ODesc* d) const;
 
-	bool Serialize(SerialInfo* info) const;
-	static Location* Unserialize(UnserialInfo* info);
-
 	bool operator==(const Location& l) const;
 	bool operator!=(const Location& l) const
 		{ return ! (*this == l); }
@@ -59,8 +52,6 @@ public:
 	// Timestamp and text for compatibility with Bison's default yyltype.
 	int timestamp;
 	char* text;
-protected:
-	DECLARE_SERIAL(Location);
 };
 
 #define YYLTYPE yyltype
@@ -86,7 +77,7 @@ inline void set_location(const Location start, const Location end)
 	end_location = end;
 	}
 
-class BroObj : public SerialObj {
+class BroObj {
 public:
 	BroObj()
 		{
@@ -112,15 +103,15 @@ public:
 			SetLocationInfo(&start_location, &end_location);
 		}
 
-	~BroObj() override;
+	virtual ~BroObj();
 
 	// Report user warnings/errors.  If obj2 is given, then it's
 	// included in the message, though if pinpoint_only is non-zero,
 	// then obj2 is only used to pinpoint the location.
 	void Warn(const char* msg, const BroObj* obj2 = 0,
-			int pinpoint_only = 0) const;
+			int pinpoint_only = 0, const Location* expr_location = 0) const;
 	void Error(const char* msg, const BroObj* obj2 = 0,
-			int pinpoint_only = 0) const;
+			int pinpoint_only = 0, const Location* expr_location = 0) const;
 
 	// Report internal errors.
 	void BadTag(const char* msg, const char* t1 = 0,
@@ -168,17 +159,13 @@ public:
 	bool in_ser_cache;
 
 protected:
-	friend class SerializationCache;
-
-	DECLARE_ABSTRACT_SERIAL(BroObj);
-
 	Location* location;	// all that matters in real estate
 
 private:
 	friend class SuppressErrors;
 
 	void DoMsg(ODesc* d, const char s1[], const BroObj* obj2 = 0,
-			int pinpoint_only = 0) const;
+			int pinpoint_only = 0, const Location* expr_location = 0) const;
 	void PinPoint(ODesc* d, const BroObj* obj2 = 0,
 			int pinpoint_only = 0) const;
 

src/OpaqueVal.cc

@@ -3,10 +3,151 @@
 #include "OpaqueVal.h"
 #include "NetVar.h"
 #include "Reporter.h"
-#include "Serializer.h"
 #include "probabilistic/BloomFilter.h"
 #include "probabilistic/CardinalityCounter.h"
 
+// Helper to retrieve a broker value out of a broker::vector at a specified
+// index, and casted to the expected destination type.
+template<typename S, typename V, typename D>
+inline bool get_vector_idx(const V& v, unsigned int i, D* dst)
+	{
+	if ( i >= v.size() )
+		return false;
+
+	auto x = caf::get_if<S>(&v[i]);
+	if ( ! x )
+		return false;
+
+	*dst = static_cast<D>(*x);
+	return true;
+	}
+
+OpaqueMgr* OpaqueMgr::mgr()
+	{
+	static OpaqueMgr mgr;
+	return &mgr;
+	}
+
+OpaqueVal::OpaqueVal(OpaqueType* t) : Val(t)
+	{
+	}
+
+OpaqueVal::~OpaqueVal()
+	{
+	}
+
+const std::string& OpaqueMgr::TypeID(const OpaqueVal* v) const
+	{
+	auto x = _types.find(v->OpaqueName());
+
+	if ( x == _types.end() )
+		reporter->InternalError("OpaqueMgr::TypeID: opaque type %s not registered",
+					v->OpaqueName());
+
+	return x->first;
+	}
+
+OpaqueVal* OpaqueMgr::Instantiate(const std::string& id) const
+	{
+	auto x = _types.find(id);
+	return x != _types.end() ? (*x->second)() : nullptr;
+	}
+
+broker::expected<broker::data> OpaqueVal::Serialize() const
+	{
+	auto type = OpaqueMgr::mgr()->TypeID(this);
+
+	auto d = DoSerialize();
+	if ( ! d )
+		return d.error();
+
+	return {broker::vector{std::move(type), std::move(*d)}};
+	}
+
+OpaqueVal* OpaqueVal::Unserialize(const broker::data& data)
+	{
+	auto v = caf::get_if<broker::vector>(&data);
+
+	if ( ! (v && v->size() == 2) )
+		return nullptr;
+
+	auto type = caf::get_if<std::string>(&(*v)[0]);
+	if ( ! type )
+		return nullptr;
+
+	auto val = OpaqueMgr::mgr()->Instantiate(*type);
+	if ( ! val )
+		return nullptr;
+
+	if ( ! val->DoUnserialize((*v)[1]) )
+		{
+		Unref(val);
+		return nullptr;
+		}
+
+	return val;
+	}
+
+broker::expected<broker::data> OpaqueVal::SerializeType(BroType* t)
+	{
+	if ( t->InternalType() == TYPE_INTERNAL_ERROR )
+		return broker::ec::invalid_data;
+
+	if ( t->InternalType() == TYPE_INTERNAL_OTHER )
+		{
+		// Serialize by name.
+		assert(t->GetName().size());
+		return {broker::vector{true, t->GetName()}};
+		}
+
+	// A base type.
+	return {broker::vector{false, static_cast<uint64>(t->Tag())}};
+	}
+
+BroType* OpaqueVal::UnserializeType(const broker::data& data)
+	{
+	auto v = caf::get_if<broker::vector>(&data);
+	if ( ! (v && v->size() == 2) )
+		return nullptr;
+
+	auto by_name = caf::get_if<bool>(&(*v)[0]);
+	if ( ! by_name )
+		return nullptr;
+
+	if ( *by_name )
+		{
+		auto name = caf::get_if<std::string>(&(*v)[1]);
+		if ( ! name )
+			return nullptr;
+
+		ID* id = global_scope()->Lookup(name->c_str());
+		if ( ! id )
+			return nullptr;
+
+		BroType* t = id->AsType();
+		if ( ! t )
+			return nullptr;
+
+		return t->Ref();
+		}
+
+	auto tag = caf::get_if<uint64>(&(*v)[1]);
+	if ( ! tag )
+		return nullptr;
+
+	return base_type(static_cast<TypeTag>(*tag));
+	}
+
+Val* OpaqueVal::DoClone(CloneState* state)
+	{
+	auto d = OpaqueVal::Serialize();
+	if ( ! d )
+		return nullptr;
+
+	auto rval = OpaqueVal::Unserialize(std::move(*d));
+	return state->NewClone(this, rval);
+	}
+
 bool HashVal::IsValid() const
 	{
 	return valid;
@@ -63,20 +204,6 @@ HashVal::HashVal(OpaqueType* t) : OpaqueVal(t)
 	valid = false;
 	}
 
-IMPLEMENT_SERIAL(HashVal, SER_HASH_VAL);
-
-bool HashVal::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_HASH_VAL, OpaqueVal);
-	return SERIALIZE(valid);
-	}
-
-bool HashVal::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(OpaqueVal);
-	return UNSERIALIZE(&valid);
-	}
-
 MD5Val::MD5Val() : HashVal(md5_type)
 	{
 	}
@@ -97,7 +224,7 @@ Val* MD5Val::DoClone(CloneState* state)
 		EVP_MD_CTX_copy_ex(out->ctx, ctx);
 		}
 
-	return out;
+	return state->NewClone(this, out);
 	}
 
 void MD5Val::digest(val_list& vlist, u_char result[MD5_DIGEST_LENGTH])
@@ -160,64 +287,72 @@ StringVal* MD5Val::DoGet()
 	return new StringVal(md5_digest_print(digest));
 	}
 
-IMPLEMENT_SERIAL(MD5Val, SER_MD5_VAL);
+IMPLEMENT_OPAQUE_VALUE(MD5Val)
 
-bool MD5Val::DoSerialize(SerialInfo* info) const
+broker::expected<broker::data> MD5Val::DoSerialize() const
 	{
-	DO_SERIALIZE(SER_MD5_VAL, HashVal);
-
 	if ( ! IsValid() )
-		return true;
+		return {broker::vector{false}};
 
 	MD5_CTX* md = (MD5_CTX*) EVP_MD_CTX_md_data(ctx);
 
-	if ( ! (SERIALIZE(md->A) &&
+	broker::vector d = {
-		SERIALIZE(md->B) &&
+	    true,
-		SERIALIZE(md->C) &&
+	    static_cast<uint64>(md->A),
-		SERIALIZE(md->D) &&
+	    static_cast<uint64>(md->B),
-		SERIALIZE(md->Nl) &&
+	    static_cast<uint64>(md->C),
-		SERIALIZE(md->Nh)) )
+	    static_cast<uint64>(md->D),
+	    static_cast<uint64>(md->Nl),
+	    static_cast<uint64>(md->Nh),
+	    static_cast<uint64>(md->num)
+	};
+
+	for ( int i = 0; i < MD5_LBLOCK; ++i )
+		d.emplace_back(static_cast<uint64>(md->data[i]));
+
+	return {std::move(d)};
+	}
+
+bool MD5Val::DoUnserialize(const broker::data& data)
+	{
+	auto d = caf::get_if<broker::vector>(&data);
+	if ( ! d )
+		return false;
+
+	auto valid = caf::get_if<bool>(&(*d)[0]);
+	if ( ! valid )
+		return false;
+
+	if ( ! *valid )
+		{
+		assert(! IsValid()); // default set by ctor
+		return true;
+		}
+
+	Init();
+	MD5_CTX* md = (MD5_CTX*) EVP_MD_CTX_md_data(ctx);
+
+	if ( ! get_vector_idx<uint64_t>(*d, 1, &md->A) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 2, &md->B) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 3, &md->C) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 4, &md->D) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 5, &md->Nl) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 6, &md->Nh) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 7, &md->num) )
 		return false;
 
 	for ( int i = 0; i < MD5_LBLOCK; ++i )
 		{
-		if ( ! SERIALIZE(md->data[i]) )
+		if ( ! get_vector_idx<uint64_t>(*d, 8 + i, &md->data[i]) )
 			return false;
 		}
 
-	if ( ! SERIALIZE(md->num) )
-		return false;
-
-	return true;
-	}
-
-bool MD5Val::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(HashVal);
-
-	if ( ! IsValid() )
-		return true;
-
-	ctx = hash_init(Hash_MD5);
-	MD5_CTX* md = (MD5_CTX*) EVP_MD_CTX_md_data(ctx);
-
-	if ( ! (UNSERIALIZE(&md->A) &&
-		UNSERIALIZE(&md->B) &&
-		UNSERIALIZE(&md->C) &&
-		UNSERIALIZE(&md->D) &&
-		UNSERIALIZE(&md->Nl) &&
-		UNSERIALIZE(&md->Nh)) )
-		return false;
-
-	for ( int i = 0; i < MD5_LBLOCK; ++i )
-		{
-		if ( ! UNSERIALIZE(&md->data[i]) )
-			return false;
-		}
-
-	if ( ! UNSERIALIZE(&md->num) )
-		return false;
-
 	return true;
 	}
 
@@ -241,7 +376,7 @@ Val* SHA1Val::DoClone(CloneState* state)
 		EVP_MD_CTX_copy_ex(out->ctx, ctx);
 		}
 
-	return out;
+	return state->NewClone(this, out);
 	}
 
 void SHA1Val::digest(val_list& vlist, u_char result[SHA_DIGEST_LENGTH])
@@ -293,66 +428,75 @@ StringVal* SHA1Val::DoGet()
 	return new StringVal(sha1_digest_print(digest));
 	}
 
-IMPLEMENT_SERIAL(SHA1Val, SER_SHA1_VAL);
+IMPLEMENT_OPAQUE_VALUE(SHA1Val)
 
-bool SHA1Val::DoSerialize(SerialInfo* info) const
+broker::expected<broker::data> SHA1Val::DoSerialize() const
 	{
-	DO_SERIALIZE(SER_SHA1_VAL, HashVal);
-
 	if ( ! IsValid() )
-		return true;
+		return {broker::vector{false}};
 
 	SHA_CTX* md = (SHA_CTX*) EVP_MD_CTX_md_data(ctx);
 
-	if ( ! (SERIALIZE(md->h0) &&
+	broker::vector d = {
-		SERIALIZE(md->h1) &&
+	    true,
-		SERIALIZE(md->h2) &&
+	    static_cast<uint64>(md->h0),
-		SERIALIZE(md->h3) &&
+	    static_cast<uint64>(md->h1),
-		SERIALIZE(md->h4) &&
+	    static_cast<uint64>(md->h2),
-		SERIALIZE(md->Nl) &&
+	    static_cast<uint64>(md->h3),
-		SERIALIZE(md->Nh)) )
+	    static_cast<uint64>(md->h4),
+	    static_cast<uint64>(md->Nl),
+	    static_cast<uint64>(md->Nh),
+	    static_cast<uint64>(md->num)
+	};
+
+	for ( int i = 0; i < SHA_LBLOCK; ++i )
+		d.emplace_back(static_cast<uint64>(md->data[i]));
+
+	return {std::move(d)};
+	}
+
+bool SHA1Val::DoUnserialize(const broker::data& data)
+	{
+	auto d = caf::get_if<broker::vector>(&data);
+	if ( ! d )
+		return false;
+
+	auto valid = caf::get_if<bool>(&(*d)[0]);
+	if ( ! valid )
+		return false;
+
+	if ( ! *valid )
+		{
+		assert(! IsValid()); // default set by ctor
+		return true;
+		}
+
+	Init();
+	SHA_CTX* md = (SHA_CTX*) EVP_MD_CTX_md_data(ctx);
+
+	if ( ! get_vector_idx<uint64_t>(*d, 1, &md->h0) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 2, &md->h1) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 3, &md->h2) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 4, &md->h3) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 5, &md->h4) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 6, &md->Nl) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 7, &md->Nh) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 8, &md->num) )
 		return false;
 
 	for ( int i = 0; i < SHA_LBLOCK; ++i )
 		{
-		if ( ! SERIALIZE(md->data[i]) )
+		if ( ! get_vector_idx<uint64_t>(*d, 9 + i, &md->data[i]) )
 			return false;
 		}
 
-	if ( ! SERIALIZE(md->num) )
-		return false;
-
-	return true;
-	}
-
-bool SHA1Val::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(HashVal);
-
-	if ( ! IsValid() )
-		return true;
-
-	ctx = hash_init(Hash_SHA1);
-	SHA_CTX* md = (SHA_CTX*) EVP_MD_CTX_md_data(ctx);
-
-	if ( ! (UNSERIALIZE(&md->h0) &&
-		UNSERIALIZE(&md->h1) &&
-		UNSERIALIZE(&md->h2) &&
-		UNSERIALIZE(&md->h3) &&
-		UNSERIALIZE(&md->h4) &&
-		UNSERIALIZE(&md->Nl) &&
-		UNSERIALIZE(&md->Nh)) )
-		return false;
-
-	for ( int i = 0; i < SHA_LBLOCK; ++i )
-		{
-		if ( ! UNSERIALIZE(&md->data[i]) )
-			return false;
-		}
-
-	if ( ! UNSERIALIZE(&md->num) )
-		return false;
-
 	return true;
 	}
 
@@ -376,7 +520,7 @@ Val* SHA256Val::DoClone(CloneState* state)
 		EVP_MD_CTX_copy_ex(out->ctx, ctx);
 		}
 
-	return out;
+	return state->NewClone(this, out);
 	}
 
 void SHA256Val::digest(val_list& vlist, u_char result[SHA256_DIGEST_LENGTH])
@@ -428,71 +572,72 @@ StringVal* SHA256Val::DoGet()
 	return new StringVal(sha256_digest_print(digest));
 	}
 
-IMPLEMENT_SERIAL(SHA256Val, SER_SHA256_VAL);
+IMPLEMENT_OPAQUE_VALUE(SHA256Val)
 
-bool SHA256Val::DoSerialize(SerialInfo* info) const
+broker::expected<broker::data> SHA256Val::DoSerialize() const
 	{
-	DO_SERIALIZE(SER_SHA256_VAL, HashVal);
-
 	if ( ! IsValid() )
-		return true;
+		return {broker::vector{false}};
 
 	SHA256_CTX* md = (SHA256_CTX*) EVP_MD_CTX_md_data(ctx);
 
-	for ( int i = 0; i < 8; ++i )
+	broker::vector d = {
-		{
+	    true,
-		if ( ! SERIALIZE(md->h[i]) )
+	    static_cast<uint64>(md->Nl),
-			return false;
+	    static_cast<uint64>(md->Nh),
-		}
+	    static_cast<uint64>(md->num),
+	    static_cast<uint64>(md->md_len)
+	};
 
-	if ( ! (SERIALIZE(md->Nl) &&
+	for ( int i = 0; i < 8; ++i )
-		SERIALIZE(md->Nh)) )
+		d.emplace_back(static_cast<uint64>(md->h[i]));
-		return false;
 
 	for ( int i = 0; i < SHA_LBLOCK; ++i )
-		{
+		d.emplace_back(static_cast<uint64>(md->data[i]));
-		if ( ! SERIALIZE(md->data[i]) )
+
-			return false;
+	return {std::move(d)};
 	}
 
-	if ( ! (SERIALIZE(md->num) &&
+bool SHA256Val::DoUnserialize(const broker::data& data)
-		SERIALIZE(md->md_len)) )
+	{
+	auto d = caf::get_if<broker::vector>(&data);
+	if ( ! d )
 		return false;
 
+	auto valid = caf::get_if<bool>(&(*d)[0]);
+	if ( ! valid )
+		return false;
+
+	if ( ! *valid )
+		{
+		assert(! IsValid()); // default set by ctor
 		return true;
 		}
 
-bool SHA256Val::DoUnserialize(UnserialInfo* info)
+	Init();
-	{
-	DO_UNSERIALIZE(HashVal);
-
-	if ( ! IsValid() )
-		return true;
-
-	ctx = hash_init(Hash_SHA256);
 	SHA256_CTX* md = (SHA256_CTX*) EVP_MD_CTX_md_data(ctx);
 
+	if ( ! get_vector_idx<uint64_t>(*d, 1, &md->Nl) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 2, &md->Nh) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 3, &md->num) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 4, &md->md_len) )
+		return false;
+
 	for ( int i = 0; i < 8; ++i )
 		{
-		if ( ! UNSERIALIZE(&md->h[i]) )
+		if ( ! get_vector_idx<uint64_t>(*d, 5 + i, &md->h[i]) )
 			return false;
 		}
 
-	if ( ! (UNSERIALIZE(&md->Nl) &&
-		UNSERIALIZE(&md->Nh)) )
-	     return false;
-
 	for ( int i = 0; i < SHA_LBLOCK; ++i )
 		{
-		if ( ! UNSERIALIZE(&md->data[i]) )
+		if ( ! get_vector_idx<uint64_t>(*d, 13 + i, &md->data[i]) )
 			return false;
 		}
 
-
-	if ( ! (UNSERIALIZE(&md->num) &&
-		UNSERIALIZE(&md->md_len)) )
-		return false;
-
 	return true;
 	}
 
@@ -500,26 +645,6 @@ EntropyVal::EntropyVal() : OpaqueVal(entropy_type)
 	{
 	}
 
-Val* EntropyVal::DoClone(CloneState* state)
-	{
-	SerializationFormat* form = new BinarySerializationFormat();
-	form->StartWrite();
-	CloneSerializer ss(form);
-	SerialInfo sinfo(&ss);
-	sinfo.cache = false;
-	sinfo.include_locations = false;
-	if ( ! this->Serialize(&sinfo) )
-		return nullptr;
-	char* data;
-	uint32 len = form->EndWrite(&data);
-	form->StartRead(data, len);
-	UnserialInfo uinfo(&ss);
-	uinfo.cache = false;
-	Val* clone = Unserialize(&uinfo, type);
-	free(data);
-	return clone;
-	}
-
 bool EntropyVal::Feed(const void* data, size_t size)
 	{
 	state.add(data, size);
@@ -533,79 +658,86 @@ bool EntropyVal::Get(double *r_ent, double *r_chisq, double *r_mean,
 	return true;
 	}
 
-IMPLEMENT_SERIAL(EntropyVal, SER_ENTROPY_VAL);
+IMPLEMENT_OPAQUE_VALUE(EntropyVal)
 
-bool EntropyVal::DoSerialize(SerialInfo* info) const
+broker::expected<broker::data> EntropyVal::DoSerialize() const
 	{
-	DO_SERIALIZE(SER_ENTROPY_VAL, OpaqueVal);
+	broker::vector d =
+		{
+		static_cast<uint64>(state.totalc),
+		static_cast<uint64>(state.mp),
+		static_cast<uint64>(state.sccfirst),
+		static_cast<uint64>(state.inmont),
+		static_cast<uint64>(state.mcount),
+		static_cast<uint64>(state.cexp),
+		static_cast<uint64>(state.montex),
+		static_cast<uint64>(state.montey),
+		static_cast<uint64>(state.montepi),
+		static_cast<uint64>(state.sccu0),
+		static_cast<uint64>(state.scclast),
+		static_cast<uint64>(state.scct1),
+		static_cast<uint64>(state.scct2),
+		static_cast<uint64>(state.scct3),
+		};
+
+	d.reserve(256 + 3 + RT_MONTEN + 11);
+
+	for ( int i = 0; i < 256; ++i )
+		d.emplace_back(static_cast<uint64>(state.ccount[i]));
+
+        for ( int i = 0; i < RT_MONTEN; ++i )
+		d.emplace_back(static_cast<uint64>(state.monte[i]));
+
+	return {std::move(d)};
+	}
+
+bool EntropyVal::DoUnserialize(const broker::data& data)
+	{
+	auto d = caf::get_if<broker::vector>(&data);
+	if ( ! d )
+		return false;
+
+	if ( ! get_vector_idx<uint64_t>(*d, 0, &state.totalc) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 1, &state.mp) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 2, &state.sccfirst) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 3, &state.inmont) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 4, &state.mcount) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 5, &state.cexp) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 6, &state.montex) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 7, &state.montey) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 8, &state.montepi) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 9, &state.sccu0) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 10, &state.scclast) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 11, &state.scct1) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 12, &state.scct2) )
+		return false;
+	if ( ! get_vector_idx<uint64_t>(*d, 13, &state.scct3) )
+		return false;
 
 	for ( int i = 0; i < 256; ++i )
 		{
-		if ( ! SERIALIZE(state.ccount[i]) )
+		if ( ! get_vector_idx<uint64_t>(*d, 14 + i, &state.ccount[i]) )
 			return false;
 		}
 
-	if ( ! (SERIALIZE(state.totalc) &&
-		SERIALIZE(state.mp) &&
-		SERIALIZE(state.sccfirst)) )
-		return false;
-
 	for ( int i = 0; i < RT_MONTEN; ++i )
 		{
-		if ( ! SERIALIZE(state.monte[i]) )
+		if ( ! get_vector_idx<uint64_t>(*d, 14 + 256 + i, &state.monte[i]) )
 			return false;
 		}
 
-	if ( ! (SERIALIZE(state.inmont) &&
-		SERIALIZE(state.mcount) &&
-		SERIALIZE(state.cexp) &&
-		SERIALIZE(state.montex) &&
-		SERIALIZE(state.montey) &&
-		SERIALIZE(state.montepi) &&
-		SERIALIZE(state.sccu0) &&
-		SERIALIZE(state.scclast) &&
-		SERIALIZE(state.scct1) &&
-		SERIALIZE(state.scct2) &&
-		SERIALIZE(state.scct3)) )
-		return false;
-
-	return true;
-	}
-
-bool EntropyVal::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(OpaqueVal);
-
-	for ( int i = 0; i < 256; ++i )
-		{
-		if ( ! UNSERIALIZE(&state.ccount[i]) )
-			return false;
-		}
-
-	if ( ! (UNSERIALIZE(&state.totalc) &&
-		UNSERIALIZE(&state.mp) &&
-		UNSERIALIZE(&state.sccfirst)) )
-		return false;
-
-	for ( int i = 0; i < RT_MONTEN; ++i )
-		{
-		if ( ! UNSERIALIZE(&state.monte[i]) )
-			return false;
-		}
-
-	if ( ! (UNSERIALIZE(&state.inmont) &&
-		UNSERIALIZE(&state.mcount) &&
-		UNSERIALIZE(&state.cexp) &&
-		UNSERIALIZE(&state.montex) &&
-		UNSERIALIZE(&state.montey) &&
-		UNSERIALIZE(&state.montepi) &&
-		UNSERIALIZE(&state.sccu0) &&
-		UNSERIALIZE(&state.scclast) &&
-		UNSERIALIZE(&state.scct1) &&
-		UNSERIALIZE(&state.scct2) &&
-		UNSERIALIZE(&state.scct3)) )
-		return false;
-
 	return true;
 	}
 
@@ -639,10 +771,10 @@ Val* BloomFilterVal::DoClone(CloneState* state)
 		{
 		auto bf = new BloomFilterVal(bloom_filter->Clone());
 		bf->Typify(type);
-		return bf;
+		return state->NewClone(this, bf);
 		}
 
-	return new BloomFilterVal();
+	return state->NewClone(this, new BloomFilterVal());
 	}
 
 bool BloomFilterVal::Typify(BroType* arg_type)
@@ -739,42 +871,52 @@ BloomFilterVal::~BloomFilterVal()
 	delete bloom_filter;
 	}
 
-IMPLEMENT_SERIAL(BloomFilterVal, SER_BLOOMFILTER_VAL);
+IMPLEMENT_OPAQUE_VALUE(BloomFilterVal)
 
-bool BloomFilterVal::DoSerialize(SerialInfo* info) const
+broker::expected<broker::data> BloomFilterVal::DoSerialize() const
 	{
-	DO_SERIALIZE(SER_BLOOMFILTER_VAL, OpaqueVal);
+	broker::vector d;
 
-	bool is_typed = (type != 0);
+	if ( type )
+		{
+		auto t = SerializeType(type);
+		if ( ! t )
+			return broker::ec::invalid_data;
 
-	if ( ! SERIALIZE(is_typed) )
+		d.emplace_back(std::move(*t));
-		return false;
+		}
+	else
+		d.emplace_back(broker::none());
 
-	if ( is_typed && ! type->Serialize(info) )
+	auto bf = bloom_filter->Serialize();
-		return false;
+	if ( ! bf )
+		return broker::ec::invalid_data; // Cannot serialize;
 
-	return bloom_filter->Serialize(info);
+	d.emplace_back(*bf);
+	return {std::move(d)};
 	}
 
-bool BloomFilterVal::DoUnserialize(UnserialInfo* info)
+bool BloomFilterVal::DoUnserialize(const broker::data& data)
 	{
-	DO_UNSERIALIZE(OpaqueVal);
+	auto v = caf::get_if<broker::vector>(&data);
 
-	bool is_typed;
+	if ( ! (v && v->size() == 2) )
-	if ( ! UNSERIALIZE(&is_typed) )
 		return false;
 
-	if ( is_typed )
+	auto no_type = caf::get_if<broker::none>(&(*v)[0]);
+	if ( ! no_type )
 		{
-		BroType* t = BroType::Unserialize(info);
+		BroType* t = UnserializeType((*v)[0]);
-		if ( ! Typify(t) )
+		if ( ! (t && Typify(t)) )
 			return false;
-
-		Unref(t);
 		}
 
-	bloom_filter = probabilistic::BloomFilter::Unserialize(info);
+	auto bf = probabilistic::BloomFilter::Unserialize((*v)[1]);
-	return bloom_filter != 0;
+	if ( ! bf )
+		return false;
+
+	bloom_filter = bf.release();
+	return true;
 	}
 
 CardinalityVal::CardinalityVal() : OpaqueVal(cardinality_type)
@@ -801,45 +943,8 @@ CardinalityVal::~CardinalityVal()
 
 Val* CardinalityVal::DoClone(CloneState* state)
 	{
-	return new CardinalityVal(new probabilistic::CardinalityCounter(*c));
+	return state->NewClone(this,
-	}
+			       new CardinalityVal(new probabilistic::CardinalityCounter(*c)));
-
-IMPLEMENT_SERIAL(CardinalityVal, SER_CARDINALITY_VAL);
-
-bool CardinalityVal::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_CARDINALITY_VAL, OpaqueVal);
-
-	bool valid = true;
-	bool is_typed = (type != 0);
-
-	valid &= SERIALIZE(is_typed);
-
-	if ( is_typed )
-		valid &= type->Serialize(info);
-
-	return c->Serialize(info);
-	}
-
-bool CardinalityVal::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(OpaqueVal);
-
-	bool is_typed;
-	if ( ! UNSERIALIZE(&is_typed) )
-		return false;
-
-	if ( is_typed )
-		{
-		BroType* t = BroType::Unserialize(info);
-		if ( ! Typify(t) )
-			return false;
-
-		Unref(t);
-		}
-
-	c = probabilistic::CardinalityCounter::Unserialize(info);
-	return c != 0;
 	}
 
 bool CardinalityVal::Typify(BroType* arg_type)
@@ -870,6 +975,53 @@ void CardinalityVal::Add(const Val* val)
 	delete key;
 	}
 
+IMPLEMENT_OPAQUE_VALUE(CardinalityVal)
+
+broker::expected<broker::data> CardinalityVal::DoSerialize() const
+	{
+	broker::vector d;
+
+	if ( type )
+		{
+		auto t = SerializeType(type);
+		if ( ! t )
+			return broker::ec::invalid_data;
+
+		d.emplace_back(std::move(*t));
+		}
+	else
+		d.emplace_back(broker::none());
+
+	auto cs = c->Serialize();
+	if ( ! cs )
+		return broker::ec::invalid_data;
+
+	d.emplace_back(*cs);
+	return {std::move(d)};
+	}
+
+bool CardinalityVal::DoUnserialize(const broker::data& data)
+	{
+	auto v = caf::get_if<broker::vector>(&data);
+
+	if ( ! (v && v->size() == 2) )
+		return false;
+
+	auto no_type = caf::get_if<broker::none>(&(*v)[0]);
+	if ( ! no_type )
+		{
+		BroType* t = UnserializeType((*v)[0]);
+                if ( ! (t && Typify(t)) )
+			return false;
+		}
+
+	auto cu = probabilistic::CardinalityCounter::Unserialize((*v)[1]);
+	if ( ! cu )
+		return false;
+
+	c = cu.release();
+	return true;
+	}
 
 ParaglobVal::ParaglobVal(std::unique_ptr<paraglob::Paraglob> p)
 : OpaqueVal(paraglob_type)

src/OpaqueVal.h

@@ -3,14 +3,158 @@
 #ifndef OPAQUEVAL_H
 #define OPAQUEVAL_H
 
+<<<<<<< HEAD
 #include <typeinfo>
 #include <memory> // std::unique_ptr
 
+=======
+>>>>>>> upstream/master
 #include "RandTest.h"
 #include "Val.h"
 #include "digest.h"
 #include "src/paraglob.h"
 
+class OpaqueVal;
+
+/**
+  * Singleton that registers all available all available types of opaque
+  * values. This faciliates their serialization into Broker values.
+  */
+class OpaqueMgr {
+public:
+	using Factory = OpaqueVal* ();
+
+	/**
+	 * Return's a unique ID for the type of an opaque value.
+	 * @param v opaque value to return type for; its class must have been
+	 * registered with the manager, otherwise this method will abort
+	 * execution.
+	 *
+	 * @return type ID, which can used with *Instantiate()* to create a
+	 * new instance of the same type.
+	 */
+	const std::string& TypeID(const OpaqueVal* v) const;
+
+	/**
+	 * Instantiates a new opaque value of a specific opaque type.
+	 *
+	 * @param id unique type ID for the class to instantiate; this will
+	 * normally have been returned earlier by *TypeID()*.
+	 *
+	 * @return A freshly instantiated value of the OpaqueVal-derived
+	 * classes that *id* specifies, with reference count at +1. If *id*
+	 * is unknown, this will return null.
+	 *
+	 */
+	OpaqueVal* Instantiate(const std::string& id) const;
+
+	/** Returns the global manager singleton object. */
+	static OpaqueMgr* mgr();
+
+	/**
+	 * Internal helper class to register an OpaqueVal-derived classes
+	 * with the manager.
+	 */
+	template<class T>
+	class Register {
+	public:
+		Register(const char* id)
+			{ OpaqueMgr::mgr()->_types.emplace(id, &T::OpaqueInstantiate); }
+	};
+
+private:
+	std::unordered_map<std::string, Factory*> _types;
+};
+
+/** Macro to insert into an OpaqueVal-derived class's declaration. */
+#define DECLARE_OPAQUE_VALUE(T)                            \
+    friend class OpaqueMgr::Register<T>;                   \
+    broker::expected<broker::data> DoSerialize() const override;             \
+    bool DoUnserialize(const broker::data& data) override; \
+    const char* OpaqueName() const override { return #T; } \
+    static OpaqueVal* OpaqueInstantiate() { return new T(); }
+
+#define __OPAQUE_MERGE(a, b) a ## b
+#define __OPAQUE_ID(x) __OPAQUE_MERGE(_opaque, x)
+
+/** Macro to insert into an OpaqueVal-derived class's implementation file. */
+#define IMPLEMENT_OPAQUE_VALUE(T) static OpaqueMgr::Register<T> __OPAQUE_ID(__LINE__)(#T);
+
+/**
+ * Base class for all opaque values. Opaque values are types that are managed
+ * completely internally, with no further script-level operators provided
+ * (other than bif functions). See OpaqueVal.h for derived classes.
+ */
+class OpaqueVal : public Val {
+public:
+	explicit OpaqueVal(OpaqueType* t);
+	~OpaqueVal() override;
+
+	/**
+	 * Serializes the value into a Broker representation.
+	 *
+	 * @return the broker representation, or an error if serialization
+	 * isn't supported or failed.
+	 */
+	broker::expected<broker::data> Serialize() const;
+
+	/**
+	 * Reinstantiates a value from its serialized Broker representation.
+	 *
+	 * @param data Broker representation as returned by *Serialize()*.
+	 * @return unserialized instances with reference count at +1
+	 */
+	static OpaqueVal* Unserialize(const broker::data& data);
+
+protected:
+	friend class Val;
+	friend class OpaqueMgr;
+	OpaqueVal() { }
+
+	/**
+	 * Must be overridden to provide a serialized version of the derived
+	 * class' state.
+	 *
+	 * @return the serialized data or an error if serialization
+	 * isn't supported or failed.
+	 */
+	virtual broker::expected<broker::data> DoSerialize() const = 0;
+
+	/**
+	 * Must be overridden to recreate the the derived class' state from a
+	 * serialization.
+	 *
+	 * @return true if successful.
+	 */
+	virtual bool DoUnserialize(const broker::data& data) = 0;
+
+	/**
+	 * Internal helper for the serialization machinery. Automatically
+	 * overridden by the `DECLARE_OPAQUE_VALUE` macro.
+	 */
+	virtual const char* OpaqueName() const = 0;
+
+	/**
+	 * Provides an implementation of *Val::DoClone()* that leverages the
+	 * serialization methods to deep-copy an instance. Derived classes
+	 * may also override this with a more efficient custom clone
+	 * implementation of their own.
+	 */
+	Val* DoClone(CloneState* state) override;
+
+	/**
+	 * Helper function for derived class that need to record a type
+	 * during serialization.
+	 */
+	static broker::expected<broker::data> SerializeType(BroType* t);
+
+	/**
+	 * Helper function for derived class that need to restore a type
+	 * during unserialization. Returns the type at reference count +1.
+	 */
+	static BroType* UnserializeType(const broker::data& data);
+};
+
 namespace probabilistic {
 	class BloomFilter;
 	class CardinalityCounter;
@@ -24,15 +168,13 @@ public:
 	virtual StringVal* Get();
 
 protected:
-	HashVal() { };
+	HashVal()	{ valid = false; }
 	explicit HashVal(OpaqueType* t);
 
 	virtual bool DoInit();
 	virtual bool DoFeed(const void* data, size_t size);
 	virtual StringVal* DoGet();
 
-	DECLARE_SERIAL(HashVal);
-
 private:
 	// This flag exists because Get() can only be called once.
 	bool valid;
@@ -58,8 +200,7 @@ protected:
 	bool DoFeed(const void* data, size_t size) override;
 	StringVal* DoGet() override;
 
-	DECLARE_SERIAL(MD5Val);
+	DECLARE_OPAQUE_VALUE(MD5Val)
-
 private:
 	EVP_MD_CTX* ctx;
 };
@@ -80,8 +221,7 @@ protected:
 	bool DoFeed(const void* data, size_t size) override;
 	StringVal* DoGet() override;
 
-	DECLARE_SERIAL(SHA1Val);
+	DECLARE_OPAQUE_VALUE(SHA1Val)
-
 private:
 	EVP_MD_CTX* ctx;
 };
@@ -102,8 +242,7 @@ protected:
 	bool DoFeed(const void* data, size_t size) override;
 	StringVal* DoGet() override;
 
-	DECLARE_SERIAL(SHA256Val);
+	DECLARE_OPAQUE_VALUE(SHA256Val)
-
 private:
 	EVP_MD_CTX* ctx;
 };
@@ -112,8 +251,6 @@ class EntropyVal : public OpaqueVal {
 public:
 	EntropyVal();
 
-	Val* DoClone(CloneState* state) override;
-
 	bool Feed(const void* data, size_t size);
 	bool Get(double *r_ent, double *r_chisq, double *r_mean,
 		 double *r_montepicalc, double *r_scc);
@@ -121,8 +258,7 @@ public:
 protected:
 	friend class Val;
 
-	DECLARE_SERIAL(EntropyVal);
+	DECLARE_OPAQUE_VALUE(EntropyVal)
-
 private:
 	RandTest state;
 };
@@ -151,8 +287,7 @@ protected:
 	BloomFilterVal();
 	explicit BloomFilterVal(OpaqueType* t);
 
-	DECLARE_SERIAL(BloomFilterVal);
+	DECLARE_OPAQUE_VALUE(BloomFilterVal)
-
 private:
 	// Disable.
 	BloomFilterVal(const BloomFilterVal&);
@@ -176,18 +311,16 @@ public:
 	BroType* Type() const;
 	bool Typify(BroType* type);
 
-
 	probabilistic::CardinalityCounter* Get()	{ return c; };
 
 protected:
 	CardinalityVal();
 
+	DECLARE_OPAQUE_VALUE(CardinalityVal)
 private:
 	BroType* type;
 	CompositeHash* hash;
 	probabilistic::CardinalityCounter* c;
-
-	DECLARE_SERIAL(CardinalityVal);
 };
 
 class ParaglobVal : public OpaqueVal {

src/RE.cc

@@ -9,7 +9,7 @@
 #include "DFA.h"
 #include "CCL.h"
 #include "EquivClass.h"
-#include "Serializer.h"
+#include "Reporter.h"
 
 CCL* curr_ccl = 0;
 
@@ -469,57 +469,6 @@ int RE_Matcher::Compile(int lazy)
 	return re_anywhere->Compile(lazy) && re_exact->Compile(lazy);
 	}
 
-bool RE_Matcher::Serialize(SerialInfo* info) const
-	{
-	return SerialObj::Serialize(info);
-	}
-
-RE_Matcher* RE_Matcher::Unserialize(UnserialInfo* info)
-	{
-	return (RE_Matcher*) SerialObj::Unserialize(info, SER_RE_MATCHER);
-	}
-
-IMPLEMENT_SERIAL(RE_Matcher, SER_RE_MATCHER);
-
-bool RE_Matcher::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_RE_MATCHER, SerialObj);
-	return SERIALIZE(re_anywhere->PatternText())
-			&& SERIALIZE(re_exact->PatternText());
-	}
-
-bool RE_Matcher::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(SerialObj);
-
-	re_anywhere = new Specific_RE_Matcher(MATCH_ANYWHERE);
-	re_exact = new Specific_RE_Matcher(MATCH_EXACTLY);
-
-	const char* pat;
-	if ( ! UNSERIALIZE_STR(&pat, 0) )
-		return false;
-
-	re_anywhere->SetPat(pat);
-	if ( ! re_anywhere->Compile() )
-		{
-		info->s->Error(fmt("Can't compile regexp '%s'", pat));
-		return false;
-		}
-
-	if ( ! UNSERIALIZE_STR(&pat, 0) )
-		return false;
-
-	re_exact->SetPat(pat);
-	if ( ! re_exact->Compile() )
-		{
-		info->s->Error(fmt("Can't compile regexp '%s'", pat));
-		return false;
-		}
-
-	return true;
-	}
-
-
 static RE_Matcher* matcher_merge(const RE_Matcher* re1, const RE_Matcher* re2,
 				const char* merge_op)
 	{

src/RE.h

@@ -171,12 +171,12 @@ protected:
 	int current_pos;
 };
 
-class RE_Matcher : SerialObj {
+class RE_Matcher {
 public:
 	RE_Matcher();
 	explicit RE_Matcher(const char* pat);
 	RE_Matcher(const char* exact_pat, const char* anywhere_pat);
-	virtual ~RE_Matcher() override;
+	virtual ~RE_Matcher();
 
 	void AddPat(const char* pat);
 
@@ -212,9 +212,6 @@ public:
 	const char* PatternText() const	{ return re_exact->PatternText(); }
 	const char* AnywherePatternText() const	{ return re_anywhere->PatternText(); }
 
-	bool Serialize(SerialInfo* info) const;
-	static RE_Matcher* Unserialize(UnserialInfo* info);
-
 	unsigned int MemoryAllocation() const
 		{
 		return padded_sizeof(*this)
@@ -223,8 +220,6 @@ public:
 		}
 
 protected:
-	DECLARE_SERIAL(RE_Matcher);
-
 	Specific_RE_Matcher* re_anywhere;
 	Specific_RE_Matcher* re_exact;
 };

src/Reassem.cc

@@ -6,7 +6,6 @@
 #include "zeek-config.h"
 
 #include "Reassem.h"
-#include "Serializer.h"
 
 static const bool DEBUG_reassem = false;
 
@@ -357,37 +356,3 @@ uint64 Reassembler::MemoryAllocation(ReassemblerType rtype)
 	return Reassembler::sizes[rtype];
 	}
 
-bool Reassembler::Serialize(SerialInfo* info) const
-	{
-	return SerialObj::Serialize(info);
-	}
-
-Reassembler* Reassembler::Unserialize(UnserialInfo* info)
-	{
-	return (Reassembler*) SerialObj::Unserialize(info, SER_REASSEMBLER);
-	}
-
-bool Reassembler::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_REASSEMBLER, BroObj);
-
-	// I'm not sure if it makes sense to actually save the buffered data.
-	// For now, we just remember the seq numbers so that we don't get
-	// complaints about missing content.
-	return SERIALIZE(trim_seq) && SERIALIZE(int(0));
-	}
-
-bool Reassembler::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(BroObj);
-
-	blocks = last_block = 0;
-
-	int dummy; // For backwards compatibility.
-	if ( ! UNSERIALIZE(&trim_seq) || ! UNSERIALIZE(&dummy) )
-		return false;
-
-	last_reassem_seq = trim_seq;
-
-	return  true;
-	}

src/Reassem.h

@@ -62,9 +62,6 @@ public:
 
 	void Describe(ODesc* d) const override;
 
-	bool Serialize(SerialInfo* info) const;
-	static Reassembler* Unserialize(UnserialInfo* info);
-
 	// Sum over all data buffered in some reassembler.
 	static uint64 TotalMemoryAllocation()	{ return total_size; }
 
@@ -76,8 +73,6 @@ public:
 protected:
 	Reassembler()	{ }
 
-	DECLARE_ABSTRACT_SERIAL(Reassembler);
-
 	friend class DataBlock;
 
 	virtual void Undelivered(uint64 up_to_seq);

src/SerialInfo.h

@@ -3,13 +3,10 @@
 #ifndef serialinfo_h
 #define serialinfo_h
 
-#include "ChunkedIO.h"
-
 class SerialInfo {
 public:
 	SerialInfo(Serializer* arg_s)
 		{
-		chunk = 0;
 		s = arg_s;
 		may_suspend = clear_containers = false;
 		cache = globals_as_names = true;
@@ -21,7 +18,6 @@ public:
 
 	SerialInfo(const SerialInfo& info)
 		{
-		chunk = info.chunk;
 		s = info.s;
 		may_suspend = info.may_suspend;
 		cache = info.cache;
@@ -49,8 +45,6 @@ public:
 	// If true, we support keeping objs in cache permanently.
 	bool new_cache_strategy;
 
-	ChunkedIO::Chunk* chunk; // chunk written right before the serialization
-
 	// Attributes set during serialization.
 	SerialType type;	// type of currently serialized object
 
@@ -65,7 +59,6 @@ public:
 		s = arg_s;
 		cache = true;
 		type = SER_NONE;
-		chunk = 0;
 		install_globals = install_conns = true;
 		install_uniques = false;
 		ignore_callbacks = false;
@@ -80,7 +73,6 @@ public:
 		s = info.s;
 		cache = info.cache;
 		type = info.type;
-		chunk = info.chunk;
 		install_globals = info.install_globals;
 		install_uniques = info.install_uniques;
 		install_conns = info.install_conns;
@@ -96,8 +88,6 @@ public:
 	bool cache;	// if true,  object caching is ok
 	FILE* print;	// print read objects to given file (human-readable)
 
-	ChunkedIO::Chunk* chunk; // chunk to parse (rather than reading one)
-
 	bool install_globals;	// if true, install unknown globals
 				// in global scope
 	bool install_conns;	// if true, add connections to session table

src/SerialObj.cc

@@ -1,277 +0,0 @@
-#include "SerialObj.h"
-#include "Serializer.h"
-
-TransientID::ID TransientID::counter = 0;
-
-SerialObj::FactoryMap* SerialObj::factories = 0;
-SerialObj::ClassNameMap* SerialObj::names = 0;
-uint64 SerialObj::time_counter = NEVER + ALWAYS + 1;
-
-SerialObj* SerialObj::Instantiate(SerialType type)
-	{
-	FactoryMap::iterator f = factories->find(type & SER_TYPE_MASK_EXACT);
-	if ( f != factories->end() )
-		{
-		SerialObj* o = (SerialObj*) (*f->second)();
-#ifdef DEBUG
-		o->serial_type = o->GetSerialType();
-#endif
-		return o;
-		}
-
-	reporter->Error("Unknown object type 0x%08x", type);
-	return 0;
-	}
-
-const char* SerialObj::ClassName(SerialType type)
-	{
-	ClassNameMap::iterator f = names->find(type);
-	if ( f != names->end() )
-		return f->second;
-
-	reporter->Error("Unknown object type 0x%08x", type);
-	return "<no-class-name>";
-	}
-
-void SerialObj::Register(SerialType type, FactoryFunc f, const char* name)
-	{
-	if ( ! factories )
-		{
-		factories = new FactoryMap;
-		names = new ClassNameMap;
-		}
-
-	type = type & SER_TYPE_MASK_EXACT;
-
-	FactoryMap::iterator i = factories->find(type);
-	if ( i != factories->end() )
-		reporter->InternalError("SerialType 0x%08x registered twice", type);
-
-	(*factories)[type] = f;
-	(*names)[type] = name;
-	}
-
-inline bool SerializePID(SerialInfo* info, bool full, SerializationCache::PermanentID pid)
-	{
-	if ( ! SERIALIZE(full) )
-		return false;
-
-	if ( ! info->pid_32bit )
-		return SERIALIZE(pid);
-
-	// Broccoli compatibility mode with 32bit pids.
-	uint32 tmp = uint32(pid);
-	return SERIALIZE(tmp);
-	}
-
-bool SerialObj::Serialize(SerialInfo* info) const
-	{
-	assert(info);
-
-	if ( info->cont.NewInstance() )
-		{
-		SerializationCache::PermanentID pid = SerializationCache::NONE;
-
-		const TransientID* tid = GetTID();
-
-		if ( ! tid )
-			reporter->InternalError("no tid - missing DECLARE_SERIAL?");
-
-		if ( info->cache )
-			pid = info->s->Cache()->Lookup(*tid);
-
-		if ( pid != SerializationCache::NONE && info->cache )
-			{
-			DBG_LOG(DBG_SERIAL, "%s [%p, ref pid %lld, tid %lld]", __PRETTY_FUNCTION__, this, (long long) pid, tid->Value() );
-
-			DBG_LOG(DBG_SERIAL, "-- Caching");
-			DBG_PUSH(DBG_SERIAL);
-
-			if ( ! SerializePID(info, false, pid) )
-				{
-				DBG_POP(DBG_SERIAL);
-				return false;
-				}
-
-			DBG_POP(DBG_SERIAL);
-			return true;
-			}
-
-		if ( info->cache )
-			pid = info->s->Cache()->Register(this,
-						SerializationCache::NONE,
-						info->new_cache_strategy);
-
-		DBG_LOG(DBG_SERIAL, "%s [%p, new pid %lld, tid %lld]", __PRETTY_FUNCTION__, this, (long long) pid, tid->Value() );
-		DBG_LOG(DBG_SERIAL, "-- Caching");
-		DBG_PUSH(DBG_SERIAL);
-
-		if ( ! SerializePID(info, true, pid) )
-			{
-			DBG_POP(DBG_SERIAL);
-			return false;
-			}
-
-		info->type = SER_NONE;
-		DBG_POP(DBG_SERIAL);
-		}
-
-	DBG_PUSH(DBG_SERIAL);
-	info->cont.SaveContext();
-	bool ret = DoSerialize(info);
-	info->cont.RestoreContext();
-	DBG_POP(DBG_SERIAL);
-
-	if ( info->cont.ChildSuspended() )
-		return ret;
-
-#ifdef DEBUG
-	if ( debug_logger.IsEnabled(DBG_SERIAL) && IsBroObj(serial_type) )
-		{
-		ODesc desc(DESC_READABLE);
-		((BroObj*)this)->Describe(&desc);
-		DBG_LOG(DBG_SERIAL, "-- Desc: %s", desc.Description());
-		}
-#endif
-
-	return ret;
-	}
-
-SerialObj* SerialObj::Unserialize(UnserialInfo* info, SerialType type)
-	{
-	SerializationCache::PermanentID pid = SerializationCache::NONE;
-
-	DBG_LOG(DBG_SERIAL, "%s", __PRETTY_FUNCTION__);
-
-	bool full_obj;
-
-	DBG_LOG(DBG_SERIAL, "-- Caching");
-	DBG_PUSH(DBG_SERIAL);
-
-	bool result;
-
-	if ( ! info->pid_32bit )
-		result = UNSERIALIZE(&full_obj) && UNSERIALIZE(&pid);
-	else
-		{
-		// Broccoli compatibility mode with 32bit pids.
-		uint32 tmp = 0;
-		result = UNSERIALIZE(&full_obj) && UNSERIALIZE(&tmp);
-		pid = tmp;
-		}
-
-	if ( ! result )
-		{
-		DBG_POP(DBG_SERIAL);
-		return 0;
-		}
-
-	DBG_POP(DBG_SERIAL);
-
-	DBG_LOG(DBG_SERIAL, "-- [%s pid %lld]", full_obj ? "obj" : "ref", (long long) pid);
-
-	if ( ! full_obj )
-		{
-		// FIXME: Yet another const_cast to check eventually...
-		SerialObj* obj =
-			const_cast<SerialObj*>(info->s->Cache()->Lookup(pid));
-		if ( obj )
-			{
-			if ( obj->IsBroObj() )
-				Ref((BroObj*) obj);
-			return obj;
-			}
-
-		// In the following we'd like the format specifier to match
-		// the type of pid; but pid is uint64, for which there's
-		// no portable format specifier.  So we upcast it to long long,
-		// which is at least that size, and use a matching format.
-		info->s->Error(fmt("unknown object %lld referenced",
-				(long long) pid));
-		return 0;
-		}
-
-	uint16 stype;
-	if ( ! UNSERIALIZE(&stype) )
-		return 0;
-
-	SerialObj* obj = Instantiate(SerialType(stype));
-
-	if ( ! obj )
-		{
-		info->s->Error("unknown object type");
-		return 0;
-		}
-
-#ifdef DEBUG
-	obj->serial_type = stype;
-#endif
-
-	const TransientID* tid = obj->GetTID();
-	if ( ! tid )
-		reporter->InternalError("no tid - missing DECLARE_SERIAL?");
-
-	if ( info->cache )
-		info->s->Cache()->Register(obj, pid, info->new_cache_strategy);
-
-	info->type = stype;
-
-	DBG_PUSH(DBG_SERIAL);
-	if ( ! obj->DoUnserialize(info) )
-		{
-		DBG_POP(DBG_SERIAL);
-		return 0;
-		}
-
-	DBG_POP(DBG_SERIAL);
-
-	if ( ! SerialObj::CheckTypes(stype, type) )
-		{
-		info->s->Error("type mismatch");
-		return 0;
-		}
-
-#ifdef DEBUG
-	if ( debug_logger.IsEnabled(DBG_SERIAL) && IsBroObj(stype) )
-		{
-		ODesc desc(DESC_READABLE);
-		((BroObj*)obj)->Describe(&desc);
-		DBG_LOG(DBG_SERIAL, "-- Desc: %s", desc.Description());
-		}
-#endif
-
-	assert(obj);
-	return obj;
-	}
-
-bool SerialObj::DoSerialize(SerialInfo* info) const
-	{
-	assert(info->type != SER_NONE);
-
-#ifdef DEBUG
-	const_cast<SerialObj*>(this)->serial_type = info->type;
-#endif
-
-	DBG_LOG(DBG_SERIAL, __PRETTY_FUNCTION__);
-	DBG_PUSH(DBG_SERIAL);
-
-	uint16 stype = uint16(info->type);
-
-	if ( ! info->new_cache_strategy )
-		{
-		// This is a bit unfortunate: to make sure we're sending
-		// out the same types as in the past, we need to strip out
-		// the new cache stable bit.
-		stype &= ~SER_IS_CACHE_STABLE;
-		}
-
-	bool ret = SERIALIZE(stype);
-	DBG_POP(DBG_SERIAL);
-	return ret;
-	}
-
-bool SerialObj::DoUnserialize(UnserialInfo* info)
-	{
-	DBG_LOG(DBG_SERIAL, __PRETTY_FUNCTION__);
-	return true;
-	}

src/SerialObj.h

@@ -1,382 +0,0 @@
-// Infrastructure for serializable objects.
-//
-// How to make objects of class Foo serializable:
-//
-//    1. Derive Foo (directly or indirectly) from SerialObj.
-//    2. Add a SER_FOO constant to SerialTypes in SerialTypes.h.
-//    3. Add DECLARE_SERIAL(Foo) into class definition.
-//    4. Add a (preferably protected) default ctor if it doesn't already exist.
-//    5. For non-abstract classes, add IMPLEMENT_SERIAL(Foo, SER_FOO) to *.cc
-//    6. Add two methods like this to *.cc (keep names of arguments!)
-//
-//       bool Foo::DoSerialize(SerialInfo* info) const
-//           {
-//           DO_SERIALIZE(SER_FOO, ParentClassOfFoo);
-//           <... serialize class members via methods in Serializer ...>
-//           return true if everything ok;
-//	     }
-//
-//       bool Foo::DoUnserialize(UnserialInfo* info)
-//           {
-//           DO_UNSERIALIZE(ParentClassOfFoo);
-//           <... unserialize class members via methods in Serializer ...>
-//           return true if everything ok;
-//           }
-//
-//   (7. If no parent class of Foo already contains Serialize()/Unserialize()
-//       methods, these need to be added somewhere too. But most of the various
-//       parts of the class hierarchy already have them.)
-
-
-#ifndef SERIALOBJ_H
-#define SERIALOBJ_H
-
-#include <map>
-#include <util.h>
-
-#include "DebugLogger.h"
-#include "Continuation.h"
-#include "SerialTypes.h"
-#include "zeek-config.h"
-
-#if SIZEOF_LONG_LONG < 8
-# error "Serialization requires that sizeof(long long) is at least 8. (Remove this message only if you know what you're doing.)"
-#endif
-
-class Serializer;
-class SerialInfo;
-class UnserialInfo;
-class SerializationCache;
-
-// Per-process unique ID.
-class TransientID {
-public:
-	TransientID()	{ id = ++counter; }
-
-	typedef unsigned long long ID;
-	ID Value() const	{ return id; }
-
-private:
-	ID id;
-	static ID counter;
-};
-
-// Abstract base class for serializable objects.
-class SerialObj {
-public:
-	virtual ~SerialObj()	{ }
-
-	virtual const TransientID* GetTID() const	{ return 0; }
-
-	virtual SerialType GetSerialType() const	{ return 0; }
-
-	bool IsBroObj() const { return IsBroObj(GetSerialType()); }
-	bool IsCacheStable() const { return IsCacheStable(GetSerialType()); }
-
-	static const uint64 NEVER = 0;
-	static const uint64 ALWAYS = 1;
-
-	// Returns time of last modification. This "time" is a monotonically
-	// increasing counter which is incremented each time a modification is
-	// performed (more precisely: each time an object is modified which
-	// returns something different than NEVER). Such times can thus be
-	// compared to see whether some modification took place before another.
-	//
-	// There are two special values:
-	//    NEVER:  This object will never change.
-	//    ALWAYS: Always consider this object as changed, i.e., don't
-	//            cache it.
-	virtual uint64 LastModified() const	{ return NEVER; }
-
-	// Instantiate an object of the given type. Return nil
-	// if unknown.
-	static SerialObj* Instantiate(SerialType type);
-
-	static const char* ClassName(SerialType type);
-
-	// Associate a "factory" function with the given type.
-	// A factory is a class or function that creates instances
-	// of a certain type.
-
-	typedef SerialObj* (*FactoryFunc)();
-	static void Register(SerialType type, FactoryFunc f,
-			const char* class_name);
-
-	static bool IsBroObj(SerialType type)
-		{ return type & SER_IS_BRO_OBJ; }
-
-	static bool IsCacheStable(SerialType type)
-		{ return type & SER_IS_CACHE_STABLE; }
-
-	static bool CheckTypes(SerialType type1, SerialType type2)
-		{ return (type1 & SER_TYPE_MASK_PARENT) ==
-			 (type2 & SER_TYPE_MASK_PARENT); }
-
-protected:
-	friend class SerializationCache;
-
-	SerialObj()
-		{
-#ifdef DEBUG
-		serial_type = 0;
-#endif
-		}
-
-	// Serializes this object. If info->cache is false, we can use
-	// DECLARE_NON_CACHEABLE_SERIAL (instead of DECLARE_SERIAL) which
-	// avoids storing a per-object id.
-	bool Serialize(SerialInfo* info) const;
-
-	// Unserializes next object.
-	static SerialObj* Unserialize(UnserialInfo* info,
-					SerialType type);
-
-	virtual bool DoSerialize(SerialInfo* info) const;
-	virtual bool DoUnserialize(UnserialInfo* info);
-
-	typedef std::map<SerialType, FactoryFunc> FactoryMap;
-	static FactoryMap* factories;
-
-	typedef std::map<SerialType, const char*> ClassNameMap;
-	static ClassNameMap* names;
-
-	static uint64 time_counter;
-	static uint64 IncreaseTimeCounter()	{ return ++time_counter; }
-	static uint64 GetTimeCounter()	{ return time_counter; }
-
-#ifdef DEBUG
-	SerialType serial_type;
-#endif
-};
-
-// A class that registers a factory function upon instantiation.
-class SerialTypeRegistrator {
-public:
-	SerialTypeRegistrator(SerialType type, SerialObj::FactoryFunc func,
-			const char* class_name)
-		{
-		SerialObj::Register(type, func, class_name);
-		}
-};
-
-
-// Macro helpers.
-
-#define DECLARE_ABSTRACT_SERIAL(classname) \
-	bool DoSerialize(SerialInfo*) const override; \
-	bool DoUnserialize(UnserialInfo*) override; \
-
-#define DECLARE_SERIAL(classname) \
-	static classname* Instantiate(); \
-	static SerialTypeRegistrator register_type; \
-	bool DoSerialize(SerialInfo*) const override; \
-	bool DoUnserialize(UnserialInfo*) override; \
-	const TransientID*  GetTID() const override	{ return &tid; } \
-	SerialType GetSerialType() const override; \
-	TransientID tid;
-
-// Only needed (and usable) for non-abstract classes.
-#define IMPLEMENT_SERIAL(classname, classtype) \
-	SerialTypeRegistrator classname::register_type(classtype, \
-			FactoryFunc(&classname::Instantiate), #classname); \
-	SerialType classname::GetSerialType() const { return classtype; }; \
-	classname* classname::Instantiate()	{ return new classname(); } \
-
-// Pushes debug level on instantiation and pops when it goes out of scope.
-class AutoPush {
-public:
-	AutoPush()	{ DBG_PUSH(DBG_SERIAL); }
-	~AutoPush()	{ DBG_POP(DBG_SERIAL); }
-};
-
-// Note that by default we disable suspending.  Use DO_SERIALIZE_WITH_SUSPEND
-// to enable, but be careful to make sure that whomever calls us is aware of
-// the fact (or has already disabled suspension itself).
-#define DO_SERIALIZE(classtype, super) \
-	DBG_LOG(DBG_SERIAL, __PRETTY_FUNCTION__); \
-	if ( info->type == SER_NONE ) \
-		info->type = classtype; \
-	DisableSuspend suspend(info); \
-	AutoPush auto_push; \
-	if ( ! super::DoSerialize(info) ) \
-		return false;
-
-// Unfortunately, this is getting quite long. :-(
-#define DO_SERIALIZE_WITH_SUSPEND(classtype, super) \
-	DBG_LOG(DBG_SERIAL, __PRETTY_FUNCTION__); \
-	if ( info->type == SER_NONE ) \
-		info->type = classtype; \
-	AutoPush auto_push; \
-	\
-	bool call_super = info->cont.NewInstance(); \
-	\
-	if ( info->cont.ChildSuspended() ) \
-		{ \
-		void* user_ptr = info->cont.RestoreState(); \
-		if ( user_ptr == &call_super ) \
-			call_super = true; \
-		} \
-	\
-	if ( call_super ) \
-		{ \
-		info->cont.SaveState(&call_super); \
-		info->cont.SaveContext(); \
-		bool result = super::DoSerialize(info); \
-		info->cont.RestoreContext(); \
-		if ( ! result ) \
-			return false; \
-		if ( info->cont.ChildSuspended() ) \
-			return true; \
-		info->cont.SaveState(0); \
-		} \
-
-#define DO_UNSERIALIZE(super) \
-	DBG_LOG(DBG_SERIAL, __PRETTY_FUNCTION__); \
-	AutoPush auto_push; \
-	if ( ! super::DoUnserialize(info) ) \
-		return false;
-
-#define SERIALIZE(x) \
-	info->s->Write(x, #x)
-
-#define SERIALIZE_STR(x, y) \
-	info->s->Write(x, y, #x)
-
-#define SERIALIZE_BIT(bit) \
-	info->s->Write(bool(bit), #bit)
-
-#define UNSERIALIZE(x) \
-	info->s->Read(x, #x)
-
-#define UNSERIALIZE_STR(x, y) \
-	info->s->Read(x, y, #x)
-
-#define UNSERIALIZE_BIT(bit) \
-	{ \
-	bool tmp; \
-	if ( ! info->s->Read(&tmp, #bit) ) \
-		return false; \
-	bit = (unsigned int) tmp; \
-	}
-
-// Some helpers for pointers which may be nil.
-#define SERIALIZE_OPTIONAL(ptr) \
-	{	\
-	if ( ptr )	\
-		{	\
-		if ( ! info->cont.ChildSuspended() )	\
-			if ( ! info->s->Write(true, "has_" #ptr) )	\
-				return false;	\
-		\
-		info->cont.SaveContext();	\
-		bool result = ptr->Serialize(info);	\
-		info->cont.RestoreContext();	\
-		if ( ! result )	\
-			return false;	\
-		\
-		if ( info->cont.ChildSuspended() )	\
-			return true;	\
-		}	\
-	\
-	else if ( ! info->s->Write(false, "has_" #ptr) )	\
-		return false;	\
-	}
-
-#define SERIALIZE_OPTIONAL_STR(str) \
-	{	\
-	if ( str )	\
-		{	\
-		if ( ! (info->s->Write(true, "has_" #str) && info->s->Write(str, "str")) )	\
-			return false;	\
-		}	\
-	\
-	else if ( ! info->s->Write(false, "has_" #str) )	\
-		return false;	\
-	}
-
-#define UNSERIALIZE_OPTIONAL(dst, unserialize)	\
-	{	\
-	bool has_it;	\
-	if ( ! info->s->Read(&has_it, "has_" #dst) )	\
-		return false;	\
-	\
-	if ( has_it )	\
-		{	\
-		dst = unserialize;	\
-		if ( ! dst )	\
-			return false;	\
-		}	\
-	\
-	else	\
-		dst = 0;	\
-	}
-
-#define UNSERIALIZE_OPTIONAL_STR(dst)	\
-	{	\
-	bool has_it;	\
-	if ( ! info->s->Read(&has_it, "has_" #dst) )	\
-		return false;	\
-	\
-	if ( has_it )	\
-		{	\
-		if ( ! info->s->Read(&dst, 0, "has_" #dst) )	\
-			return false;	\
-		if ( ! dst )	\
-			return false;	\
-		}	\
-	\
-	else	\
-		dst = 0;	\
-	}
-
-#define UNSERIALIZE_OPTIONAL_STR_DEL(dst, del)	\
-	{	\
-	bool has_it;	\
-	if ( ! info->s->Read(&has_it, "has_" #dst) )	\
-		{	\
-		delete del;	\
-		return 0;	\
-		}	\
-	\
-	if ( has_it )	\
-		{	\
-		if ( ! info->s->Read(&dst, 0, "has_" #dst) )	\
-			{	\
-			delete del;	\
-			return 0;	\
-			}	\
-		if ( ! dst )	\
-			{	\
-			delete del;	\
-			return 0;	\
-			}	\
-		}	\
-	\
-	else	\
-		dst = 0;	\
-	}
-
-#define UNSERIALIZE_OPTIONAL_STATIC(dst, unserialize, del)	\
-	{	\
-	bool has_it;	\
-	if ( ! info->s->Read(&has_it, "has_" #dst) )	\
-		{	\
-		delete del;	\
-		return 0;	\
-		}	\
-	\
-	if ( has_it )	\
-		{	\
-		dst = unserialize;	\
-		if ( ! dst )	\
-			{	\
-			delete del;	\
-			return 0;	\
-			}	\
-		}	\
-	\
-	else	\
-		dst = 0;	\
-	}
-
-#endif

src/SerialTypes.h

@@ -168,6 +168,7 @@ SERIAL_EXPR(CAST_EXPR, 45)
 SERIAL_EXPR(IS_EXPR_, 46) // Name conflict with internal SER_IS_EXPR constant.
 SERIAL_EXPR(BIT_EXPR, 47)
 SERIAL_EXPR(COMPLEMENT_EXPR, 48)
+SERIAL_EXPR(INDEX_SLICE_ASSIGN_EXPR, 49)
 
 #define SERIAL_STMT(name, val) SERIAL_CONST(name, val, STMT)
 SERIAL_STMT(STMT, 1)

src/SerializationFormat.cc

@@ -2,7 +2,7 @@
 
 #include "net_util.h"
 #include "SerializationFormat.h"
-#include "Serializer.h"
+#include "DebugLogger.h"
 #include "Reporter.h"
 
 const float SerializationFormat::GROWTH_FACTOR = 2.5;

src/Serializer.h

@@ -1,366 +0,0 @@
-#ifndef SERIALIZER_H
-#define SERIALIZER_H
-
-#include <map>
-#include <list>
-#include <pcap.h>
-
-#include "ID.h"
-#include "List.h"
-#include "Expr.h"
-#include "ChunkedIO.h"
-#include "SerializationFormat.h"
-#include "StateAccess.h"
-#include "PriorityQueue.h"
-#include "SerialInfo.h"
-#include "IP.h"
-#include "Timer.h"
-#include "iosource/IOSource.h"
-#include "Reporter.h"
-
-class SerializationCache;
-class SerialInfo;
-
-class Connection;
-class Timer;
-class Packet;
-
-class Serializer {
-public:
-	// Currently ID serialization is the only method which may suspend.
-	bool Serialize(SerialInfo* info, const ID& id);
-	bool Serialize(SerialInfo* info, const char* func, val_list* args);
-	bool Serialize(SerialInfo* info, const StateAccess& s);
-	bool Serialize(SerialInfo* info, const Connection& c);
-	bool Serialize(SerialInfo* info, const Timer& t);
-	bool Serialize(SerialInfo* info, const Packet& p);
-
-	// Access to the current cache.
-	SerializationCache* Cache()	{ return current_cache; }
-	void SetCache(SerializationCache* cache)
-		{ current_cache = cache; }
-
-	// Input/output methods.
-
-#define DECLARE_READ(type) \
-	bool Read(type* v, const char* tag)	{ return format->Read(v, tag); }
-
-#define DECLARE_WRITE(type) \
-	bool Write(type v, const char* tag)	\
-		{ return format->Write(v, tag); }
-
-#define DECLARE_IO(type)	\
-	DECLARE_READ(type)	\
-	DECLARE_WRITE(type)
-
-	DECLARE_IO(int)
-	DECLARE_IO(uint16)
-	DECLARE_IO(uint32)
-	DECLARE_IO(int64)
-	DECLARE_IO(uint64)
-	DECLARE_IO(char)
-	DECLARE_IO(bool)
-	DECLARE_IO(double)
-
-	bool Read(char** str, int* len, const char* tag)
-		{ return format->Read(str, len, tag); }
-	bool Read(const char** str, int* len, const char* tag)
-		// This cast is ok.
-		{ return format->Read(const_cast<char**>(str), len, tag); }
-
-	bool Read(string* s, const char* tag);
-	bool Read(vector<uint8>* v, const char* tag) { return format->Read(v, tag); }
-	bool Read(IPAddr* a, const char* tag)	{ return format->Read(a, tag); }
-	bool Read(IPPrefix* p, const char* tag)	{ return format->Read(p, tag); }
-
-	bool Write(const char* s, const char* tag)
-		{ return format->Write(s, tag); }
-	bool Write(const char* buf, int len, const char* tag)
-		{ return format->Write(buf, len, tag); }
-	bool Write(const string& s, const char* tag)
-		{ return format->Write(s.data(), s.size(), tag); }
-	bool Write(const vector<uint8>* v, const char* tag)
-		{ return format->Write(v, tag); }
-	bool Write(const IPAddr& a, const char* tag)	{ return format->Write(a, tag); }
-	bool Write(const IPPrefix& p, const char* tag)	{ return format->Write(p, tag); }
-
-	bool WriteOpenTag(const char* tag)
-		{ return format->WriteOpenTag(tag); }
-	bool WriteCloseTag(const char* tag)
-		{ return format->WriteCloseTag(tag); }
-
-	bool WriteSeparator()	{ return format->WriteSeparator(); }
-
-	void Error(const char* msg);
-	void Warning(const char* msg);
-
-	void SetErrorDescr(const char* descr)
-		{ delete [] error_descr; error_descr = copy_string(descr); }
-
-protected:
-	// Format defaults to binary serialization.
-	explicit Serializer(SerializationFormat* format = 0);
-	virtual ~Serializer();
-
-	// Reads next object.
-	// If 'block' is true, wait until an object can be read.
-	// Returns 0 if no more object available, -1 on error.
-	int Unserialize(UnserialInfo* info, bool block = false);
-
-	// Callback for error messages.
-	virtual void ReportError(const char* msg) = 0;
-
-	// Callbacks for unserialized objects.
-
-	// id points to ID in global scope, val is unserialized value.
-	virtual void GotID(ID* id, Val* val) = 0;
-	virtual void GotEvent(const char* name, double time,
-				EventHandlerPtr event, val_list* args) = 0;
-	virtual void GotFunctionCall(const char* name, double time,
-				Func* func, val_list* args) = 0;
-	virtual void GotStateAccess(StateAccess* s) = 0;
-	virtual void GotTimer(Timer* t) = 0;
-	virtual void GotConnection(Connection* c) = 0;
-	virtual void GotPacket(Packet* packet) = 0;
-
-	// Magic to recognize state files.
-	static const uint32 MAGIC = 0x42525354;
-
-	// This will be increased whenever there is an incompatible change
-	// in the data format.
-	static const uint32 DATA_FORMAT_VERSION = 26;
-
-	ChunkedIO* io;
-
-private:
-	bool StartSerialization(SerialInfo* info, const char* descr, char tag);
-	bool EndSerialization(SerialInfo* info);
-
-	bool UnserializeID(UnserialInfo* info);
-	bool UnserializeCall(UnserialInfo* info);
-	bool UnserializeStateAccess(UnserialInfo* info);
-	bool UnserializeTimer(UnserialInfo* info);
-	bool UnserializeConnection(UnserialInfo* info);
-	bool UnserializePacket(UnserialInfo* info);
-
-	SerializationFormat* format;
-	SerializationCache* current_cache;
-	const char* error_descr;	// used in error messages
-};
-
-
-
-// We maintain an LRU-cache for some of the objects which have already been
-// serialized. For the cache, we need two types of IDs: TransientIDs (defined
-// in SerialObj.cc) uniquely reference an object during the lifetime of a
-// process.  PermanentIDs uniquely reference an object within a serialization.
-
-class SerializationCache {
-public:
-	typedef uint64 PermanentID;
-	static const PermanentID NONE = 0;
-
-	// If max_cache_size is greater than zero, we'll remove old entries
-	// automatically if limit is reached (LRU expiration).
-	explicit SerializationCache(unsigned int max_cache_size = 0);
-	~SerializationCache();
-
-	PermanentID Register(const SerialObj* obj, PermanentID pid,
-				bool new_cache_strategy);
-
-	const SerialObj* Lookup(PermanentID pid)
-		{
-		PIDMap::const_iterator i = pid_map.find(pid);
-		if ( i == pid_map.end() )
-			return 0;
-
-		assert(i->second);
-		MoveEntryToTail(i->second);
-		return i->second->obj.serial;
-		}
-
-	PermanentID Lookup(const TransientID& tid)
-		{
-		TIDMap::const_iterator i = tid_map.find(tid.Value());
-		if ( i == tid_map.end() )
-			return 0;
-
-		uint64 modified = i->second->obj.serial->LastModified();
-		if ( modified == SerialObj::ALWAYS || modified > i->second->time )
-			return 0;
-
-		assert(i->second);
-		MoveEntryToTail(i->second);
-		return i->second->pid;
-		}
-
-	unsigned int GetMaxCacheSize() const	{ return max_cache_size; }
-	void SetMaxCacheSize(unsigned int size)	{ max_cache_size = size; }
-
-	// These methods have to be called at the start/end of the
-	// serialization of an entity. The cache guarentees that objects
-	// registered after Begin() remain valid until End() is called.
-	// After End(), objects which are not derived from BroObj are
-	// discarded; others *may* remain valid.
-	void Begin(bool can_keep_in_cache)	{ End(can_keep_in_cache); }
-	void End(bool can_keep_in_cache);
-
-	void Clear();
-
-private:
-
-	struct CacheList;
-
-	struct CacheEntry {
-		union {
-			const SerialObj* serial;
-			const BroObj* bro;
-		} obj;
-
-		bool is_bro_obj;
-		PermanentID pid;
-		TransientID::ID tid;
-		uint64 time;
-		struct CacheList* cache;
-		CacheEntry* prev;
-		CacheEntry* next;
-
-		SerialType stype;	// primarily for debugging
-	};
-
-	// We maintain two LRU-sorted lists, one for often-changing objects and
-	// one for only rarely changing objects;
-	struct CacheList {
-		CacheEntry* head;
-		CacheEntry* tail;
-		unsigned int size;
-	};
-
-	void RemoveEntry(CacheEntry* e);
-	void UnlinkEntry(CacheEntry* e);
-	void MoveEntryToTail(CacheEntry* e);
-
-	unsigned int max_cache_size;
-
-	typedef map<PermanentID, CacheEntry*> PIDMap;
-	typedef map<TransientID::ID, CacheEntry*> TIDMap;
-
-	TIDMap tid_map;
-	PIDMap pid_map;
-
-	CacheList cache_stable;
-	CacheList cache_unstable;
-
-	// Objects in the cache which aren't derived from BroObj. These are
-	// always stored in the unstable cache.
-	typedef list<CacheEntry*> VolatileList;
-	VolatileList volatiles;
-
-	PermanentID next_id;
-};
-
-// A serializer for cloning objects.  Objects can be serialized into
-// the serializer and unserialized into new objects.  An absolutely
-// minimal implementation of Serializer!
-class CloneSerializer : public Serializer {
-public:
-	explicit CloneSerializer(SerializationFormat* format = 0) : Serializer(format) { }
-	~CloneSerializer() override
-		{ }
-
-protected:
-	void ReportError(const char* msg) override { reporter->Error("%s", msg); }
-	void GotID(ID* id, Val* val) override { }
-	void GotEvent(const char* name, double time, EventHandlerPtr event, val_list* args) override { }
-	void GotFunctionCall(const char* name, double time,
-				Func* func, val_list* args) override { }
-	void GotStateAccess(StateAccess* s) override { delete s; }
-	void GotTimer(Timer* t) override { }
-	void GotConnection(Connection* c) override { }
-	void GotPacket(Packet* packet) override { }
-};
-
-// Write values/events to file or fd.
-class FileSerializer : public Serializer {
-public:
-	explicit FileSerializer(SerializationFormat* format = 0);
-	~FileSerializer() override;
-
-	// Opens the file for serialization.
-	bool Open(const char* file, bool pure = false);
-	bool Close();
-
-	// Reads the file.
-	bool Read(UnserialInfo* info, const char* file, bool header = true);
-
-protected:
-	void ReportError(const char* msg) override;
-	void GotID(ID* id, Val* val) override;
-	void GotEvent(const char* name, double time,
-				EventHandlerPtr event, val_list* args) override;
-	void GotFunctionCall(const char* name, double time,
-				Func* func, val_list* args) override;
-	void GotStateAccess(StateAccess* s) override;
-	void GotTimer(Timer* t) override;
-	void GotConnection(Connection* c) override;
-	void GotPacket(Packet* packet) override;
-
-	bool OpenFile(const char* file, bool readonly, bool should_exist = false);
-	void CloseFile();
-	bool ReadFile(const char* file);
-	bool PrepareForWriting();
-	bool ReadHeader(UnserialInfo* info = 0);
-
-	SerializationCache cache;
-	const char* file;
-	int fd;
-};
-
-// Abstract interface class for external sources providing a stream of events.
-class EventSource {
-public:
-	virtual ~EventSource() { }
-
-	// Returns time of the oldest event (0 if none available).
-	virtual double NextTimestamp(double* local_network_time) = 0;
-
-	// Dispatches the oldest event and removes it.
-	virtual void DispatchNextEvent() = 0;
-
-	// Returns true if there are more events to expect from this source.
-	virtual bool IsActive() = 0;
-};
-
-// Plays a file of events back.
-class EventPlayer : public FileSerializer, public iosource::IOSource {
-public:
-	explicit EventPlayer(const char* file);
-	~EventPlayer() override;
-
-	void GetFds(iosource::FD_Set* read, iosource::FD_Set* write,
-	                    iosource::FD_Set* except) override;
-	double NextTimestamp(double* local_network_time) override;
-	void Process() override;
-	const char* Tag() override { return "EventPlayer"; }
-
-protected:
-	void GotID(ID* id, Val* val) override {}
-	void GotEvent(const char* name, double time,
-				EventHandlerPtr event, val_list* args) override;
-	void GotFunctionCall(const char* name, double time,
-				Func* func, val_list* args) override;
-
-	double stream_time;	// time of first captured event
-	double replay_time;	// network time of replay start
-
-	// Next event waiting to be dispatched.
-	double ne_time;
-	EventHandlerPtr ne_handler;
-	val_list ne_args;
-
-};
-
-extern FileSerializer* event_serializer;
-extern FileSerializer* state_serializer;
-
-#endif

src/StateAccess.cc

@@ -1,6 +1,5 @@
 #include "Val.h"
 #include "StateAccess.h"
-#include "Serializer.h"
 #include "Event.h"
 #include "NetVar.h"
 #include "DebugLogger.h"
@@ -72,7 +71,6 @@ StateAccess::StateAccess(Opcode arg_opcode,
 	}
 
 StateAccess::StateAccess(const StateAccess& sa)
-: SerialObj()
 	{
 	opcode = sa.opcode;
 	target_type = sa.target_type;
@@ -343,146 +341,6 @@ ID* StateAccess::Target() const
 	return target_type == TYPE_ID ? target.id : target.val->UniqueID();
 	}
 
-bool StateAccess::Serialize(SerialInfo* info) const
-	{
-	return SerialObj::Serialize(info);
-	}
-
-StateAccess* StateAccess::Unserialize(UnserialInfo* info)
-	{
-	StateAccess* sa =
-		(StateAccess*) SerialObj::Unserialize(info, SER_STATE_ACCESS);
-	return sa;
-	}
-
-IMPLEMENT_SERIAL(StateAccess, SER_STATE_ACCESS);
-
-bool StateAccess::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_STATE_ACCESS, SerialObj);
-
-	if ( ! SERIALIZE(char(opcode)) )
-		return false;
-
-	const ID* id =
-		target_type == TYPE_ID ? target.id : target.val->UniqueID();
-
-	if ( ! SERIALIZE(id->Name()) )
-		 return false;
-
-	if ( op1_type == TYPE_KEY )
-		{
-		Val* index =
-			id->ID_Val()->AsTableVal()->RecoverIndex(this->op1.key);
-
-		if ( ! index )
-			return false;
-		if ( ! index->Serialize(info) )
-			return false;
-
-		Unref(index);
-		}
-
-	else if ( ! op1.val->Serialize(info) )
-		return false;
-
-	// Don't send the "old" operand if we don't want consistency checks.
-	// Unfortunately, it depends on the opcode which operand that actually
-	// is.
-
-	const Val* null = 0;
-
-	switch ( opcode ) {
-	case OP_PRINT:
-	case OP_EXPIRE:
-	case OP_READ_IDX:
-		// No old.
-		SERIALIZE_OPTIONAL(null);
-		SERIALIZE_OPTIONAL(null);
-		break;
-
-	case OP_INCR:
-	case OP_INCR_IDX:
-		// Always need old.
-		SERIALIZE_OPTIONAL(op2);
-		SERIALIZE_OPTIONAL(op3);
-		break;
-
-	case OP_ASSIGN:
-	case OP_ADD:
-	case OP_DEL:
-		// Op2 is old.
-		SERIALIZE_OPTIONAL(null);
-		SERIALIZE_OPTIONAL(null);
-		break;
-
-	case OP_ASSIGN_IDX:
-		// Op3 is old.
-		SERIALIZE_OPTIONAL(op2);
-		SERIALIZE_OPTIONAL(null);
-		break;
-
-	default:
-		reporter->InternalError("StateAccess::DoSerialize: unknown opcode");
-	}
-
-	return true;
-	}
-
-bool StateAccess::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(SerialObj);
-
-	char c;
-	if ( ! UNSERIALIZE(&c) )
-		return false;
-
-	opcode = Opcode(c);
-
-	const char* name;
-	if ( ! UNSERIALIZE_STR(&name, 0) )
-		return false;
-
-	target_type = TYPE_ID;
-	target.id = global_scope()->Lookup(name);
-
-	if ( target.id )
-		// Otherwise, we'll delete it below.
-		delete [] name;
-
-	op1_type = TYPE_VAL;
-	op1.val = Val::Unserialize(info);
-	if ( ! op1.val )
-		return false;
-
-	UNSERIALIZE_OPTIONAL(op2, Val::Unserialize(info));
-	UNSERIALIZE_OPTIONAL(op3, Val::Unserialize(info));
-
-	if ( target.id )
-		Ref(target.id);
-	else
-		{
-		// This may happen as long as we haven't agreed on the
-		// unique name for an ID during initial synchronization, or if
-		// the local peer has already deleted the ID.
-		DBG_LOG(DBG_STATE, "state access referenced unknown id %s", name);
-
-		if ( info->install_uniques )
-			{
-			target.id = new ID(name, SCOPE_GLOBAL, true);
-			Ref(target.id);
-			global_scope()->Insert(name, target.id);
-#ifdef USE_PERFTOOLS_DEBUG
-			heap_checker->IgnoreObject(target.id);
-#endif
-			}
-
-		delete [] name;
-		}
-
-	return true;
-	}
-
 void StateAccess::Describe(ODesc* d) const
 	{
 	const ID* id;

src/StateAccess.h

@@ -7,14 +7,11 @@
 #include <map>
 #include <string>
 
-#include "SerialObj.h"
-
 class Val;
 class ID;
 class MutableVal;
 class HashKey;
 class ODesc;
-class Serializer;
 class TableVal;
 
 enum Opcode {	// Op1	Op2 Op3 (Vals)
@@ -30,7 +27,7 @@ enum Opcode {	// Op1	Op2 Op3 (Vals)
 	OP_READ_IDX,	// idx
 };
 
-class StateAccess : public SerialObj {
+class StateAccess {
 public:
 	StateAccess(Opcode opcode, const ID* target, const Val* op1,
 			const Val* op2 = 0, const Val* op3 = 0);
@@ -48,7 +45,7 @@ public:
 
 	StateAccess(const StateAccess& sa);
 
-	~StateAccess() override;
+	virtual ~StateAccess();
 
 	// Replays this access in the our environment.
 	void Replay();
@@ -58,9 +55,6 @@ public:
 
 	void Describe(ODesc* d) const;
 
-	bool Serialize(SerialInfo* info) const;
-	static StateAccess* Unserialize(UnserialInfo* info);
-
 	// Main entry point when StateAcesses are performed.
 	// For every state-changing operation, this has to be called.
 	static void Log(StateAccess* access);
@@ -74,8 +68,6 @@ private:
 	StateAccess()	{ target.id = 0; op1.val = op2 = op3 = 0; }
 	void RefThem();
 
-	DECLARE_SERIAL(StateAccess);
-
 	Opcode opcode;
 	union {
 		ID* id;

src/Stmt.cc

@@ -117,47 +117,6 @@ void Stmt::AccessStats(ODesc* d) const
 		}
 	}
 
-bool Stmt::Serialize(SerialInfo* info) const
-	{
-	return SerialObj::Serialize(info);
-	}
-
-Stmt* Stmt::Unserialize(UnserialInfo* info, BroStmtTag want)
-	{
-	Stmt* stmt = (Stmt*) SerialObj::Unserialize(info, SER_STMT);
-
-	if ( want != STMT_ANY && stmt->tag != want )
-		{
-		info->s->Error("wrong stmt type");
-		Unref(stmt);
-		return 0;
-		}
-
-	return stmt;
-	}
-
-bool Stmt::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_STMT, BroObj);
-
-	return SERIALIZE(char(tag)) && SERIALIZE(last_access)
-			&& SERIALIZE(access_count);
-	}
-
-bool Stmt::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(BroObj);
-
-	char c;
-	if ( ! UNSERIALIZE(&c) )
-		return 0;
-
-	tag = BroStmtTag(c);
-
-	return UNSERIALIZE(&last_access) && UNSERIALIZE(&access_count);
-	}
-
-
 ExprListStmt::ExprListStmt(BroStmtTag t, ListExpr* arg_l)
 : Stmt(t)
 	{
@@ -207,19 +166,6 @@ void ExprListStmt::PrintVals(ODesc* d, val_list* vals, int offset) const
 	describe_vals(vals, d, offset);
 	}
 
-bool ExprListStmt::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_EXPR_LIST_STMT, Stmt);
-	return l->Serialize(info);
-	}
-
-bool ExprListStmt::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(Stmt);
-	l = (ListExpr*) Expr::Unserialize(info, EXPR_LIST);
-	return l != 0;
-	}
-
 TraversalCode ExprListStmt::Traverse(TraversalCallback* cb) const
 	{
 	TraversalCode tc = cb->PreStmt(this);
@@ -305,20 +251,6 @@ Val* PrintStmt::DoExec(val_list* vals, stmt_flow_type& /* flow */) const
 	return 0;
 	}
 
-IMPLEMENT_SERIAL(PrintStmt, SER_PRINT_STMT);
-
-bool PrintStmt::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_PRINT_STMT, ExprListStmt);
-	return true;
-	}
-
-bool PrintStmt::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(ExprListStmt);
-	return true;
-	}
-
 ExprStmt::ExprStmt(Expr* arg_e) : Stmt(STMT_EXPR)
 	{
 	e = arg_e;
@@ -404,22 +336,6 @@ TraversalCode ExprStmt::Traverse(TraversalCallback* cb) const
 	HANDLE_TC_STMT_POST(tc);
 	}
 
-IMPLEMENT_SERIAL(ExprStmt, SER_EXPR_STMT);
-
-bool ExprStmt::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_EXPR_STMT, Stmt);
-	SERIALIZE_OPTIONAL(e);
-	return true;
-	}
-
-bool ExprStmt::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(Stmt);
-	UNSERIALIZE_OPTIONAL(e, Expr::Unserialize(info));
-	return true;
-	}
-
 IfStmt::IfStmt(Expr* test, Stmt* arg_s1, Stmt* arg_s2) : ExprStmt(STMT_IF, test)
 	{
 	s1 = arg_s1;
@@ -507,25 +423,6 @@ TraversalCode IfStmt::Traverse(TraversalCallback* cb) const
 	HANDLE_TC_STMT_POST(tc);
 	}
 
-IMPLEMENT_SERIAL(IfStmt, SER_IF_STMT);
-
-bool IfStmt::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_IF_STMT, ExprStmt);
-	return s1->Serialize(info) && s2->Serialize(info);
-	}
-
-bool IfStmt::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(ExprStmt);
-	s1 = Stmt::Unserialize(info);
-	if ( ! s1 )
-		return false;
-
-	s2 = Stmt::Unserialize(info);
-	return s2 != 0;
-	}
-
 static BroStmtTag get_last_stmt_tag(const Stmt* stmt)
 	{
 	if ( ! stmt )
@@ -655,67 +552,6 @@ TraversalCode Case::Traverse(TraversalCallback* cb) const
 	return TC_CONTINUE;
 	}
 
-bool Case::Serialize(SerialInfo* info) const
-	{
-	return SerialObj::Serialize(info);
-	}
-
-Case* Case::Unserialize(UnserialInfo* info)
-	{
-	return (Case*) SerialObj::Unserialize(info, SER_CASE);
-	}
-
-IMPLEMENT_SERIAL(Case, SER_CASE);
-
-bool Case::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_CASE, BroObj);
-
-	if ( ! expr_cases->Serialize(info) )
-		return false;
-
-	id_list empty;
-	id_list* types = (type_cases ? type_cases : &empty);
-
-	if ( ! SERIALIZE(types->length()) )
-		return false;
-
-	loop_over_list((*types), i)
-		{
-		if ( ! (*types)[i]->Serialize(info) )
-			return false;
-		}
-
-	return this->s->Serialize(info);
-	}
-
-bool Case::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(BroObj);
-
-	expr_cases = (ListExpr*) Expr::Unserialize(info, EXPR_LIST);
-	if ( ! expr_cases )
-		return false;
-
-	int len;
-	if ( ! UNSERIALIZE(&len) )
-		return false;
-
-	type_cases = new id_list(len);
-
-	while ( len-- )
-		{
-		ID* id = ID::Unserialize(info);
-		if ( ! id )
-			return false;
-
-		type_cases->append(id);
-		}
-
-	this->s = Stmt::Unserialize(info);
-	return this->s != 0;
-	}
-
 static void int_del_func(void* v)
 	{
 	delete (int*) v;
@@ -1028,66 +864,6 @@ TraversalCode SwitchStmt::Traverse(TraversalCallback* cb) const
 	HANDLE_TC_STMT_POST(tc);
 	}
 
-IMPLEMENT_SERIAL(SwitchStmt, SER_SWITCH_STMT);
-
-bool SwitchStmt::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_SWITCH_STMT, ExprStmt);
-
-	if ( ! SERIALIZE(cases->length()) )
-		return false;
-
-	loop_over_list((*cases), i)
-		if ( ! (*cases)[i]->Serialize(info) )
-			return false;
-
-	if ( ! SERIALIZE(default_case_idx) )
-		return false;
-
-	return true;
-	}
-
-bool SwitchStmt::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(ExprStmt);
-
-	Init();
-
-	int len;
-	if ( ! UNSERIALIZE(&len) )
-		return false;
-
-	while ( len-- )
-		{
-		Case* c = Case::Unserialize(info);
-		if ( ! c )
-			return false;
-
-		cases->append(c);
-		}
-
-	if ( ! UNSERIALIZE(&default_case_idx) )
-		return false;
-
-	loop_over_list(*cases, i)
-		{
-		const ListExpr* le = (*cases)[i]->ExprCases();
-
-		if ( ! le )
-			continue;
-
-		const expr_list& exprs = le->Exprs();
-
-		loop_over_list(exprs, j)
-			{
-			if ( ! AddCaseLabelValueMapping(exprs[j]->ExprVal(), i) )
-				return false;
-			}
-		}
-
-	return true;
-	}
-
 AddStmt::AddStmt(Expr* arg_e) : ExprStmt(STMT_ADD, arg_e)
 	{
 	if ( ! e->CanAdd() )
@@ -1121,20 +897,6 @@ TraversalCode AddStmt::Traverse(TraversalCallback* cb) const
 	HANDLE_TC_STMT_POST(tc);
 	}
 
-IMPLEMENT_SERIAL(AddStmt, SER_ADD_STMT);
-
-bool AddStmt::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_ADD_STMT, ExprStmt);
-	return true;
-	}
-
-bool AddStmt::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(ExprStmt);
-	return true;
-	}
-
 DelStmt::DelStmt(Expr* arg_e) : ExprStmt(STMT_DELETE, arg_e)
 	{
 	if ( e->IsError() )
@@ -1170,20 +932,6 @@ TraversalCode DelStmt::Traverse(TraversalCallback* cb) const
 	HANDLE_TC_STMT_POST(tc);
 	}
 
-IMPLEMENT_SERIAL(DelStmt, SER_DEL_STMT);
-
-bool DelStmt::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_DEL_STMT, ExprStmt);
-	return true;
-	}
-
-bool DelStmt::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(ExprStmt);
-	return true;
-	}
-
 EventStmt::EventStmt(EventExpr* arg_e) : ExprStmt(STMT_EVENT, arg_e)
 	{
 	event_expr = arg_e;
@@ -1218,22 +966,6 @@ TraversalCode EventStmt::Traverse(TraversalCallback* cb) const
 	HANDLE_TC_STMT_POST(tc);
 	}
 
-IMPLEMENT_SERIAL(EventStmt, SER_EVENT_STMT);
-
-bool EventStmt::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_EVENT_STMT, ExprStmt);
-	return event_expr->Serialize(info);
-	}
-
-bool EventStmt::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(ExprStmt);
-
-	event_expr = (EventExpr*) Expr::Unserialize(info, EXPR_EVENT);
-	return event_expr != 0;
-	}
-
 WhileStmt::WhileStmt(Expr* arg_loop_condition, Stmt* arg_body)
 	: loop_condition(arg_loop_condition), body(arg_body)
 	{
@@ -1319,30 +1051,6 @@ Val* WhileStmt::Exec(Frame* f, stmt_flow_type& flow) const
 	return rval;
 	}
 
-IMPLEMENT_SERIAL(WhileStmt, SER_WHILE_STMT);
-
-bool WhileStmt::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_WHILE_STMT, Stmt);
-
-	if ( ! loop_condition->Serialize(info) )
-		return false;
-
-	return body->Serialize(info);
-	}
-
-bool WhileStmt::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(Stmt);
-	loop_condition = Expr::Unserialize(info);
-
-	if ( ! loop_condition )
-		return false;
-
-	body = Stmt::Unserialize(info);
-	return body != 0;
-	}
-
 ForStmt::ForStmt(id_list* arg_loop_vars, Expr* loop_expr)
 : ExprStmt(STMT_FOR, loop_expr)
 	{
@@ -1607,47 +1315,6 @@ TraversalCode ForStmt::Traverse(TraversalCallback* cb) const
 	HANDLE_TC_STMT_POST(tc);
 	}
 
-IMPLEMENT_SERIAL(ForStmt, SER_FOR_STMT);
-
-bool ForStmt::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_FOR_STMT, ExprStmt);
-
-	if ( ! SERIALIZE(loop_vars->length()) )
-		return false;
-
-	loop_over_list((*loop_vars), i)
-		{
-		if ( ! (*loop_vars)[i]->Serialize(info) )
-			return false;
-		}
-
-	return body->Serialize(info);
-	}
-
-bool ForStmt::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(ExprStmt);
-
-	int len;
-	if ( ! UNSERIALIZE(&len) )
-		return false;
-
-	loop_vars = new id_list(len);
-
-	while ( len-- )
-		{
-		ID* id = ID::Unserialize(info);
-		if ( ! id )
-			return false;
-
-		loop_vars->append(id);
-		}
-
-	body = Stmt::Unserialize(info);
-	return body != 0;
-	}
-
 Val* NextStmt::Exec(Frame* /* f */, stmt_flow_type& flow) const
 	{
 	RegisterAccess();
@@ -1675,20 +1342,6 @@ TraversalCode NextStmt::Traverse(TraversalCallback* cb) const
 	HANDLE_TC_STMT_POST(tc);
 	}
 
-IMPLEMENT_SERIAL(NextStmt, SER_NEXT_STMT);
-
-bool NextStmt::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_NEXT_STMT, Stmt);
-	return true;
-	}
-
-bool NextStmt::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(Stmt);
-	return true;
-	}
-
 Val* BreakStmt::Exec(Frame* /* f */, stmt_flow_type& flow) const
 	{
 	RegisterAccess();
@@ -1716,20 +1369,6 @@ TraversalCode BreakStmt::Traverse(TraversalCallback* cb) const
 	HANDLE_TC_STMT_POST(tc);
 	}
 
-IMPLEMENT_SERIAL(BreakStmt, SER_BREAK_STMT);
-
-bool BreakStmt::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_BREAK_STMT, Stmt);
-	return true;
-	}
-
-bool BreakStmt::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(Stmt);
-	return true;
-	}
-
 Val* FallthroughStmt::Exec(Frame* /* f */, stmt_flow_type& flow) const
 	{
 	RegisterAccess();
@@ -1757,20 +1396,6 @@ TraversalCode FallthroughStmt::Traverse(TraversalCallback* cb) const
 	HANDLE_TC_STMT_POST(tc);
 	}
 
-IMPLEMENT_SERIAL(FallthroughStmt, SER_FALLTHROUGH_STMT);
-
-bool FallthroughStmt::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_FALLTHROUGH_STMT, Stmt);
-	return true;
-	}
-
-bool FallthroughStmt::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(Stmt);
-	return true;
-	}
-
 ReturnStmt::ReturnStmt(Expr* arg_e) : ExprStmt(STMT_RETURN, arg_e)
 	{
 	Scope* s = current_scope();
@@ -1838,20 +1463,6 @@ void ReturnStmt::Describe(ODesc* d) const
 	DescribeDone(d);
 	}
 
-IMPLEMENT_SERIAL(ReturnStmt, SER_RETURN_STMT);
-
-bool ReturnStmt::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_RETURN_STMT, ExprStmt);
-	return true;
-	}
-
-bool ReturnStmt::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(ExprStmt);
-	return true;
-	}
-
 StmtList::StmtList() : Stmt(STMT_LIST)
 	{
 	}
@@ -1941,43 +1552,6 @@ TraversalCode StmtList::Traverse(TraversalCallback* cb) const
 	HANDLE_TC_STMT_POST(tc);
 	}
 
-IMPLEMENT_SERIAL(StmtList, SER_STMT_LIST);
-
-bool StmtList::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_STMT_LIST, Stmt);
-
-	if ( ! SERIALIZE(stmts.length()) )
-		return false;
-
-	loop_over_list(stmts, i)
-		if ( ! stmts[i]->Serialize(info) )
-			return false;
-
-	return true;
-	}
-
-bool StmtList::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(Stmt);
-
-	int len;
-	if ( ! UNSERIALIZE(&len) )
-		return false;
-
-	while ( len-- )
-		{
-		Stmt* stmt = Stmt::Unserialize(info);
-		if ( ! stmt )
-			return false;
-
-		stmts.append(stmt);
-		}
-
-	return true;
-	}
-
-
 Val* EventBodyList::Exec(Frame* f, stmt_flow_type& flow) const
 	{
 	RegisterAccess();
@@ -2036,20 +1610,6 @@ void EventBodyList::Describe(ODesc* d) const
 		StmtList::Describe(d);
 	}
 
-IMPLEMENT_SERIAL(EventBodyList, SER_EVENT_BODY_LIST);
-
-bool EventBodyList::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_EVENT_BODY_LIST, StmtList);
-	return SERIALIZE(topmost);
-	}
-
-bool EventBodyList::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(StmtList);
-	return UNSERIALIZE(&topmost);
-	}
-
 InitStmt::~InitStmt()
 	{
 	loop_over_list(*inits, i)
@@ -2123,45 +1683,6 @@ TraversalCode InitStmt::Traverse(TraversalCallback* cb) const
 	HANDLE_TC_STMT_POST(tc);
 	}
 
-IMPLEMENT_SERIAL(InitStmt, SER_INIT_STMT);
-
-bool InitStmt::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_INIT_STMT, Stmt);
-
-	if ( ! SERIALIZE(inits->length()) )
-		return false;
-
-	loop_over_list((*inits), i)
-		{
-		if ( ! (*inits)[i]->Serialize(info) )
-			return false;
-		}
-
-	return true;
-	}
-
-bool InitStmt::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(Stmt);
-
-	int len;
-	if ( ! UNSERIALIZE(&len) )
-		return false;
-
-	inits = new id_list(len);
-
-	while ( len-- )
-		{
-		ID* id = ID::Unserialize(info);
-		if ( ! id )
-			return false;
-		inits->append(id);
-		}
-	return true;
-	}
-
-
 Val* NullStmt::Exec(Frame* /* f */, stmt_flow_type& flow) const
 	{
 	RegisterAccess();
@@ -2191,20 +1712,6 @@ TraversalCode NullStmt::Traverse(TraversalCallback* cb) const
 	HANDLE_TC_STMT_POST(tc);
 	}
 
-IMPLEMENT_SERIAL(NullStmt, SER_NULL_STMT);
-
-bool NullStmt::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_NULL_STMT, Stmt);
-	return true;
-	}
-
-bool NullStmt::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(Stmt);
-	return true;
-	}
-
 WhenStmt::WhenStmt(Expr* arg_cond, Stmt* arg_s1, Stmt* arg_s2,
 			Expr* arg_timeout, bool arg_is_return)
 : Stmt(STMT_WHEN)
@@ -2320,35 +1827,3 @@ TraversalCode WhenStmt::Traverse(TraversalCallback* cb) const
 	HANDLE_TC_STMT_POST(tc);
 	}
 
-IMPLEMENT_SERIAL(WhenStmt, SER_WHEN_STMT);
-
-bool WhenStmt::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_WHEN_STMT, Stmt);
-
-	if ( cond->Serialize(info) && s1->Serialize(info) )
-		return false;
-
-	SERIALIZE_OPTIONAL(s2);
-	SERIALIZE_OPTIONAL(timeout);
-
-	return true;
-	}
-
-bool WhenStmt::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(Stmt);
-
-	cond = Expr::Unserialize(info);
-	if ( ! cond )
-		return false;
-
-	s1 = Stmt::Unserialize(info);
-	if ( ! s1 )
-		return false;
-
-	UNSERIALIZE_OPTIONAL(s2, Stmt::Unserialize(info));
-	UNSERIALIZE_OPTIONAL(timeout, Expr::Unserialize(info));
-
-	return true;
-	}

src/Stmt.h

@@ -71,9 +71,6 @@ public:
 
 	virtual unsigned int BPCount() const	{ return breakpoint_count; }
 
-	bool Serialize(SerialInfo* info) const;
-	static Stmt* Unserialize(UnserialInfo* info, BroStmtTag want = STMT_ANY);
-
 	virtual TraversalCode Traverse(TraversalCallback* cb) const = 0;
 
 protected:
@@ -83,8 +80,6 @@ protected:
 	void AddTag(ODesc* d) const;
 	void DescribeDone(ODesc* d) const;
 
-	DECLARE_ABSTRACT_SERIAL(Stmt);
-
 	BroStmtTag tag;
 	int breakpoint_count;	// how many breakpoints on this statement
 
@@ -111,8 +106,6 @@ protected:
 	void Describe(ODesc* d) const override;
 	void PrintVals(ODesc* d, val_list* vals, int offset) const;
 
-	DECLARE_ABSTRACT_SERIAL(ExprListStmt);
-
 	ListExpr* l;
 };
 
@@ -125,8 +118,6 @@ protected:
 	PrintStmt()	{}
 
 	Val* DoExec(val_list* vals, stmt_flow_type& flow) const override;
-
-	DECLARE_SERIAL(PrintStmt);
 };
 
 class ExprStmt : public Stmt {
@@ -151,8 +142,6 @@ protected:
 
 	int IsPure() const override;
 
-	DECLARE_SERIAL(ExprStmt);
-
 	Expr* e;
 };
 
@@ -175,8 +164,6 @@ protected:
 	Val* DoExec(Frame* f, Val* v, stmt_flow_type& flow) const override;
 	int IsPure() const override;
 
-	DECLARE_SERIAL(IfStmt);
-
 	Stmt* s1;
 	Stmt* s2;
 };
@@ -197,17 +184,12 @@ public:
 
 	void Describe(ODesc* d) const override;
 
-	bool Serialize(SerialInfo* info) const;
-	static Case* Unserialize(UnserialInfo* info);
-
 	TraversalCode Traverse(TraversalCallback* cb) const;
 
 protected:
 	friend class Stmt;
 	Case()	{ expr_cases = 0; type_cases = 0; s = 0; }
 
-	DECLARE_SERIAL(Case);
-
 	ListExpr* expr_cases;
 	id_list* type_cases;
 	Stmt* s;
@@ -234,8 +216,6 @@ protected:
 	Val* DoExec(Frame* f, Val* v, stmt_flow_type& flow) const override;
 	int IsPure() const override;
 
-	DECLARE_SERIAL(SwitchStmt);
-
 	// Initialize composite hash and case label map.
 	void Init();
 
@@ -274,8 +254,6 @@ public:
 protected:
 	friend class Stmt;
 	AddStmt()	{}
-
-	DECLARE_SERIAL(AddStmt);
 };
 
 class DelStmt : public ExprStmt {
@@ -290,8 +268,6 @@ public:
 protected:
 	friend class Stmt;
 	DelStmt()	{}
-
-	DECLARE_SERIAL(DelStmt);
 };
 
 class EventStmt : public ExprStmt {
@@ -306,8 +282,6 @@ protected:
 	friend class Stmt;
 	EventStmt()	{ event_expr = 0; }
 
-	DECLARE_SERIAL(EventStmt);
-
 	EventExpr* event_expr;
 };
 
@@ -331,8 +305,6 @@ protected:
 
 	Val* Exec(Frame* f, stmt_flow_type& flow) const override;
 
-	DECLARE_SERIAL(WhileStmt);
-
 	Expr* loop_condition;
 	Stmt* body;
 };
@@ -362,8 +334,6 @@ protected:
 
 	Val* DoExec(Frame* f, Val* v, stmt_flow_type& flow) const override;
 
-	DECLARE_SERIAL(ForStmt);
-
 	id_list* loop_vars;
 	Stmt* body;
 	// Stores the value variable being used for a key value for loop.
@@ -383,7 +353,6 @@ public:
 	TraversalCode Traverse(TraversalCallback* cb) const override;
 
 protected:
-	DECLARE_SERIAL(NextStmt);
 };
 
 class BreakStmt : public Stmt {
@@ -398,7 +367,6 @@ public:
 	TraversalCode Traverse(TraversalCallback* cb) const override;
 
 protected:
-	DECLARE_SERIAL(BreakStmt);
 };
 
 class FallthroughStmt : public Stmt {
@@ -413,7 +381,6 @@ public:
 	TraversalCode Traverse(TraversalCallback* cb) const override;
 
 protected:
-	DECLARE_SERIAL(FallthroughStmt);
 };
 
 class ReturnStmt : public ExprStmt {
@@ -427,8 +394,6 @@ public:
 protected:
 	friend class Stmt;
 	ReturnStmt()	{}
-
-	DECLARE_SERIAL(ReturnStmt);
 };
 
 class StmtList : public Stmt {
@@ -448,8 +413,6 @@ public:
 protected:
 	int IsPure() const override;
 
-	DECLARE_SERIAL(StmtList);
-
 	stmt_list stmts;
 };
 
@@ -467,9 +430,6 @@ public:
 	// bool IsTopmost()	{ return topmost; }
 
 protected:
-
-	DECLARE_SERIAL(EventBodyList);
-
 	bool topmost;
 };
 
@@ -496,8 +456,6 @@ protected:
 	friend class Stmt;
 	InitStmt()	{ inits = 0; }
 
-	DECLARE_SERIAL(InitStmt);
-
 	id_list* inits;
 };
 
@@ -511,9 +469,6 @@ public:
 	void Describe(ODesc* d) const override;
 
 	TraversalCode Traverse(TraversalCallback* cb) const override;
-
-protected:
-	DECLARE_SERIAL(NullStmt);
 };
 
 class WhenStmt : public Stmt {
@@ -537,8 +492,6 @@ public:
 protected:
 	WhenStmt()	{ cond = 0; s1 = s2 = 0; timeout = 0; is_return = 0; }
 
-	DECLARE_SERIAL(WhenStmt);
-
 	Expr* cond;
 	Stmt* s1;
 	Stmt* s2;

src/Timer.cc

@@ -5,7 +5,6 @@
 #include "util.h"
 #include "Timer.h"
 #include "Desc.h"
-#include "Serializer.h"
 #include "broker/Manager.h"
 
 // Names of timers in same order than in TimerType.
@@ -53,41 +52,6 @@ void Timer::Describe(ODesc* d) const
 	d->Add(Time());
 	}
 
-bool Timer::Serialize(SerialInfo* info) const
-	{
-	return SerialObj::Serialize(info);
-	}
-
-Timer* Timer::Unserialize(UnserialInfo* info)
-	{
-	Timer* timer = (Timer*) SerialObj::Unserialize(info, SER_TIMER);
-	if ( ! timer )
-		return 0;
-
-	timer_mgr->Add(timer);
-
-	return timer;
-	}
-
-bool Timer::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_TIMER, SerialObj);
-	char tmp = type;
-	return SERIALIZE(tmp) && SERIALIZE(time);
-	}
-
-bool Timer::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(SerialObj);
-
-	char tmp;
-	if ( ! UNSERIALIZE(&tmp) )
-		return false;
-	type = tmp;
-
-	return UNSERIALIZE(&time);
-	}
-
 unsigned int TimerMgr::current_timers[NUM_TIMER_TYPES];
 
 TimerMgr::~TimerMgr()

src/Timer.h

@@ -6,7 +6,6 @@
 #include <string>
 
 #include <string>
-#include "SerialObj.h"
 #include "PriorityQueue.h"
 
 extern "C" {
@@ -49,10 +48,9 @@ const int NUM_TIMER_TYPES = int(TIMER_TIMERMGR_EXPIRE) + 1;
 
 extern const char* timer_type_to_string(TimerType type);
 
-class Serializer;
 class ODesc;
 
-class Timer : public SerialObj, public PQ_Element {
+class Timer : public PQ_Element {
 public:
 	Timer(double t, TimerType arg_type) : PQ_Element(t)
 		{ type = (char) arg_type; }
@@ -67,14 +65,9 @@ public:
 
 	void Describe(ODesc* d) const;
 
-	bool Serialize(SerialInfo* info) const;
-	static Timer* Unserialize(UnserialInfo* info);
-
 protected:
 	Timer()	{}
 
-	DECLARE_ABSTRACT_SERIAL(Timer);
-
 	unsigned int type:8;
 };
 

src/Type.cc

@@ -6,7 +6,6 @@
 #include "Attr.h"
 #include "Expr.h"
 #include "Scope.h"
-#include "Serializer.h"
 #include "Reporter.h"
 #include "zeekygen/Manager.h"
 #include "zeekygen/utils.h"
@@ -122,27 +121,30 @@ BroType::BroType(TypeTag t, bool arg_base_type)
 
 	}
 
-BroType* BroType::Clone() const
+BroType* BroType::ShallowClone()
 	{
-	SerializationFormat* form = new BinarySerializationFormat();
+	switch ( tag ) {
-	form->StartWrite();
+		case TYPE_VOID:
-	CloneSerializer ss(form);
+		case TYPE_BOOL:
-	SerialInfo sinfo(&ss);
+		case TYPE_INT:
-	sinfo.cache = false;
+		case TYPE_COUNT:
+		case TYPE_COUNTER:
+		case TYPE_DOUBLE:
+		case TYPE_TIME:
+		case TYPE_INTERVAL:
+		case TYPE_STRING:
+		case TYPE_PATTERN:
+		case TYPE_TIMER:
+		case TYPE_PORT:
+		case TYPE_ADDR:
+		case TYPE_SUBNET:
+		case TYPE_ANY:
+			return new BroType(tag, base_type);
 
-	this->Serialize(&sinfo);
+		default:
-	char* data;
+			reporter->InternalError("cloning illegal base BroType");
-	uint32 len = form->EndWrite(&data);
+	}
-	form->StartRead(data, len);
+	return nullptr;
-
-	UnserialInfo uinfo(&ss);
-	uinfo.cache = false;
-
-	BroType* rval = this->Unserialize(&uinfo, false);
-	assert(rval != this);
-
-	free(data);
-	return rval;
 	}
 
 int BroType::MatchesIndex(ListExpr*& index) const
@@ -203,124 +205,6 @@ unsigned int BroType::MemoryAllocation() const
 	return padded_sizeof(*this);
 	}
 
-bool BroType::Serialize(SerialInfo* info) const
-	{
-	// We always send full types (see below).
-	if ( ! SERIALIZE(true) )
-		return false;
-
-	bool ret = SerialObj::Serialize(info);
-	return ret;
-	}
-
-BroType* BroType::Unserialize(UnserialInfo* info, bool use_existing)
-	{
-	// To avoid external Broccoli clients needing to always send full type
-	// objects, we allow them to give us only the name of a type. To
-	// differentiate between the two cases, we exchange a flag first.
-	bool full_type = true;;
-	if ( ! UNSERIALIZE(&full_type) )
-		return 0;
-
-	if ( ! full_type )
-		{
-		const char* name;
-		if ( ! UNSERIALIZE_STR(&name, 0) )
-			return 0;
-
-		ID* id = global_scope()->Lookup(name);
-		if ( ! id )
-			{
-			info->s->Error(fmt("unknown type %s", name));
-			return 0;
-			}
-
-		BroType* t = id->AsType();
-		if ( ! t )
-			{
-			info->s->Error(fmt("%s is not a type", name));
-			return 0;
-			}
-
-		return t->Ref();
-		}
-
-	BroType* t = (BroType*) SerialObj::Unserialize(info, SER_BRO_TYPE);
-
-	if ( ! t || ! use_existing )
-		return t;
-
-	if ( ! t->name.empty() )
-		{
-		// Avoid creating a new type if it's known by name.
-		// Also avoids loss of base type name alias (from condition below).
-		ID* id = global_scope()->Lookup(t->name.c_str());
-		BroType* t2 = id ? id->AsType() : 0;
-
-		if ( t2 )
-			{
-			Unref(t);
-			return t2->Ref();
-			}
-		}
-
-	if ( t->base_type )
-		{
-		BroType* t2 = ::base_type(TypeTag(t->tag));
-		Unref(t);
-		assert(t2);
-		return t2;
-		}
-
-	assert(t);
-	return t;
-	}
-
-IMPLEMENT_SERIAL(BroType, SER_BRO_TYPE)
-
-bool BroType::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_BRO_TYPE, BroObj);
-
-	info->s->WriteOpenTag("Type");
-
-	if ( ! (SERIALIZE(char(tag)) && SERIALIZE(char(internal_tag))) )
-		return false;
-
-	if ( ! (SERIALIZE(is_network_order) && SERIALIZE(base_type)) )
-		return false;
-
-	SERIALIZE_STR(name.c_str(), name.size());
-
-	info->s->WriteCloseTag("Type");
-
-	return true;
-	}
-
-bool BroType::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(BroObj);
-
-	char c1, c2;
-	if ( ! (UNSERIALIZE(&c1) && UNSERIALIZE(&c2) ) )
-		return 0;
-
-	tag = (TypeTag) c1;
-	internal_tag = (InternalTypeTag) c2;
-
-	if ( ! (UNSERIALIZE(&is_network_order) && UNSERIALIZE(&base_type)) )
-		return 0;
-
-	const char* n;
-	if ( ! UNSERIALIZE_STR(&n, 0) )
-		return false;
-
-	name = n;
-	delete [] n;
-
-	return true;
-	}
-
 TypeList::~TypeList()
 	{
 	loop_over_list(types, i)
@@ -383,47 +267,6 @@ void TypeList::Describe(ODesc* d) const
 		}
 	}
 
-IMPLEMENT_SERIAL(TypeList, SER_TYPE_LIST);
-
-bool TypeList::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_TYPE_LIST, BroType);
-
-	SERIALIZE_OPTIONAL(pure_type);
-
-	if ( ! SERIALIZE(types.length()) )
-		return false;
-
-	loop_over_list(types, j)
-		{
-		if ( ! types[j]->Serialize(info) )
-			return false;
-		}
-
-	return true;
-	}
-
-bool TypeList::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(BroType);
-
-	UNSERIALIZE_OPTIONAL(pure_type, BroType::Unserialize(info));
-
-	int len;
-	if ( ! UNSERIALIZE(&len) )
-		return false;
-
-	while ( len-- )
-		{
-		BroType* t = BroType::Unserialize(info);
-		if ( ! t )
-			return false;
-
-		types.append(t);
-		}
-	return true;
-	}
-
 IndexType::~IndexType()
 	{
 	Unref(indices);
@@ -530,25 +373,6 @@ bool IndexType::IsSubNetIndex() const
 	return false;
 	}
 
-IMPLEMENT_SERIAL(IndexType, SER_INDEX_TYPE);
-
-bool IndexType::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_INDEX_TYPE, BroType);
-
-	SERIALIZE_OPTIONAL(yield_type);
-	return indices->Serialize(info);
-	}
-
-bool IndexType::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(BroType);
-
-	UNSERIALIZE_OPTIONAL(yield_type, BroType::Unserialize(info));
-	indices = (TypeList*) BroType::Unserialize(info);
-	return indices != 0;
-	}
-
 TableType::TableType(TypeList* ind, BroType* yield)
 : IndexType(TYPE_TABLE, ind, yield)
 	{
@@ -577,6 +401,16 @@ TableType::TableType(TypeList* ind, BroType* yield)
 		}
 	}
 
+TableType* TableType::ShallowClone()
+	{
+	if ( indices )
+		indices->Ref();
+	if ( yield_type )
+		yield_type->Ref();
+
+	return new TableType(indices, yield_type);
+	}
+
 bool TableType::IsUnspecifiedTable() const
 	{
 	// Unspecified types have an empty list of indices.
@@ -650,18 +484,17 @@ SetType::SetType(TypeList* ind, ListExpr* arg_elements) : TableType(ind, 0)
 		}
 	}
 
-IMPLEMENT_SERIAL(TableType, SER_TABLE_TYPE);
+SetType* SetType::ShallowClone()
-
-bool TableType::DoSerialize(SerialInfo* info) const
 	{
-	DO_SERIALIZE(SER_TABLE_TYPE, IndexType);
+	// constructor only consumes indices when elements
-	return true;
+	// is set
+	if ( elements && indices )
+		{
+		elements->Ref();
+		indices->Ref();
 		}
 
-bool TableType::DoUnserialize(UnserialInfo* info)
+	return new SetType(indices, elements);
-	{
-	DO_UNSERIALIZE(IndexType);
-	return true;
 	}
 
 SetType::~SetType()
@@ -669,24 +502,6 @@ SetType::~SetType()
 	Unref(elements);
 	}
 
-IMPLEMENT_SERIAL(SetType, SER_SET_TYPE);
-
-bool SetType::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_SET_TYPE, TableType);
-
-	SERIALIZE_OPTIONAL(elements);
-	return true;
-	}
-
-bool SetType::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(TableType);
-
-	UNSERIALIZE_OPTIONAL(elements, (ListExpr*) Expr::Unserialize(info, EXPR_LIST));
-	return true;
-	}
-
 FuncType::FuncType(RecordType* arg_args, BroType* arg_yield, function_flavor arg_flavor)
 : BroType(TYPE_FUNC)
 	{
@@ -716,6 +531,16 @@ FuncType::FuncType(RecordType* arg_args, BroType* arg_yield, function_flavor arg
 		}
 	}
 
+FuncType* FuncType::ShallowClone()
+	{
+	auto f = new FuncType();
+	f->args = args->Ref()->AsRecordType();
+	f->arg_types = arg_types->Ref()->AsTypeList();
+	f->yield = yield->Ref();
+	f->flavor = flavor;
+	return f;
+	}
+
 string FuncType::FlavorString() const
 	{
 	switch ( flavor ) {
@@ -822,80 +647,6 @@ void FuncType::DescribeReST(ODesc* d, bool roles_only) const
 		}
 	}
 
-IMPLEMENT_SERIAL(FuncType, SER_FUNC_TYPE);
-
-bool FuncType::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_FUNC_TYPE, BroType);
-
-	assert(args);
-	assert(arg_types);
-
-	SERIALIZE_OPTIONAL(yield);
-
-	int ser_flavor = 0;
-
-	switch ( flavor ) {
-
-	case FUNC_FLAVOR_FUNCTION:
-		ser_flavor = 0;
-		break;
-
-	case FUNC_FLAVOR_EVENT:
-		ser_flavor = 1;
-		break;
-
-	case FUNC_FLAVOR_HOOK:
-		ser_flavor = 2;
-		break;
-
-	default:
-		reporter->InternalError("Invalid function flavor serialization");
-		break;
-	}
-
-	return args->Serialize(info) &&
-		arg_types->Serialize(info) &&
-		SERIALIZE(ser_flavor);
-	}
-
-bool FuncType::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(BroType);
-
-	UNSERIALIZE_OPTIONAL(yield, BroType::Unserialize(info));
-
-	args = (RecordType*) BroType::Unserialize(info);
-	if ( ! args )
-		return false;
-
-	arg_types = (TypeList*) BroType::Unserialize(info);
-	if ( ! arg_types )
-		return false;
-
-	int ser_flavor = 0;
-
-	if ( ! UNSERIALIZE(&ser_flavor) )
-		return false;
-
-	switch ( ser_flavor ) {
-	case 0:
-		flavor = FUNC_FLAVOR_FUNCTION;
-		break;
-	case 1:
-		flavor = FUNC_FLAVOR_EVENT;
-		break;
-	case 2:
-		flavor = FUNC_FLAVOR_HOOK;
-		break;
-	default:
-		reporter->InternalError("Invalid function flavor unserialization");
-		break;
-	}
-
-	return true;
-	}
-
 TypeDecl::TypeDecl(BroType* t, const char* i, attr_list* arg_attrs, bool in_record)
 	{
 	type = t;
@@ -921,35 +672,6 @@ TypeDecl::~TypeDecl()
 	delete [] id;
 	}
 
-bool TypeDecl::Serialize(SerialInfo* info) const
-	{
-	assert(type);
-	assert(id);
-
-	SERIALIZE_OPTIONAL(attrs);
-
-	if ( ! (type->Serialize(info) && SERIALIZE(id)) )
-		return false;
-
-	return true;
-	}
-
-TypeDecl* TypeDecl::Unserialize(UnserialInfo* info)
-	{
-	TypeDecl* t = new TypeDecl(0, 0, 0);
-
-	UNSERIALIZE_OPTIONAL_STATIC(t->attrs, Attributes::Unserialize(info), t);
-	t->type = BroType::Unserialize(info);
-
-	if ( ! (t->type && UNSERIALIZE_STR(&t->id, 0)) )
-		{
-		delete t;
-		return 0;
-		}
-
-	return t;
-	}
-
 void TypeDecl::DescribeReST(ODesc* d, bool roles_only) const
 	{
 	d->Add(id);
@@ -977,6 +699,16 @@ RecordType::RecordType(type_decl_list* arg_types) : BroType(TYPE_RECORD)
 	num_fields = types ? types->length() : 0;
 	}
 
+// in this case the clone is actually not so shallow, since
+// it gets modified by everyone.
+RecordType* RecordType::ShallowClone()
+	{
+	auto pass = new type_decl_list();
+	loop_over_list(*types, i)
+		pass->append(new TypeDecl(*(*types)[i]));
+	return new RecordType(pass);
+	}
+
 RecordType::~RecordType()
 	{
 	if ( types )
@@ -1253,67 +985,6 @@ void RecordType::DescribeFieldsReST(ODesc* d, bool func_args) const
 		d->PopIndentNoNL();
 	}
 
-IMPLEMENT_SERIAL(RecordType, SER_RECORD_TYPE)
-
-bool RecordType::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_RECORD_TYPE, BroType);
-
-	if ( ! SERIALIZE(num_fields) )
-		return false;
-
-	if ( types )
-		{
-		if ( ! (SERIALIZE(true) && SERIALIZE(types->length())) )
-			return false;
-
-		loop_over_list(*types, i)
-			{
-			if ( ! (*types)[i]->Serialize(info) )
-				return false;
-			}
-		}
-
-	else if ( ! SERIALIZE(false) )
-		return false;
-
-	return true;
-	}
-
-bool RecordType::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(BroType);
-
-	if ( ! UNSERIALIZE(&num_fields) )
-		return false;
-
-	bool has_it;
-	if ( ! UNSERIALIZE(&has_it) )
-		return false;
-
-	if ( has_it )
-		{
-		int len;
-		if ( ! UNSERIALIZE(&len) )
-			return false;
-
-		types = new type_decl_list(len);
-
-		while ( len-- )
-			{
-			TypeDecl* t = TypeDecl::Unserialize(info);
-			if ( ! t )
-				return false;
-
-			types->append(t);
-			}
-		}
-	else
-		types = 0;
-
-	return true;
-	}
-
 SubNetType::SubNetType() : BroType(TYPE_SUBNET)
 	{
 	}
@@ -1326,20 +997,6 @@ void SubNetType::Describe(ODesc* d) const
 		d->Add(int(Tag()));
 	}
 
-IMPLEMENT_SERIAL(SubNetType, SER_SUBNET_TYPE);
-
-bool SubNetType::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_SUBNET_TYPE, BroType);
-	return true;
-	}
-
-bool SubNetType::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(BroType);
-	return true;
-	}
-
 FileType::FileType(BroType* yield_type)
 : BroType(TYPE_FILE)
 	{
@@ -1370,24 +1027,6 @@ void FileType::Describe(ODesc* d) const
 		}
 	}
 
-IMPLEMENT_SERIAL(FileType, SER_FILE_TYPE);
-
-bool FileType::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_FILE_TYPE, BroType);
-
-	assert(yield);
-	return yield->Serialize(info);
-	}
-
-bool FileType::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(BroType);
-
-	yield = BroType::Unserialize(info);
-	return yield != 0;
-	}
-
 OpaqueType::OpaqueType(const string& arg_name) : BroType(TYPE_OPAQUE)
 	{
 	name = arg_name;
@@ -1408,28 +1047,6 @@ void OpaqueType::DescribeReST(ODesc* d, bool roles_only) const
 	d->Add(fmt(":zeek:type:`%s` of %s", type_name(Tag()), name.c_str()));
 	}
 
-IMPLEMENT_SERIAL(OpaqueType, SER_OPAQUE_TYPE);
-
-bool OpaqueType::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_OPAQUE_TYPE, BroType);
-	return SERIALIZE_STR(name.c_str(), name.size());
-	}
-
-bool OpaqueType::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(BroType);
-
-	const char* n;
-	if ( ! UNSERIALIZE_STR(&n, 0) )
-		return false;
-
-	name = n;
-	delete [] n;
-
-	return true;
-	}
-
 EnumType::EnumType(const string& name)
 	: BroType(TYPE_ENUM)
 	{
@@ -1437,18 +1054,26 @@ EnumType::EnumType(const string& name)
 	SetName(name);
 	}
 
-EnumType::EnumType(EnumType* e)
+EnumType::EnumType(const EnumType* e)
 	: BroType(TYPE_ENUM)
 	{
 	counter = e->counter;
 	SetName(e->GetName());
 
-	for ( NameMap::iterator it = e->names.begin(); it != e->names.end(); ++it )
+	for ( auto it = e->names.begin(); it != e->names.end(); ++it )
 		names[it->first] = it->second;
 
 	vals = e->vals;
 	}
 
+EnumType* EnumType::ShallowClone()
+	{
+	if ( counter == 0 )
+		return new EnumType(GetName());
+
+	return new EnumType(this);
+	}
+
 EnumType::~EnumType()
 	{
 	for ( auto& kv : vals )
@@ -1672,64 +1297,16 @@ void EnumType::DescribeReST(ODesc* d, bool roles_only) const
 		}
 	}
 
-IMPLEMENT_SERIAL(EnumType, SER_ENUM_TYPE);
-
-bool EnumType::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_ENUM_TYPE, BroType);
-
-	if ( ! (SERIALIZE(counter) && SERIALIZE((unsigned int) names.size()) &&
-		// Dummy boolean for backwards compatibility.
-		SERIALIZE(false)) )
-		return false;
-
-	for ( NameMap::const_iterator iter = names.begin();
-	      iter != names.end(); ++iter )
-		{
-		if ( ! SERIALIZE(iter->first) || ! SERIALIZE(iter->second) )
-			return false;
-		}
-
-	return true;
-	}
-
-bool EnumType::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(BroType);
-
-	unsigned int len;
-	bool dummy;
-	if ( ! UNSERIALIZE(&counter) ||
-	     ! UNSERIALIZE(&len) ||
-	     // Dummy boolean for backwards compatibility.
-	     ! UNSERIALIZE(&dummy) )
-		return false;
-
-	while ( len-- )
-		{
-		const char* name;
-		bro_int_t val;
-		if ( ! (UNSERIALIZE_STR(&name, 0) && UNSERIALIZE(&val)) )
-			return false;
-
-		names[name] = val;
-		delete [] name; // names[name] converts to std::string
-		// note: the 'vals' map gets populated lazily, which works fine and
-		// also happens to avoid a leak due to circular reference between the
-		// types and vals (there's a special case for unserializing a known
-		// type that will unserialze and then immediately want to unref the
-		// type if we already have it, except that won't delete it as intended
-		// if we've already created circular references to it here).
-		}
-
-	return true;
-	}
-
 VectorType::VectorType(BroType* element_type)
     : BroType(TYPE_VECTOR), yield_type(element_type)
 	{
 	}
 
+VectorType* VectorType::ShallowClone()
+	{
+	return new VectorType(yield_type);
+	}
+
 VectorType::~VectorType()
 	{
 	Unref(yield_type);
@@ -1773,10 +1350,12 @@ int VectorType::MatchesIndex(ListExpr*& index) const
 	{
 	expr_list& el = index->Exprs();
 
-	if ( el.length() != 1 )
+	if ( el.length() != 1 && el.length() != 2)
 		return DOES_NOT_MATCH_INDEX;
 
-	if ( el[0]->Type()->Tag() == TYPE_VECTOR )
+	if ( el.length() == 2 )
+		return MATCHES_INDEX_VECTOR;
+	else if ( el[0]->Type()->Tag() == TYPE_VECTOR )
 		return (IsIntegral(el[0]->Type()->YieldType()->Tag()) ||
 			 IsBool(el[0]->Type()->YieldType()->Tag())) ?
 				MATCHES_INDEX_VECTOR : DOES_NOT_MATCH_INDEX;
@@ -1791,21 +1370,6 @@ bool VectorType::IsUnspecifiedVector() const
 	return yield_type->Tag() == TYPE_VOID;
 	}
 
-IMPLEMENT_SERIAL(VectorType, SER_VECTOR_TYPE);
-
-bool VectorType::DoSerialize(SerialInfo* info) const
-	{
-	DO_SERIALIZE(SER_VECTOR_TYPE, BroType);
-	return yield_type->Serialize(info);
-	}
-
-bool VectorType::DoUnserialize(UnserialInfo* info)
-	{
-	DO_UNSERIALIZE(BroType);
-	yield_type = BroType::Unserialize(info);
-	return yield_type != 0;
-	}
-
 void VectorType::Describe(ODesc* d) const
 	{
 	if ( d->IsReadable() )
@@ -2123,6 +1687,10 @@ int is_assignable(BroType* t)
 	return 0;
 	}
 
+#define CHECK_TYPE(t) \
+	if ( t1 == t || t2 == t ) \
+		return t;
+
 TypeTag max_type(TypeTag t1, TypeTag t2)
 	{
 	if ( t1 == TYPE_INTERVAL || t1 == TYPE_TIME )
@@ -2132,10 +1700,6 @@ TypeTag max_type(TypeTag t1, TypeTag t2)
 
 	if ( BothArithmetic(t1, t2) )
 		{
-#define CHECK_TYPE(t) \
-	if ( t1 == t || t2 == t ) \
-		return t;
-
 		CHECK_TYPE(TYPE_DOUBLE);
 		CHECK_TYPE(TYPE_INT);
 		CHECK_TYPE(TYPE_COUNT);

src/Type.h

@@ -72,7 +72,6 @@ class SubNetType;
 class FuncType;
 class ListExpr;
 class EnumType;
-class Serializer;
 class VectorType;
 class TypeType;
 class OpaqueType;
@@ -87,7 +86,15 @@ public:
 	explicit BroType(TypeTag tag, bool base_type = false);
 	~BroType() override { }
 
-	BroType* Clone() const;
+	// Performs a shallow clone operation of the Bro type.
+	// This especially means that especially for tables the types
+	// are not recursively cloned; altering one type will in this case
+	// alter one of them.
+	// The main use for this is alias tracking.
+	// Clone operations will mostly be implemented in the derived classes;
+	// in addition cloning will be limited to classes that can be reached by
+	// the script-level.
+	virtual BroType* ShallowClone();
 
 	TypeTag Tag() const		{ return tag; }
 	InternalTypeTag InternalType() const	{ return internal_tag; }
@@ -108,7 +115,7 @@ public:
 	// this type is a table[string] of port, then returns the "port"
 	// type.  Returns nil if this is not an index type.
 	virtual BroType* YieldType();
-	const BroType* YieldType() const
+	virtual const BroType* YieldType() const
 		{ return ((BroType*) this)->YieldType(); }
 
 	// Returns true if this type is a record and contains the
@@ -256,9 +263,6 @@ public:
 
 	virtual unsigned MemoryAllocation() const;
 
-	bool Serialize(SerialInfo* info) const;
-	static BroType* Unserialize(UnserialInfo* info, bool use_existing = true);
-
 	void SetName(const string& arg_name) { name = arg_name; }
 	string GetName() const { return name; }
 
@@ -275,8 +279,6 @@ protected:
 
 	void SetError();
 
-	DECLARE_SERIAL(BroType)
-
 private:
 	TypeTag tag;
 	InternalTypeTag internal_tag;
@@ -325,8 +327,6 @@ public:
 		}
 
 protected:
-	DECLARE_SERIAL(TypeList)
-
 	BroType* pure_type;
 	type_list types;
 };
@@ -338,7 +338,7 @@ public:
 	TypeList* Indices() const		{ return indices; }
 	const type_list* IndexTypes() const	{ return indices->Types(); }
 	BroType* YieldType() override;
-	const BroType* YieldType() const;
+	const BroType* YieldType() const override;
 
 	void Describe(ODesc* d) const override;
 	void DescribeReST(ODesc* d, bool roles_only = false) const override;
@@ -356,8 +356,6 @@ protected:
 		}
 	~IndexType() override;
 
-	DECLARE_SERIAL(IndexType)
-
 	TypeList* indices;
 	BroType* yield_type;
 };
@@ -366,6 +364,8 @@ class TableType : public IndexType {
 public:
 	TableType(TypeList* ind, BroType* yield);
 
+	TableType* ShallowClone() override;
+
 	// Returns true if this table type is "unspecified", which is
 	// what one gets using an empty "set()" or "table()" constructor.
 	bool IsUnspecifiedTable() const;
@@ -374,8 +374,6 @@ protected:
 	TableType()	{}
 
 	TypeList* ExpandRecordIndex(RecordType* rt) const;
-
-	DECLARE_SERIAL(TableType)
 };
 
 class SetType : public TableType {
@@ -383,25 +381,26 @@ public:
 	SetType(TypeList* ind, ListExpr* arg_elements);
 	~SetType() override;
 
+	SetType* ShallowClone() override;
+
 	ListExpr* SetElements() const	{ return elements; }
 
 protected:
 	SetType()	{}
 
 	ListExpr* elements;
-
-	DECLARE_SERIAL(SetType)
 };
 
 class FuncType : public BroType {
 public:
 	FuncType(RecordType* args, BroType* yield, function_flavor f);
+	FuncType* ShallowClone() override;
 
 	~FuncType() override;
 
 	RecordType* Args() const	{ return args; }
 	BroType* YieldType() override;
-	const BroType* YieldType() const;
+	const BroType* YieldType() const override;
 	void SetYieldType(BroType* arg_yield)	{ yield = arg_yield; }
 	function_flavor Flavor() const { return flavor; }
 	string FlavorString() const;
@@ -419,9 +418,7 @@ public:
 	void DescribeReST(ODesc* d, bool roles_only = false) const override;
 
 protected:
-	FuncType()	{ args = 0; arg_types = 0; yield = 0; flavor = FUNC_FLAVOR_FUNCTION; }
+	FuncType() : BroType(TYPE_FUNC) { args = 0; arg_types = 0; yield = 0; flavor = FUNC_FLAVOR_FUNCTION; }
-	DECLARE_SERIAL(FuncType)
-
 	RecordType* args;
 	TypeList* arg_types;
 	BroType* yield;
@@ -431,6 +428,7 @@ protected:
 class TypeType : public BroType {
 public:
 	explicit TypeType(BroType* t) : BroType(TYPE_TYPE)	{ type = t->Ref(); }
+	TypeType* ShallowClone() override { return new TypeType(type); }
 	~TypeType() override { Unref(type); }
 
 	BroType* Type()	{ return type; }
@@ -450,9 +448,6 @@ public:
 	const Attr* FindAttr(attr_tag a) const
 		{ return attrs ? attrs->FindAttr(a) : 0; }
 
-	bool Serialize(SerialInfo* info) const;
-	static TypeDecl* Unserialize(UnserialInfo* info);
-
 	virtual void DescribeReST(ODesc* d, bool roles_only = false) const;
 
 	BroType* type;
@@ -466,6 +461,7 @@ typedef PList(TypeDecl) type_decl_list;
 class RecordType : public BroType {
 public:
 	explicit RecordType(type_decl_list* types);
+	RecordType* ShallowClone() override;
 
 	~RecordType() override;
 
@@ -501,8 +497,6 @@ public:
 protected:
 	RecordType() { types = 0; }
 
-	DECLARE_SERIAL(RecordType)
-
 	int num_fields;
 	type_decl_list* types;
 };
@@ -511,13 +505,12 @@ class SubNetType : public BroType {
 public:
 	SubNetType();
 	void Describe(ODesc* d) const override;
-protected:
-	DECLARE_SERIAL(SubNetType)
 };
 
 class FileType : public BroType {
 public:
 	explicit FileType(BroType* yield_type);
+	FileType* ShallowClone() override { return new FileType(yield->Ref()); }
 	~FileType() override;
 
 	BroType* YieldType() override;
@@ -527,14 +520,13 @@ public:
 protected:
 	FileType()	{ yield = 0; }
 
-	DECLARE_SERIAL(FileType)
-
 	BroType* yield;
 };
 
 class OpaqueType : public BroType {
 public:
 	explicit OpaqueType(const string& name);
+	OpaqueType* ShallowClone() override { return new OpaqueType(name); }
 	~OpaqueType() override { };
 
 	const string& Name() const { return name; }
@@ -545,8 +537,6 @@ public:
 protected:
 	OpaqueType() { }
 
-	DECLARE_SERIAL(OpaqueType)
-
 	string name;
 };
 
@@ -554,8 +544,9 @@ class EnumType : public BroType {
 public:
 	typedef std::list<std::pair<string, bro_int_t> > enum_name_list;
 
-	explicit EnumType(EnumType* e);
+	explicit EnumType(const EnumType* e);
 	explicit EnumType(const string& arg_name);
+	EnumType* ShallowClone() override;
 	~EnumType() override;
 
 	// The value of this name is next internal counter value, starting
@@ -582,8 +573,6 @@ public:
 protected:
 	EnumType() { counter = 0; }
 
-	DECLARE_SERIAL(EnumType)
-
 	void AddNameInternal(const string& module_name,
 			const char* name, bro_int_t val, bool is_export);
 
@@ -609,9 +598,10 @@ protected:
 class VectorType : public BroType {
 public:
 	explicit VectorType(BroType* t);
+	VectorType* ShallowClone() override;
 	~VectorType() override;
 	BroType* YieldType() override;
-	const BroType* YieldType() const;
+	const BroType* YieldType() const override;
 
 	int MatchesIndex(ListExpr*& index) const override;
 
@@ -625,8 +615,6 @@ public:
 protected:
 	VectorType()	{ yield_type = 0; }
 
-	DECLARE_SERIAL(VectorType)
-
 	BroType* yield_type;
 };