diff --git a/.gitmodules b/.gitmodules index 24375ce23d..91f39e3d04 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,3 +22,6 @@ [submodule "aux/plugins"] path = aux/plugins url = git://git.bro.org/bro-plugins +[submodule "aux/broker"] + path = aux/broker + url = git://git.bro.org/broker diff --git a/CHANGES b/CHANGES index cad5e26988..6dbaea2555 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,601 @@ +2.4-beta | 2015-05-07 21:55:31 -0700 + + * Release 2.4-beta. + + * Update local-compat.test (Johanna Amann) + +2.3-913 | 2015-05-06 09:58:00 -0700 + + * Add /sbin to PATH in btest.cfg and remove duplicate default_path. + (Daniel Thayer) + +2.3-911 | 2015-05-04 09:58:09 -0700 + + * Update usage output and list of command line options. (Daniel + Thayer) + + * Fix to ssh/geo-data.bro for unset directions. (Vlad Grigorescu) + + * Improve SIP logging and remove reporter messages. (Seth Hall) + +2.3-905 | 2015-04-29 17:01:30 -0700 + + * Improve SIP logging and remove reporter messages. (Seth Hall) + +2.3-903 | 2015-04-27 17:27:59 -0700 + + * BIT-1350: Improve record coercion type checking. (Jon Siwek) + +2.3-901 | 2015-04-27 17:25:27 -0700 + + * BIT-1384: Remove -O (optimize scripts) command-line option, which + hadn't been working for a while already. (Jon Siwek) + +2.3-899 | 2015-04-27 17:22:42 -0700 + + * Fix the -J/--set-seed cmd-line option. (Daniel Thayer) + + * Remove unused -l, -L, and -Z cmd-line options. (Daniel Thayer) + +2.3-892 | 2015-04-27 08:22:22 -0700 + + * Fix typos in the Broker BIF documentation. (Daniel Thayer) + + * Update installation instructions and remove outdated references. + (Johanna Amann) + + * Easier support for systems with tcmalloc_minimal installed. (Seth + Hall) + +2.3-884 | 2015-04-23 12:30:15 -0500 + + * Fix some outdated documentation unit tests. (Jon Siwek) + +2.3-883 | 2015-04-23 07:10:36 -0700 + + * Fix -N option to work with builtin plugins as well. (Robin Sommer) + +2.3-882 | 2015-04-23 06:59:40 -0700 + + * Add missing .pac dependencies for some binpac analyzer targets. + (Jon Siwek) + +2.3-879 | 2015-04-22 10:38:07 -0500 + + * Fix compile errors. (Jon Siwek) + +2.3-878 | 2015-04-22 08:21:23 -0700 + + * Fix another compiler warning in DTLS. (Johanna Amann) + +2.3-877 | 2015-04-21 20:14:16 -0700 + + * Adding missing include. (Robin Sommer) + +2.3-876 | 2015-04-21 16:40:10 -0700 + + * Attempt at fixing a potential std::length_error exception in RDP + analyzer. Addresses BIT-1337. (Robin Sommer) + + * Fixing compile problem caused by overeager factorization. (Robin + Sommer) + +2.3-874 | 2015-04-21 16:09:20 -0700 + + * Change details of escaping when logging/printing. (Seth Hall/Robin + Sommer) + + - Log files now escape non-printable characters consistently + as "\xXX'. Furthermore, backslashes are escaped as "\\", + making the representation fully reversible. + + - When escaping via script-level functions (escape_string, + clean), we likewise now escape consistently with "\xXX" and + "\\". + + - There's no "alternative" output style anymore, i.e., fmt() + '%A' qualifier is gone. + + Addresses BIT-1333. + + * Remove several BroString escaping methods that are no longer + useful. (Seth Hall) + +2.3-864 | 2015-04-21 15:24:02 -0700 + + * A SIP protocol analyzer. (Vlad Grigorescu) + + Activity gets logged into sip.log. It generates the following + events: + + event sip_request(c: connection, method: string, original_URI: string, version: string); + event sip_reply(c: connection, version: string, code: count, reason: string); + event sip_header(c: connection, is_orig: bool, name: string, value: string); + event sip_all_headers(c: connection, is_orig: bool, hlist: mime_header_list); + event sip_begin_entity(c: connection, is_orig: bool); + event sip_end_entity(c: connection, is_orig: bool); + + The analyzer support SIP over UDP currently. + + * BIT-1343: Factor common ASN.1 code from RDP, SNMP, and Kerberos + analyzers. (Jon Siwek/Robin Sommer) + +2.3-838 | 2015-04-21 13:40:12 -0700 + + * BIT-1373: Fix vector index assignment reference count bug. (Jon Siwek) + +2.3-836 | 2015-04-21 13:37:31 -0700 + + * Fix SSH direction field being unset. Addresses BIT-1365. (Vlad + Grigorescu) + +2.3-835 | 2015-04-21 16:36:00 -0500 + + * Clarify Broker examples. (Jon Siwek) + +2.3-833 | 2015-04-21 12:38:32 -0700 + + * A Kerberos protocol analyzer. (Vlad Grigorescu) + + Activity gets logged into kerberos.log. It generates the following + events: + + event krb_as_request(c: connection, msg: KRB::KDC_Request); + event krb_as_response(c: connection, msg: KRB::KDC_Response); + event krb_tgs_request(c: connection, msg: KRB::KDC_Request); + event krb_tgs_response(c: connection, msg: KRB::KDC_Response); + event krb_ap_request(c: connection, ticket: KRB::Ticket, opts: KRB::AP_Options); + event krb_priv(c: connection, is_orig: bool); + event krb_safe(c: connection, is_orig: bool, msg: KRB::SAFE_Msg); + event krb_cred(c: connection, is_orig: bool, tickets: KRB::Ticket_Vector); + event krb_error(c: connection, msg: KRB::Error_Msg); + +2.3-793 | 2015-04-20 20:51:00 -0700 + + * Add decoding of PROXY-AUTHORIZATION header to HTTP analyze, + treating it the same as AUTHORIZATION. (Josh Liburdi) + + * Remove deprecated fields "hot" and "addl" from the connection + record. Remove the functions append_addl() and + append_addl_marker(). (Robin Sommer) + + * Removing the NetFlow analyzer, which hasn't been used anymore + since then corresponding command-line option went away. (Robin + Sommer) + +2.3-787 | 2015-04-20 19:15:23 -0700 + + * A file analyzer for Portable Executables. (Vlad Grigorescu/Seth + Hall). + + Activity gets logged into pe.log. It generates the following + events: + + event pe_dos_header(f: fa_file, h: PE::DOSHeader); + event pe_dos_code(f: fa_file, code: string); + event pe_file_header(f: fa_file, h: PE::FileHeader); + event pe_optional_header(f: fa_file, h: PE::OptionalHeader); + event pe_section_header(f: fa_file, h: PE::SectionHeader); + +2.3-741 | 2015-04-20 13:12:39 -0700 + + * API changes to file analysis mime type detection. Removed + "file_mime_type" and "file_mime_types" event, replacing them with + a new event called "file_metadata_inferred". Addresses BIT-1368. + (Jon Siwek) + + * A large series of improvements for file type identification. This + inludes a many signature updates (new types, cleanup, performance + improvments) and splitting out signatures into subfiles. (Seth + Hall) + + * Fix an issue with files having gaps before the bof_buffer is + filled, which could lead to file type identification not working + correctly. (Seth Hall) + + * Fix an issue with packet loss in HTTP file reporting for file type + identification wasn't working correctly zero-length bodies. (Seth + Hall) + + * X.509 certificates are now populating files.log with the mime type + application/pkix-cert. (Seth Hall) + + * Normalized some FILE_ANALYSIS debug messages. (Seth Hall) + +2.3-725 | 2015-04-20 12:54:54 -0700 + + * Updating submodule(s). + +2.3-724 | 2015-04-20 14:11:02 -0500 + + * Fix uninitialized field in raw input reader. (Jon Siwek) + +2.3-722 | 2015-04-20 12:59:03 -0500 + + * Remove unneeded documentation cross-referencing. (Jon Siwek) + +2.3-721 | 2015-04-20 12:47:05 -0500 + + * BIT-1380: Improve Broxygen output of &default expressions. + (Jon Siwek) + +2.3-720 | 2015-04-17 14:18:26 -0700 + + * Updating NEWS. + +2.3-716 | 2015-04-17 13:06:37 -0700 + + * Add seeking functionality to raw reader. One can now add an option + "offset" to the config map. Positive offsets are interpreted to be + from the beginning of the file, negative from the end of the file + (-1 is end of file). Only works for raw reader in streaming or + manual mode. Does not work with executables. Addresses BIT-985. + (Johanna Amann) + + * Allow setting packet and byte thresholds for connections. (Johanna Amann) + + This extends the ConnSize analyzer to be able to raise events when + each direction of a connection crosses a certain amount of bytes + or packets. + + Thresholds are set using: + - set_conn_bytes_threshold(c$id, [num-bytes], [direction]); + - set_conn_packets_threshold(c$id, [num-packets], [direction]); + + They raise the events, respectively: + - event conn_bytes_threshold_crossed(c: connection, threshold: count, is_orig: bool) + - event conn_packets_threshold_crossed(c: connection, threshold: count, is_orig: bool) + + Current thresholds can be examined using get_conn_bytes_threshold() + and get_conn_packets_threshold(). + + Only one threshold can be set per connection. + + * Add high-level API for packet/bytes thresholding in + base/protocols/conn/thresholds.bro that holds lists of thresholds + and raises an event for each threshold exactly once. (Johanna + Amann) + + * Fix a bug where child packet analyzers of the TCP analyzer + where not found using FindChild. + + * Update GridFTP analyzer to use connection thresholding instead of + polling. (Johanna Amann) + +2.3-709 | 2015-04-17 12:37:32 -0700 + + * Fix addressing the dreaded "internal error: unknown msg type 115 + in Poll()". (Jon Siwek) + + This patch removes the error handling code for overload conditions + in the main process that could cause trouble down the road. The + "chunked_io_buffer_soft_cap" script variable can now tune when the + client process begins shutting down peer connections, and the + default setting is now double what it used to be. Addresses + BIT-1376. + +2.3-707 | 2015-04-17 10:57:59 -0500 + + * Add more info about Broker to NEWS. (Jon Siwek) + +2.3-705 | 2015-04-16 08:16:45 -0700 + + * Update Mozilla CA list. (Johanna Amann) + + * Update tests to have them keep using older certificates where + appropiate. (Johanna Amann) + +2.3-699 | 2015-04-16 09:51:58 -0500 + + * Fix the to_count function to use strtoull versus strtoll. + (Jon Siwek) + +2.3-697 | 2015-04-15 09:51:15 -0700 + + * Removing error check verifying that an ASCII writer has been + properly finished. Instead of aborting, we now just clean up in + that case and proceed. Addresses BIT-1331. (Robin Sommer) + +2.3-696 | 2015-04-14 15:56:36 -0700 + + * Update sqlite to 3.8.9 + +2.3-695 | 2015-04-13 10:34:42 -0500 + + * Fix iterator invalidation in broker::Manager dtor. (Jon Siwek) + + * Add paragraph to plugin documentation. (Robin Sommer) + +2.3-693 | 2015-04-11 10:56:31 -0700 + + * BIT-1367: improve coercion of anonymous records in set constructor. + (Jon Siwek) + + * Allow to specify ports for sftp log rotator. (Johanna Amann) + +2.3-690 | 2015-04-10 21:51:10 -0700 + + * Make sure to always delete the remote serializer. Addresses + BIT-1306 and probably also BIT-1356. (Robin Sommer) + + * Cleaning up --help. -D and -Y/y were still listed, even though + they had no effect anymore. Removing some dead code along with -D. + Addresses BIT-1372. (Robin Sommer) + +2.3-688 | 2015-04-10 08:10:44 -0700 + + * Update SQLite to 3.8.8.3. + +2.3-687 | 2015-04-10 07:32:52 -0700 + + * Remove stale signature benchmarking code (-L command-line option). + (Jon Siwek) + + * BIT-844: fix UDP payload signatures to match packet-wise. (Jon + Siwek) + +2.3-682 | 2015-04-09 12:07:00 -0700 + + * Fixing input readers' component type. (Robin Sommer) + + * Tiny spelling correction. (Seth Hall) + +2.3-680 | 2015-04-06 16:02:43 -0500 + + * BIT-1371: remove CMake version check from binary package scripts. + (Jon Siwek) + +2.3-679 | 2015-04-06 10:16:36 -0500 + + * Increase some unit test timeouts. (Jon Siwek) + + * Fix Coverity warning in RDP analyzer. (Jon Siwek) + +2.3-676 | 2015-04-02 10:10:39 -0500 + + * BIT-1366: improve checksum offloading warning. + (Frank Meier, Jon Siwek) + +2.3-675 | 2015-03-30 17:05:05 -0500 + + * Add an RDP analyzer. (Josh Liburdi, Seth Hall, Johanna Amann) + +2.3-640 | 2015-03-30 13:51:51 -0500 + + * BIT-1359: Limit maximum number of DTLS fragments to 30. (Johanna Amann) + +2.3-637 | 2015-03-30 12:02:07 -0500 + + * Increase timeout duration in some broker tests. (Jon Siwek) + +2.3-636 | 2015-03-30 11:26:32 -0500 + + * Updates related to SSH analysis. (Jon Siwek) + + - Some scripts used wrong SSH module/namespace scoping on events. + - Fix outdated notice documentation related to SSH password guessing. + - Add a unit test for SSH pasword guessing notice. + +2.3-635 | 2015-03-30 11:02:45 -0500 + + * Fix outdated documentation unit tests. (Jon Siwek) + +2.3-634 | 2015-03-30 10:22:45 -0500 + + * Add a canonifier to a unit test's output. (Jon Siwek) + +2.3-633 | 2015-03-25 18:32:59 -0700 + + * Log::write in signature framework was missing timestamp. + (Andrew Benson/Michel Laterman) + +2.3-631 | 2015-03-25 11:03:12 -0700 + + * New SSH analyzer. (Vlad Grigorescu) + +2.3-600 | 2015-03-25 10:23:46 -0700 + + * Add defensive checks in code to calculate log rotation intervals. + (Pete Nelson). + +2.3-597 | 2015-03-23 12:50:04 -0700 + + * DTLS analyzer. (Johanna Amann) + + * Implement correct parsing of TLS record fragmentation. (Johanna + Amann) + +2.3-582 | 2015-03-23 11:34:25 -0700 + + * BIT-1313: In debug builds, "bro -B " now supports "all" and + "help" for "". "all" enables all debug streams. "help" prints a + list of available debug streams. (John Donnelly/Robin Sommer). + + * BIT-1324: Allow logging filters to inherit default path from + stream. This allows the path for the default filter to be + specified explicitly through $path="..." when creating a stream. + Adapted the existing Log::create_stream calls to explicitly + specify a path value. (Jon Siwek) + + * BIT-1199: Change the way the input framework deals with values it + cannot convert into BroVals, raising error messages instead of + aborting execution. (Johanna Amann) + + * BIT-788: Use DNS QR field to better identify flow direction. (Jon + Siwek) + +2.3-572 | 2015-03-23 13:04:53 -0500 + + * BIT-1226: Fix an example in quickstart docs. (Jon siwek) + +2.3-570 | 2015-03-23 09:51:20 -0500 + + * Correct a spelling error (Daniel Thayer) + + * Improvement to SSL analyzer failure mode. (Johanna Amann) + +2.3-565 | 2015-03-20 16:27:41 -0500 + + * BIT-978: Improve documentation of 'for' loop iterator invalidation. + (Jon Siwek) + +2.3-564 | 2015-03-20 11:12:02 -0500 + + * BIT-725: Remove "unmatched_HTTP_reply" weird. (Jon Siwek) + +2.3-562 | 2015-03-20 10:31:02 -0500 + + * BIT-1207: Add unit test to catch breaking changes to local.bro + (Jon Siwek) + + * Fix failing sqlite leak test (Johanna Amann) + +2.3-560 | 2015-03-19 13:17:39 -0500 + + * BIT-1255: Increase default values of + "tcp_max_above_hole_without_any_acks" and "tcp_max_initial_window" + from 4096 to 16384 bytes. (Jon Siwek) + +2.3-559 | 2015-03-19 12:14:33 -0500 + + * BIT-849: turn SMTP reporter warnings into weirds, + "smtp_nested_mail_transaction" and "smtp_unmatched_end_of_data". + (Jon Siwek) + +2.3-558 | 2015-03-18 22:50:55 -0400 + + * DNS: Log the type number for the DNS_RR_unknown_type weird. (Vlad Grigorescu) + +2.3-555 | 2015-03-17 15:57:13 -0700 + + * Splitting test-all Makefile target into Bro tests and test-aux. + (Robin Sommer) + +2.3-554 | 2015-03-17 15:40:39 -0700 + + * Deprecate &rotate_interval, &rotate_size, &encrypt. Addresses + BIT-1305. (Jon Siwek) + +2.3-549 | 2015-03-17 09:12:18 -0700 + + * BIT-1077: Fix HTTP::log_server_header_names. Before, it just + re-logged fields from the client side. (Jon Siwek) + +2.3-547 | 2015-03-17 09:07:51 -0700 + + * Update certificate validation script to cache valid intermediate + chains that it encounters on the wire and use those to try to + validate chains that might be missing intermediate certificates. + (Johanna Amann) + +2.3-541 | 2015-03-13 15:44:08 -0500 + + * Make INSTALL a symlink to doc/install/install.rst (Jon siwek) + + * Fix Broxygen coverage. (Jon Siwek) + +2.3-539 | 2015-03-13 14:19:27 -0500 + + * BIT-1335: Include timestamp in default extracted file names. + And add a policy script to extract all files. (Jon Siwek) + + * BIT-1311: Identify GRE tunnels as Tunnel::GRE, not Tunnel::IP. + (Jon Siwek) + + * BIT-1309: Add Connection class getter methods for flow labels. + (Jon Siwek) + +2.3-536 | 2015-03-12 16:16:24 -0500 + + * Fix Broker leak tests. (Jon Siwek) + +2.3-534 | 2015-03-12 10:59:49 -0500 + + * Update NEWS file. (Jon Siwek) + +2.3-533 | 2015-03-12 10:18:53 -0500 + + * Give broker python bindings default install path within --prefix. + (Jon Siwek) + +2.3-530 | 2015-03-10 13:22:39 -0500 + + * Fix broker data stores in absence of --enable-debug. (Jon Siwek) + +2.3-529 | 2015-03-09 13:14:27 -0500 + + * Fix format specifier in SSL protocol violation. (Jon Siwek) + +2.3-526 | 2015-03-06 12:48:49 -0600 + + * Fix build warnings, clarify broker requirements, update submodule. + (Jon Siwek) + + * Rename comm/ directories to broker/ (Jon Siwek) + + * Rename broker-related namespaces. (Jon Siwek) + + * Improve remote logging via broker by only sending fields w/ &log. + (Jon Siwek) + + * Disable a stream's remote logging via broker if it fails. (Jon Siwek) + + * Improve some broker communication unit tests. (Jon Siwek) + +2.3-518 | 2015-03-04 13:13:50 -0800 + + * Add bytes_recvd to stats.log recording the number of bytes + received, according to packet headers. (Mike Smiley) + +2.3-516 | 2015-03-04 12:30:06 -0800 + + * Extract most specific Common Name from SSL certificates (Johanna + Amann) + + * Send CN and SAN fields of SSL certificates to the Intel framework. + (Johanna Amann) + +2.3-511 | 2015-03-02 18:07:17 -0800 + + * Changes to plugin meta hooks for function calls. (Gilbert Clark) + + - Add frame argument. + + - Change return value to tuple unambigiously whether hook + returned a result. + +2.3-493 | 2015-03-02 17:17:32 -0800 + + * Extend the SSL weak-keys policy file to also alert when + encountering SSL connections with old versions as well as unsafe + cipher suites. (Johanna Amann) + + * Make the notice suppression handling of other SSL policy files a + tad more robust. (Johanna Amann) + +2.3-491 | 2015-03-02 17:12:56 -0800 + + * Updating docs for recent addition of local_resp. (Robin Sommer) + +2.3-489 | 2015-03-02 15:29:30 -0800 + + * Integrate Broker, Bro's new communication library. (Jon Siwek) + + See aux/broker/README for more information on Broker, and + doc/frameworks/comm.rst for the corresponding Bro script API. + + Broker support is by default off for now; it can be enabled at + configure time with --enable-broker. It requires CAF + (https://github.com/actor-framework/actor-framework); for now iot + needs CAF's "develop" branch. Broker also requires a C++11 + compiler. + + Broker will become a mandatory dependency in future Bro versions. + + * Add --enable-c++11 configure flag to compile Bro's source code in + C++11 mode with a corresponding compiler. (Jon Siwek) + 2.3-451 | 2015-02-24 16:37:08 -0800 * Updating submodule(s). @@ -245,7 +842,7 @@ 2.3-328 | 2014-12-02 08:13:10 -0500 - * Update windows-version-detection.bro to add support for + * Update windows-version-detection.bro to add support for Windows 10. (Michal Purzynski) 2.3-326 | 2014-12-01 12:10:27 -0600 @@ -315,7 +912,7 @@ 2.3-280 | 2014-11-05 09:46:33 -0500 - * Add Windows detection based on CryptoAPI HTTP traffic as a + * Add Windows detection based on CryptoAPI HTTP traffic as a software framework policy script. (Vlad Grigorescu) 2.3-278 | 2014-11-03 18:55:18 -0800 diff --git a/CMakeLists.txt b/CMakeLists.txt index 04ac197f74..2b1e060f77 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -113,7 +113,7 @@ if (NOT DISABLE_PERFTOOLS) find_package(GooglePerftools) endif () -if (GOOGLEPERFTOOLS_FOUND) +if (GOOGLEPERFTOOLS_FOUND OR TCMALLOC_FOUND) set(HAVE_PERFTOOLS true) # Non-Linux systems may not be well-supported by gperftools, so # require explicit request from user to enable it in that case. @@ -177,6 +177,17 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR}) ######################################################################## ## Recurse on sub-directories +if ( ENABLE_CXX11 ) + include(RequireCXX11) +endif () + +if ( ENABLE_BROKER ) + add_subdirectory(aux/broker) + set(brodeps ${brodeps} broker) + add_definitions(-DENABLE_BROKER) + include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/aux/broker) +endif () + add_subdirectory(src) add_subdirectory(scripts) add_subdirectory(doc) @@ -224,6 +235,7 @@ message( "\nCXXFLAGS: ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${BuildType}}" "\nCPP: ${CMAKE_CXX_COMPILER}" "\n" + "\nBroker: ${ENABLE_BROKER}" "\nBroccoli: ${INSTALL_BROCCOLI}" "\nBroctl: ${INSTALL_BROCTL}" "\nAux. Tools: ${INSTALL_AUX_TOOLS}" diff --git a/INSTALL b/INSTALL deleted file mode 100644 index 385dac93df..0000000000 --- a/INSTALL +++ /dev/null @@ -1,3 +0,0 @@ - -See doc/install/install.rst for installation instructions. - diff --git a/INSTALL b/INSTALL new file mode 120000 index 0000000000..95fcc60eda --- /dev/null +++ b/INSTALL @@ -0,0 +1 @@ +doc/install/install.rst \ No newline at end of file diff --git a/Makefile b/Makefile index 207ce72780..3efddc4dbc 100644 --- a/Makefile +++ b/Makefile @@ -51,13 +51,15 @@ distclean: $(MAKE) -C testing $@ test: - @( cd testing && make ) + -@( cd testing && make ) -test-all: test - test -d aux/broctl && ( cd aux/broctl && make test-all ) - test -d aux/btest && ( cd aux/btest && make test ) - test -d aux/bro-aux && ( cd aux/bro-aux && make test ) - test -d aux/plugins && ( cd aux/plugins && make test-all ) +test-aux: + -test -d aux/broctl && ( cd aux/broctl && make test-all ) + -test -d aux/btest && ( cd aux/btest && make test ) + -test -d aux/bro-aux && ( cd aux/bro-aux && make test ) + -test -d aux/plugins && ( cd aux/plugins && make test-all ) + +test-all: test test-aux configured: @test -d $(BUILD) || ( echo "Error: No build/ directory found. Did you run configure?" && exit 1 ) diff --git a/NEWS b/NEWS index 8af6c38201..54deaf1e25 100644 --- a/NEWS +++ b/NEWS @@ -7,29 +7,57 @@ their own ``CHANGES``.) Bro 2.4 (in progress) ===================== -Dependencies ------------- - New Functionality ----------------- - Bro now has support for external plugins that can extend its core functionality, like protocol/file analysis, via shared libraries. Plugins can be developed and distributed externally, and will be - pulled in dynamically at startup. Currently, a plugin can provide - custom protocol analyzers, file analyzers, log writers[TODO], input - readers[TODO], packet sources[TODO], and new built-in functions. A - plugin can furthermore hook into Bro's processing a number of places - to add custom logic. + pulled in dynamically at startup (the environment variables + BRO_PLUGIN_PATH and BRO_PLUGIN_ACTIVATE can be used to specify the + locations and names of plugins to activate). Currently, a plugin + can provide custom protocol analyzers, file analyzers, log writers, + input readers, packet sources and dumpers, and new built-in functions. + A plugin can furthermore hook into Bro's processing at a number of + places to add custom logic. See https://www.bro.org/sphinx-git/devel/plugins.html for more information on writing plugins. -- Bro now has supoprt for the MySQL wire protocol. Activity gets +- Bro now has support for the MySQL wire protocol. Activity gets logged into mysql.log. +- Bro now parses DTLS traffic. Activity gets logged into ssl.log. + +- Bro now has support for the Kerberos KRB5 protocol over TCP and + UDP. Activity gets logged into kerberos.log. + +- Bro now has an RDP analyzer. Activity gets logged into rdp.log. + +- Bro now has a file analyzer for Portable Executables. Activity gets + logged into pe.log. + +- Bro now has support for the SIP protocol over UDP. Activity gets + logged into sip.log. + +- Bro now features a completely rewritten, enhanced SSH analyzer. The + new analyzer is able to determine if logins failed or succeeded in + most circumstances, logs a lot more more information about SSH + sessions, supports v1, and introduces the intelligence type + ``Intel::PUBKEY_HASH`` and location ``SSH::IN_SERVER_HOST_KEY``. The + analayzer also generates a set of additional events + (``ssh_auth_successful``, ``ssh_auth_failed``, ``ssh_capabilities``, + ``ssh2_server_host_key``, ``ssh1_server_host_key``, + ``ssh_encrypted_packet``, ``ssh2_dh_server_params``, + ``ssh2_gss_error``, ``ssh2_ecc_key``). See next section for + incompatible SSH changes. + - Bro's file analysis now supports reassembly of files that are not - transferred/seen sequentially. + transferred/seen sequentially. The default file reassembly buffer + size is set with the ``Files::reassembly_buffer_size`` variable. + +- Bro's file type identification has been greatly improved (new file types, + bug fixes, and performance improvements). - Bro's scripting language now has a ``while`` statement:: @@ -39,6 +67,70 @@ New Functionality ``next`` and ``break`` can be used inside the loop's body just like with ``for`` loops. +- Bro now integrates Broker, a new communication library. See + aux/broker/README for more information on Broker, and + doc/frameworks/broker.rst for the corresponding Bro script API. + + With Broker, Bro has the similar capabilities of exchanging events and + logs with remote peers (either another Bro process or some other + application that uses Broker). It also includes a key-value store + API that can be used to share state between peers and optionally + allow data to persist on disk for longer-term storage. + + Broker support is by default off for now; it can be enabled at + configure time with --enable-broker. It requires CAF version 0.13+ + (https://github.com/actor-framework/actor-framework) as well as a + C++11 compiler (e.g. GCC 4.8+ or Clang 3.3+). + + Broker will become a mandatory dependency in future Bro versions and + replace the current communication and serialization system. + +- Add --enable-c++11 configure flag to compile Bro's source code in + C++11 mode with a corresponding compiler. Note that 2.4 will be the + last version of Bro that compiles without C++11 support. + +- The SSL analysis now alerts when encountering SSL connections with + old protocol versions or unsafe cipher suites. It also gained + extended reporting of weak keys, caching of already validated + certificates, and full support for TLS record defragmentation. SSL generally + became much more robust and added several fields to ssl.log (while + removing some others). + +- A new icmp_sent_payload event provides access to ICMP payload. + +- The input framework's raw reader now supports seeking by adding an + option "offset" to the config map. Positive offsets are interpreted + to be from the beginning of the file, negative from the end of the + file (-1 is end of file). + +- One can now raise events when a connection crosses a given size + threshold in terms of packets or bytes. The primary API for that + functionality is in base/protocols/conn/thresholds.bro. + +- There is a new command-line option -Q/--time that prints Bro's execution + time and memory usage to stderr. + +- BroControl now has a new command "deploy" which is equivalent to running + the "check", "install", "stop", and "start" commands (in that order). + +- BroControl now has a new option "StatusCmdShowAll" that controls whether + or not the broctl "status" command gathers all of the status information. + This option can be used to make the "status" command run significantly + faster (in this case, the "Peers" column will not be shown in the output). + +- BroControl now has a new option "StatsLogEnable" that controls whether + or not broctl will record information to the "stats.log" file. This option + can be used to make the "broctl cron" command run slightly faster (in this + case, "broctl cron" will also no longer send email about not seeing any + packets on the monitoring interfaces). + +- BroControl now has a new option "MailHostUpDown" which controls whether or + not the "broctl cron" command will send email when it notices that a host + in the cluster is up or down. + +- BroControl now has a new option "CommandTimeout" which specifies the number + of seconds to wait for a command that broctl ran to return results. + Changed Functionality --------------------- @@ -47,9 +139,17 @@ Changed Functionality - File analysis * Removed ``fa_file`` record's ``mime_type`` and ``mime_types`` - fields. The events ``file_mime_type`` and ``file_mime_types`` - have been added which contain the same information. The - ``mime_type`` field of ``Files::Info`` also still has this info. + fields. The event ``file_sniff`` has been added which provides + the same information. The ``mime_type`` field of ``Files::Info`` + also still has this info. + + * The earliest point that new mime type information is available is + in the ``file_sniff`` event which comes after the ``file_new`` and + ``file_over_new_connection`` events. Scripts which inspected mime + type info within those events will need to be adapted. (Note: for + users that worked w/ versions of Bro from git, for a while there was + also an event called ``file_mime_type`` which is now replaced with + the ``file_sniff`` event). * Removed ``Files::add_analyzers_for_mime_type`` function. @@ -58,15 +158,83 @@ Changed Functionality reassembly for non-sequential files, "offset" can be obtained with other information already available -- adding together ``seen_bytes`` and ``missed_bytes`` fields of the ``fa_file`` - record gives the how many bytes have been written so far (i.e. + record gives how many bytes have been written so far (i.e. the "offset"). -- has_valid_octets: now uses a string_vec parameter instead of +- The SSH changes come with a few incompatibilities. The following + events have been renamed: + + * ``SSH::heuristic_failed_login`` to ``SSH::ssh_auth_failed`` + * ``SSH::heuristic_successful_login`` to ``SSH::ssh_auth_successful`` + + The ``SSH::Info`` status field has been removed and replaced with + the ``auth_success`` field. This field has been changed from a + string that was previously ``success``, ``failure`` or + ``undetermined`` to a boolean. a boolean that is ``T``, ``F``, or + unset. + +- The has_valid_octets function now uses a string_vec parameter instead of string_array. - conn.log gained a new field local_resp that works like local_orig, just for the responder address of the connection. +- GRE tunnels are now identified as ``Tunnel::GRE`` instead of + ``Tunnel::IP``. + +- The default name for extracted files changed from extract-protocol-id + to extract-timestamp-protocol-id. + +- The weird named "unmatched_HTTP_reply" has been removed since it can + be detected at the script-layer and is handled correctly by the + default HTTP scripts. + +- When adding a logging filter to a stream, the filter can now inherit + a default ``path`` field from the associated ``Log::Stream`` record. + +- When adding a logging filter to a stream, the + ``Log::default_path_func`` is now only automatically added to the + filter if it has neither a ``path`` nor a ``path_func`` already + explicitly set. Before, the default path function would always be set + for all filters which didn't specify their own ``path_func``. + +- BroControl now establishes only one ssh connection from the manager to + each remote host in a cluster configuration (previously, there would be + one ssh connection per remote Bro process). + +- BroControl now uses SQLite to record state information instead of a + plain text file (the file "spool/broctl.dat" is no longer used). + On FreeBSD, this means that there is a new dependency on the package + "py27-sqlite3". + +- BroControl now records the expected running state of each Bro node right + before each start or stop. The "broctl cron" command uses this info to + either start or stop Bro nodes as needed so that the actual state matches + the expected state (previously, "broctl cron" could only start nodes in + the "crashed" state, and could never stop a node). + +- BroControl now sends all normal command output (i.e., not error messages) + to stdout. Error messages are still sent to stderr, however. + +- The capability of processing NetFlow input has been removed for the + time being. Therefore, the -y/--flowfile and -Y/--netflow command-line + options have been removed, and the netflow_v5_header and netflow_v5_record + events have been removed. + +- The -D/--dfa-size command-line option has been removed. + +- The -L/--rule-benchmark command-line option has been removed. + +- The -O/--optimize command-line option has been removed. + +- The deprecated fields "hot" and "addl" have been removed from the + connection record. Likewise, the functions append_addl() and + append_addl_marker() have been removed. + +- Log files now escape non-printable characters consistently as "\xXX'. + Furthermore, backslashes are escaped as "\\", making the + representation fully reversible. + Deprecated Functionality ------------------------ @@ -76,7 +244,7 @@ Deprecated Functionality concatenation/extraction functions. Note that the new functions use 0-based indexing, rather than 1-based. - The full list of now deprecation functions is: + The full list of now deprecated functions is: * split: use split_string instead. diff --git a/VERSION b/VERSION index a8a700226f..0a8901319c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-451 +2.4-beta diff --git a/aux/binpac b/aux/binpac index 33cb1f8e6b..4f33233aef 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 33cb1f8e6bf2e33c2773e86b157e1f343ee85dc6 +Subproject commit 4f33233aef5539ae4f12c6d0e4338247833c3900 diff --git a/aux/bro-aux b/aux/bro-aux index c9d340847c..a2d290a832 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit c9d340847c668590a450f1881e6e3d763abe1138 +Subproject commit a2d290a832c35ad11f3fabb19812bcae2ff089cd diff --git a/aux/broccoli b/aux/broccoli index 1d55a0a84c..74bb4bbd94 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 1d55a0a84c5b1d0aa1727829300b388c92f92daa +Subproject commit 74bb4bbd949e61e099178f8a97499d3f1355de8b diff --git a/aux/broctl b/aux/broctl index 76f99ea52c..97c17d2172 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 76f99ea52c3e021cade3d03eda7865d4f4d1793e +Subproject commit 97c17d21725e42b36f4b49579077ecdc28ddb86a diff --git a/aux/broker b/aux/broker new file mode 160000 index 0000000000..8fc6938017 --- /dev/null +++ b/aux/broker @@ -0,0 +1 @@ +Subproject commit 8fc6938017dc15acfb26fa29e6ad0933019781c5 diff --git a/aux/btest b/aux/btest index 93d4989ed1..80b42ee3e4 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit 93d4989ed1537e4d143cf09d44077159f869a4b2 +Subproject commit 80b42ee3e4503783b6720855b28e83ff1658c22b diff --git a/aux/plugins b/aux/plugins index 71d820e9d8..e1ea9f67cf 160000 --- a/aux/plugins +++ b/aux/plugins @@ -1 +1 @@ -Subproject commit 71d820e9d8ca753fea8fb34ea3987993b28d79e4 +Subproject commit e1ea9f67cfe3d6a81e0c1479ced0b9aa73e77c87 diff --git a/cmake b/cmake index ff08be5aa1..6406fb79d3 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit ff08be5aa1b8eaadbe2775cbc11b499c5f93349e +Subproject commit 6406fb79d30df8d7956110ce65a97d18e4bc8c3b diff --git a/configure b/configure index 2b1c568b26..b139ee2bec 100755 --- a/configure +++ b/configure @@ -41,6 +41,9 @@ Usage: $0 [OPTION]... [VAR=VALUE]... --enable-perftools-debug use Google's perftools for debugging --enable-jemalloc link against jemalloc --enable-ruby build ruby bindings for broccoli (deprecated) + --enable-c++11 build using the C++11 standard + --enable-broker enable use of the Broker communication library + (requires C++ Actor Framework and C++11) --disable-broccoli don't build or install the Broccoli library --disable-broctl don't install Broctl --disable-auxtools don't build or install auxiliary tools @@ -55,6 +58,8 @@ Usage: $0 [OPTION]... [VAR=VALUE]... --with-flex=PATH path to flex executable --with-bison=PATH path to bison executable --with-perl=PATH path to perl executable + --with-libcaf=PATH path to C++ Actor Framework installation + (a required Broker dependency) Optional Packages in Non-Standard Locations: --with-geoip=PATH path to the libGeoIP install root @@ -67,6 +72,8 @@ Usage: $0 [OPTION]... [VAR=VALUE]... --with-ruby-lib=PATH path to ruby library --with-ruby-inc=PATH path to ruby headers --with-swig=PATH path to SWIG executable + --with-rocksdb=PATH path to RocksDB installation + (an optional Broker dependency) Packaging Options (for developers): --binary-package toggle special logic for binary packaging @@ -142,6 +149,10 @@ while [ $# -ne 0 ]; do append_cache_entry CMAKE_INSTALL_PREFIX PATH $optarg append_cache_entry BRO_ROOT_DIR PATH $optarg append_cache_entry PY_MOD_INSTALL_DIR PATH $optarg/lib/broctl + + if [ -n "$user_enabled_broker" ]; then + append_cache_entry BROKER_PYTHON_HOME PATH $prefix + fi ;; --scriptdir=*) append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $optarg @@ -176,6 +187,15 @@ while [ $# -ne 0 ]; do --enable-jemalloc) append_cache_entry ENABLE_JEMALLOC BOOL true ;; + --enable-c++11) + append_cache_entry ENABLE_CXX11 BOOL true + ;; + --enable-broker) + append_cache_entry ENABLE_CXX11 BOOL true + append_cache_entry ENABLE_BROKER BOOL true + append_cache_entry BROKER_PYTHON_HOME PATH $prefix + user_enabled_broker="true" + ;; --disable-broccoli) append_cache_entry INSTALL_BROCCOLI BOOL false ;; @@ -248,6 +268,12 @@ while [ $# -ne 0 ]; do --with-swig=*) append_cache_entry SWIG_EXECUTABLE PATH $optarg ;; + --with-libcaf=*) + append_cache_entry LIBCAF_ROOT_DIR PATH $optarg + ;; + --with-rocksdb=*) + append_cache_entry ROCKSDB_ROOT_DIR PATH $optarg + ;; --binary-package) append_cache_entry BINARY_PACKAGING_MODE BOOL true ;; diff --git a/doc/components/broker/README.rst b/doc/components/broker/README.rst new file mode 120000 index 0000000000..eafa3b8e77 --- /dev/null +++ b/doc/components/broker/README.rst @@ -0,0 +1 @@ +../../../aux/broker/README \ No newline at end of file diff --git a/doc/components/broker/broker-manual.rst b/doc/components/broker/broker-manual.rst new file mode 120000 index 0000000000..90bf8f0833 --- /dev/null +++ b/doc/components/broker/broker-manual.rst @@ -0,0 +1 @@ +../../../aux/broker/broker-manual.rst \ No newline at end of file diff --git a/doc/components/index.rst b/doc/components/index.rst index fe05f13683..c1feda4a61 100644 --- a/doc/components/index.rst +++ b/doc/components/index.rst @@ -17,6 +17,8 @@ current, independent component releases. Broccoli - User Manual Broccoli Python Bindings Broccoli Ruby Bindings + Broker - Bro's (New) Messaging Library (README) + Broker - User Manual BroControl - Interactive Bro management shell Bro-Aux - Small auxiliary tools for Bro BTest - A unit testing framework diff --git a/doc/devel/plugins.rst b/doc/devel/plugins.rst index 5e488cfe01..5c963a1552 100644 --- a/doc/devel/plugins.rst +++ b/doc/devel/plugins.rst @@ -245,13 +245,26 @@ VERSION). You can find a full list of files installed in copies over the binary tarball in ``build/dist``. ``init-plugin`` will never overwrite existing files. If its target -directory already exists, it will be default decline to do anything. +directory already exists, it will by default decline to do anything. You can run it with ``-u`` instead to update an existing plugin, however it will never overwrite any existing files; it will only put in place files it doesn't find yet. To revert a file back to what ``init-plugin`` created originally, delete it first and then rerun with ``-u``. +``init-plugin`` puts a ``configure`` script in place that wraps +``cmake`` with a more familiar configure-style configuration. By +default, the script provides two options for specifying paths to the +Bro source (``--bro-dist``) and to the plugin's installation directory +(``--install-root``). To extend ``configure`` with plugin-specific +options (such as search paths for its dependencies) don't edit the +script directly but instead extend ``configure.plugin``, which +``configure`` includes. That way you will be able to more easily +update ``configure`` in the future when the distribution version +changes. In ``configure.plugin`` you can use the predefined shell +function ``append_cache_entry`` to seed values into the CMake cache; +see the installed skeleton version and existing plugins for examples. + Activating a Plugin =================== diff --git a/doc/frameworks/broker.rst b/doc/frameworks/broker.rst new file mode 100644 index 0000000000..3cd8dab6e3 --- /dev/null +++ b/doc/frameworks/broker.rst @@ -0,0 +1,202 @@ + +.. _brokercomm-framework: + +====================================== +Broker-Enabled Communication Framework +====================================== + +.. rst-class:: opening + + Bro can now use the `Broker Library + <../components/broker/README.html>`_ to exchange information with + other Bro processes. To enable it run Bro's ``configure`` script + with the ``--enable-broker`` option. Note that a C++11 compatible + compiler (e.g. GCC 4.8+ or Clang 3.3+) is required as well as the + `C++ Actor Framework `_. + +.. contents:: + +Connecting to Peers +=================== + +Communication via Broker must first be turned on via +:bro:see:`BrokerComm::enable`. + +Bro can accept incoming connections by calling :bro:see:`BrokerComm::listen` +and then monitor connection status updates via +:bro:see:`BrokerComm::incoming_connection_established` and +:bro:see:`BrokerComm::incoming_connection_broken`. + +.. btest-include:: ${DOC_ROOT}/frameworks/broker/connecting-listener.bro + +Bro can initiate outgoing connections by calling :bro:see:`BrokerComm::connect` +and then monitor connection status updates via +:bro:see:`BrokerComm::outgoing_connection_established`, +:bro:see:`BrokerComm::outgoing_connection_broken`, and +:bro:see:`BrokerComm::outgoing_connection_incompatible`. + +.. btest-include:: ${DOC_ROOT}/frameworks/broker/connecting-connector.bro + +Remote Printing +=============== + +To receive remote print messages, first use +:bro:see:`BrokerComm::subscribe_to_prints` to advertise to peers a topic +prefix of interest and then create an event handler for +:bro:see:`BrokerComm::print_handler` to handle any print messages that are +received. + +.. btest-include:: ${DOC_ROOT}/frameworks/broker/printing-listener.bro + +To send remote print messages, just call :bro:see:`BrokerComm::print`. + +.. btest-include:: ${DOC_ROOT}/frameworks/broker/printing-connector.bro + +Notice that the subscriber only used the prefix "bro/print/", but is +able to receive messages with full topics of "bro/print/hi", +"bro/print/stuff", and "bro/print/bye". The model here is that the +publisher of a message checks for all subscribers who advertised +interest in a prefix of that message's topic and sends it to them. + +Message Format +-------------- + +For other applications that want to exchange print messages with Bro, +the Broker message format is simply: + +.. code:: c++ + + broker::message{std::string{}}; + +Remote Events +============= + +Receiving remote events is similar to remote prints. Just use +:bro:see:`BrokerComm::subscribe_to_events` and possibly define any new events +along with handlers that peers may want to send. + +.. btest-include:: ${DOC_ROOT}/frameworks/broker/events-listener.bro + +To send events, there are two choices. The first is to use call +:bro:see:`BrokerComm::event` directly. The second option is to use +:bro:see:`BrokerComm::auto_event` to make it so a particular event is +automatically sent to peers whenever it is called locally via the normal +event invocation syntax. + +.. btest-include:: ${DOC_ROOT}/frameworks/broker/events-connector.bro + +Again, the subscription model is prefix-based. + +Message Format +-------------- + +For other applications that want to exchange event messages with Bro, +the Broker message format is: + +.. code:: c++ + + broker::message{std::string{}, ...}; + +The first parameter is the name of the event and the remaining ``...`` +are its arguments, which are any of the support Broker data types as +they correspond to the Bro types for the event named in the first +parameter of the message. + +Remote Logging +============== + +.. btest-include:: ${DOC_ROOT}/frameworks/broker/testlog.bro + +Use :bro:see:`BrokerComm::subscribe_to_logs` to advertise interest in logs +written by peers. The topic names that Bro uses are implicitly of the +form "bro/log/". + +.. btest-include:: ${DOC_ROOT}/frameworks/broker/logs-listener.bro + +To send remote logs either use :bro:see:`Log::enable_remote_logging` or +:bro:see:`BrokerComm::enable_remote_logs`. The former allows any log stream +to be sent to peers while the later toggles remote logging for +particular streams. + +.. btest-include:: ${DOC_ROOT}/frameworks/broker/logs-connector.bro + +Message Format +-------------- + +For other applications that want to exchange logs messages with Bro, +the Broker message format is: + +.. code:: c++ + + broker::message{broker::enum_value{}, broker::record{}}; + +The enum value corresponds to the stream's :bro:see:`Log::ID` value, and +the record corresponds to a single entry of that log's columns record, +in this case a ``Test::INFO`` value. + +Tuning Access Control +===================== + +By default, endpoints do not restrict the message topics that it sends +to peers and do not restrict what message topics and data store +identifiers get advertised to peers. These are the default +:bro:see:`BrokerComm::EndpointFlags` supplied to :bro:see:`BrokerComm::enable`. + +If not using the ``auto_publish`` flag, one can use the +:bro:see:`BrokerComm::publish_topic` and :bro:see:`BrokerComm::unpublish_topic` +functions to manipulate the set of message topics (must match exactly) +that are allowed to be sent to peer endpoints. These settings take +precedence over the per-message ``peers`` flag supplied to functions +that take a :bro:see:`BrokerComm::SendFlags` such as :bro:see:`BrokerComm::print`, +:bro:see:`BrokerComm::event`, :bro:see:`BrokerComm::auto_event` or +:bro:see:`BrokerComm::enable_remote_logs`. + +If not using the ``auto_advertise`` flag, one can use the +:bro:see:`BrokerComm::advertise_topic` and :bro:see:`BrokerComm::unadvertise_topic` +to manupulate the set of topic prefixes that are allowed to be +advertised to peers. If an endpoint does not advertise a topic prefix, +the only way a peers can send messages to it is via the ``unsolicited`` +flag of :bro:see:`BrokerComm::SendFlags` and choosing a topic with a matching +prefix (i.e. full topic may be longer than receivers prefix, just the +prefix needs to match). + +Distributed Data Stores +======================= + +There are three flavors of key-value data store interfaces: master, +clone, and frontend. + +A frontend is the common interface to query and modify data stores. +That is, a clone is a specific type of frontend and a master is also a +specific type of frontend, but a standalone frontend can also exist to +e.g. query and modify the contents of a remote master store without +actually "owning" any of the contents itself. + +A master data store can be be cloned from remote peers which may then +perform lightweight, local queries against the clone, which +automatically stays synchronized with the master store. Clones cannot +modify their content directly, instead they send modifications to the +centralized master store which applies them and then broadcasts them to +all clones. + +Master and clone stores get to choose what type of storage backend to +use. E.g. In-memory versus SQLite for persistence. Note that if clones +are used, data store sizes should still be able to fit within memory +regardless of the storage backend as a single snapshot of the master +store is sent in a single chunk to initialize the clone. + +Data stores also support expiration on a per-key basis either using an +absolute point in time or a relative amount of time since the entry's +last modification time. + +.. btest-include:: ${DOC_ROOT}/frameworks/broker/stores-listener.bro + +.. btest-include:: ${DOC_ROOT}/frameworks/broker/stores-connector.bro + +In the above example, if a local copy of the store contents isn't +needed, just replace the :bro:see:`BrokerStore::create_clone` call with +:bro:see:`BrokerStore::create_frontend`. Queries will then be made against +the remote master store instead of the local clone. + +Note that all queries are made within Bro's asynchrounous ``when`` +statements and must specify a timeout block. diff --git a/doc/frameworks/broker/connecting-connector.bro b/doc/frameworks/broker/connecting-connector.bro new file mode 100644 index 0000000000..a7e621e4a6 --- /dev/null +++ b/doc/frameworks/broker/connecting-connector.bro @@ -0,0 +1,19 @@ + +const broker_port: port = 9999/tcp &redef; +redef exit_only_after_terminate = T; +redef BrokerComm::endpoint_name = "connector"; + +event bro_init() + { + BrokerComm::enable(); + BrokerComm::connect("127.0.0.1", broker_port, 1sec); + } + +event BrokerComm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + print "BrokerComm::outgoing_connection_established", + peer_address, peer_port, peer_name; + terminate(); + } diff --git a/doc/frameworks/broker/connecting-listener.bro b/doc/frameworks/broker/connecting-listener.bro new file mode 100644 index 0000000000..c37af3ae4d --- /dev/null +++ b/doc/frameworks/broker/connecting-listener.bro @@ -0,0 +1,21 @@ + +const broker_port: port = 9999/tcp &redef; +redef exit_only_after_terminate = T; +redef BrokerComm::endpoint_name = "listener"; + +event bro_init() + { + BrokerComm::enable(); + BrokerComm::listen(broker_port, "127.0.0.1"); + } + +event BrokerComm::incoming_connection_established(peer_name: string) + { + print "BrokerComm::incoming_connection_established", peer_name; + } + +event BrokerComm::incoming_connection_broken(peer_name: string) + { + print "BrokerComm::incoming_connection_broken", peer_name; + terminate(); + } diff --git a/doc/frameworks/broker/events-connector.bro b/doc/frameworks/broker/events-connector.bro new file mode 100644 index 0000000000..1ad458c245 --- /dev/null +++ b/doc/frameworks/broker/events-connector.bro @@ -0,0 +1,31 @@ +const broker_port: port = 9999/tcp &redef; +redef exit_only_after_terminate = T; +redef BrokerComm::endpoint_name = "connector"; +global my_event: event(msg: string, c: count); +global my_auto_event: event(msg: string, c: count); + +event bro_init() + { + BrokerComm::enable(); + BrokerComm::connect("127.0.0.1", broker_port, 1sec); + BrokerComm::auto_event("bro/event/my_auto_event", my_auto_event); + } + +event BrokerComm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + print "BrokerComm::outgoing_connection_established", + peer_address, peer_port, peer_name; + BrokerComm::event("bro/event/my_event", BrokerComm::event_args(my_event, "hi", 0)); + event my_auto_event("stuff", 88); + BrokerComm::event("bro/event/my_event", BrokerComm::event_args(my_event, "...", 1)); + event my_auto_event("more stuff", 51); + BrokerComm::event("bro/event/my_event", BrokerComm::event_args(my_event, "bye", 2)); + } + +event BrokerComm::outgoing_connection_broken(peer_address: string, + peer_port: port) + { + terminate(); + } diff --git a/doc/frameworks/broker/events-listener.bro b/doc/frameworks/broker/events-listener.bro new file mode 100644 index 0000000000..aa6ea9ee4e --- /dev/null +++ b/doc/frameworks/broker/events-listener.bro @@ -0,0 +1,37 @@ + +const broker_port: port = 9999/tcp &redef; +redef exit_only_after_terminate = T; +redef BrokerComm::endpoint_name = "listener"; +global msg_count = 0; +global my_event: event(msg: string, c: count); +global my_auto_event: event(msg: string, c: count); + +event bro_init() + { + BrokerComm::enable(); + BrokerComm::subscribe_to_events("bro/event/"); + BrokerComm::listen(broker_port, "127.0.0.1"); + } + +event BrokerComm::incoming_connection_established(peer_name: string) + { + print "BrokerComm::incoming_connection_established", peer_name; + } + +event my_event(msg: string, c: count) + { + ++msg_count; + print "got my_event", msg, c; + + if ( msg_count == 5 ) + terminate(); + } + +event my_auto_event(msg: string, c: count) + { + ++msg_count; + print "got my_auto_event", msg, c; + + if ( msg_count == 5 ) + terminate(); + } diff --git a/doc/frameworks/broker/logs-connector.bro b/doc/frameworks/broker/logs-connector.bro new file mode 100644 index 0000000000..6089419cab --- /dev/null +++ b/doc/frameworks/broker/logs-connector.bro @@ -0,0 +1,40 @@ +@load ./testlog + +const broker_port: port = 9999/tcp &redef; +redef exit_only_after_terminate = T; +redef BrokerComm::endpoint_name = "connector"; +redef Log::enable_local_logging = F; +redef Log::enable_remote_logging = F; +global n = 0; + +event bro_init() + { + BrokerComm::enable(); + BrokerComm::enable_remote_logs(Test::LOG); + BrokerComm::connect("127.0.0.1", broker_port, 1sec); + } + +event do_write() + { + if ( n == 6 ) + return; + + Log::write(Test::LOG, [$msg = "ping", $num = n]); + ++n; + event do_write(); + } + +event BrokerComm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + print "BrokerComm::outgoing_connection_established", + peer_address, peer_port, peer_name; + event do_write(); + } + +event BrokerComm::outgoing_connection_broken(peer_address: string, + peer_port: port) + { + terminate(); + } diff --git a/doc/frameworks/broker/logs-listener.bro b/doc/frameworks/broker/logs-listener.bro new file mode 100644 index 0000000000..5c807f08b7 --- /dev/null +++ b/doc/frameworks/broker/logs-listener.bro @@ -0,0 +1,25 @@ +@load ./testlog + +const broker_port: port = 9999/tcp &redef; +redef exit_only_after_terminate = T; +redef BrokerComm::endpoint_name = "listener"; + +event bro_init() + { + BrokerComm::enable(); + BrokerComm::subscribe_to_logs("bro/log/Test::LOG"); + BrokerComm::listen(broker_port, "127.0.0.1"); + } + +event BrokerComm::incoming_connection_established(peer_name: string) + { + print "BrokerComm::incoming_connection_established", peer_name; + } + +event Test::log_test(rec: Test::Info) + { + print "wrote log", rec; + + if ( rec$num == 5 ) + terminate(); + } diff --git a/doc/frameworks/broker/printing-connector.bro b/doc/frameworks/broker/printing-connector.bro new file mode 100644 index 0000000000..2a504ffba0 --- /dev/null +++ b/doc/frameworks/broker/printing-connector.bro @@ -0,0 +1,26 @@ +const broker_port: port = 9999/tcp &redef; +redef exit_only_after_terminate = T; +redef BrokerComm::endpoint_name = "connector"; + +event bro_init() + { + BrokerComm::enable(); + BrokerComm::connect("127.0.0.1", broker_port, 1sec); + } + +event BrokerComm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + print "BrokerComm::outgoing_connection_established", + peer_address, peer_port, peer_name; + BrokerComm::print("bro/print/hi", "hello"); + BrokerComm::print("bro/print/stuff", "..."); + BrokerComm::print("bro/print/bye", "goodbye"); + } + +event BrokerComm::outgoing_connection_broken(peer_address: string, + peer_port: port) + { + terminate(); + } diff --git a/doc/frameworks/broker/printing-listener.bro b/doc/frameworks/broker/printing-listener.bro new file mode 100644 index 0000000000..080d09e8f5 --- /dev/null +++ b/doc/frameworks/broker/printing-listener.bro @@ -0,0 +1,26 @@ + +const broker_port: port = 9999/tcp &redef; +redef exit_only_after_terminate = T; +redef BrokerComm::endpoint_name = "listener"; +global msg_count = 0; + +event bro_init() + { + BrokerComm::enable(); + BrokerComm::subscribe_to_prints("bro/print/"); + BrokerComm::listen(broker_port, "127.0.0.1"); + } + +event BrokerComm::incoming_connection_established(peer_name: string) + { + print "BrokerComm::incoming_connection_established", peer_name; + } + +event BrokerComm::print_handler(msg: string) + { + ++msg_count; + print "got print message", msg; + + if ( msg_count == 3 ) + terminate(); + } diff --git a/doc/frameworks/broker/stores-connector.bro b/doc/frameworks/broker/stores-connector.bro new file mode 100644 index 0000000000..5db8657a68 --- /dev/null +++ b/doc/frameworks/broker/stores-connector.bro @@ -0,0 +1,53 @@ +const broker_port: port = 9999/tcp &redef; +redef exit_only_after_terminate = T; + +global h: opaque of BrokerStore::Handle; + +function dv(d: BrokerComm::Data): BrokerComm::DataVector + { + local rval: BrokerComm::DataVector; + rval[0] = d; + return rval; + } + +global ready: event(); + +event BrokerComm::outgoing_connection_broken(peer_address: string, + peer_port: port) + { + terminate(); + } + +event BrokerComm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + local myset: set[string] = {"a", "b", "c"}; + local myvec: vector of string = {"alpha", "beta", "gamma"}; + h = BrokerStore::create_master("mystore"); + BrokerStore::insert(h, BrokerComm::data("one"), BrokerComm::data(110)); + BrokerStore::insert(h, BrokerComm::data("two"), BrokerComm::data(223)); + BrokerStore::insert(h, BrokerComm::data("myset"), BrokerComm::data(myset)); + BrokerStore::insert(h, BrokerComm::data("myvec"), BrokerComm::data(myvec)); + BrokerStore::increment(h, BrokerComm::data("one")); + BrokerStore::decrement(h, BrokerComm::data("two")); + BrokerStore::add_to_set(h, BrokerComm::data("myset"), BrokerComm::data("d")); + BrokerStore::remove_from_set(h, BrokerComm::data("myset"), BrokerComm::data("b")); + BrokerStore::push_left(h, BrokerComm::data("myvec"), dv(BrokerComm::data("delta"))); + BrokerStore::push_right(h, BrokerComm::data("myvec"), dv(BrokerComm::data("omega"))); + + when ( local res = BrokerStore::size(h) ) + { + print "master size", res; + event ready(); + } + timeout 10sec + { print "timeout"; } + } + +event bro_init() + { + BrokerComm::enable(); + BrokerComm::connect("127.0.0.1", broker_port, 1secs); + BrokerComm::auto_event("bro/event/ready", ready); + } diff --git a/doc/frameworks/broker/stores-listener.bro b/doc/frameworks/broker/stores-listener.bro new file mode 100644 index 0000000000..454e41a8c2 --- /dev/null +++ b/doc/frameworks/broker/stores-listener.bro @@ -0,0 +1,43 @@ +const broker_port: port = 9999/tcp &redef; +redef exit_only_after_terminate = T; + +global h: opaque of BrokerStore::Handle; +global expected_key_count = 4; +global key_count = 0; + +function do_lookup(key: string) + { + when ( local res = BrokerStore::lookup(h, BrokerComm::data(key)) ) + { + ++key_count; + print "lookup", key, res; + + if ( key_count == expected_key_count ) + terminate(); + } + timeout 10sec + { print "timeout", key; } + } + +event ready() + { + h = BrokerStore::create_clone("mystore"); + + when ( local res = BrokerStore::keys(h) ) + { + print "clone keys", res; + do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 0))); + do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 1))); + do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 2))); + do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 3))); + } + timeout 10sec + { print "timeout"; } + } + +event bro_init() + { + BrokerComm::enable(); + BrokerComm::subscribe_to_events("bro/event/ready"); + BrokerComm::listen(broker_port, "127.0.0.1"); + } diff --git a/doc/frameworks/broker/testlog.bro b/doc/frameworks/broker/testlog.bro new file mode 100644 index 0000000000..f63c19ac48 --- /dev/null +++ b/doc/frameworks/broker/testlog.bro @@ -0,0 +1,19 @@ + +module Test; + +export { + redef enum Log::ID += { LOG }; + + type Info: record { + msg: string &log; + num: count &log; + }; + + global log_test: event(rec: Test::Info); +} + +event bro_init() &priority=5 + { + BrokerComm::enable(); + Log::create_stream(Test::LOG, [$columns=Test::Info, $ev=log_test, $path="test"]); + } diff --git a/doc/frameworks/file_analysis_02.bro b/doc/frameworks/file_analysis_02.bro index 141b11fca6..fd4f0e775e 100644 --- a/doc/frameworks/file_analysis_02.bro +++ b/doc/frameworks/file_analysis_02.bro @@ -1,7 +1,8 @@ -event file_mime_type(f: fa_file, mime_type: string) +event file_sniff(f: fa_file, meta: fa_metadata) { + if ( ! meta?$mime_type ) return; print "new file", f$id; - if ( mime_type == "text/plain" ) + if ( meta$mime_type == "text/plain" ) Files::add_analyzer(f, Files::ANALYZER_MD5); } diff --git a/doc/frameworks/index.rst b/doc/frameworks/index.rst index f8c681d795..028f95af21 100644 --- a/doc/frameworks/index.rst +++ b/doc/frameworks/index.rst @@ -14,4 +14,4 @@ Frameworks notice signatures sumstats - + broker diff --git a/doc/frameworks/logging.rst b/doc/frameworks/logging.rst index c64ab02489..765d7bed23 100644 --- a/doc/frameworks/logging.rst +++ b/doc/frameworks/logging.rst @@ -344,7 +344,7 @@ example for the ``Foo`` module: event bro_init() &priority=5 { # Create the stream. This also adds a default filter automatically. - Log::create_stream(Foo::LOG, [$columns=Info, $ev=log_foo]); + Log::create_stream(Foo::LOG, [$columns=Info, $ev=log_foo, $path="foo"]); } You can also add the state to the :bro:type:`connection` record to make diff --git a/doc/frameworks/notice.rst b/doc/frameworks/notice.rst index d8197c13af..eacf9c917f 100644 --- a/doc/frameworks/notice.rst +++ b/doc/frameworks/notice.rst @@ -88,15 +88,15 @@ directly make modifications to the :bro:see:`Notice::Info` record given as the argument to the hook. Here's a simple example which tells Bro to send an email for all notices of -type :bro:see:`SSH::Password_Guessing` if the server is 10.0.0.1: +type :bro:see:`SSH::Password_Guessing` if the guesser attempted to log in to +the server at 192.168.56.103: -.. code:: bro +.. btest-include:: ${DOC_ROOT}/frameworks/notice_ssh_guesser.bro - hook Notice::policy(n: Notice::Info) - { - if ( n$note == SSH::Password_Guessing && n$id$resp_h == 10.0.0.1 ) - add n$actions[Notice::ACTION_EMAIL]; - } +.. btest:: notice_ssh_guesser.bro + + @TEST-EXEC: btest-rst-cmd bro -C -r ${TRACES}/ssh/sshguess.pcap ${DOC_ROOT}/frameworks/notice_ssh_guesser.bro + @TEST-EXEC: btest-rst-cmd cat notice.log .. note:: @@ -111,10 +111,9 @@ a hook body to run before default hook bodies might look like this: .. code:: bro hook Notice::policy(n: Notice::Info) &priority=5 - { - if ( n$note == SSH::Password_Guessing && n$id$resp_h == 10.0.0.1 ) - add n$actions[Notice::ACTION_EMAIL]; - } + { + # Insert your code here. + } Hooks can also abort later hook bodies with the ``break`` keyword. This is primarily useful if one wants to completely preempt processing by diff --git a/doc/frameworks/notice_ssh_guesser.bro b/doc/frameworks/notice_ssh_guesser.bro new file mode 100644 index 0000000000..34ffe2e95e --- /dev/null +++ b/doc/frameworks/notice_ssh_guesser.bro @@ -0,0 +1,10 @@ + +@load protocols/ssh/detect-bruteforcing + +redef SSH::password_guesses_limit=10; + +hook Notice::policy(n: Notice::Info) + { + if ( n$note == SSH::Password_Guessing && /192\.168\.56\.103/ in n$sub ) + add n$actions[Notice::ACTION_EMAIL]; + } diff --git a/doc/httpmonitor/file_extraction.bro b/doc/httpmonitor/file_extraction.bro index 3860cb361e..c387156b62 100644 --- a/doc/httpmonitor/file_extraction.bro +++ b/doc/httpmonitor/file_extraction.bro @@ -7,15 +7,18 @@ global mime_to_ext: table[string] of string = { ["text/html"] = "html", }; -event file_mime_type(f: fa_file, mime_type: string) +event file_sniff(f: fa_file, meta: fa_metadata) { if ( f$source != "HTTP" ) return; - if ( mime_type !in mime_to_ext ) + if ( ! meta?$mime_type ) return; - local fname = fmt("%s-%s.%s", f$source, f$id, mime_to_ext[mime_type]); + if ( meta$mime_type !in mime_to_ext ) + return; + + local fname = fmt("%s-%s.%s", f$source, f$id, mime_to_ext[meta$mime_type]); print fmt("Extracting file %s", fname); Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]); - } \ No newline at end of file + } diff --git a/doc/install/install.rst b/doc/install/install.rst index a3531f70c3..18bacd7758 100644 --- a/doc/install/install.rst +++ b/doc/install/install.rst @@ -68,7 +68,7 @@ that ``bash`` and ``python`` are in your ``PATH``): .. console:: - sudo pkg_add -r bash cmake swig bison python perl + sudo pkg_add -r bash cmake swig bison python perl py27-sqlite3 * Mac OS X: @@ -113,19 +113,15 @@ Using Pre-Built Binary Release Packages ======================================= See the `bro downloads page`_ for currently supported/targeted -platforms for binary releases. +platforms for binary releases and for installation instructions. -* RPM +* Linux Packages - .. console:: - - sudo yum localinstall Bro-*.rpm - -* DEB - - .. console:: - - sudo gdebi Bro-*.deb + Linux based binary installations are usually performed by adding + information about the Bro packages to the respective system packaging + tool. Theen the usual system utilities such as ``apt``, ``yum`` + or ``zyppper`` are used to perforn the installation. By default, + installations of binary packages will go into ``/opt/bro``. * MacOS Disk Image with Installer @@ -133,8 +129,6 @@ platforms for binary releases. Everything installed by the package will go into ``/opt/bro``. The primary install prefix for binary packages is ``/opt/bro``. -Non-MacOS packages that include BroControl also put variable/runtime -data (e.g. Bro logs) in ``/var/opt/bro``. Installing from Source ========================== diff --git a/doc/mimestats/mimestats.bro b/doc/mimestats/mimestats.bro index b854b26c2d..dc2eeab087 100644 --- a/doc/mimestats/mimestats.bro +++ b/doc/mimestats/mimestats.bro @@ -30,7 +30,7 @@ export { event bro_init() &priority=3 { - Log::create_stream(MimeMetrics::LOG, [$columns=Info]); + Log::create_stream(MimeMetrics::LOG, [$columns=Info, $path="mime_metrics"]); local r1: SumStats::Reducer = [$stream="mime.bytes", $apply=set(SumStats::SUM)]; local r2: SumStats::Reducer = [$stream="mime.hits", diff --git a/doc/quickstart/conditional-notice.bro b/doc/quickstart/conditional-notice.bro new file mode 100644 index 0000000000..8cc01787c9 --- /dev/null +++ b/doc/quickstart/conditional-notice.bro @@ -0,0 +1,24 @@ +@load protocols/ssl/expiring-certs + +const watched_servers: set[addr] = { + 87.98.220.10, +} &redef; + +# Site::local_nets usually isn't something you need to modify if +# BroControl automatically sets it up from networks.cfg. It's +# shown here for completeness. +redef Site::local_nets += { + 87.98.0.0/16, +}; + +hook Notice::policy(n: Notice::Info) + { + if ( n$note != SSL::Certificate_Expired ) + return; + + if ( n$id$resp_h !in watched_servers ) + return; + + add n$actions[Notice::ACTION_EMAIL]; + } + diff --git a/doc/quickstart/index.rst b/doc/quickstart/index.rst index bb642ee75a..616c94c261 100644 --- a/doc/quickstart/index.rst +++ b/doc/quickstart/index.rst @@ -156,9 +156,11 @@ changes we want to make: notice that means an SSL connection was established and the server's certificate couldn't be validated using Bro's default trust roots, but we want to ignore it. -2) ``SSH::Login`` is a notice type that is triggered when an SSH connection - attempt looks like it may have been successful, and we want email when - that happens, but only for certain servers. +2) ``SSL::Certificate_Expired`` is a notice type that is triggered when + an SSL connection was established using an expired certificate. We + want email when that happens, but only for certain servers on the + local network (Bro can also proactively monitor for certs that will + soon expire, but this is just for demonstration purposes). We've defined *what* we want to do, but need to know *where* to do it. The answer is to use a script written in the Bro programming language, so @@ -203,7 +205,7 @@ the variable's value may not change at run-time, but whose initial value can be modified via the ``redef`` operator at parse-time. Let's continue on our path to modify the behavior for the two SSL -and SSH notices. Looking at :doc:`/scripts/base/frameworks/notice/main.bro`, +notices. Looking at :doc:`/scripts/base/frameworks/notice/main.bro`, we see that it advertises: .. code:: bro @@ -216,7 +218,7 @@ we see that it advertises: const ignored_types: set[Notice::Type] = {} &redef; } -That's exactly what we want to do for the SSL notice. Add to ``local.bro``: +That's exactly what we want to do for the first notice. Add to ``local.bro``: .. code:: bro @@ -248,38 +250,30 @@ is valid before installing it and then restarting the Bro instance: stopping bro ... starting bro ... -Now that the SSL notice is ignored, let's look at how to send an email on -the SSH notice. The notice framework has a similar option called -``emailed_types``, but using that would generate email for all SSH servers and -we only want email for logins to certain ones. There is a ``policy`` hook -that is actually what is used to implement the simple functionality of -``ignored_types`` and -``emailed_types``, but it's extensible such that the condition and action taken -on notices can be user-defined. +Now that the SSL notice is ignored, let's look at how to send an email +on the other notice. The notice framework has a similar option called +``emailed_types``, but using that would generate email for all SSL +servers with expired certificates and we only want email for connections +to certain ones. There is a ``policy`` hook that is actually what is +used to implement the simple functionality of ``ignored_types`` and +``emailed_types``, but it's extensible such that the condition and +action taken on notices can be user-defined. -In ``local.bro``, let's define a new ``policy`` hook handler body -that takes the email action for SSH logins only for a defined set of servers: +In ``local.bro``, let's define a new ``policy`` hook handler body: -.. code:: bro +.. btest-include:: ${DOC_ROOT}/quickstart/conditional-notice.bro - const watched_servers: set[addr] = { - 192.168.1.100, - 192.168.1.101, - 192.168.1.102, - } &redef; +.. btest:: conditional-notice - hook Notice::policy(n: Notice::Info) - { - if ( n$note == SSH::SUCCESSFUL_LOGIN && n$id$resp_h in watched_servers ) - add n$actions[Notice::ACTION_EMAIL]; - } + @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/tls/tls-expired-cert.trace ${DOC_ROOT}/quickstart/conditional-notice.bro + @TEST-EXEC: btest-rst-cmd cat notice.log You'll just have to trust the syntax for now, but what we've done is first declare our own variable to hold a set of watched addresses, -``watched_servers``; then added a hook handler body to the policy that will -generate an email whenever the notice type is an SSH login and the responding -host stored -inside the ``Info`` record's connection field is in the set of watched servers. +``watched_servers``; then added a hook handler body to the policy that +will generate an email whenever the notice type is an SSL expired +certificate and the responding host stored inside the ``Info`` record's +connection field is in the set of watched servers. .. note:: Record field member access is done with the '$' character instead of a '.' as might be expected from other languages, in diff --git a/doc/script-reference/attributes.rst b/doc/script-reference/attributes.rst index ef6c6a54a1..40646f64f4 100644 --- a/doc/script-reference/attributes.rst +++ b/doc/script-reference/attributes.rst @@ -43,8 +43,6 @@ The Bro scripting language supports the following attributes. +-----------------------------+-----------------------------------------------+ | :bro:attr:`&mergeable` |Prefer set union for synchronized state. | +-----------------------------+-----------------------------------------------+ -| :bro:attr:`&group` |Group event handlers to activate/deactivate. | -+-----------------------------+-----------------------------------------------+ | :bro:attr:`&error_handler` |Used internally for reporter framework events. | +-----------------------------+-----------------------------------------------+ | :bro:attr:`&type_column` |Used by input framework for "port" type. | @@ -198,11 +196,6 @@ Here is a more detailed explanation of each attribute: inconsistencies and can be avoided by unifying the two sets, rather than merely overwriting the old value. -.. bro:attr:: &group - - Groups event handlers such that those in the same group can be - jointly activated or deactivated. - .. bro:attr:: &error_handler Internally set on the events that are associated with the reporter diff --git a/doc/script-reference/statements.rst b/doc/script-reference/statements.rst index 2eeb1940e7..bf607ad0f9 100644 --- a/doc/script-reference/statements.rst +++ b/doc/script-reference/statements.rst @@ -294,7 +294,10 @@ Here are the statements that the Bro scripting language supports. .. bro:keyword:: for A "for" loop iterates over each element in a string, set, vector, or - table and executes a statement for each iteration. + table and executes a statement for each iteration. Currently, + modifying a container's membership while iterating over it may + result in undefined behavior, so avoid adding or removing elements + inside the loop. For each iteration of the loop, a loop variable will be assigned to an element if the expression evaluates to a string or set, or an index if diff --git a/doc/scripting/framework_logging_factorial_02.bro b/doc/scripting/framework_logging_factorial_02.bro index 02a451265c..f947314065 100644 --- a/doc/scripting/framework_logging_factorial_02.bro +++ b/doc/scripting/framework_logging_factorial_02.bro @@ -23,7 +23,7 @@ function factorial(n: count): count event bro_init() { # Create the logging stream. - Log::create_stream(LOG, [$columns=Info]); + Log::create_stream(LOG, [$columns=Info, $path="factor"]); } event bro_done() diff --git a/doc/scripting/framework_logging_factorial_03.bro b/doc/scripting/framework_logging_factorial_03.bro index c99cca2c1d..20cb14bd59 100644 --- a/doc/scripting/framework_logging_factorial_03.bro +++ b/doc/scripting/framework_logging_factorial_03.bro @@ -37,7 +37,7 @@ function mod5(id: Log::ID, path: string, rec: Factor::Info) : string event bro_init() { - Log::create_stream(LOG, [$columns=Info]); + Log::create_stream(LOG, [$columns=Info, $path="factor"]); local filter: Log::Filter = [$name="split-mod5s", $path_func=mod5]; Log::add_filter(Factor::LOG, filter); diff --git a/doc/scripting/framework_logging_factorial_04.bro b/doc/scripting/framework_logging_factorial_04.bro index a563069c46..719d8e27e1 100644 --- a/doc/scripting/framework_logging_factorial_04.bro +++ b/doc/scripting/framework_logging_factorial_04.bro @@ -22,7 +22,7 @@ function factorial(n: count): count event bro_init() { - Log::create_stream(LOG, [$columns=Info, $ev=log_factor]); + Log::create_stream(LOG, [$columns=Info, $ev=log_factor, $path="factor"]); } event bro_done() diff --git a/doc/scripting/index.rst b/doc/scripting/index.rst index fb1c1b67a1..2b5cfbb49c 100644 --- a/doc/scripting/index.rst +++ b/doc/scripting/index.rst @@ -363,7 +363,7 @@ decrypted from HTTP streams is stored in excerpt from :doc:`/scripts/base/protocols/http/main.bro` below. .. btest-include:: ${BRO_SRC_ROOT}/scripts/base/protocols/http/main.bro - :lines: 9-11,20-22,121 + :lines: 9-11,20-22,125 Because the constant was declared with the ``&redef`` attribute, if we needed to turn this option on globally, we could do so by adding the @@ -826,7 +826,7 @@ example of the ``record`` data type in the earlier sections, the ``conn.log``, is shown by the excerpt below. .. btest-include:: ${BRO_SRC_ROOT}/scripts/base/protocols/conn/main.bro - :lines: 10-12,16-17,19,21,23,25,28,31,35,38,57,63,69,92,95,99,102,106,110-111,116 + :lines: 10-12,16-17,19,21,23,25,28,31,35,38,57,63,69,75,98,101,105,108,112,116-117,122 Looking at the structure of the definition, a new collection of data types is being defined as a type called ``Info``. Since this type diff --git a/pkg/check-cmake b/pkg/check-cmake deleted file mode 100755 index 17531af2f7..0000000000 --- a/pkg/check-cmake +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -# CMake/CPack versions before 2.8.3 have bugs that can create bad packages -# Since packages will be built on several different systems, a single -# version of CMake is required to obtain consistency, but can be increased -# as new versions of CMake come out that also produce working packages. - -CMAKE_PACK_REQ="cmake version 2.8.6" -CMAKE_VER=`cmake -version` - -if [ "${CMAKE_VER}" != "${CMAKE_PACK_REQ}" ]; then - echo "Package creation requires ${CMAKE_PACK_REQ}" >&2 - exit 1 -fi diff --git a/pkg/make-deb-packages b/pkg/make-deb-packages index 0a435a756f..36bd62c19c 100755 --- a/pkg/make-deb-packages +++ b/pkg/make-deb-packages @@ -3,8 +3,6 @@ # This script generates binary DEB packages. # They can be found in ../build/ after running. -./check-cmake || { exit 1; } - # The DEB CPack generator depends on `dpkg-shlibdeps` to automatically # determine what dependencies to set for the packages type dpkg-shlibdeps > /dev/null 2>&1 || { diff --git a/pkg/make-mac-packages b/pkg/make-mac-packages index 2930f8f393..b3d200842f 100755 --- a/pkg/make-mac-packages +++ b/pkg/make-mac-packages @@ -3,14 +3,6 @@ # This script creates binary packages for Mac OS X. # They can be found in ../build/ after running. -cmake -P /dev/stdin << "EOF" -if ( ${CMAKE_VERSION} VERSION_LESS 2.8.9 ) - message(FATAL_ERROR "CMake >= 2.8.9 required to build package") -endif () -EOF - -[ $? -ne 0 ] && exit 1; - type sw_vers > /dev/null 2>&1 || { echo "Unable to get Mac OS X version" >&2; exit 1; diff --git a/pkg/make-rpm-packages b/pkg/make-rpm-packages index 43b962f417..ee09511e44 100755 --- a/pkg/make-rpm-packages +++ b/pkg/make-rpm-packages @@ -3,8 +3,6 @@ # This script generates binary RPM packages. # They can be found in ../build/ after running. -./check-cmake || { exit 1; } - # The RPM CPack generator depends on `rpmbuild` to create packages type rpmbuild > /dev/null 2>&1 || { echo "\ diff --git a/scripts/base/files/extract/main.bro b/scripts/base/files/extract/main.bro index 765263a4d8..7f68a8bcce 100644 --- a/scripts/base/files/extract/main.bro +++ b/scripts/base/files/extract/main.bro @@ -53,7 +53,8 @@ function set_limit(f: fa_file, args: Files::AnalyzerArgs, n: count): bool function on_add(f: fa_file, args: Files::AnalyzerArgs) { if ( ! args?$extract_filename ) - args$extract_filename = cat("extract-", f$source, "-", f$id); + args$extract_filename = cat("extract-", f$last_active, "-", f$source, + "-", f$id); f$info$extracted = args$extract_filename; args$extract_filename = build_path_compressed(prefix, args$extract_filename); diff --git a/scripts/base/files/pe/__load__.bro b/scripts/base/files/pe/__load__.bro new file mode 100644 index 0000000000..0098b81a7a --- /dev/null +++ b/scripts/base/files/pe/__load__.bro @@ -0,0 +1,2 @@ +@load ./consts +@load ./main \ No newline at end of file diff --git a/scripts/base/files/pe/consts.bro b/scripts/base/files/pe/consts.bro new file mode 100644 index 0000000000..35ad9c3c61 --- /dev/null +++ b/scripts/base/files/pe/consts.bro @@ -0,0 +1,184 @@ + +module PE; + +export { + const machine_types: table[count] of string = { + [0x00] = "UNKNOWN", + [0x1d3] = "AM33", + [0x8664] = "AMD64", + [0x1c0] = "ARM", + [0x1c4] = "ARMNT", + [0xaa64] = "ARM64", + [0xebc] = "EBC", + [0x14c] = "I386", + [0x200] = "IA64", + [0x9041] = "M32R", + [0x266] = "MIPS16", + [0x366] = "MIPSFPU", + [0x466] = "MIPSFPU16", + [0x1f0] = "POWERPC", + [0x1f1] = "POWERPCFP", + [0x166] = "R4000", + [0x1a2] = "SH3", + [0x1a3] = "SH3DSP", + [0x1a6] = "SH4", + [0x1a8] = "SH5", + [0x1c2] = "THUMB", + [0x169] = "WCEMIPSV2" + } &default=function(i: count):string { return fmt("unknown-%d", i); }; + + const file_characteristics: table[count] of string = { + [0x1] = "RELOCS_STRIPPED", + [0x2] = "EXECUTABLE_IMAGE", + [0x4] = "LINE_NUMS_STRIPPED", + [0x8] = "LOCAL_SYMS_STRIPPED", + [0x10] = "AGGRESSIVE_WS_TRIM", + [0x20] = "LARGE_ADDRESS_AWARE", + [0x80] = "BYTES_REVERSED_LO", + [0x100] = "32BIT_MACHINE", + [0x200] = "DEBUG_STRIPPED", + [0x400] = "REMOVABLE_RUN_FROM_SWAP", + [0x800] = "NET_RUN_FROM_SWAP", + [0x1000] = "SYSTEM", + [0x2000] = "DLL", + [0x4000] = "UP_SYSTEM_ONLY", + [0x8000] = "BYTES_REVERSED_HI" + } &default=function(i: count):string { return fmt("unknown-%d", i); }; + + const dll_characteristics: table[count] of string = { + [0x40] = "DYNAMIC_BASE", + [0x80] = "FORCE_INTEGRITY", + [0x100] = "NX_COMPAT", + [0x200] = "NO_ISOLATION", + [0x400] = "NO_SEH", + [0x800] = "NO_BIND", + [0x2000] = "WDM_DRIVER", + [0x8000] = "TERMINAL_SERVER_AWARE" + } &default=function(i: count):string { return fmt("unknown-%d", i); }; + + const windows_subsystems: table[count] of string = { + [0] = "UNKNOWN", + [1] = "NATIVE", + [2] = "WINDOWS_GUI", + [3] = "WINDOWS_CUI", + [7] = "POSIX_CUI", + [9] = "WINDOWS_CE_GUI", + [10] = "EFI_APPLICATION", + [11] = "EFI_BOOT_SERVICE_DRIVER", + [12] = "EFI_RUNTIME_
DRIVER", + [13] = "EFI_ROM", + [14] = "XBOX" + } &default=function(i: count):string { return fmt("unknown-%d", i); }; + + const directories: table[count] of string = { + [0] = "Export Table", + [1] = "Import Table", + [2] = "Resource Table", + [3] = "Exception Table", + [4] = "Certificate Table", + [5] = "Base Relocation Table", + [6] = "Debug", + [7] = "Architecture", + [8] = "Global Ptr", + [9] = "TLS Table", + [10] = "Load Config Table", + [11] = "Bound Import", + [12] = "IAT", + [13] = "Delay Import Descriptor", + [14] = "CLR Runtime Header", + [15] = "Reserved" + } &default=function(i: count):string { return fmt("unknown-%d", i); }; + + const section_characteristics: table[count] of string = { + [0x8] = "TYPE_NO_PAD", + [0x20] = "CNT_CODE", + [0x40] = "CNT_INITIALIZED_DATA", + [0x80] = "CNT_UNINITIALIZED_DATA", + [0x100] = "LNK_OTHER", + [0x200] = "LNK_INFO", + [0x800] = "LNK_REMOVE", + [0x1000] = "LNK_COMDAT", + [0x8000] = "GPREL", + [0x20000] = "MEM_16BIT", + [0x40000] = "MEM_LOCKED", + [0x80000] = "MEM_PRELOAD", + [0x100000] = "ALIGN_1BYTES", + [0x200000] = "ALIGN_2BYTES", + [0x300000] = "ALIGN_4BYTES", + [0x400000] = "ALIGN_8BYTES", + [0x500000] = "ALIGN_16BYTES", + [0x600000] = "ALIGN_32BYTES", + [0x700000] = "ALIGN_64BYTES", + [0x800000] = "ALIGN_128BYTES", + [0x900000] = "ALIGN_256BYTES", + [0xa00000] = "ALIGN_512BYTES", + [0xb00000] = "ALIGN_1024BYTES", + [0xc00000] = "ALIGN_2048BYTES", + [0xd00000] = "ALIGN_4096BYTES", + [0xe00000] = "ALIGN_8192BYTES", + [0x1000000] = "LNK_NRELOC_OVFL", + [0x2000000] = "MEM_DISCARDABLE", + [0x4000000] = "MEM_NOT_CACHED", + [0x8000000] = "MEM_NOT_PAGED", + [0x10000000] = "MEM_SHARED", + [0x20000000] = "MEM_EXECUTE", + [0x40000000] = "MEM_READ", + [0x80000000] = "MEM_WRITE" + } &default=function(i: count):string { return fmt("unknown-%d", i); }; + + const os_versions: table[count, count] of string = { + [10,0] = "Windows 10", + [6,4] = "Windows 10 Technical Preview", + [6,3] = "Windows 8.1 or Server 2012 R2", + [6,2] = "Windows 8 or Server 2012", + [6,1] = "Windows 7 or Server 2008 R2", + [6,0] = "Windows Vista or Server 2008", + [5,2] = "Windows XP x64 or Server 2003", + [5,1] = "Windows XP", + [5,0] = "Windows 2000", + [4,90] = "Windows Me", + [4,10] = "Windows 98", + [4,0] = "Windows 95 or NT 4.0", + [3,51] = "Windows NT 3.51", + [3,50] = "Windows NT 3.5", + [3,2] = "Windows 3.2", + [3,11] = "Windows for Workgroups 3.11", + [3,10] = "Windows 3.1 or NT 3.1", + [3,0] = "Windows 3.0", + [2,11] = "Windows 2.11", + [2,10] = "Windows 2.10", + [2,0] = "Windows 2.0", + [1,4] = "Windows 1.04", + [1,3] = "Windows 1.03", + [1,1] = "Windows 1.01", + [1,0] = "Windows 1.0", + } &default=function(i: count, j: count):string { return fmt("unknown-%d.%d", i, j); }; + + const section_descs: table[string] of string = { + [".bss"] = "Uninitialized data", + [".cormeta"] = "CLR metadata that indicates that the object file contains managed code", + [".data"] = "Initialized data", + [".debug$F"] = "Generated FPO debug information", + [".debug$P"] = "Precompiled debug types", + [".debug$S"] = "Debug symbols", + [".debug$T"] = "Debug types", + [".drective"] = "Linker options", + [".edata"] = "Export tables", + [".idata"] = "Import tables", + [".idlsym"] = "Includes registered SEH to support IDL attributes", + [".pdata"] = "Exception information", + [".rdata"] = "Read-only initialized data", + [".reloc"] = "Image relocations", + [".rsrc"] = "Resource directory", + [".sbss"] = "GP-relative uninitialized data", + [".sdata"] = "GP-relative initialized data", + [".srdata"] = "GP-relative read-only data", + [".sxdata"] = "Registered exception handler data", + [".text"] = "Executable code", + [".tls"] = "Thread-local storage", + [".tls$"] = "Thread-local storage", + [".vsdata"] = "GP-relative initialized data", + [".xdata"] = "Exception information", + } &default=function(i: string):string { return fmt("unknown-%s", i); }; + +} diff --git a/scripts/base/files/pe/main.bro b/scripts/base/files/pe/main.bro new file mode 100644 index 0000000000..b2723e4138 --- /dev/null +++ b/scripts/base/files/pe/main.bro @@ -0,0 +1,137 @@ +module PE; + +@load ./consts.bro + +export { + redef enum Log::ID += { LOG }; + + type Info: record { + ## Current timestamp. + ts: time &log; + ## File id of this portable executable file. + id: string &log; + ## The target machine that the file was compiled for. + machine: string &log &optional; + ## The time that the file was created at. + compile_ts: time &log &optional; + ## The required operating system. + os: string &log &optional; + ## The subsystem that is required to run this file. + subsystem: string &log &optional; + ## Is the file an executable, or just an object file? + is_exe: bool &log &default=T; + ## Is the file a 64-bit executable? + is_64bit: bool &log &default=T; + ## Does the file support Address Space Layout Randomization? + uses_aslr: bool &log &default=F; + ## Does the file support Data Execution Prevention? + uses_dep: bool &log &default=F; + ## Does the file enforce code integrity checks? + uses_code_integrity: bool &log &default=F; + ## Does the file use structured exception handing? + uses_seh: bool &log &default=T; + ## Does the file have an import table? + has_import_table: bool &log &optional; + ## Does the file have an export table? + has_export_table: bool &log &optional; + ## Does the file have an attribute certificate table? + has_cert_table: bool &log &optional; + ## Does the file have a debug table? + has_debug_data: bool &log &optional; + ## The names of the sections, in order. + section_names: vector of string &log &optional; + }; + + ## Event for accessing logged records. + global log_pe: event(rec: Info); + + ## A hook that gets called when we first see a PE file. + global set_file: hook(f: fa_file); +} + +redef record fa_file += { + pe: Info &optional; +}; + +const pe_mime_types = { "application/x-dosexec" }; + +event bro_init() &priority=5 + { + Files::register_for_mime_types(Files::ANALYZER_PE, pe_mime_types); + Log::create_stream(LOG, [$columns=Info, $ev=log_pe, $path="pe"]); + } + +hook set_file(f: fa_file) &priority=5 + { + if ( ! f?$pe ) + f$pe = [$ts=network_time(), $id=f$id]; + } + +event pe_dos_header(f: fa_file, h: PE::DOSHeader) &priority=5 + { + hook set_file(f); + } + +event pe_file_header(f: fa_file, h: PE::FileHeader) &priority=5 + { + hook set_file(f); + + f$pe$machine = machine_types[h$machine]; + f$pe$compile_ts = h$ts; + f$pe$is_exe = ( h$optional_header_size > 0 ); + + for ( c in h$characteristics ) + { + if ( file_characteristics[c] == "32BIT_MACHINE" ) + f$pe$is_64bit = F; + } + } + +event pe_optional_header(f: fa_file, h: PE::OptionalHeader) &priority=5 + { + hook set_file(f); + + # Only EXEs have optional headers + if ( ! f$pe$is_exe ) + return; + + f$pe$os = os_versions[h$os_version_major, h$os_version_minor]; + f$pe$subsystem = windows_subsystems[h$subsystem]; + + for ( c in h$dll_characteristics ) + { + if ( dll_characteristics[c] == "DYNAMIC_BASE" ) + f$pe$uses_aslr = T; + if ( dll_characteristics[c] == "FORCE_INTEGRITY" ) + f$pe$uses_code_integrity = T; + if ( dll_characteristics[c] == "NX_COMPAT" ) + f$pe$uses_dep = T; + if ( dll_characteristics[c] == "NO_SEH" ) + f$pe$uses_seh = F; + } + + f$pe$has_export_table = (|h$table_sizes| > 0 && h$table_sizes[0] > 0); + f$pe$has_import_table = (|h$table_sizes| > 1 && h$table_sizes[1] > 0); + f$pe$has_cert_table = (|h$table_sizes| > 4 && h$table_sizes[4] > 0); + f$pe$has_debug_data = (|h$table_sizes| > 6 && h$table_sizes[6] > 0); + } + +event pe_section_header(f: fa_file, h: PE::SectionHeader) &priority=5 + { + hook set_file(f); + + # Only EXEs have section headers + if ( ! f$pe$is_exe ) + return; + + if ( ! f$pe?$section_names ) + f$pe$section_names = vector(); + f$pe$section_names[|f$pe$section_names|] = h$name; + } + +event file_state_remove(f: fa_file) &priority=-5 + { + if ( f?$pe && f$pe?$machine ) + Log::write(LOG, f$pe); + } + diff --git a/scripts/base/files/unified2/main.bro b/scripts/base/files/unified2/main.bro index 73f98aa5f8..9d9ef15d79 100644 --- a/scripts/base/files/unified2/main.bro +++ b/scripts/base/files/unified2/main.bro @@ -195,7 +195,7 @@ event Input::end_of_data(name: string, source: string) event bro_init() &priority=5 { - Log::create_stream(Unified2::LOG, [$columns=Info, $ev=log_unified2]); + Log::create_stream(Unified2::LOG, [$columns=Info, $ev=log_unified2, $path="unified2"]); if ( sid_msg == "" ) { diff --git a/scripts/base/files/x509/main.bro b/scripts/base/files/x509/main.bro index 10445ad846..c097b84560 100644 --- a/scripts/base/files/x509/main.bro +++ b/scripts/base/files/x509/main.bro @@ -36,7 +36,7 @@ export { event bro_init() &priority=5 { - Log::create_stream(X509::LOG, [$columns=Info, $ev=log_x509]); + Log::create_stream(X509::LOG, [$columns=Info, $ev=log_x509, $path="x509"]); } redef record Files::Info += { @@ -47,6 +47,9 @@ redef record Files::Info += { event x509_certificate(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate) &priority=5 { + if ( ! f$info?$mime_type ) + f$info$mime_type = "application/pkix-cert"; + f$info$x509 = [$ts=f$info$ts, $id=f$id, $certificate=cert, $handle=cert_ref]; } diff --git a/scripts/base/frameworks/broker/__load__.bro b/scripts/base/frameworks/broker/__load__.bro new file mode 100644 index 0000000000..a10fe855df --- /dev/null +++ b/scripts/base/frameworks/broker/__load__.bro @@ -0,0 +1 @@ +@load ./main diff --git a/scripts/base/frameworks/broker/main.bro b/scripts/base/frameworks/broker/main.bro new file mode 100644 index 0000000000..e8b57d57d9 --- /dev/null +++ b/scripts/base/frameworks/broker/main.bro @@ -0,0 +1,103 @@ +##! Various data structure definitions for use with Bro's communication system. + +module BrokerComm; + +export { + + ## A name used to identify this endpoint to peers. + ## .. bro:see:: BrokerComm::connect BrokerComm::listen + const endpoint_name = "" &redef; + + ## Change communication behavior. + type EndpointFlags: record { + ## Whether to restrict message topics that can be published to peers. + auto_publish: bool &default = T; + ## Whether to restrict what message topics or data store identifiers + ## the local endpoint advertises to peers (e.g. subscribing to + ## events or making a master data store available). + auto_advertise: bool &default = T; + }; + + ## Fine-grained tuning of communication behavior for a particular message. + type SendFlags: record { + ## Send the message to the local endpoint. + self: bool &default = F; + ## Send the message to peer endpoints that advertise interest in + ## the topic associated with the message. + peers: bool &default = T; + ## Send the message to peer endpoints even if they don't advertise + ## interest in the topic associated with the message. + unsolicited: bool &default = F; + }; + + ## Opaque communication data. + type Data: record { + d: opaque of BrokerComm::Data &optional; + }; + + ## Opaque communication data. + type DataVector: vector of BrokerComm::Data; + + ## Opaque event communication data. + type EventArgs: record { + ## The name of the event. Not set if invalid event or arguments. + name: string &optional; + ## The arguments to the event. + args: DataVector; + }; + + ## Opaque communication data used as a convenient way to wrap key-value + ## pairs that comprise table entries. + type TableItem : record { + key: BrokerComm::Data; + val: BrokerComm::Data; + }; +} + +module BrokerStore; + +export { + + ## Whether a data store query could be completed or not. + type QueryStatus: enum { + SUCCESS, + FAILURE, + }; + + ## An expiry time for a key-value pair inserted in to a data store. + type ExpiryTime: record { + ## Absolute point in time at which to expire the entry. + absolute: time &optional; + ## A point in time relative to the last modification time at which + ## to expire the entry. New modifications will delay the expiration. + since_last_modification: interval &optional; + }; + + ## The result of a data store query. + type QueryResult: record { + ## Whether the query completed or not. + status: BrokerStore::QueryStatus; + ## The result of the query. Certain queries may use a particular + ## data type (e.g. querying store size always returns a count, but + ## a lookup may return various data types). + result: BrokerComm::Data; + }; + + ## Options to tune the SQLite storage backend. + type SQLiteOptions: record { + ## File system path of the database. + path: string &default = "store.sqlite"; + }; + + ## Options to tune the RocksDB storage backend. + type RocksDBOptions: record { + ## File system path of the database. + path: string &default = "store.rocksdb"; + }; + + ## Options to tune the particular storage backends. + type BackendOptions: record { + sqlite: SQLiteOptions &default = SQLiteOptions(); + rocksdb: RocksDBOptions &default = RocksDBOptions(); + }; +} diff --git a/scripts/base/frameworks/cluster/main.bro b/scripts/base/frameworks/cluster/main.bro index 12cc9e27d4..218e309bad 100644 --- a/scripts/base/frameworks/cluster/main.bro +++ b/scripts/base/frameworks/cluster/main.bro @@ -159,5 +159,5 @@ event bro_init() &priority=5 terminate(); } - Log::create_stream(Cluster::LOG, [$columns=Info]); + Log::create_stream(Cluster::LOG, [$columns=Info, $path="cluster"]); } diff --git a/scripts/base/frameworks/communication/main.bro b/scripts/base/frameworks/communication/main.bro index 92d527101d..af4eb9fca5 100644 --- a/scripts/base/frameworks/communication/main.bro +++ b/scripts/base/frameworks/communication/main.bro @@ -164,7 +164,7 @@ const src_names = { event bro_init() &priority=5 { - Log::create_stream(Communication::LOG, [$columns=Info]); + Log::create_stream(Communication::LOG, [$columns=Info, $path="communication"]); } function do_script_log_common(level: count, src: count, msg: string) diff --git a/scripts/base/frameworks/dpd/main.bro b/scripts/base/frameworks/dpd/main.bro index 9df8a45e5e..4586e2c02e 100644 --- a/scripts/base/frameworks/dpd/main.bro +++ b/scripts/base/frameworks/dpd/main.bro @@ -38,7 +38,7 @@ redef record connection += { event bro_init() &priority=5 { - Log::create_stream(DPD::LOG, [$columns=Info]); + Log::create_stream(DPD::LOG, [$columns=Info, $path="dpd"]); } event protocol_confirmation(c: connection, atype: Analyzer::Tag, aid: count) &priority=10 diff --git a/scripts/base/frameworks/files/magic/__load__.bro b/scripts/base/frameworks/files/magic/__load__.bro index c6ee799a53..34115f0a55 100644 --- a/scripts/base/frameworks/files/magic/__load__.bro +++ b/scripts/base/frameworks/files/magic/__load__.bro @@ -1,3 +1,9 @@ +@load-sigs ./archive +@load-sigs ./audio +@load-sigs ./font @load-sigs ./general +@load-sigs ./image @load-sigs ./msoffice -@load-sigs ./libmagic +@load-sigs ./video + +@load-sigs ./libmagic \ No newline at end of file diff --git a/scripts/base/frameworks/files/magic/archive.sig b/scripts/base/frameworks/files/magic/archive.sig new file mode 100644 index 0000000000..9b95f33b25 --- /dev/null +++ b/scripts/base/frameworks/files/magic/archive.sig @@ -0,0 +1,176 @@ + +signature file-tar { + file-magic /^[[:print:]\x00]{100}([[:digit:]\x20]{7}\x00){3}([[:digit:]\x20]{11}\x00){2}([[:digit:]\x00\x20]{7}[\x20\x00])[0-7\x00]/ + file-mime "application/x-tar", 100 +} + +# This is low priority so that files using zip as a +# container will be identified correctly. +signature file-zip { + file-mime "application/zip", 10 + file-magic /^PK\x03\x04.{2}/ +} + +# Multivolume Zip archive +signature file-multi-zip { + file-mime "application/zip", 10 + file-magic /^PK\x07\x08PK\x03\x04/ +} + +# RAR +signature file-rar { + file-mime "application/x-rar", 70 + file-magic /^Rar!/ +} + +# GZIP +signature file-gzip { + file-mime "application/x-gzip", 100 + file-magic /\x1f\x8b/ +} + +# Microsoft Cabinet +signature file-ms-cab { + file-mime "application/vnd.ms-cab-compressed", 110 + file-magic /^MSCF\x00\x00\x00\x00/ +} + +# Mac OS X DMG files +signature file-dmg { + file-magic /^(\x78\x01\x73\x0D\x62\x62\x60|\x78\xDA\x63\x60\x18\x05|\x78\x01\x63\x60\x18\x05|\x78\xDA\x73\x0D|\x78[\x01\xDA]\xED[\xD0-\xD9])/ + file-mime "application/x-dmg", 100 +} + +# XAR (eXtensible ARchive) format. +# Mac OS X uses this for the .pkg format. +signature file-xar { + file-magic /^xar\!/ + file-mime "application/x-xar", 100 +} + +# RPM +signature file-magic-auto352 { + file-mime "application/x-rpm", 70 + file-magic /^(drpm|\xed\xab\xee\xdb)/ +} + +# StuffIt +signature file-stuffit { + file-mime "application/x-stuffit", 70 + file-magic /^(SIT\x21|StuffIt)/ +} + +# Archived data +signature file-x-archive { + file-mime "application/x-archive", 70 + file-magic /^!?/ +} + +# ARC archive data +signature file-arc { + file-mime "application/x-arc", 70 + file-magic /^[\x00-\x7f]{2}[\x02-\x0a\x14\x48]\x1a/ +} + +# EET archive +signature file-eet { + file-mime "application/x-eet", 70 + file-magic /^\x1e\xe7\xff\x00/ +} + +# Zoo archive +signature file-zoo { + file-mime "application/x-zoo", 70 + file-magic /^.{20}\xdc\xa7\xc4\xfd/ +} + +# LZ4 compressed data (legacy format) +signature file-lz4-legacy { + file-mime "application/x-lz4", 70 + file-magic /(\x02\x21\x4c\x18)/ +} + +# LZ4 compressed data +signature file-lz4 { + file-mime "application/x-lz4", 70 + file-magic /^\x04\x22\x4d\x18/ +} + +# LRZIP compressed data +signature file-lrzip { + file-mime "application/x-lrzip", 1 + file-magic /^LRZI/ +} + +# LZIP compressed data +signature file-lzip { + file-mime "application/x-lzip", 70 + file-magic /^LZIP/ +} + +# Self-extracting PKZIP archive +signature file-magic-auto434 { + file-mime "application/zip", 340 + file-magic /^MZ.{28}(Copyright 1989\x2d1990 PKWARE Inc|PKLITE Copr)\x2e/ +} + +# LHA archive (LZH) +signature file-lzh { + file-mime "application/x-lzh", 80 + file-magic /^.{2}-(lh[ abcdex0-9]|lz[s2-8]|lz[s2-8]|pm[s012]|pc1)-/ +} + +# WARC Archive +signature file-warc { + file-mime "application/warc", 50 + file-magic /^WARC\x2f/ +} + +# 7-zip archive data +signature file-7zip { + file-mime "application/x-7z-compressed", 50 + file-magic /^7z\xbc\xaf\x27\x1c/ +} + +# XZ compressed data +signature file-xz { + file-mime "application/x-xz", 90 + file-magic /^\xfd7zXZ\x00/ +} + +# LHa self-extracting archive +signature file-magic-auto436 { + file-mime "application/x-lha", 120 + file-magic /^MZ.{34}LH[aA]\x27s SFX/ +} + +# ARJ archive data +signature file-arj { + file-mime "application/x-arj", 50 + file-magic /^\x60\xea/ +} + +# Byte-swapped cpio archive +signature file-bs-cpio { + file-mime "application/x-cpio", 50 + file-magic /(\x71\xc7|\xc7\x71)/ +} + +# CPIO archive +signature file-cpio { + file-mime "application/x-cpio", 50 + file-magic /^(\xc7\x71|\x71\xc7)/ +} + +# Compress'd data +signature file-compress { + file-mime "application/x-compress", 50 + file-magic /^\x1f\x9d/ +} + +# LZMA compressed data +signature file-lzma { + file-mime "application/x-lzma", 71 + file-magic /^\x5d\x00\x00/ +} + diff --git a/scripts/base/frameworks/files/magic/audio.sig b/scripts/base/frameworks/files/magic/audio.sig new file mode 100644 index 0000000000..efba99ed0d --- /dev/null +++ b/scripts/base/frameworks/files/magic/audio.sig @@ -0,0 +1,13 @@ + +# MPEG v3 audio +signature file-mpeg-audio { + file-mime "audio/mpeg", 20 + file-magic /^\xff[\xe2\xe3\xf2\xf3\xf6\xf7\xfa\xfb\xfc\xfd]/ +} + +# MPEG v4 audio +signature file-m4a { + file-mime "audio/m4a", 70 + file-magic /^....ftyp(m4a)/ +} + diff --git a/scripts/base/frameworks/files/magic/font.sig b/scripts/base/frameworks/files/magic/font.sig new file mode 100644 index 0000000000..8f2857f6e3 --- /dev/null +++ b/scripts/base/frameworks/files/magic/font.sig @@ -0,0 +1,41 @@ + +# Web Open Font Format +signature file-woff { + file-magic /^wOFF/ + file-mime "application/font-woff", 70 +} + +# TrueType font +signature file-ttf { + file-mime "application/x-font-ttf", 80 + file-magic /^\x00\x01\x00\x00\x00/ +} + +signature file-embedded-opentype { + file-mime "application/vnd.ms-fontobject", 50 + file-magic /^.{34}LP/ +} + +# X11 SNF font +signature file-snf { + file-mime "application/x-font-sfn", 70 + file-magic /^(\x04\x00\x00\x00|\x00\x00\x00\x04).{100}(\x04\x00\x00\x00|\x00\x00\x00\x04)/ +} + +# OpenType font +signature file-opentype { + file-mime "application/vnd.ms-opentype", 70 + file-magic /^OTTO/ +} + +# FrameMaker Font file +signature file-maker-screen-font { + file-mime "application/x-mif", 190 + file-magic /^\x3cMakerScreenFont/ +} + +# >0 string,=SplineFontDB: (len=13), ["Spline Font Database "], swap_endian=0 +signature file-spline-font-db { + file-mime "application/vnd.font-fontforge-sfd", 160 + file-magic /^SplineFontDB\x3a/ +} diff --git a/scripts/base/frameworks/files/magic/general.sig b/scripts/base/frameworks/files/magic/general.sig index 500c4f7be0..eb38d39c8c 100644 --- a/scripts/base/frameworks/files/magic/general.sig +++ b/scripts/base/frameworks/files/magic/general.sig @@ -1,18 +1,87 @@ # General purpose file magic signatures. +# Plaintext +# (Including BOMs for UTF-8, 16, and 32) signature file-plaintext { - file-magic /^([[:print:][:space:]]{10})/ - file-mime "text/plain", -20 + file-mime "text/plain", -20 + file-magic /^(\xef\xbb\xbf|(\x00\x00)?\xfe\xff|\xff\xfe(\x00\x00)?)?[[:space:]\x20-\x7E]{10}/ } -signature file-tar { - file-magic /^[[:print:]\x00]{100}([[:digit:]\x20]{7}\x00){3}([[:digit:]\x20]{11}\x00){2}([[:digit:]\x00\x20]{7}[\x20\x00])[0-7\x00]/ - file-mime "application/x-tar", 100 +signature file-json { + file-mime "text/json", 1 + file-magic /^(\xef\xbb\xbf)?[\x0d\x0a[:blank:]]*\{[\x0d\x0a[:blank:]]*(["][^"]{1,}["]|[a-zA-Z][a-zA-Z0-9\\_]*)[\x0d\x0a[:blank:]]*:[\x0d\x0a[:blank:]]*(["]|\[|\{|[0-9]|true|false)/ } -signature file-zip { - file-mime "application/zip", 10 - file-magic /^PK\x03\x04.{2}/ +signature file-json2 { + file-mime "text/json", 1 + file-magic /^(\xef\xbb\xbf)?[\x0d\x0a[:blank:]]*\[[\x0d\x0a[:blank:]]*(((["][^"]{1,}["]|[0-9]{1,}(\.[0-9]{1,})?|true|false)[\x0d\x0a[:blank:]]*,)|\{|\[)[\x0d\x0a[:blank:]]*/ +} + +# Match empty JSON documents. +signature file-json3 { + file-mime "text/json", 0 + file-magic /^(\xef\xbb\xbf)?[\x0d\x0a[:blank:]]*(\[\]|\{\})[\x0d\x0a[:blank:]]*$/ +} + +signature file-xml { + file-mime "application/xml", 10 + file-magic /^(\xef\xbb\xbf)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*<\?xml / +} + +signature file-xhtml { + file-mime "text/html", 100 + file-magic /^(\xef\xbb\xbf)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*<(![dD][oO][cC][tT][yY][pP][eE] {1,}[hH][tT][mM][lL]|[hH][tT][mM][lL]|[mM][eE][tT][aA] {1,}[hH][tT][tT][pP]-[eE][qQ][uU][iI][vV])/ +} + +signature file-html { + file-mime "text/html", 49 + file-magic /^(\xef\xbb\xbf)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*)?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*<([hH][eE][aA][dD]|[hH][tT][mM][lL]|[tT][iI][tT][lL][eE]|[bB][oO][dD][yY])/ +} + +signature file-rss { + file-mime "text/rss", 90 + file-magic /^(\xef\xbb\xbf)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*<[rR][sS][sS]/ +} + +signature file-atom { + file-mime "text/atom", 100 + file-magic /^(\xef\xbb\xbf)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*<([rR][sS][sS][^>]*xmlns:atom|[fF][eE][eE][dD][^>]*xmlns=["']?http:\/\/www.w3.org\/2005\/Atom["']?)/ +} + +signature file-soap { + file-mime "application/soap+xml", 49 + file-magic /^(\xef\xbb\xbf)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*<[sS][oO][aA][pP](-[eE][nN][vV])?:[eE][nN][vV][eE][lL][oO][pP][eE]/ +} + +signature file-cross-domain-policy { + file-mime "text/x-cross-domain-policy", 49 + file-magic /^([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*)?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*<[cC][rR][oO][sS][sS]-[dD][oO][mM][aA][iI][nN]-[pP][oO][lL][iI][cC][yY]/ +} + +signature file-xmlrpc { + file-mime "application/xml-rpc", 49 + file-magic /^(\xef\xbb\xbf)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*()?[\x0d\x0a[:blank:]]*)*<[mM][eE][tT][hH][oO][dD][rR][eE][sS][pP][oO][nN][sS][eE]>/ +} + +signature file-coldfusion { + file-mime "magnus-internal/cold-fusion", 20 + file-magic /^([\x0d\x0a[:blank:]]*()?)*<(CFPARAM|CFSET|CFIF)/ +} + +# Microsoft LNK files +signature file-lnk { + file-mime "application/x-ms-shortcut", 49 + file-magic /^\x4C\x00\x00\x00\x01\x14\x02\x00\x00\x00\x00\x00\xC0\x00\x00\x00\x00\x10\x00\x00\x00\x46/ } signature file-jar { @@ -21,8 +90,20 @@ signature file-jar { } signature file-java-applet { - file-magic /^\xca\xfe\xba\xbe...[\x2e-\x34]/ file-mime "application/x-java-applet", 71 + file-magic /^\xca\xfe\xba\xbe...[\x2d-\x34]/ +} + +# OCSP requests over HTTP. +signature file-ocsp-request { + file-magic /^.{11,19}\x06\x05\x2b\x0e\x03\x02\x1a/ + file-mime "application/ocsp-request", 71 +} + +# OCSP responses over HTTP. +signature file-ocsp-response { + file-magic /^.{11,19}\x06\x09\x2B\x06\x01\x05\x05\x07\x30\x01\x01/ + file-mime "application/ocsp-response", 71 } # Shockwave flash @@ -37,12 +118,6 @@ signature file-tnef { file-mime "application/vnd.ms-tnef", 100 } -# Mac OS X DMG files -signature file-dmg { - file-magic /^(\x78\x01\x73\x0D\x62\x62\x60|\x78\xDA\x63\x60\x18\x05|\x78\x01\x63\x60\x18\x05|\x78\xDA\x73\x0D|\x78[\x01\xDA]\xED[\xD0-\xD9])/ - file-mime "application/x-dmg", 100 -} - # Mac OS X Mach-O executable signature file-mach-o { file-magic /^[\xce\xcf]\xfa\xed\xfe/ @@ -55,13 +130,6 @@ signature file-mach-o-universal { file-mime "application/x-mach-o-executable", 100 } -# XAR (eXtensible ARchive) format. -# Mac OS X uses this for the .pkg format. -signature file-xar { - file-magic /^xar\!/ - file-mime "application/x-xar", 100 -} - signature file-pkcs7 { file-magic /^MIME-Version:.*protocol=\"application\/pkcs7-signature\"/ file-mime "application/pkcs7-signature", 100 @@ -79,16 +147,6 @@ signature file-jnlp { file-mime "application/x-java-jnlp-file", 100 } -signature file-ico { - file-magic /^\x00\x00\x01\x00/ - file-mime "image/x-icon", 70 -} - -signature file-cur { - file-magic /^\x00\x00\x02\x00/ - file-mime "image/x-cursor", 70 -} - signature file-pcap { file-magic /^(\xa1\xb2\xc3\xd4|\xd4\xc3\xb2\xa1)/ file-mime "application/vnd.tcpdump.pcap", 70 @@ -119,7 +177,58 @@ signature file-python { file-mime "text/x-python", 60 } +signature file-awk { + file-mime "text/x-awk", 60 + file-magic /^\x23\x21[^\n]{1,15}bin\/(env[[:space:]]+)?(g|n)?awk/ +} + +signature file-tcl { + file-mime "text/x-tcl", 60 + file-magic /^\x23\x21[^\n]{1,15}bin\/(env[[:space:]]+)?(wish|tcl)/ +} + +signature file-lua { + file-mime "text/x-lua", 49 + file-magic /^\x23\x21[^\n]{1,15}bin\/(env[[:space:]]+)?lua/ +} + +signature file-javascript { + file-mime "application/javascript", 60 + file-magic /^\x23\x21[^\n]{1,15}bin\/(env[[:space:]]+)?node(js)?/ +} + +signature file-javascript2 { + file-mime "application/javascript", 60 + file-magic /^[\x0d\x0a[:blank:]]*<[sS][cC][rR][iI][pP][tT][[:blank:]]+([tT][yY][pP][eE]|[lL][aA][nN][gG][uU][aA][gG][eE])=['"]?([tT][eE][xX][tT]\/)?[jJ][aA][vV][aA][sS][cC][rR][iI][pP][tT]/ +} + +signature file-javascript3 { + file-mime "application/javascript", 60 + # This seems to be a somewhat common idiom in javascript. + file-magic /^[\x0d\x0a[:blank:]]*for \(;;\);/ +} + +signature file-javascript4 { + file-mime "application/javascript", 60 + file-magic /^[\x0d\x0a[:blank:]]*document\.write(ln)?[:blank:]?\(/ +} + +signature file-javascript5 { + file-mime "application/javascript", 60 + file-magic /^\(function\(\)[[:blank:]\n]*\{/ +} + +signature file-javascript6 { + file-mime "application/javascript", 60 + file-magic /^[\x0d\x0a[:blank:]]*