Merge remote-tracking branch 'origin/master' into topic/vladg/file-analysis-exe-analyzer

Conflicts:
	src/types.bif

.gitmodules

@@ -19,3 +19,6 @@
 [submodule "src/3rdparty"]
 	path = src/3rdparty
 	url = git://git.bro.org/bro-3rdparty
+[submodule "aux/plugins"]
+	path = aux/plugins
+	url = git://git.bro.org/bro-plugins

CHANGES

@@ -1,4 +1,678 @@
 
+2.3-397 | 2015-01-27 10:13:10 -0600
+
+  * Handle guess_lexer exceptions in pygments reST directive (Jon Siwek)
+
+2.3-396 | 2015-01-23 10:49:15 -0600
+
+  * DNP3: fix reachable assertion and buffer over-read/overflow.
+    CVE number pending. (Travis Emmert, Jon Siwek)
+
+  * Update binpac: Fix potential out-of-bounds memory reads in generated
+    code. CVE-2014-9586. (John Villamil and Chris Rohlf - Yahoo
+    Paranoids, Jon Siwek)
+
+  * Fixing (harmless) Coverity warning. (Robin Sommer)
+
+2.3-392 | 2015-01-15 09:44:15 -0800
+
+  * Small changes to EC curve names in a newer draft. (Johanna Amann)
+
+2.3-390 | 2015-01-14 13:27:34 -0800
+
+  * Updating MySQL analyses. (Vlad Grigorescu)
+     - Use a boolean success instead of a result string.
+     - Change the affected_rows response detail string to a "rows" count.
+     - Fix the state tracking to log incomplete command.
+
+  * Extend DNP3 to support communication over UDP. (Hui Lin)
+
+  * Fix a bug in DNP3 determining the length of an object in some
+    cases. (Hui Lin)
+
+2.3-376 | 2015-01-12 09:38:10 -0600
+
+  * Improve documentation for connection_established event. (Jon Siwek)
+
+2.3-375 | 2015-01-08 13:10:09 -0600
+
+  * Increase minimum required CMake version to 2.8. (Jon Siwek)
+
+2.3-374 | 2015-01-07 10:03:17 -0600
+
+  * Improve documentation of the Intelligence Framework. (Daniel Thayer)
+
+2.3-371 | 2015-01-06 09:58:09 -0600
+
+  * Update/improve file mime type identification. (Seth Hall)
+
+     - Change to the default BOF buffer size to 3000 (was 1024).
+
+     - Reorganized MS signatures into a separate file.
+
+     - Remove all of the x-c detections.  Nearly all false positives.
+
+     - Improve TAR detections, removing old, back up TAR detections.
+
+     - Remove one of the x-elc detections that was too loose
+       and caused many false positives.
+
+     - Improved lots of the signatures and added new ones. (Seth Hall)
+
+  * Add support for file reassembly in the file analysis framework
+    (Seth Hall, Jon Siwek).
+
+     - The reassembly behavior can be modified per-file by enabling or
+       disabling the reassembler and/or modifying the size of the
+       reassembly buffer.
+
+     - Changed the file extraction analyzer to use stream-wise input to
+       avoid issues with the chunk-wise approach not immediately
+       triggering the file_new event due to mime-type detection delay.
+       Before, early chunks frequently ended up lost.  Extraction also
+       will now explicitly NUL-fill gaps in the file instead of
+       implicitly relying on pwrite to do it.
+
+2.3-349 | 2015-01-05 15:21:13 -0600
+
+  * Fix race condition in unified2 file analyzer startup. (Jon siwek)
+
+2.3-348 | 2014-12-31 09:19:34 -0800
+
+  * Changing Makefile's test-all to run test-all for broctl, which now
+    executes trace-summary tests as well. (Robin Sommer)
+
+2.3-345 | 2014-12-31 09:06:15 -0800
+
+  * Correct a typo in the Notice framework doc. (Daniel Thayer)
+
+2.3-343 | 2014-12-12 12:43:46 -0800
+
+  * Fix PIA packet replay to deliver copy of IP header. This prevented
+    one from writing a packet-wise analyzer that needs access to IP
+    headers and can be attached to a connection via signature match.
+    Addresses BIT-1298 (Jon Siwek)
+
+2.3-338 | 2014-12-08 13:56:19 -0800
+
+  * Add man page for Bro. (Raúl Benencia)
+
+  * Updating doc baselines. (Robin Sommer)
+
+2.3-334 | 2014-12-03 14:22:07 -0800
+
+  * Fix compound assignment to require proper L-value. Addresses
+    BIT-1295. (Jon Siwek)
+
+2.3-332 | 2014-12-03 14:14:11 -0800
+
+  * Make using local IDs in @if directives an error. Addresses
+    BIT-1296. (Jon Siwek)
+
+2.3-330 | 2014-12-03 14:10:39 -0800
+
+  * Fix some "make doc" warnings and update some doc tests. (Daniel
+    Thayer)
+
+2.3-328 | 2014-12-02 08:13:10 -0500
+
+  * Update windows-version-detection.bro to add support for 
+    Windows 10. (Michal Purzynski)
+
+2.3-326 | 2014-12-01 12:10:27 -0600
+
+  * BIFScanner: fix invalid characters in generated preprocessor macros.
+    (Hilko Bengen)
+
+  * BIT-1294: fix exec.bro from mutating Input::end_of_data event
+    parameters. (Johanna Amann)
+
+  * Add/invoke "distclean" for testing directories. (Raúl Benencia)
+
+  * Delete prebuilt python bytecode files from git. (Jon Siwek)
+
+  * Add Windows detection based on CryptoAPI HTTP traffic as a software
+    framework policy script. (Vlad Grigorescu)
+
+2.3-316 | 2014-11-25 17:35:06 -0800
+
+  * Make the SSL analyzer skip further processing once encountering
+    situations which are very probably non-recoverable. (Johanna
+    Amann)
+
+2.3-313 | 2014-11-25 14:27:07 -0800
+
+  * Make SSL v2 protocol tests more strict. In its former state they
+    triggered on http traffic over port 443 sometimes. Found by Michał
+    Purzyński. (Johanna Amann)
+
+  * Fix X509 analyzer to correctly return ECDSA as the key_type for
+    ECDSA certs. Bug found by Michał Purzyński. (Johanna Amann)
+
+2.3-310 | 2014-11-19 10:56:59 -0600
+
+  * Disable verbose bison output. (Jon Siwek)
+
+2.3-309 | 2014-11-18 12:17:53 -0800
+
+  * New decompose_uri() function in base/utils/urls that splits a URI
+    into its pieces. (Anthony Kasza).
+
+2.3-305 | 2014-11-18 11:09:04 -0800
+
+  * Improve coercion of &default expressions. Addresses BIT-1288. (Jon
+    Siwek)
+
+2.3-303 | 2014-11-18 10:53:04 -0800
+
+  * For DH key exchanges, use p as the parameter for weak key
+    exchanges. (Johanna Amann)
+
+2.3-301 | 2014-11-11 13:47:27 -0800
+
+  * Add builtin function enum_to_int() that converts an enum into a
+    integer. (Christian Struck)
+
+2.3-297 | 2014-11-11 11:50:47 -0800
+
+  * Removing method from SSL analyzer that's no longer used. (Robin
+    Sommer)
+
+2.3-296 | 2014-11-11 11:42:38 -0800
+
+  * A new analyzer parsing the MySQL wire protocol. Activity gets
+    logged into mysql.log. Supports protocol versions 9 and 10. (Vlad
+    Grigorescu)
+
+2.3-280 | 2014-11-05 09:46:33 -0500
+
+  * Add Windows detection based on CryptoAPI HTTP traffic as a 
+    software framework policy script. (Vlad Grigorescu)
+
+2.3-278 | 2014-11-03 18:55:18 -0800
+
+  * Add new curves from draft-ietf-tls-negotiated-ff-dhe to SSL
+    analysis. (Johanna Amann)
+
+2.3-274 | 2014-10-31 17:45:25 -0700
+
+  * Adding call to new binpac::init() function. (Robin Sommer)
+
+2.3-272 | 2014-10-31 16:29:42 -0700
+
+  * Fix segfault if when statement's RHS is unitialized. Addresses
+    BIT-1176. (Jon Siwek)
+
+  * Fix checking vector indices via "in". Addresses BIT-1280.  (Jon
+    Siwek)
+
+2.3-268 | 2014-10-31 12:12:22 -0500
+
+  * BIT-1283: Fix crash when using &encrypt. (Jon Siwek)
+
+2.3-267 | 2014-10-31 10:35:02 -0500
+
+  * BIT-1284: Allow arbitrary when statement timeout expressions
+    (Jon Siwek)
+
+2.3-266 | 2014-10-31 09:21:28 -0500
+
+  * BIT-1166: Add configure options to fine tune local state dirs used
+    by BroControl. (Jon Siwek)
+
+2.3-264 | 2014-10-30 13:25:57 -0500
+
+  * Fix some minor Coverity Scan complaints. (Jon Siwek)
+
+2.3-263 | 2014-10-28 15:09:10 -0500
+
+  * Fix checking of fwrite return values (Johanna Amann)
+
+2.3-260 | 2014-10-27 12:54:17 -0500
+
+  * Fix errors/warnings when compiling with -std=c++11 (Jon Siwek)
+
+2.3-259 | 2014-10-27 10:04:04 -0500
+
+  * Documentation fixes. (Vicente Jimenez Aguilar and Stefano Azzalini)
+
+2.3-256 | 2014-10-24 15:33:45 -0700
+
+  * Adding missing test baseline. (Robin Sommer)
+
+2.3-255 | 2014-10-24 13:39:44 -0700
+
+  * Fixing unstable active-http test. (Robin Sommer)
+
+2.3-254 | 2014-10-24 11:40:51 -0700
+
+  * Fix active-http.bro to deal reliably with empty server responses,
+    which will now be passed back as empty files. (Christian Struck)
+
+2.3-248 | 2014-10-23 14:20:59 -0700
+
+  * Change order in which a plugin's scripts are loaded at startup.
+    (Robin Sommer)
+
+2.3-247 | 2014-10-21 13:42:38 -0700
+
+  * Updates to the SSL analyzer. (Johanna Amann)
+
+        * Mark everything below 2048 bit as a weak key.
+
+        * Fix notice suppression.
+
+        * Add information about server-chosen protocol to ssl.log, if
+          provided by application_layer_next_protocol.
+
+        * Add boolean flag to ssl.log signaling if a session was
+          resumed. Remove the (usually not really that useful) session
+          ID that the client sent.
+
+2.3-240 | 2014-10-21 13:36:33 -0700
+
+  * Fix Coverity-reported issues in DNP3 analyzer. (Seth Hall)
+
+2.3-238 | 2014-10-16 06:51:49 -0700
+
+  * Fix multipart HTTP/MIME entity file analysis so that (1) singular
+    CR or LF characters in multipart body content are no longer
+    converted to a full CRLF (thus corrupting the file) and (2) it
+    also no longer considers the CRLF before the multipart boundary as
+    part of the content. Addresses BIT-1235. (Jon Siwek)
+
+2.3-235 | 2014-10-15 10:20:47 -0500
+
+  * BIT-1273: Add error message for bad enum declaration syntax.
+    (Jon Siwek)
+
+2.3-234 | 2014-10-14 14:42:09 -0500
+
+  * Documentation fixes. (Steve Smoot)
+
+2.3-233 | 2014-10-09 16:00:27 -0500
+
+  * Change find-bro-logs unit test to follow symlinks. (Jon Siwek)
+
+  * Add error checks and messages to a test script (Daniel Thayer)
+
+2.3-230 | 2014-10-08 08:15:17 -0700
+
+  * Further baseline normalization for plugin test portability. (Robin
+    Sommer)
+
+2.3-229 | 2014-10-07 20:18:11 -0700
+
+  * Fix for test portability. (Robin Sommer)
+
+2.3-228 | 2014-10-07 15:32:37 -0700
+
+  * Include plugin unit tests into the top-level btest configuration. (Robin Sommer)
+
+  * Switching the prefix separator for packet source/dumper plugins
+    once more, now to "::". Addresses BIT-1267. (Robin Sommer)
+
+  * Fix for allowing a packet source/dumper plugin to support multiple
+    prefixes with a colon. (Robin Sommer)
+
+2.3-225 | 2014-10-07 15:13:35 -0700
+
+  * Updating plugin documentation. (Robin Sommer)
+
+2.3-224 | 2014-10-07 14:32:17 -0700
+
+  * Improved the log file reference documentation. (Jeannette Dopheide
+    and Daniel Thayer)
+
+  * Improves shockwave flash file signatures. (Seth Hall)
+
+     - This moves the signatures out of the libmagic imported signatures
+       and into our own general.sig.
+
+     - Expand the detection to LZMA compressed flash files.
+
+  * Add new script language reference documentation on operators,
+    statements, and directives.  Also improved the documentation on
+    types and attributes by splitting them into two docs, and
+    providing more examples and adding a chart on the top of each page
+    with links to each type and attribute for easier access to the
+    information. (Daniel Thayer)
+
+  * Split the types and attributes reference doc into two docs.
+    (Daniel Thayer)
+
+2.3-208 | 2014-10-03 09:38:52 -0500
+
+  * BIT-1268: Fix uninitialized router_list argument in
+    dhcp_offer/dhcp_ack. (Jon Siwek)
+
+2.3-207 | 2014-10-02 16:39:17 -0700
+
+  * Updating plugin docs. (Robin Sommer)
+
+  * Fix packet sources being treated as idle when a packet is
+    available. Addresses BIT-1266. (Jon Siwek)
+
+  * Fix regression causing the main loop to spin more frequently.
+    Addresses BIT-1266. (Jon Siwek)
+
+2.3-203 | 2014-09-29 20:06:54 -0700
+
+  * Fix to use length parameter in DNP3 time conversion correctly now.
+    (Robin Sommer)
+
+2.3-202 | 2014-09-29 17:05:18 -0700
+
+  * New SSL extension type from IANA and a few other SSL const
+    changes. (Johanna Amann)
+
+  * Make unexpected pipe errors fatal as precaution. Addresses
+    BIT-1260. (Jon Siwek)
+
+  * Adding a function for DNP3 to translate the timestamp format. (Hui
+    Lin)
+
+2.3-197 | 2014-09-29 10:42:01 -0500
+
+  * Fix possible seg fault in TCP reassembler. (Jon Siwek)
+
+2.3-196 | 2014-09-25 17:53:27 -0700
+
+  * Changing prefix for packet sources/dumper from ':' to '%'.
+    Addresses BIT-1249. (Robin Sommer)
+
+  * Remove timeouts from remote communication loop. The select() now
+    blocks until there's work to do instead of relying on a small
+    timeout value which can cause unproductive use of cpu cycles. (Jon
+    Siwek)
+
+  * Improve error message when failing to activate a plugin. Also fix
+    a unit test helper script that checks plugin availability. (Jon
+    Siwek)
+
+2.3-183 | 2014-09-24 10:08:04 -0500
+
+  * Add a "node" field to Intel::Seen struture and intel.log to
+    indicate which node discovered a hit on an intel item. (Seth Hall)
+
+  * BIT-1261: Fixes to plugin quick start doc. (Jon Siwek)
+
+2.3-180 | 2014-09-22 12:52:41 -0500
+
+  * BIT-1259: Fix issue w/ duplicate TCP reassembly deliveries.
+    (Jon Siwek)
+
+2.3-178 | 2014-09-18 14:29:46 -0500
+
+  * BIT-1256: Fix file analysis events from coming after bro_done().
+    (Jon Siwek)
+
+2.3-177 | 2014-09-17 09:41:27 -0500
+
+  * Documentation fixes. (Chris Mavrakis)
+
+2.3-174 | 2014-09-17 09:37:09 -0500
+
+  * Fixed some "make doc" warnings caused by reST formatting
+    (Daniel Thayer).
+
+2.3-172 | 2014-09-15 13:38:52 -0500
+
+  * Remove unneeded allocations for HTTP messages. (Jon Siwek)
+
+2.3-171 | 2014-09-15 11:14:57 -0500
+
+  * Fix a compile error on systems without pcap-int.h. (Jon Siwek)
+
+2.3-170 | 2014-09-12 19:28:01 -0700
+
+  * Fix incorrect data delivery skips after gap in HTTP Content-Range.
+    Addresses BIT-1247. (Jon Siwek)
+
+  * Fix file analysis placement of data after gap in HTTP
+    Content-Range. Addresses BIT-1248. (Jon Siwek)
+
+  * Fix issue w/ TCP reassembler not delivering some segments.
+    Addresses BIT-1246. (Jon Siwek)
+
+  * Fix MIME entity file data/gap ordering and raise http_entity_data
+    in line with data arrival. Addresses BIT-1240. (Jon Siwek)
+
+  * Implement file ID caching for MIME_Mail. (Jon Siwek)
+
+  * Fix a compile error. (Jon Siwek)
+
+2.3-161 | 2014-09-09 12:35:38 -0500
+
+  * Bugfixes and test updates/additions. (Robin Sommer)
+
+  * Interface tweaks and docs for PktSrc/PktDumper. (Robin Sommer)
+
+  * Moving PCAP-related bifs to iosource/pcap.bif. (Robin Sommer)
+
+  * Moving some of the BPF filtering code into base class.
+    This will allow packet sources that don't support BPF natively to
+    emulate the filtering via libpcap. (Robin Sommer)
+
+  * Removing FlowSrc. (Robin Sommer)
+
+  * Removing remaining pieces of the 2ndary path, and left-over
+    files of packet sorter. (Robin Sommer)
+
+  * A bunch of infrastructure work to move IOSource, IOSourceRegistry
+    (now iosource::Manager) and PktSrc/PktDumper code into iosource/,
+    and over to a plugin structure. (Robin Sommer)
+
+2.3-137 | 2014-09-08 19:01:13 -0500
+
+  * Fix Broxygen's rendering of opaque types. (Jon Siwek)
+
+2.3-136 | 2014-09-07 20:50:46 -0700
+
+  * Change more http links to https. (Johanna Amann)
+
+2.3-134 | 2014-09-04 16:16:36 -0700
+
+  * Fixed a number of issues with OCSP reply validation. Addresses
+    BIT-1212. (Johanna Amann)
+
+  * Fix null pointer dereference in OCSP verification code in case no
+    certificate is sent as part as the ocsp reply. Addresses BIT-1212.
+    (Johanna Amann)
+
+2.3-131 | 2014-09-04 16:10:32 -0700
+
+  * Make links in documentation templates protocol relative. (Johanna
+    Amann)
+
+2.3-129 | 2014-09-02 17:21:21 -0700
+
+  * Simplify a conditional with equivalent branches. (Jon Siwek)
+
+  * Change EDNS parsing code to use rdlength more cautiously. (Jon
+    Siwek)
+
+  * Fix a memory leak when bind() fails due to EADDRINUSE. (Jon Siwek)
+
+  * Fix possible buffer over-read in DNS TSIG parsing. (Jon Siwek)
+
+2.3-124 | 2014-08-26 09:24:19 -0500
+
+  * Better documentation for sub_bytes (Jimmy Jones)
+
+  * BIT-1234: Fix build on systems that already have ntohll/htonll
+    (Jon Siwek)
+
+2.3-121 | 2014-08-22 15:22:15 -0700
+
+  * Detect functions that try to bind variables from an outer scope
+    and raise an error saying that's not supported. Addresses
+    BIT-1233. (Jon Siwek)
+
+2.3-116 | 2014-08-21 16:04:13 -0500
+
+  * Adding plugin testing to Makefile's test-all. (Robin Sommer)
+
+  * Converting log writers and input readers to plugins.
+    DataSeries and ElasticSearch plugins have moved to the new
+    bro-plugins repository, which is now a git submodule in the
+    aux/plugins directory. (Robin Sommer)
+
+2.3-98 | 2014-08-19 11:03:46 -0500
+
+  * Silence some doc-related warnings when using `bro -e`.
+    Closes BIT-1232. (Jon Siwek)
+
+  * Fix possible null ptr derefs reported by Coverity. (Jon Siwek)
+
+2.3-96 | 2014-08-01 14:35:01 -0700
+
+  * Small change to DHCP documentation. In server->client messages the
+    host name may differ from the one requested by the client.
+    (Johanna Amann)
+
+  * Split DHCP log writing from record creation. This allows users to
+    customize dhcp.log by changing the record in their own dhcp_ack
+    event. (Johanna Amann)
+
+  * Update PATH so that documentation btests can find bro-cut. (Daniel
+    Thayer)
+
+  * Remove gawk from list of optional packages in documentation.
+    (Daniel Thayer)
+
+  * Fix for redefining built-in constants. (Robin Sommer)
+
+2.3-86 | 2014-07-31 14:19:58 -0700
+
+  * Fix for redefining built-in constants. (Robin Sommer)
+
+  * Adding missing check that a plugin's API version matches what Bro
+    defines. (Robin Sommer)
+
+  * Adding NEWS entry for plugins. (Robin Sommer)
+
+2.3-83 | 2014-07-30 16:26:11 -0500
+
+  * Minor adjustments to plugin code/docs. (Jon Siwek)
+
+  * Dynamic plugin support. (Rpbin Sommer)
+
+    Bro now supports extending core functionality, like protocol and
+    file analysis, dynamically with external plugins in the form of
+    shared libraries. See doc/devel/plugins.rst for an overview of the
+    main functionality. Changes coming with this:
+
+        - Replacing the old Plugin macro magic with a new API.
+
+        - The plugin API changed to generally use std::strings instead
+          of const char*.
+
+        - There are a number of invocations of PLUGIN_HOOK_
+          {VOID,WITH_RESULT} across the code base, which allow plugins
+          to hook into the processing at those locations.
+
+        - A few new accessor methods to various classes to allow
+          plugins to get to that information.
+
+        - network_time cannot be just assigned to anymore, there's now
+          function net_update_time() for that.
+
+        - Redoing how builtin variables are initialized, so that it
+          works for plugins as well. No more init_net_var(), but
+          instead bifcl-generated code that registers them.
+
+        - Various changes for adjusting to the now dynamic generation
+          of analyzer instances.
+
+    - same_type() gets an optional extra argument allowing record type
+      comparision to ignore if field names don't match. (Robin Sommer)
+
+    - Further unify file analysis API with the protocol analyzer API
+      (assigning IDs to analyzers; adding Init()/Done() methods;
+      adding subtypes). (Robin Sommer)
+
+    - A new command line option -Q that prints some basic execution
+      time stats. (Robin Sommer)
+
+    - Add support to the file analysis for activating analyzers by
+      MIME type. (Robin Sommer)
+
+          - File::register_for_mime_type(tag: Analyzer::Tag, mt:
+            string): Associates a file analyzer with a MIME type.
+
+          - File::add_analyzers_for_mime_type(f: fa_file, mtype:
+            string): Activates all analyzers registered for a MIME
+            type for the file.
+
+          - The default file_new() handler calls
+            File::add_analyzers_for_mime_type() with the file's MIME
+            type.
+
+2.3-20 | 2014-07-22 17:41:02 -0700
+
+  * Updating submodule(s).
+
+2.3-19 | 2014-07-22 17:29:19 -0700
+
+  * Implement bytestring_to_coils() in Modbus analyzer so that coils
+    gets passed to the corresponding events. (Hui Lin)
+
+  * Add length field to ModbusHeaders. (Hui Lin)
+
+2.3-12 | 2014-07-10 19:17:37 -0500
+
+  * Include yield of vectors in Broxygen's type descriptions.
+    Addresses BIT-1217. (Jon Siwek)
+
+2.3-11 | 2014-07-10 14:49:27 -0700
+
+  * Fixing DataSeries output. It was using a now illegal value as its
+    default compression level. (Robin Sommer)
+
+2.3-7 | 2014-06-26 17:35:18 -0700
+
+  * Extending "make test-all" to include aux/bro-aux. (Robin Sommer)
+
+2.3-6 | 2014-06-26 17:24:10 -0700
+
+  * DataSeries compilation issue fixed. (mlaterman)
+
+  * Fix a reference counting bug in ListVal ctor. (Jon Siwek)
+
+2.3-3 | 2014-06-26 15:41:04 -0500
+
+  * Support tilde expansion when Bro tries to find its own path. (Jon
+    Siwek)
+
+2.3-2 | 2014-06-23 16:54:15 -0500
+
+  * Remove references to line numbers in tutorial text. (Daniel Thayer)
+
+2.3 | 2014-06-16 09:48:25 -0500
+
+  * Release 2.3.
+
+2.3-beta-33 | 2014-06-12 11:59:28 -0500
+
+  * Documentation improvements/fixes. (Daniel Thayer)
+
+2.3-beta-24 | 2014-06-11 15:35:31 -0500
+
+  * Fix SMTP state tracking when server response is missing.
+    (Robin Sommer)
+
+2.3-beta-22 | 2014-06-11 12:31:38 -0500
+
+  * Fix doc/test that broke due to a Bro script change. (Jon Siwek)
+
+  * Remove unused --with-libmagic configure option. (Jon Siwek)
+
+2.3-beta-20 | 2014-06-10 18:16:51 -0700
+
+  * Fix use-after-free in some cases of reassigning a table index.
+    Addresses BIT-1202. (Jon Siwek)
+
 2.3-beta-18 | 2014-06-06 13:11:50 -0700
 
   * Add two more SSL events, one triggered for each handshake message
@@ -82,7 +756,7 @@
     and the load balancing FAQ on the website. (Daniel Thayer)
 
   * Update some doc tests and line numbers (Daniel Thayer)
-    
+
 2.2-457 | 2014-05-16 14:38:31 -0700
 
   * New script policy/protocols/ssl/validate-ocsp.bro that adds OSCP
@@ -97,7 +771,7 @@
     Amann)
 
   * Improved Heartbleed attack/scan detection. (Bernhard Amann)
-    
+
   * Let TLS analyzer fail better when no longer in sync with the data
     stream. (Bernhard Amann)
 
@@ -134,12 +808,12 @@
     *Undelivered methods now use a uint64 in place of an int for the
     relative sequence space offset parameter.
 
-    Addresses BIT-348. 
+    Addresses BIT-348.
 
   * Fixing compiler warnings. (Robin Sommer)
-    
+
   * Update SNMP analyzer's DeliverPacket method signature. (Jon Siwek)
-    
+
 2.2-417 | 2014-05-07 10:59:22 -0500
 
   * Change handling of atypical OpenSSL error case in x509 verification. (Jon Siwek)
@@ -179,7 +853,7 @@
 2.2-397 | 2014-05-01 20:29:20 -0700
 
   * Fix reference counting for lookup_ID() usages. (Jon Siwek)
-    
+
 2.2-395 | 2014-05-01 20:25:48 -0700
 
   * Fix missing "irc-dcc-data" service field from IRC DCC connections.
@@ -192,7 +866,7 @@
     Siwek)
 
   * Improve file analysis manager shutdown/cleanup. (Jon Siwek)
-    
+
 2.2-388 | 2014-04-24 18:38:07 -0700
 
   * Fix decoding of MIME quoted-printable. (Mareq)
@@ -205,11 +879,11 @@
 2.2-381 | 2014-04-24 17:08:45 -0700
 
   * Add Java version to software framework. (Brian Little)
-    
+
 2.2-379 | 2014-04-24 17:06:21 -0700
 
   * Remove unused Val::attribs member. (Jon Siwek)
-    
+
 2.2-377 | 2014-04-24 16:57:54 -0700
 
   * A larger set of SSL improvements and extensions. Addresses
@@ -222,7 +896,7 @@
           server_name, alpn, and ec-curves.
 
         - Adds support for the heartbeat events.
-          
+
         - Add Heartbleed detector script.
 
         - Adds basic support for OCSP stapling.
@@ -233,7 +907,7 @@
 2.2-353 | 2014-04-24 16:12:30 -0700
 
   * Adapt HTTP partial content to cache file analysis IDs. (Jon Siwek)
-    
+
   * Adapt SSL analyzer to generate file analysis handles itself. (Jon
     Siwek)
 
@@ -249,11 +923,11 @@
     Siwek)
 
   * Refactor file analysis file ID lookup. (Jon Siwek)
-    
+
 2.2-344 | 2014-04-22 20:13:30 -0700
 
   * Refactor various hex escaping code. (Jon Siwek)
-    
+
 2.2-341 | 2014-04-17 18:01:41 -0500
 
   * Fix duplicate DNS log entries. (Robin Sommer)
@@ -261,9 +935,9 @@
 2.2-341 | 2014-04-17 18:01:01 -0500
 
   * Refactor initialization of ASCII log writer options. (Jon Siwek)
-    
+
   * Fix a memory leak in ASCII log writer. (Jon Siwek)
-    
+
 2.2-338 | 2014-04-17 17:48:17 -0500
 
   * Disable input/logging threads setting their names on every
@@ -275,7 +949,7 @@
 2.2-335 | 2014-04-10 15:04:57 -0700
 
   * Small logic fix for main SSL script. (Bernhard Amann)
-    
+
   * Update DPD signatures for detecting TLS 1.2. (Bernhard Amann)
 
   * Remove unused data member of SMTP_Analyzer to silence a Coverity
@@ -299,7 +973,7 @@
 
 2.2-317 | 2014-04-03 10:51:31 -0400
 
-  * Add a uid field to the signatures.log.  Addresses BIT-1171 
+  * Add a uid field to the signatures.log.  Addresses BIT-1171
     (Anthony Verez)
 
 2.2-315 | 2014-04-01 16:50:01 -0700
@@ -310,7 +984,7 @@
 2.2-313 | 2014-04-01 16:40:19 -0700
 
   * Fix a couple nits reported by Coverity.(Jon Siwek)
-    
+
   * Fix potential memory leak in IP frag reassembly reported by
     Coverity. (Jon Siwek)
 
@@ -400,14 +1074,14 @@
 2.2-271 | 2014-03-30 20:25:17 +0200
 
   * Add unit tests covering vector/set/table ctors/inits. (Jon Siwek)
-    
+
   * Fix parsing of "local" named table constructors. (Jon Siwek)
 
   * Improve type checking of records. Addresses BIT-1159. (Jon Siwek)
-    
+
 2.2-267 | 2014-03-30 20:21:43 +0200
 
-  * Improve documentation of Bro clusters. Addresses BIT-1160. 
+  * Improve documentation of Bro clusters. Addresses BIT-1160.
     (Daniel Thayer)
 
 2.2-263 | 2014-03-30 20:19:05 +0200
@@ -434,7 +1108,7 @@
 2.2-254 | 2014-03-30 19:55:22 +0200
 
   * Update instructions on how to build Bro docs. (Daniel Thayer)
-    
+
 2.2-251 | 2014-03-28 08:37:37 -0400
 
   * Quick fix to the ElasticSearch writer. (Seth Hall)
@@ -464,7 +1138,7 @@
     formatter work.
 
   * Fixing compiler error. (Robin Sommer)
-    
+
   * Fixing (very unlikely) double delete in HTTP analyzer when
     decapsulating CONNECTs. (Robin Sommer)
 
@@ -490,7 +1164,7 @@
     Addresses BIT-1134. (Jon Siwek)
 
   * Enable fake DNS mode for test suites.
-    
+
   * Improve analysis of TCP SYN/SYN-ACK reversal situations. (Jon
     Siwek)
 
@@ -522,7 +1196,7 @@
     (Jon Siwek)
 
   * Silences some documentation warnings from Sphinx. (Jon Siwek)
-    
+
 2.2-215 | 2014-03-10 11:10:15 -0700
 
   * Fix non-deterministic logging of unmatched DNS msgs. Addresses
@@ -560,7 +1234,7 @@
     HTTP traffic. (Seth Hall)
 
   * Fixing removal of support analyzers. (Robin Sommer)
-    
+
 2.2-199 | 2014-03-03 16:34:20 -0800
 
   * Allow iterating over bif functions with result type vector of any.
@@ -575,11 +1249,11 @@
 2.2-194 | 2014-02-28 14:50:53 -0800
 
   * Remove packet sorter. Addresses BIT-700. (Bernhard Amann)
-    
+
 2.2-192 | 2014-02-28 09:46:43 -0800
 
   * Update Mozilla root bundle. (Bernhard Amann)
-    
+
 2.2-190 | 2014-02-27 07:34:44 -0800
 
   * Adjust timings of a few leak tests. (Bernhard Amann)
@@ -609,7 +1283,7 @@
 2.2-177 | 2014-02-20 17:27:46 -0800
 
   * Update to libmagic version 5.17. Addresses BIT-1136. (Jon Siwek)
-    
+
 2.2-174 | 2014-02-14 12:07:04 -0800
 
   * Support for MPLS over VLAN. (Chris Kanich)
@@ -638,7 +1312,7 @@
     defined, but we see it being actively used. (Bernhard Amann)
 
   * Test baseline updates for DNS change. (Robin Sommer)
-    
+
 2.2-158 | 2014-02-09 23:45:39 -0500
 
   * Change dns.log to include only standard DNS queries. (Jon Siwek)
@@ -714,7 +1388,7 @@
 2.2-128 | 2014-01-30 15:58:47 -0800
 
   * Add leak test for Exec module. (Bernhard Amann)
-    
+
   * Fix file_over_new_connection event to trigger when entire file is
     missed. (Jon Siwek)
 
@@ -727,7 +1401,7 @@
     "detect_filtered_trace". (Jon Siwek)
 
   * Improve TCP FIN retransmission handling. (Jon Siwek)
-    
+
 2.2-120 | 2014-01-28 10:25:23 -0800
 
   * Fix and extend x509_extension() event, which now actually returns
@@ -745,10 +1419,10 @@
 2.2-115 | 2014-01-22 12:11:18 -0800
 
   * Add unit tests for new Bro Manual docs. (Jon Siwek)
-    
+
   * New content for the "Using Bro" section of the manual. (Rafael
     Bonilla/Jon Siwek)
-    
+
 2.2-105 | 2014-01-20 12:16:48 -0800
 
   * Support GRE tunnel decapsulation, including enhanced GRE headers.
@@ -757,7 +1431,7 @@
     Addresses BIT-867. (Jon Siwek)
 
   * Simplify FragReassembler memory management. (Jon Siwek)
-    
+
 2.2-102 | 2014-01-20 12:00:29 -0800
 
   * Include file information (MIME type and description) into notice
@@ -766,7 +1440,7 @@
 2.2-100 | 2014-01-20 11:54:58 -0800
 
   * Fix caching of recently validated SSL certifcates. (Justin Azoff)
-    
+
 2.2-98 | 2014-01-20 11:50:32 -0800
 
   * For notice suppresion, instead of storing the entire notice in
@@ -794,7 +1468,7 @@
     the city database instead of just the former. (Jon Siwek)
 
   * Broxygen init fixes. Addresses BIT-1110. (Jon Siwek)
-    
+
     - Don't check mtime of bro binary if BRO_DISABLE_BROXYGEN env var set.
 
     - Fix failure to locate bro binary if invoking from a relative
@@ -807,7 +1481,7 @@
     len field being set to zero. (Seth Hall)
 
   * Canonify output of a unit test. (Jon Siwek)
-    
+
   * A set of documentation updates. (Daniel Thayer)
 
       - Fix typo in Bro 2.2 NEWS on string indexing.
@@ -854,9 +1528,9 @@
     (Jon Siwek)
 
   * Close signature files after done parsing. (Jon Siwek)
-    
+
   * Fix unlikely null ptr deref in broxygen::Manager. (Jon Siwek)
-    
+
   * FreeBSD build fix addendum: unintended variable shadowing. (Jon
     Siwek)
 
@@ -875,7 +1549,7 @@
     were cleaned up. Addresses BIT-1103. (Bernhard Amann)
 
   * Minor Broxygen improvements. Addresses BIT-1098. (Jon Siwek)
-    
+
 2.2-51 | 2013-12-05 07:53:37 -0800
 
   * Improve a unit test involving 'when' conditionals. (Jon Siwek)
@@ -900,7 +1574,7 @@
 2.2-44 | 2013-12-04 12:41:51 -0800
 
   * Fix string slice notation. Addresses BIT-1097. (Jon Siwek)
-    
+
     Slice ranges were not correctly determined for negative indices
     and also off by one in general (included one more element at the
     end of the substring than what actually matched the index range).
@@ -955,7 +1629,7 @@
 2.2-11 | 2013-12-03 10:56:28 -0800
 
   * Unit test for broccoli vector support. (Jon Siwek)
-    
+
   * Changed ordering of Bro type tag enum, which was out of sync. (Jon
     Siwek)
 
@@ -967,14 +1641,14 @@
 2.2-6 | 2013-11-15 07:05:15 -0800
 
   * Make "install-example-configs" target use DESTDIR. (Jon Siwek)
-    
+
 2.2-5 | 2013-11-11 13:47:54 -0800
 
   * Fix the irc_reply event for certain server message types. (Seth
     Hall)
 
   * Fixed Segmentation fault in SQLite Writer. (Jon Crussell)
-    
+
 2.2 | 2013-11-07 10:25:50 -0800
 
   * Release 2.2.

CMakeLists.txt

@@ -1,5 +1,9 @@
 project(Bro C CXX)
-cmake_minimum_required(VERSION 2.6.3 FATAL_ERROR)
+
+# When changing the minimum version here, also adapt
+# aux/bro-aux/plugin-support/skeleton/CMakeLists.txt
+cmake_minimum_required(VERSION 2.8 FATAL_ERROR)
+
 include(cmake/CommonCMakeConfig.cmake)
 
 ########################################################################
@@ -11,17 +15,28 @@ if (NOT BRO_SCRIPT_INSTALL_PATH)
     set(BRO_SCRIPT_INSTALL_PATH ${BRO_ROOT_DIR}/share/bro)
 endif ()
 
+if (NOT BRO_MAN_INSTALL_PATH)
+    # set the default Bro man page installation path (user did not specify one)
+    set(BRO_MAN_INSTALL_PATH ${BRO_ROOT_DIR}/share/man)
+endif ()
+
 # sanitize the Bro script install directory into an absolute path
 # (CMake is confused by ~ as a representation of home directory)
 get_filename_component(BRO_SCRIPT_INSTALL_PATH ${BRO_SCRIPT_INSTALL_PATH}
     ABSOLUTE)
 
+set(BRO_PLUGIN_INSTALL_PATH ${BRO_ROOT_DIR}/lib/bro/plugins CACHE STRING "Installation path for plugins" FORCE)
+
 configure_file(bro-path-dev.in ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev)
+
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.sh
      "export BROPATH=`${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n"
+     "export BRO_PLUGIN_PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src:${BRO_PLUGIN_INSTALL_PATH}\"\n"
      "export PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n")
+
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.csh
      "setenv BROPATH `${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n"
+     "setenv BRO_PLUGIN_PATH \"${CMAKE_CURRENT_BINARY_DIR}/src:${BRO_PLUGIN_INSTALL_PATH}\"\n"
      "setenv PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n")
 
 file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1)
@@ -117,33 +132,6 @@ if (GOOGLEPERFTOOLS_FOUND)
     endif ()
 endif ()
 
-set(USE_DATASERIES false)
-find_package(Lintel)
-find_package(DataSeries)
-find_package(LibXML2)
-
-if (NOT DISABLE_DATASERIES AND
-    LINTEL_FOUND AND DATASERIES_FOUND AND LIBXML2_FOUND)
-    set(USE_DATASERIES true)
-	include_directories(BEFORE ${Lintel_INCLUDE_DIR})
-	include_directories(BEFORE ${DataSeries_INCLUDE_DIR})
-	include_directories(BEFORE ${LibXML2_INCLUDE_DIR})
-	list(APPEND OPTLIBS ${Lintel_LIBRARIES})
-	list(APPEND OPTLIBS ${DataSeries_LIBRARIES})
-	list(APPEND OPTLIBS ${LibXML2_LIBRARIES})
-endif()
-
-set(USE_ELASTICSEARCH false)
-set(USE_CURL false)
-find_package(LibCURL)
-
-if (NOT DISABLE_ELASTICSEARCH AND LIBCURL_FOUND)
-  set(USE_ELASTICSEARCH true)
-  set(USE_CURL true)
-  include_directories(BEFORE ${LibCURL_INCLUDE_DIR})
-  list(APPEND OPTLIBS ${LibCURL_LIBRARIES})
-endif()
-
 if (ENABLE_PERFTOOLS_DEBUG OR ENABLE_PERFTOOLS)
     # Just a no op to prevent CMake from complaining about manually-specified
     # ENABLE_PERFTOOLS_DEBUG or ENABLE_PERFTOOLS not being used if google
@@ -165,6 +153,8 @@ set(brodeps
 
 include(TestBigEndian)
 test_big_endian(WORDS_BIGENDIAN)
+include(CheckSymbolExists)
+check_symbol_exists(htonll arpa/inet.h HAVE_BYTEORDER_64)
 
 include(OSSpecific)
 include(CheckTypes)
@@ -174,6 +164,10 @@ include(MiscTests)
 include(PCAPTests)
 include(OpenSSLTests)
 include(CheckNameserCompat)
+include(GetArchitecture)
+
+# Tell the plugin code that we're building as part of the main tree.
+set(BRO_PLUGIN_INTERNAL_BUILD true CACHE INTERNAL "" FORCE)
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/config.h.in
                ${CMAKE_CURRENT_BINARY_DIR}/config.h)
@@ -186,6 +180,7 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
 add_subdirectory(src)
 add_subdirectory(scripts)
 add_subdirectory(doc)
+add_subdirectory(man)
 
 include(CheckOptionalBuildSources)
 
@@ -238,10 +233,6 @@ message(
     "\n        tcmalloc:  ${USE_PERFTOOLS_TCMALLOC}"
     "\n       debugging:  ${USE_PERFTOOLS_DEBUG}"
     "\njemalloc:          ${ENABLE_JEMALLOC}"
-    "\ncURL:              ${USE_CURL}"
-    "\n"
-    "\nDataSeries:        ${USE_DATASERIES}"
-    "\nElasticSearch:     ${USE_ELASTICSEARCH}"
     "\n"
     "\n================================================================\n"
 )

Makefile

@@ -48,13 +48,16 @@ bindist:
 
 distclean:
 	rm -rf $(BUILD)
+	$(MAKE) -C testing $@
 
 test:
 	@( cd testing && make )
 
 test-all: test
-	test -d aux/broctl && ( cd aux/broctl && make test )
+	test -d aux/broctl && ( cd aux/broctl && make test-all )
 	test -d aux/btest  && ( cd aux/btest && make test )
+	test -d aux/bro-aux && ( cd aux/bro-aux && make test )
+	test -d aux/plugins && ( cd aux/plugins && make test-all )
 
 configured:
 	@test -d $(BUILD) || ( echo "Error: No build/ directory found. Did you run configure?" && exit 1 )

NEWS

@@ -4,6 +4,55 @@ release. For an exhaustive list of changes, see the ``CHANGES`` file
 (note that submodules, such as BroControl and Broccoli, come with
 their own ``CHANGES``.)
 
+Bro 2.4 (in progress)
+=====================
+
+Dependencies
+------------
+
+New Functionality
+-----------------
+
+- Bro now has support for external plugins that can extend its core
+  functionality, like protocol/file analysis, via shared libraries.
+  Plugins can be developed and distributed externally, and will be
+  pulled in dynamically at startup. Currently, a plugin can provide
+  custom protocol analyzers, file analyzers, log writers[TODO], input
+  readers[TODO], packet sources[TODO], and new built-in functions. A
+  plugin can furthermore hook into Bro's processing a number of places
+  to add custom logic.
+
+  See https://www.bro.org/sphinx-git/devel/plugins.html for more
+  information on writing plugins.
+
+- Bro now has supoprt for the MySQL wire protocol. Activity gets
+  logged into mysql.log.
+
+- Bro's file analysis now supports reassembly of files that are not
+  transferred/seen sequentially.
+
+Changed Functionality
+---------------------
+
+- bro-cut has been rewritten in C, and is hence much faster.
+
+- File analysis
+
+    * Removed ``fa_file`` record's ``mime_type`` and ``mime_types``
+      fields.  The events ``file_mime_type`` and ``file_mime_types``
+      have been added which contain the same information.  The
+      ``mime_type`` field of ``Files::Info`` also still has this info.
+
+    * Removed ``Files::add_analyzers_for_mime_type`` function.
+
+    * Removed ``offset`` parameter of the ``file_extraction_limit``
+      event.  Since file extraction now internally depends on file
+      reassembly for non-sequential files, "offset" can be obtained
+      with other information already available -- adding together
+      ``seen_bytes`` and ``missed_bytes`` fields of the ``fa_file``
+      record gives the how many bytes have been written so far (i.e.
+      the "offset").
+
 Bro 2.3
 =======
 

VERSION

@@ -1 +1 @@
-2.3-beta-18
+2.3-397

aux/binpac

@@ -1 +1 @@
-Subproject commit ec1e052afd5a8cd3d1d2cbb28fcd688018e379a5
+Subproject commit 8d56b507b8b804fa83f6637f3b1f198e696cd603

aux/bro-aux

@@ -1 +1 @@
-Subproject commit 5721df4f5f6fa84de6257cca6582a28e45831786
+Subproject commit 0b713c027d3efaaca50e5df995c02656175573cd

aux/broccoli

@@ -1 +1 @@
-Subproject commit c2f5dd2cb7876158fdf9721aebd22567db840db1
+Subproject commit d43cc790e5b8709b5e032e52ad0e00936494739b

aux/broctl

@@ -1 +1 @@
-Subproject commit 1e55bff2df49fe7dc3bd54e21050b39530eeb714
+Subproject commit 8c9b87bc73e1ddaa304e3d89028c1e7b95d37a91

aux/btest

@@ -1 +1 @@
-Subproject commit 4da1bd24038d4977e655f2b210f34e37f0b73b78
+Subproject commit 93d4989ed1537e4d143cf09d44077159f869a4b2

aux/plugins

@@ -0,0 +1 @@
+Subproject commit ad600b5bdcd56a2723e323c0f2c8e1708956ca4f

cmake

@@ -1 +1 @@
-Subproject commit 0f301aa08a970150195a2ea5b3ed43d2d98b35b3
+Subproject commit 1316c07f7059647b6c4a496ea36e4b83bb5d8f0f

config.h.in

@@ -129,6 +129,9 @@
 /* whether words are stored with the most significant byte first */
 #cmakedefine WORDS_BIGENDIAN
 
+/* whether htonll/ntohll is defined in <arpa/inet.h> */
+#cmakedefine HAVE_BYTEORDER_64
+
 /* ultrix can't hack const */
 #cmakedefine NEED_ULTRIX_CONST_HACK
 #ifdef NEED_ULTRIX_CONST_HACK
@@ -209,3 +212,14 @@
 
 /* Common IPv6 extension structure */
 #cmakedefine HAVE_IP6_EXT
+
+/* String with host architecture (e.g., "linux-x86_64") */
+#define HOST_ARCHITECTURE "@HOST_ARCHITECTURE@"
+
+/* String with extension of dynamic libraries (e.g., ".so") */
+#define DYNAMIC_PLUGIN_SUFFIX "@CMAKE_SHARED_MODULE_SUFFIX@"
+
+/* True if we're building outside of the main Bro source code tree. */
+#ifndef BRO_PLUGIN_INTERNAL_BUILD
+#define BRO_PLUGIN_INTERNAL_BUILD @BRO_PLUGIN_INTERNAL_BUILD@
+#endif

configure

@@ -24,6 +24,13 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --prefix=PREFIX        installation directory [/usr/local/bro]
     --scriptdir=PATH       root installation directory for Bro scripts
                            [PREFIX/share/bro]
+    --localstatedir=PATH   when using BroControl, path to store log files
+                           and run-time data (within log/ and spool/ subdirs)
+                           [PREFIX]
+    --spooldir=PATH        when using BroControl, path to store run-time data
+                           [PREFIX/spool]
+    --logdir=PATH          when using BroControl, path to store log file
+                           [PREFIX/logs]
     --conf-files-dir=PATH  config files installation directory [PREFIX/etc]
 
   Optional Features:
@@ -39,8 +46,6 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --disable-auxtools     don't build or install auxiliary tools
     --disable-perftools    don't try to build with Google Perftools
     --disable-python       don't try to build python bindings for broccoli
-    --disable-dataseries   don't use the optional DataSeries log writer
-    --disable-elasticsearch don't use the optional ElasticSearch log writer
 
   Required Packages in Non-Standard Locations:
     --with-openssl=PATH    path to OpenSSL install root
@@ -50,7 +55,6 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --with-flex=PATH       path to flex executable
     --with-bison=PATH      path to bison executable
     --with-perl=PATH       path to perl executable
-    --with-libmagic=PATH   path to libmagic install root
 
   Optional Packages in Non-Standard Locations:
     --with-geoip=PATH      path to the libGeoIP install root
@@ -63,9 +67,6 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --with-ruby-lib=PATH   path to ruby library
     --with-ruby-inc=PATH   path to ruby headers
     --with-swig=PATH       path to SWIG executable
-    --with-dataseries=PATH path to DataSeries and Lintel libraries
-    --with-xml2=PATH       path to libxml2 installation (for DataSeries)
-    --with-curl=PATH       path to libcurl install root (for ElasticSearch)
 
   Packaging Options (for developers):
     --binary-package       toggle special logic for binary packaging
@@ -150,6 +151,15 @@ while [ $# -ne 0 ]; do
             append_cache_entry BRO_ETC_INSTALL_DIR PATH $optarg
             user_set_conffilesdir="true"
             ;;
+        --localstatedir=*)
+            append_cache_entry BRO_LOCAL_STATE_DIR  PATH   $optarg
+            ;;
+        --spooldir=*)
+            append_cache_entry BRO_SPOOL_DIR  PATH   $optarg
+            ;;
+        --logdir=*)
+            append_cache_entry BRO_LOG_DIR  PATH   $optarg
+            ;;
         --enable-debug)
             append_cache_entry ENABLE_DEBUG         BOOL   true
             ;;
@@ -184,12 +194,6 @@ while [ $# -ne 0 ]; do
         --enable-ruby)
             append_cache_entry DISABLE_RUBY_BINDINGS     BOOL   false
             ;;
-        --disable-dataseries)
-            append_cache_entry DISABLE_DATASERIES     BOOL   true
-            ;;
-        --disable-elasticsearch)
-            append_cache_entry DISABLE_ELASTICSEARCH     BOOL   true
-            ;;
         --with-openssl=*)
             append_cache_entry OpenSSL_ROOT_DIR PATH $optarg
             ;;
@@ -211,9 +215,6 @@ while [ $# -ne 0 ]; do
         --with-perl=*)
             append_cache_entry PERL_EXECUTABLE PATH $optarg
             ;;
-        --with-libmagic=*)
-            append_cache_entry LibMagic_ROOT_DIR PATH $optarg
-            ;;
         --with-geoip=*)
             append_cache_entry LibGeoIP_ROOT_DIR PATH $optarg
             ;;
@@ -247,16 +248,6 @@ while [ $# -ne 0 ]; do
         --with-swig=*)
             append_cache_entry SWIG_EXECUTABLE      PATH    $optarg
             ;;
-        --with-dataseries=*)
-            append_cache_entry DataSeries_ROOT_DIR PATH $optarg
-            append_cache_entry Lintel_ROOT_DIR PATH $optarg
-            ;;
-        --with-xml2=*)
-            append_cache_entry LibXML2_ROOT_DIR PATH $optarg
-            ;;
-        --with-curl=*)
-            append_cache_entry LibCURL_ROOT_DIR PATH $optarg
-            ;;
         --binary-package)
             append_cache_entry BINARY_PACKAGING_MODE BOOL true
             ;;

doc/_templates/layout.html

@@ -10,7 +10,7 @@
 {% endblock %}
 
 {% block header %}
-<iframe src="http://www.bro.org/frames/header-no-logo.html" width="100%" height="100px" frameborder="0" marginheight="0" scrolling="no" marginwidth="0">
+<iframe src="//www.bro.org/frames/header-no-logo.html" width="100%" height="100px" frameborder="0" marginheight="0" scrolling="no" marginwidth="0">
 </iframe>
 {% endblock %}
 
@@ -108,6 +108,6 @@
 {% endblock %}
 
 {% block footer %}
-<iframe src="http://www.bro.org/frames/footer.html" width="100%" height="420px" frameborder="0" marginheight="0" scrolling="no" marginwidth="0">
+<iframe src="//www.bro.org/frames/footer.html" width="100%" height="420px" frameborder="0" marginheight="0" scrolling="no" marginwidth="0">
 </iframe>
 {% endblock %}

doc/conf.py.in

@@ -21,7 +21,7 @@ sys.path.insert(0, os.path.abspath('sphinx_input/ext'))
 
 # ----- Begin of BTest configuration. -----
 btest = os.path.abspath("@CMAKE_SOURCE_DIR@/aux/btest")
-brocut = os.path.abspath("@CMAKE_SOURCE_DIR@/aux/bro-aux/bro-cut")
+brocut = os.path.abspath("@CMAKE_SOURCE_DIR@/build/aux/bro-aux/bro-cut")
 bro = os.path.abspath("@CMAKE_SOURCE_DIR@/build/src")
 
 os.environ["PATH"] += (":%s:%s/sphinx:%s:%s" % (btest, btest, bro, brocut))

doc/devel/plugins.rst

@@ -0,0 +1,447 @@
+
+===================
+Writing Bro Plugins
+===================
+
+Bro is internally moving to a plugin structure that enables extending
+the system dynamically, without modifying the core code base. That way
+custom code remains self-contained and can be maintained, compiled,
+and installed independently. Currently, plugins can add the following
+functionality to Bro:
+
+    - Bro scripts.
+
+    - Builtin functions/events/types for the scripting language.
+
+    - Protocol analyzers.
+
+    - File analyzers.
+
+    - Packet sources and packet dumpers.
+
+    - Logging framework backends.
+
+    - Input framework readers.
+
+A plugin's functionality is available to the user just as if Bro had
+the corresponding code built-in. Indeed, internally many of Bro's
+pieces are structured as plugins as well, they are just statically
+compiled into the binary rather than loaded dynamically at runtime.
+
+Quick Start
+===========
+
+Writing a basic plugin is quite straight-forward as long as one
+follows a few conventions. In the following we walk a simple example
+plugin that adds a new built-in function (bif) to Bro: we'll add 
+``rot13(s: string) : string``, a function that rotates every character
+in a string by 13 places.
+
+Generally, a plugin comes in the form of a directory following a
+certain structure. To get started, Bro's distribution provides a
+helper script ``aux/bro-aux/plugin-support/init-plugin`` that creates
+a skeleton plugin that can then be customized. Let's use that::
+
+    # mkdir rot13-plugin
+    # cd rot13-plugin
+    # init-plugin Demo Rot13
+
+As you can see the script takes two arguments. The first is a
+namespace the plugin will live in, and the second a descriptive name
+for the plugin itself. Bro uses the combination of the two to identify
+a plugin. The namespace serves to avoid naming conflicts between
+plugins written by independent developers; pick, e.g., the name of
+your organisation. The namespace ``Bro`` is reserved for functionality
+distributed by the Bro Project. In our example, the plugin will be
+called ``Demo::Rot13``.
+
+The ``init-plugin`` script puts a number of files in place. The full
+layout is described later. For now, all we need is
+``src/rot13.bif``. It's initially empty, but we'll add our new bif
+there as follows::
+
+    # cat src/rot13.bif
+    module CaesarCipher;
+
+    function rot13%(s: string%) : string
+        %{
+        char* rot13 = copy_string(s->CheckString());
+
+        for ( char* p = rot13; *p; p++ )
+            {
+            char b = islower(*p) ? 'a' : 'A';
+            *p  = (*p - b + 13) % 26 + b;
+            }
+
+        BroString* bs = new BroString(1, reinterpret_cast<byte_vec>(rot13),
+                                      strlen(rot13));
+        return new StringVal(bs);
+        %}
+
+The syntax of this file is just like any other ``*.bif`` file; we
+won't go into it here.
+
+Now we can already compile our plugin, we just need to tell the
+configure script put in place by ``init-plugin`` where the Bro source
+tree is located (Bro needs to have been built there first)::
+
+    # ./configure --bro-dist=/path/to/bro/dist && make
+    [... cmake output ...]
+
+Now our ``rot13-plugin`` directory has everything that it needs
+for Bro to recognize it as a dynamic plugin. Once we point Bro to it,
+it will pull it in automatically, as we can check with the ``-N``
+option::
+
+    # export BRO_PLUGIN_PATH=/path/to/rot13-plugin
+    # bro -N
+    [...]
+    Plugin: Demo::Rot13 - <Insert brief description of plugin> (dynamic, version 1)
+    [...]
+
+That looks quite good, except for the dummy description that we should
+replace with something nicer so that users will know what our plugin
+is about.  We do this by editing the ``config.description`` line in
+``src/Plugin.cc``, like this::
+
+    [...]
+    plugin::Configuration Configure()
+        {
+        plugin::Configuration config;
+        config.name = "Demo::Rot13";
+        config.description = "Caesar cipher rotating a string's characters by 13 places.";
+        config.version.major = 1;
+        config.version.minor = 0;
+        return config;
+        }
+    [...]
+
+    # make
+    [...]
+    # bro -N | grep Rot13
+    Plugin: Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 1)
+
+Better. Bro can also show us what exactly the plugin provides with the
+more verbose option ``-NN``::
+
+    # bro -NN
+    [...]
+    Plugin: Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 1)
+        [Function] CaesarCipher::rot13
+    [...]
+
+There's our function. Now let's use it::
+
+    # bro -e 'print CaesarCipher::rot13("Hello")'
+    Uryyb
+
+It works. We next install the plugin along with Bro itself, so that it
+will find it directly without needing the ``BRO_PLUGIN_PATH``
+environment variable. If we first unset the variable, the function
+will no longer be available::
+
+    # unset BRO_PLUGIN_PATH
+    # bro -e 'print CaesarCipher::rot13("Hello")'
+    error in <command line>, line 1: unknown identifier CaesarCipher::rot13, at or near "CaesarCipher::rot13"
+
+Once we install it, it works again::
+
+    # make install
+    # bro -e 'print CaesarCipher::rot13("Hello")'
+    Uryyb
+
+The installed version went into
+``<bro-install-prefix>/lib/bro/plugins/Demo_Rot13``.
+
+We can distribute the plugin in either source or binary form by using
+the Makefile's ``sdist`` and ``bdist`` target, respectively. Both
+create corrsponding tarballs::
+
+    # make sdist
+    [...]
+    Source distribution in build/sdist/Demo_Rot13.tar.gz
+
+    # make bdist
+    [...]
+    Binary distribution in build/Demo_Rot13-darwin-x86_64.tar.gz
+
+The source archive will contain everything in the plugin directory
+except any generated files. The binary archive will contain anything
+needed to install and run the plugin, i.e., just what ``make install``
+puts into place as well. As the binary distribution is
+platform-dependent, its name includes the OS and architecture the
+plugin was built on.
+
+Plugin Directory Layout
+=======================
+
+A plugin's directory needs to follow a set of conventions so that Bro
+(1) recognizes it as a plugin, and (2) knows what to load.  While
+``init-plugin`` takes care of most of this, the following is the full
+story. We'll use ``<base>`` to represent a plugin's top-level
+directory.
+
+``<base>/__bro_plugin__``
+    A file that marks a directory as containing a Bro plugin. The file
+    must exist, and its content must consist of a single line with the
+    qualified name of the plugin (e.g., "Demo::Rot13").
+
+``<base>/lib/<plugin-name>-<os>-<arch>.so``
+    The shared library containing the plugin's compiled code. Bro will
+    load this in dynamically at run-time if OS and architecture match
+    the current platform.
+
+``scripts/``
+    A directory with the plugin's custom Bro scripts. When the plugin
+    gets activated, this directory will be automatically added to
+    ``BROPATH``, so that any scripts/modules inside can be
+    "@load"ed.
+
+``scripts``/__load__.bro
+    A Bro script that will be loaded immediately when the plugin gets
+    activated. See below for more information on activating plugins.
+
+``lib/bif/``
+    Directory with auto-generated Bro scripts that declare the plugin's
+    bif elements. The files here are produced by ``bifcl``.
+
+By convention, a plugin should put its custom scripts into sub folders
+of ``scripts/``, i.e., ``scripts/<script-namespace>/<script>.bro`` to
+avoid conflicts. As usual, you can then put a ``__load__.bro`` in
+there as well so that, e.g., ``@load Demo/Rot13`` could load a whole
+module in the form of multiple individual scripts.
+
+Note that in addition to the paths above, the ``init-plugin`` helper
+puts some more files and directories in place that help with
+development and installation (e.g., ``CMakeLists.txt``, ``Makefile``,
+and source code in ``src/``). However, all these do not have a special
+meaning for Bro at runtime and aren't necessary for a plugin to
+function.
+
+``init-plugin``
+===============
+
+``init-plugin`` puts a basic plugin structure in place that follows
+the above layout and augments it with a CMake build and installation
+system. Plugins with this structure can be used both directly out of
+their source directory (after ``make`` and setting Bro's
+``BRO_PLUGIN_PATH``), and when installed alongside Bro (after ``make
+install``).
+
+``make install`` copies over the ``lib`` and ``scripts`` directories,
+as well as the ``__bro_plugin__`` magic file and the ``README`` (which
+you should customize). One can add further CMake ``install`` rules to
+install additional files if needed.
+
+``init-plugin`` will never overwrite existing files, so it's safe to
+rerun in an existing plugin directory; it only put files in place that
+don't exist yet. That also provides a convenient way to revert a file
+back to what ``init-plugin`` created originally: just delete it and
+rerun.
+
+Activating a Plugin
+===================
+
+A plugin needs to be *activated* to make it available to the user.
+Activating a plugin will:
+
+    1. Load the dynamic module
+    2. Make any bif items available
+    3. Add the ``scripts/`` directory to ``BROPATH``
+    4. Load ``scripts/__load__.bro``
+
+By default, Bro will automatically activate all dynamic plugins found
+in its search path ``BRO_PLUGIN_PATH``. However, in bare mode (``bro
+-b``), no dynamic plugins will be activated by default; instead the
+user can selectively enable individual plugins in scriptland using the
+``@load-plugin <qualified-plugin-name>`` directive (e.g.,
+``@load-plugin Demo::Rot13``). Alternatively, one can activate a
+plugin from the command-line by specifying its full name
+(``Demo::Rot13``), or set the environment variable
+``BRO_PLUGIN_ACTIVATE`` to a list of comma(!)-separated names of
+plugins to unconditionally activate, even in bare mode.
+
+``bro -N`` shows activated plugins separately from found but not yet
+activated plugins. Note that plugins compiled statically into Bro are
+always activated, and hence show up as such even in bare mode.
+
+Plugin Components
+=================
+
+The following subsections detail providing individual types of
+functionality via plugins. Note that a single plugin can provide more
+than one component type. For example, a plugin could provide multiple
+protocol analyzers at once; or both a logging backend and input reader
+at the same time.
+
+.. todo::
+
+    These subsections are mostly missing right now, as much of their
+    content isn't actually plugin-specific, but concerns generally
+    writing such functionality for Bro. The best way to get started
+    right now is to look at existing code implementing similar
+    functionality, either as a plugin or inside Bro proper. Also, for
+    each component type there's a unit test in
+    ``testing/btest/plugins`` creating a basic plugin skeleton with a
+    corresponding component.
+
+Bro Scripts
+-----------
+
+Scripts are easy: just put them into ``scripts/``, as described above.
+The CMake infrastructure will automatically install them, as well
+include them into the source and binary plugin distributions.
+
+Builtin Language Elements
+-------------------------
+
+Functions
+    TODO
+
+Events
+    TODO
+
+Types
+    TODO
+
+Protocol Analyzers
+------------------
+
+TODO.
+
+File Analyzers
+--------------
+
+TODO.
+
+Logging Writer
+--------------
+
+TODO.
+
+Input Reader
+------------
+
+TODO.
+
+Packet Sources
+--------------
+
+TODO.
+
+Packet Dumpers
+--------------
+
+TODO.
+
+Hooks
+=====
+
+TODO.
+
+Testing Plugins
+===============
+
+A plugin should come with a test suite to exercise its functionality.
+The ``init-plugin`` script puts in place a basic </btest/README> setup
+to start with. Initially, it comes with a single test that just checks
+that Bro loads the plugin correctly. It won't have a baseline yet, so
+let's get that in place::
+
+    # cd tests
+    # btest -d
+    [  0%] plugin.loading ... failed
+    % 'btest-diff output' failed unexpectedly (exit code 100)
+    % cat .diag
+    == File ===============================
+    Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 1.0)
+        [Function] CaesarCipher::rot13
+
+    == Error ===============================
+    test-diff: no baseline found.
+    =======================================
+
+    # btest -U
+    all 1 tests successful
+
+    # cd ..
+    # make test
+    make -C tests
+    make[1]: Entering directory `tests'
+    all 1 tests successful
+    make[1]: Leaving directory `tests'
+
+Now let's add a custom test that ensures that our bif works
+correctly::
+
+    # cd tests
+    # cat >plugin/rot13.bro
+
+    # @TEST-EXEC: bro %INPUT >output
+    # @TEST-EXEC: btest-diff output
+
+    event bro_init()
+        {
+        print CaesarCipher::rot13("Hello");
+        }
+
+Check the output::
+
+    # btest -d plugin/rot13.bro
+    [  0%] plugin.rot13 ... failed
+    % 'btest-diff output' failed unexpectedly (exit code 100)
+    % cat .diag
+    == File ===============================
+    Uryyb
+    == Error ===============================
+    test-diff: no baseline found.
+    =======================================
+
+    % cat .stderr
+
+    1 of 1 test failed
+
+Install the baseline::
+
+    # btest -U plugin/rot13.bro
+    all 1 tests successful
+
+Run the test-suite::
+
+    # btest
+    all 2 tests successful
+
+Debugging Plugins
+=================
+
+If your plugin isn't loading as expected, Bro's debugging facilities
+can help to illuminate what's going on. To enable, recompile Bro
+with debugging support (``./configure --enable-debug``), and
+afterwards rebuild your plugin as well. If you then run Bro with ``-B
+plugins``, it will produce a file ``debug.log`` that records details
+about the process for searching, loading, and activating plugins. 
+
+To generate your own debugging output from inside your plugin, you can
+add a custom debug stream by using the ``PLUGIN_DBG_LOG(<plugin>,
+<args>)`` macro (defined in ``DebugLogger.h``), where ``<plugin>`` is
+the ``Plugin`` instance and ``<args>`` are printf-style arguments,
+just as with Bro's standard debugging macros (grep for ``DBG_LOG`` in
+Bro's ``src/`` to see examples). At runtime, you can then activate
+your plugin's debugging output with ``-B plugin-<name>``, where
+``<name>`` is the name of the plugin as returned by its
+``Configure()`` method, yet with the namespace-separator ``::``
+replaced with a simple dash. Example: If the plugin is called
+``Bro::Demo``, use ``-B plugin-Bro-Demo``. As usual, the debugging
+output will be recorded to ``debug.log`` if Bro's compiled in debug
+mode.
+
+
+Documenting Plugins
+===================
+
+.. todo::
+
+    Integrate all this with Broxygen.
+
+
+

doc/ext/bro.py

@@ -176,6 +176,10 @@ class BroIdentifier(BroGeneric):
     def get_index_text(self, objectname, name):
         return name
 
+class BroKeyword(BroGeneric):
+    def get_index_text(self, objectname, name):
+        return name
+
 class BroAttribute(BroGeneric):
     def get_index_text(self, objectname, name):
         return _('%s (attribute)') % (name)
@@ -213,6 +217,7 @@ class BroDomain(Domain):
         'type':             ObjType(l_('type'),             'type'),
         'namespace':        ObjType(l_('namespace'),        'namespace'),
         'id':               ObjType(l_('id'),               'id'),
+        'keyword':          ObjType(l_('keyword'),          'keyword'),
         'enum':             ObjType(l_('enum'),             'enum'),
         'attr':             ObjType(l_('attr'),             'attr'),
     }
@@ -221,6 +226,7 @@ class BroDomain(Domain):
         'type':             BroGeneric,
         'namespace':        BroNamespace,
         'id':               BroIdentifier,
+        'keyword':          BroKeyword,
         'enum':             BroEnum,
         'attr':             BroAttribute,
     }
@@ -229,6 +235,7 @@ class BroDomain(Domain):
         'type':             XRefRole(),
         'namespace':        XRefRole(),
         'id':               XRefRole(),
+        'keyword':          XRefRole(),
         'enum':             XRefRole(),
         'attr':             XRefRole(),
         'see':              XRefRole(),

doc/ext/rst_directive.py

@@ -135,7 +135,10 @@ class Pygments(Directive):
                 # lexer not found, use default.
                 lexer = TextLexer()
         else:
-            lexer = guess_lexer(content)
+            try:
+                lexer = guess_lexer(content)
+            except:
+                lexer = TextLexer()
 
         # import sys
         # print >>sys.stderr, self.arguments, lexer.__class__

doc/frameworks/file_analysis_02.bro

@@ -1,7 +1,7 @@
-event file_new(f: fa_file)
+event file_mime_type(f: fa_file, mime_type: string)
     {
     print "new file", f$id;
-    if ( f?$mime_type && f$mime_type == "text/plain" )
+    if ( mime_type == "text/plain" )
         Files::add_analyzer(f, Files::ANALYZER_MD5);
     }
 

doc/frameworks/intel.rst

@@ -14,32 +14,35 @@ consume that data, make it available for matching, and provide
 infrastructure around improving performance, memory utilization, and
 generally making all of this easier.
 
-Data in the Intelligence Framework is the atomic piece of intelligence
+Data in the Intelligence Framework is an atomic piece of intelligence
 such as an IP address or an e-mail address along with a suite of
 metadata about it such as a freeform source field, a freeform
 descriptive field and a URL which might lead to more information about
 the specific item.  The metadata in the default scripts has been
 deliberately kept minimal so that the community can find the
-appropriate fields that need added by writing scripts which extend the
+appropriate fields that need to be added by writing scripts which extend the
 base record using the normal record extension mechanism.
 
 Quick Start
 -----------
 
-Load the package of scripts that sends data into the Intelligence
-Framework to be checked by loading this script in local.bro::
-
-	@load policy/frameworks/intel/seen
-
 Refer to the "Loading Intelligence" section below to see the format
 for Intelligence Framework text files, then load those text files with
 this line in local.bro::
 
 	redef Intel::read_files += { "/somewhere/yourdata.txt" };
 
-The data itself only needs to reside on the manager if running in a
+The text files need to reside only on the manager if running in a
 cluster.
 
+Add the following line to local.bro in order to load the scripts
+that send "seen" data into the Intelligence Framework to be checked against
+the loaded intelligence data::
+
+	@load policy/frameworks/intel/seen
+
+Intelligence data matches will be logged to the intel.log file.
+
 Architecture
 ------------
 
@@ -58,8 +61,10 @@ manager is the only node that needs the intelligence data.  The
 intelligence framework has distribution mechanisms which will push
 data out to all of the nodes that need it.
 
-Here is an example of the intelligence data format.  Note that all
+Here is an example of the intelligence data format (note that there will be
-whitespace field separators are literal tabs and fields containing only a
+additional fields if you are using CIF intelligence data or if you are
+using the policy/frameworks/intel/do_notice script).  Note that all fields
+must be separated by a single tab character and fields containing only a
 hyphen are considered to be null values. ::
 
 	#fields	indicator	indicator_type	meta.source	meta.desc	meta.url
@@ -69,8 +74,21 @@ hyphen are considered to be null values. ::
 For a list of all built-in `indicator_type` values, please refer to the
 documentation of :bro:see:`Intel::Type`.
 
-To load the data once files are created, use the following example
+Note that if you are using data from the Collective Intelligence Framework,
-code to define files to load with your own file names of course::
+then you will need to add the following line to your local.bro in order
+to support additional metadata fields used by CIF::
+
+	@load policy/integration/collective-intel
+
+There is a simple mechanism to raise a Bro notice (of type Intel::Notice)
+for user-specified intelligence matches.  To use this feature, add the
+following line to local.bro in order to support additional metadata fields
+(documented in the :bro:see:`Intel::MetaData` record)::
+
+	@load policy/frameworks/intel/do_notice
+
+To load the data once the files are created, use the following example
+to specify which files to load (with your own file names of course)::
 
 	redef Intel::read_files += {
 		"/somewhere/feed1.txt",
@@ -85,24 +103,23 @@ Seen Data
 
 When some bit of data is extracted (such as an email address in the
 "From" header in a message over SMTP), the Intelligence Framework
-needs to be informed that this data was discovered and it's presence
+needs to be informed that this data was discovered so that its presence
-should be checked within the intelligence data set.  This is
+will be checked within the loaded intelligence data.  This is
-accomplished through the :bro:see:`Intel::seen` function.
+accomplished through the :bro:see:`Intel::seen` function, however
+typically users won't need to work with this function due to the
+scripts included with Bro that will call this function.
 
-Typically users won't need to work with this function due to built in
+To load all of the scripts included with Bro for sending "seen" data to
-hook scripts that Bro ships with that will "see" data and send it into
+the intelligence framework, just add this line to local.bro::
-the intelligence framework.  A user may only need to load the entire
+
-package of hook scripts as a module or pick and choose specific
+	@load policy/frameworks/intel/seen
-scripts to load.  Keep in mind that as more data is sent into the
+
+Alternatively, specific scripts in that directory can be loaded.
+Keep in mind that as more data is sent into the
 intelligence framework, the CPU load consumed by Bro will increase
 depending on how many times the :bro:see:`Intel::seen` function is
 being called which is heavily traffic dependent.
 
-The full package of hook scripts that Bro ships with for sending this
-"seen" data into the intelligence framework can be loading by adding
-this line to local.bro::
-
-	@load policy/frameworks/intel/seen
 
 Intelligence Matches
 ********************
@@ -111,6 +128,7 @@ Against all hopes, most networks will eventually have a hit on
 intelligence data which could indicate a possible compromise or other
 unwanted activity.  The Intelligence Framework provides an event that
 is generated whenever a match is discovered named :bro:see:`Intel::match`.
+
 Due to design restrictions placed upon
 the intelligence framework, there is no assurance as to where this
 event will be generated.  It could be generated on the worker where
@@ -119,3 +137,7 @@ handled, only the data given as event arguments to the event can be
 assured since the host where the data was seen may not be where
 ``Intel::match`` is handled.
 
+Intelligence matches are logged to the intel.log file.  For a description of
+each field in that file, see the documentation for the :bro:see:`Intel::Info`
+record.
+

doc/frameworks/logging-dataseries.rst

@@ -1,186 +0,0 @@
-
-=============================
-Binary Output with DataSeries
-=============================
-
-.. rst-class:: opening
-
-   Bro's default ASCII log format is not exactly the most efficient
-   way for storing and searching large volumes of data. An an
-   alternative, Bro comes with experimental support for `DataSeries
-   <http://www.hpl.hp.com/techreports/2009/HPL-2009-323.html>`_
-   output, an efficient binary format for recording structured bulk
-   data. DataSeries is developed and maintained at HP Labs.
-
-.. contents::
-
-Installing DataSeries
----------------------
-
-To use DataSeries, its libraries must be available at compile-time,
-along with the supporting *Lintel* package. Generally, both are
-distributed on `HP Labs' web site
-<http://tesla.hpl.hp.com/opensource/>`_. Currently, however, you need
-to use recent development versions for both packages, which you can
-download from github like this::
-
-    git clone http://github.com/dataseries/Lintel
-    git clone http://github.com/dataseries/DataSeries
-
-To build and install the two into ``<prefix>``, do::
-
-    ( cd Lintel     && mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX=<prefix> .. && make && make install )
-    ( cd DataSeries && mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX=<prefix> .. && make && make install )
-
-Please refer to the packages' documentation for more information about
-the installation process. In particular, there's more information on
-required and optional `dependencies for Lintel
-<https://raw.github.com/dataseries/Lintel/master/doc/dependencies.txt>`_
-and `dependencies for DataSeries
-<https://raw.github.com/dataseries/DataSeries/master/doc/dependencies.txt>`_.
-For users on RedHat-style systems, you'll need the following::
-
-    yum install libxml2-devel boost-devel
-
-Compiling Bro with DataSeries Support
--------------------------------------
-
-Once you have installed DataSeries, Bro's ``configure`` should pick it
-up automatically as long as it finds it in a standard system location.
-Alternatively, you can specify the DataSeries installation prefix
-manually with ``--with-dataseries=<prefix>``. Keep an eye on
-``configure``'s summary output, if it looks like the following, Bro
-found DataSeries and will compile in the support::
-
-    # ./configure --with-dataseries=/usr/local
-    [...]
-    ====================|  Bro Build Summary  |=====================
-    [...]
-    DataSeries:        true
-    [...]
-    ================================================================
-
-Activating DataSeries
----------------------
-
-The direct way to use DataSeries is to switch *all* log files over to
-the binary format. To do that, just add ``redef
-Log::default_writer=Log::WRITER_DATASERIES;`` to your ``local.bro``.
-For testing, you can also just pass that on the command line::
-
-    bro -r trace.pcap Log::default_writer=Log::WRITER_DATASERIES
-
-With that, Bro will now write all its output into DataSeries files
-``*.ds``. You can inspect these using DataSeries's set of command line
-tools, which its installation process installs into ``<prefix>/bin``.
-For example, to convert a file back into an ASCII representation::
-
-    $ ds2txt conn.log
-    [... We skip a bunch of metadata here ...]
-    ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes
-    1300475167.096535 CRCC5OdDlXe 141.142.220.202 5353 224.0.0.251 5353 udp dns 0.000000 0 0 S0 F 0 D 1 73 0 0
-    1300475167.097012 o7XBsfvo3U1 fe80::217:f2ff:fed7:cf65 5353 ff02::fb 5353 udp  0.000000 0 0 S0 F 0 D 1 199 0 0
-    1300475167.099816 pXPi1kPMgxb 141.142.220.50 5353 224.0.0.251 5353 udp  0.000000 0 0 S0 F 0 D 1 179 0 0
-    1300475168.853899 R7sOc16woCj 141.142.220.118 43927 141.142.2.2 53 udp dns 0.000435 38 89 SF F 0 Dd 1 66 1 117
-    1300475168.854378 Z6dfHVmt0X7 141.142.220.118 37676 141.142.2.2 53 udp dns 0.000420 52 99 SF F 0 Dd 1 80 1 127
-    1300475168.854837 k6T92WxgNAh 141.142.220.118 40526 141.142.2.2 53 udp dns 0.000392 38 183 SF F 0 Dd 1 66 1 211
-    [...]
-
-(``--skip-all`` suppresses the metadata.)
-
-Note that the ASCII conversion is *not* equivalent to Bro's default
-output format.
-
-You can also switch only individual files over to DataSeries by adding
-code like this to your ``local.bro``:
-
-.. code:: bro
-
-    event bro_init()
-        {
-        local f = Log::get_filter(Conn::LOG, "default"); # Get default filter for connection log.
-        f$writer = Log::WRITER_DATASERIES;               # Change writer type.
-        Log::add_filter(Conn::LOG, f);                   # Replace filter with adapted version.
-        }
-
-Bro's DataSeries writer comes with a few tuning options, see
-:doc:`/scripts/base/frameworks/logging/writers/dataseries.bro`.
-
-Working with DataSeries
-=======================
-
-Here are a few examples of using DataSeries command line tools to work
-with the output files.
-
-* Printing CSV::
-
-    $ ds2txt --csv conn.log
-    ts,uid,id.orig_h,id.orig_p,id.resp_h,id.resp_p,proto,service,duration,orig_bytes,resp_bytes,conn_state,local_orig,missed_bytes,history,orig_pkts,orig_ip_bytes,resp_pkts,resp_ip_bytes
-    1258790493.773208,ZTtgbHvf4s3,192.168.1.104,137,192.168.1.255,137,udp,dns,3.748891,350,0,S0,F,0,D,7,546,0,0
-    1258790451.402091,pOY6Rw7lhUd,192.168.1.106,138,192.168.1.255,138,udp,,0.000000,0,0,S0,F,0,D,1,229,0,0
-    1258790493.787448,pn5IiEslca9,192.168.1.104,138,192.168.1.255,138,udp,,2.243339,348,0,S0,F,0,D,2,404,0,0
-    1258790615.268111,D9slyIu3hFj,192.168.1.106,137,192.168.1.255,137,udp,dns,3.764626,350,0,S0,F,0,D,7,546,0,0
-    [...]
-
-  Add ``--separator=X`` to set a different separator.
-
-* Extracting a subset of columns::
-
-    $ ds2txt --select '*' ts,id.resp_h,id.resp_p --skip-all conn.log
-    1258790493.773208 192.168.1.255 137
-    1258790451.402091 192.168.1.255 138
-    1258790493.787448 192.168.1.255 138
-    1258790615.268111 192.168.1.255 137
-    1258790615.289842 192.168.1.255 138
-    [...]
-
-* Filtering rows::
-
-    $ ds2txt --where '*' 'duration > 5 && id.resp_p > 1024' --skip-all  conn.ds
-    1258790631.532888 V8mV5WLITu5 192.168.1.105 55890 239.255.255.250 1900 udp  15.004568 798 0 S0 F 0 D 6 966 0 0
-    1258792413.439596 tMcWVWQptvd 192.168.1.105 55890 239.255.255.250 1900 udp  15.004581 798 0 S0 F 0 D 6 966 0 0
-    1258794195.346127 cQwQMRdBrKa 192.168.1.105 55890 239.255.255.250 1900 udp  15.005071 798 0 S0 F 0 D 6 966 0 0
-    1258795977.253200 i8TEjhWd2W8 192.168.1.105 55890 239.255.255.250 1900 udp  15.004824 798 0 S0 F 0 D 6 966 0 0
-    1258797759.160217 MsLsBA8Ia49 192.168.1.105 55890 239.255.255.250 1900 udp  15.005078 798 0 S0 F 0 D 6 966 0 0
-    1258799541.068452 TsOxRWJRGwf 192.168.1.105 55890 239.255.255.250 1900 udp  15.004082 798 0 S0 F 0 D 6 966 0 0
-    [...]
-
-* Calculate some statistics:
-
-    Mean/stddev/min/max over a column::
-
-        $ dsstatgroupby '*' basic duration from conn.ds
-        # Begin DSStatGroupByModule
-        # processed 2159 rows, where clause eliminated 0 rows
-        # count(*), mean(duration), stddev, min, max
-        2159, 42.7938, 1858.34, 0, 86370
-        [...]
-
-    Quantiles of total connection volume::
-
-        $ dsstatgroupby '*' quantile 'orig_bytes + resp_bytes' from conn.ds
-        [...]
-        2159 data points, mean 24616 +- 343295 [0,1.26615e+07]
-        quantiles about every 216 data points:
-        10%: 0, 124, 317, 348, 350, 350, 601, 798, 1469
-        tails: 90%: 1469, 95%: 7302, 99%: 242629, 99.5%: 1226262
-        [...]
-
-The ``man`` pages for these tools show further options, and their
-``-h`` option gives some more information (either can be a bit cryptic
-unfortunately though).
-
-Deficiencies
-------------
-
-Due to limitations of the DataSeries format, one cannot inspect its
-files before they have been fully written. In other words, when using
-DataSeries, it's currently not possible to inspect the live log
-files inside the spool directory before they are rotated to their
-final location. It seems that this could be fixed with some effort,
-and we will work with DataSeries development team on that if the
-format gains traction among Bro users.
-
-Likewise, we're considering writing custom command line tools for
-interacting with DataSeries files, making that a bit more convenient
-than what the standard utilities provide.

doc/frameworks/logging-elasticsearch.rst

@@ -1,89 +0,0 @@
-
-=========================================
-Indexed Logging Output with ElasticSearch
-=========================================
-
-.. rst-class:: opening
-
-   Bro's default ASCII log format is not exactly the most efficient
-   way for searching large volumes of data. ElasticSearch
-   is a new data storage technology for dealing with tons of data.
-   It's also a search engine built on top of Apache's Lucene
-   project. It scales very well, both for distributed indexing and 
-   distributed searching.
-
-.. contents::
-
-Warning
--------
-
-This writer plugin is still in testing and is not yet recommended for
-production use!  The approach to how logs are handled in the plugin is "fire
-and forget" at this time, there is no error handling if the server fails to
-respond successfully to the insertion request.
-
-Installing ElasticSearch
-------------------------
-
-Download the latest version from: http://www.elasticsearch.org/download/.
-Once extracted, start ElasticSearch with::
-
-# ./bin/elasticsearch
-
-For more detailed information, refer to the ElasticSearch installation
-documentation: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/setup.html
-
-Compiling Bro with ElasticSearch Support
-----------------------------------------
-
-First, ensure that you have libcurl installed then run configure::
-
-    # ./configure
-    [...]
-    ====================|  Bro Build Summary  |=====================
-    [...]
-    cURL:              true
-    [...]
-    ElasticSearch:     true
-    [...]
-    ================================================================
-
-Activating ElasticSearch
-------------------------
-
-The easiest way to enable ElasticSearch output is to load the
-tuning/logs-to-elasticsearch.bro script.  If you are using BroControl,
-the following line in local.bro will enable it:
-
-.. console::
-
-    @load tuning/logs-to-elasticsearch
-
-With that, Bro will now write most of its logs into ElasticSearch in addition
-to maintaining the Ascii logs like it would do by default.  That script has
-some tunable options for choosing which logs to send to ElasticSearch, refer
-to the autogenerated script documentation for those options.
-
-There is an interface being written specifically to integrate with the data
-that Bro outputs into ElasticSearch named Brownian.  It can be found here::
-
-    https://github.com/grigorescu/Brownian
-
-Tuning
-------
-
-A common problem encountered with ElasticSearch is too many files being held
-open.  The ElasticSearch website has some suggestions on how to increase the
-open file limit.
-
-  - http://www.elasticsearch.org/tutorials/too-many-open-files/
-
-TODO
-----
-
-Lots.
-
-- Perform multicast discovery for server.
-- Better error detection.
-- Better defaults (don't index loaded-plugins, for instance).
-- 

doc/frameworks/logging.rst

@@ -38,7 +38,7 @@ Bro's logging interface is built around three main abstractions:
     Writers
         A writer defines the actual output format for the information
         being logged. At the moment, Bro comes with only one type of
-        writer, which produces tab separated ASCII files. In the 
+        writer, which produces tab separated ASCII files. In the
         future we will add further writers, like for binary output and
         direct logging into a database.
 
@@ -98,7 +98,7 @@ Note the fields that are set for the filter:
     ``include``
         A set limiting the fields to the ones given. The names
         correspond to those in the :bro:type:`Conn::Info` record, with
-        sub-records unrolled by concatenating fields (separated with 
+        sub-records unrolled by concatenating fields (separated with
         dots).
 
 Using the code above, you will now get a new log file ``origs.log``
@@ -155,7 +155,7 @@ that returns the desired path:
         {
         local filter: Log::Filter = [$name="conn-split", $path_func=split_log, $include=set("ts", "id.orig_h")];
         Log::add_filter(Conn::LOG, filter);
-        }   
+        }
 
 Running this will now produce two files, ``local.log`` and
 ``remote.log``, with the corresponding entries. One could extend this
@@ -263,7 +263,7 @@ specific destination exceeds a certain duration:
 .. code:: bro
 
     redef enum Notice::Type += {
-        ## Indicates that a connection remained established longer 
+        ## Indicates that a connection remained established longer
         ## than 5 minutes.
         Long_Conn_Found
     };
@@ -271,8 +271,8 @@ specific destination exceeds a certain duration:
     event Conn::log_conn(rec: Conn::Info)
         {
         if ( rec$duration > 5mins )
-            NOTICE([$note=Long_Conn_Found, 
+            NOTICE([$note=Long_Conn_Found,
-                    $msg=fmt("unusually long conn to %s", rec$id$resp_h), 
+                    $msg=fmt("unusually long conn to %s", rec$id$resp_h),
                     $id=rec$id]);
         }
 
@@ -335,11 +335,11 @@ example for the ``Foo`` module:
         # Define a hook event. By convention, this is called
         # "log_<stream>".
         global log_foo: event(rec: Info);
-        
+
     }
 
     # This event should be handled at a higher priority so that when
-    # users modify your stream later and they do it at priority 0, 
+    # users modify your stream later and they do it at priority 0,
     # their code runs after this.
     event bro_init() &priority=5
         {
@@ -356,7 +356,7 @@ it easily accessible across event handlers:
         foo: Info &optional;
         }
 
-Now you can use the :bro:id:`Log::write` method to output log records and 
+Now you can use the :bro:id:`Log::write` method to output log records and
 save the logged ``Foo::Info`` record into the connection record:
 
 .. code:: bro
@@ -380,11 +380,11 @@ uncommon to need to delete that data before the end of the connection.
 Other Writers
 -------------
 
-Bro supports the following output formats other than ASCII:
+Bro supports the following built-in output formats other than ASCII:
 
 .. toctree::
    :maxdepth: 1
 
-   logging-dataseries
-   logging-elasticsearch
    logging-input-sqlite
+
+Further formats are available as external plugins.

doc/frameworks/notice.rst

@@ -271,7 +271,7 @@ script that is generating the notice has indicated to the notice framework how
 to identify notices that are intrinsically the same. Identification of these
 "intrinsically duplicate" notices is implemented with an optional field in
 :bro:see:`Notice::Info` records named ``$identifier`` which is a simple string.
-If the ``$identifier`` and ``$type`` fields are the same for two notices, the
+If the ``$identifier`` and ``$note`` fields are the same for two notices, the
 notice framework actually considers them to be the same thing and can use that
 information to suppress duplicates for a configurable period of time.
 

doc/httpmonitor/file_extraction.bro

@@ -7,18 +7,15 @@ global mime_to_ext: table[string] of string = {
 	["text/html"] = "html",
 };
 
-event file_new(f: fa_file)
+event file_mime_type(f: fa_file, mime_type: string)
 	{
 	if ( f$source != "HTTP" )
 		return;
 
-	if ( ! f?$mime_type )
+	if ( mime_type !in mime_to_ext )
 		return;
 
-	if ( f$mime_type !in mime_to_ext )
+	local fname = fmt("%s-%s.%s", f$source, f$id, mime_to_ext[mime_type]);
-		return;
-
-	local fname = fmt("%s-%s.%s", f$source, f$id, mime_to_ext[f$mime_type]);
 	print fmt("Extracting file %s", fname);
 	Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]);
-	}
+	}

doc/index.rst

@@ -31,6 +31,7 @@ Using Bro Section
    httpmonitor/index.rst
    broids/index.rst
    mimestats/index.rst
+   scripting/index.rst
 
 ..
 
@@ -40,12 +41,17 @@ Reference Section
 .. toctree::
    :maxdepth: 2
 
-   scripting/index.rst
    frameworks/index.rst
    script-reference/index.rst
    components/index.rst
 
-..
+Development
+===========
+
+.. toctree::
+   :maxdepth: 2
+
+   devel/plugins.rst
 
 * :ref:`General Index <genindex>`
 * :ref:`search`

doc/install/install.rst

@@ -35,7 +35,7 @@ before you begin:
 
 To build Bro from source, the following additional dependencies are required:
 
-    * CMake 2.6.3 or greater            (http://www.cmake.org)
+    * CMake 2.8 or greater              (http://www.cmake.org)
     * Make
     * C/C++ compiler
     * SWIG                              (http://www.swig.org)
@@ -91,7 +91,6 @@ build time:
 
     * LibGeoIP (for geolocating IP addresses)
     * sendmail (enables Bro and BroControl to send mail)
-    * gawk (enables all features of bro-cut)
     * curl (used by a Bro script that implements active HTTP)
     * gperftools (tcmalloc is used to improve memory and CPU usage)
     * ipsumdump (for trace-summary; http://www.cs.ucla.edu/~kohler/ipsumdump)
@@ -181,7 +180,7 @@ automatically.  Finally, use ``make install-aux`` to install some of
 the other programs that are in the ``aux/bro-aux`` directory.
 
 OpenBSD users, please see our `FAQ
-<http://www.bro.org/documentation/faq.html>`_ if you are having
+<//www.bro.org/documentation/faq.html>`_ if you are having
 problems installing Bro.
 
 Finally, if you want to build the Bro documentation (not required, because

doc/logs/index.rst

@@ -111,7 +111,9 @@ default, including:
     such "crud" that is usually not worth following up on.
 
 As you can see, some log files are specific to a particular protocol,
-while others aggregate information across different types of activity.
+while others aggregate information across different types of activity. 
+For a complete list of log files and a description of its purpose, 
+see :doc:`Log Files <../script-reference/log-files>`.
 
 .. _bro-cut:
 
@@ -162,8 +164,8 @@ tools like ``awk`` allow you to indicate the log file as a command
 line option, bro-cut only takes input through redirection such as
 ``|`` and ``<``.  There are a couple of ways to direct log file data
 into ``bro-cut``, each dependent upon the type of log file you're
-processing.  A caveat of its use, however, is that the 8 lines of
+processing.  A caveat of its use, however, is that all of the
-header data must be present.
+header lines must be present.
 
 .. note::
 
@@ -177,8 +179,8 @@ moving the current log file into a directory with format
 ``YYYY-MM-DD`` and gzip compressing the file with a file format that
 includes the log file type and time range of the file.  In the case of
 processing a compressed log file you simply adjust your command line
-tools to use the complementary ``z*`` versions of commands such as cat
+tools to use the complementary ``z*`` versions of commands such as ``cat``
-(``zcat``), ``grep`` (``zgrep``), and ``head`` (``zhead``).
+(``zcat``) or ``grep`` (``zgrep``).
 
 Working with Timestamps
 -----------------------
@@ -250,44 +252,3 @@ protocol, it can have multiple ``GET``/``POST``/etc requests in a
 stream and Bro is able to extract and track that information for you,
 giving you an in-depth and structured view into HTTP traffic on your
 network.
-
------------------------
-Common Log Files
------------------------
-As a monitoring tool, Bro records a detailed view of the traffic inspected 
-and the events generated in a series of relevant log files. These files can 
-later be reviewed for monitoring, auditing and troubleshooting purposes.
-
-In this section we present a brief explanation of the most commonly used log 
-files generated by Bro including links to descriptions of some of the fields 
-for each log type.
-
-+-----------------+---------------------------------------+------------------------------+
-| Log File        | Description                           | Field Descriptions           |
-+=================+=======================================+==============================+
-| http.log        | Shows all HTTP requests and replies   | :bro:type:`HTTP::Info`       |
-+-----------------+---------------------------------------+------------------------------+
-| ftp.log         | Records FTP activity                  | :bro:type:`FTP::Info`        |
-+-----------------+---------------------------------------+------------------------------+
-| ssl.log         | Records SSL sessions including        | :bro:type:`SSL::Info`        |
-|                 | certificates used                     |                              |
-+-----------------+---------------------------------------+------------------------------+
-| known_certs.log | Includes SSL certificates used        | :bro:type:`Known::CertsInfo` |
-+-----------------+---------------------------------------+------------------------------+
-| smtp.log        | Summarizes SMTP traffic on a network  | :bro:type:`SMTP::Info`       |
-+-----------------+---------------------------------------+------------------------------+
-| dns.log         | Shows all DNS activity on a network   | :bro:type:`DNS::Info`        |
-+-----------------+---------------------------------------+------------------------------+
-| conn.log        | Records all connections seen by Bro   | :bro:type:`Conn::Info`       |
-+-----------------+---------------------------------------+------------------------------+
-| dpd.log         | Shows network activity on             | :bro:type:`DPD::Info`        |
-|                 | non-standard ports                    |                              |
-+-----------------+---------------------------------------+------------------------------+
-| files.log       | Records information about all files   | :bro:type:`Files::Info`      |
-|                 | transmitted over the network          |                              |
-+-----------------+---------------------------------------+------------------------------+
-| weird.log       | Records unexpected protocol-level     | :bro:type:`Weird::Info`      |
-|                 | activity                              |                              |
-+-----------------+---------------------------------------+------------------------------+
-
-

doc/quickstart/index.rst

@@ -1,5 +1,5 @@
 
-.. _FAQ: http://www.bro.org/documentation/faq.html
+.. _FAQ: //www.bro.org/documentation/faq.html
 
 .. _quickstart:
 
@@ -234,7 +234,7 @@ is valid before installing it and then restarting the Bro instance:
 .. console::
 
    [BroControl] > check
-   bro is ok.
+   bro scripts are ok.
    [BroControl] > install
    removing old policies in /usr/local/bro/spool/policy/site ... done.
    removing old policies in /usr/local/bro/spool/policy/auto ... done.
@@ -250,15 +250,15 @@ is valid before installing it and then restarting the Bro instance:
 
 Now that the SSL notice is ignored, let's look at how to send an email on
 the SSH notice.  The notice framework has a similar option called
-``emailed_types``, but that can't differentiate between SSH servers and we
+``emailed_types``, but using that would generate email for all SSH servers and
-only want email for logins to certain ones.  Then we come to the ``PolicyItem``
+we only want email for logins to certain ones.  There is a ``policy`` hook
-record and ``policy`` set and realize that those are actually what get used
+that is actually what is used to implement the simple functionality of
-to implement the simple functionality of ``ignored_types`` and
+``ignored_types`` and
 ``emailed_types``, but it's extensible such that the condition and action taken
 on notices can be user-defined.
 
-In ``local.bro``, let's add a new ``PolicyItem`` record to the ``policy`` set
+In ``local.bro``, let's define a new ``policy`` hook handler body
-that only takes the email action for SSH logins to a defined set of servers:
+that takes the email action for SSH logins only for a defined set of servers:
 
 .. code:: bro
 
@@ -276,9 +276,9 @@ that only takes the email action for SSH logins to a defined set of servers:
 
 You'll just have to trust the syntax for now, but what we've done is
 first declare our own variable to hold a set of watched addresses,
-``watched_servers``; then added a record to the policy that will generate
+``watched_servers``; then added a hook handler body to the policy that will
-an email on the condition that the predicate function evaluates to true, which
+generate an email whenever the notice type is an SSH login and the responding
-is whenever the notice type is an SSH login and the responding host stored
+host stored
 inside the ``Info`` record's connection field is in the set of watched servers.
 
 .. note:: Record field member access is done with the '$' character

doc/script-reference/attributes.rst

@@ -0,0 +1,232 @@
+Attributes
+==========
+
+The Bro scripting language supports the following attributes.
+
++-----------------------------+-----------------------------------------------+
+| Name                        | Description                                   |
++=============================+===============================================+
+| :bro:attr:`&redef`          |Redefine a global constant or extend a type.   |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&priority`       |Specify priority for event handler or hook.    |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&log`            |Mark a record field as to be written to a log. |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&optional`       |Allow a record field value to be missing.      |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&default`        |Specify a default value.                       |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&add_func`       |Specify a function to call for each "redef +=".|
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&delete_func`    |Same as "&add_func", except for "redef -=".    |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&expire_func`    |Specify a function to call when container      |
+|                             |element expires.                               |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&read_expire`    |Specify a read timeout interval.               |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&write_expire`   |Specify a write timeout interval.              |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&create_expire`  |Specify a creation timeout interval.           |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&synchronized`   |Synchronize a variable across nodes.           |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&persistent`     |Make a variable persistent (written to disk).  |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&rotate_interval`|Rotate a file after specified interval.        |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&rotate_size`    |Rotate a file after specified file size.       |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&encrypt`        |Encrypt a file when writing to disk.           |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&raw_output`     |Open file in raw mode (chars. are not escaped).|
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&mergeable`      |Prefer set union for synchronized state.       |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&group`          |Group event handlers to activate/deactivate.   |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&error_handler`  |Used internally for reporter framework events. |
++-----------------------------+-----------------------------------------------+
+| :bro:attr:`&type_column`    |Used by input framework for "port" type.       |
++-----------------------------+-----------------------------------------------+
+
+Here is a more detailed explanation of each attribute:
+
+.. bro:attr:: &redef
+
+    Allows for redefinition of initial values of global objects declared as
+    constant.
+
+    In this example, the constant (assuming it is global) can be redefined
+    with a :bro:keyword:`redef` at some later point::
+
+        const clever = T &redef;
+
+.. bro:attr:: &priority
+
+    Specifies the execution priority (as a signed integer) of a hook or
+    event handler. Higher values are executed before lower ones. The
+    default value is 0.  Example::
+
+        event bro_init() &priority=10
+        {
+            print "high priority";
+        }
+
+.. bro:attr:: &log
+
+    Writes a :bro:type:`record` field to the associated log stream.
+
+.. bro:attr:: &optional
+
+    Allows a record field value to be missing (i.e., neither initialized nor
+    ever assigned a value).
+
+    In this example, the record could be instantiated with either
+    "myrec($a=127.0.0.1)" or "myrec($a=127.0.0.1, $b=80/tcp)"::
+
+        type myrec: record { a: addr; b: port &optional; };
+
+    The ``?$`` operator can be used to check if a record field has a value or
+    not (it returns a ``bool`` value of ``T`` if the field has a value,
+    and ``F`` if not).
+
+.. bro:attr:: &default
+
+    Specifies a default value for a record field, container element, or a
+    function/hook/event parameter.
+
+    In this example, the record could be instantiated with either
+    "myrec($a=5, $c=3.14)" or "myrec($a=5, $b=53/udp, $c=3.14)"::
+
+        type myrec: record { a: count; b: port &default=80/tcp; c: double; };
+
+    In this example, the table will return the string ``"foo"`` for any
+    attempted access to a non-existing index::
+
+        global mytable: table[count] of string &default="foo";
+
+    When used with function/hook/event parameters, all of the parameters
+    with the "&default" attribute must come after all other parameters.
+    For example, the following function could be called either as "myfunc(5)"
+    or as "myfunc(5, 53/udp)"::
+
+        function myfunc(a: count, b: port &default=80/tcp)
+        {
+            print a, b;
+        }
+
+.. bro:attr:: &add_func
+
+    Can be applied to an identifier with &redef to specify a function to
+    be called any time a "redef <id> += ..." declaration is parsed.  The
+    function takes two arguments of the same type as the identifier, the first
+    being the old value of the variable and the second being the new
+    value given after the "+=" operator in the "redef" declaration.  The
+    return value of the function will be the actual new value of the
+    variable after the "redef" declaration is parsed.
+
+.. bro:attr:: &delete_func
+
+    Same as :bro:attr:`&add_func`, except for :bro:keyword:`redef` declarations
+    that use the "-=" operator.
+
+.. bro:attr:: &expire_func
+
+    Called right before a container element expires.  The function's
+    first parameter is of the same type of the container and the second
+    parameter the same type of the container's index.  The return
+    value is an :bro:type:`interval` indicating the amount of additional
+    time to wait before expiring the container element at the given
+    index (which will trigger another execution of this function).
+
+.. bro:attr:: &read_expire
+
+    Specifies a read expiration timeout for container elements. That is,
+    the element expires after the given amount of time since the last
+    time it has been read. Note that a write also counts as a read.
+
+.. bro:attr:: &write_expire
+
+    Specifies a write expiration timeout for container elements. That
+    is, the element expires after the given amount of time since the
+    last time it has been written.
+
+.. bro:attr:: &create_expire
+
+    Specifies a creation expiration timeout for container elements. That
+    is, the element expires after the given amount of time since it has
+    been inserted into the container, regardless of any reads or writes.
+
+.. bro:attr:: &synchronized
+
+    Synchronizes variable accesses across nodes. The value of a
+    ``&synchronized`` variable is automatically propagated to all peers
+    when it changes.
+
+.. bro:attr:: &persistent
+
+    Makes a variable persistent, i.e., its value is written to disk (per
+    default at shutdown time).
+
+.. bro:attr:: &rotate_interval
+
+    Rotates a file after a specified interval.
+
+.. bro:attr:: &rotate_size
+
+    Rotates a file after it has reached a given size in bytes.
+
+.. bro:attr:: &encrypt
+
+    Encrypts files right before writing them to disk.
+
+.. bro:attr:: &raw_output
+
+    Opens a file in raw mode, i.e., non-ASCII characters are not
+    escaped.
+
+.. bro:attr:: &mergeable
+
+    Prefers merging sets on assignment for synchronized state. This
+    attribute is used in conjunction with :bro:attr:`&synchronized`
+    container types: when the same container is updated at two peers
+    with different values, the propagation of the state causes a race
+    condition, where the last update succeeds. This can cause
+    inconsistencies and can be avoided by unifying the two sets, rather
+    than merely overwriting the old value.
+
+.. bro:attr:: &group
+
+    Groups event handlers such that those in the same group can be
+    jointly activated or deactivated.
+
+.. bro:attr:: &error_handler
+
+    Internally set on the events that are associated with the reporter
+    framework: :bro:id:`reporter_info`, :bro:id:`reporter_warning`, and
+    :bro:id:`reporter_error`.  It prevents any handlers of those events
+    from being able to generate reporter messages that go through any of
+    those events (i.e., it prevents an infinite event recursion).  Instead,
+    such nested reporter messages are output to stderr.
+
+.. bro:attr:: &type_column
+
+    Used by the input framework. It can be used on columns of type
+    :bro:type:`port` (such a column only contains the port number) and
+    specifies the name of an additional column in
+    the input file which specifies the protocol of the port (tcp/udp/icmp).
+
+    In the following example, the input file would contain four columns
+    named "ip", "srcp", "proto", and "msg"::
+
+        type Idx: record {
+            ip: addr;
+        };
+
+
+        type Val: record {
+            srcp: port &type_column = "proto";
+            msg: string;
+        };
+

doc/script-reference/directives.rst

@@ -0,0 +1,173 @@
+Directives
+==========
+
+The Bro scripting language supports a number of directives that can
+affect which scripts will be loaded or which lines in a script will be
+executed.  Directives are evaluated before script execution begins.
+
+.. bro:keyword:: @DEBUG
+
+    TODO
+
+
+.. bro:keyword:: @DIR
+
+    Expands to the directory pathname where the current script is located.
+
+    Example::
+
+        print "Directory:", @DIR;
+
+
+.. bro:keyword:: @FILENAME
+
+    Expands to the filename of the current script.
+
+    Example::
+
+        print "File:", @FILENAME;
+
+.. bro:keyword:: @load
+
+    Loads the specified Bro script, specified as the relative pathname
+    of the file (relative to one of the directories in Bro's file search path).
+    If the Bro script filename ends with ".bro", then you don't need to
+    specify the file extension.  The filename cannot contain any whitespace.
+
+    In this example, Bro will try to load a script
+    "policy/misc/capture-loss.bro" by looking in each directory in the file
+    search path (the file search path can be changed by setting the BROPATH
+    environment variable)::
+
+        @load policy/misc/capture-loss
+
+    If you specify the name of a directory instead of a filename, then
+    Bro will try to load a file in that directory called "__load__.bro"
+    (presumably that file will contain additional "@load" directives).
+
+    In this example, Bro will try to load a file "tuning/defaults/__load__.bro"
+    by looking in each directory in the file search path::
+
+        @load tuning/defaults
+
+    The purpose of this directive is to ensure that all script dependencies
+    are satisfied, and to avoid having to list every needed Bro script
+    on the command-line.  Bro keeps track of which scripts have been
+    loaded, so it is not an error to load a script more than once (once
+    a script has been loaded, any subsequent "@load" directives
+    for that script are ignored).
+
+
+.. bro:keyword:: @load-sigs
+
+    This works similarly to "@load", except that in this case the filename
+    represents a signature file (not a Bro script).  If the signature filename
+    ends with ".sig", then you don't need to specify the file extension
+    in the "@load-sigs" directive.  The filename cannot contain any
+    whitespace.
+
+    In this example, Bro will try to load a signature file
+    "base/protocols/ssl/dpd.sig"::
+
+        @load-sigs base/protocols/ssl/dpd
+
+    The format for a signature file is explained in the documentation for the
+    `Signature Framework <../frameworks/signatures.html>`_.
+
+
+.. bro:keyword:: @unload
+
+    This specifies a Bro script that we don't want to load (so a subsequent
+    attempt to load the specified script will be skipped).  However,
+    if the specified script has already been loaded, then this directive
+    has no affect.
+
+    In the following example, if the "policy/misc/capture-loss.bro" script
+    has not been loaded yet, then Bro will not load it::
+
+        @unload policy/misc/capture-loss
+
+
+.. bro:keyword:: @prefixes
+
+    Specifies a filename prefix to use when looking for script files
+    to load automatically.  The prefix cannot contain any whitespace.
+
+    In the following example, the prefix "cluster" is used and all prefixes
+    that were previously specified are not used::
+
+        @prefixes = cluster
+
+    In the following example, the prefix "cluster-manager" is used in
+    addition to any previously-specified prefixes::
+
+        @prefixes += cluster-manager
+
+    The way this works is that after Bro parses all script files, then for each
+    loaded script Bro will take the absolute path of the script and then
+    it removes the portion of the directory path that is in Bro's file
+    search path.  Then it replaces each "/" character with a period "."
+    and then prepends the prefix (specified in the "@prefixes" directive)
+    followed by a period.  The resulting filename is searched for in each
+    directory in Bro's file search path.  If a matching file is found, then
+    the file is automatically loaded.
+
+    For example, if a script called "local.bro" has been loaded, and a prefix
+    of "test" was specified, then Bro will look for a file named
+    "test.local.bro" in each directory of Bro's file search path.
+
+    An alternative way to specify prefixes is to use the "-p" Bro
+    command-line option.
+
+.. bro:keyword:: @if
+
+    The specified expression must evaluate to type :bro:type:`bool`.  If the
+    value is true, then the following script lines (up to the next "@else"
+    or "@endif") are available to be executed.
+
+    Example::
+
+        @if ( ver == 2 )
+            print "version 2 detected";
+        @endif
+
+.. bro:keyword:: @ifdef
+
+    This works like "@if", except that the result is true if the specified
+    identifier is defined.
+
+    Example::
+
+        @ifdef ( pi )
+            print "pi is defined";
+        @endif
+
+.. bro:keyword:: @ifndef
+
+    This works exactly like "@ifdef", except that the result is true if the
+    specified identifier is not defined.
+
+    Example::
+
+        @ifndef ( pi )
+            print "pi is not defined";
+        @endif
+
+.. bro:keyword:: @else
+
+    This directive is optional after an "@if", "@ifdef", or
+    "@ifndef".  If present, it provides an else clause.
+
+    Example::
+
+        @ifdef ( pi )
+            print "pi is defined";
+        @else
+            print "pi is not defined";
+        @endif
+
+.. bro:keyword:: @endif
+
+    This directive is required to terminate each "@if", "@ifdef", or
+    "@ifndef".
+

doc/script-reference/index.rst

@@ -5,10 +5,17 @@ Script Reference
 .. toctree::
    :maxdepth: 1
 
+   operators
+   types
+   attributes
+   statements
+   directives
+   log-files
    notices
    proto-analyzers
    file-analyzers
-   builtins
    packages
    scripts
    Broxygen Example Script </scripts/broxygen/example.bro>
+
+

doc/script-reference/log-files.rst

@@ -0,0 +1,148 @@
+=========
+Log Files
+=========
+
+Listed below are the log files generated by Bro, including a brief description
+of the log file and links to descriptions of the fields for each log
+type.
+
+Network Protocols
+-----------------
+
++----------------------------+---------------------------------------+---------------------------------+
+| Log File                   | Description                           | Field Descriptions              |
++============================+=======================================+=================================+
+| conn.log                   | TCP/UDP/ICMP connections              | :bro:type:`Conn::Info`          |
++----------------------------+---------------------------------------+---------------------------------+
+| dhcp.log                   | DHCP leases                           | :bro:type:`DHCP::Info`          |
++----------------------------+---------------------------------------+---------------------------------+
+| dnp3.log                   | DNP3 requests and replies             | :bro:type:`DNP3::Info`          |
++----------------------------+---------------------------------------+---------------------------------+
+| dns.log                    | DNS activity                          | :bro:type:`DNS::Info`           |
++----------------------------+---------------------------------------+---------------------------------+
+| ftp.log                    | FTP activity                          | :bro:type:`FTP::Info`           |
++----------------------------+---------------------------------------+---------------------------------+
+| http.log                   | HTTP requests and replies             | :bro:type:`HTTP::Info`          |
++----------------------------+---------------------------------------+---------------------------------+
+| irc.log                    | IRC commands and responses            | :bro:type:`IRC::Info`           |
++----------------------------+---------------------------------------+---------------------------------+
+| modbus.log                 | Modbus commands and responses         | :bro:type:`Modbus::Info`        |
++----------------------------+---------------------------------------+---------------------------------+
+| modbus_register_change.log | Tracks changes to Modbus holding      | :bro:type:`Modbus::MemmapInfo`  |
+|                            | registers                             |                                 |
++----------------------------+---------------------------------------+---------------------------------+
+| radius.log                 | RADIUS authentication attempts        | :bro:type:`RADIUS::Info`        |
++----------------------------+---------------------------------------+---------------------------------+
+| smtp.log                   | SMTP transactions                     | :bro:type:`SMTP::Info`          |
++----------------------------+---------------------------------------+---------------------------------+
+| snmp.log                   | SNMP messages                         | :bro:type:`SNMP::Info`          |
++----------------------------+---------------------------------------+---------------------------------+
+| socks.log                  | SOCKS proxy requests                  | :bro:type:`SOCKS::Info`         |
++----------------------------+---------------------------------------+---------------------------------+
+| ssh.log                    | SSH connections                       | :bro:type:`SSH::Info`           |
++----------------------------+---------------------------------------+---------------------------------+
+| ssl.log                    | SSL/TLS handshake info                | :bro:type:`SSL::Info`           |
++----------------------------+---------------------------------------+---------------------------------+
+| syslog.log                 | Syslog messages                       | :bro:type:`Syslog::Info`        |
++----------------------------+---------------------------------------+---------------------------------+
+| tunnel.log                 | Tunneling protocol events             | :bro:type:`Tunnel::Info`        |
++----------------------------+---------------------------------------+---------------------------------+
+
+Files
+-----
+
++----------------------------+---------------------------------------+---------------------------------+
+| Log File                   | Description                           | Field Descriptions              |
++============================+=======================================+=================================+
+| files.log                  | File analysis results                 | :bro:type:`Files::Info`         |
++----------------------------+---------------------------------------+---------------------------------+
+| x509.log                   | X.509 certificate info                | :bro:type:`X509::Info`          |
++----------------------------+---------------------------------------+---------------------------------+
+
+Detection
+---------
+
++----------------------------+---------------------------------------+---------------------------------+
+| Log File                   | Description                           | Field Descriptions              |
++============================+=======================================+=================================+
+| intel.log                  | Intelligence data matches             | :bro:type:`Intel::Info`         |
++----------------------------+---------------------------------------+---------------------------------+
+| notice.log                 | Bro notices                           | :bro:type:`Notice::Info`        |
++----------------------------+---------------------------------------+---------------------------------+
+| notice_alarm.log           | The alarm stream                      | :bro:enum:`Notice::ACTION_ALARM`|
++----------------------------+---------------------------------------+---------------------------------+
+| signatures.log             | Signature matches                     | :bro:type:`Signatures::Info`    |
++----------------------------+---------------------------------------+---------------------------------+
+| traceroute.log             | Traceroute detection                  | :bro:type:`Traceroute::Info`    |
++----------------------------+---------------------------------------+---------------------------------+
+
+
+Network Observations
+--------------------
+
++----------------------------+---------------------------------------+---------------------------------+
+| Log File                   | Description                           | Field Descriptions              |
++============================+=======================================+=================================+
+| app_stats.log              | Web app usage statistics              | :bro:type:`AppStats::Info`      |
++----------------------------+---------------------------------------+---------------------------------+
+| known_certs.log            | SSL certificates                      | :bro:type:`Known::CertsInfo`    |
++----------------------------+---------------------------------------+---------------------------------+
+| known_devices.log          | MAC addresses of devices on the       | :bro:type:`Known::DevicesInfo`  |
+|                            | network                               |                                 |
++----------------------------+---------------------------------------+---------------------------------+
+| known_hosts.log            | Hosts that have completed TCP         | :bro:type:`Known::HostsInfo`    |
+|                            | handshakes                            |                                 |
++----------------------------+---------------------------------------+---------------------------------+
+| known_modbus.log           | Modbus masters and slaves             | :bro:type:`Known::ModbusInfo`   |
++----------------------------+---------------------------------------+---------------------------------+
+| known_services.log         | Services running on hosts             | :bro:type:`Known::ServicesInfo` |
++----------------------------+---------------------------------------+---------------------------------+
+| software.log               | Software being used on the network    | :bro:type:`Software::Info`      |
++----------------------------+---------------------------------------+---------------------------------+
+
+Miscellaneous
+-------------
+
++----------------------------+---------------------------------------+---------------------------------+
+| Log File                   | Description                           | Field Descriptions              |
++============================+=======================================+=================================+
+| barnyard2.log              | Alerts received from Barnyard2        | :bro:type:`Barnyard2::Info`     |
++----------------------------+---------------------------------------+---------------------------------+
+| dpd.log                    | Dynamic protocol detection failures   | :bro:type:`DPD::Info`           |
++----------------------------+---------------------------------------+---------------------------------+
+| unified2.log               | Interprets Snort's unified output     | :bro:type:`Unified2::Info`      |
++----------------------------+---------------------------------------+---------------------------------+
+| weird.log                  | Unexpected network-level activity     | :bro:type:`Weird::Info`         |
++----------------------------+---------------------------------------+---------------------------------+
+
+Bro Diagnostics
+---------------
+
++----------------------------+---------------------------------------+---------------------------------+
+| Log File                   | Description                           | Field Descriptions              |
++============================+=======================================+=================================+
+| capture_loss.log           | Packet loss rate                      | :bro:type:`CaptureLoss::Info`   |
++----------------------------+---------------------------------------+---------------------------------+
+| cluster.log                | Bro cluster messages                  | :bro:type:`Cluster::Info`       |
++----------------------------+---------------------------------------+---------------------------------+
+| communication.log          | Communication events between Bro or   | :bro:type:`Communication::Info` |
+|                            | Broccoli instances                    |                                 |
++----------------------------+---------------------------------------+---------------------------------+
+| loaded_scripts.log         | Shows all scripts loaded by Bro       | :bro:type:`LoadedScripts::Info` |
++----------------------------+---------------------------------------+---------------------------------+
+| packet_filter.log          | List packet filters that were applied | :bro:type:`PacketFilter::Info`  |
++----------------------------+---------------------------------------+---------------------------------+
+| prof.log                   | Profiling statistics (to create this  | N/A                             |
+|                            | log, load policy/misc/profiling.bro)  |                                 |
++----------------------------+---------------------------------------+---------------------------------+
+| reporter.log               | Internal error/warning/info messages  | :bro:type:`Reporter::Info`      |
++----------------------------+---------------------------------------+---------------------------------+
+| stats.log                  | Memory/event/packet/lag statistics    | :bro:type:`Stats::Info`         |
++----------------------------+---------------------------------------+---------------------------------+
+| stderr.log                 | Captures standard error when Bro is   | N/A                             |
+|                            | started from BroControl               |                                 |
++----------------------------+---------------------------------------+---------------------------------+
+| stdout.log                 | Captures standard output when Bro is  | N/A                             |
+|                            | started from BroControl               |                                 |
++----------------------------+---------------------------------------+---------------------------------+
+

doc/script-reference/operators.rst

@@ -0,0 +1,191 @@
+Operators
+=========
+
+The Bro scripting language supports the following operators.  Note that
+each data type only supports a subset of these operators.  For more
+details, see the documentation about the `data types <types.html>`_.
+
+Relational operators
+--------------------
+
+The relational operators evaluate to type :bro:type:`bool`.
+
++------------------------------+--------------+
+| Name                         | Syntax       |
++==============================+==============+
+| Equality                     | *a* == *b*   |
++------------------------------+--------------+
+| Inequality                   | *a* != *b*   |
++------------------------------+--------------+
+| Less than                    | *a* < *b*    |
++------------------------------+--------------+
+| Less than or equal           | *a* <= *b*   |
++------------------------------+--------------+
+| Greater than                 | *a* > *b*    |
++------------------------------+--------------+
+| Greater than or equal        | *a* >= *b*   |
++------------------------------+--------------+
+
+
+Logical operators
+-----------------
+
+The logical operators require operands of type :bro:type:`bool`, and
+evaluate to type :bro:type:`bool`.
+
++------------------------------+--------------+
+| Name                         | Syntax       |
++==============================+==============+
+| Logical AND                  | *a* && *b*   |
++------------------------------+--------------+
+| Logical OR                   | *a* \|\| *b* |
++------------------------------+--------------+
+| Logical NOT                  | ! *a*        |
++------------------------------+--------------+
+
+
+Arithmetic operators
+--------------------
+
++------------------------------+-------------+-------------------------------+
+| Name                         | Syntax      | Notes                         |
++==============================+=============+===============================+
+| Addition                     | *a* + *b*   | For :bro:type:`string`        |
+|                              |             | operands, this performs       |
+|                              |             | string concatenation.         |
++------------------------------+-------------+-------------------------------+
+| Subtraction                  | *a* - *b*   |                               |
++------------------------------+-------------+-------------------------------+
+| Multiplication               | *a* \* *b*  |                               |
++------------------------------+-------------+-------------------------------+
+| Division                     | *a* / *b*   | For :bro:type:`int` or        |
+|                              |             | :bro:type:`count` operands,   |
+|                              |             | the fractional part of the    |
+|                              |             | result is dropped.            |
++------------------------------+-------------+-------------------------------+
+| Modulo                       | *a* % *b*   | Operand types cannot be       |
+|                              |             | "double".                     |
++------------------------------+-------------+-------------------------------+
+| Unary plus                   | \+ *a*      |                               |
++------------------------------+-------------+-------------------------------+
+| Unary minus                  | \- *a*      |                               |
++------------------------------+-------------+-------------------------------+
+| Pre-increment                | ++ *a*      | Operand type cannot be        |
+|                              |             | "double".                     |
++------------------------------+-------------+-------------------------------+
+| Pre-decrement                | ``--`` *a*  | Operand type cannot be        |
+|                              |             | "double".                     |
++------------------------------+-------------+-------------------------------+
+| Absolute value               | \| *a* \|   | If operand is                 |
+|                              |             | :bro:type:`string`,           |
+|                              |             | :bro:type:`set`,              |
+|                              |             | :bro:type:`table`, or         |
+|                              |             | :bro:type:`vector`, this      |
+|                              |             | evaluates to number           |
+|                              |             | of elements.                  |
++------------------------------+-------------+-------------------------------+
+
+
+Assignment operators
+--------------------
+
+The assignment operators evaluate to the result of the assignment.
+
++------------------------------+-------------+
+| Name                         | Syntax      |
++==============================+=============+
+| Assignment                   | *a* = *b*   |
++------------------------------+-------------+
+| Addition assignment          | *a* += *b*  |
++------------------------------+-------------+
+| Subtraction assignment       | *a* -= *b*  |
++------------------------------+-------------+
+
+
+Record field operators
+----------------------
+
+The record field operators take a :bro:type:`record` as the first operand,
+and a field name as the second operand.  For both operators, the specified
+field name must be in the declaration of the record type.
+
++------------------------------+-------------+-------------------------------+
+| Name                         | Syntax      | Notes                         |
++==============================+=============+===============================+
+| Field access                 | *a* $ *b*   |                               |
++------------------------------+-------------+-------------------------------+
+| Field value existence test   | *a* ?$ *b*  | Evaluates to type             |
+|                              |             | :bro:type:`bool`.             |
+|                              |             | True if the specified field   |
+|                              |             | has been assigned a value, or |
+|                              |             | false if not.                 |
++------------------------------+-------------+-------------------------------+
+
+
+Other operators
+---------------
+
++--------------------------------+-------------------+------------------------+
+| Name                           | Syntax            | Notes                  |
++================================+===================+========================+
+| Membership test                | *a* in *b*        |Evaluates to type       |
+|                                |                   |:bro:type:`bool`. Do not|
+|                                |                   |confuse this use of "in"|
+|                                |                   |with that used in a     |
+|                                |                   |:bro:keyword:`for`      |
+|                                |                   |statement.              |
++--------------------------------+-------------------+------------------------+
+| Non-membership test            | *a* !in *b*       |This is the logical NOT |
+|                                |                   |of the "in" operator.   |
+|                                |                   |For example: "a !in b"  |
+|                                |                   |is equivalent to        |
+|                                |                   |"!(a in b)".            |
++--------------------------------+-------------------+------------------------+
+| Table or vector element access | *a* [ *b* ]       |This operator can also  |
+|                                |                   |be used with a          |
+|                                |                   |:bro:type:`set`, but    |
+|                                |                   |only with the           |
+|                                |                   |:bro:keyword:`add` or   |
+|                                |                   |:bro:keyword:`delete`   |
+|                                |                   |statement.              |
++--------------------------------+-------------------+------------------------+
+| Substring extraction           | *a* [ *b* : *c* ] |See the                 |
+|                                |                   |:bro:type:`string` type |
+|                                |                   |for more details.       |
++--------------------------------+-------------------+------------------------+
+| Create a deep copy             | copy ( *a* )      |This is relevant only   |
+|                                |                   |for data types that are |
+|                                |                   |assigned by reference,  |
+|                                |                   |such as                 |
+|                                |                   |:bro:type:`vector`,     |
+|                                |                   |:bro:type:`set`,        |
+|                                |                   |:bro:type:`table`,      |
+|                                |                   |and :bro:type:`record`. |
++--------------------------------+-------------------+------------------------+
+| Module namespace access        | *a* \:\: *b*      |The first operand is the|
+|                                |                   |module name, and the    |
+|                                |                   |second operand is an    |
+|                                |                   |identifier that refers  |
+|                                |                   |to a global variable,   |
+|                                |                   |enumeration constant, or|
+|                                |                   |user-defined type that  |
+|                                |                   |was exported from the   |
+|                                |                   |module.                 |
++--------------------------------+-------------------+------------------------+
+| Conditional                    | *a* ? *b* : *c*   |The first operand must  |
+|                                |                   |evaluate to type        |
+|                                |                   |:bro:type:`bool`.       |
+|                                |                   |If true, then the       |
+|                                |                   |second expression is    |
+|                                |                   |evaluated and is the    |
+|                                |                   |result of the entire    |
+|                                |                   |expression.  Otherwise, |
+|                                |                   |the third expression is |
+|                                |                   |evaluated and is the    |
+|                                |                   |result of the entire    |
+|                                |                   |expression. The types of|
+|                                |                   |the second and third    |
+|                                |                   |operands must be        |
+|                                |                   |compatible.             |
++--------------------------------+-------------------+------------------------+
+

doc/script-reference/statements.rst

@@ -0,0 +1,602 @@
+Declarations and Statements
+===========================
+
+The Bro scripting language supports the following declarations and
+statements.
+
+
+Declarations
+~~~~~~~~~~~~
+
++----------------------------+-----------------------------+
+| Name                       | Description                 |
++============================+=============================+
+| :bro:keyword:`module`      | Change the current module   |
++----------------------------+-----------------------------+
+| :bro:keyword:`export`      | Export identifiers from the |
+|                            | current module              |
++----------------------------+-----------------------------+
+| :bro:keyword:`global`      | Declare a global variable   |
++----------------------------+-----------------------------+
+| :bro:keyword:`const`       | Declare a constant          |
++----------------------------+-----------------------------+
+| :bro:keyword:`type`        | Declare a user-defined type |
++----------------------------+-----------------------------+
+| :bro:keyword:`redef`       | Redefine a global value or  |
+|                            | extend a user-defined type  |
++----------------------------+-----------------------------+
+| `function/event/hook`_     | Declare a function, event   |
+|                            | handler, or hook            |
++----------------------------+-----------------------------+
+
+Statements
+~~~~~~~~~~
+
++----------------------------+------------------------+
+| Name                       | Description            |
++============================+========================+
+| :bro:keyword:`local`       | Declare a local        |
+|                            | variable               |
++----------------------------+------------------------+
+| :bro:keyword:`add`,        | Add or delete          |
+| :bro:keyword:`delete`      | elements               |
++----------------------------+------------------------+
+| :bro:keyword:`print`       | Print to stdout or a   |
+|                            | file                   |
++----------------------------+------------------------+
+| :bro:keyword:`for`,        | Loop over each         |
+| :bro:keyword:`next`,       | element in a container |
+| :bro:keyword:`break`       | object                 |
++----------------------------+------------------------+
+| :bro:keyword:`if`          | Evaluate boolean       |
+|                            | expression and if true,|
+|                            | execute a statement    |
++----------------------------+------------------------+
+| :bro:keyword:`switch`,     | Evaluate expression    |
+| :bro:keyword:`break`,      | and execute statement  |
+| :bro:keyword:`fallthrough` | with a matching value  |
++----------------------------+------------------------+
+| :bro:keyword:`when`        | Asynchronous execution |
++----------------------------+------------------------+
+| :bro:keyword:`event`,      | Invoke or schedule     |
+| :bro:keyword:`schedule`    | an event handler       |
++----------------------------+------------------------+
+| :bro:keyword:`return`      | Return from function,  |
+|                            | hook, or event handler |
++----------------------------+------------------------+
+
+Declarations
+------------
+
+The following global declarations cannot occur within a function, hook, or
+event handler.  Also, these declarations cannot appear after any statements
+that are outside of a function, hook, or event handler.
+
+.. bro:keyword:: module
+
+    The "module" keyword is used to change the current module.  This
+    affects the scope of any subsequently declared global identifiers.
+
+    Example::
+
+        module mymodule;
+
+    If a global identifier is declared after a "module" declaration,
+    then its scope ends at the end of the current Bro script or at the
+    next "module" declaration, whichever comes first.  However, if a
+    global identifier is declared after a "module" declaration, but inside
+    an :bro:keyword:`export` block, then its scope ends at the end of the
+    last loaded Bro script, but it must be referenced using the namespace
+    operator (``::``) in other modules.
+
+    There can be any number of "module" declarations in a Bro script.
+    The same "module" declaration can appear in any number of different
+    Bro scripts.
+
+
+.. bro:keyword:: export
+
+    An "export" block contains one or more declarations
+    (no statements are allowed in an "export" block) that the current
+    module is exporting.  This enables these global identifiers to be visible
+    in other modules (but not prior to their declaration) via the namespace
+    operator (``::``).  See the :bro:keyword:`module` keyword for a more
+    detailed explanation.
+
+    Example::
+
+        export {
+            redef enum Log::ID += { LOG };
+
+            type Info: record {
+                ts: time &log;
+                uid: string &log;
+            };
+
+            const conntime = 30sec &redef;
+        }
+
+    Note that the braces in an "export" block are always required
+    (they do not indicate a compound statement).  Also, no semicolon is
+    needed to terminate an "export" block.
+
+.. bro:keyword:: global
+
+    Variables declared with the "global" keyword will be global.
+    If a type is not specified, then an initializer is required so that
+    the type can be inferred.  Likewise, if an initializer is not supplied,
+    then the type must be specified.  Example::
+
+        global pi = 3.14;
+        global hosts: set[addr];
+        global ciphers: table[string] of string = table();
+
+    Variable declarations outside of any function, hook, or event handler are
+    required to use this keyword (unless they are declared with the
+    :bro:keyword:`const` keyword).  Definitions of functions, hooks, and
+    event handlers are not allowed to use the "global"
+    keyword (they already have global scope), except function declarations
+    where no function body is supplied use the "global" keyword.
+
+    The scope of a global variable begins where the declaration is located,
+    and extends through all remaining Bro scripts that are loaded (however,
+    see the :bro:keyword:`module` keyword for an explanation of how modules
+    change the visibility of global identifiers).
+
+
+.. bro:keyword:: const
+
+    A variable declared with the "const" keyword will be constant.
+    Variables declared as constant are required to be initialized at the
+    time of declaration.  Example::
+
+        const pi = 3.14;
+        const ssh_port: port = 22/tcp;
+
+    The value of a constant cannot be changed later (the only
+    exception is if the variable is global and has the :bro:attr:`&redef`
+    attribute, then its value can be changed only with a :bro:keyword:`redef`).
+
+    The scope of a constant is local if the declaration is in a
+    function, hook, or event handler, and global otherwise.
+    Note that the "const" keyword cannot be used with either the "local"
+    or "global" keywords (i.e., "const" replaces "local" and "global").
+
+
+.. bro:keyword:: type
+
+   The "type" keyword is used to declare a user-defined type.  The name
+   of this new type has global scope and can be used anywhere a built-in
+   type name can occur.
+
+   The "type" keyword is most commonly used when defining a
+   :bro:type:`record` or an :bro:type:`enum`, but is also useful when
+   dealing with more complex types.
+
+   Example::
+
+       type mytype: table[count] of table[addr, port] of string;
+       global myvar: mytype;
+
+.. bro:keyword:: redef
+
+    There are three ways that "redef" can be used:  to change the value of
+    a global variable, to extend a record type or enum type, or to specify
+    a new event handler body that replaces all those that were previously
+    defined.
+
+    If you're using "redef" to change a global variable (defined using either
+    :bro:keyword:`const` or :bro:keyword:`global`), then the variable that you
+    want to change must have the :bro:attr:`&redef` attribute.  If the variable
+    you're changing is a table, set, or pattern, you can use ``+=`` to add
+    new elements, or you can use ``=`` to specify a new value (all previous
+    contents of the object are removed).  If the variable you're changing is a
+    set or table, then you can use the ``-=`` operator to remove the
+    specified elements (nothing happens for specified elements that don't
+    exist).  If the variable you are changing is not a table, set, or pattern,
+    then you must use the ``=`` operator.
+
+    Examples::
+
+        redef pi = 3.14;
+
+    If you're using "redef" to extend a record or enum, then you must
+    use the ``+=`` assignment operator.
+    For an enum, you can add more enumeration constants, and for a record
+    you can add more record fields (however, each record field in the "redef"
+    must have either the :bro:attr:`&optional` or :bro:attr:`&default`
+    attribute).
+
+    Examples::
+
+        redef enum color += { Blue, Red };
+        redef record MyRecord += { n2:int &optional; s2:string &optional; };
+
+    If you're using "redef" to specify a new event handler body that
+    replaces all those that were previously defined (i.e., any subsequently
+    defined event handler body will not be affected by this "redef"), then
+    the syntax is the same as a regular event handler definition except for
+    the presence of the "redef" keyword.
+
+    Example::
+
+        redef event myevent(s:string) { print "Redefined", s; }
+
+
+.. _function/event/hook:
+
+**function/event/hook**
+    For details on how to declare a :bro:type:`function`,
+    :bro:type:`event` handler, or :bro:type:`hook`,
+    see the documentation for those types.
+
+
+Statements
+----------
+
+Each statement in a Bro script must be terminated with a semicolon (with a
+few exceptions noted below).  An individual statement can span multiple
+lines.
+
+All statements (except those contained within a function, hook, or event
+handler) must appear after all global declarations.
+
+Here are the statements that the Bro scripting language supports.
+
+.. bro:keyword:: add
+
+    The "add" statement is used to add an element to a :bro:type:`set`.
+    Nothing happens if the specified element already exists in the set.
+
+    Example::
+
+        local myset: set[string];
+        add myset["test"];
+
+.. bro:keyword:: break
+
+    The "break" statement is used to break out of a :bro:keyword:`switch` or
+    :bro:keyword:`for` statement.
+
+
+.. bro:keyword:: delete
+
+    The "delete" statement is used to remove an element from a
+    :bro:type:`set` or :bro:type:`table`.  Nothing happens if the
+    specified element does not exist in the set or table.
+
+    Example::
+
+        local myset = set("this", "test");
+        local mytable = table(["key1"] = 80/tcp, ["key2"] = 53/udp);
+        delete myset["test"];
+        delete mytable["key1"];
+
+.. bro:keyword:: event
+
+    The "event" statement immediately queues invocation of an event handler.
+
+    Example::
+
+        event myevent("test", 5);
+
+.. bro:keyword:: fallthrough
+
+    The "fallthrough" statement can be used as the last statement in a
+    "case" block to indicate that execution should continue into the
+    next "case" or "default" label.
+
+    For an example, see the :bro:keyword:`switch` statement.
+
+.. bro:keyword:: for
+
+    A "for" loop iterates over each element in a string, set, vector, or
+    table and executes a statement for each iteration.
+
+    For each iteration of the loop, a loop variable will be assigned to an
+    element if the expression evaluates to a string or set, or an index if
+    the expression evaluates to a vector or table.  Then the statement
+    is executed.  However, the statement will not be executed if the expression
+    evaluates to an object with no elements.
+
+    If the expression is a table or a set with more than one index, then the
+    loop variable must be specified as a comma-separated list of different
+    loop variables (one for each index), enclosed in brackets.
+
+    A :bro:keyword:`break` statement can be used at any time to immediately
+    terminate the "for" loop, and a :bro:keyword:`next` statement can be
+    used to skip to the next loop iteration.
+
+    Note that the loop variable in a "for" statement is not allowed to be
+    a global variable, and it does not need to be declared prior to the "for"
+    statement.  The type will be inferred from the elements of the
+    expression.
+
+    Example::
+
+        local myset = set(80/tcp, 81/tcp);
+        local mytable = table([10.0.0.1, 80/tcp]="s1", [10.0.0.2, 81/tcp]="s2");
+
+        for (p in myset)
+            print p;
+
+        for ([i,j] in mytable) {
+            if (mytable[i,j] == "done")
+                break;
+            if (mytable[i,j] == "skip")
+                next;
+            print i,j;
+        }
+
+
+.. bro:keyword:: if
+
+    Evaluates a given expression, which must yield a :bro:type:`bool` value.
+    If true, then a specified statement is executed.  If false, then
+    the statement is not executed.  Example::
+
+        if ( x == 2 ) print "x is 2";
+
+
+    However, if the expression evaluates to false and if an "else" is
+    provided, then the statement following the "else" is executed.  Example::
+
+        if ( x == 2 )
+            print "x is 2";
+        else
+            print "x is not 2";
+
+.. bro:keyword:: local
+
+    A variable declared with the "local" keyword will be local.  If a type
+    is not specified, then an initializer is required so that the type can
+    be inferred.  Likewise, if an initializer is not supplied, then the
+    type must be specified.
+
+    Examples::
+
+        local x1 = 5.7;
+        local x2: double;
+        local x3: double = 5.7;
+
+    Variable declarations inside a function, hook, or event handler are
+    required to use this keyword (the only two exceptions are variables
+    declared with :bro:keyword:`const`, and variables implicitly declared in a
+    :bro:keyword:`for` statement).
+
+    The scope of a local variable starts at the location where it is declared
+    and persists to the end of the function, hook,
+    or event handler in which it is declared (this is true even if the
+    local variable was declared within a `compound statement`_ or is the loop
+    variable in a "for" statement).
+
+
+.. bro:keyword:: next
+
+    The "next" statement can only appear within a :bro:keyword:`for` loop.
+    It causes execution to skip to the next iteration.
+
+    For an example, see the :bro:keyword:`for` statement.
+
+.. bro:keyword:: print
+
+    The "print" statement takes a comma-separated list of one or more
+    expressions.  Each expression in the list is evaluated and then converted
+    to a string.  Then each string is printed, with each string separated by
+    a comma in the output.
+
+    Examples::
+
+        print 3.14;
+        print "Results", x, y;
+
+    By default, the "print" statement writes to the standard
+    output (stdout).  However, if the first expression is of type
+    :bro:type:`file`, then "print" writes to that file.
+
+    If a string contains non-printable characters (i.e., byte values that are
+    not in the range 32 - 126), then the "print" statement converts each
+    non-printable character to an escape sequence before it is printed.
+
+    For more control over how the strings are formatted, see the :bro:id:`fmt`
+    function.
+
+.. bro:keyword:: return
+
+    The "return" statement immediately exits the current function, hook, or
+    event handler.  For a function, the specified expression (if any) is
+    evaluated and returned.  A "return" statement in a hook or event handler
+    cannot return a value because event handlers and hooks do not have
+    return types.
+
+    Examples::
+
+        function my_func(): string
+        {
+            return "done";
+        }
+
+        event my_event(n: count)
+        {
+            if ( n == 0 ) return;
+
+            print n;
+        }
+
+    There is a special form of the "return" statement that is only allowed
+    in functions.  Syntactically, it looks like a :bro:keyword:`when` statement
+    immediately preceded by the "return" keyword.  This form of the "return"
+    statement is used to specify a function that delays its result (such a
+    function can only be called in the expression of a :bro:keyword:`when`
+    statement).  The function returns at the time the "when"
+    statement's condition becomes true, and the function returns the value
+    that the "when" statement's body returns (or if the condition does
+    not become true within the specified timeout interval, then the function
+    returns the value that the "timeout" block returns).
+
+    Example::
+
+      global X: table[string] of count;
+
+      function a() : count
+            {
+            # This delays until condition becomes true.
+            return when ( "a" in X )
+                  {
+                  return X["a"];
+                  }
+            timeout 30 sec
+                  {
+                  return 0;
+                  }
+            }
+
+      event bro_init()
+            {
+            # Installs a trigger which fires if a() returns 42.
+            when ( a() == 42 )
+                print "expected result";
+
+            print "Waiting for a() to return...";
+            X["a"] = 42;
+            }
+
+
+.. bro:keyword:: schedule
+
+    The "schedule" statement is used to raise a specified event with
+    specified parameters at a later time specified as an :bro:type:`interval`.
+
+    Example::
+
+        schedule 30sec { myevent(x, y, z) };
+
+    Note that the braces are always required (they do not indicate a
+    `compound statement`_).
+
+    Note that "schedule" is actually an expression that returns a value
+    of type "timer", but in practice the return value is not used.
+
+.. bro:keyword:: switch
+
+    A "switch" statement evaluates a given expression and jumps to
+    the first "case" label which contains a matching value (the result of the
+    expression must be type-compatible with all of the values in all of the
+    "case" labels).  If there is no matching value, then execution jumps to
+    the "default" label instead, and if there is no "default" label then
+    execution jumps out of the "switch" block.
+
+    Here is an example (assuming that "get_day_of_week" is a
+    function that returns a string)::
+
+        switch get_day_of_week()
+            {
+            case "Sa", "Su":
+                print "weekend";
+                fallthrough;
+            case "Mo", "Tu", "We", "Th", "Fr":
+                print "valid result";
+                break;
+            default:
+                print "invalid result";
+                break;
+            }
+
+    A "switch" block can have any number of "case" labels, and one
+    optional "default" label.
+
+    A "case" label can have a comma-separated list of
+    more than one value.  A value in a "case" label can be an expression,
+    but it must be a constant expression (i.e., the expression can consist
+    only of constants).
+
+    Each "case" and the "default" block must
+    end with either a :bro:keyword:`break`, :bro:keyword:`fallthrough`, or
+    :bro:keyword:`return` statement (although "return" is allowed only
+    if the "switch" statement is inside a function, hook, or event handler).
+    If a "case" (or "default") block contain more than one statement, then
+    there is no need to wrap them in braces.
+
+    Note that the braces in a "switch" statement are always required (these
+    do not indicate the presence of a `compound statement`_), and that no
+    semicolon is needed at the end of a "switch" statement.
+
+
+.. bro:keyword:: when
+
+    Evaluates a given expression, which must result in a value of type
+    :bro:type:`bool`.  When the value of the expression becomes available
+    and if the result is true, then a specified statement is executed.
+
+    In the following example, if the expression evaluates to true, then
+    the "print" statement is executed::
+
+        when ( (local x = foo()) && x == 42 )
+            print x;
+
+    However, if a timeout is specified, and if the expression does not
+    evaluate to true within the specified timeout interval, then the
+    statement following the "timeout" keyword is executed::
+
+        when ( (local x = foo()) && x == 42 )
+            print x;
+        timeout 5sec {
+            print "timeout";
+        }
+
+    Note that when a timeout is specified the braces are
+    always required (these do not indicate a `compound statement`_).
+
+    The expression in a "when" statement can contain a declaration of a local
+    variable but only if the declaration is written in the form
+    "local *var* = *init*" (example: "local x = myfunction()").  This form
+    of a local declaration is actually an expression, the result of which
+    is always a boolean true value.
+
+    The expression in a "when" statement can contain an asynchronous function
+    call such as :bro:id:`lookup_hostname` (in fact, this is the only place
+    such a function can be called), but it can also contain an ordinary
+    function call.  When an asynchronous function call is in the expression,
+    then Bro will continue processing statements in the script following
+    the "when" statement, and when the result of the function call is available
+    Bro will finish evaluating the expression in the "when" statement.
+    See the :bro:keyword:`return` statement for an explanation of how to
+    create an asynchronous function in a Bro script.
+
+
+.. _compound statement:
+
+**compound statement**
+    A compound statement is created by wrapping zero or more statements in
+    braces ``{ }``.  Individual statements inside the braces need to be
+    terminated by a semicolon, but a semicolon is not needed at the end
+    (outside of the braces) of a compound statement.
+
+    A compound statement is required in order to execute more than one
+    statement in the body of a :bro:keyword:`for`, :bro:keyword:`if`, or
+    :bro:keyword:`when` statement.
+
+    Example::
+
+        if ( x == 2 ) {
+            print "x is 2";
+            ++x;
+        }
+
+    Note that there are other places in the Bro scripting language that use
+    braces, but that do not indicate the presence of a compound
+    statement (these are noted in the documentation).
+
+.. _null:
+
+**null statement**
+    The null statement (executing it has no effect) consists of just a
+    semicolon.  This might be useful during testing or debugging a Bro script
+    in places where a statement is required, but it is probably not useful
+    otherwise.
+
+    Example::
+
+        if ( x == 2 )
+            ;
+

doc/script-reference/types.rst

@@ -1,106 +1,128 @@
-Types and Attributes
-====================
-
 Types
------
+=====
 
-Every value in a Bro script has a type (see below for a list of all built-in
+The Bro scripting language supports the following built-in types:
-types).  Although Bro variables have static types (meaning that their type
-is fixed), their type is inferred from the value to which they are
-initially assigned when the variable is declared without an explicit type
-name.
 
-Automatic conversions happen when a binary operator has operands of 
++-----------------------+--------------------+
-different types.  Automatic conversions are limited to converting between
+| Name                  | Description        |
-numeric types.  The numeric types are ``int``, ``count``, and ``double``
++=======================+====================+
-(``bool`` is not a numeric type).
+| :bro:type:`bool`      | Boolean            |
-When an automatic conversion occurs, values are promoted to the "highest"
++-----------------------+--------------------+
-type in the expression. In general, this promotion follows a simple
+| :bro:type:`count`,    | Numeric types      |
-hierarchy: ``double`` is highest, ``int`` comes next, and ``count`` is
+| :bro:type:`int`,      |                    |
-lowest.
+| :bro:type:`double`    |                    |
++-----------------------+--------------------+
+| :bro:type:`time`,     | Time types         |
+| :bro:type:`interval`  |                    |
++-----------------------+--------------------+
+| :bro:type:`string`    | String             |
++-----------------------+--------------------+
+| :bro:type:`pattern`   | Regular expression |
++-----------------------+--------------------+
+| :bro:type:`port`,     | Network types      |
+| :bro:type:`addr`,     |                    |
+| :bro:type:`subnet`    |                    |
++-----------------------+--------------------+
+| :bro:type:`enum`      | Enumeration        |
+|                       | (user-defined type)|
++-----------------------+--------------------+
+| :bro:type:`table`,    | Container types    |
+| :bro:type:`set`,      |                    |
+| :bro:type:`vector`,   |                    |
+| :bro:type:`record`    |                    |
++-----------------------+--------------------+
+| :bro:type:`function`, | Executable types   |
+| :bro:type:`event`,    |                    |
+| :bro:type:`hook`      |                    |
++-----------------------+--------------------+
+| :bro:type:`file`      | File type (only    |
+|                       | for writing)       |
++-----------------------+--------------------+
+| :bro:type:`opaque`    | Opaque type (for   |
+|                       | some built-in      |
+|                       | functions)         |
++-----------------------+--------------------+
+| :bro:type:`any`       | Any type (for      |
+|                       | functions or       |
+|                       | containers)        |
++-----------------------+--------------------+
 
-The Bro scripting language supports the following built-in types.
+Here is a more detailed description of each type:
-
-.. bro:type:: void
-
-    An internal Bro type (i.e., "void" is not a reserved keyword in the Bro
-    scripting language) representing the absence of a return type for a
-    function.
 
 .. bro:type:: bool
 
     Reflects a value with one of two meanings: true or false.  The two
-    ``bool`` constants are ``T`` and ``F``.
+    "bool" constants are ``T`` and ``F``.
 
-    The ``bool`` type supports the following operators: equality/inequality
+    The "bool" type supports the following operators: equality/inequality
     (``==``, ``!=``), logical and/or (``&&``, ``||``), logical
-    negation (``!``), and absolute value (where ``|T|`` is 1, and ``|F|`` is 0).
+    negation (``!``), and absolute value (where ``|T|`` is 1, and ``|F|`` is 0,
+    and in both cases the result type is :bro:type:`count`).
 
 .. bro:type:: int
 
-    A numeric type representing a 64-bit signed integer.  An ``int`` constant
+    A numeric type representing a 64-bit signed integer.  An "int" constant
-    is a string of digits preceded by a ``+`` or ``-`` sign, e.g.
+    is a string of digits preceded by a "+" or "-" sign, e.g.
     ``-42`` or ``+5`` (the "+" sign is optional but see note about type
-    inferencing below).  An ``int`` constant can also be written in
+    inferencing below).  An "int" constant can also be written in
     hexadecimal notation (in which case "0x" must be between the sign and
     the hex digits), e.g. ``-0xFF`` or ``+0xabc123``.
 
-    The ``int`` type supports the following operators:  arithmetic
+    The "int" type supports the following operators:  arithmetic
     operators (``+``, ``-``, ``*``, ``/``, ``%``), comparison operators
     (``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``), assignment operators
     (``=``, ``+=``, ``-=``), pre-increment (``++``), pre-decrement
-    (``--``), and absolute value (e.g., ``|-3|`` is 3).
+    (``--``), unary plus and minus (``+``, ``-``), and absolute value
+    (e.g., ``|-3|`` is 3, but the result type is :bro:type:`count`).
 
     When using type inferencing use care so that the
-    intended type is inferred, e.g. ``local size_difference = 0`` will
+    intended type is inferred, e.g. "local size_difference = 0" will
-    infer :bro:type:`count`, while ``local size_difference = +0``
+    infer ":bro:type:`count`", while "local size_difference = +0"
-    will infer :bro:type:`int`.
+    will infer "int".
 
 .. bro:type:: count
 
-    A numeric type representing a 64-bit unsigned integer.  A ``count``
+    A numeric type representing a 64-bit unsigned integer.  A "count"
-    constant is a string of digits, e.g. ``1234`` or ``0``.  A ``count``
+    constant is a string of digits, e.g. ``1234`` or ``0``.  A "count"
     can also be written in hexadecimal notation (in which case "0x" must
     precede the hex digits), e.g. ``0xff`` or ``0xABC123``.
 
-    The ``count`` type supports the same operators as the :bro:type:`int`
+    The "count" type supports the same operators as the ":bro:type:`int`"
-    type.  A unary plus or minus applied to a ``count`` results in an ``int``.
+    type, but a unary plus or minus applied to a "count" results in an
-
+    "int".
-.. bro:type:: counter
-
-    An alias to :bro:type:`count`.
 
 .. bro:type:: double
 
     A numeric type representing a double-precision floating-point
     number.  Floating-point constants are written as a string of digits
     with an optional decimal point, optional scale-factor in scientific
-    notation, and optional ``+`` or ``-`` sign.  Examples are ``-1234``,
+    notation, and optional "+" or "-" sign.  Examples are ``-1234``,
     ``-1234e0``, ``3.14159``, and ``.003E-23``.
 
-    The ``double`` type supports the following operators:  arithmetic
+    The "double" type supports the following operators:  arithmetic
     operators (``+``, ``-``, ``*``, ``/``), comparison operators
     (``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``), assignment operators
-    (``=``, ``+=``, ``-=``), and absolute value (e.g., ``|-3.14|`` is 3.14).
+    (``=``, ``+=``, ``-=``), unary plus and minus (``+``, ``-``), and
+    absolute value (e.g., ``|-3.14|`` is 3.14).
 
     When using type inferencing use care so that the
-    intended type is inferred, e.g. ``local size_difference = 5`` will
+    intended type is inferred, e.g. "local size_difference = 5" will
-    infer :bro:type:`count`, while ``local size_difference = 5.0``
+    infer ":bro:type:`count`", while "local size_difference = 5.0"
-    will infer :bro:type:`double`.
+    will infer "double".
 
 .. bro:type:: time
 
     A temporal type representing an absolute time.  There is currently
     no way to specify a ``time`` constant, but one can use the
     :bro:id:`double_to_time`, :bro:id:`current_time`, or :bro:id:`network_time`
-    built-in functions to assign a value to a ``time``-typed variable.  
+    built-in functions to assign a value to a ``time``-typed variable.
 
     Time values support the comparison operators (``==``, ``!=``, ``<``,
     ``<=``, ``>``, ``>=``).  A ``time`` value can be subtracted from
-    another ``time`` value to produce an ``interval`` value.  An ``interval``
+    another ``time`` value to produce an :bro:type:`interval` value.  An
-    value can be added to, or subtracted from, a ``time`` value to produce a
+    ``interval`` value can be added to, or subtracted from, a ``time`` value
-    ``time`` value.  The absolute value of a ``time`` value is a ``double``
+    to produce a ``time`` value.  The absolute value of a ``time`` value is
-    with the same numeric value.
+    a :bro:type:`double` with the same numeric value.
 
 .. bro:type:: interval
 
@@ -115,52 +137,58 @@ The Bro scripting language supports the following built-in types.
     ``3.5mins``.  An ``interval`` can also be negated, for example
     ``-12 hr`` represents "twelve hours in the past".
 
-    Intervals support addition and subtraction.  Intervals also support
+    Intervals support addition and subtraction, the comparison operators
-    division (in which case the result is a ``double`` value), the
+    (``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``), the assignment
-    comparison operators (``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``),
+    operators (``=``, ``+=``, ``-=``), and unary plus and minus (``+``, ``-``).
-    and the assignment operators (``=``, ``+=``, ``-=``).  Also, an
+
-    ``interval`` can be multiplied or divided by an arithmetic type
+    Intervals also support division (in which case the result is a
-    (``count``, ``int``, or ``double``) to produce an ``interval`` value.
+    :bro:type:`double` value).  An ``interval`` can be multiplied or divided
-    The absolute value of an ``interval`` is a ``double`` value equal to the
+    by an arithmetic type (``count``, ``int``, or ``double``) to produce
-    number of seconds in the ``interval`` (e.g., ``|-1 min|`` is 60).
+    an ``interval`` value.  The absolute value of an ``interval`` is a
+    ``double`` value equal to the number of seconds in the ``interval``
+    (e.g., ``|-1 min|`` is 60.0).
 
 .. bro:type:: string
 
-    A type used to hold character-string values which represent text.
+    A type used to hold character-string values which represent text, although
-    String constants are created by enclosing text in double quotes (")
+    strings in a Bro script can actually contain any arbitrary binary data.
-    and the backslash character (\\) introduces escape sequences (all of
+
-    the C-style escape sequences are supported).
+    String constants are created by enclosing text within a pair of double
+    quotes (").  A string constant cannot span multiple lines in a Bro script.
+    The backslash character (\\) introduces escape sequences.  The
+    following escape sequences are recognized: ``\n``, ``\t``, ``\v``, ``\b``,
+    ``\r``, ``\f``, ``\a``, ``\ooo`` (where each 'o' is an octal digit),
+    ``\xhh`` (where each 'h' is a hexadecimal digit).  For escape sequences
+    that don't match any of these, Bro will just remove the backslash (so
+    to represent a literal backslash in a string constant, you just use
+    two consecutive backslashes).
 
     Strings support concatenation (``+``), and assignment (``=``, ``+=``).
     Strings also support the comparison operators (``==``, ``!=``, ``<``,
     ``<=``, ``>``, ``>=``).  The number of characters in a string can be
     found by enclosing the string within pipe characters (e.g., ``|"abc"|``
-    is 3).
+    is 3).  Substring searching can be performed using the "in" or "!in"
-
-    The subscript operator can extract an individual character or a substring
-    of a string (string indexing is zero-based, but an index of
-    -1 refers to the last character in the string, and -2 refers to the
-    second-to-last character, etc.).  When extracting a substring, the
-    starting and ending index values are separated by a colon.  For example::
-
-        local orig = "0123456789";
-        local third_char = orig[2];
-        local last_char = orig[-1];
-        local first_three_chars = orig[0:2];
-
-    Substring searching can be performed using the "in" or "!in"
     operators (e.g., "bar" in "foobar" yields true).
 
-    Note that Bro represents strings internally as a count and vector of
+    The subscript operator can extract a substring of a string.  To do this,
-    bytes rather than a NUL-terminated byte string (although string
+    specify the starting index to extract (if the starting index is omitted,
-    constants are also automatically NUL-terminated).  This is because
+    then zero is assumed), followed by a colon and index
-    network traffic can easily introduce NULs into strings either by
+    one past the last character to extract (if the last index is omitted,
-    nature of an application, inadvertently, or maliciously.  And while
+    then the extracted substring will go to the end of the original string).
-    NULs are allowed in Bro strings, when present in strings passed as
+    However, if both the colon and last index are omitted, then a string of
-    arguments to many functions, a run-time error can occur as their
+    length one is extracted.  String indexing is zero-based, but an index
-    presence likely indicates a sort of problem.  In that case, the
+    of -1 refers to the last character in the string, and -2 refers to the
-    string will also only be represented to the user as the literal
+    second-to-last character, etc.  Here are a few examples::
-    "<string-with-NUL>" string.
+
+        local orig = "0123456789";
+        local second_char = orig[1];
+        local last_char = orig[-1];
+        local first_two_chars = orig[:2];
+        local last_two_chars = orig[8:];
+        local no_first_and_last = orig[1:9];
+
+    Note that the subscript operator cannot be used to modify a string (i.e.,
+    it cannot be on the left side of an assignment operator).
 
 .. bro:type:: pattern
 
@@ -174,7 +202,7 @@ The Bro scripting language supports the following built-in types.
     and embedded.
 
     In exact matching the ``==`` equality relational operator is used
-    with one :bro:type:`pattern` operand and one :bro:type:`string`
+    with one "pattern" operand and one ":bro:type:`string`"
     operand (order of operands does not matter) to check whether the full
     string exactly matches the pattern.  In exact matching, the ``^``
     beginning-of-line and ``$`` end-of-line anchors are redundant since
@@ -190,8 +218,8 @@ The Bro scripting language supports the following built-in types.
     yields false.  The ``!=`` operator would yield the negation of ``==``.
 
     In embedded matching the ``in`` operator is used with one
-    :bro:type:`pattern` operand (which must be on the left-hand side) and
+    "pattern" operand (which must be on the left-hand side) and
-    one :bro:type:`string` operand, but tests whether the pattern
+    one ":bro:type:`string`" operand, but tests whether the pattern
     appears anywhere within the given string.  For example::
 
         /foo|bar/ in "foobar"
@@ -203,27 +231,12 @@ The Bro scripting language supports the following built-in types.
     is false since "oob" does not appear at the start of "foobar".  The
     ``!in`` operator would yield the negation of ``in``.
 
-.. bro:type:: enum
-
-    A type allowing the specification of a set of related values that
-    have no further structure.  An example declaration:
-
-    .. code:: bro
-
-        type color: enum { Red, White, Blue, };
-
-    The last comma after ``Blue`` is optional.
-
-    The only operations allowed on enumerations are equality comparisons
-    (``==``, ``!=``) and assignment (``=``).
-    Enumerations do not have associated values or ordering.
-
 .. bro:type:: port
 
-    A type representing transport-level port numbers.  Besides TCP and
+    A type representing transport-level port numbers (besides TCP and
     UDP ports, there is a concept of an ICMP "port" where the source
     port is the ICMP message type and the destination port the ICMP
-    message code.  A ``port`` constant is written as an unsigned integer
+    message code).  A ``port`` constant is written as an unsigned integer
     followed by one of ``/tcp``, ``/udp``, ``/icmp``, or ``/unknown``.
 
     Ports support the comparison operators (``==``, ``!=``, ``<``, ``<=``,
@@ -255,14 +268,6 @@ The Bro scripting language supports the following built-in types.
     address) are treated internally as IPv4 addresses (for example,
     ``[::ffff:192.168.1.100]`` is equal to ``192.168.1.100``).
 
-    Hostname constants can also be used, but since a hostname can
-    correspond to multiple IP addresses, the type of such a variable is a
-    :bro:type:`set` of :bro:type:`addr` elements. For example:
-
-    .. code:: bro
-
-        local a = www.google.com;
-
     Addresses can be compared for equality (``==``, ``!=``),
     and also for ordering (``<``, ``<=``, ``>``, ``>=``).  The absolute value
     of an address gives the size in bits (32 for IPv4, and 128 for IPv6).
@@ -285,9 +290,17 @@ The Bro scripting language supports the following built-in types.
         if ( a in s )
             print "true";
 
-    Note that you can check if a given ``addr`` is IPv4 or IPv6 using
+    You can check if a given ``addr`` is IPv4 or IPv6 using
     the :bro:id:`is_v4_addr` and :bro:id:`is_v6_addr` built-in functions.
 
+    Note that hostname constants can also be used, but since a hostname can
+    correspond to multiple IP addresses, the type of such a variable is
+    "set[addr]". For example:
+
+    .. code:: bro
+
+        local a = www.google.com;
+
 .. bro:type:: subnet
 
     A type representing a block of IP addresses in CIDR notation.  A
@@ -296,13 +309,24 @@ The Bro scripting language supports the following built-in types.
     number.  For example, ``192.168.0.0/16`` or ``[fe80::]/64``.
 
     Subnets can be compared for equality (``==``, ``!=``).  An
-    :bro:type:`addr` can be checked for inclusion in a subnet using
+    "addr" can be checked for inclusion in a subnet using
-    the "in" or "!in" operators.
+    the ``in`` or ``!in`` operators.
 
-.. bro:type:: any
+.. bro:type:: enum
 
-    Used to bypass strong typing.  For example, a function can take an
+    A type allowing the specification of a set of related values that
-    argument of type ``any`` when it may be of different types.
+    have no further structure.  An example declaration:
+
+    .. code:: bro
+
+        type color: enum { Red, White, Blue, };
+
+    The last comma after ``Blue`` is optional.  Both the type name ``color``
+    and the individual values (``Red``, etc.) have global scope.
+
+    Enumerations do not have associated values or ordering.
+    The only operations allowed on enumerations are equality comparisons
+    (``==``, ``!=``) and assignment (``=``).
 
 .. bro:type:: table
 
@@ -316,24 +340,25 @@ The Bro scripting language supports the following built-in types.
 
         table [ type^+ ] of type
 
-    where *type^+* is one or more types, separated by commas.  For example:
+    where *type^+* is one or more types, separated by commas.
+    For example:
 
     .. code:: bro
 
         global a: table[count] of string;
 
-    declares a table indexed by :bro:type:`count` values and yielding
+    declares a table indexed by "count" values and yielding
-    :bro:type:`string` values.  The yield type can also be more complex:
+    "string" values.  The yield type can also be more complex:
 
     .. code:: bro
 
         global a: table[count] of table[addr, port] of string;
 
-    which declares a table indexed by :bro:type:`count` and yielding
+    which declares a table indexed by "count" and yielding
-    another :bro:type:`table` which is indexed by an :bro:type:`addr`
+    another "table" which is indexed by an "addr"
-    and :bro:type:`port` to yield a :bro:type:`string`.
+    and "port" to yield a "string".
 
-    Initialization of tables occurs by enclosing a set of initializers within
+    One way to initialize a table is by enclosing a set of initializers within
     braces, for example:
 
     .. code:: bro
@@ -343,18 +368,17 @@ The Bro scripting language supports the following built-in types.
             [5] = "five",
         };
 
-    A table constructor (equivalent to above example) can also be used
+    A table constructor can also be used to create a table:
-    to create a table:
 
     .. code:: bro
 
-        global t2: table[count] of string = table(
+        global t2 = table(
-            [11] = "eleven",
+            [192.168.0.2, 22/tcp] = "ssh",
-            [5] = "five"
+            [192.168.0.3, 80/tcp] = "http"
         );
 
     Table constructors can also be explicitly named by a type, which is
-    useful for when a more complex index type could otherwise be
+    useful when a more complex index type could otherwise be
     ambiguous:
 
     .. code:: bro
@@ -381,17 +405,7 @@ The Bro scripting language supports the following built-in types.
 
         if ( 13 in t )
             ...
-
+        if ( [192.168.0.2, 22/tcp] in t2 )
-    Iterate over tables with a ``for`` loop:
-
-    .. code:: bro
-
-        local t: table[count] of string;
-        for ( n in t )
-            ...
-
-        local services: table[addr, port] of string;
-        for ( [a, p] in services )
             ...
 
     Add or overwrite individual table elements by assignment:
@@ -400,7 +414,7 @@ The Bro scripting language supports the following built-in types.
 
         t[13] = "thirteen";
 
-    Remove individual table elements with ``delete``:
+    Remove individual table elements with :bro:keyword:`delete`:
 
     .. code:: bro
 
@@ -416,6 +430,9 @@ The Bro scripting language supports the following built-in types.
 
         |t|
 
+    See the :bro:keyword:`for` statement for info on how to iterate over
+    the elements in a table.
+
 .. bro:type:: set
 
     A set is like a :bro:type:`table`, but it is a collection of indices
@@ -426,25 +443,22 @@ The Bro scripting language supports the following built-in types.
 
     where *type^+* is one or more types separated by commas.
 
-    Sets are initialized by listing elements enclosed by curly braces:
+    Sets can be initialized by listing elements enclosed by curly braces:
 
     .. code:: bro
 
         global s: set[port] = { 21/tcp, 23/tcp, 80/tcp, 443/tcp };
         global s2: set[port, string] = { [21/tcp, "ftp"], [23/tcp, "telnet"] };
 
-    The types are explicitly shown in the example above, but they could
-    have been left to type inference.
-
     A set constructor (equivalent to above example) can also be used to
     create a set:
 
     .. code:: bro
 
-        global s3: set[port] = set(21/tcp, 23/tcp, 80/tcp, 443/tcp);
+        global s3 = set(21/tcp, 23/tcp, 80/tcp, 443/tcp);
 
     Set constructors can also be explicitly named by a type, which is
-    useful for when a more complex index type could otherwise be
+    useful when a more complex index type could otherwise be
     ambiguous:
 
     .. code:: bro
@@ -465,18 +479,10 @@ The Bro scripting language supports the following built-in types.
         if ( 21/tcp in s )
             ...
 
-        if ( 21/tcp !in s )
+        if ( [21/tcp, "ftp"] !in s2 )
             ...
 
-    Iterate over a set with a ``for`` loop:
+    Elements are added with :bro:keyword:`add`:
-
-    .. code:: bro
-
-        local s: set[port];
-        for ( p in s )
-            ...
-
-    Elements are added with ``add``:
 
     .. code:: bro
 
@@ -485,7 +491,7 @@ The Bro scripting language supports the following built-in types.
     Nothing happens if the element with value ``22/tcp`` was already present in
     the set.
 
-    And removed with ``delete``:
+    And removed with :bro:keyword:`delete`:
 
     .. code:: bro
 
@@ -501,6 +507,9 @@ The Bro scripting language supports the following built-in types.
 
         |s|
 
+    See the :bro:keyword:`for` statement for info on how to iterate over
+    the elements in a set.
+
 .. bro:type:: vector
 
     A vector is like a :bro:type:`table`, except it's always indexed by a
@@ -515,7 +524,7 @@ The Bro scripting language supports the following built-in types.
 
     .. code:: bro
 
-        global v: vector of string = vector("one", "two", "three");
+        local v = vector("one", "two", "three");
 
     Vector constructors can also be explicitly named by a type, which
     is useful for when a more complex yield type could otherwise be
@@ -539,14 +548,6 @@ The Bro scripting language supports the following built-in types.
 
         print v[2];
 
-    Iterate over a vector with a ``for`` loop:
-
-    .. code:: bro
-
-        local v: vector of string;
-        for ( n in v )
-            ...
-
     An element can be added to a vector by assigning the value (a value
     that already exists at that index will be overwritten):
 
@@ -577,11 +578,17 @@ The Bro scripting language supports the following built-in types.
     The resulting vector of bool is the logical "and" (or logical "or") of
     each element of the operand vectors.
 
+    See the :bro:keyword:`for` statement for info on how to iterate over
+    the elements in a vector.
+
 .. bro:type:: record
 
-    A ``record`` is a collection of values.  Each value has a field name
+    A "record" is a collection of values.  Each value has a field name
     and a type.  Values do not need to have the same type and the types
-    have no restrictions.  An example record type definition:
+    have no restrictions.  Field names must follow the same syntax as
+    regular variable names (except that field names are allowed to be the
+    same as local or global variables).  An example record type
+    definition:
 
     .. code:: bro
 
@@ -590,85 +597,44 @@ The Bro scripting language supports the following built-in types.
             s: string &optional;
         };
 
-    Access to a record field uses the dollar sign (``$``) operator:
+    Records can be initialized or assigned as a whole in three different ways.
-
-    .. code:: bro
-
-        global r: MyRecordType;
-        r$c = 13;
-
-    Record assignment can be done field by field or as a whole like:
-
-    .. code:: bro
-
-        r = [$c = 13, $s = "thirteen"];
-
     When assigning a whole record value, all fields that are not
     :bro:attr:`&optional` or have a :bro:attr:`&default` attribute must
-    be specified.
+    be specified.  First, there's a constructor syntax:
-
-    To test for existence of a field that is :bro:attr:`&optional`, use the
-    ``?$`` operator:
 
     .. code:: bro
 
-        if ( r?$s )
+        local r: MyRecordType = record($c = 7);
-            ...
-
-    Records can also be created using a constructor syntax:
-
-    .. code:: bro
-
-        global r2: MyRecordType = record($c = 7);
 
     And the constructor can be explicitly named by type, too, which
-    is arguably more readable code:
+    is arguably more readable:
 
     .. code:: bro
 
-        global r3 = MyRecordType($c = 42);
+        local r = MyRecordType($c = 42);
 
-.. bro:type:: opaque
+    And the third way is like this:
-
-    A data type whose actual representation/implementation is
-    intentionally hidden, but whose values may be passed to certain
-    functions that can actually access the internal/hidden resources.
-    Opaque types are differentiated from each other by qualifying them
-    like ``opaque of md5`` or ``opaque of sha1``.  Any valid identifier
-    can be used as the type qualifier.
-
-    An example use of this type is the set of built-in functions which
-    perform hashing:
 
     .. code:: bro
 
-        local handle: opaque of md5 = md5_hash_init();
+        local r: MyRecordType = [$c = 13, $s = "thirteen"];
-        md5_hash_update(handle, "test");
-        md5_hash_update(handle, "testing");
-        print md5_hash_finish(handle);
 
-    Here the opaque type is used to provide a handle to a particular
+    Access to a record field uses the dollar sign (``$``) operator, and
-    resource which is calculating an MD5 checksum incrementally over
+    record fields can be assigned with this:
-    time, but the details of that resource aren't relevant, it's only
-    necessary to have a handle as a way of identifying it and
-    distinguishing it from other such resources.
-
-.. bro:type:: file
-
-    Bro supports writing to files, but not reading from them.  Files
-    can be opened using either the :bro:id:`open` or :bro:id:`open_for_append`
-    built-in functions, and closed using the :bro:id:`close` built-in
-    function. For example, declare, open, and write to a file
-    and finally close it like:
 
     .. code:: bro
 
-        global f: file = open("myfile");
+        local r: MyRecordType;
-        print f, "hello, world";
+        r$c = 13;
-        close(f);
 
-    Writing to files like this for logging usually isn't recommended, for better
+    To test if a field that is :bro:attr:`&optional` has been assigned a
-    logging support see :doc:`/frameworks/logging`.
+    value, use the ``?$`` operator (it returns a :bro:type:`bool` value of
+    ``T`` if the field has been assigned a value, or ``F`` if not):
+
+    .. code:: bro
+
+        if ( r ?$ s )
+            ...
 
 .. bro:type:: function
 
@@ -700,6 +666,16 @@ The Bro scripting language supports the following built-in types.
     type, but when it is, the return type and argument list (including the
     name of each argument) must match exactly.
 
+    Here is an example function that takes no parameters and does not
+    return a value:
+
+    .. code:: bro
+
+        function my_func()
+            {
+            print "my_func";
+            }
+
     Function types don't need to have a name and can be assigned anonymously:
 
     .. code:: bro
@@ -742,9 +718,20 @@ The Bro scripting language supports the following built-in types.
     Event handlers are nearly identical in both syntax and semantics to
     a :bro:type:`function`, with the two differences being that event
     handlers have no return type since they never return a value, and
-    you cannot call an event handler.  Instead of directly calling an
+    you cannot call an event handler.
-    event handler from a script, event handler bodies are executed when
+
-    they are invoked by one of three different methods:
+    Example:
+
+    .. code:: bro
+
+        event my_event(r: bool, s: string)
+        {
+            print "my_event", r, s;
+        }
+
+    Instead of directly calling an event handler from a script, event
+    handler bodies are executed when they are invoked by one of three
+    different methods:
 
     - From the event engine
 
@@ -765,7 +752,7 @@ The Bro scripting language supports the following built-in types.
         This assumes that ``password_exposed`` was previously declared
         as an event handler type with compatible arguments.
 
-    - Via the ``schedule`` expression in a script
+    - Via the :bro:keyword:`schedule` expression in a script
 
         This delays the invocation of event handlers until some time in
         the future.  For example:
@@ -789,8 +776,8 @@ The Bro scripting language supports the following built-in types.
     immediate and they do not get scheduled through an event queue.
     Also, a unique feature of a hook is that a given hook handler body
     can short-circuit the execution of remaining hook handlers simply by
-    exiting from the body as a result of a ``break`` statement (as
+    exiting from the body as a result of a :bro:keyword:`break` statement (as
-    opposed to a ``return`` or just reaching the end of the body).
+    opposed to a :bro:keyword:`return` or just reaching the end of the body).
 
     A hook type is declared like::
 
@@ -859,142 +846,60 @@ The Bro scripting language supports the following built-in types.
     executed due to one handler body exiting as a result of a ``break``
     statement.
 
-Attributes
+.. bro:type:: file
-----------
 
-Attributes occur at the end of type/event declarations and change their
+    Bro supports writing to files, but not reading from them (to read from
-behavior. The syntax is ``&key`` or ``&key=val``, e.g., ``type T:
+    files see the :doc:`/frameworks/input`).  Files
-set[count] &read_expire=5min`` or ``event foo() &priority=-3``.  The Bro
+    can be opened using either the :bro:id:`open` or :bro:id:`open_for_append`
-scripting language supports the following built-in attributes.
+    built-in functions, and closed using the :bro:id:`close` built-in
+    function.  For example, declare, open, and write to a file and finally
+    close it like:
 
-.. bro:attr:: &optional
+    .. code:: bro
 
-    Allows a record field to be missing. For example the type ``record {
+        local f = open("myfile");
-    a: addr; b: port &optional; }`` could be instantiated both as
+        print f, "hello, world";
-    singleton ``[$a=127.0.0.1]`` or pair ``[$a=127.0.0.1, $b=80/tcp]``.
+        close(f);
 
-.. bro:attr:: &default
+    Writing to files like this for logging usually isn't recommended, for better
+    logging support see :doc:`/frameworks/logging`.
 
-    Uses a default value for a record field, a function/hook/event
+.. bro:type:: opaque
-    parameter, or container elements.  For example, ``table[int] of
-    string &default="foo"`` would create a table that returns the
-    :bro:type:`string` ``"foo"`` for any non-existing index.
 
-.. bro:attr:: &redef
+    A data type whose actual representation/implementation is
+    intentionally hidden, but whose values may be passed to certain
+    built-in functions that can actually access the internal/hidden resources.
+    Opaque types are differentiated from each other by qualifying them
+    like "opaque of md5" or "opaque of sha1".
 
-    Allows for redefinition of initial object values. This is typically
+    An example use of this type is the set of built-in functions which
-    used with constants, for example, ``const clever = T &redef;`` would
+    perform hashing:
-    allow the constant to be redefined at some later point during script
-    execution.
 
-.. bro:attr:: &rotate_interval
+    .. code:: bro
 
-    Rotates a file after a specified interval.
+        local handle = md5_hash_init();
+        md5_hash_update(handle, "test");
+        md5_hash_update(handle, "testing");
+        print md5_hash_finish(handle);
 
-.. bro:attr:: &rotate_size
+    Here the opaque type is used to provide a handle to a particular
+    resource which is calculating an MD5 hash incrementally over
+    time, but the details of that resource aren't relevant, it's only
+    necessary to have a handle as a way of identifying it and
+    distinguishing it from other such resources.
 
-    Rotates a file after it has reached a given size in bytes.
+.. bro:type:: any
 
-.. bro:attr:: &add_func
+    Used to bypass strong typing.  For example, a function can take an
+    argument of type ``any`` when it may be of different types.
+    The only operation allowed on a variable of type ``any`` is assignment.
 
-    Can be applied to an identifier with &redef to specify a function to
+    Note that users aren't expected to use this type.  It's provided mainly
-    be called any time a "redef <id> += ..." declaration is parsed.  The
+    for use by some built-in functions and scripts included with Bro.
-    function takes two arguments of the same type as the identifier, the first
-    being the old value of the variable and the second being the new
-    value given after the "+=" operator in the "redef" declaration.  The
-    return value of the function will be the actual new value of the
-    variable after the "redef" declaration is parsed.
 
-.. bro:attr:: &delete_func
+.. bro:type:: void
 
-    Same as &add_func, except for "redef" declarations that use the "-="
+    An internal Bro type (i.e., "void" is not a reserved keyword in the Bro
-    operator.
+    scripting language) representing the absence of a return type for a
+    function.
 
-.. bro:attr:: &expire_func
-
-    Called right before a container element expires.  The function's
-    first parameter is of the same type of the container and the second
-    parameter the same type of the container's index.  The return
-    value is an :bro:type:`interval` indicating the amount of additional
-    time to wait before expiring the container element at the given
-    index (which will trigger another execution of this function).
-
-.. bro:attr:: &read_expire
-
-    Specifies a read expiration timeout for container elements. That is,
-    the element expires after the given amount of time since the last
-    time it has been read. Note that a write also counts as a read.
-
-.. bro:attr:: &write_expire
-
-    Specifies a write expiration timeout for container elements. That
-    is, the element expires after the given amount of time since the
-    last time it has been written.
-
-.. bro:attr:: &create_expire
-
-    Specifies a creation expiration timeout for container elements. That
-    is, the element expires after the given amount of time since it has
-    been inserted into the container, regardless of any reads or writes.
-
-.. bro:attr:: &persistent
-
-    Makes a variable persistent, i.e., its value is written to disk (per
-    default at shutdown time).
-
-.. bro:attr:: &synchronized
-
-    Synchronizes variable accesses across nodes. The value of a
-    ``&synchronized`` variable is automatically propagated to all peers
-    when it changes.
-
-.. bro:attr:: &encrypt
-
-    Encrypts files right before writing them to disk.
-
-.. TODO: needs to be documented in more detail.
-
-.. bro:attr:: &raw_output
-
-    Opens a file in raw mode, i.e., non-ASCII characters are not
-    escaped.
-
-.. bro:attr:: &mergeable
-
-    Prefers set union to assignment for synchronized state. This
-    attribute is used in conjunction with :bro:attr:`&synchronized`
-    container types: when the same container is updated at two peers
-    with different value, the propagation of the state causes a race
-    condition, where the last update succeeds. This can cause
-    inconsistencies and can be avoided by unifying the two sets, rather
-    than merely overwriting the old value.
-
-.. bro:attr:: &priority
-
-    Specifies the execution priority (as a signed integer) of a hook or
-    event handler. Higher values are executed before lower ones. The
-    default value is 0.
-
-.. bro:attr:: &group
-
-    Groups event handlers such that those in the same group can be
-    jointly activated or deactivated.
-
-.. bro:attr:: &log
-
-    Writes a record field to the associated log stream.
-
-.. bro:attr:: &error_handler
-
-    Internally set on the events that are associated with the reporter
-    framework: :bro:id:`reporter_info`, :bro:id:`reporter_warning`, and
-    :bro:id:`reporter_error`.  It prevents any handlers of those events
-    from being able to generate reporter messages that go through any of
-    those events (i.e., it prevents an infinite event recursion).  Instead,
-    such nested reporter messages are output to stderr.
-
-.. bro:attr:: &type_column
-
-    Used by the input framework. It can be used on columns of type
-    :bro:type:`port` and specifies the name of an additional column in
-    the input file which specifies the protocol of the port (tcp/udp/icmp).

doc/scripting/data_struct_table_declaration.bro

@@ -1,13 +1,19 @@
 event bro_init()
     {
+    # Declaration of the table.
     local ssl_services: table[string] of port;
-    
+
+    # Initialize the table.
     ssl_services = table(["SSH"] = 22/tcp, ["HTTPS"] = 443/tcp);
+
+    # Insert one key-yield pair into the table.
     ssl_services["IMAPS"] = 993/tcp;
 
+    # Check if the key "SMTPS" is not in the table.
     if ( "SMTPS" !in ssl_services )
         ssl_services["SMTPS"] = 587/tcp;
 
+    # Iterate over each key in the table.
     for ( k in ssl_services )
         print fmt("Service Name:  %s - Common Port: %s", k, ssl_services[k]);
     }

doc/scripting/data_struct_vector_declaration.bro

@@ -10,6 +10,6 @@ event bro_init()
     
     print fmt("contents of v1: %s", v1);
     print fmt("length of v1: %d", |v1|);
-    print fmt("contents of v1: %s", v2);
+    print fmt("contents of v2: %s", v2);
     print fmt("length of v2: %d", |v2|);
     }

doc/scripting/data_type_pattern_01.bro

@@ -1,6 +1,6 @@
 event bro_init()
     {
-    local test_string = "The quick brown fox jumped over the lazy dog.";
+    local test_string = "The quick brown fox jumps over the lazy dog.";
     local test_pattern = /quick|lazy/;
     
     if ( test_pattern in test_string )

doc/scripting/framework_logging_factorial_02.bro

@@ -1,8 +1,10 @@
 module Factor;
 
 export {
+    # Append the value LOG to the Log::ID enumerable.
     redef enum Log::ID += { LOG };
 
+    # Define a new type called Factor::Info.
     type Info: record {
         num:           count &log;
         factorial_num: count &log;
@@ -20,6 +22,7 @@ function factorial(n: count): count
 
 event bro_init()
     {
+    # Create the logging stream.
     Log::create_stream(LOG, [$columns=Info]);
     }
 

doc/scripting/framework_notice_shortcuts_01.bro

@@ -2,7 +2,6 @@
 @load base/protocols/ssh/
 
 redef Notice::emailed_types += {
-    SSH::Interesting_Hostname_Login,
+    SSH::Interesting_Hostname_Login
-    SSH::Login
 };
 

doc/scripting/framework_notice_shortcuts_02.bro

@@ -3,5 +3,4 @@
 
 redef Notice::type_suppression_intervals += {
     [SSH::Interesting_Hostname_Login] = 1day,
-    [SSH::Login] = 12hrs,
 };

doc/scripting/index.rst

@@ -54,7 +54,8 @@ script and much more in following sections.
 .. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/frameworks/files/detect-MHR.bro
    :lines: 4-6
 
-Lines 3 to 5 of the script process the ``__load__.bro`` script in the
+The first part of the script consists of ``@load`` directives which 
+process the ``__load__.bro`` script in the
 respective directories being loaded.  The ``@load`` directives are
 often considered good practice or even just good manners when writing
 Bro scripts to make sure they can be used on their own. While it's unlikely that in a
@@ -78,29 +79,37 @@ of the :bro:id:`NOTICE` function to generate notices of type
 ``TeamCymruMalwareHashRegistry::Match`` as done in the next section.  Notices
 allow Bro to generate some kind of extra notification beyond its
 default log types.  Often times, this extra notification comes in the
-form of an email generated and sent to a preconfigured address, but can be altered
+form of an email generated and sent to a preconfigured address, but can
-depending on the needs of the deployment.  The export section is finished off with
+be altered depending on the needs of the deployment.  The export section
-the definition of two constants that list the kind of files we want to match against and
+is finished off with the definition of a few constants that list the kind
-the minimum percentage of detection threshold in which we are interested.
+of files we want to match against and the minimum percentage of
+detection threshold in which we are interested.
 
-Up until this point, the script has merely done some basic setup.  With the next section,
+Up until this point, the script has merely done some basic setup.  With
-the script starts to define instructions to take in a given event.
+the next section, the script starts to define instructions to take in
+a given event.
 
 .. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/frameworks/files/detect-MHR.bro
    :lines: 38-71
 
 The workhorse of the script is contained in the event handler for
 ``file_hash``.  The :bro:see:`file_hash` event allows scripts to access
-the information associated with a file for which Bro's file analysis framework has
+the information associated with a file for which Bro's file analysis
-generated a hash.  The event handler is passed the file itself as ``f``, the type of digest
+framework has generated a hash.  The event handler is passed the
-algorithm used as ``kind`` and the hash generated as ``hash``.
+file itself as ``f``, the type of digest algorithm used as ``kind``
+and the hash generated as ``hash``.
 
-On line 3, an ``if`` statement is used to check for the correct type of hash, in this case
+In the ``file_hash`` event handler, there is an ``if`` statement that is used
-a SHA1 hash.  It also checks for a mime type we've defined as being of interest as defined in the
+to check for the correct type of hash, in this case
-constant ``match_file_types``.  The comparison is made against the expression ``f$mime_type``, which uses
+a SHA1 hash.  It also checks for a mime type we've defined as
-the ``$`` dereference operator to check the value ``mime_type`` inside the variable ``f``.  Once both
+being of interest as defined in the constant ``match_file_types``.
-values resolve to true, a local variable is defined to hold a string comprised of the SHA1 hash concatenated
+The comparison is made against the expression ``f$info$mime_type``, which uses
-with ``.malware.hash.cymru.com``; this value will be the domain queried in the malware hash registry.
+the ``$`` dereference operator to check the value ``mime_type``
+inside the variable ``f$info``.  If the entire expression evaluates to true,
+then a helper function is called to do the rest of the work.  In that
+function, a local variable is defined to hold a string comprised of
+the SHA1 hash concatenated with ``.malware.hash.cymru.com``; this
+value will be the domain queried in the malware hash registry.
 
 The rest of the script is contained within a ``when`` block.  In
 short, a ``when`` block is used when Bro needs to perform asynchronous
@@ -111,24 +120,28 @@ this event continues and upon receipt of the values returned by
 :bro:id:`lookup_hostname_txt`, the ``when`` block is executed.  The
 ``when`` block splits the string returned into a portion for the date on which
 the malware was first detected and the detection rate by splitting on an text space
-and storing the values returned in a local table variable.  In line 12, if the table
+and storing the values returned in a local table variable.
-returned by ``split1`` has two entries, indicating a successful split, we store the detection
+In the ``do_mhr_lookup`` function, if the table
-date in ``mhr_first_detected`` and the rate in ``mhr_detect_rate`` on lines 14 and 15 respectively
+returned by ``split1`` has two entries, indicating a successful split, we
+store the detection
+date in ``mhr_first_detected`` and the rate in ``mhr_detect_rate``
 using the appropriate conversion functions.  From this point on, Bro knows it has seen a file
 transmitted which has a hash that has been seen by the Team Cymru Malware Hash Registry, the rest
 of the script is dedicated to producing a notice.
 
-On line 17, the detection time is processed into a string representation and stored in
+The detection time is processed into a string representation and stored in
-``readable_first_detected``.  The script then compares the detection rate against the
+``readable_first_detected``.  The script then compares the detection rate
-``notice_threshold`` that was defined earlier.  If the detection rate is high enough, the script
+against the ``notice_threshold`` that was defined earlier.  If the
-creates a concise description of the notice on line 22, a possible URL to check the sample against
+detection rate is high enough, the script creates a concise description
-``virustotal.com``'s database, and makes the call to :bro:id:`NOTICE` to hand the relevant information
+of the notice and stores it in the ``message`` variable.  It also
-off to the Notice framework.
+creates a possible URL to check the sample against
+``virustotal.com``'s database, and makes the call to :bro:id:`NOTICE`
+to hand the relevant information off to the Notice framework.
 
-In approximately 25 lines of code, Bro provides an amazing
+In approximately a few dozen lines of code, Bro provides an amazing
 utility that would be incredibly difficult to implement and deploy
-with other products.  In truth, claiming that Bro does this in 25
+with other products.  In truth, claiming that Bro does this in such a small
-lines is a misdirection; there is a truly massive number of things
+number of lines is a misdirection; there is a truly massive number of things
 going on behind-the-scenes in Bro, but it is the inclusion of the
 scripting language that gives analysts access to those underlying
 layers in a succinct and well defined manner.
@@ -180,7 +193,7 @@ event definition used by Bro.  As Bro detects DNS requests being
 issued by an originator, it issues this event and any number of
 scripts then have access to the data Bro passes along with the event.
 In this example, Bro passes not only the message, the query, query
-type and query class for the DNS request, but also a then record used
+type and query class for the DNS request, but also a record used
 for the connection itself.
 
 The Connection Record Data Type
@@ -210,8 +223,7 @@ into the connection record data type will be
 :bro:id:`connection_state_remove` .  As detailed in the in-line
 documentation, Bro generates this event just before it decides to
 remove this event from memory, effectively forgetting about it.  Let's
-take a look at a simple script, stored as
+take a look at a simple example script, that will output the connection record
-``connection_record_01.bro``, that will output the connection record
 for a single connection.
 
 .. btest-include:: ${DOC_ROOT}/scripting/connection_record_01.bro
@@ -243,12 +255,12 @@ of reference for accessing data in a script.
 Bro makes extensive use of nested data structures to store state and
 information gleaned from the analysis of a connection as a complete
 unit.  To break down this collection of information, you will have to
-make use of use Bro's field delimiter ``$``.  For example, the
+make use of Bro's field delimiter ``$``.  For example, the
 originating host is referenced by ``c$id$orig_h`` which if given a
 narrative relates to ``orig_h`` which is a member of ``id`` which is
 a member of the data structure referred to as ``c`` that was passed
-into the event handler." Given that the responder port
+into the event handler. Given that the responder port
-(``c$id$resp_p``) is ``53/tcp``, it's likely that Bro's base HTTP scripts
+``c$id$resp_p`` is ``80/tcp``, it's likely that Bro's base HTTP scripts
 can further populate the connection record.  Let's load the
 ``base/protocols/http`` scripts and check the output of our script. 
 
@@ -276,7 +288,7 @@ As mentioned above, including the appropriate ``@load`` statements is
 not only good practice, but can also help to indicate which
 functionalities are being used in a script.  Take a second to run the
 script without the ``-b`` flag and check the output when all of Bro's
-functionality is applied to the tracefile.  
+functionality is applied to the trace file.  
 
 Data Types and Data Structures
 ==============================
@@ -384,9 +396,12 @@ which it was declared.  Local variables tend to be used for values
 that are only needed within a specific scope and once the processing
 of a script passes beyond that scope and no longer used, the variable
 is deleted. Bro maintains names of locals separately from globally
-visible ones, an example of which is illustrated below.  The script
+visible ones, an example of which is illustrated below.
-executes the event handler :bro:id:`bro_init` which in turn calls the
+
-function ``add_two(i: count)`` with an argument of ``10``.  Once Bro
+.. btest-include:: ${DOC_ROOT}/scripting/data_type_local.bro
+
+The script executes the event handler :bro:id:`bro_init` which in turn calls
+the function ``add_two(i: count)`` with an argument of ``10``.  Once Bro
 enters the ``add_two`` function, it provisions a locally scoped
 variable called ``added_two`` to hold the value of ``i+2``, in this
 case, ``12``.  The ``add_two`` function then prints the value of the
@@ -398,8 +413,6 @@ processing the ``bro_init`` function, the variable called ``test`` is
 no longer in scope and, since there exist no other references to the
 value ``12``, the value is also deleted.  
 
-.. btest-include:: ${DOC_ROOT}/scripting/data_type_local.bro
-
 
 Data Structures
 ---------------
@@ -506,20 +519,7 @@ Tables
 
 A table in Bro is a mapping of a key to a value or yield.  While the
 values don't have to be unique, each key in the table must be unique
-to preserve a one-to-one mapping of keys to values.  In the example
+to preserve a one-to-one mapping of keys to values.
-below, we've compiled a table of SSL-enabled services and their common
-ports.  The explicit declaration and constructor for the table on
-lines 5 and 7 lay out the data types of the keys (strings) and the
-data types of the yields (ports) and then fill in some sample key and
-yield pairs.  Line 8 shows how to use a table accessor to insert one
-key-yield pair into the table.  When using the ``in`` operator on a table,
-you are effectively working with the keys of the table.  In the case
-of an ``if`` statement, the ``in`` operator will check for membership among
-the set of keys and return a true or false value.  As seen on line 10,
-we are checking if ``SMTPS`` is not in the set of keys for the
-ssl_services table and if the condition holds true, we add the
-key-yield pair to the table.  Line 13 shows the use of a ``for`` statement
-to iterate over each key currently in the table.  
 
 .. btest-include:: ${DOC_ROOT}/scripting/data_struct_table_declaration.bro
 
@@ -527,6 +527,21 @@ to iterate over each key currently in the table.
 
     @TEST-EXEC: btest-rst-cmd bro ${DOC_ROOT}/scripting/data_struct_table_declaration.bro
 
+In this example,
+we've compiled a table of SSL-enabled services and their common
+ports.  The explicit declaration and constructor for the table are on
+two different lines and lay out the data types of the keys (strings) and the
+data types of the yields (ports) and then fill in some sample key and
+yield pairs.  You can also use a table accessor to insert one
+key-yield pair into the table.  When using the ``in``
+operator on a table, you are effectively working with the keys of the table.
+In the case of an ``if`` statement, the ``in`` operator will check for
+membership among the set of keys and return a true or false value.
+The example shows how to check if ``SMTPS`` is not in the set
+of keys for the ``ssl_services`` table and if the condition holds true,
+we add the key-yield pair to the table.  Finally, the example shows how
+to use a ``for`` statement to iterate over each key currently in the table.  
+
 Simple examples aside, tables can become extremely complex as the keys
 and values for the table become more intricate.  Tables can have keys
 comprised of multiple data types and even a series of elements called
@@ -535,9 +550,15 @@ Bro implies a cost in complexity for the person writing the scripts
 but pays off in effectiveness given the power of Bro as a network
 security platform.
 
-The script below shows a sample table of strings indexed by two
+.. btest-include:: ${DOC_ROOT}/scripting/data_struct_table_complex.bro
+
+.. btest:: data_struct_table_complex
+
+    @TEST-EXEC: btest-rst-cmd bro -b ${DOC_ROOT}/scripting/data_struct_table_complex.bro
+
+This script shows a sample table of strings indexed by two
 strings, a count, and a final string.  With a tuple acting as an
-aggregate key, the order is the important as a change in order would
+aggregate key, the order is important as a change in order would
 result in a new key.  Here, we're using the table to track the
 director, studio, year or release, and lead actor in a series of
 samurai flicks.  It's important to note that in the case of the ``for``
@@ -546,14 +567,9 @@ iterate over, say, the directors; we have to iterate with the exact
 format as the keys themselves.  In this case, we need squared brackets
 surrounding four temporary variables to act as a collection for our
 iteration.  While this is a contrived example, we could easily have
-had keys containing IP addresses (``addr``), ports (``port``) and even a ``string``
+had keys containing IP addresses (``addr``), ports (``port``) and even
-calculated as the result of a reverse hostname lookup.
+a ``string`` calculated as the result of a reverse hostname lookup.
 
-.. btest-include:: ${DOC_ROOT}/scripting/data_struct_table_complex.bro
-
-.. btest:: data_struct_table_complex
-
-    @TEST-EXEC: btest-rst-cmd bro -b ${DOC_ROOT}/scripting/data_struct_table_complex.bro
 
 Vectors
 ~~~~~~~
@@ -657,7 +673,7 @@ using a 20 bit subnet mask.
 
 Because this is a script that doesn't use any kind of network
 analysis, we can handle the event :bro:id:`bro_init` which is always
-generated by Bro's core upon startup.  On lines six and seven, two
+generated by Bro's core upon startup.  In the example script, two
 locally scoped vectors are created to hold our lists of subnets and IP
 addresses respectively.  Then, using a set of nested ``for`` loops, we
 iterate over every subnet and every IP address and use an ``if``
@@ -714,7 +730,7 @@ Bro supports ``usec``, ``msec``, ``sec``, ``min``, ``hr``, or ``day`` which repr
 microseconds, milliseconds, seconds, minutes, hours, and days
 respectively.  In fact, the interval data type allows for a surprising
 amount of variation in its definitions.  There can be a space between
-the numeric constant or they can crammed together like a temporal
+the numeric constant or they can be crammed together like a temporal
 portmanteau.  The time unit can be either singular or plural.  All of
 this adds up to to the fact that both ``42hrs`` and ``42 hr`` are
 perfectly valid and logically equivalent in Bro.  The point, however,
@@ -760,7 +776,7 @@ string against which it will be tested to be on the right.
 In the sample above, two local variables are declared to hold our
 sample sentence and regular expression.  Our regular expression in
 this case will return true if the string contains either the word
-``quick`` or the word ``fox``. The ``if`` statement on line six uses
+``quick`` or the word ``fox``. The ``if`` statement in the script uses
 embedded matching and the ``in`` operator to check for the existence
 of the pattern within the string.  If the statement resolves to true,
 :bro:id:`split` is called to break the string into separate pieces.
@@ -768,8 +784,8 @@ of the pattern within the string.  If the statement resolves to true,
 table of strings indexed by a count.  Each element of the table will
 be the segments before and after any matches against the pattern but
 excluding the actual matches.  In this case, our pattern matches
-twice, and results in a table with three entries.  Lines 11 through 13
+twice, and results in a table with three entries.  The ``print`` statements
-print the contents of the table in order.  
+in the script will print the contents of the table in order.  
 
 .. btest:: data_type_pattern
 
@@ -780,7 +796,7 @@ inequality operators through the ``==`` and ``!=`` operators
 respectively. When used in this manner however, the string must match
 entirely to resolve to true.  For example, the script below uses two
 ternary conditional statements to illustrate the use of the ``==``
-operators with patterns.  On lines 8 and 11 the output is altered based
+operator with patterns.  The output is altered based
 on the result of the comparison between the pattern and the string.  
 
 .. btest-include:: ${DOC_ROOT}/scripting/data_type_pattern_02.bro
@@ -803,7 +819,7 @@ with the ``typedef`` and ``struct`` keywords, Bro allows you to cobble
 together new data types to suit the needs of your situation.
 
 When combined with the ``type`` keyword, ``record`` can generate a
-composite type.  We have, in fact, already encountered a a complex
+composite type.  We have, in fact, already encountered a complex
 example of the ``record`` data type in the earlier sections, the
 :bro:type:`connection` record passed to many events. Another one,
 :bro:type:`Conn::Info`, which corresponds to the fields logged into
@@ -915,11 +931,7 @@ through a contrived example of simply logging the digits 1 through 10
 and their corresponding factorial to the default ASCII log writer. 
 It's always best to work through the problem once, simulating the
 desired output with ``print`` and ``fmt`` before attempting to dive
-into the Logging Framework.  Below is a script that defines a
+into the Logging Framework.
-factorial function to recursively calculate the factorial of a
-unsigned integer passed as an argument to the function.  Using
-``print`` and  :bro:id:`fmt` we can ensure that Bro can perform these
-calculations correctly as well get an idea of the answers ourselves.
 
 .. btest-include:: ${DOC_ROOT}/scripting/framework_logging_factorial_01.bro
 
@@ -927,19 +939,28 @@ calculations correctly as well get an idea of the answers ourselves.
 
    @TEST-EXEC: btest-rst-cmd bro ${DOC_ROOT}/scripting/framework_logging_factorial_01.bro
 
+This script defines a factorial function to recursively calculate the
+factorial of a unsigned integer passed as an argument to the function.  Using
+``print`` and  :bro:id:`fmt` we can ensure that Bro can perform these
+calculations correctly as well get an idea of the answers ourselves.
+
 The output of the script aligns with what we expect so now it's time
-to integrate the Logging Framework.  As mentioned above we have to
+to integrate the Logging Framework.
-perform a few steps before we can issue the :bro:id:`Log::write` 
+
-method and produce a logfile.  As we are working within a namespace
+.. btest-include:: ${DOC_ROOT}/scripting/framework_logging_factorial_02.bro
-and informing an outside entity of workings and data internal to the
+
-namespace, we use an ``export`` block.  First we need to inform Bro
+As mentioned above we have to perform a few steps before we can
+issue the :bro:id:`Log::write` method and produce a logfile.
+As we are working within a namespace and informing an outside
+entity of workings and data internal to the namespace, we use
+an ``export`` block.  First we need to inform Bro
 that we are going to be adding another Log Stream by adding a value to
-the :bro:type:`Log::ID` enumerable.  In line 6 of the script, we append the
+the :bro:type:`Log::ID` enumerable.  In this script, we append the
 value ``LOG`` to the ``Log::ID`` enumerable, however due to this being in
 an export block the value appended to ``Log::ID`` is actually
 ``Factor::Log``.  Next, we need to define the name and value pairs
-that make up the data of our logs and dictate its format.  Lines 8
+that make up the data of our logs and dictate its format.  This script
-through 11 define a new datatype called an ``Info`` record (actually,
+defines a new record datatype called ``Info`` (actually,
 ``Factor::Info``) with two fields, both unsigned integers. Each of the
 fields in the ``Factor::Log`` record type include the ``&log``
 attribute, indicating that these fields should be passed to the
@@ -947,16 +968,14 @@ Logging Framework when ``Log::write`` is called.  Were there to be
 any name value pairs without the ``&log`` attribute, those fields
 would simply be ignored during logging but remain available for the
 lifespan of the variable.  The next step is to create the logging
-stream with :bro:id:`Log::create_stream` which takes a Log::ID and a
+stream with :bro:id:`Log::create_stream` which takes a ``Log::ID`` and a
-record as its arguments.  In this example, on line 25, we call the
+record as its arguments.  In this example, we call the
 ``Log::create_stream`` method and pass ``Factor::LOG`` and the
 ``Factor::Info`` record as arguments. From here on out, if we issue
 the ``Log::write`` command with the correct ``Log::ID`` and a properly
 formatted ``Factor::Info`` record, a log entry will be generated.  
 
-.. btest-include:: ${DOC_ROOT}/scripting/framework_logging_factorial_02.bro
+Now, if we run this script, instead of generating
-
-Now, if we run the new version of the script, instead of generating
 logging information to stdout, no output is created.  Instead the
 output is all in ``factor.log``, properly formatted and organized.
 
@@ -995,13 +1014,13 @@ remaining logs to factor.log.
    :lines: 38-62
    :linenos:
 
-To dynamically alter the file in which a stream writes its logs a
+To dynamically alter the file in which a stream writes its logs, a
-filter can specify function returns a string to be used as the
+filter can specify a function that returns a string to be used as the
 filename for the current call to ``Log::write``. The definition for
 this function has to take as its parameters a ``Log::ID`` called id, a
 string called ``path`` and the appropriate record type for the logs called
-``rec``.  You can see the definition of ``mod5`` used in this example on
+``rec``.  You can see the definition of ``mod5`` used in this example
-line one conforms to that requirement.  The function simply returns
+conforms to that requirement.  The function simply returns
 ``factor-mod5`` if the factorial is divisible evenly by 5, otherwise, it
 returns ``factor-non5``.  In the additional ``bro_init`` event
 handler, we define a locally scoped ``Log::Filter`` and assign it a
@@ -1074,7 +1093,8 @@ make a call to :bro:id:`NOTICE` supplying it with an appropriate
 :bro:type:`Notice::Info` record.  Often times the call to ``NOTICE``
 includes just the ``Notice::Type``, and a concise message.  There are
 however, significantly more options available when raising notices as
-seen in the table below.  The only field in the table below whose
+seen in the definition of :bro:type:`Notice::Info`.  The only field in
+``Notice::Info`` whose
 attributes make it a required field is the ``note`` field.  Still,
 good manners are always important and including a concise message in
 ``$msg`` and, where necessary, the contents of the connection record
@@ -1086,57 +1106,6 @@ that are commonly included, ``$identifier`` and ``$suppress_for`` are
 built around the automated suppression feature of the Notice Framework
 which we will cover shortly.  
 
-.. todo::
-
-    Once the link to ``Notice::Info`` work I think we should take out
-    the table. That's too easy to get out of date.
-
-+---------------------+------------------------------------------------------------------+----------------+----------------------------------------+
-| Field               | Type                                                             | Attributes     | Use                                    |
-+=====================+==================================================================+================+========================================+
-| ts                  | time                                                             | &log &optional | The time of the notice                 |
-+---------------------+------------------------------------------------------------------+----------------+----------------------------------------+
-| uid                 | string                                                           | &log &optional | A unique connection ID                 |
-+---------------------+------------------------------------------------------------------+----------------+----------------------------------------+
-| id                  | conn_id                                                          | &log &optional | A 4-tuple to identify endpoints        |
-+---------------------+------------------------------------------------------------------+----------------+----------------------------------------+
-| conn                | connection                                                       | &optional      | Shorthand for the uid and id           |
-+---------------------+------------------------------------------------------------------+----------------+----------------------------------------+
-| iconn               | icmp_conn                                                        | &optional      | Shorthand for the uid and id           |
-+---------------------+------------------------------------------------------------------+----------------+----------------------------------------+
-| proto               | transport_proto                                                  | &log &optional | Transport protocol                     |
-+---------------------+------------------------------------------------------------------+----------------+----------------------------------------+
-| note                | Notice::Type                                                     | &log           | The Notice::Type of the notice         |
-+---------------------+------------------------------------------------------------------+----------------+----------------------------------------+
-| msg                 | string                                                           | &log &optional | Human readable message                 |
-+---------------------+------------------------------------------------------------------+----------------+----------------------------------------+
-| sub                 | string                                                           | &log &optional | Human readable message                 |
-+---------------------+------------------------------------------------------------------+----------------+----------------------------------------+
-| src                 | addr                                                             | &log &optional | Source address if no conn_id           |
-+---------------------+------------------------------------------------------------------+----------------+----------------------------------------+
-| dst                 | addr                                                             | &log &optional | Destination addr if no conn_id         |
-+---------------------+------------------------------------------------------------------+----------------+----------------------------------------+
-| p                   | port                                                             | &log &optional | Port if no conn_id                     |
-+---------------------+------------------------------------------------------------------+----------------+----------------------------------------+
-| n                   | count                                                            | &log &optional | Count or status code                   |
-+---------------------+------------------------------------------------------------------+----------------+----------------------------------------+
-| src_peer            | event_peer                                                       | &log &optional | Peer that raised the notice            |
-+---------------------+------------------------------------------------------------------+----------------+----------------------------------------+
-| peer_descr          | string                                                           | &log &optional | Text description of the src_peer       |
-+---------------------+------------------------------------------------------------------+----------------+----------------------------------------+
-| actions             | set[Notice::Action]                                              | &log &optional | Actions applied to the notice          |
-+---------------------+------------------------------------------------------------------+----------------+----------------------------------------+
-| policy_items        | set[count]                                                       | &log &optional | Policy items that have been applied    |
-+---------------------+------------------------------------------------------------------+----------------+----------------------------------------+
-| email_body_sections | vector                                                           | &optional       | Body of the email for email notices.  |
-+---------------------+------------------------------------------------------------------+----------------+----------------------------------------+
-| email_delay_tokens  | set[string]                                                      | &optional      | Delay functionality for email notices. |
-+---------------------+------------------------------------------------------------------+----------------+----------------------------------------+
-| identifier          | string                                                           | &optional      | A unique string identifier             |
-+---------------------+------------------------------------------------------------------+----------------+----------------------------------------+
-| suppress_for        | interval                                                         | &log &optional | Length of time to suppress a notice.   |
-+---------------------+------------------------------------------------------------------+----------------+----------------------------------------+
-
 One of the default policy scripts raises a notice when an SSH login
 has been heuristically detected and the originating hostname is one
 that would raise suspicion.  Effectively, the script attempts to
@@ -1153,15 +1122,15 @@ possible while staying concise.
 
 While much of the script relates to the actual detection, the parts
 specific to the Notice Framework are actually quite interesting in
-themselves.  On line 18 the script's ``export`` block adds the value
+themselves.  The script's ``export`` block adds the value
 ``SSH::Interesting_Hostname_Login`` to the enumerable constant
 ``Notice::Type`` to indicate to the Bro core that a new type of notice
 is being defined.  The script then calls ``NOTICE`` and defines the
 ``$note``, ``$msg``, ``$sub`` and ``$conn`` fields of the
-:bro:type:`Notice::Info` record.  Line 42 also includes a ternary if
+:bro:type:`Notice::Info` record.  There are two ternary if
-statement that modifies the ``$msg`` text depending on whether the
+statements that modify the ``$msg`` text depending on whether the
 host is a local address and whether it is the client or the server.
-This use of :bro:id:`fmt` and a ternary operators is a concise way to
+This use of :bro:id:`fmt` and ternary operators is a concise way to
 lend readability to the notices that are generated without the need
 for branching ``if`` statements that each raise a specific notice.
 
@@ -1222,7 +1191,7 @@ from the connection relative to the behavior that has been observed by
 Bro.  
 
 .. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ssl/expiring-certs.bro
-   :lines: 60-63
+   :lines: 64-68
 
 In the :doc:`/scripts/policy/protocols/ssl/expiring-certs.bro` script
 which identifies when SSL certificates are set to expire and raises
@@ -1302,9 +1271,9 @@ in the call to ``NOTICE``.
 
 .. btest-include:: ${DOC_ROOT}/scripting/framework_notice_shortcuts_01.bro
 
-The Notice Policy shortcut above adds the ``Notice::Types`` of
+The Notice Policy shortcut above adds the ``Notice::Type`` of
-SSH::Interesting_Hostname_Login and SSH::Login to the
+``SSH::Interesting_Hostname_Login`` to the
-Notice::emailed_types set while the shortcut below alters the length
+``Notice::emailed_types`` set while the shortcut below alters the length
 of time for which those notices will be suppressed.
 
 .. btest-include:: ${DOC_ROOT}/scripting/framework_notice_shortcuts_02.bro

man/CMakeLists.txt

@@ -0,0 +1,5 @@
+
+install(DIRECTORY . DESTINATION ${BRO_MAN_INSTALL_PATH}/man8 FILES_MATCHING
+        PATTERN "*.8"
+)
+

man/bro.8

@@ -0,0 +1,164 @@
+.TH BRO "8" "November 2014" "bro" "System Administration Utilities"
+.SH NAME
+bro \- passive network traffic analyzer
+.SH SYNOPSIS
+.B bro
+\/\fP [\fIoptions\fR] [\fIfile\fR ...]
+.SH DESCRIPTION
+Bro is primarily a security monitor that inspects all traffic on a link in
+depth for signs of suspicious activity. More generally, however, Bro
+supports a wide range of traffic analysis tasks even outside of the
+security domain, including performance measurements and helping with
+trouble-shooting.
+
+Bro comes with built-in functionality for a range of analysis and detection
+tasks, including detecting malware by interfacing to external registries,
+reporting vulnerable versions of software seen on the network, identifying
+popular web applications, detecting SSH brute-forcing, validating SSL
+certificate chains, among others.
+.SH OPTIONS
+.TP
+.B <file>
+policy file, or read stdin
+.TP
+\fB\-a\fR,\ \-\-parse\-only
+exit immediately after parsing scripts
+.TP
+\fB\-b\fR,\ \-\-bare\-mode
+don't load scripts from the base/ directory
+.TP
+\fB\-d\fR,\ \-\-debug\-policy
+activate policy file debugging
+.TP
+\fB\-e\fR,\ \-\-exec <bro code>
+augment loaded policies by given code
+.TP
+\fB\-f\fR,\ \-\-filter <filter>
+tcpdump filter
+.TP
+\fB\-g\fR,\ \-\-dump\-config
+dump current config into .state dir
+.TP
+\fB\-h\fR,\ \-\-help|\-?
+command line help
+.TP
+\fB\-i\fR,\ \-\-iface <interface>
+read from given interface
+.TP
+\fB\-p\fR,\ \-\-prefix <prefix>
+add given prefix to policy file resolution
+.TP
+\fB\-r\fR,\ \-\-readfile <readfile>
+read from given tcpdump file
+.TP
+\fB\-y\fR,\ \-\-flowfile <file>[=<ident>]
+read from given flow file
+.TP
+\fB\-Y\fR,\ \-\-netflow <ip>:<prt>[=<id>]
+read flow from socket
+.TP
+\fB\-s\fR,\ \-\-rulefile <rulefile>
+read rules from given file
+.TP
+\fB\-t\fR,\ \-\-tracefile <tracefile>
+activate execution tracing
+.TP
+\fB\-w\fR,\ \-\-writefile <writefile>
+write to given tcpdump file
+.TP
+\fB\-v\fR,\ \-\-version
+print version and exit
+.TP
+\fB\-x\fR,\ \-\-print\-state <file.bst>
+print contents of state file
+.TP
+\fB\-z\fR,\ \-\-analyze <analysis>
+run the specified policy file analysis
+.TP
+\fB\-C\fR,\ \-\-no\-checksums
+ignore checksums
+.TP
+\fB\-D\fR,\ \-\-dfa\-size <size>
+DFA state cache size
+.TP
+\fB\-F\fR,\ \-\-force\-dns
+force DNS
+.TP
+\fB\-I\fR,\ \-\-print\-id <ID name>
+print out given ID
+.TP
+\fB\-K\fR,\ \-\-md5\-hashkey <hashkey>
+set key for MD5\-keyed hashing
+.TP
+\fB\-L\fR,\ \-\-rule\-benchmark
+benchmark for rules
+.TP
+\fB\-N\fR,\ \-\-print\-plugins
+print available plugins and exit (\fB\-NN\fR for verbose)
+.TP
+\fB\-O\fR,\ \-\-optimize
+optimize policy script
+.TP
+\fB\-P\fR,\ \-\-prime\-dns
+prime DNS
+.TP
+\fB\-Q\fR,\ \-\-time
+print execution time summary to stderr
+.TP
+\fB\-R\fR,\ \-\-replay <events.bst>
+replay events
+.TP
+\fB\-S\fR,\ \-\-debug\-rules
+enable rule debugging
+.TP
+\fB\-T\fR,\ \-\-re\-level <level>
+set 'RE_level' for rules
+.TP
+\fB\-U\fR,\ \-\-status\-file <file>
+Record process status in file
+.TP
+\fB\-W\fR,\ \-\-watchdog
+activate watchdog timer
+.TP
+\fB\-X\fR,\ \-\-broxygen
+generate documentation based on config file
+.TP
+\fB\-\-pseudo\-realtime[=\fR<speedup>]
+enable pseudo\-realtime for performance evaluation (default 1)
+.TP
+\fB\-\-load\-seeds\fR <file>
+load seeds from given file
+.TP
+\fB\-\-save\-seeds\fR <file>
+save seeds to given file
+.SH ENVIRONMENT
+.TP
+.B BROPATH
+file search path
+.TP
+.B BRO_PLUGIN_PATH
+plugin search path
+.TP
+.B BRO_PLUGIN_ACTIVATE
+plugins to always activate
+.TP
+.B BRO_PREFIXES
+prefix list
+.TP
+.B BRO_DNS_FAKE
+disable DNS lookups
+.TP
+.B BRO_SEED_FILE
+file to load seeds from
+.TP
+.B BRO_LOG_SUFFIX
+ASCII log file extension
+.TP
+.B BRO_PROFILER_FILE
+Output file for script execution statistics
+.TP
+.B BRO_DISABLE_BROXYGEN
+Disable Broxygen documentation support
+.SH AUTHOR
+.B bro
+was written by The Bro Project <info@bro.org>.

pkg/make-deb-packages

@@ -16,6 +16,7 @@ the 'dpkg-dev' package, please install it first.
 }
 
 prefix=/opt/bro
+localstatedir=/var/opt/bro
 
 # During the packaging process, `dpkg-shlibs` will fail if used on a library
 # that links to other internal/project libraries unless an RPATH is used or
@@ -31,7 +32,7 @@ cd ..
 ( cd build && make package )
 
 # Full Bro package
-./configure --prefix=${prefix} --pkg-name-prefix=Bro --binary-package
+./configure --prefix=${prefix} --localstatedir=${localstatedir} --pkg-name-prefix=Bro --binary-package
 ( cd build && make package )
 
 # Broccoli
@@ -42,6 +43,6 @@ cd ../..
 
 # Broctl
 cd aux/broctl
-./configure --prefix=${prefix} --binary-package
+./configure --prefix=${prefix} --localstatedir=${localstatedir} --binary-package
 ( cd build && make package && mv *.deb ../../../build/ )
 cd ../..

pkg/make-rpm-packages

@@ -15,6 +15,7 @@ the 'rpm-build' package, please install it first.
 }
 
 prefix=/opt/bro
+localstatedir=/var/opt/bro
 
 cd ..
 
@@ -24,7 +25,7 @@ cd ..
 ( cd build && make package )
 
 # Full Bro package
-./configure --prefix=${prefix} --pkg-name-prefix=Bro --binary-package
+./configure --prefix=${prefix} --localstatedir=${localstatedir} --pkg-name-prefix=Bro --binary-package
 ( cd build && make package )
 
 # Broccoli
@@ -35,6 +36,6 @@ cd ../..
 
 # Broctl
 cd aux/broctl
-./configure --prefix=${prefix} --binary-package
+./configure --prefix=${prefix} --localstatedir=${localstatedir} --binary-package
 ( cd build && make package && mv *.rpm ../../../build/ )
 cd ../..

scripts/base/files/unified2/main.bro

@@ -71,11 +71,50 @@ global classification_map: table[count] of string;
 global sid_map: table[count] of string;
 global gen_map: table[count] of string;
 
+global num_classification_map_reads = 0;
+global num_sid_map_reads = 0;
+global num_gen_map_reads = 0;
+global watching = F;
+
 # For reading in config files.
 type OneLine: record {
 	line: string;
 };
 
+function mappings_initialized(): bool
+	{
+	return num_classification_map_reads > 0 &&
+	       num_sid_map_reads > 0 &&
+	       num_gen_map_reads > 0;
+	}
+
+function start_watching()
+	{
+	if ( watching )
+		return;
+
+	watching = T;
+
+	if ( watch_dir != "" )
+		{
+		Dir::monitor(watch_dir, function(fname: string)
+			{
+			Input::add_analysis([$source=fname,
+			                     $reader=Input::READER_BINARY,
+			                     $mode=Input::STREAM,
+			                     $name=fname]);
+			}, 10secs);
+		}
+
+	if ( watch_file != "" )
+		{
+		Input::add_analysis([$source=watch_file,
+		                     $reader=Input::READER_BINARY,
+		                     $mode=Input::STREAM,
+		                     $name=watch_file]);
+		}
+	}
+
 function create_info(ev: IDSEvent): Info
 	{
 	local info = Info($ts=ev$ts,
@@ -136,11 +175,33 @@ event Unified2::read_classification_line(desc: Input::EventDescription, tpe: Inp
 		}
 	}
 
+event Input::end_of_data(name: string, source: string)
+	{
+	if ( name == classification_config )
+		++num_classification_map_reads;
+	else if ( name == sid_msg )
+		++num_sid_map_reads;
+	else if ( name == gen_msg )
+		++num_gen_map_reads;
+	else
+		return;
+
+	if ( watching )
+		return;
+
+	if ( mappings_initialized() )
+		start_watching();
+	}
+
 event bro_init() &priority=5
 	{
 	Log::create_stream(Unified2::LOG, [$columns=Info, $ev=log_unified2]);
 
-	if ( sid_msg != "" )
+	if ( sid_msg == "" )
+		{
+		num_sid_map_reads = 1;
+		}
+	else
 		{
 		Input::add_event([$source=sid_msg,
 		                  $reader=Input::READER_RAW,
@@ -151,7 +212,11 @@ event bro_init() &priority=5
 		                  $ev=Unified2::read_sid_msg_line]);
 		}
 
-	if ( gen_msg != "" )
+	if ( gen_msg == "" )
+		{
+		num_gen_map_reads = 1;
+		}
+	else
 		{
 		Input::add_event([$source=gen_msg,
 		                  $name=gen_msg,
@@ -162,7 +227,11 @@ event bro_init() &priority=5
 		                  $ev=Unified2::read_gen_msg_line]);
 		}
 
-	if ( classification_config != "" )
+	if ( classification_config == "" )
+		{
+		num_classification_map_reads = 1;
+		}
+	else
 		{
 		Input::add_event([$source=classification_config,
 		                  $name=classification_config,
@@ -173,24 +242,8 @@ event bro_init() &priority=5
 		                  $ev=Unified2::read_classification_line]);
 		}
 
-	if ( watch_dir != "" )
+	if ( mappings_initialized() )
-		{
+		start_watching();
-		Dir::monitor(watch_dir, function(fname: string)
-			{
-			Input::add_analysis([$source=fname,
-			                     $reader=Input::READER_BINARY,
-			                     $mode=Input::STREAM,
-			                     $name=fname]);
-			}, 10secs);
-		}
-
-	if ( watch_file != "" )
-		{
-		Input::add_analysis([$source=watch_file,
-		                     $reader=Input::READER_BINARY,
-		                     $mode=Input::STREAM,
-		                     $name=watch_file]);
-		}
 	}
 
 event file_new(f: fa_file)

scripts/base/frameworks/files/magic/__load__.bro

@@ -1,2 +1,3 @@
 @load-sigs ./general
+@load-sigs ./msoffice
 @load-sigs ./libmagic

scripts/base/frameworks/files/magic/general.sig

@@ -1,11 +1,137 @@
 # General purpose file magic signatures.
 
 signature file-plaintext {
-    file-magic /([[:print:][:space:]]{10})/
+    file-magic /^([[:print:][:space:]]{10})/
     file-mime "text/plain", -20
 }
 
 signature file-tar {
-    file-magic /([[:print:]\x00]){100}(([[:digit:]\x00\x20]){8}){3}/
+    file-magic /^[[:print:]\x00]{100}([[:digit:]\x20]{7}\x00){3}([[:digit:]\x20]{11}\x00){2}([[:digit:]\x00\x20]{7}[\x20\x00])[0-7\x00]/
-    file-mime "application/x-tar", 150
+    file-mime "application/x-tar", 100
+}
+
+signature file-zip {
+	file-mime "application/zip", 10
+	file-magic /^PK\x03\x04.{2}/
+}
+
+signature file-jar {
+	file-mime "application/java-archive", 100
+	file-magic /^PK\x03\x04.{1,200}\x14\x00..META-INF\/MANIFEST\.MF/
+}
+
+signature file-java-applet {
+	file-magic /^\xca\xfe\xba\xbe...[\x2e-\x34]/
+	file-mime "application/x-java-applet", 71
+}
+
+# Shockwave flash
+signature file-swf {
+	file-magic /^(F|C|Z)WS/
+	file-mime "application/x-shockwave-flash", 60
+}
+
+# Microsoft Outlook's Transport Neutral Encapsulation Format
+signature file-tnef {
+	file-magic /^\x78\x9f\x3e\x22/
+	file-mime "application/vnd.ms-tnef", 100
+}
+
+# Mac OS X DMG files
+signature file-dmg {
+	file-magic /^(\x78\x01\x73\x0D\x62\x62\x60|\x78\xDA\x63\x60\x18\x05|\x78\x01\x63\x60\x18\x05|\x78\xDA\x73\x0D|\x78[\x01\xDA]\xED[\xD0-\xD9])/
+	file-mime "application/x-dmg", 100
+}
+
+# Mac OS X Mach-O executable
+signature file-mach-o {
+	file-magic /^[\xce\xcf]\xfa\xed\xfe/
+	file-mime "application/x-mach-o-executable", 100
+}
+
+# Mac OS X Universal Mach-O executable
+signature file-mach-o-universal {
+	file-magic /^\xca\xfe\xba\xbe..\x00[\x01-\x14]/
+	file-mime "application/x-mach-o-executable", 100
+}
+
+# XAR (eXtensible ARchive) format. 
+# Mac OS X uses this for the .pkg format.
+signature file-xar {
+	file-magic /^xar\!/
+	file-mime "application/x-xar", 100
+}
+
+signature file-pkcs7 {
+	file-magic /^MIME-Version:.*protocol=\"application\/pkcs7-signature\"/
+	file-mime "application/pkcs7-signature", 100
+}
+
+# Concatenated X.509 certificates in textual format.
+signature file-pem {
+	file-magic /^-----BEGIN CERTIFICATE-----/
+	file-mime "application/x-pem"
+}
+
+# Java Web Start file.
+signature file-jnlp {
+	file-magic /^\<jnlp\x20/
+	file-mime "application/x-java-jnlp-file", 100
+}
+
+signature file-ico {
+	file-magic /^\x00\x00\x01\x00/
+	file-mime "image/x-icon", 70
+}
+
+signature file-cur {
+	file-magic /^\x00\x00\x02\x00/
+	file-mime "image/x-cursor", 70
+}
+
+signature file-pcap {
+	file-magic /^(\xa1\xb2\xc3\xd4|\xd4\xc3\xb2\xa1)/
+	file-mime "application/vnd.tcpdump.pcap", 70
+}
+
+signature file-pcap-ng {
+	file-magic /^\x0a\x0d\x0d\x0a.{4}(\x1a\x2b\x3c\x4d|\x4d\x3c\x2b\x1a)/
+	file-mime "application/vnd.tcpdump.pcap", 100
+}
+
+signature file-shellscript {
+	file-mime "text/x-shellscript", 250
+	file-magic /^\x23\x21[^\n]{1,15}bin\/(env[[:space:]]+)?(ba|tc|c|z|fa|ae|k)?sh/
+}
+
+signature file-perl {
+	file-magic /^\x23\x21[^\n]{1,15}bin\/(env[[:space:]]+)?perl/
+	file-mime "text/x-perl", 60
+}
+
+signature file-ruby {
+	file-magic /^\x23\x21[^\n]{1,15}bin\/(env[[:space:]]+)?ruby/
+	file-mime "text/x-ruby", 60
+}
+
+signature file-python {
+	file-magic /^\x23\x21[^\n]{1,15}bin\/(env[[:space:]]+)?python/
+	file-mime "text/x-python", 60
+}
+
+signature file-php {
+	file-magic /^.*<\?php/
+	file-mime "text/x-php", 40
+}
+
+# Stereolithography ASCII format
+signature file-stl-ascii {
+	file-magic /^solid\x20/
+	file-mime "application/sla", 10
+}
+
+# Sketchup model file
+signature file-skp {
+	file-magic /^\xFF\xFE\xFF\x0E\x53\x00\x6B\x00\x65\x00\x74\x00\x63\x00\x68\x00\x55\x00\x70\x00\x20\x00\x4D\x00\x6F\x00\x64\x00\x65\x00\x6C\x00/
+	file-mime "application/skp", 100
 }

scripts/base/frameworks/files/magic/libmagic.sig

@@ -7,42 +7,18 @@
 # The instrumented version of the `file` command used to generate these
 # is located at: https://github.com/jsiwek/file/tree/bro-signatures.
 
-# >2080  string,=Foglio di lavoro Microsoft Exce (len=31), ["%s"], swap_endian=0
-signature file-magic-auto0 {
-	file-mime "application/vnd.ms-excel", 340
-	file-magic /(.{2080})(Foglio di lavoro Microsoft Exce)/
-}
-
 # >2  string,=---BEGIN PGP PUBLIC KEY BLOCK- (len=30), ["PGP public key block"], swap_endian=0
 signature file-magic-auto1 {
 	file-mime "application/pgp-keys", 330
 	file-magic /(.{2})(\x2d\x2d\x2dBEGIN PGP PUBLIC KEY BLOCK\x2d)/
 }
 
-# >2080  string,=Microsoft Excel 5.0 Worksheet (len=29), ["%s"], swap_endian=0
-signature file-magic-auto2 {
-	file-mime "application/vnd.ms-excel", 320
-	file-magic /(.{2080})(Microsoft Excel 5\x2e0 Worksheet)/
-}
-
 # >11  string,=must be converted with BinHex (len=29), ["BinHex binary text"], swap_endian=0
 signature file-magic-auto3 {
 	file-mime "application/mac-binhex40", 320
 	file-magic /(.{11})(must be converted with BinHex)/
 }
 
-# >2080  string,=Microsoft Word 6.0 Document (len=27), ["%s"], swap_endian=0
-signature file-magic-auto4 {
-	file-mime "application/msword", 300
-	file-magic /(.{2080})(Microsoft Word 6\x2e0 Document)/
-}
-
-# >2080  string,=Documento Microsoft Word 6 (len=26), ["Spanish Microsoft Word 6 document data"], swap_endian=0
-signature file-magic-auto5 {
-	file-mime "application/msword", 290
-	file-magic /(.{2080})(Documento Microsoft Word 6)/
-}
-
 # >0  string,=-----BEGIN PGP SIGNATURE- (len=25), ["PGP signature"], swap_endian=0
 signature file-magic-auto6 {
 	file-mime "application/pgp-signature", 280
@@ -92,36 +68,6 @@ signature file-magic-auto13 {
 	file-magic /(\x23\x21 ?\x2fusr\x2flocal\x2fbin\x2fgawk)/
 }
 
-# >0  string/wt,=#! /usr/local/bin/bash (len=22), ["Bourne-Again shell script text executable"], swap_endian=0
-signature file-magic-auto14 {
-	file-mime "text/x-shellscript", 250
-	file-magic /(\x23\x21 ?\x2fusr\x2flocal\x2fbin\x2fbash)/
-}
-
-# >0  string/wt,=#! /usr/local/bin/tcsh (len=22), ["Tenex C shell script text executable"], swap_endian=0
-signature file-magic-auto15 {
-	file-mime "text/x-shellscript", 250
-	file-magic /(\x23\x21 ?\x2fusr\x2flocal\x2fbin\x2ftcsh)/
-}
-
-# >0  string/wt,=#! /usr/local/bin/zsh (len=21), ["Paul Falstad's zsh script text executable"], swap_endian=0
-signature file-magic-auto16 {
-	file-mime "text/x-shellscript", 240
-	file-magic /(\x23\x21 ?\x2fusr\x2flocal\x2fbin\x2fzsh)/
-}
-
-# >0  string/wt,=#! /usr/local/bin/ash (len=21), ["Neil Brown's ash script text executable"], swap_endian=0
-signature file-magic-auto17 {
-	file-mime "text/x-shellscript", 240
-	file-magic /(\x23\x21 ?\x2fusr\x2flocal\x2fbin\x2fash)/
-}
-
-# >0  string/wt,=#! /usr/local/bin/ae (len=20), ["Neil Brown's ae script text executable"], swap_endian=0
-signature file-magic-auto18 {
-	file-mime "text/x-shellscript", 230
-	file-magic /(\x23\x21 ?\x2fusr\x2flocal\x2fbin\x2fae)/
-}
-
 # >0  string,=# PaCkAgE DaTaStReAm (len=20), ["pkg Datastream (SVR4)"], swap_endian=0
 signature file-magic-auto19 {
 	file-mime "application/x-svr4-package", 230
@@ -140,30 +86,12 @@ signature file-magic-auto21 {
 	file-magic /(\x5bKDE Desktop Entry\x5d)/
 }
 
-# >512  string,=R\000o\000o\000t\000 \000E\000n\000t\000r\000y (len=19), ["Microsoft Word Document"], swap_endian=0
-signature file-magic-auto22 {
-	file-mime "application/msword", 220
-	file-magic /(.{512})(R\x00o\x00o\x00t\x00 \x00E\x00n\x00t\x00r\x00y)/
-}
-
 # >0  string,=!<arch>\n__________E (len=19), ["MIPS archive"], swap_endian=0
 signature file-magic-auto23 {
 	file-mime "application/x-archive", 220
 	file-magic /(\x21\x3carch\x3e\x0a\x5f\x5f\x5f\x5f\x5f\x5f\x5f\x5f\x5f\x5fE)/
 }
 
-# >0  string/wt,=#! /usr/local/tcsh (len=18), ["Tenex C shell script text executable"], swap_endian=0
-signature file-magic-auto24 {
-	file-mime "text/x-shellscript", 210
-	file-magic /(\x23\x21 ?\x2fusr\x2flocal\x2ftcsh)/
-}
-
-# >0  string/wt,=#! /usr/local/bash (len=18), ["Bourne-Again shell script text executable"], swap_endian=0
-signature file-magic-auto25 {
-	file-mime "text/x-shellscript", 210
-	file-magic /(\x23\x21 ?\x2fusr\x2flocal\x2fbash)/
-}
-
 # >0  string/t,=# KDE Config File (len=17), ["KDE config file"], swap_endian=0
 signature file-magic-auto26 {
 	file-mime "application/x-kdelnk", 200
@@ -189,12 +117,6 @@ signature file-magic-auto29 {
 	file-magic /(\x23\x21 ?\x2fusr\x2fbin\x2fnawk)/
 }
 
-# >0  string/wt,=#! /usr/bin/tcsh (len=16), ["Tenex C shell script text executable"], swap_endian=0
-signature file-magic-auto30 {
-	file-mime "text/x-shellscript", 190
-	file-magic /(\x23\x21 ?\x2fusr\x2fbin\x2ftcsh)/
-}
-
 # >0  string/wt,=#! /usr/bin/gawk (len=16), ["GNU awk script text executable"], swap_endian=0
 signature file-magic-auto31 {
 	file-mime "text/x-gawk", 190
@@ -207,12 +129,6 @@ signature file-magic-auto32 {
 	file-magic /(.{369})(MICROSOFT PIFEX\x00)/
 }
 
-# >0  string/wt,=#! /usr/bin/bash (len=16), ["Bourne-Again shell script text executable"], swap_endian=0
-signature file-magic-auto33 {
-	file-mime "text/x-shellscript", 190
-	file-magic /(\x23\x21 ?\x2fusr\x2fbin\x2fbash)/
-}
-
 # >0  string/w,=#VRML V1.0 ascii (len=16), ["VRML 1 file"], swap_endian=0
 signature file-magic-auto34 {
 	file-mime "model/vrml", 190
@@ -334,12 +250,6 @@ signature file-magic-auto51 {
 	file-magic /(\x23\x21 ?\x2fusr\x2fbin\x2fawk)/
 }
 
-# >0  string/wt,=#! /usr/bin/zsh (len=15), ["Paul Falstad's zsh script text executable"], swap_endian=0
-signature file-magic-auto52 {
-	file-mime "text/x-shellscript", 180
-	file-magic /(\x23\x21 ?\x2fusr\x2fbin\x2fzsh)/
-}
-
 # >0  string,=MAS_UTrack_V00 (len=14), [""], swap_endian=0
 # >>14  string,>/0 (len=2), ["ultratracker V1.%.1s module sound data"], swap_endian=0
 signature file-magic-auto53 {
@@ -457,12 +367,6 @@ signature file-magic-auto70 {
 	file-magic /(\x3cmap ?version)/
 }
 
-# >0  string/wt,=#! /bin/tcsh (len=12), ["Tenex C shell script text executable"], swap_endian=0
-signature file-magic-auto71 {
-	file-mime "text/x-shellscript", 150
-	file-magic /(\x23\x21 ?\x2fbin\x2ftcsh)/
-}
-
 # >0  string/wt,=#! /bin/nawk (len=12), ["new awk script text executable"], swap_endian=0
 signature file-magic-auto72 {
 	file-mime "text/x-nawk", 150
@@ -475,12 +379,6 @@ signature file-magic-auto73 {
 	file-magic /(\x23\x21 ?\x2fbin\x2fgawk)/
 }
 
-# >0  string/wt,=#! /bin/bash (len=12), ["Bourne-Again shell script text executable"], swap_endian=0
-signature file-magic-auto74 {
-	file-mime "text/x-shellscript", 150
-	file-magic /(\x23\x21 ?\x2fbin\x2fbash)/
-}
-
 # >0  string/wt,=#! /bin/awk (len=11), ["awk script text executable"], swap_endian=0
 signature file-magic-auto75 {
 	file-mime "text/x-awk", 140
@@ -505,24 +403,6 @@ signature file-magic-auto78 {
 	file-magic /(d8\x3aannounce)/
 }
 
-# >0  string/wt,=#! /bin/csh (len=11), ["C shell script text executable"], swap_endian=0
-signature file-magic-auto79 {
-	file-mime "text/x-shellscript", 140
-	file-magic /(\x23\x21 ?\x2fbin\x2fcsh)/
-}
-
-# >0  string/wt,=#! /bin/ksh (len=11), ["Korn shell script text executable"], swap_endian=0
-signature file-magic-auto80 {
-	file-mime "text/x-shellscript", 140
-	file-magic /(\x23\x21 ?\x2fbin\x2fksh)/
-}
-
-# >0  string/wt,=#! /bin/zsh (len=11), ["Paul Falstad's zsh script text executable"], swap_endian=0
-signature file-magic-auto81 {
-	file-mime "text/x-shellscript", 140
-	file-magic /(\x23\x21 ?\x2fbin\x2fzsh)/
-}
-
 # >0  string/c,=BEGIN:VCARD (len=11), ["vCard visiting card"], swap_endian=0
 signature file-magic-auto82 {
 	file-mime "text/x-vcard", 140
@@ -545,12 +425,6 @@ signature file-magic-auto84 {
 	file-magic /(Forward to)/
 }
 
-# >0  string/wt,=#! /bin/sh (len=10), ["POSIX shell script text executable"], swap_endian=0
-signature file-magic-auto85 {
-	file-mime "text/x-shellscript", 130
-	file-magic /(\x23\x21 ?\x2fbin\x2fsh)/
-}
-
 # >0  string,=II*\000\020\000\000\000CR (len=10), ["Canon CR2 raw image data"], swap_endian=0
 signature file-magic-auto86 {
 	file-mime "image/x-canon-cr2", 130
@@ -585,12 +459,6 @@ signature file-magic-auto90 {
 	file-magic /(\x3cBookFile)/
 }
 
-# >2112  string,=MSWordDoc (len=9), ["Microsoft Word document data"], swap_endian=0
-signature file-magic-auto91 {
-	file-mime "application/msword", 120
-	file-magic /(.{2112})(MSWordDoc)/
-}
-
 # >0  string/t,=N#! rnews (len=9), ["mailed, batched news text"], swap_endian=0
 signature file-magic-auto92 {
 	file-mime "message/rfc822", 120
@@ -656,12 +524,6 @@ signature file-magic-auto100 {
 	file-magic /(MSCF\x00\x00\x00\x00)/
 }
 
-# >0  string/b,=\320\317\021\340\241\261\032\341 (len=8), ["Microsoft Office Document"], swap_endian=0
-signature file-magic-auto101 {
-	file-mime "application/msword", 110
-	file-magic /(\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1)/
-}
-
 # >21  string/c,=!SCREAM! (len=8), ["Screamtracker 2 module sound data"], swap_endian=0
 signature file-magic-auto102 {
 	file-mime "audio/x-mod", 110
@@ -754,10 +616,10 @@ signature file-magic-auto116 {
 }
 
 # >257  string,=ustar  \000 (len=8), ["GNU tar archive"], swap_endian=0
-signature file-magic-auto117 {
+#signature file-magic-auto117 {
-	file-mime "application/x-tar", 110
+#	file-mime "application/x-tar", 110
-	file-magic /(.{257})(ustar  \x00)/
+#	file-magic /(.{257})(ustar  \x00)/
-}
+#}
 
 # >0  string,=<MIFFile (len=8), ["FrameMaker MIF (ASCII) file"], swap_endian=0
 signature file-magic-auto118 {
@@ -771,12 +633,6 @@ signature file-magic-auto119 {
 	file-magic /(PK\x07\x08PK\x03\x04)/
 }
 
-# >0  string/b,=\t\004\006\000\000\000\020\000 (len=8), ["Microsoft Excel Worksheet"], swap_endian=0
-signature file-magic-auto120 {
-	file-mime "application/vnd.ms-excel", 110
-	file-magic /(\x09\x04\x06\x00\x00\x00\x10\x00)/
-}
-
 # >0  string/b,=WordPro\000 (len=8), ["Lotus WordPro"], swap_endian=0
 signature file-magic-auto121 {
 	file-mime "application/vnd.lotus-wordpro", 110
@@ -844,10 +700,10 @@ signature file-magic-auto130 {
 }
 
 # >257  string,=ustar\000 (len=6), ["POSIX tar archive"], swap_endian=0
-signature file-magic-auto131 {
+#signature file-magic-auto131 {
-	file-mime "application/x-tar", 90
+#	file-mime "application/x-tar", 90
-	file-magic /(.{257})(ustar\x00)/
+#	file-magic /(.{257})(ustar\x00)/
-}
+#}
 
 # >0  string,=AC1.40 (len=6), ["DWG AutoDesk AutoCAD Release 1.40"], swap_endian=0
 signature file-magic-auto132 {
@@ -994,12 +850,6 @@ signature file-magic-auto155 {
 	file-magic /(\x23 xmcd)/
 }
 
-# >0  string/b,=\333\245-\000\000\000 (len=6), ["Microsoft Office Document"], swap_endian=0
-signature file-magic-auto156 {
-	file-mime "application/msword", 90
-	file-magic /(\xdb\xa5\x2d\x00\x00\x00)/
-}
-
 # >2  string,=MMXPR3 (len=6), ["Motorola Quark Express Document (English)"], swap_endian=0
 signature file-magic-auto157 {
 	file-mime "application/x-quark-xpress-3", 90
@@ -1046,36 +896,6 @@ signature file-magic-auto162 {
 	file-magic /(\x3c\x3fxml)(.{15})(.*)( xmlns\x3d)(['"]http:\x2f\x2fwww.opengis.net\x2fkml)/
 }
 
-# >0  string,=PK\003\004 (len=4), [""], swap_endian=0
-# >>30  regex,=[Content_Types].xml|_rels/.rels (len=31), [""], swap_endian=0
-# >>>18 (lelong,+49), search/2000,=PK\003\004 (len=4), [""], swap_endian=0
-# >>>>&26  search/1000,=PK\003\004 (len=4), [""], swap_endian=0
-# >>>>>&26  string,=word/ (len=5), ["Microsoft Word 2007+"], swap_endian=0
-signature file-magic-auto163 {
-	file-mime "application/vnd.openxmlformats-officedocument.wordprocessingml.document", 80
-	file-magic /(PK\x03\x04)(.{26})(\[Content_Types\].xml|_rels\x2f.rels)(.*)(PK\x03\x04)(.{26})(.*)(PK\x03\x04)(.{26})(word\x2f)/
-}
-
-# >0  string,=PK\003\004 (len=4), [""], swap_endian=0
-# >>30  regex,=[Content_Types].xml|_rels/.rels (len=31), [""], swap_endian=0
-# >>>18 (lelong,+49), search/2000,=PK\003\004 (len=4), [""], swap_endian=0
-# >>>>&26  search/1000,=PK\003\004 (len=4), [""], swap_endian=0
-# >>>>>&26  string,=ppt/ (len=4), ["Microsoft PowerPoint 2007+"], swap_endian=0
-signature file-magic-auto164 {
-	file-mime "application/vnd.openxmlformats-officedocument.presentationml.presentation", 70
-	file-magic /(PK\x03\x04)(.{26})(\[Content_Types\].xml|_rels\x2f.rels)(.*)(PK\x03\x04)(.{26})(.*)(PK\x03\x04)(.{26})(ppt\x2f)/
-}
-
-# >0  string,=PK\003\004 (len=4), [""], swap_endian=0
-# >>30  regex,=[Content_Types].xml|_rels/.rels (len=31), [""], swap_endian=0
-# >>>18 (lelong,+49), search/2000,=PK\003\004 (len=4), [""], swap_endian=0
-# >>>>&26  search/1000,=PK\003\004 (len=4), [""], swap_endian=0
-# >>>>>&26  string,=xl/ (len=3), ["Microsoft Excel 2007+"], swap_endian=0
-signature file-magic-auto165 {
-	file-mime "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", 60
-	file-magic /(PK\x03\x04)(.{26})(\[Content_Types\].xml|_rels\x2f.rels)(.*)(PK\x03\x04)(.{26})(.*)(PK\x03\x04)(.{26})(xl\x2f)/
-}
-
 # >60  string,=RINEX (len=5), [""], swap_endian=0
 # >>80  search/256,=XXRINEXB (len=8), ["RINEX Data, GEO SBAS Broadcast"], swap_endian=0
 # >>>5  string,x, [", version %6.6s"], swap_endian=0
@@ -1229,30 +1049,12 @@ signature file-magic-auto187 {
 	file-magic /(\x00\x01\x00\x00\x00)/
 }
 
-# >0  string/b,=PO^Q` (len=5), ["Microsoft Word 6.0 Document"], swap_endian=0
-signature file-magic-auto188 {
-	file-mime "application/msword", 80
-	file-magic /(PO\x5eQ\x60)/
-}
-
 # >0  string,=%PDF- (len=5), ["PDF document"], swap_endian=0
 signature file-magic-auto189 {
 	file-mime "application/pdf", 80
 	file-magic /(\x25PDF\x2d)/
 }
 
-# >2114  string,=Biff5 (len=5), ["Microsoft Excel 5.0 Worksheet"], swap_endian=0
-signature file-magic-auto190 {
-	file-mime "application/vnd.ms-excel", 80
-	file-magic /(.{2114})(Biff5)/
-}
-
-# >2121  string,=Biff5 (len=5), ["Microsoft Excel 5.0 Worksheet"], swap_endian=0
-signature file-magic-auto191 {
-	file-mime "application/vnd.ms-excel", 80
-	file-magic /(.{2121})(Biff5)/
-}
-
 # >0  string/t,=Path: (len=5), ["news text"], swap_endian=0
 signature file-magic-auto192 {
 	file-mime "message/news", 80
@@ -1383,12 +1185,6 @@ signature file-magic-auto211 {
 	file-magic /(\x00\x00\x00\x01)([\x07\x27\x47\x67\x87\xa7\xc7\xe7])/
 }
 
-# >0  belong&,=-889275714 (0xcafebabe), [""], swap_endian=0
-signature file-magic-auto212 {
-	file-mime "application/x-java-applet", 71
-	file-magic /(\xca\xfe\xba\xbe)/
-}
-
 # >0  belong&ffffffffffffff00,=256 (0x00000100), [""], swap_endian=0
 # >>3  byte&,=0xba, ["MPEG sequence"], swap_endian=0
 signature file-magic-auto213 {
@@ -1706,46 +1502,6 @@ signature file-magic-auto245 {
 	file-magic /(PK\x03\x04)(.{22})(\x08\x00\x00\x00mimetypeapplication\x2f)(epub\x2bzip)/
 }
 
-# Seems redundant with other zip signature below.
-# >0  string,=PK\003\004 (len=4), [""], swap_endian=0
-# >>26  string,=\b\000\000\000mimetypeapplication/ (len=24), [""], swap_endian=0
-# >>>50  string,!epub+zip (len=8), [""], swap_endian=0
-# >>>>50  string,!vnd.oasis.opendocument. (len=23), [""], swap_endian=0
-# >>>>>50  string,!vnd.sun.xml. (len=12), [""], swap_endian=0
-# >>>>>>50  string,!vnd.kde. (len=8), [""], swap_endian=0
-# >>>>>>>38  regex,=[!-OQ-~]+ (len=9), ["Zip data (MIME type "%s"?)"], swap_endian=0
-#signature file-magic-auto246 {
-#	file-mime "application/zip", 39
-#	file-magic /(PK\x03\x04)(.{22})(\x08\x00\x00\x00mimetypeapplication\x2f)/
-#}
-
-# >0  string,=PK\003\004 (len=4), [""], swap_endian=0
-# >>26  string,=\b\000\000\000mimetype (len=12), [""], swap_endian=0
-# >>>38  string,!application/ (len=12), [""], swap_endian=0
-# >>>>38  regex,=[!-OQ-~]+ (len=9), ["Zip data (MIME type "%s"?)"], swap_endian=0
-signature file-magic-auto247 {
-	file-mime "application/zip", 39
-	file-magic /(PK\x03\x04)(.{22})(\x08\x00\x00\x00mimetype)/
-}
-
-# The indirect offset makes this difficult to convert.
-# The (.*) may be too generous.
-# >0  string,=PK\003\004 (len=4), [""], swap_endian=0
-# >>26 (leshort,+30), leshort&,=-13570 (0xcafe), ["Java archive data (JAR)"], swap_endian=0
-signature file-magic-auto248 {
-	file-mime "application/java-archive", 50
-	file-magic /(PK\x03\x04)(.*)(\xfe\xca)/
-}
-
-# The indeirect offset and string inequality make this difficult to convert.
-# >0  string,=PK\003\004 (len=4), [""], swap_endian=0
-# >>26 (leshort,+30), leshort&,!-13570 (0xcafe), [""], swap_endian=0
-# >>>26  string,!\b\000\000\000mimetype (len=12), ["Zip archive data"], swap_endian=0
-signature file-magic-auto249 {
-	file-mime "application/zip", 10
-	file-magic /(PK\x03\x04)(.{2})/
-}
-
 # >0  belong&,=442 (0x000001ba), [""], swap_endian=0
 # >>4  byte&,&0x40, [""], swap_endian=0
 signature file-magic-auto250 {
@@ -2065,18 +1821,6 @@ signature file-magic-auto299 {
 	file-magic /(PDN3)/
 }
 
-# >0  ulelong&,=2712847316 (0xa1b2c3d4), ["tcpdump capture file (little-endian)"], swap_endian=0
-signature file-magic-auto300 {
-	file-mime "application/vnd.tcpdump.pcap", 70
-	file-magic /(\xd4\xc3\xb2\xa1)/
-}
-
-# >0  ubelong&,=2712847316 (0xa1b2c3d4), ["tcpdump capture file (big-endian)"], swap_endian=0
-signature file-magic-auto301 {
-	file-mime "application/vnd.tcpdump.pcap", 70
-	file-magic /(\xa1\xb2\xc3\xd4)/
-}
-
 # >0  belong&,=-17957139 (0xfeedfeed), ["Java KeyStore"], swap_endian=0
 signature file-magic-auto302 {
 	file-mime "application/x-java-keystore", 70
@@ -2297,12 +2041,6 @@ signature file-magic-auto335 {
 	file-magic /(SIT\x21)/
 }
 
-# >0  lelong&,=574529400 (0x223e9f78), ["Transport Neutral Encapsulation Format"], swap_endian=0
-signature file-magic-auto336 {
-	file-mime "application/vnd.ms-tnef", 70
-	file-magic /(\x78\x9f\x3e\x22)/
-}
-
 # >0  string,=<ar> (len=4), ["System V Release 1 ar archive"], swap_endian=0
 signature file-magic-auto337 {
 	file-mime "application/x-archive", 70
@@ -2433,48 +2171,6 @@ signature file-magic-auto357 {
 	file-magic /(RIFF)(.{4})(AVI )/
 }
 
-# >0  belong&,=834535424 (0x31be0000), ["Microsoft Word Document"], swap_endian=0
-signature file-magic-auto358 {
-	file-mime "application/msword", 70
-	file-magic /(\x31\xbe\x00\x00)/
-}
-
-# >0  string/b,=\3767\000# (len=4), ["Microsoft Office Document"], swap_endian=0
-signature file-magic-auto359 {
-	file-mime "application/msword", 70
-	file-magic /(\xfe7\x00\x23)/
-}
-
-# >0  string/b,=\333\245-\000 (len=4), ["Microsoft WinWord 2.0 Document"], swap_endian=0
-signature file-magic-auto360 {
-	file-mime "application/msword", 70
-	file-magic /(\xdb\xa5\x2d\x00)/
-}
-
-# >0  string/b,=\333\245-\000 (len=4), ["Microsoft WinWord 2.0 Document"], swap_endian=0
-signature file-magic-auto361 {
-	file-mime "application/msword", 70
-	file-magic /(\xdb\xa5\x2d\x00)/
-}
-
-# >0  belong&,=6656 (0x00001a00), ["Lotus 1-2-3"], swap_endian=0
-signature file-magic-auto362 {
-	file-mime "application/x-123", 70
-	file-magic /(\x00\x00\x1a\x00)/
-}
-
-# >0  belong&,=512 (0x00000200), ["Lotus 1-2-3"], swap_endian=0
-signature file-magic-auto363 {
-	file-mime "application/x-123", 70
-	file-magic /(\x00\x00\x02\x00)/
-}
-
-# >0  string/b,=\000\000\001\000 (len=4), ["MS Windows icon resource"], swap_endian=0
-signature file-magic-auto364 {
-	file-mime "image/x-icon", 70
-	file-magic /(\x00\x00\x01\x00)/
-}
-
 # >0  lelong&,=268435536 (0x10000050), ["Psion Series 5"], swap_endian=0
 # >>4  lelong&,=268435565 (0x1000006d), ["database"], swap_endian=0
 # >>>8  lelong&,=268435588 (0x10000084), ["Agenda file"], swap_endian=0
@@ -2737,12 +2433,6 @@ signature file-magic-auto403 {
 	file-magic /(SBI)/
 }
 
-# >0  string/b,=\224\246. (len=3), ["Microsoft Word Document"], swap_endian=0
-signature file-magic-auto404 {
-	file-mime "application/msword", 60
-	file-magic /(\x94\xa6\x2e)/
-}
-
 # >0  string,=\004%! (len=3), ["PostScript document text"], swap_endian=0
 signature file-magic-auto405 {
 	file-mime "application/postscript", 60
@@ -2763,30 +2453,11 @@ signature file-magic-auto407 {
 	file-magic /(.*)([ \x09]*(class|module)[ \x09][A-Z])((modul|includ)e [A-Z]|def [a-z])(^[ \x09]*end([ \x09]*[;#].*)?$)/
 }
 
-# >512  string/b,=\354\245\301 (len=3), ["Microsoft Word Document"], swap_endian=0
-signature file-magic-auto408 {
-	file-mime "application/msword", 60
-	file-magic /(.{512})(\xec\xa5\xc1)/
-}
-
-# >0  string,=FWS (len=3), ["Macromedia Flash data,"], swap_endian=0
-# >>3  byte&,x, ["version %d"], swap_endian=0
-signature file-magic-auto409 {
-	file-mime "application/x-shockwave-flash", 1
-	file-magic /(FWS)(.{1})/
-}
-
-# >0  string,=CWS (len=3), ["Macromedia Flash data (compressed),"], swap_endian=0
-signature file-magic-auto410 {
-	file-mime "application/x-shockwave-flash", 60
-	file-magic /(CWS)/
-}
-
 # >0  regex/20,=^\.[A-Za-z0-9][A-Za-z0-9][ \t] (len=29), ["troff or preprocessor input text"], swap_endian=0
-signature file-magic-auto411 {
+#signature file-magic-auto411 {
-	file-mime "text/troff", 59
+#	file-mime "text/troff", 59
-	file-magic /(^\.[A-Za-z0-9][A-Za-z0-9][ \x09])/
+#	file-magic /(^\.[A-Za-z0-9][A-Za-z0-9][ \x09])/
-}
+#}
 
 # >0  search/4096,=\documentclass (len=14), ["LaTeX 2e document text"], swap_endian=0
 signature file-magic-auto412 {
@@ -2819,10 +2490,10 @@ signature file-magic-auto416 {
 }
 
 # >0  regex/20,=^\.[A-Za-z0-9][A-Za-z0-9]$ (len=26), ["troff or preprocessor input text"], swap_endian=0
-signature file-magic-auto417 {
+#signature file-magic-auto417 {
-	file-mime "text/troff", 56
+#	file-mime "text/troff", 56
-	file-magic /(^\.[A-Za-z0-9][A-Za-z0-9]$)/
+#	file-magic /(^\.[A-Za-z0-9][A-Za-z0-9]$)/
-}
+#}
 
 # >0  search/w/1,=#! /usr/bin/php (len=15), ["PHP script text executable"], swap_endian=0
 signature file-magic-auto418 {
@@ -2842,30 +2513,12 @@ signature file-magic-auto420 {
 	file-magic /(.*)(eval \x22exec \x2fusr\x2fbin\x2fperl)/
 }
 
-# >0  search/w/1,=#! /usr/local/bin/python (len=24), ["Python script text executable"], swap_endian=0
-signature file-magic-auto421 {
-	file-mime "text/x-python", 54
-	file-magic /(.*)(\x23\x21 ?\x2fusr\x2flocal\x2fbin\x2fpython)/
-}
-
 # >0  search/1,=Common subdirectories:  (len=23), ["diff output text"], swap_endian=0
 signature file-magic-auto422 {
 	file-mime "text/x-diff", 53
 	file-magic /(.*)(Common subdirectories\x3a )/
 }
 
-# >0  search/1,=#! /usr/bin/env python (len=22), ["Python script text executable"], swap_endian=0
-signature file-magic-auto423 {
-	file-mime "text/x-python", 52
-	file-magic /(.*)(\x23\x21 \x2fusr\x2fbin\x2fenv python)/
-}
-
-# >0  search/w/1,=#! /usr/local/bin/ruby (len=22), ["Ruby script text executable"], swap_endian=0
-signature file-magic-auto424 {
-	file-mime "text/x-ruby", 52
-	file-magic /(.*)(\x23\x21 ?\x2fusr\x2flocal\x2fbin\x2fruby)/
-}
-
 # >0  search/w/1,=#! /usr/local/bin/wish (len=22), ["Tcl/Tk script text executable"], swap_endian=0
 signature file-magic-auto425 {
 	file-mime "text/x-tcl", 52
@@ -2884,12 +2537,6 @@ signature file-magic-auto427 {
 	file-magic /(\xff\xd8)/
 }
 
-# >0  search/1,=#!/usr/bin/env python (len=21), ["Python script text executable"], swap_endian=0
-signature file-magic-auto428 {
-	file-mime "text/x-python", 51
-	file-magic /(.*)(\x23\x21\x2fusr\x2fbin\x2fenv python)/
-}
-
 # >0  search/1,=#!/usr/bin/env nodejs (len=21), ["Node.js script text executable"], swap_endian=0
 signature file-magic-auto429 {
 	file-mime "application/javascript", 51
@@ -3202,12 +2849,6 @@ signature file-magic-auto474 {
 	file-magic /(\x25\x21)/
 }
 
-# >0  search/1,=#! /usr/bin/env ruby (len=20), ["Ruby script text executable"], swap_endian=0
-signature file-magic-auto475 {
-	file-mime "text/x-ruby", 50
-	file-magic /(.*)(\x23\x21 \x2fusr\x2fbin\x2fenv ruby)/
-}
-
 # >0  regex/1,=(^[0-9]{5})[acdn][w] (len=20), ["MARC21 Classification"], swap_endian=0
 signature file-magic-auto476 {
 	file-mime "application/marc", 50
@@ -3241,10 +2882,10 @@ signature file-magic-auto480 {
 }
 
 # >0  string,=\n( (len=2), ["Emacs v18 byte-compiled Lisp data"], swap_endian=0
-signature file-magic-auto481 {
+#signature file-magic-auto481 {
-	file-mime "application/x-elc", 50
+#	file-mime "application/x-elc", 50
-	file-magic /(\x0a\x28)/
+#	file-magic /(\x0a\x28)/
-}
+#}
 
 # >0  string,=\021\t (len=2), ["Award BIOS Logo, 136 x 126"], swap_endian=0
 signature file-magic-auto482 {
@@ -3318,17 +2959,17 @@ signature file-magic-auto493 {
 	file-magic /(\xf7\x02)/
 }
 
-# >2  string,=\000\021 (len=2), ["TeX font metric data"], swap_endian=0
+## >2  string,=\000\021 (len=2), ["TeX font metric data"], swap_endian=0
-signature file-magic-auto494 {
+#signature file-magic-auto494 {
-	file-mime "application/x-tex-tfm", 50
+#	file-mime "application/x-tex-tfm", 50
-	file-magic /(.{2})(\x00\x11)/
+#	file-magic /(.{2})(\x00\x11)/
-}
+#}
-
+#
-# >2  string,=\000\022 (len=2), ["TeX font metric data"], swap_endian=0
+## >2  string,=\000\022 (len=2), ["TeX font metric data"], swap_endian=0
-signature file-magic-auto495 {
+#signature file-magic-auto495 {
-	file-mime "application/x-tex-tfm", 50
+#	file-mime "application/x-tex-tfm", 50
-	file-magic /(.{2})(\x00\x12)/
+#	file-magic /(.{2})(\x00\x12)/
-}
+#}
 
 # >0  beshort&,=-31486 (0x8502), ["GPG encrypted data"], swap_endian=0
 signature file-magic-auto496 {
@@ -3483,12 +3124,6 @@ signature file-magic-auto514 {
 	file-magic /(.*)(\x23\x21 \x2fusr\x2fbin\x2fenv lua)/
 }
 
-# >0  search/1,=#!/usr/bin/env ruby (len=19), ["Ruby script text executable"], swap_endian=0
-signature file-magic-auto515 {
-	file-mime "text/x-ruby", 49
-	file-magic /(.*)(\x23\x21\x2fusr\x2fbin\x2fenv ruby)/
-}
-
 # >0  search/1,=#! /usr/bin/env tcl (len=19), ["Tcl script text executable"], swap_endian=0
 signature file-magic-auto516 {
 	file-mime "text/x-tcl", 49
@@ -3506,12 +3141,6 @@ signature file-magic-auto519 {
 	file-magic /(.*)(\x23\x21\x2fusr\x2fbin\x2fenv lua)/
 }
 
-# >0  search/w/1,=#! /usr/bin/python (len=18), ["Python script text executable"], swap_endian=0
-signature file-magic-auto520 {
-	file-mime "text/x-python", 48
-	file-magic /(.*)(\x23\x21 ?\x2fusr\x2fbin\x2fpython)/
-}
-
 # >0  search/w/1,=#!/usr/bin/nodejs (len=17), ["Node.js script text executable"], swap_endian=0
 signature file-magic-auto521 {
 	file-mime "application/javascript", 47
@@ -3519,10 +3148,10 @@ signature file-magic-auto521 {
 }
 
 # >0  regex,=^class[ \t\n]+ (len=12), ["C++ source text"], swap_endian=0
-signature file-magic-auto522 {
+#signature file-magic-auto522 {
-	file-mime "text/x-c++", 47
+#	file-mime "text/x-c++", 47
-	file-magic /(.*)(class[ \x09\x0a]+[[:alnum:]_]+)(.*)(\x7b)(.*)(public:)/
+#	file-magic /(.*)(class[ \x09\x0a]+[[:alnum:]_]+)(.*)(\x7b)(.*)(public:)/
-}
+#}
 
 # >0  search/1,=This is Info file (len=17), ["GNU Info text"], swap_endian=0
 signature file-magic-auto528 {
@@ -3671,12 +3300,6 @@ signature file-magic-auto545 {
 	file-magic /(.*)(\x23\x21 ?\x2fusr\x2fbin\x2fwish)/
 }
 
-# >0  search/w/1,=#! /usr/bin/ruby (len=16), ["Ruby script text executable"], swap_endian=0
-signature file-magic-auto546 {
-	file-mime "text/x-ruby", 46
-	file-magic /(.*)(\x23\x21 ?\x2fusr\x2fbin\x2fruby)/
-}
-
 # >0  search/w/1,=#! /usr/bin/lua (len=15), ["Lua script text executable"], swap_endian=0
 signature file-magic-auto547 {
 	file-mime "text/x-lua", 45
@@ -3740,10 +3363,10 @@ signature file-magic-auto556 {
 }
 
 # >0  regex,=^extern[ \t\n]+ (len=13), ["C source text"], swap_endian=0
-signature file-magic-auto557 {
+#signature file-magic-auto557 {
-	file-mime "text/x-c", 43
+#	file-mime "text/x-c", 43
-	file-magic /(.*)(extern[ \x09\x0a]+)/
+#	file-magic /(.*)(extern[ \x09\x0a]+)/
-}
+#}
 
 # >0  search/4096,=% -*-latex-*- (len=13), ["LaTeX document text"], swap_endian=0
 signature file-magic-auto558 {
@@ -3759,10 +3382,10 @@ signature file-magic-auto558 {
 #}
 
 # >0  regex,=^struct[ \t\n]+ (len=13), ["C source text"], swap_endian=0
-signature file-magic-auto560 {
+#signature file-magic-auto560 {
-	file-mime "text/x-c", 43
+#	file-mime "text/x-c", 43
-	file-magic /(.*)(struct[ \x09\x0a]+)/
+#	file-magic /(.*)(struct[ \x09\x0a]+)/
-}
+#}
 
 # >0  search/w/1,=#!/bin/nodejs (len=13), ["Node.js script text executable"], swap_endian=0
 signature file-magic-auto561 {
@@ -3815,10 +3438,10 @@ signature file-magic-auto567 {
 }
 
 # >0  regex,=^char[ \t\n]+ (len=11), ["C source text"], swap_endian=0
-signature file-magic-auto568 {
+#signature file-magic-auto568 {
-	file-mime "text/x-c", 41
+#	file-mime "text/x-c", 41
-	file-magic /(.*)(char[ \x09\x0a]+)/
+#	file-magic /(.*)(char[ \x09\x0a]+)/
-}
+#}
 
 # >0  search/1,=#! (len=2), [""], swap_endian=0
 # >>0  regex,=^#!.*/bin/perl$ (len=15), ["Perl script text executable"], swap_endian=0
@@ -3900,23 +3523,11 @@ signature file-magic-auto578 {
 	file-magic /(^dnl )/
 }
 
-# >0  regex,=^all: (len=5), ["makefile script text"], swap_endian=0
-signature file-magic-auto579 {
-	file-mime "text/x-makefile", 40
-	file-magic /(^all:)/
-}
-
-# >0  regex,=^.PRECIOUS (len=10), ["makefile script text"], swap_endian=0
-signature file-magic-auto580 {
-	file-mime "text/x-makefile", 40
-	file-magic /(^.PRECIOUS)/
-}
-
 # >0  search/8192,=main( (len=5), ["C source text"], swap_endian=0
-signature file-magic-auto581 {
+#signature file-magic-auto581 {
-	file-mime "text/x-c", 40
+#	file-mime "text/x-c", 40
-	file-magic /(.*)(main\x28)/
+#	file-magic /(.*)(main\x28)/
-}
+#}
 
 # Not specific enough.
 # >0  search/1,=\" (len=2), ["troff or preprocessor input text"], swap_endian=0
@@ -3945,22 +3556,22 @@ signature file-magic-auto584 {
 #}
 
 # >0  regex,=^#include (len=9), ["C source text"], swap_endian=0
-signature file-magic-auto586 {
+#signature file-magic-auto586 {
-	file-mime "text/x-c", 39
+#	file-mime "text/x-c", 39
-	file-magic /(.*)(#include)/
+#	file-magic /(.*)(#include)/
-}
+#}
 
 # >0  search/1,=.\" (len=3), ["troff or preprocessor input text"], swap_endian=0
-signature file-magic-auto587 {
+#signature file-magic-auto587 {
-	file-mime "text/troff", 39
+#	file-mime "text/troff", 39
-	file-magic /(.*)(\x2e\x5c\x22)/
+#	file-magic /(.*)(\x2e\x5c\x22)/
-}
+#}
 
 # >0  search/1,='\" (len=3), ["troff or preprocessor input text"], swap_endian=0
-signature file-magic-auto588 {
+#signature file-magic-auto588 {
-	file-mime "text/troff", 39
+#	file-mime "text/troff", 39
-	file-magic /(.*)(\x27\x5c\x22)/
+#	file-magic /(.*)(\x27\x5c\x22)/
-}
+#}
 
 # >0  search/1,=<TeXmacs| (len=9), ["TeXmacs document text"], swap_endian=0
 signature file-magic-auto589 {
@@ -3987,10 +3598,10 @@ signature file-magic-auto592 {
 }
 
 # >0  search/1,=''' (len=3), ["troff or preprocessor input text"], swap_endian=0
-signature file-magic-auto593 {
+#signature file-magic-auto593 {
-	file-mime "text/troff", 39
+#	file-mime "text/troff", 39
-	file-magic /(.*)(\x27\x27\x27)/
+#	file-magic /(.*)(\x27\x27\x27)/
-}
+#}
 
 # >0  search/4096,=try: (len=4), [""], swap_endian=0
 # >>&0  regex,=^\s*except.*: (len=13), ["Python script text executable"], swap_endian=0
@@ -4012,12 +3623,6 @@ signature file-magic-auto596 {
 	file-magic /(.*)(\x22LIBHDR\x22)/
 }
 
-# >0  regex,=^SUBDIRS (len=8), ["automake makefile script text"], swap_endian=0
-signature file-magic-auto597 {
-	file-mime "text/x-makefile", 38
-	file-magic /(.*)(SUBDIRS)/
-}
-
 # >0  search/4096,=(defvar  (len=8), ["Lisp/Scheme program text"], swap_endian=0
 signature file-magic-auto598 {
 	file-mime "text/x-lisp", 38
@@ -4044,19 +3649,6 @@ signature file-magic-auto600 {
 #	file-magic /(.*)(\x2a\x2a\x2a )/
 #}
 
-# >0  search/1,='.\" (len=4), ["troff or preprocessor input text"], swap_endian=0
-signature file-magic-auto602 {
-	file-mime "text/troff", 38
-	file-magic /(.*)(\x27\x2e\x5c\x22)/
-}
-
-# LDFLAGS appears in other contexts, e.g. shell script.
-# >0  regex,=^LDFLAGS (len=8), ["makefile script text"], swap_endian=0
-#signature file-magic-auto603 {
-#	file-mime "text/x-makefile", 38
-#	file-magic /(.*)(LDFLAGS)/
-#}
-
 # >0  search/8192,="libhdr" (len=8), ["BCPL source text"], swap_endian=0
 signature file-magic-auto604 {
 	file-mime "text/x-bcpl", 38
@@ -4070,12 +3662,6 @@ signature file-magic-auto604 {
 #	file-magic /(^record)/
 #}
 
-# >0  regex,=^CFLAGS (len=7), ["makefile script text"], swap_endian=0
-signature file-magic-auto606 {
-	file-mime "text/x-makefile", 37
-	file-magic /(.*)(CFLAGS)/
-}
-
 # >0  search/4096,=(defun  (len=7), ["Lisp/Scheme program text"], swap_endian=0
 signature file-magic-auto607 {
 	file-mime "text/x-lisp", 37

scripts/base/frameworks/files/magic/msoffice.sig

@@ -0,0 +1,28 @@
+
+# This signature is non-specific and terrible but after
+# searching for a long time there doesn't seem to be a 
+# better option.  
+signature file-msword {
+	file-magic /^\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1/
+	file-mime "application/msword", 50
+}
+
+signature file-ooxml {
+	file-magic /^PK\x03\x04\x14\x00\x06\x00/
+	file-mime "application/vnd.openxmlformats-officedocument", 50
+}
+
+signature file-docx {
+	file-magic /^PK\x03\x04.{26}(\[Content_Types\]\.xml|_rels\x2f\.rels|word\x2f).*PK\x03\x04.{26}word\x2f/
+	file-mime "application/vnd.openxmlformats-officedocument.wordprocessingml.document", 80
+}
+
+signature file-xlsx {
+	file-magic /^PK\x03\x04.{26}(\[Content_Types\]\.xml|_rels\x2f\.rels|xl\2f).*PK\x03\x04.{26}xl\x2f/
+	file-mime "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", 80
+}
+
+signature file-pptx {
+	file-magic /^PK\x03\x04.{26}(\[Content_Types\]\.xml|_rels\x2f\.rels|ppt\x2f).*PK\x03\x04.{26}ppt\x2f/
+	file-mime "application/vnd.openxmlformats-officedocument.presentationml.presentation", 80
+}

scripts/base/frameworks/files/main.bro

@@ -56,7 +56,7 @@ export {
 		## local file path which was read, or some other input source.
 		source: string &log &optional;
 
-		## A value to represent the depth of this file in relation 
+		## A value to represent the depth of this file in relation
 		## to its source.  In SMTP, it is the depth of the MIME
 		## attachment on the message.  In HTTP, it is the depth of the
 		## request within the TCP connection.
@@ -73,7 +73,7 @@ export {
 		mime_type: string &log &optional;
 
 		## A filename for the file if one is available from the source
-		## for the file.  These will frequently come from 
+		## for the file.  These will frequently come from
 		## "Content-Disposition" headers in network protocols.
 		filename: string &log &optional;
 
@@ -100,8 +100,9 @@ export {
 		## during the process of analysis e.g. due to dropped packets.
 		missing_bytes: count &log &default=0;
 
-		## The number of not all-in-sequence bytes in the file stream that
+		## The number of bytes in the file stream that were not delivered to
-		## were delivered to file analyzers due to reassembly buffer overflow.
+		## stream file analyzers.  This could be overlapping bytes or 
+		## bytes that couldn't be reassembled.
 		overflow_bytes: count &log &default=0;
 
 		## Whether the file analysis timed out at least once for the file.
@@ -124,6 +125,37 @@ export {
 	## generate two handles that would hash to the same file id.
 	const salt = "I recommend changing this." &redef;
 
+	## Decide if you want to automatically attached analyzers to 
+	## files based on the detected mime type of the file.
+	const analyze_by_mime_type_automatically = T &redef;
+
+	## The default setting for if the file reassembler is enabled for 
+	## each file.
+	const enable_reassembler = T &redef;
+
+	## The default per-file reassembly buffer size.
+	const reassembly_buffer_size = 1048576 &redef;
+
+	## Allows the file reassembler to be used if it's necessary because the
+	## file is transferred out of order.
+	##
+	## f: the file.
+	global enable_reassembly: function(f: fa_file);
+
+	## Disables the file reassembler on this file.  If the file is not 
+	## transferred out of order this will have no effect.
+	##
+	## f: the file.
+	global disable_reassembly: function(f: fa_file);
+
+	## Set the maximum size the reassembly buffer is allowed to grow
+	## for the given file.
+	##
+	## f: the file.
+	##
+	## max: Maximum allowed size of the reassembly buffer.
+	global set_reassembly_buffer_size: function(f: fa_file, max: count);
+
 	## Sets the *timeout_interval* field of :bro:see:`fa_file`, which is
 	## used to determine the length of inactivity that is allowed for a file
 	## before internal state related to it is cleaned up.  When used within
@@ -149,8 +181,8 @@ export {
 	## Returns: true if the analyzer will be added, or false if analysis
 	##          for the file isn't currently active or the *args*
 	##          were invalid for the analyzer type.
-	global add_analyzer: function(f: fa_file, 
+	global add_analyzer: function(f: fa_file,
-	                              tag: Files::Tag, 
+	                              tag: Files::Tag,
 	                              args: AnalyzerArgs &default=AnalyzerArgs()): bool;
 
 	## Removes an analyzer from the analysis of a given file.
@@ -196,7 +228,7 @@ export {
 		## A callback to generate a file handle on demand when
 		## one is needed by the core.
 		get_file_handle: function(c: connection, is_orig: bool): string;
-		
+
 		## A callback to "describe" a file.  In the case of an HTTP
 		## transfer the most obvious description would be the URL.
 		## It's like an extremely compressed version of the normal log.
@@ -207,7 +239,7 @@ export {
 	## Register callbacks for protocols that work with the Files framework.
 	## The callbacks must uniquely identify a file and each protocol can 
 	## only have a single callback registered for it.
-	## 
+	##
 	## tag: Tag for the protocol analyzer having a callback being registered.
 	##
 	## reg: A :bro:see:`Files::ProtoRegistration` record.
@@ -225,6 +257,42 @@ export {
 	## callback: Function to execute when the given file analyzer is being added.
 	global register_analyzer_add_callback: function(tag: Files::Tag, callback: function(f: fa_file, args: AnalyzerArgs));
 
+	## Registers a set of MIME types for an analyzer. If a future connection on one of
+	## these types is seen, the analyzer will be automatically assigned to parsing it.
+	## The function *adds* to all MIME types already registered, it doesn't replace
+	## them.
+	##
+	## tag: The tag of the analyzer.
+	##
+	## mts: The set of MIME types, each in the form "foo/bar" (case-insensitive).
+	##
+	## Returns: True if the MIME types were successfully registered.
+	global register_for_mime_types: function(tag: Analyzer::Tag, mts: set[string]) : bool;
+
+	## Registers a MIME type for an analyzer. If a future file with this type is seen,
+	## the analyzer will be automatically assigned to parsing it. The function *adds*
+	## to all MIME types already registered, it doesn't replace them.
+	##
+	## tag: The tag of the analyzer.
+	##
+	## mt: The MIME type in the form "foo/bar" (case-insensitive).
+	##
+	## Returns: True if the MIME type was successfully registered.
+	global register_for_mime_type: function(tag: Analyzer::Tag, mt: string) : bool;
+
+	## Returns a set of all MIME types currently registered for a specific analyzer.
+	##
+	## tag: The tag of the analyzer.
+	##
+	## Returns: The set of MIME types.
+	global registered_mime_types: function(tag: Analyzer::Tag) : set[string];
+
+	## Returns a table of all MIME-type-to-analyzer mappings currently registered.
+	##
+	## Returns: A table mapping each analyzer to the set of MIME types
+	##          registered for it.
+	global all_registered_mime_types: function() : table[Analyzer::Tag] of set[string];
+
 	## Event that can be handled to access the Info record as it is sent on
 	## to the logging framework.
 	global log_files: event(rec: Info);
@@ -237,6 +305,10 @@ redef record fa_file += {
 # Store the callbacks for protocol analyzers that have files.
 global registered_protocols: table[Analyzer::Tag] of ProtoRegistration = table();
 
+# Store the MIME type to analyzer mappings.
+global mime_types: table[Analyzer::Tag] of set[string];
+global mime_type_to_analyzers: table[string] of set[Analyzer::Tag];
+
 global analyzer_add_callbacks: table[Files::Tag] of function(f: fa_file, args: AnalyzerArgs) = table();
 
 event bro_init() &priority=5
@@ -259,14 +331,12 @@ function set_info(f: fa_file)
 		f$info$source = f$source;
 	f$info$duration = f$last_active - f$info$ts;
 	f$info$seen_bytes = f$seen_bytes;
-	if ( f?$total_bytes ) 
+	if ( f?$total_bytes )
 		f$info$total_bytes = f$total_bytes;
 	f$info$missing_bytes = f$missing_bytes;
 	f$info$overflow_bytes = f$overflow_bytes;
 	if ( f?$is_orig )
 		f$info$is_orig = f$is_orig;
-	if ( f?$mime_type ) 
-		f$info$mime_type = f$mime_type;
 	}
 
 function set_timeout_interval(f: fa_file, t: interval): bool
@@ -274,6 +344,21 @@ function set_timeout_interval(f: fa_file, t: interval): bool
 	return __set_timeout_interval(f$id, t);
 	}
 
+function enable_reassembly(f: fa_file)
+	{
+	__enable_reassembly(f$id);
+	}
+
+function disable_reassembly(f: fa_file)
+	{
+	__disable_reassembly(f$id);
+	}
+
+function set_reassembly_buffer_size(f: fa_file, max: count)
+	{
+	__set_reassembly_buffer(f$id, max);
+	}
+
 function add_analyzer(f: fa_file, tag: Files::Tag, args: AnalyzerArgs): bool
 	{
 	add f$info$analyzers[Files::analyzer_name(tag)];
@@ -309,39 +394,6 @@ function analyzer_name(tag: Files::Tag): string
 	return __analyzer_name(tag);
 	}
 
-event file_new(f: fa_file) &priority=10
-	{
-	set_info(f);
-	}
-
-event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=10
-	{
-	set_info(f);
-	add f$info$conn_uids[c$uid];
-	local cid = c$id;
-	add f$info$tx_hosts[f$is_orig ? cid$orig_h : cid$resp_h];
-	if( |Site::local_nets| > 0 )
-		f$info$local_orig=Site::is_local_addr(f$is_orig ? cid$orig_h : cid$resp_h);
-
-	add f$info$rx_hosts[f$is_orig ? cid$resp_h : cid$orig_h];
-	}
-
-event file_timeout(f: fa_file) &priority=10
-	{
-	set_info(f);
-	f$info$timedout = T;
-	}
-
-event file_state_remove(f: fa_file) &priority=10
-	{
-	set_info(f);
-	}
-
-event file_state_remove(f: fa_file) &priority=-10
-	{
-	Log::write(Files::LOG, f$info);
-	}
-
 function register_protocol(tag: Analyzer::Tag, reg: ProtoRegistration): bool
 	{
 	local result = (tag !in registered_protocols);
@@ -349,6 +401,46 @@ function register_protocol(tag: Analyzer::Tag, reg: ProtoRegistration): bool
 	return result;
 	}
 
+function register_for_mime_types(tag: Analyzer::Tag, mime_types: set[string]) : bool
+	{
+	local rc = T;
+
+	for ( mt in mime_types )
+		{
+		if ( ! register_for_mime_type(tag, mt) )
+			rc = F;
+		}
+
+	return rc;
+	}
+
+function register_for_mime_type(tag: Analyzer::Tag, mt: string) : bool
+	{
+	if ( tag !in mime_types )
+		{
+		mime_types[tag] = set();
+		}
+	add mime_types[tag][mt];
+
+	if ( mt !in mime_type_to_analyzers )
+		{
+		mime_type_to_analyzers[mt] = set();
+		}
+	add mime_type_to_analyzers[mt][tag];
+
+	return T;
+	}
+
+function registered_mime_types(tag: Analyzer::Tag) : set[string]
+	{
+	return tag in mime_types ? mime_types[tag] : set();
+	}
+
+function all_registered_mime_types(): table[Analyzer::Tag] of set[string]
+	{
+	return mime_types;
+	}
+
 function describe(f: fa_file): string
 	{
 	local tag = Analyzer::get_tag(f$source);
@@ -367,3 +459,61 @@ event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) &priorit
 	local handler = registered_protocols[tag];
 	set_file_handle(handler$get_file_handle(c, is_orig));
 	}
+
+event file_new(f: fa_file) &priority=10
+	{
+	set_info(f);
+
+	if ( enable_reassembler )
+		{
+		Files::enable_reassembly(f);
+		Files::set_reassembly_buffer_size(f, reassembly_buffer_size);
+		}
+	}
+
+event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=10
+	{
+	set_info(f);
+
+	add f$info$conn_uids[c$uid];
+	local cid = c$id;
+	add f$info$tx_hosts[f$is_orig ? cid$orig_h : cid$resp_h];
+	if( |Site::local_nets| > 0 )
+		f$info$local_orig=Site::is_local_addr(f$is_orig ? cid$orig_h : cid$resp_h);
+
+	add f$info$rx_hosts[f$is_orig ? cid$resp_h : cid$orig_h];
+	}
+
+event file_mime_type(f: fa_file, mime_type: string) &priority=10
+	{
+	set_info(f);
+
+	f$info$mime_type = mime_type;
+
+	if ( analyze_by_mime_type_automatically &&
+	     mime_type in mime_type_to_analyzers )
+		{
+		local analyzers = mime_type_to_analyzers[mime_type];
+		for ( a in analyzers )
+			{
+			add f$info$analyzers[Files::analyzer_name(a)];
+			Files::add_analyzer(f, a);
+			}
+		}
+	}
+
+event file_timeout(f: fa_file) &priority=10
+	{
+	set_info(f);
+	f$info$timedout = T;
+	}
+
+event file_state_remove(f: fa_file) &priority=10
+	{
+	set_info(f);
+	}
+
+event file_state_remove(f: fa_file) &priority=-10
+	{
+	Log::write(Files::LOG, f$info);
+	}

scripts/base/frameworks/input/main.bro

@@ -4,6 +4,17 @@
 module Input;
 
 export {
+	type Event: enum {
+		EVENT_NEW = 0,
+		EVENT_CHANGED = 1,
+		EVENT_REMOVED = 2,
+	};
+
+	type Mode: enum {
+		MANUAL = 0,
+		REREAD = 1,
+		STREAM = 2
+	};
 
 	## The default input reader used. Defaults to `READER_ASCII`.
 	const default_reader = READER_ASCII &redef;

scripts/base/frameworks/intel/main.bro

@@ -67,6 +67,7 @@ export {
 		IN_ANYWHERE,
 	};
 
+	## Information about a piece of "seen" data.
 	type Seen: record {
 		## The string if the data is about a string.
 		indicator:       string        &log &optional;
@@ -81,6 +82,9 @@ export {
 		## Where the data was discovered.
 		where:           Where         &log;
 		
+		## The name of the node where the match was discovered.
+		node:            string        &optional &log;
+
 		## If the data was discovered within a connection, the 
 		## connection record should go here to give context to the data.
 		conn:            connection    &optional;
@@ -121,7 +125,7 @@ export {
 		sources:  set[string]    &log &default=string_set();
 	};
 
-	## Intelligence data manipulation functions.
+	## Intelligence data manipulation function.
 	global insert: function(item: Item);
 
 	## Function to declare discovery of a piece of data in order to check
@@ -240,6 +244,11 @@ function Intel::seen(s: Seen)
 			s$indicator_type = Intel::ADDR;
 			}
 
+		if ( ! s?$node )
+			{
+			s$node = peer_description;
+			}
+
 		if ( have_full_data )
 			{
 			local items = get_items(s);
@@ -281,8 +290,8 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5
 		if ( ! info?$fuid )
 			info$fuid = s$f$id;
 
-		if ( ! info?$file_mime_type && s$f?$mime_type )
+		if ( ! info?$file_mime_type && s$f?$info && s$f$info?$mime_type )
-			info$file_mime_type = s$f$mime_type;
+			info$file_mime_type = s$f$info$mime_type;
 
 		if ( ! info?$file_desc )
 			info$file_desc = Files::describe(s$f);

scripts/base/frameworks/logging/__load__.bro

@@ -1,7 +1,5 @@
 @load ./main
 @load ./postprocessors
 @load ./writers/ascii
-@load ./writers/dataseries
 @load ./writers/sqlite
-@load ./writers/elasticsearch
 @load ./writers/none

scripts/base/frameworks/logging/main.bro

@@ -5,9 +5,15 @@
 
 module Log;
 
-# Log::ID and Log::Writer are defined in types.bif due to circular dependencies.
-
 export {
+	## Type that defines an ID unique to each log stream. Scripts creating new log
+	## streams need to redef this enum to add their own specific log ID. The log ID
+	## implicitly determines the default name of the generated log file.
+	type Log::ID: enum {
+		## Dummy place-holder.
+		UNKNOWN
+	};
+
 	## If true, local logging is by default enabled for all filters.
 	const enable_local_logging = T &redef;
 
@@ -27,13 +33,13 @@ export {
 	const set_separator = "," &redef;
 
 	## String to use for empty fields. This should be different from
-        ## *unset_field* to make the output unambiguous. 
+	## *unset_field* to make the output unambiguous.
 	## Can be overwritten by individual writers.
 	const empty_field = "(empty)" &redef;
 
 	## String to use for an unset &optional field.
 	## Can be overwritten by individual writers.
-	const unset_field = "-" &redef;	
+	const unset_field = "-" &redef;
 
 	## Type defining the content of a logging stream.
 	type Stream: record {

scripts/base/frameworks/logging/writers/ascii.bro

@@ -26,20 +26,20 @@ export {
 	## This option is also available as a per-filter ``$config`` option.
 	const use_json = F &redef;
 
-	## Format of timestamps when writing out JSON. By default, the JSON formatter will
+	## Format of timestamps when writing out JSON. By default, the JSON
-	## use double values for timestamps which represent the number of seconds from the
+	## formatter will use double values for timestamps which represent the
-	## UNIX epoch.
+	## number of seconds from the UNIX epoch.
 	const json_timestamps: JSON::TimestampFormat = JSON::TS_EPOCH &redef;
 
 	## If true, include lines with log meta information such as column names
 	## with types, the values of ASCII logging options that are in use, and
 	## the time when the file was opened and closed (the latter at the end).
-        ##
+	##
 	## If writing in JSON format, this is implicitly disabled.
 	const include_meta = T &redef;
 
 	## Prefix for lines with meta information.
-        ##
+	##
 	## This option is also available as a per-filter ``$config`` option.
 	const meta_prefix = "#" &redef;
 

scripts/base/frameworks/logging/writers/dataseries.bro

@@ -1,60 +0,0 @@
-##! Interface for the DataSeries log writer.
-
-module LogDataSeries;
-
-export {
-	## Compression to use with the DS output file.  Options are:
-	##
-	## 'none' -- No compression.
-	## 'lzf'  -- LZF compression (very quick, but leads to larger output files).
-	## 'lzo'  -- LZO compression (very fast decompression times).
-	## 'gz'   -- GZIP compression (slower than LZF, but also produces smaller output).
-	## 'bz2'  -- BZIP2 compression (slower than GZIP, but also produces smaller output).
-	const compression = "gz" &redef;
-
-	## The extent buffer size.
-	## Larger values here lead to better compression and more efficient writes,
-	## but also increase the lag between the time events are received and
-	## the time they are actually written to disk.
-	const extent_size = 65536 &redef;
-
-	## Should we dump the XML schema we use for this DS file to disk?
-	## If yes, the XML schema shares the name of the logfile, but has
-	## an XML ending.
-	const dump_schema = F &redef;
-
-	## How many threads should DataSeries spawn to perform compression?
-	## Note that this dictates the number of threads per log stream.  If
-	## you're using a lot of streams, you may want to keep this number
-	## relatively small.
-	##
-	## Default value is 1, which will spawn one thread / stream.
-	##
-	## Maximum is 128, minimum is 1.
-	const num_threads = 1 &redef;
-
-	## Should time be stored as an integer or a double?
-	## Storing time as a double leads to possible precision issues and
-	## can (significantly) increase the size of the resulting DS log.
-	## That said, timestamps stored in double form are consistent
-	## with the rest of Bro, including the standard ASCII log. Hence, we
-	## use them by default.
-	const use_integer_for_time = F &redef;
-}
-
-# Default function to postprocess a rotated DataSeries log file. It moves the
-# rotated file to a new name that includes a timestamp with the opening time,
-# and then runs the writer's default postprocessor command on it.
-function default_rotation_postprocessor_func(info: Log::RotationInfo) : bool
-	{
-	# Move file to name including both opening and closing time.
-	local dst = fmt("%s.%s.ds", info$path,
-			strftime(Log::default_rotation_date_format, info$open));
-
-	system(fmt("/bin/mv %s %s", info$fname, dst));
-
-	# Run default postprocessor.
-	return Log::run_rotation_postprocessor_cmd(info, dst);
-	}
-
-redef Log::default_rotation_postprocessors += { [Log::WRITER_DATASERIES] = default_rotation_postprocessor_func };

scripts/base/frameworks/logging/writers/elasticsearch.bro

@@ -1,48 +0,0 @@
-##! Log writer for sending logs to an ElasticSearch server.
-##!
-##! Note: This module is in testing and is not yet considered stable!
-##!
-##! There is one known memory issue.  If your elasticsearch server is
-##! running slowly and taking too long to return from bulk insert
-##! requests, the message queue to the writer thread will continue
-##! growing larger and larger giving the appearance of a memory leak.
-
-module LogElasticSearch;
-
-export {
-	## Name of the ES cluster.
-	const cluster_name = "elasticsearch" &redef;
-
-	## ES server.
-	const server_host = "127.0.0.1" &redef;
-
-	## ES port.
-	const server_port = 9200 &redef;
-
-	## Name of the ES index.
-	const index_prefix = "bro" &redef;
-
-	## The ES type prefix comes before the name of the related log.
-	## e.g. prefix = "bro\_" would create types of bro_dns, bro_software, etc.
-	const type_prefix = "" &redef;
-
-	## The time before an ElasticSearch transfer will timeout. Note that
-	## the fractional part of the timeout will be ignored. In particular,
-	## time specifications less than a second result in a timeout value of
-	## 0, which means "no timeout."
-	const transfer_timeout = 2secs;
-
-	## The batch size is the number of messages that will be queued up before
-	## they are sent to be bulk indexed.
-	const max_batch_size = 1000 &redef;
-
-	## The maximum amount of wall-clock time that is allowed to pass without
-	## finishing a bulk log send.  This represents the maximum delay you
-	## would like to have with your logs before they are sent to ElasticSearch.
-	const max_batch_interval = 1min &redef;
-
-	## The maximum byte size for a buffered JSON string to send to the bulk
-	## insert API.
-	const max_byte_size = 1024 * 1024 &redef;
-}
-

scripts/base/frameworks/notice/main.bro

@@ -20,7 +20,8 @@ export {
 	## category along with the specific notice separating words with
 	## underscores and using leading capitals on each word except for
 	## abbreviations which are kept in all capitals.  For example,
-	## SSH::Login is for heuristically guessed successful SSH logins.
+	## SSH::Password_Guessing is for hosts that have crossed a threshold of
+	## heuristically determined failed SSH logins.
 	type Type: enum {
 		## Notice reporting a count of how often a notice occurred.
 		Tally,
@@ -530,8 +531,8 @@ function create_file_info(f: fa_file): Notice::FileInfo
 	local fi: Notice::FileInfo = Notice::FileInfo($fuid = f$id,
 	                                              $desc = Files::describe(f));
 
-	if ( f?$mime_type )
+	if ( f?$info && f$info?$mime_type )
-		fi$mime = f$mime_type;
+		fi$mime = f$info$mime_type;
 
 	if ( f?$conns && |f$conns| == 1 )
 		for ( id in f$conns )

scripts/base/frameworks/signatures/main.bro

@@ -71,7 +71,7 @@ export {
 		## to be logged has occurred.
 		ts:         time         &log;
 		## A unique identifier of the connection which triggered the
-		## signature match event
+		## signature match event.
 		uid:        string       &log &optional;
 		## The host which triggered the signature match event.
 		src_addr:   addr         &log &optional;

scripts/base/init-bare.bro

@@ -75,6 +75,13 @@ type addr_vec: vector of addr;
 ##    directly and then remove this alias.
 type table_string_of_string: table[string] of string;
 
+## A set of file analyzer tags.
+##
+## .. todo:: We need this type definition only for declaring builtin functions
+##    via ``bifcl``. We should extend ``bifcl`` to understand composite types
+##    directly and then remove this alias.
+type files_tag_set: set[Files::Tag];
+
 ## A structure indicating a MIME type and strength of a match against
 ## file magic signatures.
 ##
@@ -346,9 +353,10 @@ type connection: record {
 ## gives up and discards any internal state related to the file.
 const default_file_timeout_interval: interval = 2 mins &redef;
 
-## Default amount of bytes that file analysis will buffer before raising
+## Default amount of bytes that file analysis will buffer in order to use
-## :bro:see:`file_new`.
+## for mime type matching.  File analyzers attached at the time of mime type
-const default_file_bof_buffer_size: count = 1024 &redef;
+## matching or later, will receive a copy of this buffer.
+const default_file_bof_buffer_size: count = 4096 &redef;
 
 ## A file that Bro is analyzing.  This is Bro's type for describing the basic
 ## internal metadata collected about a "file", which is essentially just a
@@ -387,8 +395,10 @@ type fa_file: record {
 	## during the process of analysis e.g. due to dropped packets.
 	missing_bytes: count &default=0;
 
-	## The number of not all-in-sequence bytes in the file stream that
+	## The number of bytes in the file stream that were not delivered to
-	## were delivered to file analyzers due to reassembly buffer overflow.
+	## stream file analyzers.  Generally, this consists of bytes that
+	## couldn't be reassembled, either because reassembly simply isn't
+	## enabled, or due to size limitations of the reassembly buffer.
 	overflow_bytes: count &default=0;
 
 	## The amount of time between receiving new data for this file that
@@ -402,16 +412,6 @@ type fa_file: record {
 	## The content of the beginning of a file up to *bof_buffer_size* bytes.
 	## This is also the buffer that's used for file/mime type detection.
 	bof_buffer: string &optional;
-
-	## The mime type of the strongest file magic signature matches against
-	## the data chunk in *bof_buffer*, or in the cases where no buffering
-	## of the beginning of file occurs, an initial guess of the mime type
-	## based on the first data seen.
-	mime_type: string &optional;
-
-	## All mime types that matched file magic signatures against the data
-	## chunk in *bof_buffer*, in order of their strength value.
-	mime_types: mime_matches &optional;
 } &redef;
 
 ## Fields of a SYN packet.
@@ -2478,8 +2478,7 @@ type http_message_stat: record {
 	header_length: count;
 };
 
-## Maximum number of HTTP entity data delivered to events. The amount of data
+## Maximum number of HTTP entity data delivered to events.
-## can be limited for better performance, zero disables truncation.
 ##
 ## .. bro:see:: http_entity_data skip_http_entity_data skip_http_data
 global http_entity_data_delivery_size = 1500 &redef;
@@ -2806,6 +2805,7 @@ type ModbusRegisters: vector of count;
 type ModbusHeaders: record {
 	tid:           count;
 	pid:           count;
+	len:           count;
 	uid:           count;
 	function_code: count;
 };
@@ -3431,9 +3431,6 @@ const global_hash_seed: string = "" &redef;
 ## The maximum is currently 128 bits.
 const bits_per_uid: count = 96 &redef;
 
-# Load BiFs defined by plugins.
-@load base/bif/plugins
-
 # Load these frameworks here because they use fairly deep integration with
 # BiFs and script-land defined types.
 @load base/frameworks/logging
@@ -3442,3 +3439,7 @@ const bits_per_uid: count = 96 &redef;
 @load base/frameworks/files
 
 @load base/bif
+
+# Load BiFs defined by plugins.
+@load base/bif/plugins
+

scripts/base/init-default.bro

@@ -46,6 +46,7 @@
 @load base/protocols/http
 @load base/protocols/irc
 @load base/protocols/modbus
+@load base/protocols/mysql
 @load base/protocols/pop3
 @load base/protocols/radius
 @load base/protocols/snmp

scripts/base/protocols/dhcp/main.bro

@@ -47,13 +47,13 @@ redef record connection += {
 const ports = { 67/udp, 68/udp };
 redef likely_server_ports += { 67/udp };
 
-event bro_init()
+event bro_init() &priority=5
 	{
 	Log::create_stream(DHCP::LOG, [$columns=Info, $ev=log_dhcp]);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_DHCP, ports);
 	}
 
-event dhcp_ack(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr, host_name: string)
+event dhcp_ack(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr, host_name: string) &priority=5
 	{
 	local info: Info;
 	info$ts          = network_time();
@@ -71,6 +71,9 @@ event dhcp_ack(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_lis
 		info$assigned_ip = c$id$orig_h;
 
 	c$dhcp = info;
+	}
 
+event dhcp_ack(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr, host_name: string) &priority=-5
+	{
 	Log::write(DHCP::LOG, c$dhcp);
 	}

scripts/base/protocols/dnp3/dpd.sig

@@ -5,5 +5,11 @@ signature dpd_dnp3_server {
 	ip-proto == tcp
 	payload /\x05\x64/
 	tcp-state responder
- 	enable "dnp3"
+ 	enable "dnp3_tcp"
+}
+
+signature dpd_dnp3_server_udp {
+	ip-proto == udp
+	payload /\x05\x64/
+	enable "dnp3_udp"
 }

scripts/base/protocols/dnp3/main.bro

@@ -31,16 +31,16 @@ redef record connection += {
 	dnp3: Info &optional;
 };
 
-const ports = { 20000/tcp };
+const ports = { 20000/tcp , 20000/udp };
 redef likely_server_ports += { ports };
 
 event bro_init() &priority=5
 	{
 	Log::create_stream(DNP3::LOG, [$columns=Info, $ev=log_dnp3]);
-	Analyzer::register_for_ports(Analyzer::ANALYZER_DNP3, ports);
+	Analyzer::register_for_ports(Analyzer::ANALYZER_DNP3_TCP, ports);
 	}
 
-event dnp3_application_request_header(c: connection, is_orig: bool, fc: count)
+event dnp3_application_request_header(c: connection, is_orig: bool, application_control: count, fc: count)
 	{
 	if ( ! c?$dnp3 )
 		c$dnp3 = [$ts=network_time(), $uid=c$uid, $id=c$id];
@@ -49,7 +49,7 @@ event dnp3_application_request_header(c: connection, is_orig: bool, fc: count)
 	c$dnp3$fc_request = function_codes[fc];
 	}
 
-event dnp3_application_response_header(c: connection, is_orig: bool, fc: count, iin: count)
+event dnp3_application_response_header(c: connection, is_orig: bool, application_control: count, fc: count, iin: count)
 	{
 	if ( ! c?$dnp3 )
 		c$dnp3 = [$ts=network_time(), $uid=c$uid, $id=c$id];

scripts/base/protocols/ftp/files.bro

@@ -17,6 +17,10 @@ export {
 
 	## Describe the file being transferred.
 	global describe_file: function(f: fa_file): string;
+
+	redef record fa_file += { 
+		ftp: FTP::Info &optional;
+	};
 }
 
 function get_file_handle(c: connection, is_orig: bool): string
@@ -48,7 +52,6 @@ event bro_init() &priority=5
 	                          $describe        = FTP::describe_file]);
 	}
 
-
 event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5
 	{
 	if ( [c$id$resp_h, c$id$resp_p] !in ftp_data_expected ) 
@@ -56,6 +59,14 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori
 
 	local ftp = ftp_data_expected[c$id$resp_h, c$id$resp_p];
 	ftp$fuid = f$id;
-	if ( f?$mime_type )
+
-		ftp$mime_type = f$mime_type;
+	f$ftp = ftp;
+	}
+
+event file_mime_type(f: fa_file, mime_type: string) &priority=5
+	{
+	if ( ! f?$ftp )
+		return;
+
+	f$ftp$mime_type = mime_type;
 	}

scripts/base/protocols/http/entities.bro

@@ -35,6 +35,10 @@ export {
 		## body.
 		resp_mime_depth: count            &default=0;
 	};
+
+	redef record fa_file += {
+		http: HTTP::Info &optional;
+	};
 }
 
 event http_begin_entity(c: connection, is_orig: bool) &priority=10
@@ -67,6 +71,8 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori
 	{
 	if ( f$source == "HTTP" && c?$http ) 
 		{
+		f$http = c$http;
+
 		if ( c$http?$current_entity && c$http$current_entity?$filename )
 			f$info$filename = c$http$current_entity$filename;
 
@@ -76,14 +82,6 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori
 				c$http$orig_fuids = string_vec(f$id);
 			else
 				c$http$orig_fuids[|c$http$orig_fuids|] = f$id;
-
-			if ( f?$mime_type )
-				{
-				if ( ! c$http?$orig_mime_types )
-					c$http$orig_mime_types = string_vec(f$mime_type);
-				else
-					c$http$orig_mime_types[|c$http$orig_mime_types|] = f$mime_type;
-				}
 			}
 		else
 			{
@@ -91,17 +89,29 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori
 				c$http$resp_fuids = string_vec(f$id);
 			else
 				c$http$resp_fuids[|c$http$resp_fuids|] = f$id;
-
-			if ( f?$mime_type )
-				{
-				if ( ! c$http?$resp_mime_types )
-					c$http$resp_mime_types = string_vec(f$mime_type);
-				else
-					c$http$resp_mime_types[|c$http$resp_mime_types|] = f$mime_type;
-				}
 			}
 		}
+	}
 
+event file_mime_type(f: fa_file, mime_type: string) &priority=5
+	{
+	if ( ! f?$http || ! f?$is_orig )
+		return;
+
+	if ( f$is_orig )
+		{
+		if ( ! f$http?$orig_mime_types )
+			f$http$orig_mime_types = string_vec(mime_type);
+		else
+			f$http$orig_mime_types[|f$http$orig_mime_types|] = mime_type;
+		}
+	else
+		{
+		if ( ! f$http?$resp_mime_types )
+			f$http$resp_mime_types = string_vec(mime_type);
+		else
+			f$http$resp_mime_types[|f$http$resp_mime_types|] = mime_type;
+		}
 	}
 
 event http_end_entity(c: connection, is_orig: bool) &priority=5

scripts/base/protocols/irc/files.bro

@@ -12,6 +12,10 @@ export {
 
 	## Default file handle provider for IRC.
 	global get_file_handle: function(c: connection, is_orig: bool): string;
+
+	redef record fa_file += {
+		irc: IRC::Info &optional;
+	};
 }
 
 function get_file_handle(c: connection, is_orig: bool): string
@@ -34,6 +38,12 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori
 	irc$fuid = f$id;
 	if ( irc?$dcc_file_name )
 		f$info$filename = irc$dcc_file_name;
-	if ( f?$mime_type )
+
-		irc$dcc_mime_type = f$mime_type;
+	f$irc = irc;
 	}
+
+event file_mime_type(f: fa_file, mime_type: string) &priority=5
+	{
+	if ( f?$irc )
+		f$irc$dcc_mime_type = mime_type;
+	}

scripts/base/protocols/mysql/__load__.bro

@@ -0,0 +1 @@
+@load ./main

scripts/base/protocols/mysql/consts.bro

@@ -0,0 +1,38 @@
+module MySQL;
+
+export {
+	const commands: table[count] of string = {
+		[0]  = "sleep",
+		[1]  = "quit",
+		[2]  = "init_db",
+		[3]  = "query",
+		[4]  = "field_list",
+		[5]  = "create_db",
+		[6]  = "drop_db",
+		[7]  = "refresh",
+		[8]  = "shutdown",
+		[9]  = "statistics",
+		[10] = "process_info",
+		[11] = "connect",
+		[12] = "process_kill",
+		[13] = "debug",
+		[14] = "ping",
+		[15] = "time",
+		[16] = "delayed_insert",
+		[17] = "change_user",
+		[18] = "binlog_dump",
+		[19] = "table_dump",
+		[20] = "connect_out",
+		[21] = "register_slave",
+		[22] = "stmt_prepare",
+		[23] = "stmt_execute",
+		[24] = "stmt_send_long_data",
+		[25] = "stmt_close",
+		[26] = "stmt_reset",
+		[27] = "set_option",
+		[28] = "stmt_fetch",
+		[29] = "daemon",
+		[30] = "binlog_dump_gtid",
+		[31] = "reset_connection",
+	} &default=function(i: count): string { return fmt("unknown-%d", i); };
+}

scripts/base/protocols/mysql/main.bro

@@ -0,0 +1,132 @@
+##! Implements base functionality for MySQL analysis. Generates the mysql.log file.
+
+module MySQL;
+
+@load ./consts
+
+export {
+	redef enum Log::ID += { mysql::LOG };
+
+	type Info: record {
+		## Timestamp for when the event happened.
+		ts:     time    &log;
+		## Unique ID for the connection.
+		uid:    string  &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
+		id:     conn_id &log;
+		## The command that was issued
+		cmd:	string	&log;
+		## The argument issued to the command
+		arg:	string	&log;
+		## Did the server tell us that the command succeeded?
+		success: bool &log &optional;
+		## The number of affected rows, if any
+		rows: count &log &optional;
+		## Server message, if any
+		response: string &log &optional;
+	};
+
+	## Event that can be handled to access the MySQL record as it is sent on
+	## to the logging framework.
+	global log_mysql: event(rec: Info);
+}
+
+redef record connection += {
+	mysql: Info &optional;
+};
+
+const ports = { 1434/tcp, 3306/tcp };
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(mysql::LOG, [$columns=Info, $ev=log_mysql]);
+	Analyzer::register_for_ports(Analyzer::ANALYZER_MYSQL, ports);
+	}
+
+event mysql_handshake(c: connection, username: string)
+	{
+	if ( ! c?$mysql )
+		{
+		local info: Info;
+		info$ts = network_time();
+		info$uid = c$uid;
+		info$id = c$id;
+		info$cmd = "login";
+		info$arg = username;
+		c$mysql = info;
+		}
+	}
+
+event mysql_command_request(c: connection, command: count, arg: string) &priority=5
+	{
+	if ( c?$mysql )
+		{
+		# We got a request, but we haven't logged our
+		# previous request yet, so let's do that now.
+		Log::write(mysql::LOG, c$mysql);
+		delete c$mysql;
+		}
+
+	local info: Info;
+	info$ts = network_time();
+	info$uid = c$uid;
+	info$id = c$id;
+	info$cmd = commands[command];
+	info$arg = sub(arg, /\0$/, "");
+	c$mysql = info;
+	}
+
+event mysql_command_request(c: connection, command: count, arg: string) &priority=-5
+	{
+	if ( c?$mysql && c$mysql?$cmd && c$mysql$cmd == "quit" )
+		{
+		# We get no response for quits, so let's just log it now.
+		Log::write(mysql::LOG, c$mysql);
+		delete c$mysql;
+		}
+	}
+
+event mysql_error(c: connection, code: count, msg: string) &priority=5
+	{
+	if ( c?$mysql )
+		{
+		c$mysql$success = F;
+		c$mysql$response = msg;
+		}
+	}
+
+event mysql_error(c: connection, code: count, msg: string) &priority=-5
+	{
+	if ( c?$mysql )
+		{
+		Log::write(mysql::LOG, c$mysql);
+		delete c$mysql;
+		}
+	}
+
+event mysql_ok(c: connection, affected_rows: count) &priority=5
+	{
+	if ( c?$mysql )
+		{
+		c$mysql$success = T;
+		c$mysql$rows = affected_rows;
+		}
+	}
+
+event mysql_ok(c: connection, affected_rows: count) &priority=-5
+	{
+	if ( c?$mysql )
+		{
+		Log::write(mysql::LOG, c$mysql);
+		delete c$mysql;
+		}
+	}
+
+event connection_state_remove(c: connection) &priority=-5
+	{
+	if ( c?$mysql )
+		{
+		Log::write(mysql::LOG, c$mysql);
+		delete c$mysql;
+		}
+	}

scripts/base/protocols/smtp/main.bro

@@ -58,31 +58,31 @@ export {
 		## Indicates if client activity has been seen, but not yet logged.
 		has_client_activity:  bool            &default=F;
 	};
-	
+
 	type State: record {
 		helo:                     string    &optional;
 		## Count the number of individual messages transmitted during
 		## this SMTP session.  Note, this is not the number of
 		## recipients, but the number of message bodies transferred.
 		messages_transferred:     count     &default=0;
-		
+
 		pending_messages:         set[Info] &optional;
 	};
-	
+
 	## Direction to capture the full "Received from" path.
 	##    REMOTE_HOSTS - only capture the path until an internal host is found.
 	##    LOCAL_HOSTS - only capture the path until the external host is discovered.
 	##    ALL_HOSTS - always capture the entire path.
 	##    NO_HOSTS - never capture the path.
 	const mail_path_capture = ALL_HOSTS &redef;
-	
+
 	## Create an extremely shortened representation of a log line.
 	global describe: function(rec: Info): string;
 
 	global log_smtp: event(rec: Info);
 }
 
-redef record connection += { 
+redef record connection += {
 	smtp:       Info  &optional;
 	smtp_state: State &optional;
 };
@@ -95,7 +95,7 @@ event bro_init() &priority=5
 	Log::create_stream(SMTP::LOG, [$columns=SMTP::Info, $ev=log_smtp]);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_SMTP, ports);
 	}
-	
+
 function find_address_in_smtp_header(header: string): string
 {
 	local ips = find_ip_addresses(header);
@@ -116,17 +116,17 @@ function new_smtp_log(c: connection): Info
 	l$ts=network_time();
 	l$uid=c$uid;
 	l$id=c$id;
-	# The messages_transferred count isn't incremented until the message is 
+	# The messages_transferred count isn't incremented until the message is
 	# finished so we need to increment the count by 1 here.
 	l$trans_depth = c$smtp_state$messages_transferred+1;
-	
+
 	if ( c$smtp_state?$helo )
 		l$helo = c$smtp_state$helo;
-	
+
 	# The path will always end with the hosts involved in this connection.
 	# The lower values in the vector are the end of the path.
 	l$path = vector(c$id$resp_h, c$id$orig_h);
-	
+
 	return l;
 	}
 
@@ -134,7 +134,7 @@ function set_smtp_session(c: connection)
 	{
 	if ( ! c?$smtp_state )
 		c$smtp_state = [];
-	
+
 	if ( ! c?$smtp )
 		c$smtp = new_smtp_log(c);
 	}
@@ -142,17 +142,17 @@ function set_smtp_session(c: connection)
 function smtp_message(c: connection)
 	{
 	if ( c$smtp$has_client_activity )
+		{
 		Log::write(SMTP::LOG, c$smtp);
+		c$smtp = new_smtp_log(c);
+		}
 	}
-	
+
 event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=5
 	{
 	set_smtp_session(c);
 	local upper_command = to_upper(command);
 
-	if ( upper_command != "QUIT" )
-		c$smtp$has_client_activity = T;
-	
 	if ( upper_command == "HELO" || upper_command == "EHLO" )
 		{
 		c$smtp_state$helo = arg;
@@ -161,23 +161,28 @@ event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &
 
 	else if ( upper_command == "RCPT" && /^[tT][oO]:/ in arg )
 		{
-		if ( ! c$smtp?$rcptto ) 
+		if ( ! c$smtp?$rcptto )
 			c$smtp$rcptto = set();
 		add c$smtp$rcptto[split1(arg, /:[[:blank:]]*/)[2]];
+		c$smtp$has_client_activity = T;
 		}
 
 	else if ( upper_command == "MAIL" && /^[fF][rR][oO][mM]:/ in arg )
 		{
+		# Flush last message in case we didn't see the server's acknowledgement.
+		smtp_message(c);
+
 		local partially_done = split1(arg, /:[[:blank:]]*/)[2];
 		c$smtp$mailfrom = split1(partially_done, /[[:blank:]]?/)[1];
+		c$smtp$has_client_activity = T;
 		}
 	}
-	
+
 event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
                  msg: string, cont_resp: bool) &priority=5
 	{
 	set_smtp_session(c);
-	
+
 	# This continually overwrites, but we want the last reply,
 	# so this actually works fine.
 	c$smtp$last_reply = fmt("%d %s", code, msg);
@@ -198,7 +203,6 @@ event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
 event mime_one_header(c: connection, h: mime_header_rec) &priority=5
 	{
 	if ( ! c?$smtp ) return;
-	c$smtp$has_client_activity = T;
 
 	if ( h$name == "MESSAGE-ID" )
 		c$smtp$msg_id = h$value;
@@ -241,19 +245,19 @@ event mime_one_header(c: connection, h: mime_header_rec) &priority=5
 		if ( 1 in addresses )
 			c$smtp$x_originating_ip = to_addr(addresses[1]);
 		}
-	
+
 	else if ( h$name == "X-MAILER" ||
 	          h$name == "USER-AGENT" ||
 	          h$name == "X-USER-AGENT" )
 		c$smtp$user_agent = h$value;
 	}
-	
+
-# This event handler builds the "Received From" path by reading the 
+# This event handler builds the "Received From" path by reading the
 # headers in the mail
 event mime_one_header(c: connection, h: mime_header_rec) &priority=3
 	{
 	# If we've decided that we're done watching the received headers for
-	# whatever reason, we're done.  Could be due to only watching until 
+	# whatever reason, we're done.  Could be due to only watching until
 	# local addresses are seen in the received from headers.
 	if ( ! c?$smtp || h$name != "RECEIVED" || ! c$smtp$process_received_from )
 		return;
@@ -263,7 +267,7 @@ event mime_one_header(c: connection, h: mime_header_rec) &priority=3
 		return;
 	local ip = to_addr(text_ip);
 
-	if ( ! addr_matches_host(ip, mail_path_capture) && 
+	if ( ! addr_matches_host(ip, mail_path_capture) &&
 	     ! Site::is_private_addr(ip) )
 		{
 		c$smtp$process_received_from = F;
@@ -281,7 +285,10 @@ event connection_state_remove(c: connection) &priority=-5
 event smtp_starttls(c: connection) &priority=5
 	{
 	if ( c?$smtp )
+		{
 		c$smtp$tls = T;
+		c$smtp$has_client_activity = T;
+		}
 	}
 
 function describe(rec: Info): string

scripts/base/protocols/ssl/consts.bro

@@ -30,6 +30,7 @@ export {
 	const HELLO_REQUEST       = 0;
 	const CLIENT_HELLO        = 1;
 	const SERVER_HELLO        = 2;
+	const HELLO_VERIFY_REQUEST = 3; # RFC 6347
 	const SESSION_TICKET      = 4; # RFC 5077
 	const CERTIFICATE         = 11;
 	const SERVER_KEY_EXCHANGE = 12;
@@ -40,6 +41,7 @@ export {
 	const FINISHED            = 20;
 	const CERTIFICATE_URL     = 21; # RFC 3546
 	const CERTIFICATE_STATUS  = 22; # RFC 3546
+	const SUPPLEMENTAL_DATA   = 23; # RFC 4680
 
 	## Mapping between numeric codes and human readable strings for alert
 	## levels.
@@ -112,7 +114,8 @@ export {
 		[19] = "client_certificate_type",
 		[20] = "server_certificate_type",
 		[21] = "padding", # temporary till 2015-03-12
-		[22] = "encrypt_then_mac", # temporary till 2015-06-05
+		[22] = "encrypt_then_mac",
+		[23] = "extended_master_secret", # temporary till 2015-09-26
 		[35] = "SessionTicket TLS",
 		[40] = "extended_random",
 		[13172] = "next_protocol_negotiation",
@@ -155,6 +158,11 @@ export {
 		[26] = "brainpoolP256r1",
 		[27] = "brainpoolP384r1",
 		[28] = "brainpoolP512r1",
+		# draft-ietf-tls-negotiated-ff-dhe-05
+		[256] = "ffdhe2048",
+		[257] = "ffdhe3072",
+		[258] = "ffdhe4096",
+		[259] = "ffdhe8192",
 		[0xFF01] = "arbitrary_explicit_prime_curves",
 		[0xFF02] = "arbitrary_explicit_char2_curves"
 	} &default=function(i: count):string { return fmt("unknown-%d", i); };

scripts/base/protocols/ssl/main.bro

@@ -12,7 +12,7 @@ export {
 		## Time when the SSL connection was first detected.
 		ts:               time             &log;
 		## Unique ID for the connection.
-		uid:         string          &log;
+		uid:              string           &log;
 		## The connection's 4-tuple of endpoint addresses/ports.
 		id:               conn_id          &log;
 		## SSL/TLS version that the server offered.
@@ -25,9 +25,25 @@ export {
 		## indicates the server name that the client was requesting.
 		server_name:      string           &log &optional;
 		## Session ID offered by the client for session resumption.
-		session_id:       string           &log &optional;
+		## Not used for logging.
+		session_id:       string           &optional;
+		## Flag to indicate if the session was resumed reusing
+		## the key material exchanged in an earlier connection.
+		resumed:          bool             &log &default=F;
+		## Flag to indicate if we saw a non-empty session ticket being
+		## sent by the client using an empty session ID. This value
+		## is used to determine if a session is being resumed. It's
+		## not logged.
+		client_ticket_empty_session_seen: bool &default=F;
+		## Flag to indicate if we saw a client key exchange message sent
+		## by the client. This value is used to determine if a session
+		## is being resumed. It's not logged.
+		client_key_exchange_seen: bool     &default=F;
 		## Last alert that was seen during the connection.
 		last_alert:       string           &log &optional;
+		## Next protocol the server chose using the application layer
+		## next protocol extension, if present.
+		next_protocol:    string           &log &optional;
 
 		## The analyzer ID used for the analyzer instance attached
 		## to each connection.  It is not used for logging since it's a
@@ -36,11 +52,11 @@ export {
 
 		## Flag to indicate if this ssl session has been established
 		## succesfully, or if it was aborted during the handshake.
-		established:  bool &log &default=F;
+		established:      bool             &log &default=F;
 
 		## Flag to indicate if this record already has been logged, to
 		## prevent duplicates.
-		logged:  bool  &default=F;
+		logged:           bool             &default=F;
 	};
 
 	## The default root CA bundle.  By default, the mozilla-ca-list.bro
@@ -149,8 +165,11 @@ event ssl_client_hello(c: connection, version: count, possible_ts: time, client_
 	set_session(c);
 
 	# Save the session_id if there is one set.
-	if ( session_id != /^\x00{32}$/ )
+	if ( |session_id| > 0 && session_id != /^\x00{32}$/ )
+		{
 		c$ssl$session_id = bytestring_to_hexstr(session_id);
+		c$ssl$client_ticket_empty_session_seen = F;
+		}
 	}
 
 event ssl_server_hello(c: connection, version: count, possible_ts: time, server_random: string, session_id: string, cipher: count, comp_method: count) &priority=5
@@ -159,6 +178,9 @@ event ssl_server_hello(c: connection, version: count, possible_ts: time, server_
 
 	c$ssl$version = version_strings[version];
 	c$ssl$cipher = cipher_desc[cipher];
+
+	if ( c$ssl?$session_id && c$ssl$session_id == bytestring_to_hexstr(session_id) )
+		c$ssl$resumed = T;
 	}
 
 event ssl_server_curve(c: connection, curve: count) &priority=5
@@ -180,6 +202,45 @@ event ssl_extension_server_name(c: connection, is_orig: bool, names: string_vec)
 		}
 	}
 
+event ssl_extension_application_layer_protocol_negotiation(c: connection, is_orig: bool, protocols: string_vec)
+	{
+	set_session(c);
+
+	if ( is_orig )
+		return;
+
+	if ( |protocols| > 0 )
+		c$ssl$next_protocol = protocols[0];
+	}
+
+event ssl_handshake_message(c: connection, is_orig: bool, msg_type: count, length: count) &priority=5
+	{
+	set_session(c);
+
+	if ( is_orig && msg_type == SSL::CLIENT_KEY_EXCHANGE )
+		c$ssl$client_key_exchange_seen = T;
+	}
+
+# Extension event is fired _before_ the respective client or server hello.
+# Important for client_ticket_empty_session_seen.
+event ssl_extension(c: connection, is_orig: bool, code: count, val: string) &priority=5
+	{
+	set_session(c);
+
+	if ( is_orig && SSL::extensions[code] == "SessionTicket TLS" && |val| > 0 )
+		# In this case, we might have an empty ID. Set back to F in client_hello event
+		# if it is not empty after all.
+		c$ssl$client_ticket_empty_session_seen = T;
+	}
+
+event ssl_change_cipher_spec(c: connection, is_orig: bool) &priority=5
+	{
+	set_session(c);
+
+	if ( is_orig && c$ssl$client_ticket_empty_session_seen && ! c$ssl$client_key_exchange_seen )
+		c$ssl$resumed = T;
+	}
+
 event ssl_alert(c: connection, is_orig: bool, level: count, desc: count) &priority=5
 	{
 	set_session(c);

scripts/base/utils/active-http.bro

@@ -65,12 +65,14 @@ function request2curl(r: Request, bodyfile: string, headersfile: string): string
 	cmd = fmt("%s -m %.0f", cmd, r$max_time);
 
 	if ( r?$client_data )
-		cmd = fmt("%s -d -", cmd);
+		cmd = fmt("%s -d @-", cmd);
 
 	if ( r?$addl_curl_args )
 		cmd = fmt("%s %s", cmd, r$addl_curl_args);
 
 	cmd = fmt("%s \"%s\"", cmd, str_shell_escape(r$url));
+	# Make sure file will exist even if curl did not write one.
+	cmd = fmt("%s && touch %s", cmd, str_shell_escape(bodyfile));
 	return cmd;
 	}
 

scripts/base/utils/exec.bro

@@ -96,8 +96,9 @@ event Exec::file_line(description: Input::EventDescription, tpe: Input::Event, s
 		result$files[track_file][|result$files[track_file]|] = s;
 	}
 
-event Input::end_of_data(name: string, source:string)
+event Input::end_of_data(orig_name: string, source:string)
 	{
+	local name = orig_name;
 	local parts = split1(name, /_/);
 	name = parts[1];
 
@@ -106,6 +107,15 @@ event Input::end_of_data(name: string, source:string)
 
 	local track_file = parts[2];
 
+	# If the file is empty, still add it to the result$files table. This is needed
+	# because it is expected that the file was read even if it was empty.
+	local result = results[name];
+	if ( ! result?$files )
+		result$files = table();
+
+	if ( track_file !in result$files )
+		result$files[track_file] = vector();
+
 	Input::remove(name);
 
 	if ( name !in pending_files )

scripts/base/utils/urls.bro

@@ -3,6 +3,28 @@
 ## A regular expression for matching and extracting URLs.
 const url_regex = /^([a-zA-Z\-]{3,5})(:\/\/[^\/?#"'\r\n><]*)([^?#"'\r\n><]*)([^[:blank:]\r\n"'><]*|\??[^"'\r\n><]*)/ &redef;
 
+## A URI, as parsed by :bro:id:`decompose_uri`.
+type URI: record {
+	## The URL's scheme..
+	scheme:		string &optional;
+	## The location, which could be a domain name or an IP address. Left empty if not
+	## specified.
+	netlocation:	string;
+	## Port number, if included in URI.
+	portnum:	count &optional;
+	## Full including the file name. Will be '/' if there's not path given.
+	path:		string;
+	## Full file name, including extension, if there is a file name.
+	file_name:	string &optional;
+	## The base filename, without extension, if there is a file name.
+	file_base:	string &optional;
+	## The filename's extension, if there is a file name.
+	file_ext:	string &optional;
+	## A table of all query parameters, mapping their keys to values, if there's a
+	## query.
+	params:	table[string] of string &optional;
+};
+
 ## Extracts URLs discovered in arbitrary text.
 function find_all_urls(s: string): string_set
 	{
@@ -23,3 +45,84 @@ function find_all_urls_without_scheme(s: string): string_set
 
 	return return_urls;
 	}
+
+function decompose_uri(s: string): URI
+	{
+	local parts: string_array;
+	local u: URI = [$netlocation="", $path="/"];
+
+	if ( /\?/ in s)
+		{
+		# Parse query.
+		u$params = table();
+
+		parts = split1(s, /\?/);
+		s = parts[1];
+		local query: string = parts[2];
+
+		if ( /&/ in query )
+			{
+			local opv: table[count] of string = split(query, /&/);
+
+			for ( each in opv )
+				{
+				if ( /=/ in opv[each] )
+					{
+					parts = split1(opv[each], /=/);
+					u$params[parts[1]] = parts[2];
+					}
+				}
+			}
+		else
+			{
+			parts = split1(query, /=/);
+			u$params[parts[1]] = parts[2];
+			}
+		}
+
+	if ( /:\/\// in s )
+		{
+		# Parse scheme and remove from s.
+		parts = split1(s, /:\/\//);
+		u$scheme = parts[1];
+		s = parts[2];
+		}
+
+	if ( /\// in s )
+		{
+		# Parse path and remove from s.
+		parts = split1(s, /\//);
+		s = parts[1];
+		u$path = fmt("/%s", parts[2]);
+
+		if ( |u$path| > 1 && u$path[|u$path| - 1] != "/" )
+			{
+			local last_token: string = find_last(u$path, /\/.+/);
+			local full_filename = split1(last_token, /\//)[2];
+
+			if ( /\./ in full_filename )
+				{
+				u$file_name = full_filename;
+				u$file_base = split1(full_filename, /\./)[1];
+				u$file_ext = split1(full_filename, /\./)[2];
+				}
+			else
+				{
+				u$file_name = full_filename;
+				u$file_base = full_filename;
+				}
+			}
+		}
+
+	if ( /:/ in s )
+		{
+		# Parse location and port.
+		parts = split1(s, /:/);
+		u$netlocation = parts[1];
+		u$portnum = to_count(parts[2]);
+		}
+	else
+		u$netlocation = s;
+
+	return u;
+	}

scripts/policy/frameworks/files/detect-MHR.bro

@@ -66,6 +66,7 @@ function do_mhr_lookup(hash: string, fi: Notice::FileInfo)
 
 event file_hash(f: fa_file, kind: string, hash: string)
 	{
-	if ( kind == "sha1" && f?$mime_type && match_file_types in f$mime_type )
+	if ( kind == "sha1" && f?$info && f$info?$mime_type && 
+	     match_file_types in f$info$mime_type )
 		do_mhr_lookup(hash, Notice::create_file_info(f));
 	}

scripts/policy/frameworks/software/windows-version-detection.bro

@@ -0,0 +1,69 @@
+##! Windows systems access a Microsoft Certificate Revocation List (CRL) periodically. The
+##! user agent for these requests reveals which version of Crypt32.dll installed on the system,
+##! which can uniquely identify the version of Windows that's running.
+##!
+##! This script will log the version of Windows that was identified to the Software framework.
+
+@load base/protocols/http
+@load base/frameworks/software
+
+module OS;
+
+export {
+        redef enum Software::Type += {
+                ## Identifier for Windows operating system versions
+                WINDOWS,
+        };
+
+        type Software::name_and_version: record {
+                name   : string;
+                version: Software::Version;
+        };
+
+        const crypto_api_mapping: table[string] of Software::name_and_version = {
+                ["Microsoft-CryptoAPI/5.131.2195.6661"] = [$name="Windows", $version=[$major=5, $minor=131, $minor2=2195, $minor3=6661, $addl="2000 SP4"]],
+                ["Microsoft-CryptoAPI/5.131.2195.6824"] = [$name="Windows", $version=[$major=5, $minor=131, $minor2=2195, $minor3=6824, $addl="2000 with MS04-11"]],
+                ["Microsoft-CryptoAPI/5.131.2195.6926"] = [$name="Windows", $version=[$major=5, $minor=131, $minor2=2195, $minor3=6926, $addl="2000 with Hotfix 98830"]],
+
+                ["Microsoft-CryptoAPI/5.131.2600.0"]    = [$name="Windows", $version=[$major=5, $minor=131, $minor2=2600, $minor3=0,    $addl="XP SP0"]],
+                ["Microsoft-CryptoAPI/5.131.2600.1106"] = [$name="Windows", $version=[$major=5, $minor=131, $minor2=2600, $minor3=1106, $addl="XP SP1"]],
+                ["Microsoft-CryptoAPI/5.131.2600.2180"] = [$name="Windows", $version=[$major=5, $minor=131, $minor2=2600, $minor3=2180, $addl="XP SP2"]],
+                ["Microsoft-CryptoAPI/5.131.2600.3180"] = [$name="Windows", $version=[$major=5, $minor=131, $minor2=2600, $minor3=3180, $addl="XP SP3 Beta 1"]],
+                ["Microsoft-CryptoAPI/5.131.2600.3205"] = [$name="Windows", $version=[$major=5, $minor=131, $minor2=2600, $minor3=3205, $addl="XP SP3 Beta 2"]],
+                ["Microsoft-CryptoAPI/5.131.2600.3249"] = [$name="Windows", $version=[$major=5, $minor=131, $minor2=2600, $minor3=3249, $addl="XP SP3 RC Beta"]],
+                ["Microsoft-CryptoAPI/5.131.2600.3264"] = [$name="Windows", $version=[$major=5, $minor=131, $minor2=2600, $minor3=3264, $addl="XP SP3 RC1"]],
+                ["Microsoft-CryptoAPI/5.131.2600.3282"] = [$name="Windows", $version=[$major=5, $minor=131, $minor2=2600, $minor3=3282, $addl="XP SP3 RC1 Update"]],
+                ["Microsoft-CryptoAPI/5.131.2600.3300"] = [$name="Windows", $version=[$major=5, $minor=131, $minor2=2600, $minor3=3300, $addl="XP SP3 RC2"]],
+                ["Microsoft-CryptoAPI/5.131.2600.3311"] = [$name="Windows", $version=[$major=5, $minor=131, $minor2=2600, $minor3=3311, $addl="XP SP3 RC2 Update"]],
+                ["Microsoft-CryptoAPI/5.131.2600.5508"] = [$name="Windows", $version=[$major=5, $minor=131, $minor2=2600, $minor3=5508, $addl="XP SP3 RC2 Update 2"]],
+                ["Microsoft-CryptoAPI/5.131.2600.5512"] = [$name="Windows", $version=[$major=5, $minor=131, $minor2=2600, $minor3=5512, $addl="XP SP3"]],
+
+                ["Microsoft-CryptoAPI/5.131.3790.0"]    = [$name="Windows", $version=[$major=5, $minor=131, $minor2=3790, $minor3=0,    $addl="XP x64 or Server 2003 SP0"]],
+                ["Microsoft-CryptoAPI/5.131.3790.1830"] = [$name="Windows", $version=[$major=5, $minor=131, $minor2=3790, $minor3=1830, $addl="XP x64 or Server 2003 SP1"]],
+                ["Microsoft-CryptoAPI/5.131.3790.3959"] = [$name="Windows", $version=[$major=5, $minor=131, $minor2=3790, $minor3=3959, $addl="XP x64 or Server 2003 SP2"]],
+                ["Microsoft-CryptoAPI/5.131.3790.5235"] = [$name="Windows", $version=[$major=5, $minor=131, $minor2=3790, $minor3=5235, $addl="XP x64 or Server 2003 with MS13-095"]],
+
+                ["Microsoft-CryptoAPI/6.0"]             = [$name="Windows", $version=[$major=6, $minor=0, $addl="Vista or Server 2008"]],
+                ["Microsoft-CryptoAPI/6.1"]             = [$name="Windows", $version=[$major=6, $minor=1, $addl="7 or Server 2008 R2"]],
+                ["Microsoft-CryptoAPI/6.2"]             = [$name="Windows", $version=[$major=6, $minor=2, $addl="8 or Server 2012"]],
+                ["Microsoft-CryptoAPI/6.3"]             = [$name="Windows", $version=[$major=6, $minor=3, $addl="8.1 or Server 2012 R2"]],
+                ["Microsoft-CryptoAPI/6.4"]             = [$name="Windows", $version=[$major=6, $minor=4, $addl="10 Technical Preview"]],
+        } &redef;
+}
+
+event HTTP::log_http(rec: HTTP::Info) &priority=5
+        {
+        if ( rec?$host && rec?$user_agent && rec$host == "crl.microsoft.com" &&
+             /Microsoft-CryptoAPI\// in rec$user_agent )
+                {
+                if ( rec$user_agent !in crypto_api_mapping )
+                        {
+                        Software::found(rec$id, [$unparsed_version=sub(rec$user_agent, /Microsoft-CryptoAPI/, "Unknown CryptoAPI Version"), $host=rec$id$orig_h, $software_type=WINDOWS]);
+                        }
+                else
+                        {
+                        local result = crypto_api_mapping[rec$user_agent];
+                        Software::found(rec$id, [$version=result$version, $name=result$name, $host=rec$id$orig_h, $software_type=WINDOWS]);
+                        }
+                }
+        }

scripts/policy/protocols/mysql/software.bro

@@ -0,0 +1,20 @@
+##! Software identification and extraction for MySQL traffic.
+
+@load base/frameworks/software
+
+module MySQL;
+
+export {
+	redef enum Software::Type += {
+		## Identifier for MySQL servers in the software framework.
+		SERVER,
+	};
+}
+
+event mysql_server_version(c: connection, ver: string)
+	{
+	if ( ver == "" )
+		return;
+
+	Software::found(c$id, [$unparsed_version=ver, $host=c$id$resp_h, $software_type=SERVER]);
+	}

scripts/policy/protocols/ssl/weak-keys.bro

@@ -22,7 +22,7 @@ export {
 
 	## The minimal key length in bits that is considered to be safe. Any shorter
 	## (non-EC) key lengths will trigger the notice.
-	const notify_minimal_key_length = 1024 &redef;
+	const notify_minimal_key_length = 2048 &redef;
 
 	## Warn if the DH key length is smaller than the certificate key length. This is
 	## potentially unsafe because it gives a wrong impression of safety due to the
@@ -56,7 +56,7 @@ event ssl_established(c: connection) &priority=3
 		NOTICE([$note=Weak_Key,
 			$msg=fmt("Host uses weak certificate with %d bit key", key_length),
 			$conn=c, $suppress_for=1day,
-			$identifier=cat(c$id$orig_h, c$id$orig_p, key_length)
+			$identifier=cat(c$id$resp_h, c$id$resp_h, key_length)
 		]);
 	}
 
@@ -65,13 +65,13 @@ event ssl_dh_server_params(c: connection, p: string, q: string, Ys: string) &pri
 	if ( ! addr_matches_host(c$id$resp_h, notify_weak_keys) )
 		return;
 
-	local key_length = |Ys| * 8; # key length in bits
+	local key_length = |p| * 8; # length of the used prime number in bits
- 
+
 	if ( key_length < notify_minimal_key_length )
 		NOTICE([$note=Weak_Key,
 			$msg=fmt("Host uses weak DH parameters with %d key bits", key_length),
 			$conn=c, $suppress_for=1day,
-			$identifier=cat(c$id$orig_h, c$id$orig_p, key_length)
+			$identifier=cat(c$id$resp_h, c$id$resp_p, key_length)
 		]);
 
 	if ( notify_dh_length_shorter_cert_length &&
@@ -86,7 +86,7 @@ event ssl_dh_server_params(c: connection, p: string, q: string, Ys: string) &pri
 				$msg=fmt("DH key length of %d bits is smaller certificate key length of %d bits",
 					 key_length, c$ssl$cert_chain[0]$x509$certificate$key_length),
 				$conn=c, $suppress_for=1day,
-				$identifier=cat(c$id$orig_h, c$id$orig_p)
+				$identifier=cat(c$id$resp_h, c$id$resp_p)
 			       ]);
 		}
 	}

scripts/policy/tuning/logs-to-elasticsearch.bro

@@ -1,36 +0,0 @@
-##! Load this script to enable global log output to an ElasticSearch database.
-
-module LogElasticSearch;
-
-export {
-	## An elasticsearch specific rotation interval.
-	const rotation_interval = 3hr &redef;
-
-	## Optionally ignore any :bro:type:`Log::ID` from being sent to
-	## ElasticSearch with this script.
-	const excluded_log_ids: set[Log::ID] &redef;
-
-	## If you want to explicitly only send certain :bro:type:`Log::ID` 
-	## streams, add them to this set.  If the set remains empty, all will 
-	## be sent.  The :bro:id:`LogElasticSearch::excluded_log_ids` option
-	## will remain in effect as well.
-	const send_logs: set[Log::ID] &redef;
-}
-
-event bro_init() &priority=-5
-	{
-	if ( server_host == "" )
-		return;
-	
-	for ( stream_id in Log::active_streams )
-		{
-		if ( stream_id in excluded_log_ids ||
-		     (|send_logs| > 0 && stream_id !in send_logs) )
-			next;
-
-		local filter: Log::Filter = [$name = "default-es",
-		                             $writer = Log::WRITER_ELASTICSEARCH,
-		                             $interv = LogElasticSearch::rotation_interval];
-		Log::add_filter(stream_id, filter);
-		}
-	}

scripts/test-all-policy.bro

@@ -32,6 +32,7 @@
 @load frameworks/packet-filter/shunt.bro
 @load frameworks/software/version-changes.bro
 @load frameworks/software/vulnerable.bro
+@load frameworks/software/windows-version-detection.bro
 @load integration/barnyard2/__load__.bro
 @load integration/barnyard2/main.bro
 @load integration/barnyard2/types.bro
@@ -75,6 +76,7 @@
 @load protocols/http/var-extraction-uri.bro
 @load protocols/modbus/known-masters-slaves.bro
 @load protocols/modbus/track-memmap.bro
+@load protocols/mysql/software.bro
 @load protocols/smtp/blocklists.bro
 @load protocols/smtp/detect-suspicious-orig.bro
 @load protocols/smtp/entities-excerpt.bro
@@ -98,7 +100,4 @@
 @load tuning/defaults/packet-fragments.bro
 @load tuning/defaults/warnings.bro
 @load tuning/json-logs.bro
-@load tuning/logs-to-elasticsearch.bro
 @load tuning/track-all-assets.bro
-
-redef LogElasticSearch::server_host = "";

src/Attr.cc

@@ -265,6 +265,14 @@ void Attributes::CheckAttr(Attr* a)
 				// Ok.
 				break;
 
+			Expr* e = a->AttrExpr();
+			if ( check_and_promote_expr(e, type) )
+				{
+				a->SetAttrExpr(e);
+				// Ok.
+				break;
+				}
+
 			a->AttrExpr()->Error("&default value has inconsistent type", type);
 			}
 
@@ -297,8 +305,11 @@ void Attributes::CheckAttr(Attr* a)
 
 				Expr* e = a->AttrExpr();
 				if ( check_and_promote_expr(e, ytype) )
+					{
+					a->SetAttrExpr(e);
 					// Ok.
 					break;
+					}
 
 				Error("&default value has inconsistent type 2");
 				}

src/Attr.h

@@ -45,6 +45,13 @@ public:
 	attr_tag Tag() const	{ return tag; }
 	Expr* AttrExpr() const	{ return expr; }
 
+	// Up to the caller to decide if previous expr can be unref'd since it may
+	// not always be safe; e.g. expressions (at time of writing) don't always
+	// keep careful track of referencing their operands, so doing something
+	// like SetAttrExpr(coerce(AttrExpr())) must not completely unref the
+	// previous expr as the new expr depends on it.
+	void SetAttrExpr(Expr* e) { expr = e; }
+
 	int RedundantAttrOkay() const
 		{ return tag == ATTR_REDEF || tag == ATTR_OPTIONAL; }
 

src/Brofiler.cc

@@ -96,7 +96,7 @@ bool Brofiler::WriteStats()
 	map<pair<string, string>, uint64 >::const_iterator it;
 	for ( it = usage_map.begin(); it != usage_map.end(); ++it )
 		{
-		fprintf(f, "%"PRIu64"%c%s%c%s\n", it->second, delim,
+		fprintf(f, "%" PRIu64"%c%s%c%s\n", it->second, delim,
 				it->first.first.c_str(), delim, it->first.second.c_str());
 		}
 

src/CMakeLists.txt

@@ -7,7 +7,8 @@ include_directories(BEFORE
 set(bro_ALL_GENERATED_OUTPUTS  CACHE INTERNAL "automatically generated files" FORCE)
 
 # This collects bif inputs that we'll load automatically.
-set(bro_AUTO_BIFS CACHE INTERNAL "BIFs for automatic inclusion" FORCE)
+set(bro_AUTO_BIFS     CACHE INTERNAL "BIFs for automatic inclusion" FORCE)
+set(bro_REGISTER_BIFS CACHE INTERNAL "BIFs for automatic registering" FORCE)
 
 set(bro_BASE_BIF_SCRIPTS CACHE INTERNAL "Bro script stubs for BIFs in base distribution of Bro" FORCE)
 set(bro_PLUGIN_BIF_SCRIPTS CACHE INTERNAL "Bro script stubs for BIFs in Bro plugins" FORCE)
@@ -47,7 +48,7 @@ set(BISON_FLAGS "--debug")
 bison_target(BIFParser builtin-func.y
              ${CMAKE_CURRENT_BINARY_DIR}/bif_parse.cc
              HEADER ${CMAKE_CURRENT_BINARY_DIR}/bif_parse.h
-             VERBOSE ${CMAKE_CURRENT_BINARY_DIR}/bif_parse.output
+             #VERBOSE ${CMAKE_CURRENT_BINARY_DIR}/bif_parse.output
              COMPILE_FLAGS "${BISON_FLAGS}")
 flex_target(BIFScanner builtin-func.l ${CMAKE_CURRENT_BINARY_DIR}/bif_lex.cc)
 add_flex_bison_dependency(BIFScanner BIFParser)
@@ -56,7 +57,7 @@ add_flex_bison_dependency(BIFScanner BIFParser)
 bison_target(RuleParser rule-parse.y
              ${CMAKE_CURRENT_BINARY_DIR}/rup.cc
              HEADER ${CMAKE_CURRENT_BINARY_DIR}/rup.h
-             VERBOSE ${CMAKE_CURRENT_BINARY_DIR}/rule_parse.output
+             #VERBOSE ${CMAKE_CURRENT_BINARY_DIR}/rule_parse.output
              COMPILE_FLAGS "${BISON_FLAGS}")
 replace_yy_prefix_target(${CMAKE_CURRENT_BINARY_DIR}/rup.cc
                          ${CMAKE_CURRENT_BINARY_DIR}/rule-parse.cc
@@ -71,7 +72,7 @@ flex_target(RuleScanner rule-scan.l ${CMAKE_CURRENT_BINARY_DIR}/rule-scan.cc
 bison_target(REParser re-parse.y
              ${CMAKE_CURRENT_BINARY_DIR}/rep.cc
              HEADER ${CMAKE_CURRENT_BINARY_DIR}/re-parse.h
-             VERBOSE ${CMAKE_CURRENT_BINARY_DIR}/re_parse.output
+             #VERBOSE ${CMAKE_CURRENT_BINARY_DIR}/re_parse.output
              COMPILE_FLAGS "${BISON_FLAGS}")
 replace_yy_prefix_target(${CMAKE_CURRENT_BINARY_DIR}/rep.cc
                          ${CMAKE_CURRENT_BINARY_DIR}/re-parse.cc
@@ -84,7 +85,7 @@ add_flex_bison_dependency(REScanner REParser)
 bison_target(Parser parse.y
              ${CMAKE_CURRENT_BINARY_DIR}/p.cc
              HEADER ${CMAKE_CURRENT_BINARY_DIR}/broparse.h
-             VERBOSE ${CMAKE_CURRENT_BINARY_DIR}/parse.output
+             #VERBOSE ${CMAKE_CURRENT_BINARY_DIR}/parse.output
              COMPILE_FLAGS "${BISON_FLAGS}")
 replace_yy_prefix_target(${CMAKE_CURRENT_BINARY_DIR}/p.cc
                          ${CMAKE_CURRENT_BINARY_DIR}/parse.cc
@@ -117,8 +118,6 @@ include(BifCl)
 
 set(BIF_SRCS
     bro.bif
-    logging.bif
-    input.bif
     event.bif
     const.bif
     types.bif
@@ -155,21 +154,25 @@ set(bro_SUBDIR_LIBS CACHE INTERNAL "subdir libraries" FORCE)
 set(bro_PLUGIN_LIBS CACHE INTERNAL "plugin libraries" FORCE)
 
 add_subdirectory(analyzer)
-add_subdirectory(file_analysis)
-add_subdirectory(probabilistic)
 add_subdirectory(broxygen)
+add_subdirectory(file_analysis)
+add_subdirectory(input)
+add_subdirectory(iosource)
+add_subdirectory(logging)
+add_subdirectory(probabilistic)
 
 set(bro_SUBDIRS
-    ${bro_SUBDIR_LIBS}
+    # Order is important here.
     ${bro_PLUGIN_LIBS}
+    ${bro_SUBDIR_LIBS}
 )
 
 if ( NOT bro_HAVE_OBJECT_LIBRARIES )
     foreach (_plugin ${bro_PLUGIN_LIBS})
         string(REGEX REPLACE "plugin-" "" _plugin "${_plugin}")
         string(REGEX REPLACE "-" "_" _plugin "${_plugin}")
-        set(_decl "namespace plugin { namespace ${_plugin} { class Plugin; extern Plugin __plugin; } };")
+        set(_decl "namespace plugin { namespace ${_plugin} { class Plugin; extern Plugin plugin; } };")
-        set(_use "i += (size_t)(&(plugin::${_plugin}::__plugin));")
+        set(_use "i += (size_t)(&(plugin::${_plugin}::plugin));")
         set(__BRO_DECL_PLUGINS "${__BRO_DECL_PLUGINS}${_decl}\n")
         set(__BRO_USE_PLUGINS "${__BRO_USE_PLUGINS}${_use}\n")
     endforeach()
@@ -252,7 +255,6 @@ set(bro_SRCS
     Anon.cc
     Attr.cc
     Base64.cc
-    BPF_Program.cc
     Brofiler.cc
     BroString.cc
     CCL.cc
@@ -277,14 +279,13 @@ set(bro_SRCS
     EventRegistry.cc
     Expr.cc
     File.cc
-    FlowSrc.cc
+    Flare.cc
     Frag.cc
     Frame.cc
     Func.cc
     Hash.cc
     ID.cc
     IntSet.cc
-    IOSource.cc
     IP.cc
     IPAddr.cc
     List.cc
@@ -297,7 +298,7 @@ set(bro_SRCS
     OSFinger.cc
     PacketFilter.cc
     PersistenceSerializer.cc
-    PktSrc.cc
+    Pipe.cc
     PolicyFile.cc
     PrefixTable.cc
     PriorityQueue.cc
@@ -346,24 +347,6 @@ set(bro_SRCS
     threading/formatters/Ascii.cc
     threading/formatters/JSON.cc
 
-    logging/Manager.cc
-    logging/WriterBackend.cc
-    logging/WriterFrontend.cc
-    logging/writers/Ascii.cc
-    logging/writers/DataSeries.cc
-    logging/writers/SQLite.cc
-    logging/writers/ElasticSearch.cc
-    logging/writers/None.cc
-
-    input/Manager.cc
-    input/ReaderBackend.cc
-    input/ReaderFrontend.cc
-    input/readers/Ascii.cc
-    input/readers/Raw.cc
-    input/readers/Benchmark.cc
-    input/readers/Binary.cc
-    input/readers/SQLite.cc
-
     3rdparty/sqlite3.c
 
     plugin/Component.cc
@@ -371,7 +354,6 @@ set(bro_SRCS
     plugin/TaggedComponent.h
     plugin/Manager.cc
     plugin/Plugin.cc
-    plugin/Macros.h
 
     nb_dns.c
     digest.h
@@ -387,22 +369,31 @@ else ()
     target_link_libraries(bro ${bro_SUBDIRS} ${brodeps} ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS})
 endif ()
 
+if ( NOT "${bro_LINKER_FLAGS}" STREQUAL "" )
+    set_target_properties(bro PROPERTIES LINK_FLAGS "${bro_LINKER_FLAGS}")
+endif ()
+
 install(TARGETS bro DESTINATION bin)
 
 set(BRO_EXE bro
     CACHE STRING "Bro executable binary" FORCE)
 
+set(BRO_EXE_PATH ${CMAKE_CURRENT_BINARY_DIR}/bro
+    CACHE STRING "Path to Bro executable binary" FORCE)
+
 # Target to create all the autogenerated files.
 add_custom_target(generate_outputs_stage1)
 add_dependencies(generate_outputs_stage1 ${bro_ALL_GENERATED_OUTPUTS})
 
 # Target to create the joint includes files that pull in the bif code.
-bro_bif_create_includes(generate_outputs_stage2 ${CMAKE_CURRENT_BINARY_DIR} "${bro_AUTO_BIFS}")
+bro_bif_create_includes(generate_outputs_stage2a ${CMAKE_CURRENT_BINARY_DIR} "${bro_AUTO_BIFS}")
-add_dependencies(generate_outputs_stage2 generate_outputs_stage1)
+bro_bif_create_register(generate_outputs_stage2b ${CMAKE_CURRENT_BINARY_DIR} "${bro_REGISTER_BIFS}")
+add_dependencies(generate_outputs_stage2a generate_outputs_stage1)
+add_dependencies(generate_outputs_stage2b generate_outputs_stage1)
 
 # Global target to trigger creation of autogenerated code.
 add_custom_target(generate_outputs)
-add_dependencies(generate_outputs generate_outputs_stage2)
+add_dependencies(generate_outputs generate_outputs_stage2a generate_outputs_stage2b)
 
 # Build __load__.bro files for standard *.bif.bro.
 bro_bif_create_loader(bif_loader "${bro_BASE_BIF_SCRIPTS}")

src/ChunkedIO.cc

@@ -210,6 +210,7 @@ bool ChunkedIOFd::WriteChunk(Chunk* chunk, bool partial)
 	else
 		pending_head = pending_tail = q;
 
+	write_flare.Fire();
 	return Flush();
 	}
 
@@ -232,6 +233,7 @@ bool ChunkedIOFd::PutIntoWriteBuffer(Chunk* chunk)
 	write_len += len;
 
 	delete chunk;
+	write_flare.Fire();
 
 	if ( network_time - last_flush > 0.005 )
 		FlushWriteBuffer();
@@ -269,6 +271,10 @@ bool ChunkedIOFd::FlushWriteBuffer()
 		if ( unsigned(written) == len )
 			{
 			write_pos = write_len = 0;
+
+			if ( ! pending_head )
+				write_flare.Extinguish();
+
 			return true;
 			}
 
@@ -318,7 +324,12 @@ bool ChunkedIOFd::Flush()
 			}
 		}
 
-	return FlushWriteBuffer();
+	bool rval = FlushWriteBuffer();
+
+	if ( ! pending_head && write_len == 0 )
+		write_flare.Extinguish();
+
+	return rval;
 	}
 
 uint32 ChunkedIOFd::ChunkAvailable()
@@ -394,6 +405,9 @@ bool ChunkedIOFd::Read(Chunk** chunk, bool may_block)
 #ifdef DEBUG_COMMUNICATION
 		AddToBuffer("<false:read-chunk>", true);
 #endif
+		if ( ! ChunkAvailable() )
+			read_flare.Extinguish();
+
 		return false;
 		}
 
@@ -402,9 +416,15 @@ bool ChunkedIOFd::Read(Chunk** chunk, bool may_block)
 #ifdef DEBUG_COMMUNICATION
 		AddToBuffer("<null:no-data>", true);
 #endif
+		read_flare.Extinguish();
 		return true;
 		}
 
+	if ( ChunkAvailable() )
+		read_flare.Fire();
+	else
+		read_flare.Extinguish();
+
 #ifdef DEBUG
 	if ( *chunk )
 		DBG_LOG(DBG_CHUNKEDIO, "read of size %d %s[%s]",
@@ -481,6 +501,9 @@ bool ChunkedIOFd::ReadChunk(Chunk** chunk, bool may_block)
 	read_pos = 0;
 	read_len = bytes_left;
 
+	if ( ! ChunkAvailable() )
+		read_flare.Extinguish();
+
 	// If allowed, wait a bit for something to read.
 	if ( may_block )
 		{
@@ -607,6 +630,14 @@ bool ChunkedIOFd::IsFillingUp()
 	return stats.pending > MAX_BUFFERED_CHUNKS_SOFT;
 	}
 
+iosource::FD_Set ChunkedIOFd::ExtraReadFDs() const
+	{
+	iosource::FD_Set rval;
+	rval.Insert(write_flare.FD());
+	rval.Insert(read_flare.FD());
+	return rval;
+	}
+
 void ChunkedIOFd::Clear()
 	{
 	while ( pending_head )
@@ -618,6 +649,9 @@ void ChunkedIOFd::Clear()
 		}
 
 	pending_head = pending_tail = 0;
+
+	if ( write_len == 0 )
+		write_flare.Extinguish();
 	}
 
 const char* ChunkedIOFd::Error()
@@ -830,6 +864,7 @@ bool ChunkedIOSSL::Write(Chunk* chunk)
 	else
 		write_head = write_tail = q;
 
+	write_flare.Fire();
 	Flush();
 	return true;
 	}
@@ -935,6 +970,7 @@ bool ChunkedIOSSL::Flush()
 		write_state = LEN;
 		}
 
+	write_flare.Extinguish();
 	return true;
 	}
 
@@ -1104,6 +1140,13 @@ bool ChunkedIOSSL::IsFillingUp()
 	return false;
 	}
 
+iosource::FD_Set ChunkedIOSSL::ExtraReadFDs() const
+	{
+	iosource::FD_Set rval;
+	rval.Insert(write_flare.FD());
+	return rval;
+	}
+
 void ChunkedIOSSL::Clear()
 	{
 	while ( write_head )
@@ -1114,6 +1157,7 @@ void ChunkedIOSSL::Clear()
 		write_head = next;
 		}
 	write_head = write_tail = 0;
+	write_flare.Extinguish();
 	}
 
 const char* ChunkedIOSSL::Error()

src/ChunkedIO.h

@@ -6,7 +6,8 @@
 #include "config.h"
 #include "List.h"
 #include "util.h"
-
+#include "Flare.h"
+#include "iosource/FD_Set.h"
 #include <list>
 
 #ifdef NEED_KRB5_H
@@ -95,6 +96,11 @@ public:
 	// Returns underlying fd if available, -1 otherwise.
 	virtual int Fd()	{ return -1; }
 
+	// Returns supplementary file descriptors that become read-ready in order
+	// to signal that there is some work that can be performed.
+	virtual iosource::FD_Set ExtraReadFDs() const
+		{ return iosource::FD_Set(); }
+
 	// Makes sure that no additional protocol data is written into
 	// the output stream.  If this is activated, the output cannot
 	// be read again by any of these classes!
@@ -177,6 +183,7 @@ public:
 	virtual void Clear();
 	virtual bool Eof()	{ return eof; }
 	virtual int Fd()	{ return fd; }
+	virtual iosource::FD_Set ExtraReadFDs() const;
 	virtual void Stats(char* buffer, int length);
 
 private:
@@ -240,6 +247,8 @@ private:
 	ChunkQueue* pending_tail;
 
 	pid_t pid;
+	bro::Flare write_flare;
+	bro::Flare read_flare;
 };
 
 // Chunked I/O using an SSL connection.
@@ -262,6 +271,7 @@ public:
 	virtual void Clear();
 	virtual bool Eof()	{ return eof; }
 	virtual int Fd()	{ return socket; }
+	virtual iosource::FD_Set ExtraReadFDs() const;
 	virtual void Stats(char* buffer, int length);
 
 private:
@@ -303,6 +313,8 @@ private:
 
 	// One SSL for all connections.
 	static SSL_CTX* ctx;
+
+	bro::Flare write_flare;
 };
 
 #include <zlib.h>
@@ -328,6 +340,8 @@ public:
 
 	virtual bool Eof()	{ return io->Eof(); }
 	virtual int Fd()	{ return io->Fd(); }
+	virtual iosource::FD_Set ExtraReadFDs() const
+		{ return io->ExtraReadFDs(); }
 	virtual void Stats(char* buffer, int length);
 
 	void EnableCompression(int level)

src/DNS_Mgr.cc

@@ -35,6 +35,7 @@
 #include "Net.h"
 #include "Var.h"
 #include "Reporter.h"
+#include "iosource/Manager.h"
 
 extern "C" {
 extern int select(int, fd_set *, fd_set *, fd_set *, struct timeval *);
@@ -213,7 +214,7 @@ DNS_Mapping::DNS_Mapping(FILE* f)
 	char req_buf[512+1], name_buf[512+1];
 	int is_req_host;
 
-	if ( sscanf(buf, "%lf %d %512s %d %512s %d %d %"PRIu32, &creation_time,
+	if ( sscanf(buf, "%lf %d %512s %d %512s %d %d %" PRIu32, &creation_time,
 	     &is_req_host, req_buf, &failed, name_buf, &map_type, &num_addrs,
 	     &req_ttl) != 8 )
 		return;
@@ -359,7 +360,7 @@ void DNS_Mapping::Clear()
 
 void DNS_Mapping::Save(FILE* f) const
 	{
-	fprintf(f, "%.0f %d %s %d %s %d %d %"PRIu32"\n", creation_time, req_host != 0,
+	fprintf(f, "%.0f %d %s %d %s %d %d %" PRIu32"\n", creation_time, req_host != 0,
 		req_host ? req_host : req_addr.AsString().c_str(),
 		failed, (names && names[0]) ? names[0] : "*",
 		map_type, num_addrs, req_ttl);
@@ -404,17 +405,17 @@ DNS_Mgr::~DNS_Mgr()
 	delete [] dir;
 	}
 
-bool DNS_Mgr::Init()
+void DNS_Mgr::InitPostScript()
 	{
 	if ( did_init )
-		return true;
+		return;
 
 	const char* cache_dir = dir ? dir : ".";
 
 	if ( mode == DNS_PRIME && ! ensure_dir(cache_dir) )
 		{
 		did_init = 0;
-		return false;
+		return;
 		}
 
 	cache_name = new char[strlen(cache_dir) + 64];
@@ -433,14 +434,12 @@ bool DNS_Mgr::Init()
 
 	did_init = 1;
 
-	io_sources.Register(this, true);
+	iosource_mgr->Register(this, true);
 
 	// We never set idle to false, having the main loop only calling us from
 	// time to time. If we're issuing more DNS requests than we can handle
 	// in this way, we are having problems anyway ...
-	idle = true;
+	SetIdle(true);
-
-	return true;
 	}
 
 static TableVal* fake_name_lookup_result(const char* name)
@@ -1217,9 +1216,10 @@ void DNS_Mgr::IssueAsyncRequests()
 		}
 	}
 
-void  DNS_Mgr::GetFds(int* read, int* write, int* except)
+void DNS_Mgr::GetFds(iosource::FD_Set* read, iosource::FD_Set* write,
+                     iosource::FD_Set* except)
 	{
-	*read = nb_dns_fd(nb_dns);
+	read->Insert(nb_dns_fd(nb_dns));
 	}
 
 double DNS_Mgr::NextTimestamp(double* network_time)

src/DNS_Mgr.h

@@ -12,7 +12,7 @@
 #include "BroList.h"
 #include "Dict.h"
 #include "EventHandler.h"
-#include "IOSource.h"
+#include "iosource/IOSource.h"
 #include "IPAddr.h"
 
 class Val;
@@ -40,12 +40,12 @@ enum DNS_MgrMode {
 // Number of seconds we'll wait for a reply.
 #define DNS_TIMEOUT 5
 
-class DNS_Mgr : public IOSource {
+class DNS_Mgr : public iosource::IOSource {
 public:
 	DNS_Mgr(DNS_MgrMode mode);
 	virtual ~DNS_Mgr();
 
-	bool Init();
+	void InitPostScript();
 	void Flush();
 
 	// Looks up the address or addresses of the given host, and returns
@@ -132,7 +132,8 @@ protected:
 	void DoProcess(bool flush);
 
 	// IOSource interface.
-	virtual void GetFds(int* read, int* write, int* except);
+	virtual void GetFds(iosource::FD_Set* read, iosource::FD_Set* write,
+	                    iosource::FD_Set* except);
 	virtual double NextTimestamp(double* network_time);
 	virtual void Process();
 	virtual const char* Tag()	{ return "DNS_Mgr"; }

src/DebugLogger.cc

@@ -5,6 +5,7 @@
 
 #include "DebugLogger.h"
 #include "Net.h"
+#include "plugin/Plugin.h"
 
 DebugLogger debug_logger("debug");
 
@@ -15,9 +16,10 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = {
 	{ "compressor", 0, false }, {"string", 0, false },
 	{ "notifiers", 0, false },  { "main-loop", 0, false },
 	{ "dpd", 0, false }, { "tm", 0, false },
-	{ "logging", 0, false }, {"input", 0, false }, 
+	{ "logging", 0, false }, {"input", 0, false },
 	{ "threading", 0, false }, { "file_analysis", 0, false },
-	{ "plugins", 0, false }, { "broxygen", 0, false }
+	{ "plugins", 0, false }, { "broxygen", 0, false },
+	{ "pktio", 0, false}
 };
 
 DebugLogger::DebugLogger(const char* filename)
@@ -73,10 +75,12 @@ void DebugLogger::EnableStreams(const char* s)
 			{
 			if ( strcasecmp("verbose", tok) == 0 )
 				verbose = true;
-			else
+			else if ( strncmp(tok, "plugin-", 7) != 0 )
 				reporter->FatalError("unknown debug stream %s\n", tok);
 			}
 
+		enabled_streams.insert(tok);
+
 		tok = strtok(0, ",");
 		}
 
@@ -105,4 +109,24 @@ void DebugLogger::Log(DebugStream stream, const char* fmt, ...)
 	fflush(file);
 	}
 
+void DebugLogger::Log(const plugin::Plugin& plugin, const char* fmt, ...)
+	{
+	string tok = string("plugin-") + plugin.Name();
+	tok = strreplace(tok, "::", "-");
+
+	if ( enabled_streams.find(tok) == enabled_streams.end() )
+		return;
+
+	fprintf(file, "%17.06f/%17.06f [plugin %s] ",
+			network_time, current_time(true), plugin.Name().c_str());
+
+	va_list ap;
+	va_start(ap, fmt);
+	vfprintf(file, fmt, ap);
+	va_end(ap);
+
+	fputc('\n', file);
+	fflush(file);
+	}
+
 #endif

src/DebugLogger.h

@@ -7,6 +7,8 @@
 #ifdef DEBUG
 
 #include <stdio.h>
+#include <string>
+#include <set>
 
 // To add a new debugging stream, add a constant here as well as
 // an entry to DebugLogger::streams in DebugLogger.cc.
@@ -27,8 +29,9 @@ enum DebugStream {
 	DBG_INPUT,	// Input streams
 	DBG_THREADING,	// Threading system
 	DBG_FILE_ANALYSIS,	// File analysis
-	DBG_PLUGINS,
+	DBG_PLUGINS,	// Plugin system
-	DBG_BROXYGEN,
+	DBG_BROXYGEN,	// Broxygen
+	DBG_PKTIO,	// Packet sources and dumpers.
 
 	NUM_DBGS // Has to be last
 };
@@ -42,6 +45,10 @@ enum DebugStream {
 #define DBG_PUSH(stream) debug_logger.PushIndent(stream)
 #define DBG_POP(stream) debug_logger.PopIndent(stream)
 
+#define PLUGIN_DBG_LOG(plugin, args...) debug_logger.Log(plugin, args)
+
+namespace plugin { class Plugin; }
+
 class DebugLogger {
 public:
 	// Output goes to stderr per default.
@@ -49,6 +56,7 @@ public:
 	~DebugLogger();
 
 	void Log(DebugStream stream, const char* fmt, ...);
+	void Log(const plugin::Plugin& plugin, const char* fmt, ...);
 
 	void PushIndent(DebugStream stream)
 		{ ++streams[int(stream)].indent; }
@@ -79,6 +87,8 @@ private:
 		bool enabled;
 	};
 
+	std::set<std::string> enabled_streams;
+
 	static Stream streams[NUM_DBGS];
 };
 
@@ -89,6 +99,7 @@ extern DebugLogger debug_logger;
 #define DBG_LOG_VERBOSE(args...)
 #define DBG_PUSH(stream)
 #define DBG_POP(stream)
+#define PLUGIN_DBG_LOG(plugin, args...)
 #endif
 
 #endif

src/Event.cc

@@ -6,6 +6,7 @@
 #include "Func.h"
 #include "NetVar.h"
 #include "Trigger.h"
+#include "plugin/Manager.h"
 
 EventMgr mgr;
 
@@ -77,6 +78,11 @@ EventMgr::~EventMgr()
 
 void EventMgr::QueueEvent(Event* event)
 	{
+	bool done = PLUGIN_HOOK_WITH_RESULT(HOOK_QUEUE_EVENT, HookQueueEvent(event), false);
+
+	if ( done )
+		return;
+
 	if ( ! head )
 		head = tail = event;
 	else
@@ -115,6 +121,8 @@ void EventMgr::Drain()
 
 	SegmentProfiler(segment_logger, "draining-events");
 
+	PLUGIN_HOOK_VOID(HOOK_DRAIN_EVENTS, HookDrainEvents());
+
 	draining = true;
 	while ( head )
 		Dispatch();