Merge remote-tracking branch 'origin/master' into topic/seth/analyzer-framework

Conflicts: scripts/base/init-default.bro scripts/base/protocols/dns/main.bro scripts/base/protocols/ftp/main.bro scripts/base/protocols/http/main.bro scripts/base/protocols/irc/main.bro scripts/base/protocols/smtp/main.bro scripts/base/protocols/ssh/main.bro scripts/base/protocols/ssl/main.bro scripts/base/protocols/syslog/main.bro src/main.cc testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
2025-10-06 16:48:19 +00:00 · 2013-07-04 23:07:52 -04:00 · 2013-07-04 23:07:52 -04:00 · 5f8ee93ef0
commit 5f8ee93ef0
parent 3546d93f36 fefef47f30
1249 changed files with 267087 additions and 176962 deletions
--- a/726
+++ b/726
@ -1,4 +1,730 @@

+2.1-784 | 2013-07-04 22:28:48 -0400
+
+  * Add a call to lookup_connection in SSH scripts to update connval. (Seth Hall)
+
+  * Updating submodule(s). (Robin Sommer)
+
+2.1-782 | 2013-07-03 17:00:39 -0700
+
+  * Remove the SSL log queueing mechanism that was included with the
+    log delay mechanism. (Seth Hall)
+
+2.1-780 | 2013-07-03 16:46:26 -0700
+
+  * Rewrite of the RAW input reader for improved robustness and new
+    features. (Bernhard Amann) This includes:
+
+        - Send "end_of_data" event for all kind of streams.
+        - Send "process_finished" event with exit code of child
+          process at process termination.
+        - Expose name of input stream to readers.
+        - Better error handling.
+        - New "force_kill" option which SIGKILLs processes on reader termination.
+        - Supports reading from stdout and stderr simultaneously.
+        - Support sending data to stdin of child process.
+        - Streaming reads from external commands work without blocking.
+
+2.1-762 | 2013-07-03 16:33:22 -0700
+
+  * Fix to correct support for TLS 1.2. Addresses #1020. (Seth Hall,
+    with help from Rafal Lesniak).
+
+2.1-760 | 2013-07-03 16:31:36 -0700
+
+  * Teach broxygen to generate protocol analyzer plugin reference.
+    (Jon Siwek)
+
+  * Adding 'const' to a number of C++ methods. (Jon Siwek)
+
+2.1-757 | 2013-07-03 16:28:10 -0700
+
+  * Fix redef of table index from clearing table.
+
+    `redef foo["x"] = 1` now acts like `redef foo += { ["x"] = 1 }`
+    instead of `redef foo = { ["x"] = 1 }`.
+
+    Addresses #1013. (Jon Siwek)
+
+
+2.1-755 | 2013-07-03 16:22:43 -0700
+
+  * Add a general file analysis overview/how-to document. (Jon Siwek)
+
+  * Improve file analysis doxygen comments. (Jon Siwek)
+
+  * Improve tracking of HTTP file extraction. http.log now has files
+    taken from request and response bodies in different fields for
+    each, and can now track multiple files per body. That is, the
+    "extraction_file" field is now "extracted_request_files" and
+    "extracted_response_files". Addresses #988. (Jon Siwek)
+
+  * Fix HTTP multipart body file analysis. Each part now gets assigned
+    a different file handle/id. (Jon Siwek)
+
+  * Remove logging of analyzers field of FileAnalysis::Info. (Jon
+    Siwek)
+
+  * Remove extraction counter in default file extraction scripts. (Jon
+    Siwek)
+
+  * Remove FileAnalysis::postpone_timeout.
+    FileAnalysis::set_timeout_interval can now perform same function.
+    (Jon Siwek)
+
+  * Make default get_file_handle handlers &priority=5 so they're
+    easier to override. (Jon Siwek)
+
+  * Add input interface to forward data for file analysis. The new
+    Input::add_analysis function is used to automatically forward
+    input data on to the file analysis framework. (Jon Siwek)
+
+  * File analysis framework interface simplifications. (Jon Siwek)
+    
+    - Remove script-layer data input interface (will be managed directly
+      by input framework later).
+
+    - Only track files internally by file id hash.  Chance of collision
+      too small to justify also tracking unique file string. 
+
+
+2.1-741 | 2013-06-07 17:28:50 -0700
+
+  * Fixing typo that could cause an assertion to falsely trigger.
+    (Robin Sommer)
+
+2.1-740 | 2013-06-07 16:37:32 -0700
+
+  * Fix for CMake 2.6.x. (Robin Sommer)
+
+2.1-738 | 2013-06-07 08:38:13 -0700
+
+  * Remove invalid free on non-allocated pointer in hash function
+    object. Addresses #1018. (Matthias Vallentin)
+
+2.1-736 | 2013-06-06 10:05:20 -0700
+
+  * New "magic constants" @DIR and @FILENAME that expand to the
+    directory path of the current script and just the script file name
+    without path, respectively. (Jon Siwek)
+
+2.1-731 | 2013-06-04 21:19:08 -0700
+
+  * Reorginization of internal protocol analyzer code. We're moving
+    them to a modularized structure, based on a plugin model. Along
+    with this change comes generic plugin infrastructure that we'll
+    later extend to other Bro component as well. For now all plugins
+    are compiled in statically, but in the future we plan to also
+    enable dynamic loading at run time. (Robin Sommer)
+
+  * Ignoring file ids in external tests. (Robin Sommer)
+
+2.1-675 | 2013-06-02 20:03:19 -0700
+
+  * Fix a compiler warning. (Robin Sommer)
+
+  * Allow named vector/set/table/record constructors. Addresses #983.
+    (Jon Siwek)
+
+  * Adding Makefile target test-all that also runs the BroControl test
+    suite.  (Robin Sommer)
+
+2.1-664 | 2013-05-28 21:37:46 -0700
+
+  * Dangling pointer fix. Addresses #1004. (Jon Siwek)
+
+2.1-659 | 2013-05-24 17:24:18 -0700
+
+  * Fix broken/missing documentation. (Jon Siwek)
+    
+  * Fixing test that would fail without ES/curl support. (Robin
+    Sommer)
+
+2.1-656 | 2013-05-17 15:58:07 -0700
+
+  * Fix mutex lock problem for writers. (Bernhard Amann)
+    
+2.1-654 | 2013-05-17 13:49:52 -0700
+
+  * Tweaks to sqlite3 configuration to address threading issues.
+    (Bernhard Amann)
+
+2.1-651 | 2013-05-17 13:37:16 -0700
+
+  * Fix uninitialized DPM member. (Jon Siwek)
+
+  * Fix issue with transaction ID reuse in a single DNS connection. (Seth Hall)
+
+  * New function added to the queue.bro script to support peeking at
+    the new gettable item in the queue without removing it. (Seth Hall)
+
+2.1-647 | 2013-05-17 07:47:14 -0700
+
+  * Fixing Broxygen generation to have BROMAGIC set. (Robin Sommer)
+    
+  * Fix for 'fchmod undeclared here' on FreeBSD. (Robin Sommer)
+    
+  * CMake policy fix to avoid errors with older versions. (Robin
+    Sommer)
+
+2.1-641 | 2013-05-15 18:15:09 -0700
+
+  * Test update. (Robin Sommer)
+
+2.1-640 | 2013-05-15 17:24:09 -0700
+
+  * Support for cleaning up threads that have terminated. (Bernhard
+    Amann and Robin Sommer). Includes:
+
+      - Both logging and input frameworks now clean up threads once
+        they aren't further needed anymnore.
+
+      - New function Log::remove_stream() that removes a logging
+        stream, stopping all writer threads that are associated with
+        it. Note, however, that removing a *filter* from a stream
+        still doesn't clean up any threads. The problem is that
+        because of the output paths potentially being created
+        dynamically it's unclear if the writer thread will still be
+        needed in the future.
+
+2.1-626 | 2013-05-15 16:09:31 -0700
+
+  * Add "reservoir" sampler for SumStats framework. This maintains
+    a set of N uniquely distributed random samples. (Bernhard Amann)
+
+2.1-619 | 2013-05-15 16:01:42 -0700
+
+  * SQLite reader and writer combo. This allows to read/write
+    persistent data from on disk SQLite databases. The current
+    interface is quite low-level, we'll add higher-level abstractions
+    in the future. (Bernhard Amann)
+
+2.1-576 | 2013-05-15 14:29:09 -0700
+
+  * Initial version of new file analysis framework. This moves most of
+    the processing of file content from script-land into the core,
+    where it belongs. Much of this is an internal change, and at this
+    point the new code has essentially feature-equality with the old
+    one. More script-level changes to come. (Jon Siwek)
+
+2.1-502 | 2013-05-10 19:29:37 -0700
+
+  * Allow default function/hook/event parameters. Addresses #972. (Jon
+    Siwek)
+
+  * Change the endianness parameter of bytestring_to_count() BIF to
+    default to false (big endian). (Jon Siwek)
+
+2.1-500 | 2013-05-10 19:22:24 -0700
+
+  * Fix to prevent merge-hook of SumStat's unique plugin from damaging
+    source data. (Bernhard Amann)
+
+2.1-498 | 2013-05-03 17:44:08 -0700
+
+  * Table lookups return copy of non-const &default vals. This
+    prevents unintentional modifications to the &default value itself.
+    Addresses #981.  (Jon Siwek)
+
+2.1-496 | 2013-05-03 15:54:47 -0700
+
+  * Fix memory leak and unnecessary allocations in OpaqueVal.
+    Addresses #986. (Matthias Vallentin)
+
+2.1-492 | 2013-05-02 12:46:26 -0700
+
+  * Work-around for sumstats framework not propagating updates after
+    intermediate check in cluster environments. (Bernhard Amann)
+
+  * Always apply tcp_connection_attempt. Before this change it was
+    only applied when a connection_attempt() event handler was
+    defined. (Robin Sommer)
+
+  * Fixing coverage.bare-mode-errors test. (Robin Sommer)
+
+2.1-487 | 2013-05-01 18:03:22 -0700
+
+  * Always apply tcp_connection_attempt timer, even if no
+    connection_attempt() event handler is defined. (Robin Sommer)
+
+2.1-486 | 2013-05-01 15:28:45 -0700
+
+  * New framework for computing summary statistics in
+    base/framework/sumstats. This replaces the metrics frameworks, and
+    comes with a number of applications build on top, see NEWS. More
+    documentation to follow. (Seth Hall)
+
+2.1-397 | 2013-04-29 21:19:00 -0700
+
+  * Fixing memory leaks in CompHash implementation. Addresses #987.
+    (Robin Sommer)
+
+2.1-394 | 2013-04-27 15:02:31 -0700
+
+  * Fixed a bug in the vulnerable software script and added a test.
+    (Seth Hall)
+
+  * Fix schedule statements used outside event handlers. Addresses
+    #974. (Jon Siwek)
+
+  * Fix record coercion for default inner record fields. Addresses
+    #973. (Jon Siwek)
+
+  * Add bytestring_to_count function to bro.bif. Addresses #968. (Yun
+    Zheng Hu)
+
+2.1-386 | 2013-03-22 12:41:50 -0700
+
+  * Added reverse() function to strings.bif. (Yun Zheng Hu)
+    
+2.1-384 | 2013-03-22 12:10:14 -0700
+
+  * Fix record constructors in table initializer indices.  Addresses
+    #660. (Jon Siwek)
+
+2.1-382 | 2013-03-22 12:01:34 -0700
+
+  * Add support for 802.1ah (Q-in-Q). Addresses #641. (Seth Hall)
+    
+2.1-380 | 2013-03-18 12:18:10 -0700
+
+  * Fix gcc compile warnings in base64 encoder and benchmark reader.
+    (Bernhard Amann)
+    
+2.1-377 | 2013-03-17 17:36:09 -0700
+
+  * Fixing potential leak in DNS error case. (Vlad Grigorescu)
+    
+2.1-375 | 2013-03-17 13:14:26 -0700
+
+  * Add base64 encoding functionality, including new BiFs
+	encode_base64() and encode_base64_custom(). (Bernhard Amann)
+
+  * Replace call to external "openssl" in extract-certs-pem.bro with
+	that encode_base64(). (Bernhard Amann)
+
+  * Adding a test for extract-certs-pem.pem. (Robin Sommer)
+
+  * Renaming Base64Decoder to Base64Converter. (Robin Sommer)
+    
+2.1-366 | 2013-03-17 12:35:59 -0700
+
+  * Correctly handle DNS lookups for software version ranges. (Seth
+    Hall)
+
+  * Improvements to vulnerable software detection. (Seth Hall)
+    
+     - Add a DNS based updating method.  This needs to be tested
+       still.
+
+     - Vulnerable version ranges are used now instead of only single
+       versions.  This can deal with software with multiple stable
+       major versions.
+
+  * Update software version parsing and comparison to account for a
+    third numeric subversion. Also, $addl is now compared numerically
+    if the value is actually numeric. (Seth Hall)
+
+2.1-361 | 2013-03-13 07:18:22 -0700
+
+  * Add check for truncated link frames. Addresses #962. (Jacob
+    Baines)
+
+  * Fix large memory allocation in IP fragment reassembly. Addresses
+    #961. (Jacob Baines)
+
+2.1-357 | 2013-03-08 09:18:35 -0800
+
+  * Fix race-condition in table-event test. (Bernhard Amann)
+
+  * s/bro-ids.org/bro.org/g. (Robin Sommer)
+
+2.1-353 | 2013-03-07 13:31:37 -0800
+
+  * Fix function type-equivalence requiring same parameter names.
+    Addresses #957. (Jon Siwek)
+
+2.1-351 | 2013-03-07 13:27:29 -0800
+
+  * Fix new/delete mismatch. Addresses #958. (Jacob Baines)
+    
+  * Fix compiler warnings. (Jon Siwek)
+    
+2.1-347 | 2013-03-06 16:48:44 -0800
+
+  * Remove unused parameter from vector assignment method. (Bernhard Amann)
+
+  * Remove the byte_len() and length() bifs. (Bernhard Amann)
+
+2.1-342 | 2013-03-06 15:42:52 -0800
+
+  * Moved the Notice::notice event and Notice::policy table to both be
+    hooks. See documentation and NEWS for information. (Seth Hall).
+
+2.1-338 | 2013-03-06 15:10:43 -0800
+
+  * Fix init of local sets/vectors via curly brace initializer lists.
+    (Jon Siwek)
+
+2.1-336 | 2013-03-06 15:08:06 -0800
+
+  * Fix memory leaks resulting from 'when' and 'return when'
+    statements. Addresses #946. (Jon Siwek)
+
+  * Fix three bugs with 'when' and 'return when' statements. Addresses
+    #946. (Jon Siwek)
+
+2.1-333 | 2013-03-06 14:59:47 -0800
+
+  * Add parsing for GTPv1 extension headers and control messages.  (Jon Siwek)
+
+    This includes:
+
+      - A new generic gtpv1_message() event generated for any GTP
+        message type.
+
+      - Specific events for the create/update/delete PDP context
+        request/response messages.
+
+    Addresses #934.
+
+2.1-331 | 2013-03-06 14:54:33 -0800
+
+  * Fix possible null pointer dereference in identify_data BIF. Also
+    centralized libmagic calls for consistent error handling/output.
+    (Jon Siwek)
+
+  * Fix build on OpenBSD 5.2. (Jon Siwek)
+
+2.1-328 | 2013-02-05 01:34:29 -0500
+
+  * New script to query the ICSI Certificate Notary 
+    (http://notary.icsi.berkeley.edu/) over DNS and add information
+    to the SSL log at runtime. (Matthias Vallentin) 
+
+  * Add delayed logging to SSL base scripts. (Matthias Vallentin)
+
+2.1-319 | 2013-02-04 09:45:34 -0800
+
+  * Update input tests to use exit_only_after_terminate. (Bernhard
+    Amann)
+
+  * New option exit_only_after_terminate to prevent Bro from exiting.
+    If set, the main loop won't terminate before somebody calls
+    terminate(). (Robin Sommer)
+
+2.1-311 | 2013-02-01 08:03:01 -0800
+
+  * Updating submodule(s).
+
+2.1-310 | 2013-01-30 20:09:27 -0800
+
+  * Add an error for record coercions that would orphan a field. (Jon
+    Siwek)
+
+  * Fixing several scripts where a field in an inlined record was
+    never removed after a code refactor. (Jon Siwek)
+
+2.1-307 | 2013-01-25 13:50:57 -0800
+
+  * Fix runaway reference counting bug in record coercion. (Jon Siwek)
+
+  * Fix memory leak in some reporter messaging cases. (Jon Siwek)
+
+2.1-304 | 2013-01-23 19:43:27 -0800
+
+  * Making a test portable. (Robin Sommer)
+
+2.1-302 | 2013-01-23 16:17:29 -0800
+
+  * Refactoring ASCII formatting/parsing from loggers/readers into a
+    separate AsciiFormatter class. (Bernhard Amann)
+
+  * Fix uninitialized locals in event/hook handlers from having a
+    value. Addresses #932. (Jon Siwek)
+
+  * Add a null value check in CompositeHash::ComputeHash. Addresses
+    #930. (Jon Siwek)
+
+  * Change reporter messages to more reliably print to stderr.
+    Addressed #930 (and revisits #836). (Jon Siwek)
+
+  * Changing test=suite's btest call to use "-j" instead of "-j 5".
+    (Robin Sommer)
+    
+  * Require "case" blocks to end with either "break", "return", or a
+    new "fallthrough" statement that passes control on to the
+    subsequent case. This gives us the best mix of safety,
+    readability, and flexibility. Addresses #754. (Jon Siwek)
+
+2.1-279 | 2013-01-18 17:18:22 -0800
+
+  * Revert "Trick for parallelizing input framework unit tests." The
+    old way of doing the tests seems more reliable for now. (Jon
+    Siwek)
+
+  * Fixing variable size issues with http response code in
+    ElasticSearch writer. (Gilbert Clark)
+
+  * Removing unused class member. (Robin Sommer)
+    
+  * Add opaque type-ignoring for the accept_unsupported_types input
+    framework option. (Bernhard Amann)
+
+2.1-271 | 2013-01-08 10:18:57 -0800
+
+  * Change substring index notation to use a colon.  String slice
+    notation is now written as `s[1:2]`. Addresses #422. (Jon Siwek)
+
+2.1-268 | 2013-01-07 09:43:44 -0800
+
+  * Fix memory leak in OpaqueType::DoUnserialize. (Jon Siwek)
+
+2.1-265 | 2012-12-20 17:38:42 -0800
+
+  * Add array-style index accessor for strings.  Addresses #422. (Jon
+    Siwek)
+
+    The index expression can take up to two indices for the start and
+    end index of the substring to return (e.g. "mystring[1,3]").
+    Negative indices are allowed, with -1 representing the last
+    character in the string.  The indexing is not cyclic -- if the
+    starting index is >= the length of the string an empty string is
+    returned, and if the ending index is >= the length of the string
+    then it's interpreted as the last index of the string.  Assigning
+    to substrings accessed like this isn't allowed.
+
+2.1-263 | 2012-12-20 16:22:09 -0800
+
+  * Bro's language now has a new set of types "opaque of X". (Matthias
+    Vallentin)
+
+    Opaque values can be passed around like other values but they can
+    only be manipulated with BiF functions, not with other operators.
+    Currently, the following opaque types are supported:
+
+        - opaque of md5
+        - opaque of sha1
+        - opaque of sha256
+        - opaquey of entropy.
+
+    They go along with the corrsponding BiF functions md5_*, sha1_*,
+    sha256_*, and entropy_*, respectively. Note that these functions
+    have changed their signatures to work with opaques types rather
+    than global state as it was before.
+    
+2.1-240 | 2012-12-20 15:21:07 -0800
+
+  * Improve error for invalid use of types as values. Addresses #923.
+    (Jon Siwek)
+
+2.1-238 | 2012-12-20 15:11:25 -0800
+
+  * Finish implementation of script-layer switch statement. Addresses
+    #754. (Jon Siwek)
+
+    They behave like C-style switches except case labels can be
+    comprised of multiple literal constants delimited by commas.  Only
+    atomic types are allowed for now.  Case label bodies that don't
+    execute a "return" or "break" statement will fall through to
+    subsequent cases.  A default case label is allowed.
+
+  * Fix a case where c$resp$size is misrepresented.  Addresses #730.
+    (Jon Siwek)
+
+2.1-234 | 2012-12-20 12:12:19 -0800
+
+  * Fix return value of hook calls that have no handlers. For this
+    case, the return value is always true. (Jon Siwek)
+
+  * Fix to_port() BIF for port strings with a port number of zero.
+    (Jon Siwek)
+
+2.1-231 | 2012-12-14 14:51:35 -0800
+
+  * Make const variables actually constant. Both local and global
+    variables declared with "const" could be modified, but now
+    expressions that would modify them generate an error message at
+    parse-time. Addresses #922. (Jon Siwek)
+
+2.1-229 | 2012-12-14 14:46:12 -0800
+
+  * Fix memory leak in ASCII reader when encoutering errors in input.
+    (Bernhard Amann)
+
+  * Improvements for the "bad checksums" detector to make it detect
+    bad TCP checksums. (Seth Hall)
+
+2.1-223 | 2012-12-12 14:25:15 -0800
+
+  * Trick for parallelizing input framework unit tests. Instead of
+    loading listen.bro to block until files are read, just read a pcap
+    file in pseudo-realtime. (Jon Siwek)
+
+  * Fix reliability of a unit test that relies on when statements.
+    (Jon Siwek)
+
+  * Remove unused attributes. (Daniel Thayer)
+        - Removed attributes &postprocessor and &match from documentation and source code.
+        - Removed undocumented attribute &attr from source code.
+        - Removed internal attribute "(&tracked)" from documentation.
+
+2.1-218 | 2012-12-10 14:45:04 -0800
+
+  * Add GPRS Tunnelling Protocol (GTPv1) decapsulation. This currently
+    supports automatic decapsulation of GTP-U packets on UDP port 2152.
+    The GTPv1 headers for such tunnels can be inspected by handling
+    the "gtpv1_g_pdu_packet" event, which has a parameter of type
+    "gtpv1_hdr". Addresses #690. (Jon Siwek; derived from patch by
+    Carsten Langer)
+
+  * Change BinPAC exceptions in AYIYA/GTP analyzers to do
+    "protocol_violation". (Jon Siwek)
+
+2.1-212 | 2012-12-07 19:42:03 -0800
+
+  * Changing the HTTP parser to accept request methods in alignment
+    with the RFC. (Robin Sommer)
+
+2.1-209 | 2012-12-05 16:44:04 -0800
+
+  * Adapting the HTTP request line parsing to only accept methods
+    consisting of letters [A-Za-z]. (Robin Sommer)
+
+2.1-207 | 2012-12-05 15:47:32 -0800
+
+  * Reporting warnings if kill/waitpid fail in communication system.
+    (Bill Parker)
+
+  * Replace() bzero with memset(). (Bill Parker)
+
+  * Merge remote-tracking branch 'vlad/topic/vladg/http-verbs'
+
+    * vlad/topic/vladg/http-verbs:
+      A test for HTTP methods, including some horribly illegal requests.
+      Remove hardcoded HTTP verbs from the analyzer (#741)
+
+    I added a "bad_HTTP_request" weird for HTTP request lines that don't
+    have more than a single word.
+
+    Closes #741. (Robin Sommer)
+
+  * A test for HTTP methods, including some horribly illegal requests. (Vlad Grigorescu)
+
+  * Remove hardcoded HTTP verbs from the analyzer (#741) (Vlad Grigorescu)
+
+
+2.1-203 | 2012-12-05 14:36:56 -0800
+
+  * Fix segfault: Synchronization of state between connecting peers
+    now skips over identifiers that aren't initialized with a value
+    yet. Addresses #66. (Jon Siwek)
+
+  * Fix segfault: Delete correct entry in error case in input
+    framework. (Bernhard Amann)
+
+  * Bad record constructor initializers now give an error. Addresses
+    #34. (Jon Siwek)
+
+  * Invalid vector indices now generate error message. Addresses #24.
+    (Jon Siwek)
+
+  * Bump CPack RPM package requirement to Python >= 2.6.0. (Jon Siwek)
+
+  * Interpreter exceptions occurring in "when" blocks are now handled.
+    Addresses #779 (Jon Siwek)
+
+2.1-195 | 2012-12-03 14:50:33 -0800
+
+  * Catching out-of-memory in patricia tree code. (Bill Parker)
+    
+2.1-194 | 2012-12-03 14:36:26 -0800
+
+  * Renaming ASCII writer filter option 'only_single_header_row' to
+    'tsv'. Also clarifying usage. Closes #912. (Robin Sommer)
+
+2.1-193 | 2012-12-03 14:11:14 -0800
+
+  * Fix a set of bugs with table/set attributes. (Jon Siwek)
+
+    - Identifiers that are initialized with set()/table() constructor
+      expressions now inherit attributes from the expression.  Before,
+      statements like
+
+         const i: set[string] = set() &redef;
+
+      associated the attribute with the set() constructor, but not the
+      "i" identifier, preventing redefinition.  Addresses #866.
+
+    - Allow &default attribute to apply to tables initialized as empty
+      (via either "{ }" or "table()") or if the expression supplied to it
+      can evaluate to a type that's promotable to the same yield type as
+      the table.
+
+2.1-191 | 2012-12-03 14:08:56 -0800
+
+  * Add test of record() constructor to table initializer unit test.
+    (Jon Siwek)
+
+  * Fix table(), set(), vector() constructors in table initializer
+    lists. Also adds type checking of yield values to table()
+    constructor and fixes the type checking of yield values in
+    vector() constructor. Addresses #5. (Jon Siwek)
+
+2.1-188 | 2012-12-03 14:04:29 -0800
+
+  * Hook functions now callable with "hook" expression (i.e., hook is
+    no longer a statement). The return value of the call is an
+    implicit boolean value of T if all hook handlers ran, or F if one
+    hook handler exited as a result of a break statement and
+    potentially prevented other handlers from running.
+
+    Scripts don't need to declare hooks with an explicit return type of bool
+    (internally, that's assumed), and any values given to (optional) return
+    statements in handler definitions are just ignored.
+
+    Addresses #918. (Jon Siwek)
+
+  * Clarification in hook documentation. (Jon Siwek)
+
+2.1-184 | 2012-12-03 13:59:50 -0800
+
+  * Slightly fix up file name extraction from Content-Disposition
+    headers. (Seth Hall)
+
+  * Adding -b flag to bro in unit tests so they run faster.
+
+  * Fixed a DNS attribute issue. Reported by Matt Thompson. (Seth
+    Hall)
+
+  * Adding NEWS placeholder for hooks and CSV mode. (Robin Sommer)
+    
+2.1-178 | 2012-11-23 19:35:32 -0800
+
+  * The ASCII writer now supports a new filter config option
+    "only_single_header_row" that turns the output into CSV format
+    when set to "T". (Carsten Langer)
+
+  * Add new function flavor called a "hook". This new flavor of
+    function behaves like a "synchronous event". See
+    doc/scripts/builtins.rst more details on usage. (Jon Siwek)
+
+  * Improve auto-generated enum documentation. The names of enum types
+    are tracked so that variables holding a value of a given enum type
+    can generate a reference to it instead of just listing the type as
+    a generic "enum". (Jon Siwek)
+
+2.1-171 | 2012-11-23 18:24:15 -0800
+
+  * Fix ambiguity between composite table index and record ctor
+    expressions. If a table type is "global t = table[conn_id, bool]
+    of count", then checking membership like "[c$id, is_orig] in t"
+    now works. Addresses #80. (Jon Siwek)
+
+2.1-169 | 2012-11-23 18:21:32 -0800
+
+  * Fix some warnings from sphinx when building docs. (Jon Siwek)
+
 2.1-167 | 2012-11-14 13:19:17 -0800

  * Add a new BIF "bytestring_to_double" for converting from a binary
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -17,12 +17,17 @@ set(BRO_SCRIPT_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/scripts)
 get_filename_component(BRO_SCRIPT_INSTALL_PATH ${BRO_SCRIPT_INSTALL_PATH}
    ABSOLUTE)

+set(BRO_MAGIC_INSTALL_PATH ${BRO_ROOT_DIR}/share/bro/magic)
+set(BRO_MAGIC_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/magic)
+
 configure_file(bro-path-dev.in ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev)
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.sh
     "export BROPATH=`${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n"
+     "export BROMAGIC=\"${BRO_MAGIC_SOURCE_PATH}\"\n"
     "export PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n")
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.csh
     "setenv BROPATH `${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n"
+     "setenv BROMAGIC \"${BRO_MAGIC_SOURCE_PATH}\"\n"
     "setenv PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n")

 file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1)
@ -69,6 +74,12 @@ if (MISSING_PREREQS)
    message(FATAL_ERROR "Configuration aborted due to missing prerequisites")
 endif ()

+set(libmagic_req 5.04)
+if ( LibMagic_VERSION VERSION_LESS ${libmagic_req} )
+    message(FATAL_ERROR "libmagic of at least version ${libmagic_req} required "
+                        "(found ${LibMagic_VERSION})")
+endif ()
+
 include_directories(BEFORE
                    ${PCAP_INCLUDE_DIR}
                    ${OpenSSL_INCLUDE_DIR}
@ -190,12 +201,17 @@ CheckOptionalBuildSources(aux/broctl   Broctl   INSTALL_BROCTL)
 CheckOptionalBuildSources(aux/bro-aux  Bro-Aux  INSTALL_AUX_TOOLS)
 CheckOptionalBuildSources(aux/broccoli Broccoli INSTALL_BROCCOLI)

+install(DIRECTORY ./magic/ DESTINATION ${BRO_MAGIC_INSTALL_PATH} FILES_MATCHING
+        PATTERN "COPYING" EXCLUDE
+        PATTERN "*"
+)
+
 ########################################################################
 ## Packaging Setup

 if (INSTALL_BROCTL)
    # CPack RPM Generator may not automatically detect this
-    set(CPACK_RPM_PACKAGE_REQUIRES "python >= 2.4.0")
+    set(CPACK_RPM_PACKAGE_REQUIRES "python >= 2.6.0")
 endif ()

 # If this CMake project is a sub-project of another, we will not
--- a/DocSourcesList.cmake
+++ b/DocSourcesList.cmake
@ -1,144 +0,0 @@
-# DO NOT EDIT
-# This file is auto-generated from the genDocSourcesList.sh script.
-#
-# This is a list of Bro script sources for which to generate reST documentation.
-# It will be included inline in the CMakeLists.txt found in the same directory
-# in order to create Makefile targets that define how to generate reST from
-# a given Bro script.
-#
-# Note: any path prefix of the script (2nd argument of rest_target macro)
-# will be used to derive what path under scripts/ the generated documentation
-# will be placed.
-
-set(psd ${PROJECT_SOURCE_DIR}/scripts)
-
-rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
-rest_target(${psd} base/init-default.bro internal)
-rest_target(${psd} base/init-bare.bro internal)
-
-rest_target(${CMAKE_BINARY_DIR}/src base/bro.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/const.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/event.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/logging.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/reporter.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/strings.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/types.bif.bro)
-rest_target(${psd} base/frameworks/cluster/main.bro)
-rest_target(${psd} base/frameworks/cluster/nodes/manager.bro)
-rest_target(${psd} base/frameworks/cluster/nodes/proxy.bro)
-rest_target(${psd} base/frameworks/cluster/nodes/worker.bro)
-rest_target(${psd} base/frameworks/cluster/setup-connections.bro)
-rest_target(${psd} base/frameworks/communication/main.bro)
-rest_target(${psd} base/frameworks/control/main.bro)
-rest_target(${psd} base/frameworks/dpd/main.bro)
-rest_target(${psd} base/frameworks/intel/main.bro)
-rest_target(${psd} base/frameworks/logging/main.bro)
-rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro)
-rest_target(${psd} base/frameworks/logging/postprocessors/sftp.bro)
-rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
-rest_target(${psd} base/frameworks/logging/writers/dataseries.bro)
-rest_target(${psd} base/frameworks/metrics/cluster.bro)
-rest_target(${psd} base/frameworks/metrics/main.bro)
-rest_target(${psd} base/frameworks/metrics/non-cluster.bro)
-rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
-rest_target(${psd} base/frameworks/notice/actions/drop.bro)
-rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
-rest_target(${psd} base/frameworks/notice/actions/page.bro)
-rest_target(${psd} base/frameworks/notice/actions/pp-alarms.bro)
-rest_target(${psd} base/frameworks/notice/cluster.bro)
-rest_target(${psd} base/frameworks/notice/extend-email/hostnames.bro)
-rest_target(${psd} base/frameworks/notice/main.bro)
-rest_target(${psd} base/frameworks/notice/weird.bro)
-rest_target(${psd} base/frameworks/packet-filter/main.bro)
-rest_target(${psd} base/frameworks/packet-filter/netstats.bro)
-rest_target(${psd} base/frameworks/reporter/main.bro)
-rest_target(${psd} base/frameworks/signatures/main.bro)
-rest_target(${psd} base/frameworks/software/main.bro)
-rest_target(${psd} base/protocols/conn/contents.bro)
-rest_target(${psd} base/protocols/conn/inactivity.bro)
-rest_target(${psd} base/protocols/conn/main.bro)
-rest_target(${psd} base/protocols/dns/consts.bro)
-rest_target(${psd} base/protocols/dns/main.bro)
-rest_target(${psd} base/protocols/ftp/file-extract.bro)
-rest_target(${psd} base/protocols/ftp/main.bro)
-rest_target(${psd} base/protocols/ftp/utils-commands.bro)
-rest_target(${psd} base/protocols/http/file-extract.bro)
-rest_target(${psd} base/protocols/http/file-hash.bro)
-rest_target(${psd} base/protocols/http/file-ident.bro)
-rest_target(${psd} base/protocols/http/main.bro)
-rest_target(${psd} base/protocols/http/utils.bro)
-rest_target(${psd} base/protocols/irc/dcc-send.bro)
-rest_target(${psd} base/protocols/irc/main.bro)
-rest_target(${psd} base/protocols/smtp/entities-excerpt.bro)
-rest_target(${psd} base/protocols/smtp/entities.bro)
-rest_target(${psd} base/protocols/smtp/main.bro)
-rest_target(${psd} base/protocols/ssh/main.bro)
-rest_target(${psd} base/protocols/ssl/consts.bro)
-rest_target(${psd} base/protocols/ssl/main.bro)
-rest_target(${psd} base/protocols/ssl/mozilla-ca-list.bro)
-rest_target(${psd} base/protocols/syslog/consts.bro)
-rest_target(${psd} base/protocols/syslog/main.bro)
-rest_target(${psd} base/utils/addrs.bro)
-rest_target(${psd} base/utils/conn-ids.bro)
-rest_target(${psd} base/utils/directions-and-hosts.bro)
-rest_target(${psd} base/utils/files.bro)
-rest_target(${psd} base/utils/numbers.bro)
-rest_target(${psd} base/utils/paths.bro)
-rest_target(${psd} base/utils/patterns.bro)
-rest_target(${psd} base/utils/site.bro)
-rest_target(${psd} base/utils/strings.bro)
-rest_target(${psd} base/utils/thresholds.bro)
-rest_target(${psd} policy/frameworks/communication/listen.bro)
-rest_target(${psd} policy/frameworks/control/controllee.bro)
-rest_target(${psd} policy/frameworks/control/controller.bro)
-rest_target(${psd} policy/frameworks/dpd/detect-protocols.bro)
-rest_target(${psd} policy/frameworks/dpd/packet-segment-logging.bro)
-rest_target(${psd} policy/frameworks/metrics/conn-example.bro)
-rest_target(${psd} policy/frameworks/metrics/http-example.bro)
-rest_target(${psd} policy/frameworks/metrics/ssl-example.bro)
-rest_target(${psd} policy/frameworks/software/version-changes.bro)
-rest_target(${psd} policy/frameworks/software/vulnerable.bro)
-rest_target(${psd} policy/integration/barnyard2/main.bro)
-rest_target(${psd} policy/integration/barnyard2/types.bro)
-rest_target(${psd} policy/misc/analysis-groups.bro)
-rest_target(${psd} policy/misc/capture-loss.bro)
-rest_target(${psd} policy/misc/loaded-scripts.bro)
-rest_target(${psd} policy/misc/profiling.bro)
-rest_target(${psd} policy/misc/stats.bro)
-rest_target(${psd} policy/misc/trim-trace-file.bro)
-rest_target(${psd} policy/protocols/conn/known-hosts.bro)
-rest_target(${psd} policy/protocols/conn/known-services.bro)
-rest_target(${psd} policy/protocols/conn/weirds.bro)
-rest_target(${psd} policy/protocols/dns/auth-addl.bro)
-rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
-rest_target(${psd} policy/protocols/ftp/detect.bro)
-rest_target(${psd} policy/protocols/ftp/software.bro)
-rest_target(${psd} policy/protocols/http/detect-MHR.bro)
-rest_target(${psd} policy/protocols/http/detect-intel.bro)
-rest_target(${psd} policy/protocols/http/detect-sqli.bro)
-rest_target(${psd} policy/protocols/http/detect-webapps.bro)
-rest_target(${psd} policy/protocols/http/header-names.bro)
-rest_target(${psd} policy/protocols/http/software-browser-plugins.bro)
-rest_target(${psd} policy/protocols/http/software.bro)
-rest_target(${psd} policy/protocols/http/var-extraction-cookies.bro)
-rest_target(${psd} policy/protocols/http/var-extraction-uri.bro)
-rest_target(${psd} policy/protocols/smtp/blocklists.bro)
-rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro)
-rest_target(${psd} policy/protocols/smtp/software.bro)
-rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro)
-rest_target(${psd} policy/protocols/ssh/geo-data.bro)
-rest_target(${psd} policy/protocols/ssh/interesting-hostnames.bro)
-rest_target(${psd} policy/protocols/ssh/software.bro)
-rest_target(${psd} policy/protocols/ssl/cert-hash.bro)
-rest_target(${psd} policy/protocols/ssl/expiring-certs.bro)
-rest_target(${psd} policy/protocols/ssl/extract-certs-pem.bro)
-rest_target(${psd} policy/protocols/ssl/known-certs.bro)
-rest_target(${psd} policy/protocols/ssl/validate-certs.bro)
-rest_target(${psd} policy/tuning/defaults/packet-fragments.bro)
-rest_target(${psd} policy/tuning/defaults/warnings.bro)
-rest_target(${psd} policy/tuning/track-all-assets.bro)
-rest_target(${psd} site/local-manager.bro)
-rest_target(${psd} site/local-proxy.bro)
-rest_target(${psd} site/local-worker.bro)
-rest_target(${psd} site/local.bro)
-rest_target(${psd} test-all-policy.bro)
--- a/12
+++ b/12
@ -4,7 +4,7 @@
 .. _MacPorts: http://www.macports.org
 .. _Fink: http://www.finkproject.org
 .. _Homebrew: http://mxcl.github.com/homebrew
-.. _bro downloads page: http://bro-ids.org/download/index.html
+.. _bro downloads page: http://bro.org/download/index.html

 ==============
 Installing Bro
@ -189,15 +189,15 @@ Bro releases are bundled into source packages for convenience and
 available from the `bro downloads page`_.

 Alternatively, the latest Bro development version can be obtained through git
-repositories hosted at `git.bro-ids.org <http://git.bro-ids.org>`_.  See
+repositories hosted at `git.bro.org <http://git.bro.org>`_.  See
 our `git development documentation
-<http://bro-ids.org/development/process.html>`_ for comprehensive
+<http://bro.org/development/process.html>`_ for comprehensive
 information on Bro's use of git revision control, but the short story
 for downloading the full source code experience for Bro via git is:

 .. console::

-    git clone --recursive git://git.bro-ids.org/bro
+    git clone --recursive git://git.bro.org/bro

 .. note:: If you choose to clone the ``bro`` repository non-recursively for
   a "minimal Bro experience", be aware that compiling it depends on
@ -230,7 +230,7 @@ automatically.  Finally, use ``make install-aux`` to install some of
 the other programs that are in the ``aux/bro-aux`` directory.

 OpenBSD users, please see our FAQ at
-http://www.bro-ids.org/documentation/faq.html if you are having
+http://www.bro.org/documentation/faq.html if you are having
 problems installing Bro.


@ -298,7 +298,7 @@ Running Bro

 Bro is a complex program and it takes a bit of time to get familiar
 with it.  A good place for newcomers to start is the Quick Start Guide
-at http://www.bro-ids.org/documentation/quickstart.html.
+at http://www.bro.org/documentation/quickstart.html.

 For developers that wish to run Bro directly from the ``build/``
 directory (i.e., without performing ``make install``), they will have
--- a/5
+++ b/5
@ -61,7 +61,10 @@ distclean:
 	rm -rf $(BUILD)

 test:
-	@(cd testing && make )
+	@( cd testing && make )
+
+test-all: test
+	test -d aux/broctl && ( cd aux/broctl && make test )

 configured:
 	@test -d $(BUILD) || ( echo "Error: No build/ directory found. Did you run configure?" && exit 1 )
--- a/163
+++ b/163
@ -13,10 +13,97 @@ Bro 2.2
 New Functionality
 ~~~~~~~~~~~~~~~~~

+- GPRS Tunnelling Protocol (GTPv1) decapsulation.
+
 - GridFTP support. TODO: Extend.

 - ssl.log now also records the subject client and issuer certificates.

+- Hooks: TODO: Briefly summarize the documention from
+  doc/scripts/builtins.rst here.
+
+- The ASCII writer can now output CSV files on a per filter basis.
+
+- Bro's language now has a working "switch" statement that generally
+  behaves like C-style switches except case labels can be comprised of
+  multiple literal constants delimited by commas.  Only atomic types
+  are allowed for now.  Case label bodies that don't execute a
+  "return" or "break" statement will fall through to subsequent cases.
+  A default case label is allowed.
+
+- Bro's language now has a new set of types "opaque of X". Opaque
+  values can be passed around like other values but they can only be
+  manipulated with BiF functions, not with other operators. Currently,
+  the following opaque types are supported:
+
+        - opaque of md5
+        - opaque of sha1
+        - opaque of sha256
+        - opaquey of entropy.
+
+  They go along with the corrsponding BiF functions md5_*, sha1_*,
+  sha256_*, and entropy_*, respectively. Note that these functions
+  have changed their signatures to work with opaques types rather
+  than global state as it was before.
+
+- The scripting language now supports a constructing sets, tables,
+  vectors, and records by name:
+
+        type MyRecordType: record {
+            c: count;
+            s: string &optional;
+        };
+
+        global r: MyRecordType = record($c = 7);
+
+        type MySet: set[MyRec];
+        global s = MySet([$c=1], [$c=2]);
+
+- Strings now support the subscript operator to extract individual
+  characters and substrings (e.g., s[4], s[1,5]). The index expression
+  can take up to two indices for the start and end index of the
+  substring to return (e.g. "mystring[1,3]").
+
+- Functions now support default parameters, e.g.:
+
+        global foo: function(s: string, t: string &default="abc", u: count &default=0);
+
+- Scripts can now use two new "magic constants" @DIR and @FILENAME
+  that expand to the directory path of the current script and just the
+  script file name without path, respectively. (Jon Siwek)
+
+- The new file analysis framework moves most of the processing of file
+  content from script-land into the core, where it belongs. See
+  doc/file-analysis.rst for more information.
+
+  Much of this is an internal change, but the framework also comes
+  with the following user-visibible functionality (some of that was
+  already available before, but done differently):
+
+  [TODO: This will probably change with further script updates.]
+
+      - A binary input reader interfaces the input framework with file
+        analysis, allowing to inject files on disk into Bro's
+        processing.
+
+      - Supports for analyzing data transfereed via HTTP range
+        requests.
+
+      - HTTP:
+        * Identify MIME type of message.
+        * Extract message to disk.
+        * Compute MD5 for messages.
+
+      - SMTP:
+        * Identify MIME type of message.
+        * Extract message to disk.
+        * Compute MD5 for messages.
+        * Provide access to start of entity data.
+
+      - FTP data transfers: Identify MIME type; record to disk.
+
+      - IRC DCC transfers: Record to disk.
+
 Changed Functionality
 ~~~~~~~~~~~~~~~~~~~~~

@ -30,6 +117,10 @@ Changed Functionality
          make_connection_persistent(), generate_idmef(),
          split_complete()

+        - md5_*, sha1_*, sha256_*, and entropy_* have all changed
+        their signatures to work with opaque types (see above).
+
+
 - Removed a now unused argument from "do_split" helper function.

 - "this" is no longer a reserved keyword.
@ -44,6 +135,53 @@ Changed Functionality
  value can now be set with the new broctl.cfg option
  "MailAlarmsInterval".

+- We have completely reworded the "notice_policy" mechanism. It now no
+  linger uses a record of policy items but a "hook", a new language
+  element that's roughly equivalent to a function with multiple
+  bodies. The documentation [TODO: insert link] describes how to use
+  the new notice policy. For existing code, the two main changes are:
+
+    - What used to be a "redef" of "Notice::policy" now becomes a hook
+      implementation. Example:
+
+      Old:
+
+        redef Notice::policy += {
+            [$pred(n: Notice::Info) = {
+                return n$note == SSH::Login && n$id$resp_h == 10.0.0.1;
+                },
+            $action = Notice::ACTION_EMAIL]
+            };
+
+      New:
+
+        hook Notice::policy(n: Notice::Info)
+            {
+            if ( n$note == SSH::Login && n$id$resp_h == 10.0.0.1 )
+                add n$actions[Notice::ACTION_EMAIL];
+            }
+
+    - notice() is now likewise a hook, no longer an event. If you have
+      handlers for that event, you'll likely just need to change the
+      type accordingly. Example:
+
+      Old:
+
+        event notice(n: Notice::Info) { ... }
+
+      New:
+
+        hook notice(n: Notice::Info) { ... }
+
+- The notice_policy.log is gone. That's a result of the new notice
+  policy setup.
+
+- Removed the byte_len() and length() bif functions. Use the "|...|"
+  operator instead.
+
+- The SSH::Login notice has been superseded by an corresponding
+  intelligence framework observation (SSH::SUCCESSFUL_LOGIN).
+
 Bro 2.1
 -------

@ -127,6 +265,27 @@ New Functionality
  outputs. We do not yet recommend them for production (but welcome
  feedback!)

+- Summary statistics framework. [Extend]
+
+- A number of new applications build on top of the summary statistics
+  framework:
+
+    * Scan detection: Detectors for port and address scans return. See
+      policy/misc/scan.bro.
+
+    * Tracerouter detector: policy/misc/detect-traceroute
+
+    * Web application detection/measurement: policy/misc/app-metrics.bro
+
+    * FTP brute-forcing detector: policy/protocols/ftp/detect-bruteforcing.bro
+
+    * HTTP-based SQL injection detector: policy/protocols/http/detect-sqli.bro
+      (existed before, but now ported to the new framework)
+
+    * SSH brute-forcing detector feeding the intelligence framework:
+      policy/protocols/ssh/detect-bruteforcing.bro
+
+

 Changed Functionality
 ~~~~~~~~~~~~~~~~~~~~~
@ -210,7 +369,7 @@ Bro 2.0
 As the version number jump suggests, Bro 2.0 is a major upgrade and
 lots of things have changed. We have assembled a separate upgrade
 guide with the most important changes compared to Bro 1.5 at
-http://www.bro-ids.org/documentation/upgrade.html. You can find
+http://www.bro.org/documentation/upgrade.html. You can find
 the offline version of that document in ``doc/upgrade.rst.``.

 Compared to the earlier 2.0 Beta version, the major changes in the
@ -218,7 +377,7 @@ final release are:

    * The default scripts now come with complete reference
      documentation. See
-      http://www.bro-ids.org/documentation/index.html.
+      http://www.bro.org/documentation/index.html.

    * libz and libmagic are now required dependencies.

--- a/2
+++ b/2
@ -11,7 +11,7 @@ Please see COPYING for licensing information.
 For more documentation, research publications, and community contact
 information, please see Bro's home page:

-    http://www.bro-ids.org
+    http://www.bro.org

 On behalf of the Bro Development Team,

--- a/2
+++ b/2
@ -1 +1 @@
-2.1-167
+2.1-784
--- a/aux/binpac
+++ b/aux/binpac
@ -1 +1 @@
-Subproject commit 2fd9086c9dc0e76f6ff1ae04a60cbbce60507aab
+Subproject commit c39bd478b9d0ecd05b1b83aa9d09a7887893977c
--- a/aux/bro-aux
+++ b/aux/bro-aux
@ -1 +1 @@
-Subproject commit bea556198b69d30d64c0cf1b594e6de71176df6f
+Subproject commit a9942558c7d3dfd80148b8aaded64c82ade3d117
--- a/aux/broccoli
+++ b/aux/broccoli
@ -1 +1 @@
-Subproject commit 907210ce1470724fb386f939cc1b10a4caa2ae39
+Subproject commit 889f9c65944ceac20ad9230efc39d33e6e1221c3
--- a/aux/broctl
+++ b/aux/broctl
@ -1 +1 @@
-Subproject commit 834131cd0ec0f63cce9de818726fe6167dedbf34
+Subproject commit 017e7732446b36af935c26834394b51829335e7c
--- a/aux/btest
+++ b/aux/btest
@ -1 +1 @@
-Subproject commit d83e10c5f76cbfdf81c843575351fbc7b544fc93
+Subproject commit d5b8df42cb9c398142e02d4bf8ede835fd0227f4
--- a/bro-path-dev.in
+++ b/bro-path-dev.in
@ -12,7 +12,7 @@

 broPolicies=${BRO_SCRIPT_SOURCE_PATH}:${BRO_SCRIPT_SOURCE_PATH}/policy:${BRO_SCRIPT_SOURCE_PATH}/site

-broGenPolicies=${CMAKE_BINARY_DIR}/src
+broGenPolicies=${CMAKE_BINARY_DIR}/scripts

 installedPolicies=${BRO_SCRIPT_INSTALL_PATH}:${BRO_SCRIPT_INSTALL_PATH}/site

--- a/2
+++ b/2
@ -1 +1 @@
-Subproject commit 14537f56d66b18ab9d5024f798caf4d1f356fc67
+Subproject commit 0187b33a29d5ec824f940feff60dc5d8c2fe314f
--- a/doc/_templates/layout.html
+++ b/doc/_templates/layout.html
@ -10,7 +10,7 @@
 {% endblock %}

 {% block header %}
-<iframe src="http://www.bro-ids.org/frames/header-no-logo.html" width="100%" height="100px" frameborder="0" marginheight="0" scrolling="no" marginwidth="0">
+<iframe src="http://www.bro.org/frames/header-no-logo.html" width="100%" height="100px" frameborder="0" marginheight="0" scrolling="no" marginwidth="0">
 </iframe>
 {% endblock %}

@ -108,6 +108,6 @@
 {% endblock %}

 {% block footer %}
-<iframe src="http://www.bro-ids.org/frames/footer.html" width="100%" height="420px" frameborder="0" marginheight="0" scrolling="no" marginwidth="0">
+<iframe src="http://www.bro.org/frames/footer.html" width="100%" height="420px" frameborder="0" marginheight="0" scrolling="no" marginwidth="0">
 </iframe>
 {% endblock %}
--- a/doc/ext/bro.py
+++ b/doc/ext/bro.py
@ -82,7 +82,8 @@ class BroGeneric(ObjectDescription):

            objects = self.env.domaindata['bro']['objects']
            key = (self.objtype, name)
-            if key in objects:
+            if ( key in objects and self.objtype != "id" and
+                 self.objtype != "type" ):
                self.env.warn(self.env.docname,
                              'duplicate description of %s %s, ' %
                              (self.objtype, name) +
@ -150,6 +151,12 @@ class BroEnum(BroGeneric):
        #self.indexnode['entries'].append(('single', indextext,
        #                                  targetname, targetname))
        m = sig.split()
+
+        if len(m) < 2:
+            self.env.warn(self.env.docname,
+                          "bro:enum directive missing argument(s)")
+            return
+
        if m[1] == "Notice::Type":
            if 'notices' not in self.env.domaindata['bro']:
                self.env.domaindata['bro']['notices'] = []
--- a/doc/file-analysis.rst
+++ b/doc/file-analysis.rst
@ -0,0 +1,184 @@
+=============
+File Analysis
+=============
+
+.. rst-class:: opening
+
+    In the past, writing Bro scripts with the intent of analyzing file
+    content could be cumbersome because of the fact that the content
+    would be presented in different ways, via events, at the
+    script-layer depending on which network protocol was involved in the
+    file transfer.  Scripts written to analyze files over one protocol
+    would have to be copied and modified to fit other protocols.  The
+    file analysis framework (FAF) instead provides a generalized
+    presentation of file-related information.  The information regarding
+    the protocol involved in transporting a file over the network is
+    still available, but it no longer has to dictate how one organizes
+    their scripting logic to handle it.  A goal of the FAF is to
+    provide analysis specifically for files that is analogous to the
+    analysis Bro provides for network connections.
+
+.. contents::
+
+File Lifecycle Events
+=====================
+
+The key events that may occur during the lifetime of a file are:
+:bro:see:`file_new`, :bro:see:`file_over_new_connection`,
+:bro:see:`file_timeout`, :bro:see:`file_gap`, and
+:bro:see:`file_state_remove`.  Handling any of these events provides
+some information about the file such as which network
+:bro:see:`connection` and protocol are transporting the file, how many
+bytes have been transferred so far, and its MIME type.
+
+.. code:: bro
+
+    event connection_state_remove(c: connection)
+        {
+        print "connection_state_remove";
+        print c$uid;
+        print c$id;
+        for ( s in c$service )
+            print s;
+        }
+
+    event file_state_remove(f: fa_file)
+        {
+        print "file_state_remove";
+        print f$id;
+        for ( cid in f$conns )
+            {
+            print f$conns[cid]$uid;
+            print cid;
+            }
+        print f$source;
+        }
+
+might give output like::
+
+    file_state_remove
+    Cx92a0ym5R8
+    REs2LQfVW2j
+    [orig_h=10.0.0.7, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]
+    HTTP
+    connection_state_remove
+    REs2LQfVW2j
+    [orig_h=10.0.0.7, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]
+    HTTP
+
+This doesn't perform any interesting analysis yet, but does highlight
+the similarity between analysis of connections and files.  Connections
+are identified by the usual 5-tuple or a convenient UID string while
+files are identified just by a string of the same format as the
+connection UID.  So there's unique ways to identify both files and
+connections and files hold references to a connection (or connections)
+that transported it.
+
+Adding Analysis
+===============
+
+There are builtin file analyzers which can be attached to files.  Once
+attached, they start receiving the contents of the file as Bro extracts
+it from an ongoing network connection.  What they do with the file
+contents is up to the particular file analyzer implementation, but
+they'll typically either report further information about the file via
+events (e.g. :bro:see:`FileAnalysis::ANALYZER_MD5` will report the
+file's MD5 checksum via :bro:see:`file_hash` once calculated) or they'll
+have some side effect (e.g. :bro:see:`FileAnalysis::ANALYZER_EXTRACT`
+will write the contents of the file out to the local file system).
+
+In the future there may be file analyzers that automatically attach to
+files based on heuristics, similar to the Dynamic Protocol Detection
+(DPD) framework for connections, but many will always require an
+explicit attachment decision:
+
+.. code:: bro
+
+    event file_new(f: fa_file)
+        {
+        print "new file", f$id;
+        if ( f?$mime_type && f$mime_type == "text/plain" )
+            FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_MD5]);
+        }
+
+    event file_hash(f: fa_file, kind: string, hash: string)
+        {
+        print "file_hash", f$id, kind, hash;
+        }
+
+this script calculates MD5s for all plain text files and might give
+output::
+
+    new file, Cx92a0ym5R8
+    file_hash, Cx92a0ym5R8, md5, 397168fd09991a0e712254df7bc639ac
+
+Some file analyzers might have tunable parameters that need to be
+specified in the call to :bro:see:`FileAnalysis::add_analyzer`:
+
+.. code:: bro
+
+    event file_new(f: fa_file)
+        {
+        FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT,
+                                       $extract_filename="./myfile"]);
+        }
+
+In this case, the file extraction analyzer doesn't generate any further
+events, but does have the side effect of writing out the file contents
+to the local file system at the specified location of ``./myfile``.  Of
+course, for a network with more than a single file being transferred,
+it's probably preferable to specify a different extraction path for each
+file, unlike this example.
+
+Regardless of which file analyzers end up acting on a file, general
+information about the file (e.g. size, time of last data transferred,
+MIME type, etc.) are logged in ``file_analysis.log``.
+
+Input Framework Integration
+===========================
+
+The FAF comes with a simple way to integrate with the :doc:`Input
+Framework <input>`, so that Bro can analyze files from external sources
+in the same way it analyzes files that it sees coming over traffic from
+a network interface it's monitoring.  It only requires a call to
+:bro:see:`Input::add_analysis`:
+
+.. code:: bro
+
+    redef exit_only_after_terminate = T;
+
+    event file_new(f: fa_file)
+        {
+        print "new file", f$id;
+        FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_MD5]);
+        }
+
+    event file_state_remove(f: fa_file)
+        {
+        Input::remove(f$source);
+        terminate();
+        }
+
+    event file_hash(f: fa_file, kind: string, hash: string)
+        {
+        print "file_hash", f$id, kind, hash;
+        }
+
+    event bro_init()
+        {
+        local source: string = "./myfile";
+        Input::add_analysis([$source=source, $name=source]);
+        }
+
+Note that the "source" field of :bro:see:`fa_file` corresponds to the
+"name" field of :bro:see:`Input::AnalysisDescription` since that is what
+the input framework uses to uniquely identify an input stream.
+
+The output of the above script may be::
+
+    new file, G1fS2xthS4l
+    file_hash, G1fS2xthS4l, md5, 54098b367d2e87b078671fad4afb9dbb
+
+Nothing that special, but it at least verifies the MD5 file analyzer
+saw all the bytes of the input file and calculated the checksum
+correctly!
--- a/doc/index.rst
+++ b/doc/index.rst
@ -25,6 +25,7 @@ Frameworks
   notice
   logging
   input
+   file-analysis
   cluster
   signatures

@ -45,7 +46,7 @@ Script Reference
   scripts/packages
   scripts/index
   scripts/builtins
-   scripts/bifs
+   scripts/proto-analyzers

 Other Bro Components
 --------------------
@ -53,7 +54,7 @@ Other Bro Components
 The following are snapshots of documentation for components that come
 with this version of Bro (|version|). Since they can also be used
 independently, see the `download page
-<http://bro-ids.org/download/index.html>`_ for documentation of any
+<http://bro.org/download/index.html>`_ for documentation of any
 current, independent component releases.

 .. toctree::
--- a/doc/logging.rst
+++ b/doc/logging.rst
@ -89,8 +89,7 @@ Note the fields that are set for the filter:
        are generated by taking the stream's ID and munging it slightly.
        :bro:enum:`Conn::LOG` is converted into ``conn``,
        :bro:enum:`PacketFilter::LOG` is converted into
-        ``packet_filter``, and :bro:enum:`Notice::POLICY_LOG` is
-        converted into ``notice_policy``.
+        ``packet_filter``.

    ``include``
        A set limiting the fields to the ones given. The names
--- a/doc/notice.rst
+++ b/doc/notice.rst
@ -6,7 +6,7 @@ Notice Framework

    One of the easiest ways to customize Bro is writing a local notice
    policy. Bro can detect a large number of potentially interesting
-    situations, and the notice policy tells which of them the user wants to be
+    situations, and the notice policy hook which of them the user wants to be
    acted upon in some manner. In particular, the notice policy can specify
    actions to be taken, such as sending an email or compiling regular
    alarm emails.  This page gives an introduction into writing such a notice
@ -24,8 +24,8 @@ of interest for the user. However, none of these scripts determines the
 importance of what it finds itself. Instead, the scripts only flag situations
 as *potentially* interesting, leaving it to the local configuration to define
 which of them are in fact actionable. This decoupling of detection and
-reporting allows Bro to address the different needs that sites have:
-definitions of what constitutes an attack or even a compromise differ quite a
+reporting allows Bro to address the different needs that sites have.
+Definitions of what constitutes an attack or even a compromise differ quite a
 bit between environments, and activity deemed malicious at one site might be
 fully acceptable at another.

@ -40,7 +40,7 @@ More information about raising notices can be found in the `Raising Notices`_
 section.

 Once a notice is raised, it can have any number of actions applied to it by
-the :bro:see:`Notice::policy` set which is described in the `Notice Policy`_
+writing :bro:see:`Notice::policy` hooks which is described in the `Notice Policy`_
 section below. Such actions can be to send a mail to the configured
 address(es) or to simply ignore the notice. Currently, the following actions
 are defined:
@ -68,12 +68,6 @@ are defined:
      - Send an email to the email address or addresses given in the
        :bro:see:`Notice::mail_page_dest` variable.

-    * - Notice::ACTION_NO_SUPPRESS
-      - This action will disable the built in notice suppression for the
-        notice. Keep in mind that this action will need to be applied to
-        every notice that shouldn't be suppressed including each of the future
-        notices that would have normally been suppressed.
-
 How these notice actions are applied to notices is discussed in the
 `Notice Policy`_ and `Notice Policy Shortcuts`_ sections.

@ -83,105 +77,46 @@ Processing Notices
 Notice Policy
 *************

-The predefined set :bro:see:`Notice::policy` provides the mechanism for
-applying actions and other behavior modifications to notices. Each entry
-of :bro:see:`Notice::policy` is a record of the type
-:bro:see:`Notice::PolicyItem` which defines a condition to be matched
-against all raised notices and one or more of a variety of behavior
-modifiers. The notice policy is defined by adding any number of
-:bro:see:`Notice::PolicyItem` records to the :bro:see:`Notice::policy`
-set.
+The hook :bro:see:`Notice::policy` provides the mechanism for applying
+actions and generally modifying the notice before it's sent onward to
+the action plugins.  Hooks can be thought of as multi-bodied functions
+and using them looks very similar to handling events.  The difference
+is that they don't go through the event queue like events.  Users should
+directly make modifications to the :bro:see:`Notice::Info` record
+given as the argument to the hook.

 Here's a simple example which tells Bro to send an email for all notices of
-type :bro:see:`SSH::Login` if the server is 10.0.0.1:
+type :bro:see:`SSH::Password_Guessing` if the server is 10.0.0.1:

 .. code:: bro

-    redef Notice::policy += {
-      [$pred(n: Notice::Info) = {
-         return n$note == SSH::Login && n$id$resp_h == 10.0.0.1;
-       },
-       $action = Notice::ACTION_EMAIL]
-      };
+    hook Notice::policy(n: Notice::Info)
+      {
+      if ( n$note == SSH::Password_Guessing && n$id$resp_h == 10.0.0.1 )
+        add n$actions[Notice::ACTION_EMAIL];
+      }

 .. note::

-    Keep in mind that the semantics of the SSH::Login notice are
-    such that it is only raised when Bro heuristically detects a successful
-    login. No apparently failed logins will raise this notice.
-
-While the syntax might look a bit convoluted at first, it provides a lot of
-flexibility due to having access to Bro's full programming language.
-
-Predicate Field
-^^^^^^^^^^^^^^^
-
-The :bro:see:`Notice::PolicyItem` record type has a field name ``$pred``
-which defines the entry's condition in the form of a predicate written
-as a Bro function. The function is passed the notice as a
-:bro:see:`Notice::Info` record and it returns a boolean value indicating
-if the entry is applicable to that particular notice.
-
-.. note::
-
-    The lack of a predicate in a ``Notice::PolicyItem`` is implicitly true
-    (``T``) since an implicit false (``F``) value would never be used.
-
-Bro evaluates the predicates of each entry in the order defined by the
-``$priority`` field in :bro:see:`Notice::PolicyItem` records. The valid
-values are 0-10 with 10 being earliest evaluated. If ``$priority`` is
-omitted, the default priority is 5.
-
-Behavior Modification Fields
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-There are a set of fields in the :bro:see:`Notice::PolicyItem` record type that
-indicate ways that either the notice or notice processing should be modified
-if the predicate field (``$pred``) evaluated to true (``T``). Those fields are
-explained in more detail in the following table.
-
-.. list-table::
-    :widths: 20 30 20
-    :header-rows: 1
-
-    * - Field
-      - Description
-      - Example
-
-    * - ``$action=<Notice::Action>``
-      - Each :bro:see:`Notice::PolicyItem` can have a single action
-        applied to the notice with this field.
-      - ``$action = Notice::ACTION_EMAIL``
-
-    * - ``$suppress_for=<interval>`` 
-      - This field makes it possible for a user to modify the behavior of the
-        notice framework's automated suppression of intrinsically similar
-        notices. More information about the notice framework's automated
-        suppression can be found in the `Automated Suppression`_ section of
-        this document.
-      - ``$suppress_for = 10mins``
-
-    * - ``$halt=<bool>``
-      - This field can be used for modification of the notice policy
-        evaluation. To stop processing of notice policy items before
-        evaluating all of them, set this field to ``T`` and make the ``$pred``
-        field return ``T``. :bro:see:`Notice::PolicyItem` records defined at
-        a higher priority as defined by the ``$priority`` field will still be
-        evaluated but those at a lower priority won't.
-      - ``$halt = T``
-
+    Keep in mind that the semantics of the SSH::Password_Guessing notice are
+    such that it is only raised when Bro heuristically detects a failed
+    login.

+Hooks can also have priorities applied to order their execution like events
+with a default priority of 0.  Greater values are executed first.  Setting
+a hook body to run before default hook bodies might look like this:

 .. code:: bro

-    redef Notice::policy += {
-      [$pred(n: Notice::Info) = {
-         return n$note == SSH::Login && n$id$resp_h == 10.0.0.1;
-       },
-       $action = Notice::ACTION_EMAIL,
-       $priority=5]
-      };
+    hook Notice::policy(n: Notice::Info) &priority=5
+      {
+      if ( n$note == SSH::Password_Guessing && n$id$resp_h == 10.0.0.1 )
+        add n$actions[Notice::ACTION_EMAIL];
+      }

+Hooks can also abort later hook bodies with the ``break`` keyword. This
+is primarily useful if one wants to completely preempt processing by
+lower priority :bro:see:`Notice::policy` hooks.

 Notice Policy Shortcuts
 ***********************
@ -189,7 +124,7 @@ Notice Policy Shortcuts
 Although the notice framework provides a great deal of flexibility and
 configurability there are many times that the full expressiveness isn't needed
 and actually becomes a hindrance to achieving results. The framework provides
-a default :bro:see:`Notice::policy` suite as a way of giving users the
+a default :bro:see:`Notice::policy` hook body as a way of giving users the
 shortcuts to easily apply many common actions to notices.

 These are implemented as sets and tables indexed with a
@ -238,16 +173,16 @@ Raising Notices

 A script should raise a notice for any occurrence that a user may want
 to be notified about or take action on. For example, whenever the base
-SSH analysis scripts sees an SSH session where it is heuristically
-guessed to be a successful login, it raises a Notice of the type
-:bro:see:`SSH::Login`. The code in the base SSH analysis script looks
-like this:
+SSH analysis scripts sees enough failed logins to a given host, it
+raises a notice of the type :bro:see:`SSH::Password_Guessing`.  The code
+in the base SSH analysis script which raises the notice looks like this:

 .. code:: bro

-    NOTICE([$note=SSH::Login, 
-            $msg="Heuristically detected successful SSH login.",
-            $conn=c]);
+    NOTICE([$note=Password_Guessing,
+            $msg=fmt("%s appears to be guessing SSH passwords (seen in %d connections).", key$host, r$num),
+            $src=key$host,
+            $identifier=cat(key$host)]);

 :bro:see:`NOTICE` is a normal function in the global namespace which
 wraps a function within the ``Notice`` namespace. It takes a single
@ -377,19 +312,45 @@ Setting the ``$identifier`` field is left to those raising notices because
 it's assumed that the script author who is raising the notice understands the
 full problem set and edge cases of the notice which may not be readily
 apparent to users. If users don't want the suppression to take place or simply
-want a different interval, they can always modify it with the
-:bro:see:`Notice::policy`.
+want a different interval, they can set a notice's suppression
+interval to ``0secs`` or delete the value from the ``$identifier`` field in
+a :bro:see:`Notice::policy` hook.


 Extending Notice Framework
 --------------------------

-Adding Custom Notice Actions
-****************************
+There are a couple of mechanism currently for extending the notice framework
+and adding new capability.

 Extending Notice Emails
 ***********************

+If there is extra information that you would like to add to emails, that is
+possible to add by writing :bro:see:`Notice::policy` hooks.
+
+There is a field in the :bro:see:`Notice::Info` record named
+``$email_body_sections`` which will be included verbatim when email is being
+sent. An example of including some information from an HTTP request is
+included below.
+
+.. code:: bro
+
+    hook Notice::policy(n: Notice::Info)
+      {
+      if ( n?$conn && n$conn?$http && n$conn$http?$host )
+        n$email_body_sections[|email_body_sections|] = fmt("HTTP host header: %s", n$conn$http$host);
+      }
+
+
 Cluster Considerations
 ----------------------

+As a user/developer of Bro, the main cluster concern with the notice framework
+is understanding what runs where. When a notice is generated on a worker, the
+worker checks to see if the notice shoudl be suppressed based on information
+locally maintained in the worker process. If it's not being
+suppressed, the worker forwards the notice directly to the manager and does no more
+local processing. The manager then runs the :bro:see:`Notice::policy` hook and
+executes all of the actions determined to be run.
+
--- a/doc/quickstart.rst
+++ b/doc/quickstart.rst
@ -111,7 +111,7 @@ protocol-dependent activity that's occurring.  E.g. ``http.log``'s next few
 columns (shortened for brevity) show a request to the root of Bro website::

    # method   host         uri  referrer  user_agent
-    GET        bro-ids.org  /    -         <...>Chrome/12.0.742.122<...>
+    GET        bro.org  /    -         <...>Chrome/12.0.742.122<...>

 Some logs are worth explicit mention:

--- a/doc/reporting-problems.rst
+++ b/doc/reporting-problems.rst
@ -19,7 +19,7 @@ Reporting Problems

 Generally, when you encounter a problem with Bro, the best thing to do
 is opening a new ticket in `Bro's issue tracker
-<http://tracker.bro-ids.org/>`__ and include information on how to
+<http://tracker.bro.org/>`__ and include information on how to
 reproduce the issue. Ideally, your ticket should come with the
 following:

--- a/doc/scripts/CMakeLists.txt
+++ b/doc/scripts/CMakeLists.txt
@ -15,11 +15,11 @@ endif ()
 #
 # srcDir: the directory which contains broInput
 # broInput: the file name of a bro policy script, any path prefix of this
-#     argument will be used to derive what path under policy/ the generated
+#     argument will be used to derive what path under scripts/ the generated
 #     documentation will be placed.
 # group: optional name of group that the script documentation will belong to.
-#     If this is not given, .bif files automatically get their own group or
-#     the group is automatically by any path portion of the broInput argument.
+#     If this is not given, the group is automatically set to any path portion
+#     of the broInput argument.
 #
 # In addition to adding the makefile target, several CMake variables are set:
 #
@ -45,12 +45,6 @@ macro(REST_TARGET srcDir broInput)

    set(sumTextSrc ${absSrcPath})
    set(ogSourceFile ${absSrcPath})
-    if (${extension} STREQUAL ".bif.bro")
-        set(ogSourceFile ${BIF_SRC_DIR}/${basename})
-        # the summary text is taken at configure time, but .bif.bro files
-        # may not have been generated yet, so read .bif file instead
-        set(sumTextSrc ${ogSourceFile})
-    endif ()

    if (NOT relDstDir)
        set(docName "${basename}")
@ -70,8 +64,6 @@ macro(REST_TARGET srcDir broInput)

    if (NOT "${ARGN}" STREQUAL "")
        set(group ${ARGN})
-    elseif (${extension} STREQUAL ".bif.bro")
-        set(group bifs)
    elseif (relDstDir)
        set(group ${relDstDir}/index)
        # add package index to master package list if not already in it
@ -107,7 +99,7 @@ macro(REST_TARGET srcDir broInput)
        COMMAND "${CMAKE_COMMAND}"
        ARGS -E remove_directory .state
        # generate the reST documentation using bro
-        COMMAND BROPATH=${BROPATH}:${srcDir} ${CMAKE_BINARY_DIR}/src/bro
+        COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic ${CMAKE_BINARY_DIR}/src/bro
        ARGS -b -Z ${broInput} || (rm -rf .state *.log *.rst && exit 1)
        # move generated doc into a new directory tree that
        # defines the final structure of documents
@ -132,6 +124,29 @@ endmacro(REST_TARGET)
 # Schedule Bro scripts for which to generate documentation.
 include(DocSourcesList.cmake)

+# This reST target is independent of a particular Bro script...
+add_custom_command(OUTPUT proto-analyzers.rst
+    # delete any leftover state from previous bro runs
+    COMMAND "${CMAKE_COMMAND}"
+    ARGS -E remove_directory .state
+    # generate the reST documentation using bro
+    COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic ${CMAKE_BINARY_DIR}/src/bro
+    ARGS -b -Z base/init-bare.bro || (rm -rf .state *.log *.rst && exit 1)
+    # move generated doc into a new directory tree that
+    # defines the final structure of documents
+    COMMAND "${CMAKE_COMMAND}"
+    ARGS -E make_directory ${dstDir}
+    COMMAND "${CMAKE_COMMAND}"
+    ARGS -E copy proto-analyzers.rst ${dstDir}
+    # clean up the build directory
+    COMMAND rm
+    ARGS -rf .state *.log *.rst
+    DEPENDS bro
+    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+    COMMENT "[Bro] Generating reST docs for proto-analyzers.rst"
+)
+list(APPEND ALL_REST_OUTPUTS proto-analyzers.rst)
+
 # create temporary list of all docs to include in the master policy/index file
 file(WRITE ${MASTER_POLICY_INDEX} "${MASTER_POLICY_INDEX_TEXT}")

--- a/doc/scripts/DocSourcesList.cmake
+++ b/doc/scripts/DocSourcesList.cmake
@ -16,14 +16,64 @@ rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
 rest_target(${psd} base/init-default.bro internal)
 rest_target(${psd} base/init-bare.bro internal)

-rest_target(${CMAKE_BINARY_DIR}/src base/bro.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/const.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/event.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/input.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/logging.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/reporter.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/strings.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/types.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/analyzer.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/bro.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/const.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/event.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/file_analysis.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/input.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/logging.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ARP.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_AYIYA.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_BackDoor.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_BitTorrent.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ConnSize.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DCE_RPC.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DHCP.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DNS.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FTP.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FTP.functions.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_File.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FileHash.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Finger.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_GTPv1.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Gnutella.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_HTTP.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_HTTP.functions.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ICMP.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_IRC.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Ident.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_InterConn.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Login.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Login.functions.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_MIME.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Modbus.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NCP.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NTP.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NetBIOS.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NetBIOS.functions.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NetFlow.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_PIA.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_POP3.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_RPC.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SMB.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SMTP.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SMTP.functions.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SOCKS.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SSH.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SSL.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SSL.functions.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SteppingStone.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Syslog.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_TCP.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_TCP.functions.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Teredo.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_UDP.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ZIP.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/reporter.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/strings.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/types.bif.bro)
+rest_target(${psd} base/frameworks/analyzer/main.bro)
 rest_target(${psd} base/frameworks/cluster/main.bro)
 rest_target(${psd} base/frameworks/cluster/nodes/manager.bro)
 rest_target(${psd} base/frameworks/cluster/nodes/proxy.bro)
@ -32,10 +82,13 @@ rest_target(${psd} base/frameworks/cluster/setup-connections.bro)
 rest_target(${psd} base/frameworks/communication/main.bro)
 rest_target(${psd} base/frameworks/control/main.bro)
 rest_target(${psd} base/frameworks/dpd/main.bro)
+rest_target(${psd} base/frameworks/file-analysis/main.bro)
 rest_target(${psd} base/frameworks/input/main.bro)
 rest_target(${psd} base/frameworks/input/readers/ascii.bro)
 rest_target(${psd} base/frameworks/input/readers/benchmark.bro)
+rest_target(${psd} base/frameworks/input/readers/binary.bro)
 rest_target(${psd} base/frameworks/input/readers/raw.bro)
+rest_target(${psd} base/frameworks/input/readers/sqlite.bro)
 rest_target(${psd} base/frameworks/intel/cluster.bro)
 rest_target(${psd} base/frameworks/intel/input.bro)
 rest_target(${psd} base/frameworks/intel/main.bro)
@ -46,9 +99,7 @@ rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
 rest_target(${psd} base/frameworks/logging/writers/dataseries.bro)
 rest_target(${psd} base/frameworks/logging/writers/elasticsearch.bro)
 rest_target(${psd} base/frameworks/logging/writers/none.bro)
-rest_target(${psd} base/frameworks/metrics/cluster.bro)
-rest_target(${psd} base/frameworks/metrics/main.bro)
-rest_target(${psd} base/frameworks/metrics/non-cluster.bro)
+rest_target(${psd} base/frameworks/logging/writers/sqlite.bro)
 rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
 rest_target(${psd} base/frameworks/notice/actions/drop.bro)
 rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
@ -57,12 +108,25 @@ rest_target(${psd} base/frameworks/notice/actions/pp-alarms.bro)
 rest_target(${psd} base/frameworks/notice/cluster.bro)
 rest_target(${psd} base/frameworks/notice/extend-email/hostnames.bro)
 rest_target(${psd} base/frameworks/notice/main.bro)
+rest_target(${psd} base/frameworks/notice/non-cluster.bro)
 rest_target(${psd} base/frameworks/notice/weird.bro)
 rest_target(${psd} base/frameworks/packet-filter/main.bro)
 rest_target(${psd} base/frameworks/packet-filter/netstats.bro)
 rest_target(${psd} base/frameworks/reporter/main.bro)
 rest_target(${psd} base/frameworks/signatures/main.bro)
 rest_target(${psd} base/frameworks/software/main.bro)
+rest_target(${psd} base/frameworks/sumstats/cluster.bro)
+rest_target(${psd} base/frameworks/sumstats/main.bro)
+rest_target(${psd} base/frameworks/sumstats/non-cluster.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/average.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/last.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/max.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/min.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/sample.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/std-dev.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/sum.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/unique.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/variance.bro)
 rest_target(${psd} base/frameworks/tunnels/main.bro)
 rest_target(${psd} base/misc/find-checksum-offloading.bro)
 rest_target(${psd} base/protocols/conn/contents.bro)
@ -71,21 +135,25 @@ rest_target(${psd} base/protocols/conn/main.bro)
 rest_target(${psd} base/protocols/conn/polling.bro)
 rest_target(${psd} base/protocols/dns/consts.bro)
 rest_target(${psd} base/protocols/dns/main.bro)
+rest_target(${psd} base/protocols/ftp/file-analysis.bro)
 rest_target(${psd} base/protocols/ftp/file-extract.bro)
 rest_target(${psd} base/protocols/ftp/gridftp.bro)
 rest_target(${psd} base/protocols/ftp/main.bro)
 rest_target(${psd} base/protocols/ftp/utils-commands.bro)
+rest_target(${psd} base/protocols/http/file-analysis.bro)
 rest_target(${psd} base/protocols/http/file-extract.bro)
 rest_target(${psd} base/protocols/http/file-hash.bro)
 rest_target(${psd} base/protocols/http/file-ident.bro)
 rest_target(${psd} base/protocols/http/main.bro)
 rest_target(${psd} base/protocols/http/utils.bro)
 rest_target(${psd} base/protocols/irc/dcc-send.bro)
+rest_target(${psd} base/protocols/irc/file-analysis.bro)
 rest_target(${psd} base/protocols/irc/main.bro)
 rest_target(${psd} base/protocols/modbus/consts.bro)
 rest_target(${psd} base/protocols/modbus/main.bro)
 rest_target(${psd} base/protocols/smtp/entities-excerpt.bro)
 rest_target(${psd} base/protocols/smtp/entities.bro)
+rest_target(${psd} base/protocols/smtp/file-analysis.bro)
 rest_target(${psd} base/protocols/smtp/main.bro)
 rest_target(${psd} base/protocols/socks/consts.bro)
 rest_target(${psd} base/protocols/socks/main.bro)
@ -102,9 +170,11 @@ rest_target(${psd} base/utils/files.bro)
 rest_target(${psd} base/utils/numbers.bro)
 rest_target(${psd} base/utils/paths.bro)
 rest_target(${psd} base/utils/patterns.bro)
+rest_target(${psd} base/utils/queue.bro)
 rest_target(${psd} base/utils/site.bro)
 rest_target(${psd} base/utils/strings.bro)
 rest_target(${psd} base/utils/thresholds.bro)
+rest_target(${psd} base/utils/time.bro)
 rest_target(${psd} base/utils/urls.bro)
 rest_target(${psd} policy/frameworks/communication/listen.bro)
 rest_target(${psd} policy/frameworks/control/controllee.bro)
@ -120,18 +190,17 @@ rest_target(${psd} policy/frameworks/intel/smtp-url-extraction.bro)
 rest_target(${psd} policy/frameworks/intel/smtp.bro)
 rest_target(${psd} policy/frameworks/intel/ssl.bro)
 rest_target(${psd} policy/frameworks/intel/where-locations.bro)
-rest_target(${psd} policy/frameworks/metrics/conn-example.bro)
-rest_target(${psd} policy/frameworks/metrics/http-example.bro)
-rest_target(${psd} policy/frameworks/metrics/ssl-example.bro)
 rest_target(${psd} policy/frameworks/software/version-changes.bro)
 rest_target(${psd} policy/frameworks/software/vulnerable.bro)
 rest_target(${psd} policy/integration/barnyard2/main.bro)
 rest_target(${psd} policy/integration/barnyard2/types.bro)
 rest_target(${psd} policy/integration/collective-intel/main.bro)
-rest_target(${psd} policy/misc/analysis-groups.bro)
+rest_target(${psd} policy/misc/app-metrics.bro)
 rest_target(${psd} policy/misc/capture-loss.bro)
+rest_target(${psd} policy/misc/detect-traceroute/main.bro)
 rest_target(${psd} policy/misc/loaded-scripts.bro)
 rest_target(${psd} policy/misc/profiling.bro)
+rest_target(${psd} policy/misc/scan.bro)
 rest_target(${psd} policy/misc/stats.bro)
 rest_target(${psd} policy/misc/trim-trace-file.bro)
 rest_target(${psd} policy/protocols/conn/known-hosts.bro)
@ -139,6 +208,7 @@ rest_target(${psd} policy/protocols/conn/known-services.bro)
 rest_target(${psd} policy/protocols/conn/weirds.bro)
 rest_target(${psd} policy/protocols/dns/auth-addl.bro)
 rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
+rest_target(${psd} policy/protocols/ftp/detect-bruteforcing.bro)
 rest_target(${psd} policy/protocols/ftp/detect.bro)
 rest_target(${psd} policy/protocols/ftp/software.bro)
 rest_target(${psd} policy/protocols/http/detect-MHR.bro)
@ -162,6 +232,7 @@ rest_target(${psd} policy/protocols/ssl/cert-hash.bro)
 rest_target(${psd} policy/protocols/ssl/expiring-certs.bro)
 rest_target(${psd} policy/protocols/ssl/extract-certs-pem.bro)
 rest_target(${psd} policy/protocols/ssl/known-certs.bro)
+rest_target(${psd} policy/protocols/ssl/notary.bro)
 rest_target(${psd} policy/protocols/ssl/validate-certs.bro)
 rest_target(${psd} policy/tuning/defaults/packet-fragments.bro)
 rest_target(${psd} policy/tuning/defaults/warnings.bro)
--- a/doc/scripts/bifs.rst
+++ b/doc/scripts/bifs.rst
@ -1,5 +0,0 @@
-.. This is a stub doc to which broxygen appends during the build process
-
-Built-In Functions (BIFs)
-=========================
-
--- a/doc/scripts/builtins.rst
+++ b/doc/scripts/builtins.rst
@ -246,6 +246,31 @@ The Bro scripting language supports the following built-in types.
            [5] = "five",
        };

+    A table constructor (equivalent to above example) can also be used
+    to create a table:
+
+    .. code:: bro
+
+        global t2: table[count] of string = table(
+            [11] = "eleven",
+            [5] = "five"
+        );
+
+    Table constructors can also be explicitly named by a type, which is
+    useful for when a more complex index type could otherwise be
+    ambiguous:
+
+    .. code:: bro
+
+        type MyRec: record {
+            a: count &optional;
+            b: count;
+        };
+
+        type MyTable: table[MyRec] of string;
+
+        global t3 = MyTable([[$b=5]] = "b5", [[$b=7]] = "b7");
+
    Accessing table elements if provided by enclosing values within square
    brackets (``[]``), for example:

@ -308,6 +333,28 @@ The Bro scripting language supports the following built-in types.
    The types are explicitly shown in the example above, but they could
    have been left to type inference.

+    A set constructor (equivalent to above example) can also be used to
+    create a set:
+
+    .. code:: bro
+
+        global s3: set[port] = set(21/tcp, 23/tcp, 80/tcp, 443/tcp);
+
+    Set constructors can also be explicitly named by a type, which is
+    useful for when a more complex index type could otherwise be
+    ambiguous:
+
+    .. code:: bro
+
+        type MyRec: record {
+            a: count &optional;
+            b: count;
+        };
+
+        type MySet: set[MyRec];
+
+        global s4 = MySet([$b=1], [$b=2]);
+
    Set membership is tested with ``in``:

    .. code:: bro
@ -349,6 +396,21 @@ The Bro scripting language supports the following built-in types.

        global v: vector of string = vector("one", "two", "three");

+    Vector constructors can also be explicitly named by a type, which
+    is useful for when a more complex yield type could otherwise be
+    ambiguous.
+
+    .. code:: bro
+
+        type MyRec: record {
+            a: count &optional;
+            b: count;
+        };
+
+        type MyVec: vector of MyRec;
+
+        global v2 = MyVec([$b=1], [$b=2], [$b=3]);
+
    Adding an element to a vector involves accessing/assigning it:

    .. code:: bro
@ -402,6 +464,44 @@ The Bro scripting language supports the following built-in types.
        if ( r?$s )
            ...

+    Records can also be created using a constructor syntax:
+
+    .. code:: bro
+
+        global r2: MyRecordType = record($c = 7);
+
+    And the constructor can be explicitly named by type, too, which
+    is arguably more readable code:
+
+    .. code:: bro
+
+        global r3 = MyRecordType($c = 42);
+
+.. bro:type:: opaque
+
+    A data type whose actual representation/implementation is
+    intentionally hidden, but whose values may be passed to certain
+    functions that can actually access the internal/hidden resources.
+    Opaque types are differentiated from each other by qualifying them
+    like ``opaque of md5`` or ``opaque of sha1``.  Any valid identifier
+    can be used as the type qualifier.
+
+    An example use of this type is the set of built-in functions which
+    perform hashing:
+
+    .. code:: bro
+
+        local handle: opaque of md5 = md5_hash_init();
+        md5_hash_update(handle, "test");
+        md5_hash_update(handle, "testing");
+        print md5_hash_finish(handle);
+
+    Here the opaque type is used to provide a handle to a particular
+    resource which is calculating an MD5 checksum incrementally over
+    time, but the details of that resource aren't relevant, it's only
+    necessary to have a handle as a way of identifying it and
+    distinguishing it from other such resources.
+
 .. bro:type:: file

    Bro supports writing to files, but not reading from them.  For
@ -417,10 +517,6 @@ The Bro scripting language supports the following built-in types.
    Writing to files like this for logging usually isn't recommended, for better
    logging support see :doc:`/logging`.

-.. bro:type:: func
-
-    See :bro:type:`function`.
-
 .. bro:type:: function

    Function types in Bro are declared using::
@ -463,6 +559,31 @@ The Bro scripting language supports the following built-in types.

        print greeting("Dave");

+    Function parameters may specify default values as long as they appear
+    last in the parameter list:
+
+    .. code:: bro
+
+        global foo: function(s: string, t: string &default="abc", u: count &default=0);
+
+    If a function was previously declared with default parameters, the
+    default expressions can be omitted when implementing the function
+    body and they will still be used for function calls that lack those
+    arguments.
+
+    .. code:: bro
+
+        function foo(s: string, t: string, u: count)
+            {
+            print s, t, u;
+            }
+
+    And calls to the function may omit the defaults from the argument list:
+
+    .. code:: bro
+
+        foo("test");
+
 .. bro:type:: event

    Event handlers are nearly identical in both syntax and semantics to
@ -504,6 +625,87 @@ The Bro scripting language supports the following built-in types.
    identifier and the body of each will be executed in turn.  Ordering
    of execution can be influenced with :bro:attr:`&priority`.

+.. bro:type:: hook
+
+    A hook is another flavor of function that shares characteristics of
+    both a :bro:type:`function` and a :bro:type:`event`.  They are like
+    events in that many handler bodies can be defined for the same hook
+    identifier and the order of execution can be enforced with
+    :bro:attr:`&priority`.  They are more like functions in the way they
+    are invoked/called, because, unlike events, their execution is
+    immediate and they do not get scheduled through an event queue.
+    Also, a unique feature of a hook is that a given hook handler body
+    can short-circuit the execution of remaining hook handlers simply by
+    exiting from the body as a result of a ``break`` statement (as
+    opposed to a ``return`` or just reaching the end of the body).
+
+    A hook type is declared like::
+
+        hook( argument* )
+
+    where *argument* is a (possibly empty) comma-separated list of
+    arguments.  For example:
+
+    .. code:: bro
+
+        global myhook: hook(s: string)
+
+    Here ``myhook`` is the hook type identifier and no hook handler
+    bodies have been defined for it yet.  To define some hook handler
+    bodies the syntax looks like:
+
+    .. code:: bro
+
+        hook myhook(s: string) &priority=10
+            {
+            print "priority 10 myhook handler", s;
+            s = "bye";
+            }
+
+        hook myhook(s: string)
+            {
+            print "break out of myhook handling", s;
+            break;
+            }
+
+        hook myhook(s: string) &priority=-5
+            {
+            print "not going to happen", s;
+            }
+
+    Note that the first (forward) declaration of ``myhook`` as a hook
+    type isn't strictly required.  Argument types must match for all
+    hook handlers and any forward declaration of a given hook.
+
+    To invoke immediate execution of all hook handler bodies, they
+    are called similarly to a function, except preceded by the ``hook``
+    keyword:
+
+    .. code:: bro
+
+        hook myhook("hi");
+
+    or
+
+    .. code:: bro
+
+        if ( hook myhook("hi") )
+            print "all handlers ran";
+
+    And the output would look like::
+
+        priority 10 myhook handler, hi
+        break out of myhook handling, bye
+
+    Note how the modification to arguments can be seen by remaining
+    hook handlers.
+
+    The return value of a hook call is an implicit :bro:type:`bool`
+    value with ``T`` meaning that all handlers for the hook were
+    executed and ``F`` meaning that only some of the handlers may have
+    executed due to one handler body exiting as a result of a ``break``
+    statement.
+
 Attributes
 ----------

@ -520,10 +722,10 @@ scripting language supports the following built-in attributes.

 .. bro:attr:: &default

-    Uses a default value for a record field or container elements. For
-    example, ``table[int] of string &default="foo" }`` would create a
-    table that returns the :bro:type:`string` ``"foo"`` for any
-    non-existing index.
+    Uses a default value for a record field, a function/hook/event
+    parameter, or container elements.  For example, ``table[int] of
+    string &default="foo" }`` would create a table that returns the
+    :bro:type:`string` ``"foo"`` for any non-existing index.

 .. bro:attr:: &redef

@ -586,20 +788,12 @@ scripting language supports the following built-in attributes.
    ``&synchronized`` variable is automatically propagated to all peers
    when it changes.

-.. bro:attr:: &postprocessor
-
-.. TODO: needs to be documented.
-
 .. bro:attr:: &encrypt

    Encrypts files right before writing them to disk.

 .. TODO: needs to be documented in more detail.

-.. bro:attr:: &match
-
-.. TODO: needs to be documented.
-
 .. bro:attr:: &raw_output

    Opens a file in raw mode, i.e., non-ASCII characters are not
@ -633,6 +827,3 @@ scripting language supports the following built-in attributes.

 .. TODO: needs documented

-.. bro:attr:: (&tracked)
-
-.. TODO: needs documented or removed if it's not used anywhere.
--- a/doc/scripts/example.bro
+++ b/doc/scripts/example.bro
@ -54,11 +54,11 @@ global example_ports = {
    443/tcp, 562/tcp,
 } &redef;

-# redefinitions of "dpd_config" are self-documenting and
-# go into the generated doc's "Port Analysis" section
-redef dpd_config += {
-    [ANALYZER_SSL] = [$ports = example_ports]
-};
+
+event bro_init()
+	{
+	Analyzer::register_for_ports(Analyzer::ANALYZER_SSL, example_ports);
+	}

 # redefinitions of "Notice::Type" are self-documenting, but
 # more information can be supplied in two different ways
--- a/doc/scripts/genDocSourcesList.sh
+++ b/doc/scripts/genDocSourcesList.sh
@ -67,12 +67,12 @@ sourcedir=${thisdir}/../..

 echo "$statictext" > $outfile

-bifs=`( cd ${sourcedir}/src && find . -name \*\.bif | sort )`
+bifs=`( cd ${sourcedir}/build/scripts/base && find . -name \*\.bif.bro | sort )`

 for file in $bifs
 do
-    f=${file:2}.bro
-    echo "rest_target(\${CMAKE_BINARY_DIR}/src base/$f)" >> $outfile
+    f=${file:2}
+    echo "rest_target(\${CMAKE_BINARY_DIR}/scripts base/$f)" >> $outfile
 done

 scriptfiles=`( cd ${sourcedir}/scripts && find . -name \*\.bro | sort )`
--- a/doc/upgrade.rst
+++ b/doc/upgrade.rst
@ -254,7 +254,7 @@ Variable Naming

 - Identifiers may have been renamed to conform to new `scripting
  conventions
-  <http://www.bro-ids.org/development/script-conventions.html>`_
+  <http://www.bro.org/development/script-conventions.html>`_


 BroControl
@ -296,7 +296,7 @@ Development Infrastructure
 Bro development has moved from using SVN to Git for revision control.
 Users that want to use the latest Bro development snapshot by checking it out
 from the source repositories should see the `development process
-<http://www.bro-ids.org/development/process.html>`_. Note that all the various
+<http://www.bro.org/development/process.html>`_. Note that all the various
 sub-components now reside in their own repositories. However, the
 top-level Bro repository includes them as git submodules so it's easy
 to check them all out simultaneously.
--- a/magic/COPYING
+++ b/magic/COPYING
@ -0,0 +1,29 @@
+# $File: LEGAL.NOTICE,v 1.15 2006/05/03 18:48:33 christos Exp $
+# Copyright (c) Ian F. Darwin 1986, 1987, 1989, 1990, 1991, 1992, 1994, 1995.
+# Software written by Ian F. Darwin and others;
+# maintained 1994- Christos Zoulas.
+# 
+# This software is not subject to any export provision of the United States
+# Department of Commerce, and may be exported to any country or planet.
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice immediately at the beginning of the file, without modification,
+#    this list of conditions, and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
--- a/magic/animation
+++ b/magic/animation
@ -0,0 +1,208 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: animation,v 1.47 2013/02/06 14:18:52 christos Exp $
+# animation:  file(1) magic for animation/movie formats
+#
+# animation formats
+# MPEG, FLI, DL originally from vax@ccwf.cc.utexas.edu (VaX#n8)
+# FLC, SGI, Apple originally from Daniel Quinlan (quinlan@yggdrasil.com)
+
+# SGI and Apple formats
+0	string		MOVI		Silicon Graphics movie file
+!:mime	video/x-sgi-movie
+4       string          moov            Apple QuickTime
+!:mime	video/quicktime
+4       string          mdat            Apple QuickTime movie (unoptimized)
+!:mime	video/quicktime
+#4       string          wide            Apple QuickTime movie (unoptimized)
+#!:mime	video/quicktime
+#4       string          skip            Apple QuickTime movie (modified)
+#!:mime	video/quicktime
+#4       string          free            Apple QuickTime movie (modified)
+#!:mime	video/quicktime
+4       string          idsc            Apple QuickTime image (fast start)
+!:mime	image/x-quicktime
+#4       string          idat            Apple QuickTime image (unoptimized)
+#!:mime	image/x-quicktime
+4       string          pckg            Apple QuickTime compressed archive
+!:mime	application/x-quicktime-player
+4	string/W	jP		JPEG 2000 image
+!:mime	image/jp2
+4	string		ftyp		ISO Media
+>8	string		isom		\b, MPEG v4 system, version 1
+!:mime	video/mp4
+>8	string		mp41		\b, MPEG v4 system, version 1
+!:mime	video/mp4
+>8	string		mp42		\b, MPEG v4 system, version 2
+!:mime	video/mp4
+>8	string/W	jp2		\b, JPEG 2000
+!:mime	image/jp2
+>8	string		3ge		\b, MPEG v4 system, 3GPP
+!:mime	video/3gpp
+>8	string		3gg		\b, MPEG v4 system, 3GPP
+!:mime	video/3gpp
+>8	string		3gp		\b, MPEG v4 system, 3GPP
+!:mime	video/3gpp
+>8	string		3gs		\b, MPEG v4 system, 3GPP
+!:mime	video/3gpp
+>8	string		3g2		\b, MPEG v4 system, 3GPP2
+!:mime	video/3gpp2
+>8	string		mmp4		\b, MPEG v4 system, 3GPP Mobile
+!:mime	video/mp4
+>8	string		avc1		\b, MPEG v4 system, 3GPP JVT AVC
+!:mime	video/3gpp
+>8	string/W	M4A		\b, MPEG v4 system, iTunes AAC-LC
+!:mime	audio/mp4
+>8	string/W	M4V		\b, MPEG v4 system, iTunes AVC-LC
+!:mime	video/mp4
+>8	string/W	qt		\b, Apple QuickTime movie
+!:mime	video/quicktime
+
+# MPEG sequences
+# Scans for all common MPEG header start codes
+0        belong&0xFFFFFF00  0x00000100     
+>3       byte               0xBA           MPEG sequence
+!:mime  video/mpeg
+# GRR too general as it catches also FoxPro Memo example NG.FPT
+>3       byte               0xB0           MPEG sequence, v4
+!:mime  video/mpeg4-generic
+>3       byte               0xB5           MPEG sequence, v4
+!:mime  video/mpeg4-generic
+>3       byte               0xB3           MPEG sequence
+!:mime  video/mpeg
+
+# MPEG ADTS Audio (*.mpx/mxa/aac)
+# from dreesen@math.fu-berlin.de
+# modified to fully support MPEG ADTS
+
+# MP3, M1A
+# modified by Joerg Jenderek
+# GRR the original test are too common for many DOS files
+# so don't accept as MP3 until we've tested the rate
+0       beshort&0xFFFE  0xFFFA
+# rates
+>2      byte&0xF0       0x10           MPEG ADTS, layer III, v1,  32 kbps
+!:mime	audio/mpeg
+>2      byte&0xF0       0x20           MPEG ADTS, layer III, v1,  40 kbps
+!:mime	audio/mpeg
+>2      byte&0xF0       0x30           MPEG ADTS, layer III, v1,  48 kbps
+!:mime	audio/mpeg
+>2      byte&0xF0       0x40           MPEG ADTS, layer III, v1,  56 kbps
+!:mime	audio/mpeg
+>2      byte&0xF0       0x50           MPEG ADTS, layer III, v1,  64 kbps
+!:mime	audio/mpeg
+>2      byte&0xF0       0x60           MPEG ADTS, layer III, v1,  80 kbps
+!:mime	audio/mpeg
+>2      byte&0xF0       0x70           MPEG ADTS, layer III, v1,  96 kbps
+!:mime	audio/mpeg
+>2      byte&0xF0       0x80           MPEG ADTS, layer III, v1, 112 kbps
+!:mime	audio/mpeg
+>2      byte&0xF0       0x90           MPEG ADTS, layer III, v1, 128 kbps
+!:mime	audio/mpeg
+>2      byte&0xF0       0xA0           MPEG ADTS, layer III, v1, 160 kbps
+!:mime	audio/mpeg
+>2      byte&0xF0       0xB0           MPEG ADTS, layer III, v1, 192 kbps
+!:mime	audio/mpeg
+>2      byte&0xF0       0xC0           MPEG ADTS, layer III, v1, 224 kbps
+!:mime	audio/mpeg
+>2      byte&0xF0       0xD0           MPEG ADTS, layer III, v1, 256 kbps
+!:mime	audio/mpeg
+>2      byte&0xF0       0xE0           MPEG ADTS, layer III, v1, 320 kbps
+!:mime	audio/mpeg
+
+# MP2, M1A
+0       beshort&0xFFFE  0xFFFC         MPEG ADTS, layer II, v1
+!:mime	audio/mpeg
+
+# MP3, M2A
+0       beshort&0xFFFE  0xFFF2         MPEG ADTS, layer III, v2
+!:mime	audio/mpeg
+
+# MPA, M2A
+0       beshort&0xFFFE  0xFFF6         MPEG ADTS, layer I, v2
+!:mime	audio/mpeg
+
+# MP3, M25A
+0       beshort&0xFFFE  0xFFE2         MPEG ADTS, layer III,  v2.5
+!:mime	audio/mpeg
+
+# Stored AAC streams (instead of the MP4 format)
+0       string          ADIF           MPEG ADIF, AAC
+!:mime	audio/x-hx-aac-adif
+
+# Live or stored single AAC stream (used with MPEG-2 systems)
+0       beshort&0xFFF6  0xFFF0         MPEG ADTS, AAC
+!:mime	audio/x-hx-aac-adts
+
+# Live MPEG-4 audio streams (instead of RTP FlexMux)
+0       beshort&0xFFE0  0x56E0         MPEG-4 LOAS
+!:mime	audio/x-mp4a-latm
+
+# This magic isn't strong enough (matches plausible ISO-8859-1 text)
+#0       beshort         0x4DE1         MPEG-4 LO-EP audio stream
+#!:mime	audio/x-mp4a-latm
+
+# Summary: FLI animation format
+# Created by: Daniel Quinlan <quinlan@yggdrasil.com>
+# Modified by (1): Abel Cheung <abelcheung@gmail.com> (avoid over-generic detection)
+4	leshort		0xAF11
+# standard FLI always has 320x200 resolution and 8 bit color
+>8	leshort		320
+>>10	leshort		200
+>>>12	leshort		8			FLI animation, 320x200x8
+!:mime	video/x-fli
+
+# Summary: FLC animation format
+# Created by: Daniel Quinlan <quinlan@yggdrasil.com>
+# Modified by (1): Abel Cheung <abelcheung@gmail.com> (avoid over-generic detection)
+4	leshort		0xAF12
+# standard FLC always use 8 bit color
+>12	leshort		8			FLC animation
+!:mime	video/x-flc
+
+# Microsoft Advanced Streaming Format (ASF) <mpruett@sgi.com>
+0	belong			0x3026b275	Microsoft ASF
+!:mime  video/x-ms-asf
+
+# MNG Video Format, <URL:http://www.libpng.org/pub/mng/spec/>
+0	string			\x8aMNG		MNG video data,
+!:mime	video/x-mng
+
+# JNG Video Format, <URL:http://www.libpng.org/pub/mng/spec/>
+0	string			\x8bJNG		JNG video data,
+!:mime	video/x-jng
+
+# VRML (Virtual Reality Modelling Language)
+0       string/w        #VRML\ V1.0\ ascii	VRML 1 file
+!:mime	model/vrml
+0	string/w	#VRML\ V2.0\ utf8	ISO/IEC 14772 VRML 97 file
+!:mime	model/vrml
+
+# X3D (Extensible 3D) [http://www.web3d.org/specifications/x3d-3.0.dtd]
+# From Michel Briand <michelbriand@free.fr>
+0	string/t		\<?xml\ version="
+!:strength +1
+>20	search/1000/cw  \<!DOCTYPE\ X3D		X3D (Extensible 3D) model xml text
+!:mime model/x3d
+
+#						MPEG file
+# MPEG sequences
+# FIXME: This section is from the old magic.mime file and needs integrating with the rest
+0       belong             0x000001BA
+>4      byte               &0x40
+!:mime	video/mp2p
+>4      byte               ^0x40
+!:mime	video/mpeg
+0       belong             0x000001BB
+!:mime	video/mpeg
+0       belong             0x000001B0
+!:mime	video/mp4v-es
+0       belong             0x000001B5
+!:mime	video/mp4v-es
+0       belong             0x000001B3
+!:mime	video/mpv
+0       belong&0xFF5FFF1F  0x47400010
+!:mime	video/mp2t
+0       belong             0x00000001
+>4      byte&0x1F	   0x07
+!:mime	video/h264
--- a/magic/archive
+++ b/magic/archive
@ -0,0 +1,242 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: archive,v 1.78 2013/02/06 14:18:52 christos Exp $
+# archive:  file(1) magic for archive formats (see also "msdos" for self-
+#           extracting compressed archives)
+#
+# cpio, ar, arc, arj, hpack, lha/lharc, rar, squish, uc2, zip, zoo, etc.
+# pre-POSIX "tar" archives are handled in the C code.
+
+# POSIX tar archives
+257	string		ustar\0		POSIX tar archive
+!:mime	application/x-tar # encoding: posix
+257	string		ustar\040\040\0	GNU tar archive
+!:mime	application/x-tar # encoding: gnu
+
+# cpio archives
+#
+# Yes, the top two "cpio archive" formats *are* supposed to just be "short".
+# The idea is to indicate archives produced on machines with the same
+# byte order as the machine running "file" with "cpio archive", and
+# to indicate archives produced on machines with the opposite byte order
+# from the machine running "file" with "byte-swapped cpio archive".
+#
+# The SVR4 "cpio(4)" hints that there are additional formats, but they
+# are defined as "short"s; I think all the new formats are
+# character-header formats and thus are strings, not numbers.
+0	short		070707		cpio archive
+!:mime	application/x-cpio
+0	short		0143561		byte-swapped cpio archive
+!:mime	application/x-cpio # encoding: swapped
+
+#
+# System V Release 1 portable(?) archive format.
+#
+0	string		=<ar>		System V Release 1 ar archive
+!:mime	application/x-archive
+
+#
+# Debian package; it's in the portable archive format, and needs to go
+# before the entry for regular portable archives, as it's recognized as
+# a portable archive whose first member has a name beginning with
+# "debian".
+#
+0	string		=!<arch>\ndebian
+!:mime	application/x-debian-package
+
+#
+# MIPS archive; they're in the portable archive format, and need to go
+# before the entry for regular portable archives, as it's recognized as
+# a portable archive whose first member has a name beginning with
+# "__________E".
+#
+0	string	=!<arch>\n__________E	MIPS archive
+!:mime	application/x-archive
+
+#
+# BSD/SVR2-and-later portable archive formats.
+#
+0	string		=!<arch>		current ar archive
+!:mime	application/x-archive
+
+# ARC archiver, from Daniel Quinlan (quinlan@yggdrasil.com)
+#
+# The first byte is the magic (0x1a), byte 2 is the compression type for
+# the first file (0x01 through 0x09), and bytes 3 to 15 are the MS-DOS
+# filename of the first file (null terminated).  Since some types collide
+# we only test some types on basis of frequency: 0x08 (83%), 0x09 (5%),
+# 0x02 (5%), 0x03 (3%), 0x04 (2%), 0x06 (2%).  0x01 collides with terminfo.
+0	lelong&0x8080ffff	0x0000081a	ARC archive data, dynamic LZW
+!:mime	application/x-arc
+0	lelong&0x8080ffff	0x0000091a	ARC archive data, squashed
+!:mime	application/x-arc
+0	lelong&0x8080ffff	0x0000021a	ARC archive data, uncompressed
+!:mime	application/x-arc
+0	lelong&0x8080ffff	0x0000031a	ARC archive data, packed
+!:mime	application/x-arc
+0	lelong&0x8080ffff	0x0000041a	ARC archive data, squeezed
+!:mime	application/x-arc
+0	lelong&0x8080ffff	0x0000061a	ARC archive data, crunched
+!:mime	application/x-arc
+# [JW] stuff taken from idarc, obviously ARC successors:
+0	lelong&0x8080ffff	0x00000a1a	PAK archive data
+!:mime	application/x-arc
+0	lelong&0x8080ffff	0x0000141a	ARC+ archive data
+!:mime	application/x-arc
+0	lelong&0x8080ffff	0x0000481a	HYP archive data
+!:mime	application/x-arc
+
+# ARJ archiver (jason@jarthur.Claremont.EDU)
+0	leshort		0xea60		ARJ archive data
+!:mime	application/x-arj
+
+# LHARC/LHA archiver (Greg Roelofs, newt@uchicago.edu)
+2	string		-lh0-		LHarc 1.x/ARX archive data [lh0]
+!:mime	application/x-lharc
+2	string		-lh1-		LHarc 1.x/ARX archive data [lh1]
+!:mime	application/x-lharc
+2	string		-lz4-		LHarc 1.x archive data [lz4]
+!:mime	application/x-lharc
+2	string		-lz5-		LHarc 1.x archive data [lz5]
+!:mime	application/x-lharc
+#	[never seen any but the last; -lh4- reported in comp.compression:]
+2	string		-lzs-		LHa/LZS archive data [lzs]
+!:mime	application/x-lha
+2	string		-lh\40-		LHa 2.x? archive data [lh ]
+!:mime	application/x-lha
+2	string		-lhd-		LHa 2.x? archive data [lhd]
+!:mime	application/x-lha
+2	string		-lh2-		LHa 2.x? archive data [lh2]
+!:mime	application/x-lha
+2	string		-lh3-		LHa 2.x? archive data [lh3]
+!:mime	application/x-lha
+2	string		-lh4-		LHa (2.x) archive data [lh4]
+!:mime	application/x-lha
+2	string		-lh5-		LHa (2.x) archive data [lh5]
+!:mime	application/x-lha
+2	string		-lh6-		LHa (2.x) archive data [lh6]
+!:mime	application/x-lha
+2	string		-lh7-		LHa (2.x)/LHark archive data [lh7]
+!:mime	application/x-lha
+
+# RAR archiver (Greg Roelofs, newt@uchicago.edu)
+0	string		Rar!		RAR archive data,
+!:mime	application/x-rar
+
+# PKZIP multi-volume archive
+0	string		PK\x07\x08PK\x03\x04	Zip multi-volume archive data, at least PKZIP v2.50 to extract
+!:mime	application/zip
+
+# Zip archives (Greg Roelofs, c/o zip-bugs@wkuvx1.wku.edu)
+0	string		PK\003\004
+
+# Specialised zip formats which start with a member named 'mimetype'
+# (stored uncompressed, with no 'extra field') containing the file's MIME type.
+# Check for have 8-byte name, 0-byte extra field, name "mimetype", and
+#  contents starting with "application/":
+>26	string		\x8\0\0\0mimetypeapplication/
+
+#   OpenDocument formats (for OpenOffice 2.x / StarOffice >= 8)
+#    http://lists.oasis-open.org/archives/office/200505/msg00006.html
+#    (mimetype contains "application/vnd.oasis.opendocument.<SUBTYPE>")
+>>50	string	vnd.oasis.opendocument.	OpenDocument
+>>>73	string	text
+>>>>77	byte	!0x2d			Text
+!:mime	application/vnd.oasis.opendocument.text
+>>>>77	string	-template		Text Template
+!:mime	application/vnd.oasis.opendocument.text-template
+>>>>77	string	-web			HTML Document Template
+!:mime	application/vnd.oasis.opendocument.text-web
+>>>>77	string	-master			Master Document
+!:mime	application/vnd.oasis.opendocument.text-master
+>>>73	string	graphics
+>>>>81	byte	!0x2d			Drawing
+!:mime	application/vnd.oasis.opendocument.graphics
+>>>>81	string	-template		Template
+!:mime	application/vnd.oasis.opendocument.graphics-template
+>>>73	string	presentation
+>>>>85	byte	!0x2d			Presentation
+!:mime	application/vnd.oasis.opendocument.presentation
+>>>>85	string	-template		Template
+!:mime	application/vnd.oasis.opendocument.presentation-template
+>>>73	string	spreadsheet
+>>>>84	byte	!0x2d			Spreadsheet
+!:mime	application/vnd.oasis.opendocument.spreadsheet
+>>>>84	string	-template		Template
+!:mime	application/vnd.oasis.opendocument.spreadsheet-template
+>>>73	string	chart
+>>>>78	byte	!0x2d			Chart
+!:mime	application/vnd.oasis.opendocument.chart
+>>>>78	string	-template		Template
+!:mime	application/vnd.oasis.opendocument.chart-template
+>>>73	string	formula
+>>>>80	byte	!0x2d			Formula
+!:mime	application/vnd.oasis.opendocument.formula
+>>>>80	string	-template		Template
+!:mime	application/vnd.oasis.opendocument.formula-template
+>>>73	string	database		Database
+!:mime	application/vnd.oasis.opendocument.database
+>>>73	string	image
+>>>>78	byte	!0x2d			Image
+!:mime	application/vnd.oasis.opendocument.image
+>>>>78	string	-template		Template
+!:mime	application/vnd.oasis.opendocument.image-template
+
+#  EPUB (OEBPS) books using OCF (OEBPS Container Format)
+#    http://www.idpf.org/ocf/ocf1.0/download/ocf10.htm, section 4.
+#    From: Ralf Brown <ralf.brown@gmail.com>
+>0x1E	string	mimetypeapplication/epub+zip	EPUB document
+!:mime application/epub+zip
+
+#  Catch other ZIP-with-mimetype formats
+#	In a ZIP file, the bytes immediately after a member's contents are
+#	always "PK". The 2 regex rules here print the "mimetype" member's
+#	contents up to the first 'P'. Luckily, most MIME types don't contain
+#	any capital 'P's. This is a kludge.
+#    (mimetype contains "application/<OTHER>")
+>>50		string	!epub+zip
+>>>50		string	!vnd.oasis.opendocument.
+>>>>50		string	!vnd.sun.xml.
+>>>>>50		string	!vnd.kde.
+>>>>>>38	regex	[!-OQ-~]+		Zip data (MIME type "%s"?)
+!:mime	application/zip
+#    (mimetype contents other than "application/*")
+>26		string	\x8\0\0\0mimetype
+>>38		string	!application/
+>>>38		regex	[!-OQ-~]+		Zip data (MIME type "%s"?)
+!:mime	application/zip
+
+# Java Jar files
+>(26.s+30)	leshort	0xcafe		Java Jar file data (zip)
+!:mime	application/jar
+
+# Generic zip archives (Greg Roelofs, c/o zip-bugs@wkuvx1.wku.edu)
+#   Next line excludes specialized formats:
+>(26.s+30)	leshort	!0xcafe
+>>26    string          !\x8\0\0\0mimetype	Zip archive data
+!:mime	application/zip
+
+# Zoo archiver
+20	lelong		0xfdc4a7dc	Zoo archive data
+!:mime	application/x-zoo
+
+# Shell archives
+10	string		#\ This\ is\ a\ shell\ archive	shell archive text
+!:mime	application/octet-stream
+
+# Felix von Leitner <felix-file@fefe.de>
+0	string	d8:announce	BitTorrent file
+!:mime	application/x-bittorrent
+
+# EET archive
+# From: Tilman Sauerbeck <tilman@code-monkey.de>
+0	belong	0x1ee7ff00	EET archive
+!:mime	application/x-eet
+
+# Symbian installation files
+#  http://www.thouky.co.uk/software/psifs/sis.html
+#  http://developer.symbian.com/main/downloads/papers/SymbianOSv91/softwareinstallsis.pdf
+8	lelong	0x10000419	Symbian installation file
+!:mime	application/vnd.symbian.install
+0	lelong	0x10201A7A	Symbian installation file (Symbian OS 9.x)
+!:mime	x-epoc/x-sisx-app
--- a/magic/assembler
+++ b/magic/assembler
@ -0,0 +1,19 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: assembler,v 1.3 2013/01/04 17:23:28 christos Exp $
+# make:  file(1) magic for assembler source
+#
+0	regex	\^[\020\t]*\\.asciiz		assembler source text
+!:mime	text/x-asm
+0	regex	\^[\020\t]*\\.byte		assembler source text
+!:mime	text/x-asm
+0	regex	\^[\020\t]*\\.even		assembler source text
+!:mime	text/x-asm
+0	regex	\^[\020\t]*\\.globl		assembler source text
+!:mime	text/x-asm
+0	regex	\^[\020\t]*\\.text		assembler source text
+!:mime	text/x-asm
+0	regex	\^[\020\t]*\\.file		assembler source text
+!:mime	text/x-asm
+0	regex	\^[\020\t]*\\.type		assembler source text
+!:mime	text/x-asm
--- a/magic/audio
+++ b/magic/audio
@ -0,0 +1,149 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: audio,v 1.65 2012/10/31 13:38:40 christos Exp $
+# audio:  file(1) magic for sound formats (see also "iff")
+#
+# Jan Nicolai Langfeldt (janl@ifi.uio.no), Dan Quinlan (quinlan@yggdrasil.com),
+# and others
+#
+
+# Sun/NeXT audio data
+0	string		.snd		Sun/NeXT audio data:
+>12	belong		1		8-bit ISDN mu-law,
+!:mime	audio/basic
+>12	belong		2		8-bit linear PCM [REF-PCM],
+!:mime	audio/basic
+>12	belong		3		16-bit linear PCM,
+!:mime	audio/basic
+>12	belong		4		24-bit linear PCM,
+!:mime	audio/basic
+>12	belong		5		32-bit linear PCM,
+!:mime	audio/basic
+>12	belong		6		32-bit IEEE floating point,
+!:mime	audio/basic
+>12	belong		7		64-bit IEEE floating point,
+!:mime	audio/basic
+>12	belong		23		8-bit ISDN mu-law compressed (CCITT G.721 ADPCM voice enc.),
+!:mime  audio/x-adpcm
+
+# DEC systems (e.g. DECstation 5000) use a variant of the Sun/NeXT format
+# that uses little-endian encoding and has a different magic number
+0	lelong		0x0064732E	DEC audio data:
+>12	lelong		1		8-bit ISDN mu-law,
+!:mime	audio/x-dec-basic
+>12	lelong		2		8-bit linear PCM [REF-PCM],
+!:mime	audio/x-dec-basic
+>12	lelong		3		16-bit linear PCM,
+!:mime	audio/x-dec-basic
+>12	lelong		4		24-bit linear PCM,
+!:mime	audio/x-dec-basic
+>12	lelong		5		32-bit linear PCM,
+!:mime	audio/x-dec-basic
+>12	lelong		6		32-bit IEEE floating point,
+!:mime	audio/x-dec-basic
+>12	lelong		7		64-bit IEEE floating point,
+!:mime	audio/x-dec-basic
+>12	lelong		23		8-bit ISDN mu-law compressed (CCITT G.721 ADPCM voice enc.),
+!:mime	audio/x-dec-basic
+
+# Creative Labs AUDIO stuff
+0	string	MThd			Standard MIDI data
+!:mime	audio/midi
+
+0	string	CTMF			Creative Music (CMF) data
+!:mime	audio/x-unknown
+0	string	SBI			SoundBlaster instrument data
+!:mime	audio/x-unknown
+0	string	Creative\ Voice\ File	Creative Labs voice data
+!:mime	audio/x-unknown
+
+# Real Audio (Magic .ra\0375)
+0	belong		0x2e7261fd	RealAudio sound file
+!:mime	audio/x-pn-realaudio
+0	string		.RMF\0\0\0	RealMedia file
+!:mime	application/vnd.rn-realmedia
+
+# mime types according to http://www.geocities.com/nevilo/mod.htm:
+#	audio/it	.it
+#	audio/x-zipped-it	.itz
+#	audio/xm	fasttracker modules
+#	audio/x-s3m	screamtracker modules
+#	audio/s3m	screamtracker modules
+#	audio/x-zipped-mod	mdz
+#	audio/mod	mod
+#	audio/x-mod	All modules (mod, s3m, 669, mtm, med, xm, it, mdz, stm, itz, xmz, s3z)
+
+#
+# Taken from loader code from mikmod version 2.14
+# by Steve McIntyre (stevem@chiark.greenend.org.uk)
+# <doj@cubic.org> added title printing on 2003-06-24
+0	string	MAS_UTrack_V00
+>14	string	>/0		ultratracker V1.%.1s module sound data
+!:mime	audio/x-mod
+#audio/x-tracker-module
+
+0	string	Extended\ Module: Fasttracker II module sound data
+!:mime	audio/x-mod
+#audio/x-tracker-module
+
+21	string/c	=!SCREAM!	Screamtracker 2 module sound data
+!:mime	audio/x-mod
+#audio/x-screamtracker-module
+21	string	BMOD2STM	Screamtracker 2 module sound data
+!:mime	audio/x-mod
+#audio/x-screamtracker-module
+1080	string	M.K.		4-channel Protracker module sound data
+!:mime	audio/x-mod
+#audio/x-protracker-module
+1080	string	M!K!		4-channel Protracker module sound data
+!:mime	audio/x-mod
+#audio/x-protracker-module
+1080	string	FLT4		4-channel Startracker module sound data
+!:mime	audio/x-mod
+#audio/x-startracker-module
+1080	string	FLT8		8-channel Startracker module sound data
+!:mime	audio/x-mod
+#audio/x-startracker-module
+1080	string	4CHN		4-channel Fasttracker module sound data
+!:mime	audio/x-mod
+#audio/x-fasttracker-module
+1080	string	6CHN		6-channel Fasttracker module sound data
+!:mime	audio/x-mod
+#audio/x-fasttracker-module
+1080	string	8CHN		8-channel Fasttracker module sound data
+!:mime	audio/x-mod
+#audio/x-fasttracker-module
+1080	string	CD81		8-channel Octalyser module sound data
+!:mime	audio/x-mod
+#audio/x-octalysertracker-module
+1080	string	OKTA		8-channel Octalyzer module sound data
+!:mime	audio/x-mod
+#audio/x-octalysertracker-module
+# Not good enough.
+#1082	string	CH
+#>1080	string	>/0		%.2s-channel Fasttracker "oktalyzer" module sound data
+1080	string	16CN		16-channel Taketracker module sound data
+!:mime	audio/x-mod
+#audio/x-taketracker-module
+1080	string	32CN		32-channel Taketracker module sound data
+!:mime	audio/x-mod
+#audio/x-taketracker-module
+
+# Impulse tracker module (audio/x-it)
+0	string		IMPM		Impulse Tracker module sound data -
+!:mime	audio/x-mod
+
+# Free lossless audio codec <http://flac.sourceforge.net>
+# From: Przemyslaw Augustyniak <silvathraec@rpg.pl>
+0	string			fLaC		FLAC audio bitstream data
+!:mime	audio/x-flac
+
+# Monkey's Audio compressed audio format (.ape)
+# From danny.milo@gmx.net (Danny Milosavljevic)
+# New version from Abel Cheung <abel (@) oaka.org>
+0		string		MAC\040		Monkey's Audio compressed format
+!:mime audio/x-ape
+
+# musepak support From: "Jiri Pejchal" <jiri.pejchal@gmail.com>
+0       string          MP+     Musepack audio
+!:mime	audio/x-musepack
--- a/magic/c-lang
+++ b/magic/c-lang
@ -0,0 +1,47 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: c-lang,v 1.16 2011/12/09 08:02:16 rrt Exp $
+# c-lang:  file(1) magic for C and related languages programs
+#
+
+# BCPL
+0	search/8192	"libhdr"	BCPL source text
+!:mime	text/x-bcpl
+0	search/8192	"LIBHDR"	BCPL source text
+!:mime	text/x-bcpl
+
+# C
+0	regex	\^#include	C source text
+!:mime	text/x-c
+0	regex	\^char		C source text
+!:mime	text/x-c
+0	regex	\^double		C source text
+!:mime	text/x-c
+0	regex	\^extern		C source text
+!:mime	text/x-c
+0	regex	\^float		C source text
+!:mime	text/x-c
+0	regex	\^struct		C source text
+!:mime	text/x-c
+0	regex	\^union		C source text
+!:mime	text/x-c
+0	search/8192	main(		C source text
+!:mime	text/x-c
+
+# C++
+# The strength of these rules is increased so they beat the C rules above
+0	regex	\^template	C++ source text
+!:strength + 5
+!:mime	text/x-c++
+0	regex	\^virtual		C++ source text
+!:strength + 5
+!:mime	text/x-c++
+0	regex	\^class		C++ source text
+!:strength + 5
+!:mime	text/x-c++
+0	regex	\^public:		C++ source text
+!:strength + 5
+!:mime	text/x-c++
+0	regex	\^private:		C++ source text
+!:strength + 5
+!:mime	text/x-c++
--- a/magic/cafebabe
+++ b/magic/cafebabe
@ -0,0 +1,31 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: cafebabe,v 1.13 2013/02/26 21:04:38 christos Exp $
+# Cafe Babes unite!
+#
+# Since Java bytecode and Mach-O universal binaries have the same magic number,
+# the test must be performed in the same "magic" sequence to get both right.
+# The long at offset 4 in a Mach-O universal binary tells the number of
+# architectures; the short at offset 4 in a Java bytecode file is the JVM minor
+# version and the short at offset 6 is the JVM major version.  Since there are only 
+# only 18 labeled Mach-O architectures at current, and the first released 
+# Java class format was version 43.0, we can safely choose any number
+# between 18 and 39 to test the number of architectures against
+# (and use as a hack). Let's not use 18, because the Mach-O people
+# might add another one or two as time goes by...
+#
+### JAVA START ###
+0	belong		0xcafebabe
+!:mime	application/x-java-applet
+
+0	belong		0xcafed00d	JAR compressed with pack200,
+>5	byte		x		version %d.
+>4	byte		x		\b%d
+!:mime	application/x-java-pack200
+
+0	belong		0xcafed00d	JAR compressed with pack200,
+>5	byte		x		version %d.
+>4	byte		x		\b%d
+!:mime	application/x-java-pack200
+
+### JAVA END ###
--- a/magic/commands
+++ b/magic/commands
@ -0,0 +1,82 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: commands,v 1.44 2013/02/05 15:20:47 christos Exp $
+# commands:  file(1) magic for various shells and interpreters
+#
+#0	string/w	:			shell archive or script for antique kernel text
+0	string/wt	#!\ /bin/sh		POSIX shell script text executable
+!:mime	text/x-shellscript
+0	string/wt	#!\ /bin/csh		C shell script text executable
+!:mime	text/x-shellscript
+# korn shell magic, sent by George Wu, gwu@clyde.att.com
+0	string/wt	#!\ /bin/ksh		Korn shell script text executable
+!:mime	text/x-shellscript
+0	string/wt 	#!\ /bin/tcsh		Tenex C shell script text executable
+!:mime	text/x-shellscript
+0	string/wt	#!\ /usr/bin/tcsh	Tenex C shell script text executable
+!:mime	text/x-shellscript
+0	string/wt 	#!\ /usr/local/tcsh	Tenex C shell script text executable
+!:mime	text/x-shellscript
+0	string/wt	#!\ /usr/local/bin/tcsh	Tenex C shell script text executable
+!:mime	text/x-shellscript
+
+#
+# zsh/ash/ae/nawk/gawk magic from cameron@cs.unsw.oz.au (Cameron Simpson)
+0	string/wt	#!\ /bin/zsh		Paul Falstad's zsh script text executable
+!:mime	text/x-shellscript
+0	string/wt	#!\ /usr/bin/zsh	Paul Falstad's zsh script text executable
+!:mime	text/x-shellscript
+0	string/wt	#!\ /usr/local/bin/zsh	Paul Falstad's zsh script text executable
+!:mime	text/x-shellscript
+0	string/wt	#!\ /usr/local/bin/ash	Neil Brown's ash script text executable
+!:mime	text/x-shellscript
+0	string/wt	#!\ /usr/local/bin/ae	Neil Brown's ae script text executable
+!:mime	text/x-shellscript
+0	string/wt	#!\ /bin/nawk		new awk script text executable
+!:mime	text/x-nawk
+0	string/wt	#!\ /usr/bin/nawk	new awk script text executable
+!:mime	text/x-nawk
+0	string/wt	#!\ /usr/local/bin/nawk	new awk script text executable
+!:mime	text/x-nawk
+0	string/wt	#!\ /bin/gawk		GNU awk script text executable
+!:mime	text/x-gawk
+0	string/wt	#!\ /usr/bin/gawk	GNU awk script text executable
+!:mime	text/x-gawk
+0	string/wt	#!\ /usr/local/bin/gawk	GNU awk script text executable
+!:mime	text/x-gawk
+#
+0	string/wt	#!\ /bin/awk		awk script text executable
+!:mime	text/x-awk
+0	string/wt	#!\ /usr/bin/awk	awk script text executable
+!:mime	text/x-awk
+
+# bash shell magic, from Peter Tobias (tobias@server.et-inf.fho-emden.de)
+0	string/wt	#!\ /bin/bash	Bourne-Again shell script text executable
+!:mime	text/x-shellscript
+0	string/wt	#!\ /usr/bin/bash	Bourne-Again shell script text executable
+!:mime	text/x-shellscript
+0	string/wt	#!\ /usr/local/bash	Bourne-Again shell script text executable
+!:mime	text/x-shellscript
+0	string/wt	#!\ /usr/local/bin/bash	Bourne-Again shell script text executable
+!:mime	text/x-shellscript
+
+# PHP scripts
+# Ulf Harnhammar <ulfh@update.uu.se>
+0	search/1/c	=<?php			PHP script text
+!:strength + 10
+!:mime	text/x-php
+0	search/1	=<?\n			PHP script text
+!:mime	text/x-php
+0	search/1	=<?\r			PHP script text
+!:mime	text/x-php
+0	search/1/w	#!\ /usr/local/bin/php	PHP script text executable
+!:strength + 10
+!:mime	text/x-php
+0	search/1/w	#!\ /usr/bin/php	PHP script text executable
+!:strength + 10
+!:mime	text/x-php
+# Smarty compiled template, http://www.smarty.net/
+# Elan Ruusamae <glen@delfi.ee>
+0	string	=<?php\ /*\ Smarty\ version	Smarty compiled template
+>24	regex	[0-9.]+				\b, version %s
+!:mime	text/x-php
--- a/magic/compress
+++ b/magic/compress
@ -0,0 +1,77 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: compress,v 1.48 2011/12/07 18:39:43 christos Exp $
+# compress:  file(1) magic for pure-compression formats (no archives)
+#
+# compress, gzip, pack, compact, huf, squeeze, crunch, freeze, yabba, etc.
+#
+# Formats for various forms of compressed data
+# Formats for "compress" proper have been moved into "compress.c",
+# because it tries to uncompress it to figure out what's inside.
+
+# standard unix compress
+0	string		\037\235	compress'd data
+!:mime	application/x-compress
+!:apple	LZIVZIVU
+
+# gzip (GNU zip, not to be confused with Info-ZIP or PKWARE zip archiver)
+#   Edited by Chris Chittleborough <cchittleborough@yahoo.com.au>, March 2002
+#	* Original filename is only at offset 10 if "extra field" absent
+#	* Produce shorter output - notably, only report compression methods
+#         other than 8 ("deflate", the only method defined in RFC 1952).
+0       string          \037\213        gzip compressed data
+!:mime	application/x-gzip
+
+# packed data, Huffman (minimum redundancy) codes on a byte-by-byte basis
+0	string		\037\036	packed data
+!:mime	application/octet-stream
+
+#
+# This magic number is byte-order-independent.
+0	short		0x1f1f		old packed data
+!:mime	application/octet-stream
+
+# XXX - why *two* entries for "compacted data", one of which is
+# byte-order independent, and one of which is byte-order dependent?
+#
+0	short		0x1fff		compacted data
+!:mime	application/octet-stream
+# This string is valid for SunOS (BE) and a matching "short" is listed
+# in the Ultrix (LE) magic file.
+0	string		\377\037	compacted data
+!:mime	application/octet-stream
+0	short		0145405		huf output
+!:mime	application/octet-stream
+
+# bzip2
+0	string		BZh		bzip2 compressed data
+!:mime	application/x-bzip2
+
+# lzip
+0	string		LZIP		lzip compressed data
+!:mime application/x-lzip
+
+# 7-zip archiver, from Thomas Klausner (wiz@danbala.tuwien.ac.at)
+# http://www.7-zip.org or DOC/7zFormat.txt
+#
+0	string		7z\274\257\047\034	7-zip archive data,
+>6	byte		x			version %d
+>7	byte		x			\b.%d
+!:mime	application/x-7z-compressed
+
+# Type: LZMA
+0	lelong&0xffffff	=0x5d
+>12	leshort		=0xff			LZMA compressed data,
+>>5	lequad		=0xffffffffffffffff	streamed
+>>5	lequad		!0xffffffffffffffff	non-streamed, size %lld
+!:mime	application/x-lzma
+
+# http://tukaani.org/xz/xz-file-format.txt
+0	ustring		\xFD7zXZ\x00		XZ compressed data
+!:mime	application/x-xz
+
+# https://github.com/ckolivas/lrzip/blob/master/doc/magic.header.txt
+0	string		LRZI			LRZIP compressed data
+>4	byte		x			- version %d
+>5	byte		x			\b.%d
+!:mime	application/x-lrzip
--- a/magic/database
+++ b/magic/database
@ -0,0 +1,47 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: database,v 1.32 2013/02/06 14:18:52 christos Exp $
+# database:  file(1) magic for various databases
+#
+# extracted from header/code files by Graeme Wilford (eep2gw@ee.surrey.ac.uk)
+#
+#
+# GDBM magic numbers
+#  Will be maintained as part of the GDBM distribution in the future.
+#  <downsj@teeny.org>
+0	belong	0x13579ace	GNU dbm 1.x or ndbm database, big endian
+!:mime	application/x-gdbm
+0	lelong	0x13579ace	GNU dbm 1.x or ndbm database, little endian
+!:mime	application/x-gdbm
+0	string	GDBM		GNU dbm 2.x database
+!:mime	application/x-gdbm
+#
+# Berkeley DB
+#
+# Ian Darwin's file /etc/magic files: big/little-endian version.
+#
+# Hash 1.85/1.86 databases store metadata in network byte order.
+# Btree 1.85/1.86 databases store the metadata in host byte order.
+# Hash and Btree 2.X and later databases store the metadata in host byte order.
+
+0	long	0x00061561	Berkeley DB
+!:mime	application/x-dbm
+
+# MS Access database
+4	string	Standard\ Jet\ DB	Microsoft Access Database
+!:mime	application/x-msaccess
+4	string	Standard\ ACE\ DB	Microsoft Access Database
+!:mime	application/x-msaccess
+
+# Tokyo Cabinet magic data
+# http://tokyocabinet.sourceforge.net/index.html
+0	string		ToKyO\ CaBiNeT\n	Tokyo Cabinet
+>14	string		x			\b (%s)
+>32	byte		0			\b, Hash
+!:mime	application/x-tokyocabinet-hash
+>32	byte		1			\b, B+ tree
+!:mime	application/x-tokyocabinet-btree
+>32	byte		2			\b, Fixed-length
+!:mime	application/x-tokyocabinet-fixed
+>32	byte		3			\b, Table
+!:mime	application/x-tokyocabinet-table
--- a/magic/diff
+++ b/magic/diff
@ -0,0 +1,25 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: diff,v 1.13 2012/06/16 14:43:36 christos Exp $
+# diff:  file(1) magic for diff(1) output
+#
+0	search/1	diff\ 		diff output text
+!:mime	text/x-diff
+0	search/1	***\ 		diff output text
+!:mime	text/x-diff
+0	search/1	Only\ in\ 	diff output text
+!:mime	text/x-diff
+0	search/1	Common\ subdirectories:\ 	diff output text
+!:mime	text/x-diff
+
+0	search/1	Index:		RCS/CVS diff output text
+!:mime	text/x-diff
+
+# unified diff
+0	search/4096	---\ 
+>&0	search/1024 \n
+>>&0	search/1 +++\ 
+>>>&0	search/1024 \n
+>>>>&0	search/1 @@	unified diff output text
+!:mime	text/x-diff
+!:strength + 90
--- a/magic/elf
+++ b/magic/elf
@ -0,0 +1,43 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# elf:  file(1) magic for ELF executables
+#
+# We have to check the byte order flag to see what byte order all the
+# other stuff in the header is in.
+#
+# What're the correct byte orders for the nCUBE and the Fujitsu VPP500?
+#
+# Created by: unknown
+# Modified by (1): Daniel Quinlan <quinlan@yggdrasil.com>
+# Modified by (2): Peter Tobias <tobias@server.et-inf.fho-emden.de> (core support)
+# Modified by (3): Christian 'Dr. Disk' Hechelmann <drdisk@ds9.au.s.shuttle.de> (fix of core support)
+# Modified by (4): <gerardo.cacciari@gmail.com> (VMS Itanium)
+# Modified by (5): Matthias Urlichs <smurf@debian.org> (Listing of many architectures)
+0	string		\177ELF		ELF
+>4	byte		0		invalid class
+>4	byte		1		32-bit
+>4	byte		2		64-bit
+>5	byte		0		invalid byte order
+>5	byte		1		LSB
+>>16	leshort		0		no file type,
+!:strength *2
+!:mime	application/octet-stream
+>>16	leshort		1		relocatable,
+!:mime	application/x-object
+>>16	leshort		2		executable,
+!:mime	application/x-executable
+>>16	leshort		3		shared object,
+!:mime	application/x-sharedlib
+>>16	leshort		4		core file
+!:mime	application/x-coredump
+>5	byte		2		MSB
+>>16	beshort		0		no file type,
+!:mime	application/octet-stream
+>>16	beshort		1		relocatable,
+!:mime	application/x-object
+>>16	beshort		2		executable,
+!:mime	application/x-executable
+>>16	beshort		3		shared object,
+!:mime	application/x-sharedlib
+>>16	beshort		4		core file,
+!:mime	application/x-coredump
--- a/magic/epoc
+++ b/magic/epoc
@ -0,0 +1,34 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: epoc,v 1.7 2009/09/19 16:28:09 christos Exp $
+# EPOC : file(1) magic for EPOC documents [Psion Series 5/Osaris/Geofox 1]
+# Stefan Praszalowicz <hpicollo@worldnet.fr> and Peter Breitenlohner <peb@mppmu.mpg.de>
+# Useful information for improving this file can be found at:
+# http://software.frodo.looijaard.name/psiconv/formats/Index.html
+#------------------------------------------------------------------------------
+0	lelong		0x10000037	Psion Series 5
+>4	lelong		0x10000042	multi-bitmap image
+!:mime image/x-epoc-mbm
+>4	lelong		0x1000006D
+>>8	lelong		0x1000007D	Sketch image
+!:mime image/x-epoc-sketch
+>>8	lelong		0x1000007F	Word file
+!:mime application/x-epoc-word
+>>8	lelong		0x10000085	OPL program (TextEd)
+!:mime application/x-epoc-opl
+>>8	lelong		0x10000088	Sheet file
+!:mime application/x-epoc-sheet
+>4	lelong		0x10000073	OPO module
+!:mime application/x-epoc-opo
+>4	lelong		0x10000074	OPL application
+!:mime application/x-epoc-app
+
+
+0	lelong		0x10000050	Psion Series 5
+>4	lelong		0x1000006D	database
+>>8	lelong		0x10000084	Agenda file
+!:mime application/x-epoc-agenda
+>>8	lelong		0x10000086	Data file
+!:mime application/x-epoc-data
+>>8	lelong		0x10000CEA	Jotter file
+!:mime application/x-epoc-jotter
--- a/magic/filesystems
+++ b/magic/filesystems
@ -0,0 +1,12 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: filesystems,v 1.76 2013/02/18 18:45:41 christos Exp $
+# filesystems:  file(1) magic for different filesystems
+#
+
+# CDROM Filesystems
+# Modified for UDF by gerardo.cacciari@gmail.com
+32769	string    CD001     #
+!:mime	application/x-iso9660-image
+37633	string    CD001     ISO 9660 CD-ROM filesystem data (raw 2352 byte sectors)
+!:mime	application/x-iso9660-image
--- a/magic/flash
+++ b/magic/flash
@ -0,0 +1,18 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: flash,v 1.8 2009/09/19 16:28:09 christos Exp $
+# flash:	file(1) magic for Macromedia Flash file format
+#
+# See
+#
+#	http://www.macromedia.com/software/flash/open/
+#
+0	string		FWS		Macromedia Flash data,
+>3	byte		x		version %d
+!:mime	application/x-shockwave-flash
+0	string		CWS		Macromedia Flash data (compressed),
+!:mime	application/x-shockwave-flash
+
+# From: Cal Peake <cp@absolutedigital.net>
+0	string		FLV		Macromedia Flash Video
+!:mime	video/x-flv
--- a/magic/fonts
+++ b/magic/fonts
@ -0,0 +1,32 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: fonts,v 1.25 2013/02/06 14:18:52 christos Exp $
+# fonts:  file(1) magic for font data
+#
+
+# X11 font files in SNF (Server Natural Format) format
+# updated by Joerg Jenderek at Feb 2013
+# http://computer-programming-forum.com/51-perl/8f22fb96d2e34bab.htm
+0	belong		00000004		X11 SNF font data, MSB first
+#>104	belong		00000004		X11 SNF font data, MSB first
+!:mime	application/x-font-sfn
+# GRR: line below too general as it catches also Xbase index file t3-CHAR.NDX
+0	lelong		00000004		
+>104	lelong		00000004		X11 SNF font data, LSB first
+!:mime	application/x-font-sfn
+
+# True Type fonts
+0	string	\000\001\000\000\000	TrueType font data
+!:mime application/x-font-ttf
+
+# Opentype font data from Avi Bercovich
+0	string		OTTO		OpenType font data
+!:mime application/vnd.ms-opentype
+
+# Gurkan Sengun <gurkan@linuks.mine.nu>, www.linuks.mine.nu 
+0	string		SplineFontDB:	Spline Font Database 
+!:mime application/vnd.font-fontforge-sfd
+
+# EOT
+34	string		LP		Embedded OpenType (EOT)
+!:mime application/vnd.ms-fontobject
--- a/magic/fortran
+++ b/magic/fortran
@ -0,0 +1,7 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: fortran,v 1.6 2009/09/19 16:28:09 christos Exp $
+# FORTRAN source
+0	regex/100	\^[Cc][\ \t]	FORTRAN program
+!:mime	text/x-fortran
+!:strength - 5
--- a/magic/frame
+++ b/magic/frame
@ -0,0 +1,31 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File$
+# frame:  file(1) magic for FrameMaker files
+#
+# This stuff came on a FrameMaker demo tape, most of which is
+# copyright, but this file is "published" as witness the following:
+#
+# Note that this is the Framemaker Maker Interchange Format, not the
+# Normal format which would be application/vnd.framemaker.
+#
+0	string		\<MakerFile	FrameMaker document
+!:mime	application/x-mif
+0	string		\<MIFFile	FrameMaker MIF (ASCII) file
+!:mime	application/x-mif
+0	search/1	\<MakerDictionary	FrameMaker Dictionary text
+!:mime	application/x-mif
+0	string		\<MakerScreenFont	FrameMaker Font file
+!:mime	application/x-mif
+0	string		\<MML		FrameMaker MML file
+!:mime	application/x-mif
+0	string		\<BookFile	FrameMaker Book file
+!:mime	application/x-mif
+# XXX - this book entry should be verified, if you find one, uncomment this
+#0	string		\<Book\ 	FrameMaker Book (ASCII) file
+#!:mime	application/x-mif
+#>6	string		3.0		 (3.0)
+#>6	string		2.0		 (2.0)
+#>6	string		1.0		 (1.0)
+0	string		\<Maker	Intermediate Print File	FrameMaker IPL file
+!:mime	application/x-mif
--- a/magic/gimp
+++ b/magic/gimp
@ -0,0 +1,13 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: gimp,v 1.6 2009/09/19 16:28:09 christos Exp $
+# GIMP Gradient: file(1) magic for the GIMP's gradient data files
+# by Federico Mena <federico@nuclecu.unam.mx>
+
+#------------------------------------------------------------------------------
+# XCF:  file(1) magic for the XCF image format used in the GIMP developed
+#       by Spencer Kimball and Peter Mattis
+#       ('Bucky' LaDieu, nega@vt.edu)
+
+0	string		gimp\ xcf	GIMP XCF image data,
+!:mime	image/x-xcf
--- a/magic/gnu
+++ b/magic/gnu
@ -0,0 +1,23 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: gnu,v 1.13 2012/01/03 17:16:54 christos Exp $
+# gnu:  file(1) magic for various GNU tools
+#
+# GNU nlsutils message catalog file format
+#
+# GNU message catalog (.mo and .gmo files)
+
+# GnuPG
+# The format is very similar to pgp
+# Note: magic.mime had 0x8501 for the next line instead of 0x8502
+0	beshort		0x8502			GPG encrypted data
+!:mime	text/PGP # encoding: data
+
+# This magic is not particularly good, as the keyrings don't have true
+# magic. Nevertheless, it covers many keyrings.
+0       beshort         0x9901                  GPG key public ring
+!:mime	application/x-gnupg-keyring
+
+# gettext message catalogue
+0	regex	\^msgid\ 		GNU gettext message catalogue text
+!:mime text/x-po
--- a/magic/gnumeric
+++ b/magic/gnumeric
@ -0,0 +1,8 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File$
+# gnumeric:  file(1) magic for Gnumeric spreadsheet
+# This entry is only semi-helpful, as Gnumeric compresses its files, so
+# they will ordinarily reported as "compressed", but at least -z helps
+39	string	=<gmr:Workbook	Gnumeric spreadsheet
+!:mime	application/x-gnumeric
--- a/magic/icc
+++ b/magic/icc
@ -0,0 +1,51 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File$
+# icc:  file(1) magic for International Color Consortium file formats
+
+#
+# Color profiles as per the ICC's "Image technology colour management -
+# Architecture, profile format, and data structure" specification.
+# See
+#
+#	http://www.color.org/specification/ICC1v43_2010-12.pdf
+#
+# for Specification ICC.1:2010 (Profile version 4.3.0.0).
+#
+# Bytes 36 to 39 contain a generic profile file signature of "acsp";
+# bytes 40 to 43 "may be used to identify the primary platform/operating
+# system framework for which the profile was created".
+#
+# There are other fields that might be worth dumping as well.
+#
+
+# This appears to be what's used for Apple ColorSync profiles.
+# Instead of adding that, Apple just changed the generic "acsp" entry
+# to be for "ColorSync ICC Color Profile" rather than "Kodak Color
+# Management System, ICC Profile".
+# Yes, it's "APPL", not "AAPL"; see the spec.
+36	string		acspAPPL	ColorSync ICC Profile
+!:mime	application/vnd.iccprofile
+
+# Microsoft ICM color profile
+36	string		acspMSFT	Microsoft ICM Color Profile
+!:mime	application/vnd.iccprofile
+
+# Yes, that's a blank after "SGI".
+36	string		acspSGI\ 	SGI ICC Profile
+!:mime	application/vnd.iccprofile
+
+# XXX - is this what's used for the Sun KCMS or not?  The standard file
+# uses just "acsp" for that, but Apple's file uses it for "ColorSync",
+# and there *is* an identified "primary platform" value of SUNW.
+36	string		acspSUNW	Sun KCMS ICC Profile
+!:mime	application/vnd.iccprofile
+
+# Any other profile.
+# XXX - should we use "acsp\0\0\0\0" for "no primary platform" profiles,
+# and use "acsp" for everything else and dump the "primary platform"
+# string in those cases?
+36	string		acsp		ICC Profile
+!:mime	application/vnd.iccprofile
+
+
--- a/magic/iff
+++ b/magic/iff
@ -0,0 +1,21 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: iff,v 1.12 2009/09/19 16:28:09 christos Exp $
+# iff:	file(1) magic for Interchange File Format (see also "audio" & "images")
+#
+# Daniel Quinlan (quinlan@yggdrasil.com) -- IFF was designed by Electronic
+# Arts for file interchange.  It has also been used by Apple, SGI, and
+# especially Commodore-Amiga.
+#
+# IFF files begin with an 8 byte FORM header, followed by a 4 character
+# FORM type, which is followed by the first chunk in the FORM.
+
+0	string		FORM		IFF data
+#>4	belong		x		\b, FORM is %d bytes long
+# audio formats
+>8	string		AIFF		\b, AIFF audio
+!:mime	audio/x-aiff
+>8	string		AIFC		\b, AIFF-C compressed audio
+!:mime	audio/x-aiff
+>8	string		8SVX		\b, 8SVX 8-bit sampled sound voice
+!:mime	audio/x-aiff
--- a/magic/images
+++ b/magic/images
@ -0,0 +1,255 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: images,v 1.80 2013/02/06 14:18:52 christos Exp $
+# images:  file(1) magic for image formats (see also "iff", and "c-lang" for
+# XPM bitmaps)
+#
+# originally from jef@helios.ee.lbl.gov (Jef Poskanzer),
+# additions by janl@ifi.uio.no as well as others. Jan also suggested
+# merging several one- and two-line files into here.
+#
+# little magic: PCX (first byte is 0x0a)
+
+# PBMPLUS images
+# The next byte following the magic is always whitespace.
+# strength is changed to try these patterns before "x86 boot sector"
+0	search/1	P1		
+>3	regex		=[0-9]*\ [0-9]*		Netpbm PBM image text
+>3	regex		=[0-9]+\ 		\b, size = %sx
+>>3	regex		=\ [0-9]+	\b%s
+!:strength + 45
+!:mime	image/x-portable-bitmap
+0	search/1	P2		
+>3	regex		=[0-9]*\ [0-9]*		Netpbm PGM image text
+>3	regex		=[0-9]+\ 		\b, size = %sx
+>>3	regex		=\ [0-9]+	\b%s
+!:strength + 45
+!:mime	image/x-portable-greymap
+0	search/1	P3		Netpbm PPM image text
+>3	regex		=[0-9]*\ [0-9]*		Netpbm PPM image text
+>3	regex		=[0-9]+\ 		\b, size = %sx
+>>3	regex		=\ [0-9]+	\b%s
+!:strength + 45
+!:mime	image/x-portable-pixmap
+0	string		P4		
+>3	regex		=[0-9]*\ [0-9]*		Netpbm PBM "rawbits" image data
+>3	regex		=[0-9]+\ 		\b, size = %sx
+>>3	regex		=\ [0-9]+	\b%s
+!:strength + 45
+!:mime	image/x-portable-bitmap
+0	string		P5		
+>3	regex		=[0-9]*\ [0-9]*		Netpbm PGM "rawbits" image data
+>3	regex		=[0-9]+\ 		\b, size = %sx
+>>3	regex		=\ [0-9]+	\b%s
+!:strength + 45
+!:mime	image/x-portable-greymap
+0	string		P6		
+>3	regex		=[0-9]*\ [0-9]*		Netpbm PPM "rawbits" image data
+>3	regex		=[0-9]+\ 		\b, size = %sx
+>>3	regex		=\ [0-9]+	\b%s
+!:strength + 45
+!:mime	image/x-portable-pixmap
+0	string		P7		Netpbm PAM image file
+!:mime	image/x-portable-pixmap
+
+# NIFF (Navy Interchange File Format, a modification of TIFF) images
+# [GRR:  this *must* go before TIFF]
+0	string		IIN1		NIFF image data
+!:mime	image/x-niff
+
+# Canon RAW version 1 (CRW) files are a type of Canon Image File Format
+# (CIFF) file. These are apparently all little-endian.
+# From: Adam Buchbinder <adam.buchbinder@gmail.com>
+# URL: http://www.sno.phy.queensu.ca/~phil/exiftool/canon_raw.html
+0	string		II\x1a\0\0\0HEAPCCDR	Canon CIFF raw image data
+!:mime	image/x-canon-crw
+
+# Canon RAW version 2 (CR2) files are a kind of TIFF with an extra magic
+# number. Put this above the TIFF test to make sure we detect them.
+# These are apparently all little-endian.
+# From: Adam Buchbinder <adam.buchbinder@gmail.com>
+# URL: http://libopenraw.freedesktop.org/wiki/Canon_CR2
+0	string		II\x2a\0\x10\0\0\0CR	Canon CR2 raw image data
+!:mime	image/x-canon-cr2
+
+# Tag Image File Format, from Daniel Quinlan (quinlan@yggdrasil.com)
+# The second word of TIFF files is the TIFF version number, 42, which has
+# never changed.  The TIFF specification recommends testing for it.
+0	string		MM\x00\x2a	TIFF image data, big-endian
+!:mime	image/tiff
+0	string		II\x2a\x00	TIFF image data, little-endian
+!:mime	image/tiff
+
+0	string		MM\x00\x2b	Big TIFF image data, big-endian
+!:mime	image/tiff
+0	string		II\x2b\x00	Big TIFF image data, little-endian
+!:mime	image/tiff
+
+# PNG [Portable Network Graphics, or "PNG's Not GIF"] images
+# (Greg Roelofs, newt@uchicago.edu)
+# (Albert Cahalan, acahalan@cs.uml.edu)
+#
+# 137 P N G \r \n ^Z \n [4-byte length] H E A D [HEAD data] [HEAD crc] ...
+#
+0	string		\x89PNG\x0d\x0a\x1a\x0a		PNG image data
+!:mime	image/png
+
+# possible GIF replacements; none yet released!
+# (Greg Roelofs, newt@uchicago.edu)
+#
+# GRR 950115:  this was mine ("Zip GIF"):
+0	string		GIF94z		ZIF image (GIF+deflate alpha)
+!:mime	image/x-unknown
+#
+# GRR 950115:  this is Jeremy Wohl's Free Graphics Format (better):
+#					
+0	string		FGF95a		FGF image (GIF+deflate beta)
+!:mime	image/x-unknown
+#
+# GRR 950115:  this is Thomas Boutell's Portable Bitmap Format proposal
+# (best; not yet implemented):
+#					
+0	string		PBF		PBF image (deflate compression)
+!:mime	image/x-unknown
+
+# GIF
+0	string		GIF8		GIF image data
+!:mime	image/gif
+!:apple	8BIMGIFf
+
+# From: Joerg Jenderek <joerg.jen.der.ek@gmx.net>
+# most files with the extension .EPA and some with .BMP
+0	string		\x11\x06	Award BIOS Logo, 136 x 84
+!:mime	image/x-award-bioslogo
+0	string		\x11\x09	Award BIOS Logo, 136 x 126
+!:mime	image/x-award-bioslogo
+#0	string		\x07\x1f	BIOS Logo corrupted?
+# http://www.blackfiveservices.co.uk/awbmtools.shtml
+# http://biosgfx.narod.ru/v3/
+# http://biosgfx.narod.ru/abr-2/
+0	string		AWBM		
+>4	leshort		<1981		Award BIOS bitmap
+!:mime	image/x-award-bmp
+
+# PC bitmaps (OS/2, Windows BMP files)  (Greg Roelofs, newt@uchicago.edu)
+0	string		BM
+>14	leshort		12		PC bitmap, OS/2 1.x format
+!:mime	image/x-ms-bmp
+>14	leshort		64		PC bitmap, OS/2 2.x format
+!:mime	image/x-ms-bmp
+>14	leshort		40		PC bitmap, Windows 3.x format
+!:mime	image/x-ms-bmp
+>14	leshort		128		PC bitmap, Windows NT/2000 format
+!:mime	image/x-ms-bmp
+
+# XPM icons (Greg Roelofs, newt@uchicago.edu)
+0	search/1	/*\ XPM\ */	X pixmap image text
+!:mime	image/x-xpmi
+
+# DICOM medical imaging data
+128	string	DICM			DICOM medical imaging data
+!:mime	application/dicom
+
+# XWD - X Window Dump file.
+#   As described in /usr/X11R6/include/X11/XWDFile.h
+#   used by the xwd program.
+#   Bradford Castalia, idaeim, 1/01
+#   updated by Adam Buchbinder, 2/09
+# The following assumes version 7 of the format; the first long is the length
+# of the header, which is at least 25 4-byte longs, and the one at offset 8
+# is a constant which is always either 1 or 2. Offset 12 is the pixmap depth,
+# which is a maximum of 32.
+0	belong	>100
+>8	belong	<3
+>>12	belong	<33
+>>>4	belong	7			XWD X Window Dump image data
+!:mime	image/x-xwindowdump
+
+# PCX image files
+# From: Dan Fandrich <dan@coneharvesters.com>
+# updated by Joerg Jenderek at Feb 2013 by http://de.wikipedia.org/wiki/PCX
+# http://web.archive.org/web/20100206055706/http://www.qzx.com/pc-gpe/pcx.txt
+# GRR: original test was still too general as it catches xbase examples T5.DBT,T6.DBT with 0xa000000
+# test for bytes 0x0a,version byte (0,2,3,4,5),compression byte flag(0,1), bit depth (>0) of PCX or T5.DBT,T6.DBT
+0	ubelong&0xffF8fe00	0x0a000000	
+# for PCX bit depth > 0 
+>3	ubyte		>0	
+# test for valid versions
+>>1	ubyte		<6	
+>>>1	ubyte		!1	PCX
+!:mime	image/x-pcx
+
+# Adobe Photoshop
+# From: Asbjoern Sloth Toennesen <asbjorn@lila.io>
+0	string		8BPS Adobe Photoshop Image
+!:mime	image/vnd.adobe.photoshop
+
+# Summary: DjVu image / document
+# Extension: .djvu
+# Reference: http://djvu.org/docs/DjVu3Spec.djvu
+# Submitted by: Stephane Loeuillet <stephane.loeuillet@tiscali.fr>
+# Modified by (1): Abel Cheung <abelcheung@gmail.com>
+0	string	AT&TFORM
+>12	string	DJVM		DjVu multiple page document
+!:mime	image/vnd.djvu
+>12	string	DJVU		DjVu image or single page document
+!:mime	image/vnd.djvu
+>12	string	DJVI		DjVu shared document
+!:mime	image/vnd.djvu
+>12	string	THUM		DjVu page thumbnails
+!:mime	image/vnd.djvu
+
+# Originally by Marc Espie
+# Modified by Robert Minsk <robertminsk at yahoo.com>
+# http://www.openexr.com/openexrfilelayout.pdf
+0	lelong		20000630	OpenEXR image data,
+!:mime image/x-exr
+
+# SMPTE Digital Picture Exchange Format, SMPTE DPX
+#
+# ANSI/SMPTE 268M-1994, SMPTE Standard for File Format for Digital
+# Moving-Picture Exchange (DPX), v1.0, 18 February 1994
+# Robert Minsk <robertminsk at yahoo.com>
+0	string		SDPX	DPX image data, big-endian,
+!:mime image/x-dpx
+
+#-----------------------------------------------------------------------
+# Hierarchical Data Format, used to facilitate scientific data exchange
+# specifications at http://hdf.ncsa.uiuc.edu/
+0	belong	0x0e031301	Hierarchical Data Format (version 4) data
+!:mime	application/x-hdf
+0	string	\211HDF\r\n\032\n	Hierarchical Data Format (version 5) data
+!:mime	application/x-hdf
+
+# http://www.cartesianinc.com/Tech/
+0	string	CPC\262		Cartesian Perceptual Compression image
+!:mime	image/x-cpi
+
+
+# Polar Monitor Bitmap (.pmb) used as logo for Polar Electro watches
+# From: Markus Heidelberg <markus.heidelberg at web.de>
+0	string/t	[BitmapInfo2]	Polar Monitor Bitmap text
+!:mime	image/x-polar-monitor-bitmap
+
+# Type:	Olympus ORF raw images.
+# URL:	http://libopenraw.freedesktop.org/wiki/Olympus_ORF
+# From:	Adam Buchbinder <adam.buchbinder@gmail.com>
+0	string		MMOR		Olympus ORF raw image data, big-endian
+!:mime	image/x-olympus-orf
+0	string		IIRO		Olympus ORF raw image data, little-endian
+!:mime	image/x-olympus-orf
+0	string		IIRS		Olympus ORF raw image data, little-endian
+!:mime	image/x-olympus-orf
+
+# Type: Foveon X3F
+# URL:  http://www.photofo.com/downloads/x3f-raw-format.pdf
+# From: Adam Buchbinder <adam.buchbinder@gmail.com>
+# Note that the MIME type isn't defined anywhere that I can find; if
+# there's a canonical type for this format, it should replace this one.
+0	string	FOVb	Foveon X3F raw image data
+!:mime	image/x-x3f
+
+# Paint.NET file
+# From Adam Buchbinder <adam.buchbinder@gmail.com>
+0	string	PDN3	Paint.NET image data
+!:mime	image/x-paintnet
--- a/magic/java
+++ b/magic/java
@ -0,0 +1,16 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------
+# $File: java,v 1.13 2011/12/08 12:12:46 rrt Exp $
+# Java ByteCode and Mach-O binaries (e.g., Mac OS X) use the
+# same magic number, 0xcafebabe, so they are both handled
+# in the entry called "cafebabe".
+#------------------------------------------------------------
+
+0	belong		0xfeedfeed	Java KeyStore
+!:mime	application/x-java-keystore
+0	belong		0xcececece	Java JCE KeyStore
+!:mime	application/x-java-jce-keystore
+
+# Java source
+0	regex	^import.*;$	Java source
+!:mime	text/x-java
--- a/magic/javascript
+++ b/magic/javascript
@ -0,0 +1,17 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: $
+# javascript:  magic for javascript and node.js scripts.
+#
+0	search/1/w	#!/bin/node		Node.js script text executable
+!:mime application/javascript
+0	search/1/w	#!/usr/bin/node		Node.js script text executable
+!:mime application/javascript
+0	search/1/w	#!/bin/nodejs		Node.js script text executable
+!:mime application/javascript
+0	search/1/w	#!/usr/bin/nodejs	Node.js script text executable
+!:mime application/javascript
+0	search/1	#!/usr/bin/env\ node	Node.js script text executable
+!:mime application/javascript
+0	search/1	#!/usr/bin/env\ nodejs	Node.js script text executable
+!:mime application/javascript
--- a/magic/jpeg
+++ b/magic/jpeg
@ -0,0 +1,31 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: jpeg,v 1.18 2012/08/01 12:12:36 christos Exp $
+# JPEG images
+# SunOS 5.5.1 had
+#
+#	0	string		\377\330\377\340	JPEG file
+#	0	string		\377\330\377\356	JPG file
+#
+# both of which turn into "JPEG image data" here.
+#
+0	beshort		0xffd8		JPEG image data
+!:mime	image/jpeg
+!:apple	8BIMJPEG
+!:strength +2
+
+# From: David Santinoli <david@santinoli.com>
+0	string		\x00\x00\x00\x0C\x6A\x50\x20\x20\x0D\x0A\x87\x0A	JPEG 2000
+# From: Johan van der Knijff <johan.vanderknijff@kb.nl>
+# Added sub-entries for JP2, JPX, JPM and MJ2 formats; added mimetypes
+# https://github.com/bitsgalore/jp2kMagic
+#
+# Now read value of 'Brand' field, which yields a few possibilities:
+>20	string		\x6a\x70\x32\x20	Part 1 (JP2)
+!:mime	image/jp2
+>20	string		\x6a\x70\x78\x20	Part 2 (JPX)
+!:mime	image/jpx
+>20	string		\x6a\x70\x6d\x20	Part 6 (JPM)
+!:mime	image/jpm
+>20	string		\x6d\x6a\x70\x32	Part 3 (MJ2)
+!:mime	video/mj2
--- a/magic/kde
+++ b/magic/kde
@ -0,0 +1,11 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: kde,v 1.4 2009/09/19 16:28:10 christos Exp $
+# kde:  file(1) magic for KDE
+
+0		string/t	[KDE\ Desktop\ Entry]	KDE desktop entry
+!:mime	application/x-kdelnk
+0		string/t	#\ KDE\ Config\ File	KDE config file
+!:mime	application/x-kdelnk
+0		string/t	#\ xmcd	xmcd database file for kscd
+!:mime	text/x-xmcd
--- a/magic/kml
+++ b/magic/kml
@ -0,0 +1,30 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: kml,v 1.2 2009/09/19 16:28:10 christos Exp $
+# Type: Google KML, formerly Keyhole Markup Language
+# Future development of this format has been handed
+# over to the Open Geospatial Consortium.
+# http://www.opengeospatial.org/standards/kml/
+# From: Asbjoern Sloth Toennesen <asbjorn@lila.io>
+0 string/t    \<?xml
+>20  search/400 \ xmlns= 
+>>&0 regex ['"]http://earth.google.com/kml Google KML document
+!:mime application/vnd.google-earth.kml+xml
+
+#------------------------------------------------------------------------------
+# Type: OpenGIS KML, formerly Keyhole Markup Language
+# This standard is maintained by the
+# Open Geospatial Consortium.
+# http://www.opengeospatial.org/standards/kml/
+# From: Asbjoern Sloth Toennesen <asbjorn@lila.io>
+>>&0 regex ['"]http://www.opengis.net/kml OpenGIS KML document
+!:mime application/vnd.google-earth.kml+xml
+
+#------------------------------------------------------------------------------
+# Type: Google KML Archive (ZIP based) 
+# http://code.google.com/apis/kml/documentation/kml_tut.html
+# From: Asbjoern Sloth Toennesen <asbjorn@lila.io>
+0 string    PK\003\004
+>4  byte    0x14
+>>30  string doc.kml Compressed Google KML Document, including resources.
+!:mime application/vnd.google-earth.kmz
--- a/magic/linux
+++ b/magic/linux
@ -0,0 +1,22 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: linux,v 1.46 2013/01/06 21:26:48 christos Exp $
+# linux:  file(1) magic for Linux files
+#
+# Values for Linux/i386 binaries, from Daniel Quinlan <quinlan@yggdrasil.com>
+# The following basic Linux magic is useful for reference, but using
+# "long" magic is a better practice in order to avoid collisions.
+#
+# 2	leshort		100		Linux/i386
+# >0	leshort		0407		impure executable (OMAGIC)
+# >0	leshort		0410		pure executable (NMAGIC)
+# >0	leshort		0413		demand-paged executable (ZMAGIC)
+# >0	leshort		0314		demand-paged executable (QMAGIC)
+#
+
+# SYSLINUX boot logo files (from 'ppmtolss16' sources)
+# http://www.syslinux.org/wiki/index.php/SYSLINUX#Display_graphic_from_filename:
+# file extension .lss .16
+0	lelong	=0x1413f33d		SYSLINUX' LSS16 image data
+# syslinux-4.05/mime/image/x-lss16.xml
+!:mime image/x-lss16
--- a/magic/lisp
+++ b/magic/lisp
@ -0,0 +1,42 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File$
+# lisp:  file(1) magic for lisp programs
+#
+# various lisp types, from Daniel Quinlan (quinlan@yggdrasil.com)
+
+# updated by Joerg Jenderek
+# GRR: This lot is too weak
+#0	string	;;			
+# windows INF files often begin with semicolon and use CRLF as line end
+# lisp files are mainly created on unix system with LF as line end
+#>2	search/4096	!\r		Lisp/Scheme program text
+#>2	search/4096	\r		Windows INF file
+
+0	search/4096	(setq\ 			Lisp/Scheme program text
+!:mime	text/x-lisp
+0	search/4096	(defvar\ 		Lisp/Scheme program text
+!:mime	text/x-lisp
+0	search/4096	(defparam\ 		Lisp/Scheme program text
+!:mime	text/x-lisp
+0	search/4096	(defun\  		Lisp/Scheme program text
+!:mime	text/x-lisp
+0	search/4096	(autoload\ 		Lisp/Scheme program text
+!:mime	text/x-lisp
+0	search/4096	(custom-set-variables\ 	Lisp/Scheme program text
+!:mime	text/x-lisp
+
+# Emacs 18 - this is always correct, but not very magical.
+0	string	\012(			Emacs v18 byte-compiled Lisp data
+!:mime	application/x-elc
+# Emacs 19+ - ver. recognition added by Ian Springer
+# Also applies to XEmacs 19+ .elc files; could tell them apart with regexs
+# - Chris Chittleborough <cchittleborough@yahoo.com.au>
+0	string	;ELC	
+>4	byte	>18			
+>4	byte    <32			Emacs/XEmacs v%d byte-compiled Lisp data
+!:mime	application/x-elc		
+
+# From: David Allouche <david@allouche.net>
+0	search/1	\<TeXmacs|	TeXmacs document text
+!:mime	text/texmacs
--- a/magic/lua
+++ b/magic/lua
@ -0,0 +1,17 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: lua,v 1.5 2009/09/19 16:28:10 christos Exp $
+# lua:  file(1) magic for Lua scripting language
+# URL:  http://www.lua.org/
+# From: Reuben Thomas <rrt@sc3d.org>, Seo Sanghyeon <tinuviel@sparcs.kaist.ac.kr>
+
+# Lua scripts
+0	search/1/w	#!\ /usr/bin/lua	Lua script text executable
+!:mime	text/x-lua
+0	search/1/w	#!\ /usr/local/bin/lua	Lua script text executable
+!:mime	text/x-lua
+0	search/1	#!/usr/bin/env\ lua	Lua script text executable
+!:mime	text/x-lua
+0	search/1	#!\ /usr/bin/env\ lua	Lua script text executable
+!:mime	text/x-lua
+
--- a/magic/m4
+++ b/magic/m4
@ -0,0 +1,7 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File$
+# make:  file(1) magic for M4 scripts
+#
+0	regex	\^dnl\ 		M4 macro processor script text
+!:mime	text/x-m4
--- a/magic/macintosh
+++ b/magic/macintosh
@ -0,0 +1,21 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: macintosh,v 1.21 2010/09/20 19:19:17 rrt Exp $
+# macintosh description
+#
+# BinHex is the Macintosh ASCII-encoded file format (see also "apple")
+# Daniel Quinlan, quinlan@yggdrasil.com
+11	string	must\ be\ converted\ with\ BinHex	BinHex binary text
+!:mime	application/mac-binhex40
+
+# Stuffit archives are the de facto standard of compression for Macintosh
+# files obtained from most archives. (franklsm@tuns.ca)
+0	string		SIT!			StuffIt Archive (data)
+!:mime	application/x-stuffit
+!:apple	SIT!SIT!
+
+# Newer StuffIt archives (grant@netbsd.org)
+0	string		StuffIt			StuffIt Archive
+!:mime	application/x-stuffit
+!:apple	SIT!SIT!
+#>162	string		>0			: %s
--- a/magic/mail.news
+++ b/magic/mail.news
@ -0,0 +1,35 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: mail.news,v 1.21 2012/06/21 01:44:52 christos Exp $
+# mail.news:  file(1) magic for mail and news
+#
+# Unfortunately, saved netnews also has From line added in some news software.
+#0	string		From 		mail text
+0	string/t		Relay-Version: 	old news text
+!:mime	message/rfc822
+0	string/t		#!\ rnews	batched news text
+!:mime	message/rfc822
+0	string/t		N#!\ rnews	mailed, batched news text
+!:mime	message/rfc822
+0	string/t		Forward\ to 	mail forwarding text
+!:mime	message/rfc822
+0	string/t		Pipe\ to 	mail piping text
+!:mime	message/rfc822
+0	string/tc		delivered-to:	SMTP mail text
+!:mime	message/rfc822
+0	string/tc		return-path:	SMTP mail text
+!:mime	message/rfc822
+0	string/t		Path:		news text
+!:mime	message/news
+0	string/t		Xref:		news text
+!:mime	message/news
+0	string/t		From:		news or mail text
+!:mime	message/rfc822
+0	string/t		Article 	saved news text
+!:mime	message/news
+0	string/t		Received:	RFC 822 mail text
+!:mime	message/rfc822
+
+# TNEF files...
+0	lelong		0x223E9F78	Transport Neutral Encapsulation Format
+!:mime	application/vnd.ms-tnef
--- a/magic/make
+++ b/magic/make
@ -0,0 +1,16 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File$
+# make:  file(1) magic for makefiles
+#
+0	regex	\^CFLAGS	makefile script text
+!:mime	text/x-makefile
+0	regex	\^LDFLAGS	makefile script text
+!:mime	text/x-makefile
+0	regex	\^all:	makefile script text
+!:mime	text/x-makefile
+0	regex	\^.PRECIOUS	makefile script text
+!:mime	text/x-makefile
+
+0	regex	\^SUBDIRS	automake makefile script text
+!:mime	text/x-makefile
--- a/magic/marc21
+++ b/magic/marc21
@ -0,0 +1,29 @@
+# See COPYING file in this directory for original libmagic copyright.
+#--------------------------------------------
+# marc21: file(1) magic for MARC 21 Format
+#
+# Kevin Ford (kefo@loc.gov)
+# 
+# MARC21 formats are for the representation and communication
+# of bibliographic and related information in machine-readable
+# form.  For more info, see http://www.loc.gov/marc/
+
+
+# leader position 20-21 must be 45
+20	string	45	
+
+# leader starts with 5 digits, followed by codes specific to MARC format
+>0	regex/1	(^[0-9]{5})[acdnp][^bhlnqsu-z]	MARC21 Bibliographic
+!:mime	application/marc
+>0	regex/1	(^[0-9]{5})[acdnosx][z]	MARC21 Authority
+!:mime	application/marc
+>0	regex/1	(^[0-9]{5})[cdn][uvxy]	MARC21 Holdings
+!:mime	application/marc
+0	regex/1	(^[0-9]{5})[acdn][w]	MARC21 Classification
+!:mime	application/marc
+>0	regex/1	(^[0-9]{5})[cdn][q]	MARC21 Community
+!:mime	application/marc
+
+# leader position 22-23, should be "00" but is it?
+>0	regex/1	(^.{21})([^0]{2})	(non-conforming)
+!:mime	application/marc
--- a/magic/matroska
+++ b/magic/matroska
@ -0,0 +1,17 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: matroska,v 1.7 2012/08/26 10:06:15 christos Exp $
+# matroska:  file(1) magic for Matroska files
+#
+# See http://www.matroska.org/
+#
+
+# EBML id:
+0		belong		0x1a45dfa3
+# DocType id:
+>4		search/4096 	\x42\x82
+# DocType contents:
+>>&1		string		webm		WebM
+!:mime  video/webm
+>>&1		string		matroska	Matroska data
+!:mime  video/x-matroska
--- a/magic/misctools
+++ b/magic/misctools
@ -0,0 +1,9 @@
+# See COPYING file in this directory for original libmagic copyright.
+#-----------------------------------------------------------------------------
+# $File: misctools,v 1.12 2010/09/29 18:36:49 rrt Exp $
+# misctools:  file(1) magic for miscellaneous UNIX tools.
+#
+0	string/c	BEGIN:VCALENDAR		vCalendar calendar file
+!:mime	text/calendar
+0	string/c	BEGIN:VCARD		vCard visiting card
+!:mime	text/x-vcard
--- a/magic/msdos
+++ b/magic/msdos
@ -0,0 +1,368 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: msdos,v 1.84 2013/02/05 13:55:22 christos Exp $
+# msdos:  file(1) magic for MS-DOS files
+#
+
+# .BAT files (Daniel Quinlan, quinlan@yggdrasil.com)
+# updated by Joerg Jenderek at Oct 2008,Apr 2011
+0	string/t	@			
+>1	string/cW	\ echo\ off	DOS batch file text
+!:mime	text/x-msdos-batch
+>1	string/cW	echo\ off	DOS batch file text
+!:mime	text/x-msdos-batch
+>1	string/cW	rem		DOS batch file text
+!:mime	text/x-msdos-batch
+>1	string/cW	set\ 		DOS batch file text
+!:mime	text/x-msdos-batch
+
+# Tests for various EXE types.
+#
+# Many of the compressed formats were extraced from IDARC 1.23 source code.
+#
+0	string/b	MZ DOS MZ
+!:mime	application/x-dosexec
+# All non-DOS EXE extensions have the relocation table more than 0x40 bytes into the file.
+>0x18	leshort <0x40 MS-DOS executable
+# These traditional tests usually work but not always.  When test quality support is
+# implemented these can be turned on.
+#>>0x18	leshort	0x1c	(Borland compiler)
+#>>0x18	leshort	0x1e	(MS compiler)
+
+# If the relocation table is 0x40 or more bytes into the file, it's definitely
+# not a DOS EXE.
+>0x18  leshort >0x3f
+
+# Maybe it's a PE?
+>>(0x3c.l) string PE\0\0 PE
+>>>(0x3c.l+24)	leshort		0x010b	\b32 executable
+>>>(0x3c.l+24)	leshort		0x020b	\b32+ executable
+>>>(0x3c.l+24)	leshort		0x0107	ROM image
+>>>(0x3c.l+24)	default		x	Unknown PE signature
+>>>>&0 		leshort		x	0x%x
+>>>(0x3c.l+22)	leshort&0x2000	>0	(DLL)
+>>>(0x3c.l+92)	leshort		1	(native)
+>>>(0x3c.l+92)	leshort		2	(GUI)
+>>>(0x3c.l+92)	leshort		3	(console)
+>>>(0x3c.l+92)	leshort		7	(POSIX)
+>>>(0x3c.l+92)	leshort		9	(Windows CE)
+>>>(0x3c.l+92)	leshort		10	(EFI application)
+>>>(0x3c.l+92)	leshort		11	(EFI boot service driver)
+>>>(0x3c.l+92)	leshort		12	(EFI runtime driver)
+>>>(0x3c.l+92)	leshort		13	(EFI ROM)
+>>>(0x3c.l+92)	leshort		14	(XBOX)
+>>>(0x3c.l+92)	leshort		15	(Windows boot application)
+>>>(0x3c.l+92)	default		x	(Unknown subsystem
+>>>>&0		leshort		x	0x%x)
+>>>(0x3c.l+4)	leshort		0x14c	Intel 80386
+>>>(0x3c.l+4)	leshort		0x166	MIPS R4000
+>>>(0x3c.l+4)	leshort		0x168	MIPS R10000
+>>>(0x3c.l+4)	leshort		0x184	Alpha
+>>>(0x3c.l+4)	leshort		0x1a2	Hitachi SH3
+>>>(0x3c.l+4)	leshort		0x1a6	Hitachi SH4
+>>>(0x3c.l+4)	leshort		0x1c0	ARM
+>>>(0x3c.l+4)	leshort		0x1c2	ARM Thumb
+>>>(0x3c.l+4)	leshort		0x1c4	ARMv7 Thumb
+>>>(0x3c.l+4)	leshort		0x1f0	PowerPC
+>>>(0x3c.l+4)	leshort		0x200	Intel Itanium
+>>>(0x3c.l+4)	leshort		0x266	MIPS16
+>>>(0x3c.l+4)	leshort		0x268	Motorola 68000
+>>>(0x3c.l+4)	leshort		0x290	PA-RISC
+>>>(0x3c.l+4)	leshort		0x366	MIPSIV
+>>>(0x3c.l+4)	leshort		0x466	MIPS16 with FPU
+>>>(0x3c.l+4)	leshort		0xebc	EFI byte code
+>>>(0x3c.l+4)	leshort		0x8664	x86-64
+>>>(0x3c.l+4)	leshort		0xc0ee	MSIL
+>>>(0x3c.l+4)	default		x	Unknown processor type
+>>>>&0		leshort		x	0x%x
+>>>(0x3c.l+22)	leshort&0x0200	>0	(stripped to external PDB)
+>>>(0x3c.l+22)	leshort&0x1000	>0	system file
+>>>(0x3c.l+24)	leshort		0x010b
+>>>>(0x3c.l+232) lelong	>0	Mono/.Net assembly
+>>>(0x3c.l+24)	leshort		0x020b
+>>>>(0x3c.l+248) lelong	>0	Mono/.Net assembly
+
+# hooray, there's a DOS extender using the PE format, with a valid PE
+# executable inside (which just prints a message and exits if run in win)
+>>>(8.s*16)		string		32STUB	\b, 32rtm DOS extender
+>>>(8.s*16)		string		!32STUB	\b, for MS Windows
+>>>(0x3c.l+0xf8)	string		UPX0 \b, UPX compressed
+>>>(0x3c.l+0xf8)	search/0x140	PEC2 \b, PECompact2 compressed
+>>>(0x3c.l+0xf8)	search/0x140	UPX2
+>>>>(&0x10.l+(-4))	string		PK\3\4 \b, ZIP self-extracting archive (Info-Zip)
+>>>(0x3c.l+0xf8)	search/0x140	.idata
+>>>>(&0xe.l+(-4))	string		PK\3\4 \b, ZIP self-extracting archive (Info-Zip)
+>>>>(&0xe.l+(-4))	string		ZZ0 \b, ZZip self-extracting archive
+>>>>(&0xe.l+(-4))	string		ZZ1 \b, ZZip self-extracting archive
+>>>(0x3c.l+0xf8)	search/0x140	.rsrc
+>>>>(&0x0f.l+(-4))	string		a\\\4\5 \b, WinHKI self-extracting archive
+>>>>(&0x0f.l+(-4))	string		Rar! \b, RAR self-extracting archive
+>>>>(&0x0f.l+(-4))	search/0x3000	MSCF \b, InstallShield self-extracting archive
+>>>>(&0x0f.l+(-4))	search/32	Nullsoft \b, Nullsoft Installer self-extracting archive
+>>>(0x3c.l+0xf8)	search/0x140	.data
+>>>>(&0x0f.l)		string		WEXTRACT \b, MS CAB-Installer self-extracting archive
+>>>(0x3c.l+0xf8)	search/0x140	.petite\0 \b, Petite compressed
+>>>>(0x3c.l+0xf7)	byte		x
+>>>>>(&0x104.l+(-4))	string		=!sfx! \b, ACE self-extracting archive
+>>>(0x3c.l+0xf8)	search/0x140	.WISE \b, WISE installer self-extracting archive
+>>>(0x3c.l+0xf8)	search/0x140	.dz\0\0\0 \b, Dzip self-extracting archive
+>>>&(0x3c.l+0xf8)	search/0x100	_winzip_ \b, ZIP self-extracting archive (WinZip)
+>>>&(0x3c.l+0xf8)	search/0x100	SharedD \b, Microsoft Installer self-extracting archive
+>>>0x30			string		Inno \b, InnoSetup self-extracting archive
+
+# Hmm, not a PE but the relocation table is too high for a traditional DOS exe,
+# must be one of the unusual subformats.
+>>(0x3c.l) string !PE\0\0 MS-DOS executable
+
+>>(0x3c.l)		string		NE \b, NE
+>>>(0x3c.l+0x36)	byte		1 for OS/2 1.x
+>>>(0x3c.l+0x36)	byte		2 for MS Windows 3.x
+>>>(0x3c.l+0x36)	byte		3 for MS-DOS
+>>>(0x3c.l+0x36)	byte		4 for Windows 386
+>>>(0x3c.l+0x36)	byte		5 for Borland Operating System Services
+>>>(0x3c.l+0x36)	default		x
+>>>>(0x3c.l+0x36)	byte		x (unknown OS %x)
+>>>(0x3c.l+0x36)	byte		0x81 for MS-DOS, Phar Lap DOS extender
+>>>(0x3c.l+0x0c)	leshort&0x8003	0x8002 (DLL)
+>>>(0x3c.l+0x0c)	leshort&0x8003	0x8001 (driver)
+>>>&(&0x24.s-1)		string		ARJSFX \b, ARJ self-extracting archive
+>>>(0x3c.l+0x70)	search/0x80	WinZip(R)\ Self-Extractor \b, ZIP self-extracting archive (WinZip)
+
+>>(0x3c.l)		string		LX\0\0 \b, LX
+>>>(0x3c.l+0x0a)	leshort		<1 (unknown OS)
+>>>(0x3c.l+0x0a)	leshort		1 for OS/2
+>>>(0x3c.l+0x0a)	leshort		2 for MS Windows
+>>>(0x3c.l+0x0a)	leshort		3 for DOS
+>>>(0x3c.l+0x0a)	leshort		>3 (unknown OS)
+>>>(0x3c.l+0x10)	lelong&0x28000	=0x8000 (DLL)
+>>>(0x3c.l+0x10)	lelong&0x20000	>0 (device driver)
+>>>(0x3c.l+0x10)	lelong&0x300	0x300 (GUI)
+>>>(0x3c.l+0x10)	lelong&0x28300	<0x300 (console)
+>>>(0x3c.l+0x08)	leshort		1 i80286
+>>>(0x3c.l+0x08)	leshort		2 i80386
+>>>(0x3c.l+0x08)	leshort		3 i80486
+>>>(8.s*16)		string		emx \b, emx
+>>>>&1			string		x %s
+>>>&(&0x54.l-3)		string		arjsfx \b, ARJ self-extracting archive
+
+# MS Windows system file, supposedly a collection of LE executables
+>>(0x3c.l)		string		W3 \b, W3 for MS Windows
+
+>>(0x3c.l)		string		LE\0\0 \b, LE executable
+>>>(0x3c.l+0x0a)	leshort		1
+# some DOS extenders use LE files with OS/2 header
+>>>>0x240		search/0x100	DOS/4G for MS-DOS, DOS4GW DOS extender
+>>>>0x240		search/0x200	WATCOM\ C/C++ for MS-DOS, DOS4GW DOS extender
+>>>>0x440		search/0x100	CauseWay\ DOS\ Extender for MS-DOS, CauseWay DOS extender
+>>>>0x40		search/0x40	PMODE/W for MS-DOS, PMODE/W DOS extender
+>>>>0x40		search/0x40	STUB/32A for MS-DOS, DOS/32A DOS extender (stub)
+>>>>0x40		search/0x80	STUB/32C for MS-DOS, DOS/32A DOS extender (configurable stub)
+>>>>0x40		search/0x80	DOS/32A for MS-DOS, DOS/32A DOS extender (embedded)
+# this is a wild guess; hopefully it is a specific signature
+>>>>&0x24		lelong		<0x50
+>>>>>(&0x4c.l)		string		\xfc\xb8WATCOM
+>>>>>>&0		search/8	3\xdbf\xb9 \b, 32Lite compressed
+# another wild guess: if real OS/2 LE executables exist, they probably have higher start EIP
+#>>>>(0x3c.l+0x1c)	lelong		>0x10000 for OS/2
+# fails with DOS-Extenders.
+>>>(0x3c.l+0x0a)	leshort		2 for MS Windows
+>>>(0x3c.l+0x0a)	leshort		3 for DOS
+>>>(0x3c.l+0x0a)	leshort		4 for MS Windows (VxD)
+>>>(&0x7c.l+0x26)	string		UPX \b, UPX compressed
+>>>&(&0x54.l-3)		string		UNACE \b, ACE self-extracting archive
+
+# looks like ASCII, probably some embedded copyright message.
+# and definitely not NE/LE/LX/PE
+>>0x3c		lelong	>0x20000000
+>>>(4.s*512)	leshort !0x014c \b, MZ for MS-DOS
+# header data too small for extended executable
+>2		long	!0
+>>0x18		leshort <0x40
+>>>(4.s*512)	leshort !0x014c
+
+>>>>&(2.s-514)	string	!LE
+>>>>>&-2	string	!BW \b, MZ for MS-DOS
+>>>>&(2.s-514)	string	LE \b, LE
+>>>>>0x240	search/0x100	DOS/4G for MS-DOS, DOS4GW DOS extender
+# educated guess since indirection is still not capable enough for complex offset
+# calculations (next embedded executable would be at &(&2*512+&0-2)
+# I suspect there are only LE executables in these multi-exe files
+>>>>&(2.s-514)	string	BW
+>>>>>0x240	search/0x100	DOS/4G ,\b LE for MS-DOS, DOS4GW DOS extender (embedded)
+>>>>>0x240	search/0x100	!DOS/4G ,\b BW collection for MS-DOS
+
+# This sequence skips to the first COFF segment, usually .text
+>(4.s*512)	leshort		0x014c \b, COFF
+>>(8.s*16)	string		go32stub for MS-DOS, DJGPP go32 DOS extender
+>>(8.s*16)	string		emx
+>>>&1		string		x for DOS, Win or OS/2, emx %s
+>>&(&0x42.l-3)	byte		x 
+>>>&0x26	string		UPX \b, UPX compressed
+# and yet another guess: small .text, and after large .data is unusal, could be 32lite
+>>&0x2c		search/0xa0	.text
+>>>&0x0b	lelong		<0x2000
+>>>>&0		lelong		>0x6000 \b, 32lite compressed
+
+>(8.s*16) string $WdX \b, WDos/X DOS extender
+
+# By now an executable type should have been printed out.  The executable
+# may be a self-uncompressing archive, so look for evidence of that and 
+# print it out.  
+#
+# Some signatures below from Greg Roelofs, newt@uchicago.edu.
+#
+>0x35	string	\x8e\xc0\xb9\x08\x00\xf3\xa5\x4a\x75\xeb\x8e\xc3\x8e\xd8\x33\xff\xbe\x30\x00\x05 \b, aPack compressed
+>0xe7	string	LH/2\ 	Self-Extract \b, %s
+>0x1c	string	UC2X	\b, UCEXE compressed
+>0x1c	string	WWP\ 	\b, WWPACK compressed
+>0x1c	string	RJSX 	\b, ARJ self-extracting archive
+>0x1c	string	diet 	\b, diet compressed
+>0x1c	string	LZ09 	\b, LZEXE v0.90 compressed
+>0x1c	string	LZ91 	\b, LZEXE v0.91 compressed
+>0x1c	string	tz 	\b, TinyProg compressed
+>0x1e	string	Copyright\ 1989-1990\ PKWARE\ Inc.	Self-extracting PKZIP archive
+!:mime	application/zip
+# Yes, this really is "Copr", not "Corp."
+>0x1e	string	PKLITE\ Copr.	Self-extracting PKZIP archive
+!:mime	application/zip
+# winarj stores a message in the stub instead of the sig in the MZ header
+>0x20	search/0xe0	aRJsfX \b, ARJ self-extracting archive
+>0x20	string AIN
+>>0x23	string 2	\b, AIN 2.x compressed
+>>0x23	string <2	\b, AIN 1.x compressed
+>>0x23	string >2	\b, AIN 1.x compressed
+>0x24	string	LHa's\ SFX \b, LHa self-extracting archive
+!:mime	application/x-lha
+>0x24	string	LHA's\ SFX \b, LHa self-extracting archive
+!:mime	application/x-lha
+>0x24	string	\ $ARX \b, ARX self-extracting archive
+>0x24	string	\ $LHarc \b, LHarc self-extracting archive
+>0x20	string	SFX\ by\ LARC \b, LARC self-extracting archive
+>0x40	string aPKG \b, aPackage self-extracting archive
+>0x64	string	W\ Collis\0\0 \b, Compack compressed
+>0x7a	string		Windows\ self-extracting\ ZIP	\b, ZIP self-extracting archive
+>>&0xf4 search/0x140 \x0\x40\x1\x0
+>>>(&0.l+(4)) string MSCF \b, WinHKI CAB self-extracting archive
+>1638	string	-lh5- \b, LHa self-extracting archive v2.13S
+>0x17888 string Rar! \b, RAR self-extracting archive
+
+# Skip to the end of the EXE.  This will usually work fine in the PE case
+# because the MZ image is hardcoded into the toolchain and almost certainly
+# won't match any of these signatures.
+>(4.s*512)	long	x 
+>>&(2.s-517)	byte	x 
+>>>&0	string		PK\3\4 \b, ZIP self-extracting archive
+>>>&0	string		Rar! \b, RAR self-extracting archive
+>>>&0	string		=!\x11 \b, AIN 2.x self-extracting archive
+>>>&0	string		=!\x12 \b, AIN 2.x self-extracting archive
+>>>&0	string		=!\x17 \b, AIN 1.x self-extracting archive
+>>>&0	string		=!\x18 \b, AIN 1.x self-extracting archive
+>>>&7	search/400	**ACE** \b, ACE self-extracting archive
+>>>&0	search/0x480	UC2SFX\ Header \b, UC2 self-extracting archive
+
+# a few unknown ZIP sfxes, no idea if they are needed or if they are
+# already captured by the generic patterns above
+>(8.s*16)	search/0x20	PKSFX \b, ZIP self-extracting archive (PKZIP)
+# TODO: how to add this? >FileSize-34 string Windows\ Self-Installing\ Executable \b, ZIP self-extracting archive
+#
+
+# TELVOX Teleinformatica CODEC self-extractor for OS/2:
+>49801	string	\x79\xff\x80\xff\x76\xff	\b, CODEC archive v3.21
+>>49824 leshort		=1			\b, 1 file
+>>49824 leshort		>1			\b, %u files
+
+# Popular applications
+2080	string	Microsoft\ Word\ 6.0\ Document	%s
+!:mime	application/msword
+2080	string	Documento\ Microsoft\ Word\ 6 Spanish Microsoft Word 6 document data
+!:mime	application/msword
+# Pawel Wiecek <coven@i17linuxb.ists.pwr.wroc.pl> (for polish Word)
+2112	string	MSWordDoc			Microsoft Word document data
+!:mime	application/msword
+#
+0	belong	0x31be0000			Microsoft Word Document
+!:mime	application/msword
+#
+0	string/b	PO^Q`				Microsoft Word 6.0 Document
+!:mime	application/msword
+#
+0	string/b	\376\067\0\043			Microsoft Office Document
+!:mime	application/msword
+0	string/b	\333\245-\0\0\0			Microsoft Office Document
+!:mime	application/msword
+512	string/b	\354\245\301			Microsoft Word Document
+!:mime	application/msword
+
+#
+0	string/b	\xDB\xA5\x2D\x00		Microsoft WinWord 2.0 Document
+!:mime application/msword
+#
+2080	string	Microsoft\ Excel\ 5.0\ Worksheet	%s
+!:mime	application/vnd.ms-excel
+#
+0	string/b	\xDB\xA5\x2D\x00		Microsoft WinWord 2.0 Document
+!:mime application/msword
+
+2080	string	Foglio\ di\ lavoro\ Microsoft\ Exce	%s
+!:mime	application/vnd.ms-excel
+#
+# Pawel Wiecek <coven@i17linuxb.ists.pwr.wroc.pl> (for polish Excel)
+2114	string	Biff5		Microsoft Excel 5.0 Worksheet
+!:mime	application/vnd.ms-excel
+# Italian MS-Excel
+2121	string	Biff5		Microsoft Excel 5.0 Worksheet
+!:mime	application/vnd.ms-excel
+0	string/b	\x09\x04\x06\x00\x00\x00\x10\x00	Microsoft Excel Worksheet
+!:mime	application/vnd.ms-excel
+#
+0	belong	0x00001a00	Lotus 1-2-3
+!:mime	application/x-123
+#
+0	belong	0x00000200	Lotus 1-2-3
+!:mime	application/x-123
+0	string/b		WordPro\0	Lotus WordPro
+!:mime	application/vnd.lotus-wordpro
+0	string/b		WordPro\r\373	Lotus WordPro
+!:mime	application/vnd.lotus-wordpro
+
+# Windows icons (Ian Springer <ips@fpk.hp.com>)
+0	string/b	\000\000\001\000	MS Windows icon resource
+!:mime	image/x-icon
+
+# .PIF files added by Joerg Jenderek from http://smsoft.ru/en/pifdoc.htm
+# only for windows versions equal or greater 3.0
+0x171	string	MICROSOFT\ PIFEX\0	Windows Program Information File
+!:mime	application/x-dosexec
+
+# TNEF magic From "Joomy" <joomy@se-ed.net> 
+# Microsoft Outlook's Transport Neutral Encapsulation Format (TNEF)
+0	leshort		0x223e9f78	TNEF
+!:mime	application/vnd.ms-tnef
+
+#------------------------------------------------------------------------------
+# From Stuart Caie <kyzer@4u.net> (developer of cabextract)
+# Microsoft Cabinet files
+0	string/b	MSCF\0\0\0\0	Microsoft Cabinet archive data
+!:mime application/vnd.ms-cab-compressed
+
+# from http://filext.com by Derek M Jones <derek@knosof.co.uk>
+# False positive with PPT (also currently this string is too long)
+#0	string/b	\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x3E\x00\x03\x00\xFE\xFF\x09\x00\x06	Microsoft Installer
+0	string/b	\320\317\021\340\241\261\032\341	Microsoft Office Document
+#>48	byte	0x1B					Excel Document
+#!:mime application/vnd.ms-excel
+>546	string	bjbj			Microsoft Word Document
+!:mime	application/msword
+>546	string	jbjb			Microsoft Word Document
+!:mime	application/msword
+
+0	string/b	\224\246\056		Microsoft Word Document
+!:mime	application/msword
+
+512	string	R\0o\0o\0t\0\ \0E\0n\0t\0r\0y	Microsoft Word Document
+!:mime	application/msword
+
+# MS eBook format (.lit)
+0	string/b	ITOLITLS		Microsoft Reader eBook Data
+>8	lelong	x			\b, version %u
+!:mime					application/x-ms-reader
--- a/magic/neko
+++ b/magic/neko
@ -0,0 +1,12 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------
+# $File: java,v 1.12 2009/09/19 16:28:10 christos Exp $
+
+# From: Mikhail Gusarov <dottedmag@dottedmag.net>
+# NekoVM (http://nekovm.org/) bytecode
+0	string		NEKO	NekoVM bytecode
+>4	lelong		x	(%d global symbols,
+>8	lelong		x	%d global fields,
+>12	lelong		x	%d bytecode ops)
+!:mime	application/x-nekovm-bytecode
+
--- a/magic/pascal
+++ b/magic/pascal
@ -0,0 +1,11 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File$
+# pascal:  file(1) magic for Pascal source
+#
+0	search/8192	(input,		Pascal source text
+!:mime	text/x-pascal
+0	regex		\^program	Pascal source text
+!:mime	text/x-pascal
+0	regex           	\^record		Pascal source text
+!:mime	text/x-pascal
--- a/magic/pdf
+++ b/magic/pdf
@ -0,0 +1,8 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File$
+# pdf:  file(1) magic for Portable Document Format
+#
+
+0	string		%PDF-		PDF document
+!:mime	application/pdf
--- a/magic/perl
+++ b/magic/perl
@ -0,0 +1,26 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: perl,v 1.19 2012/06/20 21:16:25 christos Exp $
+# perl:  file(1) magic for Larry Wall's perl language.
+#
+# The `eval' lines recognizes an outrageously clever hack.
+# Keith Waclena <keith@cerberus.uchicago.edu>
+# Send additions to <perl5-porters@perl.org>
+0	search/1/w	#!\ /bin/perl			Perl script text executable
+!:mime	text/x-perl
+0	search/1	eval\ "exec\ /bin/perl		Perl script text
+!:mime	text/x-perl
+0	search/1/w	#!\ /usr/bin/perl		Perl script text executable
+!:mime	text/x-perl
+0	search/1	eval\ "exec\ /usr/bin/perl	Perl script text
+!:mime	text/x-perl
+0	search/1/w	#!\ /usr/local/bin/perl		Perl script text executable
+!:mime	text/x-perl
+0	search/1	eval\ "exec\ /usr/local/bin/perl	Perl script text
+!:mime	text/x-perl
+0	search/1	eval\ '(exit\ $?0)'\ &&\ eval\ 'exec	Perl script text
+!:mime	text/x-perl
+0	search/1	#!/usr/bin/env\ perl	Perl script text executable
+!:mime	text/x-perl
+0	search/1	#!\ /usr/bin/env\ perl	Perl script text executable
+!:mime	text/x-perl
--- a/magic/pgp
+++ b/magic/pgp
@ -0,0 +1,27 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File$
+# pgp:  file(1) magic for Pretty Good Privacy
+# see http://lists.gnupg.org/pipermail/gnupg-devel/1999-September/016052.html
+#
+0       beshort         0x9900                  PGP key public ring
+!:mime	application/x-pgp-keyring
+0       beshort         0x9501                  PGP key security ring
+!:mime	application/x-pgp-keyring
+0       beshort         0x9500                  PGP key security ring
+!:mime	application/x-pgp-keyring
+0	beshort		0xa600			PGP encrypted data
+#!:mime	application/pgp-encrypted
+#0	string		-----BEGIN\040PGP	text/PGP armored data
+!:mime	text/PGP # encoding: armored data
+#>15	string	PUBLIC\040KEY\040BLOCK-	public key block
+#>15	string	MESSAGE-		message
+#>15	string	SIGNED\040MESSAGE-	signed message
+#>15	string	PGP\040SIGNATURE-	signature
+
+2	string	---BEGIN\ PGP\ PUBLIC\ KEY\ BLOCK-	PGP public key block
+!:mime	application/pgp-keys
+0	string	-----BEGIN\040PGP\40MESSAGE-		PGP message
+!:mime	application/pgp
+0	string	-----BEGIN\040PGP\40SIGNATURE-		PGP signature
+!:mime	application/pgp-signature
--- a/magic/pkgadd
+++ b/magic/pkgadd
@ -0,0 +1,7 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File$
+# pkgadd:  file(1) magic for SysV R4 PKG Datastreams
+#
+0       string          #\ PaCkAgE\ DaTaStReAm  pkg Datastream (SVR4)
+!:mime	application/x-svr4-package
--- a/magic/printer
+++ b/magic/printer
@ -0,0 +1,14 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: printer,v 1.24 2011/05/08 16:34:51 christos Exp $
+# printer:  file(1) magic for printer-formatted files
+#
+
+# PostScript, updated by Daniel Quinlan (quinlan@yggdrasil.com)
+0	string		%!		PostScript document text
+!:mime	application/postscript
+!:apple	ASPSTEXT
+# Some PCs have the annoying habit of adding a ^D as a document separator
+0	string		\004%!		PostScript document text
+!:mime	application/postscript
+!:apple	ASPSTEXT
--- a/magic/python
+++ b/magic/python
@ -0,0 +1,46 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: python,v 1.21 2012/06/21 01:12:51 christos Exp $
+# python:  file(1) magic for python
+#
+
+0	search/1/w	#!\ /usr/bin/python	Python script text executable
+!:mime text/x-python
+0	search/1/w	#!\ /usr/local/bin/python	Python script text executable
+!:mime text/x-python
+0	search/1	#!/usr/bin/env\ python	Python script text executable
+!:mime text/x-python
+0	search/1	#!\ /usr/bin/env\ python	Python script text executable
+!:mime text/x-python
+
+# from module.submodule import func1, func2
+0	regex	\^from\\s+(\\w|\\.)+\\s+import.*$	Python script text executable
+!:mime text/x-python
+
+# def __init__ (self, ...):
+0	search/4096	def\ __init__
+>&0	search/64 self	Python script text executable
+!:mime text/x-python
+
+# comments
+0	search/4096	'''
+>&0	regex	.*'''$	Python script text executable
+!:mime text/x-python
+
+0	search/4096	"""
+>&0	regex	.*"""$	Python script text executable
+!:mime text/x-python
+
+# try:
+# except: or finally:
+# block
+0	search/4096	try:
+>&0	regex	\^\\s*except.*:	Python script text executable
+!:mime text/x-python
+>&0	search/4096	finally:	Python script text executable
+!:mime text/x-python
+
+# def name(args, args):
+0	regex	 \^(\ |\\t)*def\ +[a-zA-Z]+
+>&0	regex	\ *\\(([a-zA-Z]|,|\ )*\\):$ Python script text executable
+!:mime text/x-python
--- a/magic/riff
+++ b/magic/riff
@ -0,0 +1,36 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: riff,v 1.22 2011/09/06 11:00:06 christos Exp $
+# riff:  file(1) magic for RIFF format
+# See
+#
+#	http://www.seanet.com/users/matts/riffmci/riffmci.htm
+#
+# AVI section extended by Patrik Radman <patrik+file-magic@iki.fi>
+#
+0	string		RIFF		RIFF (little-endian) data
+# Microsoft WAVE format (*.wav)
+>8	string		WAVE		\b, WAVE audio
+!:mime	audio/x-wav
+# Corel Draw Picture
+>8	string		CDRA		\b, Corel Draw Picture
+!:mime	image/x-coreldraw
+# AVI == Audio Video Interleave
+>8	string		AVI\040		\b, AVI
+!:mime	video/x-msvideo
+
+#------------------------------------------------------------------------------
+# Sony Wave64
+# see http://www.vcs.de/fileadmin/user_upload/MBS/PDF/Whitepaper/Informations_about_Sony_Wave64.pdf
+# 128 bit RIFF-GUID { 66666972-912E-11CF-A5D6-28DB04C10000 } in little-endian 
+0	string	riff\x2E\x91\xCF\x11\xA5\xD6\x28\xDB\x04\xC1\x00\x00		Sony Wave64 RIFF data
+# 128 bit + total file size (64 bits) so 24 bytes
+# then WAVE-GUID { 65766177-ACF3-11D3-8CD1-00C04F8EDB8A }
+>24	string		wave\xF3\xAC\xD3\x11\x8C\xD1\x00\xC0\x4F\x8E\xDB\x8A		\b, WAVE 64 audio
+!:mime	audio/x-w64
+
+#------------------------------------------------------------------------------
+# MBWF/RF64
+# see EBU TECH 3306 http://tech.ebu.ch/docs/tech/tech3306-2009.pdf
+0	string	RF64\xff\xff\xff\xffWAVEds64		MBWF/RF64 audio
+!:mime	audio/x-wav
--- a/magic/rpm
+++ b/magic/rpm
@ -0,0 +1,12 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: rpm,v 1.11 2011/06/14 12:47:41 christos Exp $
+#
+# RPM: file(1) magic for Red Hat Packages   Erik Troan (ewt@redhat.com)
+#
+0	belong		0xedabeedb	RPM
+!:mime	application/x-rpm
+
+#delta RPM    Daniel Novotny (dnovotny@redhat.com)
+0	string		drpm		Delta RPM
+!:mime  application/x-rpm
--- a/magic/rtf
+++ b/magic/rtf
@ -0,0 +1,9 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File$
+# rtf:	file(1) magic for Rich Text Format (RTF)
+#
+# Duncan P. Simpson, D.P.Simpson@dcs.warwick.ac.uk
+#
+0	string		{\\rtf		Rich Text Format data,
+!:mime	text/rtf
--- a/magic/ruby
+++ b/magic/ruby
@ -0,0 +1,28 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: ruby,v 1.4 2010/07/08 20:24:13 christos Exp $
+# ruby:  file(1) magic for Ruby scripting language
+# URL:  http://www.ruby-lang.org/
+# From: Reuben Thomas <rrt@sc3d.org>
+
+# Ruby scripts
+0	search/1/w	#!\ /usr/bin/ruby	Ruby script text executable
+!:mime text/x-ruby
+0	search/1/w	#!\ /usr/local/bin/ruby	Ruby script text executable
+!:mime text/x-ruby
+0	search/1	#!/usr/bin/env\ ruby	Ruby script text executable
+!:mime text/x-ruby
+0	search/1	#!\ /usr/bin/env\ ruby	Ruby script text executable
+!:mime text/x-ruby
+
+# What looks like ruby, but does not have a shebang
+# (modules and such)
+# From: Lubomir Rintel <lkundrak@v3.sk>
+0	regex		\^[\ \t]*require[\ \t]'[A-Za-z_/]+'
+>0	regex		include\ [A-Z]|def\ [a-z]|\ do$
+>>0	regex		\^[\ \t]*end([\ \t]*[;#].*)?$		Ruby script text
+!:mime	text/x-ruby
+0	regex		\^[\ \t]*(class|module)[\ \t][A-Z]
+>0	regex		(modul|includ)e\ [A-Z]|def\ [a-z]
+>>0	regex		\^[\ \t]*end([\ \t]*[;#].*)?$		Ruby module source text
+!:mime	text/x-ruby
--- a/magic/sc
+++ b/magic/sc
@ -0,0 +1,7 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File$
+# sc:  file(1) magic for "sc" spreadsheet
+#
+38	string		Spreadsheet	sc spreadsheet file
+!:mime	application/x-sc
--- a/magic/sgml
+++ b/magic/sgml
@ -0,0 +1,82 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: sgml,v 1.28 2012/04/28 21:20:26 christos Exp $
+# Type:	SVG Vectorial Graphics
+# From:	Noel Torres <tecnico@ejerciciosresueltos.com>
+0	string		\<?xml\ version="
+>15	string		>\0
+>>19	search/4096	\<svg			SVG Scalable Vector Graphics image
+!:mime	image/svg+xml
+>>19	search/4096	\<gnc-v2		GnuCash file
+!:mime	application/x-gnucash
+
+# Sitemap file
+0	string/t		\<?xml\ version="
+>15	string		>\0
+>>19	search/4096	\<urlset		XML Sitemap document text
+!:mime	application/xml-sitemap
+
+# xhtml
+0	string/t		\<?xml\ version="
+>15	string		>\0
+>>19	search/4096/cWbt	\<!doctype\ html	XHTML document text
+!:mime	text/html
+0	string/t		\<?xml\ version='
+>15	string		>\0
+>>19	search/4096/cWbt	\<!doctype\ html	XHTML document text
+!:mime	text/html
+0	string/t		\<?xml\ version="
+>15	string		>\0
+>>19	search/4096/cWbt	\<html	broken XHTML document text
+!:mime	text/html
+
+#------------------------------------------------------------------------------
+# sgml:  file(1) magic for Standard Generalized Markup Language
+# HyperText Markup Language (HTML) is an SGML document type,
+# from Daniel Quinlan (quinlan@yggdrasil.com)
+# adapted to string extenstions by Anthon van der Neut <anthon@mnt.org)
+0	search/4096/cWt	\<!doctype\ html	HTML document text
+!:mime	text/html
+!:strength + 5
+0	search/4096/cwt	\<head			HTML document text
+!:mime	text/html
+!:strength + 5
+0	search/4096/cwt	\<title			HTML document text
+!:mime	text/html
+!:strength + 5
+0	search/4096/cwt	\<html			HTML document text
+!:mime	text/html
+!:strength + 5
+0	search/4096/cwt	\<script 		HTML document text
+!:mime	text/html
+!:strength + 5
+0	search/4096/cwt	\<style 		HTML document text
+!:mime	text/html
+!:strength + 5
+0	search/4096/cwt	\<table			HTML document text
+!:mime	text/html
+!:strength + 5
+0	search/4096/cwt	\<a\ href=		HTML document text
+!:mime	text/html
+!:strength + 5
+
+# Extensible markup language (XML), a subset of SGML
+# from Marc Prud'hommeaux (marc@apocalypse.org)
+0	search/1/cwt	\<?xml			XML document text
+!:mime	application/xml
+!:strength + 5
+0	string/t		\<?xml\ version\ "	XML
+!:mime	application/xml
+!:strength + 5
+0	string/t		\<?xml\ version="	XML
+!:mime	application/xml
+!:strength + 5
+0	string		\<?xml\ version='	XML
+!:mime	application/xml
+!:strength + 5
+0	search/1/wbt	\<?xml			XML document text
+!:mime	application/xml
+!:strength - 10
+0	search/1/wt	\<?XML			broken XML document text
+!:mime	application/xml
+!:strength - 10
--- a/magic/sniffer
+++ b/magic/sniffer
@ -0,0 +1,17 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# sniffer:  file(1) magic for packet capture files
+#
+# From: guy@alum.mit.edu (Guy Harris)
+#
+
+#
+# "libpcap" capture files.
+# (We call them "tcpdump capture file(s)" for now, as "tcpdump" is
+# the main program that uses that format, but there are other programs
+# that use "libpcap", or that use the same capture file format.)
+#
+0	ubelong		0xa1b2c3d4	tcpdump capture file (big-endian)
+!:mime  application/vnd.tcpdump.pcap
+0	ulelong		0xa1b2c3d4	tcpdump capture file (little-endian)
+!:mime  application/vnd.tcpdump.pcap
--- a/magic/tcl
+++ b/magic/tcl
@ -0,0 +1,23 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# file:  file(1) magic for Tcl scripting language
+# URL:  http://www.tcl.tk/
+# From: gustaf neumann
+
+# Tcl scripts
+0	search/1/w	#!\ /usr/bin/tcl	Tcl script text executable
+!:mime	text/x-lua
+0	search/1/w	#!\ /usr/local/bin/tcl	Tcl script text executable
+!:mime	text/x-tcl
+0	search/1	#!/usr/bin/env\ tcl	Tcl script text executable
+!:mime	text/x-tcl
+0	search/1	#!\ /usr/bin/env\ tcl	Tcl script text executable
+!:mime	text/x-tcl
+0	search/1/w	#!\ /usr/bin/wish	Tcl/Tk script text executable
+!:mime	text/x-tcl
+0	search/1/w	#!\ /usr/local/bin/wish	Tcl/Tk script text executable
+!:mime	text/x-tcl
+0	search/1	#!/usr/bin/env\ wish	Tcl/Tk script text executable
+!:mime	text/x-tcl
+0	search/1	#!\ /usr/bin/env\ wish	Tcl/Tk script text executable
+!:mime	text/x-tcl
--- a/magic/tex
+++ b/magic/tex
@ -0,0 +1,56 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: tex,v 1.17 2010/09/20 19:19:17 rrt Exp $
+# tex:  file(1) magic for TeX files
+#
+# XXX - needs byte-endian stuff (big-endian and little-endian DVI?)
+#
+# From <conklin@talisman.kaleida.com>
+
+# Although we may know the offset of certain text fields in TeX DVI
+# and font files, we can't use them reliably because they are not
+# zero terminated. [but we do anyway, christos]
+0	string		\367\002	TeX DVI file
+!:mime	application/x-dvi
+
+# There is no way to detect TeX Font Metric (*.tfm) files without
+# breaking them apart and reading the data.  The following patterns
+# match most *.tfm files generated by METAFONT or afm2tfm.
+2	string		\000\021	TeX font metric data
+!:mime	application/x-tex-tfm
+2	string		\000\022	TeX font metric data
+!:mime	application/x-tex-tfm
+
+# Texinfo and GNU Info, from Daniel Quinlan (quinlan@yggdrasil.com)
+0	search/1	\\input\ texinfo	Texinfo source text
+!:mime	text/x-texinfo
+0	search/1	This\ is\ Info\ file	GNU Info text
+!:mime	text/x-info
+
+# TeX documents, from Daniel Quinlan (quinlan@yggdrasil.com)
+0	search/4096	\\input		TeX document text
+!:mime	text/x-tex
+!:strength + 15
+0	search/4096	\\section	LaTeX document text
+!:mime	text/x-tex
+!:strength + 18
+0	search/4096	\\setlength	LaTeX document text
+!:mime	text/x-tex
+!:strength + 15
+0	search/4096	\\documentstyle	LaTeX document text
+!:mime	text/x-tex
+!:strength + 18
+0	search/4096	\\chapter	LaTeX document text
+!:mime	text/x-tex
+!:strength + 18
+0	search/4096	\\documentclass	LaTeX 2e document text
+!:mime	text/x-tex
+!:strength + 15
+0	search/4096	\\relax		LaTeX auxiliary file
+!:mime	text/x-tex
+!:strength + 15
+0	search/4096	\\contentsline	LaTeX table of contents
+!:mime	text/x-tex
+!:strength + 15
+0	search/4096	%\ -*-latex-*-	LaTeX document text
+!:mime	text/x-tex
--- a/magic/troff
+++ b/magic/troff
@ -0,0 +1,22 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File$
+# troff:  file(1) magic for *roff
+#
+# updated by Daniel Quinlan (quinlan@yggdrasil.com)
+
+# troff input
+0	search/1	.\\"		troff or preprocessor input text
+!:mime	text/troff
+0	search/1	'\\"		troff or preprocessor input text
+!:mime	text/troff
+0	search/1	'.\\"		troff or preprocessor input text
+!:mime	text/troff
+0	search/1	\\"		troff or preprocessor input text
+!:mime	text/troff
+0	search/1	'''		troff or preprocessor input text
+!:mime	text/troff
+0	regex/20	\^\\.[A-Za-z0-9][A-Za-z0-9][\ \t]	troff or preprocessor input text
+!:mime	text/troff
+0	regex/20	\^\\.[A-Za-z0-9][A-Za-z0-9]$	troff or preprocessor input text
+!:mime	text/troff
--- a/magic/vorbis
+++ b/magic/vorbis
@ -0,0 +1,26 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File$
+# vorbis:  file(1) magic for Ogg/Vorbis files
+#
+# From Felix von Leitner <leitner@fefe.de>
+# Extended by Beni Cherniavsky <cben@crosswinds.net>
+# Further extended by Greg Wooledge <greg@wooledge.org>
+#
+# Most (everything but the number of channels and bitrate) is commented
+# out with `##' as it's not interesting to the average user.  The most
+# probable things advanced users would want to uncomment are probably
+# the number of comments and the encoder version.
+#
+# FIXME: The first match has been made a search, so that it can skip
+# over prepended ID3 tags. This will work for MIME type detection, but
+# won't work for detecting other properties of the file (they all need
+# to be made relative to the search). In any case, if the file has ID3
+# tags, the ID3 information will be printed, not the Ogg information,
+# so until that's fixed, this doesn't matter.
+# FIXME[2]: Disable the above for now, since search assumes text mode.
+#
+# --- Ogg Framing ---
+#0		search/1000	OggS		Ogg data
+0		string	OggS		Ogg data
+!:mime		application/ogg
--- a/magic/warc
+++ b/magic/warc
@ -0,0 +1,14 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: warc,v 1.2 2009/09/19 16:28:13 christos Exp $
+# warc:  file(1) magic for WARC files
+
+0	string	WARC/	WARC Archive
+>5	string	x	version %.4s
+!:mime application/warc
+
+#------------------------------------------------------------------------------
+# Arc File Format from Internet Archive
+# see http://www.archive.org/web/researcher/ArcFileFormat.php
+0      string          filedesc://     Internet Archive File
+!:mime application/x-ia-arc
--- a/magic/windows
+++ b/magic/windows
@ -0,0 +1,19 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: windows,v 1.4 2009/09/19 16:28:13 christos Exp $
+# windows:  file(1) magic for Microsoft Windows
+#
+# This file is mainly reserved for files where programs
+# using them are run almost always on MS Windows 3.x or
+# above, or files only used exclusively in Windows OS,
+# where there is no better category to allocate for.
+# For example, even though WinZIP almost run on Windows
+# only, it is better to treat them as "archive" instead.
+# For format usable in DOS, such as generic executable
+# format, please specify under "msdos" file.
+#
+
+# From: Pal Tamas <folti@balabit.hu>
+# Autorun File
+0       string/c          [autorun]\r\n   Microsoft Windows Autorun file.
+!:mime	application/x-setupscript. 
--- a/magic/wordprocessors
+++ b/magic/wordprocessors
@ -0,0 +1,43 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: wordprocessors,v 1.16 2012/10/29 17:36:49 christos Exp $
+# wordprocessors:  file(1) magic fo word processors.
+#
+
+# Hangul (Korean) Word Processor File
+# From: Won-Kyu Park <wkpark@kldp.org>
+512	string		R\0o\0o\0t\0	Hangul (Korean) Word Processor File 2000
+!:mime	application/x-hwp
+
+# Quark Express from http://www.garykessler.net/library/file_sigs.html
+2	string	MMXPR3			Motorola Quark Express Document (English)
+!:mime	application/x-quark-xpress-3
+
+#------------------------------------------------------------------------------
+# ichitaro456: file(1) magic for Just System Word Processor Ichitaro
+#
+# Contributor kenzo-:
+# Reversed-engineered JS Ichitaro magic numbers
+#
+
+0	string		DOC
+>43	byte		0x14	Just System Word Processor Ichitaro v4
+!:mime	application/x-ichitaro4
+
+0	string		DOC
+>43	byte		0x15	Just System Word Processor Ichitaro v5
+!:mime	application/x-ichitaro5
+
+0	string		DOC
+>43	byte		0x16	Just System Word Processor Ichitaro v6
+!:mime	application/x-ichitaro6
+
+# Type: Freemind mindmap documents
+# From: Jamie Thompson <debian-bugs@jamie-thompson.co.uk>
+0	string/w	\<map\ version	Freemind document
+!:mime	application/x-freemind
+
+# Type:        Scribus
+# From:        Werner Fink <werner@suse.de>
+0	string	\<SCRIBUSUTF8NEW\ Version	Scribus Document
+!:mime	application/x-scribus
--- a/magic/xwindows
+++ b/magic/xwindows
@ -0,0 +1,11 @@
+# See COPYING file in this directory for original libmagic copyright.
+#------------------------------------------------------------------------------
+# $File: xwindows,v 1.7 2011/05/03 01:44:17 christos Exp $
+# xwindows:  file(1) magic for various X/Window system file formats.
+
+# Xcursor data
+# X11 mouse cursor format defined in libXcursor, see
+# http://www.x.org/archive/X11R6.8.1/doc/Xcursor.3.html
+# http://cgit.freedesktop.org/xorg/lib/libXcursor/tree/include/X11/Xcursor/Xcursor.h
+0	string		Xcur		Xcursor data
+!:mime	image/x-xcursor
--- a/scripts/base/frameworks/analyzer/load.bro
+++ b/scripts/base/frameworks/analyzer/load.bro
@ -0,0 +1 @@
+@load ./main
--- a/scripts/base/frameworks/analyzer/main.bro
+++ b/scripts/base/frameworks/analyzer/main.bro
@ -0,0 +1,179 @@
+##! Framework for managing Bro's protocol analyzers.
+##!
+##! The analyzer framework allows to dynamically enable or disable analyzers, as
+##! well as to manage the well-known ports which automatically activate a
+##! particular analyzer for new connections.
+##!
+##! Protocol analyzers are identified by unique tags of type
+##! :bro:type:`Analyzer::Tag`, such as :bro:enum:`Analyzer::ANALYZER_HTTP` and
+##! :bro:enum:`Analyzer::ANALYZER_HTTP`. These tags are defined internally by
+##! the analyzers themselves, and documented in their analyzer-specific
+##! description along with the events that they generate.
+
+module Analyzer;
+
+export {
+	## If true, all available analyzers are initially disabled at startup. One
+	## can then selectively enable them with
+	## :bro:id:`Analyzer::enable_analyzer`.
+	global disable_all = F &redef;
+
+	## Enables an analyzer. Once enabled, the analyzer may be used for analysis
+	## of future connections as decided by Bro's dynamic protocol detection.
+	##
+	## tag: The tag of the analyzer to enable.
+	##
+	## Returns: True if the analyzer was successfully enabled.
+	global enable_analyzer: function(tag: Analyzer::Tag) : bool;
+
+	## Disables an analyzer. Once disabled, the analyzer will not be used
+	## further for analysis of future connections.
+	##
+	## tag: The tag of the analyzer to disable.
+	##
+	## Returns: True if the analyzer was successfully disabled.
+	global disable_analyzer: function(tag: Analyzer::Tag) : bool;
+
+	## Registers a set of well-known ports for an analyzer. If a future
+	## connection on one of these ports is seen, the analyzer will be
+	## automatically assigned to parsing it. The function *adds* to all ports
+	## already registered, it doesn't replace them.
+	##
+	## tag: The tag of the analyzer.
+	##
+	## ports: The set of well-known ports to associate with the analyzer.
+	##
+	## Returns: True if the ports were sucessfully registered.
+	global register_for_ports: function(tag: Analyzer::Tag, ports: set[port]) : bool;
+
+	## Registers an individual well-known port for an analyzer. If a future
+	## connection on this port is seen, the analyzer will be automatically
+	## assigned to parsing it. The function *adds* to all ports already
+	## registered, it doesn't replace them.
+	##
+	## tag: The tag of the analyzer.
+	##
+	## p: The well-known port to associate with the analyzer.
+	##
+	## Returns: True if the port was sucessfully registered.
+	global register_for_port: function(tag: Analyzer::Tag, p: port) : bool;
+
+	## Returns a set of all well-known ports currently registered for a
+	## specific analyzer.
+	##
+	## tag: The tag of the analyzer.
+	##
+	## Returns: The set of ports.
+	global registered_ports: function(tag: Analyzer::Tag) : set[port];
+
+	## Returns a table of all ports-to-analyzer mappings currently registered.
+	##
+	## Returns: A table mapping each analyzer to the set of ports
+	##          registered for it.
+	global all_registered_ports: function() : table[Analyzer::Tag] of set[port];
+
+	## Translates an analyzer type to a string with the analyzer's name.
+	##
+	## tag: The analyzer tag.
+	##
+	## Returns: The analyzer name corresponding to the tag.
+	global name: function(tag: Analyzer::Tag) : string;
+
+	## Schedules an analyzer for a future connection originating from a given IP
+	## address and port.
+	##
+	## orig: The IP address originating a connection in the future.
+	##       0.0.0.0 can be used as a wildcard to match any originator address.
+	##
+	## resp: The IP address responding to a connection from *orig*.
+	##
+	## resp_p: The destination port at *resp*.
+	##
+	## analyzer: The analyzer ID.
+	##
+	## tout: A timeout interval after which the scheduling request will be
+	##       discarded if the connection has not yet been seen.
+	##
+	## Returns: True if succesful.
+	global schedule_analyzer: function(orig: addr, resp: addr, resp_p: port,
+					   analyzer: Analyzer::Tag, tout: interval) : bool;
+
+	## A set of analyzers to disable by default at startup. The default set
+	## contains legacy analyzers that are no longer supported.
+	global disabled_analyzers: set[Analyzer::Tag] = {
+		ANALYZER_INTERCONN,
+		ANALYZER_STEPPINGSTONE,
+		ANALYZER_BACKDOOR,
+		ANALYZER_TCPSTATS,
+	} &redef;
+}
+
+@load base/bif/analyzer.bif
+
+global ports: table[Analyzer::Tag] of set[port];
+
+event bro_init() &priority=5
+	{
+	if ( disable_all )
+		__disable_all_analyzers();
+
+	for ( a in disabled_analyzers )
+		disable_analyzer(a);
+	}
+
+function enable_analyzer(tag: Analyzer::Tag) : bool
+	{
+	return __enable_analyzer(tag);
+	}
+
+function disable_analyzer(tag: Analyzer::Tag) : bool
+	{
+	return __disable_analyzer(tag);
+	}
+
+function register_for_ports(tag: Analyzer::Tag, ports: set[port]) : bool
+	{
+	local rc = T;
+
+	for ( p in ports )
+		{
+		if ( ! register_for_port(tag, p) )
+			rc = F;
+		}
+
+	return rc;
+	}
+
+function register_for_port(tag: Analyzer::Tag, p: port) : bool
+	{
+	if ( ! __register_for_port(tag, p) )
+		return F;
+
+	if ( tag !in ports )
+		ports[tag] = set();
+
+	add ports[tag][p];
+	return T;
+	}
+
+function registered_ports(tag: Analyzer::Tag) : set[port]
+	{
+	return tag in ports ? ports[tag] : set();
+	}
+
+function all_registered_ports(): table[Analyzer::Tag] of set[port]
+	{
+	return ports;
+	}
+
+function name(atype: Analyzer::Tag) : string
+	{
+	return __name(atype);
+	}
+
+function schedule_analyzer(orig: addr, resp: addr, resp_p: port,
+			   analyzer: Analyzer::Tag, tout: interval) : bool
+	{
+	return __schedule_analyzer(orig, resp, resp_p, analyzer, tout);
+	}
+
--- a/scripts/base/frameworks/cluster/main.bro
+++ b/scripts/base/frameworks/cluster/main.bro
@ -39,7 +39,7 @@ export {
 		## The node type doing all the actual traffic analysis.
 		WORKER,
 		## A node acting as a traffic recorder using the
-		## `Time Machine <http://tracker.bro-ids.org/time-machine>`_ software.
+		## `Time Machine <http://tracker.bro.org/time-machine>`_ software.
 		TIME_MACHINE,
 	};
 	
--- a/scripts/base/frameworks/dpd/main.bro
+++ b/scripts/base/frameworks/dpd/main.bro
@ -41,22 +41,11 @@ redef record connection += {
 event bro_init() &priority=5
 	{
 	Log::create_stream(DPD::LOG, [$columns=Info]);
-	
-	# Populate the internal DPD analysis variable.
-	for ( a in dpd_config )
-		{
-		for ( p in dpd_config[a]$ports )
-			{
-			if ( p !in dpd_analyzer_ports )
-				dpd_analyzer_ports[p] = set();
-			add dpd_analyzer_ports[p][a];
-			}
-		}
 	}

-event protocol_confirmation(c: connection, atype: count, aid: count) &priority=10
+event protocol_confirmation(c: connection, atype: Analyzer::Tag, aid: count) &priority=10
 	{
-	local analyzer = analyzer_name(atype);
+	local analyzer = Analyzer::name(atype);

 	if ( fmt("-%s",analyzer) in c$service )
 		delete c$service[fmt("-%s", analyzer)];
@ -64,10 +53,10 @@ event protocol_confirmation(c: connection, atype: count, aid: count) &priority=1
 	add c$service[analyzer];
 	}

-event protocol_violation(c: connection, atype: count, aid: count,
+event protocol_violation(c: connection, atype: Analyzer::Tag, aid: count,
                         reason: string) &priority=10
 	{
-	local analyzer = analyzer_name(atype);
+	local analyzer = Analyzer::name(atype);
 	# If the service hasn't been confirmed yet, don't generate a log message
 	# for the protocol violation.
 	if ( analyzer !in c$service )
@ -86,7 +75,7 @@ event protocol_violation(c: connection, atype: count, aid: count,
 	c$dpd = info;
 	}

-event protocol_violation(c: connection, atype: count, aid: count, reason: string) &priority=5
+event protocol_violation(c: connection, atype: Analyzer::Tag, aid: count, reason: string) &priority=5
 	{
 	if ( !c?$dpd || aid in c$dpd$disabled_aids )
 		return;
@ -100,7 +89,7 @@ event protocol_violation(c: connection, atype: count, aid: count, reason: string
 	add c$dpd$disabled_aids[aid];
 	}

-event protocol_violation(c: connection, atype: count, aid: count,
+event protocol_violation(c: connection, atype: Analyzer::Tag, aid: count,
 				reason: string) &priority=-5
 	{
 	if ( c?$dpd )
--- a/scripts/base/frameworks/file-analysis/load.bro
+++ b/scripts/base/frameworks/file-analysis/load.bro
@ -0,0 +1 @@
+@load ./main.bro
--- a/scripts/base/frameworks/file-analysis/main.bro
+++ b/scripts/base/frameworks/file-analysis/main.bro
@ -0,0 +1,261 @@
+##! An interface for driving the analysis of files, possibly independent of
+##! any network protocol over which they're transported.
+
+@load base/bif/file_analysis.bif
+@load base/frameworks/logging
+
+module FileAnalysis;
+
+export {
+	redef enum Log::ID += {
+		## Logging stream for file analysis.
+		LOG
+	};
+
+	## A structure which represents a desired type of file analysis.
+	type AnalyzerArgs: record {
+		## The type of analysis.
+		tag: FileAnalysis::Tag;
+
+		## The local filename to which to write an extracted file.  Must be
+		## set when *tag* is :bro:see:`FileAnalysis::ANALYZER_EXTRACT`.
+		extract_filename: string &optional;
+
+		## An event which will be generated for all new file contents,
+		## chunk-wise.  Used when *tag* is
+		## :bro:see:`FileAnalysis::ANALYZER_DATA_EVENT`.
+		chunk_event: event(f: fa_file, data: string, off: count) &optional;
+
+		## An event which will be generated for all new file contents,
+		## stream-wise.  Used when *tag* is
+		## :bro:see:`FileAnalysis::ANALYZER_DATA_EVENT`.
+		stream_event: event(f: fa_file, data: string) &optional;
+	} &redef;
+
+	## Contains all metadata related to the analysis of a given file.
+	## For the most part, fields here are derived from ones of the same name
+	## in :bro:see:`fa_file`.
+	type Info: record {
+		## An identifier associated with a single file.
+		id: string &log;
+
+		## Identifier associated with a container file from which this one was
+		## extracted as part of the file analysis.
+		parent_id: string &log &optional;
+
+		## An identification of the source of the file data.  E.g. it may be
+		## a network protocol over which it was transferred, or a local file
+		## path which was read, or some other input source.
+		source: string &log &optional;
+
+		## If the source of this file is is a network connection, this field
+		## may be set to indicate the directionality.
+		is_orig: bool &log &optional;
+
+		## The time at which the last activity for the file was seen.
+		last_active: time &log;
+
+		## Number of bytes provided to the file analysis engine for the file.
+		seen_bytes: count &log &default=0;
+
+		## Total number of bytes that are supposed to comprise the full file.
+		total_bytes: count &log &optional;
+
+		## The number of bytes in the file stream that were completely missed
+		## during the process of analysis e.g. due to dropped packets.
+		missing_bytes: count &log &default=0;
+
+		## The number of not all-in-sequence bytes in the file stream that
+		## were delivered to file analyzers due to reassembly buffer overflow.
+		overflow_bytes: count &log &default=0;
+
+		## The amount of time between receiving new data for this file that
+		## the analysis engine will wait before giving up on it.
+		timeout_interval: interval &log &optional;
+
+		## The number of bytes at the beginning of a file to save for later
+		## inspection in *bof_buffer* field.
+		bof_buffer_size: count &log &optional;
+
+		## A mime type provided by libmagic against the *bof_buffer*, or
+		## in the cases where no buffering of the beginning of file occurs,
+		## an initial guess of the mime type based on the first data seen.
+		mime_type: string &log &optional;
+
+		## Whether the file analysis timed out at least once for the file.
+		timedout: bool &log &default=F;
+
+		## Connection UIDS over which the file was transferred.
+		conn_uids: set[string] &log;
+
+		## A set of analysis types done during the file analysis.
+		analyzers: set[FileAnalysis::Tag];
+
+		## Local filenames of extracted files.
+		extracted_files: set[string] &log;
+
+		## An MD5 digest of the file contents.
+		md5: string &log &optional;
+
+		## A SHA1 digest of the file contents.
+		sha1: string &log &optional;
+
+		## A SHA256 digest of the file contents.
+		sha256: string &log &optional;
+	} &redef;
+
+	## A table that can be used to disable file analysis completely for
+	## any files transferred over given network protocol analyzers.
+	const disable: table[Analyzer::Tag] of bool = table() &redef;
+
+	## Event that can be handled to access the Info record as it is sent on
+	## to the logging framework.
+	global log_file_analysis: event(rec: Info);
+
+	## The salt concatenated to unique file handle strings generated by
+	## :bro:see:`get_file_handle` before hashing them in to a file id
+	## (the *id* field of :bro:see:`fa_file`).
+	## Provided to help mitigate the possiblility of manipulating parts of
+	## network connections that factor in to the file handle in order to
+	## generate two handles that would hash to the same file id.
+	const salt = "I recommend changing this." &redef;
+
+	## Sets the *timeout_interval* field of :bro:see:`fa_file`, which is
+	## used to determine the length of inactivity that is allowed for a file
+	## before internal state related to it is cleaned up.  When used within a
+	## :bro:see:`file_timeout` handler, the analysis will delay timing out
+	## again for the period specified by *t*.
+	##
+	## f: the file.
+	##
+	## t: the amount of time the file can remain inactive before discarding.
+	##
+	## Returns: true if the timeout interval was set, or false if analysis
+	##          for the *id* isn't currently active.
+	global set_timeout_interval: function(f: fa_file, t: interval): bool;
+
+	## Adds an analyzer to the analysis of a given file.
+	##
+	## f: the file.
+	##
+	## args: the analyzer type to add along with any arguments it takes.
+	##
+	## Returns: true if the analyzer will be added, or false if analysis
+	##          for the *id* isn't currently active or the *args*
+	##          were invalid for the analyzer type.
+	global add_analyzer: function(f: fa_file, args: AnalyzerArgs): bool;
+
+	## Removes an analyzer from the analysis of a given file.
+	##
+	## f: the file.
+	##
+	## args: the analyzer (type and args) to remove.
+	##
+	## Returns: true if the analyzer will be removed, or false if analysis
+	##          for the *id* isn't currently active.
+	global remove_analyzer: function(f: fa_file, args: AnalyzerArgs): bool;
+
+	## Stops/ignores any further analysis of a given file.
+	##
+	## f: the file.
+	##
+	## Returns: true if analysis for the given file will be ignored for the
+	##          rest of it's contents, or false if analysis for the *id*
+	##          isn't currently active.
+	global stop: function(f: fa_file): bool;
+}
+
+redef record fa_file += {
+	info: Info &optional;
+};
+
+function set_info(f: fa_file)
+	{
+	if ( ! f?$info )
+		{
+		local tmp: Info;
+		f$info = tmp;
+		}
+
+	f$info$id = f$id;
+	if ( f?$parent_id ) f$info$parent_id = f$parent_id;
+	if ( f?$source ) f$info$source = f$source;
+	if ( f?$is_orig ) f$info$is_orig = f$is_orig;
+	f$info$last_active = f$last_active;
+	f$info$seen_bytes = f$seen_bytes;
+	if ( f?$total_bytes ) f$info$total_bytes = f$total_bytes;
+	f$info$missing_bytes = f$missing_bytes;
+	f$info$overflow_bytes = f$overflow_bytes;
+	f$info$timeout_interval = f$timeout_interval;
+	f$info$bof_buffer_size = f$bof_buffer_size;
+	if ( f?$mime_type ) f$info$mime_type = f$mime_type;
+	if ( f?$conns )
+		for ( cid in f$conns )
+			add f$info$conn_uids[f$conns[cid]$uid];
+	}
+
+function set_timeout_interval(f: fa_file, t: interval): bool
+	{
+	return __set_timeout_interval(f$id, t);
+	}
+
+function add_analyzer(f: fa_file, args: AnalyzerArgs): bool
+	{
+	if ( ! __add_analyzer(f$id, args) ) return F;
+
+	set_info(f);
+	add f$info$analyzers[args$tag];
+
+	if ( args$tag == FileAnalysis::ANALYZER_EXTRACT )
+		add f$info$extracted_files[args$extract_filename];
+
+	return T;
+	}
+
+function remove_analyzer(f: fa_file, args: AnalyzerArgs): bool
+	{
+	return __remove_analyzer(f$id, args);
+	}
+
+function stop(f: fa_file): bool
+	{
+	return __stop(f$id);
+	}
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(FileAnalysis::LOG,
+	                   [$columns=Info, $ev=log_file_analysis]);
+	}
+
+event file_timeout(f: fa_file) &priority=5
+	{
+	set_info(f);
+	f$info$timedout = T;
+	}
+
+event file_hash(f: fa_file, kind: string, hash: string) &priority=5
+	{
+	set_info(f);
+	switch ( kind ) {
+	case "md5":
+		f$info$md5 = hash;
+		break;
+	case "sha1":
+		f$info$sha1 = hash;
+		break;
+	case "sha256":
+		f$info$sha256 = hash;
+		break;
+	}
+	}
+
+event file_state_remove(f: fa_file) &priority=5
+	{
+	set_info(f);
+	}
+
+event file_state_remove(f: fa_file) &priority=-5
+	{
+	Log::write(FileAnalysis::LOG, f$info);
+	}
--- a/scripts/base/frameworks/input/load.bro
+++ b/scripts/base/frameworks/input/load.bro
@ -2,4 +2,5 @@
@load ./readers/ascii
@load ./readers/raw
@load ./readers/benchmark
-
+@load ./readers/binary
+@load ./readers/sqlite
--- a/Show more
+++ b/Show more
 @ -1 +1 @@
 .1-167
 .1-784