mirror of
https://github.com/zeek/zeek.git
synced 2025-10-02 14:48:21 +00:00
Merge remote branch 'origin/master' into topic/bernhard/input-update
This commit is contained in:
Bernhard Amann
commit
3719524a6a
423 changed files with 240373 additions and 161770 deletions
158
CHANGES
158
CHANGES
|
|
@ -1,4 +1,162 @@
|
|||
|
||||
2.1-659 | 2013-05-24 17:24:18 -0700
|
||||
|
||||
* Fix broken/missing documentation. (Jon Siwek)
|
||||
|
||||
* Fixing test that would fail without ES/curl support. (Robin
|
||||
Sommer)
|
||||
|
||||
2.1-656 | 2013-05-17 15:58:07 -0700
|
||||
|
||||
* Fix mutex lock problem for writers. (Bernhard Amann)
|
||||
|
||||
2.1-654 | 2013-05-17 13:49:52 -0700
|
||||
|
||||
* Tweaks to sqlite3 configuration to address threading issues.
|
||||
(Bernhard Amann)
|
||||
|
||||
2.1-651 | 2013-05-17 13:37:16 -0700
|
||||
|
||||
* Fix uninitialized DPM member. (Jon Siwek)
|
||||
|
||||
* Fix issue with transaction ID reuse in a single DNS connection. (Seth Hall)
|
||||
|
||||
* New function added to the queue.bro script to support peeking at
|
||||
the new gettable item in the queue without removing it. (Seth Hall)
|
||||
|
||||
2.1-647 | 2013-05-17 07:47:14 -0700
|
||||
|
||||
* Fixing Broxygen generation to have BROMAGIC set. (Robin Sommer)
|
||||
|
||||
* Fix for 'fchmod undeclared here' on FreeBSD. (Robin Sommer)
|
||||
|
||||
* CMake policy fix to avoid errors with older versions. (Robin
|
||||
Sommer)
|
||||
|
||||
2.1-641 | 2013-05-15 18:15:09 -0700
|
||||
|
||||
* Test update. (Robin Sommer)
|
||||
|
||||
2.1-640 | 2013-05-15 17:24:09 -0700
|
||||
|
||||
* Support for cleaning up threads that have terminated. (Bernhard
|
||||
Amann and Robin Sommer). Includes:
|
||||
|
||||
- Both logging and input frameworks now clean up threads once
|
||||
they aren't further needed anymnore.
|
||||
|
||||
- New function Log::remove_stream() that removes a logging
|
||||
stream, stopping all writer threads that are associated with
|
||||
it. Note, however, that removing a *filter* from a stream
|
||||
still doesn't clean up any threads. The problem is that
|
||||
because of the output paths potentially being created
|
||||
dynamically it's unclear if the writer thread will still be
|
||||
needed in the future.
|
||||
|
||||
2.1-626 | 2013-05-15 16:09:31 -0700
|
||||
|
||||
* Add "reservoir" sampler for SumStats framework. This maintains
|
||||
a set of N uniquely distributed random samples. (Bernhard Amann)
|
||||
|
||||
2.1-619 | 2013-05-15 16:01:42 -0700
|
||||
|
||||
* SQLite reader and writer combo. This allows to read/write
|
||||
persistent data from on disk SQLite databases. The current
|
||||
interface is quite low-level, we'll add higher-level abstractions
|
||||
in the future. (Bernhard Amann)
|
||||
|
||||
2.1-576 | 2013-05-15 14:29:09 -0700
|
||||
|
||||
* Initial version of new file analysis framework. This moves most of
|
||||
the processing of file content from script-land into the core,
|
||||
where it belongs. Much of this is an internal change, and at this
|
||||
point the new code has essentially feature-equality with the old
|
||||
one. More script-level changes to come. (Jon Siwek)
|
||||
|
||||
2.1-502 | 2013-05-10 19:29:37 -0700
|
||||
|
||||
* Allow default function/hook/event parameters. Addresses #972. (Jon
|
||||
Siwek)
|
||||
|
||||
* Change the endianness parameter of bytestring_to_count() BIF to
|
||||
default to false (big endian). (Jon Siwek)
|
||||
|
||||
2.1-500 | 2013-05-10 19:22:24 -0700
|
||||
|
||||
* Fix to prevent merge-hook of SumStat's unique plugin from damaging
|
||||
source data. (Bernhard Amann)
|
||||
|
||||
2.1-498 | 2013-05-03 17:44:08 -0700
|
||||
|
||||
* Table lookups return copy of non-const &default vals. This
|
||||
prevents unintentional modifications to the &default value itself.
|
||||
Addresses #981. (Jon Siwek)
|
||||
|
||||
2.1-496 | 2013-05-03 15:54:47 -0700
|
||||
|
||||
* Fix memory leak and unnecessary allocations in OpaqueVal.
|
||||
Addresses #986. (Matthias Vallentin)
|
||||
|
||||
2.1-492 | 2013-05-02 12:46:26 -0700
|
||||
|
||||
* Work-around for sumstats framework not propagating updates after
|
||||
intermediate check in cluster environments. (Bernhard Amann)
|
||||
|
||||
* Always apply tcp_connection_attempt. Before this change it was
|
||||
only applied when a connection_attempt() event handler was
|
||||
defined. (Robin Sommer)
|
||||
|
||||
* Fixing coverage.bare-mode-errors test. (Robin Sommer)
|
||||
|
||||
2.1-487 | 2013-05-01 18:03:22 -0700
|
||||
|
||||
* Always apply tcp_connection_attempt timer, even if no
|
||||
connection_attempt() event handler is defined. (Robin Sommer)
|
||||
|
||||
2.1-486 | 2013-05-01 15:28:45 -0700
|
||||
|
||||
* New framework for computing summary statistics in
|
||||
base/framework/sumstats. This replaces the metrics frameworks, and
|
||||
comes with a number of applications build on top, see NEWS. More
|
||||
documentation to follow. (Seth Hall)
|
||||
|
||||
2.1-397 | 2013-04-29 21:19:00 -0700
|
||||
|
||||
* Fixing memory leaks in CompHash implementation. Addresses #987.
|
||||
(Robin Sommer)
|
||||
|
||||
2.1-394 | 2013-04-27 15:02:31 -0700
|
||||
|
||||
* Fixed a bug in the vulnerable software script and added a test.
|
||||
(Seth Hall)
|
||||
|
||||
* Fix schedule statements used outside event handlers. Addresses
|
||||
#974. (Jon Siwek)
|
||||
|
||||
* Fix record coercion for default inner record fields. Addresses
|
||||
#973. (Jon Siwek)
|
||||
|
||||
* Add bytestring_to_count function to bro.bif. Addresses #968. (Yun
|
||||
Zheng Hu)
|
||||
|
||||
2.1-386 | 2013-03-22 12:41:50 -0700
|
||||
|
||||
* Added reverse() function to strings.bif. (Yun Zheng Hu)
|
||||
|
||||
2.1-384 | 2013-03-22 12:10:14 -0700
|
||||
|
||||
* Fix record constructors in table initializer indices. Addresses
|
||||
#660. (Jon Siwek)
|
||||
|
||||
2.1-382 | 2013-03-22 12:01:34 -0700
|
||||
|
||||
* Add support for 802.1ah (Q-in-Q). Addresses #641. (Seth Hall)
|
||||
|
||||
2.1-380 | 2013-03-18 12:18:10 -0700
|
||||
|
||||
* Fix gcc compile warnings in base64 encoder and benchmark reader.
|
||||
(Bernhard Amann)
|
||||
|
||||
2.1-377 | 2013-03-17 17:36:09 -0700
|
||||
|
||||
* Fixing potential leak in DNS error case. (Vlad Grigorescu)
|
||||
|
|
|
|||
|
|
@ -17,12 +17,17 @@ set(BRO_SCRIPT_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/scripts)
|
|||
get_filename_component(BRO_SCRIPT_INSTALL_PATH ${BRO_SCRIPT_INSTALL_PATH}
|
||||
ABSOLUTE)
|
||||
|
||||
set(BRO_MAGIC_INSTALL_PATH ${BRO_ROOT_DIR}/share/bro/magic)
|
||||
set(BRO_MAGIC_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/magic)
|
||||
|
||||
configure_file(bro-path-dev.in ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev)
|
||||
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.sh
|
||||
"export BROPATH=`${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n"
|
||||
"export BROMAGIC=\"${BRO_MAGIC_SOURCE_PATH}\"\n"
|
||||
"export PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n")
|
||||
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.csh
|
||||
"setenv BROPATH `${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n"
|
||||
"setenv BROMAGIC \"${BRO_MAGIC_SOURCE_PATH}\"\n"
|
||||
"setenv PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n")
|
||||
|
||||
file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1)
|
||||
|
|
@ -69,6 +74,12 @@ if (MISSING_PREREQS)
|
|||
message(FATAL_ERROR "Configuration aborted due to missing prerequisites")
|
||||
endif ()
|
||||
|
||||
set(libmagic_req 5.04)
|
||||
if ( LibMagic_VERSION VERSION_LESS ${libmagic_req} )
|
||||
message(FATAL_ERROR "libmagic of at least version ${libmagic_req} required "
|
||||
"(found ${LibMagic_VERSION})")
|
||||
endif ()
|
||||
|
||||
include_directories(BEFORE
|
||||
${PCAP_INCLUDE_DIR}
|
||||
${OpenSSL_INCLUDE_DIR}
|
||||
|
|
@ -190,6 +201,11 @@ CheckOptionalBuildSources(aux/broctl Broctl INSTALL_BROCTL)
|
|||
CheckOptionalBuildSources(aux/bro-aux Bro-Aux INSTALL_AUX_TOOLS)
|
||||
CheckOptionalBuildSources(aux/broccoli Broccoli INSTALL_BROCCOLI)
|
||||
|
||||
install(DIRECTORY ./magic/ DESTINATION ${BRO_MAGIC_INSTALL_PATH} FILES_MATCHING
|
||||
PATTERN "COPYING" EXCLUDE
|
||||
PATTERN "*"
|
||||
)
|
||||
|
||||
########################################################################
|
||||
## Packaging Setup
|
||||
|
||||
|
|
|
|||
|
|
@ -1,144 +0,0 @@
|
|||
# DO NOT EDIT
|
||||
# This file is auto-generated from the genDocSourcesList.sh script.
|
||||
#
|
||||
# This is a list of Bro script sources for which to generate reST documentation.
|
||||
# It will be included inline in the CMakeLists.txt found in the same directory
|
||||
# in order to create Makefile targets that define how to generate reST from
|
||||
# a given Bro script.
|
||||
#
|
||||
# Note: any path prefix of the script (2nd argument of rest_target macro)
|
||||
# will be used to derive what path under scripts/ the generated documentation
|
||||
# will be placed.
|
||||
|
||||
set(psd ${PROJECT_SOURCE_DIR}/scripts)
|
||||
|
||||
rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
|
||||
rest_target(${psd} base/init-default.bro internal)
|
||||
rest_target(${psd} base/init-bare.bro internal)
|
||||
|
||||
rest_target(${CMAKE_BINARY_DIR}/src base/bro.bif.bro)
|
||||
rest_target(${CMAKE_BINARY_DIR}/src base/const.bif.bro)
|
||||
rest_target(${CMAKE_BINARY_DIR}/src base/event.bif.bro)
|
||||
rest_target(${CMAKE_BINARY_DIR}/src base/logging.bif.bro)
|
||||
rest_target(${CMAKE_BINARY_DIR}/src base/reporter.bif.bro)
|
||||
rest_target(${CMAKE_BINARY_DIR}/src base/strings.bif.bro)
|
||||
rest_target(${CMAKE_BINARY_DIR}/src base/types.bif.bro)
|
||||
rest_target(${psd} base/frameworks/cluster/main.bro)
|
||||
rest_target(${psd} base/frameworks/cluster/nodes/manager.bro)
|
||||
rest_target(${psd} base/frameworks/cluster/nodes/proxy.bro)
|
||||
rest_target(${psd} base/frameworks/cluster/nodes/worker.bro)
|
||||
rest_target(${psd} base/frameworks/cluster/setup-connections.bro)
|
||||
rest_target(${psd} base/frameworks/communication/main.bro)
|
||||
rest_target(${psd} base/frameworks/control/main.bro)
|
||||
rest_target(${psd} base/frameworks/dpd/main.bro)
|
||||
rest_target(${psd} base/frameworks/intel/main.bro)
|
||||
rest_target(${psd} base/frameworks/logging/main.bro)
|
||||
rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro)
|
||||
rest_target(${psd} base/frameworks/logging/postprocessors/sftp.bro)
|
||||
rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
|
||||
rest_target(${psd} base/frameworks/logging/writers/dataseries.bro)
|
||||
rest_target(${psd} base/frameworks/metrics/cluster.bro)
|
||||
rest_target(${psd} base/frameworks/metrics/main.bro)
|
||||
rest_target(${psd} base/frameworks/metrics/non-cluster.bro)
|
||||
rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
|
||||
rest_target(${psd} base/frameworks/notice/actions/drop.bro)
|
||||
rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
|
||||
rest_target(${psd} base/frameworks/notice/actions/page.bro)
|
||||
rest_target(${psd} base/frameworks/notice/actions/pp-alarms.bro)
|
||||
rest_target(${psd} base/frameworks/notice/cluster.bro)
|
||||
rest_target(${psd} base/frameworks/notice/extend-email/hostnames.bro)
|
||||
rest_target(${psd} base/frameworks/notice/main.bro)
|
||||
rest_target(${psd} base/frameworks/notice/weird.bro)
|
||||
rest_target(${psd} base/frameworks/packet-filter/main.bro)
|
||||
rest_target(${psd} base/frameworks/packet-filter/netstats.bro)
|
||||
rest_target(${psd} base/frameworks/reporter/main.bro)
|
||||
rest_target(${psd} base/frameworks/signatures/main.bro)
|
||||
rest_target(${psd} base/frameworks/software/main.bro)
|
||||
rest_target(${psd} base/protocols/conn/contents.bro)
|
||||
rest_target(${psd} base/protocols/conn/inactivity.bro)
|
||||
rest_target(${psd} base/protocols/conn/main.bro)
|
||||
rest_target(${psd} base/protocols/dns/consts.bro)
|
||||
rest_target(${psd} base/protocols/dns/main.bro)
|
||||
rest_target(${psd} base/protocols/ftp/file-extract.bro)
|
||||
rest_target(${psd} base/protocols/ftp/main.bro)
|
||||
rest_target(${psd} base/protocols/ftp/utils-commands.bro)
|
||||
rest_target(${psd} base/protocols/http/file-extract.bro)
|
||||
rest_target(${psd} base/protocols/http/file-hash.bro)
|
||||
rest_target(${psd} base/protocols/http/file-ident.bro)
|
||||
rest_target(${psd} base/protocols/http/main.bro)
|
||||
rest_target(${psd} base/protocols/http/utils.bro)
|
||||
rest_target(${psd} base/protocols/irc/dcc-send.bro)
|
||||
rest_target(${psd} base/protocols/irc/main.bro)
|
||||
rest_target(${psd} base/protocols/smtp/entities-excerpt.bro)
|
||||
rest_target(${psd} base/protocols/smtp/entities.bro)
|
||||
rest_target(${psd} base/protocols/smtp/main.bro)
|
||||
rest_target(${psd} base/protocols/ssh/main.bro)
|
||||
rest_target(${psd} base/protocols/ssl/consts.bro)
|
||||
rest_target(${psd} base/protocols/ssl/main.bro)
|
||||
rest_target(${psd} base/protocols/ssl/mozilla-ca-list.bro)
|
||||
rest_target(${psd} base/protocols/syslog/consts.bro)
|
||||
rest_target(${psd} base/protocols/syslog/main.bro)
|
||||
rest_target(${psd} base/utils/addrs.bro)
|
||||
rest_target(${psd} base/utils/conn-ids.bro)
|
||||
rest_target(${psd} base/utils/directions-and-hosts.bro)
|
||||
rest_target(${psd} base/utils/files.bro)
|
||||
rest_target(${psd} base/utils/numbers.bro)
|
||||
rest_target(${psd} base/utils/paths.bro)
|
||||
rest_target(${psd} base/utils/patterns.bro)
|
||||
rest_target(${psd} base/utils/site.bro)
|
||||
rest_target(${psd} base/utils/strings.bro)
|
||||
rest_target(${psd} base/utils/thresholds.bro)
|
||||
rest_target(${psd} policy/frameworks/communication/listen.bro)
|
||||
rest_target(${psd} policy/frameworks/control/controllee.bro)
|
||||
rest_target(${psd} policy/frameworks/control/controller.bro)
|
||||
rest_target(${psd} policy/frameworks/dpd/detect-protocols.bro)
|
||||
rest_target(${psd} policy/frameworks/dpd/packet-segment-logging.bro)
|
||||
rest_target(${psd} policy/frameworks/metrics/conn-example.bro)
|
||||
rest_target(${psd} policy/frameworks/metrics/http-example.bro)
|
||||
rest_target(${psd} policy/frameworks/metrics/ssl-example.bro)
|
||||
rest_target(${psd} policy/frameworks/software/version-changes.bro)
|
||||
rest_target(${psd} policy/frameworks/software/vulnerable.bro)
|
||||
rest_target(${psd} policy/integration/barnyard2/main.bro)
|
||||
rest_target(${psd} policy/integration/barnyard2/types.bro)
|
||||
rest_target(${psd} policy/misc/analysis-groups.bro)
|
||||
rest_target(${psd} policy/misc/capture-loss.bro)
|
||||
rest_target(${psd} policy/misc/loaded-scripts.bro)
|
||||
rest_target(${psd} policy/misc/profiling.bro)
|
||||
rest_target(${psd} policy/misc/stats.bro)
|
||||
rest_target(${psd} policy/misc/trim-trace-file.bro)
|
||||
rest_target(${psd} policy/protocols/conn/known-hosts.bro)
|
||||
rest_target(${psd} policy/protocols/conn/known-services.bro)
|
||||
rest_target(${psd} policy/protocols/conn/weirds.bro)
|
||||
rest_target(${psd} policy/protocols/dns/auth-addl.bro)
|
||||
rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
|
||||
rest_target(${psd} policy/protocols/ftp/detect.bro)
|
||||
rest_target(${psd} policy/protocols/ftp/software.bro)
|
||||
rest_target(${psd} policy/protocols/http/detect-MHR.bro)
|
||||
rest_target(${psd} policy/protocols/http/detect-intel.bro)
|
||||
rest_target(${psd} policy/protocols/http/detect-sqli.bro)
|
||||
rest_target(${psd} policy/protocols/http/detect-webapps.bro)
|
||||
rest_target(${psd} policy/protocols/http/header-names.bro)
|
||||
rest_target(${psd} policy/protocols/http/software-browser-plugins.bro)
|
||||
rest_target(${psd} policy/protocols/http/software.bro)
|
||||
rest_target(${psd} policy/protocols/http/var-extraction-cookies.bro)
|
||||
rest_target(${psd} policy/protocols/http/var-extraction-uri.bro)
|
||||
rest_target(${psd} policy/protocols/smtp/blocklists.bro)
|
||||
rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro)
|
||||
rest_target(${psd} policy/protocols/smtp/software.bro)
|
||||
rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro)
|
||||
rest_target(${psd} policy/protocols/ssh/geo-data.bro)
|
||||
rest_target(${psd} policy/protocols/ssh/interesting-hostnames.bro)
|
||||
rest_target(${psd} policy/protocols/ssh/software.bro)
|
||||
rest_target(${psd} policy/protocols/ssl/cert-hash.bro)
|
||||
rest_target(${psd} policy/protocols/ssl/expiring-certs.bro)
|
||||
rest_target(${psd} policy/protocols/ssl/extract-certs-pem.bro)
|
||||
rest_target(${psd} policy/protocols/ssl/known-certs.bro)
|
||||
rest_target(${psd} policy/protocols/ssl/validate-certs.bro)
|
||||
rest_target(${psd} policy/tuning/defaults/packet-fragments.bro)
|
||||
rest_target(${psd} policy/tuning/defaults/warnings.bro)
|
||||
rest_target(${psd} policy/tuning/track-all-assets.bro)
|
||||
rest_target(${psd} site/local-manager.bro)
|
||||
rest_target(${psd} site/local-proxy.bro)
|
||||
rest_target(${psd} site/local-worker.bro)
|
||||
rest_target(${psd} site/local.bro)
|
||||
rest_target(${psd} test-all-policy.bro)
|
||||
58
NEWS
58
NEWS
|
|
@ -51,6 +51,40 @@ New Functionality
|
|||
can take up to two indices for the start and end index of the
|
||||
substring to return (e.g. "mystring[1,3]").
|
||||
|
||||
- Functions now support default parameters, e.g.:
|
||||
|
||||
global foo: function(s: string, t: string &default="abc", u: count &default=0);
|
||||
|
||||
- The new file analysis framework moves most of the processing of file
|
||||
content from script-land into the core, where it belongs. Much of
|
||||
this is an internal change, the framework comes with the following
|
||||
user-visibible functionality (some of that was already available
|
||||
before, but done differently):
|
||||
|
||||
[TODO: This will probably change with further script updates.]
|
||||
|
||||
- A binary input reader interfaces the input framework with file
|
||||
analysis, allowing to inject files on disk into Bro's
|
||||
processing.
|
||||
|
||||
- Supports for analyzing data transfereed via HTTP range
|
||||
requests.
|
||||
|
||||
- HTTP:
|
||||
* Identify MIME type of message.
|
||||
* Extract message to disk.
|
||||
* Compute MD5 for messages.
|
||||
|
||||
- SMTP:
|
||||
* Identify MIME type of message.
|
||||
* Extract message to disk.
|
||||
* Compute MD5 for messages.
|
||||
* Provide access to start of entity data.
|
||||
|
||||
- FTP data transfers: Identify MIME type; record to disk.
|
||||
|
||||
- IRC DCC transfers: Record to disk.
|
||||
|
||||
Changed Functionality
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
|
@ -126,6 +160,9 @@ Changed Functionality
|
|||
- Removed the byte_len() and length() bif functions. Use the "|...|"
|
||||
operator instead.
|
||||
|
||||
- The SSH::Login notice has been superseded by an corresponding
|
||||
intelligence framework observation (SSH::SUCCESSFUL_LOGIN).
|
||||
|
||||
Bro 2.1
|
||||
-------
|
||||
|
||||
|
|
@ -209,6 +246,27 @@ New Functionality
|
|||
outputs. We do not yet recommend them for production (but welcome
|
||||
feedback!)
|
||||
|
||||
- Summary statistics framework. [Extend]
|
||||
|
||||
- A number of new applications build on top of the summary statistics
|
||||
framework:
|
||||
|
||||
* Scan detection: Detectors for port and address scans return. See
|
||||
policy/misc/scan.bro.
|
||||
|
||||
* Tracerouter detector: policy/misc/detect-traceroute
|
||||
|
||||
* Web application detection/measurement: policy/misc/app-metrics.bro
|
||||
|
||||
* FTP brute-forcing detector: policy/protocols/ftp/detect-bruteforcing.bro
|
||||
|
||||
* HTTP-based SQL injection detector: policy/protocols/http/detect-sqli.bro
|
||||
(existed before, but now ported to the new framework)
|
||||
|
||||
* SSH brute-forcing detector feeding the intelligence framework:
|
||||
policy/protocols/ssh/detect-bruteforcing.bro
|
||||
|
||||
|
||||
|
||||
Changed Functionality
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
|
|
|||
2
VERSION
2
VERSION
|
|
@ -1 +1 @@
|
|||
2.1-377
|
||||
2.1-659
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
Subproject commit 72d121ade5a37df83d3252646de51cb77ce69a89
|
||||
Subproject commit f86a3169b8d49189d264cbc1a7507260cd9ff51d
|
||||
|
|
@ -1 +1 @@
|
|||
Subproject commit ae14da422bfb252c8a53bd00d3e5fd7da8bc112e
|
||||
Subproject commit cfaf4eea788bdac4ebfe9e46e3de2cd74b0bc068
|
||||
|
|
@ -1 +1 @@
|
|||
Subproject commit e64204fec55759c614a276c1933bbff2069a63db
|
||||
Subproject commit 8955807b0f4151f5f6aca2e68d353b9b341d9f86
|
||||
|
|
@ -1 +1 @@
|
|||
Subproject commit 3e3ada3c2efebeda1278b8897859dd7c7d61e671
|
||||
Subproject commit 4d0b75afadd6a3c6507e8ca18cb1913faa93a3b0
|
||||
2
cmake
2
cmake
|
|
@ -1 +1 @@
|
|||
Subproject commit 94e72a3075bb0b9550ad05758963afda394bfb2c
|
||||
Subproject commit e1a7fd00a0a66d6831a239fe84f5fcfaa54e2c35
|
||||
|
|
@ -89,8 +89,7 @@ Note the fields that are set for the filter:
|
|||
are generated by taking the stream's ID and munging it slightly.
|
||||
:bro:enum:`Conn::LOG` is converted into ``conn``,
|
||||
:bro:enum:`PacketFilter::LOG` is converted into
|
||||
``packet_filter``, and :bro:enum:`Notice::POLICY_LOG` is
|
||||
converted into ``notice_policy``.
|
||||
``packet_filter``.
|
||||
|
||||
``include``
|
||||
A set limiting the fields to the ones given. The names
|
||||
|
|
|
|||
|
|
@ -86,21 +86,21 @@ directly make modifications to the :bro:see:`Notice::Info` record
|
|||
given as the argument to the hook.
|
||||
|
||||
Here's a simple example which tells Bro to send an email for all notices of
|
||||
type :bro:see:`SSH::Login` if the server is 10.0.0.1:
|
||||
type :bro:see:`SSH::Password_Guessing` if the server is 10.0.0.1:
|
||||
|
||||
.. code:: bro
|
||||
|
||||
hook Notice::policy(n: Notice::Info)
|
||||
{
|
||||
if ( n$note == SSH::Login && n$id$resp_h == 10.0.0.1 )
|
||||
if ( n$note == SSH::Password_Guessing && n$id$resp_h == 10.0.0.1 )
|
||||
add n$actions[Notice::ACTION_EMAIL];
|
||||
}
|
||||
|
||||
.. note::
|
||||
|
||||
Keep in mind that the semantics of the SSH::Login notice are
|
||||
such that it is only raised when Bro heuristically detects a successful
|
||||
login. No apparently failed logins will raise this notice.
|
||||
Keep in mind that the semantics of the SSH::Password_Guessing notice are
|
||||
such that it is only raised when Bro heuristically detects a failed
|
||||
login.
|
||||
|
||||
Hooks can also have priorities applied to order their execution like events
|
||||
with a default priority of 0. Greater values are executed first. Setting
|
||||
|
|
@ -110,7 +110,7 @@ a hook body to run before default hook bodies might look like this:
|
|||
|
||||
hook Notice::policy(n: Notice::Info) &priority=5
|
||||
{
|
||||
if ( n$note == SSH::Login && n$id$resp_h == 10.0.0.1 )
|
||||
if ( n$note == SSH::Password_Guessing && n$id$resp_h == 10.0.0.1 )
|
||||
add n$actions[Notice::ACTION_EMAIL];
|
||||
}
|
||||
|
||||
|
|
@ -173,16 +173,16 @@ Raising Notices
|
|||
|
||||
A script should raise a notice for any occurrence that a user may want
|
||||
to be notified about or take action on. For example, whenever the base
|
||||
SSH analysis scripts sees an SSH session where it is heuristically
|
||||
guessed to be a successful login, it raises a Notice of the type
|
||||
:bro:see:`SSH::Login`. The code in the base SSH analysis script looks
|
||||
like this:
|
||||
SSH analysis scripts sees enough failed logins to a given host, it
|
||||
raises a notice of the type :bro:see:`SSH::Password_Guessing`. The code
|
||||
in the base SSH analysis script which raises the notice looks like this:
|
||||
|
||||
.. code:: bro
|
||||
|
||||
NOTICE([$note=SSH::Login,
|
||||
$msg="Heuristically detected successful SSH login.",
|
||||
$conn=c]);
|
||||
NOTICE([$note=Password_Guessing,
|
||||
$msg=fmt("%s appears to be guessing SSH passwords (seen in %d connections).", key$host, r$num),
|
||||
$src=key$host,
|
||||
$identifier=cat(key$host)]);
|
||||
|
||||
:bro:see:`NOTICE` is a normal function in the global namespace which
|
||||
wraps a function within the ``Notice`` namespace. It takes a single
|
||||
|
|
|
|||
|
|
@ -107,7 +107,7 @@ macro(REST_TARGET srcDir broInput)
|
|||
COMMAND "${CMAKE_COMMAND}"
|
||||
ARGS -E remove_directory .state
|
||||
# generate the reST documentation using bro
|
||||
COMMAND BROPATH=${BROPATH}:${srcDir} ${CMAKE_BINARY_DIR}/src/bro
|
||||
COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic ${CMAKE_BINARY_DIR}/src/bro
|
||||
ARGS -b -Z ${broInput} || (rm -rf .state *.log *.rst && exit 1)
|
||||
# move generated doc into a new directory tree that
|
||||
# defines the final structure of documents
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ rest_target(${psd} base/init-bare.bro internal)
|
|||
rest_target(${CMAKE_BINARY_DIR}/src base/bro.bif.bro)
|
||||
rest_target(${CMAKE_BINARY_DIR}/src base/const.bif.bro)
|
||||
rest_target(${CMAKE_BINARY_DIR}/src base/event.bif.bro)
|
||||
rest_target(${CMAKE_BINARY_DIR}/src base/file_analysis.bif.bro)
|
||||
rest_target(${CMAKE_BINARY_DIR}/src base/input.bif.bro)
|
||||
rest_target(${CMAKE_BINARY_DIR}/src base/logging.bif.bro)
|
||||
rest_target(${CMAKE_BINARY_DIR}/src base/reporter.bif.bro)
|
||||
|
|
@ -32,10 +33,13 @@ rest_target(${psd} base/frameworks/cluster/setup-connections.bro)
|
|||
rest_target(${psd} base/frameworks/communication/main.bro)
|
||||
rest_target(${psd} base/frameworks/control/main.bro)
|
||||
rest_target(${psd} base/frameworks/dpd/main.bro)
|
||||
rest_target(${psd} base/frameworks/file-analysis/main.bro)
|
||||
rest_target(${psd} base/frameworks/input/main.bro)
|
||||
rest_target(${psd} base/frameworks/input/readers/ascii.bro)
|
||||
rest_target(${psd} base/frameworks/input/readers/benchmark.bro)
|
||||
rest_target(${psd} base/frameworks/input/readers/binary.bro)
|
||||
rest_target(${psd} base/frameworks/input/readers/raw.bro)
|
||||
rest_target(${psd} base/frameworks/input/readers/sqlite.bro)
|
||||
rest_target(${psd} base/frameworks/intel/cluster.bro)
|
||||
rest_target(${psd} base/frameworks/intel/input.bro)
|
||||
rest_target(${psd} base/frameworks/intel/main.bro)
|
||||
|
|
@ -46,9 +50,7 @@ rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
|
|||
rest_target(${psd} base/frameworks/logging/writers/dataseries.bro)
|
||||
rest_target(${psd} base/frameworks/logging/writers/elasticsearch.bro)
|
||||
rest_target(${psd} base/frameworks/logging/writers/none.bro)
|
||||
rest_target(${psd} base/frameworks/metrics/cluster.bro)
|
||||
rest_target(${psd} base/frameworks/metrics/main.bro)
|
||||
rest_target(${psd} base/frameworks/metrics/non-cluster.bro)
|
||||
rest_target(${psd} base/frameworks/logging/writers/sqlite.bro)
|
||||
rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
|
||||
rest_target(${psd} base/frameworks/notice/actions/drop.bro)
|
||||
rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
|
||||
|
|
@ -64,6 +66,18 @@ rest_target(${psd} base/frameworks/packet-filter/netstats.bro)
|
|||
rest_target(${psd} base/frameworks/reporter/main.bro)
|
||||
rest_target(${psd} base/frameworks/signatures/main.bro)
|
||||
rest_target(${psd} base/frameworks/software/main.bro)
|
||||
rest_target(${psd} base/frameworks/sumstats/cluster.bro)
|
||||
rest_target(${psd} base/frameworks/sumstats/main.bro)
|
||||
rest_target(${psd} base/frameworks/sumstats/non-cluster.bro)
|
||||
rest_target(${psd} base/frameworks/sumstats/plugins/average.bro)
|
||||
rest_target(${psd} base/frameworks/sumstats/plugins/last.bro)
|
||||
rest_target(${psd} base/frameworks/sumstats/plugins/max.bro)
|
||||
rest_target(${psd} base/frameworks/sumstats/plugins/min.bro)
|
||||
rest_target(${psd} base/frameworks/sumstats/plugins/sample.bro)
|
||||
rest_target(${psd} base/frameworks/sumstats/plugins/std-dev.bro)
|
||||
rest_target(${psd} base/frameworks/sumstats/plugins/sum.bro)
|
||||
rest_target(${psd} base/frameworks/sumstats/plugins/unique.bro)
|
||||
rest_target(${psd} base/frameworks/sumstats/plugins/variance.bro)
|
||||
rest_target(${psd} base/frameworks/tunnels/main.bro)
|
||||
rest_target(${psd} base/misc/find-checksum-offloading.bro)
|
||||
rest_target(${psd} base/protocols/conn/contents.bro)
|
||||
|
|
@ -72,21 +86,25 @@ rest_target(${psd} base/protocols/conn/main.bro)
|
|||
rest_target(${psd} base/protocols/conn/polling.bro)
|
||||
rest_target(${psd} base/protocols/dns/consts.bro)
|
||||
rest_target(${psd} base/protocols/dns/main.bro)
|
||||
rest_target(${psd} base/protocols/ftp/file-analysis.bro)
|
||||
rest_target(${psd} base/protocols/ftp/file-extract.bro)
|
||||
rest_target(${psd} base/protocols/ftp/gridftp.bro)
|
||||
rest_target(${psd} base/protocols/ftp/main.bro)
|
||||
rest_target(${psd} base/protocols/ftp/utils-commands.bro)
|
||||
rest_target(${psd} base/protocols/http/file-analysis.bro)
|
||||
rest_target(${psd} base/protocols/http/file-extract.bro)
|
||||
rest_target(${psd} base/protocols/http/file-hash.bro)
|
||||
rest_target(${psd} base/protocols/http/file-ident.bro)
|
||||
rest_target(${psd} base/protocols/http/main.bro)
|
||||
rest_target(${psd} base/protocols/http/utils.bro)
|
||||
rest_target(${psd} base/protocols/irc/dcc-send.bro)
|
||||
rest_target(${psd} base/protocols/irc/file-analysis.bro)
|
||||
rest_target(${psd} base/protocols/irc/main.bro)
|
||||
rest_target(${psd} base/protocols/modbus/consts.bro)
|
||||
rest_target(${psd} base/protocols/modbus/main.bro)
|
||||
rest_target(${psd} base/protocols/smtp/entities-excerpt.bro)
|
||||
rest_target(${psd} base/protocols/smtp/entities.bro)
|
||||
rest_target(${psd} base/protocols/smtp/file-analysis.bro)
|
||||
rest_target(${psd} base/protocols/smtp/main.bro)
|
||||
rest_target(${psd} base/protocols/socks/consts.bro)
|
||||
rest_target(${psd} base/protocols/socks/main.bro)
|
||||
|
|
@ -103,9 +121,11 @@ rest_target(${psd} base/utils/files.bro)
|
|||
rest_target(${psd} base/utils/numbers.bro)
|
||||
rest_target(${psd} base/utils/paths.bro)
|
||||
rest_target(${psd} base/utils/patterns.bro)
|
||||
rest_target(${psd} base/utils/queue.bro)
|
||||
rest_target(${psd} base/utils/site.bro)
|
||||
rest_target(${psd} base/utils/strings.bro)
|
||||
rest_target(${psd} base/utils/thresholds.bro)
|
||||
rest_target(${psd} base/utils/time.bro)
|
||||
rest_target(${psd} base/utils/urls.bro)
|
||||
rest_target(${psd} policy/frameworks/communication/listen.bro)
|
||||
rest_target(${psd} policy/frameworks/control/controllee.bro)
|
||||
|
|
@ -121,18 +141,18 @@ rest_target(${psd} policy/frameworks/intel/smtp-url-extraction.bro)
|
|||
rest_target(${psd} policy/frameworks/intel/smtp.bro)
|
||||
rest_target(${psd} policy/frameworks/intel/ssl.bro)
|
||||
rest_target(${psd} policy/frameworks/intel/where-locations.bro)
|
||||
rest_target(${psd} policy/frameworks/metrics/conn-example.bro)
|
||||
rest_target(${psd} policy/frameworks/metrics/http-example.bro)
|
||||
rest_target(${psd} policy/frameworks/metrics/ssl-example.bro)
|
||||
rest_target(${psd} policy/frameworks/software/version-changes.bro)
|
||||
rest_target(${psd} policy/frameworks/software/vulnerable.bro)
|
||||
rest_target(${psd} policy/integration/barnyard2/main.bro)
|
||||
rest_target(${psd} policy/integration/barnyard2/types.bro)
|
||||
rest_target(${psd} policy/integration/collective-intel/main.bro)
|
||||
rest_target(${psd} policy/misc/analysis-groups.bro)
|
||||
rest_target(${psd} policy/misc/app-metrics.bro)
|
||||
rest_target(${psd} policy/misc/capture-loss.bro)
|
||||
rest_target(${psd} policy/misc/detect-traceroute/main.bro)
|
||||
rest_target(${psd} policy/misc/loaded-scripts.bro)
|
||||
rest_target(${psd} policy/misc/profiling.bro)
|
||||
rest_target(${psd} policy/misc/scan.bro)
|
||||
rest_target(${psd} policy/misc/stats.bro)
|
||||
rest_target(${psd} policy/misc/trim-trace-file.bro)
|
||||
rest_target(${psd} policy/protocols/conn/known-hosts.bro)
|
||||
|
|
@ -140,6 +160,7 @@ rest_target(${psd} policy/protocols/conn/known-services.bro)
|
|||
rest_target(${psd} policy/protocols/conn/weirds.bro)
|
||||
rest_target(${psd} policy/protocols/dns/auth-addl.bro)
|
||||
rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
|
||||
rest_target(${psd} policy/protocols/ftp/detect-bruteforcing.bro)
|
||||
rest_target(${psd} policy/protocols/ftp/detect.bro)
|
||||
rest_target(${psd} policy/protocols/ftp/software.bro)
|
||||
rest_target(${psd} policy/protocols/http/detect-MHR.bro)
|
||||
|
|
|
|||
|
|
@ -402,6 +402,31 @@ The Bro scripting language supports the following built-in types.
|
|||
if ( r?$s )
|
||||
...
|
||||
|
||||
.. bro:type:: opaque
|
||||
|
||||
A data type whose actual representation/implementation is
|
||||
intentionally hidden, but whose values may be passed to certain
|
||||
functions that can actually access the internal/hidden resources.
|
||||
Opaque types are differentiated from each other by qualifying them
|
||||
like ``opaque of md5`` or ``opaque of sha1``. Any valid identifier
|
||||
can be used as the type qualifier.
|
||||
|
||||
An example use of this type is the set of built-in functions which
|
||||
perform hashing:
|
||||
|
||||
.. code:: bro
|
||||
|
||||
local handle: opaque of md5 = md5_hash_init();
|
||||
md5_hash_update(handle, "test");
|
||||
md5_hash_update(handle, "testing");
|
||||
print md5_hash_finish(handle);
|
||||
|
||||
Here the opaque type is used to provide a handle to a particular
|
||||
resource which is calculating an MD5 checksum incrementally over
|
||||
time, but the details of that resource aren't relevant, it's only
|
||||
necessary to have a handle as a way of identifying it and
|
||||
distinguishing it from other such resources.
|
||||
|
||||
.. bro:type:: file
|
||||
|
||||
Bro supports writing to files, but not reading from them. For
|
||||
|
|
@ -459,6 +484,31 @@ The Bro scripting language supports the following built-in types.
|
|||
|
||||
print greeting("Dave");
|
||||
|
||||
Function parameters may specify default values as long as they appear
|
||||
last in the parameter list:
|
||||
|
||||
.. code:: bro
|
||||
|
||||
global foo: function(s: string, t: string &default="abc", u: count &default=0);
|
||||
|
||||
If a function was previously declared with default parameters, the
|
||||
default expressions can be omitted when implementing the function
|
||||
body and they will still be used for function calls that lack those
|
||||
arguments.
|
||||
|
||||
.. code:: bro
|
||||
|
||||
function foo(s: string, t: string, u: count)
|
||||
{
|
||||
print s, t, u;
|
||||
}
|
||||
|
||||
And calls to the function may omit the defaults from the argument list:
|
||||
|
||||
.. code:: bro
|
||||
|
||||
foo("test");
|
||||
|
||||
.. bro:type:: event
|
||||
|
||||
Event handlers are nearly identical in both syntax and semantics to
|
||||
|
|
@ -597,10 +647,10 @@ scripting language supports the following built-in attributes.
|
|||
|
||||
.. bro:attr:: &default
|
||||
|
||||
Uses a default value for a record field or container elements. For
|
||||
example, ``table[int] of string &default="foo" }`` would create a
|
||||
table that returns the :bro:type:`string` ``"foo"`` for any
|
||||
non-existing index.
|
||||
Uses a default value for a record field, a function/hook/event
|
||||
parameter, or container elements. For example, ``table[int] of
|
||||
string &default="foo" }`` would create a table that returns the
|
||||
:bro:type:`string` ``"foo"`` for any non-existing index.
|
||||
|
||||
.. bro:attr:: &redef
|
||||
|
||||
|
|
|
|||
29
magic/COPYING
Normal file
29
magic/COPYING
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
# $File: LEGAL.NOTICE,v 1.15 2006/05/03 18:48:33 christos Exp $
|
||||
# Copyright (c) Ian F. Darwin 1986, 1987, 1989, 1990, 1991, 1992, 1994, 1995.
|
||||
# Software written by Ian F. Darwin and others;
|
||||
# maintained 1994- Christos Zoulas.
|
||||
#
|
||||
# This software is not subject to any export provision of the United States
|
||||
# Department of Commerce, and may be exported to any country or planet.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions
|
||||
# are met:
|
||||
# 1. Redistributions of source code must retain the above copyright
|
||||
# notice immediately at the beginning of the file, without modification,
|
||||
# this list of conditions, and the following disclaimer.
|
||||
# 2. Redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in the
|
||||
# documentation and/or other materials provided with the distribution.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
|
||||
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
# SUCH DAMAGE.
|
||||
208
magic/animation
Normal file
208
magic/animation
Normal file
|
|
@ -0,0 +1,208 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: animation,v 1.47 2013/02/06 14:18:52 christos Exp $
|
||||
# animation: file(1) magic for animation/movie formats
|
||||
#
|
||||
# animation formats
|
||||
# MPEG, FLI, DL originally from vax@ccwf.cc.utexas.edu (VaX#n8)
|
||||
# FLC, SGI, Apple originally from Daniel Quinlan (quinlan@yggdrasil.com)
|
||||
|
||||
# SGI and Apple formats
|
||||
0 string MOVI Silicon Graphics movie file
|
||||
!:mime video/x-sgi-movie
|
||||
4 string moov Apple QuickTime
|
||||
!:mime video/quicktime
|
||||
4 string mdat Apple QuickTime movie (unoptimized)
|
||||
!:mime video/quicktime
|
||||
#4 string wide Apple QuickTime movie (unoptimized)
|
||||
#!:mime video/quicktime
|
||||
#4 string skip Apple QuickTime movie (modified)
|
||||
#!:mime video/quicktime
|
||||
#4 string free Apple QuickTime movie (modified)
|
||||
#!:mime video/quicktime
|
||||
4 string idsc Apple QuickTime image (fast start)
|
||||
!:mime image/x-quicktime
|
||||
#4 string idat Apple QuickTime image (unoptimized)
|
||||
#!:mime image/x-quicktime
|
||||
4 string pckg Apple QuickTime compressed archive
|
||||
!:mime application/x-quicktime-player
|
||||
4 string/W jP JPEG 2000 image
|
||||
!:mime image/jp2
|
||||
4 string ftyp ISO Media
|
||||
>8 string isom \b, MPEG v4 system, version 1
|
||||
!:mime video/mp4
|
||||
>8 string mp41 \b, MPEG v4 system, version 1
|
||||
!:mime video/mp4
|
||||
>8 string mp42 \b, MPEG v4 system, version 2
|
||||
!:mime video/mp4
|
||||
>8 string/W jp2 \b, JPEG 2000
|
||||
!:mime image/jp2
|
||||
>8 string 3ge \b, MPEG v4 system, 3GPP
|
||||
!:mime video/3gpp
|
||||
>8 string 3gg \b, MPEG v4 system, 3GPP
|
||||
!:mime video/3gpp
|
||||
>8 string 3gp \b, MPEG v4 system, 3GPP
|
||||
!:mime video/3gpp
|
||||
>8 string 3gs \b, MPEG v4 system, 3GPP
|
||||
!:mime video/3gpp
|
||||
>8 string 3g2 \b, MPEG v4 system, 3GPP2
|
||||
!:mime video/3gpp2
|
||||
>8 string mmp4 \b, MPEG v4 system, 3GPP Mobile
|
||||
!:mime video/mp4
|
||||
>8 string avc1 \b, MPEG v4 system, 3GPP JVT AVC
|
||||
!:mime video/3gpp
|
||||
>8 string/W M4A \b, MPEG v4 system, iTunes AAC-LC
|
||||
!:mime audio/mp4
|
||||
>8 string/W M4V \b, MPEG v4 system, iTunes AVC-LC
|
||||
!:mime video/mp4
|
||||
>8 string/W qt \b, Apple QuickTime movie
|
||||
!:mime video/quicktime
|
||||
|
||||
# MPEG sequences
|
||||
# Scans for all common MPEG header start codes
|
||||
0 belong&0xFFFFFF00 0x00000100
|
||||
>3 byte 0xBA MPEG sequence
|
||||
!:mime video/mpeg
|
||||
# GRR too general as it catches also FoxPro Memo example NG.FPT
|
||||
>3 byte 0xB0 MPEG sequence, v4
|
||||
!:mime video/mpeg4-generic
|
||||
>3 byte 0xB5 MPEG sequence, v4
|
||||
!:mime video/mpeg4-generic
|
||||
>3 byte 0xB3 MPEG sequence
|
||||
!:mime video/mpeg
|
||||
|
||||
# MPEG ADTS Audio (*.mpx/mxa/aac)
|
||||
# from dreesen@math.fu-berlin.de
|
||||
# modified to fully support MPEG ADTS
|
||||
|
||||
# MP3, M1A
|
||||
# modified by Joerg Jenderek
|
||||
# GRR the original test are too common for many DOS files
|
||||
# so don't accept as MP3 until we've tested the rate
|
||||
0 beshort&0xFFFE 0xFFFA
|
||||
# rates
|
||||
>2 byte&0xF0 0x10 MPEG ADTS, layer III, v1, 32 kbps
|
||||
!:mime audio/mpeg
|
||||
>2 byte&0xF0 0x20 MPEG ADTS, layer III, v1, 40 kbps
|
||||
!:mime audio/mpeg
|
||||
>2 byte&0xF0 0x30 MPEG ADTS, layer III, v1, 48 kbps
|
||||
!:mime audio/mpeg
|
||||
>2 byte&0xF0 0x40 MPEG ADTS, layer III, v1, 56 kbps
|
||||
!:mime audio/mpeg
|
||||
>2 byte&0xF0 0x50 MPEG ADTS, layer III, v1, 64 kbps
|
||||
!:mime audio/mpeg
|
||||
>2 byte&0xF0 0x60 MPEG ADTS, layer III, v1, 80 kbps
|
||||
!:mime audio/mpeg
|
||||
>2 byte&0xF0 0x70 MPEG ADTS, layer III, v1, 96 kbps
|
||||
!:mime audio/mpeg
|
||||
>2 byte&0xF0 0x80 MPEG ADTS, layer III, v1, 112 kbps
|
||||
!:mime audio/mpeg
|
||||
>2 byte&0xF0 0x90 MPEG ADTS, layer III, v1, 128 kbps
|
||||
!:mime audio/mpeg
|
||||
>2 byte&0xF0 0xA0 MPEG ADTS, layer III, v1, 160 kbps
|
||||
!:mime audio/mpeg
|
||||
>2 byte&0xF0 0xB0 MPEG ADTS, layer III, v1, 192 kbps
|
||||
!:mime audio/mpeg
|
||||
>2 byte&0xF0 0xC0 MPEG ADTS, layer III, v1, 224 kbps
|
||||
!:mime audio/mpeg
|
||||
>2 byte&0xF0 0xD0 MPEG ADTS, layer III, v1, 256 kbps
|
||||
!:mime audio/mpeg
|
||||
>2 byte&0xF0 0xE0 MPEG ADTS, layer III, v1, 320 kbps
|
||||
!:mime audio/mpeg
|
||||
|
||||
# MP2, M1A
|
||||
0 beshort&0xFFFE 0xFFFC MPEG ADTS, layer II, v1
|
||||
!:mime audio/mpeg
|
||||
|
||||
# MP3, M2A
|
||||
0 beshort&0xFFFE 0xFFF2 MPEG ADTS, layer III, v2
|
||||
!:mime audio/mpeg
|
||||
|
||||
# MPA, M2A
|
||||
0 beshort&0xFFFE 0xFFF6 MPEG ADTS, layer I, v2
|
||||
!:mime audio/mpeg
|
||||
|
||||
# MP3, M25A
|
||||
0 beshort&0xFFFE 0xFFE2 MPEG ADTS, layer III, v2.5
|
||||
!:mime audio/mpeg
|
||||
|
||||
# Stored AAC streams (instead of the MP4 format)
|
||||
0 string ADIF MPEG ADIF, AAC
|
||||
!:mime audio/x-hx-aac-adif
|
||||
|
||||
# Live or stored single AAC stream (used with MPEG-2 systems)
|
||||
0 beshort&0xFFF6 0xFFF0 MPEG ADTS, AAC
|
||||
!:mime audio/x-hx-aac-adts
|
||||
|
||||
# Live MPEG-4 audio streams (instead of RTP FlexMux)
|
||||
0 beshort&0xFFE0 0x56E0 MPEG-4 LOAS
|
||||
!:mime audio/x-mp4a-latm
|
||||
|
||||
# This magic isn't strong enough (matches plausible ISO-8859-1 text)
|
||||
#0 beshort 0x4DE1 MPEG-4 LO-EP audio stream
|
||||
#!:mime audio/x-mp4a-latm
|
||||
|
||||
# Summary: FLI animation format
|
||||
# Created by: Daniel Quinlan <quinlan@yggdrasil.com>
|
||||
# Modified by (1): Abel Cheung <abelcheung@gmail.com> (avoid over-generic detection)
|
||||
4 leshort 0xAF11
|
||||
# standard FLI always has 320x200 resolution and 8 bit color
|
||||
>8 leshort 320
|
||||
>>10 leshort 200
|
||||
>>>12 leshort 8 FLI animation, 320x200x8
|
||||
!:mime video/x-fli
|
||||
|
||||
# Summary: FLC animation format
|
||||
# Created by: Daniel Quinlan <quinlan@yggdrasil.com>
|
||||
# Modified by (1): Abel Cheung <abelcheung@gmail.com> (avoid over-generic detection)
|
||||
4 leshort 0xAF12
|
||||
# standard FLC always use 8 bit color
|
||||
>12 leshort 8 FLC animation
|
||||
!:mime video/x-flc
|
||||
|
||||
# Microsoft Advanced Streaming Format (ASF) <mpruett@sgi.com>
|
||||
0 belong 0x3026b275 Microsoft ASF
|
||||
!:mime video/x-ms-asf
|
||||
|
||||
# MNG Video Format, <URL:http://www.libpng.org/pub/mng/spec/>
|
||||
0 string \x8aMNG MNG video data,
|
||||
!:mime video/x-mng
|
||||
|
||||
# JNG Video Format, <URL:http://www.libpng.org/pub/mng/spec/>
|
||||
0 string \x8bJNG JNG video data,
|
||||
!:mime video/x-jng
|
||||
|
||||
# VRML (Virtual Reality Modelling Language)
|
||||
0 string/w #VRML\ V1.0\ ascii VRML 1 file
|
||||
!:mime model/vrml
|
||||
0 string/w #VRML\ V2.0\ utf8 ISO/IEC 14772 VRML 97 file
|
||||
!:mime model/vrml
|
||||
|
||||
# X3D (Extensible 3D) [http://www.web3d.org/specifications/x3d-3.0.dtd]
|
||||
# From Michel Briand <michelbriand@free.fr>
|
||||
0 string/t \<?xml\ version="
|
||||
!:strength +1
|
||||
>20 search/1000/cw \<!DOCTYPE\ X3D X3D (Extensible 3D) model xml text
|
||||
!:mime model/x3d
|
||||
|
||||
# MPEG file
|
||||
# MPEG sequences
|
||||
# FIXME: This section is from the old magic.mime file and needs integrating with the rest
|
||||
0 belong 0x000001BA
|
||||
>4 byte &0x40
|
||||
!:mime video/mp2p
|
||||
>4 byte ^0x40
|
||||
!:mime video/mpeg
|
||||
0 belong 0x000001BB
|
||||
!:mime video/mpeg
|
||||
0 belong 0x000001B0
|
||||
!:mime video/mp4v-es
|
||||
0 belong 0x000001B5
|
||||
!:mime video/mp4v-es
|
||||
0 belong 0x000001B3
|
||||
!:mime video/mpv
|
||||
0 belong&0xFF5FFF1F 0x47400010
|
||||
!:mime video/mp2t
|
||||
0 belong 0x00000001
|
||||
>4 byte&0x1F 0x07
|
||||
!:mime video/h264
|
||||
242
magic/archive
Normal file
242
magic/archive
Normal file
|
|
@ -0,0 +1,242 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: archive,v 1.78 2013/02/06 14:18:52 christos Exp $
|
||||
# archive: file(1) magic for archive formats (see also "msdos" for self-
|
||||
# extracting compressed archives)
|
||||
#
|
||||
# cpio, ar, arc, arj, hpack, lha/lharc, rar, squish, uc2, zip, zoo, etc.
|
||||
# pre-POSIX "tar" archives are handled in the C code.
|
||||
|
||||
# POSIX tar archives
|
||||
257 string ustar\0 POSIX tar archive
|
||||
!:mime application/x-tar # encoding: posix
|
||||
257 string ustar\040\040\0 GNU tar archive
|
||||
!:mime application/x-tar # encoding: gnu
|
||||
|
||||
# cpio archives
|
||||
#
|
||||
# Yes, the top two "cpio archive" formats *are* supposed to just be "short".
|
||||
# The idea is to indicate archives produced on machines with the same
|
||||
# byte order as the machine running "file" with "cpio archive", and
|
||||
# to indicate archives produced on machines with the opposite byte order
|
||||
# from the machine running "file" with "byte-swapped cpio archive".
|
||||
#
|
||||
# The SVR4 "cpio(4)" hints that there are additional formats, but they
|
||||
# are defined as "short"s; I think all the new formats are
|
||||
# character-header formats and thus are strings, not numbers.
|
||||
0 short 070707 cpio archive
|
||||
!:mime application/x-cpio
|
||||
0 short 0143561 byte-swapped cpio archive
|
||||
!:mime application/x-cpio # encoding: swapped
|
||||
|
||||
#
|
||||
# System V Release 1 portable(?) archive format.
|
||||
#
|
||||
0 string =<ar> System V Release 1 ar archive
|
||||
!:mime application/x-archive
|
||||
|
||||
#
|
||||
# Debian package; it's in the portable archive format, and needs to go
|
||||
# before the entry for regular portable archives, as it's recognized as
|
||||
# a portable archive whose first member has a name beginning with
|
||||
# "debian".
|
||||
#
|
||||
0 string =!<arch>\ndebian
|
||||
!:mime application/x-debian-package
|
||||
|
||||
#
|
||||
# MIPS archive; they're in the portable archive format, and need to go
|
||||
# before the entry for regular portable archives, as it's recognized as
|
||||
# a portable archive whose first member has a name beginning with
|
||||
# "__________E".
|
||||
#
|
||||
0 string =!<arch>\n__________E MIPS archive
|
||||
!:mime application/x-archive
|
||||
|
||||
#
|
||||
# BSD/SVR2-and-later portable archive formats.
|
||||
#
|
||||
0 string =!<arch> current ar archive
|
||||
!:mime application/x-archive
|
||||
|
||||
# ARC archiver, from Daniel Quinlan (quinlan@yggdrasil.com)
|
||||
#
|
||||
# The first byte is the magic (0x1a), byte 2 is the compression type for
|
||||
# the first file (0x01 through 0x09), and bytes 3 to 15 are the MS-DOS
|
||||
# filename of the first file (null terminated). Since some types collide
|
||||
# we only test some types on basis of frequency: 0x08 (83%), 0x09 (5%),
|
||||
# 0x02 (5%), 0x03 (3%), 0x04 (2%), 0x06 (2%). 0x01 collides with terminfo.
|
||||
0 lelong&0x8080ffff 0x0000081a ARC archive data, dynamic LZW
|
||||
!:mime application/x-arc
|
||||
0 lelong&0x8080ffff 0x0000091a ARC archive data, squashed
|
||||
!:mime application/x-arc
|
||||
0 lelong&0x8080ffff 0x0000021a ARC archive data, uncompressed
|
||||
!:mime application/x-arc
|
||||
0 lelong&0x8080ffff 0x0000031a ARC archive data, packed
|
||||
!:mime application/x-arc
|
||||
0 lelong&0x8080ffff 0x0000041a ARC archive data, squeezed
|
||||
!:mime application/x-arc
|
||||
0 lelong&0x8080ffff 0x0000061a ARC archive data, crunched
|
||||
!:mime application/x-arc
|
||||
# [JW] stuff taken from idarc, obviously ARC successors:
|
||||
0 lelong&0x8080ffff 0x00000a1a PAK archive data
|
||||
!:mime application/x-arc
|
||||
0 lelong&0x8080ffff 0x0000141a ARC+ archive data
|
||||
!:mime application/x-arc
|
||||
0 lelong&0x8080ffff 0x0000481a HYP archive data
|
||||
!:mime application/x-arc
|
||||
|
||||
# ARJ archiver (jason@jarthur.Claremont.EDU)
|
||||
0 leshort 0xea60 ARJ archive data
|
||||
!:mime application/x-arj
|
||||
|
||||
# LHARC/LHA archiver (Greg Roelofs, newt@uchicago.edu)
|
||||
2 string -lh0- LHarc 1.x/ARX archive data [lh0]
|
||||
!:mime application/x-lharc
|
||||
2 string -lh1- LHarc 1.x/ARX archive data [lh1]
|
||||
!:mime application/x-lharc
|
||||
2 string -lz4- LHarc 1.x archive data [lz4]
|
||||
!:mime application/x-lharc
|
||||
2 string -lz5- LHarc 1.x archive data [lz5]
|
||||
!:mime application/x-lharc
|
||||
# [never seen any but the last; -lh4- reported in comp.compression:]
|
||||
2 string -lzs- LHa/LZS archive data [lzs]
|
||||
!:mime application/x-lha
|
||||
2 string -lh\40- LHa 2.x? archive data [lh ]
|
||||
!:mime application/x-lha
|
||||
2 string -lhd- LHa 2.x? archive data [lhd]
|
||||
!:mime application/x-lha
|
||||
2 string -lh2- LHa 2.x? archive data [lh2]
|
||||
!:mime application/x-lha
|
||||
2 string -lh3- LHa 2.x? archive data [lh3]
|
||||
!:mime application/x-lha
|
||||
2 string -lh4- LHa (2.x) archive data [lh4]
|
||||
!:mime application/x-lha
|
||||
2 string -lh5- LHa (2.x) archive data [lh5]
|
||||
!:mime application/x-lha
|
||||
2 string -lh6- LHa (2.x) archive data [lh6]
|
||||
!:mime application/x-lha
|
||||
2 string -lh7- LHa (2.x)/LHark archive data [lh7]
|
||||
!:mime application/x-lha
|
||||
|
||||
# RAR archiver (Greg Roelofs, newt@uchicago.edu)
|
||||
0 string Rar! RAR archive data,
|
||||
!:mime application/x-rar
|
||||
|
||||
# PKZIP multi-volume archive
|
||||
0 string PK\x07\x08PK\x03\x04 Zip multi-volume archive data, at least PKZIP v2.50 to extract
|
||||
!:mime application/zip
|
||||
|
||||
# Zip archives (Greg Roelofs, c/o zip-bugs@wkuvx1.wku.edu)
|
||||
0 string PK\003\004
|
||||
|
||||
# Specialised zip formats which start with a member named 'mimetype'
|
||||
# (stored uncompressed, with no 'extra field') containing the file's MIME type.
|
||||
# Check for have 8-byte name, 0-byte extra field, name "mimetype", and
|
||||
# contents starting with "application/":
|
||||
>26 string \x8\0\0\0mimetypeapplication/
|
||||
|
||||
# OpenDocument formats (for OpenOffice 2.x / StarOffice >= 8)
|
||||
# http://lists.oasis-open.org/archives/office/200505/msg00006.html
|
||||
# (mimetype contains "application/vnd.oasis.opendocument.<SUBTYPE>")
|
||||
>>50 string vnd.oasis.opendocument. OpenDocument
|
||||
>>>73 string text
|
||||
>>>>77 byte !0x2d Text
|
||||
!:mime application/vnd.oasis.opendocument.text
|
||||
>>>>77 string -template Text Template
|
||||
!:mime application/vnd.oasis.opendocument.text-template
|
||||
>>>>77 string -web HTML Document Template
|
||||
!:mime application/vnd.oasis.opendocument.text-web
|
||||
>>>>77 string -master Master Document
|
||||
!:mime application/vnd.oasis.opendocument.text-master
|
||||
>>>73 string graphics
|
||||
>>>>81 byte !0x2d Drawing
|
||||
!:mime application/vnd.oasis.opendocument.graphics
|
||||
>>>>81 string -template Template
|
||||
!:mime application/vnd.oasis.opendocument.graphics-template
|
||||
>>>73 string presentation
|
||||
>>>>85 byte !0x2d Presentation
|
||||
!:mime application/vnd.oasis.opendocument.presentation
|
||||
>>>>85 string -template Template
|
||||
!:mime application/vnd.oasis.opendocument.presentation-template
|
||||
>>>73 string spreadsheet
|
||||
>>>>84 byte !0x2d Spreadsheet
|
||||
!:mime application/vnd.oasis.opendocument.spreadsheet
|
||||
>>>>84 string -template Template
|
||||
!:mime application/vnd.oasis.opendocument.spreadsheet-template
|
||||
>>>73 string chart
|
||||
>>>>78 byte !0x2d Chart
|
||||
!:mime application/vnd.oasis.opendocument.chart
|
||||
>>>>78 string -template Template
|
||||
!:mime application/vnd.oasis.opendocument.chart-template
|
||||
>>>73 string formula
|
||||
>>>>80 byte !0x2d Formula
|
||||
!:mime application/vnd.oasis.opendocument.formula
|
||||
>>>>80 string -template Template
|
||||
!:mime application/vnd.oasis.opendocument.formula-template
|
||||
>>>73 string database Database
|
||||
!:mime application/vnd.oasis.opendocument.database
|
||||
>>>73 string image
|
||||
>>>>78 byte !0x2d Image
|
||||
!:mime application/vnd.oasis.opendocument.image
|
||||
>>>>78 string -template Template
|
||||
!:mime application/vnd.oasis.opendocument.image-template
|
||||
|
||||
# EPUB (OEBPS) books using OCF (OEBPS Container Format)
|
||||
# http://www.idpf.org/ocf/ocf1.0/download/ocf10.htm, section 4.
|
||||
# From: Ralf Brown <ralf.brown@gmail.com>
|
||||
>0x1E string mimetypeapplication/epub+zip EPUB document
|
||||
!:mime application/epub+zip
|
||||
|
||||
# Catch other ZIP-with-mimetype formats
|
||||
# In a ZIP file, the bytes immediately after a member's contents are
|
||||
# always "PK". The 2 regex rules here print the "mimetype" member's
|
||||
# contents up to the first 'P'. Luckily, most MIME types don't contain
|
||||
# any capital 'P's. This is a kludge.
|
||||
# (mimetype contains "application/<OTHER>")
|
||||
>>50 string !epub+zip
|
||||
>>>50 string !vnd.oasis.opendocument.
|
||||
>>>>50 string !vnd.sun.xml.
|
||||
>>>>>50 string !vnd.kde.
|
||||
>>>>>>38 regex [!-OQ-~]+ Zip data (MIME type "%s"?)
|
||||
!:mime application/zip
|
||||
# (mimetype contents other than "application/*")
|
||||
>26 string \x8\0\0\0mimetype
|
||||
>>38 string !application/
|
||||
>>>38 regex [!-OQ-~]+ Zip data (MIME type "%s"?)
|
||||
!:mime application/zip
|
||||
|
||||
# Java Jar files
|
||||
>(26.s+30) leshort 0xcafe Java Jar file data (zip)
|
||||
!:mime application/jar
|
||||
|
||||
# Generic zip archives (Greg Roelofs, c/o zip-bugs@wkuvx1.wku.edu)
|
||||
# Next line excludes specialized formats:
|
||||
>(26.s+30) leshort !0xcafe
|
||||
>>26 string !\x8\0\0\0mimetype Zip archive data
|
||||
!:mime application/zip
|
||||
|
||||
# Zoo archiver
|
||||
20 lelong 0xfdc4a7dc Zoo archive data
|
||||
!:mime application/x-zoo
|
||||
|
||||
# Shell archives
|
||||
10 string #\ This\ is\ a\ shell\ archive shell archive text
|
||||
!:mime application/octet-stream
|
||||
|
||||
# Felix von Leitner <felix-file@fefe.de>
|
||||
0 string d8:announce BitTorrent file
|
||||
!:mime application/x-bittorrent
|
||||
|
||||
# EET archive
|
||||
# From: Tilman Sauerbeck <tilman@code-monkey.de>
|
||||
0 belong 0x1ee7ff00 EET archive
|
||||
!:mime application/x-eet
|
||||
|
||||
# Symbian installation files
|
||||
# http://www.thouky.co.uk/software/psifs/sis.html
|
||||
# http://developer.symbian.com/main/downloads/papers/SymbianOSv91/softwareinstallsis.pdf
|
||||
8 lelong 0x10000419 Symbian installation file
|
||||
!:mime application/vnd.symbian.install
|
||||
0 lelong 0x10201A7A Symbian installation file (Symbian OS 9.x)
|
||||
!:mime x-epoc/x-sisx-app
|
||||
19
magic/assembler
Normal file
19
magic/assembler
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: assembler,v 1.3 2013/01/04 17:23:28 christos Exp $
|
||||
# make: file(1) magic for assembler source
|
||||
#
|
||||
0 regex \^[\020\t]*\\.asciiz assembler source text
|
||||
!:mime text/x-asm
|
||||
0 regex \^[\020\t]*\\.byte assembler source text
|
||||
!:mime text/x-asm
|
||||
0 regex \^[\020\t]*\\.even assembler source text
|
||||
!:mime text/x-asm
|
||||
0 regex \^[\020\t]*\\.globl assembler source text
|
||||
!:mime text/x-asm
|
||||
0 regex \^[\020\t]*\\.text assembler source text
|
||||
!:mime text/x-asm
|
||||
0 regex \^[\020\t]*\\.file assembler source text
|
||||
!:mime text/x-asm
|
||||
0 regex \^[\020\t]*\\.type assembler source text
|
||||
!:mime text/x-asm
|
||||
149
magic/audio
Normal file
149
magic/audio
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: audio,v 1.65 2012/10/31 13:38:40 christos Exp $
|
||||
# audio: file(1) magic for sound formats (see also "iff")
|
||||
#
|
||||
# Jan Nicolai Langfeldt (janl@ifi.uio.no), Dan Quinlan (quinlan@yggdrasil.com),
|
||||
# and others
|
||||
#
|
||||
|
||||
# Sun/NeXT audio data
|
||||
0 string .snd Sun/NeXT audio data:
|
||||
>12 belong 1 8-bit ISDN mu-law,
|
||||
!:mime audio/basic
|
||||
>12 belong 2 8-bit linear PCM [REF-PCM],
|
||||
!:mime audio/basic
|
||||
>12 belong 3 16-bit linear PCM,
|
||||
!:mime audio/basic
|
||||
>12 belong 4 24-bit linear PCM,
|
||||
!:mime audio/basic
|
||||
>12 belong 5 32-bit linear PCM,
|
||||
!:mime audio/basic
|
||||
>12 belong 6 32-bit IEEE floating point,
|
||||
!:mime audio/basic
|
||||
>12 belong 7 64-bit IEEE floating point,
|
||||
!:mime audio/basic
|
||||
>12 belong 23 8-bit ISDN mu-law compressed (CCITT G.721 ADPCM voice enc.),
|
||||
!:mime audio/x-adpcm
|
||||
|
||||
# DEC systems (e.g. DECstation 5000) use a variant of the Sun/NeXT format
|
||||
# that uses little-endian encoding and has a different magic number
|
||||
0 lelong 0x0064732E DEC audio data:
|
||||
>12 lelong 1 8-bit ISDN mu-law,
|
||||
!:mime audio/x-dec-basic
|
||||
>12 lelong 2 8-bit linear PCM [REF-PCM],
|
||||
!:mime audio/x-dec-basic
|
||||
>12 lelong 3 16-bit linear PCM,
|
||||
!:mime audio/x-dec-basic
|
||||
>12 lelong 4 24-bit linear PCM,
|
||||
!:mime audio/x-dec-basic
|
||||
>12 lelong 5 32-bit linear PCM,
|
||||
!:mime audio/x-dec-basic
|
||||
>12 lelong 6 32-bit IEEE floating point,
|
||||
!:mime audio/x-dec-basic
|
||||
>12 lelong 7 64-bit IEEE floating point,
|
||||
!:mime audio/x-dec-basic
|
||||
>12 lelong 23 8-bit ISDN mu-law compressed (CCITT G.721 ADPCM voice enc.),
|
||||
!:mime audio/x-dec-basic
|
||||
|
||||
# Creative Labs AUDIO stuff
|
||||
0 string MThd Standard MIDI data
|
||||
!:mime audio/midi
|
||||
|
||||
0 string CTMF Creative Music (CMF) data
|
||||
!:mime audio/x-unknown
|
||||
0 string SBI SoundBlaster instrument data
|
||||
!:mime audio/x-unknown
|
||||
0 string Creative\ Voice\ File Creative Labs voice data
|
||||
!:mime audio/x-unknown
|
||||
|
||||
# Real Audio (Magic .ra\0375)
|
||||
0 belong 0x2e7261fd RealAudio sound file
|
||||
!:mime audio/x-pn-realaudio
|
||||
0 string .RMF\0\0\0 RealMedia file
|
||||
!:mime application/vnd.rn-realmedia
|
||||
|
||||
# mime types according to http://www.geocities.com/nevilo/mod.htm:
|
||||
# audio/it .it
|
||||
# audio/x-zipped-it .itz
|
||||
# audio/xm fasttracker modules
|
||||
# audio/x-s3m screamtracker modules
|
||||
# audio/s3m screamtracker modules
|
||||
# audio/x-zipped-mod mdz
|
||||
# audio/mod mod
|
||||
# audio/x-mod All modules (mod, s3m, 669, mtm, med, xm, it, mdz, stm, itz, xmz, s3z)
|
||||
|
||||
#
|
||||
# Taken from loader code from mikmod version 2.14
|
||||
# by Steve McIntyre (stevem@chiark.greenend.org.uk)
|
||||
# <doj@cubic.org> added title printing on 2003-06-24
|
||||
0 string MAS_UTrack_V00
|
||||
>14 string >/0 ultratracker V1.%.1s module sound data
|
||||
!:mime audio/x-mod
|
||||
#audio/x-tracker-module
|
||||
|
||||
0 string Extended\ Module: Fasttracker II module sound data
|
||||
!:mime audio/x-mod
|
||||
#audio/x-tracker-module
|
||||
|
||||
21 string/c =!SCREAM! Screamtracker 2 module sound data
|
||||
!:mime audio/x-mod
|
||||
#audio/x-screamtracker-module
|
||||
21 string BMOD2STM Screamtracker 2 module sound data
|
||||
!:mime audio/x-mod
|
||||
#audio/x-screamtracker-module
|
||||
1080 string M.K. 4-channel Protracker module sound data
|
||||
!:mime audio/x-mod
|
||||
#audio/x-protracker-module
|
||||
1080 string M!K! 4-channel Protracker module sound data
|
||||
!:mime audio/x-mod
|
||||
#audio/x-protracker-module
|
||||
1080 string FLT4 4-channel Startracker module sound data
|
||||
!:mime audio/x-mod
|
||||
#audio/x-startracker-module
|
||||
1080 string FLT8 8-channel Startracker module sound data
|
||||
!:mime audio/x-mod
|
||||
#audio/x-startracker-module
|
||||
1080 string 4CHN 4-channel Fasttracker module sound data
|
||||
!:mime audio/x-mod
|
||||
#audio/x-fasttracker-module
|
||||
1080 string 6CHN 6-channel Fasttracker module sound data
|
||||
!:mime audio/x-mod
|
||||
#audio/x-fasttracker-module
|
||||
1080 string 8CHN 8-channel Fasttracker module sound data
|
||||
!:mime audio/x-mod
|
||||
#audio/x-fasttracker-module
|
||||
1080 string CD81 8-channel Octalyser module sound data
|
||||
!:mime audio/x-mod
|
||||
#audio/x-octalysertracker-module
|
||||
1080 string OKTA 8-channel Octalyzer module sound data
|
||||
!:mime audio/x-mod
|
||||
#audio/x-octalysertracker-module
|
||||
# Not good enough.
|
||||
#1082 string CH
|
||||
#>1080 string >/0 %.2s-channel Fasttracker "oktalyzer" module sound data
|
||||
1080 string 16CN 16-channel Taketracker module sound data
|
||||
!:mime audio/x-mod
|
||||
#audio/x-taketracker-module
|
||||
1080 string 32CN 32-channel Taketracker module sound data
|
||||
!:mime audio/x-mod
|
||||
#audio/x-taketracker-module
|
||||
|
||||
# Impulse tracker module (audio/x-it)
|
||||
0 string IMPM Impulse Tracker module sound data -
|
||||
!:mime audio/x-mod
|
||||
|
||||
# Free lossless audio codec <http://flac.sourceforge.net>
|
||||
# From: Przemyslaw Augustyniak <silvathraec@rpg.pl>
|
||||
0 string fLaC FLAC audio bitstream data
|
||||
!:mime audio/x-flac
|
||||
|
||||
# Monkey's Audio compressed audio format (.ape)
|
||||
# From danny.milo@gmx.net (Danny Milosavljevic)
|
||||
# New version from Abel Cheung <abel (@) oaka.org>
|
||||
0 string MAC\040 Monkey's Audio compressed format
|
||||
!:mime audio/x-ape
|
||||
|
||||
# musepak support From: "Jiri Pejchal" <jiri.pejchal@gmail.com>
|
||||
0 string MP+ Musepack audio
|
||||
!:mime audio/x-musepack
|
||||
47
magic/c-lang
Normal file
47
magic/c-lang
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: c-lang,v 1.16 2011/12/09 08:02:16 rrt Exp $
|
||||
# c-lang: file(1) magic for C and related languages programs
|
||||
#
|
||||
|
||||
# BCPL
|
||||
0 search/8192 "libhdr" BCPL source text
|
||||
!:mime text/x-bcpl
|
||||
0 search/8192 "LIBHDR" BCPL source text
|
||||
!:mime text/x-bcpl
|
||||
|
||||
# C
|
||||
0 regex \^#include C source text
|
||||
!:mime text/x-c
|
||||
0 regex \^char C source text
|
||||
!:mime text/x-c
|
||||
0 regex \^double C source text
|
||||
!:mime text/x-c
|
||||
0 regex \^extern C source text
|
||||
!:mime text/x-c
|
||||
0 regex \^float C source text
|
||||
!:mime text/x-c
|
||||
0 regex \^struct C source text
|
||||
!:mime text/x-c
|
||||
0 regex \^union C source text
|
||||
!:mime text/x-c
|
||||
0 search/8192 main( C source text
|
||||
!:mime text/x-c
|
||||
|
||||
# C++
|
||||
# The strength of these rules is increased so they beat the C rules above
|
||||
0 regex \^template C++ source text
|
||||
!:strength + 5
|
||||
!:mime text/x-c++
|
||||
0 regex \^virtual C++ source text
|
||||
!:strength + 5
|
||||
!:mime text/x-c++
|
||||
0 regex \^class C++ source text
|
||||
!:strength + 5
|
||||
!:mime text/x-c++
|
||||
0 regex \^public: C++ source text
|
||||
!:strength + 5
|
||||
!:mime text/x-c++
|
||||
0 regex \^private: C++ source text
|
||||
!:strength + 5
|
||||
!:mime text/x-c++
|
||||
31
magic/cafebabe
Normal file
31
magic/cafebabe
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: cafebabe,v 1.13 2013/02/26 21:04:38 christos Exp $
|
||||
# Cafe Babes unite!
|
||||
#
|
||||
# Since Java bytecode and Mach-O universal binaries have the same magic number,
|
||||
# the test must be performed in the same "magic" sequence to get both right.
|
||||
# The long at offset 4 in a Mach-O universal binary tells the number of
|
||||
# architectures; the short at offset 4 in a Java bytecode file is the JVM minor
|
||||
# version and the short at offset 6 is the JVM major version. Since there are only
|
||||
# only 18 labeled Mach-O architectures at current, and the first released
|
||||
# Java class format was version 43.0, we can safely choose any number
|
||||
# between 18 and 39 to test the number of architectures against
|
||||
# (and use as a hack). Let's not use 18, because the Mach-O people
|
||||
# might add another one or two as time goes by...
|
||||
#
|
||||
### JAVA START ###
|
||||
0 belong 0xcafebabe
|
||||
!:mime application/x-java-applet
|
||||
|
||||
0 belong 0xcafed00d JAR compressed with pack200,
|
||||
>5 byte x version %d.
|
||||
>4 byte x \b%d
|
||||
!:mime application/x-java-pack200
|
||||
|
||||
0 belong 0xcafed00d JAR compressed with pack200,
|
||||
>5 byte x version %d.
|
||||
>4 byte x \b%d
|
||||
!:mime application/x-java-pack200
|
||||
|
||||
### JAVA END ###
|
||||
82
magic/commands
Normal file
82
magic/commands
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: commands,v 1.44 2013/02/05 15:20:47 christos Exp $
|
||||
# commands: file(1) magic for various shells and interpreters
|
||||
#
|
||||
#0 string/w : shell archive or script for antique kernel text
|
||||
0 string/wt #!\ /bin/sh POSIX shell script text executable
|
||||
!:mime text/x-shellscript
|
||||
0 string/wt #!\ /bin/csh C shell script text executable
|
||||
!:mime text/x-shellscript
|
||||
# korn shell magic, sent by George Wu, gwu@clyde.att.com
|
||||
0 string/wt #!\ /bin/ksh Korn shell script text executable
|
||||
!:mime text/x-shellscript
|
||||
0 string/wt #!\ /bin/tcsh Tenex C shell script text executable
|
||||
!:mime text/x-shellscript
|
||||
0 string/wt #!\ /usr/bin/tcsh Tenex C shell script text executable
|
||||
!:mime text/x-shellscript
|
||||
0 string/wt #!\ /usr/local/tcsh Tenex C shell script text executable
|
||||
!:mime text/x-shellscript
|
||||
0 string/wt #!\ /usr/local/bin/tcsh Tenex C shell script text executable
|
||||
!:mime text/x-shellscript
|
||||
|
||||
#
|
||||
# zsh/ash/ae/nawk/gawk magic from cameron@cs.unsw.oz.au (Cameron Simpson)
|
||||
0 string/wt #!\ /bin/zsh Paul Falstad's zsh script text executable
|
||||
!:mime text/x-shellscript
|
||||
0 string/wt #!\ /usr/bin/zsh Paul Falstad's zsh script text executable
|
||||
!:mime text/x-shellscript
|
||||
0 string/wt #!\ /usr/local/bin/zsh Paul Falstad's zsh script text executable
|
||||
!:mime text/x-shellscript
|
||||
0 string/wt #!\ /usr/local/bin/ash Neil Brown's ash script text executable
|
||||
!:mime text/x-shellscript
|
||||
0 string/wt #!\ /usr/local/bin/ae Neil Brown's ae script text executable
|
||||
!:mime text/x-shellscript
|
||||
0 string/wt #!\ /bin/nawk new awk script text executable
|
||||
!:mime text/x-nawk
|
||||
0 string/wt #!\ /usr/bin/nawk new awk script text executable
|
||||
!:mime text/x-nawk
|
||||
0 string/wt #!\ /usr/local/bin/nawk new awk script text executable
|
||||
!:mime text/x-nawk
|
||||
0 string/wt #!\ /bin/gawk GNU awk script text executable
|
||||
!:mime text/x-gawk
|
||||
0 string/wt #!\ /usr/bin/gawk GNU awk script text executable
|
||||
!:mime text/x-gawk
|
||||
0 string/wt #!\ /usr/local/bin/gawk GNU awk script text executable
|
||||
!:mime text/x-gawk
|
||||
#
|
||||
0 string/wt #!\ /bin/awk awk script text executable
|
||||
!:mime text/x-awk
|
||||
0 string/wt #!\ /usr/bin/awk awk script text executable
|
||||
!:mime text/x-awk
|
||||
|
||||
# bash shell magic, from Peter Tobias (tobias@server.et-inf.fho-emden.de)
|
||||
0 string/wt #!\ /bin/bash Bourne-Again shell script text executable
|
||||
!:mime text/x-shellscript
|
||||
0 string/wt #!\ /usr/bin/bash Bourne-Again shell script text executable
|
||||
!:mime text/x-shellscript
|
||||
0 string/wt #!\ /usr/local/bash Bourne-Again shell script text executable
|
||||
!:mime text/x-shellscript
|
||||
0 string/wt #!\ /usr/local/bin/bash Bourne-Again shell script text executable
|
||||
!:mime text/x-shellscript
|
||||
|
||||
# PHP scripts
|
||||
# Ulf Harnhammar <ulfh@update.uu.se>
|
||||
0 search/1/c =<?php PHP script text
|
||||
!:strength + 10
|
||||
!:mime text/x-php
|
||||
0 search/1 =<?\n PHP script text
|
||||
!:mime text/x-php
|
||||
0 search/1 =<?\r PHP script text
|
||||
!:mime text/x-php
|
||||
0 search/1/w #!\ /usr/local/bin/php PHP script text executable
|
||||
!:strength + 10
|
||||
!:mime text/x-php
|
||||
0 search/1/w #!\ /usr/bin/php PHP script text executable
|
||||
!:strength + 10
|
||||
!:mime text/x-php
|
||||
# Smarty compiled template, http://www.smarty.net/
|
||||
# Elan Ruusamae <glen@delfi.ee>
|
||||
0 string =<?php\ /*\ Smarty\ version Smarty compiled template
|
||||
>24 regex [0-9.]+ \b, version %s
|
||||
!:mime text/x-php
|
||||
77
magic/compress
Normal file
77
magic/compress
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: compress,v 1.48 2011/12/07 18:39:43 christos Exp $
|
||||
# compress: file(1) magic for pure-compression formats (no archives)
|
||||
#
|
||||
# compress, gzip, pack, compact, huf, squeeze, crunch, freeze, yabba, etc.
|
||||
#
|
||||
# Formats for various forms of compressed data
|
||||
# Formats for "compress" proper have been moved into "compress.c",
|
||||
# because it tries to uncompress it to figure out what's inside.
|
||||
|
||||
# standard unix compress
|
||||
0 string \037\235 compress'd data
|
||||
!:mime application/x-compress
|
||||
!:apple LZIVZIVU
|
||||
|
||||
# gzip (GNU zip, not to be confused with Info-ZIP or PKWARE zip archiver)
|
||||
# Edited by Chris Chittleborough <cchittleborough@yahoo.com.au>, March 2002
|
||||
# * Original filename is only at offset 10 if "extra field" absent
|
||||
# * Produce shorter output - notably, only report compression methods
|
||||
# other than 8 ("deflate", the only method defined in RFC 1952).
|
||||
0 string \037\213 gzip compressed data
|
||||
!:mime application/x-gzip
|
||||
|
||||
# packed data, Huffman (minimum redundancy) codes on a byte-by-byte basis
|
||||
0 string \037\036 packed data
|
||||
!:mime application/octet-stream
|
||||
|
||||
#
|
||||
# This magic number is byte-order-independent.
|
||||
0 short 0x1f1f old packed data
|
||||
!:mime application/octet-stream
|
||||
|
||||
# XXX - why *two* entries for "compacted data", one of which is
|
||||
# byte-order independent, and one of which is byte-order dependent?
|
||||
#
|
||||
0 short 0x1fff compacted data
|
||||
!:mime application/octet-stream
|
||||
# This string is valid for SunOS (BE) and a matching "short" is listed
|
||||
# in the Ultrix (LE) magic file.
|
||||
0 string \377\037 compacted data
|
||||
!:mime application/octet-stream
|
||||
0 short 0145405 huf output
|
||||
!:mime application/octet-stream
|
||||
|
||||
# bzip2
|
||||
0 string BZh bzip2 compressed data
|
||||
!:mime application/x-bzip2
|
||||
|
||||
# lzip
|
||||
0 string LZIP lzip compressed data
|
||||
!:mime application/x-lzip
|
||||
|
||||
# 7-zip archiver, from Thomas Klausner (wiz@danbala.tuwien.ac.at)
|
||||
# http://www.7-zip.org or DOC/7zFormat.txt
|
||||
#
|
||||
0 string 7z\274\257\047\034 7-zip archive data,
|
||||
>6 byte x version %d
|
||||
>7 byte x \b.%d
|
||||
!:mime application/x-7z-compressed
|
||||
|
||||
# Type: LZMA
|
||||
0 lelong&0xffffff =0x5d
|
||||
>12 leshort =0xff LZMA compressed data,
|
||||
>>5 lequad =0xffffffffffffffff streamed
|
||||
>>5 lequad !0xffffffffffffffff non-streamed, size %lld
|
||||
!:mime application/x-lzma
|
||||
|
||||
# http://tukaani.org/xz/xz-file-format.txt
|
||||
0 ustring \xFD7zXZ\x00 XZ compressed data
|
||||
!:mime application/x-xz
|
||||
|
||||
# https://github.com/ckolivas/lrzip/blob/master/doc/magic.header.txt
|
||||
0 string LRZI LRZIP compressed data
|
||||
>4 byte x - version %d
|
||||
>5 byte x \b.%d
|
||||
!:mime application/x-lrzip
|
||||
47
magic/database
Normal file
47
magic/database
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: database,v 1.32 2013/02/06 14:18:52 christos Exp $
|
||||
# database: file(1) magic for various databases
|
||||
#
|
||||
# extracted from header/code files by Graeme Wilford (eep2gw@ee.surrey.ac.uk)
|
||||
#
|
||||
#
|
||||
# GDBM magic numbers
|
||||
# Will be maintained as part of the GDBM distribution in the future.
|
||||
# <downsj@teeny.org>
|
||||
0 belong 0x13579ace GNU dbm 1.x or ndbm database, big endian
|
||||
!:mime application/x-gdbm
|
||||
0 lelong 0x13579ace GNU dbm 1.x or ndbm database, little endian
|
||||
!:mime application/x-gdbm
|
||||
0 string GDBM GNU dbm 2.x database
|
||||
!:mime application/x-gdbm
|
||||
#
|
||||
# Berkeley DB
|
||||
#
|
||||
# Ian Darwin's file /etc/magic files: big/little-endian version.
|
||||
#
|
||||
# Hash 1.85/1.86 databases store metadata in network byte order.
|
||||
# Btree 1.85/1.86 databases store the metadata in host byte order.
|
||||
# Hash and Btree 2.X and later databases store the metadata in host byte order.
|
||||
|
||||
0 long 0x00061561 Berkeley DB
|
||||
!:mime application/x-dbm
|
||||
|
||||
# MS Access database
|
||||
4 string Standard\ Jet\ DB Microsoft Access Database
|
||||
!:mime application/x-msaccess
|
||||
4 string Standard\ ACE\ DB Microsoft Access Database
|
||||
!:mime application/x-msaccess
|
||||
|
||||
# Tokyo Cabinet magic data
|
||||
# http://tokyocabinet.sourceforge.net/index.html
|
||||
0 string ToKyO\ CaBiNeT\n Tokyo Cabinet
|
||||
>14 string x \b (%s)
|
||||
>32 byte 0 \b, Hash
|
||||
!:mime application/x-tokyocabinet-hash
|
||||
>32 byte 1 \b, B+ tree
|
||||
!:mime application/x-tokyocabinet-btree
|
||||
>32 byte 2 \b, Fixed-length
|
||||
!:mime application/x-tokyocabinet-fixed
|
||||
>32 byte 3 \b, Table
|
||||
!:mime application/x-tokyocabinet-table
|
||||
25
magic/diff
Normal file
25
magic/diff
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: diff,v 1.13 2012/06/16 14:43:36 christos Exp $
|
||||
# diff: file(1) magic for diff(1) output
|
||||
#
|
||||
0 search/1 diff\ diff output text
|
||||
!:mime text/x-diff
|
||||
0 search/1 ***\ diff output text
|
||||
!:mime text/x-diff
|
||||
0 search/1 Only\ in\ diff output text
|
||||
!:mime text/x-diff
|
||||
0 search/1 Common\ subdirectories:\ diff output text
|
||||
!:mime text/x-diff
|
||||
|
||||
0 search/1 Index: RCS/CVS diff output text
|
||||
!:mime text/x-diff
|
||||
|
||||
# unified diff
|
||||
0 search/4096 ---\
|
||||
>&0 search/1024 \n
|
||||
>>&0 search/1 +++\
|
||||
>>>&0 search/1024 \n
|
||||
>>>>&0 search/1 @@ unified diff output text
|
||||
!:mime text/x-diff
|
||||
!:strength + 90
|
||||
43
magic/elf
Normal file
43
magic/elf
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# elf: file(1) magic for ELF executables
|
||||
#
|
||||
# We have to check the byte order flag to see what byte order all the
|
||||
# other stuff in the header is in.
|
||||
#
|
||||
# What're the correct byte orders for the nCUBE and the Fujitsu VPP500?
|
||||
#
|
||||
# Created by: unknown
|
||||
# Modified by (1): Daniel Quinlan <quinlan@yggdrasil.com>
|
||||
# Modified by (2): Peter Tobias <tobias@server.et-inf.fho-emden.de> (core support)
|
||||
# Modified by (3): Christian 'Dr. Disk' Hechelmann <drdisk@ds9.au.s.shuttle.de> (fix of core support)
|
||||
# Modified by (4): <gerardo.cacciari@gmail.com> (VMS Itanium)
|
||||
# Modified by (5): Matthias Urlichs <smurf@debian.org> (Listing of many architectures)
|
||||
0 string \177ELF ELF
|
||||
>4 byte 0 invalid class
|
||||
>4 byte 1 32-bit
|
||||
>4 byte 2 64-bit
|
||||
>5 byte 0 invalid byte order
|
||||
>5 byte 1 LSB
|
||||
>>16 leshort 0 no file type,
|
||||
!:strength *2
|
||||
!:mime application/octet-stream
|
||||
>>16 leshort 1 relocatable,
|
||||
!:mime application/x-object
|
||||
>>16 leshort 2 executable,
|
||||
!:mime application/x-executable
|
||||
>>16 leshort 3 shared object,
|
||||
!:mime application/x-sharedlib
|
||||
>>16 leshort 4 core file
|
||||
!:mime application/x-coredump
|
||||
>5 byte 2 MSB
|
||||
>>16 beshort 0 no file type,
|
||||
!:mime application/octet-stream
|
||||
>>16 beshort 1 relocatable,
|
||||
!:mime application/x-object
|
||||
>>16 beshort 2 executable,
|
||||
!:mime application/x-executable
|
||||
>>16 beshort 3 shared object,
|
||||
!:mime application/x-sharedlib
|
||||
>>16 beshort 4 core file,
|
||||
!:mime application/x-coredump
|
||||
34
magic/epoc
Normal file
34
magic/epoc
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: epoc,v 1.7 2009/09/19 16:28:09 christos Exp $
|
||||
# EPOC : file(1) magic for EPOC documents [Psion Series 5/Osaris/Geofox 1]
|
||||
# Stefan Praszalowicz <hpicollo@worldnet.fr> and Peter Breitenlohner <peb@mppmu.mpg.de>
|
||||
# Useful information for improving this file can be found at:
|
||||
# http://software.frodo.looijaard.name/psiconv/formats/Index.html
|
||||
#------------------------------------------------------------------------------
|
||||
0 lelong 0x10000037 Psion Series 5
|
||||
>4 lelong 0x10000042 multi-bitmap image
|
||||
!:mime image/x-epoc-mbm
|
||||
>4 lelong 0x1000006D
|
||||
>>8 lelong 0x1000007D Sketch image
|
||||
!:mime image/x-epoc-sketch
|
||||
>>8 lelong 0x1000007F Word file
|
||||
!:mime application/x-epoc-word
|
||||
>>8 lelong 0x10000085 OPL program (TextEd)
|
||||
!:mime application/x-epoc-opl
|
||||
>>8 lelong 0x10000088 Sheet file
|
||||
!:mime application/x-epoc-sheet
|
||||
>4 lelong 0x10000073 OPO module
|
||||
!:mime application/x-epoc-opo
|
||||
>4 lelong 0x10000074 OPL application
|
||||
!:mime application/x-epoc-app
|
||||
|
||||
|
||||
0 lelong 0x10000050 Psion Series 5
|
||||
>4 lelong 0x1000006D database
|
||||
>>8 lelong 0x10000084 Agenda file
|
||||
!:mime application/x-epoc-agenda
|
||||
>>8 lelong 0x10000086 Data file
|
||||
!:mime application/x-epoc-data
|
||||
>>8 lelong 0x10000CEA Jotter file
|
||||
!:mime application/x-epoc-jotter
|
||||
12
magic/filesystems
Normal file
12
magic/filesystems
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: filesystems,v 1.76 2013/02/18 18:45:41 christos Exp $
|
||||
# filesystems: file(1) magic for different filesystems
|
||||
#
|
||||
|
||||
# CDROM Filesystems
|
||||
# Modified for UDF by gerardo.cacciari@gmail.com
|
||||
32769 string CD001 #
|
||||
!:mime application/x-iso9660-image
|
||||
37633 string CD001 ISO 9660 CD-ROM filesystem data (raw 2352 byte sectors)
|
||||
!:mime application/x-iso9660-image
|
||||
18
magic/flash
Normal file
18
magic/flash
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: flash,v 1.8 2009/09/19 16:28:09 christos Exp $
|
||||
# flash: file(1) magic for Macromedia Flash file format
|
||||
#
|
||||
# See
|
||||
#
|
||||
# http://www.macromedia.com/software/flash/open/
|
||||
#
|
||||
0 string FWS Macromedia Flash data,
|
||||
>3 byte x version %d
|
||||
!:mime application/x-shockwave-flash
|
||||
0 string CWS Macromedia Flash data (compressed),
|
||||
!:mime application/x-shockwave-flash
|
||||
|
||||
# From: Cal Peake <cp@absolutedigital.net>
|
||||
0 string FLV Macromedia Flash Video
|
||||
!:mime video/x-flv
|
||||
32
magic/fonts
Normal file
32
magic/fonts
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: fonts,v 1.25 2013/02/06 14:18:52 christos Exp $
|
||||
# fonts: file(1) magic for font data
|
||||
#
|
||||
|
||||
# X11 font files in SNF (Server Natural Format) format
|
||||
# updated by Joerg Jenderek at Feb 2013
|
||||
# http://computer-programming-forum.com/51-perl/8f22fb96d2e34bab.htm
|
||||
0 belong 00000004 X11 SNF font data, MSB first
|
||||
#>104 belong 00000004 X11 SNF font data, MSB first
|
||||
!:mime application/x-font-sfn
|
||||
# GRR: line below too general as it catches also Xbase index file t3-CHAR.NDX
|
||||
0 lelong 00000004
|
||||
>104 lelong 00000004 X11 SNF font data, LSB first
|
||||
!:mime application/x-font-sfn
|
||||
|
||||
# True Type fonts
|
||||
0 string \000\001\000\000\000 TrueType font data
|
||||
!:mime application/x-font-ttf
|
||||
|
||||
# Opentype font data from Avi Bercovich
|
||||
0 string OTTO OpenType font data
|
||||
!:mime application/vnd.ms-opentype
|
||||
|
||||
# Gurkan Sengun <gurkan@linuks.mine.nu>, www.linuks.mine.nu
|
||||
0 string SplineFontDB: Spline Font Database
|
||||
!:mime application/vnd.font-fontforge-sfd
|
||||
|
||||
# EOT
|
||||
34 string LP Embedded OpenType (EOT)
|
||||
!:mime application/vnd.ms-fontobject
|
||||
7
magic/fortran
Normal file
7
magic/fortran
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: fortran,v 1.6 2009/09/19 16:28:09 christos Exp $
|
||||
# FORTRAN source
|
||||
0 regex/100 \^[Cc][\ \t] FORTRAN program
|
||||
!:mime text/x-fortran
|
||||
!:strength - 5
|
||||
31
magic/frame
Normal file
31
magic/frame
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File$
|
||||
# frame: file(1) magic for FrameMaker files
|
||||
#
|
||||
# This stuff came on a FrameMaker demo tape, most of which is
|
||||
# copyright, but this file is "published" as witness the following:
|
||||
#
|
||||
# Note that this is the Framemaker Maker Interchange Format, not the
|
||||
# Normal format which would be application/vnd.framemaker.
|
||||
#
|
||||
0 string \<MakerFile FrameMaker document
|
||||
!:mime application/x-mif
|
||||
0 string \<MIFFile FrameMaker MIF (ASCII) file
|
||||
!:mime application/x-mif
|
||||
0 search/1 \<MakerDictionary FrameMaker Dictionary text
|
||||
!:mime application/x-mif
|
||||
0 string \<MakerScreenFont FrameMaker Font file
|
||||
!:mime application/x-mif
|
||||
0 string \<MML FrameMaker MML file
|
||||
!:mime application/x-mif
|
||||
0 string \<BookFile FrameMaker Book file
|
||||
!:mime application/x-mif
|
||||
# XXX - this book entry should be verified, if you find one, uncomment this
|
||||
#0 string \<Book\ FrameMaker Book (ASCII) file
|
||||
#!:mime application/x-mif
|
||||
#>6 string 3.0 (3.0)
|
||||
#>6 string 2.0 (2.0)
|
||||
#>6 string 1.0 (1.0)
|
||||
0 string \<Maker Intermediate Print File FrameMaker IPL file
|
||||
!:mime application/x-mif
|
||||
13
magic/gimp
Normal file
13
magic/gimp
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: gimp,v 1.6 2009/09/19 16:28:09 christos Exp $
|
||||
# GIMP Gradient: file(1) magic for the GIMP's gradient data files
|
||||
# by Federico Mena <federico@nuclecu.unam.mx>
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# XCF: file(1) magic for the XCF image format used in the GIMP developed
|
||||
# by Spencer Kimball and Peter Mattis
|
||||
# ('Bucky' LaDieu, nega@vt.edu)
|
||||
|
||||
0 string gimp\ xcf GIMP XCF image data,
|
||||
!:mime image/x-xcf
|
||||
23
magic/gnu
Normal file
23
magic/gnu
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: gnu,v 1.13 2012/01/03 17:16:54 christos Exp $
|
||||
# gnu: file(1) magic for various GNU tools
|
||||
#
|
||||
# GNU nlsutils message catalog file format
|
||||
#
|
||||
# GNU message catalog (.mo and .gmo files)
|
||||
|
||||
# GnuPG
|
||||
# The format is very similar to pgp
|
||||
# Note: magic.mime had 0x8501 for the next line instead of 0x8502
|
||||
0 beshort 0x8502 GPG encrypted data
|
||||
!:mime text/PGP # encoding: data
|
||||
|
||||
# This magic is not particularly good, as the keyrings don't have true
|
||||
# magic. Nevertheless, it covers many keyrings.
|
||||
0 beshort 0x9901 GPG key public ring
|
||||
!:mime application/x-gnupg-keyring
|
||||
|
||||
# gettext message catalogue
|
||||
0 regex \^msgid\ GNU gettext message catalogue text
|
||||
!:mime text/x-po
|
||||
8
magic/gnumeric
Normal file
8
magic/gnumeric
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File$
|
||||
# gnumeric: file(1) magic for Gnumeric spreadsheet
|
||||
# This entry is only semi-helpful, as Gnumeric compresses its files, so
|
||||
# they will ordinarily reported as "compressed", but at least -z helps
|
||||
39 string =<gmr:Workbook Gnumeric spreadsheet
|
||||
!:mime application/x-gnumeric
|
||||
51
magic/icc
Normal file
51
magic/icc
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File$
|
||||
# icc: file(1) magic for International Color Consortium file formats
|
||||
|
||||
#
|
||||
# Color profiles as per the ICC's "Image technology colour management -
|
||||
# Architecture, profile format, and data structure" specification.
|
||||
# See
|
||||
#
|
||||
# http://www.color.org/specification/ICC1v43_2010-12.pdf
|
||||
#
|
||||
# for Specification ICC.1:2010 (Profile version 4.3.0.0).
|
||||
#
|
||||
# Bytes 36 to 39 contain a generic profile file signature of "acsp";
|
||||
# bytes 40 to 43 "may be used to identify the primary platform/operating
|
||||
# system framework for which the profile was created".
|
||||
#
|
||||
# There are other fields that might be worth dumping as well.
|
||||
#
|
||||
|
||||
# This appears to be what's used for Apple ColorSync profiles.
|
||||
# Instead of adding that, Apple just changed the generic "acsp" entry
|
||||
# to be for "ColorSync ICC Color Profile" rather than "Kodak Color
|
||||
# Management System, ICC Profile".
|
||||
# Yes, it's "APPL", not "AAPL"; see the spec.
|
||||
36 string acspAPPL ColorSync ICC Profile
|
||||
!:mime application/vnd.iccprofile
|
||||
|
||||
# Microsoft ICM color profile
|
||||
36 string acspMSFT Microsoft ICM Color Profile
|
||||
!:mime application/vnd.iccprofile
|
||||
|
||||
# Yes, that's a blank after "SGI".
|
||||
36 string acspSGI\ SGI ICC Profile
|
||||
!:mime application/vnd.iccprofile
|
||||
|
||||
# XXX - is this what's used for the Sun KCMS or not? The standard file
|
||||
# uses just "acsp" for that, but Apple's file uses it for "ColorSync",
|
||||
# and there *is* an identified "primary platform" value of SUNW.
|
||||
36 string acspSUNW Sun KCMS ICC Profile
|
||||
!:mime application/vnd.iccprofile
|
||||
|
||||
# Any other profile.
|
||||
# XXX - should we use "acsp\0\0\0\0" for "no primary platform" profiles,
|
||||
# and use "acsp" for everything else and dump the "primary platform"
|
||||
# string in those cases?
|
||||
36 string acsp ICC Profile
|
||||
!:mime application/vnd.iccprofile
|
||||
|
||||
|
||||
21
magic/iff
Normal file
21
magic/iff
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: iff,v 1.12 2009/09/19 16:28:09 christos Exp $
|
||||
# iff: file(1) magic for Interchange File Format (see also "audio" & "images")
|
||||
#
|
||||
# Daniel Quinlan (quinlan@yggdrasil.com) -- IFF was designed by Electronic
|
||||
# Arts for file interchange. It has also been used by Apple, SGI, and
|
||||
# especially Commodore-Amiga.
|
||||
#
|
||||
# IFF files begin with an 8 byte FORM header, followed by a 4 character
|
||||
# FORM type, which is followed by the first chunk in the FORM.
|
||||
|
||||
0 string FORM IFF data
|
||||
#>4 belong x \b, FORM is %d bytes long
|
||||
# audio formats
|
||||
>8 string AIFF \b, AIFF audio
|
||||
!:mime audio/x-aiff
|
||||
>8 string AIFC \b, AIFF-C compressed audio
|
||||
!:mime audio/x-aiff
|
||||
>8 string 8SVX \b, 8SVX 8-bit sampled sound voice
|
||||
!:mime audio/x-aiff
|
||||
255
magic/images
Normal file
255
magic/images
Normal file
|
|
@ -0,0 +1,255 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: images,v 1.80 2013/02/06 14:18:52 christos Exp $
|
||||
# images: file(1) magic for image formats (see also "iff", and "c-lang" for
|
||||
# XPM bitmaps)
|
||||
#
|
||||
# originally from jef@helios.ee.lbl.gov (Jef Poskanzer),
|
||||
# additions by janl@ifi.uio.no as well as others. Jan also suggested
|
||||
# merging several one- and two-line files into here.
|
||||
#
|
||||
# little magic: PCX (first byte is 0x0a)
|
||||
|
||||
# PBMPLUS images
|
||||
# The next byte following the magic is always whitespace.
|
||||
# strength is changed to try these patterns before "x86 boot sector"
|
||||
0 search/1 P1
|
||||
>3 regex =[0-9]*\ [0-9]* Netpbm PBM image text
|
||||
>3 regex =[0-9]+\ \b, size = %sx
|
||||
>>3 regex =\ [0-9]+ \b%s
|
||||
!:strength + 45
|
||||
!:mime image/x-portable-bitmap
|
||||
0 search/1 P2
|
||||
>3 regex =[0-9]*\ [0-9]* Netpbm PGM image text
|
||||
>3 regex =[0-9]+\ \b, size = %sx
|
||||
>>3 regex =\ [0-9]+ \b%s
|
||||
!:strength + 45
|
||||
!:mime image/x-portable-greymap
|
||||
0 search/1 P3 Netpbm PPM image text
|
||||
>3 regex =[0-9]*\ [0-9]* Netpbm PPM image text
|
||||
>3 regex =[0-9]+\ \b, size = %sx
|
||||
>>3 regex =\ [0-9]+ \b%s
|
||||
!:strength + 45
|
||||
!:mime image/x-portable-pixmap
|
||||
0 string P4
|
||||
>3 regex =[0-9]*\ [0-9]* Netpbm PBM "rawbits" image data
|
||||
>3 regex =[0-9]+\ \b, size = %sx
|
||||
>>3 regex =\ [0-9]+ \b%s
|
||||
!:strength + 45
|
||||
!:mime image/x-portable-bitmap
|
||||
0 string P5
|
||||
>3 regex =[0-9]*\ [0-9]* Netpbm PGM "rawbits" image data
|
||||
>3 regex =[0-9]+\ \b, size = %sx
|
||||
>>3 regex =\ [0-9]+ \b%s
|
||||
!:strength + 45
|
||||
!:mime image/x-portable-greymap
|
||||
0 string P6
|
||||
>3 regex =[0-9]*\ [0-9]* Netpbm PPM "rawbits" image data
|
||||
>3 regex =[0-9]+\ \b, size = %sx
|
||||
>>3 regex =\ [0-9]+ \b%s
|
||||
!:strength + 45
|
||||
!:mime image/x-portable-pixmap
|
||||
0 string P7 Netpbm PAM image file
|
||||
!:mime image/x-portable-pixmap
|
||||
|
||||
# NIFF (Navy Interchange File Format, a modification of TIFF) images
|
||||
# [GRR: this *must* go before TIFF]
|
||||
0 string IIN1 NIFF image data
|
||||
!:mime image/x-niff
|
||||
|
||||
# Canon RAW version 1 (CRW) files are a type of Canon Image File Format
|
||||
# (CIFF) file. These are apparently all little-endian.
|
||||
# From: Adam Buchbinder <adam.buchbinder@gmail.com>
|
||||
# URL: http://www.sno.phy.queensu.ca/~phil/exiftool/canon_raw.html
|
||||
0 string II\x1a\0\0\0HEAPCCDR Canon CIFF raw image data
|
||||
!:mime image/x-canon-crw
|
||||
|
||||
# Canon RAW version 2 (CR2) files are a kind of TIFF with an extra magic
|
||||
# number. Put this above the TIFF test to make sure we detect them.
|
||||
# These are apparently all little-endian.
|
||||
# From: Adam Buchbinder <adam.buchbinder@gmail.com>
|
||||
# URL: http://libopenraw.freedesktop.org/wiki/Canon_CR2
|
||||
0 string II\x2a\0\x10\0\0\0CR Canon CR2 raw image data
|
||||
!:mime image/x-canon-cr2
|
||||
|
||||
# Tag Image File Format, from Daniel Quinlan (quinlan@yggdrasil.com)
|
||||
# The second word of TIFF files is the TIFF version number, 42, which has
|
||||
# never changed. The TIFF specification recommends testing for it.
|
||||
0 string MM\x00\x2a TIFF image data, big-endian
|
||||
!:mime image/tiff
|
||||
0 string II\x2a\x00 TIFF image data, little-endian
|
||||
!:mime image/tiff
|
||||
|
||||
0 string MM\x00\x2b Big TIFF image data, big-endian
|
||||
!:mime image/tiff
|
||||
0 string II\x2b\x00 Big TIFF image data, little-endian
|
||||
!:mime image/tiff
|
||||
|
||||
# PNG [Portable Network Graphics, or "PNG's Not GIF"] images
|
||||
# (Greg Roelofs, newt@uchicago.edu)
|
||||
# (Albert Cahalan, acahalan@cs.uml.edu)
|
||||
#
|
||||
# 137 P N G \r \n ^Z \n [4-byte length] H E A D [HEAD data] [HEAD crc] ...
|
||||
#
|
||||
0 string \x89PNG\x0d\x0a\x1a\x0a PNG image data
|
||||
!:mime image/png
|
||||
|
||||
# possible GIF replacements; none yet released!
|
||||
# (Greg Roelofs, newt@uchicago.edu)
|
||||
#
|
||||
# GRR 950115: this was mine ("Zip GIF"):
|
||||
0 string GIF94z ZIF image (GIF+deflate alpha)
|
||||
!:mime image/x-unknown
|
||||
#
|
||||
# GRR 950115: this is Jeremy Wohl's Free Graphics Format (better):
|
||||
#
|
||||
0 string FGF95a FGF image (GIF+deflate beta)
|
||||
!:mime image/x-unknown
|
||||
#
|
||||
# GRR 950115: this is Thomas Boutell's Portable Bitmap Format proposal
|
||||
# (best; not yet implemented):
|
||||
#
|
||||
0 string PBF PBF image (deflate compression)
|
||||
!:mime image/x-unknown
|
||||
|
||||
# GIF
|
||||
0 string GIF8 GIF image data
|
||||
!:mime image/gif
|
||||
!:apple 8BIMGIFf
|
||||
|
||||
# From: Joerg Jenderek <joerg.jen.der.ek@gmx.net>
|
||||
# most files with the extension .EPA and some with .BMP
|
||||
0 string \x11\x06 Award BIOS Logo, 136 x 84
|
||||
!:mime image/x-award-bioslogo
|
||||
0 string \x11\x09 Award BIOS Logo, 136 x 126
|
||||
!:mime image/x-award-bioslogo
|
||||
#0 string \x07\x1f BIOS Logo corrupted?
|
||||
# http://www.blackfiveservices.co.uk/awbmtools.shtml
|
||||
# http://biosgfx.narod.ru/v3/
|
||||
# http://biosgfx.narod.ru/abr-2/
|
||||
0 string AWBM
|
||||
>4 leshort <1981 Award BIOS bitmap
|
||||
!:mime image/x-award-bmp
|
||||
|
||||
# PC bitmaps (OS/2, Windows BMP files) (Greg Roelofs, newt@uchicago.edu)
|
||||
0 string BM
|
||||
>14 leshort 12 PC bitmap, OS/2 1.x format
|
||||
!:mime image/x-ms-bmp
|
||||
>14 leshort 64 PC bitmap, OS/2 2.x format
|
||||
!:mime image/x-ms-bmp
|
||||
>14 leshort 40 PC bitmap, Windows 3.x format
|
||||
!:mime image/x-ms-bmp
|
||||
>14 leshort 128 PC bitmap, Windows NT/2000 format
|
||||
!:mime image/x-ms-bmp
|
||||
|
||||
# XPM icons (Greg Roelofs, newt@uchicago.edu)
|
||||
0 search/1 /*\ XPM\ */ X pixmap image text
|
||||
!:mime image/x-xpmi
|
||||
|
||||
# DICOM medical imaging data
|
||||
128 string DICM DICOM medical imaging data
|
||||
!:mime application/dicom
|
||||
|
||||
# XWD - X Window Dump file.
|
||||
# As described in /usr/X11R6/include/X11/XWDFile.h
|
||||
# used by the xwd program.
|
||||
# Bradford Castalia, idaeim, 1/01
|
||||
# updated by Adam Buchbinder, 2/09
|
||||
# The following assumes version 7 of the format; the first long is the length
|
||||
# of the header, which is at least 25 4-byte longs, and the one at offset 8
|
||||
# is a constant which is always either 1 or 2. Offset 12 is the pixmap depth,
|
||||
# which is a maximum of 32.
|
||||
0 belong >100
|
||||
>8 belong <3
|
||||
>>12 belong <33
|
||||
>>>4 belong 7 XWD X Window Dump image data
|
||||
!:mime image/x-xwindowdump
|
||||
|
||||
# PCX image files
|
||||
# From: Dan Fandrich <dan@coneharvesters.com>
|
||||
# updated by Joerg Jenderek at Feb 2013 by http://de.wikipedia.org/wiki/PCX
|
||||
# http://web.archive.org/web/20100206055706/http://www.qzx.com/pc-gpe/pcx.txt
|
||||
# GRR: original test was still too general as it catches xbase examples T5.DBT,T6.DBT with 0xa000000
|
||||
# test for bytes 0x0a,version byte (0,2,3,4,5),compression byte flag(0,1), bit depth (>0) of PCX or T5.DBT,T6.DBT
|
||||
0 ubelong&0xffF8fe00 0x0a000000
|
||||
# for PCX bit depth > 0
|
||||
>3 ubyte >0
|
||||
# test for valid versions
|
||||
>>1 ubyte <6
|
||||
>>>1 ubyte !1 PCX
|
||||
!:mime image/x-pcx
|
||||
|
||||
# Adobe Photoshop
|
||||
# From: Asbjoern Sloth Toennesen <asbjorn@lila.io>
|
||||
0 string 8BPS Adobe Photoshop Image
|
||||
!:mime image/vnd.adobe.photoshop
|
||||
|
||||
# Summary: DjVu image / document
|
||||
# Extension: .djvu
|
||||
# Reference: http://djvu.org/docs/DjVu3Spec.djvu
|
||||
# Submitted by: Stephane Loeuillet <stephane.loeuillet@tiscali.fr>
|
||||
# Modified by (1): Abel Cheung <abelcheung@gmail.com>
|
||||
0 string AT&TFORM
|
||||
>12 string DJVM DjVu multiple page document
|
||||
!:mime image/vnd.djvu
|
||||
>12 string DJVU DjVu image or single page document
|
||||
!:mime image/vnd.djvu
|
||||
>12 string DJVI DjVu shared document
|
||||
!:mime image/vnd.djvu
|
||||
>12 string THUM DjVu page thumbnails
|
||||
!:mime image/vnd.djvu
|
||||
|
||||
# Originally by Marc Espie
|
||||
# Modified by Robert Minsk <robertminsk at yahoo.com>
|
||||
# http://www.openexr.com/openexrfilelayout.pdf
|
||||
0 lelong 20000630 OpenEXR image data,
|
||||
!:mime image/x-exr
|
||||
|
||||
# SMPTE Digital Picture Exchange Format, SMPTE DPX
|
||||
#
|
||||
# ANSI/SMPTE 268M-1994, SMPTE Standard for File Format for Digital
|
||||
# Moving-Picture Exchange (DPX), v1.0, 18 February 1994
|
||||
# Robert Minsk <robertminsk at yahoo.com>
|
||||
0 string SDPX DPX image data, big-endian,
|
||||
!:mime image/x-dpx
|
||||
|
||||
#-----------------------------------------------------------------------
|
||||
# Hierarchical Data Format, used to facilitate scientific data exchange
|
||||
# specifications at http://hdf.ncsa.uiuc.edu/
|
||||
0 belong 0x0e031301 Hierarchical Data Format (version 4) data
|
||||
!:mime application/x-hdf
|
||||
0 string \211HDF\r\n\032\n Hierarchical Data Format (version 5) data
|
||||
!:mime application/x-hdf
|
||||
|
||||
# http://www.cartesianinc.com/Tech/
|
||||
0 string CPC\262 Cartesian Perceptual Compression image
|
||||
!:mime image/x-cpi
|
||||
|
||||
|
||||
# Polar Monitor Bitmap (.pmb) used as logo for Polar Electro watches
|
||||
# From: Markus Heidelberg <markus.heidelberg at web.de>
|
||||
0 string/t [BitmapInfo2] Polar Monitor Bitmap text
|
||||
!:mime image/x-polar-monitor-bitmap
|
||||
|
||||
# Type: Olympus ORF raw images.
|
||||
# URL: http://libopenraw.freedesktop.org/wiki/Olympus_ORF
|
||||
# From: Adam Buchbinder <adam.buchbinder@gmail.com>
|
||||
0 string MMOR Olympus ORF raw image data, big-endian
|
||||
!:mime image/x-olympus-orf
|
||||
0 string IIRO Olympus ORF raw image data, little-endian
|
||||
!:mime image/x-olympus-orf
|
||||
0 string IIRS Olympus ORF raw image data, little-endian
|
||||
!:mime image/x-olympus-orf
|
||||
|
||||
# Type: Foveon X3F
|
||||
# URL: http://www.photofo.com/downloads/x3f-raw-format.pdf
|
||||
# From: Adam Buchbinder <adam.buchbinder@gmail.com>
|
||||
# Note that the MIME type isn't defined anywhere that I can find; if
|
||||
# there's a canonical type for this format, it should replace this one.
|
||||
0 string FOVb Foveon X3F raw image data
|
||||
!:mime image/x-x3f
|
||||
|
||||
# Paint.NET file
|
||||
# From Adam Buchbinder <adam.buchbinder@gmail.com>
|
||||
0 string PDN3 Paint.NET image data
|
||||
!:mime image/x-paintnet
|
||||
16
magic/java
Normal file
16
magic/java
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------
|
||||
# $File: java,v 1.13 2011/12/08 12:12:46 rrt Exp $
|
||||
# Java ByteCode and Mach-O binaries (e.g., Mac OS X) use the
|
||||
# same magic number, 0xcafebabe, so they are both handled
|
||||
# in the entry called "cafebabe".
|
||||
#------------------------------------------------------------
|
||||
|
||||
0 belong 0xfeedfeed Java KeyStore
|
||||
!:mime application/x-java-keystore
|
||||
0 belong 0xcececece Java JCE KeyStore
|
||||
!:mime application/x-java-jce-keystore
|
||||
|
||||
# Java source
|
||||
0 regex ^import.*;$ Java source
|
||||
!:mime text/x-java
|
||||
17
magic/javascript
Normal file
17
magic/javascript
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: $
|
||||
# javascript: magic for javascript and node.js scripts.
|
||||
#
|
||||
0 search/1/w #!/bin/node Node.js script text executable
|
||||
!:mime application/javascript
|
||||
0 search/1/w #!/usr/bin/node Node.js script text executable
|
||||
!:mime application/javascript
|
||||
0 search/1/w #!/bin/nodejs Node.js script text executable
|
||||
!:mime application/javascript
|
||||
0 search/1/w #!/usr/bin/nodejs Node.js script text executable
|
||||
!:mime application/javascript
|
||||
0 search/1 #!/usr/bin/env\ node Node.js script text executable
|
||||
!:mime application/javascript
|
||||
0 search/1 #!/usr/bin/env\ nodejs Node.js script text executable
|
||||
!:mime application/javascript
|
||||
31
magic/jpeg
Normal file
31
magic/jpeg
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: jpeg,v 1.18 2012/08/01 12:12:36 christos Exp $
|
||||
# JPEG images
|
||||
# SunOS 5.5.1 had
|
||||
#
|
||||
# 0 string \377\330\377\340 JPEG file
|
||||
# 0 string \377\330\377\356 JPG file
|
||||
#
|
||||
# both of which turn into "JPEG image data" here.
|
||||
#
|
||||
0 beshort 0xffd8 JPEG image data
|
||||
!:mime image/jpeg
|
||||
!:apple 8BIMJPEG
|
||||
!:strength +2
|
||||
|
||||
# From: David Santinoli <david@santinoli.com>
|
||||
0 string \x00\x00\x00\x0C\x6A\x50\x20\x20\x0D\x0A\x87\x0A JPEG 2000
|
||||
# From: Johan van der Knijff <johan.vanderknijff@kb.nl>
|
||||
# Added sub-entries for JP2, JPX, JPM and MJ2 formats; added mimetypes
|
||||
# https://github.com/bitsgalore/jp2kMagic
|
||||
#
|
||||
# Now read value of 'Brand' field, which yields a few possibilities:
|
||||
>20 string \x6a\x70\x32\x20 Part 1 (JP2)
|
||||
!:mime image/jp2
|
||||
>20 string \x6a\x70\x78\x20 Part 2 (JPX)
|
||||
!:mime image/jpx
|
||||
>20 string \x6a\x70\x6d\x20 Part 6 (JPM)
|
||||
!:mime image/jpm
|
||||
>20 string \x6d\x6a\x70\x32 Part 3 (MJ2)
|
||||
!:mime video/mj2
|
||||
11
magic/kde
Normal file
11
magic/kde
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: kde,v 1.4 2009/09/19 16:28:10 christos Exp $
|
||||
# kde: file(1) magic for KDE
|
||||
|
||||
0 string/t [KDE\ Desktop\ Entry] KDE desktop entry
|
||||
!:mime application/x-kdelnk
|
||||
0 string/t #\ KDE\ Config\ File KDE config file
|
||||
!:mime application/x-kdelnk
|
||||
0 string/t #\ xmcd xmcd database file for kscd
|
||||
!:mime text/x-xmcd
|
||||
30
magic/kml
Normal file
30
magic/kml
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: kml,v 1.2 2009/09/19 16:28:10 christos Exp $
|
||||
# Type: Google KML, formerly Keyhole Markup Language
|
||||
# Future development of this format has been handed
|
||||
# over to the Open Geospatial Consortium.
|
||||
# http://www.opengeospatial.org/standards/kml/
|
||||
# From: Asbjoern Sloth Toennesen <asbjorn@lila.io>
|
||||
0 string/t \<?xml
|
||||
>20 search/400 \ xmlns=
|
||||
>>&0 regex ['"]http://earth.google.com/kml Google KML document
|
||||
!:mime application/vnd.google-earth.kml+xml
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Type: OpenGIS KML, formerly Keyhole Markup Language
|
||||
# This standard is maintained by the
|
||||
# Open Geospatial Consortium.
|
||||
# http://www.opengeospatial.org/standards/kml/
|
||||
# From: Asbjoern Sloth Toennesen <asbjorn@lila.io>
|
||||
>>&0 regex ['"]http://www.opengis.net/kml OpenGIS KML document
|
||||
!:mime application/vnd.google-earth.kml+xml
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Type: Google KML Archive (ZIP based)
|
||||
# http://code.google.com/apis/kml/documentation/kml_tut.html
|
||||
# From: Asbjoern Sloth Toennesen <asbjorn@lila.io>
|
||||
0 string PK\003\004
|
||||
>4 byte 0x14
|
||||
>>30 string doc.kml Compressed Google KML Document, including resources.
|
||||
!:mime application/vnd.google-earth.kmz
|
||||
22
magic/linux
Normal file
22
magic/linux
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: linux,v 1.46 2013/01/06 21:26:48 christos Exp $
|
||||
# linux: file(1) magic for Linux files
|
||||
#
|
||||
# Values for Linux/i386 binaries, from Daniel Quinlan <quinlan@yggdrasil.com>
|
||||
# The following basic Linux magic is useful for reference, but using
|
||||
# "long" magic is a better practice in order to avoid collisions.
|
||||
#
|
||||
# 2 leshort 100 Linux/i386
|
||||
# >0 leshort 0407 impure executable (OMAGIC)
|
||||
# >0 leshort 0410 pure executable (NMAGIC)
|
||||
# >0 leshort 0413 demand-paged executable (ZMAGIC)
|
||||
# >0 leshort 0314 demand-paged executable (QMAGIC)
|
||||
#
|
||||
|
||||
# SYSLINUX boot logo files (from 'ppmtolss16' sources)
|
||||
# http://www.syslinux.org/wiki/index.php/SYSLINUX#Display_graphic_from_filename:
|
||||
# file extension .lss .16
|
||||
0 lelong =0x1413f33d SYSLINUX' LSS16 image data
|
||||
# syslinux-4.05/mime/image/x-lss16.xml
|
||||
!:mime image/x-lss16
|
||||
42
magic/lisp
Normal file
42
magic/lisp
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File$
|
||||
# lisp: file(1) magic for lisp programs
|
||||
#
|
||||
# various lisp types, from Daniel Quinlan (quinlan@yggdrasil.com)
|
||||
|
||||
# updated by Joerg Jenderek
|
||||
# GRR: This lot is too weak
|
||||
#0 string ;;
|
||||
# windows INF files often begin with semicolon and use CRLF as line end
|
||||
# lisp files are mainly created on unix system with LF as line end
|
||||
#>2 search/4096 !\r Lisp/Scheme program text
|
||||
#>2 search/4096 \r Windows INF file
|
||||
|
||||
0 search/4096 (setq\ Lisp/Scheme program text
|
||||
!:mime text/x-lisp
|
||||
0 search/4096 (defvar\ Lisp/Scheme program text
|
||||
!:mime text/x-lisp
|
||||
0 search/4096 (defparam\ Lisp/Scheme program text
|
||||
!:mime text/x-lisp
|
||||
0 search/4096 (defun\ Lisp/Scheme program text
|
||||
!:mime text/x-lisp
|
||||
0 search/4096 (autoload\ Lisp/Scheme program text
|
||||
!:mime text/x-lisp
|
||||
0 search/4096 (custom-set-variables\ Lisp/Scheme program text
|
||||
!:mime text/x-lisp
|
||||
|
||||
# Emacs 18 - this is always correct, but not very magical.
|
||||
0 string \012( Emacs v18 byte-compiled Lisp data
|
||||
!:mime application/x-elc
|
||||
# Emacs 19+ - ver. recognition added by Ian Springer
|
||||
# Also applies to XEmacs 19+ .elc files; could tell them apart with regexs
|
||||
# - Chris Chittleborough <cchittleborough@yahoo.com.au>
|
||||
0 string ;ELC
|
||||
>4 byte >18
|
||||
>4 byte <32 Emacs/XEmacs v%d byte-compiled Lisp data
|
||||
!:mime application/x-elc
|
||||
|
||||
# From: David Allouche <david@allouche.net>
|
||||
0 search/1 \<TeXmacs| TeXmacs document text
|
||||
!:mime text/texmacs
|
||||
17
magic/lua
Normal file
17
magic/lua
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: lua,v 1.5 2009/09/19 16:28:10 christos Exp $
|
||||
# lua: file(1) magic for Lua scripting language
|
||||
# URL: http://www.lua.org/
|
||||
# From: Reuben Thomas <rrt@sc3d.org>, Seo Sanghyeon <tinuviel@sparcs.kaist.ac.kr>
|
||||
|
||||
# Lua scripts
|
||||
0 search/1/w #!\ /usr/bin/lua Lua script text executable
|
||||
!:mime text/x-lua
|
||||
0 search/1/w #!\ /usr/local/bin/lua Lua script text executable
|
||||
!:mime text/x-lua
|
||||
0 search/1 #!/usr/bin/env\ lua Lua script text executable
|
||||
!:mime text/x-lua
|
||||
0 search/1 #!\ /usr/bin/env\ lua Lua script text executable
|
||||
!:mime text/x-lua
|
||||
|
||||
7
magic/m4
Normal file
7
magic/m4
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File$
|
||||
# make: file(1) magic for M4 scripts
|
||||
#
|
||||
0 regex \^dnl\ M4 macro processor script text
|
||||
!:mime text/x-m4
|
||||
21
magic/macintosh
Normal file
21
magic/macintosh
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: macintosh,v 1.21 2010/09/20 19:19:17 rrt Exp $
|
||||
# macintosh description
|
||||
#
|
||||
# BinHex is the Macintosh ASCII-encoded file format (see also "apple")
|
||||
# Daniel Quinlan, quinlan@yggdrasil.com
|
||||
11 string must\ be\ converted\ with\ BinHex BinHex binary text
|
||||
!:mime application/mac-binhex40
|
||||
|
||||
# Stuffit archives are the de facto standard of compression for Macintosh
|
||||
# files obtained from most archives. (franklsm@tuns.ca)
|
||||
0 string SIT! StuffIt Archive (data)
|
||||
!:mime application/x-stuffit
|
||||
!:apple SIT!SIT!
|
||||
|
||||
# Newer StuffIt archives (grant@netbsd.org)
|
||||
0 string StuffIt StuffIt Archive
|
||||
!:mime application/x-stuffit
|
||||
!:apple SIT!SIT!
|
||||
#>162 string >0 : %s
|
||||
35
magic/mail.news
Normal file
35
magic/mail.news
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: mail.news,v 1.21 2012/06/21 01:44:52 christos Exp $
|
||||
# mail.news: file(1) magic for mail and news
|
||||
#
|
||||
# Unfortunately, saved netnews also has From line added in some news software.
|
||||
#0 string From mail text
|
||||
0 string/t Relay-Version: old news text
|
||||
!:mime message/rfc822
|
||||
0 string/t #!\ rnews batched news text
|
||||
!:mime message/rfc822
|
||||
0 string/t N#!\ rnews mailed, batched news text
|
||||
!:mime message/rfc822
|
||||
0 string/t Forward\ to mail forwarding text
|
||||
!:mime message/rfc822
|
||||
0 string/t Pipe\ to mail piping text
|
||||
!:mime message/rfc822
|
||||
0 string/tc delivered-to: SMTP mail text
|
||||
!:mime message/rfc822
|
||||
0 string/tc return-path: SMTP mail text
|
||||
!:mime message/rfc822
|
||||
0 string/t Path: news text
|
||||
!:mime message/news
|
||||
0 string/t Xref: news text
|
||||
!:mime message/news
|
||||
0 string/t From: news or mail text
|
||||
!:mime message/rfc822
|
||||
0 string/t Article saved news text
|
||||
!:mime message/news
|
||||
0 string/t Received: RFC 822 mail text
|
||||
!:mime message/rfc822
|
||||
|
||||
# TNEF files...
|
||||
0 lelong 0x223E9F78 Transport Neutral Encapsulation Format
|
||||
!:mime application/vnd.ms-tnef
|
||||
16
magic/make
Normal file
16
magic/make
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File$
|
||||
# make: file(1) magic for makefiles
|
||||
#
|
||||
0 regex \^CFLAGS makefile script text
|
||||
!:mime text/x-makefile
|
||||
0 regex \^LDFLAGS makefile script text
|
||||
!:mime text/x-makefile
|
||||
0 regex \^all: makefile script text
|
||||
!:mime text/x-makefile
|
||||
0 regex \^.PRECIOUS makefile script text
|
||||
!:mime text/x-makefile
|
||||
|
||||
0 regex \^SUBDIRS automake makefile script text
|
||||
!:mime text/x-makefile
|
||||
29
magic/marc21
Normal file
29
magic/marc21
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#--------------------------------------------
|
||||
# marc21: file(1) magic for MARC 21 Format
|
||||
#
|
||||
# Kevin Ford (kefo@loc.gov)
|
||||
#
|
||||
# MARC21 formats are for the representation and communication
|
||||
# of bibliographic and related information in machine-readable
|
||||
# form. For more info, see http://www.loc.gov/marc/
|
||||
|
||||
|
||||
# leader position 20-21 must be 45
|
||||
20 string 45
|
||||
|
||||
# leader starts with 5 digits, followed by codes specific to MARC format
|
||||
>0 regex/1 (^[0-9]{5})[acdnp][^bhlnqsu-z] MARC21 Bibliographic
|
||||
!:mime application/marc
|
||||
>0 regex/1 (^[0-9]{5})[acdnosx][z] MARC21 Authority
|
||||
!:mime application/marc
|
||||
>0 regex/1 (^[0-9]{5})[cdn][uvxy] MARC21 Holdings
|
||||
!:mime application/marc
|
||||
0 regex/1 (^[0-9]{5})[acdn][w] MARC21 Classification
|
||||
!:mime application/marc
|
||||
>0 regex/1 (^[0-9]{5})[cdn][q] MARC21 Community
|
||||
!:mime application/marc
|
||||
|
||||
# leader position 22-23, should be "00" but is it?
|
||||
>0 regex/1 (^.{21})([^0]{2}) (non-conforming)
|
||||
!:mime application/marc
|
||||
17
magic/matroska
Normal file
17
magic/matroska
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: matroska,v 1.7 2012/08/26 10:06:15 christos Exp $
|
||||
# matroska: file(1) magic for Matroska files
|
||||
#
|
||||
# See http://www.matroska.org/
|
||||
#
|
||||
|
||||
# EBML id:
|
||||
0 belong 0x1a45dfa3
|
||||
# DocType id:
|
||||
>4 search/4096 \x42\x82
|
||||
# DocType contents:
|
||||
>>&1 string webm WebM
|
||||
!:mime video/webm
|
||||
>>&1 string matroska Matroska data
|
||||
!:mime video/x-matroska
|
||||
9
magic/misctools
Normal file
9
magic/misctools
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#-----------------------------------------------------------------------------
|
||||
# $File: misctools,v 1.12 2010/09/29 18:36:49 rrt Exp $
|
||||
# misctools: file(1) magic for miscellaneous UNIX tools.
|
||||
#
|
||||
0 string/c BEGIN:VCALENDAR vCalendar calendar file
|
||||
!:mime text/calendar
|
||||
0 string/c BEGIN:VCARD vCard visiting card
|
||||
!:mime text/x-vcard
|
||||
368
magic/msdos
Normal file
368
magic/msdos
Normal file
|
|
@ -0,0 +1,368 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: msdos,v 1.84 2013/02/05 13:55:22 christos Exp $
|
||||
# msdos: file(1) magic for MS-DOS files
|
||||
#
|
||||
|
||||
# .BAT files (Daniel Quinlan, quinlan@yggdrasil.com)
|
||||
# updated by Joerg Jenderek at Oct 2008,Apr 2011
|
||||
0 string/t @
|
||||
>1 string/cW \ echo\ off DOS batch file text
|
||||
!:mime text/x-msdos-batch
|
||||
>1 string/cW echo\ off DOS batch file text
|
||||
!:mime text/x-msdos-batch
|
||||
>1 string/cW rem DOS batch file text
|
||||
!:mime text/x-msdos-batch
|
||||
>1 string/cW set\ DOS batch file text
|
||||
!:mime text/x-msdos-batch
|
||||
|
||||
# Tests for various EXE types.
|
||||
#
|
||||
# Many of the compressed formats were extraced from IDARC 1.23 source code.
|
||||
#
|
||||
0 string/b MZ DOS MZ
|
||||
!:mime application/x-dosexec
|
||||
# All non-DOS EXE extensions have the relocation table more than 0x40 bytes into the file.
|
||||
>0x18 leshort <0x40 MS-DOS executable
|
||||
# These traditional tests usually work but not always. When test quality support is
|
||||
# implemented these can be turned on.
|
||||
#>>0x18 leshort 0x1c (Borland compiler)
|
||||
#>>0x18 leshort 0x1e (MS compiler)
|
||||
|
||||
# If the relocation table is 0x40 or more bytes into the file, it's definitely
|
||||
# not a DOS EXE.
|
||||
>0x18 leshort >0x3f
|
||||
|
||||
# Maybe it's a PE?
|
||||
>>(0x3c.l) string PE\0\0 PE
|
||||
>>>(0x3c.l+24) leshort 0x010b \b32 executable
|
||||
>>>(0x3c.l+24) leshort 0x020b \b32+ executable
|
||||
>>>(0x3c.l+24) leshort 0x0107 ROM image
|
||||
>>>(0x3c.l+24) default x Unknown PE signature
|
||||
>>>>&0 leshort x 0x%x
|
||||
>>>(0x3c.l+22) leshort&0x2000 >0 (DLL)
|
||||
>>>(0x3c.l+92) leshort 1 (native)
|
||||
>>>(0x3c.l+92) leshort 2 (GUI)
|
||||
>>>(0x3c.l+92) leshort 3 (console)
|
||||
>>>(0x3c.l+92) leshort 7 (POSIX)
|
||||
>>>(0x3c.l+92) leshort 9 (Windows CE)
|
||||
>>>(0x3c.l+92) leshort 10 (EFI application)
|
||||
>>>(0x3c.l+92) leshort 11 (EFI boot service driver)
|
||||
>>>(0x3c.l+92) leshort 12 (EFI runtime driver)
|
||||
>>>(0x3c.l+92) leshort 13 (EFI ROM)
|
||||
>>>(0x3c.l+92) leshort 14 (XBOX)
|
||||
>>>(0x3c.l+92) leshort 15 (Windows boot application)
|
||||
>>>(0x3c.l+92) default x (Unknown subsystem
|
||||
>>>>&0 leshort x 0x%x)
|
||||
>>>(0x3c.l+4) leshort 0x14c Intel 80386
|
||||
>>>(0x3c.l+4) leshort 0x166 MIPS R4000
|
||||
>>>(0x3c.l+4) leshort 0x168 MIPS R10000
|
||||
>>>(0x3c.l+4) leshort 0x184 Alpha
|
||||
>>>(0x3c.l+4) leshort 0x1a2 Hitachi SH3
|
||||
>>>(0x3c.l+4) leshort 0x1a6 Hitachi SH4
|
||||
>>>(0x3c.l+4) leshort 0x1c0 ARM
|
||||
>>>(0x3c.l+4) leshort 0x1c2 ARM Thumb
|
||||
>>>(0x3c.l+4) leshort 0x1c4 ARMv7 Thumb
|
||||
>>>(0x3c.l+4) leshort 0x1f0 PowerPC
|
||||
>>>(0x3c.l+4) leshort 0x200 Intel Itanium
|
||||
>>>(0x3c.l+4) leshort 0x266 MIPS16
|
||||
>>>(0x3c.l+4) leshort 0x268 Motorola 68000
|
||||
>>>(0x3c.l+4) leshort 0x290 PA-RISC
|
||||
>>>(0x3c.l+4) leshort 0x366 MIPSIV
|
||||
>>>(0x3c.l+4) leshort 0x466 MIPS16 with FPU
|
||||
>>>(0x3c.l+4) leshort 0xebc EFI byte code
|
||||
>>>(0x3c.l+4) leshort 0x8664 x86-64
|
||||
>>>(0x3c.l+4) leshort 0xc0ee MSIL
|
||||
>>>(0x3c.l+4) default x Unknown processor type
|
||||
>>>>&0 leshort x 0x%x
|
||||
>>>(0x3c.l+22) leshort&0x0200 >0 (stripped to external PDB)
|
||||
>>>(0x3c.l+22) leshort&0x1000 >0 system file
|
||||
>>>(0x3c.l+24) leshort 0x010b
|
||||
>>>>(0x3c.l+232) lelong >0 Mono/.Net assembly
|
||||
>>>(0x3c.l+24) leshort 0x020b
|
||||
>>>>(0x3c.l+248) lelong >0 Mono/.Net assembly
|
||||
|
||||
# hooray, there's a DOS extender using the PE format, with a valid PE
|
||||
# executable inside (which just prints a message and exits if run in win)
|
||||
>>>(8.s*16) string 32STUB \b, 32rtm DOS extender
|
||||
>>>(8.s*16) string !32STUB \b, for MS Windows
|
||||
>>>(0x3c.l+0xf8) string UPX0 \b, UPX compressed
|
||||
>>>(0x3c.l+0xf8) search/0x140 PEC2 \b, PECompact2 compressed
|
||||
>>>(0x3c.l+0xf8) search/0x140 UPX2
|
||||
>>>>(&0x10.l+(-4)) string PK\3\4 \b, ZIP self-extracting archive (Info-Zip)
|
||||
>>>(0x3c.l+0xf8) search/0x140 .idata
|
||||
>>>>(&0xe.l+(-4)) string PK\3\4 \b, ZIP self-extracting archive (Info-Zip)
|
||||
>>>>(&0xe.l+(-4)) string ZZ0 \b, ZZip self-extracting archive
|
||||
>>>>(&0xe.l+(-4)) string ZZ1 \b, ZZip self-extracting archive
|
||||
>>>(0x3c.l+0xf8) search/0x140 .rsrc
|
||||
>>>>(&0x0f.l+(-4)) string a\\\4\5 \b, WinHKI self-extracting archive
|
||||
>>>>(&0x0f.l+(-4)) string Rar! \b, RAR self-extracting archive
|
||||
>>>>(&0x0f.l+(-4)) search/0x3000 MSCF \b, InstallShield self-extracting archive
|
||||
>>>>(&0x0f.l+(-4)) search/32 Nullsoft \b, Nullsoft Installer self-extracting archive
|
||||
>>>(0x3c.l+0xf8) search/0x140 .data
|
||||
>>>>(&0x0f.l) string WEXTRACT \b, MS CAB-Installer self-extracting archive
|
||||
>>>(0x3c.l+0xf8) search/0x140 .petite\0 \b, Petite compressed
|
||||
>>>>(0x3c.l+0xf7) byte x
|
||||
>>>>>(&0x104.l+(-4)) string =!sfx! \b, ACE self-extracting archive
|
||||
>>>(0x3c.l+0xf8) search/0x140 .WISE \b, WISE installer self-extracting archive
|
||||
>>>(0x3c.l+0xf8) search/0x140 .dz\0\0\0 \b, Dzip self-extracting archive
|
||||
>>>&(0x3c.l+0xf8) search/0x100 _winzip_ \b, ZIP self-extracting archive (WinZip)
|
||||
>>>&(0x3c.l+0xf8) search/0x100 SharedD \b, Microsoft Installer self-extracting archive
|
||||
>>>0x30 string Inno \b, InnoSetup self-extracting archive
|
||||
|
||||
# Hmm, not a PE but the relocation table is too high for a traditional DOS exe,
|
||||
# must be one of the unusual subformats.
|
||||
>>(0x3c.l) string !PE\0\0 MS-DOS executable
|
||||
|
||||
>>(0x3c.l) string NE \b, NE
|
||||
>>>(0x3c.l+0x36) byte 1 for OS/2 1.x
|
||||
>>>(0x3c.l+0x36) byte 2 for MS Windows 3.x
|
||||
>>>(0x3c.l+0x36) byte 3 for MS-DOS
|
||||
>>>(0x3c.l+0x36) byte 4 for Windows 386
|
||||
>>>(0x3c.l+0x36) byte 5 for Borland Operating System Services
|
||||
>>>(0x3c.l+0x36) default x
|
||||
>>>>(0x3c.l+0x36) byte x (unknown OS %x)
|
||||
>>>(0x3c.l+0x36) byte 0x81 for MS-DOS, Phar Lap DOS extender
|
||||
>>>(0x3c.l+0x0c) leshort&0x8003 0x8002 (DLL)
|
||||
>>>(0x3c.l+0x0c) leshort&0x8003 0x8001 (driver)
|
||||
>>>&(&0x24.s-1) string ARJSFX \b, ARJ self-extracting archive
|
||||
>>>(0x3c.l+0x70) search/0x80 WinZip(R)\ Self-Extractor \b, ZIP self-extracting archive (WinZip)
|
||||
|
||||
>>(0x3c.l) string LX\0\0 \b, LX
|
||||
>>>(0x3c.l+0x0a) leshort <1 (unknown OS)
|
||||
>>>(0x3c.l+0x0a) leshort 1 for OS/2
|
||||
>>>(0x3c.l+0x0a) leshort 2 for MS Windows
|
||||
>>>(0x3c.l+0x0a) leshort 3 for DOS
|
||||
>>>(0x3c.l+0x0a) leshort >3 (unknown OS)
|
||||
>>>(0x3c.l+0x10) lelong&0x28000 =0x8000 (DLL)
|
||||
>>>(0x3c.l+0x10) lelong&0x20000 >0 (device driver)
|
||||
>>>(0x3c.l+0x10) lelong&0x300 0x300 (GUI)
|
||||
>>>(0x3c.l+0x10) lelong&0x28300 <0x300 (console)
|
||||
>>>(0x3c.l+0x08) leshort 1 i80286
|
||||
>>>(0x3c.l+0x08) leshort 2 i80386
|
||||
>>>(0x3c.l+0x08) leshort 3 i80486
|
||||
>>>(8.s*16) string emx \b, emx
|
||||
>>>>&1 string x %s
|
||||
>>>&(&0x54.l-3) string arjsfx \b, ARJ self-extracting archive
|
||||
|
||||
# MS Windows system file, supposedly a collection of LE executables
|
||||
>>(0x3c.l) string W3 \b, W3 for MS Windows
|
||||
|
||||
>>(0x3c.l) string LE\0\0 \b, LE executable
|
||||
>>>(0x3c.l+0x0a) leshort 1
|
||||
# some DOS extenders use LE files with OS/2 header
|
||||
>>>>0x240 search/0x100 DOS/4G for MS-DOS, DOS4GW DOS extender
|
||||
>>>>0x240 search/0x200 WATCOM\ C/C++ for MS-DOS, DOS4GW DOS extender
|
||||
>>>>0x440 search/0x100 CauseWay\ DOS\ Extender for MS-DOS, CauseWay DOS extender
|
||||
>>>>0x40 search/0x40 PMODE/W for MS-DOS, PMODE/W DOS extender
|
||||
>>>>0x40 search/0x40 STUB/32A for MS-DOS, DOS/32A DOS extender (stub)
|
||||
>>>>0x40 search/0x80 STUB/32C for MS-DOS, DOS/32A DOS extender (configurable stub)
|
||||
>>>>0x40 search/0x80 DOS/32A for MS-DOS, DOS/32A DOS extender (embedded)
|
||||
# this is a wild guess; hopefully it is a specific signature
|
||||
>>>>&0x24 lelong <0x50
|
||||
>>>>>(&0x4c.l) string \xfc\xb8WATCOM
|
||||
>>>>>>&0 search/8 3\xdbf\xb9 \b, 32Lite compressed
|
||||
# another wild guess: if real OS/2 LE executables exist, they probably have higher start EIP
|
||||
#>>>>(0x3c.l+0x1c) lelong >0x10000 for OS/2
|
||||
# fails with DOS-Extenders.
|
||||
>>>(0x3c.l+0x0a) leshort 2 for MS Windows
|
||||
>>>(0x3c.l+0x0a) leshort 3 for DOS
|
||||
>>>(0x3c.l+0x0a) leshort 4 for MS Windows (VxD)
|
||||
>>>(&0x7c.l+0x26) string UPX \b, UPX compressed
|
||||
>>>&(&0x54.l-3) string UNACE \b, ACE self-extracting archive
|
||||
|
||||
# looks like ASCII, probably some embedded copyright message.
|
||||
# and definitely not NE/LE/LX/PE
|
||||
>>0x3c lelong >0x20000000
|
||||
>>>(4.s*512) leshort !0x014c \b, MZ for MS-DOS
|
||||
# header data too small for extended executable
|
||||
>2 long !0
|
||||
>>0x18 leshort <0x40
|
||||
>>>(4.s*512) leshort !0x014c
|
||||
|
||||
>>>>&(2.s-514) string !LE
|
||||
>>>>>&-2 string !BW \b, MZ for MS-DOS
|
||||
>>>>&(2.s-514) string LE \b, LE
|
||||
>>>>>0x240 search/0x100 DOS/4G for MS-DOS, DOS4GW DOS extender
|
||||
# educated guess since indirection is still not capable enough for complex offset
|
||||
# calculations (next embedded executable would be at &(&2*512+&0-2)
|
||||
# I suspect there are only LE executables in these multi-exe files
|
||||
>>>>&(2.s-514) string BW
|
||||
>>>>>0x240 search/0x100 DOS/4G ,\b LE for MS-DOS, DOS4GW DOS extender (embedded)
|
||||
>>>>>0x240 search/0x100 !DOS/4G ,\b BW collection for MS-DOS
|
||||
|
||||
# This sequence skips to the first COFF segment, usually .text
|
||||
>(4.s*512) leshort 0x014c \b, COFF
|
||||
>>(8.s*16) string go32stub for MS-DOS, DJGPP go32 DOS extender
|
||||
>>(8.s*16) string emx
|
||||
>>>&1 string x for DOS, Win or OS/2, emx %s
|
||||
>>&(&0x42.l-3) byte x
|
||||
>>>&0x26 string UPX \b, UPX compressed
|
||||
# and yet another guess: small .text, and after large .data is unusal, could be 32lite
|
||||
>>&0x2c search/0xa0 .text
|
||||
>>>&0x0b lelong <0x2000
|
||||
>>>>&0 lelong >0x6000 \b, 32lite compressed
|
||||
|
||||
>(8.s*16) string $WdX \b, WDos/X DOS extender
|
||||
|
||||
# By now an executable type should have been printed out. The executable
|
||||
# may be a self-uncompressing archive, so look for evidence of that and
|
||||
# print it out.
|
||||
#
|
||||
# Some signatures below from Greg Roelofs, newt@uchicago.edu.
|
||||
#
|
||||
>0x35 string \x8e\xc0\xb9\x08\x00\xf3\xa5\x4a\x75\xeb\x8e\xc3\x8e\xd8\x33\xff\xbe\x30\x00\x05 \b, aPack compressed
|
||||
>0xe7 string LH/2\ Self-Extract \b, %s
|
||||
>0x1c string UC2X \b, UCEXE compressed
|
||||
>0x1c string WWP\ \b, WWPACK compressed
|
||||
>0x1c string RJSX \b, ARJ self-extracting archive
|
||||
>0x1c string diet \b, diet compressed
|
||||
>0x1c string LZ09 \b, LZEXE v0.90 compressed
|
||||
>0x1c string LZ91 \b, LZEXE v0.91 compressed
|
||||
>0x1c string tz \b, TinyProg compressed
|
||||
>0x1e string Copyright\ 1989-1990\ PKWARE\ Inc. Self-extracting PKZIP archive
|
||||
!:mime application/zip
|
||||
# Yes, this really is "Copr", not "Corp."
|
||||
>0x1e string PKLITE\ Copr. Self-extracting PKZIP archive
|
||||
!:mime application/zip
|
||||
# winarj stores a message in the stub instead of the sig in the MZ header
|
||||
>0x20 search/0xe0 aRJsfX \b, ARJ self-extracting archive
|
||||
>0x20 string AIN
|
||||
>>0x23 string 2 \b, AIN 2.x compressed
|
||||
>>0x23 string <2 \b, AIN 1.x compressed
|
||||
>>0x23 string >2 \b, AIN 1.x compressed
|
||||
>0x24 string LHa's\ SFX \b, LHa self-extracting archive
|
||||
!:mime application/x-lha
|
||||
>0x24 string LHA's\ SFX \b, LHa self-extracting archive
|
||||
!:mime application/x-lha
|
||||
>0x24 string \ $ARX \b, ARX self-extracting archive
|
||||
>0x24 string \ $LHarc \b, LHarc self-extracting archive
|
||||
>0x20 string SFX\ by\ LARC \b, LARC self-extracting archive
|
||||
>0x40 string aPKG \b, aPackage self-extracting archive
|
||||
>0x64 string W\ Collis\0\0 \b, Compack compressed
|
||||
>0x7a string Windows\ self-extracting\ ZIP \b, ZIP self-extracting archive
|
||||
>>&0xf4 search/0x140 \x0\x40\x1\x0
|
||||
>>>(&0.l+(4)) string MSCF \b, WinHKI CAB self-extracting archive
|
||||
>1638 string -lh5- \b, LHa self-extracting archive v2.13S
|
||||
>0x17888 string Rar! \b, RAR self-extracting archive
|
||||
|
||||
# Skip to the end of the EXE. This will usually work fine in the PE case
|
||||
# because the MZ image is hardcoded into the toolchain and almost certainly
|
||||
# won't match any of these signatures.
|
||||
>(4.s*512) long x
|
||||
>>&(2.s-517) byte x
|
||||
>>>&0 string PK\3\4 \b, ZIP self-extracting archive
|
||||
>>>&0 string Rar! \b, RAR self-extracting archive
|
||||
>>>&0 string =!\x11 \b, AIN 2.x self-extracting archive
|
||||
>>>&0 string =!\x12 \b, AIN 2.x self-extracting archive
|
||||
>>>&0 string =!\x17 \b, AIN 1.x self-extracting archive
|
||||
>>>&0 string =!\x18 \b, AIN 1.x self-extracting archive
|
||||
>>>&7 search/400 **ACE** \b, ACE self-extracting archive
|
||||
>>>&0 search/0x480 UC2SFX\ Header \b, UC2 self-extracting archive
|
||||
|
||||
# a few unknown ZIP sfxes, no idea if they are needed or if they are
|
||||
# already captured by the generic patterns above
|
||||
>(8.s*16) search/0x20 PKSFX \b, ZIP self-extracting archive (PKZIP)
|
||||
# TODO: how to add this? >FileSize-34 string Windows\ Self-Installing\ Executable \b, ZIP self-extracting archive
|
||||
#
|
||||
|
||||
# TELVOX Teleinformatica CODEC self-extractor for OS/2:
|
||||
>49801 string \x79\xff\x80\xff\x76\xff \b, CODEC archive v3.21
|
||||
>>49824 leshort =1 \b, 1 file
|
||||
>>49824 leshort >1 \b, %u files
|
||||
|
||||
# Popular applications
|
||||
2080 string Microsoft\ Word\ 6.0\ Document %s
|
||||
!:mime application/msword
|
||||
2080 string Documento\ Microsoft\ Word\ 6 Spanish Microsoft Word 6 document data
|
||||
!:mime application/msword
|
||||
# Pawel Wiecek <coven@i17linuxb.ists.pwr.wroc.pl> (for polish Word)
|
||||
2112 string MSWordDoc Microsoft Word document data
|
||||
!:mime application/msword
|
||||
#
|
||||
0 belong 0x31be0000 Microsoft Word Document
|
||||
!:mime application/msword
|
||||
#
|
||||
0 string/b PO^Q` Microsoft Word 6.0 Document
|
||||
!:mime application/msword
|
||||
#
|
||||
0 string/b \376\067\0\043 Microsoft Office Document
|
||||
!:mime application/msword
|
||||
0 string/b \333\245-\0\0\0 Microsoft Office Document
|
||||
!:mime application/msword
|
||||
512 string/b \354\245\301 Microsoft Word Document
|
||||
!:mime application/msword
|
||||
|
||||
#
|
||||
0 string/b \xDB\xA5\x2D\x00 Microsoft WinWord 2.0 Document
|
||||
!:mime application/msword
|
||||
#
|
||||
2080 string Microsoft\ Excel\ 5.0\ Worksheet %s
|
||||
!:mime application/vnd.ms-excel
|
||||
#
|
||||
0 string/b \xDB\xA5\x2D\x00 Microsoft WinWord 2.0 Document
|
||||
!:mime application/msword
|
||||
|
||||
2080 string Foglio\ di\ lavoro\ Microsoft\ Exce %s
|
||||
!:mime application/vnd.ms-excel
|
||||
#
|
||||
# Pawel Wiecek <coven@i17linuxb.ists.pwr.wroc.pl> (for polish Excel)
|
||||
2114 string Biff5 Microsoft Excel 5.0 Worksheet
|
||||
!:mime application/vnd.ms-excel
|
||||
# Italian MS-Excel
|
||||
2121 string Biff5 Microsoft Excel 5.0 Worksheet
|
||||
!:mime application/vnd.ms-excel
|
||||
0 string/b \x09\x04\x06\x00\x00\x00\x10\x00 Microsoft Excel Worksheet
|
||||
!:mime application/vnd.ms-excel
|
||||
#
|
||||
0 belong 0x00001a00 Lotus 1-2-3
|
||||
!:mime application/x-123
|
||||
#
|
||||
0 belong 0x00000200 Lotus 1-2-3
|
||||
!:mime application/x-123
|
||||
0 string/b WordPro\0 Lotus WordPro
|
||||
!:mime application/vnd.lotus-wordpro
|
||||
0 string/b WordPro\r\373 Lotus WordPro
|
||||
!:mime application/vnd.lotus-wordpro
|
||||
|
||||
# Windows icons (Ian Springer <ips@fpk.hp.com>)
|
||||
0 string/b \000\000\001\000 MS Windows icon resource
|
||||
!:mime image/x-icon
|
||||
|
||||
# .PIF files added by Joerg Jenderek from http://smsoft.ru/en/pifdoc.htm
|
||||
# only for windows versions equal or greater 3.0
|
||||
0x171 string MICROSOFT\ PIFEX\0 Windows Program Information File
|
||||
!:mime application/x-dosexec
|
||||
|
||||
# TNEF magic From "Joomy" <joomy@se-ed.net>
|
||||
# Microsoft Outlook's Transport Neutral Encapsulation Format (TNEF)
|
||||
0 leshort 0x223e9f78 TNEF
|
||||
!:mime application/vnd.ms-tnef
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# From Stuart Caie <kyzer@4u.net> (developer of cabextract)
|
||||
# Microsoft Cabinet files
|
||||
0 string/b MSCF\0\0\0\0 Microsoft Cabinet archive data
|
||||
!:mime application/vnd.ms-cab-compressed
|
||||
|
||||
# from http://filext.com by Derek M Jones <derek@knosof.co.uk>
|
||||
# False positive with PPT (also currently this string is too long)
|
||||
#0 string/b \xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x3E\x00\x03\x00\xFE\xFF\x09\x00\x06 Microsoft Installer
|
||||
0 string/b \320\317\021\340\241\261\032\341 Microsoft Office Document
|
||||
#>48 byte 0x1B Excel Document
|
||||
#!:mime application/vnd.ms-excel
|
||||
>546 string bjbj Microsoft Word Document
|
||||
!:mime application/msword
|
||||
>546 string jbjb Microsoft Word Document
|
||||
!:mime application/msword
|
||||
|
||||
0 string/b \224\246\056 Microsoft Word Document
|
||||
!:mime application/msword
|
||||
|
||||
512 string R\0o\0o\0t\0\ \0E\0n\0t\0r\0y Microsoft Word Document
|
||||
!:mime application/msword
|
||||
|
||||
# MS eBook format (.lit)
|
||||
0 string/b ITOLITLS Microsoft Reader eBook Data
|
||||
>8 lelong x \b, version %u
|
||||
!:mime application/x-ms-reader
|
||||
12
magic/neko
Normal file
12
magic/neko
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------
|
||||
# $File: java,v 1.12 2009/09/19 16:28:10 christos Exp $
|
||||
|
||||
# From: Mikhail Gusarov <dottedmag@dottedmag.net>
|
||||
# NekoVM (http://nekovm.org/) bytecode
|
||||
0 string NEKO NekoVM bytecode
|
||||
>4 lelong x (%d global symbols,
|
||||
>8 lelong x %d global fields,
|
||||
>12 lelong x %d bytecode ops)
|
||||
!:mime application/x-nekovm-bytecode
|
||||
|
||||
11
magic/pascal
Normal file
11
magic/pascal
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File$
|
||||
# pascal: file(1) magic for Pascal source
|
||||
#
|
||||
0 search/8192 (input, Pascal source text
|
||||
!:mime text/x-pascal
|
||||
0 regex \^program Pascal source text
|
||||
!:mime text/x-pascal
|
||||
0 regex \^record Pascal source text
|
||||
!:mime text/x-pascal
|
||||
8
magic/pdf
Normal file
8
magic/pdf
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File$
|
||||
# pdf: file(1) magic for Portable Document Format
|
||||
#
|
||||
|
||||
0 string %PDF- PDF document
|
||||
!:mime application/pdf
|
||||
26
magic/perl
Normal file
26
magic/perl
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: perl,v 1.19 2012/06/20 21:16:25 christos Exp $
|
||||
# perl: file(1) magic for Larry Wall's perl language.
|
||||
#
|
||||
# The `eval' lines recognizes an outrageously clever hack.
|
||||
# Keith Waclena <keith@cerberus.uchicago.edu>
|
||||
# Send additions to <perl5-porters@perl.org>
|
||||
0 search/1/w #!\ /bin/perl Perl script text executable
|
||||
!:mime text/x-perl
|
||||
0 search/1 eval\ "exec\ /bin/perl Perl script text
|
||||
!:mime text/x-perl
|
||||
0 search/1/w #!\ /usr/bin/perl Perl script text executable
|
||||
!:mime text/x-perl
|
||||
0 search/1 eval\ "exec\ /usr/bin/perl Perl script text
|
||||
!:mime text/x-perl
|
||||
0 search/1/w #!\ /usr/local/bin/perl Perl script text executable
|
||||
!:mime text/x-perl
|
||||
0 search/1 eval\ "exec\ /usr/local/bin/perl Perl script text
|
||||
!:mime text/x-perl
|
||||
0 search/1 eval\ '(exit\ $?0)'\ &&\ eval\ 'exec Perl script text
|
||||
!:mime text/x-perl
|
||||
0 search/1 #!/usr/bin/env\ perl Perl script text executable
|
||||
!:mime text/x-perl
|
||||
0 search/1 #!\ /usr/bin/env\ perl Perl script text executable
|
||||
!:mime text/x-perl
|
||||
27
magic/pgp
Normal file
27
magic/pgp
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File$
|
||||
# pgp: file(1) magic for Pretty Good Privacy
|
||||
# see http://lists.gnupg.org/pipermail/gnupg-devel/1999-September/016052.html
|
||||
#
|
||||
0 beshort 0x9900 PGP key public ring
|
||||
!:mime application/x-pgp-keyring
|
||||
0 beshort 0x9501 PGP key security ring
|
||||
!:mime application/x-pgp-keyring
|
||||
0 beshort 0x9500 PGP key security ring
|
||||
!:mime application/x-pgp-keyring
|
||||
0 beshort 0xa600 PGP encrypted data
|
||||
#!:mime application/pgp-encrypted
|
||||
#0 string -----BEGIN\040PGP text/PGP armored data
|
||||
!:mime text/PGP # encoding: armored data
|
||||
#>15 string PUBLIC\040KEY\040BLOCK- public key block
|
||||
#>15 string MESSAGE- message
|
||||
#>15 string SIGNED\040MESSAGE- signed message
|
||||
#>15 string PGP\040SIGNATURE- signature
|
||||
|
||||
2 string ---BEGIN\ PGP\ PUBLIC\ KEY\ BLOCK- PGP public key block
|
||||
!:mime application/pgp-keys
|
||||
0 string -----BEGIN\040PGP\40MESSAGE- PGP message
|
||||
!:mime application/pgp
|
||||
0 string -----BEGIN\040PGP\40SIGNATURE- PGP signature
|
||||
!:mime application/pgp-signature
|
||||
7
magic/pkgadd
Normal file
7
magic/pkgadd
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File$
|
||||
# pkgadd: file(1) magic for SysV R4 PKG Datastreams
|
||||
#
|
||||
0 string #\ PaCkAgE\ DaTaStReAm pkg Datastream (SVR4)
|
||||
!:mime application/x-svr4-package
|
||||
14
magic/printer
Normal file
14
magic/printer
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: printer,v 1.24 2011/05/08 16:34:51 christos Exp $
|
||||
# printer: file(1) magic for printer-formatted files
|
||||
#
|
||||
|
||||
# PostScript, updated by Daniel Quinlan (quinlan@yggdrasil.com)
|
||||
0 string %! PostScript document text
|
||||
!:mime application/postscript
|
||||
!:apple ASPSTEXT
|
||||
# Some PCs have the annoying habit of adding a ^D as a document separator
|
||||
0 string \004%! PostScript document text
|
||||
!:mime application/postscript
|
||||
!:apple ASPSTEXT
|
||||
46
magic/python
Normal file
46
magic/python
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: python,v 1.21 2012/06/21 01:12:51 christos Exp $
|
||||
# python: file(1) magic for python
|
||||
#
|
||||
|
||||
0 search/1/w #!\ /usr/bin/python Python script text executable
|
||||
!:mime text/x-python
|
||||
0 search/1/w #!\ /usr/local/bin/python Python script text executable
|
||||
!:mime text/x-python
|
||||
0 search/1 #!/usr/bin/env\ python Python script text executable
|
||||
!:mime text/x-python
|
||||
0 search/1 #!\ /usr/bin/env\ python Python script text executable
|
||||
!:mime text/x-python
|
||||
|
||||
# from module.submodule import func1, func2
|
||||
0 regex \^from\\s+(\\w|\\.)+\\s+import.*$ Python script text executable
|
||||
!:mime text/x-python
|
||||
|
||||
# def __init__ (self, ...):
|
||||
0 search/4096 def\ __init__
|
||||
>&0 search/64 self Python script text executable
|
||||
!:mime text/x-python
|
||||
|
||||
# comments
|
||||
0 search/4096 '''
|
||||
>&0 regex .*'''$ Python script text executable
|
||||
!:mime text/x-python
|
||||
|
||||
0 search/4096 """
|
||||
>&0 regex .*"""$ Python script text executable
|
||||
!:mime text/x-python
|
||||
|
||||
# try:
|
||||
# except: or finally:
|
||||
# block
|
||||
0 search/4096 try:
|
||||
>&0 regex \^\\s*except.*: Python script text executable
|
||||
!:mime text/x-python
|
||||
>&0 search/4096 finally: Python script text executable
|
||||
!:mime text/x-python
|
||||
|
||||
# def name(args, args):
|
||||
0 regex \^(\ |\\t)*def\ +[a-zA-Z]+
|
||||
>&0 regex \ *\\(([a-zA-Z]|,|\ )*\\):$ Python script text executable
|
||||
!:mime text/x-python
|
||||
36
magic/riff
Normal file
36
magic/riff
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: riff,v 1.22 2011/09/06 11:00:06 christos Exp $
|
||||
# riff: file(1) magic for RIFF format
|
||||
# See
|
||||
#
|
||||
# http://www.seanet.com/users/matts/riffmci/riffmci.htm
|
||||
#
|
||||
# AVI section extended by Patrik Radman <patrik+file-magic@iki.fi>
|
||||
#
|
||||
0 string RIFF RIFF (little-endian) data
|
||||
# Microsoft WAVE format (*.wav)
|
||||
>8 string WAVE \b, WAVE audio
|
||||
!:mime audio/x-wav
|
||||
# Corel Draw Picture
|
||||
>8 string CDRA \b, Corel Draw Picture
|
||||
!:mime image/x-coreldraw
|
||||
# AVI == Audio Video Interleave
|
||||
>8 string AVI\040 \b, AVI
|
||||
!:mime video/x-msvideo
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Sony Wave64
|
||||
# see http://www.vcs.de/fileadmin/user_upload/MBS/PDF/Whitepaper/Informations_about_Sony_Wave64.pdf
|
||||
# 128 bit RIFF-GUID { 66666972-912E-11CF-A5D6-28DB04C10000 } in little-endian
|
||||
0 string riff\x2E\x91\xCF\x11\xA5\xD6\x28\xDB\x04\xC1\x00\x00 Sony Wave64 RIFF data
|
||||
# 128 bit + total file size (64 bits) so 24 bytes
|
||||
# then WAVE-GUID { 65766177-ACF3-11D3-8CD1-00C04F8EDB8A }
|
||||
>24 string wave\xF3\xAC\xD3\x11\x8C\xD1\x00\xC0\x4F\x8E\xDB\x8A \b, WAVE 64 audio
|
||||
!:mime audio/x-w64
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# MBWF/RF64
|
||||
# see EBU TECH 3306 http://tech.ebu.ch/docs/tech/tech3306-2009.pdf
|
||||
0 string RF64\xff\xff\xff\xffWAVEds64 MBWF/RF64 audio
|
||||
!:mime audio/x-wav
|
||||
12
magic/rpm
Normal file
12
magic/rpm
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: rpm,v 1.11 2011/06/14 12:47:41 christos Exp $
|
||||
#
|
||||
# RPM: file(1) magic for Red Hat Packages Erik Troan (ewt@redhat.com)
|
||||
#
|
||||
0 belong 0xedabeedb RPM
|
||||
!:mime application/x-rpm
|
||||
|
||||
#delta RPM Daniel Novotny (dnovotny@redhat.com)
|
||||
0 string drpm Delta RPM
|
||||
!:mime application/x-rpm
|
||||
9
magic/rtf
Normal file
9
magic/rtf
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File$
|
||||
# rtf: file(1) magic for Rich Text Format (RTF)
|
||||
#
|
||||
# Duncan P. Simpson, D.P.Simpson@dcs.warwick.ac.uk
|
||||
#
|
||||
0 string {\\rtf Rich Text Format data,
|
||||
!:mime text/rtf
|
||||
28
magic/ruby
Normal file
28
magic/ruby
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: ruby,v 1.4 2010/07/08 20:24:13 christos Exp $
|
||||
# ruby: file(1) magic for Ruby scripting language
|
||||
# URL: http://www.ruby-lang.org/
|
||||
# From: Reuben Thomas <rrt@sc3d.org>
|
||||
|
||||
# Ruby scripts
|
||||
0 search/1/w #!\ /usr/bin/ruby Ruby script text executable
|
||||
!:mime text/x-ruby
|
||||
0 search/1/w #!\ /usr/local/bin/ruby Ruby script text executable
|
||||
!:mime text/x-ruby
|
||||
0 search/1 #!/usr/bin/env\ ruby Ruby script text executable
|
||||
!:mime text/x-ruby
|
||||
0 search/1 #!\ /usr/bin/env\ ruby Ruby script text executable
|
||||
!:mime text/x-ruby
|
||||
|
||||
# What looks like ruby, but does not have a shebang
|
||||
# (modules and such)
|
||||
# From: Lubomir Rintel <lkundrak@v3.sk>
|
||||
0 regex \^[\ \t]*require[\ \t]'[A-Za-z_/]+'
|
||||
>0 regex include\ [A-Z]|def\ [a-z]|\ do$
|
||||
>>0 regex \^[\ \t]*end([\ \t]*[;#].*)?$ Ruby script text
|
||||
!:mime text/x-ruby
|
||||
0 regex \^[\ \t]*(class|module)[\ \t][A-Z]
|
||||
>0 regex (modul|includ)e\ [A-Z]|def\ [a-z]
|
||||
>>0 regex \^[\ \t]*end([\ \t]*[;#].*)?$ Ruby module source text
|
||||
!:mime text/x-ruby
|
||||
7
magic/sc
Normal file
7
magic/sc
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File$
|
||||
# sc: file(1) magic for "sc" spreadsheet
|
||||
#
|
||||
38 string Spreadsheet sc spreadsheet file
|
||||
!:mime application/x-sc
|
||||
82
magic/sgml
Normal file
82
magic/sgml
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: sgml,v 1.28 2012/04/28 21:20:26 christos Exp $
|
||||
# Type: SVG Vectorial Graphics
|
||||
# From: Noel Torres <tecnico@ejerciciosresueltos.com>
|
||||
0 string \<?xml\ version="
|
||||
>15 string >\0
|
||||
>>19 search/4096 \<svg SVG Scalable Vector Graphics image
|
||||
!:mime image/svg+xml
|
||||
>>19 search/4096 \<gnc-v2 GnuCash file
|
||||
!:mime application/x-gnucash
|
||||
|
||||
# Sitemap file
|
||||
0 string/t \<?xml\ version="
|
||||
>15 string >\0
|
||||
>>19 search/4096 \<urlset XML Sitemap document text
|
||||
!:mime application/xml-sitemap
|
||||
|
||||
# xhtml
|
||||
0 string/t \<?xml\ version="
|
||||
>15 string >\0
|
||||
>>19 search/4096/cWbt \<!doctype\ html XHTML document text
|
||||
!:mime text/html
|
||||
0 string/t \<?xml\ version='
|
||||
>15 string >\0
|
||||
>>19 search/4096/cWbt \<!doctype\ html XHTML document text
|
||||
!:mime text/html
|
||||
0 string/t \<?xml\ version="
|
||||
>15 string >\0
|
||||
>>19 search/4096/cWbt \<html broken XHTML document text
|
||||
!:mime text/html
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# sgml: file(1) magic for Standard Generalized Markup Language
|
||||
# HyperText Markup Language (HTML) is an SGML document type,
|
||||
# from Daniel Quinlan (quinlan@yggdrasil.com)
|
||||
# adapted to string extenstions by Anthon van der Neut <anthon@mnt.org)
|
||||
0 search/4096/cWt \<!doctype\ html HTML document text
|
||||
!:mime text/html
|
||||
!:strength + 5
|
||||
0 search/4096/cwt \<head HTML document text
|
||||
!:mime text/html
|
||||
!:strength + 5
|
||||
0 search/4096/cwt \<title HTML document text
|
||||
!:mime text/html
|
||||
!:strength + 5
|
||||
0 search/4096/cwt \<html HTML document text
|
||||
!:mime text/html
|
||||
!:strength + 5
|
||||
0 search/4096/cwt \<script HTML document text
|
||||
!:mime text/html
|
||||
!:strength + 5
|
||||
0 search/4096/cwt \<style HTML document text
|
||||
!:mime text/html
|
||||
!:strength + 5
|
||||
0 search/4096/cwt \<table HTML document text
|
||||
!:mime text/html
|
||||
!:strength + 5
|
||||
0 search/4096/cwt \<a\ href= HTML document text
|
||||
!:mime text/html
|
||||
!:strength + 5
|
||||
|
||||
# Extensible markup language (XML), a subset of SGML
|
||||
# from Marc Prud'hommeaux (marc@apocalypse.org)
|
||||
0 search/1/cwt \<?xml XML document text
|
||||
!:mime application/xml
|
||||
!:strength + 5
|
||||
0 string/t \<?xml\ version\ " XML
|
||||
!:mime application/xml
|
||||
!:strength + 5
|
||||
0 string/t \<?xml\ version=" XML
|
||||
!:mime application/xml
|
||||
!:strength + 5
|
||||
0 string \<?xml\ version=' XML
|
||||
!:mime application/xml
|
||||
!:strength + 5
|
||||
0 search/1/wbt \<?xml XML document text
|
||||
!:mime application/xml
|
||||
!:strength - 10
|
||||
0 search/1/wt \<?XML broken XML document text
|
||||
!:mime application/xml
|
||||
!:strength - 10
|
||||
17
magic/sniffer
Normal file
17
magic/sniffer
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# sniffer: file(1) magic for packet capture files
|
||||
#
|
||||
# From: guy@alum.mit.edu (Guy Harris)
|
||||
#
|
||||
|
||||
#
|
||||
# "libpcap" capture files.
|
||||
# (We call them "tcpdump capture file(s)" for now, as "tcpdump" is
|
||||
# the main program that uses that format, but there are other programs
|
||||
# that use "libpcap", or that use the same capture file format.)
|
||||
#
|
||||
0 ubelong 0xa1b2c3d4 tcpdump capture file (big-endian)
|
||||
!:mime application/vnd.tcpdump.pcap
|
||||
0 ulelong 0xa1b2c3d4 tcpdump capture file (little-endian)
|
||||
!:mime application/vnd.tcpdump.pcap
|
||||
23
magic/tcl
Normal file
23
magic/tcl
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# file: file(1) magic for Tcl scripting language
|
||||
# URL: http://www.tcl.tk/
|
||||
# From: gustaf neumann
|
||||
|
||||
# Tcl scripts
|
||||
0 search/1/w #!\ /usr/bin/tcl Tcl script text executable
|
||||
!:mime text/x-lua
|
||||
0 search/1/w #!\ /usr/local/bin/tcl Tcl script text executable
|
||||
!:mime text/x-tcl
|
||||
0 search/1 #!/usr/bin/env\ tcl Tcl script text executable
|
||||
!:mime text/x-tcl
|
||||
0 search/1 #!\ /usr/bin/env\ tcl Tcl script text executable
|
||||
!:mime text/x-tcl
|
||||
0 search/1/w #!\ /usr/bin/wish Tcl/Tk script text executable
|
||||
!:mime text/x-tcl
|
||||
0 search/1/w #!\ /usr/local/bin/wish Tcl/Tk script text executable
|
||||
!:mime text/x-tcl
|
||||
0 search/1 #!/usr/bin/env\ wish Tcl/Tk script text executable
|
||||
!:mime text/x-tcl
|
||||
0 search/1 #!\ /usr/bin/env\ wish Tcl/Tk script text executable
|
||||
!:mime text/x-tcl
|
||||
56
magic/tex
Normal file
56
magic/tex
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: tex,v 1.17 2010/09/20 19:19:17 rrt Exp $
|
||||
# tex: file(1) magic for TeX files
|
||||
#
|
||||
# XXX - needs byte-endian stuff (big-endian and little-endian DVI?)
|
||||
#
|
||||
# From <conklin@talisman.kaleida.com>
|
||||
|
||||
# Although we may know the offset of certain text fields in TeX DVI
|
||||
# and font files, we can't use them reliably because they are not
|
||||
# zero terminated. [but we do anyway, christos]
|
||||
0 string \367\002 TeX DVI file
|
||||
!:mime application/x-dvi
|
||||
|
||||
# There is no way to detect TeX Font Metric (*.tfm) files without
|
||||
# breaking them apart and reading the data. The following patterns
|
||||
# match most *.tfm files generated by METAFONT or afm2tfm.
|
||||
2 string \000\021 TeX font metric data
|
||||
!:mime application/x-tex-tfm
|
||||
2 string \000\022 TeX font metric data
|
||||
!:mime application/x-tex-tfm
|
||||
|
||||
# Texinfo and GNU Info, from Daniel Quinlan (quinlan@yggdrasil.com)
|
||||
0 search/1 \\input\ texinfo Texinfo source text
|
||||
!:mime text/x-texinfo
|
||||
0 search/1 This\ is\ Info\ file GNU Info text
|
||||
!:mime text/x-info
|
||||
|
||||
# TeX documents, from Daniel Quinlan (quinlan@yggdrasil.com)
|
||||
0 search/4096 \\input TeX document text
|
||||
!:mime text/x-tex
|
||||
!:strength + 15
|
||||
0 search/4096 \\section LaTeX document text
|
||||
!:mime text/x-tex
|
||||
!:strength + 18
|
||||
0 search/4096 \\setlength LaTeX document text
|
||||
!:mime text/x-tex
|
||||
!:strength + 15
|
||||
0 search/4096 \\documentstyle LaTeX document text
|
||||
!:mime text/x-tex
|
||||
!:strength + 18
|
||||
0 search/4096 \\chapter LaTeX document text
|
||||
!:mime text/x-tex
|
||||
!:strength + 18
|
||||
0 search/4096 \\documentclass LaTeX 2e document text
|
||||
!:mime text/x-tex
|
||||
!:strength + 15
|
||||
0 search/4096 \\relax LaTeX auxiliary file
|
||||
!:mime text/x-tex
|
||||
!:strength + 15
|
||||
0 search/4096 \\contentsline LaTeX table of contents
|
||||
!:mime text/x-tex
|
||||
!:strength + 15
|
||||
0 search/4096 %\ -*-latex-*- LaTeX document text
|
||||
!:mime text/x-tex
|
||||
22
magic/troff
Normal file
22
magic/troff
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File$
|
||||
# troff: file(1) magic for *roff
|
||||
#
|
||||
# updated by Daniel Quinlan (quinlan@yggdrasil.com)
|
||||
|
||||
# troff input
|
||||
0 search/1 .\\" troff or preprocessor input text
|
||||
!:mime text/troff
|
||||
0 search/1 '\\" troff or preprocessor input text
|
||||
!:mime text/troff
|
||||
0 search/1 '.\\" troff or preprocessor input text
|
||||
!:mime text/troff
|
||||
0 search/1 \\" troff or preprocessor input text
|
||||
!:mime text/troff
|
||||
0 search/1 ''' troff or preprocessor input text
|
||||
!:mime text/troff
|
||||
0 regex/20 \^\\.[A-Za-z0-9][A-Za-z0-9][\ \t] troff or preprocessor input text
|
||||
!:mime text/troff
|
||||
0 regex/20 \^\\.[A-Za-z0-9][A-Za-z0-9]$ troff or preprocessor input text
|
||||
!:mime text/troff
|
||||
26
magic/vorbis
Normal file
26
magic/vorbis
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File$
|
||||
# vorbis: file(1) magic for Ogg/Vorbis files
|
||||
#
|
||||
# From Felix von Leitner <leitner@fefe.de>
|
||||
# Extended by Beni Cherniavsky <cben@crosswinds.net>
|
||||
# Further extended by Greg Wooledge <greg@wooledge.org>
|
||||
#
|
||||
# Most (everything but the number of channels and bitrate) is commented
|
||||
# out with `##' as it's not interesting to the average user. The most
|
||||
# probable things advanced users would want to uncomment are probably
|
||||
# the number of comments and the encoder version.
|
||||
#
|
||||
# FIXME: The first match has been made a search, so that it can skip
|
||||
# over prepended ID3 tags. This will work for MIME type detection, but
|
||||
# won't work for detecting other properties of the file (they all need
|
||||
# to be made relative to the search). In any case, if the file has ID3
|
||||
# tags, the ID3 information will be printed, not the Ogg information,
|
||||
# so until that's fixed, this doesn't matter.
|
||||
# FIXME[2]: Disable the above for now, since search assumes text mode.
|
||||
#
|
||||
# --- Ogg Framing ---
|
||||
#0 search/1000 OggS Ogg data
|
||||
0 string OggS Ogg data
|
||||
!:mime application/ogg
|
||||
14
magic/warc
Normal file
14
magic/warc
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: warc,v 1.2 2009/09/19 16:28:13 christos Exp $
|
||||
# warc: file(1) magic for WARC files
|
||||
|
||||
0 string WARC/ WARC Archive
|
||||
>5 string x version %.4s
|
||||
!:mime application/warc
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Arc File Format from Internet Archive
|
||||
# see http://www.archive.org/web/researcher/ArcFileFormat.php
|
||||
0 string filedesc:// Internet Archive File
|
||||
!:mime application/x-ia-arc
|
||||
19
magic/windows
Normal file
19
magic/windows
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: windows,v 1.4 2009/09/19 16:28:13 christos Exp $
|
||||
# windows: file(1) magic for Microsoft Windows
|
||||
#
|
||||
# This file is mainly reserved for files where programs
|
||||
# using them are run almost always on MS Windows 3.x or
|
||||
# above, or files only used exclusively in Windows OS,
|
||||
# where there is no better category to allocate for.
|
||||
# For example, even though WinZIP almost run on Windows
|
||||
# only, it is better to treat them as "archive" instead.
|
||||
# For format usable in DOS, such as generic executable
|
||||
# format, please specify under "msdos" file.
|
||||
#
|
||||
|
||||
# From: Pal Tamas <folti@balabit.hu>
|
||||
# Autorun File
|
||||
0 string/c [autorun]\r\n Microsoft Windows Autorun file.
|
||||
!:mime application/x-setupscript.
|
||||
43
magic/wordprocessors
Normal file
43
magic/wordprocessors
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: wordprocessors,v 1.16 2012/10/29 17:36:49 christos Exp $
|
||||
# wordprocessors: file(1) magic fo word processors.
|
||||
#
|
||||
|
||||
# Hangul (Korean) Word Processor File
|
||||
# From: Won-Kyu Park <wkpark@kldp.org>
|
||||
512 string R\0o\0o\0t\0 Hangul (Korean) Word Processor File 2000
|
||||
!:mime application/x-hwp
|
||||
|
||||
# Quark Express from http://www.garykessler.net/library/file_sigs.html
|
||||
2 string MMXPR3 Motorola Quark Express Document (English)
|
||||
!:mime application/x-quark-xpress-3
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# ichitaro456: file(1) magic for Just System Word Processor Ichitaro
|
||||
#
|
||||
# Contributor kenzo-:
|
||||
# Reversed-engineered JS Ichitaro magic numbers
|
||||
#
|
||||
|
||||
0 string DOC
|
||||
>43 byte 0x14 Just System Word Processor Ichitaro v4
|
||||
!:mime application/x-ichitaro4
|
||||
|
||||
0 string DOC
|
||||
>43 byte 0x15 Just System Word Processor Ichitaro v5
|
||||
!:mime application/x-ichitaro5
|
||||
|
||||
0 string DOC
|
||||
>43 byte 0x16 Just System Word Processor Ichitaro v6
|
||||
!:mime application/x-ichitaro6
|
||||
|
||||
# Type: Freemind mindmap documents
|
||||
# From: Jamie Thompson <debian-bugs@jamie-thompson.co.uk>
|
||||
0 string/w \<map\ version Freemind document
|
||||
!:mime application/x-freemind
|
||||
|
||||
# Type: Scribus
|
||||
# From: Werner Fink <werner@suse.de>
|
||||
0 string \<SCRIBUSUTF8NEW\ Version Scribus Document
|
||||
!:mime application/x-scribus
|
||||
11
magic/xwindows
Normal file
11
magic/xwindows
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
# See COPYING file in this directory for original libmagic copyright.
|
||||
#------------------------------------------------------------------------------
|
||||
# $File: xwindows,v 1.7 2011/05/03 01:44:17 christos Exp $
|
||||
# xwindows: file(1) magic for various X/Window system file formats.
|
||||
|
||||
# Xcursor data
|
||||
# X11 mouse cursor format defined in libXcursor, see
|
||||
# http://www.x.org/archive/X11R6.8.1/doc/Xcursor.3.html
|
||||
# http://cgit.freedesktop.org/xorg/lib/libXcursor/tree/include/X11/Xcursor/Xcursor.h
|
||||
0 string Xcur Xcursor data
|
||||
!:mime image/x-xcursor
|
||||
1
scripts/base/frameworks/file-analysis/__load__.bro
Normal file
1
scripts/base/frameworks/file-analysis/__load__.bro
Normal file
|
|
@ -0,0 +1 @@
|
|||
@load ./main.bro
|
||||
351
scripts/base/frameworks/file-analysis/main.bro
Normal file
351
scripts/base/frameworks/file-analysis/main.bro
Normal file
|
|
@ -0,0 +1,351 @@
|
|||
##! An interface for driving the analysis of files, possibly independent of
|
||||
##! any network protocol over which they're transported.
|
||||
|
||||
@load base/file_analysis.bif
|
||||
@load base/frameworks/logging
|
||||
|
||||
module FileAnalysis;
|
||||
|
||||
export {
|
||||
redef enum Log::ID += {
|
||||
## Logging stream for file analysis.
|
||||
LOG
|
||||
};
|
||||
|
||||
## A structure which represents a desired type of file analysis.
|
||||
type AnalyzerArgs: record {
|
||||
## The type of analysis.
|
||||
tag: Analyzer;
|
||||
|
||||
## The local filename to which to write an extracted file. Must be
|
||||
## set when *tag* is :bro:see:`FileAnalysis::ANALYZER_EXTRACT`.
|
||||
extract_filename: string &optional;
|
||||
|
||||
## An event which will be generated for all new file contents,
|
||||
## chunk-wise.
|
||||
chunk_event: event(f: fa_file, data: string, off: count) &optional;
|
||||
|
||||
## An event which will be generated for all new file contents,
|
||||
## stream-wise.
|
||||
stream_event: event(f: fa_file, data: string) &optional;
|
||||
} &redef;
|
||||
|
||||
## Contains all metadata related to the analysis of a given file.
|
||||
## For the most part, fields here are derived from ones of the same name
|
||||
## in :bro:see:`fa_file`.
|
||||
type Info: record {
|
||||
## An identifier associated with a single file.
|
||||
id: string &log;
|
||||
|
||||
## Identifier associated with a container file from which this one was
|
||||
## extracted as part of the file analysis.
|
||||
parent_id: string &log &optional;
|
||||
|
||||
## An identification of the source of the file data. E.g. it may be
|
||||
## a network protocol over which it was transferred, or a local file
|
||||
## path which was read, or some other input source.
|
||||
source: string &log &optional;
|
||||
|
||||
## If the source of this file is is a network connection, this field
|
||||
## may be set to indicate the directionality.
|
||||
is_orig: bool &log &optional;
|
||||
|
||||
## The time at which the last activity for the file was seen.
|
||||
last_active: time &log;
|
||||
|
||||
## Number of bytes provided to the file analysis engine for the file.
|
||||
seen_bytes: count &log &default=0;
|
||||
|
||||
## Total number of bytes that are supposed to comprise the full file.
|
||||
total_bytes: count &log &optional;
|
||||
|
||||
## The number of bytes in the file stream that were completely missed
|
||||
## during the process of analysis e.g. due to dropped packets.
|
||||
missing_bytes: count &log &default=0;
|
||||
|
||||
## The number of not all-in-sequence bytes in the file stream that
|
||||
## were delivered to file analyzers due to reassembly buffer overflow.
|
||||
overflow_bytes: count &log &default=0;
|
||||
|
||||
## The amount of time between receiving new data for this file that
|
||||
## the analysis engine will wait before giving up on it.
|
||||
timeout_interval: interval &log &optional;
|
||||
|
||||
## The number of bytes at the beginning of a file to save for later
|
||||
## inspection in *bof_buffer* field.
|
||||
bof_buffer_size: count &log &optional;
|
||||
|
||||
## A mime type provided by libmagic against the *bof_buffer*, or
|
||||
## in the cases where no buffering of the beginning of file occurs,
|
||||
## an initial guess of the mime type based on the first data seen.
|
||||
mime_type: string &log &optional;
|
||||
|
||||
## Whether the file analysis timed out at least once for the file.
|
||||
timedout: bool &log &default=F;
|
||||
|
||||
## Connection UIDS over which the file was transferred.
|
||||
conn_uids: set[string] &log;
|
||||
|
||||
## A set of analysis types done during the file analysis.
|
||||
analyzers: set[Analyzer] &log;
|
||||
|
||||
## Local filenames of extracted files.
|
||||
extracted_files: set[string] &log;
|
||||
|
||||
## An MD5 digest of the file contents.
|
||||
md5: string &log &optional;
|
||||
|
||||
## A SHA1 digest of the file contents.
|
||||
sha1: string &log &optional;
|
||||
|
||||
## A SHA256 digest of the file contents.
|
||||
sha256: string &log &optional;
|
||||
} &redef;
|
||||
|
||||
## A table that can be used to disable file analysis completely for
|
||||
## any files transferred over given network protocol analyzers.
|
||||
const disable: table[AnalyzerTag] of bool = table() &redef;
|
||||
|
||||
## Event that can be handled to access the Info record as it is sent on
|
||||
## to the logging framework.
|
||||
global log_file_analysis: event(rec: Info);
|
||||
|
||||
## The salt concatenated to unique file handle strings generated by
|
||||
## :bro:see:`get_file_handle` before hashing them in to a file id
|
||||
## (the *id* field of :bro:see:`fa_file`).
|
||||
## Provided to help mitigate the possiblility of manipulating parts of
|
||||
## network connections that factor in to the file handle in order to
|
||||
## generate two handles that would hash to the same file id.
|
||||
const salt = "I recommend changing this." &redef;
|
||||
|
||||
## Sets the *timeout_interval* field of :bro:see:`fa_file`, which is
|
||||
## used to determine the length of inactivity that is allowed for a file
|
||||
## before internal state related to it is cleaned up.
|
||||
##
|
||||
## f: the file.
|
||||
##
|
||||
## t: the amount of time the file can remain inactive before discarding.
|
||||
##
|
||||
## Returns: true if the timeout interval was set, or false if analysis
|
||||
## for the *id* isn't currently active.
|
||||
global set_timeout_interval: function(f: fa_file, t: interval): bool;
|
||||
|
||||
## Postpones the timeout of file analysis for a given file.
|
||||
## When used within a :bro:see:`file_timeout` handler for, the analysis
|
||||
## the analysis will delay timing out for the period of time indicated by
|
||||
## the *timeout_interval* field of :bro:see:`fa_file`, which can be set
|
||||
## with :bro:see:`FileAnalysis::set_timeout_interval`.
|
||||
##
|
||||
## f: the file.
|
||||
##
|
||||
## Returns: true if the timeout will be postponed, or false if analysis
|
||||
## for the *id* isn't currently active.
|
||||
global postpone_timeout: function(f: fa_file): bool;
|
||||
|
||||
## Adds an analyzer to the analysis of a given file.
|
||||
##
|
||||
## f: the file.
|
||||
##
|
||||
## args: the analyzer type to add along with any arguments it takes.
|
||||
##
|
||||
## Returns: true if the analyzer will be added, or false if analysis
|
||||
## for the *id* isn't currently active or the *args*
|
||||
## were invalid for the analyzer type.
|
||||
global add_analyzer: function(f: fa_file, args: AnalyzerArgs): bool;
|
||||
|
||||
## Removes an analyzer from the analysis of a given file.
|
||||
##
|
||||
## f: the file.
|
||||
##
|
||||
## args: the analyzer (type and args) to remove.
|
||||
##
|
||||
## Returns: true if the analyzer will be removed, or false if analysis
|
||||
## for the *id* isn't currently active.
|
||||
global remove_analyzer: function(f: fa_file, args: AnalyzerArgs): bool;
|
||||
|
||||
## Stops/ignores any further analysis of a given file.
|
||||
##
|
||||
## f: the file.
|
||||
##
|
||||
## Returns: true if analysis for the given file will be ignored for the
|
||||
## rest of it's contents, or false if analysis for the *id*
|
||||
## isn't currently active.
|
||||
global stop: function(f: fa_file): bool;
|
||||
|
||||
## Sends a sequential stream of data in for file analysis.
|
||||
## Meant for use when providing external file analysis input (e.g.
|
||||
## from the input framework).
|
||||
##
|
||||
## source: a string that uniquely identifies the logical file that the
|
||||
## data is a part of and describes its source.
|
||||
##
|
||||
## data: bytestring contents of the file to analyze.
|
||||
global data_stream: function(source: string, data: string);
|
||||
|
||||
## Sends a non-sequential chunk of data in for file analysis.
|
||||
## Meant for use when providing external file analysis input (e.g.
|
||||
## from the input framework).
|
||||
##
|
||||
## source: a string that uniquely identifies the logical file that the
|
||||
## data is a part of and describes its source.
|
||||
##
|
||||
## data: bytestring contents of the file to analyze.
|
||||
##
|
||||
## offset: the offset within the file that this chunk starts.
|
||||
global data_chunk: function(source: string, data: string, offset: count);
|
||||
|
||||
## Signals a content gap in the file bytestream.
|
||||
## Meant for use when providing external file analysis input (e.g.
|
||||
## from the input framework).
|
||||
##
|
||||
## source: a string that uniquely identifies the logical file that the
|
||||
## data is a part of and describes its source.
|
||||
##
|
||||
## offset: the offset within the file that this gap starts.
|
||||
##
|
||||
## len: the number of bytes that are missing.
|
||||
global gap: function(source: string, offset: count, len: count);
|
||||
|
||||
## Signals the total size of a file.
|
||||
## Meant for use when providing external file analysis input (e.g.
|
||||
## from the input framework).
|
||||
##
|
||||
## source: a string that uniquely identifies the logical file that the
|
||||
## data is a part of and describes its source.
|
||||
##
|
||||
## size: the number of bytes that comprise the full file.
|
||||
global set_size: function(source: string, size: count);
|
||||
|
||||
## Signals the end of a file.
|
||||
## Meant for use when providing external file analysis input (e.g.
|
||||
## from the input framework).
|
||||
##
|
||||
## source: a string that uniquely identifies the logical file that the
|
||||
## data is a part of and describes its source.
|
||||
global eof: function(source: string);
|
||||
}
|
||||
|
||||
redef record fa_file += {
|
||||
info: Info &optional;
|
||||
};
|
||||
|
||||
function set_info(f: fa_file)
|
||||
{
|
||||
if ( ! f?$info )
|
||||
{
|
||||
local tmp: Info;
|
||||
f$info = tmp;
|
||||
}
|
||||
|
||||
f$info$id = f$id;
|
||||
if ( f?$parent_id ) f$info$parent_id = f$parent_id;
|
||||
if ( f?$source ) f$info$source = f$source;
|
||||
if ( f?$is_orig ) f$info$is_orig = f$is_orig;
|
||||
f$info$last_active = f$last_active;
|
||||
f$info$seen_bytes = f$seen_bytes;
|
||||
if ( f?$total_bytes ) f$info$total_bytes = f$total_bytes;
|
||||
f$info$missing_bytes = f$missing_bytes;
|
||||
f$info$overflow_bytes = f$overflow_bytes;
|
||||
f$info$timeout_interval = f$timeout_interval;
|
||||
f$info$bof_buffer_size = f$bof_buffer_size;
|
||||
if ( f?$mime_type ) f$info$mime_type = f$mime_type;
|
||||
if ( f?$conns )
|
||||
for ( cid in f$conns )
|
||||
add f$info$conn_uids[f$conns[cid]$uid];
|
||||
}
|
||||
|
||||
function set_timeout_interval(f: fa_file, t: interval): bool
|
||||
{
|
||||
return __set_timeout_interval(f$id, t);
|
||||
}
|
||||
|
||||
function postpone_timeout(f: fa_file): bool
|
||||
{
|
||||
return __postpone_timeout(f$id);
|
||||
}
|
||||
|
||||
function add_analyzer(f: fa_file, args: AnalyzerArgs): bool
|
||||
{
|
||||
if ( ! __add_analyzer(f$id, args) ) return F;
|
||||
|
||||
set_info(f);
|
||||
add f$info$analyzers[args$tag];
|
||||
|
||||
if ( args$tag == FileAnalysis::ANALYZER_EXTRACT )
|
||||
add f$info$extracted_files[args$extract_filename];
|
||||
|
||||
return T;
|
||||
}
|
||||
|
||||
function remove_analyzer(f: fa_file, args: AnalyzerArgs): bool
|
||||
{
|
||||
return __remove_analyzer(f$id, args);
|
||||
}
|
||||
|
||||
function stop(f: fa_file): bool
|
||||
{
|
||||
return __stop(f$id);
|
||||
}
|
||||
|
||||
function data_stream(source: string, data: string)
|
||||
{
|
||||
__data_stream(source, data);
|
||||
}
|
||||
|
||||
function data_chunk(source: string, data: string, offset: count)
|
||||
{
|
||||
__data_chunk(source, data, offset);
|
||||
}
|
||||
|
||||
function gap(source: string, offset: count, len: count)
|
||||
{
|
||||
__gap(source, offset, len);
|
||||
}
|
||||
|
||||
function set_size(source: string, size: count)
|
||||
{
|
||||
__set_size(source, size);
|
||||
}
|
||||
|
||||
function eof(source: string)
|
||||
{
|
||||
__eof(source);
|
||||
}
|
||||
|
||||
event bro_init() &priority=5
|
||||
{
|
||||
Log::create_stream(FileAnalysis::LOG,
|
||||
[$columns=Info, $ev=log_file_analysis]);
|
||||
}
|
||||
|
||||
event file_timeout(f: fa_file) &priority=5
|
||||
{
|
||||
set_info(f);
|
||||
f$info$timedout = T;
|
||||
}
|
||||
|
||||
event file_hash(f: fa_file, kind: string, hash: string) &priority=5
|
||||
{
|
||||
set_info(f);
|
||||
switch ( kind ) {
|
||||
case "md5":
|
||||
f$info$md5 = hash;
|
||||
break;
|
||||
case "sha1":
|
||||
f$info$sha1 = hash;
|
||||
break;
|
||||
case "sha256":
|
||||
f$info$sha256 = hash;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
event file_state_remove(f: fa_file) &priority=5
|
||||
{
|
||||
set_info(f);
|
||||
}
|
||||
|
||||
event file_state_remove(f: fa_file) &priority=-5
|
||||
{
|
||||
Log::write(FileAnalysis::LOG, f$info);
|
||||
}
|
||||
|
|
@ -2,4 +2,5 @@
|
|||
@load ./readers/ascii
|
||||
@load ./readers/raw
|
||||
@load ./readers/benchmark
|
||||
|
||||
@load ./readers/binary
|
||||
@load ./readers/sqlite
|
||||
|
|
|
|||
8
scripts/base/frameworks/input/readers/binary.bro
Normal file
8
scripts/base/frameworks/input/readers/binary.bro
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
##! Interface for the binary input reader.
|
||||
|
||||
module InputBinary;
|
||||
|
||||
export {
|
||||
## Size of data chunks to read from the input file at a time.
|
||||
const chunk_size = 1024 &redef;
|
||||
}
|
||||
17
scripts/base/frameworks/input/readers/sqlite.bro
Normal file
17
scripts/base/frameworks/input/readers/sqlite.bro
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
##! Interface for the SQLite input reader.
|
||||
##!
|
||||
##! The defaults are set to match Bro's ASCII output.
|
||||
|
||||
module InputSQLite;
|
||||
|
||||
export {
|
||||
## Separator between set elements.
|
||||
## Please note that the separator has to be exactly one character long.
|
||||
const set_separator = Input::set_separator &redef;
|
||||
|
||||
## String to use for an unset &optional field.
|
||||
const unset_field = Input::unset_field &redef;
|
||||
|
||||
## String to use for empty fields.
|
||||
const empty_field = Input::empty_field &redef;
|
||||
}
|
||||
|
|
@ -2,5 +2,6 @@
|
|||
@load ./postprocessors
|
||||
@load ./writers/ascii
|
||||
@load ./writers/dataseries
|
||||
@load ./writers/sqlite
|
||||
@load ./writers/elasticsearch
|
||||
@load ./writers/none
|
||||
|
|
|
|||
|
|
@ -189,6 +189,15 @@ export {
|
|||
## .. bro:see:: Log::add_default_filter Log::remove_default_filter
|
||||
global create_stream: function(id: ID, stream: Stream) : bool;
|
||||
|
||||
## Removes a logging stream completely, stopping all the threads.
|
||||
##
|
||||
## id: The ID enum to be associated with the new logging stream.
|
||||
##
|
||||
## Returns: True if a new stream was successfully removed.
|
||||
##
|
||||
## .. bro:see:: Log::create_stream
|
||||
global remove_stream: function(id: ID) : bool;
|
||||
|
||||
## Enables a previously disabled logging stream. Disabled streams
|
||||
## will not be written to until they are enabled again. New streams
|
||||
## are enabled by default.
|
||||
|
|
@ -442,6 +451,12 @@ function create_stream(id: ID, stream: Stream) : bool
|
|||
return add_default_filter(id);
|
||||
}
|
||||
|
||||
function remove_stream(id: ID) : bool
|
||||
{
|
||||
delete active_streams[id];
|
||||
return __remove_stream(id);
|
||||
}
|
||||
|
||||
function disable_stream(id: ID) : bool
|
||||
{
|
||||
delete active_streams[id];
|
||||
|
|
|
|||
17
scripts/base/frameworks/logging/writers/sqlite.bro
Normal file
17
scripts/base/frameworks/logging/writers/sqlite.bro
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
##! Interface for the SQLite log writer. Redefinable options are available
|
||||
##! to tweak the output format of the SQLite reader.
|
||||
|
||||
module LogSQLite;
|
||||
|
||||
export {
|
||||
## Separator between set elements.
|
||||
const set_separator = Log::set_separator &redef;
|
||||
|
||||
## String to use for an unset &optional field.
|
||||
const unset_field = Log::unset_field &redef;
|
||||
|
||||
## String to use for empty fields. This should be different from
|
||||
## *unset_field* to make the output non-ambigious.
|
||||
const empty_field = Log::empty_field &redef;
|
||||
}
|
||||
|
||||
|
|
@ -1,264 +0,0 @@
|
|||
##! This implements transparent cluster support for the metrics framework.
|
||||
##! Do not load this file directly. It's only meant to be loaded automatically
|
||||
##! and will be depending on if the cluster framework has been enabled.
|
||||
##! The goal of this script is to make metric calculation completely and
|
||||
##! transparently automated when running on a cluster.
|
||||
##!
|
||||
##! Events defined here are not exported deliberately because they are meant
|
||||
##! to be an internal implementation detail.
|
||||
|
||||
@load base/frameworks/cluster
|
||||
@load ./main
|
||||
|
||||
module Metrics;
|
||||
|
||||
export {
|
||||
## Allows a user to decide how large of result groups the
|
||||
## workers should transmit values for cluster metric aggregation.
|
||||
const cluster_send_in_groups_of = 50 &redef;
|
||||
|
||||
## The percent of the full threshold value that needs to be met
|
||||
## on a single worker for that worker to send the value to its manager in
|
||||
## order for it to request a global view for that value. There is no
|
||||
## requirement that the manager requests a global view for the index
|
||||
## since it may opt not to if it requested a global view for the index
|
||||
## recently.
|
||||
const cluster_request_global_view_percent = 0.1 &redef;
|
||||
|
||||
## Event sent by the manager in a cluster to initiate the
|
||||
## collection of metrics values for a filter.
|
||||
global cluster_filter_request: event(uid: string, id: ID, filter_name: string);
|
||||
|
||||
## Event sent by nodes that are collecting metrics after receiving
|
||||
## a request for the metric filter from the manager.
|
||||
global cluster_filter_response: event(uid: string, id: ID, filter_name: string, data: MetricTable, done: bool);
|
||||
|
||||
## This event is sent by the manager in a cluster to initiate the
|
||||
## collection of a single index value from a filter. It's typically
|
||||
## used to get intermediate updates before the break interval triggers
|
||||
## to speed detection of a value crossing a threshold.
|
||||
global cluster_index_request: event(uid: string, id: ID, filter_name: string, index: Index);
|
||||
|
||||
## This event is sent by nodes in response to a
|
||||
## :bro:id:`Metrics::cluster_index_request` event.
|
||||
global cluster_index_response: event(uid: string, id: ID, filter_name: string, index: Index, val: count);
|
||||
|
||||
## This is sent by workers to indicate that they crossed the percent of the
|
||||
## current threshold by the percentage defined globally in
|
||||
## :bro:id:`Metrics::cluster_request_global_view_percent`
|
||||
global cluster_index_intermediate_response: event(id: Metrics::ID, filter_name: string, index: Metrics::Index, val: count);
|
||||
|
||||
## This event is scheduled internally on workers to send result chunks.
|
||||
global send_data: event(uid: string, id: ID, filter_name: string, data: MetricTable);
|
||||
|
||||
}
|
||||
|
||||
|
||||
# This is maintained by managers so they can know what data they requested and
|
||||
# when they requested it.
|
||||
global requested_results: table[string] of time = table() &create_expire=5mins;
|
||||
|
||||
# TODO: The next 4 variables make the assumption that a value never
|
||||
# takes longer than 5 minutes to transmit from workers to manager. This needs to
|
||||
# be tunable or self-tuning. These should also be restructured to be
|
||||
# maintained within a single variable.
|
||||
|
||||
# This variable is maintained by manager nodes as they collect and aggregate
|
||||
# results.
|
||||
global filter_results: table[string, ID, string] of MetricTable &create_expire=5mins;
|
||||
|
||||
# This variable is maintained by manager nodes to track how many "dones" they
|
||||
# collected per collection unique id. Once the number of results for a uid
|
||||
# matches the number of peer nodes that results should be coming from, the
|
||||
# result is written out and deleted from here.
|
||||
# TODO: add an &expire_func in case not all results are received.
|
||||
global done_with: table[string] of count &create_expire=5mins &default=0;
|
||||
|
||||
# This variable is maintained by managers to track intermediate responses as
|
||||
# they are getting a global view for a certain index.
|
||||
global index_requests: table[string, ID, string, Index] of count &create_expire=5mins &default=0;
|
||||
|
||||
# This variable is maintained by all hosts for different purposes. Non-managers
|
||||
# maintain it to know what indexes they have recently sent as intermediate
|
||||
# updates so they don't overwhelm their manager. Managers maintain it so they
|
||||
# don't overwhelm workers with intermediate index requests. The count that is
|
||||
# yielded is the number of times the percentage threshold has been crossed and
|
||||
# an intermediate result has been received. The manager may optionally request
|
||||
# the index again before data expires from here if too many workers are crossing
|
||||
# the percentage threshold (not implemented yet!).
|
||||
global recent_global_view_indexes: table[ID, string, Index] of count &create_expire=5mins &default=0;
|
||||
|
||||
# Add events to the cluster framework to make this work.
|
||||
redef Cluster::manager2worker_events += /Metrics::cluster_(filter_request|index_request)/;
|
||||
redef Cluster::worker2manager_events += /Metrics::cluster_(filter_response|index_response|index_intermediate_response)/;
|
||||
|
||||
@if ( Cluster::local_node_type() != Cluster::MANAGER )
|
||||
# This is done on all non-manager node types in the event that a metric is
|
||||
# being collected somewhere other than a worker.
|
||||
function data_added(filter: Filter, index: Index, val: count)
|
||||
{
|
||||
# If an intermediate update for this value was sent recently, don't send
|
||||
# it again.
|
||||
if ( [filter$id, filter$name, index] in recent_global_view_indexes )
|
||||
return;
|
||||
|
||||
# If val is 5 and global view % is 0.1 (10%), pct_val will be 50. If that
|
||||
# crosses the full threshold then it's a candidate to send as an
|
||||
# intermediate update.
|
||||
local pct_val = double_to_count(val / cluster_request_global_view_percent);
|
||||
|
||||
if ( check_notice(filter, index, pct_val) )
|
||||
{
|
||||
# kick off intermediate update
|
||||
event Metrics::cluster_index_intermediate_response(filter$id, filter$name, index, val);
|
||||
|
||||
++recent_global_view_indexes[filter$id, filter$name, index];
|
||||
}
|
||||
}
|
||||
|
||||
event Metrics::send_data(uid: string, id: ID, filter_name: string, data: MetricTable)
|
||||
{
|
||||
#print fmt("WORKER %s: sending data for uid %s...", Cluster::node, uid);
|
||||
|
||||
local local_data: MetricTable;
|
||||
local num_added = 0;
|
||||
for ( index in data )
|
||||
{
|
||||
local_data[index] = data[index];
|
||||
delete data[index];
|
||||
|
||||
# Only send cluster_send_in_groups_of at a time. Queue another
|
||||
# event to send the next group.
|
||||
if ( cluster_send_in_groups_of == ++num_added )
|
||||
break;
|
||||
}
|
||||
|
||||
local done = F;
|
||||
# If data is empty, this metric is done.
|
||||
if ( |data| == 0 )
|
||||
done = T;
|
||||
|
||||
event Metrics::cluster_filter_response(uid, id, filter_name, local_data, done);
|
||||
if ( ! done )
|
||||
event Metrics::send_data(uid, id, filter_name, data);
|
||||
}
|
||||
|
||||
event Metrics::cluster_filter_request(uid: string, id: ID, filter_name: string)
|
||||
{
|
||||
#print fmt("WORKER %s: received the cluster_filter_request event.", Cluster::node);
|
||||
|
||||
# Initiate sending all of the data for the requested filter.
|
||||
event Metrics::send_data(uid, id, filter_name, store[id, filter_name]);
|
||||
|
||||
# Lookup the actual filter and reset it, the reference to the data
|
||||
# currently stored will be maintained interally by the send_data event.
|
||||
reset(filter_store[id, filter_name]);
|
||||
}
|
||||
|
||||
event Metrics::cluster_index_request(uid: string, id: ID, filter_name: string, index: Index)
|
||||
{
|
||||
local val=0;
|
||||
if ( index in store[id, filter_name] )
|
||||
val = store[id, filter_name][index];
|
||||
|
||||
# fmt("WORKER %s: received the cluster_index_request event for %s=%d.", Cluster::node, index2str(index), val);
|
||||
event Metrics::cluster_index_response(uid, id, filter_name, index, val);
|
||||
}
|
||||
|
||||
@endif
|
||||
|
||||
|
||||
@if ( Cluster::local_node_type() == Cluster::MANAGER )
|
||||
|
||||
# Manager's handle logging.
|
||||
event Metrics::log_it(filter: Filter)
|
||||
{
|
||||
#print fmt("%.6f MANAGER: breaking %s filter for %s metric", network_time(), filter$name, filter$id);
|
||||
|
||||
local uid = unique_id("");
|
||||
|
||||
# Set some tracking variables.
|
||||
requested_results[uid] = network_time();
|
||||
filter_results[uid, filter$id, filter$name] = table();
|
||||
|
||||
# Request data from peers.
|
||||
event Metrics::cluster_filter_request(uid, filter$id, filter$name);
|
||||
# Schedule the log_it event for the next break period.
|
||||
schedule filter$break_interval { Metrics::log_it(filter) };
|
||||
}
|
||||
|
||||
# This is unlikely to be called often, but it's here in case there are metrics
|
||||
# being collected by managers.
|
||||
function data_added(filter: Filter, index: Index, val: count)
|
||||
{
|
||||
if ( check_notice(filter, index, val) )
|
||||
do_notice(filter, index, val);
|
||||
}
|
||||
|
||||
event Metrics::cluster_index_response(uid: string, id: ID, filter_name: string, index: Index, val: count)
|
||||
{
|
||||
#print fmt("%0.6f MANAGER: receiving index data from %s", network_time(), get_event_peer()$descr);
|
||||
|
||||
if ( [uid, id, filter_name, index] !in index_requests )
|
||||
index_requests[uid, id, filter_name, index] = 0;
|
||||
|
||||
index_requests[uid, id, filter_name, index] += val;
|
||||
local ir = index_requests[uid, id, filter_name, index];
|
||||
|
||||
++done_with[uid];
|
||||
if ( Cluster::worker_count == done_with[uid] )
|
||||
{
|
||||
if ( check_notice(filter_store[id, filter_name], index, ir) )
|
||||
do_notice(filter_store[id, filter_name], index, ir);
|
||||
delete done_with[uid];
|
||||
delete index_requests[uid, id, filter_name, index];
|
||||
}
|
||||
}
|
||||
|
||||
# Managers handle intermediate updates here.
|
||||
event Metrics::cluster_index_intermediate_response(id: ID, filter_name: string, index: Index, val: count)
|
||||
{
|
||||
#print fmt("MANAGER: receiving intermediate index data from %s", get_event_peer()$descr);
|
||||
#print fmt("MANAGER: requesting index data for %s", index2str(index));
|
||||
|
||||
local uid = unique_id("");
|
||||
event Metrics::cluster_index_request(uid, id, filter_name, index);
|
||||
++recent_global_view_indexes[id, filter_name, index];
|
||||
}
|
||||
|
||||
event Metrics::cluster_filter_response(uid: string, id: ID, filter_name: string, data: MetricTable, done: bool)
|
||||
{
|
||||
#print fmt("MANAGER: receiving results from %s", get_event_peer()$descr);
|
||||
|
||||
local local_data = filter_results[uid, id, filter_name];
|
||||
for ( index in data )
|
||||
{
|
||||
if ( index !in local_data )
|
||||
local_data[index] = 0;
|
||||
local_data[index] += data[index];
|
||||
}
|
||||
|
||||
# Mark another worker as being "done" for this uid.
|
||||
if ( done )
|
||||
++done_with[uid];
|
||||
|
||||
# If the data has been collected from all peers, we are done and ready to log.
|
||||
if ( Cluster::worker_count == done_with[uid] )
|
||||
{
|
||||
local ts = network_time();
|
||||
# Log the time this was initially requested if it's available.
|
||||
if ( uid in requested_results )
|
||||
{
|
||||
ts = requested_results[uid];
|
||||
delete requested_results[uid];
|
||||
}
|
||||
|
||||
write_log(ts, filter_store[id, filter_name], local_data);
|
||||
|
||||
# Clean up
|
||||
delete filter_results[uid, id, filter_name];
|
||||
delete done_with[uid];
|
||||
}
|
||||
}
|
||||
|
||||
@endif
|
||||
|
|
@ -1,320 +0,0 @@
|
|||
##! The metrics framework provides a way to count and measure data.
|
||||
|
||||
@load base/frameworks/notice
|
||||
|
||||
module Metrics;
|
||||
|
||||
export {
|
||||
## The metrics logging stream identifier.
|
||||
redef enum Log::ID += { LOG };
|
||||
|
||||
## Identifiers for metrics to collect.
|
||||
type ID: enum {
|
||||
## Blank placeholder value.
|
||||
NOTHING,
|
||||
};
|
||||
|
||||
## The default interval used for "breaking" metrics and writing the
|
||||
## current value to the logging stream.
|
||||
const default_break_interval = 15mins &redef;
|
||||
|
||||
## This is the interval for how often threshold based notices will happen
|
||||
## after they have already fired.
|
||||
const renotice_interval = 1hr &redef;
|
||||
|
||||
## Represents a thing which is having metrics collected for it. An instance
|
||||
## of this record type and a :bro:type:`Metrics::ID` together represent a
|
||||
## single measurement.
|
||||
type Index: record {
|
||||
## Host is the value to which this metric applies.
|
||||
host: addr &optional;
|
||||
|
||||
## A non-address related metric or a sub-key for an address based metric.
|
||||
## An example might be successful SSH connections by client IP address
|
||||
## where the client string would be the index value.
|
||||
## Another example might be number of HTTP requests to a particular
|
||||
## value in a Host header. This is an example of a non-host based
|
||||
## metric since multiple IP addresses could respond for the same Host
|
||||
## header value.
|
||||
str: string &optional;
|
||||
|
||||
## The CIDR block that this metric applies to. This is typically
|
||||
## only used internally for host based aggregation.
|
||||
network: subnet &optional;
|
||||
} &log;
|
||||
|
||||
## The record type that is used for logging metrics.
|
||||
type Info: record {
|
||||
## Timestamp at which the metric was "broken".
|
||||
ts: time &log;
|
||||
## What measurement the metric represents.
|
||||
metric_id: ID &log;
|
||||
## The name of the filter being logged. :bro:type:`Metrics::ID` values
|
||||
## can have multiple filters which represent different perspectives on
|
||||
## the data so this is necessary to understand the value.
|
||||
filter_name: string &log;
|
||||
## What the metric value applies to.
|
||||
index: Index &log;
|
||||
## The simple numeric value of the metric.
|
||||
value: count &log;
|
||||
};
|
||||
|
||||
# TODO: configure a metrics filter logging stream to log the current
|
||||
# metrics configuration in case someone is looking through
|
||||
# old logs and the configuration has changed since then.
|
||||
|
||||
## Filters define how the data from a metric is aggregated and handled.
|
||||
## Filters can be used to set how often the measurements are cut or "broken"
|
||||
## and logged or how the data within them is aggregated. It's also
|
||||
## possible to disable logging and use filters for thresholding.
|
||||
type Filter: record {
|
||||
## The :bro:type:`Metrics::ID` that this filter applies to.
|
||||
id: ID &optional;
|
||||
## The name for this filter so that multiple filters can be
|
||||
## applied to a single metrics to get a different view of the same
|
||||
## metric data being collected (different aggregation, break, etc).
|
||||
name: string &default="default";
|
||||
## A predicate so that you can decide per index if you would like
|
||||
## to accept the data being inserted.
|
||||
pred: function(index: Index): bool &optional;
|
||||
## Global mask by which you'd like to aggregate traffic.
|
||||
aggregation_mask: count &optional;
|
||||
## This is essentially a mapping table between addresses and subnets.
|
||||
aggregation_table: table[subnet] of subnet &optional;
|
||||
## The interval at which this filter should be "broken" and written
|
||||
## to the logging stream. The counters are also reset to zero at
|
||||
## this time so any threshold based detection needs to be set to a
|
||||
## number that should be expected to happen within this period.
|
||||
break_interval: interval &default=default_break_interval;
|
||||
## This determines if the result of this filter is sent to the metrics
|
||||
## logging stream. One use for the logging framework is as an internal
|
||||
## thresholding and statistics gathering utility that is meant to
|
||||
## never log but rather to generate notices and derive data.
|
||||
log: bool &default=T;
|
||||
## If this and a $notice_threshold value are set, this notice type
|
||||
## will be generated by the metrics framework.
|
||||
note: Notice::Type &optional;
|
||||
## A straight threshold for generating a notice.
|
||||
notice_threshold: count &optional;
|
||||
## A series of thresholds at which to generate notices.
|
||||
notice_thresholds: vector of count &optional;
|
||||
## How often this notice should be raised for this filter. It
|
||||
## will be generated everytime it crosses a threshold, but if the
|
||||
## $break_interval is set to 5mins and this is set to 1hr the notice
|
||||
## only be generated once per hour even if something crosses the
|
||||
## threshold in every break interval.
|
||||
notice_freq: interval &optional;
|
||||
};
|
||||
|
||||
## Function to associate a metric filter with a metric ID.
|
||||
##
|
||||
## id: The metric ID that the filter should be associated with.
|
||||
##
|
||||
## filter: The record representing the filter configuration.
|
||||
global add_filter: function(id: ID, filter: Filter);
|
||||
|
||||
## Add data into a :bro:type:`Metrics::ID`. This should be called when
|
||||
## a script has measured some point value and is ready to increment the
|
||||
## counters.
|
||||
##
|
||||
## id: The metric ID that the data represents.
|
||||
##
|
||||
## index: The metric index that the value is to be added to.
|
||||
##
|
||||
## increment: How much to increment the counter by.
|
||||
global add_data: function(id: ID, index: Index, increment: count);
|
||||
|
||||
## Helper function to represent a :bro:type:`Metrics::Index` value as
|
||||
## a simple string
|
||||
##
|
||||
## index: The metric index that is to be converted into a string.
|
||||
##
|
||||
## Returns: A string reprentation of the metric index.
|
||||
global index2str: function(index: Index): string;
|
||||
|
||||
## Event that is used to "finish" metrics and adapt the metrics
|
||||
## framework for clustered or non-clustered usage.
|
||||
##
|
||||
## ..note: This is primarily intended for internal use.
|
||||
global log_it: event(filter: Filter);
|
||||
|
||||
## Event to access metrics records as they are passed to the logging framework.
|
||||
global log_metrics: event(rec: Info);
|
||||
|
||||
## Type to store a table of metrics values. Interal use only!
|
||||
type MetricTable: table[Index] of count &default=0;
|
||||
}
|
||||
|
||||
redef record Notice::Info += {
|
||||
metric_index: Index &log &optional;
|
||||
};
|
||||
|
||||
global metric_filters: table[ID] of vector of Filter = table();
|
||||
global filter_store: table[ID, string] of Filter = table();
|
||||
|
||||
# This is indexed by metric ID and stream filter name.
|
||||
global store: table[ID, string] of MetricTable = table() &default=table();
|
||||
|
||||
# This function checks if a threshold has been crossed and generates a
|
||||
# notice if it has. It is also used as a method to implement
|
||||
# mid-break-interval threshold crossing detection for cluster deployments.
|
||||
global check_notice: function(filter: Filter, index: Index, val: count): bool;
|
||||
|
||||
# This is hook for watching thresholds being crossed. It is called whenever
|
||||
# index values are updated and the new val is given as the `val` argument.
|
||||
global data_added: function(filter: Filter, index: Index, val: count);
|
||||
|
||||
# This stores the current threshold index for filters using the
|
||||
# $notice_threshold and $notice_thresholds elements.
|
||||
global thresholds: table[ID, string, Index] of count = {} &create_expire=renotice_interval &default=0;
|
||||
|
||||
event bro_init() &priority=5
|
||||
{
|
||||
Log::create_stream(Metrics::LOG, [$columns=Info, $ev=log_metrics]);
|
||||
}
|
||||
|
||||
function index2str(index: Index): string
|
||||
{
|
||||
local out = "";
|
||||
if ( index?$host )
|
||||
out = fmt("%shost=%s", out, index$host);
|
||||
if ( index?$network )
|
||||
out = fmt("%s%snetwork=%s", out, |out|==0 ? "" : ", ", index$network);
|
||||
if ( index?$str )
|
||||
out = fmt("%s%sstr=%s", out, |out|==0 ? "" : ", ", index$str);
|
||||
return fmt("metric_index(%s)", out);
|
||||
}
|
||||
|
||||
function write_log(ts: time, filter: Filter, data: MetricTable)
|
||||
{
|
||||
for ( index in data )
|
||||
{
|
||||
local val = data[index];
|
||||
local m: Info = [$ts=ts,
|
||||
$metric_id=filter$id,
|
||||
$filter_name=filter$name,
|
||||
$index=index,
|
||||
$value=val];
|
||||
|
||||
if ( filter$log )
|
||||
Log::write(Metrics::LOG, m);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function reset(filter: Filter)
|
||||
{
|
||||
store[filter$id, filter$name] = table();
|
||||
}
|
||||
|
||||
function add_filter(id: ID, filter: Filter)
|
||||
{
|
||||
if ( filter?$aggregation_table && filter?$aggregation_mask )
|
||||
{
|
||||
print "INVALID Metric filter: Defined $aggregation_table and $aggregation_mask.";
|
||||
return;
|
||||
}
|
||||
if ( [id, filter$name] in store )
|
||||
{
|
||||
print fmt("INVALID Metric filter: Filter with name \"%s\" already exists.", filter$name);
|
||||
return;
|
||||
}
|
||||
if ( filter?$notice_threshold && filter?$notice_thresholds )
|
||||
{
|
||||
print "INVALID Metric filter: Defined both $notice_threshold and $notice_thresholds";
|
||||
return;
|
||||
}
|
||||
|
||||
if ( ! filter?$id )
|
||||
filter$id = id;
|
||||
|
||||
if ( id !in metric_filters )
|
||||
metric_filters[id] = vector();
|
||||
metric_filters[id][|metric_filters[id]|] = filter;
|
||||
|
||||
filter_store[id, filter$name] = filter;
|
||||
store[id, filter$name] = table();
|
||||
|
||||
schedule filter$break_interval { Metrics::log_it(filter) };
|
||||
}
|
||||
|
||||
function add_data(id: ID, index: Index, increment: count)
|
||||
{
|
||||
if ( id !in metric_filters )
|
||||
return;
|
||||
|
||||
local filters = metric_filters[id];
|
||||
|
||||
# Try to add the data to all of the defined filters for the metric.
|
||||
for ( filter_id in filters )
|
||||
{
|
||||
local filter = filters[filter_id];
|
||||
|
||||
# If this filter has a predicate, run the predicate and skip this
|
||||
# index if the predicate return false.
|
||||
if ( filter?$pred && ! filter$pred(index) )
|
||||
next;
|
||||
|
||||
if ( index?$host )
|
||||
{
|
||||
if ( filter?$aggregation_mask )
|
||||
{
|
||||
index$network = mask_addr(index$host, filter$aggregation_mask);
|
||||
delete index$host;
|
||||
}
|
||||
else if ( filter?$aggregation_table )
|
||||
{
|
||||
# Don't add the data if the aggregation table doesn't include
|
||||
# the given host address.
|
||||
if ( index$host !in filter$aggregation_table )
|
||||
return;
|
||||
index$network = filter$aggregation_table[index$host];
|
||||
delete index$host;
|
||||
}
|
||||
}
|
||||
|
||||
local metric_tbl = store[id, filter$name];
|
||||
if ( index !in metric_tbl )
|
||||
metric_tbl[index] = 0;
|
||||
metric_tbl[index] += increment;
|
||||
|
||||
data_added(filter, index, metric_tbl[index]);
|
||||
}
|
||||
}
|
||||
|
||||
function check_notice(filter: Filter, index: Index, val: count): bool
|
||||
{
|
||||
if ( (filter?$notice_threshold &&
|
||||
[filter$id, filter$name, index] !in thresholds &&
|
||||
val >= filter$notice_threshold) ||
|
||||
(filter?$notice_thresholds &&
|
||||
|filter$notice_thresholds| <= thresholds[filter$id, filter$name, index] &&
|
||||
val >= filter$notice_thresholds[thresholds[filter$id, filter$name, index]]) )
|
||||
return T;
|
||||
else
|
||||
return F;
|
||||
}
|
||||
|
||||
function do_notice(filter: Filter, index: Index, val: count)
|
||||
{
|
||||
# We include $peer_descr here because the a manager count have actually
|
||||
# generated the notice even though the current remote peer for the event
|
||||
# calling this could be a worker if this is running as a cluster.
|
||||
local n: Notice::Info = [$note=filter$note,
|
||||
$n=val,
|
||||
$metric_index=index,
|
||||
$peer_descr=peer_description];
|
||||
n$msg = fmt("Threshold crossed by %s %d/%d", index2str(index), val, filter$notice_threshold);
|
||||
if ( index?$str )
|
||||
n$sub = index$str;
|
||||
if ( index?$host )
|
||||
n$src = index$host;
|
||||
# TODO: not sure where to put the network yet.
|
||||
|
||||
NOTICE(n);
|
||||
|
||||
# This just needs set to some value so that it doesn't refire the
|
||||
# notice until it expires from the table or it crosses the next
|
||||
# threshold in the case of vectors of thresholds.
|
||||
++thresholds[filter$id, filter$name, index];
|
||||
}
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
@load ./main
|
||||
|
||||
module Metrics;
|
||||
|
||||
event Metrics::log_it(filter: Filter)
|
||||
{
|
||||
local id = filter$id;
|
||||
local name = filter$name;
|
||||
|
||||
write_log(network_time(), filter, store[id, name]);
|
||||
reset(filter);
|
||||
|
||||
schedule filter$break_interval { Metrics::log_it(filter) };
|
||||
}
|
||||
|
||||
|
||||
function data_added(filter: Filter, index: Index, val: count)
|
||||
{
|
||||
if ( check_notice(filter, index, val) )
|
||||
do_notice(filter, index, val);
|
||||
}
|
||||
|
|
@ -431,9 +431,6 @@ hook Notice::notice(n: Notice::Info) &priority=-5
|
|||
}
|
||||
}
|
||||
|
||||
## This determines if a notice is being suppressed. It is only used
|
||||
## internally as part of the mechanics for the global :bro:id:`NOTICE`
|
||||
## function.
|
||||
function is_being_suppressed(n: Notice::Info): bool
|
||||
{
|
||||
if ( n?$identifier && [n$note, n$identifier] in suppressing )
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
@load ./main
|
||||
@load ./plugins
|
||||
|
||||
# The cluster framework must be loaded first.
|
||||
@load base/frameworks/cluster
|
||||
346
scripts/base/frameworks/sumstats/cluster.bro
Normal file
346
scripts/base/frameworks/sumstats/cluster.bro
Normal file
|
|
@ -0,0 +1,346 @@
|
|||
##! This implements transparent cluster support for the SumStats framework.
|
||||
##! Do not load this file directly. It's only meant to be loaded automatically
|
||||
##! and will be depending on if the cluster framework has been enabled.
|
||||
##! The goal of this script is to make sumstats calculation completely and
|
||||
##! transparently automated when running on a cluster.
|
||||
|
||||
@load base/frameworks/cluster
|
||||
@load ./main
|
||||
|
||||
module SumStats;
|
||||
|
||||
export {
|
||||
## Allows a user to decide how large of result groups the workers should transmit
|
||||
## values for cluster stats aggregation.
|
||||
const cluster_send_in_groups_of = 50 &redef;
|
||||
|
||||
## The percent of the full threshold value that needs to be met on a single worker
|
||||
## for that worker to send the value to its manager in order for it to request a
|
||||
## global view for that value. There is no requirement that the manager requests
|
||||
## a global view for the key since it may opt not to if it requested a global view
|
||||
## for the key recently.
|
||||
const cluster_request_global_view_percent = 0.2 &redef;
|
||||
|
||||
## This is to deal with intermediate update overload. A manager will only allow
|
||||
## this many intermediate update requests to the workers to be inflight at any
|
||||
## given time. Requested intermediate updates are currently thrown out and not
|
||||
## performed. In practice this should hopefully have a minimal effect.
|
||||
const max_outstanding_global_views = 10 &redef;
|
||||
|
||||
## Intermediate updates can cause overload situations on very large clusters. This
|
||||
## option may help reduce load and correct intermittent problems. The goal for this
|
||||
## option is also meant to be temporary.
|
||||
const enable_intermediate_updates = T &redef;
|
||||
|
||||
## Event sent by the manager in a cluster to initiate the collection of values for
|
||||
## a sumstat.
|
||||
global cluster_ss_request: event(uid: string, ssid: string);
|
||||
|
||||
## Event sent by nodes that are collecting sumstats after receiving a request for
|
||||
## the sumstat from the manager.
|
||||
global cluster_ss_response: event(uid: string, ssid: string, data: ResultTable, done: bool);
|
||||
|
||||
## This event is sent by the manager in a cluster to initiate the collection of
|
||||
## a single key value from a sumstat. It's typically used to get intermediate
|
||||
## updates before the break interval triggers to speed detection of a value
|
||||
## crossing a threshold.
|
||||
global cluster_key_request: event(uid: string, ssid: string, key: Key);
|
||||
|
||||
## This event is sent by nodes in response to a
|
||||
## :bro:id:`SumStats::cluster_key_request` event.
|
||||
global cluster_key_response: event(uid: string, ssid: string, key: Key, result: Result);
|
||||
|
||||
## This is sent by workers to indicate that they crossed the percent
|
||||
## of the current threshold by the percentage defined globally in
|
||||
## :bro:id:`SumStats::cluster_request_global_view_percent`
|
||||
global cluster_key_intermediate_response: event(ssid: string, key: SumStats::Key);
|
||||
|
||||
## This event is scheduled internally on workers to send result chunks.
|
||||
global send_data: event(uid: string, ssid: string, data: ResultTable);
|
||||
|
||||
## This event is generated when a threshold is crossed.
|
||||
global cluster_threshold_crossed: event(ssid: string, key: SumStats::Key, thold: Thresholding);
|
||||
}
|
||||
|
||||
# Add events to the cluster framework to make this work.
|
||||
redef Cluster::manager2worker_events += /SumStats::cluster_(ss_request|key_request|threshold_crossed)/;
|
||||
redef Cluster::manager2worker_events += /SumStats::thresholds_reset/;
|
||||
redef Cluster::worker2manager_events += /SumStats::cluster_(ss_response|key_response|key_intermediate_response)/;
|
||||
|
||||
@if ( Cluster::local_node_type() != Cluster::MANAGER )
|
||||
# This variable is maintained to know what keys have recently sent as
|
||||
# intermediate updates so they don't overwhelm their manager. The count that is
|
||||
# yielded is the number of times the percentage threshold has been crossed and
|
||||
# an intermediate result has been received.
|
||||
global recent_global_view_keys: table[string, Key] of count &create_expire=1min &default=0;
|
||||
|
||||
event bro_init() &priority=-100
|
||||
{
|
||||
# The manager is the only host allowed to track these.
|
||||
stats_store = table();
|
||||
reducer_store = table();
|
||||
}
|
||||
|
||||
# This is done on all non-manager node types in the event that a sumstat is
|
||||
# being collected somewhere other than a worker.
|
||||
function data_added(ss: SumStat, key: Key, result: Result)
|
||||
{
|
||||
# If an intermediate update for this value was sent recently, don't send
|
||||
# it again.
|
||||
if ( [ss$id, key] in recent_global_view_keys )
|
||||
return;
|
||||
|
||||
# If val is 5 and global view % is 0.1 (10%), pct_val will be 50. If that
|
||||
# crosses the full threshold then it's a candidate to send as an
|
||||
# intermediate update.
|
||||
if ( enable_intermediate_updates &&
|
||||
check_thresholds(ss, key, result, cluster_request_global_view_percent) )
|
||||
{
|
||||
# kick off intermediate update
|
||||
event SumStats::cluster_key_intermediate_response(ss$id, key);
|
||||
++recent_global_view_keys[ss$id, key];
|
||||
}
|
||||
}
|
||||
|
||||
event SumStats::send_data(uid: string, ssid: string, data: ResultTable)
|
||||
{
|
||||
#print fmt("WORKER %s: sending data for uid %s...", Cluster::node, uid);
|
||||
|
||||
local local_data: ResultTable = table();
|
||||
local num_added = 0;
|
||||
for ( key in data )
|
||||
{
|
||||
local_data[key] = data[key];
|
||||
delete data[key];
|
||||
|
||||
# Only send cluster_send_in_groups_of at a time. Queue another
|
||||
# event to send the next group.
|
||||
if ( cluster_send_in_groups_of == ++num_added )
|
||||
break;
|
||||
}
|
||||
|
||||
local done = F;
|
||||
# If data is empty, this sumstat is done.
|
||||
if ( |data| == 0 )
|
||||
done = T;
|
||||
|
||||
# Note: copy is needed to compensate serialization caching issue. This should be
|
||||
# changed to something else later.
|
||||
event SumStats::cluster_ss_response(uid, ssid, copy(local_data), done);
|
||||
if ( ! done )
|
||||
schedule 0.01 sec { SumStats::send_data(uid, ssid, data) };
|
||||
}
|
||||
|
||||
event SumStats::cluster_ss_request(uid: string, ssid: string)
|
||||
{
|
||||
#print fmt("WORKER %s: received the cluster_ss_request event for %s.", Cluster::node, id);
|
||||
|
||||
# Initiate sending all of the data for the requested stats.
|
||||
if ( ssid in result_store )
|
||||
event SumStats::send_data(uid, ssid, result_store[ssid]);
|
||||
else
|
||||
event SumStats::send_data(uid, ssid, table());
|
||||
|
||||
# Lookup the actual sumstats and reset it, the reference to the data
|
||||
# currently stored will be maintained internally by the send_data event.
|
||||
if ( ssid in stats_store )
|
||||
reset(stats_store[ssid]);
|
||||
}
|
||||
|
||||
event SumStats::cluster_key_request(uid: string, ssid: string, key: Key)
|
||||
{
|
||||
if ( ssid in result_store && key in result_store[ssid] )
|
||||
{
|
||||
#print fmt("WORKER %s: received the cluster_key_request event for %s=%s.", Cluster::node, key2str(key), data);
|
||||
|
||||
# Note: copy is needed to compensate serialization caching issue. This should be
|
||||
# changed to something else later.
|
||||
event SumStats::cluster_key_response(uid, ssid, key, copy(result_store[ssid][key]));
|
||||
}
|
||||
else
|
||||
{
|
||||
# We need to send an empty response if we don't have the data so that the manager
|
||||
# can know that it heard back from all of the workers.
|
||||
event SumStats::cluster_key_response(uid, ssid, key, table());
|
||||
}
|
||||
}
|
||||
|
||||
event SumStats::cluster_threshold_crossed(ssid: string, key: SumStats::Key, thold: Thresholding)
|
||||
{
|
||||
if ( ssid !in threshold_tracker )
|
||||
threshold_tracker[ssid] = table();
|
||||
|
||||
threshold_tracker[ssid][key] = thold;
|
||||
}
|
||||
|
||||
event SumStats::thresholds_reset(ssid: string)
|
||||
{
|
||||
threshold_tracker[ssid] = table();
|
||||
}
|
||||
|
||||
@endif
|
||||
|
||||
|
||||
@if ( Cluster::local_node_type() == Cluster::MANAGER )
|
||||
|
||||
# This variable is maintained by manager nodes as they collect and aggregate
|
||||
# results.
|
||||
# Index on a uid.
|
||||
global stats_results: table[string] of ResultTable &read_expire=1min;
|
||||
|
||||
# This variable is maintained by manager nodes to track how many "dones" they
|
||||
# collected per collection unique id. Once the number of results for a uid
|
||||
# matches the number of peer nodes that results should be coming from, the
|
||||
# result is written out and deleted from here.
|
||||
# Indexed on a uid.
|
||||
# TODO: add an &expire_func in case not all results are received.
|
||||
global done_with: table[string] of count &read_expire=1min &default=0;
|
||||
|
||||
# This variable is maintained by managers to track intermediate responses as
|
||||
# they are getting a global view for a certain key.
|
||||
# Indexed on a uid.
|
||||
global key_requests: table[string] of Result &read_expire=1min;
|
||||
|
||||
# This variable is maintained by managers to prevent overwhelming communication due
|
||||
# to too many intermediate updates. Each sumstat is tracked separately so that
|
||||
# one won't overwhelm and degrade other quieter sumstats.
|
||||
# Indexed on a sumstat id.
|
||||
global outstanding_global_views: table[string] of count &default=0;
|
||||
|
||||
const zero_time = double_to_time(0.0);
|
||||
# Managers handle logging.
|
||||
event SumStats::finish_epoch(ss: SumStat)
|
||||
{
|
||||
if ( network_time() > zero_time )
|
||||
{
|
||||
#print fmt("%.6f MANAGER: breaking %s sumstat for %s sumstat", network_time(), ss$name, ss$id);
|
||||
local uid = unique_id("");
|
||||
|
||||
if ( uid in stats_results )
|
||||
delete stats_results[uid];
|
||||
stats_results[uid] = table();
|
||||
|
||||
# Request data from peers.
|
||||
event SumStats::cluster_ss_request(uid, ss$id);
|
||||
}
|
||||
|
||||
# Schedule the next finish_epoch event.
|
||||
schedule ss$epoch { SumStats::finish_epoch(ss) };
|
||||
}
|
||||
|
||||
# This is unlikely to be called often, but it's here in
|
||||
# case there are sumstats being collected by managers.
|
||||
function data_added(ss: SumStat, key: Key, result: Result)
|
||||
{
|
||||
if ( check_thresholds(ss, key, result, 1.0) )
|
||||
{
|
||||
threshold_crossed(ss, key, result);
|
||||
event SumStats::cluster_threshold_crossed(ss$id, key, threshold_tracker[ss$id][key]);
|
||||
}
|
||||
}
|
||||
|
||||
event SumStats::cluster_key_response(uid: string, ssid: string, key: Key, result: Result)
|
||||
{
|
||||
#print fmt("%0.6f MANAGER: receiving key data from %s - %s=%s", network_time(), get_event_peer()$descr, key2str(key), result);
|
||||
|
||||
# We only want to try and do a value merge if there are actually measured datapoints
|
||||
# in the Result.
|
||||
if ( uid in key_requests )
|
||||
key_requests[uid] = compose_results(key_requests[uid], result);
|
||||
else
|
||||
key_requests[uid] = result;
|
||||
|
||||
# Mark that a worker is done.
|
||||
++done_with[uid];
|
||||
|
||||
#print fmt("worker_count:%d :: done_with:%d", Cluster::worker_count, done_with[uid]);
|
||||
if ( Cluster::worker_count == done_with[uid] )
|
||||
{
|
||||
local ss = stats_store[ssid];
|
||||
local ir = key_requests[uid];
|
||||
if ( check_thresholds(ss, key, ir, 1.0) )
|
||||
{
|
||||
threshold_crossed(ss, key, ir);
|
||||
event SumStats::cluster_threshold_crossed(ss$id, key, threshold_tracker[ss$id][key]);
|
||||
}
|
||||
|
||||
delete done_with[uid];
|
||||
delete key_requests[uid];
|
||||
# Check that there is an outstanding view before subtracting.
|
||||
if ( outstanding_global_views[ssid] > 0 )
|
||||
--outstanding_global_views[ssid];
|
||||
}
|
||||
}
|
||||
|
||||
# Managers handle intermediate updates here.
|
||||
event SumStats::cluster_key_intermediate_response(ssid: string, key: Key)
|
||||
{
|
||||
#print fmt("MANAGER: receiving intermediate key data from %s", get_event_peer()$descr);
|
||||
#print fmt("MANAGER: requesting key data for %s", key2str(key));
|
||||
|
||||
if ( ssid in outstanding_global_views &&
|
||||
|outstanding_global_views[ssid]| > max_outstanding_global_views )
|
||||
{
|
||||
# Don't do this intermediate update. Perhaps at some point in the future
|
||||
# we will queue and randomly select from these ignored intermediate
|
||||
# update requests.
|
||||
return;
|
||||
}
|
||||
|
||||
++outstanding_global_views[ssid];
|
||||
|
||||
local uid = unique_id("");
|
||||
event SumStats::cluster_key_request(uid, ssid, key);
|
||||
}
|
||||
|
||||
event SumStats::cluster_ss_response(uid: string, ssid: string, data: ResultTable, done: bool)
|
||||
{
|
||||
#print fmt("MANAGER: receiving results from %s", get_event_peer()$descr);
|
||||
|
||||
# Mark another worker as being "done" for this uid.
|
||||
if ( done )
|
||||
++done_with[uid];
|
||||
|
||||
local local_data = stats_results[uid];
|
||||
local ss = stats_store[ssid];
|
||||
|
||||
for ( key in data )
|
||||
{
|
||||
if ( key in local_data )
|
||||
local_data[key] = compose_results(local_data[key], data[key]);
|
||||
else
|
||||
local_data[key] = data[key];
|
||||
|
||||
# If a stat is done being collected, thresholds for each key
|
||||
# need to be checked so we're doing it here to avoid doubly
|
||||
# iterating over each key.
|
||||
if ( Cluster::worker_count == done_with[uid] )
|
||||
{
|
||||
if ( check_thresholds(ss, key, local_data[key], 1.0) )
|
||||
{
|
||||
threshold_crossed(ss, key, local_data[key]);
|
||||
event SumStats::cluster_threshold_crossed(ss$id, key, threshold_tracker[ss$id][key]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# If the data has been collected from all peers, we are done and ready to finish.
|
||||
if ( Cluster::worker_count == done_with[uid] )
|
||||
{
|
||||
if ( ss?$epoch_finished )
|
||||
ss$epoch_finished(local_data);
|
||||
|
||||
# Clean up
|
||||
delete stats_results[uid];
|
||||
delete done_with[uid];
|
||||
# Not sure I need to reset the sumstat on the manager.
|
||||
reset(ss);
|
||||
}
|
||||
}
|
||||
|
||||
event remote_connection_handshake_done(p: event_peer) &priority=5
|
||||
{
|
||||
send_id(p, "SumStats::stats_store");
|
||||
send_id(p, "SumStats::reducer_store");
|
||||
}
|
||||
@endif
|
||||
436
scripts/base/frameworks/sumstats/main.bro
Normal file
436
scripts/base/frameworks/sumstats/main.bro
Normal file
|
|
@ -0,0 +1,436 @@
|
|||
##! The summary statistics framework provides a way to
|
||||
##! summarize large streams of data into simple reduced
|
||||
##! measurements.
|
||||
|
||||
module SumStats;
|
||||
|
||||
export {
|
||||
## The various calculations are all defined as plugins.
|
||||
type Calculation: enum {
|
||||
PLACEHOLDER
|
||||
};
|
||||
|
||||
## Represents a thing which is having summarization
|
||||
## results collected for it.
|
||||
type Key: record {
|
||||
## A non-address related summarization or a sub-key for
|
||||
## an address based summarization. An example might be
|
||||
## successful SSH connections by client IP address
|
||||
## where the client string would be the key value.
|
||||
## Another example might be number of HTTP requests to
|
||||
## a particular value in a Host header. This is an
|
||||
## example of a non-host based metric since multiple
|
||||
## IP addresses could respond for the same Host
|
||||
## header value.
|
||||
str: string &optional;
|
||||
|
||||
## Host is the value to which this metric applies.
|
||||
host: addr &optional;
|
||||
};
|
||||
|
||||
## Represents data being added for a single observation.
|
||||
## Only supply a single field at a time!
|
||||
type Observation: record {
|
||||
## Count value.
|
||||
num: count &optional;
|
||||
## Double value.
|
||||
dbl: double &optional;
|
||||
## String value.
|
||||
str: string &optional;
|
||||
};
|
||||
|
||||
type Reducer: record {
|
||||
## Observation stream identifier for the reducer
|
||||
## to attach to.
|
||||
stream: string;
|
||||
|
||||
## The calculations to perform on the data points.
|
||||
apply: set[Calculation];
|
||||
|
||||
## A predicate so that you can decide per key if you
|
||||
## would like to accept the data being inserted.
|
||||
pred: function(key: SumStats::Key, obs: SumStats::Observation): bool &optional;
|
||||
|
||||
## A function to normalize the key. This can be used to aggregate or
|
||||
## normalize the entire key.
|
||||
normalize_key: function(key: SumStats::Key): Key &optional;
|
||||
};
|
||||
|
||||
## Value calculated for an observation stream fed into a reducer.
|
||||
## Most of the fields are added by plugins.
|
||||
type ResultVal: record {
|
||||
## The time when the first observation was added to
|
||||
## this result value.
|
||||
begin: time;
|
||||
|
||||
## The time when the last observation was added to
|
||||
## this result value.
|
||||
end: time;
|
||||
|
||||
## The number of observations received.
|
||||
num: count &default=0;
|
||||
};
|
||||
|
||||
## Type to store results for multiple reducers.
|
||||
type Result: table[string] of ResultVal;
|
||||
|
||||
## Type to store a table of sumstats results indexed
|
||||
## by keys.
|
||||
type ResultTable: table[Key] of Result;
|
||||
|
||||
## SumStats represent an aggregation of reducers along with
|
||||
## mechanisms to handle various situations like the epoch ending
|
||||
## or thresholds being crossed.
|
||||
##
|
||||
## It's best to not access any global state outside
|
||||
## of the variables given to the callbacks because there
|
||||
## is no assurance provided as to where the callbacks
|
||||
## will be executed on clusters.
|
||||
type SumStat: record {
|
||||
## The interval at which this filter should be "broken"
|
||||
## and the '$epoch_finished' callback called. The
|
||||
## results are also reset at this time so any threshold
|
||||
## based detection needs to be set to a
|
||||
## value that should be expected to happen within
|
||||
## this epoch.
|
||||
epoch: interval;
|
||||
|
||||
## The reducers for the SumStat
|
||||
reducers: set[Reducer];
|
||||
|
||||
## Provide a function to calculate a value from the
|
||||
## :bro:see:`SumStats::Result` structure which will be used
|
||||
## for thresholding.
|
||||
## This is required if a $threshold value is given.
|
||||
threshold_val: function(key: SumStats::Key, result: SumStats::Result): count &optional;
|
||||
|
||||
## The threshold value for calling the
|
||||
## $threshold_crossed callback.
|
||||
threshold: count &optional;
|
||||
|
||||
## A series of thresholds for calling the
|
||||
## $threshold_crossed callback.
|
||||
threshold_series: vector of count &optional;
|
||||
|
||||
## A callback that is called when a threshold is crossed.
|
||||
threshold_crossed: function(key: SumStats::Key, result: SumStats::Result) &optional;
|
||||
|
||||
## A callback with the full collection of Results for
|
||||
## this SumStat.
|
||||
epoch_finished: function(rt: SumStats::ResultTable) &optional;
|
||||
};
|
||||
|
||||
## Create a summary statistic.
|
||||
global create: function(ss: SumStats::SumStat);
|
||||
|
||||
## Add data into an observation stream. This should be
|
||||
## called when a script has measured some point value.
|
||||
##
|
||||
## id: The observation stream identifier that the data
|
||||
## point represents.
|
||||
##
|
||||
## key: The key that the value is related to.
|
||||
##
|
||||
## obs: The data point to send into the stream.
|
||||
global observe: function(id: string, key: SumStats::Key, obs: SumStats::Observation);
|
||||
|
||||
## This record is primarily used for internal threshold tracking.
|
||||
type Thresholding: record {
|
||||
# Internal use only. Indicates if a simple threshold was already crossed.
|
||||
is_threshold_crossed: bool &default=F;
|
||||
|
||||
# Internal use only. Current key for threshold series.
|
||||
threshold_series_index: count &default=0;
|
||||
};
|
||||
|
||||
## This event is generated when thresholds are reset for a SumStat.
|
||||
##
|
||||
## ssid: SumStats ID that thresholds were reset for.
|
||||
global thresholds_reset: event(ssid: string);
|
||||
|
||||
## Helper function to represent a :bro:type:`SumStats::Key` value as
|
||||
## a simple string.
|
||||
##
|
||||
## key: The metric key that is to be converted into a string.
|
||||
##
|
||||
## Returns: A string representation of the metric key.
|
||||
global key2str: function(key: SumStats::Key): string;
|
||||
}
|
||||
|
||||
redef record Reducer += {
|
||||
# Internal use only. Provides a reference back to the related SumStats by it's ID.
|
||||
sid: string &optional;
|
||||
};
|
||||
|
||||
# Internal use only. For tracking thresholds per sumstat and key.
|
||||
global threshold_tracker: table[string] of table[Key] of Thresholding &optional;
|
||||
|
||||
redef record SumStat += {
|
||||
# Internal use only (mostly for cluster coherency).
|
||||
id: string &optional;
|
||||
};
|
||||
|
||||
# Store of sumstats indexed on the sumstat id.
|
||||
global stats_store: table[string] of SumStat = table();
|
||||
|
||||
# Store of reducers indexed on the data point stream id.
|
||||
global reducer_store: table[string] of set[Reducer] = table();
|
||||
|
||||
# Store of results indexed on the measurement id.
|
||||
global result_store: table[string] of ResultTable = table();
|
||||
|
||||
# Store of threshold information.
|
||||
global thresholds_store: table[string, Key] of bool = table();
|
||||
|
||||
# This is called whenever key values are updated and the new val is given as the
|
||||
# `val` argument. It's only prototyped here because cluster and non-cluster have
|
||||
# separate implementations.
|
||||
global data_added: function(ss: SumStat, key: Key, result: Result);
|
||||
|
||||
# Prototype the hook point for plugins to do calculations.
|
||||
global observe_hook: hook(r: Reducer, val: double, data: Observation, rv: ResultVal);
|
||||
|
||||
# Prototype the hook point for plugins to initialize any result values.
|
||||
global init_resultval_hook: hook(r: Reducer, rv: ResultVal);
|
||||
|
||||
# Prototype the hook point for plugins to merge Results.
|
||||
global compose_resultvals_hook: hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal);
|
||||
|
||||
# Event that is used to "finish" measurements and adapt the measurement
|
||||
# framework for clustered or non-clustered usage.
|
||||
global finish_epoch: event(ss: SumStat);
|
||||
|
||||
function key2str(key: Key): string
|
||||
{
|
||||
local out = "";
|
||||
if ( key?$host )
|
||||
out = fmt("%shost=%s", out, key$host);
|
||||
if ( key?$str )
|
||||
out = fmt("%s%sstr=%s", out, |out|==0 ? "" : ", ", key$str);
|
||||
return fmt("sumstats_key(%s)", out);
|
||||
}
|
||||
|
||||
function init_resultval(r: Reducer): ResultVal
|
||||
{
|
||||
local rv: ResultVal = [$begin=network_time(), $end=network_time()];
|
||||
hook init_resultval_hook(r, rv);
|
||||
return rv;
|
||||
}
|
||||
|
||||
function compose_resultvals(rv1: ResultVal, rv2: ResultVal): ResultVal
|
||||
{
|
||||
local result: ResultVal;
|
||||
|
||||
result$begin = (rv1$begin < rv2$begin) ? rv1$begin : rv2$begin;
|
||||
result$end = (rv1$end > rv2$end) ? rv1$end : rv2$end;
|
||||
result$num = rv1$num + rv2$num;
|
||||
|
||||
# Run the plugin composition hooks.
|
||||
hook compose_resultvals_hook(result, rv1, rv2);
|
||||
return result;
|
||||
}
|
||||
|
||||
function compose_results(r1: Result, r2: Result): Result
|
||||
{
|
||||
local result: Result = table();
|
||||
|
||||
if ( |r1| > |r2| )
|
||||
{
|
||||
for ( data_id in r1 )
|
||||
{
|
||||
if ( data_id in r2 )
|
||||
result[data_id] = compose_resultvals(r1[data_id], r2[data_id]);
|
||||
else
|
||||
result[data_id] = r1[data_id];
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
for ( data_id in r2 )
|
||||
{
|
||||
if ( data_id in r1 )
|
||||
result[data_id] = compose_resultvals(r1[data_id], r2[data_id]);
|
||||
else
|
||||
result[data_id] = r2[data_id];
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
function reset(ss: SumStat)
|
||||
{
|
||||
if ( ss$id in result_store )
|
||||
delete result_store[ss$id];
|
||||
|
||||
result_store[ss$id] = table();
|
||||
|
||||
if ( ss?$threshold || ss?$threshold_series )
|
||||
{
|
||||
threshold_tracker[ss$id] = table();
|
||||
event SumStats::thresholds_reset(ss$id);
|
||||
}
|
||||
}
|
||||
|
||||
function create(ss: SumStat)
|
||||
{
|
||||
if ( (ss?$threshold || ss?$threshold_series) && ! ss?$threshold_val )
|
||||
{
|
||||
Reporter::error("SumStats given a threshold with no $threshold_val function");
|
||||
}
|
||||
|
||||
if ( ! ss?$id )
|
||||
ss$id=unique_id("");
|
||||
threshold_tracker[ss$id] = table();
|
||||
stats_store[ss$id] = ss;
|
||||
|
||||
for ( reducer in ss$reducers )
|
||||
{
|
||||
reducer$sid = ss$id;
|
||||
if ( reducer$stream !in reducer_store )
|
||||
reducer_store[reducer$stream] = set();
|
||||
add reducer_store[reducer$stream][reducer];
|
||||
}
|
||||
|
||||
reset(ss);
|
||||
schedule ss$epoch { SumStats::finish_epoch(ss) };
|
||||
}
|
||||
|
||||
function observe(id: string, key: Key, obs: Observation)
|
||||
{
|
||||
if ( id !in reducer_store )
|
||||
return;
|
||||
|
||||
# Try to add the data to all of the defined reducers.
|
||||
for ( r in reducer_store[id] )
|
||||
{
|
||||
if ( r?$normalize_key )
|
||||
key = r$normalize_key(copy(key));
|
||||
|
||||
# If this reducer has a predicate, run the predicate
|
||||
# and skip this key if the predicate return false.
|
||||
if ( r?$pred && ! r$pred(key, obs) )
|
||||
next;
|
||||
|
||||
local ss = stats_store[r$sid];
|
||||
|
||||
# If there is a threshold and no epoch_finished callback
|
||||
# we don't need to continue counting since the data will
|
||||
# never be accessed. This was leading
|
||||
# to some state management issues when measuring
|
||||
# uniqueness.
|
||||
# NOTE: this optimization could need removed in the
|
||||
# future if on demand access is provided to the
|
||||
# SumStats results.
|
||||
if ( ! ss?$epoch_finished &&
|
||||
r$sid in threshold_tracker &&
|
||||
key in threshold_tracker[r$sid] &&
|
||||
( ss?$threshold &&
|
||||
threshold_tracker[r$sid][key]$is_threshold_crossed ) ||
|
||||
( ss?$threshold_series &&
|
||||
threshold_tracker[r$sid][key]$threshold_series_index+1 == |ss$threshold_series| ) )
|
||||
next;
|
||||
|
||||
if ( r$sid !in result_store )
|
||||
result_store[ss$id] = table();
|
||||
local results = result_store[r$sid];
|
||||
|
||||
if ( key !in results )
|
||||
results[key] = table();
|
||||
local result = results[key];
|
||||
|
||||
if ( id !in result )
|
||||
result[id] = init_resultval(r);
|
||||
local result_val = result[id];
|
||||
|
||||
++result_val$num;
|
||||
# Continually update the $end field.
|
||||
result_val$end=network_time();
|
||||
|
||||
# If a string was given, fall back to 1.0 as the value.
|
||||
local val = 1.0;
|
||||
if ( obs?$num || obs?$dbl )
|
||||
val = obs?$dbl ? obs$dbl : obs$num;
|
||||
|
||||
hook observe_hook(r, val, obs, result_val);
|
||||
data_added(ss, key, result);
|
||||
}
|
||||
}
|
||||
|
||||
# This function checks if a threshold has been crossed. It is also used as a method to implement
|
||||
# mid-break-interval threshold crossing detection for cluster deployments.
|
||||
function check_thresholds(ss: SumStat, key: Key, result: Result, modify_pct: double): bool
|
||||
{
|
||||
if ( ! (ss?$threshold || ss?$threshold_series) )
|
||||
return F;
|
||||
|
||||
# Add in the extra ResultVals to make threshold_vals easier to write.
|
||||
if ( |ss$reducers| != |result| )
|
||||
{
|
||||
for ( reducer in ss$reducers )
|
||||
{
|
||||
if ( reducer$stream !in result )
|
||||
result[reducer$stream] = init_resultval(reducer);
|
||||
}
|
||||
}
|
||||
|
||||
local watch = ss$threshold_val(key, result);
|
||||
|
||||
if ( modify_pct < 1.0 && modify_pct > 0.0 )
|
||||
watch = double_to_count(floor(watch/modify_pct));
|
||||
|
||||
if ( ss$id !in threshold_tracker )
|
||||
threshold_tracker[ss$id] = table();
|
||||
local t_tracker = threshold_tracker[ss$id];
|
||||
|
||||
if ( key !in t_tracker )
|
||||
{
|
||||
local ttmp: Thresholding;
|
||||
t_tracker[key] = ttmp;
|
||||
}
|
||||
local tt = t_tracker[key];
|
||||
|
||||
if ( ss?$threshold && ! tt$is_threshold_crossed && watch >= ss$threshold )
|
||||
{
|
||||
# Value crossed the threshold.
|
||||
return T;
|
||||
}
|
||||
|
||||
if ( ss?$threshold_series &&
|
||||
|ss$threshold_series| >= tt$threshold_series_index &&
|
||||
watch >= ss$threshold_series[tt$threshold_series_index] )
|
||||
{
|
||||
# A threshold series was given and the value crossed the next
|
||||
# value in the series.
|
||||
return T;
|
||||
}
|
||||
|
||||
return F;
|
||||
}
|
||||
|
||||
function threshold_crossed(ss: SumStat, key: Key, result: Result)
|
||||
{
|
||||
# If there is no callback, there is no point in any of this.
|
||||
if ( ! ss?$threshold_crossed )
|
||||
return;
|
||||
|
||||
# Add in the extra ResultVals to make threshold_crossed callbacks easier to write.
|
||||
if ( |ss$reducers| != |result| )
|
||||
{
|
||||
for ( reducer in ss$reducers )
|
||||
{
|
||||
if ( reducer$stream !in result )
|
||||
result[reducer$stream] = init_resultval(reducer);
|
||||
}
|
||||
}
|
||||
|
||||
ss$threshold_crossed(key, result);
|
||||
local tt = threshold_tracker[ss$id][key];
|
||||
tt$is_threshold_crossed = T;
|
||||
|
||||
# Bump up to the next threshold series index if a threshold series is being used.
|
||||
if ( ss?$threshold_series )
|
||||
++tt$threshold_series_index;
|
||||
}
|
||||
|
||||
24
scripts/base/frameworks/sumstats/non-cluster.bro
Normal file
24
scripts/base/frameworks/sumstats/non-cluster.bro
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
@load ./main
|
||||
|
||||
module SumStats;
|
||||
|
||||
event SumStats::finish_epoch(ss: SumStat)
|
||||
{
|
||||
if ( ss$id in result_store )
|
||||
{
|
||||
local data = result_store[ss$id];
|
||||
if ( ss?$epoch_finished )
|
||||
ss$epoch_finished(data);
|
||||
|
||||
reset(ss);
|
||||
}
|
||||
|
||||
schedule ss$epoch { SumStats::finish_epoch(ss) };
|
||||
}
|
||||
|
||||
|
||||
function data_added(ss: SumStat, key: Key, result: Result)
|
||||
{
|
||||
if ( check_thresholds(ss, key, result, 1.0) )
|
||||
threshold_crossed(ss, key, result);
|
||||
}
|
||||
9
scripts/base/frameworks/sumstats/plugins/__load__.bro
Normal file
9
scripts/base/frameworks/sumstats/plugins/__load__.bro
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
@load ./average
|
||||
@load ./last
|
||||
@load ./max
|
||||
@load ./min
|
||||
@load ./sample
|
||||
@load ./std-dev
|
||||
@load ./sum
|
||||
@load ./unique
|
||||
@load ./variance
|
||||
36
scripts/base/frameworks/sumstats/plugins/average.bro
Normal file
36
scripts/base/frameworks/sumstats/plugins/average.bro
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
@load base/frameworks/sumstats/main
|
||||
|
||||
module SumStats;
|
||||
|
||||
export {
|
||||
redef enum Calculation += {
|
||||
## Calculate the average of the values.
|
||||
AVERAGE
|
||||
};
|
||||
|
||||
redef record ResultVal += {
|
||||
## For numeric data, this calculates the average of all values.
|
||||
average: double &optional;
|
||||
};
|
||||
}
|
||||
|
||||
hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
|
||||
{
|
||||
if ( AVERAGE in r$apply )
|
||||
{
|
||||
if ( ! rv?$average )
|
||||
rv$average = val;
|
||||
else
|
||||
rv$average += (val - rv$average) / rv$num;
|
||||
}
|
||||
}
|
||||
|
||||
hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
|
||||
{
|
||||
if ( rv1?$average && rv2?$average )
|
||||
result$average = ((rv1$average*rv1$num) + (rv2$average*rv2$num))/(rv1$num+rv2$num);
|
||||
else if ( rv1?$average )
|
||||
result$average = rv1$average;
|
||||
else if ( rv2?$average )
|
||||
result$average = rv2$average;
|
||||
}
|
||||
55
scripts/base/frameworks/sumstats/plugins/last.bro
Normal file
55
scripts/base/frameworks/sumstats/plugins/last.bro
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
@load base/frameworks/sumstats
|
||||
@load base/utils/queue
|
||||
|
||||
module SumStats;
|
||||
|
||||
export {
|
||||
redef enum Calculation += {
|
||||
## Keep last X observations in a queue
|
||||
LAST
|
||||
};
|
||||
|
||||
redef record Reducer += {
|
||||
## number of elements to keep.
|
||||
num_last_elements: count &default=0;
|
||||
};
|
||||
|
||||
redef record ResultVal += {
|
||||
## This is the queue where elements are maintained. Use the
|
||||
## :bro:see:`SumStats::get_last` function to get a vector of
|
||||
## the current element values.
|
||||
last_elements: Queue::Queue &optional;
|
||||
};
|
||||
|
||||
## Get a vector of element values from a ResultVal.
|
||||
global get_last: function(rv: ResultVal): vector of Observation;
|
||||
}
|
||||
|
||||
function get_last(rv: ResultVal): vector of Observation
|
||||
{
|
||||
local s: vector of Observation = vector();
|
||||
if ( rv?$last_elements )
|
||||
Queue::get_vector(rv$last_elements, s);
|
||||
return s;
|
||||
}
|
||||
|
||||
hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
|
||||
{
|
||||
if ( LAST in r$apply && r$num_last_elements > 0 )
|
||||
{
|
||||
if ( ! rv?$last_elements )
|
||||
rv$last_elements = Queue::init([$max_len=r$num_last_elements]);
|
||||
Queue::put(rv$last_elements, obs);
|
||||
}
|
||||
}
|
||||
|
||||
hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
|
||||
{
|
||||
# Merge $samples
|
||||
if ( rv1?$last_elements && rv2?$last_elements )
|
||||
result$last_elements = Queue::merge(rv1$last_elements, rv2$last_elements);
|
||||
else if ( rv1?$last_elements )
|
||||
result$last_elements = rv1$last_elements;
|
||||
else if ( rv2?$last_elements )
|
||||
result$last_elements = rv2$last_elements;
|
||||
}
|
||||
38
scripts/base/frameworks/sumstats/plugins/max.bro
Normal file
38
scripts/base/frameworks/sumstats/plugins/max.bro
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
@load base/frameworks/sumstats/main
|
||||
|
||||
module SumStats;
|
||||
|
||||
export {
|
||||
redef enum Calculation += {
|
||||
## Find the maximum value.
|
||||
MAX
|
||||
};
|
||||
|
||||
redef record ResultVal += {
|
||||
## For numeric data, this tracks the maximum value given.
|
||||
max: double &optional;
|
||||
};
|
||||
}
|
||||
|
||||
hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
|
||||
{
|
||||
if ( MAX in r$apply )
|
||||
{
|
||||
if ( ! rv?$max )
|
||||
rv$max = val;
|
||||
else if ( val > rv$max )
|
||||
rv$max = val;
|
||||
}
|
||||
}
|
||||
|
||||
hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
|
||||
{
|
||||
if ( rv1?$max && rv2?$max )
|
||||
result$max = (rv1$max > rv2$max) ? rv1$max : rv2$max;
|
||||
else if ( rv1?$max )
|
||||
result$max = rv1$max;
|
||||
else if ( rv2?$max )
|
||||
result$max = rv2$max;
|
||||
}
|
||||
|
||||
|
||||
36
scripts/base/frameworks/sumstats/plugins/min.bro
Normal file
36
scripts/base/frameworks/sumstats/plugins/min.bro
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
@load base/frameworks/sumstats/main
|
||||
|
||||
module SumStats;
|
||||
|
||||
export {
|
||||
redef enum Calculation += {
|
||||
## Find the minimum value.
|
||||
MIN
|
||||
};
|
||||
|
||||
redef record ResultVal += {
|
||||
## For numeric data, this tracks the minimum value given.
|
||||
min: double &optional;
|
||||
};
|
||||
}
|
||||
|
||||
hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
|
||||
{
|
||||
if ( MIN in r$apply )
|
||||
{
|
||||
if ( ! rv?$min )
|
||||
rv$min = val;
|
||||
else if ( val < rv$min )
|
||||
rv$min = val;
|
||||
}
|
||||
}
|
||||
|
||||
hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
|
||||
{
|
||||
if ( rv1?$min && rv2?$min )
|
||||
result$min = (rv1$min < rv2$min) ? rv1$min : rv2$min;
|
||||
else if ( rv1?$min )
|
||||
result$min = rv1$min;
|
||||
else if ( rv2?$min )
|
||||
result$min = rv2$min;
|
||||
}
|
||||
120
scripts/base/frameworks/sumstats/plugins/sample.bro
Normal file
120
scripts/base/frameworks/sumstats/plugins/sample.bro
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
@load base/frameworks/sumstats/main
|
||||
|
||||
module SumStats;
|
||||
|
||||
export {
|
||||
redef enum Calculation += {
|
||||
## Get uniquely distributed random samples from the observation stream.
|
||||
SAMPLE
|
||||
};
|
||||
|
||||
redef record Reducer += {
|
||||
## A number of sample Observations to collect.
|
||||
num_samples: count &default=0;
|
||||
};
|
||||
|
||||
redef record ResultVal += {
|
||||
## This is the vector in which the samples are maintained.
|
||||
samples: vector of Observation &default=vector();
|
||||
|
||||
## Number of total observed elements.
|
||||
sample_elements: count &default=0;
|
||||
};
|
||||
}
|
||||
|
||||
redef record ResultVal += {
|
||||
# Internal use only. This is not meant to be publically available
|
||||
# and just a copy of num_samples from the Reducer. Needed for availability
|
||||
# in the compose hook.
|
||||
num_samples: count &default=0;
|
||||
};
|
||||
|
||||
hook init_resultval_hook(r: Reducer, rv: ResultVal)
|
||||
{
|
||||
if ( SAMPLE in r$apply )
|
||||
rv$num_samples = r$num_samples;
|
||||
}
|
||||
|
||||
function sample_add_sample(obs:Observation, rv: ResultVal)
|
||||
{
|
||||
++rv$sample_elements;
|
||||
|
||||
if ( |rv$samples| < rv$num_samples )
|
||||
rv$samples[|rv$samples|] = obs;
|
||||
else
|
||||
{
|
||||
local ra = rand(rv$sample_elements);
|
||||
if ( ra < rv$num_samples )
|
||||
rv$samples[ra] = obs;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
|
||||
{
|
||||
if ( SAMPLE in r$apply )
|
||||
{
|
||||
sample_add_sample(obs, rv);
|
||||
}
|
||||
}
|
||||
|
||||
hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
|
||||
{
|
||||
if ( rv1$num_samples != rv2$num_samples )
|
||||
{
|
||||
Reporter::error("Merging sample sets with differing sizes is not supported");
|
||||
return;
|
||||
}
|
||||
|
||||
local num_samples = rv1$num_samples;
|
||||
result$num_samples = num_samples;
|
||||
|
||||
if ( |rv1$samples| > num_samples || |rv2$samples| > num_samples )
|
||||
{
|
||||
Reporter::error("Sample vector with too many elements. Aborting.");
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
if ( |rv1$samples| != num_samples && |rv2$samples| < num_samples )
|
||||
{
|
||||
if ( |rv1$samples| != rv1$sample_elements || |rv2$samples| < rv2$sample_elements )
|
||||
{
|
||||
Reporter::error("Mismatch in sample element size and tracking. Aborting merge");
|
||||
return;
|
||||
}
|
||||
|
||||
for ( i in rv1$samples )
|
||||
sample_add_sample(rv1$samples[i], result);
|
||||
|
||||
for ( i in rv2$samples)
|
||||
sample_add_sample(rv2$samples[i], result);
|
||||
}
|
||||
else
|
||||
{
|
||||
local other_vector: vector of Observation;
|
||||
local othercount: count;
|
||||
|
||||
if ( rv1$sample_elements > rv2$sample_elements )
|
||||
{
|
||||
result$samples = copy(rv1$samples);
|
||||
other_vector = rv2$samples;
|
||||
othercount = rv2$sample_elements;
|
||||
}
|
||||
else
|
||||
{
|
||||
result$samples = copy(rv2$samples);
|
||||
other_vector = rv1$samples;
|
||||
othercount = rv1$sample_elements;
|
||||
}
|
||||
|
||||
local totalcount = rv1$sample_elements + rv2$sample_elements;
|
||||
result$sample_elements = totalcount;
|
||||
|
||||
for ( i in other_vector )
|
||||
{
|
||||
if ( rand(totalcount) <= othercount )
|
||||
result$samples[i] = other_vector[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue