diff --git a/.gitmodules b/.gitmodules index 95053091cf..2ede715f49 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,3 +16,6 @@ [submodule "cmake"] path = cmake url = git://git.bro-ids.org/cmake +[submodule "magic"] + path = magic + url = git://git.bro.org/bromagic diff --git a/CHANGES b/CHANGES index 912d7d301f..693dc36dac 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,90 @@ +2.1-945 | 2013-07-30 10:05:10 -0700 + + * Make hashers serializable. (Matthias Vallentin) + + * Add docs and use default value for hasher names. (Matthias + Vallentin) + +2.1-939 | 2013-07-29 15:42:38 -0700 + + * Added Exec, Dir, and ActiveHTTP modules. (Seth Hall) + + base/utils/exec.bro provides a module to start external processes + asynchronously and retrieve their output on termination. + base/utils/dir.bro uses it to monitor a directory for changes, and + base/utils/active-http.bro for providing an interface for querying + remote web servers. + +2.1-930 | 2013-07-29 15:06:07 -0700 + + * Major file analysis overhaul in naming and appearance, along with + fixes and test updates. (Seth Hall and Jon Siwek) + + Includes: + + * Added protocol description functions that provide a super + compressed log representation. (Seth Hall) + + * Added mime types to http.log (Seth Hall) + + * Add jar files to the default MHR lookups. (Seth Hall) + + * Adding CAB files for MHR checking. (Seth Hall) + + * Improve malware hash registry script. + + - Include a link to a virustotal search in the notice sub message field. + - Give all information returned from Team Cymru in the notice message. + - Add more file types to match on to the default set. + + * Make the custom libmagic database a git submodule. + + * Add an is_orig parameter to file_over_new_connection event. + + * Recorrected the module name to Files. + + * Added Files::analyzer_name to get a more readable name for a + file analyzer. + + * Improved and just overall better handled multipart mime + transfers in HTTP and SMTP. HTTP now has orig_fuids and + resp_fuids log fields since multiple "files" can be transferred + with multipart mime in a single request/response pair. SMTP has + an fuids field which has file unique IDs for all parts + transferred. FTP and IRC have a log field named fuid added + because only a single file can be transferred per irc and ftp + log line. + +2.1-895 | 2013-07-29 14:07:35 -0700 + + * Adding a test for a DNSKEY RR. (Robin Sommer) + +2.1-894 | 2013-07-29 16:44:41 -0400 + + * Updates for the Intel Framework. (Seth Hall) + + - policy/frameworks/intel/seen is the new location for the + scripts that push data into the intel framework for checking. + + - The new policy/frameworks/intel/do_notice script adds an + example mechanism for data driven notices. + + - Remove the Intel insertion after heuristically detecting SSH + bruteforcing. + + - Intel importing format has changed (refer to docs). + + - All string matching is now case insensitive. + + - SMTP intel script has been updated to extract email + addresses correctly. + + - Small fix sneaking into the smtp base script to actually + extract individual email addresses in the To: field + correctly. + + 2.1-888 | 2013-07-25 12:02:41 -0700 * Protection about broken traces with empty pcap headers. (Matt diff --git a/CMakeLists.txt b/CMakeLists.txt index b95b637770..0f64f304b8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,7 +18,7 @@ get_filename_component(BRO_SCRIPT_INSTALL_PATH ${BRO_SCRIPT_INSTALL_PATH} ABSOLUTE) set(BRO_MAGIC_INSTALL_PATH ${BRO_ROOT_DIR}/share/bro/magic) -set(BRO_MAGIC_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/magic) +set(BRO_MAGIC_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/magic/database) configure_file(bro-path-dev.in ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev) file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.sh @@ -201,9 +201,8 @@ CheckOptionalBuildSources(aux/broctl Broctl INSTALL_BROCTL) CheckOptionalBuildSources(aux/bro-aux Bro-Aux INSTALL_AUX_TOOLS) CheckOptionalBuildSources(aux/broccoli Broccoli INSTALL_BROCCOLI) -install(DIRECTORY ./magic/ DESTINATION ${BRO_MAGIC_INSTALL_PATH} FILES_MATCHING - PATTERN "COPYING" EXCLUDE - PATTERN "*" +install(DIRECTORY ./magic/database/ + DESTINATION ${BRO_MAGIC_INSTALL_PATH} ) ######################################################################## diff --git a/NEWS b/NEWS index c3eabf5554..c421e7d675 100644 --- a/NEWS +++ b/NEWS @@ -80,7 +80,7 @@ New Functionality with the following user-visibible functionality (some of that was already available before, but done differently): - [TODO: This will probably change with further script updates.] + [TODO: Update with changes from 984e9793db56.] - A binary input reader interfaces the input framework with file analysis, allowing to inject files on disk into Bro's @@ -121,6 +121,12 @@ New Functionality See for full documentation. +- base/utils/exec.bro provides a module to start external processes + asynchronously and retrieve their output on termination. + base/utils/dir.bro uses it to monitor a directory for changes, and + base/utils/active-http.bro for providing an interface for querying + remote web servers. + Changed Functionality ~~~~~~~~~~~~~~~~~~~~~ diff --git a/VERSION b/VERSION index 4f0ea7a5ac..6b605113b8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1-888 +2.1-945 diff --git a/aux/binpac b/aux/binpac index 896ddedde5..314fa8f65f 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 896ddedde55c48ec2163577fc258b49c418abb3e +Subproject commit 314fa8f65fc240e960c23c3bba98623436a72b98 diff --git a/aux/bro-aux b/aux/bro-aux index a9942558c7..91d258cc8b 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit a9942558c7d3dfd80148b8aaded64c82ade3d117 +Subproject commit 91d258cc8b2f74cd02fc93dfe61f73ec9f0dd489 diff --git a/aux/broccoli b/aux/broccoli index 889f9c6594..d59c73b6e0 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 889f9c65944ceac20ad9230efc39d33e6e1221c3 +Subproject commit d59c73b6e0966ad63bbc63a35741b5f68263e7b1 diff --git a/aux/broctl b/aux/broctl index 0cd102805e..52fd91261f 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 0cd102805e73343cab3f9fd4a76552e13940dad9 +Subproject commit 52fd91261f41fa1528f7b964837a364d7991889e diff --git a/doc/intel.rst b/doc/intel.rst index 390313461a..787524a417 100644 --- a/doc/intel.rst +++ b/doc/intel.rst @@ -27,10 +27,7 @@ Quick Start Load the package of scripts that sends data into the Intelligence Framework to be checked by loading this script in local.bro:: - @load policy/frameworks/intel - -(TODO: find some good mechanism for getting setup with good data -quickly) + @load policy/frameworks/intel/seen Refer to the "Loading Intelligence" section below to see the format for Intelligence Framework text files, then load those text files with @@ -61,16 +58,14 @@ data out to all of the nodes that need it. Here is an example of the intelligence data format. Note that all whitespace separators are literal tabs and fields containing only a -hyphen a considered to be null values.:: +hyphen are considered to be null values.:: - #fields host net str str_type meta.source meta.desc meta.url - 1.2.3.4 - - - source1 Sending phishing email http://source1.com/badhosts/1.2.3.4 - - 31.131.248.0/21 - - spamhaus-drop SBL154982 - - - - - a.b.com Intel::DOMAIN source2 Name used for data exfiltration - + #fields indicator indicator_type meta.source meta.desc meta.url + 1.2.3.4 Intel::ADDR source1 Sending phishing email http://source1.com/badhosts/1.2.3.4 + a.b.com Intel::DOMAIN source2 Name used for data exfiltration - -For more examples of built in `str_type` values, please refer to the -autogenerated documentation for the intelligence framework (TODO: -figure out how to do this link). +For more examples of built in `indicator_type` values, please refer to the +autogenerated documentation for the intelligence framework. To load the data once files are created, use the following example code to define files to load with your own file names of course:: @@ -90,8 +85,7 @@ When some bit of data is extracted (such as an email address in the "From" header in a message over SMTP), the Intelligence Framework needs to be informed that this data was discovered and it's presence should be checked within the intelligence data set. This is -accomplished through the Intel::seen (TODO: do a reference link) -function. +accomplished through the Intel::seen function. Typically users won't need to work with this function due to built in hook scripts that Bro ships with that will "see" data and send it into @@ -106,7 +100,7 @@ The full package of hook scripts that Bro ships with for sending this "seen" data into the intelligence framework can be loading by adding this line to local.bro:: - @load policy/frameworks/intel + @load policy/frameworks/intel/seen Intelligence Matches ******************** diff --git a/doc/scripts/CMakeLists.txt b/doc/scripts/CMakeLists.txt index ddb09bb29c..e7e39d0b3f 100644 --- a/doc/scripts/CMakeLists.txt +++ b/doc/scripts/CMakeLists.txt @@ -99,7 +99,7 @@ macro(REST_TARGET srcDir broInput) COMMAND "${CMAKE_COMMAND}" ARGS -E remove_directory .state # generate the reST documentation using bro - COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic ${CMAKE_BINARY_DIR}/src/bro + COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic/database ${CMAKE_BINARY_DIR}/src/bro ARGS -b -Z ${broInput} || (rm -rf .state *.log *.rst && exit 1) # move generated doc into a new directory tree that # defines the final structure of documents @@ -130,7 +130,7 @@ add_custom_command(OUTPUT proto-analyzers.rst COMMAND "${CMAKE_COMMAND}" ARGS -E remove_directory .state # generate the reST documentation using bro - COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic ${CMAKE_BINARY_DIR}/src/bro + COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic/database ${CMAKE_BINARY_DIR}/src/bro ARGS -b -Z base/init-bare.bro || (rm -rf .state *.log *.rst && exit 1) # move generated doc into a new directory tree that # defines the final structure of documents diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake index 26a88027ef..2efa45ef38 100644 --- a/doc/scripts/DocSourcesList.cmake +++ b/doc/scripts/DocSourcesList.cmake @@ -74,6 +74,8 @@ rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ZIP.events.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/reporter.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/strings.bif.bro) rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/types.bif.bro) +rest_target(${psd} base/files/extract/main.bro) +rest_target(${psd} base/files/hash/main.bro) rest_target(${psd} base/frameworks/analyzer/main.bro) rest_target(${psd} base/frameworks/cluster/main.bro) rest_target(${psd} base/frameworks/cluster/nodes/manager.bro) @@ -83,7 +85,7 @@ rest_target(${psd} base/frameworks/cluster/setup-connections.bro) rest_target(${psd} base/frameworks/communication/main.bro) rest_target(${psd} base/frameworks/control/main.bro) rest_target(${psd} base/frameworks/dpd/main.bro) -rest_target(${psd} base/frameworks/file-analysis/main.bro) +rest_target(${psd} base/frameworks/files/main.bro) rest_target(${psd} base/frameworks/input/main.bro) rest_target(${psd} base/frameworks/input/readers/ascii.bro) rest_target(${psd} base/frameworks/input/readers/benchmark.bro) @@ -137,25 +139,22 @@ rest_target(${psd} base/protocols/conn/main.bro) rest_target(${psd} base/protocols/conn/polling.bro) rest_target(${psd} base/protocols/dns/consts.bro) rest_target(${psd} base/protocols/dns/main.bro) -rest_target(${psd} base/protocols/ftp/file-analysis.bro) -rest_target(${psd} base/protocols/ftp/file-extract.bro) +rest_target(${psd} base/protocols/ftp/files.bro) rest_target(${psd} base/protocols/ftp/gridftp.bro) rest_target(${psd} base/protocols/ftp/main.bro) rest_target(${psd} base/protocols/ftp/utils-commands.bro) -rest_target(${psd} base/protocols/http/file-analysis.bro) -rest_target(${psd} base/protocols/http/file-extract.bro) -rest_target(${psd} base/protocols/http/file-hash.bro) -rest_target(${psd} base/protocols/http/file-ident.bro) +rest_target(${psd} base/protocols/ftp/utils.bro) +rest_target(${psd} base/protocols/http/entities.bro) +rest_target(${psd} base/protocols/http/files.bro) rest_target(${psd} base/protocols/http/main.bro) rest_target(${psd} base/protocols/http/utils.bro) rest_target(${psd} base/protocols/irc/dcc-send.bro) -rest_target(${psd} base/protocols/irc/file-analysis.bro) +rest_target(${psd} base/protocols/irc/files.bro) rest_target(${psd} base/protocols/irc/main.bro) rest_target(${psd} base/protocols/modbus/consts.bro) rest_target(${psd} base/protocols/modbus/main.bro) -rest_target(${psd} base/protocols/smtp/entities-excerpt.bro) rest_target(${psd} base/protocols/smtp/entities.bro) -rest_target(${psd} base/protocols/smtp/file-analysis.bro) +rest_target(${psd} base/protocols/smtp/files.bro) rest_target(${psd} base/protocols/smtp/main.bro) rest_target(${psd} base/protocols/socks/consts.bro) rest_target(${psd} base/protocols/socks/main.bro) @@ -165,9 +164,12 @@ rest_target(${psd} base/protocols/ssl/main.bro) rest_target(${psd} base/protocols/ssl/mozilla-ca-list.bro) rest_target(${psd} base/protocols/syslog/consts.bro) rest_target(${psd} base/protocols/syslog/main.bro) +rest_target(${psd} base/utils/active-http.bro) rest_target(${psd} base/utils/addrs.bro) rest_target(${psd} base/utils/conn-ids.bro) +rest_target(${psd} base/utils/dir.bro) rest_target(${psd} base/utils/directions-and-hosts.bro) +rest_target(${psd} base/utils/exec.bro) rest_target(${psd} base/utils/files.bro) rest_target(${psd} base/utils/numbers.bro) rest_target(${psd} base/utils/paths.bro) @@ -183,15 +185,18 @@ rest_target(${psd} policy/frameworks/control/controllee.bro) rest_target(${psd} policy/frameworks/control/controller.bro) rest_target(${psd} policy/frameworks/dpd/detect-protocols.bro) rest_target(${psd} policy/frameworks/dpd/packet-segment-logging.bro) -rest_target(${psd} policy/frameworks/intel/conn-established.bro) -rest_target(${psd} policy/frameworks/intel/dns.bro) -rest_target(${psd} policy/frameworks/intel/http-host-header.bro) -rest_target(${psd} policy/frameworks/intel/http-url.bro) -rest_target(${psd} policy/frameworks/intel/http-user-agents.bro) -rest_target(${psd} policy/frameworks/intel/smtp-url-extraction.bro) -rest_target(${psd} policy/frameworks/intel/smtp.bro) -rest_target(${psd} policy/frameworks/intel/ssl.bro) -rest_target(${psd} policy/frameworks/intel/where-locations.bro) +rest_target(${psd} policy/frameworks/files/detect-MHR.bro) +rest_target(${psd} policy/frameworks/files/hash-all-files.bro) +rest_target(${psd} policy/frameworks/intel/do_notice.bro) +rest_target(${psd} policy/frameworks/intel/seen/conn-established.bro) +rest_target(${psd} policy/frameworks/intel/seen/dns.bro) +rest_target(${psd} policy/frameworks/intel/seen/http-host-header.bro) +rest_target(${psd} policy/frameworks/intel/seen/http-url.bro) +rest_target(${psd} policy/frameworks/intel/seen/http-user-agents.bro) +rest_target(${psd} policy/frameworks/intel/seen/smtp-url-extraction.bro) +rest_target(${psd} policy/frameworks/intel/seen/smtp.bro) +rest_target(${psd} policy/frameworks/intel/seen/ssl.bro) +rest_target(${psd} policy/frameworks/intel/seen/where-locations.bro) rest_target(${psd} policy/frameworks/packet-filter/shunt.bro) rest_target(${psd} policy/frameworks/software/version-changes.bro) rest_target(${psd} policy/frameworks/software/vulnerable.bro) @@ -215,7 +220,6 @@ rest_target(${psd} policy/protocols/dns/detect-external-names.bro) rest_target(${psd} policy/protocols/ftp/detect-bruteforcing.bro) rest_target(${psd} policy/protocols/ftp/detect.bro) rest_target(${psd} policy/protocols/ftp/software.bro) -rest_target(${psd} policy/protocols/http/detect-MHR.bro) rest_target(${psd} policy/protocols/http/detect-sqli.bro) rest_target(${psd} policy/protocols/http/detect-webapps.bro) rest_target(${psd} policy/protocols/http/header-names.bro) @@ -227,6 +231,7 @@ rest_target(${psd} policy/protocols/modbus/known-masters-slaves.bro) rest_target(${psd} policy/protocols/modbus/track-memmap.bro) rest_target(${psd} policy/protocols/smtp/blocklists.bro) rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro) +rest_target(${psd} policy/protocols/smtp/entities-excerpt.bro) rest_target(${psd} policy/protocols/smtp/software.bro) rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro) rest_target(${psd} policy/protocols/ssh/geo-data.bro) diff --git a/magic b/magic new file mode 160000 index 0000000000..e87fe13a7b --- /dev/null +++ b/magic @@ -0,0 +1 @@ +Subproject commit e87fe13a7b776182ffc8c75076d42702f5c28fed diff --git a/magic/COPYING b/magic/COPYING deleted file mode 100644 index 7d2bf1e711..0000000000 --- a/magic/COPYING +++ /dev/null @@ -1,29 +0,0 @@ -# $File: LEGAL.NOTICE,v 1.15 2006/05/03 18:48:33 christos Exp $ -# Copyright (c) Ian F. Darwin 1986, 1987, 1989, 1990, 1991, 1992, 1994, 1995. -# Software written by Ian F. Darwin and others; -# maintained 1994- Christos Zoulas. -# -# This software is not subject to any export provision of the United States -# Department of Commerce, and may be exported to any country or planet. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# 1. Redistributions of source code must retain the above copyright -# notice immediately at the beginning of the file, without modification, -# this list of conditions, and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR -# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -# SUCH DAMAGE. diff --git a/magic/animation b/magic/animation deleted file mode 100644 index 0cec03d511..0000000000 --- a/magic/animation +++ /dev/null @@ -1,208 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: animation,v 1.47 2013/02/06 14:18:52 christos Exp $ -# animation: file(1) magic for animation/movie formats -# -# animation formats -# MPEG, FLI, DL originally from vax@ccwf.cc.utexas.edu (VaX#n8) -# FLC, SGI, Apple originally from Daniel Quinlan (quinlan@yggdrasil.com) - -# SGI and Apple formats -0 string MOVI Silicon Graphics movie file -!:mime video/x-sgi-movie -4 string moov Apple QuickTime -!:mime video/quicktime -4 string mdat Apple QuickTime movie (unoptimized) -!:mime video/quicktime -#4 string wide Apple QuickTime movie (unoptimized) -#!:mime video/quicktime -#4 string skip Apple QuickTime movie (modified) -#!:mime video/quicktime -#4 string free Apple QuickTime movie (modified) -#!:mime video/quicktime -4 string idsc Apple QuickTime image (fast start) -!:mime image/x-quicktime -#4 string idat Apple QuickTime image (unoptimized) -#!:mime image/x-quicktime -4 string pckg Apple QuickTime compressed archive -!:mime application/x-quicktime-player -4 string/W jP JPEG 2000 image -!:mime image/jp2 -4 string ftyp ISO Media ->8 string isom \b, MPEG v4 system, version 1 -!:mime video/mp4 ->8 string mp41 \b, MPEG v4 system, version 1 -!:mime video/mp4 ->8 string mp42 \b, MPEG v4 system, version 2 -!:mime video/mp4 ->8 string/W jp2 \b, JPEG 2000 -!:mime image/jp2 ->8 string 3ge \b, MPEG v4 system, 3GPP -!:mime video/3gpp ->8 string 3gg \b, MPEG v4 system, 3GPP -!:mime video/3gpp ->8 string 3gp \b, MPEG v4 system, 3GPP -!:mime video/3gpp ->8 string 3gs \b, MPEG v4 system, 3GPP -!:mime video/3gpp ->8 string 3g2 \b, MPEG v4 system, 3GPP2 -!:mime video/3gpp2 ->8 string mmp4 \b, MPEG v4 system, 3GPP Mobile -!:mime video/mp4 ->8 string avc1 \b, MPEG v4 system, 3GPP JVT AVC -!:mime video/3gpp ->8 string/W M4A \b, MPEG v4 system, iTunes AAC-LC -!:mime audio/mp4 ->8 string/W M4V \b, MPEG v4 system, iTunes AVC-LC -!:mime video/mp4 ->8 string/W qt \b, Apple QuickTime movie -!:mime video/quicktime - -# MPEG sequences -# Scans for all common MPEG header start codes -0 belong&0xFFFFFF00 0x00000100 ->3 byte 0xBA MPEG sequence -!:mime video/mpeg -# GRR too general as it catches also FoxPro Memo example NG.FPT ->3 byte 0xB0 MPEG sequence, v4 -!:mime video/mpeg4-generic ->3 byte 0xB5 MPEG sequence, v4 -!:mime video/mpeg4-generic ->3 byte 0xB3 MPEG sequence -!:mime video/mpeg - -# MPEG ADTS Audio (*.mpx/mxa/aac) -# from dreesen@math.fu-berlin.de -# modified to fully support MPEG ADTS - -# MP3, M1A -# modified by Joerg Jenderek -# GRR the original test are too common for many DOS files -# so don't accept as MP3 until we've tested the rate -0 beshort&0xFFFE 0xFFFA -# rates ->2 byte&0xF0 0x10 MPEG ADTS, layer III, v1, 32 kbps -!:mime audio/mpeg ->2 byte&0xF0 0x20 MPEG ADTS, layer III, v1, 40 kbps -!:mime audio/mpeg ->2 byte&0xF0 0x30 MPEG ADTS, layer III, v1, 48 kbps -!:mime audio/mpeg ->2 byte&0xF0 0x40 MPEG ADTS, layer III, v1, 56 kbps -!:mime audio/mpeg ->2 byte&0xF0 0x50 MPEG ADTS, layer III, v1, 64 kbps -!:mime audio/mpeg ->2 byte&0xF0 0x60 MPEG ADTS, layer III, v1, 80 kbps -!:mime audio/mpeg ->2 byte&0xF0 0x70 MPEG ADTS, layer III, v1, 96 kbps -!:mime audio/mpeg ->2 byte&0xF0 0x80 MPEG ADTS, layer III, v1, 112 kbps -!:mime audio/mpeg ->2 byte&0xF0 0x90 MPEG ADTS, layer III, v1, 128 kbps -!:mime audio/mpeg ->2 byte&0xF0 0xA0 MPEG ADTS, layer III, v1, 160 kbps -!:mime audio/mpeg ->2 byte&0xF0 0xB0 MPEG ADTS, layer III, v1, 192 kbps -!:mime audio/mpeg ->2 byte&0xF0 0xC0 MPEG ADTS, layer III, v1, 224 kbps -!:mime audio/mpeg ->2 byte&0xF0 0xD0 MPEG ADTS, layer III, v1, 256 kbps -!:mime audio/mpeg ->2 byte&0xF0 0xE0 MPEG ADTS, layer III, v1, 320 kbps -!:mime audio/mpeg - -# MP2, M1A -0 beshort&0xFFFE 0xFFFC MPEG ADTS, layer II, v1 -!:mime audio/mpeg - -# MP3, M2A -0 beshort&0xFFFE 0xFFF2 MPEG ADTS, layer III, v2 -!:mime audio/mpeg - -# MPA, M2A -0 beshort&0xFFFE 0xFFF6 MPEG ADTS, layer I, v2 -!:mime audio/mpeg - -# MP3, M25A -0 beshort&0xFFFE 0xFFE2 MPEG ADTS, layer III, v2.5 -!:mime audio/mpeg - -# Stored AAC streams (instead of the MP4 format) -0 string ADIF MPEG ADIF, AAC -!:mime audio/x-hx-aac-adif - -# Live or stored single AAC stream (used with MPEG-2 systems) -0 beshort&0xFFF6 0xFFF0 MPEG ADTS, AAC -!:mime audio/x-hx-aac-adts - -# Live MPEG-4 audio streams (instead of RTP FlexMux) -0 beshort&0xFFE0 0x56E0 MPEG-4 LOAS -!:mime audio/x-mp4a-latm - -# This magic isn't strong enough (matches plausible ISO-8859-1 text) -#0 beshort 0x4DE1 MPEG-4 LO-EP audio stream -#!:mime audio/x-mp4a-latm - -# Summary: FLI animation format -# Created by: Daniel Quinlan -# Modified by (1): Abel Cheung (avoid over-generic detection) -4 leshort 0xAF11 -# standard FLI always has 320x200 resolution and 8 bit color ->8 leshort 320 ->>10 leshort 200 ->>>12 leshort 8 FLI animation, 320x200x8 -!:mime video/x-fli - -# Summary: FLC animation format -# Created by: Daniel Quinlan -# Modified by (1): Abel Cheung (avoid over-generic detection) -4 leshort 0xAF12 -# standard FLC always use 8 bit color ->12 leshort 8 FLC animation -!:mime video/x-flc - -# Microsoft Advanced Streaming Format (ASF) -0 belong 0x3026b275 Microsoft ASF -!:mime video/x-ms-asf - -# MNG Video Format, -0 string \x8aMNG MNG video data, -!:mime video/x-mng - -# JNG Video Format, -0 string \x8bJNG JNG video data, -!:mime video/x-jng - -# VRML (Virtual Reality Modelling Language) -0 string/w #VRML\ V1.0\ ascii VRML 1 file -!:mime model/vrml -0 string/w #VRML\ V2.0\ utf8 ISO/IEC 14772 VRML 97 file -!:mime model/vrml - -# X3D (Extensible 3D) [http://www.web3d.org/specifications/x3d-3.0.dtd] -# From Michel Briand -0 string/t \20 search/1000/cw \4 byte &0x40 -!:mime video/mp2p ->4 byte ^0x40 -!:mime video/mpeg -0 belong 0x000001BB -!:mime video/mpeg -0 belong 0x000001B0 -!:mime video/mp4v-es -0 belong 0x000001B5 -!:mime video/mp4v-es -0 belong 0x000001B3 -!:mime video/mpv -0 belong&0xFF5FFF1F 0x47400010 -!:mime video/mp2t -0 belong 0x00000001 ->4 byte&0x1F 0x07 -!:mime video/h264 diff --git a/magic/archive b/magic/archive deleted file mode 100644 index 35cbef4012..0000000000 --- a/magic/archive +++ /dev/null @@ -1,242 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: archive,v 1.78 2013/02/06 14:18:52 christos Exp $ -# archive: file(1) magic for archive formats (see also "msdos" for self- -# extracting compressed archives) -# -# cpio, ar, arc, arj, hpack, lha/lharc, rar, squish, uc2, zip, zoo, etc. -# pre-POSIX "tar" archives are handled in the C code. - -# POSIX tar archives -257 string ustar\0 POSIX tar archive -!:mime application/x-tar # encoding: posix -257 string ustar\040\040\0 GNU tar archive -!:mime application/x-tar # encoding: gnu - -# cpio archives -# -# Yes, the top two "cpio archive" formats *are* supposed to just be "short". -# The idea is to indicate archives produced on machines with the same -# byte order as the machine running "file" with "cpio archive", and -# to indicate archives produced on machines with the opposite byte order -# from the machine running "file" with "byte-swapped cpio archive". -# -# The SVR4 "cpio(4)" hints that there are additional formats, but they -# are defined as "short"s; I think all the new formats are -# character-header formats and thus are strings, not numbers. -0 short 070707 cpio archive -!:mime application/x-cpio -0 short 0143561 byte-swapped cpio archive -!:mime application/x-cpio # encoding: swapped - -# -# System V Release 1 portable(?) archive format. -# -0 string = System V Release 1 ar archive -!:mime application/x-archive - -# -# Debian package; it's in the portable archive format, and needs to go -# before the entry for regular portable archives, as it's recognized as -# a portable archive whose first member has a name beginning with -# "debian". -# -0 string =!\ndebian -!:mime application/x-debian-package - -# -# MIPS archive; they're in the portable archive format, and need to go -# before the entry for regular portable archives, as it's recognized as -# a portable archive whose first member has a name beginning with -# "__________E". -# -0 string =!\n__________E MIPS archive -!:mime application/x-archive - -# -# BSD/SVR2-and-later portable archive formats. -# -0 string =! current ar archive -!:mime application/x-archive - -# ARC archiver, from Daniel Quinlan (quinlan@yggdrasil.com) -# -# The first byte is the magic (0x1a), byte 2 is the compression type for -# the first file (0x01 through 0x09), and bytes 3 to 15 are the MS-DOS -# filename of the first file (null terminated). Since some types collide -# we only test some types on basis of frequency: 0x08 (83%), 0x09 (5%), -# 0x02 (5%), 0x03 (3%), 0x04 (2%), 0x06 (2%). 0x01 collides with terminfo. -0 lelong&0x8080ffff 0x0000081a ARC archive data, dynamic LZW -!:mime application/x-arc -0 lelong&0x8080ffff 0x0000091a ARC archive data, squashed -!:mime application/x-arc -0 lelong&0x8080ffff 0x0000021a ARC archive data, uncompressed -!:mime application/x-arc -0 lelong&0x8080ffff 0x0000031a ARC archive data, packed -!:mime application/x-arc -0 lelong&0x8080ffff 0x0000041a ARC archive data, squeezed -!:mime application/x-arc -0 lelong&0x8080ffff 0x0000061a ARC archive data, crunched -!:mime application/x-arc -# [JW] stuff taken from idarc, obviously ARC successors: -0 lelong&0x8080ffff 0x00000a1a PAK archive data -!:mime application/x-arc -0 lelong&0x8080ffff 0x0000141a ARC+ archive data -!:mime application/x-arc -0 lelong&0x8080ffff 0x0000481a HYP archive data -!:mime application/x-arc - -# ARJ archiver (jason@jarthur.Claremont.EDU) -0 leshort 0xea60 ARJ archive data -!:mime application/x-arj - -# LHARC/LHA archiver (Greg Roelofs, newt@uchicago.edu) -2 string -lh0- LHarc 1.x/ARX archive data [lh0] -!:mime application/x-lharc -2 string -lh1- LHarc 1.x/ARX archive data [lh1] -!:mime application/x-lharc -2 string -lz4- LHarc 1.x archive data [lz4] -!:mime application/x-lharc -2 string -lz5- LHarc 1.x archive data [lz5] -!:mime application/x-lharc -# [never seen any but the last; -lh4- reported in comp.compression:] -2 string -lzs- LHa/LZS archive data [lzs] -!:mime application/x-lha -2 string -lh\40- LHa 2.x? archive data [lh ] -!:mime application/x-lha -2 string -lhd- LHa 2.x? archive data [lhd] -!:mime application/x-lha -2 string -lh2- LHa 2.x? archive data [lh2] -!:mime application/x-lha -2 string -lh3- LHa 2.x? archive data [lh3] -!:mime application/x-lha -2 string -lh4- LHa (2.x) archive data [lh4] -!:mime application/x-lha -2 string -lh5- LHa (2.x) archive data [lh5] -!:mime application/x-lha -2 string -lh6- LHa (2.x) archive data [lh6] -!:mime application/x-lha -2 string -lh7- LHa (2.x)/LHark archive data [lh7] -!:mime application/x-lha - -# RAR archiver (Greg Roelofs, newt@uchicago.edu) -0 string Rar! RAR archive data, -!:mime application/x-rar - -# PKZIP multi-volume archive -0 string PK\x07\x08PK\x03\x04 Zip multi-volume archive data, at least PKZIP v2.50 to extract -!:mime application/zip - -# Zip archives (Greg Roelofs, c/o zip-bugs@wkuvx1.wku.edu) -0 string PK\003\004 - -# Specialised zip formats which start with a member named 'mimetype' -# (stored uncompressed, with no 'extra field') containing the file's MIME type. -# Check for have 8-byte name, 0-byte extra field, name "mimetype", and -# contents starting with "application/": ->26 string \x8\0\0\0mimetypeapplication/ - -# OpenDocument formats (for OpenOffice 2.x / StarOffice >= 8) -# http://lists.oasis-open.org/archives/office/200505/msg00006.html -# (mimetype contains "application/vnd.oasis.opendocument.") ->>50 string vnd.oasis.opendocument. OpenDocument ->>>73 string text ->>>>77 byte !0x2d Text -!:mime application/vnd.oasis.opendocument.text ->>>>77 string -template Text Template -!:mime application/vnd.oasis.opendocument.text-template ->>>>77 string -web HTML Document Template -!:mime application/vnd.oasis.opendocument.text-web ->>>>77 string -master Master Document -!:mime application/vnd.oasis.opendocument.text-master ->>>73 string graphics ->>>>81 byte !0x2d Drawing -!:mime application/vnd.oasis.opendocument.graphics ->>>>81 string -template Template -!:mime application/vnd.oasis.opendocument.graphics-template ->>>73 string presentation ->>>>85 byte !0x2d Presentation -!:mime application/vnd.oasis.opendocument.presentation ->>>>85 string -template Template -!:mime application/vnd.oasis.opendocument.presentation-template ->>>73 string spreadsheet ->>>>84 byte !0x2d Spreadsheet -!:mime application/vnd.oasis.opendocument.spreadsheet ->>>>84 string -template Template -!:mime application/vnd.oasis.opendocument.spreadsheet-template ->>>73 string chart ->>>>78 byte !0x2d Chart -!:mime application/vnd.oasis.opendocument.chart ->>>>78 string -template Template -!:mime application/vnd.oasis.opendocument.chart-template ->>>73 string formula ->>>>80 byte !0x2d Formula -!:mime application/vnd.oasis.opendocument.formula ->>>>80 string -template Template -!:mime application/vnd.oasis.opendocument.formula-template ->>>73 string database Database -!:mime application/vnd.oasis.opendocument.database ->>>73 string image ->>>>78 byte !0x2d Image -!:mime application/vnd.oasis.opendocument.image ->>>>78 string -template Template -!:mime application/vnd.oasis.opendocument.image-template - -# EPUB (OEBPS) books using OCF (OEBPS Container Format) -# http://www.idpf.org/ocf/ocf1.0/download/ocf10.htm, section 4. -# From: Ralf Brown ->0x1E string mimetypeapplication/epub+zip EPUB document -!:mime application/epub+zip - -# Catch other ZIP-with-mimetype formats -# In a ZIP file, the bytes immediately after a member's contents are -# always "PK". The 2 regex rules here print the "mimetype" member's -# contents up to the first 'P'. Luckily, most MIME types don't contain -# any capital 'P's. This is a kludge. -# (mimetype contains "application/") ->>50 string !epub+zip ->>>50 string !vnd.oasis.opendocument. ->>>>50 string !vnd.sun.xml. ->>>>>50 string !vnd.kde. ->>>>>>38 regex [!-OQ-~]+ Zip data (MIME type "%s"?) -!:mime application/zip -# (mimetype contents other than "application/*") ->26 string \x8\0\0\0mimetype ->>38 string !application/ ->>>38 regex [!-OQ-~]+ Zip data (MIME type "%s"?) -!:mime application/zip - -# Java Jar files ->(26.s+30) leshort 0xcafe Java Jar file data (zip) -!:mime application/jar - -# Generic zip archives (Greg Roelofs, c/o zip-bugs@wkuvx1.wku.edu) -# Next line excludes specialized formats: ->(26.s+30) leshort !0xcafe ->>26 string !\x8\0\0\0mimetype Zip archive data -!:mime application/zip - -# Zoo archiver -20 lelong 0xfdc4a7dc Zoo archive data -!:mime application/x-zoo - -# Shell archives -10 string #\ This\ is\ a\ shell\ archive shell archive text -!:mime application/octet-stream - -# Felix von Leitner -0 string d8:announce BitTorrent file -!:mime application/x-bittorrent - -# EET archive -# From: Tilman Sauerbeck -0 belong 0x1ee7ff00 EET archive -!:mime application/x-eet - -# Symbian installation files -# http://www.thouky.co.uk/software/psifs/sis.html -# http://developer.symbian.com/main/downloads/papers/SymbianOSv91/softwareinstallsis.pdf -8 lelong 0x10000419 Symbian installation file -!:mime application/vnd.symbian.install -0 lelong 0x10201A7A Symbian installation file (Symbian OS 9.x) -!:mime x-epoc/x-sisx-app diff --git a/magic/assembler b/magic/assembler deleted file mode 100644 index 242b6e19e2..0000000000 --- a/magic/assembler +++ /dev/null @@ -1,19 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: assembler,v 1.3 2013/01/04 17:23:28 christos Exp $ -# make: file(1) magic for assembler source -# -0 regex \^[\020\t]*\\.asciiz assembler source text -!:mime text/x-asm -0 regex \^[\020\t]*\\.byte assembler source text -!:mime text/x-asm -0 regex \^[\020\t]*\\.even assembler source text -!:mime text/x-asm -0 regex \^[\020\t]*\\.globl assembler source text -!:mime text/x-asm -0 regex \^[\020\t]*\\.text assembler source text -!:mime text/x-asm -0 regex \^[\020\t]*\\.file assembler source text -!:mime text/x-asm -0 regex \^[\020\t]*\\.type assembler source text -!:mime text/x-asm diff --git a/magic/audio b/magic/audio deleted file mode 100644 index 75a9dc536c..0000000000 --- a/magic/audio +++ /dev/null @@ -1,149 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: audio,v 1.65 2012/10/31 13:38:40 christos Exp $ -# audio: file(1) magic for sound formats (see also "iff") -# -# Jan Nicolai Langfeldt (janl@ifi.uio.no), Dan Quinlan (quinlan@yggdrasil.com), -# and others -# - -# Sun/NeXT audio data -0 string .snd Sun/NeXT audio data: ->12 belong 1 8-bit ISDN mu-law, -!:mime audio/basic ->12 belong 2 8-bit linear PCM [REF-PCM], -!:mime audio/basic ->12 belong 3 16-bit linear PCM, -!:mime audio/basic ->12 belong 4 24-bit linear PCM, -!:mime audio/basic ->12 belong 5 32-bit linear PCM, -!:mime audio/basic ->12 belong 6 32-bit IEEE floating point, -!:mime audio/basic ->12 belong 7 64-bit IEEE floating point, -!:mime audio/basic ->12 belong 23 8-bit ISDN mu-law compressed (CCITT G.721 ADPCM voice enc.), -!:mime audio/x-adpcm - -# DEC systems (e.g. DECstation 5000) use a variant of the Sun/NeXT format -# that uses little-endian encoding and has a different magic number -0 lelong 0x0064732E DEC audio data: ->12 lelong 1 8-bit ISDN mu-law, -!:mime audio/x-dec-basic ->12 lelong 2 8-bit linear PCM [REF-PCM], -!:mime audio/x-dec-basic ->12 lelong 3 16-bit linear PCM, -!:mime audio/x-dec-basic ->12 lelong 4 24-bit linear PCM, -!:mime audio/x-dec-basic ->12 lelong 5 32-bit linear PCM, -!:mime audio/x-dec-basic ->12 lelong 6 32-bit IEEE floating point, -!:mime audio/x-dec-basic ->12 lelong 7 64-bit IEEE floating point, -!:mime audio/x-dec-basic ->12 lelong 23 8-bit ISDN mu-law compressed (CCITT G.721 ADPCM voice enc.), -!:mime audio/x-dec-basic - -# Creative Labs AUDIO stuff -0 string MThd Standard MIDI data -!:mime audio/midi - -0 string CTMF Creative Music (CMF) data -!:mime audio/x-unknown -0 string SBI SoundBlaster instrument data -!:mime audio/x-unknown -0 string Creative\ Voice\ File Creative Labs voice data -!:mime audio/x-unknown - -# Real Audio (Magic .ra\0375) -0 belong 0x2e7261fd RealAudio sound file -!:mime audio/x-pn-realaudio -0 string .RMF\0\0\0 RealMedia file -!:mime application/vnd.rn-realmedia - -# mime types according to http://www.geocities.com/nevilo/mod.htm: -# audio/it .it -# audio/x-zipped-it .itz -# audio/xm fasttracker modules -# audio/x-s3m screamtracker modules -# audio/s3m screamtracker modules -# audio/x-zipped-mod mdz -# audio/mod mod -# audio/x-mod All modules (mod, s3m, 669, mtm, med, xm, it, mdz, stm, itz, xmz, s3z) - -# -# Taken from loader code from mikmod version 2.14 -# by Steve McIntyre (stevem@chiark.greenend.org.uk) -# added title printing on 2003-06-24 -0 string MAS_UTrack_V00 ->14 string >/0 ultratracker V1.%.1s module sound data -!:mime audio/x-mod -#audio/x-tracker-module - -0 string Extended\ Module: Fasttracker II module sound data -!:mime audio/x-mod -#audio/x-tracker-module - -21 string/c =!SCREAM! Screamtracker 2 module sound data -!:mime audio/x-mod -#audio/x-screamtracker-module -21 string BMOD2STM Screamtracker 2 module sound data -!:mime audio/x-mod -#audio/x-screamtracker-module -1080 string M.K. 4-channel Protracker module sound data -!:mime audio/x-mod -#audio/x-protracker-module -1080 string M!K! 4-channel Protracker module sound data -!:mime audio/x-mod -#audio/x-protracker-module -1080 string FLT4 4-channel Startracker module sound data -!:mime audio/x-mod -#audio/x-startracker-module -1080 string FLT8 8-channel Startracker module sound data -!:mime audio/x-mod -#audio/x-startracker-module -1080 string 4CHN 4-channel Fasttracker module sound data -!:mime audio/x-mod -#audio/x-fasttracker-module -1080 string 6CHN 6-channel Fasttracker module sound data -!:mime audio/x-mod -#audio/x-fasttracker-module -1080 string 8CHN 8-channel Fasttracker module sound data -!:mime audio/x-mod -#audio/x-fasttracker-module -1080 string CD81 8-channel Octalyser module sound data -!:mime audio/x-mod -#audio/x-octalysertracker-module -1080 string OKTA 8-channel Octalyzer module sound data -!:mime audio/x-mod -#audio/x-octalysertracker-module -# Not good enough. -#1082 string CH -#>1080 string >/0 %.2s-channel Fasttracker "oktalyzer" module sound data -1080 string 16CN 16-channel Taketracker module sound data -!:mime audio/x-mod -#audio/x-taketracker-module -1080 string 32CN 32-channel Taketracker module sound data -!:mime audio/x-mod -#audio/x-taketracker-module - -# Impulse tracker module (audio/x-it) -0 string IMPM Impulse Tracker module sound data - -!:mime audio/x-mod - -# Free lossless audio codec -# From: Przemyslaw Augustyniak -0 string fLaC FLAC audio bitstream data -!:mime audio/x-flac - -# Monkey's Audio compressed audio format (.ape) -# From danny.milo@gmx.net (Danny Milosavljevic) -# New version from Abel Cheung -0 string MAC\040 Monkey's Audio compressed format -!:mime audio/x-ape - -# musepak support From: "Jiri Pejchal" -0 string MP+ Musepack audio -!:mime audio/x-musepack diff --git a/magic/c-lang b/magic/c-lang deleted file mode 100644 index 525dc6b599..0000000000 --- a/magic/c-lang +++ /dev/null @@ -1,47 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: c-lang,v 1.16 2011/12/09 08:02:16 rrt Exp $ -# c-lang: file(1) magic for C and related languages programs -# - -# BCPL -0 search/8192 "libhdr" BCPL source text -!:mime text/x-bcpl -0 search/8192 "LIBHDR" BCPL source text -!:mime text/x-bcpl - -# C -0 regex \^#include C source text -!:mime text/x-c -0 regex \^char C source text -!:mime text/x-c -0 regex \^double C source text -!:mime text/x-c -0 regex \^extern C source text -!:mime text/x-c -0 regex \^float C source text -!:mime text/x-c -0 regex \^struct C source text -!:mime text/x-c -0 regex \^union C source text -!:mime text/x-c -0 search/8192 main( C source text -!:mime text/x-c - -# C++ -# The strength of these rules is increased so they beat the C rules above -0 regex \^template C++ source text -!:strength + 5 -!:mime text/x-c++ -0 regex \^virtual C++ source text -!:strength + 5 -!:mime text/x-c++ -0 regex \^class C++ source text -!:strength + 5 -!:mime text/x-c++ -0 regex \^public: C++ source text -!:strength + 5 -!:mime text/x-c++ -0 regex \^private: C++ source text -!:strength + 5 -!:mime text/x-c++ diff --git a/magic/cafebabe b/magic/cafebabe deleted file mode 100644 index 29fefd5f1e..0000000000 --- a/magic/cafebabe +++ /dev/null @@ -1,31 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: cafebabe,v 1.13 2013/02/26 21:04:38 christos Exp $ -# Cafe Babes unite! -# -# Since Java bytecode and Mach-O universal binaries have the same magic number, -# the test must be performed in the same "magic" sequence to get both right. -# The long at offset 4 in a Mach-O universal binary tells the number of -# architectures; the short at offset 4 in a Java bytecode file is the JVM minor -# version and the short at offset 6 is the JVM major version. Since there are only -# only 18 labeled Mach-O architectures at current, and the first released -# Java class format was version 43.0, we can safely choose any number -# between 18 and 39 to test the number of architectures against -# (and use as a hack). Let's not use 18, because the Mach-O people -# might add another one or two as time goes by... -# -### JAVA START ### -0 belong 0xcafebabe -!:mime application/x-java-applet - -0 belong 0xcafed00d JAR compressed with pack200, ->5 byte x version %d. ->4 byte x \b%d -!:mime application/x-java-pack200 - -0 belong 0xcafed00d JAR compressed with pack200, ->5 byte x version %d. ->4 byte x \b%d -!:mime application/x-java-pack200 - -### JAVA END ### diff --git a/magic/commands b/magic/commands deleted file mode 100644 index 6ad7699c5e..0000000000 --- a/magic/commands +++ /dev/null @@ -1,82 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: commands,v 1.44 2013/02/05 15:20:47 christos Exp $ -# commands: file(1) magic for various shells and interpreters -# -#0 string/w : shell archive or script for antique kernel text -0 string/wt #!\ /bin/sh POSIX shell script text executable -!:mime text/x-shellscript -0 string/wt #!\ /bin/csh C shell script text executable -!:mime text/x-shellscript -# korn shell magic, sent by George Wu, gwu@clyde.att.com -0 string/wt #!\ /bin/ksh Korn shell script text executable -!:mime text/x-shellscript -0 string/wt #!\ /bin/tcsh Tenex C shell script text executable -!:mime text/x-shellscript -0 string/wt #!\ /usr/bin/tcsh Tenex C shell script text executable -!:mime text/x-shellscript -0 string/wt #!\ /usr/local/tcsh Tenex C shell script text executable -!:mime text/x-shellscript -0 string/wt #!\ /usr/local/bin/tcsh Tenex C shell script text executable -!:mime text/x-shellscript - -# -# zsh/ash/ae/nawk/gawk magic from cameron@cs.unsw.oz.au (Cameron Simpson) -0 string/wt #!\ /bin/zsh Paul Falstad's zsh script text executable -!:mime text/x-shellscript -0 string/wt #!\ /usr/bin/zsh Paul Falstad's zsh script text executable -!:mime text/x-shellscript -0 string/wt #!\ /usr/local/bin/zsh Paul Falstad's zsh script text executable -!:mime text/x-shellscript -0 string/wt #!\ /usr/local/bin/ash Neil Brown's ash script text executable -!:mime text/x-shellscript -0 string/wt #!\ /usr/local/bin/ae Neil Brown's ae script text executable -!:mime text/x-shellscript -0 string/wt #!\ /bin/nawk new awk script text executable -!:mime text/x-nawk -0 string/wt #!\ /usr/bin/nawk new awk script text executable -!:mime text/x-nawk -0 string/wt #!\ /usr/local/bin/nawk new awk script text executable -!:mime text/x-nawk -0 string/wt #!\ /bin/gawk GNU awk script text executable -!:mime text/x-gawk -0 string/wt #!\ /usr/bin/gawk GNU awk script text executable -!:mime text/x-gawk -0 string/wt #!\ /usr/local/bin/gawk GNU awk script text executable -!:mime text/x-gawk -# -0 string/wt #!\ /bin/awk awk script text executable -!:mime text/x-awk -0 string/wt #!\ /usr/bin/awk awk script text executable -!:mime text/x-awk - -# bash shell magic, from Peter Tobias (tobias@server.et-inf.fho-emden.de) -0 string/wt #!\ /bin/bash Bourne-Again shell script text executable -!:mime text/x-shellscript -0 string/wt #!\ /usr/bin/bash Bourne-Again shell script text executable -!:mime text/x-shellscript -0 string/wt #!\ /usr/local/bash Bourne-Again shell script text executable -!:mime text/x-shellscript -0 string/wt #!\ /usr/local/bin/bash Bourne-Again shell script text executable -!:mime text/x-shellscript - -# PHP scripts -# Ulf Harnhammar -0 search/1/c = -0 string =24 regex [0-9.]+ \b, version %s -!:mime text/x-php diff --git a/magic/compress b/magic/compress deleted file mode 100644 index f2598b783f..0000000000 --- a/magic/compress +++ /dev/null @@ -1,77 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: compress,v 1.48 2011/12/07 18:39:43 christos Exp $ -# compress: file(1) magic for pure-compression formats (no archives) -# -# compress, gzip, pack, compact, huf, squeeze, crunch, freeze, yabba, etc. -# -# Formats for various forms of compressed data -# Formats for "compress" proper have been moved into "compress.c", -# because it tries to uncompress it to figure out what's inside. - -# standard unix compress -0 string \037\235 compress'd data -!:mime application/x-compress -!:apple LZIVZIVU - -# gzip (GNU zip, not to be confused with Info-ZIP or PKWARE zip archiver) -# Edited by Chris Chittleborough , March 2002 -# * Original filename is only at offset 10 if "extra field" absent -# * Produce shorter output - notably, only report compression methods -# other than 8 ("deflate", the only method defined in RFC 1952). -0 string \037\213 gzip compressed data -!:mime application/x-gzip - -# packed data, Huffman (minimum redundancy) codes on a byte-by-byte basis -0 string \037\036 packed data -!:mime application/octet-stream - -# -# This magic number is byte-order-independent. -0 short 0x1f1f old packed data -!:mime application/octet-stream - -# XXX - why *two* entries for "compacted data", one of which is -# byte-order independent, and one of which is byte-order dependent? -# -0 short 0x1fff compacted data -!:mime application/octet-stream -# This string is valid for SunOS (BE) and a matching "short" is listed -# in the Ultrix (LE) magic file. -0 string \377\037 compacted data -!:mime application/octet-stream -0 short 0145405 huf output -!:mime application/octet-stream - -# bzip2 -0 string BZh bzip2 compressed data -!:mime application/x-bzip2 - -# lzip -0 string LZIP lzip compressed data -!:mime application/x-lzip - -# 7-zip archiver, from Thomas Klausner (wiz@danbala.tuwien.ac.at) -# http://www.7-zip.org or DOC/7zFormat.txt -# -0 string 7z\274\257\047\034 7-zip archive data, ->6 byte x version %d ->7 byte x \b.%d -!:mime application/x-7z-compressed - -# Type: LZMA -0 lelong&0xffffff =0x5d ->12 leshort =0xff LZMA compressed data, ->>5 lequad =0xffffffffffffffff streamed ->>5 lequad !0xffffffffffffffff non-streamed, size %lld -!:mime application/x-lzma - -# http://tukaani.org/xz/xz-file-format.txt -0 ustring \xFD7zXZ\x00 XZ compressed data -!:mime application/x-xz - -# https://github.com/ckolivas/lrzip/blob/master/doc/magic.header.txt -0 string LRZI LRZIP compressed data ->4 byte x - version %d ->5 byte x \b.%d -!:mime application/x-lrzip diff --git a/magic/database b/magic/database deleted file mode 100644 index f1c09c0629..0000000000 --- a/magic/database +++ /dev/null @@ -1,47 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: database,v 1.32 2013/02/06 14:18:52 christos Exp $ -# database: file(1) magic for various databases -# -# extracted from header/code files by Graeme Wilford (eep2gw@ee.surrey.ac.uk) -# -# -# GDBM magic numbers -# Will be maintained as part of the GDBM distribution in the future. -# -0 belong 0x13579ace GNU dbm 1.x or ndbm database, big endian -!:mime application/x-gdbm -0 lelong 0x13579ace GNU dbm 1.x or ndbm database, little endian -!:mime application/x-gdbm -0 string GDBM GNU dbm 2.x database -!:mime application/x-gdbm -# -# Berkeley DB -# -# Ian Darwin's file /etc/magic files: big/little-endian version. -# -# Hash 1.85/1.86 databases store metadata in network byte order. -# Btree 1.85/1.86 databases store the metadata in host byte order. -# Hash and Btree 2.X and later databases store the metadata in host byte order. - -0 long 0x00061561 Berkeley DB -!:mime application/x-dbm - -# MS Access database -4 string Standard\ Jet\ DB Microsoft Access Database -!:mime application/x-msaccess -4 string Standard\ ACE\ DB Microsoft Access Database -!:mime application/x-msaccess - -# Tokyo Cabinet magic data -# http://tokyocabinet.sourceforge.net/index.html -0 string ToKyO\ CaBiNeT\n Tokyo Cabinet ->14 string x \b (%s) ->32 byte 0 \b, Hash -!:mime application/x-tokyocabinet-hash ->32 byte 1 \b, B+ tree -!:mime application/x-tokyocabinet-btree ->32 byte 2 \b, Fixed-length -!:mime application/x-tokyocabinet-fixed ->32 byte 3 \b, Table -!:mime application/x-tokyocabinet-table diff --git a/magic/diff b/magic/diff deleted file mode 100644 index b6504f17a0..0000000000 --- a/magic/diff +++ /dev/null @@ -1,25 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: diff,v 1.13 2012/06/16 14:43:36 christos Exp $ -# diff: file(1) magic for diff(1) output -# -0 search/1 diff\ diff output text -!:mime text/x-diff -0 search/1 ***\ diff output text -!:mime text/x-diff -0 search/1 Only\ in\ diff output text -!:mime text/x-diff -0 search/1 Common\ subdirectories:\ diff output text -!:mime text/x-diff - -0 search/1 Index: RCS/CVS diff output text -!:mime text/x-diff - -# unified diff -0 search/4096 ---\ ->&0 search/1024 \n ->>&0 search/1 +++\ ->>>&0 search/1024 \n ->>>>&0 search/1 @@ unified diff output text -!:mime text/x-diff -!:strength + 90 diff --git a/magic/elf b/magic/elf deleted file mode 100644 index aaf80cf10e..0000000000 --- a/magic/elf +++ /dev/null @@ -1,43 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# elf: file(1) magic for ELF executables -# -# We have to check the byte order flag to see what byte order all the -# other stuff in the header is in. -# -# What're the correct byte orders for the nCUBE and the Fujitsu VPP500? -# -# Created by: unknown -# Modified by (1): Daniel Quinlan -# Modified by (2): Peter Tobias (core support) -# Modified by (3): Christian 'Dr. Disk' Hechelmann (fix of core support) -# Modified by (4): (VMS Itanium) -# Modified by (5): Matthias Urlichs (Listing of many architectures) -0 string \177ELF ELF ->4 byte 0 invalid class ->4 byte 1 32-bit ->4 byte 2 64-bit ->5 byte 0 invalid byte order ->5 byte 1 LSB ->>16 leshort 0 no file type, -!:strength *2 -!:mime application/octet-stream ->>16 leshort 1 relocatable, -!:mime application/x-object ->>16 leshort 2 executable, -!:mime application/x-executable ->>16 leshort 3 shared object, -!:mime application/x-sharedlib ->>16 leshort 4 core file -!:mime application/x-coredump ->5 byte 2 MSB ->>16 beshort 0 no file type, -!:mime application/octet-stream ->>16 beshort 1 relocatable, -!:mime application/x-object ->>16 beshort 2 executable, -!:mime application/x-executable ->>16 beshort 3 shared object, -!:mime application/x-sharedlib ->>16 beshort 4 core file, -!:mime application/x-coredump diff --git a/magic/epoc b/magic/epoc deleted file mode 100644 index d7397145fb..0000000000 --- a/magic/epoc +++ /dev/null @@ -1,34 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: epoc,v 1.7 2009/09/19 16:28:09 christos Exp $ -# EPOC : file(1) magic for EPOC documents [Psion Series 5/Osaris/Geofox 1] -# Stefan Praszalowicz and Peter Breitenlohner -# Useful information for improving this file can be found at: -# http://software.frodo.looijaard.name/psiconv/formats/Index.html -#------------------------------------------------------------------------------ -0 lelong 0x10000037 Psion Series 5 ->4 lelong 0x10000042 multi-bitmap image -!:mime image/x-epoc-mbm ->4 lelong 0x1000006D ->>8 lelong 0x1000007D Sketch image -!:mime image/x-epoc-sketch ->>8 lelong 0x1000007F Word file -!:mime application/x-epoc-word ->>8 lelong 0x10000085 OPL program (TextEd) -!:mime application/x-epoc-opl ->>8 lelong 0x10000088 Sheet file -!:mime application/x-epoc-sheet ->4 lelong 0x10000073 OPO module -!:mime application/x-epoc-opo ->4 lelong 0x10000074 OPL application -!:mime application/x-epoc-app - - -0 lelong 0x10000050 Psion Series 5 ->4 lelong 0x1000006D database ->>8 lelong 0x10000084 Agenda file -!:mime application/x-epoc-agenda ->>8 lelong 0x10000086 Data file -!:mime application/x-epoc-data ->>8 lelong 0x10000CEA Jotter file -!:mime application/x-epoc-jotter diff --git a/magic/filesystems b/magic/filesystems deleted file mode 100644 index d2178296e0..0000000000 --- a/magic/filesystems +++ /dev/null @@ -1,12 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: filesystems,v 1.76 2013/02/18 18:45:41 christos Exp $ -# filesystems: file(1) magic for different filesystems -# - -# CDROM Filesystems -# Modified for UDF by gerardo.cacciari@gmail.com -32769 string CD001 # -!:mime application/x-iso9660-image -37633 string CD001 ISO 9660 CD-ROM filesystem data (raw 2352 byte sectors) -!:mime application/x-iso9660-image diff --git a/magic/flash b/magic/flash deleted file mode 100644 index b64761b12d..0000000000 --- a/magic/flash +++ /dev/null @@ -1,18 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: flash,v 1.8 2009/09/19 16:28:09 christos Exp $ -# flash: file(1) magic for Macromedia Flash file format -# -# See -# -# http://www.macromedia.com/software/flash/open/ -# -0 string FWS Macromedia Flash data, ->3 byte x version %d -!:mime application/x-shockwave-flash -0 string CWS Macromedia Flash data (compressed), -!:mime application/x-shockwave-flash - -# From: Cal Peake -0 string FLV Macromedia Flash Video -!:mime video/x-flv diff --git a/magic/fonts b/magic/fonts deleted file mode 100644 index 8189131d15..0000000000 --- a/magic/fonts +++ /dev/null @@ -1,32 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: fonts,v 1.25 2013/02/06 14:18:52 christos Exp $ -# fonts: file(1) magic for font data -# - -# X11 font files in SNF (Server Natural Format) format -# updated by Joerg Jenderek at Feb 2013 -# http://computer-programming-forum.com/51-perl/8f22fb96d2e34bab.htm -0 belong 00000004 X11 SNF font data, MSB first -#>104 belong 00000004 X11 SNF font data, MSB first -!:mime application/x-font-sfn -# GRR: line below too general as it catches also Xbase index file t3-CHAR.NDX -0 lelong 00000004 ->104 lelong 00000004 X11 SNF font data, LSB first -!:mime application/x-font-sfn - -# True Type fonts -0 string \000\001\000\000\000 TrueType font data -!:mime application/x-font-ttf - -# Opentype font data from Avi Bercovich -0 string OTTO OpenType font data -!:mime application/vnd.ms-opentype - -# Gurkan Sengun , www.linuks.mine.nu -0 string SplineFontDB: Spline Font Database -!:mime application/vnd.font-fontforge-sfd - -# EOT -34 string LP Embedded OpenType (EOT) -!:mime application/vnd.ms-fontobject diff --git a/magic/fortran b/magic/fortran deleted file mode 100644 index 498eeacf8a..0000000000 --- a/magic/fortran +++ /dev/null @@ -1,7 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: fortran,v 1.6 2009/09/19 16:28:09 christos Exp $ -# FORTRAN source -0 regex/100 \^[Cc][\ \t] FORTRAN program -!:mime text/x-fortran -!:strength - 5 diff --git a/magic/frame b/magic/frame deleted file mode 100644 index b42943bfcd..0000000000 --- a/magic/frame +++ /dev/null @@ -1,31 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File$ -# frame: file(1) magic for FrameMaker files -# -# This stuff came on a FrameMaker demo tape, most of which is -# copyright, but this file is "published" as witness the following: -# -# Note that this is the Framemaker Maker Interchange Format, not the -# Normal format which would be application/vnd.framemaker. -# -0 string \6 string 3.0 (3.0) -#>6 string 2.0 (2.0) -#>6 string 1.0 (1.0) -0 string \ - -#------------------------------------------------------------------------------ -# XCF: file(1) magic for the XCF image format used in the GIMP developed -# by Spencer Kimball and Peter Mattis -# ('Bucky' LaDieu, nega@vt.edu) - -0 string gimp\ xcf GIMP XCF image data, -!:mime image/x-xcf diff --git a/magic/gnu b/magic/gnu deleted file mode 100644 index bf1f631751..0000000000 --- a/magic/gnu +++ /dev/null @@ -1,23 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: gnu,v 1.13 2012/01/03 17:16:54 christos Exp $ -# gnu: file(1) magic for various GNU tools -# -# GNU nlsutils message catalog file format -# -# GNU message catalog (.mo and .gmo files) - -# GnuPG -# The format is very similar to pgp -# Note: magic.mime had 0x8501 for the next line instead of 0x8502 -0 beshort 0x8502 GPG encrypted data -!:mime text/PGP # encoding: data - -# This magic is not particularly good, as the keyrings don't have true -# magic. Nevertheless, it covers many keyrings. -0 beshort 0x9901 GPG key public ring -!:mime application/x-gnupg-keyring - -# gettext message catalogue -0 regex \^msgid\ GNU gettext message catalogue text -!:mime text/x-po diff --git a/magic/gnumeric b/magic/gnumeric deleted file mode 100644 index b5edca93c1..0000000000 --- a/magic/gnumeric +++ /dev/null @@ -1,8 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File$ -# gnumeric: file(1) magic for Gnumeric spreadsheet -# This entry is only semi-helpful, as Gnumeric compresses its files, so -# they will ordinarily reported as "compressed", but at least -z helps -39 string =4 belong x \b, FORM is %d bytes long -# audio formats ->8 string AIFF \b, AIFF audio -!:mime audio/x-aiff ->8 string AIFC \b, AIFF-C compressed audio -!:mime audio/x-aiff ->8 string 8SVX \b, 8SVX 8-bit sampled sound voice -!:mime audio/x-aiff diff --git a/magic/images b/magic/images deleted file mode 100644 index 281aba4706..0000000000 --- a/magic/images +++ /dev/null @@ -1,255 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: images,v 1.80 2013/02/06 14:18:52 christos Exp $ -# images: file(1) magic for image formats (see also "iff", and "c-lang" for -# XPM bitmaps) -# -# originally from jef@helios.ee.lbl.gov (Jef Poskanzer), -# additions by janl@ifi.uio.no as well as others. Jan also suggested -# merging several one- and two-line files into here. -# -# little magic: PCX (first byte is 0x0a) - -# PBMPLUS images -# The next byte following the magic is always whitespace. -# strength is changed to try these patterns before "x86 boot sector" -0 search/1 P1 ->3 regex =[0-9]*\ [0-9]* Netpbm PBM image text ->3 regex =[0-9]+\ \b, size = %sx ->>3 regex =\ [0-9]+ \b%s -!:strength + 45 -!:mime image/x-portable-bitmap -0 search/1 P2 ->3 regex =[0-9]*\ [0-9]* Netpbm PGM image text ->3 regex =[0-9]+\ \b, size = %sx ->>3 regex =\ [0-9]+ \b%s -!:strength + 45 -!:mime image/x-portable-greymap -0 search/1 P3 Netpbm PPM image text ->3 regex =[0-9]*\ [0-9]* Netpbm PPM image text ->3 regex =[0-9]+\ \b, size = %sx ->>3 regex =\ [0-9]+ \b%s -!:strength + 45 -!:mime image/x-portable-pixmap -0 string P4 ->3 regex =[0-9]*\ [0-9]* Netpbm PBM "rawbits" image data ->3 regex =[0-9]+\ \b, size = %sx ->>3 regex =\ [0-9]+ \b%s -!:strength + 45 -!:mime image/x-portable-bitmap -0 string P5 ->3 regex =[0-9]*\ [0-9]* Netpbm PGM "rawbits" image data ->3 regex =[0-9]+\ \b, size = %sx ->>3 regex =\ [0-9]+ \b%s -!:strength + 45 -!:mime image/x-portable-greymap -0 string P6 ->3 regex =[0-9]*\ [0-9]* Netpbm PPM "rawbits" image data ->3 regex =[0-9]+\ \b, size = %sx ->>3 regex =\ [0-9]+ \b%s -!:strength + 45 -!:mime image/x-portable-pixmap -0 string P7 Netpbm PAM image file -!:mime image/x-portable-pixmap - -# NIFF (Navy Interchange File Format, a modification of TIFF) images -# [GRR: this *must* go before TIFF] -0 string IIN1 NIFF image data -!:mime image/x-niff - -# Canon RAW version 1 (CRW) files are a type of Canon Image File Format -# (CIFF) file. These are apparently all little-endian. -# From: Adam Buchbinder -# URL: http://www.sno.phy.queensu.ca/~phil/exiftool/canon_raw.html -0 string II\x1a\0\0\0HEAPCCDR Canon CIFF raw image data -!:mime image/x-canon-crw - -# Canon RAW version 2 (CR2) files are a kind of TIFF with an extra magic -# number. Put this above the TIFF test to make sure we detect them. -# These are apparently all little-endian. -# From: Adam Buchbinder -# URL: http://libopenraw.freedesktop.org/wiki/Canon_CR2 -0 string II\x2a\0\x10\0\0\0CR Canon CR2 raw image data -!:mime image/x-canon-cr2 - -# Tag Image File Format, from Daniel Quinlan (quinlan@yggdrasil.com) -# The second word of TIFF files is the TIFF version number, 42, which has -# never changed. The TIFF specification recommends testing for it. -0 string MM\x00\x2a TIFF image data, big-endian -!:mime image/tiff -0 string II\x2a\x00 TIFF image data, little-endian -!:mime image/tiff - -0 string MM\x00\x2b Big TIFF image data, big-endian -!:mime image/tiff -0 string II\x2b\x00 Big TIFF image data, little-endian -!:mime image/tiff - -# PNG [Portable Network Graphics, or "PNG's Not GIF"] images -# (Greg Roelofs, newt@uchicago.edu) -# (Albert Cahalan, acahalan@cs.uml.edu) -# -# 137 P N G \r \n ^Z \n [4-byte length] H E A D [HEAD data] [HEAD crc] ... -# -0 string \x89PNG\x0d\x0a\x1a\x0a PNG image data -!:mime image/png - -# possible GIF replacements; none yet released! -# (Greg Roelofs, newt@uchicago.edu) -# -# GRR 950115: this was mine ("Zip GIF"): -0 string GIF94z ZIF image (GIF+deflate alpha) -!:mime image/x-unknown -# -# GRR 950115: this is Jeremy Wohl's Free Graphics Format (better): -# -0 string FGF95a FGF image (GIF+deflate beta) -!:mime image/x-unknown -# -# GRR 950115: this is Thomas Boutell's Portable Bitmap Format proposal -# (best; not yet implemented): -# -0 string PBF PBF image (deflate compression) -!:mime image/x-unknown - -# GIF -0 string GIF8 GIF image data -!:mime image/gif -!:apple 8BIMGIFf - -# From: Joerg Jenderek -# most files with the extension .EPA and some with .BMP -0 string \x11\x06 Award BIOS Logo, 136 x 84 -!:mime image/x-award-bioslogo -0 string \x11\x09 Award BIOS Logo, 136 x 126 -!:mime image/x-award-bioslogo -#0 string \x07\x1f BIOS Logo corrupted? -# http://www.blackfiveservices.co.uk/awbmtools.shtml -# http://biosgfx.narod.ru/v3/ -# http://biosgfx.narod.ru/abr-2/ -0 string AWBM ->4 leshort <1981 Award BIOS bitmap -!:mime image/x-award-bmp - -# PC bitmaps (OS/2, Windows BMP files) (Greg Roelofs, newt@uchicago.edu) -0 string BM ->14 leshort 12 PC bitmap, OS/2 1.x format -!:mime image/x-ms-bmp ->14 leshort 64 PC bitmap, OS/2 2.x format -!:mime image/x-ms-bmp ->14 leshort 40 PC bitmap, Windows 3.x format -!:mime image/x-ms-bmp ->14 leshort 128 PC bitmap, Windows NT/2000 format -!:mime image/x-ms-bmp - -# XPM icons (Greg Roelofs, newt@uchicago.edu) -0 search/1 /*\ XPM\ */ X pixmap image text -!:mime image/x-xpmi - -# DICOM medical imaging data -128 string DICM DICOM medical imaging data -!:mime application/dicom - -# XWD - X Window Dump file. -# As described in /usr/X11R6/include/X11/XWDFile.h -# used by the xwd program. -# Bradford Castalia, idaeim, 1/01 -# updated by Adam Buchbinder, 2/09 -# The following assumes version 7 of the format; the first long is the length -# of the header, which is at least 25 4-byte longs, and the one at offset 8 -# is a constant which is always either 1 or 2. Offset 12 is the pixmap depth, -# which is a maximum of 32. -0 belong >100 ->8 belong <3 ->>12 belong <33 ->>>4 belong 7 XWD X Window Dump image data -!:mime image/x-xwindowdump - -# PCX image files -# From: Dan Fandrich -# updated by Joerg Jenderek at Feb 2013 by http://de.wikipedia.org/wiki/PCX -# http://web.archive.org/web/20100206055706/http://www.qzx.com/pc-gpe/pcx.txt -# GRR: original test was still too general as it catches xbase examples T5.DBT,T6.DBT with 0xa000000 -# test for bytes 0x0a,version byte (0,2,3,4,5),compression byte flag(0,1), bit depth (>0) of PCX or T5.DBT,T6.DBT -0 ubelong&0xffF8fe00 0x0a000000 -# for PCX bit depth > 0 ->3 ubyte >0 -# test for valid versions ->>1 ubyte <6 ->>>1 ubyte !1 PCX -!:mime image/x-pcx - -# Adobe Photoshop -# From: Asbjoern Sloth Toennesen -0 string 8BPS Adobe Photoshop Image -!:mime image/vnd.adobe.photoshop - -# Summary: DjVu image / document -# Extension: .djvu -# Reference: http://djvu.org/docs/DjVu3Spec.djvu -# Submitted by: Stephane Loeuillet -# Modified by (1): Abel Cheung -0 string AT&TFORM ->12 string DJVM DjVu multiple page document -!:mime image/vnd.djvu ->12 string DJVU DjVu image or single page document -!:mime image/vnd.djvu ->12 string DJVI DjVu shared document -!:mime image/vnd.djvu ->12 string THUM DjVu page thumbnails -!:mime image/vnd.djvu - -# Originally by Marc Espie -# Modified by Robert Minsk -# http://www.openexr.com/openexrfilelayout.pdf -0 lelong 20000630 OpenEXR image data, -!:mime image/x-exr - -# SMPTE Digital Picture Exchange Format, SMPTE DPX -# -# ANSI/SMPTE 268M-1994, SMPTE Standard for File Format for Digital -# Moving-Picture Exchange (DPX), v1.0, 18 February 1994 -# Robert Minsk -0 string SDPX DPX image data, big-endian, -!:mime image/x-dpx - -#----------------------------------------------------------------------- -# Hierarchical Data Format, used to facilitate scientific data exchange -# specifications at http://hdf.ncsa.uiuc.edu/ -0 belong 0x0e031301 Hierarchical Data Format (version 4) data -!:mime application/x-hdf -0 string \211HDF\r\n\032\n Hierarchical Data Format (version 5) data -!:mime application/x-hdf - -# http://www.cartesianinc.com/Tech/ -0 string CPC\262 Cartesian Perceptual Compression image -!:mime image/x-cpi - - -# Polar Monitor Bitmap (.pmb) used as logo for Polar Electro watches -# From: Markus Heidelberg -0 string/t [BitmapInfo2] Polar Monitor Bitmap text -!:mime image/x-polar-monitor-bitmap - -# Type: Olympus ORF raw images. -# URL: http://libopenraw.freedesktop.org/wiki/Olympus_ORF -# From: Adam Buchbinder -0 string MMOR Olympus ORF raw image data, big-endian -!:mime image/x-olympus-orf -0 string IIRO Olympus ORF raw image data, little-endian -!:mime image/x-olympus-orf -0 string IIRS Olympus ORF raw image data, little-endian -!:mime image/x-olympus-orf - -# Type: Foveon X3F -# URL: http://www.photofo.com/downloads/x3f-raw-format.pdf -# From: Adam Buchbinder -# Note that the MIME type isn't defined anywhere that I can find; if -# there's a canonical type for this format, it should replace this one. -0 string FOVb Foveon X3F raw image data -!:mime image/x-x3f - -# Paint.NET file -# From Adam Buchbinder -0 string PDN3 Paint.NET image data -!:mime image/x-paintnet diff --git a/magic/java b/magic/java deleted file mode 100644 index 481ffec160..0000000000 --- a/magic/java +++ /dev/null @@ -1,16 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------ -# $File: java,v 1.13 2011/12/08 12:12:46 rrt Exp $ -# Java ByteCode and Mach-O binaries (e.g., Mac OS X) use the -# same magic number, 0xcafebabe, so they are both handled -# in the entry called "cafebabe". -#------------------------------------------------------------ - -0 belong 0xfeedfeed Java KeyStore -!:mime application/x-java-keystore -0 belong 0xcececece Java JCE KeyStore -!:mime application/x-java-jce-keystore - -# Java source -0 regex ^import.*;$ Java source -!:mime text/x-java diff --git a/magic/javascript b/magic/javascript deleted file mode 100644 index a1311d0e71..0000000000 --- a/magic/javascript +++ /dev/null @@ -1,17 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: $ -# javascript: magic for javascript and node.js scripts. -# -0 search/1/w #!/bin/node Node.js script text executable -!:mime application/javascript -0 search/1/w #!/usr/bin/node Node.js script text executable -!:mime application/javascript -0 search/1/w #!/bin/nodejs Node.js script text executable -!:mime application/javascript -0 search/1/w #!/usr/bin/nodejs Node.js script text executable -!:mime application/javascript -0 search/1 #!/usr/bin/env\ node Node.js script text executable -!:mime application/javascript -0 search/1 #!/usr/bin/env\ nodejs Node.js script text executable -!:mime application/javascript diff --git a/magic/jpeg b/magic/jpeg deleted file mode 100644 index 55fedae4b4..0000000000 --- a/magic/jpeg +++ /dev/null @@ -1,31 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: jpeg,v 1.18 2012/08/01 12:12:36 christos Exp $ -# JPEG images -# SunOS 5.5.1 had -# -# 0 string \377\330\377\340 JPEG file -# 0 string \377\330\377\356 JPG file -# -# both of which turn into "JPEG image data" here. -# -0 beshort 0xffd8 JPEG image data -!:mime image/jpeg -!:apple 8BIMJPEG -!:strength +2 - -# From: David Santinoli -0 string \x00\x00\x00\x0C\x6A\x50\x20\x20\x0D\x0A\x87\x0A JPEG 2000 -# From: Johan van der Knijff -# Added sub-entries for JP2, JPX, JPM and MJ2 formats; added mimetypes -# https://github.com/bitsgalore/jp2kMagic -# -# Now read value of 'Brand' field, which yields a few possibilities: ->20 string \x6a\x70\x32\x20 Part 1 (JP2) -!:mime image/jp2 ->20 string \x6a\x70\x78\x20 Part 2 (JPX) -!:mime image/jpx ->20 string \x6a\x70\x6d\x20 Part 6 (JPM) -!:mime image/jpm ->20 string \x6d\x6a\x70\x32 Part 3 (MJ2) -!:mime video/mj2 diff --git a/magic/kde b/magic/kde deleted file mode 100644 index 2b66ee611d..0000000000 --- a/magic/kde +++ /dev/null @@ -1,11 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: kde,v 1.4 2009/09/19 16:28:10 christos Exp $ -# kde: file(1) magic for KDE - -0 string/t [KDE\ Desktop\ Entry] KDE desktop entry -!:mime application/x-kdelnk -0 string/t #\ KDE\ Config\ File KDE config file -!:mime application/x-kdelnk -0 string/t #\ xmcd xmcd database file for kscd -!:mime text/x-xmcd diff --git a/magic/kml b/magic/kml deleted file mode 100644 index 608ff0e1b0..0000000000 --- a/magic/kml +++ /dev/null @@ -1,30 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: kml,v 1.2 2009/09/19 16:28:10 christos Exp $ -# Type: Google KML, formerly Keyhole Markup Language -# Future development of this format has been handed -# over to the Open Geospatial Consortium. -# http://www.opengeospatial.org/standards/kml/ -# From: Asbjoern Sloth Toennesen -0 string/t \20 search/400 \ xmlns= ->>&0 regex ['"]http://earth.google.com/kml Google KML document -!:mime application/vnd.google-earth.kml+xml - -#------------------------------------------------------------------------------ -# Type: OpenGIS KML, formerly Keyhole Markup Language -# This standard is maintained by the -# Open Geospatial Consortium. -# http://www.opengeospatial.org/standards/kml/ -# From: Asbjoern Sloth Toennesen ->>&0 regex ['"]http://www.opengis.net/kml OpenGIS KML document -!:mime application/vnd.google-earth.kml+xml - -#------------------------------------------------------------------------------ -# Type: Google KML Archive (ZIP based) -# http://code.google.com/apis/kml/documentation/kml_tut.html -# From: Asbjoern Sloth Toennesen -0 string PK\003\004 ->4 byte 0x14 ->>30 string doc.kml Compressed Google KML Document, including resources. -!:mime application/vnd.google-earth.kmz diff --git a/magic/linux b/magic/linux deleted file mode 100644 index 4a5c935760..0000000000 --- a/magic/linux +++ /dev/null @@ -1,22 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: linux,v 1.46 2013/01/06 21:26:48 christos Exp $ -# linux: file(1) magic for Linux files -# -# Values for Linux/i386 binaries, from Daniel Quinlan -# The following basic Linux magic is useful for reference, but using -# "long" magic is a better practice in order to avoid collisions. -# -# 2 leshort 100 Linux/i386 -# >0 leshort 0407 impure executable (OMAGIC) -# >0 leshort 0410 pure executable (NMAGIC) -# >0 leshort 0413 demand-paged executable (ZMAGIC) -# >0 leshort 0314 demand-paged executable (QMAGIC) -# - -# SYSLINUX boot logo files (from 'ppmtolss16' sources) -# http://www.syslinux.org/wiki/index.php/SYSLINUX#Display_graphic_from_filename: -# file extension .lss .16 -0 lelong =0x1413f33d SYSLINUX' LSS16 image data -# syslinux-4.05/mime/image/x-lss16.xml -!:mime image/x-lss16 diff --git a/magic/lisp b/magic/lisp deleted file mode 100644 index f5a06c8964..0000000000 --- a/magic/lisp +++ /dev/null @@ -1,42 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File$ -# lisp: file(1) magic for lisp programs -# -# various lisp types, from Daniel Quinlan (quinlan@yggdrasil.com) - -# updated by Joerg Jenderek -# GRR: This lot is too weak -#0 string ;; -# windows INF files often begin with semicolon and use CRLF as line end -# lisp files are mainly created on unix system with LF as line end -#>2 search/4096 !\r Lisp/Scheme program text -#>2 search/4096 \r Windows INF file - -0 search/4096 (setq\ Lisp/Scheme program text -!:mime text/x-lisp -0 search/4096 (defvar\ Lisp/Scheme program text -!:mime text/x-lisp -0 search/4096 (defparam\ Lisp/Scheme program text -!:mime text/x-lisp -0 search/4096 (defun\ Lisp/Scheme program text -!:mime text/x-lisp -0 search/4096 (autoload\ Lisp/Scheme program text -!:mime text/x-lisp -0 search/4096 (custom-set-variables\ Lisp/Scheme program text -!:mime text/x-lisp - -# Emacs 18 - this is always correct, but not very magical. -0 string \012( Emacs v18 byte-compiled Lisp data -!:mime application/x-elc -# Emacs 19+ - ver. recognition added by Ian Springer -# Also applies to XEmacs 19+ .elc files; could tell them apart with regexs -# - Chris Chittleborough -0 string ;ELC ->4 byte >18 ->4 byte <32 Emacs/XEmacs v%d byte-compiled Lisp data -!:mime application/x-elc - -# From: David Allouche -0 search/1 \, Seo Sanghyeon - -# Lua scripts -0 search/1/w #!\ /usr/bin/lua Lua script text executable -!:mime text/x-lua -0 search/1/w #!\ /usr/local/bin/lua Lua script text executable -!:mime text/x-lua -0 search/1 #!/usr/bin/env\ lua Lua script text executable -!:mime text/x-lua -0 search/1 #!\ /usr/bin/env\ lua Lua script text executable -!:mime text/x-lua - diff --git a/magic/m4 b/magic/m4 deleted file mode 100644 index 7262fca81b..0000000000 --- a/magic/m4 +++ /dev/null @@ -1,7 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File$ -# make: file(1) magic for M4 scripts -# -0 regex \^dnl\ M4 macro processor script text -!:mime text/x-m4 diff --git a/magic/macintosh b/magic/macintosh deleted file mode 100644 index 6398fc2ff2..0000000000 --- a/magic/macintosh +++ /dev/null @@ -1,21 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: macintosh,v 1.21 2010/09/20 19:19:17 rrt Exp $ -# macintosh description -# -# BinHex is the Macintosh ASCII-encoded file format (see also "apple") -# Daniel Quinlan, quinlan@yggdrasil.com -11 string must\ be\ converted\ with\ BinHex BinHex binary text -!:mime application/mac-binhex40 - -# Stuffit archives are the de facto standard of compression for Macintosh -# files obtained from most archives. (franklsm@tuns.ca) -0 string SIT! StuffIt Archive (data) -!:mime application/x-stuffit -!:apple SIT!SIT! - -# Newer StuffIt archives (grant@netbsd.org) -0 string StuffIt StuffIt Archive -!:mime application/x-stuffit -!:apple SIT!SIT! -#>162 string >0 : %s diff --git a/magic/mail.news b/magic/mail.news deleted file mode 100644 index c1a446d4ca..0000000000 --- a/magic/mail.news +++ /dev/null @@ -1,35 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: mail.news,v 1.21 2012/06/21 01:44:52 christos Exp $ -# mail.news: file(1) magic for mail and news -# -# Unfortunately, saved netnews also has From line added in some news software. -#0 string From mail text -0 string/t Relay-Version: old news text -!:mime message/rfc822 -0 string/t #!\ rnews batched news text -!:mime message/rfc822 -0 string/t N#!\ rnews mailed, batched news text -!:mime message/rfc822 -0 string/t Forward\ to mail forwarding text -!:mime message/rfc822 -0 string/t Pipe\ to mail piping text -!:mime message/rfc822 -0 string/tc delivered-to: SMTP mail text -!:mime message/rfc822 -0 string/tc return-path: SMTP mail text -!:mime message/rfc822 -0 string/t Path: news text -!:mime message/news -0 string/t Xref: news text -!:mime message/news -0 string/t From: news or mail text -!:mime message/rfc822 -0 string/t Article saved news text -!:mime message/news -0 string/t Received: RFC 822 mail text -!:mime message/rfc822 - -# TNEF files... -0 lelong 0x223E9F78 Transport Neutral Encapsulation Format -!:mime application/vnd.ms-tnef diff --git a/magic/make b/magic/make deleted file mode 100644 index 83d6a012dd..0000000000 --- a/magic/make +++ /dev/null @@ -1,16 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File$ -# make: file(1) magic for makefiles -# -0 regex \^CFLAGS makefile script text -!:mime text/x-makefile -0 regex \^LDFLAGS makefile script text -!:mime text/x-makefile -0 regex \^all: makefile script text -!:mime text/x-makefile -0 regex \^.PRECIOUS makefile script text -!:mime text/x-makefile - -0 regex \^SUBDIRS automake makefile script text -!:mime text/x-makefile diff --git a/magic/marc21 b/magic/marc21 deleted file mode 100644 index 26899d2e70..0000000000 --- a/magic/marc21 +++ /dev/null @@ -1,29 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#-------------------------------------------- -# marc21: file(1) magic for MARC 21 Format -# -# Kevin Ford (kefo@loc.gov) -# -# MARC21 formats are for the representation and communication -# of bibliographic and related information in machine-readable -# form. For more info, see http://www.loc.gov/marc/ - - -# leader position 20-21 must be 45 -20 string 45 - -# leader starts with 5 digits, followed by codes specific to MARC format ->0 regex/1 (^[0-9]{5})[acdnp][^bhlnqsu-z] MARC21 Bibliographic -!:mime application/marc ->0 regex/1 (^[0-9]{5})[acdnosx][z] MARC21 Authority -!:mime application/marc ->0 regex/1 (^[0-9]{5})[cdn][uvxy] MARC21 Holdings -!:mime application/marc -0 regex/1 (^[0-9]{5})[acdn][w] MARC21 Classification -!:mime application/marc ->0 regex/1 (^[0-9]{5})[cdn][q] MARC21 Community -!:mime application/marc - -# leader position 22-23, should be "00" but is it? ->0 regex/1 (^.{21})([^0]{2}) (non-conforming) -!:mime application/marc diff --git a/magic/matroska b/magic/matroska deleted file mode 100644 index c1791413cb..0000000000 --- a/magic/matroska +++ /dev/null @@ -1,17 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: matroska,v 1.7 2012/08/26 10:06:15 christos Exp $ -# matroska: file(1) magic for Matroska files -# -# See http://www.matroska.org/ -# - -# EBML id: -0 belong 0x1a45dfa3 -# DocType id: ->4 search/4096 \x42\x82 -# DocType contents: ->>&1 string webm WebM -!:mime video/webm ->>&1 string matroska Matroska data -!:mime video/x-matroska diff --git a/magic/misctools b/magic/misctools deleted file mode 100644 index 35fddaa61a..0000000000 --- a/magic/misctools +++ /dev/null @@ -1,9 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#----------------------------------------------------------------------------- -# $File: misctools,v 1.12 2010/09/29 18:36:49 rrt Exp $ -# misctools: file(1) magic for miscellaneous UNIX tools. -# -0 string/c BEGIN:VCALENDAR vCalendar calendar file -!:mime text/calendar -0 string/c BEGIN:VCARD vCard visiting card -!:mime text/x-vcard diff --git a/magic/msdos b/magic/msdos deleted file mode 100644 index 59a9d2caac..0000000000 --- a/magic/msdos +++ /dev/null @@ -1,368 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: msdos,v 1.84 2013/02/05 13:55:22 christos Exp $ -# msdos: file(1) magic for MS-DOS files -# - -# .BAT files (Daniel Quinlan, quinlan@yggdrasil.com) -# updated by Joerg Jenderek at Oct 2008,Apr 2011 -0 string/t @ ->1 string/cW \ echo\ off DOS batch file text -!:mime text/x-msdos-batch ->1 string/cW echo\ off DOS batch file text -!:mime text/x-msdos-batch ->1 string/cW rem DOS batch file text -!:mime text/x-msdos-batch ->1 string/cW set\ DOS batch file text -!:mime text/x-msdos-batch - -# Tests for various EXE types. -# -# Many of the compressed formats were extraced from IDARC 1.23 source code. -# -0 string/b MZ DOS MZ -!:mime application/x-dosexec -# All non-DOS EXE extensions have the relocation table more than 0x40 bytes into the file. ->0x18 leshort <0x40 MS-DOS executable -# These traditional tests usually work but not always. When test quality support is -# implemented these can be turned on. -#>>0x18 leshort 0x1c (Borland compiler) -#>>0x18 leshort 0x1e (MS compiler) - -# If the relocation table is 0x40 or more bytes into the file, it's definitely -# not a DOS EXE. ->0x18 leshort >0x3f - -# Maybe it's a PE? ->>(0x3c.l) string PE\0\0 PE ->>>(0x3c.l+24) leshort 0x010b \b32 executable ->>>(0x3c.l+24) leshort 0x020b \b32+ executable ->>>(0x3c.l+24) leshort 0x0107 ROM image ->>>(0x3c.l+24) default x Unknown PE signature ->>>>&0 leshort x 0x%x ->>>(0x3c.l+22) leshort&0x2000 >0 (DLL) ->>>(0x3c.l+92) leshort 1 (native) ->>>(0x3c.l+92) leshort 2 (GUI) ->>>(0x3c.l+92) leshort 3 (console) ->>>(0x3c.l+92) leshort 7 (POSIX) ->>>(0x3c.l+92) leshort 9 (Windows CE) ->>>(0x3c.l+92) leshort 10 (EFI application) ->>>(0x3c.l+92) leshort 11 (EFI boot service driver) ->>>(0x3c.l+92) leshort 12 (EFI runtime driver) ->>>(0x3c.l+92) leshort 13 (EFI ROM) ->>>(0x3c.l+92) leshort 14 (XBOX) ->>>(0x3c.l+92) leshort 15 (Windows boot application) ->>>(0x3c.l+92) default x (Unknown subsystem ->>>>&0 leshort x 0x%x) ->>>(0x3c.l+4) leshort 0x14c Intel 80386 ->>>(0x3c.l+4) leshort 0x166 MIPS R4000 ->>>(0x3c.l+4) leshort 0x168 MIPS R10000 ->>>(0x3c.l+4) leshort 0x184 Alpha ->>>(0x3c.l+4) leshort 0x1a2 Hitachi SH3 ->>>(0x3c.l+4) leshort 0x1a6 Hitachi SH4 ->>>(0x3c.l+4) leshort 0x1c0 ARM ->>>(0x3c.l+4) leshort 0x1c2 ARM Thumb ->>>(0x3c.l+4) leshort 0x1c4 ARMv7 Thumb ->>>(0x3c.l+4) leshort 0x1f0 PowerPC ->>>(0x3c.l+4) leshort 0x200 Intel Itanium ->>>(0x3c.l+4) leshort 0x266 MIPS16 ->>>(0x3c.l+4) leshort 0x268 Motorola 68000 ->>>(0x3c.l+4) leshort 0x290 PA-RISC ->>>(0x3c.l+4) leshort 0x366 MIPSIV ->>>(0x3c.l+4) leshort 0x466 MIPS16 with FPU ->>>(0x3c.l+4) leshort 0xebc EFI byte code ->>>(0x3c.l+4) leshort 0x8664 x86-64 ->>>(0x3c.l+4) leshort 0xc0ee MSIL ->>>(0x3c.l+4) default x Unknown processor type ->>>>&0 leshort x 0x%x ->>>(0x3c.l+22) leshort&0x0200 >0 (stripped to external PDB) ->>>(0x3c.l+22) leshort&0x1000 >0 system file ->>>(0x3c.l+24) leshort 0x010b ->>>>(0x3c.l+232) lelong >0 Mono/.Net assembly ->>>(0x3c.l+24) leshort 0x020b ->>>>(0x3c.l+248) lelong >0 Mono/.Net assembly - -# hooray, there's a DOS extender using the PE format, with a valid PE -# executable inside (which just prints a message and exits if run in win) ->>>(8.s*16) string 32STUB \b, 32rtm DOS extender ->>>(8.s*16) string !32STUB \b, for MS Windows ->>>(0x3c.l+0xf8) string UPX0 \b, UPX compressed ->>>(0x3c.l+0xf8) search/0x140 PEC2 \b, PECompact2 compressed ->>>(0x3c.l+0xf8) search/0x140 UPX2 ->>>>(&0x10.l+(-4)) string PK\3\4 \b, ZIP self-extracting archive (Info-Zip) ->>>(0x3c.l+0xf8) search/0x140 .idata ->>>>(&0xe.l+(-4)) string PK\3\4 \b, ZIP self-extracting archive (Info-Zip) ->>>>(&0xe.l+(-4)) string ZZ0 \b, ZZip self-extracting archive ->>>>(&0xe.l+(-4)) string ZZ1 \b, ZZip self-extracting archive ->>>(0x3c.l+0xf8) search/0x140 .rsrc ->>>>(&0x0f.l+(-4)) string a\\\4\5 \b, WinHKI self-extracting archive ->>>>(&0x0f.l+(-4)) string Rar! \b, RAR self-extracting archive ->>>>(&0x0f.l+(-4)) search/0x3000 MSCF \b, InstallShield self-extracting archive ->>>>(&0x0f.l+(-4)) search/32 Nullsoft \b, Nullsoft Installer self-extracting archive ->>>(0x3c.l+0xf8) search/0x140 .data ->>>>(&0x0f.l) string WEXTRACT \b, MS CAB-Installer self-extracting archive ->>>(0x3c.l+0xf8) search/0x140 .petite\0 \b, Petite compressed ->>>>(0x3c.l+0xf7) byte x ->>>>>(&0x104.l+(-4)) string =!sfx! \b, ACE self-extracting archive ->>>(0x3c.l+0xf8) search/0x140 .WISE \b, WISE installer self-extracting archive ->>>(0x3c.l+0xf8) search/0x140 .dz\0\0\0 \b, Dzip self-extracting archive ->>>&(0x3c.l+0xf8) search/0x100 _winzip_ \b, ZIP self-extracting archive (WinZip) ->>>&(0x3c.l+0xf8) search/0x100 SharedD \b, Microsoft Installer self-extracting archive ->>>0x30 string Inno \b, InnoSetup self-extracting archive - -# Hmm, not a PE but the relocation table is too high for a traditional DOS exe, -# must be one of the unusual subformats. ->>(0x3c.l) string !PE\0\0 MS-DOS executable - ->>(0x3c.l) string NE \b, NE ->>>(0x3c.l+0x36) byte 1 for OS/2 1.x ->>>(0x3c.l+0x36) byte 2 for MS Windows 3.x ->>>(0x3c.l+0x36) byte 3 for MS-DOS ->>>(0x3c.l+0x36) byte 4 for Windows 386 ->>>(0x3c.l+0x36) byte 5 for Borland Operating System Services ->>>(0x3c.l+0x36) default x ->>>>(0x3c.l+0x36) byte x (unknown OS %x) ->>>(0x3c.l+0x36) byte 0x81 for MS-DOS, Phar Lap DOS extender ->>>(0x3c.l+0x0c) leshort&0x8003 0x8002 (DLL) ->>>(0x3c.l+0x0c) leshort&0x8003 0x8001 (driver) ->>>&(&0x24.s-1) string ARJSFX \b, ARJ self-extracting archive ->>>(0x3c.l+0x70) search/0x80 WinZip(R)\ Self-Extractor \b, ZIP self-extracting archive (WinZip) - ->>(0x3c.l) string LX\0\0 \b, LX ->>>(0x3c.l+0x0a) leshort <1 (unknown OS) ->>>(0x3c.l+0x0a) leshort 1 for OS/2 ->>>(0x3c.l+0x0a) leshort 2 for MS Windows ->>>(0x3c.l+0x0a) leshort 3 for DOS ->>>(0x3c.l+0x0a) leshort >3 (unknown OS) ->>>(0x3c.l+0x10) lelong&0x28000 =0x8000 (DLL) ->>>(0x3c.l+0x10) lelong&0x20000 >0 (device driver) ->>>(0x3c.l+0x10) lelong&0x300 0x300 (GUI) ->>>(0x3c.l+0x10) lelong&0x28300 <0x300 (console) ->>>(0x3c.l+0x08) leshort 1 i80286 ->>>(0x3c.l+0x08) leshort 2 i80386 ->>>(0x3c.l+0x08) leshort 3 i80486 ->>>(8.s*16) string emx \b, emx ->>>>&1 string x %s ->>>&(&0x54.l-3) string arjsfx \b, ARJ self-extracting archive - -# MS Windows system file, supposedly a collection of LE executables ->>(0x3c.l) string W3 \b, W3 for MS Windows - ->>(0x3c.l) string LE\0\0 \b, LE executable ->>>(0x3c.l+0x0a) leshort 1 -# some DOS extenders use LE files with OS/2 header ->>>>0x240 search/0x100 DOS/4G for MS-DOS, DOS4GW DOS extender ->>>>0x240 search/0x200 WATCOM\ C/C++ for MS-DOS, DOS4GW DOS extender ->>>>0x440 search/0x100 CauseWay\ DOS\ Extender for MS-DOS, CauseWay DOS extender ->>>>0x40 search/0x40 PMODE/W for MS-DOS, PMODE/W DOS extender ->>>>0x40 search/0x40 STUB/32A for MS-DOS, DOS/32A DOS extender (stub) ->>>>0x40 search/0x80 STUB/32C for MS-DOS, DOS/32A DOS extender (configurable stub) ->>>>0x40 search/0x80 DOS/32A for MS-DOS, DOS/32A DOS extender (embedded) -# this is a wild guess; hopefully it is a specific signature ->>>>&0x24 lelong <0x50 ->>>>>(&0x4c.l) string \xfc\xb8WATCOM ->>>>>>&0 search/8 3\xdbf\xb9 \b, 32Lite compressed -# another wild guess: if real OS/2 LE executables exist, they probably have higher start EIP -#>>>>(0x3c.l+0x1c) lelong >0x10000 for OS/2 -# fails with DOS-Extenders. ->>>(0x3c.l+0x0a) leshort 2 for MS Windows ->>>(0x3c.l+0x0a) leshort 3 for DOS ->>>(0x3c.l+0x0a) leshort 4 for MS Windows (VxD) ->>>(&0x7c.l+0x26) string UPX \b, UPX compressed ->>>&(&0x54.l-3) string UNACE \b, ACE self-extracting archive - -# looks like ASCII, probably some embedded copyright message. -# and definitely not NE/LE/LX/PE ->>0x3c lelong >0x20000000 ->>>(4.s*512) leshort !0x014c \b, MZ for MS-DOS -# header data too small for extended executable ->2 long !0 ->>0x18 leshort <0x40 ->>>(4.s*512) leshort !0x014c - ->>>>&(2.s-514) string !LE ->>>>>&-2 string !BW \b, MZ for MS-DOS ->>>>&(2.s-514) string LE \b, LE ->>>>>0x240 search/0x100 DOS/4G for MS-DOS, DOS4GW DOS extender -# educated guess since indirection is still not capable enough for complex offset -# calculations (next embedded executable would be at &(&2*512+&0-2) -# I suspect there are only LE executables in these multi-exe files ->>>>&(2.s-514) string BW ->>>>>0x240 search/0x100 DOS/4G ,\b LE for MS-DOS, DOS4GW DOS extender (embedded) ->>>>>0x240 search/0x100 !DOS/4G ,\b BW collection for MS-DOS - -# This sequence skips to the first COFF segment, usually .text ->(4.s*512) leshort 0x014c \b, COFF ->>(8.s*16) string go32stub for MS-DOS, DJGPP go32 DOS extender ->>(8.s*16) string emx ->>>&1 string x for DOS, Win or OS/2, emx %s ->>&(&0x42.l-3) byte x ->>>&0x26 string UPX \b, UPX compressed -# and yet another guess: small .text, and after large .data is unusal, could be 32lite ->>&0x2c search/0xa0 .text ->>>&0x0b lelong <0x2000 ->>>>&0 lelong >0x6000 \b, 32lite compressed - ->(8.s*16) string $WdX \b, WDos/X DOS extender - -# By now an executable type should have been printed out. The executable -# may be a self-uncompressing archive, so look for evidence of that and -# print it out. -# -# Some signatures below from Greg Roelofs, newt@uchicago.edu. -# ->0x35 string \x8e\xc0\xb9\x08\x00\xf3\xa5\x4a\x75\xeb\x8e\xc3\x8e\xd8\x33\xff\xbe\x30\x00\x05 \b, aPack compressed ->0xe7 string LH/2\ Self-Extract \b, %s ->0x1c string UC2X \b, UCEXE compressed ->0x1c string WWP\ \b, WWPACK compressed ->0x1c string RJSX \b, ARJ self-extracting archive ->0x1c string diet \b, diet compressed ->0x1c string LZ09 \b, LZEXE v0.90 compressed ->0x1c string LZ91 \b, LZEXE v0.91 compressed ->0x1c string tz \b, TinyProg compressed ->0x1e string Copyright\ 1989-1990\ PKWARE\ Inc. Self-extracting PKZIP archive -!:mime application/zip -# Yes, this really is "Copr", not "Corp." ->0x1e string PKLITE\ Copr. Self-extracting PKZIP archive -!:mime application/zip -# winarj stores a message in the stub instead of the sig in the MZ header ->0x20 search/0xe0 aRJsfX \b, ARJ self-extracting archive ->0x20 string AIN ->>0x23 string 2 \b, AIN 2.x compressed ->>0x23 string <2 \b, AIN 1.x compressed ->>0x23 string >2 \b, AIN 1.x compressed ->0x24 string LHa's\ SFX \b, LHa self-extracting archive -!:mime application/x-lha ->0x24 string LHA's\ SFX \b, LHa self-extracting archive -!:mime application/x-lha ->0x24 string \ $ARX \b, ARX self-extracting archive ->0x24 string \ $LHarc \b, LHarc self-extracting archive ->0x20 string SFX\ by\ LARC \b, LARC self-extracting archive ->0x40 string aPKG \b, aPackage self-extracting archive ->0x64 string W\ Collis\0\0 \b, Compack compressed ->0x7a string Windows\ self-extracting\ ZIP \b, ZIP self-extracting archive ->>&0xf4 search/0x140 \x0\x40\x1\x0 ->>>(&0.l+(4)) string MSCF \b, WinHKI CAB self-extracting archive ->1638 string -lh5- \b, LHa self-extracting archive v2.13S ->0x17888 string Rar! \b, RAR self-extracting archive - -# Skip to the end of the EXE. This will usually work fine in the PE case -# because the MZ image is hardcoded into the toolchain and almost certainly -# won't match any of these signatures. ->(4.s*512) long x ->>&(2.s-517) byte x ->>>&0 string PK\3\4 \b, ZIP self-extracting archive ->>>&0 string Rar! \b, RAR self-extracting archive ->>>&0 string =!\x11 \b, AIN 2.x self-extracting archive ->>>&0 string =!\x12 \b, AIN 2.x self-extracting archive ->>>&0 string =!\x17 \b, AIN 1.x self-extracting archive ->>>&0 string =!\x18 \b, AIN 1.x self-extracting archive ->>>&7 search/400 **ACE** \b, ACE self-extracting archive ->>>&0 search/0x480 UC2SFX\ Header \b, UC2 self-extracting archive - -# a few unknown ZIP sfxes, no idea if they are needed or if they are -# already captured by the generic patterns above ->(8.s*16) search/0x20 PKSFX \b, ZIP self-extracting archive (PKZIP) -# TODO: how to add this? >FileSize-34 string Windows\ Self-Installing\ Executable \b, ZIP self-extracting archive -# - -# TELVOX Teleinformatica CODEC self-extractor for OS/2: ->49801 string \x79\xff\x80\xff\x76\xff \b, CODEC archive v3.21 ->>49824 leshort =1 \b, 1 file ->>49824 leshort >1 \b, %u files - -# Popular applications -2080 string Microsoft\ Word\ 6.0\ Document %s -!:mime application/msword -2080 string Documento\ Microsoft\ Word\ 6 Spanish Microsoft Word 6 document data -!:mime application/msword -# Pawel Wiecek (for polish Word) -2112 string MSWordDoc Microsoft Word document data -!:mime application/msword -# -0 belong 0x31be0000 Microsoft Word Document -!:mime application/msword -# -0 string/b PO^Q` Microsoft Word 6.0 Document -!:mime application/msword -# -0 string/b \376\067\0\043 Microsoft Office Document -!:mime application/msword -0 string/b \333\245-\0\0\0 Microsoft Office Document -!:mime application/msword -512 string/b \354\245\301 Microsoft Word Document -!:mime application/msword - -# -0 string/b \xDB\xA5\x2D\x00 Microsoft WinWord 2.0 Document -!:mime application/msword -# -2080 string Microsoft\ Excel\ 5.0\ Worksheet %s -!:mime application/vnd.ms-excel -# -0 string/b \xDB\xA5\x2D\x00 Microsoft WinWord 2.0 Document -!:mime application/msword - -2080 string Foglio\ di\ lavoro\ Microsoft\ Exce %s -!:mime application/vnd.ms-excel -# -# Pawel Wiecek (for polish Excel) -2114 string Biff5 Microsoft Excel 5.0 Worksheet -!:mime application/vnd.ms-excel -# Italian MS-Excel -2121 string Biff5 Microsoft Excel 5.0 Worksheet -!:mime application/vnd.ms-excel -0 string/b \x09\x04\x06\x00\x00\x00\x10\x00 Microsoft Excel Worksheet -!:mime application/vnd.ms-excel -# -0 belong 0x00001a00 Lotus 1-2-3 -!:mime application/x-123 -# -0 belong 0x00000200 Lotus 1-2-3 -!:mime application/x-123 -0 string/b WordPro\0 Lotus WordPro -!:mime application/vnd.lotus-wordpro -0 string/b WordPro\r\373 Lotus WordPro -!:mime application/vnd.lotus-wordpro - -# Windows icons (Ian Springer ) -0 string/b \000\000\001\000 MS Windows icon resource -!:mime image/x-icon - -# .PIF files added by Joerg Jenderek from http://smsoft.ru/en/pifdoc.htm -# only for windows versions equal or greater 3.0 -0x171 string MICROSOFT\ PIFEX\0 Windows Program Information File -!:mime application/x-dosexec - -# TNEF magic From "Joomy" -# Microsoft Outlook's Transport Neutral Encapsulation Format (TNEF) -0 leshort 0x223e9f78 TNEF -!:mime application/vnd.ms-tnef - -#------------------------------------------------------------------------------ -# From Stuart Caie (developer of cabextract) -# Microsoft Cabinet files -0 string/b MSCF\0\0\0\0 Microsoft Cabinet archive data -!:mime application/vnd.ms-cab-compressed - -# from http://filext.com by Derek M Jones -# False positive with PPT (also currently this string is too long) -#0 string/b \xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x3E\x00\x03\x00\xFE\xFF\x09\x00\x06 Microsoft Installer -0 string/b \320\317\021\340\241\261\032\341 Microsoft Office Document -#>48 byte 0x1B Excel Document -#!:mime application/vnd.ms-excel ->546 string bjbj Microsoft Word Document -!:mime application/msword ->546 string jbjb Microsoft Word Document -!:mime application/msword - -0 string/b \224\246\056 Microsoft Word Document -!:mime application/msword - -512 string R\0o\0o\0t\0\ \0E\0n\0t\0r\0y Microsoft Word Document -!:mime application/msword - -# MS eBook format (.lit) -0 string/b ITOLITLS Microsoft Reader eBook Data ->8 lelong x \b, version %u -!:mime application/x-ms-reader diff --git a/magic/neko b/magic/neko deleted file mode 100644 index 50163a0861..0000000000 --- a/magic/neko +++ /dev/null @@ -1,12 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------ -# $File: java,v 1.12 2009/09/19 16:28:10 christos Exp $ - -# From: Mikhail Gusarov -# NekoVM (http://nekovm.org/) bytecode -0 string NEKO NekoVM bytecode ->4 lelong x (%d global symbols, ->8 lelong x %d global fields, ->12 lelong x %d bytecode ops) -!:mime application/x-nekovm-bytecode - diff --git a/magic/pascal b/magic/pascal deleted file mode 100644 index 911eea3c0c..0000000000 --- a/magic/pascal +++ /dev/null @@ -1,11 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File$ -# pascal: file(1) magic for Pascal source -# -0 search/8192 (input, Pascal source text -!:mime text/x-pascal -0 regex \^program Pascal source text -!:mime text/x-pascal -0 regex \^record Pascal source text -!:mime text/x-pascal diff --git a/magic/pdf b/magic/pdf deleted file mode 100644 index 761006ffe6..0000000000 --- a/magic/pdf +++ /dev/null @@ -1,8 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File$ -# pdf: file(1) magic for Portable Document Format -# - -0 string %PDF- PDF document -!:mime application/pdf diff --git a/magic/perl b/magic/perl deleted file mode 100644 index 12ec33b73a..0000000000 --- a/magic/perl +++ /dev/null @@ -1,26 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: perl,v 1.19 2012/06/20 21:16:25 christos Exp $ -# perl: file(1) magic for Larry Wall's perl language. -# -# The `eval' lines recognizes an outrageously clever hack. -# Keith Waclena -# Send additions to -0 search/1/w #!\ /bin/perl Perl script text executable -!:mime text/x-perl -0 search/1 eval\ "exec\ /bin/perl Perl script text -!:mime text/x-perl -0 search/1/w #!\ /usr/bin/perl Perl script text executable -!:mime text/x-perl -0 search/1 eval\ "exec\ /usr/bin/perl Perl script text -!:mime text/x-perl -0 search/1/w #!\ /usr/local/bin/perl Perl script text executable -!:mime text/x-perl -0 search/1 eval\ "exec\ /usr/local/bin/perl Perl script text -!:mime text/x-perl -0 search/1 eval\ '(exit\ $?0)'\ &&\ eval\ 'exec Perl script text -!:mime text/x-perl -0 search/1 #!/usr/bin/env\ perl Perl script text executable -!:mime text/x-perl -0 search/1 #!\ /usr/bin/env\ perl Perl script text executable -!:mime text/x-perl diff --git a/magic/pgp b/magic/pgp deleted file mode 100644 index 2bdfb77981..0000000000 --- a/magic/pgp +++ /dev/null @@ -1,27 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File$ -# pgp: file(1) magic for Pretty Good Privacy -# see http://lists.gnupg.org/pipermail/gnupg-devel/1999-September/016052.html -# -0 beshort 0x9900 PGP key public ring -!:mime application/x-pgp-keyring -0 beshort 0x9501 PGP key security ring -!:mime application/x-pgp-keyring -0 beshort 0x9500 PGP key security ring -!:mime application/x-pgp-keyring -0 beshort 0xa600 PGP encrypted data -#!:mime application/pgp-encrypted -#0 string -----BEGIN\040PGP text/PGP armored data -!:mime text/PGP # encoding: armored data -#>15 string PUBLIC\040KEY\040BLOCK- public key block -#>15 string MESSAGE- message -#>15 string SIGNED\040MESSAGE- signed message -#>15 string PGP\040SIGNATURE- signature - -2 string ---BEGIN\ PGP\ PUBLIC\ KEY\ BLOCK- PGP public key block -!:mime application/pgp-keys -0 string -----BEGIN\040PGP\40MESSAGE- PGP message -!:mime application/pgp -0 string -----BEGIN\040PGP\40SIGNATURE- PGP signature -!:mime application/pgp-signature diff --git a/magic/pkgadd b/magic/pkgadd deleted file mode 100644 index 602b4ec21d..0000000000 --- a/magic/pkgadd +++ /dev/null @@ -1,7 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File$ -# pkgadd: file(1) magic for SysV R4 PKG Datastreams -# -0 string #\ PaCkAgE\ DaTaStReAm pkg Datastream (SVR4) -!:mime application/x-svr4-package diff --git a/magic/printer b/magic/printer deleted file mode 100644 index cdce275b12..0000000000 --- a/magic/printer +++ /dev/null @@ -1,14 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: printer,v 1.24 2011/05/08 16:34:51 christos Exp $ -# printer: file(1) magic for printer-formatted files -# - -# PostScript, updated by Daniel Quinlan (quinlan@yggdrasil.com) -0 string %! PostScript document text -!:mime application/postscript -!:apple ASPSTEXT -# Some PCs have the annoying habit of adding a ^D as a document separator -0 string \004%! PostScript document text -!:mime application/postscript -!:apple ASPSTEXT diff --git a/magic/python b/magic/python deleted file mode 100644 index 1cd724bc59..0000000000 --- a/magic/python +++ /dev/null @@ -1,46 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: python,v 1.21 2012/06/21 01:12:51 christos Exp $ -# python: file(1) magic for python -# - -0 search/1/w #!\ /usr/bin/python Python script text executable -!:mime text/x-python -0 search/1/w #!\ /usr/local/bin/python Python script text executable -!:mime text/x-python -0 search/1 #!/usr/bin/env\ python Python script text executable -!:mime text/x-python -0 search/1 #!\ /usr/bin/env\ python Python script text executable -!:mime text/x-python - -# from module.submodule import func1, func2 -0 regex \^from\\s+(\\w|\\.)+\\s+import.*$ Python script text executable -!:mime text/x-python - -# def __init__ (self, ...): -0 search/4096 def\ __init__ ->&0 search/64 self Python script text executable -!:mime text/x-python - -# comments -0 search/4096 ''' ->&0 regex .*'''$ Python script text executable -!:mime text/x-python - -0 search/4096 """ ->&0 regex .*"""$ Python script text executable -!:mime text/x-python - -# try: -# except: or finally: -# block -0 search/4096 try: ->&0 regex \^\\s*except.*: Python script text executable -!:mime text/x-python ->&0 search/4096 finally: Python script text executable -!:mime text/x-python - -# def name(args, args): -0 regex \^(\ |\\t)*def\ +[a-zA-Z]+ ->&0 regex \ *\\(([a-zA-Z]|,|\ )*\\):$ Python script text executable -!:mime text/x-python diff --git a/magic/riff b/magic/riff deleted file mode 100644 index 929dc9aa89..0000000000 --- a/magic/riff +++ /dev/null @@ -1,36 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: riff,v 1.22 2011/09/06 11:00:06 christos Exp $ -# riff: file(1) magic for RIFF format -# See -# -# http://www.seanet.com/users/matts/riffmci/riffmci.htm -# -# AVI section extended by Patrik Radman -# -0 string RIFF RIFF (little-endian) data -# Microsoft WAVE format (*.wav) ->8 string WAVE \b, WAVE audio -!:mime audio/x-wav -# Corel Draw Picture ->8 string CDRA \b, Corel Draw Picture -!:mime image/x-coreldraw -# AVI == Audio Video Interleave ->8 string AVI\040 \b, AVI -!:mime video/x-msvideo - -#------------------------------------------------------------------------------ -# Sony Wave64 -# see http://www.vcs.de/fileadmin/user_upload/MBS/PDF/Whitepaper/Informations_about_Sony_Wave64.pdf -# 128 bit RIFF-GUID { 66666972-912E-11CF-A5D6-28DB04C10000 } in little-endian -0 string riff\x2E\x91\xCF\x11\xA5\xD6\x28\xDB\x04\xC1\x00\x00 Sony Wave64 RIFF data -# 128 bit + total file size (64 bits) so 24 bytes -# then WAVE-GUID { 65766177-ACF3-11D3-8CD1-00C04F8EDB8A } ->24 string wave\xF3\xAC\xD3\x11\x8C\xD1\x00\xC0\x4F\x8E\xDB\x8A \b, WAVE 64 audio -!:mime audio/x-w64 - -#------------------------------------------------------------------------------ -# MBWF/RF64 -# see EBU TECH 3306 http://tech.ebu.ch/docs/tech/tech3306-2009.pdf -0 string RF64\xff\xff\xff\xffWAVEds64 MBWF/RF64 audio -!:mime audio/x-wav diff --git a/magic/rpm b/magic/rpm deleted file mode 100644 index 2558ebeef1..0000000000 --- a/magic/rpm +++ /dev/null @@ -1,12 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: rpm,v 1.11 2011/06/14 12:47:41 christos Exp $ -# -# RPM: file(1) magic for Red Hat Packages Erik Troan (ewt@redhat.com) -# -0 belong 0xedabeedb RPM -!:mime application/x-rpm - -#delta RPM Daniel Novotny (dnovotny@redhat.com) -0 string drpm Delta RPM -!:mime application/x-rpm diff --git a/magic/rtf b/magic/rtf deleted file mode 100644 index 0719264e47..0000000000 --- a/magic/rtf +++ /dev/null @@ -1,9 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File$ -# rtf: file(1) magic for Rich Text Format (RTF) -# -# Duncan P. Simpson, D.P.Simpson@dcs.warwick.ac.uk -# -0 string {\\rtf Rich Text Format data, -!:mime text/rtf diff --git a/magic/ruby b/magic/ruby deleted file mode 100644 index 41682a89ad..0000000000 --- a/magic/ruby +++ /dev/null @@ -1,28 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: ruby,v 1.4 2010/07/08 20:24:13 christos Exp $ -# ruby: file(1) magic for Ruby scripting language -# URL: http://www.ruby-lang.org/ -# From: Reuben Thomas - -# Ruby scripts -0 search/1/w #!\ /usr/bin/ruby Ruby script text executable -!:mime text/x-ruby -0 search/1/w #!\ /usr/local/bin/ruby Ruby script text executable -!:mime text/x-ruby -0 search/1 #!/usr/bin/env\ ruby Ruby script text executable -!:mime text/x-ruby -0 search/1 #!\ /usr/bin/env\ ruby Ruby script text executable -!:mime text/x-ruby - -# What looks like ruby, but does not have a shebang -# (modules and such) -# From: Lubomir Rintel -0 regex \^[\ \t]*require[\ \t]'[A-Za-z_/]+' ->0 regex include\ [A-Z]|def\ [a-z]|\ do$ ->>0 regex \^[\ \t]*end([\ \t]*[;#].*)?$ Ruby script text -!:mime text/x-ruby -0 regex \^[\ \t]*(class|module)[\ \t][A-Z] ->0 regex (modul|includ)e\ [A-Z]|def\ [a-z] ->>0 regex \^[\ \t]*end([\ \t]*[;#].*)?$ Ruby module source text -!:mime text/x-ruby diff --git a/magic/sc b/magic/sc deleted file mode 100644 index 75333b3916..0000000000 --- a/magic/sc +++ /dev/null @@ -1,7 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File$ -# sc: file(1) magic for "sc" spreadsheet -# -38 string Spreadsheet sc spreadsheet file -!:mime application/x-sc diff --git a/magic/sgml b/magic/sgml deleted file mode 100644 index 64efa2c153..0000000000 --- a/magic/sgml +++ /dev/null @@ -1,82 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: sgml,v 1.28 2012/04/28 21:20:26 christos Exp $ -# Type: SVG Vectorial Graphics -# From: Noel Torres -0 string \15 string >\0 ->>19 search/4096 \>19 search/4096 \15 string >\0 ->>19 search/4096/cWbt \15 string >\0 ->>19 search/4096/cWbt \15 string >\0 ->>19 search/4096/cWbt \ - -# Although we may know the offset of certain text fields in TeX DVI -# and font files, we can't use them reliably because they are not -# zero terminated. [but we do anyway, christos] -0 string \367\002 TeX DVI file -!:mime application/x-dvi - -# There is no way to detect TeX Font Metric (*.tfm) files without -# breaking them apart and reading the data. The following patterns -# match most *.tfm files generated by METAFONT or afm2tfm. -2 string \000\021 TeX font metric data -!:mime application/x-tex-tfm -2 string \000\022 TeX font metric data -!:mime application/x-tex-tfm - -# Texinfo and GNU Info, from Daniel Quinlan (quinlan@yggdrasil.com) -0 search/1 \\input\ texinfo Texinfo source text -!:mime text/x-texinfo -0 search/1 This\ is\ Info\ file GNU Info text -!:mime text/x-info - -# TeX documents, from Daniel Quinlan (quinlan@yggdrasil.com) -0 search/4096 \\input TeX document text -!:mime text/x-tex -!:strength + 15 -0 search/4096 \\section LaTeX document text -!:mime text/x-tex -!:strength + 18 -0 search/4096 \\setlength LaTeX document text -!:mime text/x-tex -!:strength + 15 -0 search/4096 \\documentstyle LaTeX document text -!:mime text/x-tex -!:strength + 18 -0 search/4096 \\chapter LaTeX document text -!:mime text/x-tex -!:strength + 18 -0 search/4096 \\documentclass LaTeX 2e document text -!:mime text/x-tex -!:strength + 15 -0 search/4096 \\relax LaTeX auxiliary file -!:mime text/x-tex -!:strength + 15 -0 search/4096 \\contentsline LaTeX table of contents -!:mime text/x-tex -!:strength + 15 -0 search/4096 %\ -*-latex-*- LaTeX document text -!:mime text/x-tex diff --git a/magic/troff b/magic/troff deleted file mode 100644 index 7f60b1d9b3..0000000000 --- a/magic/troff +++ /dev/null @@ -1,22 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File$ -# troff: file(1) magic for *roff -# -# updated by Daniel Quinlan (quinlan@yggdrasil.com) - -# troff input -0 search/1 .\\" troff or preprocessor input text -!:mime text/troff -0 search/1 '\\" troff or preprocessor input text -!:mime text/troff -0 search/1 '.\\" troff or preprocessor input text -!:mime text/troff -0 search/1 \\" troff or preprocessor input text -!:mime text/troff -0 search/1 ''' troff or preprocessor input text -!:mime text/troff -0 regex/20 \^\\.[A-Za-z0-9][A-Za-z0-9][\ \t] troff or preprocessor input text -!:mime text/troff -0 regex/20 \^\\.[A-Za-z0-9][A-Za-z0-9]$ troff or preprocessor input text -!:mime text/troff diff --git a/magic/vorbis b/magic/vorbis deleted file mode 100644 index 4d25c3c3cd..0000000000 --- a/magic/vorbis +++ /dev/null @@ -1,26 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File$ -# vorbis: file(1) magic for Ogg/Vorbis files -# -# From Felix von Leitner -# Extended by Beni Cherniavsky -# Further extended by Greg Wooledge -# -# Most (everything but the number of channels and bitrate) is commented -# out with `##' as it's not interesting to the average user. The most -# probable things advanced users would want to uncomment are probably -# the number of comments and the encoder version. -# -# FIXME: The first match has been made a search, so that it can skip -# over prepended ID3 tags. This will work for MIME type detection, but -# won't work for detecting other properties of the file (they all need -# to be made relative to the search). In any case, if the file has ID3 -# tags, the ID3 information will be printed, not the Ogg information, -# so until that's fixed, this doesn't matter. -# FIXME[2]: Disable the above for now, since search assumes text mode. -# -# --- Ogg Framing --- -#0 search/1000 OggS Ogg data -0 string OggS Ogg data -!:mime application/ogg diff --git a/magic/warc b/magic/warc deleted file mode 100644 index 2a2aeb6fae..0000000000 --- a/magic/warc +++ /dev/null @@ -1,14 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: warc,v 1.2 2009/09/19 16:28:13 christos Exp $ -# warc: file(1) magic for WARC files - -0 string WARC/ WARC Archive ->5 string x version %.4s -!:mime application/warc - -#------------------------------------------------------------------------------ -# Arc File Format from Internet Archive -# see http://www.archive.org/web/researcher/ArcFileFormat.php -0 string filedesc:// Internet Archive File -!:mime application/x-ia-arc diff --git a/magic/windows b/magic/windows deleted file mode 100644 index 6a529782a9..0000000000 --- a/magic/windows +++ /dev/null @@ -1,19 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: windows,v 1.4 2009/09/19 16:28:13 christos Exp $ -# windows: file(1) magic for Microsoft Windows -# -# This file is mainly reserved for files where programs -# using them are run almost always on MS Windows 3.x or -# above, or files only used exclusively in Windows OS, -# where there is no better category to allocate for. -# For example, even though WinZIP almost run on Windows -# only, it is better to treat them as "archive" instead. -# For format usable in DOS, such as generic executable -# format, please specify under "msdos" file. -# - -# From: Pal Tamas -# Autorun File -0 string/c [autorun]\r\n Microsoft Windows Autorun file. -!:mime application/x-setupscript. diff --git a/magic/wordprocessors b/magic/wordprocessors deleted file mode 100644 index 7de3413c0a..0000000000 --- a/magic/wordprocessors +++ /dev/null @@ -1,43 +0,0 @@ -# See COPYING file in this directory for original libmagic copyright. -#------------------------------------------------------------------------------ -# $File: wordprocessors,v 1.16 2012/10/29 17:36:49 christos Exp $ -# wordprocessors: file(1) magic fo word processors. -# - -# Hangul (Korean) Word Processor File -# From: Won-Kyu Park -512 string R\0o\0o\0t\0 Hangul (Korean) Word Processor File 2000 -!:mime application/x-hwp - -# Quark Express from http://www.garykessler.net/library/file_sigs.html -2 string MMXPR3 Motorola Quark Express Document (English) -!:mime application/x-quark-xpress-3 - -#------------------------------------------------------------------------------ -# ichitaro456: file(1) magic for Just System Word Processor Ichitaro -# -# Contributor kenzo-: -# Reversed-engineered JS Ichitaro magic numbers -# - -0 string DOC ->43 byte 0x14 Just System Word Processor Ichitaro v4 -!:mime application/x-ichitaro4 - -0 string DOC ->43 byte 0x15 Just System Word Processor Ichitaro v5 -!:mime application/x-ichitaro5 - -0 string DOC ->43 byte 0x16 Just System Word Processor Ichitaro v6 -!:mime application/x-ichitaro6 - -# Type: Freemind mindmap documents -# From: Jamie Thompson -0 string/w \ -0 string \ 0 ) + f$info$local_orig=Site::is_local_addr(f$is_orig ? cid$orig_h : cid$resp_h); + + add f$info$rx_hosts[f$is_orig ? cid$resp_h : cid$orig_h]; + } + +event file_timeout(f: fa_file) &priority=10 + { + set_info(f); + f$info$timedout = T; + } + +event file_state_remove(f: fa_file) &priority=10 + { + set_info(f); + } + +event file_state_remove(f: fa_file) &priority=-10 + { + Log::write(Files::LOG, f$info); + } + +function register_protocol(tag: Analyzer::Tag, reg: ProtoRegistration): bool + { + local result = (tag !in registered_protocols); + registered_protocols[tag] = reg; + return result; + } + +function describe(f: fa_file): string + { + local tag = Analyzer::get_tag(f$source); + if ( tag !in registered_protocols ) + return ""; + + local handler = registered_protocols[tag]; + return handler$describe(f); + } + +event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) &priority=5 + { + if ( tag !in registered_protocols ) + return; + + local handler = registered_protocols[tag]; + set_file_handle(handler$get_file_handle(c, is_orig)); + } diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro index aeb7bf4bfc..a201a7a041 100644 --- a/scripts/base/frameworks/intel/main.bro +++ b/scripts/base/frameworks/intel/main.bro @@ -10,13 +10,14 @@ module Intel; export { redef enum Log::ID += { LOG }; - ## String data needs to be further categoried since it could represent - ## and number of types of data. - type StrType: enum { + ## Enum type to represent various types of intelligence data. + type Type: enum { + ## An IP address. + ADDR, ## A complete URL without the prefix "http://". URL, - ## User-Agent string, typically HTTP or mail message body. - USER_AGENT, + ## Software name. + SOFTWARE, ## Email address. EMAIL, ## DNS domain name. @@ -44,18 +45,15 @@ export { ## Represents a piece of intelligence. type Item: record { - ## The IP address if the intelligence is about an IP address. - host: addr &optional; - ## The network if the intelligence is about a CIDR block. - net: subnet &optional; - ## The string if the intelligence is about a string. - str: string &optional; - ## The type of data that is in the string if the $str field is set. - str_type: StrType &optional; + ## The intelligence indicator. + indicator: string; + + ## The type of data that the indicator field represents. + indicator_type: Type; - ## Metadata for the item. Typically represents more deeply \ + ## Metadata for the item. Typically represents more deeply ## descriptive data for a piece of intelligence. - meta: MetaData; + meta: MetaData; }; ## Enum to represent where data came from when it was discovered. @@ -65,23 +63,23 @@ export { IN_ANYWHERE, }; - ## The $host field and combination of $str and $str_type fields are mutually - ## exclusive. These records *must* represent either an IP address being - ## seen or a string being seen. type Seen: record { - ## The IP address if the data seen is an IP address. - host: addr &log &optional; ## The string if the data is about a string. - str: string &log &optional; - ## The type of data that is in the string if the $str field is set. - str_type: StrType &log &optional; + indicator: string &log &optional; + + ## The type of data that the indicator represents. + indicator_type: Type &log &optional; + + ## If the indicator type was :bro:enum:`Intel::ADDR`, then this + ## field will be present. + host: addr &optional; ## Where the data was discovered. - where: Where &log; + where: Where &log; ## If the data was discovered within a connection, the ## connection record should go into get to give context to the data. - conn: connection &optional; + conn: connection &optional; }; ## Record used for the logging framework representing a positive @@ -100,7 +98,7 @@ export { ## Where the data was seen. seen: Seen &log; ## Sources which supplied data that resulted in this match. - sources: set[string] &log; + sources: set[string] &log &default=string_set(); }; ## Intelligence data manipulation functions. @@ -135,8 +133,8 @@ const have_full_data = T &redef; # The in memory data structure for holding intelligence. type DataStore: record { - net_data: table[subnet] of set[MetaData]; - string_data: table[string, StrType] of set[MetaData]; + host_data: table[addr] of set[MetaData]; + string_data: table[string, Type] of set[MetaData]; }; global data_store: DataStore &redef; @@ -144,8 +142,8 @@ global data_store: DataStore &redef; # This is primarily for workers to do the initial quick matches and store # a minimal amount of data for the full match to happen on the manager. type MinDataStore: record { - net_data: set[subnet]; - string_data: set[string, StrType]; + host_data: set[addr]; + string_data: set[string, Type]; }; global min_data_store: MinDataStore &redef; @@ -157,15 +155,13 @@ event bro_init() &priority=5 function find(s: Seen): bool { - if ( s?$host && - ((have_full_data && s$host in data_store$net_data) || - (s$host in min_data_store$net_data))) + if ( s?$host ) { - return T; + return ((s$host in min_data_store$host_data) || + (have_full_data && s$host in data_store$host_data)); } - else if ( s?$str && s?$str_type && - ((have_full_data && [s$str, s$str_type] in data_store$string_data) || - ([s$str, s$str_type] in min_data_store$string_data))) + else if ( ([to_lower(s$indicator), s$indicator_type] in min_data_store$string_data) || + (have_full_data && [to_lower(s$indicator), s$indicator_type] in data_store$string_data) ) { return T; } @@ -177,8 +173,7 @@ function find(s: Seen): bool function get_items(s: Seen): set[Item] { - local item: Item; - local return_data: set[Item] = set(); + local return_data: set[Item]; if ( ! have_full_data ) { @@ -191,26 +186,23 @@ function get_items(s: Seen): set[Item] if ( s?$host ) { # See if the host is known about and it has meta values - if ( s$host in data_store$net_data ) + if ( s$host in data_store$host_data ) { - for ( m in data_store$net_data[s$host] ) + for ( m in data_store$host_data[s$host] ) { - # TODO: the lookup should be finding all and not just most specific - # and $host/$net should have the correct value. - item = [$host=s$host, $meta=m]; - add return_data[item]; + add return_data[Item($indicator=cat(s$host), $indicator_type=ADDR, $meta=m)]; } } } - else if ( s?$str && s?$str_type ) + else { + local lower_indicator = to_lower(s$indicator); # See if the string is known about and it has meta values - if ( [s$str, s$str_type] in data_store$string_data ) + if ( [lower_indicator, s$indicator_type] in data_store$string_data ) { - for ( m in data_store$string_data[s$str, s$str_type] ) + for ( m in data_store$string_data[lower_indicator, s$indicator_type] ) { - item = [$str=s$str, $str_type=s$str_type, $meta=m]; - add return_data[item]; + add return_data[Item($indicator=s$indicator, $indicator_type=s$indicator_type, $meta=m)]; } } } @@ -222,6 +214,12 @@ function Intel::seen(s: Seen) { if ( find(s) ) { + if ( s?$host ) + { + s$indicator = cat(s$host); + s$indicator_type = Intel::ADDR; + } + if ( have_full_data ) { local items = get_items(s); @@ -250,8 +248,7 @@ function has_meta(check: MetaData, metas: set[MetaData]): bool event Intel::match(s: Seen, items: set[Item]) &priority=5 { - local empty_set: set[string] = set(); - local info: Info = [$ts=network_time(), $seen=s, $sources=empty_set]; + local info: Info = [$ts=network_time(), $seen=s]; if ( s?$conn ) { @@ -267,52 +264,37 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5 function insert(item: Item) { - if ( item?$str && !item?$str_type ) - { - event reporter_warning(network_time(), fmt("You must provide a str_type for strings or this item doesn't make sense. Item: %s", item), ""); - return; - } - # Create and fill out the meta data item. local meta = item$meta; local metas: set[MetaData]; - if ( item?$host ) + # All intelligence is case insensitive at the moment. + local lower_indicator = to_lower(item$indicator); + + if ( item$indicator_type == ADDR ) { - local host = mask_addr(item$host, is_v4_addr(item$host) ? 32 : 128); + local host = to_addr(item$indicator); if ( have_full_data ) { - if ( host !in data_store$net_data ) - data_store$net_data[host] = set(); + if ( host !in data_store$host_data ) + data_store$host_data[host] = set(); - metas = data_store$net_data[host]; + metas = data_store$host_data[host]; } - add min_data_store$net_data[host]; + add min_data_store$host_data[host]; } - else if ( item?$net ) + else { if ( have_full_data ) { - if ( item$net !in data_store$net_data ) - data_store$net_data[item$net] = set(); + if ( [lower_indicator, item$indicator_type] !in data_store$string_data ) + data_store$string_data[lower_indicator, item$indicator_type] = set(); - metas = data_store$net_data[item$net]; + metas = data_store$string_data[lower_indicator, item$indicator_type]; } - add min_data_store$net_data[item$net]; - } - else if ( item?$str ) - { - if ( have_full_data ) - { - if ( [item$str, item$str_type] !in data_store$string_data ) - data_store$string_data[item$str, item$str_type] = set(); - - metas = data_store$string_data[item$str, item$str_type]; - } - - add min_data_store$string_data[item$str, item$str_type]; + add min_data_store$string_data[lower_indicator, item$indicator_type]; } local updated = F; diff --git a/scripts/base/frameworks/notice/main.bro b/scripts/base/frameworks/notice/main.bro index 30e0013517..d046e6b076 100644 --- a/scripts/base/frameworks/notice/main.bro +++ b/scripts/base/frameworks/notice/main.bro @@ -68,6 +68,25 @@ export { ## the notice policy. iconn: icmp_conn &optional; + ## A file record if the notice is relted to a file. The + ## reference to the actual fa_file record will be deleted after applying + ## the notice policy. + f: fa_file &optional; + + ## A file unique ID if this notice is related to a file. If the $f + ## field is provided, this will be automatically filled out. + fuid: string &log &optional; + + ## A mime type if the notice is related to a file. If the $f field + ## is provided, this will be automatically filled out. + file_mime_type: string &log &optional; + + ## Frequently files can be "described" to give a bit more context. + ## This field will typically be automatically filled out from an + ## fa_file record. For example, if a notice was related to a + ## file over HTTP, the URL of the request would be shown. + file_desc: string &log &optional; + ## The transport protocol. Filled automatically when either conn, iconn ## or p is specified. proto: transport_proto &log &optional; @@ -460,10 +479,28 @@ function apply_policy(n: Notice::Info) if ( ! n?$ts ) n$ts = network_time(); + if ( n?$f ) + { + if ( ! n?$fuid ) + n$fuid = n$f$id; + + if ( ! n?$file_mime_type && n$f?$mime_type ) + n$file_mime_type = n$f$mime_type; + + n$file_desc = Files::describe(n$f); + + if ( n$f?$conns && |n$f$conns| == 1 ) + { + for ( id in n$f$conns ) + n$conn = n$f$conns[id]; + } + } + if ( n?$conn ) { if ( ! n?$id ) n$id = n$conn$id; + if ( ! n?$uid ) n$uid = n$conn$uid; } @@ -513,13 +550,15 @@ function apply_policy(n: Notice::Info) if ( ! n?$suppress_for ) n$suppress_for = default_suppression_interval; - # Delete the connection record if it's there so we aren't sending that - # to remote machines. It can cause problems due to the size of the - # connection record. + # Delete the connection and file records if they're there so we + # aren't sending that to remote machines. It can cause problems + # due to the size of those records. if ( n?$conn ) delete n$conn; if ( n?$iconn ) delete n$iconn; + if ( n?$f ) + delete n$f; } function internal_NOTICE(n: Notice::Info) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index cffa6d80f1..7ee4c627b3 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -328,7 +328,7 @@ type fa_file: record { ## An identification of the source of the file data. E.g. it may be ## a network protocol over which it was transferred, or a local file ## path which was read, or some other input source. - source: string &optional; + source: string; ## If the source of this file is is a network connection, this field ## may be set to indicate the directionality. @@ -3049,6 +3049,6 @@ const snaplen = 8192 &redef; @load base/frameworks/logging @load base/frameworks/input @load base/frameworks/analyzer -@load base/frameworks/file-analysis +@load base/frameworks/files @load base/bif diff --git a/scripts/base/init-default.bro b/scripts/base/init-default.bro index 6aa8ff5e26..6e348cfffd 100644 --- a/scripts/base/init-default.bro +++ b/scripts/base/init-default.bro @@ -5,9 +5,12 @@ ##! you actually want. @load base/utils/site +@load base/utils/active-http @load base/utils/addrs @load base/utils/conn-ids +@load base/utils/dir @load base/utils/directions-and-hosts +@load base/utils/exec @load base/utils/files @load base/utils/numbers @load base/utils/paths @@ -49,4 +52,7 @@ @load base/protocols/syslog @load base/protocols/tunnels +@load base/files/hash +@load base/files/extract + @load base/misc/find-checksum-offloading diff --git a/scripts/base/protocols/ftp/__load__.bro b/scripts/base/protocols/ftp/__load__.bro index f3226de69d..ebb09e702c 100644 --- a/scripts/base/protocols/ftp/__load__.bro +++ b/scripts/base/protocols/ftp/__load__.bro @@ -1,7 +1,7 @@ @load ./utils-commands @load ./main -@load ./file-analysis -@load ./file-extract +@load ./utils +@load ./files @load ./gridftp -@load-sigs ./dpd.sig \ No newline at end of file +@load-sigs ./dpd.sig diff --git a/scripts/base/protocols/ftp/file-analysis.bro b/scripts/base/protocols/ftp/file-analysis.bro deleted file mode 100644 index 2d7609197a..0000000000 --- a/scripts/base/protocols/ftp/file-analysis.bro +++ /dev/null @@ -1,48 +0,0 @@ -@load ./main -@load base/utils/conn-ids -@load base/frameworks/file-analysis/main - -module FTP; - -export { - ## Default file handle provider for FTP. - global get_file_handle: function(c: connection, is_orig: bool): string; -} - -function get_handle_string(c: connection): string - { - return cat(Analyzer::ANALYZER_FTP_DATA, " ", c$start_time, " ", id_string(c$id)); - } - -function get_file_handle(c: connection, is_orig: bool): string - { - if ( [c$id$resp_h, c$id$resp_p] !in ftp_data_expected ) return ""; - - local info: FTP::Info = ftp_data_expected[c$id$resp_h, c$id$resp_p]; - - if ( info$passive ) - # FTP client initiates data channel. - if ( is_orig ) - # Don't care about FTP client data. - return ""; - else - # Do care about FTP server data. - return get_handle_string(c); - else - # FTP server initiates dta channel. - if ( is_orig ) - # Do care about FTP server data. - return get_handle_string(c); - else - # Don't care about FTP client data. - return ""; - } - -module GLOBAL; - -event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) - &priority=5 - { - if ( tag != Analyzer::ANALYZER_FTP_DATA ) return; - set_file_handle(FTP::get_file_handle(c, is_orig)); - } diff --git a/scripts/base/protocols/ftp/file-extract.bro b/scripts/base/protocols/ftp/file-extract.bro deleted file mode 100644 index 2b7bb8cd50..0000000000 --- a/scripts/base/protocols/ftp/file-extract.bro +++ /dev/null @@ -1,90 +0,0 @@ -##! File extraction support for FTP. - -@load ./main -@load base/utils/files - -module FTP; - -export { - ## Pattern of file mime types to extract from FTP transfers. - const extract_file_types = /NO_DEFAULT/ &redef; - - ## The on-disk prefix for files to be extracted from FTP-data transfers. - const extraction_prefix = "ftp-item" &redef; -} - -redef record Info += { - ## On disk file where it was extracted to. - extraction_file: string &log &optional; - - ## Indicates if the current command/response pair should attempt to - ## extract the file if a file was transferred. - extract_file: bool &default=F; -}; - -function get_extraction_name(f: fa_file): string - { - local r = fmt("%s-%s.dat", extraction_prefix, f$id); - return r; - } - -event file_new(f: fa_file) &priority=5 - { - if ( ! f?$source ) return; - if ( f$source != "FTP_DATA" ) return; - - if ( f?$mime_type && extract_file_types in f$mime_type ) - { - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT, - $extract_filename=get_extraction_name(f)]); - return; - } - - if ( ! f?$conns ) return; - - for ( cid in f$conns ) - { - local c: connection = f$conns[cid]; - - if ( [cid$resp_h, cid$resp_p] !in ftp_data_expected ) next; - - local s = ftp_data_expected[cid$resp_h, cid$resp_p]; - - if ( ! s$extract_file ) next; - - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT, - $extract_filename=get_extraction_name(f)]); - return; - } - } - -event file_state_remove(f: fa_file) &priority=4 - { - if ( ! f?$source ) return; - if ( f$source != "FTP_DATA" ) return; - if ( ! f?$info ) return; - - for ( filename in f$info$extracted_files ) - { - local s: FTP::Info; - s$ts = network_time(); - s$tags = set(); - s$user = ""; - s$extraction_file = filename; - - if ( f?$conns ) - for ( cid in f$conns ) - { - s$uid = f$conns[cid]$uid; - s$id = cid; - } - - Log::write(FTP::LOG, s); - } - } - -event log_ftp(rec: Info) &priority=-10 - { - delete rec$extraction_file; - delete rec$extract_file; - } diff --git a/scripts/base/protocols/ftp/files.bro b/scripts/base/protocols/ftp/files.bro new file mode 100644 index 0000000000..9ed17ab2a4 --- /dev/null +++ b/scripts/base/protocols/ftp/files.bro @@ -0,0 +1,60 @@ +@load ./main +@load ./utils +@load base/utils/conn-ids +@load base/frameworks/files + +module FTP; + +export { + redef record Info += { + ## File unique ID. + fuid: string &optional &log; + }; + + ## Default file handle provider for FTP. + global get_file_handle: function(c: connection, is_orig: bool): string; + + ## Describe the file being transferred. + global describe_file: function(f: fa_file): string; +} + +function get_file_handle(c: connection, is_orig: bool): string + { + if ( [c$id$resp_h, c$id$resp_p] !in ftp_data_expected ) + return ""; + + return cat(Analyzer::ANALYZER_FTP_DATA, c$start_time, c$id, is_orig); + } + +function describe_file(f: fa_file): string + { + # This shouldn't be needed, but just in case... + if ( f$source != "FTP" ) + return ""; + + for ( cid in f$conns ) + { + if ( f$conns[cid]?$ftp ) + return FTP::describe(f$conns[cid]$ftp); + } + return ""; + } + +event bro_init() &priority=5 + { + Files::register_protocol(Analyzer::ANALYZER_FTP_DATA, + [$get_file_handle = FTP::get_file_handle, + $describe = FTP::describe_file]); + } + + +event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5 + { + if ( [c$id$resp_h, c$id$resp_p] !in ftp_data_expected ) + return; + + local ftp = ftp_data_expected[c$id$resp_h, c$id$resp_p]; + ftp$fuid = f$id; + if ( f?$mime_type ) + ftp$mime_type = f$mime_type; + } diff --git a/scripts/base/protocols/ftp/main.bro b/scripts/base/protocols/ftp/main.bro index 7bf9d6cc4c..7e66b63f40 100644 --- a/scripts/base/protocols/ftp/main.bro +++ b/scripts/base/protocols/ftp/main.bro @@ -63,8 +63,6 @@ export { reply_code: count &log &optional; ## Reply message from the server in response to the command. reply_msg: string &log &optional; - ## Arbitrary tags that may indicate a particular attribute of this command. - tags: set[string] &log; ## Expected FTP data channel. data_channel: ExpectedDataChannel &log &optional; @@ -104,6 +102,8 @@ export { global log_ftp: event(rec: Info); } +@load ./utils + # Add the state tracking information variable to the connection record redef record connection += { ftp: Info &optional; @@ -171,37 +171,26 @@ function set_ftp_session(c: connection) function ftp_message(s: Info) { - # If it either has a tag associated with it (something detected) - # or it's a deliberately logged command. - if ( |s$tags| > 0 || (s?$cmdarg && s$cmdarg$cmd in logged_commands) ) + s$ts=s$cmdarg$ts; + s$command=s$cmdarg$cmd; + + s$arg = s$cmdarg$arg; + if ( s$cmdarg$cmd in file_cmds ) + s$arg = build_url_ftp(s); + + if ( s$arg == "" ) + delete s$arg; + + if ( s?$password && + ! s$capture_password && + to_lower(s$user) !in guest_ids ) { - if ( s?$password && - ! s$capture_password && - to_lower(s$user) !in guest_ids ) - { - s$password = ""; - } - - local arg = s$cmdarg$arg; - if ( s$cmdarg$cmd in file_cmds ) - { - local comp_path = build_path_compressed(s$cwd, arg); - if ( comp_path[0] != "/" ) - comp_path = cat("/", comp_path); - - arg = fmt("ftp://%s%s", addr_to_uri(s$id$resp_h), comp_path); - } - - s$ts=s$cmdarg$ts; - s$command=s$cmdarg$cmd; - if ( arg == "" ) - delete s$arg; - else - s$arg=arg; - - Log::write(FTP::LOG, s); + s$password = ""; } + if ( s?$cmdarg && s$command in logged_commands) + Log::write(FTP::LOG, s); + # The MIME and file_size fields are specific to file transfer commands # and may not be used in all commands so they need reset to "blank" # values after logging. @@ -209,8 +198,6 @@ function ftp_message(s: Info) delete s$file_size; # Same with data channel. delete s$data_channel; - # Tags are cleared everytime too. - s$tags = set(); } function add_expected_data_channel(s: Info, chan: ExpectedDataChannel) @@ -218,8 +205,9 @@ function add_expected_data_channel(s: Info, chan: ExpectedDataChannel) s$passive = chan$passive; s$data_channel = chan; ftp_data_expected[chan$resp_h, chan$resp_p] = s; - Analyzer::schedule_analyzer(chan$orig_h, chan$resp_h, chan$resp_p, Analyzer::ANALYZER_FTP_DATA, - 5mins); + Analyzer::schedule_analyzer(chan$orig_h, chan$resp_h, chan$resp_p, + Analyzer::ANALYZER_FTP_DATA, + 5mins); } event ftp_request(c: connection, command: string, arg: string) &priority=5 diff --git a/scripts/base/protocols/ftp/utils.bro b/scripts/base/protocols/ftp/utils.bro new file mode 100644 index 0000000000..8b92a37764 --- /dev/null +++ b/scripts/base/protocols/ftp/utils.bro @@ -0,0 +1,47 @@ +##! Utilities specific for FTP processing. + +@load ./main +@load base/utils/addrs + +module FTP; + +export { + ## Creates a URL from an :bro:type:`FTP::Info` record. + ## + ## rec: An :bro:type:`FTP::Info` record. + ## + ## Returns: A URL, not prefixed by "ftp://". + global build_url: function(rec: Info): string; + + ## Creates a URL from an :bro:type:`FTP::Info` record. + ## + ## rec: An :bro:type:`FTP::Info` record. + ## + ## Returns: A URL prefixed with "ftp://". + global build_url_ftp: function(rec: Info): string; + + ## Create an extremely shortened representation of a log line. + global describe: function(rec: Info): string; +} + +function build_url(rec: Info): string + { + if ( !rec?$arg ) + return ""; + + local comp_path = build_path_compressed(rec$cwd, rec$arg); + if ( comp_path[0] != "/" ) + comp_path = cat("/", comp_path); + + return fmt("%s%s", addr_to_uri(rec$id$resp_h), comp_path); + } + +function build_url_ftp(rec: Info): string + { + return fmt("ftp://%s", build_url(rec)); + } + +function describe(rec: Info): string + { + return build_url_ftp(rec); + } diff --git a/scripts/base/protocols/http/__load__.bro b/scripts/base/protocols/http/__load__.bro index 8f426c1521..20fcd5f1ec 100644 --- a/scripts/base/protocols/http/__load__.bro +++ b/scripts/base/protocols/http/__load__.bro @@ -1,8 +1,6 @@ @load ./main +@load ./entities @load ./utils -@load ./file-analysis -@load ./file-ident -@load ./file-hash -@load ./file-extract +@load ./files @load-sigs ./dpd.sig \ No newline at end of file diff --git a/scripts/base/protocols/http/entities.bro b/scripts/base/protocols/http/entities.bro new file mode 100644 index 0000000000..e9376a0c0c --- /dev/null +++ b/scripts/base/protocols/http/entities.bro @@ -0,0 +1,109 @@ +##! Analysis and logging for MIME entities found in HTTP sessions. + +@load base/frameworks/files +@load base/utils/strings +@load base/utils/files +@load ./main + +module HTTP; + +export { + type Entity: record { + ## Filename for the entity if discovered from a header. + filename: string &optional; + }; + + redef record Info += { + ## An ordered vector of file unique IDs. + orig_fuids: vector of string &log &optional; + + ## An ordered vector of mime types. + orig_mime_types: vector of string &log &optional; + + ## An ordered vector of file unique IDs. + resp_fuids: vector of string &log &optional; + + ## An ordered vector of mime types. + resp_mime_types: vector of string &log &optional; + + ## The current entity. + current_entity: Entity &optional; + ## Current number of MIME entities in the HTTP request message body. + orig_mime_depth: count &default=0; + ## Current number of MIME entities in the HTTP response message body. + resp_mime_depth: count &default=0; + }; +} + +event http_begin_entity(c: connection, is_orig: bool) &priority=10 + { + set_state(c, F, is_orig); + + if ( is_orig ) + ++c$http$orig_mime_depth; + else + ++c$http$resp_mime_depth; + + c$http$current_entity = Entity(); + } + +event http_header(c: connection, is_orig: bool, name: string, value: string) &priority=3 + { + if ( name == "CONTENT-DISPOSITION" && + /[fF][iI][lL][eE][nN][aA][mM][eE]/ in value ) + { + c$http$current_entity$filename = extract_filename_from_content_disposition(value); + } + else if ( name == "CONTENT-TYPE" && + /[nN][aA][mM][eE][:blank:]*=/ in value ) + { + c$http$current_entity$filename = extract_filename_from_content_disposition(value); + } + } + +event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5 + { + if ( f$source == "HTTP" && c?$http ) + { + if ( c$http?$current_entity && c$http$current_entity?$filename ) + f$info$filename = c$http$current_entity$filename; + + if ( f$is_orig ) + { + if ( ! c$http?$orig_mime_types ) + c$http$orig_fuids = string_vec(f$id); + else + c$http$orig_fuids[|c$http$orig_fuids|] = f$id; + + if ( f?$mime_type ) + { + if ( ! c$http?$orig_mime_types ) + c$http$orig_mime_types = string_vec(f$mime_type); + else + c$http$orig_mime_types[|c$http$orig_mime_types|] = f$mime_type; + } + } + else + { + if ( ! c$http?$resp_mime_types ) + c$http$resp_fuids = string_vec(f$id); + else + c$http$resp_fuids[|c$http$resp_fuids|] = f$id; + + if ( f?$mime_type ) + { + if ( ! c$http?$resp_mime_types ) + c$http$resp_mime_types = string_vec(f$mime_type); + else + c$http$resp_mime_types[|c$http$resp_mime_types|] = f$mime_type; + } + } + } + + } + +event http_end_entity(c: connection, is_orig: bool) &priority=5 + { + if ( c?$http && c$http?$current_entity ) + delete c$http$current_entity; + } diff --git a/scripts/base/protocols/http/file-analysis.bro b/scripts/base/protocols/http/file-analysis.bro deleted file mode 100644 index d6da8c4f69..0000000000 --- a/scripts/base/protocols/http/file-analysis.bro +++ /dev/null @@ -1,54 +0,0 @@ -@load ./main -@load ./utils -@load base/utils/conn-ids -@load base/frameworks/file-analysis/main - -module HTTP; - -export { - redef record HTTP::Info += { - ## Number of MIME entities in the HTTP request message body so far. - request_mime_level: count &default=0; - ## Number of MIME entities in the HTTP response message body so far. - response_mime_level: count &default=0; - }; - - ## Default file handle provider for HTTP. - global get_file_handle: function(c: connection, is_orig: bool): string; -} - -event http_begin_entity(c: connection, is_orig: bool) &priority=5 - { - if ( ! c?$http ) - return; - - if ( is_orig ) - ++c$http$request_mime_level; - else - ++c$http$response_mime_level; - } - -function get_file_handle(c: connection, is_orig: bool): string - { - if ( ! c?$http ) return ""; - - local mime_level: count = - is_orig ? c$http$request_mime_level : c$http$response_mime_level; - local mime_level_str: string = mime_level > 1 ? cat(mime_level) : ""; - - if ( c$http$range_request ) - return cat(Analyzer::ANALYZER_HTTP, " ", is_orig, " ", c$id$orig_h, " ", - build_url(c$http)); - - return cat(Analyzer::ANALYZER_HTTP, " ", c$start_time, " ", is_orig, " ", - c$http$trans_depth, mime_level_str, " ", id_string(c$id)); - } - -module GLOBAL; - -event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) - &priority=5 - { - if ( tag != Analyzer::ANALYZER_HTTP ) return; - set_file_handle(HTTP::get_file_handle(c, is_orig)); - } diff --git a/scripts/base/protocols/http/file-extract.bro b/scripts/base/protocols/http/file-extract.bro deleted file mode 100644 index a8c6039395..0000000000 --- a/scripts/base/protocols/http/file-extract.bro +++ /dev/null @@ -1,100 +0,0 @@ -##! Extracts the items from HTTP traffic, one per file. At this time only -##! the message body from the server can be extracted with this script. - -@load ./main -@load ./file-analysis - -module HTTP; - -export { - ## Pattern of file mime types to extract from HTTP response entity bodies. - const extract_file_types = /NO_DEFAULT/ &redef; - - ## The on-disk prefix for files to be extracted from HTTP entity bodies. - const extraction_prefix = "http-item" &redef; - - redef record Info += { - ## On-disk location where files in request body were extracted. - extracted_request_files: vector of string &log &optional; - - ## On-disk location where files in response body were extracted. - extracted_response_files: vector of string &log &optional; - - ## Indicates if the response body is to be extracted or not. Must be - ## set before or by the first :bro:see:`file_new` for the file content. - extract_file: bool &default=F; - }; -} - -function get_extraction_name(f: fa_file): string - { - local r = fmt("%s-%s.dat", extraction_prefix, f$id); - return r; - } - -function add_extraction_file(c: connection, is_orig: bool, fn: string) - { - if ( is_orig ) - { - if ( ! c$http?$extracted_request_files ) - c$http$extracted_request_files = vector(); - c$http$extracted_request_files[|c$http$extracted_request_files|] = fn; - } - else - { - if ( ! c$http?$extracted_response_files ) - c$http$extracted_response_files = vector(); - c$http$extracted_response_files[|c$http$extracted_response_files|] = fn; - } - } - -event file_new(f: fa_file) &priority=5 - { - if ( ! f?$source ) return; - if ( f$source != "HTTP" ) return; - if ( ! f?$conns ) return; - - local fname: string; - local c: connection; - - if ( f?$mime_type && extract_file_types in f$mime_type ) - { - fname = get_extraction_name(f); - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT, - $extract_filename=fname]); - - for ( cid in f$conns ) - { - c = f$conns[cid]; - if ( ! c?$http ) next; - add_extraction_file(c, f$is_orig, fname); - } - - return; - } - - local extracting: bool = F; - - for ( cid in f$conns ) - { - c = f$conns[cid]; - - if ( ! c?$http ) next; - - if ( ! c$http$extract_file ) next; - - fname = get_extraction_name(f); - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT, - $extract_filename=fname]); - extracting = T; - break; - } - - if ( extracting ) - for ( cid in f$conns ) - { - c = f$conns[cid]; - if ( ! c?$http ) next; - add_extraction_file(c, f$is_orig, fname); - } - } diff --git a/scripts/base/protocols/http/file-hash.bro b/scripts/base/protocols/http/file-hash.bro deleted file mode 100644 index 34d91e45bb..0000000000 --- a/scripts/base/protocols/http/file-hash.bro +++ /dev/null @@ -1,68 +0,0 @@ -##! Calculate hashes for HTTP body transfers. - -@load ./main -@load ./file-analysis - -module HTTP; - -export { - redef record Info += { - ## MD5 sum for a file transferred over HTTP calculated from the - ## response body. - md5: string &log &optional; - - ## This value can be set per-transfer to determine per request - ## if a file should have an MD5 sum generated. It must be - ## set to T at the time of or before the first chunk of body data. - calc_md5: bool &default=F; - }; - - ## Generate MD5 sums for these filetypes. - const generate_md5 = /application\/x-dosexec/ # Windows and DOS executables - | /application\/x-executable/ # *NIX executable binary - &redef; -} - -event file_new(f: fa_file) &priority=5 - { - if ( ! f?$source ) return; - if ( f$source != "HTTP" ) return; - - if ( f?$mime_type && generate_md5 in f$mime_type ) - { - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_MD5]); - return; - } - - if ( ! f?$conns ) return; - - for ( cid in f$conns ) - { - local c: connection = f$conns[cid]; - - if ( ! c?$http ) next; - - if ( ! c$http$calc_md5 ) next; - - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_MD5]); - return; - } - } - -event file_state_remove(f: fa_file) &priority=4 - { - if ( ! f?$source ) return; - if ( f$source != "HTTP" ) return; - if ( ! f?$conns ) return; - if ( ! f?$info ) return; - if ( ! f$info?$md5 ) return; - - for ( cid in f$conns ) - { - local c: connection = f$conns[cid]; - - if ( ! c?$http ) next; - - c$http$md5 = f$info$md5; - } - } diff --git a/scripts/base/protocols/http/file-ident.bro b/scripts/base/protocols/http/file-ident.bro deleted file mode 100644 index 7ed4b58a37..0000000000 --- a/scripts/base/protocols/http/file-ident.bro +++ /dev/null @@ -1,105 +0,0 @@ -##! Identification of file types in HTTP response bodies with file content sniffing. - -@load base/frameworks/notice -@load ./main -@load ./utils -@load ./file-analysis - -module HTTP; - -export { - redef enum Notice::Type += { - ## Indicates when the file extension doesn't seem to match the file - ## contents. - Incorrect_File_Type, - }; - - redef record Info += { - ## Mime type of response body identified by content sniffing. - mime_type: string &log &optional; - }; - - ## Mapping between mime type strings (without character set) and - ## regular expressions for URLs. - ## The :bro:enum:`HTTP::Incorrect_File_Type` notice is generated if the - ## pattern doesn't match the mime type that was discovered. - const mime_types_extensions: table[string] of pattern = { - ["application/x-dosexec"] = /\.([eE][xX][eE]|[dD][lL][lL])/, - } &redef; - - ## A pattern for filtering out :bro:enum:`HTTP::Incorrect_File_Type` urls - ## that are not noteworthy before a notice is created. Each - ## pattern added should match the complete URL (the matched URLs include - ## "http://" at the beginning). - const ignored_incorrect_file_type_urls = /^$/ &redef; -} - -event file_new(f: fa_file) &priority=5 - { - if ( ! f?$source ) return; - if ( f$source != "HTTP" ) return; - if ( ! f?$mime_type ) return; - if ( ! f?$conns ) return; - - for ( cid in f$conns ) - { - local c: connection = f$conns[cid]; - - if ( ! c?$http ) next; - - c$http$mime_type = f$mime_type; - - local mime_str: string = c$http$mime_type; - - if ( mime_str !in mime_types_extensions ) next; - if ( ! c$http?$uri ) next; - if ( mime_types_extensions[mime_str] in c$http$uri ) next; - - local url = build_url_http(c$http); - - if ( url == ignored_incorrect_file_type_urls ) next; - - local message = fmt("%s %s %s", mime_str, c$http$method, url); - NOTICE([$note=Incorrect_File_Type, - $msg=message, - $conn=c]); - } - } - -event file_over_new_connection(f: fa_file, c: connection) &priority=5 - { - if ( ! f?$source ) return; - if ( f$source != "HTTP" ) return; - if ( ! f?$mime_type ) return; - if ( ! c?$http ) return; - - # Spread the mime around (e.g. for partial content, file_type event only - # happens once for the first connection, but if there's subsequent - # connections to transfer the same file, they'll be lacking the mime_type - # field if we don't do this). - c$http$mime_type = f$mime_type; - } - -# Tracks byte-range request / partial content response mime types, indexed -# by [connection, uri] pairs. This is needed because a person can pipeline -# byte-range requests over multiple connections to the same uri. Without -# the tracking, only the first request in the pipeline for each connection -# would get a mime_type field assigned to it (by the FileAnalysis policy hooks). -global partial_types: table[conn_id, string] of string &read_expire=5mins; - -# Priority 4 so that it runs before the handler that will write to http.log. -event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) - &priority=4 - { - if ( ! c$http$range_request ) return; - if ( ! c$http?$uri ) return; - - if ( c$http?$mime_type ) - { - partial_types[c$id, c$http$uri] = c$http$mime_type; - return; - } - - if ( [c$id, c$http$uri] in partial_types ) - c$http$mime_type = partial_types[c$id, c$http$uri]; - } diff --git a/scripts/base/protocols/http/files.bro b/scripts/base/protocols/http/files.bro new file mode 100644 index 0000000000..840b5a2372 --- /dev/null +++ b/scripts/base/protocols/http/files.bro @@ -0,0 +1,56 @@ +@load ./main +@load ./entities +@load ./utils +@load base/utils/conn-ids +@load base/frameworks/files + +module HTTP; + +export { + ## Default file handle provider for HTTP. + global get_file_handle: function(c: connection, is_orig: bool): string; + + ## Default file describer for HTTP. + global describe_file: function(f: fa_file): string; +} + +function get_file_handle(c: connection, is_orig: bool): string + { + if ( ! c?$http ) + return ""; + + if ( c$http$range_request && ! is_orig ) + { + # Any multipart responses from the server are pieces of same file + # that correspond to range requests, so don't use mime depth to + # identify the file. + return cat(Analyzer::ANALYZER_HTTP, is_orig, c$id$orig_h, build_url(c$http)); + } + else + { + local mime_depth = is_orig ? c$http$orig_mime_depth : c$http$resp_mime_depth; + return cat(Analyzer::ANALYZER_HTTP, c$start_time, is_orig, + c$http$trans_depth, mime_depth, id_string(c$id)); + } + } + +function describe_file(f: fa_file): string + { + # This shouldn't be needed, but just in case... + if ( f$source != "HTTP" ) + return ""; + + for ( cid in f$conns ) + { + if ( f$conns[cid]?$http ) + return build_url_http(f$conns[cid]$http); + } + return ""; + } + +event bro_init() &priority=5 + { + Files::register_protocol(Analyzer::ANALYZER_HTTP, + [$get_file_handle = HTTP::get_file_handle, + $describe = HTTP::describe_file]); + } diff --git a/scripts/base/protocols/http/main.bro b/scripts/base/protocols/http/main.bro index 6d06376183..e103bc6b41 100644 --- a/scripts/base/protocols/http/main.bro +++ b/scripts/base/protocols/http/main.bro @@ -1,5 +1,5 @@ -##! Implements base functionality for HTTP analysis. The logging model is -##! to log request/response pairs and all relevant metadata together in +##! Implements base functionality for HTTP analysis. The logging model is +##! to log request/response pairs and all relevant metadata together in ##! a single record. @load base/utils/numbers @@ -15,10 +15,10 @@ export { ## Placeholder. EMPTY }; - + ## This setting changes if passwords used in Basic-Auth are captured or not. const default_capture_password = F &redef; - + type Info: record { ## Timestamp for when the request happened. ts: time &log; @@ -26,7 +26,7 @@ export { uid: string &log; ## The connection's 4-tuple of endpoint addresses/ports. id: conn_id &log; - ## Represents the pipelined depth into the connection of this + ## Represents the pipelined depth into the connection of this ## request/response transaction. trans_depth: count &log; ## Verb used in the HTTP request (GET, POST, HEAD, etc.). @@ -60,24 +60,24 @@ export { ## A set of indicators of various attributes discovered and ## related to a particular request/response pair. tags: set[Tags] &log; - + ## Username if basic-auth is performed for the request. username: string &log &optional; ## Password if basic-auth is performed for the request. password: string &log &optional; - + ## Determines if the password will be captured for this request. capture_password: bool &default=default_capture_password; - + ## All of the headers that may indicate if the request was proxied. proxied: set[string] &log &optional; ## Indicates if this request can assume 206 partial content in ## response. - range_request: bool &default=F; + range_request: bool &default=F; }; - - ## Structure to maintain state for an HTTP connection with multiple + + ## Structure to maintain state for an HTTP connection with multiple ## requests and responses. type State: record { ## Pending requests. @@ -87,7 +87,7 @@ export { ## Current response in the pending queue. current_response: count &default=0; }; - + ## A list of HTTP headers typically used to indicate proxied requests. const proxy_headers: set[string] = { "FORWARDED", @@ -100,8 +100,8 @@ export { } &redef; ## A list of HTTP methods. Other methods will generate a weird. Note - ## that the HTTP analyzer will only accept methods consisting solely - ## of letters ``[A-Za-z]``. + ## that the HTTP analyzer will only accept methods consisting solely + ## of letters ``[A-Za-z]``. const http_methods: set[string] = { "GET", "POST", "HEAD", "OPTIONS", "PUT", "DELETE", "TRACE", "CONNECT", @@ -111,8 +111,8 @@ export { "POLL", "REPORT", "SUBSCRIBE", "BMOVE", "SEARCH" } &redef; - - ## Event that can be handled to access the HTTP record as it is sent on + + ## Event that can be handled to access the HTTP record as it is sent on ## to the logging framework. global log_http: event(rec: Info); } @@ -147,12 +147,12 @@ function new_http_session(c: connection): Info tmp$ts=network_time(); tmp$uid=c$uid; tmp$id=c$id; - # $current_request is set prior to the Info record creation so we + # $current_request is set prior to the Info record creation so we # can use the value directly here. tmp$trans_depth = c$http_state$current_request; return tmp; } - + function set_state(c: connection, request: bool, is_orig: bool) { if ( ! c?$http_state ) @@ -160,19 +160,19 @@ function set_state(c: connection, request: bool, is_orig: bool) local s: State; c$http_state = s; } - + # These deal with new requests and responses. if ( request || c$http_state$current_request !in c$http_state$pending ) c$http_state$pending[c$http_state$current_request] = new_http_session(c); if ( ! is_orig && c$http_state$current_response !in c$http_state$pending ) c$http_state$pending[c$http_state$current_response] = new_http_session(c); - + if ( is_orig ) c$http = c$http_state$pending[c$http_state$current_request]; else c$http = c$http_state$pending[c$http_state$current_response]; } - + event http_request(c: connection, method: string, original_URI: string, unescaped_URI: string, version: string) &priority=5 { @@ -181,17 +181,17 @@ event http_request(c: connection, method: string, original_URI: string, local s: State; c$http_state = s; } - + ++c$http_state$current_request; set_state(c, T, T); - + c$http$method = method; c$http$uri = unescaped_URI; if ( method !in http_methods ) event conn_weird("unknown_HTTP_method", c, method); } - + event http_reply(c: connection, version: string, code: count, reason: string) &priority=5 { if ( ! c?$http_state ) @@ -199,7 +199,7 @@ event http_reply(c: connection, version: string, code: count, reason: string) &p local s: State; c$http_state = s; } - + # If the last response was an informational 1xx, we're still expecting # the real response to the request, so don't create a new Info record yet. if ( c$http_state$current_response !in c$http_state$pending || @@ -207,7 +207,7 @@ event http_reply(c: connection, version: string, code: count, reason: string) &p ! code_in_range(c$http_state$pending[c$http_state$current_response]$status_code, 100, 199)) ) ++c$http_state$current_response; set_state(c, F, F); - + c$http$status_code = code; c$http$status_msg = reason; if ( code_in_range(code, 100, 199) ) @@ -216,33 +216,33 @@ event http_reply(c: connection, version: string, code: count, reason: string) &p c$http$info_msg = reason; } } - + event http_header(c: connection, is_orig: bool, name: string, value: string) &priority=5 { set_state(c, F, is_orig); - + if ( is_orig ) # client headers { if ( name == "REFERER" ) c$http$referrer = value; - + else if ( name == "HOST" ) # The split is done to remove the occasional port value that shows up here. c$http$host = split1(value, /:/)[1]; else if ( name == "RANGE" ) c$http$range_request = T; - + else if ( name == "USER-AGENT" ) c$http$user_agent = value; - + else if ( name in proxy_headers ) { if ( ! c$http?$proxied ) c$http$proxied = set(); add c$http$proxied[fmt("%s -> %s", name, value)]; } - + else if ( name == "AUTHORIZATION" ) { if ( /^[bB][aA][sS][iI][cC] / in value ) @@ -264,25 +264,19 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr } } } - - else # server headers - { - if ( name == "CONTENT-DISPOSITION" && - /[fF][iI][lL][eE][nN][aA][mM][eE]/ in value ) - c$http$filename = extract_filename_from_content_disposition(value); - } + } - + event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &priority = 5 { set_state(c, F, is_orig); - + if ( is_orig ) c$http$request_body_len = stat$body_length; else c$http$response_body_len = stat$body_length; } - + event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &priority = -5 { # The reply body is done so we're ready to log. @@ -311,4 +305,4 @@ event connection_state_remove(c: connection) &priority=-5 } } } - + diff --git a/scripts/base/protocols/http/utils.bro b/scripts/base/protocols/http/utils.bro index a74a2fe696..fe8c076780 100644 --- a/scripts/base/protocols/http/utils.bro +++ b/scripts/base/protocols/http/utils.bro @@ -32,6 +32,9 @@ export { ## ## Returns: A URL prefixed with "http://". global build_url_http: function(rec: Info): string; + + ## Create an extremely shortened representation of a log line. + global describe: function(rec: Info): string; } @@ -62,3 +65,8 @@ function build_url_http(rec: Info): string { return fmt("http://%s", build_url(rec)); } + +function describe(rec: Info): string + { + return build_url_http(rec); + } diff --git a/scripts/base/protocols/irc/__load__.bro b/scripts/base/protocols/irc/__load__.bro index 2e60cda0a6..0834604683 100644 --- a/scripts/base/protocols/irc/__load__.bro +++ b/scripts/base/protocols/irc/__load__.bro @@ -1,5 +1,5 @@ @load ./main @load ./dcc-send -@load ./file-analysis +@load ./files @load-sigs ./dpd.sig \ No newline at end of file diff --git a/scripts/base/protocols/irc/dcc-send.bro b/scripts/base/protocols/irc/dcc-send.bro index 3194766946..d95eb97517 100644 --- a/scripts/base/protocols/irc/dcc-send.bro +++ b/scripts/base/protocols/irc/dcc-send.bro @@ -2,7 +2,7 @@ ##! ##! There is a major problem with this script in the cluster context because ##! we might see A send B a message that a DCC connection is to be expected, -##! but that connection will actually be between B and C which could be +##! but that connection will actually be between B and C which could be ##! analyzed on a different worker. ##! @@ -15,12 +15,6 @@ module IRC; export { - ## Pattern of file mime types to extract from IRC DCC file transfers. - const extract_file_types = /NO_DEFAULT/ &redef; - - ## On-disk prefix for files to be extracted from IRC DCC file transfers. - const extraction_prefix = "irc-dcc-item" &redef; - redef record Info += { ## DCC filename requested. dcc_file_name: string &log &optional; @@ -28,101 +22,10 @@ export { dcc_file_size: count &log &optional; ## Sniffed mime type of the file. dcc_mime_type: string &log &optional; - - ## The file handle for the file to be extracted - extraction_file: string &log &optional; - - ## A boolean to indicate if the current file transfer should be extracted. - extract_file: bool &default=F; }; } -global dcc_expected_transfers: table[addr, port] of Info &read_expire=5mins; - -function set_dcc_mime(f: fa_file) - { - if ( ! f?$conns ) return; - - for ( cid in f$conns ) - { - local c: connection = f$conns[cid]; - - if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next; - - local s = dcc_expected_transfers[cid$resp_h, cid$resp_p]; - - s$dcc_mime_type = f$mime_type; - } - } - -function set_dcc_extraction_file(f: fa_file, filename: string) - { - if ( ! f?$conns ) return; - - for ( cid in f$conns ) - { - local c: connection = f$conns[cid]; - - if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next; - - local s = dcc_expected_transfers[cid$resp_h, cid$resp_p]; - - s$extraction_file = filename; - } - } - -function get_extraction_name(f: fa_file): string - { - local r = fmt("%s-%s.dat", extraction_prefix, f$id); - return r; - } - -# this handler sets the IRC::Info mime type -event file_new(f: fa_file) &priority=5 - { - if ( ! f?$source ) return; - if ( f$source != "IRC_DATA" ) return; - if ( ! f?$mime_type ) return; - - set_dcc_mime(f); - } - -# this handler check if file extraction is desired -event file_new(f: fa_file) &priority=5 - { - if ( ! f?$source ) return; - if ( f$source != "IRC_DATA" ) return; - - local fname: string; - - if ( f?$mime_type && extract_file_types in f$mime_type ) - { - fname = get_extraction_name(f); - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT, - $extract_filename=fname]); - set_dcc_extraction_file(f, fname); - return; - } - - if ( ! f?$conns ) return; - - for ( cid in f$conns ) - { - local c: connection = f$conns[cid]; - - if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next; - - local s = dcc_expected_transfers[cid$resp_h, cid$resp_p]; - - if ( ! s$extract_file ) next; - - fname = get_extraction_name(f); - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT, - $extract_filename=fname]); - s$extraction_file = fname; - return; - } - } +global dcc_expected_transfers: table[addr, port] of Info &synchronized &read_expire=5mins; function log_dcc(f: fa_file) { @@ -141,24 +44,21 @@ function log_dcc(f: fa_file) Log::write(IRC::LOG, irc); irc$command = tmp; - # Delete these values in case another DCC transfer + # Delete these values in case another DCC transfer # happens during the IRC session. - delete irc$extract_file; - delete irc$extraction_file; delete irc$dcc_file_name; delete irc$dcc_file_size; delete irc$dcc_mime_type; + delete dcc_expected_transfers[cid$resp_h, cid$resp_p]; return; } } event file_new(f: fa_file) &priority=-5 { - if ( ! f?$source ) return; - if ( f$source != "IRC_DATA" ) return; - - log_dcc(f); + if ( f$source == "IRC_DATA" ) + log_dcc(f); } event irc_dcc_message(c: connection, is_orig: bool, diff --git a/scripts/base/protocols/irc/file-analysis.bro b/scripts/base/protocols/irc/file-analysis.bro deleted file mode 100644 index 89cbe7990c..0000000000 --- a/scripts/base/protocols/irc/file-analysis.bro +++ /dev/null @@ -1,25 +0,0 @@ -@load ./dcc-send.bro -@load base/utils/conn-ids -@load base/frameworks/file-analysis/main - -module IRC; - -export { - ## Default file handle provider for IRC. - global get_file_handle: function(c: connection, is_orig: bool): string; -} - -function get_file_handle(c: connection, is_orig: bool): string - { - if ( is_orig ) return ""; - return cat(Analyzer::ANALYZER_IRC_DATA, " ", c$start_time, " ", id_string(c$id)); - } - -module GLOBAL; - -event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) - &priority=5 - { - if ( tag != Analyzer::ANALYZER_IRC_DATA ) return; - set_file_handle(IRC::get_file_handle(c, is_orig)); - } diff --git a/scripts/base/protocols/irc/files.bro b/scripts/base/protocols/irc/files.bro new file mode 100644 index 0000000000..7e077c8331 --- /dev/null +++ b/scripts/base/protocols/irc/files.bro @@ -0,0 +1,39 @@ +@load ./dcc-send +@load base/utils/conn-ids +@load base/frameworks/files + +module IRC; + +export { + redef record Info += { + ## File unique ID. + fuid: string &log &optional; + }; + + ## Default file handle provider for IRC. + global get_file_handle: function(c: connection, is_orig: bool): string; +} + +function get_file_handle(c: connection, is_orig: bool): string + { + return cat(Analyzer::ANALYZER_IRC_DATA, c$start_time, c$id, is_orig); + } + +event bro_init() &priority=5 + { + Files::register_protocol(Analyzer::ANALYZER_IRC_DATA, + [$get_file_handle = IRC::get_file_handle]); + } + +event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5 + { + if ( [c$id$resp_h, c$id$resp_p] !in dcc_expected_transfers ) + return; + + local irc = dcc_expected_transfers[c$id$resp_h, c$id$resp_p]; + irc$fuid = f$id; + if ( irc?$dcc_file_name ) + f$info$filename = irc$dcc_file_name; + if ( f?$mime_type ) + irc$dcc_mime_type = f$mime_type; + } diff --git a/scripts/base/protocols/smtp/__load__.bro b/scripts/base/protocols/smtp/__load__.bro index 3e3fde6947..bf37447ecb 100644 --- a/scripts/base/protocols/smtp/__load__.bro +++ b/scripts/base/protocols/smtp/__load__.bro @@ -1,6 +1,5 @@ @load ./main @load ./entities -@load ./entities-excerpt -@load ./file-analysis +@load ./files @load-sigs ./dpd.sig \ No newline at end of file diff --git a/scripts/base/protocols/smtp/entities.bro b/scripts/base/protocols/smtp/entities.bro index b58766e51d..d21ba8722d 100644 --- a/scripts/base/protocols/smtp/entities.bro +++ b/scripts/base/protocols/smtp/entities.bro @@ -1,5 +1,6 @@ ##! Analysis and logging for MIME entities found in SMTP sessions. +@load base/frameworks/files @load base/utils/strings @load base/utils/files @load ./main @@ -7,217 +8,55 @@ module SMTP; export { - redef enum Log::ID += { ENTITIES_LOG }; - - type EntityInfo: record { - ## This is the timestamp of when the MIME content transfer began. - ts: time &log; - uid: string &log; - id: conn_id &log; - ## A count to represent the depth of this message transaction in a - ## single connection where multiple messages were transferred. - trans_depth: count &log; - ## The filename seen in the Content-Disposition header. - filename: string &log &optional; - ## Track how many bytes of the MIME encoded file have been seen. - content_len: count &log &default=0; - ## The mime type of the entity discovered through magic bytes identification. - mime_type: string &log &optional; - - ## The calculated MD5 sum for the MIME entity. - md5: string &log &optional; - ## Optionally calculate the file's MD5 sum. Must be set prior to the - ## first data chunk being see in an event. - calc_md5: bool &default=F; - - ## Optionally write the file to disk. Must be set prior to first - ## data chunk being seen in an event. - extract_file: bool &default=F; - ## Store the file handle here for the file currently being extracted. - extraction_file: string &log &optional; + type Entity: record { + ## Filename for the entity if discovered from a header. + filename: string &optional; }; redef record Info += { - ## The in-progress entity information. - current_entity: EntityInfo &optional; + ## The current entity being seen. + entity: Entity &optional; }; redef record State += { - ## Track the number of MIME encoded files transferred during a session. - mime_level: count &default=0; + ## Track the number of MIME encoded files transferred + ## during a session. + mime_depth: count &default=0; }; - - ## Generate MD5 sums for these filetypes. - const generate_md5 = /application\/x-dosexec/ # Windows and DOS executables - | /application\/x-executable/ # *NIX executable binary - &redef; - - ## Pattern of file mime types to extract from MIME bodies. - const extract_file_types = /NO_DEFAULT/ &redef; - - ## The on-disk prefix for files to be extracted from MIME entity bodies. - const extraction_prefix = "smtp-entity" &redef; - - ## If set, never generate MD5s. This is mainly for testing purposes to create - ## reproducable output in the case that the decision whether to create - ## checksums depends on environment specifics. - const never_calc_md5 = F &redef; - - global log_mime: event(rec: EntityInfo); } -event bro_init() &priority=5 - { - Log::create_stream(SMTP::ENTITIES_LOG, [$columns=EntityInfo, $ev=log_mime]); - } - -function set_session(c: connection, new_entity: bool) - { - if ( ! c$smtp?$current_entity || new_entity ) - { - local info: EntityInfo; - info$ts=network_time(); - info$uid=c$uid; - info$id=c$id; - info$trans_depth=c$smtp$trans_depth; - - c$smtp$current_entity = info; - ++c$smtp_state$mime_level; - } - } - -function get_extraction_name(f: fa_file): string - { - local r = fmt("%s-%s.dat", extraction_prefix, f$id); - return r; - } - event mime_begin_entity(c: connection) &priority=10 { - if ( ! c?$smtp ) return; - - set_session(c, T); + c$smtp$entity = Entity(); + ++c$smtp_state$mime_depth; } -event file_new(f: fa_file) &priority=5 +event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5 { - if ( ! f?$source ) return; - if ( f$source != "SMTP" ) return; - if ( ! f?$conns ) return; - - local fname: string; - local extracting: bool = F; - - for ( cid in f$conns ) + if ( f$source == "SMTP" && c?$smtp ) { - local c: connection = f$conns[cid]; - - if ( ! c?$smtp ) next; - if ( ! c$smtp?$current_entity ) next; - - if ( c$smtp$current_entity$extract_file ) - { - if ( ! extracting ) - { - fname = get_extraction_name(f); - FileAnalysis::add_analyzer(f, - [$tag=FileAnalysis::ANALYZER_EXTRACT, - $extract_filename=fname]); - extracting = T; - } - - c$smtp$current_entity$extraction_file = fname; - } - - if ( c$smtp$current_entity$calc_md5 ) - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_MD5]); + if ( c$smtp?$entity && c$smtp$entity?$filename ) + f$info$filename = c$smtp$entity$filename; + f$info$depth = c$smtp_state$mime_depth; } } -function check_extract_by_type(f: fa_file) +event mime_one_header(c: connection, h: mime_header_rec) &priority=5 { - if ( extract_file_types !in f$mime_type ) return; - - if ( f?$info && FileAnalysis::ANALYZER_EXTRACT in f$info$analyzers ) + if ( ! c?$smtp ) return; - local fname: string = get_extraction_name(f); - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT, - $extract_filename=fname]); - - if ( ! f?$conns ) return; - - for ( cid in f$conns ) - { - local c: connection = f$conns[cid]; - if ( ! c?$smtp ) next; - c$smtp$current_entity$extraction_file = fname; - } - } - -function check_md5_by_type(f: fa_file) - { - if ( never_calc_md5 ) return; - if ( generate_md5 !in f$mime_type ) return; - - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_MD5]); - } - -event file_new(f: fa_file) &priority=5 - { - if ( ! f?$source ) return; - if ( f$source != "SMTP" ) return; - if ( ! f?$mime_type ) return; - - if ( f?$conns ) - for ( cid in f$conns ) - { - local c: connection = f$conns[cid]; - - if ( ! c?$smtp ) next; - if ( ! c$smtp?$current_entity ) next; - - c$smtp$current_entity$mime_type = f$mime_type; - } - - check_extract_by_type(f); - check_md5_by_type(f); - } - -event file_state_remove(f: fa_file) &priority=4 - { - if ( ! f?$source ) return; - if ( f$source != "SMTP" ) return; - if ( ! f?$conns ) return; - - for ( cid in f$conns ) - { - local c: connection = f$conns[cid]; - - if ( ! c?$smtp ) next; - if ( ! c$smtp?$current_entity ) next; - # Only log if there was some content. - if ( f$seen_bytes == 0 ) next; - - if ( f?$info && f$info?$md5 ) - c$smtp$current_entity$md5 = f$info$md5; - - c$smtp$current_entity$content_len = f$seen_bytes; - Log::write(SMTP::ENTITIES_LOG, c$smtp$current_entity); - delete c$smtp$current_entity; - return; - } - } - -event mime_one_header(c: connection, h: mime_header_rec) - { - if ( ! c?$smtp ) return; - if ( h$name == "CONTENT-DISPOSITION" && /[fF][iI][lL][eE][nN][aA][mM][eE]/ in h$value ) - c$smtp$current_entity$filename = extract_filename_from_content_disposition(h$value); + c$smtp$entity$filename = extract_filename_from_content_disposition(h$value); if ( h$name == "CONTENT-TYPE" && /[nN][aA][mM][eE][:blank:]*=/ in h$value ) - c$smtp$current_entity$filename = extract_filename_from_content_disposition(h$value); + c$smtp$entity$filename = extract_filename_from_content_disposition(h$value); + } + +event mime_end_entity(c: connection) &priority=5 + { + if ( c?$smtp && c$smtp?$entity ) + delete c$smtp$entity; } diff --git a/scripts/base/protocols/smtp/file-analysis.bro b/scripts/base/protocols/smtp/file-analysis.bro deleted file mode 100644 index 68ec6390dd..0000000000 --- a/scripts/base/protocols/smtp/file-analysis.bro +++ /dev/null @@ -1,27 +0,0 @@ -@load ./main -@load ./entities -@load base/utils/conn-ids -@load base/frameworks/file-analysis/main - -module SMTP; - -export { - ## Default file handle provider for SMTP. - global get_file_handle: function(c: connection, is_orig: bool): string; -} - -function get_file_handle(c: connection, is_orig: bool): string - { - if ( ! c?$smtp ) return ""; - return cat(Analyzer::ANALYZER_SMTP, " ", c$start_time, " ", c$smtp$trans_depth, " ", - c$smtp_state$mime_level); - } - -module GLOBAL; - -event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) - &priority=5 - { - if ( tag != Analyzer::ANALYZER_SMTP ) return; - set_file_handle(SMTP::get_file_handle(c, is_orig)); - } diff --git a/scripts/base/protocols/smtp/files.bro b/scripts/base/protocols/smtp/files.bro new file mode 100644 index 0000000000..f9ae2ab05f --- /dev/null +++ b/scripts/base/protocols/smtp/files.bro @@ -0,0 +1,53 @@ +@load ./main +@load ./entities +@load base/utils/conn-ids +@load base/frameworks/files + +module SMTP; + +export { + redef record Info += { + ## An ordered vector of file unique IDs seen attached to + ## the message. + fuids: vector of string &log &default=string_vec(); + }; + + ## Default file handle provider for SMTP. + global get_file_handle: function(c: connection, is_orig: bool): string; + + ## Default file describer for SMTP. + global describe_file: function(f: fa_file): string; +} + +function get_file_handle(c: connection, is_orig: bool): string + { + return cat(Analyzer::ANALYZER_SMTP, c$start_time, c$smtp$trans_depth, + c$smtp_state$mime_depth); + } + +function describe_file(f: fa_file): string + { + # This shouldn't be needed, but just in case... + if ( f$source != "SMTP" ) + return ""; + + for ( cid in f$conns ) + { + local c = f$conns[cid]; + return SMTP::describe(c$smtp); + } + return ""; + } + +event bro_init() &priority=5 + { + Files::register_protocol(Analyzer::ANALYZER_SMTP, + [$get_file_handle = SMTP::get_file_handle, + $describe = SMTP::describe_file]); + } + +event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5 + { + if ( c?$smtp ) + c$smtp$fuids[|c$smtp$fuids|] = f$id; + } diff --git a/scripts/base/protocols/smtp/main.bro b/scripts/base/protocols/smtp/main.bro index d53128b06c..f54fcdd4c8 100644 --- a/scripts/base/protocols/smtp/main.bro +++ b/scripts/base/protocols/smtp/main.bro @@ -72,7 +72,10 @@ export { ## ALL_HOSTS - always capture the entire path. ## NO_HOSTS - never capture the path. const mail_path_capture = ALL_HOSTS &redef; - + + ## Create an extremely shortened representation of a log line. + global describe: function(rec: Info): string; + global log_smtp: event(rec: Info); } @@ -223,7 +226,10 @@ event mime_one_header(c: connection, h: mime_header_rec) &priority=5 { if ( ! c$smtp?$to ) c$smtp$to = set(); - add c$smtp$to[h$value]; + + local to_parts = split(h$value, /[[:blank:]]*,[[:blank:]]*/); + for ( i in to_parts ) + add c$smtp$to[to_parts[i]]; } else if ( h$name == "X-ORIGINATING-IP" ) @@ -268,3 +274,29 @@ event connection_state_remove(c: connection) &priority=-5 if ( c?$smtp ) smtp_message(c); } + +function describe(rec: Info): string + { + if ( rec?$mailfrom && rec?$rcptto ) + { + local one_to = ""; + for ( to in rec$rcptto ) + { + one_to = to; + break; + } + local abbrev_subject = ""; + if ( rec?$subject ) + { + if ( |rec$subject| > 20 ) + { + abbrev_subject = rec$subject[0:20] + "..."; + } + } + + return fmt("%s -> %s%s%s", rec$mailfrom, one_to, + (|rec$rcptto|>1 ? fmt(" (plus %d others)", |rec$rcptto|-1) : ""), + (abbrev_subject != "" ? fmt(": %s", abbrev_subject) : "")); + } + return ""; + } diff --git a/scripts/base/utils/active-http.bro b/scripts/base/utils/active-http.bro new file mode 100644 index 0000000000..eb9a212221 --- /dev/null +++ b/scripts/base/utils/active-http.bro @@ -0,0 +1,123 @@ +##! A module for performing active HTTP requests and +##! getting the reply at runtime. + +@load ./exec + +module ActiveHTTP; + +export { + ## The default timeout for HTTP requests. + const default_max_time = 1min &redef; + + ## The default HTTP method/verb to use for requests. + const default_method = "GET" &redef; + + type Response: record { + ## Numeric response code from the server. + code: count; + ## String response message from the server. + msg: string; + ## Full body of the response. + body: string &optional; + ## All headers returned by the server. + headers: table[string] of string &optional; + }; + + type Request: record { + ## The URL being requested. + url: string; + ## The HTTP method/verb to use for the request. + method: string &default=default_method; + ## Data to send to the server in the client body. Keep in + ## mind that you will probably need to set the *method* field + ## to "POST" or "PUT". + client_data: string &optional; + ## Arbitrary headers to pass to the server. Some headers + ## will be included by libCurl. + #custom_headers: table[string] of string &optional; + ## Timeout for the request. + max_time: interval &default=default_max_time; + ## Additional curl command line arguments. Be very careful + ## with this option since shell injection could take place + ## if careful handling of untrusted data is not applied. + addl_curl_args: string &optional; + }; + + ## Perform an HTTP request according to the :bro:type:`Request` record. + ## This is an asynchronous function and must be called within a "when" + ## statement. + ## + ## req: A record instance representing all options for an HTTP request. + ## + ## Returns: A record with the full response message. + global request: function(req: ActiveHTTP::Request): ActiveHTTP::Response; +} + +function request2curl(r: Request, bodyfile: string, headersfile: string): string + { + local cmd = fmt("curl -s -g -o \"%s\" -D \"%s\" -X \"%s\"", + str_shell_escape(bodyfile), + str_shell_escape(headersfile), + str_shell_escape(r$method)); + + cmd = fmt("%s -m %.0f", cmd, r$max_time); + + if ( r?$client_data ) + cmd = fmt("%s -d -", cmd); + + if ( r?$addl_curl_args ) + cmd = fmt("%s %s", cmd, r$addl_curl_args); + + cmd = fmt("%s \"%s\"", cmd, str_shell_escape(r$url)); + return cmd; + } + +function request(req: Request): ActiveHTTP::Response + { + local tmpfile = "/tmp/bro-activehttp-" + unique_id(""); + local bodyfile = fmt("%s_body", tmpfile); + local headersfile = fmt("%s_headers", tmpfile); + + local cmd = request2curl(req, bodyfile, headersfile); + local stdin_data = req?$client_data ? req$client_data : ""; + + local resp: Response; + resp$code = 0; + resp$msg = ""; + resp$body = ""; + resp$headers = table(); + return when ( local result = Exec::run([$cmd=cmd, $stdin=stdin_data, $read_files=set(bodyfile, headersfile)]) ) + { + # If there is no response line then nothing else will work either. + if ( ! (result?$files && headersfile in result$files) ) + { + Reporter::error(fmt("There was a failure when requesting \"%s\" with ActiveHTTP.", req$url)); + return resp; + } + + local headers = result$files[headersfile]; + for ( i in headers ) + { + # The reply is the first line. + if ( i == 0 ) + { + local response_line = split_n(headers[0], /[[:blank:]]+/, F, 2); + if ( |response_line| != 3 ) + return resp; + + resp$code = to_count(response_line[2]); + resp$msg = response_line[3]; + resp$body = join_string_vec(result$files[bodyfile], ""); + } + else + { + local line = headers[i]; + local h = split1(line, /:/); + if ( |h| != 2 ) + next; + resp$headers[h[1]] = sub_bytes(h[2], 0, |h[2]|-1); + } + } + return resp; + } + } diff --git a/scripts/base/utils/dir.bro b/scripts/base/utils/dir.bro new file mode 100644 index 0000000000..4f3ee94945 --- /dev/null +++ b/scripts/base/utils/dir.bro @@ -0,0 +1,66 @@ +@load base/utils/exec +@load base/frameworks/reporter +@load base/utils/paths + +module Dir; + +export { + ## The default interval this module checks for files in directories when + ## using the :bro:see:`Dir::monitor` function. + const polling_interval = 30sec &redef; + + ## Register a directory to monitor with a callback that is called + ## every time a previously unseen file is seen. If a file is deleted + ## and seen to be gone, the file is available for being seen again in + ## the future. + ## + ## dir: The directory to monitor for files. + ## + ## callback: Callback that gets executed with each file name + ## that is found. Filenames are provided with the full path. + ## + ## poll_interval: An interval at which to check for new files. + global monitor: function(dir: string, callback: function(fname: string), + poll_interval: interval &default=polling_interval); +} + +event Dir::monitor_ev(dir: string, last_files: set[string], + callback: function(fname: string), + poll_interval: interval) + { + when ( local result = Exec::run([$cmd=fmt("ls -i \"%s/\"", str_shell_escape(dir))]) ) + { + if ( result$exit_code != 0 ) + { + Reporter::warning(fmt("Requested monitoring of non-existent directory (%s).", dir)); + return; + } + + local current_files: set[string] = set(); + local files: vector of string = vector(); + + if ( result?$stdout ) + files = result$stdout; + + for ( i in files ) + { + local parts = split1(files[i], / /); + if ( parts[1] !in last_files ) + callback(build_path_compressed(dir, parts[2])); + add current_files[parts[1]]; + } + + schedule poll_interval + { + Dir::monitor_ev(dir, current_files, callback, poll_interval) + }; + } + } + +function monitor(dir: string, callback: function(fname: string), + poll_interval: interval &default=polling_interval) + { + event Dir::monitor_ev(dir, set(), callback, poll_interval); + } + + diff --git a/scripts/base/utils/exec.bro b/scripts/base/utils/exec.bro new file mode 100644 index 0000000000..732bbcf34c --- /dev/null +++ b/scripts/base/utils/exec.bro @@ -0,0 +1,185 @@ +##! A module for executing external command line programs. + +@load base/frameworks/input + +module Exec; + +export { + type Command: record { + ## The command line to execute. Use care to avoid injection attacks. + ## I.e. if the command uses untrusted/variable data, sanitize + ## it with str_shell_escape(). + cmd: string; + ## Provide standard in to the program as a string. + stdin: string &default=""; + ## If additional files are required to be read in as part of the output + ## of the command they can be defined here. + read_files: set[string] &optional; + # The unique id for tracking executors. + uid: string &default=unique_id(""); + }; + + type Result: record { + ## Exit code from the program. + exit_code: count &default=0; + ## True if the command was terminated with a signal. + signal_exit: bool &default=F; + ## Each line of standard out. + stdout: vector of string &optional; + ## Each line of standard error. + stderr: vector of string &optional; + ## If additional files were requested to be read in + ## the content of the files will be available here. + files: table[string] of string_vec &optional; + }; + + ## Function for running command line programs and getting + ## output. This is an asynchronous function which is meant + ## to be run with the `when` statement. + ## + ## cmd: The command to run. Use care to avoid injection attacks! + ## + ## returns: A record representing the full results from the + ## external program execution. + global run: function(cmd: Command): Result; + + ## The system directory for temp files. + const tmp_dir = "/tmp" &redef; +} + +# Indexed by command uid. +global results: table[string] of Result; +global pending_commands: set[string]; +global pending_files: table[string] of set[string]; + +type OneLine: record { + s: string; + is_stderr: bool; +}; + +type FileLine: record { + s: string; +}; + +event Exec::line(description: Input::EventDescription, tpe: Input::Event, s: string, is_stderr: bool) + { + local result = results[description$name]; + if ( is_stderr ) + { + if ( ! result?$stderr ) + result$stderr = vector(s); + else + result$stderr[|result$stderr|] = s; + } + else + { + if ( ! result?$stdout ) + result$stdout = vector(s); + else + result$stdout[|result$stdout|] = s; + } + } + +event Exec::file_line(description: Input::EventDescription, tpe: Input::Event, s: string) + { + local parts = split1(description$name, /_/); + local name = parts[1]; + local track_file = parts[2]; + + local result = results[name]; + if ( ! result?$files ) + result$files = table(); + + if ( track_file !in result$files ) + result$files[track_file] = vector(s); + else + result$files[track_file][|result$files[track_file]|] = s; + } + +event Input::end_of_data(name: string, source:string) + { + local parts = split1(name, /_/); + name = parts[1]; + + if ( name !in pending_commands || |parts| < 2 ) + return; + + local track_file = parts[2]; + + Input::remove(name); + + if ( name !in pending_files ) + delete pending_commands[name]; + else + { + delete pending_files[name][track_file]; + if ( |pending_files[name]| == 0 ) + delete pending_commands[name]; + system(fmt("rm \"%s\"", str_shell_escape(track_file))); + } + } + +event InputRaw::process_finished(name: string, source:string, exit_code:count, signal_exit:bool) + { + if ( name !in pending_commands ) + return; + + Input::remove(name); + results[name]$exit_code = exit_code; + results[name]$signal_exit = signal_exit; + + if ( name !in pending_files || |pending_files[name]| == 0 ) + # No extra files to read, command is done. + delete pending_commands[name]; + else + for ( read_file in pending_files[name] ) + Input::add_event([$source=fmt("%s", read_file), + $name=fmt("%s_%s", name, read_file), + $reader=Input::READER_RAW, + $want_record=F, + $fields=FileLine, + $ev=Exec::file_line]); + } + +function run(cmd: Command): Result + { + add pending_commands[cmd$uid]; + results[cmd$uid] = []; + + if ( cmd?$read_files ) + { + for ( read_file in cmd$read_files ) + { + if ( cmd$uid !in pending_files ) + pending_files[cmd$uid] = set(); + add pending_files[cmd$uid][read_file]; + } + } + + local config_strings: table[string] of string = { + ["stdin"] = cmd$stdin, + ["read_stderr"] = "1", + }; + Input::add_event([$name=cmd$uid, + $source=fmt("%s |", cmd$cmd), + $reader=Input::READER_RAW, + $fields=Exec::OneLine, + $ev=Exec::line, + $want_record=F, + $config=config_strings]); + + return when ( cmd$uid !in pending_commands ) + { + local result = results[cmd$uid]; + delete results[cmd$uid]; + return result; + } + } + +event bro_done() + { + # We are punting here and just deleting any unprocessed files. + for ( uid in pending_files ) + for ( fname in pending_files[uid] ) + system(fmt("rm \"%s\"", str_shell_escape(fname))); + } diff --git a/scripts/base/utils/files.bro b/scripts/base/utils/files.bro index 76d2ecea4f..b88ae5763e 100644 --- a/scripts/base/utils/files.bro +++ b/scripts/base/utils/files.bro @@ -6,22 +6,28 @@ function generate_extraction_filename(prefix: string, c: connection, suffix: str { local conn_info = fmt("%s:%d-%s:%d", addr_to_uri(c$id$orig_h), c$id$orig_p, addr_to_uri(c$id$resp_h), c$id$resp_p); - + if ( prefix != "" ) conn_info = fmt("%s_%s", prefix, conn_info); if ( suffix != "" ) conn_info = fmt("%s_%s", conn_info, suffix); - + return conn_info; } - -## For CONTENT-DISPOSITION headers, this function can be used to extract + +## For CONTENT-DISPOSITION headers, this function can be used to extract ## the filename. function extract_filename_from_content_disposition(data: string): string { - local filename = sub(data, /^.*[nN][aA][mM][eE][[:blank:]]*=[[:blank:]]*/, ""); + local filename = sub(data, /^.*[nN][aA][mM][eE][[:blank:]]*\*?=[[:blank:]]*/, ""); + # Remove quotes around the filename if they are there. if ( /^\"/ in filename ) - filename = split_n(filename, /\"/, F, 2)[2]; - return filename; + filename = split_n(filename, /\"/, F, 2)[2]; + + # Remove the language and encoding if it's there. + if ( /^[a-zA-Z0-9\!#$%&+-^_`{}~]+'[a-zA-Z0-9\!#$%&+-^_`{}~]*'/ in filename ) + filename = sub(filename, /^.+'.*'/, ""); + + return unescape_URI(filename); } diff --git a/scripts/policy/frameworks/files/detect-MHR.bro b/scripts/policy/frameworks/files/detect-MHR.bro new file mode 100644 index 0000000000..8a2e33b7f4 --- /dev/null +++ b/scripts/policy/frameworks/files/detect-MHR.bro @@ -0,0 +1,57 @@ +##! Detect file downloads that have hash values matching files in Team +##! Cymru's Malware Hash Registry (http://www.team-cymru.org/Services/MHR/). + +@load base/frameworks/files +@load base/frameworks/notice +@load frameworks/files/hash-all-files + +module TeamCymruMalwareHashRegistry; + +export { + redef enum Notice::Type += { + ## The hash value of a file transferred over HTTP matched in the + ## malware hash registry. + Match + }; + + ## File types to attempt matching against the Malware Hash Registry. + const match_file_types = /application\/x-dosexec/ | + /application\/vnd.ms-cab-compressed/ | + /application\/pdf/ | + /application\/x-shockwave-flash/ | + /application\/x-java-applet/ | + /application\/jar/ | + /video\/mp4/ &redef; + + ## The malware hash registry runs each malware sample through several A/V engines. + ## Team Cymru returns a percentage to indicate how many A/V engines flagged the + ## sample as malicious. This threshold allows you to require a minimum detection + ## rate. + const notice_threshold = 10 &redef; +} + +event file_hash(f: fa_file, kind: string, hash: string) + { + if ( kind=="sha1" && match_file_types in f$mime_type ) + { + local hash_domain = fmt("%s.malware.hash.cymru.com", hash); + when ( local MHR_result = lookup_hostname_txt(hash_domain) ) + { + # Data is returned as " " + local MHR_answer = split1(MHR_result, / /); + if ( |MHR_answer| == 2 ) + { + local mhr_first_detected = double_to_time(to_double(MHR_answer[1])); + local mhr_detect_rate = to_count(MHR_answer[2]); + + local readable_first_detected = strftime("%Y-%m-%d %H:%M:%S", mhr_first_detected); + if ( mhr_detect_rate >= notice_threshold ) + { + local message = fmt("Malware Hash Registry Detection rate: %d%% Last seen: %s", mhr_detect_rate, readable_first_detected); + local virustotal_url = fmt("https://www.virustotal.com/en/file/%s/analysis/", hash); + NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]); + } + } + } + } + } diff --git a/scripts/policy/frameworks/files/hash-all-files.bro b/scripts/policy/frameworks/files/hash-all-files.bro new file mode 100644 index 0000000000..931857c2bc --- /dev/null +++ b/scripts/policy/frameworks/files/hash-all-files.bro @@ -0,0 +1,7 @@ +# Perform MD5 and SHA1 hashing on all files. + +event file_new(f: fa_file) + { + Files::add_analyzer(f, Files::ANALYZER_MD5); + Files::add_analyzer(f, Files::ANALYZER_SHA1); + } diff --git a/scripts/policy/frameworks/intel/conn-established.bro b/scripts/policy/frameworks/intel/conn-established.bro deleted file mode 100644 index a2e67b292b..0000000000 --- a/scripts/policy/frameworks/intel/conn-established.bro +++ /dev/null @@ -1,8 +0,0 @@ -@load base/frameworks/intel -@load ./where-locations - -event connection_established(c: connection) - { - Intel::seen([$host=c$id$orig_h, $conn=c, $where=Conn::IN_ORIG]); - Intel::seen([$host=c$id$resp_h, $conn=c, $where=Conn::IN_RESP]); - } diff --git a/scripts/policy/frameworks/intel/do_notice.bro b/scripts/policy/frameworks/intel/do_notice.bro new file mode 100644 index 0000000000..720e29c35c --- /dev/null +++ b/scripts/policy/frameworks/intel/do_notice.bro @@ -0,0 +1,44 @@ + +@load base/frameworks/intel +@load base/frameworks/notice + +module Intel; + +export { + redef enum Notice::Type += { + ## Intel::Notice is a notice that happens when an intelligence + ## indicator is denoted to be notice-worthy. + Intel::Notice + }; + + redef record Intel::MetaData += { + ## A boolean value to allow the data itself to represent + ## if the indicator that this metadata is attached to + ## is notice worthy. + do_notice: bool &default=F; + + ## Restrictions on when notices are created to only create + ## them if the do_notice field is T and the notice was + ## seen in the indicated location. + if_in: Intel::Where &optional; + }; +} + +event Intel::match(s: Seen, items: set[Item]) + { + for ( item in items ) + { + if ( item$meta$do_notice && + (! item$meta?$if_in || s$where == item$meta$if_in) ) + { + local n = Notice::Info($note=Intel::Notice, + $msg=fmt("Intel hit on %s at %s", s$indicator, s$where), + $sub=s$indicator); + + if ( s?$conn ) + n$conn = s$conn; + + NOTICE(n); + } + } + } diff --git a/scripts/policy/frameworks/intel/__load__.bro b/scripts/policy/frameworks/intel/seen/__load__.bro similarity index 100% rename from scripts/policy/frameworks/intel/__load__.bro rename to scripts/policy/frameworks/intel/seen/__load__.bro diff --git a/scripts/policy/frameworks/intel/seen/conn-established.bro b/scripts/policy/frameworks/intel/seen/conn-established.bro new file mode 100644 index 0000000000..20cec43e04 --- /dev/null +++ b/scripts/policy/frameworks/intel/seen/conn-established.bro @@ -0,0 +1,12 @@ +@load base/frameworks/intel +@load ./where-locations + +event connection_established(c: connection) + { + if ( c$orig$state == TCP_ESTABLISHED && + c$resp$state == TCP_ESTABLISHED ) + { + Intel::seen([$host=c$id$orig_h, $conn=c, $where=Conn::IN_ORIG]); + Intel::seen([$host=c$id$resp_h, $conn=c, $where=Conn::IN_RESP]); + } + } diff --git a/scripts/policy/frameworks/intel/dns.bro b/scripts/policy/frameworks/intel/seen/dns.bro similarity index 73% rename from scripts/policy/frameworks/intel/dns.bro rename to scripts/policy/frameworks/intel/seen/dns.bro index a0dee47acf..9218586c95 100644 --- a/scripts/policy/frameworks/intel/dns.bro +++ b/scripts/policy/frameworks/intel/seen/dns.bro @@ -3,8 +3,8 @@ event dns_request(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count) { - Intel::seen([$str=query, - $str_type=Intel::DOMAIN, + Intel::seen([$indicator=query, + $indicator_type=Intel::DOMAIN, $conn=c, $where=DNS::IN_REQUEST]); } diff --git a/scripts/policy/frameworks/intel/http-host-header.bro b/scripts/policy/frameworks/intel/seen/http-host-header.bro similarity index 75% rename from scripts/policy/frameworks/intel/http-host-header.bro rename to scripts/policy/frameworks/intel/seen/http-host-header.bro index f16b1628aa..3fd28b8ef9 100644 --- a/scripts/policy/frameworks/intel/http-host-header.bro +++ b/scripts/policy/frameworks/intel/seen/http-host-header.bro @@ -4,8 +4,8 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) { if ( is_orig && name == "HOST" ) - Intel::seen([$str=value, - $str_type=Intel::DOMAIN, + Intel::seen([$indicator=value, + $indicator_type=Intel::DOMAIN, $conn=c, $where=HTTP::IN_HOST_HEADER]); } diff --git a/scripts/policy/frameworks/intel/http-url.bro b/scripts/policy/frameworks/intel/seen/http-url.bro similarity index 73% rename from scripts/policy/frameworks/intel/http-url.bro rename to scripts/policy/frameworks/intel/seen/http-url.bro index feef4f0dac..340ae3c5ab 100644 --- a/scripts/policy/frameworks/intel/http-url.bro +++ b/scripts/policy/frameworks/intel/seen/http-url.bro @@ -5,8 +5,8 @@ event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) { if ( is_orig && c?$http ) - Intel::seen([$str=HTTP::build_url(c$http), - $str_type=Intel::URL, + Intel::seen([$indicator=HTTP::build_url(c$http), + $indicator_type=Intel::URL, $conn=c, $where=HTTP::IN_URL]); } diff --git a/scripts/policy/frameworks/intel/http-user-agents.bro b/scripts/policy/frameworks/intel/seen/http-user-agents.bro similarity index 75% rename from scripts/policy/frameworks/intel/http-user-agents.bro rename to scripts/policy/frameworks/intel/seen/http-user-agents.bro index 93445c1e43..7c4558d2a5 100644 --- a/scripts/policy/frameworks/intel/http-user-agents.bro +++ b/scripts/policy/frameworks/intel/seen/http-user-agents.bro @@ -4,8 +4,8 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) { if ( is_orig && name == "USER-AGENT" ) - Intel::seen([$str=value, - $str_type=Intel::USER_AGENT, + Intel::seen([$indicator=value, + $indicator_type=Intel::SOFTWARE, $conn=c, $where=HTTP::IN_USER_AGENT_HEADER]); } diff --git a/scripts/policy/frameworks/intel/seen/smtp-url-extraction.bro b/scripts/policy/frameworks/intel/seen/smtp-url-extraction.bro new file mode 100644 index 0000000000..aa9c322bcf --- /dev/null +++ b/scripts/policy/frameworks/intel/seen/smtp-url-extraction.bro @@ -0,0 +1,29 @@ +@load base/frameworks/intel +@load base/protocols/smtp +@load base/utils/urls +@load ./where-locations + +event intel_mime_data(f: fa_file, data: string) + { + if ( ! f?$conns ) + return; + + for ( cid in f$conns ) + { + local c: connection = f$conns[cid]; + local urls = find_all_urls_without_scheme(data); + for ( url in urls ) + { + Intel::seen([$indicator=url, + $indicator_type=Intel::URL, + $conn=c, + $where=SMTP::IN_MESSAGE]); + } + } + } + +event file_new(f: fa_file) + { + if ( f$source == "SMTP" ) + Files::add_analyzer(f, Files::ANALYZER_DATA_EVENT, [$stream_event=intel_mime_data]); + } diff --git a/scripts/policy/frameworks/intel/seen/smtp.bro b/scripts/policy/frameworks/intel/seen/smtp.bro new file mode 100644 index 0000000000..d760995e51 --- /dev/null +++ b/scripts/policy/frameworks/intel/seen/smtp.bro @@ -0,0 +1,97 @@ +@load base/frameworks/intel +@load base/protocols/smtp +@load ./where-locations + +event mime_end_entity(c: connection) + { + if ( c?$smtp ) + { + if ( c$smtp?$path ) + { + local path = c$smtp$path; + for ( i in path ) + { + Intel::seen([$host=path[i], + $conn=c, + $where=SMTP::IN_RECEIVED_HEADER]); + } + } + + if ( c$smtp?$user_agent ) + Intel::seen([$indicator=c$smtp$user_agent, + $indicator_type=Intel::SOFTWARE, + $conn=c, + $where=SMTP::IN_HEADER]); + + if ( c$smtp?$x_originating_ip ) + Intel::seen([$host=c$smtp$x_originating_ip, + $conn=c, + $where=SMTP::IN_X_ORIGINATING_IP_HEADER]); + + if ( c$smtp?$mailfrom ) + { + local mailfromparts = split_n(c$smtp$mailfrom, /<.+>/, T, 1); + if ( |mailfromparts| > 2 ) + { + Intel::seen([$indicator=mailfromparts[2][1:-2], + $indicator_type=Intel::EMAIL, + $conn=c, + $where=SMTP::IN_MAIL_FROM]); + } + } + + if ( c$smtp?$rcptto ) + { + for ( rcptto in c$smtp$rcptto ) + { + local rcpttoparts = split_n(rcptto, /<.+>/, T, 1); + if ( |rcpttoparts| > 2 ) + { + Intel::seen([$indicator=rcpttoparts[2][1:-2], + $indicator_type=Intel::EMAIL, + $conn=c, + $where=SMTP::IN_RCPT_TO]); + } + } + } + + if ( c$smtp?$from ) + { + local fromparts = split_n(c$smtp$from, /<.+>/, T, 1); + if ( |fromparts| > 2 ) + { + Intel::seen([$indicator=fromparts[2][1:-2], + $indicator_type=Intel::EMAIL, + $conn=c, + $where=SMTP::IN_FROM]); + } + } + + if ( c$smtp?$to ) + { + for ( email_to in c$smtp$to ) + { + local toparts = split_n(email_to, /<.+>/, T, 1); + if ( |toparts| > 2 ) + { + Intel::seen([$indicator=toparts[2][1:-2], + $indicator_type=Intel::EMAIL, + $conn=c, + $where=SMTP::IN_TO]); + } + } + } + + if ( c$smtp?$reply_to ) + { + local replytoparts = split_n(c$smtp$reply_to, /<.+>/, T, 1); + if ( |replytoparts| > 2 ) + { + Intel::seen([$indicator=replytoparts[2][1:-2], + $indicator_type=Intel::EMAIL, + $conn=c, + $where=SMTP::IN_REPLY_TO]); + } + } + } + } diff --git a/scripts/policy/frameworks/intel/ssl.bro b/scripts/policy/frameworks/intel/seen/ssl.bro similarity index 75% rename from scripts/policy/frameworks/intel/ssl.bro rename to scripts/policy/frameworks/intel/seen/ssl.bro index 3f18a11e6e..e404c39e5b 100644 --- a/scripts/policy/frameworks/intel/ssl.bro +++ b/scripts/policy/frameworks/intel/seen/ssl.bro @@ -10,14 +10,14 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: coun { local email = sub(cert$subject, /^.*emailAddress=/, ""); email = sub(email, /,.*$/, ""); - Intel::seen([$str=email, - $str_type=Intel::EMAIL, + Intel::seen([$indicator=email, + $indicator_type=Intel::EMAIL, $conn=c, $where=(is_orig ? SSL::IN_CLIENT_CERT : SSL::IN_SERVER_CERT)]); } - Intel::seen([$str=sha1_hash(der_cert), - $str_type=Intel::CERT_HASH, + Intel::seen([$indicator=sha1_hash(der_cert), + $indicator_type=Intel::CERT_HASH, $conn=c, $where=(is_orig ? SSL::IN_CLIENT_CERT : SSL::IN_SERVER_CERT)]); } @@ -27,8 +27,8 @@ event ssl_extension(c: connection, is_orig: bool, code: count, val: string) { if ( is_orig && SSL::extensions[code] == "server_name" && c?$ssl && c$ssl?$server_name ) - Intel::seen([$str=c$ssl$server_name, - $str_type=Intel::DOMAIN, + Intel::seen([$indicator=c$ssl$server_name, + $indicator_type=Intel::DOMAIN, $conn=c, $where=SSL::IN_SERVER_NAME]); } diff --git a/scripts/policy/frameworks/intel/where-locations.bro b/scripts/policy/frameworks/intel/seen/where-locations.bro similarity index 100% rename from scripts/policy/frameworks/intel/where-locations.bro rename to scripts/policy/frameworks/intel/seen/where-locations.bro diff --git a/scripts/policy/frameworks/intel/smtp-url-extraction.bro b/scripts/policy/frameworks/intel/smtp-url-extraction.bro deleted file mode 100644 index 2b87f809a6..0000000000 --- a/scripts/policy/frameworks/intel/smtp-url-extraction.bro +++ /dev/null @@ -1,31 +0,0 @@ -@load base/frameworks/intel -@load base/protocols/smtp/file-analysis -@load base/utils/urls -@load ./where-locations - -event intel_mime_data(f: fa_file, data: string) - { - if ( ! f?$conns ) return; - - for ( cid in f$conns ) - { - local c: connection = f$conns[cid]; - local urls = find_all_urls_without_scheme(data); - for ( url in urls ) - { - Intel::seen([$str=url, - $str_type=Intel::URL, - $conn=c, - $where=SMTP::IN_MESSAGE]); - } - } - } - -event file_new(f: fa_file) &priority=5 - { - if ( ! f?$source ) return; - if ( f$source != "SMTP" ) return; - - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_DATA_EVENT, - $stream_event=intel_mime_data]); - } diff --git a/scripts/policy/frameworks/intel/smtp.bro b/scripts/policy/frameworks/intel/smtp.bro deleted file mode 100644 index 02e97ea54a..0000000000 --- a/scripts/policy/frameworks/intel/smtp.bro +++ /dev/null @@ -1,71 +0,0 @@ -@load base/frameworks/intel -@load base/protocols/smtp -@load ./where-locations - -event mime_end_entity(c: connection) - { - if ( c?$smtp ) - { - if ( c$smtp?$path ) - { - local path = c$smtp$path; - for ( i in path ) - { - Intel::seen([$host=path[i], - $conn=c, - $where=SMTP::IN_RECEIVED_HEADER]); - } - } - - if ( c$smtp?$user_agent ) - Intel::seen([$str=c$smtp$user_agent, - $str_type=Intel::USER_AGENT, - $conn=c, - $where=SMTP::IN_HEADER]); - - if ( c$smtp?$x_originating_ip ) - Intel::seen([$host=c$smtp$x_originating_ip, - $conn=c, - $where=SMTP::IN_X_ORIGINATING_IP_HEADER]); - - if ( c$smtp?$mailfrom ) - Intel::seen([$str=c$smtp$mailfrom, - $str_type=Intel::EMAIL, - $conn=c, - $where=SMTP::IN_MAIL_FROM]); - - if ( c$smtp?$rcptto ) - { - for ( rcptto in c$smtp$rcptto ) - { - Intel::seen([$str=rcptto, - $str_type=Intel::EMAIL, - $conn=c, - $where=SMTP::IN_RCPT_TO]); - } - } - - if ( c$smtp?$from ) - Intel::seen([$str=c$smtp$from, - $str_type=Intel::EMAIL, - $conn=c, - $where=SMTP::IN_FROM]); - - if ( c$smtp?$to ) - { - for ( email_to in c$smtp$to ) - { - Intel::seen([$str=email_to, - $str_type=Intel::EMAIL, - $conn=c, - $where=SMTP::IN_TO]); - } - } - - if ( c$smtp?$reply_to ) - Intel::seen([$str=c$smtp$reply_to, - $str_type=Intel::EMAIL, - $conn=c, - $where=SMTP::IN_REPLY_TO]); - } - } diff --git a/scripts/policy/protocols/http/detect-MHR.bro b/scripts/policy/protocols/http/detect-MHR.bro deleted file mode 100644 index 0594276c93..0000000000 --- a/scripts/policy/protocols/http/detect-MHR.bro +++ /dev/null @@ -1,44 +0,0 @@ -##! Detect file downloads over HTTP that have MD5 sums matching files in Team -##! Cymru's Malware Hash Registry (http://www.team-cymru.org/Services/MHR/). -##! By default, not all file transfers will have MD5 sums calculated. Read the -##! documentation for the :doc:base/protocols/http/file-hash.bro script to see -##! how to configure which transfers will have hashes calculated. - -@load base/frameworks/notice -@load base/protocols/http - -module HTTP; - -export { - redef enum Notice::Type += { - ## The MD5 sum of a file transferred over HTTP matched in the - ## malware hash registry. - Malware_Hash_Registry_Match - }; - - ## The malware hash registry runs each malware sample through several A/V engines. - ## Team Cymru returns a percentage to indicate how many A/V engines flagged the - ## sample as malicious. This threshold allows you to require a minimum detection - ## rate (default: 50%). - const MHR_threshold = 50 &redef; -} - -event log_http(rec: HTTP::Info) - { - if ( rec?$md5 ) - { - local hash_domain = fmt("%s.malware.hash.cymru.com", rec$md5); - when ( local MHR_result = lookup_hostname_txt(hash_domain) ) - { - # Data is returned as " " - local MHR_answer = split1(MHR_result, / /); - if ( |MHR_answer| == 2 && to_count(MHR_answer[2]) >= MHR_threshold ) - { - local url = HTTP::build_url_http(rec); - local message = fmt("%s %s %s", rec$id$orig_h, rec$md5, url); - NOTICE([$note=Malware_Hash_Registry_Match, - $msg=message, $id=rec$id]); - } - } - } - } diff --git a/scripts/base/protocols/smtp/entities-excerpt.bro b/scripts/policy/protocols/smtp/entities-excerpt.bro similarity index 83% rename from scripts/base/protocols/smtp/entities-excerpt.bro rename to scripts/policy/protocols/smtp/entities-excerpt.bro index 1ecd100571..0e596ebdd1 100644 --- a/scripts/base/protocols/smtp/entities-excerpt.bro +++ b/scripts/policy/protocols/smtp/entities-excerpt.bro @@ -1,16 +1,16 @@ ##! This script is for optionally adding a body excerpt to the SMTP ##! entities log. -@load ./entities +@load base/protocols/smtp/entities module SMTP; export { - redef record SMTP::EntityInfo += { + redef record SMTP::Entity+= { ## The entity body excerpt. excerpt: string &log &default=""; }; - + ## This is the default value for how much of the entity body should be ## included for all MIME entities. The lesser of this value and ## :bro:see:`default_file_bof_buffer_size` will be used. @@ -31,7 +31,6 @@ event file_new(f: fa_file) &priority=5 if ( ! c?$smtp ) next; if ( default_entity_excerpt_len > 0 ) - c$smtp$current_entity$excerpt = - f$bof_buffer[0:default_entity_excerpt_len]; + c$smtp$entity$excerpt = f$bof_buffer[0:default_entity_excerpt_len]; } } diff --git a/scripts/policy/protocols/ssh/detect-bruteforcing.bro b/scripts/policy/protocols/ssh/detect-bruteforcing.bro index 309905e939..ada418e61f 100644 --- a/scripts/policy/protocols/ssh/detect-bruteforcing.bro +++ b/scripts/policy/protocols/ssh/detect-bruteforcing.bro @@ -58,10 +58,6 @@ event bro_init() $msg=fmt("%s appears to be guessing SSH passwords (seen in %d connections).", key$host, r$num), $src=key$host, $identifier=cat(key$host)]); - # Insert the guesser into the intel framework. - Intel::insert([$host=key$host, - $meta=[$source="local", - $desc=fmt("Bro observed %d apparently failed SSH connections.", r$num)]]); }]); } diff --git a/scripts/site/local.bro b/scripts/site/local.bro index dfebd9923a..e4b3a44e7a 100644 --- a/scripts/site/local.bro +++ b/scripts/site/local.bro @@ -64,7 +64,14 @@ # Detect logins using "interesting" hostnames. @load protocols/ssh/interesting-hostnames -# Detect MD5 sums in Team Cymru's Malware Hash Registry. -@load protocols/http/detect-MHR # Detect SQL injection attacks. @load protocols/http/detect-sqli + +#### Network File Handling #### + +# Enable MD5 and SHA1 hashing for all files. +@load frameworks/files/hash-all-files + +# Detect SHA1 sums in Team Cymru's Malware Hash Registry. +@load frameworks/files/detect-MHR + diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro index 1fd34d6f2f..dcf50b538e 100644 --- a/scripts/test-all-policy.bro +++ b/scripts/test-all-policy.bro @@ -14,16 +14,19 @@ # @load frameworks/control/controller.bro @load frameworks/dpd/detect-protocols.bro @load frameworks/dpd/packet-segment-logging.bro -@load frameworks/intel/__load__.bro -@load frameworks/intel/conn-established.bro -@load frameworks/intel/dns.bro -@load frameworks/intel/http-host-header.bro -@load frameworks/intel/http-url.bro -@load frameworks/intel/http-user-agents.bro -@load frameworks/intel/smtp-url-extraction.bro -@load frameworks/intel/smtp.bro -@load frameworks/intel/ssl.bro -@load frameworks/intel/where-locations.bro +@load frameworks/intel/do_notice.bro +@load frameworks/intel/seen/__load__.bro +@load frameworks/intel/seen/conn-established.bro +@load frameworks/intel/seen/dns.bro +@load frameworks/intel/seen/http-host-header.bro +@load frameworks/intel/seen/http-url.bro +@load frameworks/intel/seen/http-user-agents.bro +@load frameworks/intel/seen/smtp-url-extraction.bro +@load frameworks/intel/seen/smtp.bro +@load frameworks/intel/seen/ssl.bro +@load frameworks/intel/seen/where-locations.bro +@load frameworks/files/detect-MHR.bro +@load frameworks/files/hash-all-files.bro @load frameworks/packet-filter/shunt.bro @load frameworks/software/version-changes.bro @load frameworks/software/vulnerable.bro @@ -50,7 +53,6 @@ @load protocols/ftp/detect-bruteforcing.bro @load protocols/ftp/detect.bro @load protocols/ftp/software.bro -@load protocols/http/detect-MHR.bro @load protocols/http/detect-sqli.bro @load protocols/http/detect-webapps.bro @load protocols/http/header-names.bro @@ -62,6 +64,7 @@ @load protocols/modbus/track-memmap.bro @load protocols/smtp/blocklists.bro @load protocols/smtp/detect-suspicious-orig.bro +@load protocols/smtp/entities-excerpt.bro @load protocols/smtp/software.bro @load protocols/ssh/detect-bruteforcing.bro @load protocols/ssh/geo-data.bro diff --git a/src/SerialTypes.h b/src/SerialTypes.h index 50629302ec..b008763bbf 100644 --- a/src/SerialTypes.h +++ b/src/SerialTypes.h @@ -52,6 +52,7 @@ SERIAL_IS(RE_MATCHER, 0x1400) SERIAL_IS(BITVECTOR, 0x1500) SERIAL_IS(COUNTERVECTOR, 0x1600) SERIAL_IS(BLOOMFILTER, 0x1700) +SERIAL_IS(HASHER, 0x1800) // These are the externally visible types. const SerialType SER_NONE = 0; @@ -207,6 +208,11 @@ SERIAL_BLOOMFILTER(BLOOMFILTER, 1) SERIAL_BLOOMFILTER(BASICBLOOMFILTER, 2) SERIAL_BLOOMFILTER(COUNTINGBLOOMFILTER, 3) +#define SERIAL_HASHER(name, val) SERIAL_CONST(name, val, HASHER) +SERIAL_HASHER(HASHER, 1) +SERIAL_HASHER(DEFAULTHASHER, 2) +SERIAL_HASHER(DOUBLEHASHER, 3) + SERIAL_CONST2(ID) SERIAL_CONST2(STATE_ACCESS) SERIAL_CONST2(CASE) diff --git a/src/analyzer/analyzer.bif b/src/analyzer/analyzer.bif index 7f3cc6ed94..4d70816075 100644 --- a/src/analyzer/analyzer.bif +++ b/src/analyzer/analyzer.bif @@ -43,3 +43,9 @@ function __name%(atype: Analyzer::Tag%) : string %{ return new StringVal(analyzer_mgr->GetAnalyzerName(atype)); %} + +function __tag%(name: string%) : Analyzer::Tag + %{ + analyzer::Tag t = analyzer_mgr->GetAnalyzerTag(name->CheckString()); + return t.AsEnumVal()->Ref(); + %} diff --git a/src/event.bif b/src/event.bif index 6f363cb961..e4d6f8c844 100644 --- a/src/event.bif +++ b/src/event.bif @@ -897,7 +897,7 @@ event event_queue_flush_point%(%); event get_file_handle%(tag: Analyzer::Tag, c: connection, is_orig: bool%); ## Indicates that an analysis of a new file has begun. The analysis can be -## augmented at this time via :bro:see:`FileAnalysis::add_analyzer`. +## augmented at this time via :bro:see:`Files::add_analyzer`. ## ## f: The file. ## @@ -911,8 +911,10 @@ event file_new%(f: fa_file%); ## ## c: The new connection over which the file is seen being transferred. ## +## is_orig: true if the originator of *c* is the one sending the file. +## ## .. bro:see:: file_new file_timeout file_gap file_state_remove -event file_over_new_connection%(f: fa_file, c: connection%); +event file_over_new_connection%(f: fa_file, c: connection, is_orig: bool%); ## Indicates that file analysis has timed out because no activity was seen ## for the file in a while. @@ -920,8 +922,8 @@ event file_over_new_connection%(f: fa_file, c: connection%); ## f: The file. ## ## .. bro:see:: file_new file_over_new_connection file_gap file_state_remove -## default_file_timeout_interval FileAnalysis::set_timeout_interval -## FileAnalysis::set_timeout_interval +## default_file_timeout_interval Files::set_timeout_interval +## Files::set_timeout_interval event file_timeout%(f: fa_file%); ## Indicates that a chunk of the file is missing. diff --git a/src/file_analysis/Analyzer.h b/src/file_analysis/Analyzer.h index 2589dfbe35..0a5aa9e25c 100644 --- a/src/file_analysis/Analyzer.h +++ b/src/file_analysis/Analyzer.h @@ -96,7 +96,7 @@ public: */ static FA_Tag ArgsTag(const RecordVal* args) { - using BifType::Record::FileAnalysis::AnalyzerArgs; + using BifType::Record::Files::AnalyzerArgs; return args->Lookup(AnalyzerArgs->FieldOffset("tag"))->AsEnum(); } diff --git a/src/file_analysis/AnalyzerSet.cc b/src/file_analysis/AnalyzerSet.cc index e350e8b0d8..c710d8b085 100644 --- a/src/file_analysis/AnalyzerSet.cc +++ b/src/file_analysis/AnalyzerSet.cc @@ -15,7 +15,7 @@ static void analyzer_del_func(void* v) AnalyzerSet::AnalyzerSet(File* arg_file) : file(arg_file) { TypeList* t = new TypeList(); - t->Append(BifType::Record::FileAnalysis::AnalyzerArgs->Ref()); + t->Append(BifType::Record::Files::AnalyzerArgs->Ref()); analyzer_hash = new CompositeHash(t); Unref(t); analyzer_map.SetDeleteFunc(analyzer_del_func); diff --git a/src/file_analysis/File.cc b/src/file_analysis/File.cc index ee590a23a7..9e44e327e3 100644 --- a/src/file_analysis/File.cc +++ b/src/file_analysis/File.cc @@ -75,7 +75,8 @@ void File::StaticInit() File::File(const string& file_id, Connection* conn, analyzer::Tag tag, bool is_orig) : id(file_id), val(0), postpone_timeout(false), first_chunk(true), - missed_bof(false), need_reassembly(false), done(false), analyzers(this) + missed_bof(false), need_reassembly(false), done(false), + did_file_new_event(false), analyzers(this) { StaticInit(); @@ -89,7 +90,7 @@ File::File(const string& file_id, Connection* conn, analyzer::Tag tag, // add source, connection, is_orig fields SetSource(analyzer_mgr->GetAnalyzerName(tag)); val->Assign(is_orig_idx, new Val(is_orig, TYPE_BOOL)); - UpdateConnectionFields(conn); + UpdateConnectionFields(conn, is_orig); } UpdateLastActivityTime(); @@ -99,6 +100,13 @@ File::~File() { DBG_LOG(DBG_FILE_ANALYSIS, "Destroying File object %s", id.c_str()); Unref(val); + + // Queue may not be empty in the case where only content gaps were seen. + while ( ! fonc_queue.empty() ) + { + delete_vals(fonc_queue.front().second); + fonc_queue.pop(); + } } void File::UpdateLastActivityTime() @@ -111,18 +119,15 @@ double File::GetLastActivityTime() const return val->Lookup(last_active_idx)->AsTime(); } -void File::UpdateConnectionFields(Connection* conn) +void File::UpdateConnectionFields(Connection* conn, bool is_orig) { if ( ! conn ) return; Val* conns = val->Lookup(conns_idx); - bool is_first = false; - if ( ! conns ) { - is_first = true; conns = empty_connection_table(); val->Assign(conns_idx, conns); } @@ -133,12 +138,18 @@ void File::UpdateConnectionFields(Connection* conn) Val* conn_val = conn->BuildConnVal(); conns->AsTableVal()->Assign(idx, conn_val); - if ( ! is_first && FileEventAvailable(file_over_new_connection) ) + if ( FileEventAvailable(file_over_new_connection) ) { val_list* vl = new val_list(); vl->append(val->Ref()); vl->append(conn_val->Ref()); - FileEvent(file_over_new_connection, vl); + vl->append(new Val(is_orig, TYPE_BOOL)); + + if ( did_file_new_event ) + FileEvent(file_over_new_connection, vl); + else + fonc_queue.push(pair( + file_over_new_connection, vl)); } } @@ -435,6 +446,18 @@ void File::FileEvent(EventHandlerPtr h, val_list* vl) { mgr.QueueEvent(h, vl); + if ( h == file_new ) + { + did_file_new_event = true; + + while ( ! fonc_queue.empty() ) + { + pair p = fonc_queue.front(); + mgr.QueueEvent(p.first, p.second); + fonc_queue.pop(); + } + } + if ( h == file_new || h == file_timeout ) { // immediate feedback is required for these events. diff --git a/src/file_analysis/File.h b/src/file_analysis/File.h index ac54c75bc5..794734d24b 100644 --- a/src/file_analysis/File.h +++ b/src/file_analysis/File.h @@ -3,7 +3,9 @@ #ifndef FILE_ANALYSIS_FILE_H #define FILE_ANALYSIS_FILE_H +#include #include +#include #include #include "Conn.h" @@ -171,8 +173,9 @@ protected: * Updates the "conn_ids" and "conn_uids" fields in #val record with the * \c conn_id and UID taken from \a conn. * @param conn the connection over which a part of the file has been seen. + * @param is_orig true if the connection originator is sending the file. */ - void UpdateConnectionFields(Connection* conn); + void UpdateConnectionFields(Connection* conn, bool is_orig); /** * Increment a byte count field of #val record by \a size. @@ -239,7 +242,9 @@ private: bool missed_bof; /**< Flags that we missed start of file. */ bool need_reassembly; /**< Whether file stream reassembly is needed. */ bool done; /**< If this object is about to be deleted. */ + bool did_file_new_event; /**< Whether the file_new event has been done. */ AnalyzerSet analyzers; /**< A set of attached file analyzer. */ + queue > fonc_queue; struct BOF_Buffer { BOF_Buffer() : full(false), replayed(false), size(0) {} diff --git a/src/file_analysis/Manager.cc b/src/file_analysis/Manager.cc index a7f7a29c18..7a92e92109 100644 --- a/src/file_analysis/Manager.cc +++ b/src/file_analysis/Manager.cc @@ -19,8 +19,8 @@ string Manager::salt; Manager::Manager() { - tag_enum_type = new EnumType("FileAnalysis::Tag"); - ::ID* id = install_ID("Tag", "FileAnalysis", true, true); + tag_enum_type = new EnumType("Files::Tag"); + ::ID* id = install_ID("Tag", "Files", true, true); add_type(id, tag_enum_type, 0, 0); } @@ -42,7 +42,7 @@ void Manager::RegisterAnalyzerComponent(Component* component) { const char* cname = component->CanonicalName(); - if ( tag_enum_type->Lookup("FileAnalysis", cname) != -1 ) + if ( tag_enum_type->Lookup("Files", cname) != -1 ) reporter->FatalError("File Analyzer %s defined more than once", cname); DBG_LOG(DBG_FILE_ANALYSIS, "Registering analyzer %s (tag %s)", @@ -54,7 +54,7 @@ void Manager::RegisterAnalyzerComponent(Component* component) component->Tag().AsEnumVal()->InternalInt(), component)); string id = fmt("ANALYZER_%s", cname); - tag_enum_type->AddName("FileAnalysis", id.c_str(), + tag_enum_type->AddName("Files", id.c_str(), component->Tag().AsEnumVal()->InternalInt(), true); } @@ -76,7 +76,7 @@ void Manager::Terminate() string Manager::HashHandle(const string& handle) const { if ( salt.empty() ) - salt = BifConst::FileAnalysis::salt->CheckString(); + salt = BifConst::Files::salt->CheckString(); char tmp[20]; uint64 hash[2]; @@ -249,7 +249,7 @@ File* Manager::GetFile(const string& file_id, Connection* conn, rval->UpdateLastActivityTime(); if ( update_conn ) - rval->UpdateConnectionFields(conn); + rval->UpdateConnectionFields(conn, is_orig); } return rval; @@ -351,7 +351,7 @@ void Manager::GetFileHandle(analyzer::Tag tag, Connection* c, bool is_orig) bool Manager::IsDisabled(analyzer::Tag tag) { if ( ! disabled ) - disabled = internal_const_val("FileAnalysis::disable")->AsTableVal(); + disabled = internal_const_val("Files::disable")->AsTableVal(); Val* index = new Val(tag, TYPE_COUNT); Val* yield = disabled->Lookup(index); diff --git a/src/file_analysis/analyzer/data_event/DataEvent.cc b/src/file_analysis/analyzer/data_event/DataEvent.cc index 159c8c19cd..1b04111c44 100644 --- a/src/file_analysis/analyzer/data_event/DataEvent.cc +++ b/src/file_analysis/analyzer/data_event/DataEvent.cc @@ -17,7 +17,7 @@ DataEvent::DataEvent(RecordVal* args, File* file, file_analysis::Analyzer* DataEvent::Instantiate(RecordVal* args, File* file) { - using BifType::Record::FileAnalysis::AnalyzerArgs; + using BifType::Record::Files::AnalyzerArgs; int chunk_off = AnalyzerArgs->FieldOffset("chunk_event"); int stream_off = AnalyzerArgs->FieldOffset("stream_event"); diff --git a/src/file_analysis/analyzer/extract/Extract.cc b/src/file_analysis/analyzer/extract/Extract.cc index cbe176d4ca..ef37425003 100644 --- a/src/file_analysis/analyzer/extract/Extract.cc +++ b/src/file_analysis/analyzer/extract/Extract.cc @@ -29,7 +29,7 @@ Extract::~Extract() file_analysis::Analyzer* Extract::Instantiate(RecordVal* args, File* file) { - using BifType::Record::FileAnalysis::AnalyzerArgs; + using BifType::Record::Files::AnalyzerArgs; Val* v = args->Lookup(AnalyzerArgs->FieldOffset("extract_filename")); if ( ! v ) diff --git a/src/file_analysis/analyzer/hash/events.bif b/src/file_analysis/analyzer/hash/events.bif index b4a8de1c74..e03cbf359a 100644 --- a/src/file_analysis/analyzer/hash/events.bif +++ b/src/file_analysis/analyzer/hash/events.bif @@ -7,6 +7,6 @@ ## ## hash: The result of the hashing. ## -## .. bro:see:: FileAnalysis::add_analyzer FileAnalysis::ANALYZER_MD5 -## FileAnalysis::ANALYZER_SHA1 FileAnalysis::ANALYZER_SHA256 +## .. bro:see:: Files::add_analyzer Files::ANALYZER_MD5 +## Files::ANALYZER_SHA1 Files::ANALYZER_SHA256 event file_hash%(f: fa_file, kind: string, hash: string%); diff --git a/src/file_analysis/file_analysis.bif b/src/file_analysis/file_analysis.bif index 06ae9450dd..b6c80ac800 100644 --- a/src/file_analysis/file_analysis.bif +++ b/src/file_analysis/file_analysis.bif @@ -1,6 +1,6 @@ ##! Internal functions and types used by the file analysis framework. -module FileAnalysis; +module Files; %%{ #include "file_analysis/Manager.h" @@ -8,40 +8,46 @@ module FileAnalysis; type AnalyzerArgs: record; -## :bro:see:`FileAnalysis::set_timeout_interval`. -function FileAnalysis::__set_timeout_interval%(file_id: string, t: interval%): bool +## :bro:see:`Files::set_timeout_interval`. +function Files::__set_timeout_interval%(file_id: string, t: interval%): bool %{ bool result = file_mgr->SetTimeoutInterval(file_id->CheckString(), t); return new Val(result, TYPE_BOOL); %} -## :bro:see:`FileAnalysis::add_analyzer`. -function FileAnalysis::__add_analyzer%(file_id: string, args: any%): bool +## :bro:see:`Files::add_analyzer`. +function Files::__add_analyzer%(file_id: string, args: any%): bool %{ - using BifType::Record::FileAnalysis::AnalyzerArgs; + using BifType::Record::Files::AnalyzerArgs; RecordVal* rv = args->AsRecordVal()->CoerceTo(AnalyzerArgs); bool result = file_mgr->AddAnalyzer(file_id->CheckString(), rv); Unref(rv); return new Val(result, TYPE_BOOL); %} -## :bro:see:`FileAnalysis::remove_analyzer`. -function FileAnalysis::__remove_analyzer%(file_id: string, args: any%): bool +## :bro:see:`Files::remove_analyzer`. +function Files::__remove_analyzer%(file_id: string, args: any%): bool %{ - using BifType::Record::FileAnalysis::AnalyzerArgs; + using BifType::Record::Files::AnalyzerArgs; RecordVal* rv = args->AsRecordVal()->CoerceTo(AnalyzerArgs); bool result = file_mgr->RemoveAnalyzer(file_id->CheckString(), rv); Unref(rv); return new Val(result, TYPE_BOOL); %} -## :bro:see:`FileAnalysis::stop`. -function FileAnalysis::__stop%(file_id: string%): bool +## :bro:see:`Files::stop`. +function Files::__stop%(file_id: string%): bool %{ bool result = file_mgr->IgnoreFile(file_id->CheckString()); return new Val(result, TYPE_BOOL); %} +## :bro:see:`Files::analyzer_name`. +function Files::__analyzer_name%(tag: Files::Tag%) : string + %{ + return new StringVal(file_mgr->GetAnalyzerName(tag->InternalInt())); + %} + module GLOBAL; ## For use within a :bro:see:`get_file_handle` handler to set a unique @@ -58,4 +64,4 @@ function set_file_handle%(handle: string%): any return 0; %} -const FileAnalysis::salt: string; +const Files::salt: string; diff --git a/src/probabilistic/BloomFilter.cc b/src/probabilistic/BloomFilter.cc index db768ed934..23b812269c 100644 --- a/src/probabilistic/BloomFilter.cc +++ b/src/probabilistic/BloomFilter.cc @@ -40,28 +40,15 @@ bool BloomFilter::DoSerialize(SerialInfo* info) const { DO_SERIALIZE(SER_BLOOMFILTER, SerialObj); - if ( ! SERIALIZE(static_cast(hasher->K())) ) - return false; - - return SERIALIZE_STR(hasher->Name().c_str(), hasher->Name().size()); + return hasher->Serialize(info); } bool BloomFilter::DoUnserialize(UnserialInfo* info) { DO_UNSERIALIZE(SerialObj); - uint16 k; - if ( ! UNSERIALIZE(&k) ) - return false; - - const char* name; - if ( ! UNSERIALIZE_STR(&name, 0) ) - return false; - - hasher = Hasher::Create(k, name); - - delete [] name; - return true; + hasher = Hasher::Unserialize(info); + return hasher != 0; } size_t BasicBloomFilter::M(double fp, size_t capacity) diff --git a/src/probabilistic/BloomFilter.h b/src/probabilistic/BloomFilter.h index b6cf18672f..4865ae145c 100644 --- a/src/probabilistic/BloomFilter.h +++ b/src/probabilistic/BloomFilter.h @@ -13,9 +13,6 @@ class CounterVector; /** * The abstract base class for Bloom filters. - * - * At this point we won't let the user choose the hasher, but we might open - * up the interface in the future. */ class BloomFilter : public SerialObj { public: diff --git a/src/probabilistic/Hasher.cc b/src/probabilistic/Hasher.cc index f9ce7bdd6b..17597b9a82 100644 --- a/src/probabilistic/Hasher.cc +++ b/src/probabilistic/Hasher.cc @@ -4,9 +4,59 @@ #include "Hasher.h" #include "digest.h" +#include "Serializer.h" using namespace probabilistic; +bool Hasher::Serialize(SerialInfo* info) const + { + return SerialObj::Serialize(info); + } + +Hasher* Hasher::Unserialize(UnserialInfo* info) + { + return reinterpret_cast(SerialObj::Unserialize(info, SER_HASHER)); + } + +bool Hasher::DoSerialize(SerialInfo* info) const + { + DO_SERIALIZE(SER_HASHER, SerialObj); + + if ( ! SERIALIZE(static_cast(k)) ) + return false; + + return SERIALIZE_STR(name.c_str(), name.size()); + } + +bool Hasher::DoUnserialize(UnserialInfo* info) + { + DO_UNSERIALIZE(SerialObj); + + uint16 serial_k; + if ( ! UNSERIALIZE(&serial_k) ) + return false; + + k = serial_k; + assert(k > 0); + + const char* serial_name; + if ( ! UNSERIALIZE_STR(&serial_name, 0) ) + return false; + + name = serial_name; + delete [] serial_name; + + return true; + } + +Hasher::Hasher(size_t k, const std::string& arg_name) + : k(k) + { + k = k; + name = arg_name; + } + + UHF::UHF(size_t seed, const std::string& extra) : h(compute_seed(seed, extra)) { @@ -40,17 +90,6 @@ size_t UHF::compute_seed(size_t seed, const std::string& extra) return *reinterpret_cast(buf); } -Hasher* Hasher::Create(size_t k, const std::string& name) - { - return new DefaultHasher(k, name); - } - -Hasher::Hasher(size_t k, const std::string& arg_name) - : k(k) - { - name = arg_name; - } - DefaultHasher::DefaultHasher(size_t k, const std::string& name) : Hasher(k, name) { @@ -82,6 +121,27 @@ bool DefaultHasher::Equals(const Hasher* other) const return hash_functions == o->hash_functions; } +IMPLEMENT_SERIAL(DefaultHasher, SER_DEFAULTHASHER) + +bool DefaultHasher::DoSerialize(SerialInfo* info) const + { + DO_SERIALIZE(SER_DEFAULTHASHER, Hasher); + + // Nothing to do here, the base class has all we need serialized already. + return true; + } + +bool DefaultHasher::DoUnserialize(UnserialInfo* info) + { + DO_UNSERIALIZE(Hasher); + + hash_functions.clear(); + for ( size_t i = 0; i < K(); ++i ) + hash_functions.push_back(UHF(i, Name())); + + return true; + } + DoubleHasher::DoubleHasher(size_t k, const std::string& name) : Hasher(k, name), h1(1, name), h2(2, name) { @@ -112,3 +172,23 @@ bool DoubleHasher::Equals(const Hasher* other) const const DoubleHasher* o = static_cast(other); return h1 == o->h1 && h2 == o->h2; } + +IMPLEMENT_SERIAL(DoubleHasher, SER_DOUBLEHASHER) + +bool DoubleHasher::DoSerialize(SerialInfo* info) const + { + DO_SERIALIZE(SER_DOUBLEHASHER, Hasher); + + // Nothing to do here, the base class has all we need serialized already. + return true; + } + +bool DoubleHasher::DoUnserialize(UnserialInfo* info) + { + DO_UNSERIALIZE(Hasher); + + h1 = UHF(1, Name()); + h2 = UHF(2, Name()); + + return true; + } diff --git a/src/probabilistic/Hasher.h b/src/probabilistic/Hasher.h index 9f7d4ae32d..3acd5c5867 100644 --- a/src/probabilistic/Hasher.h +++ b/src/probabilistic/Hasher.h @@ -5,6 +5,7 @@ #include "Hash.h" #include "H3.h" +#include "SerialObj.h" namespace probabilistic { @@ -12,7 +13,7 @@ namespace probabilistic { * Abstract base class for hashers. A hasher creates a family of hash * functions to hash an element *k* times. */ -class Hasher { +class Hasher : public SerialObj { public: typedef hash_t digest; typedef std::vector digest_vector; @@ -63,25 +64,20 @@ public: size_t K() const { return k; } /** - * Returns the hasher's name. TODO: What's this? + * Returns the hasher's name. If not empty, the hasher uses this descriptor + * to seed its *k* hash functions. Otherwise the hasher mixes in the initial + * seed derived from the environment variable `$BRO_SEED`. */ const std::string& Name() const { return name; } - /** - * Constructs the hasher used by the implementation. This hardcodes a - * specific hashing policy. It exists only because the HashingPolicy - * class hierachy is not yet serializable. - * - * @param k The number of hash functions to apply. - * - * @param name The hasher's name. Hashers with the same name should - * provide consistent results. - * - * @return Returns a new hasher instance. - */ - static Hasher* Create(size_t k, const std::string& name); + bool Serialize(SerialInfo* info) const; + static Hasher* Unserialize(UnserialInfo* info); protected: + DECLARE_ABSTRACT_SERIAL(Hasher); + + Hasher() { } + /** * Constructor. * @@ -93,7 +89,7 @@ protected: Hasher(size_t k, const std::string& name); private: - const size_t k; + size_t k; std::string name; }; @@ -113,7 +109,7 @@ public: * seed to compute the seed for t to compute the seed NUL-terminated * string as additional seed. */ - UHF(size_t seed, const std::string& extra = ""); + UHF(size_t seed = 0, const std::string& extra = ""); template Hasher::digest operator()(const T& x) const @@ -175,14 +171,18 @@ public: * * @param name The name of the hasher. */ - DefaultHasher(size_t k, const std::string& name); + DefaultHasher(size_t k, const std::string& name = ""); // Overridden from Hasher. virtual digest_vector Hash(const void* x, size_t n) const /* final */; virtual DefaultHasher* Clone() const /* final */; virtual bool Equals(const Hasher* other) const /* final */; + DECLARE_SERIAL(DefaultHasher); + private: + DefaultHasher() { } + std::vector hash_functions; }; @@ -199,14 +199,18 @@ public: * * @param name The name of the hasher. */ - DoubleHasher(size_t k, const std::string& name); + DoubleHasher(size_t k, const std::string& name = ""); // Overridden from Hasher. virtual digest_vector Hash(const void* x, size_t n) const /* final */; virtual DoubleHasher* Clone() const /* final */; virtual bool Equals(const Hasher* other) const /* final */; + DECLARE_SERIAL(DoubleHasher); + private: + DoubleHasher() { } + UHF h1; UHF h2; }; diff --git a/src/probabilistic/bloom-filter.bif b/src/probabilistic/bloom-filter.bif index c6760f6adf..a3567ad6f7 100644 --- a/src/probabilistic/bloom-filter.bif +++ b/src/probabilistic/bloom-filter.bif @@ -48,7 +48,7 @@ function bloomfilter_basic_init%(fp: double, capacity: count, size_t cells = BasicBloomFilter::M(fp, capacity); size_t optimal_k = BasicBloomFilter::K(cells, capacity); - const Hasher* h = Hasher::Create(optimal_k, name->CheckString()); + const Hasher* h = new DefaultHasher(optimal_k, name->CheckString()); return new BloomFilterVal(new BasicBloomFilter(h, cells)); %} @@ -86,7 +86,7 @@ function bloomfilter_counting_init%(k: count, cells: count, max: count, return 0; } - const Hasher* h = Hasher::Create(k, name->CheckString()); + const Hasher* h = new DefaultHasher(k, name->CheckString()); uint16 width = 1; while ( max >>= 1 ) diff --git a/src/util.cc b/src/util.cc index 23abbacc3f..10a1ad94be 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1593,12 +1593,26 @@ void operator delete[](void* v) #endif +// Being selective of which components of MAGIC_NO_CHECK_BUILTIN are actually +// known to be problematic, but keeping rest of libmagic's builtin checks. +#define DISABLE_LIBMAGIC_BUILTIN_CHECKS ( \ +/* MAGIC_NO_CHECK_COMPRESS | */ \ +/* MAGIC_NO_CHECK_TAR | */ \ +/* MAGIC_NO_CHECK_SOFT | */ \ +/* MAGIC_NO_CHECK_APPTYPE | */ \ +/* MAGIC_NO_CHECK_ELF | */ \ +/* MAGIC_NO_CHECK_TEXT | */ \ + MAGIC_NO_CHECK_CDF | \ + MAGIC_NO_CHECK_TOKENS \ +/* MAGIC_NO_CHECK_ENCODING */ \ +) + void bro_init_magic(magic_t* cookie_ptr, int flags) { if ( ! cookie_ptr || *cookie_ptr ) return; - *cookie_ptr = magic_open(flags|MAGIC_NO_CHECK_TOKENS); + *cookie_ptr = magic_open(flags|DISABLE_LIBMAGIC_BUILTIN_CHECKS); // Use our custom database for mime types, but the default database // from libmagic for the verbose file type. diff --git a/testing/btest/Baseline/core.tunnels.ayiya/http.log b/testing/btest/Baseline/core.tunnels.ayiya/http.log index cd49c4cc89..cc0cf32148 100644 --- a/testing/btest/Baseline/core.tunnels.ayiya/http.log +++ b/testing/btest/Baseline/core.tunnels.ayiya/http.log @@ -3,10 +3,10 @@ #empty_field (empty) #unset_field - #path http -#open 2013-05-21-21-11-20 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1257655301.652206 5OKnoww6xl4 2001:4978:f:4c::2 53382 2001:4860:b002::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre) 0 10102 200 OK - - - (empty) - - - text/html - - - +#open 2013-07-25-21-12-29 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] +1257655301.652206 5OKnoww6xl4 2001:4978:f:4c::2 53382 2001:4860:b002::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre) 0 10102 200 OK - - - (empty) - - - - - meGKu6goEyd text/html 1257655302.514424 5OKnoww6xl4 2001:4978:f:4c::2 53382 2001:4860:b002::68 80 2 GET ipv6.google.com /csi?v=3&s=webhp&action=&tran=undefined&e=17259,19771,21517,21766,21887,22212&ei=BUz2Su7PMJTglQfz3NzCAw&rt=prt.77,xjs.565,ol.645 http://ipv6.google.com/ Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre) 0 0 204 No Content - - - (empty) - - - - - - - 1257655303.603569 5OKnoww6xl4 2001:4978:f:4c::2 53382 2001:4860:b002::68 80 3 GET ipv6.google.com /gen_204?atyp=i&ct=fade&cad=1254&ei=BUz2Su7PMJTglQfz3NzCAw&zx=1257655303600 http://ipv6.google.com/ Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre) 0 0 204 No Content - - - (empty) - - - - - - - -#close 2013-05-21-21-11-20 +#close 2013-07-25-21-12-29 diff --git a/testing/btest/Baseline/core.tunnels.gtp.different_dl_and_ul/http.log b/testing/btest/Baseline/core.tunnels.gtp.different_dl_and_ul/http.log index e88be88763..8f9ac07c96 100644 --- a/testing/btest/Baseline/core.tunnels.gtp.different_dl_and_ul/http.log +++ b/testing/btest/Baseline/core.tunnels.gtp.different_dl_and_ul/http.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path http -#open 2013-05-21-21-11-21 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1333458850.340368 arKYeMETxOg 10.131.17.170 51803 173.199.115.168 80 1 GET cdn.epicgameads.com /ads/flash/728x90_nx8com.swf?clickTAG=http://www.epicgameads.com/ads/bannerclickPage.php?id=e3ubwU6IF&pd=1&adid=0&icpc=1&axid=0&uctt=1&channel=4&cac=1&t=728x90&cb=1333458879 http://www.epicgameads.com/ads/banneriframe.php?id=e3ubwU6IF&t=728x90&channel=4&cb=1333458905296 Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) 0 31461 200 OK - - - (empty) - - - application/x-shockwave-flash - - - -1333458850.399501 arKYeMETxOg 10.131.17.170 51803 173.199.115.168 80 2 GET cdn.epicgameads.com /ads/flash/728x90_nx8com.swf?clickTAG=http://www.epicgameads.com/ads/bannerclickPage.php?id=e3ubwU6IF&pd=1&adid=0&icpc=1&axid=0&uctt=1&channel=0&cac=1&t=728x90&cb=1333458881 http://www.epicgameads.com/ads/banneriframe.php?id=e3ubwU6IF&t=728x90&cb=1333458920207 Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) 0 31461 200 OK - - - (empty) - - - application/x-shockwave-flash - - - -#close 2013-05-21-21-11-21 +#open 2013-07-25-16-23-41 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] +1333458850.340368 arKYeMETxOg 10.131.17.170 51803 173.199.115.168 80 1 GET cdn.epicgameads.com /ads/flash/728x90_nx8com.swf?clickTAG=http://www.epicgameads.com/ads/bannerclickPage.php?id=e3ubwU6IF&pd=1&adid=0&icpc=1&axid=0&uctt=1&channel=4&cac=1&t=728x90&cb=1333458879 http://www.epicgameads.com/ads/banneriframe.php?id=e3ubwU6IF&t=728x90&channel=4&cb=1333458905296 Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) 0 31461 200 OK - - - (empty) - - - - - 6jqjOyeITn5 application/x-shockwave-flash +1333458850.399501 arKYeMETxOg 10.131.17.170 51803 173.199.115.168 80 2 GET cdn.epicgameads.com /ads/flash/728x90_nx8com.swf?clickTAG=http://www.epicgameads.com/ads/bannerclickPage.php?id=e3ubwU6IF&pd=1&adid=0&icpc=1&axid=0&uctt=1&channel=0&cac=1&t=728x90&cb=1333458881 http://www.epicgameads.com/ads/banneriframe.php?id=e3ubwU6IF&t=728x90&cb=1333458920207 Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) 0 31461 200 OK - - - (empty) - - - - - A0xot7xPc22 application/x-shockwave-flash +#close 2013-07-25-16-23-41 diff --git a/testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log b/testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log index 8f2893caa7..45b88b7813 100644 --- a/testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log +++ b/testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2013-05-21-21-11-22 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1333458850.375568 arKYeMETxOg 10.131.47.185 1923 79.101.110.141 80 1 GET o-o.preferred.telekomrs-beg1.v2.lscache8.c.youtube.com /videoplayback?upn=MTU2MDY5NzQ5OTM0NTI3NDY4NDc&sparams=algorithm,burst,cp,factor,id,ip,ipbits,itag,source,upn,expire&fexp=912300,907210&algorithm=throttle-factor&itag=34&ip=212.0.0.0&burst=40&sver=3&signature=832FB1042E20780CFCA77A4DB5EA64AC593E8627.D1166C7E8365732E52DAFD68076DAE0146E0AE01&source=youtube&expire=1333484980&key=yt1&ipbits=8&factor=1.25&cp=U0hSSFRTUl9NSkNOMl9MTVZKOjh5eEN2SG8tZF84&id=ebf1e932d4bd1286&cm2=1 http://s.ytimg.com/yt/swfbin/watch_as3-vflqrJwOA.swf Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko; X-SBLSP) Chrome/17.0.963.83 Safari/535.11 0 56320 206 Partial Content - - - (empty) - - - application/octet-stream - - - -#close 2013-05-21-21-11-22 +#open 2013-07-25-21-12-32 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] +1333458850.375568 arKYeMETxOg 10.131.47.185 1923 79.101.110.141 80 1 GET o-o.preferred.telekomrs-beg1.v2.lscache8.c.youtube.com /videoplayback?upn=MTU2MDY5NzQ5OTM0NTI3NDY4NDc&sparams=algorithm,burst,cp,factor,id,ip,ipbits,itag,source,upn,expire&fexp=912300,907210&algorithm=throttle-factor&itag=34&ip=212.0.0.0&burst=40&sver=3&signature=832FB1042E20780CFCA77A4DB5EA64AC593E8627.D1166C7E8365732E52DAFD68076DAE0146E0AE01&source=youtube&expire=1333484980&key=yt1&ipbits=8&factor=1.25&cp=U0hSSFRTUl9NSkNOMl9MTVZKOjh5eEN2SG8tZF84&id=ebf1e932d4bd1286&cm2=1 http://s.ytimg.com/yt/swfbin/watch_as3-vflqrJwOA.swf Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko; X-SBLSP) Chrome/17.0.963.83 Safari/535.11 0 56320 206 Partial Content - - - (empty) - - - - - oypNlaRdgs7 application/octet-stream +#close 2013-07-25-21-12-32 diff --git a/testing/btest/Baseline/core.tunnels.teredo/http.log b/testing/btest/Baseline/core.tunnels.teredo/http.log index 4e3cdfd61d..1ecf0884e2 100644 --- a/testing/btest/Baseline/core.tunnels.teredo/http.log +++ b/testing/btest/Baseline/core.tunnels.teredo/http.log @@ -3,11 +3,11 @@ #empty_field (empty) #unset_field - #path http -#open 2013-05-21-21-11-21 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1210953057.917183 3PKsZ2Uye21 192.168.2.16 1578 75.126.203.78 80 1 POST download913.avast.com /cgi-bin/iavs4stats.cgi - Syncer/4.80 (av_pro-1169;f) 589 0 204 - - - (empty) - - - text/plain - - - -1210953061.585996 70MGiRM1Qf4 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 6640 200 OK - - - (empty) - - - text/html - - - -1210953073.381474 70MGiRM1Qf4 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 2 GET ipv6.google.com /search?hl=en&q=Wireshark+!&btnG=Google+Search http://ipv6.google.com/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 25119 200 OK - - - (empty) - - - text/html - - - -1210953074.674817 c4Zw9TmAE05 192.168.2.16 1580 67.228.110.120 80 1 GET www.wireshark.org / http://ipv6.google.com/search?hl=en&q=Wireshark+%21&btnG=Google+Search Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 11845 200 OK - - - (empty) - - - application/xml - - - -#close 2013-05-21-21-11-21 +#open 2013-07-25-16-23-17 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] +1210953057.917183 3PKsZ2Uye21 192.168.2.16 1578 75.126.203.78 80 1 POST download913.avast.com /cgi-bin/iavs4stats.cgi - Syncer/4.80 (av_pro-1169;f) 589 0 204 - - - (empty) - - - tZX578lAmo3 text/plain - - +1210953061.585996 70MGiRM1Qf4 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 6640 200 OK - - - (empty) - - - - - nkfWSsPnjX7 text/html +1210953073.381474 70MGiRM1Qf4 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 2 GET ipv6.google.com /search?hl=en&q=Wireshark+!&btnG=Google+Search http://ipv6.google.com/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 25119 200 OK - - - (empty) - - - - - fk5lVax7K37 text/html +1210953074.674817 c4Zw9TmAE05 192.168.2.16 1580 67.228.110.120 80 1 GET www.wireshark.org / http://ipv6.google.com/search?hl=en&q=Wireshark+%21&btnG=Google+Search Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 11845 200 OK - - - (empty) - - - - - 6wF1NFmBUza application/xml +#close 2013-07-25-16-23-17 diff --git a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log index 65ec33186e..0c8c448e30 100644 --- a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log +++ b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path http -#open 2013-05-21-21-11-22 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1340127577.361683 FrJExwHcSal 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 6640 200 OK - - - (empty) - - - text/html - - - -1340127577.379360 FrJExwHcSal 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 2 GET ipv6.google.com /search?hl=en&q=Wireshark+!&btnG=Google+Search http://ipv6.google.com/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 25119 200 OK - - - (empty) - - - text/html - - - -#close 2013-05-21-21-11-22 +#open 2013-07-25-16-22-21 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] +1340127577.361683 FrJExwHcSal 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 1 GET ipv6.google.com / - Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 6640 200 OK - - - (empty) - - - - - RzAMHHXJral text/html +1340127577.379360 FrJExwHcSal 2001:0:4137:9e50:8000:f12a:b9c8:2815 1286 2001:4860:0:2001::68 80 2 GET ipv6.google.com /search?hl=en&q=Wireshark+!&btnG=Google+Search http://ipv6.google.com/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5 0 25119 200 OK - - - (empty) - - - - - vOmb3ToMKRg text/html +#close 2013-07-25-16-22-21 diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 5879c504e2..e65b72a30b 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2013-07-25-17-54-33 +#open 2013-07-29-22-37-52 #fields name #types string scripts/base/init-bare.bro @@ -85,10 +85,12 @@ scripts/base/init-bare.bro scripts/base/frameworks/analyzer/main.bro scripts/base/frameworks/packet-filter/utils.bro build/scripts/base/bif/analyzer.bif.bro - scripts/base/frameworks/file-analysis/__load__.bro - scripts/base/frameworks/file-analysis/main.bro + scripts/base/frameworks/files/__load__.bro + scripts/base/frameworks/files/main.bro build/scripts/base/bif/file_analysis.bif.bro + scripts/base/utils/site.bro + scripts/base/utils/patterns.bro build/scripts/base/bif/__load__.bro scripts/policy/misc/loaded-scripts.bro scripts/base/utils/paths.bro -#close 2013-07-25-17-54-33 +#close 2013-07-29-22-37-52 diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 2a820f4270..dbbf689185 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2013-07-25-17-54-33 +#open 2013-07-29-22-37-53 #fields name #types string scripts/base/init-bare.bro @@ -85,19 +85,24 @@ scripts/base/init-bare.bro scripts/base/frameworks/analyzer/main.bro scripts/base/frameworks/packet-filter/utils.bro build/scripts/base/bif/analyzer.bif.bro - scripts/base/frameworks/file-analysis/__load__.bro - scripts/base/frameworks/file-analysis/main.bro + scripts/base/frameworks/files/__load__.bro + scripts/base/frameworks/files/main.bro build/scripts/base/bif/file_analysis.bif.bro + scripts/base/utils/site.bro + scripts/base/utils/patterns.bro build/scripts/base/bif/__load__.bro scripts/base/init-default.bro - scripts/base/utils/site.bro - scripts/base/utils/patterns.bro + scripts/base/utils/active-http.bro + scripts/base/utils/exec.bro scripts/base/utils/addrs.bro scripts/base/utils/conn-ids.bro + scripts/base/utils/dir.bro + scripts/base/frameworks/reporter/__load__.bro + scripts/base/frameworks/reporter/main.bro + scripts/base/utils/paths.bro scripts/base/utils/directions-and-hosts.bro scripts/base/utils/files.bro scripts/base/utils/numbers.bro - scripts/base/utils/paths.bro scripts/base/utils/queue.bro scripts/base/utils/strings.bro scripts/base/utils/thresholds.bro @@ -131,8 +136,6 @@ scripts/base/init-default.bro scripts/base/frameworks/intel/__load__.bro scripts/base/frameworks/intel/main.bro scripts/base/frameworks/intel/input.bro - scripts/base/frameworks/reporter/__load__.bro - scripts/base/frameworks/reporter/main.bro scripts/base/frameworks/sumstats/__load__.bro scripts/base/frameworks/sumstats/main.bro scripts/base/frameworks/sumstats/plugins/__load__.bro @@ -159,8 +162,8 @@ scripts/base/init-default.bro scripts/base/protocols/ftp/__load__.bro scripts/base/protocols/ftp/utils-commands.bro scripts/base/protocols/ftp/main.bro - scripts/base/protocols/ftp/file-analysis.bro - scripts/base/protocols/ftp/file-extract.bro + scripts/base/protocols/ftp/utils.bro + scripts/base/protocols/ftp/files.bro scripts/base/protocols/ftp/gridftp.bro scripts/base/protocols/ssl/__load__.bro scripts/base/protocols/ssl/consts.bro @@ -168,15 +171,13 @@ scripts/base/init-default.bro scripts/base/protocols/ssl/mozilla-ca-list.bro scripts/base/protocols/http/__load__.bro scripts/base/protocols/http/main.bro + scripts/base/protocols/http/entities.bro scripts/base/protocols/http/utils.bro - scripts/base/protocols/http/file-analysis.bro - scripts/base/protocols/http/file-ident.bro - scripts/base/protocols/http/file-hash.bro - scripts/base/protocols/http/file-extract.bro + scripts/base/protocols/http/files.bro scripts/base/protocols/irc/__load__.bro scripts/base/protocols/irc/main.bro scripts/base/protocols/irc/dcc-send.bro - scripts/base/protocols/irc/file-analysis.bro + scripts/base/protocols/irc/files.bro scripts/base/protocols/modbus/__load__.bro scripts/base/protocols/modbus/consts.bro scripts/base/protocols/modbus/main.bro @@ -184,8 +185,7 @@ scripts/base/init-default.bro scripts/base/protocols/smtp/__load__.bro scripts/base/protocols/smtp/main.bro scripts/base/protocols/smtp/entities.bro - scripts/base/protocols/smtp/entities-excerpt.bro - scripts/base/protocols/smtp/file-analysis.bro + scripts/base/protocols/smtp/files.bro scripts/base/protocols/socks/__load__.bro scripts/base/protocols/socks/consts.bro scripts/base/protocols/socks/main.bro @@ -195,6 +195,10 @@ scripts/base/init-default.bro scripts/base/protocols/syslog/consts.bro scripts/base/protocols/syslog/main.bro scripts/base/protocols/tunnels/__load__.bro + scripts/base/files/hash/__load__.bro + scripts/base/files/hash/main.bro + scripts/base/files/extract/__load__.bro + scripts/base/files/extract/main.bro scripts/base/misc/find-checksum-offloading.bro scripts/policy/misc/loaded-scripts.bro -#close 2013-07-25-17-54-33 +#close 2013-07-29-22-37-53 diff --git a/testing/btest/Baseline/istate.events-ssl/receiver.http.log b/testing/btest/Baseline/istate.events-ssl/receiver.http.log index be7e6e5692..dd61de5424 100644 --- a/testing/btest/Baseline/istate.events-ssl/receiver.http.log +++ b/testing/btest/Baseline/istate.events-ssl/receiver.http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2013-05-21-21-11-32 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1369170691.550143 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - - -#close 2013-05-21-21-11-33 +#open 2013-07-25-21-10-36 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] +1374786635.573905 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - - +#close 2013-07-25-21-10-37 diff --git a/testing/btest/Baseline/istate.events-ssl/sender.http.log b/testing/btest/Baseline/istate.events-ssl/sender.http.log index be7e6e5692..dd61de5424 100644 --- a/testing/btest/Baseline/istate.events-ssl/sender.http.log +++ b/testing/btest/Baseline/istate.events-ssl/sender.http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2013-05-21-21-11-32 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1369170691.550143 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - - -#close 2013-05-21-21-11-33 +#open 2013-07-25-21-10-36 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] +1374786635.573905 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - - +#close 2013-07-25-21-10-37 diff --git a/testing/btest/Baseline/istate.events/receiver.http.log b/testing/btest/Baseline/istate.events/receiver.http.log index ae693399c3..aebe4dea7b 100644 --- a/testing/btest/Baseline/istate.events/receiver.http.log +++ b/testing/btest/Baseline/istate.events/receiver.http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2013-05-21-21-11-40 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1369170699.511968 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - - -#close 2013-05-21-21-11-41 +#open 2013-07-25-20-26-59 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] +1374784018.898860 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - - +#close 2013-07-25-20-27-00 diff --git a/testing/btest/Baseline/istate.events/sender.http.log b/testing/btest/Baseline/istate.events/sender.http.log index ae693399c3..b70ba733bd 100644 --- a/testing/btest/Baseline/istate.events/sender.http.log +++ b/testing/btest/Baseline/istate.events/sender.http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2013-05-21-21-11-40 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1369170699.511968 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - - -#close 2013-05-21-21-11-41 +#open 2013-07-25-21-05-37 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] +1374786336.338273 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - - - +#close 2013-07-25-21-05-38 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out index ddc3449a4c..cbd60840bf 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out @@ -4,6 +4,7 @@ FILE_BOF_BUFFER ^J0.26 | 201 MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION file_stream, file #0, 1500, ^J0.26 | 2012-08-24 15:10:04 -0700^J^J * Fixing update-changes, which could pick the wrong control file. (Robin Sommer)^J^J * Fixing GPG signing script. (Robin Sommer)^J^J0.25 | 2012-08-01 13:55:46 -0500^J^J * Fix configure script to exit with non-zero status on error (Jon Siwek)^J^J0.24 | 2012-07-05 12:50:43 -0700^J^J * Raise minimum required CMake version to 2.6.3 (Jon Siwek)^J^J * Adding script to delete old fully-merged branches. (Robin Sommer)^J^J0.23-2 | 2012-01-25 13:24:01 -0800^J^J * Fix a bro-cut error message. (Daniel Thayer)^J^J0.23 | 2012-01-11 12:16:11 -0800^J^J * Tweaks to release scripts, plus a new one for signing files.^J (Robin Sommer)^J^J0.22 | 2012-01-10 16:45:19 -0800^J^J * Tweaks for OpenBSD support. (Jon Siwek)^J^J * bro-cut extensions and fixes. (Robin Sommer)^J ^J - If no field names are given on the command line, we now pass through^J all fields. Adresses #657.^J^J - Removing some GNUism from awk script. Addresses #653.^J^J - Added option for time output in UTC. Addresses #668.^J^J - Added output field separator option -F. Addresses #649.^J^J - Fixing option -c: only some header lines were passed through^J rather than all. (Robin Sommer)^J^J * Fix parallel make portability. (Jon Siwek)^J^J0.21-9 | 2011-11-07 05:44:14 -0800^J^J * Fixing compiler warnings. Addresses #388. (Jon Siwek)^J^J0.21-2 | 2011-11-02 18:12:13 -0700^J^J * Fix for misnaming temp file in update-changes script. (Robin Sommer)^J^J0.21-1 | 2011-11-02 18:10:39 -0700^J^J * Little fix for make-relea file_chunk, file #0, 1500, 0, ^J0.26 | 2012-08-24 15:10:04 -0700^J^J * Fixing update-changes, which could pick the wrong control file. (Robin Sommer)^J^J * Fixing GPG signing script. (Robin Sommer)^J^J0.25 | 2012-08-01 13:55:46 -0500^J^J * Fix configure script to exit with non-zero status on error (Jon Siwek)^J^J0.24 | 2012-07-05 12:50:43 -0700^J^J * Raise minimum required CMake version to 2.6.3 (Jon Siwek)^J^J * Adding script to delete old fully-merged branches. (Robin Sommer)^J^J0.23-2 | 2012-01-25 13:24:01 -0800^J^J * Fix a bro-cut error message. (Daniel Thayer)^J^J0.23 | 2012-01-11 12:16:11 -0800^J^J * Tweaks to release scripts, plus a new one for signing files.^J (Robin Sommer)^J^J0.22 | 2012-01-10 16:45:19 -0800^J^J * Tweaks for OpenBSD support. (Jon Siwek)^J^J * bro-cut extensions and fixes. (Robin Sommer)^J ^J - If no field names are given on the command line, we now pass through^J all fields. Adresses #657.^J^J - Removing some GNUism from awk script. Addresses #653.^J^J - Added option for time output in UTC. Addresses #668.^J^J - Added output field separator option -F. Addresses #649.^J^J - Fixing option -c: only some header lines were passed through^J rather than all. (Robin Sommer)^J^J * Fix parallel make portability. (Jon Siwek)^J^J0.21-9 | 2011-11-07 05:44:14 -0800^J^J * Fixing compiler warnings. Addresses #388. (Jon Siwek)^J^J0.21-2 | 2011-11-02 18:12:13 -0700^J^J * Fix for misnaming temp file in update-changes script. (Robin Sommer)^J^J0.21-1 | 2011-11-02 18:10:39 -0700^J^J * Little fix for make-relea file_stream, file #0, 1024, se script, which could pick out the wrong^J tag. (Robin Sommer)^J^J0.21 | 2011-10-27 17:40:45 -0700^J^J * Fixing bro-cut's usage message and argument error handling. (Robin Sommer)^J^J * Bugfix in update-changes script. (Robin Sommer)^J^J * update-changes now ignores commits it did itself. (Robin Sommer)^J^J * Fix a bug in the update-changes script. (Robin Sommer)^J^J * bro-cut now always installs to $prefix/bin by `make install`. (Jon Siwek)^J^J * Options to adjust time format for bro-cut. (Robin Sommer)^J^J The default with -d is now ISO format. The new option "-D "^J specifies a custom strftime()-style format string. Alternatively,^J the environment variable BRO_CUT_TIMEFMT can set the format as^J well.^J^J * bro-cut now understands the field separator header. (Robin Sommer)^J^J * Renaming options -h/-H -> -c/-C, and doing some general cleanup.^J^J0.2 | 2011-10-25 19:53:57 -0700^J^J * Adding support for replacing version string in a setup.py. (Robin^J Sommer)^J^J * Change generated root cert DN indices f diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out index 4b572d5df9..eb62690f91 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out @@ -4,6 +4,7 @@ FILE_BOF_BUFFER ^J0.26 | 201 MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 4705, 0 [orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp] diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.set_timeout_interval/bro..stdout b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.set_timeout_interval/bro..stdout index 160a51a543..e78f5c8c17 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.set_timeout_interval/bro..stdout +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.set_timeout_interval/bro..stdout @@ -2,6 +2,7 @@ FILE_NEW file #0, 0, 0 MIME_TYPE application/x-dosexec +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 1022920, 0 [orig_h=192.168.72.14, orig_p=3254/tcp, resp_h=65.54.95.206, resp_p=80/tcp] @@ -11,6 +12,7 @@ FILE_NEW file #1, 0, 0 MIME_TYPE application/octet-stream +FILE_OVER_NEW_CONNECTION FILE_TIMEOUT FILE_TIMEOUT FILE_STATE_REMOVE diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.stop/get.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.stop/get.out index f7182027aa..13cfe5de58 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.stop/get.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.stop/get.out @@ -4,3 +4,4 @@ FILE_BOF_BUFFER ^J0.26 | 201 MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out index 4463db6958..eba43b94a4 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out @@ -4,6 +4,7 @@ FILE_BOF_BUFFER The Nationa MIME_TYPE text/x-pascal +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 16557, 0 [orig_h=141.142.228.5, orig_p=50737/tcp, resp_h=141.142.192.162, resp_p=38141/tcp] diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out index 2b46d02042..d42db4b90a 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out @@ -4,6 +4,7 @@ FILE_BOF_BUFFER {^J "origin MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 197, 0 [orig_h=141.142.228.5, orig_p=50153/tcp, resp_h=54.243.118.187, resp_p=80/tcp] diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out index bb2f622969..219aad4eff 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out @@ -4,6 +4,7 @@ FILE_BOF_BUFFER ^J0.26 | 201 MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 4705, 0 [orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp] diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/out index 4b6fa76c0c..da42f4fd68 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/out @@ -4,6 +4,7 @@ FILE_BOF_BUFFER test^M^J MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 6, 0 [orig_h=141.142.228.5, orig_p=57262/tcp, resp_h=54.243.88.146, resp_p=80/tcp] @@ -17,6 +18,7 @@ FILE_BOF_BUFFER test2^M^J MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #1, 7, 0 [orig_h=141.142.228.5, orig_p=57262/tcp, resp_h=54.243.88.146, resp_p=80/tcp] @@ -30,6 +32,7 @@ FILE_BOF_BUFFER test3^M^J MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #2, 7, 0 [orig_h=141.142.228.5, orig_p=57262/tcp, resp_h=54.243.88.146, resp_p=80/tcp] @@ -43,6 +46,7 @@ FILE_BOF_BUFFER {^J "data": MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #3, 465, 0 [orig_h=141.142.228.5, orig_p=57262/tcp, resp_h=54.243.88.146, resp_p=80/tcp] diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out index f8f2538e92..077fb5282c 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out @@ -3,6 +3,7 @@ file #0, 0, 0 MIME_TYPE application/pdf FILE_OVER_NEW_CONNECTION +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 555523, 0 [orig_h=10.101.84.70, orig_p=10978/tcp, resp_h=129.174.93.161, resp_p=80/tcp] diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out index b2a0cb66a2..9c05f311f3 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out @@ -2,6 +2,7 @@ FILE_NEW file #0, 0, 0 MIME_TYPE application/x-dosexec +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 1022920, 0 [orig_h=192.168.72.14, orig_p=3254/tcp, resp_h=65.54.95.206, resp_p=80/tcp] @@ -11,6 +12,7 @@ FILE_NEW file #1, 0, 0 MIME_TYPE application/octet-stream +FILE_OVER_NEW_CONNECTION FILE_TIMEOUT FILE_STATE_REMOVE file #1, 206024, 0 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out index 7c5e9dfeca..d85a9de314 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out @@ -3,6 +3,7 @@ file #0, 0, 0 MIME_TYPE application/octet-stream FILE_OVER_NEW_CONNECTION +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 498702, 0 [orig_h=10.45.179.94, orig_p=19950/tcp, resp_h=129.174.93.170, resp_p=80/tcp] diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out index 02ac2f0a7e..b85485cd1a 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out @@ -4,6 +4,7 @@ FILE_BOF_BUFFER /*^J******** MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 2675, 0 [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] @@ -17,6 +18,7 @@ FILE_BOF_BUFFER //-- Google MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #1, 21421, 0 [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] @@ -30,6 +32,7 @@ FILE_BOF_BUFFER GIF89a^D\0^D\0\xb3 MIME_TYPE image/gif +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #2, 94, 0 [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] @@ -44,6 +47,7 @@ FILE_BOF_BUFFER \x89PNG^M^J^Z^J\0\0\0 MIME_TYPE image/png +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #3, 2349, 0 [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] @@ -58,6 +62,7 @@ FILE_BOF_BUFFER \x89PNG^M^J^Z^J\0\0\0 MIME_TYPE image/png +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #4, 27579, 0 [orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp] diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out index 3103ecb39e..cedc396254 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out @@ -4,6 +4,7 @@ FILE_BOF_BUFFER hello world MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 11, 0 [orig_h=141.142.228.5, orig_p=53595/tcp, resp_h=54.243.55.129, resp_p=80/tcp] @@ -18,6 +19,7 @@ FILE_BOF_BUFFER {^J "origin MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #1, 366, 0 [orig_h=141.142.228.5, orig_p=53595/tcp, resp_h=54.243.55.129, resp_p=80/tcp] diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out index 36da7bdeed..fcd30b2253 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out @@ -4,6 +4,21 @@ FILE_BOF_BUFFER PK^C^D^T\0\0\0^H\0\xae MIME_TYPE application/zip +FILE_OVER_NEW_CONNECTION +FILE_NEW +file #1, 0, 0 +FILE_BOF_BUFFER +\0\0^Ex\0\0^J\xf0\0\0^P +MIME_TYPE +application/octet-stream +FILE_OVER_NEW_CONNECTION +FILE_STATE_REMOVE +file #1, 124, 0 +[orig_h=192.168.1.77, orig_p=57655/tcp, resp_h=209.197.168.151, resp_p=1024/tcp] +source: IRC_DATA +MD5: 35288fd50a74c7d675909ff83424d7a1 +SHA1: 8a98f177cb47e6bf771bf57c2f7e94c4b5e79ffa +SHA256: b24dde52b933a0d76e885ab418cb6d697b14a4e2fef45fce66e12ecc5a6a81aa FILE_STATE_REMOVE file #0, 42208, 0 [orig_h=192.168.1.77, orig_p=57655/tcp, resp_h=209.197.168.151, resp_p=1024/tcp] diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/file_analysis.log b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/file_analysis.log deleted file mode 100644 index f95a70d50a..0000000000 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/file_analysis.log +++ /dev/null @@ -1,10 +0,0 @@ -#separator \x09 -#set_separator , -#empty_field (empty) -#unset_field - -#path file_analysis -#open 2013-06-07-18-51-45 -#fields id parent_id source is_orig last_active seen_bytes total_bytes missing_bytes overflow_bytes timeout_interval bof_buffer_size mime_type timedout conn_uids extracted_files md5 sha1 sha256 -#types string string string bool time count count count count interval count string bool table[string] table[string] string string string -BYYd1GSNX5c - HTTP F 1362692527.009775 4705 4705 0 0 120.000000 1024 text/plain F UWkUyAuUGXf BYYd1GSNX5c-file 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18 -#close 2013-06-07-18-51-46 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/files.log b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/files.log new file mode 100644 index 0000000000..2663184b88 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/files.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path files +#open 2013-07-25-16-57-31 +#fields ts fuid tx_hosts rx_hosts conn_uids source depth analyzers mime_type filename duration local_orig is_orig seen_bytes total_bytes missing_bytes overflow_bytes timedout parent_fuid md5 sha1 sha256 extracted +#types time string table[addr] table[addr] table[string] string count table[string] string string interval bool bool count count count count bool string string string string string +1362692527.009721 G75mcAsU764 192.150.187.43 141.142.228.5 UWkUyAuUGXf HTTP 0 SHA256,DATA_EVENT,MD5,EXTRACT,SHA1 text/plain - 0.000054 - F 4705 4705 0 0 F - 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18 G75mcAsU764-file +#close 2013-07-25-16-57-31 diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out index ac4e6e50fa..57f1f97b9c 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out @@ -4,6 +4,7 @@ FILE_BOF_BUFFER Hello^M^J^M^J ^M MIME_TYPE text/plain +FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE file #0, 79, 0 [orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp] @@ -17,6 +18,7 @@ FILE_BOF_BUFFER - - - - + + + + @@ -60,13 +60,13 @@ - - - - + + + + # Extent, type='http' -ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files +ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types 1300475168.784020 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 1 GET bits.wikimedia.org /skins-1.5/monobook/main.css http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 1300475168.916018 VW0XPVINV8a 141.142.220.118 49997 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/6/63/Wikipedia-logo.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 1300475168.916183 3PKsZ2Uye21 141.142.220.118 49996 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/b/bb/Wikipedia_wordmark.svg/174px-Wikipedia_wordmark.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http.log b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http.log index 6b7bea88c9..8f9d553d9a 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path http -#open 2013-05-21-21-11-23 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] +#open 2013-07-23-05-48-35 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] 1300475168.784020 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 1 GET bits.wikimedia.org /skins-1.5/monobook/main.css http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - 1300475168.916018 VW0XPVINV8a 141.142.220.118 49997 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/6/63/Wikipedia-logo.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - 1300475168.916183 3PKsZ2Uye21 141.142.220.118 49996 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/b/bb/Wikipedia_wordmark.svg/174px-Wikipedia_wordmark.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - @@ -20,4 +20,4 @@ 1300475169.014619 Tw8jXtpTGu6 141.142.220.118 50000 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/4/4a/Commons-logo.svg/35px-Commons-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - 1300475169.014593 P654jzLoe3a 141.142.220.118 49999 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/9/91/Wikiversity-logo.svg/35px-Wikiversity-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - 1300475169.014927 0Q4FH8sESw5 141.142.220.118 50001 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/7/75/Wikimedia_Community_Logo.svg/35px-Wikimedia_Community_Logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - -#close 2013-05-21-21-11-23 +#close 2013-07-23-05-48-35 diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log index e17610d69e..8f91bce3ae 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path notice -#open 2013-04-02-02-21-00 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude -#types time string addr port addr port enum enum string string addr addr port count string table[enum] interval bool string string string double double -1364869260.950557 - - - - - - Test_Notice test notice! - - - - - worker-1 Notice::ACTION_LOG 3600.000000 F - - - - - -#close 2013-04-02-02-21-00 +#open 2013-07-25-20-02-37 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude +#types time string addr port addr port string string string enum enum string string addr addr port count string table[enum] interval bool string string string double double +1374782557.074572 - - - - - - - - - Test_Notice test notice! - - - - - worker-1 Notice::ACTION_LOG 3600.000000 F - - - - - +#close 2013-07-25-20-02-37 diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log index c8b4306d22..0374dadc90 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path notice -#open 2013-04-02-02-21-29 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude -#types time string addr port addr port enum enum string string addr addr port count string table[enum] interval bool string string string double double -1364869289.545369 - - - - - - Test_Notice test notice! - - - - - worker-2 Notice::ACTION_LOG 3600.000000 F - - - - - -#close 2013-04-02-02-21-32 +#open 2013-07-25-20-24-55 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude +#types time string addr port addr port string string string enum enum string string addr addr port count string table[enum] interval bool string string string double double +1374783895.933003 - - - - - - - - - Test_Notice test notice! - - - - - worker-2 Notice::ACTION_LOG 3600.000000 F - - - - - +#close 2013-07-25-20-24-58 diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log index f7f4148548..075bd7ea0a 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path notice -#open 2013-02-11-18-32-39 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude -#types time string addr port addr port enum enum string string addr addr port count string table[enum] interval bool string string string double double -1360607559.193954 - - - - - - Test_Notice test - - - - - bro Notice::ACTION_LOG 3600.000000 F - - - - - -#close 2013-02-11-18-32-39 +#open 2013-07-25-18-56-00 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude +#types time string addr port addr port string string string enum enum string string addr addr port count string table[enum] interval bool string string string double double +1374778560.016355 - - - - - - - - - Test_Notice test - - - - - bro Notice::ACTION_LOG 3600.000000 F - - - - - +#close 2013-07-25-18-56-00 diff --git a/testing/btest/Baseline/scripts.base.protocols.dns.dns-key/dns.log b/testing/btest/Baseline/scripts.base.protocols.dns.dns-key/dns.log new file mode 100644 index 0000000000..722d2c3912 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.dns.dns-key/dns.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path dns +#open 2013-07-25-20-29-44 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto trans_id query qclass qclass_name qtype qtype_name rcode rcode_name AA TC RD RA Z answers TTLs rejected +#types time string addr port addr port enum count string count string count string count string bool bool bool bool count vector[string] vector[interval] bool +1359565680.761790 UWkUyAuUGXf 192.168.6.10 53209 192.168.129.36 53 udp 41477 paypal.com 1 C_INTERNET 48 DNSKEY 0 NOERROR F F T F 1 - - F +#close 2013-07-25-20-29-44 diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/conn.log b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/conn.log deleted file mode 100644 index 52f7d90401..0000000000 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/conn.log +++ /dev/null @@ -1,14 +0,0 @@ -#separator \x09 -#set_separator , -#empty_field (empty) -#unset_field - -#path conn -#open 2013-03-27-17-47-03 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents -#types time string addr port addr port enum string interval count count string bool count string count count count count table[string] -1329843175.736107 arKYeMETxOg 141.142.220.235 37604 199.233.217.249 56666 tcp ftp-data 0.112432 0 342 SF - 0 ShAdfFa 4 216 4 562 (empty) -1329843179.871641 k6kgXLOoSKl 141.142.220.235 59378 199.233.217.249 56667 tcp ftp-data 0.111218 0 77 SF - 0 ShAdfFa 4 216 4 297 (empty) -1329843194.151526 nQcgTWjvg4c 199.233.217.249 61920 141.142.220.235 33582 tcp ftp-data 0.056211 342 0 SF - 0 ShADaFf 5 614 3 164 (empty) -1329843197.783443 j4u32Pc5bif 199.233.217.249 61918 141.142.220.235 37835 tcp ftp-data 0.056005 77 0 SF - 0 ShADaFf 5 349 3 164 (empty) -1329843161.968492 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 tcp ftp 38.055625 180 3146 SF - 0 ShAdDfFa 38 2164 25 4458 (empty) -#close 2013-03-27-17-47-03 diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/extractions b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/extractions deleted file mode 100644 index 1933de9992..0000000000 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/extractions +++ /dev/null @@ -1,22 +0,0 @@ --rw-rw-r-- 1 600 netbsd 50158695 Feb 21 03:10 ls-lRA.gz --rw-rw-r-- 1 600 netbsd 50158695 Feb 21 03:10 ls-lRA.gz --rw-rw-r-- 1 root wheel 77 Aug 16 2009 robots.txt --rw-rw-r-- 1 root wheel 77 Aug 16 2009 robots.txt -Disallow: *.bz2 -Disallow: *.bz2 -Disallow: *.gz -Disallow: *.gz -Disallow: *.tbz -Disallow: *.tbz -Disallow: *.tgz -Disallow: *.tgz -User-agent: * -User-agent: * -drwxr-x--x 3 root wheel 512 Aug 16 2009 etc -drwxr-x--x 3 root wheel 512 Aug 16 2009 etc -drwxr-xr-x 7 root wheel 512 Aug 20 2009 pub -drwxr-xr-x 7 root wheel 512 Aug 20 2009 pub -lrwxrwxr-x 1 root wheel 32 Aug 16 2009 .message -> pub/NetBSD/README.export-control -lrwxrwxr-x 1 root wheel 32 Aug 16 2009 .message -> pub/NetBSD/README.export-control -total 98028 -total 98028 diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp.log b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp.log deleted file mode 100644 index e77f59dc44..0000000000 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-extract/ftp.log +++ /dev/null @@ -1,21 +0,0 @@ -#separator \x09 -#set_separator , -#empty_field (empty) -#unset_field - -#path ftp -#open 2013-06-07-18-57-22 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p user password command arg mime_type file_size reply_code reply_msg tags data_channel.passive data_channel.orig_h data_channel.resp_h data_channel.resp_p extraction_file -#types time string addr port addr port string string string string string count count string table[string] bool addr addr port string -1329843175.680248 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PASV - - - 227 Entering Passive Mode (199,233,217,249,221,90) (empty) T 141.142.220.235 199.233.217.249 56666 - -1329843175.791528 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test LIST - - - 226 Transfer complete. (empty) - - - - - -1329843179.815947 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PASV - - - 227 Entering Passive Mode (199,233,217,249,221,91) (empty) T 141.142.220.235 199.233.217.249 56667 - -1329843193.984222 arKYeMETxOg 141.142.220.235 37604 199.233.217.249 56666 - - - - - - - (empty) - - - - ftp-item-pVhQhhFsB2b.dat -1329843193.984222 k6kgXLOoSKl 141.142.220.235 59378 199.233.217.249 56667 - - - - - - - (empty) - - - - ftp-item-fFCPkV1sEsc.dat -1329843179.926563 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test RETR ftp://199.233.217.249/./robots.txt text/plain 77 226 Transfer complete. (empty) - - - - - -1329843194.040188 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PORT 141,142,220,235,131,46 - - 200 PORT command successful. (empty) F 199.233.217.249 141.142.220.235 33582 - -1329843194.095782 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test LIST - - - 226 Transfer complete. (empty) - - - - - -1329843197.672179 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PORT 141,142,220,235,147,203 - - 200 PORT command successful. (empty) F 199.233.217.249 141.142.220.235 37835 - -1329843199.968212 nQcgTWjvg4c 199.233.217.249 61920 141.142.220.235 33582 - - - - - - - (empty) - - - - ftp-item-g3zS3MuJFh.dat -1329843197.727769 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test RETR ftp://199.233.217.249/./robots.txt text/plain 77 226 Transfer complete. (empty) - - - - - -1329843200.079930 j4u32Pc5bif 199.233.217.249 61918 141.142.220.235 37835 - - - - - - - (empty) - - - - ftp-item-lMf4UWRkEO5.dat -#close 2013-06-07-18-57-22 diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/ftp.log b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/ftp.log index afa4c97830..4cc6d67761 100644 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/ftp.log +++ b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/ftp.log @@ -3,13 +3,13 @@ #empty_field (empty) #unset_field - #path ftp -#open 2013-04-12-16-32-25 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p user password command arg mime_type file_size reply_code reply_msg tags data_channel.passive data_channel.orig_h data_channel.resp_h data_channel.resp_p extraction_file -#types time string addr port addr port string string string string string count count string table[string] bool addr addr port string -1329843175.680248 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PASV - - - 227 Entering Passive Mode (199,233,217,249,221,90) (empty) T 141.142.220.235 199.233.217.249 56666 - -1329843179.815947 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PASV - - - 227 Entering Passive Mode (199,233,217,249,221,91) (empty) T 141.142.220.235 199.233.217.249 56667 - -1329843179.926563 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test RETR ftp://199.233.217.249/./robots.txt text/plain 77 226 Transfer complete. (empty) - - - - - -1329843194.040188 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PORT 141,142,220,235,131,46 - - 200 PORT command successful. (empty) F 199.233.217.249 141.142.220.235 33582 - -1329843197.672179 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PORT 141,142,220,235,147,203 - - 200 PORT command successful. (empty) F 199.233.217.249 141.142.220.235 37835 - -1329843197.727769 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test RETR ftp://199.233.217.249/./robots.txt text/plain 77 226 Transfer complete. (empty) - - - - - -#close 2013-04-12-16-32-25 +#open 2013-07-27-01-49-02 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p user password command arg mime_type file_size reply_code reply_msg data_channel.passive data_channel.orig_h data_channel.resp_h data_channel.resp_p fuid +#types time string addr port addr port string string string string string count count string bool addr addr port string +1329843175.680248 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PASV - - - 227 Entering Passive Mode (199,233,217,249,221,90) T 141.142.220.235 199.233.217.249 56666 - +1329843179.815947 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PASV - - - 227 Entering Passive Mode (199,233,217,249,221,91) T 141.142.220.235 199.233.217.249 56667 - +1329843179.926563 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test RETR ftp://199.233.217.249/./robots.txt text/plain 77 226 Transfer complete. - - - - 4VAnSiNGSQh +1329843194.040188 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PORT 141,142,220,235,131,46 - - 200 PORT command successful. F 199.233.217.249 141.142.220.235 33582 4VAnSiNGSQh +1329843197.672179 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test PORT 141,142,220,235,147,203 - - 200 PORT command successful. F 199.233.217.249 141.142.220.235 37835 4VAnSiNGSQh +1329843197.727769 UWkUyAuUGXf 141.142.220.235 50003 199.233.217.249 21 anonymous test RETR ftp://199.233.217.249/./robots.txt text/plain 77 226 Transfer complete. - - - - aJg8mtdsS86 +#close 2013-07-27-01-49-02 diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/ftp.log b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/ftp.log index 85207806c4..d6f57bcf45 100644 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/ftp.log +++ b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/ftp.log @@ -3,14 +3,14 @@ #empty_field (empty) #unset_field - #path ftp -#open 2013-04-12-16-32-25 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p user password command arg mime_type file_size reply_code reply_msg tags data_channel.passive data_channel.orig_h data_channel.resp_h data_channel.resp_p extraction_file -#types time string addr port addr port string string string string string count count string table[string] bool addr addr port string -1329327783.207785 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPSV - - - 229 Entering Extended Passive Mode (|||57086|) (empty) T 2001:470:1f11:81f:c999:d94:aa7c:2e3e 2001:470:4867:99::21 57086 - -1329327786.415755 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPSV - - - 229 Entering Extended Passive Mode (|||57087|) (empty) T 2001:470:1f11:81f:c999:d94:aa7c:2e3e 2001:470:4867:99::21 57087 - -1329327787.180814 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPSV - - - 229 Entering Extended Passive Mode (|||57088|) (empty) T 2001:470:1f11:81f:c999:d94:aa7c:2e3e 2001:470:4867:99::21 57088 - -1329327787.396984 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test RETR ftp://[2001:470:4867:99::21]/robots.txt - 77 226 Transfer complete. (empty) - - - - - -1329327795.355248 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPRT |2|2001:470:1f11:81f:c999:d94:aa7c:2e3e|49189| - - 200 EPRT command successful. (empty) F 2001:470:4867:99::21 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49189 - -1329327795.463946 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test RETR ftp://[2001:470:4867:99::21]/robots.txt - 77 226 Transfer complete. (empty) - - - - - -1329327799.799327 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPRT |2|2001:470:1f11:81f:c999:d94:aa7c:2e3e|49190| - - 200 EPRT command successful. (empty) F 2001:470:4867:99::21 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49190 - -#close 2013-04-12-16-32-25 +#open 2013-07-27-01-49-13 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p user password command arg mime_type file_size reply_code reply_msg data_channel.passive data_channel.orig_h data_channel.resp_h data_channel.resp_p fuid +#types time string addr port addr port string string string string string count count string bool addr addr port string +1329327783.207785 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPSV - - - 229 Entering Extended Passive Mode (|||57086|) T 2001:470:1f11:81f:c999:d94:aa7c:2e3e 2001:470:4867:99::21 57086 - +1329327786.415755 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPSV - - - 229 Entering Extended Passive Mode (|||57087|) T 2001:470:1f11:81f:c999:d94:aa7c:2e3e 2001:470:4867:99::21 57087 - +1329327787.180814 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPSV - - - 229 Entering Extended Passive Mode (|||57088|) T 2001:470:1f11:81f:c999:d94:aa7c:2e3e 2001:470:4867:99::21 57088 - +1329327787.396984 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test RETR ftp://[2001:470:4867:99::21]/robots.txt - 77 226 Transfer complete. - - - - - +1329327795.355248 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPRT |2|2001:470:1f11:81f:c999:d94:aa7c:2e3e|49189| - - 200 EPRT command successful. F 2001:470:4867:99::21 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49189 4YhNtGvCehl +1329327795.463946 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test RETR ftp://[2001:470:4867:99::21]/robots.txt - 77 226 Transfer complete. - - - - 4YhNtGvCehl +1329327799.799327 UWkUyAuUGXf 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49185 2001:470:4867:99::21 21 anonymous test EPRT |2|2001:470:1f11:81f:c999:d94:aa7c:2e3e|49190| - - 200 EPRT command successful. F 2001:470:4867:99::21 2001:470:1f11:81f:c999:d94:aa7c:2e3e 49190 4YhNtGvCehl +#close 2013-07-27-01-49-13 diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log b/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log index 051f1c6266..04c80407f6 100644 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log +++ b/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path notice -#open 2013-04-02-02-19-21 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude -#types time string addr port addr port enum enum string string addr addr port count string table[enum] interval bool string string string double double -1348168976.558309 arKYeMETxOg 192.168.57.103 35391 192.168.57.101 55968 tcp GridFTP::Data_Channel GridFTP data channel over threshold 2 bytes - 192.168.57.103 192.168.57.101 55968 - bro Notice::ACTION_LOG 3600.000000 F - - - - - -#close 2013-04-02-02-19-21 +#open 2013-07-23-05-19-25 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude +#types time string addr port addr port string string string enum enum string string addr addr port count string table[enum] interval bool string string string double double +1348168976.558309 arKYeMETxOg 192.168.57.103 35391 192.168.57.101 55968 - - - tcp GridFTP::Data_Channel GridFTP data channel over threshold 2 bytes - 192.168.57.103 192.168.57.101 55968 - bro Notice::ACTION_LOG 3600.000000 F - - - - - +#close 2013-07-23-05-19-25 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log b/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log index edbee28991..a81c0d4a2d 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2013-05-21-21-11-24 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1237440095.634312 UWkUyAuUGXf 192.168.3.103 54102 128.146.216.51 80 1 POST www.osu.edu / - curl/7.17.1 (i386-apple-darwin8.11.1) libcurl/7.17.1 zlib/1.2.3 2001 60731 200 OK 100 Continue - (empty) - - - text/html - - - -#close 2013-05-21-21-11-24 +#open 2013-07-25-19-39-08 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] +1237440095.634312 UWkUyAuUGXf 192.168.3.103 54102 128.146.216.51 80 1 POST www.osu.edu / - curl/7.17.1 (i386-apple-darwin8.11.1) libcurl/7.17.1 zlib/1.2.3 2001 60731 200 OK 100 Continue - (empty) - - - 8TXBHVmBGD7 text/plain ATGo7hdUXdi text/html +#close 2013-07-25-19-39-08 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http-item.dat b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http-item.dat deleted file mode 100644 index 73c369dd14..0000000000 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http-item.dat +++ /dev/null @@ -1,304 +0,0 @@ - -ICIR - -ICIR
-

-ICIR (The ICSI Center for Internet Research) -is a -non-profit -research institute at -ICSI -in -Berkeley, -California.
-For the three years from 1999 to 2001 we were named -ACIRI, the AT&T Center for Internet Research at ICSI, -and were funded by AT&T.
- -The goals of ICIR are to: -

    -
  • Pursue research on the Internet architecture and related networking issues, -
  • -Participate actively in the research (SIGCOMM and IRTF) and -standards (IETF) communities, -
  • Bridge the gap between the Internet research community and commercial -interests by providing a neutral forum where topics of mutual technical -interest can be addressed. -
-

- -

- -
- - - - - - - - - - -
- -

-People -

- - -		 - -

-Publications -

- - -

-Projects -

- - - -		 - -

Research

-   Transport and Congestion - - -   Traffic and Topology -
    -
  • -IDMaps -(Internet Distance Mapping). -
  • The -Internet Traffic Archive. -
  • -MINC -(Multicast-based Inference of Network-internal Characteristics). -
  • -NIMI -(National Internet Measurement Infrastructure). -
- -

- -Collaborators -

- - - -
-
- -
-

Information for visitors and local users.

-
-Last modified: June 2004. Copyright notice. - -Older versions of this web page, in its ACIRI incarnation.. -
-For more information about this server, mail www@aciri.org. -
-To report unusual activity by any of our hosts, mail abuse@aciri.org. - diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log deleted file mode 100644 index 53b80e5e9e..0000000000 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log +++ /dev/null @@ -1,10 +0,0 @@ -#separator \x09 -#set_separator , -#empty_field (empty) -#unset_field - -#path http -#open 2013-06-07-19-04-27 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1128727435.634189 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - text/html - - http-item-54zlJFqn0x6.dat -#close 2013-06-07-19-04-27 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-methods/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-methods/http.log index 54a75f4697..674e355631 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-methods/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-methods/http.log @@ -3,56 +3,56 @@ #empty_field (empty) #unset_field - #path http -#open 2013-05-21-21-11-25 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1354328870.191989 UWkUyAuUGXf 128.2.6.136 46562 173.194.75.103 80 1 OPTIONS www.google.com * - - 0 962 405 Method Not Allowed - - - (empty) - - - text/html - - - -1354328874.237327 arKYeMETxOg 128.2.6.136 46563 173.194.75.103 80 1 OPTIONS www.google.com HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328874.299063 k6kgXLOoSKl 128.2.6.136 46564 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328874.342591 nQcgTWjvg4c 128.2.6.136 46565 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328874.364020 j4u32Pc5bif 128.2.6.136 46566 173.194.75.103 80 1 GET www.google.com / - - 0 43911 200 OK - - - (empty) - - - text/html - - - -1354328878.470424 TEfuqmmG4bh 128.2.6.136 46567 173.194.75.103 80 1 GET www.google.com / - - 0 43983 200 OK - - - (empty) - - - text/html - - - -1354328882.575456 FrJExwHcSal 128.2.6.136 46568 173.194.75.103 80 1 GET www.google.com /HTTP/1.1 - - 0 1207 403 Forbidden - - - (empty) - - - text/html - - - -1354328882.928027 5OKnoww6xl4 128.2.6.136 46569 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328882.968948 3PKsZ2Uye21 128.2.6.136 46570 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328882.990373 VW0XPVINV8a 128.2.6.136 46571 173.194.75.103 80 1 GET www.google.com / - - 0 43913 200 OK - - - (empty) - - - text/html - - - -1354328887.114613 fRFu0wcOle6 128.2.6.136 46572 173.194.75.103 80 0 - - - - - 0 961 405 Method Not Allowed - - - (empty) - - - text/html - - - -1354328891.161077 qSsw6ESzHV4 128.2.6.136 46573 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328891.204740 iE6yhOq3SF 128.2.6.136 46574 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328891.245592 GSxOnSLghOa 128.2.6.136 46575 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328891.287655 qCaWGmzFtM5 128.2.6.136 46576 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328891.309065 70MGiRM1Qf4 128.2.6.136 46577 173.194.75.103 80 1 CCM_POST www.google.com / - - 0 963 405 Method Not Allowed - - - (empty) - - - text/html - - - -1354328895.355012 h5DsfNtYzi1 128.2.6.136 46578 173.194.75.103 80 1 CCM_POST www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328895.416133 P654jzLoe3a 128.2.6.136 46579 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328895.459490 Tw8jXtpTGu6 128.2.6.136 46580 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328895.480865 c4Zw9TmAE05 128.2.6.136 46581 173.194.75.103 80 1 CCM_POST www.google.com / - - 0 963 405 Method Not Allowed - - - (empty) - - - text/html - - - -1354328899.526682 EAr0uf4mhq 128.2.6.136 46582 173.194.75.103 80 1 CONNECT www.google.com / - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328903.572533 GvmoxJFXdTa 128.2.6.136 46583 173.194.75.103 80 1 CONNECT www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328903.634196 0Q4FH8sESw5 128.2.6.136 46584 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328903.676395 slFea8xwSmb 128.2.6.136 46585 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328903.697693 UfGkYA2HI2g 128.2.6.136 46586 173.194.75.103 80 1 CONNECT www.google.com / - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328907.743696 i2rO3KD1Syg 128.2.6.136 46587 173.194.75.103 80 1 TRACE www.google.com / - - 0 960 405 Method Not Allowed - - - (empty) - - - text/html - - - -1354328911.790590 2cx26uAvUPl 128.2.6.136 46588 173.194.75.103 80 1 TRACE www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328911.853464 BWaU4aSuwkc 128.2.6.136 46589 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328911.897044 10XodEwRycf 128.2.6.136 46590 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328911.918511 zno26fFZkrh 128.2.6.136 46591 173.194.75.103 80 1 TRACE www.google.com / - - 0 960 405 Method Not Allowed - - - (empty) - - - text/html - - - -1354328915.964678 v5rgkJBig5l 128.2.6.136 46592 173.194.75.103 80 1 DELETE www.google.com / - - 0 961 405 Method Not Allowed - - - (empty) - - - text/html - - - -1354328920.010458 eWZCH7OONC1 128.2.6.136 46593 173.194.75.103 80 1 DELETE www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328920.072101 0Pwk3ntf8O3 128.2.6.136 46594 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328920.114526 0HKorjr8Zp7 128.2.6.136 46595 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328920.136714 yC2d6kVg709 128.2.6.136 46596 173.194.75.103 80 1 DELETE www.google.com / - - 0 961 405 Method Not Allowed - - - (empty) - - - text/html - - - -1354328924.183211 VcgagLjnO92 128.2.6.136 46597 173.194.75.103 80 1 PUT www.google.com / - - 0 934 411 Length Required - - - (empty) - - - text/html - - - -1354328924.224567 bdRoHfaPBo3 128.2.6.136 46598 173.194.75.103 80 1 PUT www.google.com /HTTP/1.1 - - 0 934 411 Length Required - - - (empty) - - - text/html - - - -1354328924.287402 zHqb7t7kv28 128.2.6.136 46599 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328924.328257 rrZWoMUQpv8 128.2.6.136 46600 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328924.350343 xNYSS2hJkle 128.2.6.136 46601 173.194.75.103 80 1 PUT www.google.com / - - 0 934 411 Length Required - - - (empty) - - - text/html - - - -1354328924.391728 vMVjlplKKbd 128.2.6.136 46602 173.194.75.103 80 1 POST www.google.com / - - 0 934 411 Length Required - - - (empty) - - - text/html - - - -1354328924.433150 3omNawSNrxj 128.2.6.136 46603 173.194.75.103 80 1 POST www.google.com /HTTP/1.1 - - 0 934 411 Length Required - - - (empty) - - - text/html - - - -1354328924.496732 Rv8AJVfi9Zi 128.2.6.136 46604 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328924.537671 wEyF3OvvcQe 128.2.6.136 46605 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - +#open 2013-07-25-19-41-27 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] +1354328870.191989 UWkUyAuUGXf 128.2.6.136 46562 173.194.75.103 80 1 OPTIONS www.google.com * - - 0 962 405 Method Not Allowed - - - (empty) - - - - - VTrFjxi3V27 text/html +1354328874.237327 arKYeMETxOg 128.2.6.136 46563 173.194.75.103 80 1 OPTIONS www.google.com HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - - - jeoiUX9q8v9 text/html +1354328874.299063 k6kgXLOoSKl 128.2.6.136 46564 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - 6dL7NPgFhil text/html +1354328874.342591 nQcgTWjvg4c 128.2.6.136 46565 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - cix6gzDRCob text/html +1354328874.364020 j4u32Pc5bif 128.2.6.136 46566 173.194.75.103 80 1 GET www.google.com / - - 0 43911 200 OK - - - (empty) - - - - - tCZHDKUkBdi text/html +1354328878.470424 TEfuqmmG4bh 128.2.6.136 46567 173.194.75.103 80 1 GET www.google.com / - - 0 43983 200 OK - - - (empty) - - - - - iVzFNTeQnnc text/html +1354328882.575456 FrJExwHcSal 128.2.6.136 46568 173.194.75.103 80 1 GET www.google.com /HTTP/1.1 - - 0 1207 403 Forbidden - - - (empty) - - - - - boBAqw2JcFi text/html +1354328882.928027 5OKnoww6xl4 128.2.6.136 46569 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - r3w183FJvW3 text/html +1354328882.968948 3PKsZ2Uye21 128.2.6.136 46570 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - bncugeoItlf text/html +1354328882.990373 VW0XPVINV8a 128.2.6.136 46571 173.194.75.103 80 1 GET www.google.com / - - 0 43913 200 OK - - - (empty) - - - - - NkYD5vo8Gy text/html +1354328887.114613 fRFu0wcOle6 128.2.6.136 46572 173.194.75.103 80 0 - - - - - 0 961 405 Method Not Allowed - - - (empty) - - - - - S85THffBTLh text/html +1354328891.161077 qSsw6ESzHV4 128.2.6.136 46573 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - 2m6kUZZS0wd text/html +1354328891.204740 iE6yhOq3SF 128.2.6.136 46574 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - UoqtpOgJZSk text/html +1354328891.245592 GSxOnSLghOa 128.2.6.136 46575 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - mqs8p4wwsS7 text/html +1354328891.287655 qCaWGmzFtM5 128.2.6.136 46576 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - S36eCQJUY5k text/html +1354328891.309065 70MGiRM1Qf4 128.2.6.136 46577 173.194.75.103 80 1 CCM_POST www.google.com / - - 0 963 405 Method Not Allowed - - - (empty) - - - - - LeNRDWYrpS7 text/html +1354328895.355012 h5DsfNtYzi1 128.2.6.136 46578 173.194.75.103 80 1 CCM_POST www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - - - ZwKUASlWzYk text/html +1354328895.416133 P654jzLoe3a 128.2.6.136 46579 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - uj62KNQhsG3 text/html +1354328895.459490 Tw8jXtpTGu6 128.2.6.136 46580 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - taBxWzrYquk text/html +1354328895.480865 c4Zw9TmAE05 128.2.6.136 46581 173.194.75.103 80 1 CCM_POST www.google.com / - - 0 963 405 Method Not Allowed - - - (empty) - - - - - bHBxZULKI0k text/html +1354328899.526682 EAr0uf4mhq 128.2.6.136 46582 173.194.75.103 80 1 CONNECT www.google.com / - - 0 925 400 Bad Request - - - (empty) - - - - - t6k8zHaGZk5 text/html +1354328903.572533 GvmoxJFXdTa 128.2.6.136 46583 173.194.75.103 80 1 CONNECT www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - - - c11un7ZO6nc text/html +1354328903.634196 0Q4FH8sESw5 128.2.6.136 46584 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - iWCHzW5XJWk text/html +1354328903.676395 slFea8xwSmb 128.2.6.136 46585 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - dzvHktkjD9a text/html +1354328903.697693 UfGkYA2HI2g 128.2.6.136 46586 173.194.75.103 80 1 CONNECT www.google.com / - - 0 925 400 Bad Request - - - (empty) - - - - - vEO9iYqh3Zc text/html +1354328907.743696 i2rO3KD1Syg 128.2.6.136 46587 173.194.75.103 80 1 TRACE www.google.com / - - 0 960 405 Method Not Allowed - - - (empty) - - - - - 8seYaeRVuV2 text/html +1354328911.790590 2cx26uAvUPl 128.2.6.136 46588 173.194.75.103 80 1 TRACE www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - - - 0kkHkmLHFl3 text/html +1354328911.853464 BWaU4aSuwkc 128.2.6.136 46589 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - koHEYsvMVBa text/html +1354328911.897044 10XodEwRycf 128.2.6.136 46590 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - 50tlwxQjBCb text/html +1354328911.918511 zno26fFZkrh 128.2.6.136 46591 173.194.75.103 80 1 TRACE www.google.com / - - 0 960 405 Method Not Allowed - - - (empty) - - - - - DdECXqOZjXh text/html +1354328915.964678 v5rgkJBig5l 128.2.6.136 46592 173.194.75.103 80 1 DELETE www.google.com / - - 0 961 405 Method Not Allowed - - - (empty) - - - - - LIZQeBP0Coi text/html +1354328920.010458 eWZCH7OONC1 128.2.6.136 46593 173.194.75.103 80 1 DELETE www.google.com /HTTP/1.1 - - 0 925 400 Bad Request - - - (empty) - - - - - hjPo0BdP973 text/html +1354328920.072101 0Pwk3ntf8O3 128.2.6.136 46594 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - d6K2onvteNa text/html +1354328920.114526 0HKorjr8Zp7 128.2.6.136 46595 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - BY1g634OMv6 text/html +1354328920.136714 yC2d6kVg709 128.2.6.136 46596 173.194.75.103 80 1 DELETE www.google.com / - - 0 961 405 Method Not Allowed - - - (empty) - - - - - 5aAa2m40fZd text/html +1354328924.183211 VcgagLjnO92 128.2.6.136 46597 173.194.75.103 80 1 PUT www.google.com / - - 0 934 411 Length Required - - - (empty) - - - - - y3Syn85ve8e text/html +1354328924.224567 bdRoHfaPBo3 128.2.6.136 46598 173.194.75.103 80 1 PUT www.google.com /HTTP/1.1 - - 0 934 411 Length Required - - - (empty) - - - - - P92nMD5z6D4 text/html +1354328924.287402 zHqb7t7kv28 128.2.6.136 46599 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - qIPObDBIhSj text/html +1354328924.328257 rrZWoMUQpv8 128.2.6.136 46600 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - su86MWxyjne text/html +1354328924.350343 xNYSS2hJkle 128.2.6.136 46601 173.194.75.103 80 1 PUT www.google.com / - - 0 934 411 Length Required - - - (empty) - - - - - r2aysGE6ve8 text/html +1354328924.391728 vMVjlplKKbd 128.2.6.136 46602 173.194.75.103 80 1 POST www.google.com / - - 0 934 411 Length Required - - - (empty) - - - - - Zosv3c0p2Zb text/html +1354328924.433150 3omNawSNrxj 128.2.6.136 46603 173.194.75.103 80 1 POST www.google.com /HTTP/1.1 - - 0 934 411 Length Required - - - (empty) - - - - - L02QmCl2lX4 text/html +1354328924.496732 Rv8AJVfi9Zi 128.2.6.136 46604 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - uh9TwTMdWI9 text/html +1354328924.537671 wEyF3OvvcQe 128.2.6.136 46605 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - 4gLQ9WVkuYd text/html 1354328924.559704 E490YZTUozc 128.2.6.136 46606 173.194.75.103 80 1 HEAD www.google.com / - - 0 0 200 OK - - - (empty) - - - - - - - 1354328928.625437 YIeWJmXWNWj 128.2.6.136 46607 173.194.75.103 80 1 HEAD www.google.com / - - 0 0 200 OK - - - (empty) - - - - - - - 1354328932.692706 ydiZblvsYri 128.2.6.136 46608 173.194.75.103 80 1 HEAD www.google.com /HTTP/1.1 - - 0 0 400 Bad Request - - - (empty) - - - - - - - -1354328932.754657 HFYOnBqSE5e 128.2.6.136 46609 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -1354328932.796568 JcUvhfWUMgd 128.2.6.136 46610 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - text/html - - - -#close 2013-05-21-21-11-25 +1354328932.754657 HFYOnBqSE5e 128.2.6.136 46609 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - NIV5LGdqSk2 text/html +1354328932.796568 JcUvhfWUMgd 128.2.6.136 46610 173.194.75.103 80 0 - - - - - 0 925 400 Bad Request - - - (empty) - - - - - SlC7NZIgx1d text/html +#close 2013-07-25-19-41-27 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log deleted file mode 100644 index 97e797b4fb..0000000000 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log +++ /dev/null @@ -1,14 +0,0 @@ -#separator \x09 -#set_separator , -#empty_field (empty) -#unset_field - -#path http -#open 2013-05-21-21-11-25 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - (empty) - - - text/plain - - - -1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - (empty) - - - text/plain - - - -1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - (empty) - - - image/gif - - - -1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - (empty) - - - image/png e0029eea80812e9a8e57b8d05d52938a - - -1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - (empty) - - - image/png 30aa926344f58019d047e85ba049ca1e - - -#close 2013-05-21-21-11-25 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log index e22fb53103..6779485f91 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log @@ -3,12 +3,12 @@ #empty_field (empty) #unset_field - #path http -#open 2013-05-21-21-11-25 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string vector[string] vector[string] -1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - (empty) - - - - - - -1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - (empty) - - - - - - -1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - (empty) - - - - - - -1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - (empty) - - - - - - -1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - (empty) - - - - - - -#close 2013-05-21-21-11-25 +#open 2013-07-25-19-43-06 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] +1258577884.844956 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 1 GET www.mozilla.org /style/enhanced.css http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2675 200 OK - - - (empty) - - - - - XRu8VItOvLc text/plain +1258577884.960135 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 2 GET www.mozilla.org /script/urchin.js http://www.mozilla.org/projects/calendar/ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 21421 200 OK - - - (empty) - - - - - m1D1wMxW9y8 text/plain +1258577885.317160 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 3 GET www.mozilla.org /images/template/screen/bullet_utility.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 94 200 OK - - - (empty) - - - - - ZwnCaxWANNb image/gif +1258577885.349639 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 4 GET www.mozilla.org /images/template/screen/key-point-top.png http://www.mozilla.org/style/screen.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 2349 200 OK - - - (empty) - - - - - 3WVi9g0Caei image/png +1258577885.394612 UWkUyAuUGXf 192.168.1.104 1673 63.245.209.11 80 5 GET www.mozilla.org /projects/calendar/images/header-sunbird.png http://www.mozilla.org/projects/calendar/calendar.css Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5 0 27579 200 OK - - - (empty) - - - - - ta9bGBff1Wl image/png +#close 2013-07-25-19-43-06 diff --git a/testing/btest/Baseline/scripts.base.protocols.http.multipart-extract/http.log b/testing/btest/Baseline/scripts.base.protocols.http.multipart-extract/http.log index 0bd15badef..ae71680dfa 100644 --- a/testing/btest/Baseline/scripts.base.protocols.http.multipart-extract/http.log +++ b/testing/btest/Baseline/scripts.base.protocols.http.multipart-extract/http.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path http -#open 2013-06-07-19-57-15 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files -#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string vector[string] vector[string] -1369159408.455878 UWkUyAuUGXf 141.142.228.5 57262 54.243.88.146 80 1 POST httpbin.org /post - curl/7.30.0 370 465 200 OK - - - (empty) - - - text/plain - http-item-lcf92jVphSl.dat,http-item-z8gOS6arddh.dat,http-item-tBYz7eElzTb.dat http-item-GVJrSB2Vxk6.dat -#close 2013-06-07-19-57-15 +#open 2013-07-25-19-50-23 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types +#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] vector[string] vector[string] vector[string] vector[string] +1369159408.455878 UWkUyAuUGXf 141.142.228.5 57262 54.243.88.146 80 1 POST httpbin.org /post - curl/7.30.0 370 465 200 OK - - - (empty) - - - UB09X6VFGTd,wFP689pOsIa,g5yDIGBH4i5 text/plain,text/plain,text/plain yv4qm3EsdOc text/plain +#close 2013-07-25-19-50-23 diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log b/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log index 64bdb41861..8249c94938 100644 --- a/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log +++ b/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log @@ -3,11 +3,11 @@ #empty_field (empty) #unset_field - #path irc -#open 2013-03-27-18-51-40 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user command value addl dcc_file_name dcc_file_size extraction_file +#open 2013-07-25-19-51-43 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user command value addl dcc_file_name dcc_file_size fuid #types time string addr port addr port string string string string string string count string 1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - NICK bloed - - - - 1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq - - - 1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje JOIN #easymovies (empty) - - - 1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje DCC #easymovies (empty) ladyvampress-default(2011-07-07)-OS.zip 42208 - -#close 2013-03-27-18-51-40 +#close 2013-07-25-19-51-43 diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc-dcc-item.dat b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc-dcc-item.dat deleted file mode 100644 index d4ec9e374b..0000000000 Binary files a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc-dcc-item.dat and /dev/null differ diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log deleted file mode 100644 index 28ca448e05..0000000000 --- a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log +++ /dev/null @@ -1,13 +0,0 @@ -#separator \x09 -#set_separator , -#empty_field (empty) -#unset_field - -#path irc -#open 2013-06-07-19-08-42 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user command value addl dcc_file_name dcc_file_size dcc_mime_type extraction_file -#types time string addr port addr port string string string string string string count string string -1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - NICK bloed - - - - - -1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq - - - - -1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje JOIN #easymovies (empty) - - - - -1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje DCC #easymovies (empty) ladyvampress-default(2011-07-07)-OS.zip 42208 application/zip irc-dcc-item-A3OSdqG9zvk.dat -#close 2013-06-07-19-08-42 diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log b/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log index ba16578dfb..b56b8afab6 100644 --- a/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path smtp -#open 2009-10-05-06-06-12 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth helo mailfrom rcptto date from to reply_to msg_id in_reply_to subject x_originating_ip first_received second_received last_reply path user_agent -#types time string addr port addr port count string string table[string] string string table[string] string string string string addr string string string vector[addr] string -1254722768.219663 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 GP Mon, 5 Oct 2009 11:36:07 +0530 "Gurpartap Singh" - <000301ca4581$ef9e57f0$cedb07d0$@in> - SMTP - - - 250 OK id=1Mugho-0003Dg-Un 74.53.140.153,10.10.1.4 Microsoft Office Outlook 12.0 -#close 2009-10-05-06-06-16 +#open 2013-07-25-19-52-35 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth helo mailfrom rcptto date from to reply_to msg_id in_reply_to subject x_originating_ip first_received second_received last_reply path user_agent fuids +#types time string addr port addr port count string string table[string] string string table[string] string string string string addr string string string vector[addr] string vector[string] +1254722768.219663 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 GP Mon, 5 Oct 2009 11:36:07 +0530 "Gurpartap Singh" - <000301ca4581$ef9e57f0$cedb07d0$@in> - SMTP - - - 250 OK id=1Mugho-0003Dg-Un 74.53.140.153,10.10.1.4 Microsoft Office Outlook 12.0 A1IqG95k9Tk,VUcocHqaWva,JJPHrvZaGJj +#close 2013-07-25-19-52-35 diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/extractions b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/extractions deleted file mode 100644 index 45d776a8e9..0000000000 --- a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/extractions +++ /dev/null @@ -1,277 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - k6, k6-2, k6-3, athlon, athlon-tbird, athlon-4, athlon-xp, athlon-mp, winchip-c6, winchip2, k8, c3 and c3-2) - "windows.h", he gets all the WinAPI! If he adds "wx/wx.h", he gets all of - #included directly or indirectly)! - (available when right-clicking the class-browser - (still can be overriden by using "-c" command line parameter). - (the same filename as the project's but with extension ".layout"). If you - - Enable use of processor specific built-in functions (mmmx, sse, sse2, pni, 3dnow) - - Generate instructions for a specific machine (i386, i486, i586, i686, pentium, pentium-mmx, pentiumpro, pentium2, pentium3, pentium4, 20 - - Strip executable - -c - 20 - Instead open the file in an already launched Dev-C++. - It used to be a checkbox, allowing only two states (on or off), but there is - The user can define this in the class browser's context menu under "View mode". - Well, it adds caching to code-completion. Depending on the cache size, - a third relevant option now: "Project classes" so it didn't fit the purpose... - and selecting "View mode"). - cause of many errors (although it should be fixed by now), we are giving the - class inheritance and visibility (shows items only from files - code-completion and the user has all the commands (belonging to the files - compiler: -D__DEBUG__ - displayed in the editor when the mouse moves over a word. Since this was the - have your project under CVS control, you ''ll know why this had to happen... - he added in the cache) at his fingertips. If, for example, the user adds - include files can now be generated. - information definitions - it in the private resource) - its counterpart are highlighted - resource) - the program may take a bit longer to start-up, but provides very fast - the requested CVS action - then we even get a stack trace in the bug report! - user the option to disable this feature. - wxWindows! You get the picture... -* "Build priority" per-unit -* "Compile as C++" per-unit -* "Default" button in Compiler Options is back -* "Include file in compilation process" per-unit -* "Include file in linking process" per-unit -* Added "Add Library" button in Project Options -* Added "Classes" toolbar -* Added "External programs" in Tools/Environment Options (for units "Open with") -* Added "Files" tab in CVS form to allow selection of more than one file for -* Added "Open with" in project units context menu -* Added "Tip of the day" system. -* Added *working* function arguments hint -* Added CVS "login" and "logout" commands -* Added CVS commands "Add" and "Remove" -* Added ExceptionsAnalyzer. If the devcpp.map file is in the devcpp.exe directory -* Added bracket highlighting. When the caret is on a bracket, that bracket and -* Added configuration option for "Templates Directory" in "Environment Options" -* Added display of project filename, project output and a summary of the project files in Project Options General tab. -* Added doxygen-style comments in NewClass, NewMemberFunction and NewMemberVariable wizards -* Added file's date/time stamp in File/Properties window -* Added new WebUpdate module (inactive temporarily). -* Added new code for code-completion caching of files (disabled - work in progress). -* Added new compiler/linker options: 20 -* Added new file menu entry: Save Project As -* Added new option in class-browser: Use colors -* Added possibility to include in a Template the Project's directories (include, libs and ressources) -* Added support for GCC > 3.2 -* Added support for macros in the "default source code" (Tools/Editor Options/Code) -* Added support for the "interface" keyword -* Added support for the '::' member access operator in code-completion -* Added the possibility to modify the value of a variable during debugging (right click on a watch variable and select "Modify value") -* Added the possibility to specify an include directory for the code completion cache to be created at Dev-C++ first startup -* Added two new macros: and -* Allow customizing of per-unit compile command in projects -* Allow user to specify an alternate configuration file in Environment Options 20 -* Backtrace in debugging -* Big speed up in function parameters listing while editing -* Bug fixes -* Bug fixes -* Bug fixes -* Bug fixes -* Bug fixes -* Bug fixes -* Bug fixes -* Bug fixes -* Bug fixes -* Bug fixes -* Bug fixes -* Bug fixes -* Bug fixes -* Bug fixes -* Bug fixes -* Bug fixes -* Bug-fix for double quotes in devcpp.cfg file read by vUpdate -* CPU Window (still in development) -* CVS support -* Caching of result set of code-completion for speed-up. -* Changed position of compiler/linker parameters in Project Options. -* Changed tint of Class browser pictures colors to match the New Look style -* Class-parser speed-up (50% to 85% improvement timed!!!) -* Code-completion updates -* Compiler set per-project -* Compiler settings per-project -* Compiling progress window -* Current windows listing in Window menu -* Debug variable browser -* Debug variables are now resent during next debug session -* Dev-C++ now traps access violation of your programs (and of itself too ;) -* During Dev-C++ First Time COnfiguration window, users can now choose between using or not class browser and code completion features. -* Each project creates a _private.h file containing version -* Editor colors are initialized properly on Dev-C++ first-run -* Environment options : "Show progress window" and "Auto-close progress window" -* Error messages parsing improved -* Fixed many class browser bugs, including some that had to do with class folders. -* Fixed pre-compilation dependency checks to work correctly -* Fixed the "compiler-dirs-with-spaces" bug that crept-in in 4.9.7.0 -* Fixed the dreaded "Clock skew detected" compiler warning! -* Folders in Project and Class Browser -* Implemented "compiler sets" infrastructure to switch between different compilers easily (e.g. gcc-2.95 and gcc-3.2) -* Implemented new compiler settings framework -* Implemented search in help files for the word at cursor (context sensitive help) -* Implemented the "File/Export/Project to HTML" function. -* Improved Indent/Unindent and Remove Comment -* Improved WebUpdate module -* Improved automatic indent -* Improved code completion cache -* Improved editor -* Improved help file -* Improved installer -* Lots of bug fixes. -* Lots of bugfixes -* MSVC import now creates the folders structure of the original VC project -* Made whole bottom report control floating instead of only debug output. -* Makefile can now be customized. -* Many bug fixes -* Many bug fixes -* Many bug fixes -* Many bug fixes -* Many bug fixes -* Many bug fixes -* Many bug fixes -* Many code-completion updates. Now takes into account context, -* Modified the behaviour of the -c param : 20 -* Multi-select files in project-view (when "double-click to open" is configured in Environment Settings) -* Necessary UI changes in Project Options -* Nested folders in project view -* New "Abort compilation" button -* New WebUpdater module. -* New class browser option: "Show inherited members" -* New code tooltip display -* New debug feature for DLLs: attach to a running process -* New environment options : "watch variable under mouse" and "Report watch errors" -* New feature: compile current file only -* New option "Execution/Parameters" (and "Debug/Parameters"). -* New option in Editor Options (code-completion): Use code-completion cache. -* New option in Editor Options: Show editor hints. User can disable the hints -* New project option: Use custom Makefile. 20 -* New splash screen and association icons -* Now checks for vRoach existance when sending a crash report -* On Dev-C++ first time configuration dialog, a code completion cache of all the standard 20 -* Other bug fixes -* Possibility of changing compilers and tools filename. -* Printing settings are now saved -* Profiling support -* Project manager and debugging window (in Debug tab) can now be trasnformed into floating windows. -* Project version info (creates the relevant VERSIONINFO struct in the private -* Removed "Only show classes from current file" option in class browser settings. -* Resource errors are now reported in the Resource sheet -* Resource files are treated as ordinary files now -* Run to cursor -* Saving of custom syntax parameter group -* Send custom commands to GDB -* Separated C++ compiler options from C compiler options in Makefile (see bug report #654744) -* Separated C++ include dirs from C include dirs in Makefile (see bug report #654744) -* Separated layout info from project file. It is now kept in a different file -* Support XP Themes (creates the CommonControls 6.0 manifest file and includes -* Support for latest Mingw compiler system builds -* ToDo list -* Under NT, 2000 and XP, user application data directory will be used to store config files (i.e : C:\Documents and Settings\Username\Local Settings\Application Data) -* Updates in "Project Options/Files" code -* Watched Variables not in correct context are now kept and updated when it is needed -* WebUpdate should now report installation problems from PackMan -* WebUpdate will now backup downloaded DevPaks in Dev-C++\Packages directory, and Dev-C++ executable in devcpp.exe.BACKUP -* When adding debugging symbols on request, remove "-s" option from linker -* When compiling the current file only, no dependency checks are performed -* When compiling with debugging symbols, an extra definition is passed to the -* When creating a DLL, the created static lib respects now the project-defined output directory -* When running a source file in explorer, don't spawn new instance. -* Window list (in Window menu) -* XP Theme support -* added ENTER key for opening file in project browser, DEL to delete from the project. -* back to gcc 2.95.3 -* bug fixes -* bug fixes -* new update/packages checker (vUpdate) -* support for DLL application hosting, for debugging and executing DLLs under Dev-C++. -* ~300% Speed-up in class parser -Find the attachment -GPS -Hello -I send u smtp pcap file -Version 4.9.4.1 (5.0 beta 4.1): -Version 4.9.5.0 (5.0 beta 5): -Version 4.9.5.1 -Version 4.9.5.2 -Version 4.9.5.3 -Version 4.9.5.4 -Version 4.9.5.5 -Version 4.9.6.5 -Version 4.9.6.6 -Version 4.9.6.7 -Version 4.9.6.8 -Version 4.9.6.9 -Version 4.9.7.0 -Version 4.9.7.1 -Version 4.9.7.2 -Version 4.9.7.3 -Version 4.9.7.4 -Version 4.9.7.5 -Version 4.9.7.6 -Version 4.9.7.7 -Version 4.9.7.8 -Version 4.9.7.9 -Version 4.9.8.0 -Version 4.9.8.1 -Version 4.9.8.2 -Version 4.9.8.3 -Version 4.9.8.4 -Version 4.9.8.5 -Version 4.9.8.7 -Version 4.9.8.9 -Version 4.9.9.0 -Version 4.9.9.1 -version 4.9.6.1 -version 4.9.6.2 -version 4.9.6.3 -version 4.9.6.4 diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/filecount b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/filecount deleted file mode 100644 index 0cfbf08886..0000000000 --- a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/filecount +++ /dev/null @@ -1 +0,0 @@ -2 diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log deleted file mode 100644 index 865694e8a2..0000000000 --- a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log +++ /dev/null @@ -1,12 +0,0 @@ -#separator \x09 -#set_separator , -#empty_field (empty) -#unset_field - -#path smtp_entities -#open 2013-06-07-19-32-56 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth filename content_len mime_type md5 extraction_file excerpt -#types time string addr port addr port count string count string string string string -1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 79 text/plain - smtp-entity-mR3f2AAKo11.dat (empty) -1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 1918 text/html - - (empty) -1254722770.692804 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 NEWS.txt 10823 text/plain - smtp-entity-ZNp0KBSLByc.dat (empty) -#close 2013-06-07-19-32-56 diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log b/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log deleted file mode 100644 index 135c644855..0000000000 --- a/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log +++ /dev/null @@ -1,12 +0,0 @@ -#separator \x09 -#set_separator , -#empty_field (empty) -#unset_field - -#path smtp_entities -#open 2013-03-26-20-39-07 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth filename content_len mime_type md5 extraction_file excerpt -#types time string addr port addr port count string count string string string string -1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 79 text/plain 92bca2e6cdcde73647125da7dccbdd07 - (empty) -1254722770.692743 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 - 1918 text/html - - (empty) -1254722770.692804 arKYeMETxOg 10.10.1.4 1470 74.53.140.153 25 1 NEWS.txt 10823 text/plain a968bb0f9f9d95835b2e74c845877e87 - (empty) -#close 2013-03-26-20-39-07 diff --git a/testing/btest/Baseline/scripts.base.utils.active-http/bro..stdout b/testing/btest/Baseline/scripts.base.utils.active-http/bro..stdout new file mode 100644 index 0000000000..0284eb19b3 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.utils.active-http/bro..stdout @@ -0,0 +1,5 @@ +[code=200, msg=OK^M, body=It works!, headers={ +[Server] = 1.0, +[Content-type] = text/plain, +[Date] = July 22, 2013 +}] diff --git a/testing/btest/Baseline/scripts.base.utils.dir/bro..stdout b/testing/btest/Baseline/scripts.base.utils.dir/bro..stdout new file mode 100644 index 0000000000..c3103b7f64 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.utils.dir/bro..stdout @@ -0,0 +1,10 @@ +new_file1, ../testdir/bye +new_file1, ../testdir/hi +new_file1, ../testdir/howsitgoing +new_file2, ../testdir/bye +new_file2, ../testdir/hi +new_file2, ../testdir/howsitgoing +new_file1, ../testdir/bye +new_file1, ../testdir/newone +new_file2, ../testdir/bye +new_file2, ../testdir/newone diff --git a/testing/btest/Baseline/scripts.base.utils.exec/bro..stdout b/testing/btest/Baseline/scripts.base.utils.exec/bro..stdout new file mode 100644 index 0000000000..5352d15d18 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.utils.exec/bro..stdout @@ -0,0 +1,7 @@ +test1, [exit_code=0, signal_exit=F, stdout=[done, exit, stop], stderr=, files={ +[out1] = [insert text here, and here], +[out2] = [insert more text here, and there] +}] +test2, [exit_code=1, signal_exit=F, stdout=[here's something on stdout, some more stdout, last stdout], stderr=[and some stderr, more stderr, last stderr], files=] +test3, [exit_code=9, signal_exit=F, stdout=[FML], stderr=, files=] +test4, [exit_code=0, signal_exit=F, stdout=[hibye], stderr=, files=] diff --git a/testing/btest/Baseline/scripts.base.utils.files/output b/testing/btest/Baseline/scripts.base.utils.files/output index ab92c3a624..4d53bcedd3 100644 --- a/testing/btest/Baseline/scripts.base.utils.files/output +++ b/testing/btest/Baseline/scripts.base.utils.files/output @@ -1,3 +1,6 @@ +Economy +US-$ rates +\xa3 rates test-prefix_141.142.220.118:48649-208.80.152.118:80_test-suffix test-prefix_141.142.220.118:48649-208.80.152.118:80 141.142.220.118:48649-208.80.152.118:80_test-suffix diff --git a/testing/btest/Baseline/scripts.policy.frameworks.software.vulnerable/notice.log b/testing/btest/Baseline/scripts.policy.frameworks.software.vulnerable/notice.log index f2cf09cab6..54b04aafae 100644 --- a/testing/btest/Baseline/scripts.policy.frameworks.software.vulnerable/notice.log +++ b/testing/btest/Baseline/scripts.policy.frameworks.software.vulnerable/notice.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path notice -#open 2013-04-28-22-36-26 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude -#types time string addr port addr port enum enum string string addr addr port count string table[enum] interval bool string string string double double -1367188586.649122 - - - - - - Software::Vulnerable_Version 1.2.3.4 is running Java 1.7.0.15 which is vulnerable. Java 1.7.0.15 1.2.3.4 - - - bro Notice::ACTION_LOG 3600.000000 F - - - - - -1367188586.649122 - - - - - - Software::Vulnerable_Version 1.2.3.5 is running Java 1.6.0.43 which is vulnerable. Java 1.6.0.43 1.2.3.5 - - - bro Notice::ACTION_LOG 3600.000000 F - - - - - -#close 2013-04-28-22-36-26 +#open 2013-07-25-19-54-45 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude +#types time string addr port addr port string string string enum enum string string addr addr port count string table[enum] interval bool string string string double double +1374782085.726121 - - - - - - - - - Software::Vulnerable_Version 1.2.3.4 is running Java 1.7.0.15 which is vulnerable. Java 1.7.0.15 1.2.3.4 - - - bro Notice::ACTION_LOG 3600.000000 F - - - - - +1374782085.726121 - - - - - - - - - Software::Vulnerable_Version 1.2.3.5 is running Java 1.6.0.43 which is vulnerable. Java 1.6.0.43 1.2.3.5 - - - bro Notice::ACTION_LOG 3600.000000 F - - - - - +#close 2013-07-25-19-54-45 diff --git a/testing/btest/Traces/dns-dnskey.trace b/testing/btest/Traces/dns-dnskey.trace new file mode 100644 index 0000000000..c7a6448e79 Binary files /dev/null and b/testing/btest/Traces/dns-dnskey.trace differ diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 4a13833094..7ccf99eea8 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -7,7 +7,7 @@ IgnoreFiles = *.tmp *.swp #* *.trace .DS_Store [environment] BROPATH=`bash -c %(testbase)s/../../build/bro-path-dev` -BROMAGIC=%(testbase)s/../../magic +BROMAGIC=%(testbase)s/../../magic/database BRO_SEED_FILE=%(testbase)s/random.seed TZ=UTC LC_ALL=C diff --git a/testing/btest/coverage/bare-mode-errors.test b/testing/btest/coverage/bare-mode-errors.test index 34ba063081..1910ef8e17 100644 --- a/testing/btest/coverage/bare-mode-errors.test +++ b/testing/btest/coverage/bare-mode-errors.test @@ -10,5 +10,8 @@ # # @TEST-EXEC: test -d $DIST/scripts # @TEST-EXEC: for script in `find $DIST/scripts/ -name \*\.bro -not -path '*/site/*'`; do echo "=== $script" >>allerrors; if echo "$script" | egrep -q 'communication/listen|controllee'; then rm -rf load_attempt .bgprocs; btest-bg-run load_attempt bro -b $script; btest-bg-wait -k 2; cat load_attempt/.stderr >>allerrors; else bro -b $script 2>>allerrors; fi done || exit 0 -# @TEST-EXEC: cat allerrors | grep -v "received termination signal" | grep -v '===' | sort | uniq > unique_errors +# @TEST-EXEC: cat allerrors | grep -v "received termination signal" | fgrep -v -f %INPUT | grep -v '===' | sort | uniq > unique_errors # @TEST-EXEC: btest-diff unique_errors + +# White-list of tests to exclude because of cyclic load dependencies. +scripts/base/protocols/ftp/utils.bro diff --git a/testing/btest/istate/events-ssl.bro b/testing/btest/istate/events-ssl.bro index 249ebc3754..d227417c15 100644 --- a/testing/btest/istate/events-ssl.bro +++ b/testing/btest/istate/events-ssl.bro @@ -41,16 +41,25 @@ redef ssl_ca_certificate = "../ca_cert.pem"; redef ssl_private_key = "../bro.pem"; redef ssl_passphrase = "my-password"; +# Make sure the HTTP connection really gets out. +# (We still miss one final connection event because we shutdown before +# it gets propagated but that's ok.) +redef tcp_close_delay = 0secs; + # File-analysis fields in http.log won't get set on receiver side correctly, # one problem is with the way serialization may send a unique ID in place # of a full value and expect the remote side to associate that unique ID with -# a value it received at an earlier time. So sometimes modifications the sender -# makes to the value aren't seen on the receiver (in this case, the mime_type -# field). -event file_new(f: fa_file) &priority=10 +# a value it received at an earlier time. So sometimes modifications the sender# makes to the value aren't seen on the receiver. +function myfh(c: connection, is_orig: bool): string { - delete f$mime_type; - FileAnalysis::stop(f); + return ""; + } + +event bro_init() + { + # Ignore all http files. + Files::register_protocol(Analyzer::ANALYZER_HTTP, + [$get_file_handle = myfh]); } @TEST-END-FILE diff --git a/testing/btest/istate/events.bro b/testing/btest/istate/events.bro index 21f46cf4b3..1edf14fee7 100644 --- a/testing/btest/istate/events.bro +++ b/testing/btest/istate/events.bro @@ -39,12 +39,17 @@ redef tcp_close_delay = 0secs; # File-analysis fields in http.log won't get set on receiver side correctly, # one problem is with the way serialization may send a unique ID in place # of a full value and expect the remote side to associate that unique ID with -# a value it received at an earlier time. So sometimes modifications the sender# makes to the value aren't seen on the receiver (in this case, the mime_type -# field). -event file_new(f: fa_file) &priority=10 +# a value it received at an earlier time. So sometimes modifications the sender# makes to the value aren't seen on the receiver. +function myfh(c: connection, is_orig: bool): string { - delete f$mime_type; - FileAnalysis::stop(f); + return ""; + } + +event bro_init() + { + # Ignore all http files. + Files::register_protocol(Analyzer::ANALYZER_HTTP, + [$get_file_handle = myfh]); } @TEST-END-FILE diff --git a/testing/btest/scripts/base/frameworks/file-analysis/bifs/remove_action.bro b/testing/btest/scripts/base/frameworks/file-analysis/bifs/remove_action.bro index 1f15a4221f..a3704618bd 100644 --- a/testing/btest/scripts/base/frameworks/file-analysis/bifs/remove_action.bro +++ b/testing/btest/scripts/base/frameworks/file-analysis/bifs/remove_action.bro @@ -11,8 +11,8 @@ redef test_get_file_name = function(f: fa_file): string event file_new(f: fa_file) &priority=-10 { for ( tag in test_file_analyzers ) - FileAnalysis::remove_analyzer(f, tag); + Files::remove_analyzer(f, tag); local filename = test_get_file_name(f); - FileAnalysis::remove_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT, - $extract_filename=filename]); + Files::remove_analyzer(f, Files::ANALYZER_EXTRACT, + [$extract_filename=filename]); } diff --git a/testing/btest/scripts/base/frameworks/file-analysis/bifs/set_timeout_interval.bro b/testing/btest/scripts/base/frameworks/file-analysis/bifs/set_timeout_interval.bro index 8ec4704cdb..c9eac4c31d 100644 --- a/testing/btest/scripts/base/frameworks/file-analysis/bifs/set_timeout_interval.bro +++ b/testing/btest/scripts/base/frameworks/file-analysis/bifs/set_timeout_interval.bro @@ -20,7 +20,7 @@ redef default_file_timeout_interval = 2sec; event file_timeout(f: fa_file) { if ( timeout_cnt < 1 ) - FileAnalysis::set_timeout_interval(f, f$timeout_interval); + Files::set_timeout_interval(f, f$timeout_interval); else terminate(); ++timeout_cnt; diff --git a/testing/btest/scripts/base/frameworks/file-analysis/bifs/stop.bro b/testing/btest/scripts/base/frameworks/file-analysis/bifs/stop.bro index e994706010..dd40c69684 100644 --- a/testing/btest/scripts/base/frameworks/file-analysis/bifs/stop.bro +++ b/testing/btest/scripts/base/frameworks/file-analysis/bifs/stop.bro @@ -4,5 +4,5 @@ event file_new(f: fa_file) { - FileAnalysis::stop(f); + Files::stop(f); } diff --git a/testing/btest/scripts/base/frameworks/file-analysis/irc.bro b/testing/btest/scripts/base/frameworks/file-analysis/irc.bro index 2b93a59a8f..9fd8e06613 100644 --- a/testing/btest/scripts/base/frameworks/file-analysis/irc.bro +++ b/testing/btest/scripts/base/frameworks/file-analysis/irc.bro @@ -4,7 +4,17 @@ redef test_file_analysis_source = "IRC_DATA"; -redef test_get_file_name = function(f: fa_file): string +global first: bool = T; + +function myfile(f: fa_file): string { - return "thefile"; - }; + if ( first ) + { + first = F; + return "thefile"; + } + else + return ""; + } + +redef test_get_file_name = myfile; diff --git a/testing/btest/scripts/base/frameworks/file-analysis/logging.bro b/testing/btest/scripts/base/frameworks/file-analysis/logging.bro index 9792017962..1d1f5fd721 100644 --- a/testing/btest/scripts/base/frameworks/file-analysis/logging.bro +++ b/testing/btest/scripts/base/frameworks/file-analysis/logging.bro @@ -1,5 +1,5 @@ # @TEST-EXEC: bro -r $TRACES/http/get.trace $SCRIPTS/file-analysis-test.bro %INPUT -# @TEST-EXEC: btest-diff file_analysis.log +# @TEST-EXEC: btest-diff files.log redef test_file_analysis_source = "HTTP"; diff --git a/testing/btest/scripts/base/frameworks/intel/cluster-transparency.bro b/testing/btest/scripts/base/frameworks/intel/cluster-transparency.bro index 3810de5d4b..4d977d475d 100644 --- a/testing/btest/scripts/base/frameworks/intel/cluster-transparency.bro +++ b/testing/btest/scripts/base/frameworks/intel/cluster-transparency.bro @@ -28,7 +28,7 @@ event remote_connection_handshake_done(p: event_peer) # Insert the data once both workers are connected. if ( Cluster::local_node_type() == Cluster::MANAGER && Cluster::worker_count == 2 ) { - Intel::insert([$host=1.2.3.4,$meta=[$source="manager"]]); + Intel::insert([$indicator="1.2.3.4", $indicator_type=Intel::ADDR, $meta=[$source="manager"]]); } } @@ -39,7 +39,7 @@ event Intel::cluster_new_item(item: Intel::Item) if ( ! is_remote_event() ) return; - print fmt("cluster_new_item: %s inserted by %s (from peer: %s)", item$host, item$meta$source, get_event_peer()$descr); + print fmt("cluster_new_item: %s inserted by %s (from peer: %s)", item$indicator, item$meta$source, get_event_peer()$descr); if ( ! sent_data ) { @@ -47,9 +47,9 @@ event Intel::cluster_new_item(item: Intel::Item) # full cluster is constructed. sent_data = T; if ( Cluster::node == "worker-1" ) - Intel::insert([$host=123.123.123.123,$meta=[$source="worker-1"]]); + Intel::insert([$indicator="123.123.123.123", $indicator_type=Intel::ADDR, $meta=[$source="worker-1"]]); if ( Cluster::node == "worker-2" ) - Intel::insert([$host=4.3.2.1,$meta=[$source="worker-2"]]); + Intel::insert([$indicator="4.3.2.1", $indicator_type=Intel::ADDR, $meta=[$source="worker-2"]]); } # We're forcing worker-2 to do a lookup when it has three intelligence items diff --git a/testing/btest/scripts/base/frameworks/intel/input-and-match.bro b/testing/btest/scripts/base/frameworks/intel/input-and-match.bro index f77f5c0f1d..7150d30993 100644 --- a/testing/btest/scripts/base/frameworks/intel/input-and-match.bro +++ b/testing/btest/scripts/base/frameworks/intel/input-and-match.bro @@ -5,10 +5,10 @@ # @TEST-EXEC: btest-diff broproc/intel.log @TEST-START-FILE intel.dat -#fields host net str str_type meta.source meta.desc meta.url -1.2.3.4 - - - source1 this host is just plain baaad http://some-data-distributor.com/1234 -1.2.3.4 - - - source1 this host is just plain baaad http://some-data-distributor.com/1234 -- - e@mail.com Intel::EMAIL source1 Phishing email source http://some-data-distributor.com/100000 +#fields indicator indicator_type meta.source meta.desc meta.url +1.2.3.4 Intel::ADDR source1 this host is just plain baaad http://some-data-distributor.com/1234 +1.2.3.4 Intel::ADDR source1 this host is just plain baaad http://some-data-distributor.com/1234 +e@mail.com Intel::EMAIL source1 Phishing email source http://some-data-distributor.com/100000 @TEST-END-FILE @load frameworks/communication/listen @@ -18,8 +18,8 @@ redef enum Intel::Where += { SOMEWHERE }; event do_it() { - Intel::seen([$str="e@mail.com", - $str_type=Intel::EMAIL, + Intel::seen([$indicator="e@mail.com", + $indicator_type=Intel::EMAIL, $where=SOMEWHERE]); Intel::seen([$host=1.2.3.4, diff --git a/testing/btest/scripts/base/frameworks/intel/read-file-dist-cluster.bro b/testing/btest/scripts/base/frameworks/intel/read-file-dist-cluster.bro index 6838736249..f336fe24b3 100644 --- a/testing/btest/scripts/base/frameworks/intel/read-file-dist-cluster.bro +++ b/testing/btest/scripts/base/frameworks/intel/read-file-dist-cluster.bro @@ -19,10 +19,10 @@ redef Cluster::nodes = { @TEST-END-FILE @TEST-START-FILE intel.dat -#fields host net str str_type meta.source meta.desc meta.url -1.2.3.4 - - - source1 this host is just plain baaad http://some-data-distributor.com/1234 -1.2.3.4 - - - source1 this host is just plain baaad http://some-data-distributor.com/1234 -- - e@mail.com Intel::EMAIL source1 Phishing email source http://some-data-distributor.com/100000 +#fields indicator indicator_type meta.source meta.desc meta.url +1.2.3.4 Intel::ADDR source1 this host is just plain baaad http://some-data-distributor.com/1234 +1.2.3.4 Intel::ADDR source1 this host is just plain baaad http://some-data-distributor.com/1234 +e@mail.com Intel::EMAIL source1 Phishing email source http://some-data-distributor.com/100000 @TEST-END-FILE @load base/frameworks/control @@ -41,7 +41,7 @@ redef enum Intel::Where += { event do_it() { Intel::seen([$host=1.2.3.4, $where=Intel::IN_A_TEST]); - Intel::seen([$str="e@mail.com", $str_type=Intel::EMAIL, $where=Intel::IN_A_TEST]); + Intel::seen([$indicator="e@mail.com", $indicator_type=Intel::EMAIL, $where=Intel::IN_A_TEST]); } event bro_init() diff --git a/testing/btest/scripts/base/protocols/dns/dns-key.bro b/testing/btest/scripts/base/protocols/dns/dns-key.bro new file mode 100644 index 0000000000..c51788c605 --- /dev/null +++ b/testing/btest/scripts/base/protocols/dns/dns-key.bro @@ -0,0 +1,4 @@ +# Making sure DNSKEY gets logged as such. +# +# @TEST-EXEC: bro -r $TRACES/dns-dnskey.trace +# @TEST-EXEC: btest-diff dns.log diff --git a/testing/btest/scripts/base/protocols/ftp/ftp-extract.bro b/testing/btest/scripts/base/protocols/ftp/ftp-extract.bro deleted file mode 100644 index 8cbacdbf6f..0000000000 --- a/testing/btest/scripts/base/protocols/ftp/ftp-extract.bro +++ /dev/null @@ -1,10 +0,0 @@ -# This tests FTP file extraction. -# -# @TEST-EXEC: bro -r $TRACES/ftp/ipv4.trace %INPUT -# @TEST-EXEC: btest-diff conn.log -# @TEST-EXEC: btest-diff ftp.log -# @TEST-EXEC: cat ftp-item-*.dat | sort > extractions -# @TEST-EXEC: btest-diff extractions - -redef FTP::logged_commands += {"LIST"}; -redef FTP::extract_file_types=/.*/; diff --git a/testing/btest/scripts/base/protocols/http/http-extract-files.bro b/testing/btest/scripts/base/protocols/http/http-extract-files.bro deleted file mode 100644 index 6156009821..0000000000 --- a/testing/btest/scripts/base/protocols/http/http-extract-files.bro +++ /dev/null @@ -1,6 +0,0 @@ -# @TEST-EXEC: bro -C -r $TRACES/web.trace %INPUT -# @TEST-EXEC: btest-diff http.log -# @TEST-EXEC: mv http-item-*.dat http-item.dat -# @TEST-EXEC: btest-diff http-item.dat - -redef HTTP::extract_file_types += /text\/html/; diff --git a/testing/btest/scripts/base/protocols/http/http-mime-and-md5.bro b/testing/btest/scripts/base/protocols/http/http-mime-and-md5.bro deleted file mode 100644 index b35e491b4d..0000000000 --- a/testing/btest/scripts/base/protocols/http/http-mime-and-md5.bro +++ /dev/null @@ -1,6 +0,0 @@ -# This tests md5 calculation for a specified mime type. - -# @TEST-EXEC: bro -r $TRACES/http/pipelined-requests.trace %INPUT > output -# @TEST-EXEC: btest-diff http.log - -redef HTTP::generate_md5 += /image\/png/; diff --git a/testing/btest/scripts/base/protocols/http/multipart-extract.bro b/testing/btest/scripts/base/protocols/http/multipart-extract.bro index c2789750a3..a919a844b2 100644 --- a/testing/btest/scripts/base/protocols/http/multipart-extract.bro +++ b/testing/btest/scripts/base/protocols/http/multipart-extract.bro @@ -1,5 +1,9 @@ # @TEST-EXEC: bro -C -r $TRACES/http/multipart.trace %INPUT # @TEST-EXEC: btest-diff http.log -# @TEST-EXEC: cat http-item-* | sort > extractions +# @TEST-EXEC: cat extract_files/http-item-* | sort > extractions -redef HTTP::extract_file_types += /.*/; +event file_new(f: fa_file) + { + local fname = fmt("http-item-%s", f$id); + Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]); + } diff --git a/testing/btest/scripts/base/protocols/irc/dcc-extract.test b/testing/btest/scripts/base/protocols/irc/dcc-extract.test deleted file mode 100644 index cbfc6890da..0000000000 --- a/testing/btest/scripts/base/protocols/irc/dcc-extract.test +++ /dev/null @@ -1,11 +0,0 @@ -# This tests that the contents of a DCC transfer negotiated with IRC can be -# correctly extracted. - -# @TEST-EXEC: bro -r $TRACES/irc-dcc-send.trace %INPUT -# @TEST-EXEC: btest-diff irc.log -# @TEST-EXEC: mv irc-dcc-item-*.dat irc-dcc-item.dat -# @TEST-EXEC: btest-diff irc-dcc-item.dat -# @TEST-EXEC: bro -r $TRACES/irc-dcc-send.trace %INPUT IRC::extraction_prefix="test" -# @TEST-EXEC: test -e test-*.dat - -redef IRC::extract_file_types=/.*/; diff --git a/testing/btest/scripts/base/protocols/smtp/mime-extract.test b/testing/btest/scripts/base/protocols/smtp/mime-extract.test deleted file mode 100644 index 0caa5d530c..0000000000 --- a/testing/btest/scripts/base/protocols/smtp/mime-extract.test +++ /dev/null @@ -1,11 +0,0 @@ -# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT -# @TEST-EXEC: btest-diff smtp_entities.log -# @TEST-EXEC: cat smtp-entity-*.dat | sort > extractions -# @TEST-EXEC: btest-diff extractions -# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT SMTP::extraction_prefix="test" -# @TEST-EXEC: cnt=0 && for f in test-*.dat; do cnt=$((cnt+1)); done && echo $cnt >filecount -# @TEST-EXEC: btest-diff filecount - -@load base/protocols/smtp - -redef SMTP::extract_file_types=/text\/plain/; diff --git a/testing/btest/scripts/base/protocols/smtp/mime.test b/testing/btest/scripts/base/protocols/smtp/mime.test deleted file mode 100644 index 8e7a336987..0000000000 --- a/testing/btest/scripts/base/protocols/smtp/mime.test +++ /dev/null @@ -1,6 +0,0 @@ -# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT -# @TEST-EXEC: btest-diff smtp_entities.log - -@load base/protocols/smtp - -redef SMTP::generate_md5=/text\/plain/; diff --git a/testing/btest/scripts/base/utils/active-http.test b/testing/btest/scripts/base/utils/active-http.test new file mode 100644 index 0000000000..127b21d77e --- /dev/null +++ b/testing/btest/scripts/base/utils/active-http.test @@ -0,0 +1,28 @@ +# @TEST-REQUIRES: which httpd +# @TEST-REQUIRES: which python +# +# @TEST-EXEC: btest-bg-run httpd python $SCRIPTS/httpd.py --max 1 +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: btest-bg-wait 15 +# @TEST-EXEC: btest-diff bro/.stdout + +@load base/utils/active-http + +redef exit_only_after_terminate = T; + +event bro_init() + { + local req = ActiveHTTP::Request($url="localhost:32123"); + + when ( local resp = ActiveHTTP::request(req) ) + { + print resp; + terminate(); + } + timeout 1min + { + print "HTTP request timeout"; + terminate(); + } + } diff --git a/testing/btest/scripts/base/utils/dir.test b/testing/btest/scripts/base/utils/dir.test new file mode 100644 index 0000000000..44fee3860f --- /dev/null +++ b/testing/btest/scripts/base/utils/dir.test @@ -0,0 +1,58 @@ +# @TEST-EXEC: btest-bg-run bro bro -b ../dirtest.bro +# @TEST-EXEC: btest-bg-wait 10 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff bro/.stdout + +@TEST-START-FILE dirtest.bro + +@load base/utils/dir + +redef exit_only_after_terminate = T; + +global c: count = 0; + +function check_terminate_condition() + { + c += 1; + + if ( c == 10 ) + terminate(); + } + +function new_file1(fname: string) + { + print "new_file1", fname; + check_terminate_condition(); + } + +function new_file2(fname: string) + { + print "new_file2", fname; + check_terminate_condition(); + } + +event change_things() + { + system("touch ../testdir/newone"); + system("rm ../testdir/bye && touch ../testdir/bye"); + } + +event bro_init() + { + Dir::monitor("../testdir", new_file1, .5sec); + Dir::monitor("../testdir", new_file2, 1sec); + schedule 1sec { change_things() }; + } + +@TEST-END-FILE + +@TEST-START-FILE testdir/hi +123 +@TEST-END-FILE + +@TEST-START-FILE testdir/howsitgoing +abc +@TEST-END-FILE + +@TEST-START-FILE testdir/bye +!@# +@TEST-END-FILE diff --git a/testing/btest/scripts/base/utils/exec.test b/testing/btest/scripts/base/utils/exec.test new file mode 100644 index 0000000000..8876f0f49b --- /dev/null +++ b/testing/btest/scripts/base/utils/exec.test @@ -0,0 +1,74 @@ +# @TEST-EXEC: btest-bg-run bro bro -b ../exectest.bro +# @TEST-EXEC: btest-bg-wait 10 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff bro/.stdout + +@TEST-START-FILE exectest.bro + +@load base/utils/exec + +redef exit_only_after_terminate = T; + +global c: count = 0; + +function check_exit_condition() + { + c += 1; + + if ( c == 4 ) + terminate(); + } + +function test_cmd(label: string, cmd: Exec::Command) + { + when ( local result = Exec::run(cmd) ) + { + print label, result; + check_exit_condition(); + } + } + +event bro_init() + { + test_cmd("test1", [$cmd="bash ../somescript.sh", + $read_files=set("out1", "out2")]); + test_cmd("test2", [$cmd="bash ../nofiles.sh"]); + test_cmd("test3", [$cmd="bash ../suicide.sh"]); + test_cmd("test4", [$cmd="bash ../stdin.sh", $stdin="hibye"]); + } + +@TEST-END-FILE + +@TEST-START-FILE somescript.sh +#! /usr/bin/env bash +echo "insert text here" > out1 +echo "and here" >> out1 +echo "insert more text here" > out2 +echo "and there" >> out2 +echo "done" +echo "exit" +echo "stop" +@TEST-END-FILE + +@TEST-START-FILE nofiles.sh +#! /usr/bin/env bash +echo "here's something on stdout" +echo "some more stdout" +echo "last stdout" +echo "and some stderr" 1>&2 +echo "more stderr" 1>&2 +echo "last stderr" 1>&2 +exit 1 +@TEST-END-FILE + +@TEST-START-FILE suicide.sh +#! /usr/bin/env bash +echo "FML" +kill -9 $$ +echo "nope" +@TEST-END-FILE + +@TEST-START-FILE stdin.sh +#! /usr/bin/env bash +read -r line +echo "$line" +@TEST-END-FILE diff --git a/testing/btest/scripts/base/utils/files.test b/testing/btest/scripts/base/utils/files.test index 84eff13187..3324522030 100644 --- a/testing/btest/scripts/base/utils/files.test +++ b/testing/btest/scripts/base/utils/files.test @@ -11,3 +11,10 @@ event connection_established(c: connection) print generate_extraction_filename("", c, "test-suffix"); print generate_extraction_filename("", c, ""); } + +event bro_init() + { + print extract_filename_from_content_disposition("attachment; filename=Economy"); + print extract_filename_from_content_disposition("attachment; name=\"US-$ rates\""); + print extract_filename_from_content_disposition("attachment; filename*=iso-8859-1'en'%A3%20rates"); + } \ No newline at end of file diff --git a/testing/external/subdir-btest.cfg b/testing/external/subdir-btest.cfg index b631ba2457..fb5873418a 100644 --- a/testing/external/subdir-btest.cfg +++ b/testing/external/subdir-btest.cfg @@ -7,7 +7,7 @@ IgnoreFiles = *.tmp *.swp #* *.trace .gitignore *.skeleton [environment] BROPATH=`bash -c %(testbase)s/../../../build/bro-path-dev`:%(testbase)s/../scripts -BROMAGIC=%(testbase)s/../../../magic +BROMAGIC=%(testbase)s/../../../magic/database BRO_SEED_FILE=%(testbase)s/../random.seed TZ=UTC LC_ALL=C diff --git a/testing/scripts/file-analysis-test.bro b/testing/scripts/file-analysis-test.bro index 8b85ae2bd5..d84fadae5c 100644 --- a/testing/scripts/file-analysis-test.bro +++ b/testing/scripts/file-analysis-test.bro @@ -1,7 +1,11 @@ +@load base/files/extract +@load base/files/hash + +redef FileExtract::prefix = "./"; global test_file_analysis_source: string = "" &redef; -global test_file_analyzers: set[FileAnalysis::AnalyzerArgs]; +global test_file_analyzers: set[Files::Tag]; global test_get_file_name: function(f: fa_file): string = function(f: fa_file): string { return ""; } &redef; @@ -42,15 +46,15 @@ event file_new(f: fa_file) f$source == test_file_analysis_source ) { for ( tag in test_file_analyzers ) - FileAnalysis::add_analyzer(f, tag); + Files::add_analyzer(f, tag); local filename: string = test_get_file_name(f); if ( filename != "" ) - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_EXTRACT, - $extract_filename=filename]); - FileAnalysis::add_analyzer(f, [$tag=FileAnalysis::ANALYZER_DATA_EVENT, - $chunk_event=file_chunk, - $stream_event=file_stream]); + Files::add_analyzer(f, Files::ANALYZER_EXTRACT, + [$extract_filename=filename]); + Files::add_analyzer(f, Files::ANALYZER_DATA_EVENT, + [$chunk_event=file_chunk, + $stream_event=file_stream]); } if ( f?$bof_buffer ) @@ -66,7 +70,7 @@ event file_new(f: fa_file) } } -event file_over_new_connection(f: fa_file, c: connection) +event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) { print "FILE_OVER_NEW_CONNECTION"; } @@ -106,7 +110,7 @@ event file_state_remove(f: fa_file) event bro_init() { - add test_file_analyzers[[$tag=FileAnalysis::ANALYZER_MD5]]; - add test_file_analyzers[[$tag=FileAnalysis::ANALYZER_SHA1]]; - add test_file_analyzers[[$tag=FileAnalysis::ANALYZER_SHA256]]; + add test_file_analyzers[Files::ANALYZER_MD5]; + add test_file_analyzers[Files::ANALYZER_SHA1]; + add test_file_analyzers[Files::ANALYZER_SHA256]; } diff --git a/testing/scripts/httpd.py b/testing/scripts/httpd.py new file mode 100755 index 0000000000..0732614bc2 --- /dev/null +++ b/testing/scripts/httpd.py @@ -0,0 +1,40 @@ +#! /usr/bin/env python + +import BaseHTTPServer + +class MyRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): + + def do_GET(self): + self.send_response(200) + self.send_header("Content-type", "text/plain") + self.end_headers() + self.wfile.write("It works!") + + def version_string(self): + return "1.0" + + def date_time_string(self): + return "July 22, 2013" + + +if __name__ == "__main__": + from optparse import OptionParser + p = OptionParser() + p.add_option("-a", "--addr", type="string", default="localhost", + help=("listen on given address (numeric IP or host name), " + "an empty string (the default) means INADDR_ANY")) + p.add_option("-p", "--port", type="int", default=32123, + help="listen on given TCP port number") + p.add_option("-m", "--max", type="int", default=-1, + help="max number of requests to respond to, -1 means no max") + options, args = p.parse_args() + + httpd = BaseHTTPServer.HTTPServer((options.addr, options.port), + MyRequestHandler) + if options.max == -1: + httpd.serve_forever() + else: + served_count = 0 + while served_count != options.max: + httpd.handle_request() + served_count += 1